Commit a40cdad18c06fd377622c47aa34564df7ea959b5 | zig

lib/tsan/builtins/assembly.h created+293

...	@@ -0,0 +1,293 @@
1	//===-- assembly.h - compiler-rt assembler support macros -----------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file defines macros for use in compiler-rt assembler source.
10	// This file is not part of the interface of this library.
11	//
12	//===----------------------------------------------------------------------===//
13
14	#ifndef COMPILERRT_ASSEMBLY_H
15	#define COMPILERRT_ASSEMBLY_H
16
17	#if defined(__linux__) && defined(__CET__)
18	#if __has_include(<cet.h>)
19	#include <cet.h>
20	#endif
21	#endif
22
23	#if defined(__APPLE__) && defined(__aarch64__)
24	#define SEPARATOR %%
25	#else
26	#define SEPARATOR ;
27	#endif
28
29	#if defined(__APPLE__)
30	#define HIDDEN(name) .private_extern name
31	#define LOCAL_LABEL(name) L_##name
32	// tell linker it can break up file at label boundaries
33	#define FILE_LEVEL_DIRECTIVE .subsections_via_symbols
34	#define SYMBOL_IS_FUNC(name)
35	#define CONST_SECTION .const
36
37	#define NO_EXEC_STACK_DIRECTIVE
38
39	#elif defined(__ELF__)
40
41	#define HIDDEN(name) .hidden name
42	#define LOCAL_LABEL(name) .L_##name
43	#define FILE_LEVEL_DIRECTIVE
44	#if defined(__arm__) \|\| defined(__aarch64__)
45	#define SYMBOL_IS_FUNC(name) .type name,%function
46	#else
47	#define SYMBOL_IS_FUNC(name) .type name,@function
48	#endif
49	#define CONST_SECTION .section .rodata
50
51	#if defined(__GNU__) \|\| defined(__FreeBSD__) \|\| defined(__Fuchsia__) \|\| \
52	defined(__linux__)
53	#define NO_EXEC_STACK_DIRECTIVE .section .note.GNU-stack,"",%progbits
54	#else
55	#define NO_EXEC_STACK_DIRECTIVE
56	#endif
57
58	#else // !__APPLE__ && !__ELF__
59
60	#define HIDDEN(name)
61	#define LOCAL_LABEL(name) .L ## name
62	#define FILE_LEVEL_DIRECTIVE
63	#define SYMBOL_IS_FUNC(name) \
64	.def name SEPARATOR \
65	.scl 2 SEPARATOR \
66	.type 32 SEPARATOR \
67	.endef
68	#define CONST_SECTION .section .rdata,"rd"
69
70	#define NO_EXEC_STACK_DIRECTIVE
71
72	#endif
73
74	#if defined(__arm__) \|\| defined(__aarch64__)
75	#define FUNC_ALIGN \
76	.text SEPARATOR \
77	.balign 16 SEPARATOR
78	#else
79	#define FUNC_ALIGN
80	#endif
81
82	// BTI and PAC gnu property note
83	#define NT_GNU_PROPERTY_TYPE_0 5
84	#define GNU_PROPERTY_AARCH64_FEATURE_1_AND 0xc0000000
85	#define GNU_PROPERTY_AARCH64_FEATURE_1_BTI 1
86	#define GNU_PROPERTY_AARCH64_FEATURE_1_PAC 2
87
88	#if defined(__ARM_FEATURE_BTI_DEFAULT)
89	#define BTI_FLAG GNU_PROPERTY_AARCH64_FEATURE_1_BTI
90	#else
91	#define BTI_FLAG 0
92	#endif
93
94	#if __ARM_FEATURE_PAC_DEFAULT & 3
95	#define PAC_FLAG GNU_PROPERTY_AARCH64_FEATURE_1_PAC
96	#else
97	#define PAC_FLAG 0
98	#endif
99
100	#define GNU_PROPERTY(type, value) \
101	.pushsection .note.gnu.property, "a" SEPARATOR \
102	.p2align 3 SEPARATOR \
103	.word 4 SEPARATOR \
104	.word 16 SEPARATOR \
105	.word NT_GNU_PROPERTY_TYPE_0 SEPARATOR \
106	.asciz "GNU" SEPARATOR \
107	.word type SEPARATOR \
108	.word 4 SEPARATOR \
109	.word value SEPARATOR \
110	.word 0 SEPARATOR \
111	.popsection
112
113	#if BTI_FLAG != 0
114	#define BTI_C hint #34
115	#define BTI_J hint #36
116	#else
117	#define BTI_C
118	#define BTI_J
119	#endif
120
121	#if (BTI_FLAG \| PAC_FLAG) != 0
122	#define GNU_PROPERTY_BTI_PAC \
123	GNU_PROPERTY(GNU_PROPERTY_AARCH64_FEATURE_1_AND, BTI_FLAG \| PAC_FLAG)
124	#else
125	#define GNU_PROPERTY_BTI_PAC
126	#endif
127
128	#if defined(__clang__) \|\| defined(__GCC_HAVE_DWARF2_CFI_ASM)
129	#define CFI_START .cfi_startproc
130	#define CFI_END .cfi_endproc
131	#else
132	#define CFI_START
133	#define CFI_END
134	#endif
135
136	#if defined(__arm__)
137
138	// Determine actual [ARM][THUMB[1][2]] ISA using compiler predefined macros:
139	// - for '-mthumb -march=armv6' compiler defines '__thumb__'
140	// - for '-mthumb -march=armv7' compiler defines '__thumb__' and '__thumb2__'
141	#if defined(__thumb2__) \|\| defined(__thumb__)
142	#define DEFINE_CODE_STATE .thumb SEPARATOR
143	#define DECLARE_FUNC_ENCODING .thumb_func SEPARATOR
144	#if defined(__thumb2__)
145	#define USE_THUMB_2
146	#define IT(cond) it cond
147	#define ITT(cond) itt cond
148	#define ITE(cond) ite cond
149	#else
150	#define USE_THUMB_1
151	#define IT(cond)
152	#define ITT(cond)
153	#define ITE(cond)
154	#endif // defined(__thumb__2)
155	#else // !defined(__thumb2__) && !defined(__thumb__)
156	#define DEFINE_CODE_STATE .arm SEPARATOR
157	#define DECLARE_FUNC_ENCODING
158	#define IT(cond)
159	#define ITT(cond)
160	#define ITE(cond)
161	#endif
162
163	#if defined(USE_THUMB_1) && defined(USE_THUMB_2)
164	#error "USE_THUMB_1 and USE_THUMB_2 can't be defined together."
165	#endif
166
167	#if defined(__ARM_ARCH_4T__) \|\| __ARM_ARCH >= 5
168	#define ARM_HAS_BX
169	#endif
170	#if !defined(__ARM_FEATURE_CLZ) && !defined(USE_THUMB_1) && \
171	(__ARM_ARCH >= 6 \|\| (__ARM_ARCH == 5 && !defined(__ARM_ARCH_5__)))
172	#define __ARM_FEATURE_CLZ
173	#endif
174
175	#ifdef ARM_HAS_BX
176	#define JMP(r) bx r
177	#define JMPc(r, c) bx##c r
178	#else
179	#define JMP(r) mov pc, r
180	#define JMPc(r, c) mov##c pc, r
181	#endif
182
183	// pop {pc} can't switch Thumb mode on ARMv4T
184	#if __ARM_ARCH >= 5
185	#define POP_PC() pop {pc}
186	#else
187	#define POP_PC() \
188	pop {ip}; \
189	JMP(ip)
190	#endif
191
192	#if defined(USE_THUMB_2)
193	#define WIDE(op) op.w
194	#else
195	#define WIDE(op) op
196	#endif
197	#else // !defined(__arm)
198	#define DECLARE_FUNC_ENCODING
199	#define DEFINE_CODE_STATE
200	#endif
201
202	#define GLUE2_(a, b) a##b
203	#define GLUE(a, b) GLUE2_(a, b)
204	#define GLUE2(a, b) GLUE2_(a, b)
205	#define GLUE3_(a, b, c) a##b##c
206	#define GLUE3(a, b, c) GLUE3_(a, b, c)
207	#define GLUE4_(a, b, c, d) a##b##c##d
208	#define GLUE4(a, b, c, d) GLUE4_(a, b, c, d)
209
210	#define SYMBOL_NAME(name) GLUE(__USER_LABEL_PREFIX__, name)
211
212	#ifdef VISIBILITY_HIDDEN
213	#define DECLARE_SYMBOL_VISIBILITY(name) \
214	HIDDEN(SYMBOL_NAME(name)) SEPARATOR
215	#define DECLARE_SYMBOL_VISIBILITY_UNMANGLED(name) \
216	HIDDEN(name) SEPARATOR
217	#else
218	#define DECLARE_SYMBOL_VISIBILITY(name)
219	#define DECLARE_SYMBOL_VISIBILITY_UNMANGLED(name)
220	#endif
221
222	#define DEFINE_COMPILERRT_FUNCTION(name) \
223	DEFINE_CODE_STATE \
224	FILE_LEVEL_DIRECTIVE SEPARATOR \
225	.globl SYMBOL_NAME(name) SEPARATOR \
226	SYMBOL_IS_FUNC(SYMBOL_NAME(name)) SEPARATOR \
227	DECLARE_SYMBOL_VISIBILITY(name) \
228	DECLARE_FUNC_ENCODING \
229	SYMBOL_NAME(name):
230
231	#define DEFINE_COMPILERRT_THUMB_FUNCTION(name) \
232	DEFINE_CODE_STATE \
233	FILE_LEVEL_DIRECTIVE SEPARATOR \
234	.globl SYMBOL_NAME(name) SEPARATOR \
235	SYMBOL_IS_FUNC(SYMBOL_NAME(name)) SEPARATOR \
236	DECLARE_SYMBOL_VISIBILITY(name) SEPARATOR \
237	.thumb_func SEPARATOR \
238	SYMBOL_NAME(name):
239
240	#define DEFINE_COMPILERRT_PRIVATE_FUNCTION(name) \
241	DEFINE_CODE_STATE \
242	FILE_LEVEL_DIRECTIVE SEPARATOR \
243	.globl SYMBOL_NAME(name) SEPARATOR \
244	SYMBOL_IS_FUNC(SYMBOL_NAME(name)) SEPARATOR \
245	HIDDEN(SYMBOL_NAME(name)) SEPARATOR \
246	DECLARE_FUNC_ENCODING \
247	SYMBOL_NAME(name):
248
249	#define DEFINE_COMPILERRT_PRIVATE_FUNCTION_UNMANGLED(name) \
250	DEFINE_CODE_STATE \
251	.globl name SEPARATOR \
252	SYMBOL_IS_FUNC(name) SEPARATOR \
253	HIDDEN(name) SEPARATOR \
254	DECLARE_FUNC_ENCODING \
255	name:
256
257	#define DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(name) \
258	DEFINE_CODE_STATE \
259	FUNC_ALIGN \
260	.globl name SEPARATOR \
261	SYMBOL_IS_FUNC(name) SEPARATOR \
262	DECLARE_SYMBOL_VISIBILITY_UNMANGLED(name) SEPARATOR \
263	DECLARE_FUNC_ENCODING \
264	name: \
265	SEPARATOR CFI_START \
266	SEPARATOR BTI_C
267
268	#define DEFINE_COMPILERRT_FUNCTION_ALIAS(name, target) \
269	.globl SYMBOL_NAME(name) SEPARATOR \
270	SYMBOL_IS_FUNC(SYMBOL_NAME(name)) SEPARATOR \
271	DECLARE_SYMBOL_VISIBILITY(name) SEPARATOR \
272	.set SYMBOL_NAME(name), SYMBOL_NAME(target) SEPARATOR
273
274	#if defined(__ARM_EABI__)
275	#define DEFINE_AEABI_FUNCTION_ALIAS(aeabi_name, name) \
276	DEFINE_COMPILERRT_FUNCTION_ALIAS(aeabi_name, name)
277	#else
278	#define DEFINE_AEABI_FUNCTION_ALIAS(aeabi_name, name)
279	#endif
280
281	#ifdef __ELF__
282	#define END_COMPILERRT_FUNCTION(name) \
283	.size SYMBOL_NAME(name), . - SYMBOL_NAME(name)
284	#define END_COMPILERRT_OUTLINE_FUNCTION(name) \
285	CFI_END SEPARATOR \
286	.size SYMBOL_NAME(name), . - SYMBOL_NAME(name)
287	#else
288	#define END_COMPILERRT_FUNCTION(name)
289	#define END_COMPILERRT_OUTLINE_FUNCTION(name) \
290	CFI_END
291	#endif
292
293	#endif // COMPILERRT_ASSEMBLY_H

lib/tsan/interception/interception.h+24-5

 @@ -185,6 +185,11 @@ const interpose_substitution substitution_##func_name[]             \
 #  else
 #   define __ASM_WEAK_WRAPPER(func) ".weak " #func "\n"
 #  endif  // SANITIZER_FREEBSD || SANITIZER_NETBSD
+#  if defined(__arm__) || defined(__aarch64__)
+#   define ASM_TYPE_FUNCTION_STR "%function"
+#  else
+#   define ASM_TYPE_FUNCTION_STR "@function"
+#  endif
 // Keep trampoline implementation in sync with sanitizer_common/sanitizer_asm.h
 #  define DECLARE_WRAPPER(ret_type, func, ...)                                 \
      extern "C" ret_type func(__VA_ARGS__);                                    \
 @@ -196,12 +201,14 @@ const interpose_substitution substitution_##func_name[]             \
        __ASM_WEAK_WRAPPER(func)                                                \
        ".set " #func ", " SANITIZER_STRINGIFY(TRAMPOLINE(func)) "\n"           \
        ".globl " SANITIZER_STRINGIFY(TRAMPOLINE(func)) "\n"                    \
-       ".type  " SANITIZER_STRINGIFY(TRAMPOLINE(func)) ", %function\n"         \
+       ".type  " SANITIZER_STRINGIFY(TRAMPOLINE(func)) ", "                    \
+         ASM_TYPE_FUNCTION_STR "\n"                                            \
        SANITIZER_STRINGIFY(TRAMPOLINE(func)) ":\n"                             \
-       SANITIZER_STRINGIFY(CFI_STARTPROC) "\n"                                 \
+       C_ASM_STARTPROC "\n"                                                    \
-       SANITIZER_STRINGIFY(ASM_TAIL_CALL) " __interceptor_"                    \
+       C_ASM_TAIL_CALL(SANITIZER_STRINGIFY(TRAMPOLINE(func)),                  \
-         SANITIZER_STRINGIFY(ASM_PREEMPTIBLE_SYM(func)) "\n"                   \
+                       "__interceptor_"                                        \
-       SANITIZER_STRINGIFY(CFI_ENDPROC) "\n"                                   \
+                         SANITIZER_STRINGIFY(ASM_PREEMPTIBLE_SYM(func))) "\n"  \
+       C_ASM_ENDPROC "\n"                                                      \
        ".size  " SANITIZER_STRINGIFY(TRAMPOLINE(func)) ", "                    \
             ".-" SANITIZER_STRINGIFY(TRAMPOLINE(func)) "\n"                    \
      );
 @@ -341,6 +348,18 @@ typedef unsigned long long uptr;
 #else
 typedef unsigned long uptr;
 #endif  // _WIN64
+#if defined(__ELF__) && !SANITIZER_FUCHSIA
+// The use of interceptors makes many sanitizers unusable for static linking.
+// Define a function, if called, will cause a linker error (undefined _DYNAMIC).
+// However, -static-pie (which is not common) cannot be detected at link time.
+extern uptr kDynamic[] asm("_DYNAMIC");
+inline void DoesNotSupportStaticLinking() {
+  [[maybe_unused]] volatile auto x = &kDynamic;
+}
+#else
+inline void DoesNotSupportStaticLinking() {}
+#endif
 }  // namespace __interception
 #define INCLUDED_FROM_INTERCEPTION_LIB

lib/tsan/interception/interception_linux.h+9-7

 @@ -28,12 +28,14 @@ bool InterceptFunction(const char *name, const char *ver, uptr *ptr_to_real,
                        uptr func, uptr trampoline);
 }  // namespace __interception
-#define INTERCEPT_FUNCTION_LINUX_OR_FREEBSD(func) \
+// Cast func to type of REAL(func) before casting to uptr in case it is an
-  ::__interception::InterceptFunction(            \
+// overloaded function, which is the case for some glibc functions when
-      #func,                                      \
+// _FORTIFY_SOURCE is used. This disambiguates which overload to use.
-      (::__interception::uptr *)&REAL(func),      \
+#define INTERCEPT_FUNCTION_LINUX_OR_FREEBSD(func)            \
-      (::__interception::uptr)&(func),            \
+  ::__interception::InterceptFunction(                       \
-      (::__interception::uptr)&TRAMPOLINE(func))
+      #func, (::__interception::uptr *)&REAL(func),          \
+      (::__interception::uptr)(decltype(REAL(func)))&(func), \
+      (::__interception::uptr) &TRAMPOLINE(func))
 // dlvsym is a GNU extension supported by some other platforms.
 #if SANITIZER_GLIBC || SANITIZER_FREEBSD || SANITIZER_NETBSD
 @@ -41,7 +43,7 @@ bool InterceptFunction(const char *name, const char *ver, uptr *ptr_to_real,
   ::__interception::InterceptFunction(                        \
       #func, symver,                                          \
       (::__interception::uptr *)&REAL(func),                  \
-      (::__interception::uptr)&(func),                        \
+      (::__interception::uptr)(decltype(REAL(func)))&(func),  \
       (::__interception::uptr)&TRAMPOLINE(func))
 #else
 #define INTERCEPT_FUNCTION_VER_LINUX_OR_FREEBSD(func, symver) \

lib/tsan/interception/interception_win.cpp+56-27

...	@@ -1,4 +1,4 @@	...	@@ -1,4 +1,4 @@
1	//===-- interception_linux.cpp ----------------------------------- C++ --===//	1	//===-- interception_win.cpp ------------------------------------- C++ --===//
2	//	2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.	4	// See https://llvm.org/LICENSE.txt for license information.
...	@@ -339,7 +339,7 @@ struct TrampolineMemoryRegion {	...	@@ -339,7 +339,7 @@ struct TrampolineMemoryRegion {
339	uptr max_size;	339	uptr max_size;
340	};	340	};
341		341
342	UNUSED static const uptr kTrampolineScanLimitRange = 1 << 31; // 2 gig	342	UNUSED static const uptr kTrampolineScanLimitRange = 1ull << 31; // 2 gig
343	static const int kMaxTrampolineRegion = 1024;	343	static const int kMaxTrampolineRegion = 1024;
344	static TrampolineMemoryRegion TrampolineRegions[kMaxTrampolineRegion];	344	static TrampolineMemoryRegion TrampolineRegions[kMaxTrampolineRegion];
345		345
...	@@ -431,7 +431,8 @@ static uptr AllocateMemoryForTrampoline(uptr image_address, size_t size) {	...	@@ -431,7 +431,8 @@ static uptr AllocateMemoryForTrampoline(uptr image_address, size_t size) {
431	// The following prologues cannot be patched because of the short jump	431	// The following prologues cannot be patched because of the short jump
432	// jumping to the patching region.	432	// jumping to the patching region.
433		433
434	#if SANITIZER_WINDOWS64	434	// Short jump patterns below are only for x86_64.
		435	# if SANITIZER_WINDOWS_x64
435	// ntdll!wcslen in Win11	436	// ntdll!wcslen in Win11
436	// 488bc1 mov rax,rcx	437	// 488bc1 mov rax,rcx
437	// 0fb710 movzx edx,word ptr [rax]	438	// 0fb710 movzx edx,word ptr [rax]
...	@@ -457,7 +458,12 @@ static const u8 kPrologueWithShortJump2[] = {	...	@@ -457,7 +458,12 @@ static const u8 kPrologueWithShortJump2[] = {
457		458
458	// Returns 0 on error.	459	// Returns 0 on error.
459	static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {	460	static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
460	#if SANITIZER_WINDOWS64	461	#if SANITIZER_ARM64
		462	// An ARM64 instruction is 4 bytes long.
		463	return 4;
		464	#endif
		465
		466	# if SANITIZER_WINDOWS_x64
461	if (memcmp((u8*)address, kPrologueWithShortJump1,	467	if (memcmp((u8*)address, kPrologueWithShortJump1,
462	sizeof(kPrologueWithShortJump1)) == 0 \|\|	468	sizeof(kPrologueWithShortJump1)) == 0 \|\|
463	memcmp((u8*)address, kPrologueWithShortJump2,	469	memcmp((u8*)address, kPrologueWithShortJump2,
...	@@ -473,6 +479,8 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {	...	@@ -473,6 +479,8 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
473		479
474	switch ((u8)address) {	480	switch ((u8)address) {
475	case 0x90: // 90 : nop	481	case 0x90: // 90 : nop
		482	case 0xC3: // C3 : ret (for small/empty function interception
		483	case 0xCC: // CC : int 3 i.e. registering weak functions)
476	return 1;	484	return 1;
477		485
478	case 0x50: // push eax / rax	486	case 0x50: // push eax / rax
...	@@ -496,7 +504,6 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {	...	@@ -496,7 +504,6 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
496	// Cannot overwrite control-instruction. Return 0 to indicate failure.	504	// Cannot overwrite control-instruction. Return 0 to indicate failure.
497	case 0xE9: // E9 XX XX XX XX : jmp <label>	505	case 0xE9: // E9 XX XX XX XX : jmp <label>
498	case 0xE8: // E8 XX XX XX XX : call <func>	506	case 0xE8: // E8 XX XX XX XX : call <func>
499	case 0xC3: // C3 : ret
500	case 0xEB: // EB XX : jmp XX (short jump)	507	case 0xEB: // EB XX : jmp XX (short jump)
501	case 0x70: // 7Y YY : jy XX (short conditional jump)	508	case 0x70: // 7Y YY : jy XX (short conditional jump)
502	case 0x71:	509	case 0x71:
...	@@ -539,7 +546,12 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {	...	@@ -539,7 +546,12 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
539	return 7;	546	return 7;
540	}	547	}
541		548
542	#if SANITIZER_WINDOWS64	549	switch (0x000000FF & (u32 )address) {
		550	case 0xc2: // C2 XX XX : ret XX (needed for registering weak functions)
		551	return 3;
		552	}
		553
		554	# if SANITIZER_WINDOWS_x64
543	switch ((u8)address) {	555	switch ((u8)address) {
544	case 0xA1: // A1 XX XX XX XX XX XX XX XX :	556	case 0xA1: // A1 XX XX XX XX XX XX XX XX :
545	// movabs eax, dword ptr ds:[XXXXXXXX]	557	// movabs eax, dword ptr ds:[XXXXXXXX]
...	@@ -572,6 +584,7 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {	...	@@ -572,6 +584,7 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
572	case 0x018a: // mov al, byte ptr [rcx]	584	case 0x018a: // mov al, byte ptr [rcx]
573	return 2;	585	return 2;
574		586
		587	case 0x058A: // 8A 05 XX XX XX XX : mov al, byte ptr [XX XX XX XX]
575	case 0x058B: // 8B 05 XX XX XX XX : mov eax, dword ptr [XX XX XX XX]	588	case 0x058B: // 8B 05 XX XX XX XX : mov eax, dword ptr [XX XX XX XX]
576	if (rel_offset)	589	if (rel_offset)
577	*rel_offset = 2;	590	*rel_offset = 2;
...	@@ -598,6 +611,7 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {	...	@@ -598,6 +611,7 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
598	case 0xc18b4c: // 4C 8B C1 : mov r8, rcx	611	case 0xc18b4c: // 4C 8B C1 : mov r8, rcx
599	case 0xd2b60f: // 0f b6 d2 : movzx edx, dl	612	case 0xd2b60f: // 0f b6 d2 : movzx edx, dl
600	case 0xca2b48: // 48 2b ca : sub rcx, rdx	613	case 0xca2b48: // 48 2b ca : sub rcx, rdx
		614	case 0xca3b48: // 48 3b ca : cmp rcx, rdx
601	case 0x10b70f: // 0f b7 10 : movzx edx, WORD PTR [rax]	615	case 0x10b70f: // 0f b7 10 : movzx edx, WORD PTR [rax]
602	case 0xc00b4d: // 3d 0b c0 : or r8, r8	616	case 0xc00b4d: // 3d 0b c0 : or r8, r8
603	case 0xc08b41: // 41 8b c0 : mov eax, r8d	617	case 0xc08b41: // 41 8b c0 : mov eax, r8d
...	@@ -617,9 +631,11 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {	...	@@ -617,9 +631,11 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
617		631
618	case 0x058b48: // 48 8b 05 XX XX XX XX :	632	case 0x058b48: // 48 8b 05 XX XX XX XX :
619	// mov rax, QWORD PTR [rip + XXXXXXXX]	633	// mov rax, QWORD PTR [rip + XXXXXXXX]
		634	case 0x058d48: // 48 8d 05 XX XX XX XX :
		635	// lea rax, QWORD PTR [rip + XXXXXXXX]
620	case 0x25ff48: // 48 ff 25 XX XX XX XX :	636	case 0x25ff48: // 48 ff 25 XX XX XX XX :
621	// rex.W jmp QWORD PTR [rip + XXXXXXXX]	637	// rex.W jmp QWORD PTR [rip + XXXXXXXX]
622		638	case 0x158D4C: // 4c 8d 15 XX XX XX XX : lea r10, [rip + XX]
623	// Instructions having offset relative to 'rip' need offset adjustment.	639	// Instructions having offset relative to 'rip' need offset adjustment.
624	if (rel_offset)	640	if (rel_offset)
625	*rel_offset = 3;	641	*rel_offset = 3;
...	@@ -721,16 +737,22 @@ static bool CopyInstructions(uptr to, uptr from, size_t size) {	...	@@ -721,16 +737,22 @@ static bool CopyInstructions(uptr to, uptr from, size_t size) {
721	size_t instruction_size = GetInstructionSize(from + cursor, &rel_offset);	737	size_t instruction_size = GetInstructionSize(from + cursor, &rel_offset);
722	if (!instruction_size)	738	if (!instruction_size)
723	return false;	739	return false;
724	_memcpy((void)(to + cursor), (void)(from + cursor),	740	_memcpy((void )(to + cursor), (void )(from + cursor),
725	(size_t)instruction_size);	741	(size_t)instruction_size);
726	if (rel_offset) {	742	if (rel_offset) {
727	uptr delta = to - from;	743	# if SANITIZER_WINDOWS64
728	uptr relocated_offset = (u32)(to + cursor + rel_offset) - delta;	744	// we want to make sure that the new relative offset still fits in 32-bits
729	#if SANITIZER_WINDOWS64	745	// this will be untrue if relocated_offset \notin [-231, 231)
730	if (relocated_offset + 0x80000000U >= 0xFFFFFFFFU)	746	s64 delta = to - from;
		747	s64 relocated_offset = (s32 )(to + cursor + rel_offset) - delta;
		748	if (-0x8000'0000ll > relocated_offset \|\| relocated_offset > 0x7FFF'FFFFll)
731	return false;	749	return false;
732	#endif	750	# else
733	(u32)(to + cursor + rel_offset) = relocated_offset;	751	// on 32-bit, the relative offset will always be correct
		752	s32 delta = to - from;
		753	s32 relocated_offset = (s32 )(to + cursor + rel_offset) - delta;
		754	# endif
		755	(s32 )(to + cursor + rel_offset) = relocated_offset;
734	}	756	}
735	cursor += instruction_size;	757	cursor += instruction_size;
736	}	758	}
...	@@ -932,19 +954,26 @@ bool OverrideFunction(	...	@@ -932,19 +954,26 @@ bool OverrideFunction(
932		954
933	static void **InterestingDLLsAvailable() {	955	static void **InterestingDLLsAvailable() {
934	static const char *InterestingDLLs[] = {	956	static const char *InterestingDLLs[] = {
935	"kernel32.dll",	957	"kernel32.dll",
936	"msvcr100.dll", // VS2010	958	"msvcr100d.dll", // VS2010
937	"msvcr110.dll", // VS2012	959	"msvcr110d.dll", // VS2012
938	"msvcr120.dll", // VS2013	960	"msvcr120d.dll", // VS2013
939	"vcruntime140.dll", // VS2015	961	"vcruntime140d.dll", // VS2015
940	"ucrtbase.dll", // Universal CRT	962	"ucrtbased.dll", // Universal CRT
941	#if (defined(__MINGW32__) && defined(__i386__))	963	"msvcr100.dll", // VS2010
942	"libc++.dll", // libc++	964	"msvcr110.dll", // VS2012
943	"libunwind.dll", // libunwind	965	"msvcr120.dll", // VS2013
944	#endif	966	"vcruntime140.dll", // VS2015
945	// NTDLL should go last as it exports some functions that we should	967	"ucrtbase.dll", // Universal CRT
946	// override in the CRT [presumably only used internally].	968	# if (defined(__MINGW32__) && defined(__i386__))
947	"ntdll.dll", NULL};	969	"libc++.dll", // libc++
		970	"libunwind.dll", // libunwind
		971	# endif
		972	// NTDLL should go last as it exports some functions that we should
		973	// override in the CRT [presumably only used internally].
		974	"ntdll.dll",
		975	NULL
		976	};
948	static void *result[ARRAY_SIZE(InterestingDLLs)] = { 0 };	977	static void *result[ARRAY_SIZE(InterestingDLLs)] = { 0 };
949	if (!result[0]) {	978	if (!result[0]) {
950	for (size_t i = 0, j = 0; InterestingDLLs[i]; ++i) {	979	for (size_t i = 0, j = 0; InterestingDLLs[i]; ++i) {

lib/tsan/sanitizer_common/sanitizer_allocator.cpp+8-2

 @@ -25,7 +25,7 @@ namespace __sanitizer {
 const char *PrimaryAllocatorName = "SizeClassAllocator";
 const char *SecondaryAllocatorName = "LargeMmapAllocator";
-static ALIGNED(64) char internal_alloc_placeholder[sizeof(InternalAllocator)];
+alignas(64) static char internal_alloc_placeholder[sizeof(InternalAllocator)];
 static atomic_uint8_t internal_allocator_initialized;
 static StaticSpinMutex internal_alloc_init_mu;
 @@ -138,14 +138,20 @@ void InternalAllocatorUnlock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
 // LowLevelAllocator
 constexpr uptr kLowLevelAllocatorDefaultAlignment = 8;
+constexpr uptr kMinNumPagesRounded = 16;
+constexpr uptr kMinRoundedSize = 65536;
 static uptr low_level_alloc_min_alignment = kLowLevelAllocatorDefaultAlignment;
 static LowLevelAllocateCallback low_level_alloc_callback;
+static LowLevelAllocator Alloc;
+LowLevelAllocator &GetGlobalLowLevelAllocator() { return Alloc; }
 void *LowLevelAllocator::Allocate(uptr size) {
   // Align allocation size.
   size = RoundUpTo(size, low_level_alloc_min_alignment);
   if (allocated_end_ - allocated_current_ < (sptr)size) {
-    uptr size_to_allocate = RoundUpTo(size, GetPageSizeCached());
+    uptr size_to_allocate = RoundUpTo(
+        size, Min(GetPageSizeCached() * kMinNumPagesRounded, kMinRoundedSize));
     allocated_current_ = (char *)MmapOrDie(size_to_allocate, __func__);
     allocated_end_ = allocated_current_ + size_to_allocate;
     if (low_level_alloc_callback) {

lib/tsan/sanitizer_common/sanitizer_allocator_interface.h+2

...	@@ -40,6 +40,8 @@ SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE	...	@@ -40,6 +40,8 @@ SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
40	void __sanitizer_malloc_hook(void *ptr, uptr size);	40	void __sanitizer_malloc_hook(void *ptr, uptr size);
41	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE	41	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
42	void __sanitizer_free_hook(void *ptr);	42	void __sanitizer_free_hook(void *ptr);
		43	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE int
		44	__sanitizer_ignore_free_hook(void *ptr);
43		45
44	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void	46	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
45	__sanitizer_purge_allocator();	47	__sanitizer_purge_allocator();

lib/tsan/sanitizer_common/sanitizer_allocator_primary32.h+1-1

...	@@ -278,7 +278,7 @@ class SizeClassAllocator32 {	...	@@ -278,7 +278,7 @@ class SizeClassAllocator32 {
278	static const uptr kRegionSize = 1 << kRegionSizeLog;	278	static const uptr kRegionSize = 1 << kRegionSizeLog;
279	static const uptr kNumPossibleRegions = kSpaceSize / kRegionSize;	279	static const uptr kNumPossibleRegions = kSpaceSize / kRegionSize;
280		280
281	struct ALIGNED(SANITIZER_CACHE_LINE_SIZE) SizeClassInfo {	281	struct alignas(SANITIZER_CACHE_LINE_SIZE) SizeClassInfo {
282	StaticSpinMutex mutex;	282	StaticSpinMutex mutex;
283	IntrusiveList<TransferBatch> free_list;	283	IntrusiveList<TransferBatch> free_list;
284	u32 rand_state;	284	u32 rand_state;

lib/tsan/sanitizer_common/sanitizer_allocator_primary64.h+9-7

 @@ -316,13 +316,13 @@ class SizeClassAllocator64 {
     Printf(
         "%s %02zd (%6zd): mapped: %6zdK allocs: %7zd frees: %7zd inuse: %6zd "
         "num_freed_chunks %7zd avail: %6zd rss: %6zdK releases: %6zd "
-        "last released: %6lldK region: 0x%zx\n",
+        "last released: %6lldK region: %p\n",
         region->exhausted ? "F" : " ", class_id, ClassIdToSize(class_id),
         region->mapped_user >> 10, region->stats.n_allocated,
         region->stats.n_freed, in_use, region->num_freed_chunks, avail_chunks,
         rss >> 10, region->rtoi.num_releases,
         region->rtoi.last_released_bytes >> 10,
-        SpaceBeg() + kRegionSize * class_id);
+        (void *)(SpaceBeg() + kRegionSize * class_id));
   }
   void PrintStats() {
 @@ -636,15 +636,17 @@ class SizeClassAllocator64 {
   }
   uptr SpaceEnd() const { return  SpaceBeg() + kSpaceSize; }
   // kRegionSize should be able to satisfy the largest size class.
-  static_assert(kRegionSize >= SizeClassMap::kMaxSize);
+  static_assert(kRegionSize >= SizeClassMap::kMaxSize,
+                "Region size exceed largest size");
   // kRegionSize must be <= 2^36, see CompactPtrT.
-  COMPILER_CHECK((kRegionSize) <= (1ULL << (SANITIZER_WORDSIZE / 2 + 4)));
+  COMPILER_CHECK((kRegionSize) <=
+                 (1ULL << (sizeof(CompactPtrT) * 8 + kCompactPtrScale)));
   // Call mmap for user memory with at least this size.
-  static const uptr kUserMapSize = 1 << 16;
+  static const uptr kUserMapSize = 1 << 18;
   // Call mmap for metadata memory with at least this size.
   static const uptr kMetaMapSize = 1 << 16;
   // Call mmap for free array memory with at least this size.
-  static const uptr kFreeArrayMapSize = 1 << 16;
+  static const uptr kFreeArrayMapSize = 1 << 18;
   atomic_sint32_t release_to_os_interval_ms_;
 @@ -665,7 +667,7 @@ class SizeClassAllocator64 {
     u64 last_released_bytes;
   };
-  struct ALIGNED(SANITIZER_CACHE_LINE_SIZE) RegionInfo {
+  struct alignas(SANITIZER_CACHE_LINE_SIZE) RegionInfo {
     Mutex mutex;
     uptr num_freed_chunks;  // Number of elements in the freearray.
     uptr mapped_free_array;  // Bytes mapped for freearray.

lib/tsan/sanitizer_common/sanitizer_asm.h+40-3

...	@@ -42,6 +42,16 @@	...	@@ -42,6 +42,16 @@
42	# define CFI_RESTORE(reg)	42	# define CFI_RESTORE(reg)
43	#endif	43	#endif
44		44
		45	#if defined(__aarch64__) && defined(__ARM_FEATURE_BTI_DEFAULT)
		46	# define ASM_STARTPROC CFI_STARTPROC; hint #34
		47	# define C_ASM_STARTPROC SANITIZER_STRINGIFY(CFI_STARTPROC) "\nhint #34"
		48	#else
		49	# define ASM_STARTPROC CFI_STARTPROC
		50	# define C_ASM_STARTPROC SANITIZER_STRINGIFY(CFI_STARTPROC)
		51	#endif
		52	#define ASM_ENDPROC CFI_ENDPROC
		53	#define C_ASM_ENDPROC SANITIZER_STRINGIFY(CFI_ENDPROC)
		54
45	#if defined(__x86_64__) \|\| defined(__i386__) \|\| defined(__sparc__)	55	#if defined(__x86_64__) \|\| defined(__i386__) \|\| defined(__sparc__)
46	# define ASM_TAIL_CALL jmp	56	# define ASM_TAIL_CALL jmp
47	#elif defined(__arm__) \|\| defined(__aarch64__) \|\| defined(__mips__) \|\| \	57	#elif defined(__arm__) \|\| defined(__aarch64__) \|\| defined(__mips__) \|\| \
...	@@ -53,6 +63,29 @@	...	@@ -53,6 +63,29 @@
53	# define ASM_TAIL_CALL tail	63	# define ASM_TAIL_CALL tail
54	#endif	64	#endif
55		65
		66	// Currently, almost all of the shared libraries rely on the value of
		67	// $t9 to get the address of current function, instead of PCREL, even
		68	// on MIPSr6. To be compatiable with them, we have to set $t9 properly.
		69	// MIPS uses GOT to get the address of preemptible functions.
		70	#if defined(__mips64)
		71	# define C_ASM_TAIL_CALL(t_func, i_func) \
		72	"lui $t8, %hi(%neg(%gp_rel(" t_func ")))\n" \
		73	"daddu $t8, $t8, $t9\n" \
		74	"daddiu $t8, $t8, %lo(%neg(%gp_rel(" t_func ")))\n" \
		75	"ld $t9, %got_disp(" i_func ")($t8)\n" \
		76	"jr $t9\n"
		77	#elif defined(__mips__)
		78	# define C_ASM_TAIL_CALL(t_func, i_func) \
		79	".set noreorder\n" \
		80	".cpload $t9\n" \
		81	".set reorder\n" \
		82	"lw $t9, %got(" i_func ")($gp)\n" \
		83	"jr $t9\n"
		84	#elif defined(ASM_TAIL_CALL)
		85	# define C_ASM_TAIL_CALL(t_func, i_func) \
		86	SANITIZER_STRINGIFY(ASM_TAIL_CALL) " " i_func
		87	#endif
		88
56	#if defined(__ELF__) && defined(__x86_64__) \|\| defined(__i386__) \|\| \	89	#if defined(__ELF__) && defined(__x86_64__) \|\| defined(__i386__) \|\| \
57	defined(__riscv)	90	defined(__riscv)
58	# define ASM_PREEMPTIBLE_SYM(sym) sym@plt	91	# define ASM_PREEMPTIBLE_SYM(sym) sym@plt
...	@@ -62,7 +95,11 @@	...	@@ -62,7 +95,11 @@
62		95
63	#if !defined(__APPLE__)	96	#if !defined(__APPLE__)
64	# define ASM_HIDDEN(symbol) .hidden symbol	97	# define ASM_HIDDEN(symbol) .hidden symbol
65	# define ASM_TYPE_FUNCTION(symbol) .type symbol, %function	98	# if defined(__arm__) \|\| defined(__aarch64__)
		99	# define ASM_TYPE_FUNCTION(symbol) .type symbol, %function
		100	# else
		101	# define ASM_TYPE_FUNCTION(symbol) .type symbol, @function
		102	# endif
66	# define ASM_SIZE(symbol) .size symbol, .-symbol	103	# define ASM_SIZE(symbol) .size symbol, .-symbol
67	# define ASM_SYMBOL(symbol) symbol	104	# define ASM_SYMBOL(symbol) symbol
68	# define ASM_SYMBOL_INTERCEPTOR(symbol) symbol	105	# define ASM_SYMBOL_INTERCEPTOR(symbol) symbol
...	@@ -87,9 +124,9 @@	...	@@ -87,9 +124,9 @@
87	.globl __interceptor_trampoline_##name; \	124	.globl __interceptor_trampoline_##name; \
88	ASM_TYPE_FUNCTION(__interceptor_trampoline_##name); \	125	ASM_TYPE_FUNCTION(__interceptor_trampoline_##name); \
89	__interceptor_trampoline_##name: \	126	__interceptor_trampoline_##name: \
90	CFI_STARTPROC; \	127	ASM_STARTPROC; \
91	ASM_TAIL_CALL ASM_PREEMPTIBLE_SYM(__interceptor_##name); \	128	ASM_TAIL_CALL ASM_PREEMPTIBLE_SYM(__interceptor_##name); \
92	CFI_ENDPROC; \	129	ASM_ENDPROC; \
93	ASM_SIZE(__interceptor_trampoline_##name)	130	ASM_SIZE(__interceptor_trampoline_##name)
94	# define ASM_INTERCEPTOR_TRAMPOLINE_SUPPORT 1	131	# define ASM_INTERCEPTOR_TRAMPOLINE_SUPPORT 1
95	# endif // Architecture supports interceptor trampoline	132	# endif // Architecture supports interceptor trampoline

lib/tsan/sanitizer_common/sanitizer_atomic.h+13-1

 @@ -18,12 +18,24 @@
 namespace __sanitizer {
 enum memory_order {
+// If the __atomic atomic builtins are supported (Clang/GCC), use the
+// compiler provided macro values so that we can map the atomic operations
+// to __atomic_* directly.
+#ifdef __ATOMIC_SEQ_CST
+  memory_order_relaxed = __ATOMIC_RELAXED,
+  memory_order_consume = __ATOMIC_CONSUME,
+  memory_order_acquire = __ATOMIC_ACQUIRE,
+  memory_order_release = __ATOMIC_RELEASE,
+  memory_order_acq_rel = __ATOMIC_ACQ_REL,
+  memory_order_seq_cst = __ATOMIC_SEQ_CST
+#else
   memory_order_relaxed = 1 << 0,
   memory_order_consume = 1 << 1,
   memory_order_acquire = 1 << 2,
   memory_order_release = 1 << 3,
   memory_order_acq_rel = 1 << 4,
   memory_order_seq_cst = 1 << 5
+#endif
 };
 struct atomic_uint8_t {
 @@ -49,7 +61,7 @@ struct atomic_uint32_t {
 struct atomic_uint64_t {
   typedef u64 Type;
   // On 32-bit platforms u64 is not necessary aligned on 8 bytes.
-  volatile ALIGNED(8) Type val_dont_use;
+  alignas(8) volatile Type val_dont_use;
 };
 struct atomic_uintptr_t {

lib/tsan/sanitizer_common/sanitizer_atomic_clang.h+40-45

 @@ -14,60 +14,63 @@
 #ifndef SANITIZER_ATOMIC_CLANG_H
 #define SANITIZER_ATOMIC_CLANG_H
-#if defined(__i386__) || defined(__x86_64__)
-# include "sanitizer_atomic_clang_x86.h"
-#else
-# include "sanitizer_atomic_clang_other.h"
-#endif
 namespace __sanitizer {
-// We would like to just use compiler builtin atomic operations
+// We use the compiler builtin atomic operations for loads and stores, which
-// for loads and stores, but they are mostly broken in clang:
+// generates correct code for all architectures, but may require libatomic
-// - they lead to vastly inefficient code generation
+// on platforms where e.g. 64-bit atomics are not supported natively.
-// (http://llvm.org/bugs/show_bug.cgi?id=17281)
-// - 64-bit atomic operations are not implemented on x86_32
-// (http://llvm.org/bugs/show_bug.cgi?id=15034)
-// - they are not implemented on ARM
-// error: undefined reference to '__atomic_load_4'
 // See http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html
 // for mappings of the memory model to different processors.
-inline void atomic_signal_fence(memory_order) {
+inline void atomic_signal_fence(memory_order mo) { __atomic_signal_fence(mo); }
+inline void atomic_thread_fence(memory_order mo) { __atomic_thread_fence(mo); }
+inline void proc_yield(int cnt) {
+  __asm__ __volatile__("" ::: "memory");
+#if defined(__i386__) || defined(__x86_64__)
+  for (int i = 0; i < cnt; i++) __asm__ __volatile__("pause");
   __asm__ __volatile__("" ::: "memory");
+#endif
 }
-inline void atomic_thread_fence(memory_order) {
+template <typename T>
-  __sync_synchronize();
+inline typename T::Type atomic_load(const volatile T *a, memory_order mo) {
+  DCHECK(mo == memory_order_relaxed || mo == memory_order_consume ||
+         mo == memory_order_acquire || mo == memory_order_seq_cst);
+  DCHECK(!((uptr)a % sizeof(*a)));
+  return __atomic_load_n(&a->val_dont_use, mo);
 }
-template<typename T>
+template <typename T>
-inline typename T::Type atomic_fetch_add(volatile T *a,
+inline void atomic_store(volatile T *a, typename T::Type v, memory_order mo) {
-    typename T::Type v, memory_order mo) {
+  DCHECK(mo == memory_order_relaxed || mo == memory_order_release ||
-  (void)mo;
+         mo == memory_order_seq_cst);
   DCHECK(!((uptr)a % sizeof(*a)));
-  return __sync_fetch_and_add(&a->val_dont_use, v);
+  __atomic_store_n(&a->val_dont_use, v, mo);
 }
-template<typename T>
+template <typename T>
-inline typename T::Type atomic_fetch_sub(volatile T *a,
+inline typename T::Type atomic_fetch_add(volatile T *a, typename T::Type v,
-    typename T::Type v, memory_order mo) {
+                                         memory_order mo) {
+  DCHECK(!((uptr)a % sizeof(*a)));
+  return __atomic_fetch_add(&a->val_dont_use, v, mo);
+}
+template <typename T>
+inline typename T::Type atomic_fetch_sub(volatile T *a, typename T::Type v,
+                                         memory_order mo) {
   (void)mo;
   DCHECK(!((uptr)a % sizeof(*a)));
-  return __sync_fetch_and_add(&a->val_dont_use, -v);
+  return __atomic_fetch_sub(&a->val_dont_use, v, mo);
 }
-template<typename T>
+template <typename T>
-inline typename T::Type atomic_exchange(volatile T *a,
+inline typename T::Type atomic_exchange(volatile T *a, typename T::Type v,
-    typename T::Type v, memory_order mo) {
+                                        memory_order mo) {
   DCHECK(!((uptr)a % sizeof(*a)));
-  if (mo & (memory_order_release | memory_order_acq_rel | memory_order_seq_cst))
+  return __atomic_exchange_n(&a->val_dont_use, v, mo);
-    __sync_synchronize();
-  v = __sync_lock_test_and_set(&a->val_dont_use, v);
-  if (mo == memory_order_seq_cst)
-    __sync_synchronize();
-  return v;
 }
 template <typename T>
 @@ -82,9 +85,8 @@ inline bool atomic_compare_exchange_strong(volatile T *a, typename T::Type *cmp,
                                    __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
 }
-template<typename T>
+template <typename T>
-inline bool atomic_compare_exchange_weak(volatile T *a,
+inline bool atomic_compare_exchange_weak(volatile T *a, typename T::Type *cmp,
-                                         typename T::Type *cmp,
                                          typename T::Type xchg,
                                          memory_order mo) {
   return atomic_compare_exchange_strong(a, cmp, xchg, mo);
 @@ -92,13 +94,6 @@ inline bool atomic_compare_exchange_weak(volatile T *a,
 }  // namespace __sanitizer
-// This include provides explicit template instantiations for atomic_uint64_t
-// on MIPS32, which does not directly support 8 byte atomics. It has to
-// proceed the template definitions above.
-#if defined(_MIPS_SIM) && defined(_ABIO32) && _MIPS_SIM == _ABIO32
-#  include "sanitizer_atomic_clang_mips.h"
-#endif
 #undef ATOMIC_ORDER
 #endif  // SANITIZER_ATOMIC_CLANG_H

lib/tsan/sanitizer_common/sanitizer_atomic_clang_mips.h deleted-117

...	@@ -1,117 +0,0 @@
1	//===-- sanitizer_atomic_clang_mips.h ---------------------------- C++ --===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is a part of ThreadSanitizer/AddressSanitizer runtime.
10	// Not intended for direct inclusion. Include sanitizer_atomic.h.
11	//
12	//===----------------------------------------------------------------------===//
13
14	#ifndef SANITIZER_ATOMIC_CLANG_MIPS_H
15	#define SANITIZER_ATOMIC_CLANG_MIPS_H
16
17	namespace __sanitizer {
18
19	// MIPS32 does not support atomics > 4 bytes. To address this lack of
20	// functionality, the sanitizer library provides helper methods which use an
21	// internal spin lock mechanism to emulate atomic operations when the size is
22	// 8 bytes.
23	static void __spin_lock(volatile int *lock) {
24	while (__sync_lock_test_and_set(lock, 1))
25	while (*lock) {
26	}
27	}
28
29	static void __spin_unlock(volatile int *lock) { __sync_lock_release(lock); }
30
31	// Make sure the lock is on its own cache line to prevent false sharing.
32	// Put it inside a struct that is aligned and padded to the typical MIPS
33	// cacheline which is 32 bytes.
34	static struct {
35	int lock;
36	char pad[32 - sizeof(int)];
37	} __attribute__((aligned(32))) lock = {0, {0}};
38
39	template <>
40	inline atomic_uint64_t::Type atomic_fetch_add(volatile atomic_uint64_t *ptr,
41	atomic_uint64_t::Type val,
42	memory_order mo) {
43	DCHECK(mo &
44	(memory_order_relaxed \| memory_order_release \| memory_order_seq_cst));
45	DCHECK(!((uptr)ptr % sizeof(*ptr)));
46
47	atomic_uint64_t::Type ret;
48
49	__spin_lock(&lock.lock);
50	ret = (const_cast<atomic_uint64_t::Type volatile >(&ptr->val_dont_use));
51	ptr->val_dont_use = ret + val;
52	__spin_unlock(&lock.lock);
53
54	return ret;
55	}
56
57	template <>
58	inline atomic_uint64_t::Type atomic_fetch_sub(volatile atomic_uint64_t *ptr,
59	atomic_uint64_t::Type val,
60	memory_order mo) {
61	return atomic_fetch_add(ptr, -val, mo);
62	}
63
64	template <>
65	inline bool atomic_compare_exchange_strong(volatile atomic_uint64_t *ptr,
66	atomic_uint64_t::Type *cmp,
67	atomic_uint64_t::Type xchg,
68	memory_order mo) {
69	DCHECK(mo &
70	(memory_order_relaxed \| memory_order_release \| memory_order_seq_cst));
71	DCHECK(!((uptr)ptr % sizeof(*ptr)));
72
73	typedef atomic_uint64_t::Type Type;
74	Type cmpv = *cmp;
75	Type prev;
76	bool ret = false;
77
78	__spin_lock(&lock.lock);
79	prev = (const_cast<Type volatile >(&ptr->val_dont_use));
80	if (prev == cmpv) {
81	ret = true;
82	ptr->val_dont_use = xchg;
83	}
84	__spin_unlock(&lock.lock);
85
86	return ret;
87	}
88
89	template <>
90	inline atomic_uint64_t::Type atomic_load(const volatile atomic_uint64_t *ptr,
91	memory_order mo) {
92	DCHECK(mo &
93	(memory_order_relaxed \| memory_order_release \| memory_order_seq_cst));
94	DCHECK(!((uptr)ptr % sizeof(*ptr)));
95
96	atomic_uint64_t::Type zero = 0;
97	volatile atomic_uint64_t *Newptr =
98	const_cast<volatile atomic_uint64_t *>(ptr);
99	return atomic_fetch_add(Newptr, zero, mo);
100	}
101
102	template <>
103	inline void atomic_store(volatile atomic_uint64_t *ptr, atomic_uint64_t::Type v,
104	memory_order mo) {
105	DCHECK(mo &
106	(memory_order_relaxed \| memory_order_release \| memory_order_seq_cst));
107	DCHECK(!((uptr)ptr % sizeof(*ptr)));
108
109	__spin_lock(&lock.lock);
110	ptr->val_dont_use = v;
111	__spin_unlock(&lock.lock);
112	}
113
114	} // namespace __sanitizer
115
116	#endif // SANITIZER_ATOMIC_CLANG_MIPS_H
117

lib/tsan/sanitizer_common/sanitizer_atomic_clang_other.h deleted-85

...	@@ -1,85 +0,0 @@
1	//===-- sanitizer_atomic_clang_other.h --------------------------- C++ --===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is a part of ThreadSanitizer/AddressSanitizer runtime.
10	// Not intended for direct inclusion. Include sanitizer_atomic.h.
11	//
12	//===----------------------------------------------------------------------===//
13
14	#ifndef SANITIZER_ATOMIC_CLANG_OTHER_H
15	#define SANITIZER_ATOMIC_CLANG_OTHER_H
16
17	namespace __sanitizer {
18
19
20	inline void proc_yield(int cnt) {
21	__asm__ __volatile__("" ::: "memory");
22	}
23
24	template<typename T>
25	inline typename T::Type atomic_load(
26	const volatile T *a, memory_order mo) {
27	DCHECK(mo & (memory_order_relaxed \| memory_order_consume
28	\| memory_order_acquire \| memory_order_seq_cst));
29	DCHECK(!((uptr)a % sizeof(*a)));
30	typename T::Type v;
31
32	if (sizeof(a) < 8 \|\| sizeof(void) == 8) {
33	// Assume that aligned loads are atomic.
34	if (mo == memory_order_relaxed) {
35	v = a->val_dont_use;
36	} else if (mo == memory_order_consume) {
37	// Assume that processor respects data dependencies
38	// (and that compiler won't break them).
39	__asm__ __volatile__("" ::: "memory");
40	v = a->val_dont_use;
41	__asm__ __volatile__("" ::: "memory");
42	} else if (mo == memory_order_acquire) {
43	__asm__ __volatile__("" ::: "memory");
44	v = a->val_dont_use;
45	__sync_synchronize();
46	} else { // seq_cst
47	// E.g. on POWER we need a hw fence even before the store.
48	__sync_synchronize();
49	v = a->val_dont_use;
50	__sync_synchronize();
51	}
52	} else {
53	__atomic_load(const_cast<typename T::Type volatile *>(&a->val_dont_use), &v,
54	__ATOMIC_SEQ_CST);
55	}
56	return v;
57	}
58
59	template<typename T>
60	inline void atomic_store(volatile T *a, typename T::Type v, memory_order mo) {
61	DCHECK(mo & (memory_order_relaxed \| memory_order_release
62	\| memory_order_seq_cst));
63	DCHECK(!((uptr)a % sizeof(*a)));
64
65	if (sizeof(a) < 8 \|\| sizeof(void) == 8) {
66	// Assume that aligned loads are atomic.
67	if (mo == memory_order_relaxed) {
68	a->val_dont_use = v;
69	} else if (mo == memory_order_release) {
70	__sync_synchronize();
71	a->val_dont_use = v;
72	__asm__ __volatile__("" ::: "memory");
73	} else { // seq_cst
74	__sync_synchronize();
75	a->val_dont_use = v;
76	__sync_synchronize();
77	}
78	} else {
79	__atomic_store(&a->val_dont_use, &v, __ATOMIC_SEQ_CST);
80	}
81	}
82
83	} // namespace __sanitizer
84
85	#endif // #ifndef SANITIZER_ATOMIC_CLANG_OTHER_H

lib/tsan/sanitizer_common/sanitizer_atomic_clang_x86.h deleted-113

...	@@ -1,113 +0,0 @@
1	//===-- sanitizer_atomic_clang_x86.h ----------------------------- C++ --===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is a part of ThreadSanitizer/AddressSanitizer runtime.
10	// Not intended for direct inclusion. Include sanitizer_atomic.h.
11	//
12	//===----------------------------------------------------------------------===//
13
14	#ifndef SANITIZER_ATOMIC_CLANG_X86_H
15	#define SANITIZER_ATOMIC_CLANG_X86_H
16
17	namespace __sanitizer {
18
19	inline void proc_yield(int cnt) {
20	__asm__ __volatile__("" ::: "memory");
21	for (int i = 0; i < cnt; i++)
22	__asm__ __volatile__("pause");
23	__asm__ __volatile__("" ::: "memory");
24	}
25
26	template<typename T>
27	inline typename T::Type atomic_load(
28	const volatile T *a, memory_order mo) {
29	DCHECK(mo & (memory_order_relaxed \| memory_order_consume
30	\| memory_order_acquire \| memory_order_seq_cst));
31	DCHECK(!((uptr)a % sizeof(*a)));
32	typename T::Type v;
33
34	if (sizeof(a) < 8 \|\| sizeof(void) == 8) {
35	// Assume that aligned loads are atomic.
36	if (mo == memory_order_relaxed) {
37	v = a->val_dont_use;
38	} else if (mo == memory_order_consume) {
39	// Assume that processor respects data dependencies
40	// (and that compiler won't break them).
41	__asm__ __volatile__("" ::: "memory");
42	v = a->val_dont_use;
43	__asm__ __volatile__("" ::: "memory");
44	} else if (mo == memory_order_acquire) {
45	__asm__ __volatile__("" ::: "memory");
46	v = a->val_dont_use;
47	// On x86 loads are implicitly acquire.
48	__asm__ __volatile__("" ::: "memory");
49	} else { // seq_cst
50	// On x86 plain MOV is enough for seq_cst store.
51	__asm__ __volatile__("" ::: "memory");
52	v = a->val_dont_use;
53	__asm__ __volatile__("" ::: "memory");
54	}
55	} else {
56	// 64-bit load on 32-bit platform.
57	__asm__ __volatile__(
58	"movq %1, %%mm0;" // Use mmx reg for 64-bit atomic moves
59	"movq %%mm0, %0;" // (ptr could be read-only)
60	"emms;" // Empty mmx state/Reset FP regs
61	: "=m" (v)
62	: "m" (a->val_dont_use)
63	: // mark the mmx registers as clobbered
64	#ifdef __MMX__
65	"mm0", "mm1", "mm2", "mm3", "mm4", "mm5", "mm6", "mm7",
66	#endif // #ifdef __MMX__
67	"memory");
68	}
69	return v;
70	}
71
72	template<typename T>
73	inline void atomic_store(volatile T *a, typename T::Type v, memory_order mo) {
74	DCHECK(mo & (memory_order_relaxed \| memory_order_release
75	\| memory_order_seq_cst));
76	DCHECK(!((uptr)a % sizeof(*a)));
77
78	if (sizeof(a) < 8 \|\| sizeof(void) == 8) {
79	// Assume that aligned loads are atomic.
80	if (mo == memory_order_relaxed) {
81	a->val_dont_use = v;
82	} else if (mo == memory_order_release) {
83	// On x86 stores are implicitly release.
84	__asm__ __volatile__("" ::: "memory");
85	a->val_dont_use = v;
86	__asm__ __volatile__("" ::: "memory");
87	} else { // seq_cst
88	// On x86 stores are implicitly release.
89	__asm__ __volatile__("" ::: "memory");
90	a->val_dont_use = v;
91	__sync_synchronize();
92	}
93	} else {
94	// 64-bit store on 32-bit platform.
95	__asm__ __volatile__(
96	"movq %1, %%mm0;" // Use mmx reg for 64-bit atomic moves
97	"movq %%mm0, %0;"
98	"emms;" // Empty mmx state/Reset FP regs
99	: "=m" (a->val_dont_use)
100	: "m" (v)
101	: // mark the mmx registers as clobbered
102	#ifdef __MMX__
103	"mm0", "mm1", "mm2", "mm3", "mm4", "mm5", "mm6", "mm7",
104	#endif // #ifdef __MMX__
105	"memory");
106	if (mo == memory_order_seq_cst)
107	__sync_synchronize();
108	}
109	}
110
111	} // namespace __sanitizer
112
113	#endif // #ifndef SANITIZER_ATOMIC_CLANG_X86_H

lib/tsan/sanitizer_common/sanitizer_atomic_msvc.h+4-4

...	@@ -70,8 +70,8 @@ inline void proc_yield(int cnt) {	...	@@ -70,8 +70,8 @@ inline void proc_yield(int cnt) {
70	template<typename T>	70	template<typename T>
71	inline typename T::Type atomic_load(	71	inline typename T::Type atomic_load(
72	const volatile T *a, memory_order mo) {	72	const volatile T *a, memory_order mo) {
73	DCHECK(mo & (memory_order_relaxed \| memory_order_consume	73	DCHECK(mo == memory_order_relaxed \|\| mo == memory_order_consume \|\|
74	\| memory_order_acquire \| memory_order_seq_cst));	74	mo == memory_order_acquire \|\| mo == memory_order_seq_cst);
75	DCHECK(!((uptr)a % sizeof(*a)));	75	DCHECK(!((uptr)a % sizeof(*a)));
76	typename T::Type v;	76	typename T::Type v;
77	// FIXME(dvyukov): 64-bit load is not atomic on 32-bits.	77	// FIXME(dvyukov): 64-bit load is not atomic on 32-bits.
...	@@ -87,8 +87,8 @@ inline typename T::Type atomic_load(	...	@@ -87,8 +87,8 @@ inline typename T::Type atomic_load(
87		87
88	template<typename T>	88	template<typename T>
89	inline void atomic_store(volatile T *a, typename T::Type v, memory_order mo) {	89	inline void atomic_store(volatile T *a, typename T::Type v, memory_order mo) {
90	DCHECK(mo & (memory_order_relaxed \| memory_order_release	90	DCHECK(mo == memory_order_relaxed \|\| mo == memory_order_release \|\|
91	\| memory_order_seq_cst));	91	mo == memory_order_seq_cst);
92	DCHECK(!((uptr)a % sizeof(*a)));	92	DCHECK(!((uptr)a % sizeof(*a)));
93	// FIXME(dvyukov): 64-bit store is not atomic on 32-bits.	93	// FIXME(dvyukov): 64-bit store is not atomic on 32-bits.
94	if (mo == memory_order_relaxed) {	94	if (mo == memory_order_relaxed) {

lib/tsan/sanitizer_common/sanitizer_bitvector.h+4-4

...	@@ -321,23 +321,23 @@ class TwoLevelBitVector {	...	@@ -321,23 +321,23 @@ class TwoLevelBitVector {
321	};	321	};
322		322
323	private:	323	private:
324	void check(uptr idx) const { CHECK_LE(idx, size()); }	324	void check(uptr idx) const { CHECK_LT(idx, size()); }
325		325
326	uptr idx0(uptr idx) const {	326	uptr idx0(uptr idx) const {
327	uptr res = idx / (BV::kSize * BV::kSize);	327	uptr res = idx / (BV::kSize * BV::kSize);
328	CHECK_LE(res, kLevel1Size);	328	CHECK_LT(res, kLevel1Size);
329	return res;	329	return res;
330	}	330	}
331		331
332	uptr idx1(uptr idx) const {	332	uptr idx1(uptr idx) const {
333	uptr res = (idx / BV::kSize) % BV::kSize;	333	uptr res = (idx / BV::kSize) % BV::kSize;
334	CHECK_LE(res, BV::kSize);	334	CHECK_LT(res, BV::kSize);
335	return res;	335	return res;
336	}	336	}
337		337
338	uptr idx2(uptr idx) const {	338	uptr idx2(uptr idx) const {
339	uptr res = idx % BV::kSize;	339	uptr res = idx % BV::kSize;
340	CHECK_LE(res, BV::kSize);	340	CHECK_LT(res, BV::kSize);
341	return res;	341	return res;
342	}	342	}
343		343

lib/tsan/sanitizer_common/sanitizer_chained_origin_depot.cpp+4-2

 @@ -139,9 +139,11 @@ u32 ChainedOriginDepot::Get(u32 id, u32 *other) {
   return desc.here_id;
 }
-void ChainedOriginDepot::LockAll() { depot.LockAll(); }
+void ChainedOriginDepot::LockBeforeFork() { depot.LockBeforeFork(); }
-void ChainedOriginDepot::UnlockAll() { depot.UnlockAll(); }
+void ChainedOriginDepot::UnlockAfterFork(bool fork_child) {
+  depot.UnlockAfterFork(fork_child);
+}
 void ChainedOriginDepot::TestOnlyUnmap() { depot.TestOnlyUnmap(); }

lib/tsan/sanitizer_common/sanitizer_chained_origin_depot.h+2-2

...	@@ -32,8 +32,8 @@ class ChainedOriginDepot {	...	@@ -32,8 +32,8 @@ class ChainedOriginDepot {
32	// Retrieves the stored StackDepot ID for the given origin ID.	32	// Retrieves the stored StackDepot ID for the given origin ID.
33	u32 Get(u32 id, u32 *other);	33	u32 Get(u32 id, u32 *other);
34		34
35	void LockAll();	35	void LockBeforeFork();
36	void UnlockAll();	36	void UnlockAfterFork(bool fork_child);
37	void TestOnlyUnmap();	37	void TestOnlyUnmap();
38		38
39	private:	39	private:

lib/tsan/sanitizer_common/sanitizer_common.cpp+17-3

 @@ -115,8 +115,9 @@ void ReportErrorSummary(const char *error_message, const char *alt_tool_name) {
   if (!common_flags()->print_summary)
     return;
   InternalScopedString buff;
-  buff.append("SUMMARY: %s: %s",
+  buff.AppendF("SUMMARY: %s: %s",
-              alt_tool_name ? alt_tool_name : SanitizerToolName, error_message);
+               alt_tool_name ? alt_tool_name : SanitizerToolName,
+               error_message);
   __sanitizer_report_error_summary(buff.data());
 }
 @@ -346,7 +347,13 @@ void RunMallocHooks(void *ptr, uptr size) {
   }
 }
-void RunFreeHooks(void *ptr) {
+// Returns '1' if the call to free() should be ignored (based on
+// __sanitizer_ignore_free_hook), or '0' otherwise.
+int RunFreeHooks(void *ptr) {
+  if (__sanitizer_ignore_free_hook(ptr)) {
+    return 1;
+  }
   __sanitizer_free_hook(ptr);
   for (int i = 0; i < kMaxMallocFreeHooks; i++) {
     auto hook = MFHooks[i].free_hook;
 @@ -354,6 +361,8 @@ void RunFreeHooks(void *ptr) {
       break;
     hook(ptr);
   }
+  return 0;
 }
 static int InstallMallocFreeHooks(void (*malloc_hook)(const void *, uptr),
 @@ -418,4 +427,9 @@ SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_free_hook, void *ptr) {
   (void)ptr;
 }
+SANITIZER_INTERFACE_WEAK_DEF(int, __sanitizer_ignore_free_hook, void *ptr) {
+  (void)ptr;
+  return 0;
+}
 } // extern "C"

lib/tsan/sanitizer_common/sanitizer_common.h+28-17

...	@@ -32,6 +32,7 @@ struct AddressInfo;	...	@@ -32,6 +32,7 @@ struct AddressInfo;
32	struct BufferedStackTrace;	32	struct BufferedStackTrace;
33	struct SignalContext;	33	struct SignalContext;
34	struct StackTrace;	34	struct StackTrace;
		35	struct SymbolizedStack;
35		36
36	// Constants.	37	// Constants.
37	const uptr kWordSize = SANITIZER_WORDSIZE / 8;	38	const uptr kWordSize = SANITIZER_WORDSIZE / 8;
...	@@ -59,14 +60,10 @@ inline int Verbosity() {	...	@@ -59,14 +60,10 @@ inline int Verbosity() {
59	return atomic_load(&current_verbosity, memory_order_relaxed);	60	return atomic_load(&current_verbosity, memory_order_relaxed);
60	}	61	}
61		62
62	#if SANITIZER_ANDROID	63	#if SANITIZER_ANDROID && !defined(__aarch64__)
63	inline uptr GetPageSize() {	64	// 32-bit Android only has 4k pages.
64	// Android post-M sysconf(_SC_PAGESIZE) crashes if called from .preinit_array.	65	inline uptr GetPageSize() { return 4096; }
65	return 4096;	66	inline uptr GetPageSizeCached() { return 4096; }
66	}
67	inline uptr GetPageSizeCached() {
68	return 4096;
69	}
70	#else	67	#else
71	uptr GetPageSize();	68	uptr GetPageSize();
72	extern uptr PageSizeCached;	69	extern uptr PageSizeCached;
...	@@ -76,6 +73,7 @@ inline uptr GetPageSizeCached() {	...	@@ -76,6 +73,7 @@ inline uptr GetPageSizeCached() {
76	return PageSizeCached;	73	return PageSizeCached;
77	}	74	}
78	#endif	75	#endif
		76
79	uptr GetMmapGranularity();	77	uptr GetMmapGranularity();
80	uptr GetMaxVirtualAddress();	78	uptr GetMaxVirtualAddress();
81	uptr GetMaxUserVirtualAddress();	79	uptr GetMaxUserVirtualAddress();
...	@@ -90,10 +88,11 @@ void GetThreadStackAndTls(bool main, uptr stk_addr, uptr stk_size,	...	@@ -90,10 +88,11 @@ void GetThreadStackAndTls(bool main, uptr stk_addr, uptr stk_size,
90		88
91	// Memory management	89	// Memory management
92	void MmapOrDie(uptr size, const char mem_type, bool raw_report = false);	90	void MmapOrDie(uptr size, const char mem_type, bool raw_report = false);
		91
93	inline void MmapOrDieQuietly(uptr size, const char mem_type) {	92	inline void MmapOrDieQuietly(uptr size, const char mem_type) {
94	return MmapOrDie(size, mem_type, /raw_report/ true);	93	return MmapOrDie(size, mem_type, /raw_report/ true);
95	}	94	}
96	void UnmapOrDie(void *addr, uptr size);	95	void UnmapOrDie(void *addr, uptr size, bool raw_report = false);
97	// Behaves just like MmapOrDie, but tolerates out of memory condition, in that	96	// Behaves just like MmapOrDie, but tolerates out of memory condition, in that
98	// case returns nullptr.	97	// case returns nullptr.
99	void MmapOrDieOnFatalError(uptr size, const char mem_type);	98	void MmapOrDieOnFatalError(uptr size, const char mem_type);
...	@@ -138,7 +137,8 @@ void UnmapFromTo(uptr from, uptr to);	...	@@ -138,7 +137,8 @@ void UnmapFromTo(uptr from, uptr to);
138	// shadow_size_bytes bytes on the right, which on linux is mapped no access.	137	// shadow_size_bytes bytes on the right, which on linux is mapped no access.
139	// The high_mem_end may be updated if the original shadow size doesn't fit.	138	// The high_mem_end may be updated if the original shadow size doesn't fit.
140	uptr MapDynamicShadow(uptr shadow_size_bytes, uptr shadow_scale,	139	uptr MapDynamicShadow(uptr shadow_size_bytes, uptr shadow_scale,
141	uptr min_shadow_base_alignment, uptr &high_mem_end);	140	uptr min_shadow_base_alignment, uptr &high_mem_end,
		141	uptr granularity);
142		142
143	// Let S = max(shadow_size, num_aliases * alias_size, ring_buffer_size).	143	// Let S = max(shadow_size, num_aliases * alias_size, ring_buffer_size).
144	// Reserves 2*S bytes of address space to the right of the returned address and	144	// Reserves 2*S bytes of address space to the right of the returned address and
...	@@ -177,7 +177,7 @@ bool DontDumpShadowMemory(uptr addr, uptr length);	...	@@ -177,7 +177,7 @@ bool DontDumpShadowMemory(uptr addr, uptr length);
177	// Check if the built VMA size matches the runtime one.	177	// Check if the built VMA size matches the runtime one.
178	void CheckVMASize();	178	void CheckVMASize();
179	void RunMallocHooks(void *ptr, uptr size);	179	void RunMallocHooks(void *ptr, uptr size);
180	void RunFreeHooks(void *ptr);	180	int RunFreeHooks(void *ptr);
181		181
182	class ReservedAddressRange {	182	class ReservedAddressRange {
183	public:	183	public:
...	@@ -208,6 +208,11 @@ void ParseUnixMemoryProfile(fill_profile_f cb, uptr stats, char smaps,	...	@@ -208,6 +208,11 @@ void ParseUnixMemoryProfile(fill_profile_f cb, uptr stats, char smaps,
208	// Simple low-level (mmap-based) allocator for internal use. Doesn't have	208	// Simple low-level (mmap-based) allocator for internal use. Doesn't have
209	// constructor, so all instances of LowLevelAllocator should be	209	// constructor, so all instances of LowLevelAllocator should be
210	// linker initialized.	210	// linker initialized.
		211	//
		212	// NOTE: Users should instead use the singleton provided via
		213	// `GetGlobalLowLevelAllocator()` rather than create a new one. This way, the
		214	// number of mmap fragments can be reduced and use the same contiguous mmap
		215	// provided by this singleton.
211	class LowLevelAllocator {	216	class LowLevelAllocator {
212	public:	217	public:
213	// Requires an external lock.	218	// Requires an external lock.
...	@@ -224,6 +229,8 @@ typedef void (*LowLevelAllocateCallback)(uptr ptr, uptr size);	...	@@ -224,6 +229,8 @@ typedef void (*LowLevelAllocateCallback)(uptr ptr, uptr size);
224	// Passing NULL removes the callback.	229	// Passing NULL removes the callback.
225	void SetLowLevelAllocateCallback(LowLevelAllocateCallback callback);	230	void SetLowLevelAllocateCallback(LowLevelAllocateCallback callback);
226		231
		232	LowLevelAllocator &GetGlobalLowLevelAllocator();
		233
227	// IO	234	// IO
228	void CatastrophicErrorWrite(const char *buffer, uptr length);	235	void CatastrophicErrorWrite(const char *buffer, uptr length);
229	void RawWrite(const char *buffer);	236	void RawWrite(const char *buffer);
...	@@ -386,6 +393,8 @@ void ReportErrorSummary(const char *error_type, const AddressInfo &info,	...	@@ -386,6 +393,8 @@ void ReportErrorSummary(const char *error_type, const AddressInfo &info,
386	// Same as above, but obtains AddressInfo by symbolizing top stack trace frame.	393	// Same as above, but obtains AddressInfo by symbolizing top stack trace frame.
387	void ReportErrorSummary(const char error_type, const StackTrace trace,	394	void ReportErrorSummary(const char error_type, const StackTrace trace,
388	const char *alt_tool_name = nullptr);	395	const char *alt_tool_name = nullptr);
		396	// Skips frames which we consider internal and not usefull to the users.
		397	const SymbolizedStack SkipInternalFrames(const SymbolizedStack frames);
389		398
390	void ReportMmapWriteExec(int prot, int mflags);	399	void ReportMmapWriteExec(int prot, int mflags);
391		400
...	@@ -500,7 +509,7 @@ inline int ToLower(int c) {	...	@@ -500,7 +509,7 @@ inline int ToLower(int c) {
500	// A low-level vector based on mmap. May incur a significant memory overhead for	509	// A low-level vector based on mmap. May incur a significant memory overhead for
501	// small vectors.	510	// small vectors.
502	// WARNING: The current implementation supports only POD types.	511	// WARNING: The current implementation supports only POD types.
503	template<typename T>	512	template <typename T, bool raw_report = false>
504	class InternalMmapVectorNoCtor {	513	class InternalMmapVectorNoCtor {
505	public:	514	public:
506	using value_type = T;	515	using value_type = T;
...	@@ -510,7 +519,7 @@ class InternalMmapVectorNoCtor {	...	@@ -510,7 +519,7 @@ class InternalMmapVectorNoCtor {
510	data_ = 0;	519	data_ = 0;
511	reserve(initial_capacity);	520	reserve(initial_capacity);
512	}	521	}
513	void Destroy() { UnmapOrDie(data_, capacity_bytes_); }	522	void Destroy() { UnmapOrDie(data_, capacity_bytes_, raw_report); }
514	T &operator[](uptr i) {	523	T &operator[](uptr i) {
515	CHECK_LT(i, size_);	524	CHECK_LT(i, size_);
516	return data_[i];	525	return data_[i];
...	@@ -586,9 +595,10 @@ class InternalMmapVectorNoCtor {	...	@@ -586,9 +595,10 @@ class InternalMmapVectorNoCtor {
586	CHECK_LE(size_, new_capacity);	595	CHECK_LE(size_, new_capacity);
587	uptr new_capacity_bytes =	596	uptr new_capacity_bytes =
588	RoundUpTo(new_capacity * sizeof(T), GetPageSizeCached());	597	RoundUpTo(new_capacity * sizeof(T), GetPageSizeCached());
589	T new_data = (T )MmapOrDie(new_capacity_bytes, "InternalMmapVector");	598	T *new_data =
		599	(T *)MmapOrDie(new_capacity_bytes, "InternalMmapVector", raw_report);
590	internal_memcpy(new_data, data_, size_ * sizeof(T));	600	internal_memcpy(new_data, data_, size_ * sizeof(T));
591	UnmapOrDie(data_, capacity_bytes_);	601	UnmapOrDie(data_, capacity_bytes_, raw_report);
592	data_ = new_data;	602	data_ = new_data;
593	capacity_bytes_ = new_capacity_bytes;	603	capacity_bytes_ = new_capacity_bytes;
594	}	604	}
...	@@ -636,7 +646,8 @@ class InternalScopedString {	...	@@ -636,7 +646,8 @@ class InternalScopedString {
636	buffer_.resize(1);	646	buffer_.resize(1);
637	buffer_[0] = '\0';	647	buffer_[0] = '\0';
638	}	648	}
639	void append(const char *format, ...) FORMAT(2, 3);	649	void Append(const char *str);
		650	void AppendF(const char *format, ...) FORMAT(2, 3);
640	const char *data() const { return buffer_.data(); }	651	const char *data() const { return buffer_.data(); }
641	char *data() { return buffer_.data(); }	652	char *data() { return buffer_.data(); }
642		653
...	@@ -1086,7 +1097,7 @@ inline u32 GetNumberOfCPUsCached() {	...	@@ -1086,7 +1097,7 @@ inline u32 GetNumberOfCPUsCached() {
1086		1097
1087	} // namespace __sanitizer	1098	} // namespace __sanitizer
1088		1099
1089	inline void *operator new(__sanitizer::operator_new_size_type size,	1100	inline void *operator new(__sanitizer::usize size,
1090	__sanitizer::LowLevelAllocator &alloc) {	1101	__sanitizer::LowLevelAllocator &alloc) {
1091	return alloc.Allocate(size);	1102	return alloc.Allocate(size);
1092	}	1103	}

lib/tsan/sanitizer_common/sanitizer_common_interceptors.inc+137-66

...	@@ -33,16 +33,17 @@	...	@@ -33,16 +33,17 @@
33	// COMMON_INTERCEPTOR_STRERROR	33	// COMMON_INTERCEPTOR_STRERROR
34	//===----------------------------------------------------------------------===//	34	//===----------------------------------------------------------------------===//
35		35
		36	#include <stdarg.h>
		37
36	#include "interception/interception.h"	38	#include "interception/interception.h"
37	#include "sanitizer_addrhashmap.h"	39	#include "sanitizer_addrhashmap.h"
		40	#include "sanitizer_dl.h"
38	#include "sanitizer_errno.h"	41	#include "sanitizer_errno.h"
39	#include "sanitizer_placement_new.h"	42	#include "sanitizer_placement_new.h"
40	#include "sanitizer_platform_interceptors.h"	43	#include "sanitizer_platform_interceptors.h"
41	#include "sanitizer_symbolizer.h"	44	#include "sanitizer_symbolizer.h"
42	#include "sanitizer_tls_get_addr.h"	45	#include "sanitizer_tls_get_addr.h"
43		46
44	#include <stdarg.h>
45
46	#if SANITIZER_INTERCEPTOR_HOOKS	47	#if SANITIZER_INTERCEPTOR_HOOKS
47	#define CALL_WEAK_INTERCEPTOR_HOOK(f, ...) f(__VA_ARGS__);	48	#define CALL_WEAK_INTERCEPTOR_HOOK(f, ...) f(__VA_ARGS__);
48	#define DECLARE_WEAK_INTERCEPTOR_HOOK(f, ...) \	49	#define DECLARE_WEAK_INTERCEPTOR_HOOK(f, ...) \
...	@@ -445,11 +446,13 @@ INTERCEPTOR(char, textdomain, const char domainname) {	...	@@ -445,11 +446,13 @@ INTERCEPTOR(char, textdomain, const char domainname) {
445	#define INIT_TEXTDOMAIN	446	#define INIT_TEXTDOMAIN
446	#endif	447	#endif
447		448
448	#if SANITIZER_INTERCEPT_STRCMP	449	#if SANITIZER_INTERCEPT_STRCMP \|\| SANITIZER_INTERCEPT_MEMCMP
449	static inline int CharCmpX(unsigned char c1, unsigned char c2) {	450	static inline int CharCmpX(unsigned char c1, unsigned char c2) {
450	return (c1 == c2) ? 0 : (c1 < c2) ? -1 : 1;	451	return (c1 == c2) ? 0 : (c1 < c2) ? -1 : 1;
451	}	452	}
		453	#endif
452		454
		455	#if SANITIZER_INTERCEPT_STRCMP
453	DECLARE_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_strcmp, uptr called_pc,	456	DECLARE_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_strcmp, uptr called_pc,
454	const char s1, const char s2, int result)	457	const char s1, const char s2, int result)
455		458
...	@@ -971,7 +974,7 @@ INTERCEPTOR(SSIZE_T, read, int fd, void *ptr, SIZE_T count) {	...	@@ -971,7 +974,7 @@ INTERCEPTOR(SSIZE_T, read, int fd, void *ptr, SIZE_T count) {
971	// FIXME: under ASan the call below may write to freed memory and corrupt	974	// FIXME: under ASan the call below may write to freed memory and corrupt
972	// its metadata. See	975	// its metadata. See
973	// https://github.com/google/sanitizers/issues/321.	976	// https://github.com/google/sanitizers/issues/321.
974	SSIZE_T res = REAL(read)(fd, ptr, count);	977	SSIZE_T res = COMMON_INTERCEPTOR_BLOCK_REAL(read)(fd, ptr, count);
975	if (res > 0) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, res);	978	if (res > 0) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, res);
976	if (res >= 0 && fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);	979	if (res >= 0 && fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);
977	return res;	980	return res;
...	@@ -1006,7 +1009,7 @@ INTERCEPTOR(SSIZE_T, pread, int fd, void *ptr, SIZE_T count, OFF_T offset) {	...	@@ -1006,7 +1009,7 @@ INTERCEPTOR(SSIZE_T, pread, int fd, void *ptr, SIZE_T count, OFF_T offset) {
1006	// FIXME: under ASan the call below may write to freed memory and corrupt	1009	// FIXME: under ASan the call below may write to freed memory and corrupt
1007	// its metadata. See	1010	// its metadata. See
1008	// https://github.com/google/sanitizers/issues/321.	1011	// https://github.com/google/sanitizers/issues/321.
1009	SSIZE_T res = REAL(pread)(fd, ptr, count, offset);	1012	SSIZE_T res = COMMON_INTERCEPTOR_BLOCK_REAL(pread)(fd, ptr, count, offset);
1010	if (res > 0) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, res);	1013	if (res > 0) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, res);
1011	if (res >= 0 && fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);	1014	if (res >= 0 && fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);
1012	return res;	1015	return res;
...	@@ -1024,7 +1027,7 @@ INTERCEPTOR(SSIZE_T, pread64, int fd, void *ptr, SIZE_T count, OFF64_T offset) {	...	@@ -1024,7 +1027,7 @@ INTERCEPTOR(SSIZE_T, pread64, int fd, void *ptr, SIZE_T count, OFF64_T offset) {
1024	// FIXME: under ASan the call below may write to freed memory and corrupt	1027	// FIXME: under ASan the call below may write to freed memory and corrupt
1025	// its metadata. See	1028	// its metadata. See
1026	// https://github.com/google/sanitizers/issues/321.	1029	// https://github.com/google/sanitizers/issues/321.
1027	SSIZE_T res = REAL(pread64)(fd, ptr, count, offset);	1030	SSIZE_T res = COMMON_INTERCEPTOR_BLOCK_REAL(pread64)(fd, ptr, count, offset);
1028	if (res > 0) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, res);	1031	if (res > 0) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, res);
1029	if (res >= 0 && fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);	1032	if (res >= 0 && fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);
1030	return res;	1033	return res;
...	@@ -1040,7 +1043,7 @@ INTERCEPTOR_WITH_SUFFIX(SSIZE_T, readv, int fd, __sanitizer_iovec *iov,	...	@@ -1040,7 +1043,7 @@ INTERCEPTOR_WITH_SUFFIX(SSIZE_T, readv, int fd, __sanitizer_iovec *iov,
1040	void *ctx;	1043	void *ctx;
1041	COMMON_INTERCEPTOR_ENTER(ctx, readv, fd, iov, iovcnt);	1044	COMMON_INTERCEPTOR_ENTER(ctx, readv, fd, iov, iovcnt);
1042	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);	1045	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
1043	SSIZE_T res = REAL(readv)(fd, iov, iovcnt);	1046	SSIZE_T res = COMMON_INTERCEPTOR_BLOCK_REAL(readv)(fd, iov, iovcnt);
1044	if (res > 0) write_iovec(ctx, iov, iovcnt, res);	1047	if (res > 0) write_iovec(ctx, iov, iovcnt, res);
1045	if (res >= 0 && fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);	1048	if (res >= 0 && fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);
1046	return res;	1049	return res;
...	@@ -1056,7 +1059,7 @@ INTERCEPTOR(SSIZE_T, preadv, int fd, __sanitizer_iovec *iov, int iovcnt,	...	@@ -1056,7 +1059,7 @@ INTERCEPTOR(SSIZE_T, preadv, int fd, __sanitizer_iovec *iov, int iovcnt,
1056	void *ctx;	1059	void *ctx;
1057	COMMON_INTERCEPTOR_ENTER(ctx, preadv, fd, iov, iovcnt, offset);	1060	COMMON_INTERCEPTOR_ENTER(ctx, preadv, fd, iov, iovcnt, offset);
1058	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);	1061	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
1059	SSIZE_T res = REAL(preadv)(fd, iov, iovcnt, offset);	1062	SSIZE_T res = COMMON_INTERCEPTOR_BLOCK_REAL(preadv)(fd, iov, iovcnt, offset);
1060	if (res > 0) write_iovec(ctx, iov, iovcnt, res);	1063	if (res > 0) write_iovec(ctx, iov, iovcnt, res);
1061	if (res >= 0 && fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);	1064	if (res >= 0 && fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);
1062	return res;	1065	return res;
...	@@ -1072,7 +1075,8 @@ INTERCEPTOR(SSIZE_T, preadv64, int fd, __sanitizer_iovec *iov, int iovcnt,	...	@@ -1072,7 +1075,8 @@ INTERCEPTOR(SSIZE_T, preadv64, int fd, __sanitizer_iovec *iov, int iovcnt,
1072	void *ctx;	1075	void *ctx;
1073	COMMON_INTERCEPTOR_ENTER(ctx, preadv64, fd, iov, iovcnt, offset);	1076	COMMON_INTERCEPTOR_ENTER(ctx, preadv64, fd, iov, iovcnt, offset);
1074	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);	1077	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
1075	SSIZE_T res = REAL(preadv64)(fd, iov, iovcnt, offset);	1078	SSIZE_T res =
		1079	COMMON_INTERCEPTOR_BLOCK_REAL(preadv64)(fd, iov, iovcnt, offset);
1076	if (res > 0) write_iovec(ctx, iov, iovcnt, res);	1080	if (res > 0) write_iovec(ctx, iov, iovcnt, res);
1077	if (res >= 0 && fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);	1081	if (res >= 0 && fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);
1078	return res;	1082	return res;
...	@@ -1088,8 +1092,9 @@ INTERCEPTOR(SSIZE_T, write, int fd, void *ptr, SIZE_T count) {	...	@@ -1088,8 +1092,9 @@ INTERCEPTOR(SSIZE_T, write, int fd, void *ptr, SIZE_T count) {
1088	COMMON_INTERCEPTOR_ENTER(ctx, write, fd, ptr, count);	1092	COMMON_INTERCEPTOR_ENTER(ctx, write, fd, ptr, count);
1089	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);	1093	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
1090	if (fd >= 0) COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd);	1094	if (fd >= 0) COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd);
1091	SSIZE_T res = REAL(write)(fd, ptr, count);	1095	SSIZE_T res = COMMON_INTERCEPTOR_BLOCK_REAL(write)(fd, ptr, count);
1092	// FIXME: this check should be _before_ the call to REAL(write), not after	1096	// FIXME: this check should be _before_ the call to
		1097	// COMMON_INTERCEPTOR_BLOCK_REAL(write), not after
1093	if (res > 0) COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, res);	1098	if (res > 0) COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, res);
1094	return res;	1099	return res;
1095	}	1100	}
...	@@ -1118,7 +1123,7 @@ INTERCEPTOR(SSIZE_T, pwrite, int fd, void *ptr, SIZE_T count, OFF_T offset) {	...	@@ -1118,7 +1123,7 @@ INTERCEPTOR(SSIZE_T, pwrite, int fd, void *ptr, SIZE_T count, OFF_T offset) {
1118	COMMON_INTERCEPTOR_ENTER(ctx, pwrite, fd, ptr, count, offset);	1123	COMMON_INTERCEPTOR_ENTER(ctx, pwrite, fd, ptr, count, offset);
1119	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);	1124	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
1120	if (fd >= 0) COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd);	1125	if (fd >= 0) COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd);
1121	SSIZE_T res = REAL(pwrite)(fd, ptr, count, offset);	1126	SSIZE_T res = COMMON_INTERCEPTOR_BLOCK_REAL(pwrite)(fd, ptr, count, offset);
1122	if (res > 0) COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, res);	1127	if (res > 0) COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, res);
1123	return res;	1128	return res;
1124	}	1129	}
...	@@ -1134,7 +1139,7 @@ INTERCEPTOR(SSIZE_T, pwrite64, int fd, void *ptr, OFF64_T count,	...	@@ -1134,7 +1139,7 @@ INTERCEPTOR(SSIZE_T, pwrite64, int fd, void *ptr, OFF64_T count,
1134	COMMON_INTERCEPTOR_ENTER(ctx, pwrite64, fd, ptr, count, offset);	1139	COMMON_INTERCEPTOR_ENTER(ctx, pwrite64, fd, ptr, count, offset);
1135	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);	1140	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
1136	if (fd >= 0) COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd);	1141	if (fd >= 0) COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd);
1137	SSIZE_T res = REAL(pwrite64)(fd, ptr, count, offset);	1142	SSIZE_T res = COMMON_INTERCEPTOR_BLOCK_REAL(pwrite64)(fd, ptr, count, offset);
1138	if (res > 0) COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, res);	1143	if (res > 0) COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, res);
1139	return res;	1144	return res;
1140	}	1145	}
...	@@ -1150,7 +1155,7 @@ INTERCEPTOR_WITH_SUFFIX(SSIZE_T, writev, int fd, __sanitizer_iovec *iov,	...	@@ -1150,7 +1155,7 @@ INTERCEPTOR_WITH_SUFFIX(SSIZE_T, writev, int fd, __sanitizer_iovec *iov,
1150	COMMON_INTERCEPTOR_ENTER(ctx, writev, fd, iov, iovcnt);	1155	COMMON_INTERCEPTOR_ENTER(ctx, writev, fd, iov, iovcnt);
1151	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);	1156	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
1152	if (fd >= 0) COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd);	1157	if (fd >= 0) COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd);
1153	SSIZE_T res = REAL(writev)(fd, iov, iovcnt);	1158	SSIZE_T res = COMMON_INTERCEPTOR_BLOCK_REAL(writev)(fd, iov, iovcnt);
1154	if (res > 0) read_iovec(ctx, iov, iovcnt, res);	1159	if (res > 0) read_iovec(ctx, iov, iovcnt, res);
1155	return res;	1160	return res;
1156	}	1161	}
...	@@ -1166,7 +1171,7 @@ INTERCEPTOR(SSIZE_T, pwritev, int fd, __sanitizer_iovec *iov, int iovcnt,	...	@@ -1166,7 +1171,7 @@ INTERCEPTOR(SSIZE_T, pwritev, int fd, __sanitizer_iovec *iov, int iovcnt,
1166	COMMON_INTERCEPTOR_ENTER(ctx, pwritev, fd, iov, iovcnt, offset);	1171	COMMON_INTERCEPTOR_ENTER(ctx, pwritev, fd, iov, iovcnt, offset);
1167	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);	1172	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
1168	if (fd >= 0) COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd);	1173	if (fd >= 0) COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd);
1169	SSIZE_T res = REAL(pwritev)(fd, iov, iovcnt, offset);	1174	SSIZE_T res = COMMON_INTERCEPTOR_BLOCK_REAL(pwritev)(fd, iov, iovcnt, offset);
1170	if (res > 0) read_iovec(ctx, iov, iovcnt, res);	1175	if (res > 0) read_iovec(ctx, iov, iovcnt, res);
1171	return res;	1176	return res;
1172	}	1177	}
...	@@ -1182,7 +1187,8 @@ INTERCEPTOR(SSIZE_T, pwritev64, int fd, __sanitizer_iovec *iov, int iovcnt,	...	@@ -1182,7 +1187,8 @@ INTERCEPTOR(SSIZE_T, pwritev64, int fd, __sanitizer_iovec *iov, int iovcnt,
1182	COMMON_INTERCEPTOR_ENTER(ctx, pwritev64, fd, iov, iovcnt, offset);	1187	COMMON_INTERCEPTOR_ENTER(ctx, pwritev64, fd, iov, iovcnt, offset);
1183	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);	1188	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
1184	if (fd >= 0) COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd);	1189	if (fd >= 0) COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd);
1185	SSIZE_T res = REAL(pwritev64)(fd, iov, iovcnt, offset);	1190	SSIZE_T res =
		1191	COMMON_INTERCEPTOR_BLOCK_REAL(pwritev64)(fd, iov, iovcnt, offset);
1186	if (res > 0) read_iovec(ctx, iov, iovcnt, res);	1192	if (res > 0) read_iovec(ctx, iov, iovcnt, res);
1187	return res;	1193	return res;
1188	}	1194	}
...	@@ -1245,6 +1251,7 @@ INTERCEPTOR(int, prctl, int option, unsigned long arg2, unsigned long arg3,	...	@@ -1245,6 +1251,7 @@ INTERCEPTOR(int, prctl, int option, unsigned long arg2, unsigned long arg3,
1245	void *ctx;	1251	void *ctx;
1246	COMMON_INTERCEPTOR_ENTER(ctx, prctl, option, arg2, arg3, arg4, arg5);	1252	COMMON_INTERCEPTOR_ENTER(ctx, prctl, option, arg2, arg3, arg4, arg5);
1247	static const int PR_SET_NAME = 15;	1253	static const int PR_SET_NAME = 15;
		1254	static const int PR_GET_NAME = 16;
1248	static const int PR_SET_VMA = 0x53564d41;	1255	static const int PR_SET_VMA = 0x53564d41;
1249	static const int PR_SCHED_CORE = 62;	1256	static const int PR_SCHED_CORE = 62;
1250	static const int PR_SCHED_CORE_GET = 0;	1257	static const int PR_SCHED_CORE_GET = 0;
...	@@ -1258,7 +1265,11 @@ INTERCEPTOR(int, prctl, int option, unsigned long arg2, unsigned long arg3,	...	@@ -1258,7 +1265,11 @@ INTERCEPTOR(int, prctl, int option, unsigned long arg2, unsigned long arg3,
1258	internal_strncpy(buff, (char *)arg2, 15);	1265	internal_strncpy(buff, (char *)arg2, 15);
1259	buff[15] = 0;	1266	buff[15] = 0;
1260	COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, buff);	1267	COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, buff);
1261	} else if (res != -1 && option == PR_SCHED_CORE && arg2 == PR_SCHED_CORE_GET) {	1268	} else if (res == 0 && option == PR_GET_NAME) {
		1269	char name = (char )arg2;
		1270	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, name, internal_strlen(name) + 1);
		1271	} else if (res != -1 && option == PR_SCHED_CORE &&
		1272	arg2 == PR_SCHED_CORE_GET) {
1262	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, (u64*)(arg5), sizeof(u64));	1273	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, (u64*)(arg5), sizeof(u64));
1263	}	1274	}
1264	return res;	1275	return res;
...	@@ -2546,7 +2557,7 @@ INTERCEPTOR_WITH_SUFFIX(int, wait, int *status) {	...	@@ -2546,7 +2557,7 @@ INTERCEPTOR_WITH_SUFFIX(int, wait, int *status) {
2546	// FIXME: under ASan the call below may write to freed memory and corrupt	2557	// FIXME: under ASan the call below may write to freed memory and corrupt
2547	// its metadata. See	2558	// its metadata. See
2548	// https://github.com/google/sanitizers/issues/321.	2559	// https://github.com/google/sanitizers/issues/321.
2549	int res = REAL(wait)(status);	2560	int res = COMMON_INTERCEPTOR_BLOCK_REAL(wait)(status);
2550	if (res != -1 && status)	2561	if (res != -1 && status)
2551	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, status, sizeof(*status));	2562	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, status, sizeof(*status));
2552	return res;	2563	return res;
...	@@ -2564,7 +2575,7 @@ INTERCEPTOR_WITH_SUFFIX(int, waitid, int idtype, int id, void *infop,	...	@@ -2564,7 +2575,7 @@ INTERCEPTOR_WITH_SUFFIX(int, waitid, int idtype, int id, void *infop,
2564	// FIXME: under ASan the call below may write to freed memory and corrupt	2575	// FIXME: under ASan the call below may write to freed memory and corrupt
2565	// its metadata. See	2576	// its metadata. See
2566	// https://github.com/google/sanitizers/issues/321.	2577	// https://github.com/google/sanitizers/issues/321.
2567	int res = REAL(waitid)(idtype, id, infop, options);	2578	int res = COMMON_INTERCEPTOR_BLOCK_REAL(waitid)(idtype, id, infop, options);
2568	if (res != -1 && infop)	2579	if (res != -1 && infop)
2569	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, infop, siginfo_t_sz);	2580	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, infop, siginfo_t_sz);
2570	return res;	2581	return res;
...	@@ -2575,7 +2586,7 @@ INTERCEPTOR_WITH_SUFFIX(int, waitpid, int pid, int *status, int options) {	...	@@ -2575,7 +2586,7 @@ INTERCEPTOR_WITH_SUFFIX(int, waitpid, int pid, int *status, int options) {
2575	// FIXME: under ASan the call below may write to freed memory and corrupt	2586	// FIXME: under ASan the call below may write to freed memory and corrupt
2576	// its metadata. See	2587	// its metadata. See
2577	// https://github.com/google/sanitizers/issues/321.	2588	// https://github.com/google/sanitizers/issues/321.
2578	int res = REAL(waitpid)(pid, status, options);	2589	int res = COMMON_INTERCEPTOR_BLOCK_REAL(waitpid)(pid, status, options);
2579	if (res != -1 && status)	2590	if (res != -1 && status)
2580	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, status, sizeof(*status));	2591	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, status, sizeof(*status));
2581	return res;	2592	return res;
...	@@ -2586,7 +2597,7 @@ INTERCEPTOR(int, wait3, int status, int options, void rusage) {	...	@@ -2586,7 +2597,7 @@ INTERCEPTOR(int, wait3, int status, int options, void rusage) {
2586	// FIXME: under ASan the call below may write to freed memory and corrupt	2597	// FIXME: under ASan the call below may write to freed memory and corrupt
2587	// its metadata. See	2598	// its metadata. See
2588	// https://github.com/google/sanitizers/issues/321.	2599	// https://github.com/google/sanitizers/issues/321.
2589	int res = REAL(wait3)(status, options, rusage);	2600	int res = COMMON_INTERCEPTOR_BLOCK_REAL(wait3)(status, options, rusage);
2590	if (res != -1) {	2601	if (res != -1) {
2591	if (status) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, status, sizeof(*status));	2602	if (status) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, status, sizeof(*status));
2592	if (rusage) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, rusage, struct_rusage_sz);	2603	if (rusage) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, rusage, struct_rusage_sz);
...	@@ -2600,7 +2611,8 @@ INTERCEPTOR(int, __wait4, int pid, int status, int options, void rusage) {	...	@@ -2600,7 +2611,8 @@ INTERCEPTOR(int, __wait4, int pid, int status, int options, void rusage) {
2600	// FIXME: under ASan the call below may write to freed memory and corrupt	2611	// FIXME: under ASan the call below may write to freed memory and corrupt
2601	// its metadata. See	2612	// its metadata. See
2602	// https://github.com/google/sanitizers/issues/321.	2613	// https://github.com/google/sanitizers/issues/321.
2603	int res = REAL(__wait4)(pid, status, options, rusage);	2614	int res =
		2615	COMMON_INTERCEPTOR_BLOCK_REAL(__wait4)(pid, status, options, rusage);
2604	if (res != -1) {	2616	if (res != -1) {
2605	if (status) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, status, sizeof(*status));	2617	if (status) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, status, sizeof(*status));
2606	if (rusage) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, rusage, struct_rusage_sz);	2618	if (rusage) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, rusage, struct_rusage_sz);
...	@@ -2615,7 +2627,7 @@ INTERCEPTOR(int, wait4, int pid, int status, int options, void rusage) {	...	@@ -2615,7 +2627,7 @@ INTERCEPTOR(int, wait4, int pid, int status, int options, void rusage) {
2615	// FIXME: under ASan the call below may write to freed memory and corrupt	2627	// FIXME: under ASan the call below may write to freed memory and corrupt
2616	// its metadata. See	2628	// its metadata. See
2617	// https://github.com/google/sanitizers/issues/321.	2629	// https://github.com/google/sanitizers/issues/321.
2618	int res = REAL(wait4)(pid, status, options, rusage);	2630	int res = COMMON_INTERCEPTOR_BLOCK_REAL(wait4)(pid, status, options, rusage);
2619	if (res != -1) {	2631	if (res != -1) {
2620	if (status) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, status, sizeof(*status));	2632	if (status) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, status, sizeof(*status));
2621	if (rusage) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, rusage, struct_rusage_sz);	2633	if (rusage) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, rusage, struct_rusage_sz);
...	@@ -2993,7 +3005,7 @@ INTERCEPTOR(int, accept, int fd, void addr, unsigned addrlen) {	...	@@ -2993,7 +3005,7 @@ INTERCEPTOR(int, accept, int fd, void addr, unsigned addrlen) {
2993	COMMON_INTERCEPTOR_READ_RANGE(ctx, addrlen, sizeof(*addrlen));	3005	COMMON_INTERCEPTOR_READ_RANGE(ctx, addrlen, sizeof(*addrlen));
2994	addrlen0 = *addrlen;	3006	addrlen0 = *addrlen;
2995	}	3007	}
2996	int fd2 = REAL(accept)(fd, addr, addrlen);	3008	int fd2 = COMMON_INTERCEPTOR_BLOCK_REAL(accept)(fd, addr, addrlen);
2997	if (fd2 >= 0) {	3009	if (fd2 >= 0) {
2998	if (fd >= 0) COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, fd2);	3010	if (fd >= 0) COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, fd2);
2999	if (addr && addrlen)	3011	if (addr && addrlen)
...	@@ -3018,7 +3030,7 @@ INTERCEPTOR(int, accept4, int fd, void addr, unsigned addrlen, int f) {	...	@@ -3018,7 +3030,7 @@ INTERCEPTOR(int, accept4, int fd, void addr, unsigned addrlen, int f) {
3018	// FIXME: under ASan the call below may write to freed memory and corrupt	3030	// FIXME: under ASan the call below may write to freed memory and corrupt
3019	// its metadata. See	3031	// its metadata. See
3020	// https://github.com/google/sanitizers/issues/321.	3032	// https://github.com/google/sanitizers/issues/321.
3021	int fd2 = REAL(accept4)(fd, addr, addrlen, f);	3033	int fd2 = COMMON_INTERCEPTOR_BLOCK_REAL(accept4)(fd, addr, addrlen, f);
3022	if (fd2 >= 0) {	3034	if (fd2 >= 0) {
3023	if (fd >= 0) COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, fd2);	3035	if (fd >= 0) COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, fd2);
3024	if (addr && addrlen)	3036	if (addr && addrlen)
...	@@ -3042,7 +3054,7 @@ INTERCEPTOR(int, paccept, int fd, void addr, unsigned addrlen,	...	@@ -3042,7 +3054,7 @@ INTERCEPTOR(int, paccept, int fd, void addr, unsigned addrlen,
3042	addrlen0 = *addrlen;	3054	addrlen0 = *addrlen;
3043	}	3055	}
3044	if (set) COMMON_INTERCEPTOR_READ_RANGE(ctx, set, sizeof(*set));	3056	if (set) COMMON_INTERCEPTOR_READ_RANGE(ctx, set, sizeof(*set));
3045	int fd2 = REAL(paccept)(fd, addr, addrlen, set, f);	3057	int fd2 = COMMON_INTERCEPTOR_BLOCK_REAL(paccept)(fd, addr, addrlen, set, f);
3046	if (fd2 >= 0) {	3058	if (fd2 >= 0) {
3047	if (fd >= 0) COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, fd2);	3059	if (fd >= 0) COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, fd2);
3048	if (addr && addrlen)	3060	if (addr && addrlen)
...	@@ -3123,7 +3135,7 @@ INTERCEPTOR(SSIZE_T, recvmsg, int fd, struct __sanitizer_msghdr *msg,	...	@@ -3123,7 +3135,7 @@ INTERCEPTOR(SSIZE_T, recvmsg, int fd, struct __sanitizer_msghdr *msg,
3123	// FIXME: under ASan the call below may write to freed memory and corrupt	3135	// FIXME: under ASan the call below may write to freed memory and corrupt
3124	// its metadata. See	3136	// its metadata. See
3125	// https://github.com/google/sanitizers/issues/321.	3137	// https://github.com/google/sanitizers/issues/321.
3126	SSIZE_T res = REAL(recvmsg)(fd, msg, flags);	3138	SSIZE_T res = COMMON_INTERCEPTOR_BLOCK_REAL(recvmsg)(fd, msg, flags);
3127	if (res >= 0) {	3139	if (res >= 0) {
3128	if (fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);	3140	if (fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);
3129	if (msg) {	3141	if (msg) {
...	@@ -3144,7 +3156,8 @@ INTERCEPTOR(int, recvmmsg, int fd, struct __sanitizer_mmsghdr *msgvec,	...	@@ -3144,7 +3156,8 @@ INTERCEPTOR(int, recvmmsg, int fd, struct __sanitizer_mmsghdr *msgvec,
3144	void *ctx;	3156	void *ctx;
3145	COMMON_INTERCEPTOR_ENTER(ctx, recvmmsg, fd, msgvec, vlen, flags, timeout);	3157	COMMON_INTERCEPTOR_ENTER(ctx, recvmmsg, fd, msgvec, vlen, flags, timeout);
3146	if (timeout) COMMON_INTERCEPTOR_READ_RANGE(ctx, timeout, struct_timespec_sz);	3158	if (timeout) COMMON_INTERCEPTOR_READ_RANGE(ctx, timeout, struct_timespec_sz);
3147	int res = REAL(recvmmsg)(fd, msgvec, vlen, flags, timeout);	3159	int res =
		3160	COMMON_INTERCEPTOR_BLOCK_REAL(recvmmsg)(fd, msgvec, vlen, flags, timeout);
3148	if (res >= 0) {	3161	if (res >= 0) {
3149	if (fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);	3162	if (fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);
3150	for (int i = 0; i < res; ++i) {	3163	for (int i = 0; i < res; ++i) {
...	@@ -3222,7 +3235,7 @@ INTERCEPTOR(SSIZE_T, sendmsg, int fd, struct __sanitizer_msghdr *msg,	...	@@ -3222,7 +3235,7 @@ INTERCEPTOR(SSIZE_T, sendmsg, int fd, struct __sanitizer_msghdr *msg,
3222	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);	3235	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
3223	COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd);	3236	COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd);
3224	}	3237	}
3225	SSIZE_T res = REAL(sendmsg)(fd, msg, flags);	3238	SSIZE_T res = COMMON_INTERCEPTOR_BLOCK_REAL(sendmsg)(fd, msg, flags);
3226	if (common_flags()->intercept_send && res >= 0 && msg)	3239	if (common_flags()->intercept_send && res >= 0 && msg)
3227	read_msghdr(ctx, msg, res);	3240	read_msghdr(ctx, msg, res);
3228	return res;	3241	return res;
...	@@ -3241,7 +3254,7 @@ INTERCEPTOR(int, sendmmsg, int fd, struct __sanitizer_mmsghdr *msgvec,	...	@@ -3241,7 +3254,7 @@ INTERCEPTOR(int, sendmmsg, int fd, struct __sanitizer_mmsghdr *msgvec,
3241	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);	3254	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
3242	COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd);	3255	COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd);
3243	}	3256	}
3244	int res = REAL(sendmmsg)(fd, msgvec, vlen, flags);	3257	int res = COMMON_INTERCEPTOR_BLOCK_REAL(sendmmsg)(fd, msgvec, vlen, flags);
3245	if (res >= 0 && msgvec) {	3258	if (res >= 0 && msgvec) {
3246	for (int i = 0; i < res; ++i) {	3259	for (int i = 0; i < res; ++i) {
3247	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, &msgvec[i].msg_len,	3260	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, &msgvec[i].msg_len,
...	@@ -3264,7 +3277,7 @@ INTERCEPTOR(int, msgsnd, int msqid, const void *msgp, SIZE_T msgsz,	...	@@ -3264,7 +3277,7 @@ INTERCEPTOR(int, msgsnd, int msqid, const void *msgp, SIZE_T msgsz,
3264	COMMON_INTERCEPTOR_ENTER(ctx, msgsnd, msqid, msgp, msgsz, msgflg);	3277	COMMON_INTERCEPTOR_ENTER(ctx, msgsnd, msqid, msgp, msgsz, msgflg);
3265	if (msgp)	3278	if (msgp)
3266	COMMON_INTERCEPTOR_READ_RANGE(ctx, msgp, sizeof(long) + msgsz);	3279	COMMON_INTERCEPTOR_READ_RANGE(ctx, msgp, sizeof(long) + msgsz);
3267	int res = REAL(msgsnd)(msqid, msgp, msgsz, msgflg);	3280	int res = COMMON_INTERCEPTOR_BLOCK_REAL(msgsnd)(msqid, msgp, msgsz, msgflg);
3268	return res;	3281	return res;
3269	}	3282	}
3270		3283
...	@@ -3272,7 +3285,8 @@ INTERCEPTOR(SSIZE_T, msgrcv, int msqid, void *msgp, SIZE_T msgsz,	...	@@ -3272,7 +3285,8 @@ INTERCEPTOR(SSIZE_T, msgrcv, int msqid, void *msgp, SIZE_T msgsz,
3272	long msgtyp, int msgflg) {	3285	long msgtyp, int msgflg) {
3273	void *ctx;	3286	void *ctx;
3274	COMMON_INTERCEPTOR_ENTER(ctx, msgrcv, msqid, msgp, msgsz, msgtyp, msgflg);	3287	COMMON_INTERCEPTOR_ENTER(ctx, msgrcv, msqid, msgp, msgsz, msgtyp, msgflg);
3275	SSIZE_T len = REAL(msgrcv)(msqid, msgp, msgsz, msgtyp, msgflg);	3288	SSIZE_T len =
		3289	COMMON_INTERCEPTOR_BLOCK_REAL(msgrcv)(msqid, msgp, msgsz, msgtyp, msgflg);
3276	if (len != -1)	3290	if (len != -1)
3277	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, msgp, sizeof(long) + len);	3291	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, msgp, sizeof(long) + len);
3278	return len;	3292	return len;
...	@@ -6116,7 +6130,7 @@ INTERCEPTOR(int, flopen, const char *path, int flags, ...) {	...	@@ -6116,7 +6130,7 @@ INTERCEPTOR(int, flopen, const char *path, int flags, ...) {
6116	if (path) {	6130	if (path) {
6117	COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);	6131	COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
6118	}	6132	}
6119	return REAL(flopen)(path, flags, mode);	6133	return COMMON_INTERCEPTOR_BLOCK_REAL(flopen)(path, flags, mode);
6120	}	6134	}
6121		6135
6122	INTERCEPTOR(int, flopenat, int dirfd, const char *path, int flags, ...) {	6136	INTERCEPTOR(int, flopenat, int dirfd, const char *path, int flags, ...) {
...	@@ -6129,7 +6143,7 @@ INTERCEPTOR(int, flopenat, int dirfd, const char *path, int flags, ...) {	...	@@ -6129,7 +6143,7 @@ INTERCEPTOR(int, flopenat, int dirfd, const char *path, int flags, ...) {
6129	if (path) {	6143	if (path) {
6130	COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);	6144	COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
6131	}	6145	}
6132	return REAL(flopenat)(dirfd, path, flags, mode);	6146	return COMMON_INTERCEPTOR_BLOCK_REAL(flopenat)(dirfd, path, flags, mode);
6133	}	6147	}
6134		6148
6135	#define INIT_FLOPEN \	6149	#define INIT_FLOPEN \
...	@@ -6305,7 +6319,36 @@ INTERCEPTOR(int, fclose, __sanitizer_FILE *fp) {	...	@@ -6305,7 +6319,36 @@ INTERCEPTOR(int, fclose, __sanitizer_FILE *fp) {
6305	INTERCEPTOR(void, dlopen, const char filename, int flag) {	6319	INTERCEPTOR(void, dlopen, const char filename, int flag) {
6306	void *ctx;	6320	void *ctx;
6307	COMMON_INTERCEPTOR_ENTER_NOIGNORE(ctx, dlopen, filename, flag);	6321	COMMON_INTERCEPTOR_ENTER_NOIGNORE(ctx, dlopen, filename, flag);
6308	if (filename) COMMON_INTERCEPTOR_READ_STRING(ctx, filename, 0);	6322
		6323	if (filename) {
		6324	COMMON_INTERCEPTOR_READ_STRING(ctx, filename, 0);
		6325
		6326	# if !SANITIZER_DYNAMIC
		6327	// We care about a very specific use-case: dladdr on
		6328	// statically-linked ASan may return <main program>
		6329	// instead of the library.
		6330	// We therefore only take effect if the sanitizer is statically
		6331	// linked, and we don't bother canonicalizing paths because
		6332	// dladdr should return the same address both times (we assume
		6333	// the user did not canonicalize the result from dladdr).
		6334	if (common_flags()->test_only_replace_dlopen_main_program) {
		6335	VPrintf(1, "dlopen interceptor: filename: %s\n", filename);
		6336
		6337	const char *SelfFName = DladdrSelfFName();
		6338	VPrintf(1, "dlopen interceptor: DladdrSelfFName: %p %s\n",
		6339	(const void *)SelfFName, SelfFName);
		6340
		6341	if (SelfFName && internal_strcmp(SelfFName, filename) == 0) {
		6342	// It's possible they copied the string from dladdr, so
		6343	// we do a string comparison rather than pointer comparison.
		6344	VPrintf(1, "dlopen interceptor: replacing %s because it matches %s\n",
		6345	filename, SelfFName);
		6346	filename = (char *)0; // RTLD_DEFAULT
		6347	}
		6348	}
		6349	# endif // !SANITIZER_DYNAMIC
		6350	}
		6351
6309	void *res = COMMON_INTERCEPTOR_DLOPEN(filename, flag);	6352	void *res = COMMON_INTERCEPTOR_DLOPEN(filename, flag);
6310	Symbolizer::GetOrInit()->InvalidateModuleList();	6353	Symbolizer::GetOrInit()->InvalidateModuleList();
6311	COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, res);	6354	COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, res);
...	@@ -6685,7 +6728,7 @@ INTERCEPTOR(SSIZE_T, recv, int fd, void *buf, SIZE_T len, int flags) {	...	@@ -6685,7 +6728,7 @@ INTERCEPTOR(SSIZE_T, recv, int fd, void *buf, SIZE_T len, int flags) {
6685	void *ctx;	6728	void *ctx;
6686	COMMON_INTERCEPTOR_ENTER(ctx, recv, fd, buf, len, flags);	6729	COMMON_INTERCEPTOR_ENTER(ctx, recv, fd, buf, len, flags);
6687	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);	6730	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
6688	SSIZE_T res = REAL(recv)(fd, buf, len, flags);	6731	SSIZE_T res = COMMON_INTERCEPTOR_BLOCK_REAL(recv)(fd, buf, len, flags);
6689	if (res > 0) {	6732	if (res > 0) {
6690	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, Min((SIZE_T)res, len));	6733	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, Min((SIZE_T)res, len));
6691	}	6734	}
...	@@ -6702,7 +6745,8 @@ INTERCEPTOR(SSIZE_T, recvfrom, int fd, void *buf, SIZE_T len, int flags,	...	@@ -6702,7 +6745,8 @@ INTERCEPTOR(SSIZE_T, recvfrom, int fd, void *buf, SIZE_T len, int flags,
6702	SIZE_T srcaddr_sz;	6745	SIZE_T srcaddr_sz;
6703	if (srcaddr) srcaddr_sz = *addrlen;	6746	if (srcaddr) srcaddr_sz = *addrlen;
6704	(void)srcaddr_sz; // prevent "set but not used" warning	6747	(void)srcaddr_sz; // prevent "set but not used" warning
6705	SSIZE_T res = REAL(recvfrom)(fd, buf, len, flags, srcaddr, addrlen);	6748	SSIZE_T res = COMMON_INTERCEPTOR_BLOCK_REAL(recvfrom)(fd, buf, len, flags,
		6749	srcaddr, addrlen);
6706	if (res > 0)	6750	if (res > 0)
6707	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, Min((SIZE_T)res, len));	6751	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, Min((SIZE_T)res, len));
6708	if (res >= 0 && srcaddr)	6752	if (res >= 0 && srcaddr)
...	@@ -6725,7 +6769,7 @@ INTERCEPTOR(SSIZE_T, send, int fd, void *buf, SIZE_T len, int flags) {	...	@@ -6725,7 +6769,7 @@ INTERCEPTOR(SSIZE_T, send, int fd, void *buf, SIZE_T len, int flags) {
6725	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);	6769	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
6726	COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd);	6770	COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd);
6727	}	6771	}
6728	SSIZE_T res = REAL(send)(fd, buf, len, flags);	6772	SSIZE_T res = COMMON_INTERCEPTOR_BLOCK_REAL(send)(fd, buf, len, flags);
6729	if (common_flags()->intercept_send && res > 0)	6773	if (common_flags()->intercept_send && res > 0)
6730	COMMON_INTERCEPTOR_READ_RANGE(ctx, buf, Min((SIZE_T)res, len));	6774	COMMON_INTERCEPTOR_READ_RANGE(ctx, buf, Min((SIZE_T)res, len));
6731	return res;	6775	return res;
...	@@ -6740,7 +6784,8 @@ INTERCEPTOR(SSIZE_T, sendto, int fd, void *buf, SIZE_T len, int flags,	...	@@ -6740,7 +6784,8 @@ INTERCEPTOR(SSIZE_T, sendto, int fd, void *buf, SIZE_T len, int flags,
6740	COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd);	6784	COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd);
6741	}	6785	}
6742	// Can't check dstaddr as it may have uninitialized padding at the end.	6786	// Can't check dstaddr as it may have uninitialized padding at the end.
6743	SSIZE_T res = REAL(sendto)(fd, buf, len, flags, dstaddr, addrlen);	6787	SSIZE_T res = COMMON_INTERCEPTOR_BLOCK_REAL(sendto)(fd, buf, len, flags,
		6788	dstaddr, addrlen);
6744	if (common_flags()->intercept_send && res > 0)	6789	if (common_flags()->intercept_send && res > 0)
6745	COMMON_INTERCEPTOR_READ_RANGE(ctx, buf, Min((SIZE_T)res, len));	6790	COMMON_INTERCEPTOR_READ_RANGE(ctx, buf, Min((SIZE_T)res, len));
6746	return res;	6791	return res;
...	@@ -6753,25 +6798,25 @@ INTERCEPTOR(SSIZE_T, sendto, int fd, void *buf, SIZE_T len, int flags,	...	@@ -6753,25 +6798,25 @@ INTERCEPTOR(SSIZE_T, sendto, int fd, void *buf, SIZE_T len, int flags,
6753	#endif	6798	#endif
6754		6799
6755	#if SANITIZER_INTERCEPT_EVENTFD_READ_WRITE	6800	#if SANITIZER_INTERCEPT_EVENTFD_READ_WRITE
6756	INTERCEPTOR(int, eventfd_read, int fd, u64 *value) {	6801	INTERCEPTOR(int, eventfd_read, int fd, __sanitizer_eventfd_t *value) {
6757	void *ctx;	6802	void *ctx;
6758	COMMON_INTERCEPTOR_ENTER(ctx, eventfd_read, fd, value);	6803	COMMON_INTERCEPTOR_ENTER(ctx, eventfd_read, fd, value);
6759	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);	6804	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
6760	int res = REAL(eventfd_read)(fd, value);	6805	int res = COMMON_INTERCEPTOR_BLOCK_REAL(eventfd_read)(fd, value);
6761	if (res == 0) {	6806	if (res == 0) {
6762	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, value, sizeof(*value));	6807	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, value, sizeof(*value));
6763	if (fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);	6808	if (fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);
6764	}	6809	}
6765	return res;	6810	return res;
6766	}	6811	}
6767	INTERCEPTOR(int, eventfd_write, int fd, u64 value) {	6812	INTERCEPTOR(int, eventfd_write, int fd, __sanitizer_eventfd_t value) {
6768	void *ctx;	6813	void *ctx;
6769	COMMON_INTERCEPTOR_ENTER(ctx, eventfd_write, fd, value);	6814	COMMON_INTERCEPTOR_ENTER(ctx, eventfd_write, fd, value);
6770	if (fd >= 0) {	6815	if (fd >= 0) {
6771	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);	6816	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
6772	COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd);	6817	COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd);
6773	}	6818	}
6774	int res = REAL(eventfd_write)(fd, value);	6819	int res = COMMON_INTERCEPTOR_BLOCK_REAL(eventfd_write)(fd, value);
6775	return res;	6820	return res;
6776	}	6821	}
6777	#define INIT_EVENTFD_READ_WRITE \	6822	#define INIT_EVENTFD_READ_WRITE \
...	@@ -7394,7 +7439,8 @@ INTERCEPTOR(int, open_by_handle_at, int mount_fd, struct file_handle* handle,	...	@@ -7394,7 +7439,8 @@ INTERCEPTOR(int, open_by_handle_at, int mount_fd, struct file_handle* handle,
7394	COMMON_INTERCEPTOR_READ_RANGE(	7439	COMMON_INTERCEPTOR_READ_RANGE(
7395	ctx, &sanitizer_handle->f_handle, sanitizer_handle->handle_bytes);	7440	ctx, &sanitizer_handle->f_handle, sanitizer_handle->handle_bytes);
7396		7441
7397	return REAL(open_by_handle_at)(mount_fd, handle, flags);	7442	return COMMON_INTERCEPTOR_BLOCK_REAL(open_by_handle_at)(mount_fd, handle,
		7443	flags);
7398	}	7444	}
7399		7445
7400	#define INIT_OPEN_BY_HANDLE_AT COMMON_INTERCEPT_FUNCTION(open_by_handle_at)	7446	#define INIT_OPEN_BY_HANDLE_AT COMMON_INTERCEPT_FUNCTION(open_by_handle_at)
...	@@ -7609,9 +7655,9 @@ static void write_protoent(void ctx, struct __sanitizer_protoent p) {	...	@@ -7609,9 +7655,9 @@ static void write_protoent(void ctx, struct __sanitizer_protoent p) {
7609	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->p_aliases, pp_size * sizeof(char *));	7655	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->p_aliases, pp_size * sizeof(char *));
7610	}	7656	}
7611		7657
7612	INTERCEPTOR(struct __sanitizer_protoent *, getprotoent) {	7658	INTERCEPTOR(struct __sanitizer_protoent *, getprotoent,) {
7613	void *ctx;	7659	void *ctx;
7614	COMMON_INTERCEPTOR_ENTER(ctx, getprotoent);	7660	COMMON_INTERCEPTOR_ENTER(ctx, getprotoent,);
7615	struct __sanitizer_protoent *p = REAL(getprotoent)();	7661	struct __sanitizer_protoent *p = REAL(getprotoent)();
7616	if (p)	7662	if (p)
7617	write_protoent(ctx, p);	7663	write_protoent(ctx, p);
...	@@ -7698,9 +7744,9 @@ INTERCEPTOR(int, getprotobynumber_r, int num,	...	@@ -7698,9 +7744,9 @@ INTERCEPTOR(int, getprotobynumber_r, int num,
7698	#endif	7744	#endif
7699		7745
7700	#if SANITIZER_INTERCEPT_NETENT	7746	#if SANITIZER_INTERCEPT_NETENT
7701	INTERCEPTOR(struct __sanitizer_netent *, getnetent) {	7747	INTERCEPTOR(struct __sanitizer_netent *, getnetent,) {
7702	void *ctx;	7748	void *ctx;
7703	COMMON_INTERCEPTOR_ENTER(ctx, getnetent);	7749	COMMON_INTERCEPTOR_ENTER(ctx, getnetent,);
7704	struct __sanitizer_netent *n = REAL(getnetent)();	7750	struct __sanitizer_netent *n = REAL(getnetent)();
7705	if (n) {	7751	if (n) {
7706	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n, sizeof(*n));	7752	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n, sizeof(*n));
...	@@ -9862,9 +9908,9 @@ INTERCEPTOR(char , fdevname_r, int fd, char buf, SIZE_T len) {	...	@@ -9862,9 +9908,9 @@ INTERCEPTOR(char , fdevname_r, int fd, char buf, SIZE_T len) {
9862	#endif	9908	#endif
9863		9909
9864	#if SANITIZER_INTERCEPT_GETUSERSHELL	9910	#if SANITIZER_INTERCEPT_GETUSERSHELL
9865	INTERCEPTOR(char *, getusershell) {	9911	INTERCEPTOR(char *, getusershell,) {
9866	void *ctx;	9912	void *ctx;
9867	COMMON_INTERCEPTOR_ENTER(ctx, getusershell);	9913	COMMON_INTERCEPTOR_ENTER(ctx, getusershell,);
9868	char *res = REAL(getusershell)();	9914	char *res = REAL(getusershell)();
9869	if (res)	9915	if (res)
9870	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, internal_strlen(res) + 1);	9916	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, internal_strlen(res) + 1);
...	@@ -9933,7 +9979,13 @@ INTERCEPTOR(void, sl_free, void *sl, int freeall) {	...	@@ -9933,7 +9979,13 @@ INTERCEPTOR(void, sl_free, void *sl, int freeall) {
9933	INTERCEPTOR(SSIZE_T, getrandom, void *buf, SIZE_T buflen, unsigned int flags) {	9979	INTERCEPTOR(SSIZE_T, getrandom, void *buf, SIZE_T buflen, unsigned int flags) {
9934	void *ctx;	9980	void *ctx;
9935	COMMON_INTERCEPTOR_ENTER(ctx, getrandom, buf, buflen, flags);	9981	COMMON_INTERCEPTOR_ENTER(ctx, getrandom, buf, buflen, flags);
9936	SSIZE_T n = REAL(getrandom)(buf, buflen, flags);	9982	// If GRND_NONBLOCK is set in the flags, it is non blocking.
		9983	static const int grnd_nonblock = 1;
		9984	SSIZE_T n;
		9985	if ((flags & grnd_nonblock))
		9986	n = REAL(getrandom)(buf, buflen, flags);
		9987	else
		9988	n = COMMON_INTERCEPTOR_BLOCK_REAL(getrandom)(buf, buflen, flags);
9937	if (n > 0) {	9989	if (n > 0) {
9938	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, n);	9990	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, n);
9939	}	9991	}
...	@@ -10180,20 +10232,6 @@ INTERCEPTOR(int, __xuname, int size, void *utsname) {	...	@@ -10180,20 +10232,6 @@ INTERCEPTOR(int, __xuname, int size, void *utsname) {
10180	#define INIT___XUNAME	10232	#define INIT___XUNAME
10181	#endif	10233	#endif
10182		10234
10183	#if SANITIZER_INTERCEPT_HEXDUMP
10184	INTERCEPTOR(void, hexdump, const void ptr, int length, const char header, int flags) {
10185	void *ctx;
10186	COMMON_INTERCEPTOR_ENTER(ctx, hexdump, ptr, length, header, flags);
10187	COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, length);
10188	COMMON_INTERCEPTOR_READ_RANGE(ctx, header, internal_strlen(header) + 1);
10189	REAL(hexdump)(ptr, length, header, flags);
10190	}
10191
10192	#define INIT_HEXDUMP COMMON_INTERCEPT_FUNCTION(hexdump);
10193	#else
10194	#define INIT_HEXDUMP
10195	#endif
10196
10197	#if SANITIZER_INTERCEPT_ARGP_PARSE	10235	#if SANITIZER_INTERCEPT_ARGP_PARSE
10198	INTERCEPTOR(int, argp_parse, const struct argp argp, int argc, char *argv,	10236	INTERCEPTOR(int, argp_parse, const struct argp argp, int argc, char *argv,
10199	unsigned flags, int arg_index, void input) {	10237	unsigned flags, int arg_index, void input) {
...	@@ -10226,6 +10264,38 @@ INTERCEPTOR(int, cpuset_getaffinity, int level, int which, __int64_t id, SIZE_T	...	@@ -10226,6 +10264,38 @@ INTERCEPTOR(int, cpuset_getaffinity, int level, int which, __int64_t id, SIZE_T
10226	#define INIT_CPUSET_GETAFFINITY	10264	#define INIT_CPUSET_GETAFFINITY
10227	#endif	10265	#endif
10228		10266
		10267	#if SANITIZER_INTERCEPT_PREADV2
		10268	INTERCEPTOR(SSIZE_T, preadv2, int fd, __sanitizer_iovec *iov, int iovcnt,
		10269	OFF_T offset, int flags) {
		10270	void *ctx;
		10271	COMMON_INTERCEPTOR_ENTER(ctx, preadv2, fd, iov, iovcnt, offset, flags);
		10272	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
		10273	SSIZE_T res = REAL(preadv2)(fd, iov, iovcnt, offset, flags);
		10274	if (res > 0) write_iovec(ctx, iov, iovcnt, res);
		10275	if (res >= 0 && fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);
		10276	return res;
		10277	}
		10278	#define INIT_PREADV2 COMMON_INTERCEPT_FUNCTION(preadv2)
		10279	#else
		10280	#define INIT_PREADV2
		10281	#endif
		10282
		10283	#if SANITIZER_INTERCEPT_PWRITEV2
		10284	INTERCEPTOR(SSIZE_T, pwritev2, int fd, __sanitizer_iovec *iov, int iovcnt,
		10285	OFF_T offset, int flags) {
		10286	void *ctx;
		10287	COMMON_INTERCEPTOR_ENTER(ctx, pwritev2, fd, iov, iovcnt, offset, flags);
		10288	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
		10289	if (fd >= 0) COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd);
		10290	SSIZE_T res = REAL(pwritev2)(fd, iov, iovcnt, offset, flags);
		10291	if (res > 0) read_iovec(ctx, iov, iovcnt, res);
		10292	return res;
		10293	}
		10294	#define INIT_PWRITEV2 COMMON_INTERCEPT_FUNCTION(pwritev2)
		10295	#else
		10296	#define INIT_PWRITEV2
		10297	#endif
		10298
10229	#include "sanitizer_common_interceptors_netbsd_compat.inc"	10299	#include "sanitizer_common_interceptors_netbsd_compat.inc"
10230		10300
10231	namespace __sanitizer {	10301	namespace __sanitizer {
...	@@ -10543,9 +10613,10 @@ static void InitializeCommonInterceptors() {	...	@@ -10543,9 +10613,10 @@ static void InitializeCommonInterceptors() {
10543	INIT_PROCCTL	10613	INIT_PROCCTL
10544	INIT_UNAME;	10614	INIT_UNAME;
10545	INIT___XUNAME;	10615	INIT___XUNAME;
10546	INIT_HEXDUMP;
10547	INIT_ARGP_PARSE;	10616	INIT_ARGP_PARSE;
10548	INIT_CPUSET_GETAFFINITY;	10617	INIT_CPUSET_GETAFFINITY;
		10618	INIT_PREADV2;
		10619	INIT_PWRITEV2;
10549		10620
10550	INIT___PRINTF_CHK;	10621	INIT___PRINTF_CHK;
10551	}	10622	}

lib/tsan/sanitizer_common/sanitizer_common_interceptors_format.inc+8-7

 @@ -547,24 +547,25 @@ static void printf_common(void *ctx, const char *format, va_list aq) {
       continue;
     } else if (size == FSS_STRLEN) {
       if (void *argp = va_arg(aq, void *)) {
+        uptr len;
         if (dir.starredPrecision) {
           // FIXME: properly support starred precision for strings.
-          size = 0;
+          len = 0;
         } else if (dir.fieldPrecision > 0) {
           // Won't read more than "precision" symbols.
-          size = internal_strnlen((const char *)argp, dir.fieldPrecision);
+          len = internal_strnlen((const char *)argp, dir.fieldPrecision);
-          if (size < dir.fieldPrecision) size++;
+          if (len < (uptr)dir.fieldPrecision)
+            len++;
         } else {
           // Whole string will be accessed.
-          size = internal_strlen((const char *)argp) + 1;
+          len = internal_strlen((const char *)argp) + 1;
         }
-        COMMON_INTERCEPTOR_READ_RANGE(ctx, argp, size);
+        COMMON_INTERCEPTOR_READ_RANGE(ctx, argp, len);
       }
     } else if (size == FSS_WCSLEN) {
       if (void *argp = va_arg(aq, void *)) {
         // FIXME: Properly support wide-character strings (via wcsrtombs).
-        size = 0;
+        COMMON_INTERCEPTOR_READ_RANGE(ctx, argp, 0);
-        COMMON_INTERCEPTOR_READ_RANGE(ctx, argp, size);
       }
     } else {
       // Skip non-pointer args

lib/tsan/sanitizer_common/sanitizer_common_interface.inc+1

...	@@ -46,6 +46,7 @@ INTERFACE_FUNCTION(__sanitizer_purge_allocator)	...	@@ -46,6 +46,7 @@ INTERFACE_FUNCTION(__sanitizer_purge_allocator)
46	INTERFACE_FUNCTION(__sanitizer_print_memory_profile)	46	INTERFACE_FUNCTION(__sanitizer_print_memory_profile)
47	INTERFACE_WEAK_FUNCTION(__sanitizer_free_hook)	47	INTERFACE_WEAK_FUNCTION(__sanitizer_free_hook)
48	INTERFACE_WEAK_FUNCTION(__sanitizer_malloc_hook)	48	INTERFACE_WEAK_FUNCTION(__sanitizer_malloc_hook)
		49	INTERFACE_WEAK_FUNCTION(__sanitizer_ignore_free_hook)
49	// Memintrinsic functions.	50	// Memintrinsic functions.
50	INTERFACE_FUNCTION(__sanitizer_internal_memcpy)	51	INTERFACE_FUNCTION(__sanitizer_internal_memcpy)
51	INTERFACE_FUNCTION(__sanitizer_internal_memmove)	52	INTERFACE_FUNCTION(__sanitizer_internal_memmove)

lib/tsan/sanitizer_common/sanitizer_common_interface_posix.inc+1

...	@@ -9,6 +9,7 @@	...	@@ -9,6 +9,7 @@
9	//===----------------------------------------------------------------------===//	9	//===----------------------------------------------------------------------===//
10	INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_code)	10	INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_code)
11	INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_data)	11	INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_data)
		12	INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_frame)
12	INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_demangle)	13	INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_demangle)
13	INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_flush)	14	INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_flush)
14	INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_set_demangle)	15	INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_set_demangle)

lib/tsan/sanitizer_common/sanitizer_common_libcdep.cpp+6-4

...	@@ -87,8 +87,8 @@ void MaybeStartBackgroudThread() {	...	@@ -87,8 +87,8 @@ void MaybeStartBackgroudThread() {
87	if (!common_flags()->hard_rss_limit_mb &&	87	if (!common_flags()->hard_rss_limit_mb &&
88	!common_flags()->soft_rss_limit_mb &&	88	!common_flags()->soft_rss_limit_mb &&
89	!common_flags()->heap_profile) return;	89	!common_flags()->heap_profile) return;
90	if (!&real_pthread_create) {	90	if (!&internal_pthread_create) {
91	VPrintf(1, "%s: real_pthread_create undefined\n", SanitizerToolName);	91	VPrintf(1, "%s: internal_pthread_create undefined\n", SanitizerToolName);
92	return; // Can't spawn the thread anyway.	92	return; // Can't spawn the thread anyway.
93	}	93	}
94		94
...	@@ -119,8 +119,10 @@ void MaybeStartBackgroudThread() {}	...	@@ -119,8 +119,10 @@ void MaybeStartBackgroudThread() {}
119	#endif	119	#endif
120		120
121	void WriteToSyslog(const char *msg) {	121	void WriteToSyslog(const char *msg) {
		122	if (!msg)
		123	return;
122	InternalScopedString msg_copy;	124	InternalScopedString msg_copy;
123	msg_copy.append("%s", msg);	125	msg_copy.Append(msg);
124	const char *p = msg_copy.data();	126	const char *p = msg_copy.data();
125		127
126	// Print one line at a time.	128	// Print one line at a time.
...	@@ -167,7 +169,7 @@ void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name,	...	@@ -167,7 +169,7 @@ void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name,
167	: !MmapFixedNoReserve(beg, size, name)) {	169	: !MmapFixedNoReserve(beg, size, name)) {
168	Report(	170	Report(
169	"ReserveShadowMemoryRange failed while trying to map 0x%zx bytes. "	171	"ReserveShadowMemoryRange failed while trying to map 0x%zx bytes. "
170	"Perhaps you're using ulimit -v\n",	172	"Perhaps you're using ulimit -v or ulimit -d\n",
171	size);	173	size);
172	Abort();	174	Abort();
173	}	175	}

lib/tsan/sanitizer_common/sanitizer_common_syscalls.inc+35

...	@@ -38,6 +38,10 @@	...	@@ -38,6 +38,10 @@
38	// Called before fork syscall.	38	// Called before fork syscall.
39	// COMMON_SYSCALL_POST_FORK(long res)	39	// COMMON_SYSCALL_POST_FORK(long res)
40	// Called after fork syscall.	40	// Called after fork syscall.
		41	// COMMON_SYSCALL_BLOCKING_START()
		42	// Called before blocking syscall.
		43	// COMMON_SYSCALL_BLOCKING_END()
		44	// Called after blocking syscall.
41	//===----------------------------------------------------------------------===//	45	//===----------------------------------------------------------------------===//
42		46
43	#include "sanitizer_platform.h"	47	#include "sanitizer_platform.h"
...	@@ -85,6 +89,16 @@	...	@@ -85,6 +89,16 @@
85	{}	89	{}
86	# endif	90	# endif
87		91
		92	# ifndef COMMON_SYSCALL_BLOCKING_START
		93	# define COMMON_SYSCALL_BLOCKING_START() \
		94	{}
		95	# endif
		96
		97	# ifndef COMMON_SYSCALL_BLOCKING_END
		98	# define COMMON_SYSCALL_BLOCKING_END() \
		99	{}
		100	# endif
		101
88	// FIXME: do some kind of PRE_READ for all syscall arguments (int(s) and such).	102	// FIXME: do some kind of PRE_READ for all syscall arguments (int(s) and such).
89		103
90	extern "C" {	104	extern "C" {
...	@@ -2808,6 +2822,15 @@ PRE_SYSCALL(fchownat)	...	@@ -2808,6 +2822,15 @@ PRE_SYSCALL(fchownat)
2808	POST_SYSCALL(fchownat)	2822	POST_SYSCALL(fchownat)
2809	(long res, long dfd, const void *filename, long user, long group, long flag) {}	2823	(long res, long dfd, const void *filename, long user, long group, long flag) {}
2810		2824
		2825	PRE_SYSCALL(fchmodat2)(long dfd, const void *filename, long mode, long flag) {
		2826	if (filename)
		2827	PRE_READ(filename,
		2828	__sanitizer::internal_strlen((const char *)filename) + 1);
		2829	}
		2830
		2831	POST_SYSCALL(fchmodat2)
		2832	(long res, long dfd, const void *filename, long mode, long flag) {}
		2833
2811	PRE_SYSCALL(openat)(long dfd, const void *filename, long flags, long mode) {	2834	PRE_SYSCALL(openat)(long dfd, const void *filename, long flags, long mode) {
2812	if (filename)	2835	if (filename)
2813	PRE_READ(filename,	2836	PRE_READ(filename,
...	@@ -3167,6 +3190,18 @@ POST_SYSCALL(sigaltstack)(long res, void ss, void oss) {	...	@@ -3167,6 +3190,18 @@ POST_SYSCALL(sigaltstack)(long res, void ss, void oss) {
3167	}	3190	}
3168	}	3191	}
3169	}	3192	}
		3193
		3194	PRE_SYSCALL(futex)
		3195	(void uaddr, long futex_op, long val, void timeout, void *uaddr2, long val3) {
		3196	COMMON_SYSCALL_BLOCKING_START();
		3197	}
		3198
		3199	POST_SYSCALL(futex)
		3200	(long res, void uaddr, long futex_op, long val, void timeout, void *uaddr2,
		3201	long val3) {
		3202	COMMON_SYSCALL_BLOCKING_END();
		3203	}
		3204
3170	} // extern "C"	3205	} // extern "C"
3171		3206
3172	# undef PRE_SYSCALL	3207	# undef PRE_SYSCALL

lib/tsan/sanitizer_common/sanitizer_dl.cpp created+37

...	@@ -0,0 +1,37 @@
1	//===-- sanitizer_dl.cpp --------------------------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file has helper functions that depend on libc's dynamic loading
10	// introspection.
11	//
12	//===----------------------------------------------------------------------===//
13
14	#include "sanitizer_dl.h"
15
16	#include "sanitizer_common/sanitizer_platform.h"
17
18	#if SANITIZER_GLIBC
19	# include <dlfcn.h>
20	#endif
21
22	namespace __sanitizer {
23	extern const char *SanitizerToolName;
24
25	const char *DladdrSelfFName(void) {
26	#if SANITIZER_GLIBC
27	Dl_info info;
28	int ret = dladdr((void *)&SanitizerToolName, &info);
29	if (ret) {
30	return info.dli_fname;
31	}
32	#endif
33
34	return nullptr;
35	}
36
37	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_dl.h created+26

...	@@ -0,0 +1,26 @@
1	//===-- sanitizer_dl.h ----------------------------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file has helper functions that depend on libc's dynamic loading
10	// introspection.
11	//
12	//===----------------------------------------------------------------------===//
13
14	#ifndef SANITIZER_DL_H
15	#define SANITIZER_DL_H
16
17	namespace __sanitizer {
18
19	// Returns the path to the shared object or - in the case of statically linked
20	// sanitizers
21	// - the main program itself, that contains the sanitizer.
22	const char* DladdrSelfFName(void);
23
24	} // namespace __sanitizer
25
26	#endif // SANITIZER_DL_H

lib/tsan/sanitizer_common/sanitizer_file.cpp+3-1

lib/tsan/sanitizer_common/sanitizer_file.h+1-1

...	@@ -84,7 +84,7 @@ bool IsPathSeparator(const char c);	...	@@ -84,7 +84,7 @@ bool IsPathSeparator(const char c);
84	bool IsAbsolutePath(const char *path);	84	bool IsAbsolutePath(const char *path);
85	// Returns true on success, false on failure.	85	// Returns true on success, false on failure.
86	bool CreateDir(const char *pathname);	86	bool CreateDir(const char *pathname);
87	// Starts a subprocess and returs its pid.	87	// Starts a subprocess and returns its pid.
88	// If *_fd parameters are not kInvalidFd their corresponding input/output	88	// If *_fd parameters are not kInvalidFd their corresponding input/output
89	// streams will be redirect to the file. The files will always be closed	89	// streams will be redirect to the file. The files will always be closed
90	// in parent process even in case of an error.	90	// in parent process even in case of an error.

lib/tsan/sanitizer_common/sanitizer_flag_parser.cpp+3-4

...	@@ -19,8 +19,6 @@	...	@@ -19,8 +19,6 @@
19		19
20	namespace __sanitizer {	20	namespace __sanitizer {
21		21
22	LowLevelAllocator FlagParser::Alloc;
23
24	class UnknownFlags {	22	class UnknownFlags {
25	static const int kMaxUnknownFlags = 20;	23	static const int kMaxUnknownFlags = 20;
26	const char *unknown_flags_[kMaxUnknownFlags];	24	const char *unknown_flags_[kMaxUnknownFlags];
...	@@ -49,7 +47,7 @@ void ReportUnrecognizedFlags() {	...	@@ -49,7 +47,7 @@ void ReportUnrecognizedFlags() {
49		47
50	char FlagParser::ll_strndup(const char s, uptr n) {	48	char FlagParser::ll_strndup(const char s, uptr n) {
51	uptr len = internal_strnlen(s, n);	49	uptr len = internal_strnlen(s, n);
52	char s2 = (char)Alloc.Allocate(len + 1);	50	char s2 = (char )GetGlobalLowLevelAllocator().Allocate(len + 1);
53	internal_memcpy(s2, s, len);	51	internal_memcpy(s2, s, len);
54	s2[len] = 0;	52	s2[len] = 0;
55	return s2;	53	return s2;
...	@@ -185,7 +183,8 @@ void FlagParser::RegisterHandler(const char name, FlagHandlerBase handler,	...	@@ -185,7 +183,8 @@ void FlagParser::RegisterHandler(const char name, FlagHandlerBase handler,
185	}	183	}
186		184
187	FlagParser::FlagParser() : n_flags_(0), buf_(nullptr), pos_(0) {	185	FlagParser::FlagParser() : n_flags_(0), buf_(nullptr), pos_(0) {
188	flags_ = (Flag )Alloc.Allocate(sizeof(Flag) kMaxFlags);	186	flags_ =
		187	(Flag )GetGlobalLowLevelAllocator().Allocate(sizeof(Flag) kMaxFlags);
189	}	188	}
190		189
191	} // namespace __sanitizer	190	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_flag_parser.h+1-3

...	@@ -178,8 +178,6 @@ class FlagParser {	...	@@ -178,8 +178,6 @@ class FlagParser {
178	bool ParseFile(const char *path, bool ignore_missing);	178	bool ParseFile(const char *path, bool ignore_missing);
179	void PrintFlagDescriptions();	179	void PrintFlagDescriptions();
180		180
181	static LowLevelAllocator Alloc;
182
183	private:	181	private:
184	void fatal_error(const char *err);	182	void fatal_error(const char *err);
185	bool is_space(char c);	183	bool is_space(char c);
...	@@ -193,7 +191,7 @@ class FlagParser {	...	@@ -193,7 +191,7 @@ class FlagParser {
193	template <typename T>	191	template <typename T>
194	static void RegisterFlag(FlagParser parser, const char name, const char *desc,	192	static void RegisterFlag(FlagParser parser, const char name, const char *desc,
195	T *var) {	193	T *var) {
196	FlagHandler<T> *fh = new (FlagParser::Alloc) FlagHandler<T>(var);	194	FlagHandler<T> *fh = new (GetGlobalLowLevelAllocator()) FlagHandler<T>(var);
197	parser->RegisterHandler(name, fh, desc);	195	parser->RegisterHandler(name, fh, desc);
198	}	196	}
199		197

lib/tsan/sanitizer_common/sanitizer_flags.cpp+2-2

...	@@ -108,11 +108,11 @@ class FlagHandlerInclude final : public FlagHandlerBase {	...	@@ -108,11 +108,11 @@ class FlagHandlerInclude final : public FlagHandlerBase {
108	};	108	};
109		109
110	void RegisterIncludeFlags(FlagParser parser, CommonFlags cf) {	110	void RegisterIncludeFlags(FlagParser parser, CommonFlags cf) {
111	FlagHandlerInclude *fh_include = new (FlagParser::Alloc)	111	FlagHandlerInclude *fh_include = new (GetGlobalLowLevelAllocator())
112	FlagHandlerInclude(parser, /ignore_missing/ false);	112	FlagHandlerInclude(parser, /ignore_missing/ false);
113	parser->RegisterHandler("include", fh_include,	113	parser->RegisterHandler("include", fh_include,
114	"read more options from the given file");	114	"read more options from the given file");
115	FlagHandlerInclude *fh_include_if_exists = new (FlagParser::Alloc)	115	FlagHandlerInclude *fh_include_if_exists = new (GetGlobalLowLevelAllocator())
116	FlagHandlerInclude(parser, /ignore_missing/ true);	116	FlagHandlerInclude(parser, /ignore_missing/ true);
117	parser->RegisterHandler(	117	parser->RegisterHandler(
118	"include_if_exists", fh_include_if_exists,	118	"include_if_exists", fh_include_if_exists,

lib/tsan/sanitizer_common/sanitizer_flags.inc+13

...	@@ -269,3 +269,16 @@ COMMON_FLAG(bool, detect_write_exec, false,	...	@@ -269,3 +269,16 @@ COMMON_FLAG(bool, detect_write_exec, false,
269	COMMON_FLAG(bool, test_only_emulate_no_memorymap, false,	269	COMMON_FLAG(bool, test_only_emulate_no_memorymap, false,
270	"TEST ONLY fail to read memory mappings to emulate sanitized "	270	"TEST ONLY fail to read memory mappings to emulate sanitized "
271	"\"init\"")	271	"\"init\"")
		272	// With static linking, dladdr((void*)pthread_join) or similar will return the
		273	// path to the main program. This flag will replace dlopen(<main program,...>
		274	// with dlopen(NULL,...), which is the correct way to get a handle to the main
		275	// program.
		276	COMMON_FLAG(bool, test_only_replace_dlopen_main_program, false,
		277	"TEST ONLY replace dlopen(<main program>,...) with dlopen(NULL)")
		278
		279	COMMON_FLAG(bool, enable_symbolizer_markup, SANITIZER_FUCHSIA,
		280	"Use sanitizer symbolizer markup, available on Linux "
		281	"and always set true for Fuchsia.")
		282
		283	COMMON_FLAG(bool, detect_invalid_join, true,
		284	"If set, check invalid joins of threads.")

lib/tsan/sanitizer_common/sanitizer_flat_map.h+4

 @@ -109,6 +109,10 @@ class TwoLevelMap {
     return *AddressSpaceView::LoadWritable(&map2[idx % kSize2]);
   }
+  void Lock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS { mu_.Lock(); }
+  void Unlock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS { mu_.Unlock(); }
  private:
   constexpr uptr MmapSize() const {
     return RoundUpTo(kSize2 * sizeof(T), GetPageSizeCached());

lib/tsan/sanitizer_common/sanitizer_freebsd.h deleted-137

...	@@ -1,137 +0,0 @@
1	//===-- sanitizer_freebsd.h -------------------------------------- C++ --===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is a part of Sanitizer runtime. It contains FreeBSD-specific
10	// definitions.
11	//
12	//===----------------------------------------------------------------------===//
13
14	#ifndef SANITIZER_FREEBSD_H
15	#define SANITIZER_FREEBSD_H
16
17	#include "sanitizer_internal_defs.h"
18
19	// x86-64 FreeBSD 9.2 and older define 'ucontext_t' incorrectly in
20	// 32-bit mode.
21	#if SANITIZER_FREEBSD && (SANITIZER_WORDSIZE == 32)
22	#include <osreldate.h>
23	#if __FreeBSD_version <= 902001 // v9.2
24	#include <link.h>
25	#include <sys/param.h>
26	#include <ucontext.h>
27
28	namespace __sanitizer {
29
30	typedef unsigned long long __xuint64_t;
31
32	typedef __int32_t __xregister_t;
33
34	typedef struct __xmcontext {
35	__xregister_t mc_onstack;
36	__xregister_t mc_gs;
37	__xregister_t mc_fs;
38	__xregister_t mc_es;
39	__xregister_t mc_ds;
40	__xregister_t mc_edi;
41	__xregister_t mc_esi;
42	__xregister_t mc_ebp;
43	__xregister_t mc_isp;
44	__xregister_t mc_ebx;
45	__xregister_t mc_edx;
46	__xregister_t mc_ecx;
47	__xregister_t mc_eax;
48	__xregister_t mc_trapno;
49	__xregister_t mc_err;
50	__xregister_t mc_eip;
51	__xregister_t mc_cs;
52	__xregister_t mc_eflags;
53	__xregister_t mc_esp;
54	__xregister_t mc_ss;
55
56	int mc_len;
57	int mc_fpformat;
58	int mc_ownedfp;
59	__xregister_t mc_flags;
60
61	int mc_fpstate[128] __aligned(16);
62	__xregister_t mc_fsbase;
63	__xregister_t mc_gsbase;
64	__xregister_t mc_xfpustate;
65	__xregister_t mc_xfpustate_len;
66
67	int mc_spare2[4];
68	} xmcontext_t;
69
70	typedef struct __xucontext {
71	sigset_t uc_sigmask;
72	xmcontext_t uc_mcontext;
73
74	struct __ucontext *uc_link;
75	stack_t uc_stack;
76	int uc_flags;
77	int __spare__[4];
78	} xucontext_t;
79
80	struct xkinfo_vmentry {
81	int kve_structsize;
82	int kve_type;
83	__xuint64_t kve_start;
84	__xuint64_t kve_end;
85	__xuint64_t kve_offset;
86	__xuint64_t kve_vn_fileid;
87	__uint32_t kve_vn_fsid;
88	int kve_flags;
89	int kve_resident;
90	int kve_private_resident;
91	int kve_protection;
92	int kve_ref_count;
93	int kve_shadow_count;
94	int kve_vn_type;
95	__xuint64_t kve_vn_size;
96	__uint32_t kve_vn_rdev;
97	__uint16_t kve_vn_mode;
98	__uint16_t kve_status;
99	int _kve_ispare[12];
100	char kve_path[PATH_MAX];
101	};
102
103	typedef struct {
104	__uint32_t p_type;
105	__uint32_t p_offset;
106	__uint32_t p_vaddr;
107	__uint32_t p_paddr;
108	__uint32_t p_filesz;
109	__uint32_t p_memsz;
110	__uint32_t p_flags;
111	__uint32_t p_align;
112	} XElf32_Phdr;
113
114	struct xdl_phdr_info {
115	Elf_Addr dlpi_addr;
116	const char *dlpi_name;
117	const XElf32_Phdr *dlpi_phdr;
118	Elf_Half dlpi_phnum;
119	unsigned long long int dlpi_adds;
120	unsigned long long int dlpi_subs;
121	size_t dlpi_tls_modid;
122	void *dlpi_tls_data;
123	};
124
125	typedef int (__xdl_iterate_hdr_callback)(struct xdl_phdr_info , size_t,
126	void *);
127	typedef int xdl_iterate_phdr_t(__xdl_iterate_hdr_callback, void *);
128
129	#define xdl_iterate_phdr(callback, param) \
130	(((xdl_iterate_phdr_t *)dl_iterate_phdr)((callback), (param)))
131
132	} // namespace __sanitizer
133
134	#endif // __FreeBSD_version <= 902001
135	#endif // SANITIZER_FREEBSD && (SANITIZER_WORDSIZE == 32)
136
137	#endif // SANITIZER_FREEBSD_H

lib/tsan/sanitizer_common/sanitizer_fuchsia.cpp+82-25

 @@ -129,6 +129,60 @@ uptr GetMaxVirtualAddress() { return GetMaxUserVirtualAddress(); }
 bool ErrorIsOOM(error_t err) { return err == ZX_ERR_NO_MEMORY; }
+// For any sanitizer internal that needs to map something which can be unmapped
+// later, first attempt to map to a pre-allocated VMAR. This helps reduce
+// fragmentation from many small anonymous mmap calls. A good value for this
+// VMAR size would be the total size of your typical sanitizer internal objects
+// allocated in an "average" process lifetime. Examples of this include:
+// FakeStack, LowLevelAllocator mappings, TwoLevelMap, InternalMmapVector,
+// StackStore, CreateAsanThread, etc.
+//
+// This is roughly equal to the total sum of sanitizer internal mappings for a
+// large test case.
+constexpr size_t kSanitizerHeapVmarSize = 13ULL << 20;
+static zx_handle_t gSanitizerHeapVmar = ZX_HANDLE_INVALID;
+static zx_status_t GetSanitizerHeapVmar(zx_handle_t *vmar) {
+  zx_status_t status = ZX_OK;
+  if (gSanitizerHeapVmar == ZX_HANDLE_INVALID) {
+    CHECK_EQ(kSanitizerHeapVmarSize % GetPageSizeCached(), 0);
+    uintptr_t base;
+    status = _zx_vmar_allocate(
+        _zx_vmar_root_self(),
+        ZX_VM_CAN_MAP_READ | ZX_VM_CAN_MAP_WRITE | ZX_VM_CAN_MAP_SPECIFIC, 0,
+        kSanitizerHeapVmarSize, &gSanitizerHeapVmar, &base);
+  }
+  *vmar = gSanitizerHeapVmar;
+  if (status == ZX_OK)
+    CHECK_NE(gSanitizerHeapVmar, ZX_HANDLE_INVALID);
+  return status;
+}
+static zx_status_t TryVmoMapSanitizerVmar(zx_vm_option_t options,
+                                          size_t vmar_offset, zx_handle_t vmo,
+                                          size_t size, uintptr_t *addr,
+                                          zx_handle_t *vmar_used = nullptr) {
+  zx_handle_t vmar;
+  zx_status_t status = GetSanitizerHeapVmar(&vmar);
+  if (status != ZX_OK)
+    return status;
+  status = _zx_vmar_map(gSanitizerHeapVmar, options, vmar_offset, vmo,
+                        /*vmo_offset=*/0, size, addr);
+  if (vmar_used)
+    *vmar_used = gSanitizerHeapVmar;
+  if (status == ZX_ERR_NO_RESOURCES || status == ZX_ERR_INVALID_ARGS) {
+    // This means there's no space in the heap VMAR, so fallback to the root
+    // VMAR.
+    status = _zx_vmar_map(_zx_vmar_root_self(), options, vmar_offset, vmo,
+                          /*vmo_offset=*/0, size, addr);
+    if (vmar_used)
+      *vmar_used = _zx_vmar_root_self();
+  }
+  return status;
+}
 static void *DoAnonymousMmapOrDie(uptr size, const char *mem_type,
                                   bool raw_report, bool die_for_nomem) {
   size = RoundUpTo(size, GetPageSize());
 @@ -144,11 +198,9 @@ static void *DoAnonymousMmapOrDie(uptr size, const char *mem_type,
   _zx_object_set_property(vmo, ZX_PROP_NAME, mem_type,
                           internal_strlen(mem_type));
-  // TODO(mcgrathr): Maybe allocate a VMAR for all sanitizer heap and use that?
   uintptr_t addr;
-  status =
+  status = TryVmoMapSanitizerVmar(ZX_VM_PERM_READ | ZX_VM_PERM_WRITE,
-      _zx_vmar_map(_zx_vmar_root_self(), ZX_VM_PERM_READ | ZX_VM_PERM_WRITE, 0,
+                                  /*vmar_offset=*/0, vmo, size, &addr);
-                   vmo, 0, size, &addr);
   _zx_handle_close(vmo);
   if (status != ZX_OK) {
 @@ -226,27 +278,32 @@ static uptr DoMmapFixedOrDie(zx_handle_t vmar, uptr fixed_addr, uptr map_size,
 uptr ReservedAddressRange::Map(uptr fixed_addr, uptr map_size,
                                const char *name) {
-  return DoMmapFixedOrDie(os_handle_, fixed_addr, map_size, base_, name_,
+  return DoMmapFixedOrDie(os_handle_, fixed_addr, map_size, base_,
-                          false);
+                          name ? name : name_, false);
 }
 uptr ReservedAddressRange::MapOrDie(uptr fixed_addr, uptr map_size,
                                     const char *name) {
-  return DoMmapFixedOrDie(os_handle_, fixed_addr, map_size, base_, name_, true);
+  return DoMmapFixedOrDie(os_handle_, fixed_addr, map_size, base_,
+                          name ? name : name_, true);
 }
-void UnmapOrDieVmar(void *addr, uptr size, zx_handle_t target_vmar) {
+void UnmapOrDieVmar(void *addr, uptr size, zx_handle_t target_vmar,
+                    bool raw_report) {
   if (!addr || !size)
     return;
   size = RoundUpTo(size, GetPageSize());
   zx_status_t status =
       _zx_vmar_unmap(target_vmar, reinterpret_cast<uintptr_t>(addr), size);
-  if (status != ZX_OK) {
+  if (status == ZX_ERR_INVALID_ARGS && target_vmar == gSanitizerHeapVmar) {
-    Report("ERROR: %s failed to deallocate 0x%zx (%zd) bytes at address %p\n",
+    // If there wasn't any space in the heap vmar, the fallback was the root
-           SanitizerToolName, size, size, addr);
+    // vmar.
-    CHECK("unable to unmap" && 0);
+    status = _zx_vmar_unmap(_zx_vmar_root_self(),
+                            reinterpret_cast<uintptr_t>(addr), size);
   }
+  if (status != ZX_OK)
+    ReportMunmapFailureAndDie(addr, size, status, raw_report);
   DecreaseTotalMmap(size);
 }
 @@ -268,7 +325,8 @@ void ReservedAddressRange::Unmap(uptr addr, uptr size) {
   }
   // Partial unmapping does not affect the fact that the initial range is still
   // reserved, and the resulting unmapped memory can't be reused.
-  UnmapOrDieVmar(reinterpret_cast<void *>(addr), size, vmar);
+  UnmapOrDieVmar(reinterpret_cast<void *>(addr), size, vmar,
+                 /*raw_report=*/false);
 }
 // This should never be called.
 @@ -307,17 +365,16 @@ void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment,
   _zx_object_set_property(vmo, ZX_PROP_NAME, mem_type,
                           internal_strlen(mem_type));
-  // TODO(mcgrathr): Maybe allocate a VMAR for all sanitizer heap and use that?
   // Map a larger size to get a chunk of address space big enough that
   // it surely contains an aligned region of the requested size.  Then
   // overwrite the aligned middle portion with a mapping from the
   // beginning of the VMO, and unmap the excess before and after.
   size_t map_size = size + alignment;
   uintptr_t addr;
-  status =
+  zx_handle_t vmar_used;
-      _zx_vmar_map(_zx_vmar_root_self(), ZX_VM_PERM_READ | ZX_VM_PERM_WRITE, 0,
+  status = TryVmoMapSanitizerVmar(ZX_VM_PERM_READ | ZX_VM_PERM_WRITE,
-                   vmo, 0, map_size, &addr);
+                                  /*vmar_offset=*/0, vmo, map_size, &addr,
+                                  &vmar_used);
   if (status == ZX_OK) {
     uintptr_t map_addr = addr;
     uintptr_t map_end = map_addr + map_size;
 @@ -325,12 +382,12 @@ void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment,
     uintptr_t end = addr + size;
     if (addr != map_addr) {
       zx_info_vmar_t info;
-      status = _zx_object_get_info(_zx_vmar_root_self(), ZX_INFO_VMAR, &info,
+      status = _zx_object_get_info(vmar_used, ZX_INFO_VMAR, &info, sizeof(info),
-                                   sizeof(info), NULL, NULL);
+                                   NULL, NULL);
       if (status == ZX_OK) {
         uintptr_t new_addr;
         status = _zx_vmar_map(
-            _zx_vmar_root_self(),
+            vmar_used,
             ZX_VM_PERM_READ | ZX_VM_PERM_WRITE | ZX_VM_SPECIFIC_OVERWRITE,
             addr - info.base, vmo, 0, size, &new_addr);
         if (status == ZX_OK)
 @@ -338,9 +395,9 @@ void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment,
       }
     }
     if (status == ZX_OK && addr != map_addr)
-      status = _zx_vmar_unmap(_zx_vmar_root_self(), map_addr, addr - map_addr);
+      status = _zx_vmar_unmap(vmar_used, map_addr, addr - map_addr);
     if (status == ZX_OK && end != map_end)
-      status = _zx_vmar_unmap(_zx_vmar_root_self(), end, map_end - end);
+      status = _zx_vmar_unmap(vmar_used, end, map_end - end);
   }
   _zx_handle_close(vmo);
 @@ -355,8 +412,8 @@ void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment,
   return reinterpret_cast<void *>(addr);
 }
-void UnmapOrDie(void *addr, uptr size) {
+void UnmapOrDie(void *addr, uptr size, bool raw_report) {
-  UnmapOrDieVmar(addr, size, _zx_vmar_root_self());
+  UnmapOrDieVmar(addr, size, gSanitizerHeapVmar, raw_report);
 }
 void ReleaseMemoryPagesToOS(uptr beg, uptr end) {

lib/tsan/sanitizer_common/sanitizer_hash.h+1-1

...	@@ -62,6 +62,6 @@ class MurMur2Hash64Builder {	...	@@ -62,6 +62,6 @@ class MurMur2Hash64Builder {
62	return x;	62	return x;
63	}	63	}
64	};	64	};
65	} //namespace __sanitizer	65	} // namespace __sanitizer
66		66
67	#endif // SANITIZER_HASH_H	67	#endif // SANITIZER_HASH_H

lib/tsan/sanitizer_common/sanitizer_internal_defs.h+25-9

 @@ -15,6 +15,11 @@
 #include "sanitizer_platform.h"
 #include "sanitizer_redefine_builtins.h"
+// GCC does not understand __has_feature.
+#if !defined(__has_feature)
+#define __has_feature(x) 0
+#endif
 #ifndef SANITIZER_DEBUG
 # define SANITIZER_DEBUG 0
 #endif
 @@ -30,13 +35,20 @@
 # define SANITIZER_INTERFACE_ATTRIBUTE __declspec(dllexport)
 #endif
 # define SANITIZER_WEAK_ATTRIBUTE
+#  define SANITIZER_WEAK_IMPORT
 #elif SANITIZER_GO
 # define SANITIZER_INTERFACE_ATTRIBUTE
 # define SANITIZER_WEAK_ATTRIBUTE
+#  define SANITIZER_WEAK_IMPORT
 #else
 # define SANITIZER_INTERFACE_ATTRIBUTE __attribute__((visibility("default")))
 # define SANITIZER_WEAK_ATTRIBUTE  __attribute__((weak))
-#endif
+#  if SANITIZER_APPLE
+#    define SANITIZER_WEAK_IMPORT extern "C" __attribute((weak_import))
+#  else
+#    define SANITIZER_WEAK_IMPORT extern "C" SANITIZER_WEAK_ATTRIBUTE
+#  endif  // SANITIZER_APPLE
+#endif    // SANITIZER_WINDOWS
 //--------------------------- WEAK FUNCTIONS ---------------------------------//
 // When working with weak functions, to simplify the code and make it more
 @@ -179,15 +191,19 @@ typedef uptr OFF_T;
 #endif
 typedef u64  OFF64_T;
-#if (SANITIZER_WORDSIZE == 64) || SANITIZER_APPLE
+#ifdef __SIZE_TYPE__
-typedef uptr operator_new_size_type;
+typedef __SIZE_TYPE__ usize;
 #else
-# if defined(__s390__) && !defined(__s390x__)
+// Since we use this for operator new, usize must match the real size_t, but on
-// Special case: 31-bit s390 has unsigned long as size_t.
+// 32-bit Windows the definition of uptr does not actually match uintptr_t or
-typedef unsigned long operator_new_size_type;
+// size_t because we are working around typedef mismatches for the (S)SIZE_T
-# else
+// types used in interception.h.
-typedef u32 operator_new_size_type;
+// Until the definition of uptr has been fixed we have to special case Win32.
-# endif
+#  if SANITIZER_WINDOWS && SANITIZER_WORDSIZE == 32
+typedef unsigned int usize;
+#  else
+typedef uptr usize;
+#  endif
 #endif
 typedef u64 tid_t;

lib/tsan/sanitizer_common/sanitizer_libc.cpp+16

...	@@ -199,6 +199,14 @@ char internal_strncat(char dst, const char *src, uptr n) {	...	@@ -199,6 +199,14 @@ char internal_strncat(char dst, const char *src, uptr n) {
199	return dst;	199	return dst;
200	}	200	}
201		201
		202	wchar_t internal_wcscpy(wchar_t dst, const wchar_t *src) {
		203	wchar_t *dst_it = dst;
		204	do {
		205	dst_it++ = src++;
		206	} while (*src);
		207	return dst;
		208	}
		209
202	uptr internal_strlcpy(char dst, const char src, uptr maxlen) {	210	uptr internal_strlcpy(char dst, const char src, uptr maxlen) {
203	const uptr srclen = internal_strlen(src);	211	const uptr srclen = internal_strlen(src);
204	if (srclen < maxlen) {	212	if (srclen < maxlen) {
...	@@ -218,6 +226,14 @@ char internal_strncpy(char dst, const char *src, uptr n) {	...	@@ -218,6 +226,14 @@ char internal_strncpy(char dst, const char *src, uptr n) {
218	return dst;	226	return dst;
219	}	227	}
220		228
		229	wchar_t internal_wcsncpy(wchar_t dst, const wchar_t *src, uptr n) {
		230	uptr i;
		231	for (i = 0; i < n && src[i]; ++i)
		232	dst[i] = src[i];
		233	internal_memset(dst + i, 0, (n - i) * sizeof(wchar_t));
		234	return dst;
		235	}
		236
221	uptr internal_strnlen(const char *s, uptr maxlen) {	237	uptr internal_strnlen(const char *s, uptr maxlen) {
222	uptr i = 0;	238	uptr i = 0;
223	while (i < maxlen && s[i]) i++;	239	while (i < maxlen && s[i]) i++;

lib/tsan/sanitizer_common/sanitizer_libc.h+2-1

...	@@ -71,7 +71,8 @@ int internal_snprintf(char buffer, uptr length, const char format, ...)	...	@@ -71,7 +71,8 @@ int internal_snprintf(char buffer, uptr length, const char format, ...)
71	FORMAT(3, 4);	71	FORMAT(3, 4);
72	uptr internal_wcslen(const wchar_t *s);	72	uptr internal_wcslen(const wchar_t *s);
73	uptr internal_wcsnlen(const wchar_t *s, uptr maxlen);	73	uptr internal_wcsnlen(const wchar_t *s, uptr maxlen);
74		74	wchar_t internal_wcscpy(wchar_t dst, const wchar_t *src);
		75	wchar_t internal_wcsncpy(wchar_t dst, const wchar_t *src, uptr maxlen);
75	// Return true if all bytes in [mem, mem+size) are zero.	76	// Return true if all bytes in [mem, mem+size) are zero.
76	// Optimized for the case when the result is true.	77	// Optimized for the case when the result is true.
77	bool mem_is_zero(const char *mem, uptr size);	78	bool mem_is_zero(const char *mem, uptr size);

lib/tsan/sanitizer_common/sanitizer_libignore.cpp+2-2

...	@@ -105,8 +105,8 @@ void LibIgnore::OnLibraryLoaded(const char *name) {	...	@@ -105,8 +105,8 @@ void LibIgnore::OnLibraryLoaded(const char *name) {
105	continue;	105	continue;
106	if (IsPcInstrumented(range.beg) && IsPcInstrumented(range.end - 1))	106	if (IsPcInstrumented(range.beg) && IsPcInstrumented(range.end - 1))
107	continue;	107	continue;
108	VReport(1, "Adding instrumented range 0x%zx-0x%zx from library '%s'\n",	108	VReport(1, "Adding instrumented range %p-%p from library '%s'\n",
109	range.beg, range.end, mod.full_name());	109	(void )range.beg, (void )range.end, mod.full_name());
110	const uptr idx =	110	const uptr idx =
111	atomic_load(&instrumented_ranges_count_, memory_order_relaxed);	111	atomic_load(&instrumented_ranges_count_, memory_order_relaxed);
112	CHECK_LT(idx, ARRAY_SIZE(instrumented_code_ranges_));	112	CHECK_LT(idx, ARRAY_SIZE(instrumented_code_ranges_));

lib/tsan/sanitizer_common/sanitizer_linux.cpp+1009-833

...	@@ -16,101 +16,105 @@	...	@@ -16,101 +16,105 @@
16	#if SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_NETBSD \|\| \	16	#if SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_NETBSD \|\| \
17	SANITIZER_SOLARIS	17	SANITIZER_SOLARIS
18		18
19	#include "sanitizer_common.h"	19	# include "sanitizer_common.h"
20	#include "sanitizer_flags.h"	20	# include "sanitizer_flags.h"
21	#include "sanitizer_getauxval.h"	21	# include "sanitizer_getauxval.h"
22	#include "sanitizer_internal_defs.h"	22	# include "sanitizer_internal_defs.h"
23	#include "sanitizer_libc.h"	23	# include "sanitizer_libc.h"
24	#include "sanitizer_linux.h"	24	# include "sanitizer_linux.h"
25	#include "sanitizer_mutex.h"	25	# include "sanitizer_mutex.h"
26	#include "sanitizer_placement_new.h"	26	# include "sanitizer_placement_new.h"
27	#include "sanitizer_procmaps.h"	27	# include "sanitizer_procmaps.h"
28		28
29	#if SANITIZER_LINUX && !SANITIZER_GO	29	# if SANITIZER_LINUX && !SANITIZER_GO
30	#include <asm/param.h>	30	# include <asm/param.h>
31	#endif	31	# endif
32		32
33	// For mips64, syscall(__NR_stat) fills the buffer in the 'struct kernel_stat'	33	// For mips64, syscall(__NR_stat) fills the buffer in the 'struct kernel_stat'
34	// format. Struct kernel_stat is defined as 'struct stat' in asm/stat.h. To	34	// format. Struct kernel_stat is defined as 'struct stat' in asm/stat.h. To
35	// access stat from asm/stat.h, without conflicting with definition in	35	// access stat from asm/stat.h, without conflicting with definition in
36	// sys/stat.h, we use this trick.	36	// sys/stat.h, we use this trick. sparc64 is similar, using
37	#if SANITIZER_MIPS64	37	// syscall(__NR_stat64) and struct kernel_stat64.
38	#include <asm/unistd.h>	38	# if SANITIZER_LINUX && (SANITIZER_MIPS64 \|\| SANITIZER_SPARC64)
39	#include <sys/types.h>	39	# include <asm/unistd.h>
40	#define stat kernel_stat	40	# include <sys/types.h>
41	#if SANITIZER_GO	41	# define stat kernel_stat
42	#undef st_atime	42	# if SANITIZER_SPARC64
43	#undef st_mtime	43	# define stat64 kernel_stat64
44	#undef st_ctime	44	# endif
45	#define st_atime st_atim	45	# if SANITIZER_GO
46	#define st_mtime st_mtim	46	# undef st_atime
47	#define st_ctime st_ctim	47	# undef st_mtime
48	#endif	48	# undef st_ctime
49	#include <asm/stat.h>	49	# define st_atime st_atim
50	#undef stat	50	# define st_mtime st_mtim
51	#endif	51	# define st_ctime st_ctim
		52	# endif
		53	# include <asm/stat.h>
		54	# undef stat
		55	# undef stat64
		56	# endif
52		57
53	#include <dlfcn.h>	58	# include <dlfcn.h>
54	#include <errno.h>	59	# include <errno.h>
55	#include <fcntl.h>	60	# include <fcntl.h>
56	#include <link.h>	61	# include <link.h>
57	#include <pthread.h>	62	# include <pthread.h>
58	#include <sched.h>	63	# include <sched.h>
59	#include <signal.h>	64	# include <signal.h>
60	#include <sys/mman.h>	65	# include <sys/mman.h>
61	#include <sys/param.h>	66	# if !SANITIZER_SOLARIS
62	#if !SANITIZER_SOLARIS	67	# include <sys/ptrace.h>
63	#include <sys/ptrace.h>	68	# endif
64	#endif	69	# include <sys/resource.h>
65	#include <sys/resource.h>	70	# include <sys/stat.h>
66	#include <sys/stat.h>	71	# include <sys/syscall.h>
67	#include <sys/syscall.h>	72	# include <sys/time.h>
68	#include <sys/time.h>	73	# include <sys/types.h>
69	#include <sys/types.h>	74	# include <ucontext.h>
70	#include <ucontext.h>	75	# include <unistd.h>
71	#include <unistd.h>
72
73	#if SANITIZER_LINUX
74	#include <sys/utsname.h>
75	#endif
76		76
77	#if SANITIZER_LINUX && !SANITIZER_ANDROID	77	# if SANITIZER_LINUX
78	#include <sys/personality.h>	78	# include <sys/utsname.h>
79	#endif	79	# endif
80		80
81	#if SANITIZER_LINUX && defined(__loongarch__)	81	# if SANITIZER_LINUX && !SANITIZER_ANDROID
82	# include <sys/sysmacros.h>	82	# include <sys/personality.h>
83	#endif	83	# endif
84		84
85	#if SANITIZER_FREEBSD	85	# if SANITIZER_LINUX && defined(__loongarch__)
86	#include <sys/exec.h>	86	# include <sys/sysmacros.h>
87	#include <sys/procctl.h>	87	# endif
88	#include <sys/sysctl.h>	88
89	#include <machine/atomic.h>	89	# if SANITIZER_FREEBSD
		90	# include <machine/atomic.h>
		91	# include <sys/exec.h>
		92	# include <sys/procctl.h>
		93	# include <sys/sysctl.h>
90	extern "C" {	94	extern "C" {
91	// <sys/umtx.h> must be included after <errno.h> and <sys/types.h> on	95	// <sys/umtx.h> must be included after <errno.h> and <sys/types.h> on
92	// FreeBSD 9.2 and 10.0.	96	// FreeBSD 9.2 and 10.0.
93	#include <sys/umtx.h>	97	# include <sys/umtx.h>
94	}	98	}
95	#include <sys/thr.h>	99	# include <sys/thr.h>
96	#endif // SANITIZER_FREEBSD	100	# endif // SANITIZER_FREEBSD
97		101
98	#if SANITIZER_NETBSD	102	# if SANITIZER_NETBSD
99	#include <limits.h> // For NAME_MAX	103	# include <limits.h> // For NAME_MAX
100	#include <sys/sysctl.h>	104	# include <sys/exec.h>
101	#include <sys/exec.h>	105	# include <sys/sysctl.h>
102	extern struct ps_strings *__ps_strings;	106	extern struct ps_strings *__ps_strings;
103	#endif // SANITIZER_NETBSD	107	# endif // SANITIZER_NETBSD
104		108
105	#if SANITIZER_SOLARIS	109	# if SANITIZER_SOLARIS
106	#include <stdlib.h>	110	# include <stdlib.h>
107	#include <thread.h>	111	# include <thread.h>
108	#define environ _environ	112	# define environ _environ
109	#endif	113	# endif
110		114
111	extern char **environ;	115	extern char **environ;
112		116
113	#if SANITIZER_LINUX	117	# if SANITIZER_LINUX
114	// <linux/time.h>	118	// <linux/time.h>
115	struct kernel_timeval {	119	struct kernel_timeval {
116	long tv_sec;	120	long tv_sec;
...	@@ -123,36 +127,32 @@ const int FUTEX_WAKE = 1;	...	@@ -123,36 +127,32 @@ const int FUTEX_WAKE = 1;
123	const int FUTEX_PRIVATE_FLAG = 128;	127	const int FUTEX_PRIVATE_FLAG = 128;
124	const int FUTEX_WAIT_PRIVATE = FUTEX_WAIT \| FUTEX_PRIVATE_FLAG;	128	const int FUTEX_WAIT_PRIVATE = FUTEX_WAIT \| FUTEX_PRIVATE_FLAG;
125	const int FUTEX_WAKE_PRIVATE = FUTEX_WAKE \| FUTEX_PRIVATE_FLAG;	129	const int FUTEX_WAKE_PRIVATE = FUTEX_WAKE \| FUTEX_PRIVATE_FLAG;
126	#endif // SANITIZER_LINUX	130	# endif // SANITIZER_LINUX
127		131
128	// Are we using 32-bit or 64-bit Linux syscalls?	132	// Are we using 32-bit or 64-bit Linux syscalls?
129	// x32 (which defines __x86_64__) has SANITIZER_WORDSIZE == 32	133	// x32 (which defines __x86_64__) has SANITIZER_WORDSIZE == 32
130	// but it still needs to use 64-bit syscalls.	134	// but it still needs to use 64-bit syscalls.
131	#if SANITIZER_LINUX && (defined(__x86_64__) \|\| defined(__powerpc64__) \|\| \	135	# if SANITIZER_LINUX && (defined(__x86_64__) \|\| defined(__powerpc64__) \|\| \
132	SANITIZER_WORDSIZE == 64 \|\| \	136	SANITIZER_WORDSIZE == 64 \|\| \
133	(defined(__mips__) && _MIPS_SIM == _ABIN32))	137	(defined(__mips__) && _MIPS_SIM == _ABIN32))
134	# define SANITIZER_LINUX_USES_64BIT_SYSCALLS 1	138	# define SANITIZER_LINUX_USES_64BIT_SYSCALLS 1
135	#else	139	# else
136	# define SANITIZER_LINUX_USES_64BIT_SYSCALLS 0	140	# define SANITIZER_LINUX_USES_64BIT_SYSCALLS 0
137	#endif	141	# endif
138		142
139	// Note : FreeBSD had implemented both	143	// Note : FreeBSD implemented both Linux and OpenBSD apis.
140	// Linux apis, available from	144	# if SANITIZER_LINUX && defined(__NR_getrandom)
141	// future 12.x version most likely	145	# if !defined(GRND_NONBLOCK)
142	#if SANITIZER_LINUX && defined(__NR_getrandom)	146	# define GRND_NONBLOCK 1
143	# if !defined(GRND_NONBLOCK)	147	# endif
144	# define GRND_NONBLOCK 1	148	# define SANITIZER_USE_GETRANDOM 1
145	# endif	149	# else
146	# define SANITIZER_USE_GETRANDOM 1	150	# define SANITIZER_USE_GETRANDOM 0
147	#else	151	# endif // SANITIZER_LINUX && defined(__NR_getrandom)
148	# define SANITIZER_USE_GETRANDOM 0	152
149	#endif // SANITIZER_LINUX && defined(__NR_getrandom)	153	# if SANITIZER_FREEBSD
150		154	# define SANITIZER_USE_GETENTROPY 1
151	#if SANITIZER_FREEBSD && __FreeBSD_version >= 1200000	155	# endif
152	# define SANITIZER_USE_GETENTROPY 1
153	#else
154	# define SANITIZER_USE_GETENTROPY 0
155	#endif
156		156
157	namespace __sanitizer {	157	namespace __sanitizer {
158		158
...	@@ -160,6 +160,7 @@ void SetSigProcMask(__sanitizer_sigset_t set, __sanitizer_sigset_t oldset) {	...	@@ -160,6 +160,7 @@ void SetSigProcMask(__sanitizer_sigset_t set, __sanitizer_sigset_t oldset) {
160	CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, set, oldset));	160	CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, set, oldset));
161	}	161	}
162		162
		163	// Block asynchronous signals
163	void BlockSignals(__sanitizer_sigset_t *oldset) {	164	void BlockSignals(__sanitizer_sigset_t *oldset) {
164	__sanitizer_sigset_t set;	165	__sanitizer_sigset_t set;
165	internal_sigfillset(&set);	166	internal_sigfillset(&set);
...	@@ -174,7 +175,17 @@ void BlockSignals(__sanitizer_sigset_t *oldset) {	...	@@ -174,7 +175,17 @@ void BlockSignals(__sanitizer_sigset_t *oldset) {
174	// If this signal is blocked, such calls cannot be handled and the process may	175	// If this signal is blocked, such calls cannot be handled and the process may
175	// hang.	176	// hang.
176	internal_sigdelset(&set, 31);	177	internal_sigdelset(&set, 31);
		178
		179	// Don't block synchronous signals
		180	internal_sigdelset(&set, SIGSEGV);
		181	internal_sigdelset(&set, SIGBUS);
		182	internal_sigdelset(&set, SIGILL);
		183	internal_sigdelset(&set, SIGTRAP);
		184	internal_sigdelset(&set, SIGABRT);
		185	internal_sigdelset(&set, SIGFPE);
		186	internal_sigdelset(&set, SIGPIPE);
177	# endif	187	# endif
		188
178	SetSigProcMask(&set, oldset);	189	SetSigProcMask(&set, oldset);
179	}	190	}
180		191
...	@@ -203,33 +214,33 @@ ScopedBlockSignals::~ScopedBlockSignals() { SetSigProcMask(&saved_, nullptr); }	...	@@ -203,33 +214,33 @@ ScopedBlockSignals::~ScopedBlockSignals() { SetSigProcMask(&saved_, nullptr); }
203	# endif	214	# endif
204		215
205	// --------------- sanitizer_libc.h	216	// --------------- sanitizer_libc.h
206	#if !SANITIZER_SOLARIS && !SANITIZER_NETBSD	217	# if !SANITIZER_SOLARIS && !SANITIZER_NETBSD
207	#if !SANITIZER_S390	218	# if !SANITIZER_S390
208	uptr internal_mmap(void *addr, uptr length, int prot, int flags, int fd,	219	uptr internal_mmap(void *addr, uptr length, int prot, int flags, int fd,
209	u64 offset) {	220	u64 offset) {
210	#if SANITIZER_FREEBSD \|\| SANITIZER_LINUX_USES_64BIT_SYSCALLS	221	# if SANITIZER_FREEBSD \|\| SANITIZER_LINUX_USES_64BIT_SYSCALLS
211	return internal_syscall(SYSCALL(mmap), (uptr)addr, length, prot, flags, fd,	222	return internal_syscall(SYSCALL(mmap), (uptr)addr, length, prot, flags, fd,
212	offset);	223	offset);
213	#else	224	# else
214	// mmap2 specifies file offset in 4096-byte units.	225	// mmap2 specifies file offset in 4096-byte units.
215	CHECK(IsAligned(offset, 4096));	226	CHECK(IsAligned(offset, 4096));
216	return internal_syscall(SYSCALL(mmap2), addr, length, prot, flags, fd,	227	return internal_syscall(SYSCALL(mmap2), addr, length, prot, flags, fd,
217	offset / 4096);	228	(OFF_T)(offset / 4096));
218	#endif	229	# endif
219	}	230	}
220	#endif // !SANITIZER_S390	231	# endif // !SANITIZER_S390
221		232
222	uptr internal_munmap(void *addr, uptr length) {	233	uptr internal_munmap(void *addr, uptr length) {
223	return internal_syscall(SYSCALL(munmap), (uptr)addr, length);	234	return internal_syscall(SYSCALL(munmap), (uptr)addr, length);
224	}	235	}
225		236
226	#if SANITIZER_LINUX	237	# if SANITIZER_LINUX
227	uptr internal_mremap(void *old_address, uptr old_size, uptr new_size, int flags,	238	uptr internal_mremap(void *old_address, uptr old_size, uptr new_size, int flags,
228	void *new_address) {	239	void *new_address) {
229	return internal_syscall(SYSCALL(mremap), (uptr)old_address, old_size,	240	return internal_syscall(SYSCALL(mremap), (uptr)old_address, old_size,
230	new_size, flags, (uptr)new_address);	241	new_size, flags, (uptr)new_address);
231	}	242	}
232	#endif	243	# endif
233		244
234	int internal_mprotect(void *addr, uptr length, int prot) {	245	int internal_mprotect(void *addr, uptr length, int prot) {
235	return internal_syscall(SYSCALL(mprotect), (uptr)addr, length, prot);	246	return internal_syscall(SYSCALL(mprotect), (uptr)addr, length, prot);
...	@@ -239,25 +250,23 @@ int internal_madvise(uptr addr, uptr length, int advice) {	...	@@ -239,25 +250,23 @@ int internal_madvise(uptr addr, uptr length, int advice) {
239	return internal_syscall(SYSCALL(madvise), addr, length, advice);	250	return internal_syscall(SYSCALL(madvise), addr, length, advice);
240	}	251	}
241		252
242	uptr internal_close(fd_t fd) {	253	uptr internal_close(fd_t fd) { return internal_syscall(SYSCALL(close), fd); }
243	return internal_syscall(SYSCALL(close), fd);
244	}
245		254
246	uptr internal_open(const char *filename, int flags) {	255	uptr internal_open(const char *filename, int flags) {
247	# if SANITIZER_LINUX	256	# if SANITIZER_LINUX
248	return internal_syscall(SYSCALL(openat), AT_FDCWD, (uptr)filename, flags);	257	return internal_syscall(SYSCALL(openat), AT_FDCWD, (uptr)filename, flags);
249	#else	258	# else
250	return internal_syscall(SYSCALL(open), (uptr)filename, flags);	259	return internal_syscall(SYSCALL(open), (uptr)filename, flags);
251	#endif	260	# endif
252	}	261	}
253		262
254	uptr internal_open(const char *filename, int flags, u32 mode) {	263	uptr internal_open(const char *filename, int flags, u32 mode) {
255	# if SANITIZER_LINUX	264	# if SANITIZER_LINUX
256	return internal_syscall(SYSCALL(openat), AT_FDCWD, (uptr)filename, flags,	265	return internal_syscall(SYSCALL(openat), AT_FDCWD, (uptr)filename, flags,
257	mode);	266	mode);
258	#else	267	# else
259	return internal_syscall(SYSCALL(open), (uptr)filename, flags, mode);	268	return internal_syscall(SYSCALL(open), (uptr)filename, flags, mode);
260	#endif	269	# endif
261	}	270	}
262		271
263	uptr internal_read(fd_t fd, void *buf, uptr count) {	272	uptr internal_read(fd_t fd, void *buf, uptr count) {
...	@@ -276,12 +285,12 @@ uptr internal_write(fd_t fd, const void *buf, uptr count) {	...	@@ -276,12 +285,12 @@ uptr internal_write(fd_t fd, const void *buf, uptr count) {
276		285
277	uptr internal_ftruncate(fd_t fd, uptr size) {	286	uptr internal_ftruncate(fd_t fd, uptr size) {
278	sptr res;	287	sptr res;
279	HANDLE_EINTR(res, (sptr)internal_syscall(SYSCALL(ftruncate), fd,	288	HANDLE_EINTR(res,
280	(OFF_T)size));	289	(sptr)internal_syscall(SYSCALL(ftruncate), fd, (OFF_T)size));
281	return res;	290	return res;
282	}	291	}
283		292
284	#if (!SANITIZER_LINUX_USES_64BIT_SYSCALLS \|\| SANITIZER_SPARC) && SANITIZER_LINUX	293	# if !SANITIZER_LINUX_USES_64BIT_SYSCALLS && SANITIZER_LINUX
285	static void stat64_to_stat(struct stat64 in, struct stat out) {	294	static void stat64_to_stat(struct stat64 in, struct stat out) {
286	internal_memset(out, 0, sizeof(*out));	295	internal_memset(out, 0, sizeof(*out));
287	out->st_dev = in->st_dev;	296	out->st_dev = in->st_dev;
...	@@ -298,9 +307,9 @@ static void stat64_to_stat(struct stat64 in, struct stat out) {	...	@@ -298,9 +307,9 @@ static void stat64_to_stat(struct stat64 in, struct stat out) {
298	out->st_mtime = in->st_mtime;	307	out->st_mtime = in->st_mtime;
299	out->st_ctime = in->st_ctime;	308	out->st_ctime = in->st_ctime;
300	}	309	}
301	#endif	310	# endif
302		311
303	#if SANITIZER_LINUX && defined(__loongarch__)	312	# if SANITIZER_LINUX && defined(__loongarch__)
304	static void statx_to_stat(struct statx in, struct stat out) {	313	static void statx_to_stat(struct statx in, struct stat out) {
305	internal_memset(out, 0, sizeof(*out));	314	internal_memset(out, 0, sizeof(*out));
306	out->st_dev = makedev(in->stx_dev_major, in->stx_dev_minor);	315	out->st_dev = makedev(in->stx_dev_major, in->stx_dev_minor);
...	@@ -320,27 +329,32 @@ static void statx_to_stat(struct statx in, struct stat out) {	...	@@ -320,27 +329,32 @@ static void statx_to_stat(struct statx in, struct stat out) {
320	out->st_ctime = in->stx_ctime.tv_sec;	329	out->st_ctime = in->stx_ctime.tv_sec;
321	out->st_ctim.tv_nsec = in->stx_ctime.tv_nsec;	330	out->st_ctim.tv_nsec = in->stx_ctime.tv_nsec;
322	}	331	}
323	#endif	332	# endif
324		333
325	#if SANITIZER_MIPS64	334	# if SANITIZER_MIPS64 \|\| SANITIZER_SPARC64
		335	# if SANITIZER_MIPS64
		336	typedef struct kernel_stat kstat_t;
		337	# else
		338	typedef struct kernel_stat64 kstat_t;
		339	# endif
326	// Undefine compatibility macros from <sys/stat.h>	340	// Undefine compatibility macros from <sys/stat.h>
327	// so that they would not clash with the kernel_stat	341	// so that they would not clash with the kernel_stat
328	// st_[a\|m\|c]time fields	342	// st_[a\|m\|c]time fields
329	#if !SANITIZER_GO	343	# if !SANITIZER_GO
330	#undef st_atime	344	# undef st_atime
331	#undef st_mtime	345	# undef st_mtime
332	#undef st_ctime	346	# undef st_ctime
333	#endif	347	# endif
334	#if defined(SANITIZER_ANDROID)	348	# if defined(SANITIZER_ANDROID)
335	// Bionic sys/stat.h defines additional macros	349	// Bionic sys/stat.h defines additional macros
336	// for compatibility with the old NDKs and	350	// for compatibility with the old NDKs and
337	// they clash with the kernel_stat structure	351	// they clash with the kernel_stat structure
338	// st_[a\|m\|c]time_nsec fields.	352	// st_[a\|m\|c]time_nsec fields.
339	#undef st_atime_nsec	353	# undef st_atime_nsec
340	#undef st_mtime_nsec	354	# undef st_mtime_nsec
341	#undef st_ctime_nsec	355	# undef st_ctime_nsec
342	#endif	356	# endif
343	static void kernel_stat_to_stat(struct kernel_stat in, struct stat out) {	357	static void kernel_stat_to_stat(kstat_t in, struct stat out) {
344	internal_memset(out, 0, sizeof(*out));	358	internal_memset(out, 0, sizeof(*out));
345	out->st_dev = in->st_dev;	359	out->st_dev = in->st_dev;
346	out->st_ino = in->st_ino;	360	out->st_ino = in->st_ino;
...	@@ -352,96 +366,113 @@ static void kernel_stat_to_stat(struct kernel_stat in, struct stat out) {	...	@@ -352,96 +366,113 @@ static void kernel_stat_to_stat(struct kernel_stat in, struct stat out) {
352	out->st_size = in->st_size;	366	out->st_size = in->st_size;
353	out->st_blksize = in->st_blksize;	367	out->st_blksize = in->st_blksize;
354	out->st_blocks = in->st_blocks;	368	out->st_blocks = in->st_blocks;
355	#if defined(__USE_MISC) \|\| \	369	# if defined(__USE_MISC) \|\| defined(__USE_XOPEN2K8) \|\| \
356	defined(__USE_XOPEN2K8) \|\| \	370	defined(SANITIZER_ANDROID)
357	defined(SANITIZER_ANDROID)
358	out->st_atim.tv_sec = in->st_atime;	371	out->st_atim.tv_sec = in->st_atime;
359	out->st_atim.tv_nsec = in->st_atime_nsec;	372	out->st_atim.tv_nsec = in->st_atime_nsec;
360	out->st_mtim.tv_sec = in->st_mtime;	373	out->st_mtim.tv_sec = in->st_mtime;
361	out->st_mtim.tv_nsec = in->st_mtime_nsec;	374	out->st_mtim.tv_nsec = in->st_mtime_nsec;
362	out->st_ctim.tv_sec = in->st_ctime;	375	out->st_ctim.tv_sec = in->st_ctime;
363	out->st_ctim.tv_nsec = in->st_ctime_nsec;	376	out->st_ctim.tv_nsec = in->st_ctime_nsec;
364	#else	377	# else
365	out->st_atime = in->st_atime;	378	out->st_atime = in->st_atime;
366	out->st_atimensec = in->st_atime_nsec;	379	out->st_atimensec = in->st_atime_nsec;
367	out->st_mtime = in->st_mtime;	380	out->st_mtime = in->st_mtime;
368	out->st_mtimensec = in->st_mtime_nsec;	381	out->st_mtimensec = in->st_mtime_nsec;
369	out->st_ctime = in->st_ctime;	382	out->st_ctime = in->st_ctime;
370	out->st_atimensec = in->st_ctime_nsec;	383	out->st_atimensec = in->st_ctime_nsec;
371	#endif	384	# endif
372	}	385	}
373	#endif	386	# endif
374		387
375	uptr internal_stat(const char path, void buf) {	388	uptr internal_stat(const char path, void buf) {
376	# if SANITIZER_FREEBSD	389	# if SANITIZER_FREEBSD
377	return internal_syscall(SYSCALL(fstatat), AT_FDCWD, (uptr)path, (uptr)buf, 0);	390	return internal_syscall(SYSCALL(fstatat), AT_FDCWD, (uptr)path, (uptr)buf, 0);
378	# elif SANITIZER_LINUX	391	# elif SANITIZER_LINUX
379	# if defined(__loongarch__)	392	# if defined(__loongarch__)
380	struct statx bufx;	393	struct statx bufx;
381	int res = internal_syscall(SYSCALL(statx), AT_FDCWD, (uptr)path,	394	int res = internal_syscall(SYSCALL(statx), AT_FDCWD, (uptr)path,
382	AT_NO_AUTOMOUNT, STATX_BASIC_STATS, (uptr)&bufx);	395	AT_NO_AUTOMOUNT, STATX_BASIC_STATS, (uptr)&bufx);
383	statx_to_stat(&bufx, (struct stat *)buf);	396	statx_to_stat(&bufx, (struct stat *)buf);
384	return res;	397	return res;
385	# elif (SANITIZER_WORDSIZE == 64 \|\| SANITIZER_X32 \|\| \	398	# elif (SANITIZER_WORDSIZE == 64 \|\| SANITIZER_X32 \|\| \
386	(defined(__mips__) && _MIPS_SIM == _ABIN32)) && \	399	(defined(__mips__) && _MIPS_SIM == _ABIN32)) && \
387	!SANITIZER_SPARC	400	!SANITIZER_SPARC
388	return internal_syscall(SYSCALL(newfstatat), AT_FDCWD, (uptr)path, (uptr)buf,	401	return internal_syscall(SYSCALL(newfstatat), AT_FDCWD, (uptr)path, (uptr)buf,
389	0);	402	0);
390	# else	403	# elif SANITIZER_SPARC64
		404	kstat_t buf64;
		405	int res = internal_syscall(SYSCALL(fstatat64), AT_FDCWD, (uptr)path,
		406	(uptr)&buf64, 0);
		407	kernel_stat_to_stat(&buf64, (struct stat *)buf);
		408	return res;
		409	# else
391	struct stat64 buf64;	410	struct stat64 buf64;
392	int res = internal_syscall(SYSCALL(fstatat64), AT_FDCWD, (uptr)path,	411	int res = internal_syscall(SYSCALL(fstatat64), AT_FDCWD, (uptr)path,
393	(uptr)&buf64, 0);	412	(uptr)&buf64, 0);
394	stat64_to_stat(&buf64, (struct stat *)buf);	413	stat64_to_stat(&buf64, (struct stat *)buf);
395	return res;	414	return res;
396	# endif	415	# endif
397	# else	416	# else
398	struct stat64 buf64;	417	struct stat64 buf64;
399	int res = internal_syscall(SYSCALL(stat64), path, &buf64);	418	int res = internal_syscall(SYSCALL(stat64), path, &buf64);
400	stat64_to_stat(&buf64, (struct stat *)buf);	419	stat64_to_stat(&buf64, (struct stat *)buf);
401	return res;	420	return res;
402	# endif	421	# endif
403	}	422	}
404		423
405	uptr internal_lstat(const char path, void buf) {	424	uptr internal_lstat(const char path, void buf) {
406	# if SANITIZER_FREEBSD	425	# if SANITIZER_FREEBSD
407	return internal_syscall(SYSCALL(fstatat), AT_FDCWD, (uptr)path, (uptr)buf,	426	return internal_syscall(SYSCALL(fstatat), AT_FDCWD, (uptr)path, (uptr)buf,
408	AT_SYMLINK_NOFOLLOW);	427	AT_SYMLINK_NOFOLLOW);
409	# elif SANITIZER_LINUX	428	# elif SANITIZER_LINUX
410	# if defined(__loongarch__)	429	# if defined(__loongarch__)
411	struct statx bufx;	430	struct statx bufx;
412	int res = internal_syscall(SYSCALL(statx), AT_FDCWD, (uptr)path,	431	int res = internal_syscall(SYSCALL(statx), AT_FDCWD, (uptr)path,
413	AT_SYMLINK_NOFOLLOW \| AT_NO_AUTOMOUNT,	432	AT_SYMLINK_NOFOLLOW \| AT_NO_AUTOMOUNT,
414	STATX_BASIC_STATS, (uptr)&bufx);	433	STATX_BASIC_STATS, (uptr)&bufx);
415	statx_to_stat(&bufx, (struct stat *)buf);	434	statx_to_stat(&bufx, (struct stat *)buf);
416	return res;	435	return res;
417	# elif (defined(_LP64) \|\| SANITIZER_X32 \|\| \	436	# elif (defined(_LP64) \|\| SANITIZER_X32 \|\| \
418	(defined(__mips__) && _MIPS_SIM == _ABIN32)) && \	437	(defined(__mips__) && _MIPS_SIM == _ABIN32)) && \
419	!SANITIZER_SPARC	438	!SANITIZER_SPARC
420	return internal_syscall(SYSCALL(newfstatat), AT_FDCWD, (uptr)path, (uptr)buf,	439	return internal_syscall(SYSCALL(newfstatat), AT_FDCWD, (uptr)path, (uptr)buf,
421	AT_SYMLINK_NOFOLLOW);	440	AT_SYMLINK_NOFOLLOW);
422	# else	441	# elif SANITIZER_SPARC64
		442	kstat_t buf64;
		443	int res = internal_syscall(SYSCALL(fstatat64), AT_FDCWD, (uptr)path,
		444	(uptr)&buf64, AT_SYMLINK_NOFOLLOW);
		445	kernel_stat_to_stat(&buf64, (struct stat *)buf);
		446	return res;
		447	# else
423	struct stat64 buf64;	448	struct stat64 buf64;
424	int res = internal_syscall(SYSCALL(fstatat64), AT_FDCWD, (uptr)path,	449	int res = internal_syscall(SYSCALL(fstatat64), AT_FDCWD, (uptr)path,
425	(uptr)&buf64, AT_SYMLINK_NOFOLLOW);	450	(uptr)&buf64, AT_SYMLINK_NOFOLLOW);
426	stat64_to_stat(&buf64, (struct stat *)buf);	451	stat64_to_stat(&buf64, (struct stat *)buf);
427	return res;	452	return res;
428	# endif	453	# endif
429	# else	454	# else
430	struct stat64 buf64;	455	struct stat64 buf64;
431	int res = internal_syscall(SYSCALL(lstat64), path, &buf64);	456	int res = internal_syscall(SYSCALL(lstat64), path, &buf64);
432	stat64_to_stat(&buf64, (struct stat *)buf);	457	stat64_to_stat(&buf64, (struct stat *)buf);
433	return res;	458	return res;
434	# endif	459	# endif
435	}	460	}
436		461
437	uptr internal_fstat(fd_t fd, void *buf) {	462	uptr internal_fstat(fd_t fd, void *buf) {
438	#if SANITIZER_FREEBSD \|\| SANITIZER_LINUX_USES_64BIT_SYSCALLS	463	# if SANITIZER_FREEBSD \|\| SANITIZER_LINUX_USES_64BIT_SYSCALLS
439	#if SANITIZER_MIPS64	464	# if SANITIZER_MIPS64
440	// For mips64, fstat syscall fills buffer in the format of kernel_stat	465	// For mips64, fstat syscall fills buffer in the format of kernel_stat
441	struct kernel_stat kbuf;	466	kstat_t kbuf;
442	int res = internal_syscall(SYSCALL(fstat), fd, &kbuf);	467	int res = internal_syscall(SYSCALL(fstat), fd, &kbuf);
443	kernel_stat_to_stat(&kbuf, (struct stat *)buf);	468	kernel_stat_to_stat(&kbuf, (struct stat *)buf);
444	return res;	469	return res;
		470	# elif SANITIZER_LINUX && SANITIZER_SPARC64
		471	// For sparc64, fstat64 syscall fills buffer in the format of kernel_stat64
		472	kstat_t kbuf;
		473	int res = internal_syscall(SYSCALL(fstat64), fd, &kbuf);
		474	kernel_stat_to_stat(&kbuf, (struct stat *)buf);
		475	return res;
445	# elif SANITIZER_LINUX && defined(__loongarch__)	476	# elif SANITIZER_LINUX && defined(__loongarch__)
446	struct statx bufx;	477	struct statx bufx;
447	int res = internal_syscall(SYSCALL(statx), fd, "", AT_EMPTY_PATH,	478	int res = internal_syscall(SYSCALL(statx), fd, "", AT_EMPTY_PATH,
...	@@ -451,12 +482,12 @@ uptr internal_fstat(fd_t fd, void *buf) {	...	@@ -451,12 +482,12 @@ uptr internal_fstat(fd_t fd, void *buf) {
451	# else	482	# else
452	return internal_syscall(SYSCALL(fstat), fd, (uptr)buf);	483	return internal_syscall(SYSCALL(fstat), fd, (uptr)buf);
453	# endif	484	# endif
454	#else	485	# else
455	struct stat64 buf64;	486	struct stat64 buf64;
456	int res = internal_syscall(SYSCALL(fstat64), fd, &buf64);	487	int res = internal_syscall(SYSCALL(fstat64), fd, &buf64);
457	stat64_to_stat(&buf64, (struct stat *)buf);	488	stat64_to_stat(&buf64, (struct stat *)buf);
458	return res;	489	return res;
459	#endif	490	# endif
460	}	491	}
461		492
462	uptr internal_filesize(fd_t fd) {	493	uptr internal_filesize(fd_t fd) {
...	@@ -466,50 +497,46 @@ uptr internal_filesize(fd_t fd) {	...	@@ -466,50 +497,46 @@ uptr internal_filesize(fd_t fd) {
466	return (uptr)st.st_size;	497	return (uptr)st.st_size;
467	}	498	}
468		499
469	uptr internal_dup(int oldfd) {	500	uptr internal_dup(int oldfd) { return internal_syscall(SYSCALL(dup), oldfd); }
470	return internal_syscall(SYSCALL(dup), oldfd);
471	}
472		501
473	uptr internal_dup2(int oldfd, int newfd) {	502	uptr internal_dup2(int oldfd, int newfd) {
474	# if SANITIZER_LINUX	503	# if SANITIZER_LINUX
475	return internal_syscall(SYSCALL(dup3), oldfd, newfd, 0);	504	return internal_syscall(SYSCALL(dup3), oldfd, newfd, 0);
476	#else	505	# else
477	return internal_syscall(SYSCALL(dup2), oldfd, newfd);	506	return internal_syscall(SYSCALL(dup2), oldfd, newfd);
478	#endif	507	# endif
479	}	508	}
480		509
481	uptr internal_readlink(const char path, char buf, uptr bufsize) {	510	uptr internal_readlink(const char path, char buf, uptr bufsize) {
482	# if SANITIZER_LINUX	511	# if SANITIZER_LINUX
483	return internal_syscall(SYSCALL(readlinkat), AT_FDCWD, (uptr)path, (uptr)buf,	512	return internal_syscall(SYSCALL(readlinkat), AT_FDCWD, (uptr)path, (uptr)buf,
484	bufsize);	513	bufsize);
485	#else	514	# else
486	return internal_syscall(SYSCALL(readlink), (uptr)path, (uptr)buf, bufsize);	515	return internal_syscall(SYSCALL(readlink), (uptr)path, (uptr)buf, bufsize);
487	#endif	516	# endif
488	}	517	}
489		518
490	uptr internal_unlink(const char *path) {	519	uptr internal_unlink(const char *path) {
491	# if SANITIZER_LINUX	520	# if SANITIZER_LINUX
492	return internal_syscall(SYSCALL(unlinkat), AT_FDCWD, (uptr)path, 0);	521	return internal_syscall(SYSCALL(unlinkat), AT_FDCWD, (uptr)path, 0);
493	#else	522	# else
494	return internal_syscall(SYSCALL(unlink), (uptr)path);	523	return internal_syscall(SYSCALL(unlink), (uptr)path);
495	#endif	524	# endif
496	}	525	}
497		526
498	uptr internal_rename(const char oldpath, const char newpath) {	527	uptr internal_rename(const char oldpath, const char newpath) {
499	# if (defined(__riscv) \|\| defined(__loongarch__)) && defined(__linux__)	528	# if (defined(__riscv) \|\| defined(__loongarch__)) && defined(__linux__)
500	return internal_syscall(SYSCALL(renameat2), AT_FDCWD, (uptr)oldpath, AT_FDCWD,	529	return internal_syscall(SYSCALL(renameat2), AT_FDCWD, (uptr)oldpath, AT_FDCWD,
501	(uptr)newpath, 0);	530	(uptr)newpath, 0);
502	# elif SANITIZER_LINUX	531	# elif SANITIZER_LINUX
503	return internal_syscall(SYSCALL(renameat), AT_FDCWD, (uptr)oldpath, AT_FDCWD,	532	return internal_syscall(SYSCALL(renameat), AT_FDCWD, (uptr)oldpath, AT_FDCWD,
504	(uptr)newpath);	533	(uptr)newpath);
505	# else	534	# else
506	return internal_syscall(SYSCALL(rename), (uptr)oldpath, (uptr)newpath);	535	return internal_syscall(SYSCALL(rename), (uptr)oldpath, (uptr)newpath);
507	# endif	536	# endif
508	}	537	}
509		538
510	uptr internal_sched_yield() {	539	uptr internal_sched_yield() { return internal_syscall(SYSCALL(sched_yield)); }
511	return internal_syscall(SYSCALL(sched_yield));
512	}
513		540
514	void internal_usleep(u64 useconds) {	541	void internal_usleep(u64 useconds) {
515	struct timespec ts;	542	struct timespec ts;
...	@@ -523,18 +550,18 @@ uptr internal_execve(const char filename, char const argv[],	...	@@ -523,18 +550,18 @@ uptr internal_execve(const char filename, char const argv[],
523	return internal_syscall(SYSCALL(execve), (uptr)filename, (uptr)argv,	550	return internal_syscall(SYSCALL(execve), (uptr)filename, (uptr)argv,
524	(uptr)envp);	551	(uptr)envp);
525	}	552	}
526	#endif // !SANITIZER_SOLARIS && !SANITIZER_NETBSD	553	# endif // !SANITIZER_SOLARIS && !SANITIZER_NETBSD
527		554
528	#if !SANITIZER_NETBSD	555	# if !SANITIZER_NETBSD
529	void internal__exit(int exitcode) {	556	void internal__exit(int exitcode) {
530	#if SANITIZER_FREEBSD \|\| SANITIZER_SOLARIS	557	# if SANITIZER_FREEBSD \|\| SANITIZER_SOLARIS
531	internal_syscall(SYSCALL(exit), exitcode);	558	internal_syscall(SYSCALL(exit), exitcode);
532	#else	559	# else
533	internal_syscall(SYSCALL(exit_group), exitcode);	560	internal_syscall(SYSCALL(exit_group), exitcode);
534	#endif	561	# endif
535	Die(); // Unreachable.	562	Die(); // Unreachable.
536	}	563	}
537	#endif // !SANITIZER_NETBSD	564	# endif // !SANITIZER_NETBSD
538		565
539	// ----------------- sanitizer_common.h	566	// ----------------- sanitizer_common.h
540	bool FileExists(const char *filename) {	567	bool FileExists(const char *filename) {
...	@@ -556,30 +583,32 @@ bool DirExists(const char *path) {	...	@@ -556,30 +583,32 @@ bool DirExists(const char *path) {
556		583
557	# if !SANITIZER_NETBSD	584	# if !SANITIZER_NETBSD
558	tid_t GetTid() {	585	tid_t GetTid() {
559	#if SANITIZER_FREEBSD	586	# if SANITIZER_FREEBSD
560	long Tid;	587	long Tid;
561	thr_self(&Tid);	588	thr_self(&Tid);
562	return Tid;	589	return Tid;
563	#elif SANITIZER_SOLARIS	590	# elif SANITIZER_SOLARIS
564	return thr_self();	591	return thr_self();
565	#else	592	# else
566	return internal_syscall(SYSCALL(gettid));	593	return internal_syscall(SYSCALL(gettid));
567	#endif	594	# endif
568	}	595	}
569		596
570	int TgKill(pid_t pid, tid_t tid, int sig) {	597	int TgKill(pid_t pid, tid_t tid, int sig) {
571	#if SANITIZER_LINUX	598	# if SANITIZER_LINUX
572	return internal_syscall(SYSCALL(tgkill), pid, tid, sig);	599	return internal_syscall(SYSCALL(tgkill), pid, tid, sig);
573	#elif SANITIZER_FREEBSD	600	# elif SANITIZER_FREEBSD
574	return internal_syscall(SYSCALL(thr_kill2), pid, tid, sig);	601	return internal_syscall(SYSCALL(thr_kill2), pid, tid, sig);
575	#elif SANITIZER_SOLARIS	602	# elif SANITIZER_SOLARIS
576	(void)pid;	603	(void)pid;
577	return thr_kill(tid, sig);	604	errno = thr_kill(tid, sig);
578	#endif	605	// TgKill is expected to return -1 on error, not an errno.
		606	return errno != 0 ? -1 : 0;
		607	# endif
579	}	608	}
580	#endif	609	# endif
581		610
582	#if SANITIZER_GLIBC	611	# if SANITIZER_GLIBC
583	u64 NanoTime() {	612	u64 NanoTime() {
584	kernel_timeval tv;	613	kernel_timeval tv;
585	internal_memset(&tv, 0, sizeof(tv));	614	internal_memset(&tv, 0, sizeof(tv));
...	@@ -590,19 +619,19 @@ u64 NanoTime() {	...	@@ -590,19 +619,19 @@ u64 NanoTime() {
590	uptr internal_clock_gettime(__sanitizer_clockid_t clk_id, void *tp) {	619	uptr internal_clock_gettime(__sanitizer_clockid_t clk_id, void *tp) {
591	return internal_syscall(SYSCALL(clock_gettime), clk_id, tp);	620	return internal_syscall(SYSCALL(clock_gettime), clk_id, tp);
592	}	621	}
593	#elif !SANITIZER_SOLARIS && !SANITIZER_NETBSD	622	# elif !SANITIZER_SOLARIS && !SANITIZER_NETBSD
594	u64 NanoTime() {	623	u64 NanoTime() {
595	struct timespec ts;	624	struct timespec ts;
596	clock_gettime(CLOCK_REALTIME, &ts);	625	clock_gettime(CLOCK_REALTIME, &ts);
597	return (u64)ts.tv_sec * 1000 * 1000 * 1000 + ts.tv_nsec;	626	return (u64)ts.tv_sec * 1000 * 1000 * 1000 + ts.tv_nsec;
598	}	627	}
599	#endif	628	# endif
600		629
601	// Like getenv, but reads env directly from /proc (on Linux) or parses the	630	// Like getenv, but reads env directly from /proc (on Linux) or parses the
602	// 'environ' array (on some others) and does not use libc. This function	631	// 'environ' array (on some others) and does not use libc. This function
603	// should be called first inside __asan_init.	632	// should be called first inside __asan_init.
604	const char GetEnv(const char name) {	633	const char GetEnv(const char name) {
605	#if SANITIZER_FREEBSD \|\| SANITIZER_NETBSD \|\| SANITIZER_SOLARIS	634	# if SANITIZER_FREEBSD \|\| SANITIZER_NETBSD \|\| SANITIZER_SOLARIS
606	if (::environ != 0) {	635	if (::environ != 0) {
607	uptr NameLen = internal_strlen(name);	636	uptr NameLen = internal_strlen(name);
608	for (char *Env = ::environ; Env != 0; Env++) {	637	for (char *Env = ::environ; Env != 0; Env++) {
...	@@ -611,7 +640,7 @@ const char GetEnv(const char name) {	...	@@ -611,7 +640,7 @@ const char GetEnv(const char name) {
611	}	640	}
612	}	641	}
613	return 0; // Not found.	642	return 0; // Not found.
614	#elif SANITIZER_LINUX	643	# elif SANITIZER_LINUX
615	static char *environ;	644	static char *environ;
616	static uptr len;	645	static uptr len;
617	static bool inited;	646	static bool inited;
...	@@ -621,13 +650,13 @@ const char GetEnv(const char name) {	...	@@ -621,13 +650,13 @@ const char GetEnv(const char name) {
621	if (!ReadFileToBuffer("/proc/self/environ", &environ, &environ_size, &len))	650	if (!ReadFileToBuffer("/proc/self/environ", &environ, &environ_size, &len))
622	environ = nullptr;	651	environ = nullptr;
623	}	652	}
624	if (!environ \|\| len == 0) return nullptr;	653	if (!environ \|\| len == 0)
		654	return nullptr;
625	uptr namelen = internal_strlen(name);	655	uptr namelen = internal_strlen(name);
626	const char *p = environ;	656	const char *p = environ;
627	while (*p != '\0') { // will happen at the \0\0 that terminates the buffer	657	while (*p != '\0') { // will happen at the \0\0 that terminates the buffer
628	// proc file has the format NAME=value\0NAME=value\0NAME=value\0...	658	// proc file has the format NAME=value\0NAME=value\0NAME=value\0...
629	const char* endp =	659	const char endp = (char )internal_memchr(p, '\0', len - (p - environ));
630	(char*)internal_memchr(p, '\0', len - (p - environ));
631	if (!endp) // this entry isn't NUL terminated	660	if (!endp) // this entry isn't NUL terminated
632	return nullptr;	661	return nullptr;
633	else if (!internal_memcmp(p, name, namelen) && p[namelen] == '=') // Match.	662	else if (!internal_memcmp(p, name, namelen) && p[namelen] == '=') // Match.
...	@@ -635,18 +664,18 @@ const char GetEnv(const char name) {	...	@@ -635,18 +664,18 @@ const char GetEnv(const char name) {
635	p = endp + 1;	664	p = endp + 1;
636	}	665	}
637	return nullptr; // Not found.	666	return nullptr; // Not found.
638	#else	667	# else
639	#error "Unsupported platform"	668	# error "Unsupported platform"
640	#endif	669	# endif
641	}	670	}
642		671
643	#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD && !SANITIZER_GO	672	# if !SANITIZER_FREEBSD && !SANITIZER_NETBSD && !SANITIZER_GO
644	extern "C" {	673	extern "C" {
645	SANITIZER_WEAK_ATTRIBUTE extern void *__libc_stack_end;	674	SANITIZER_WEAK_ATTRIBUTE extern void *__libc_stack_end;
646	}	675	}
647	#endif	676	# endif
648		677
649	#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD	678	# if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
650	static void ReadNullSepFileToArray(const char path, char **arr,	679	static void ReadNullSepFileToArray(const char path, char **arr,
651	int arr_size) {	680	int arr_size) {
652	char *buff;	681	char *buff;
...	@@ -659,20 +688,21 @@ static void ReadNullSepFileToArray(const char path, char **arr,	...	@@ -659,20 +688,21 @@ static void ReadNullSepFileToArray(const char path, char **arr,
659	}	688	}
660	(*arr)[0] = buff;	689	(*arr)[0] = buff;
661	int count, i;	690	int count, i;
662	for (count = 1, i = 1; ; i++) {	691	for (count = 1, i = 1;; i++) {
663	if (buff[i] == 0) {	692	if (buff[i] == 0) {
664	if (buff[i+1] == 0) break;	693	if (buff[i + 1] == 0)
665	(*arr)[count] = &buff[i+1];	694	break;
		695	(*arr)[count] = &buff[i + 1];
666	CHECK_LE(count, arr_size - 1); // FIXME: make this more flexible.	696	CHECK_LE(count, arr_size - 1); // FIXME: make this more flexible.
667	count++;	697	count++;
668	}	698	}
669	}	699	}
670	(*arr)[count] = nullptr;	700	(*arr)[count] = nullptr;
671	}	701	}
672	#endif	702	# endif
673		703
674	static void GetArgsAndEnv(char *argv, char *envp) {	704	static void GetArgsAndEnv(char *argv, char *envp) {
675	#if SANITIZER_FREEBSD	705	# if SANITIZER_FREEBSD
676	// On FreeBSD, retrieving the argument and environment arrays is done via the	706	// On FreeBSD, retrieving the argument and environment arrays is done via the
677	// kern.ps_strings sysctl, which returns a pointer to a structure containing	707	// kern.ps_strings sysctl, which returns a pointer to a structure containing
678	// this information. See also <sys/exec.h>.	708	// this information. See also <sys/exec.h>.
...	@@ -684,30 +714,30 @@ static void GetArgsAndEnv(char *argv, char *envp) {	...	@@ -684,30 +714,30 @@ static void GetArgsAndEnv(char *argv, char *envp) {
684	}	714	}
685	*argv = pss->ps_argvstr;	715	*argv = pss->ps_argvstr;
686	*envp = pss->ps_envstr;	716	*envp = pss->ps_envstr;
687	#elif SANITIZER_NETBSD	717	# elif SANITIZER_NETBSD
688	*argv = __ps_strings->ps_argvstr;	718	*argv = __ps_strings->ps_argvstr;
689	*envp = __ps_strings->ps_envstr;	719	*envp = __ps_strings->ps_envstr;
690	#else // SANITIZER_FREEBSD	720	# else // SANITIZER_FREEBSD
691	#if !SANITIZER_GO	721	# if !SANITIZER_GO
692	if (&__libc_stack_end) {	722	if (&__libc_stack_end) {
693	uptr* stack_end = (uptr*)__libc_stack_end;	723	uptr stack_end = (uptr )__libc_stack_end;
694	// Normally argc can be obtained from *stack_end, however, on ARM glibc's	724	// Normally argc can be obtained from *stack_end, however, on ARM glibc's
695	// _start clobbers it:	725	// _start clobbers it:
696	// https://sourceware.org/git/?p=glibc.git;a=blob;f=sysdeps/arm/start.S;hb=refs/heads/release/2.31/master#l75	726	// https://sourceware.org/git/?p=glibc.git;a=blob;f=sysdeps/arm/start.S;hb=refs/heads/release/2.31/master#l75
697	// Do not special-case ARM and infer argc from argv everywhere.	727	// Do not special-case ARM and infer argc from argv everywhere.
698	int argc = 0;	728	int argc = 0;
699	while (stack_end[argc + 1]) argc++;	729	while (stack_end[argc + 1]) argc++;
700	argv = (char*)(stack_end + 1);	730	argv = (char *)(stack_end + 1);
701	envp = (char*)(stack_end + argc + 2);	731	envp = (char *)(stack_end + argc + 2);
702	} else {	732	} else {
703	#endif // !SANITIZER_GO	733	# endif // !SANITIZER_GO
704	static const int kMaxArgv = 2000, kMaxEnvp = 2000;	734	static const int kMaxArgv = 2000, kMaxEnvp = 2000;
705	ReadNullSepFileToArray("/proc/self/cmdline", argv, kMaxArgv);	735	ReadNullSepFileToArray("/proc/self/cmdline", argv, kMaxArgv);
706	ReadNullSepFileToArray("/proc/self/environ", envp, kMaxEnvp);	736	ReadNullSepFileToArray("/proc/self/environ", envp, kMaxEnvp);
707	#if !SANITIZER_GO	737	# if !SANITIZER_GO
708	}	738	}
709	#endif // !SANITIZER_GO	739	# endif // !SANITIZER_GO
710	#endif // SANITIZER_FREEBSD	740	# endif // SANITIZER_FREEBSD
711	}	741	}
712		742
713	char **GetArgv() {	743	char **GetArgv() {
...	@@ -722,12 +752,12 @@ char **GetEnviron() {	...	@@ -722,12 +752,12 @@ char **GetEnviron() {
722	return envp;	752	return envp;
723	}	753	}
724		754
725	#if !SANITIZER_SOLARIS	755	# if !SANITIZER_SOLARIS
726	void FutexWait(atomic_uint32_t *p, u32 cmp) {	756	void FutexWait(atomic_uint32_t *p, u32 cmp) {
727	# if SANITIZER_FREEBSD	757	# if SANITIZER_FREEBSD
728	_umtx_op(p, UMTX_OP_WAIT_UINT, cmp, 0, 0);	758	_umtx_op(p, UMTX_OP_WAIT_UINT, cmp, 0, 0);
729	# elif SANITIZER_NETBSD	759	# elif SANITIZER_NETBSD
730	sched_yield(); /* No userspace futex-like synchronization */	760	sched_yield(); /* No userspace futex-like synchronization */
731	# else	761	# else
732	internal_syscall(SYSCALL(futex), (uptr)p, FUTEX_WAIT_PRIVATE, cmp, 0, 0, 0);	762	internal_syscall(SYSCALL(futex), (uptr)p, FUTEX_WAIT_PRIVATE, cmp, 0, 0, 0);
733	# endif	763	# endif
...	@@ -737,7 +767,7 @@ void FutexWake(atomic_uint32_t *p, u32 count) {	...	@@ -737,7 +767,7 @@ void FutexWake(atomic_uint32_t *p, u32 count) {
737	# if SANITIZER_FREEBSD	767	# if SANITIZER_FREEBSD
738	_umtx_op(p, UMTX_OP_WAKE, count, 0, 0);	768	_umtx_op(p, UMTX_OP_WAKE, count, 0, 0);
739	# elif SANITIZER_NETBSD	769	# elif SANITIZER_NETBSD
740	/* No userspace futex-like synchronization */	770	/* No userspace futex-like synchronization */
741	# else	771	# else
742	internal_syscall(SYSCALL(futex), (uptr)p, FUTEX_WAKE_PRIVATE, count, 0, 0, 0);	772	internal_syscall(SYSCALL(futex), (uptr)p, FUTEX_WAKE_PRIVATE, count, 0, 0, 0);
743	# endif	773	# endif
...	@@ -749,26 +779,26 @@ void FutexWake(atomic_uint32_t *p, u32 count) {	...	@@ -749,26 +779,26 @@ void FutexWake(atomic_uint32_t *p, u32 count) {
749	// The actual size of this structure is specified by d_reclen.	779	// The actual size of this structure is specified by d_reclen.
750	// Note that getdents64 uses a different structure format. We only provide the	780	// Note that getdents64 uses a different structure format. We only provide the
751	// 32-bit syscall here.	781	// 32-bit syscall here.
752	#if SANITIZER_NETBSD	782	# if SANITIZER_NETBSD
753	// Not used	783	// Not used
754	#else	784	# else
755	struct linux_dirent {	785	struct linux_dirent {
756	# if SANITIZER_X32 \|\| SANITIZER_LINUX	786	# if SANITIZER_X32 \|\| SANITIZER_LINUX
757	u64 d_ino;	787	u64 d_ino;
758	u64 d_off;	788	u64 d_off;
759	# else	789	# else
760	unsigned long d_ino;	790	unsigned long d_ino;
761	unsigned long d_off;	791	unsigned long d_off;
762	# endif	792	# endif
763	unsigned short d_reclen;	793	unsigned short d_reclen;
764	# if SANITIZER_LINUX	794	# if SANITIZER_LINUX
765	unsigned char d_type;	795	unsigned char d_type;
766	# endif	796	# endif
767	char d_name[256];	797	char d_name[256];
768	};	798	};
769	#endif	799	# endif
770		800
771	#if !SANITIZER_SOLARIS && !SANITIZER_NETBSD	801	# if !SANITIZER_SOLARIS && !SANITIZER_NETBSD
772	// Syscall wrappers.	802	// Syscall wrappers.
773	uptr internal_ptrace(int request, int pid, void addr, void data) {	803	uptr internal_ptrace(int request, int pid, void addr, void data) {
774	return internal_syscall(SYSCALL(ptrace), request, pid, (uptr)addr,	804	return internal_syscall(SYSCALL(ptrace), request, pid, (uptr)addr,
...	@@ -780,24 +810,20 @@ uptr internal_waitpid(int pid, int *status, int options) {	...	@@ -780,24 +810,20 @@ uptr internal_waitpid(int pid, int *status, int options) {
780	0 /* rusage */);	810	0 /* rusage */);
781	}	811	}
782		812
783	uptr internal_getpid() {	813	uptr internal_getpid() { return internal_syscall(SYSCALL(getpid)); }
784	return internal_syscall(SYSCALL(getpid));
785	}
786		814
787	uptr internal_getppid() {	815	uptr internal_getppid() { return internal_syscall(SYSCALL(getppid)); }
788	return internal_syscall(SYSCALL(getppid));
789	}
790		816
791	int internal_dlinfo(void handle, int request, void p) {	817	int internal_dlinfo(void handle, int request, void p) {
792	#if SANITIZER_FREEBSD	818	# if SANITIZER_FREEBSD
793	return dlinfo(handle, request, p);	819	return dlinfo(handle, request, p);
794	#else	820	# else
795	UNIMPLEMENTED();	821	UNIMPLEMENTED();
796	#endif	822	# endif
797	}	823	}
798		824
799	uptr internal_getdents(fd_t fd, struct linux_dirent *dirp, unsigned int count) {	825	uptr internal_getdents(fd_t fd, struct linux_dirent *dirp, unsigned int count) {
800	#if SANITIZER_FREEBSD	826	# if SANITIZER_FREEBSD
801	return internal_syscall(SYSCALL(getdirentries), fd, (uptr)dirp, count, NULL);	827	return internal_syscall(SYSCALL(getdirentries), fd, (uptr)dirp, count, NULL);
802	# elif SANITIZER_LINUX	828	# elif SANITIZER_LINUX
803	return internal_syscall(SYSCALL(getdents64), fd, (uptr)dirp, count);	829	return internal_syscall(SYSCALL(getdents64), fd, (uptr)dirp, count);
...	@@ -810,7 +836,7 @@ uptr internal_lseek(fd_t fd, OFF_T offset, int whence) {	...	@@ -810,7 +836,7 @@ uptr internal_lseek(fd_t fd, OFF_T offset, int whence) {
810	return internal_syscall(SYSCALL(lseek), fd, offset, whence);	836	return internal_syscall(SYSCALL(lseek), fd, offset, whence);
811	}	837	}
812		838
813	#if SANITIZER_LINUX	839	# if SANITIZER_LINUX
814	uptr internal_prctl(int option, uptr arg2, uptr arg3, uptr arg4, uptr arg5) {	840	uptr internal_prctl(int option, uptr arg2, uptr arg3, uptr arg4, uptr arg5) {
815	return internal_syscall(SYSCALL(prctl), option, arg2, arg3, arg4, arg5);	841	return internal_syscall(SYSCALL(prctl), option, arg2, arg3, arg4, arg5);
816	}	842	}
...	@@ -827,10 +853,16 @@ uptr internal_sigaltstack(const void ss, void oss) {	...	@@ -827,10 +853,16 @@ uptr internal_sigaltstack(const void ss, void oss) {
827	return internal_syscall(SYSCALL(sigaltstack), (uptr)ss, (uptr)oss);	853	return internal_syscall(SYSCALL(sigaltstack), (uptr)ss, (uptr)oss);
828	}	854	}
829		855
		856	extern "C" pid_t __fork(void);
		857
830	int internal_fork() {	858	int internal_fork() {
831	# if SANITIZER_LINUX	859	# if SANITIZER_LINUX
832	# if SANITIZER_S390	860	# if SANITIZER_S390
833	return internal_syscall(SYSCALL(clone), 0, SIGCHLD);	861	return internal_syscall(SYSCALL(clone), 0, SIGCHLD);
		862	# elif SANITIZER_SPARC
		863	// The clone syscall interface on SPARC differs massively from the rest,
		864	// so fall back to __fork.
		865	return __fork();
834	# else	866	# else
835	return internal_syscall(SYSCALL(clone), SIGCHLD, 0);	867	return internal_syscall(SYSCALL(clone), SIGCHLD, 0);
836	# endif	868	# endif
...	@@ -839,7 +871,7 @@ int internal_fork() {	...	@@ -839,7 +871,7 @@ int internal_fork() {
839	# endif	871	# endif
840	}	872	}
841		873
842	#if SANITIZER_FREEBSD	874	# if SANITIZER_FREEBSD
843	int internal_sysctl(const int name, unsigned int namelen, void oldp,	875	int internal_sysctl(const int name, unsigned int namelen, void oldp,
844	uptr oldlenp, const void newp, uptr newlen) {	876	uptr oldlenp, const void newp, uptr newlen) {
845	return internal_syscall(SYSCALL(__sysctl), name, namelen, oldp,	877	return internal_syscall(SYSCALL(__sysctl), name, namelen, oldp,
...	@@ -854,11 +886,11 @@ int internal_sysctlbyname(const char sname, void oldp, uptr *oldlenp,	...	@@ -854,11 +886,11 @@ int internal_sysctlbyname(const char sname, void oldp, uptr *oldlenp,
854	// followed by sysctl(). To avoid calling the intercepted version and	886	// followed by sysctl(). To avoid calling the intercepted version and
855	// asserting if this happens during startup, call the real sysctlnametomib()	887	// asserting if this happens during startup, call the real sysctlnametomib()
856	// followed by internal_sysctl() if the syscall is not available.	888	// followed by internal_sysctl() if the syscall is not available.
857	#ifdef SYS___sysctlbyname	889	# ifdef SYS___sysctlbyname
858	return internal_syscall(SYSCALL(__sysctlbyname), sname,	890	return internal_syscall(SYSCALL(__sysctlbyname), sname,
859	internal_strlen(sname), oldp, (size_t *)oldlenp, newp,	891	internal_strlen(sname), oldp, (size_t *)oldlenp, newp,
860	(size_t)newlen);	892	(size_t)newlen);
861	#else	893	# else
862	static decltype(sysctlnametomib) *real_sysctlnametomib = nullptr;	894	static decltype(sysctlnametomib) *real_sysctlnametomib = nullptr;
863	if (!real_sysctlnametomib)	895	if (!real_sysctlnametomib)
864	real_sysctlnametomib =	896	real_sysctlnametomib =
...	@@ -870,12 +902,12 @@ int internal_sysctlbyname(const char sname, void oldp, uptr *oldlenp,	...	@@ -870,12 +902,12 @@ int internal_sysctlbyname(const char sname, void oldp, uptr *oldlenp,
870	if (real_sysctlnametomib(sname, oid, &len) == -1)	902	if (real_sysctlnametomib(sname, oid, &len) == -1)
871	return (-1);	903	return (-1);
872	return internal_sysctl(oid, len, oldp, oldlenp, newp, newlen);	904	return internal_sysctl(oid, len, oldp, oldlenp, newp, newlen);
873	#endif	905	# endif
874	}	906	}
875	#endif	907	# endif
876		908
877	#if SANITIZER_LINUX	909	# if SANITIZER_LINUX
878	#define SA_RESTORER 0x04000000	910	# define SA_RESTORER 0x04000000
879	// Doesn't set sa_restorer if the caller did not set it, so use with caution	911	// Doesn't set sa_restorer if the caller did not set it, so use with caution
880	//(see below).	912	//(see below).
881	int internal_sigaction_norestorer(int signum, const void act, void oldact) {	913	int internal_sigaction_norestorer(int signum, const void act, void oldact) {
...	@@ -899,15 +931,15 @@ int internal_sigaction_norestorer(int signum, const void act, void oldact) {	...	@@ -899,15 +931,15 @@ int internal_sigaction_norestorer(int signum, const void act, void oldact) {
899	// rt_sigaction, so we need to do the same (we'll need to reimplement the	931	// rt_sigaction, so we need to do the same (we'll need to reimplement the
900	// restorers; for x86_64 the restorer address can be obtained from	932	// restorers; for x86_64 the restorer address can be obtained from
901	// oldact->sa_restorer upon a call to sigaction(xxx, NULL, oldact).	933	// oldact->sa_restorer upon a call to sigaction(xxx, NULL, oldact).
902	#if !SANITIZER_ANDROID \|\| !SANITIZER_MIPS32	934	# if !SANITIZER_ANDROID \|\| !SANITIZER_MIPS32
903	k_act.sa_restorer = u_act->sa_restorer;	935	k_act.sa_restorer = u_act->sa_restorer;
904	#endif	936	# endif
905	}	937	}
906		938
907	uptr result = internal_syscall(SYSCALL(rt_sigaction), (uptr)signum,	939	uptr result = internal_syscall(SYSCALL(rt_sigaction), (uptr)signum,
908	(uptr)(u_act ? &k_act : nullptr),	940	(uptr)(u_act ? &k_act : nullptr),
909	(uptr)(u_oldact ? &k_oldact : nullptr),	941	(uptr)(u_oldact ? &k_oldact : nullptr),
910	(uptr)sizeof(__sanitizer_kernel_sigset_t));	942	(uptr)sizeof(__sanitizer_kernel_sigset_t));
911		943
912	if ((result == 0) && u_oldact) {	944	if ((result == 0) && u_oldact) {
913	u_oldact->handler = k_oldact.handler;	945	u_oldact->handler = k_oldact.handler;
...	@@ -915,24 +947,24 @@ int internal_sigaction_norestorer(int signum, const void act, void oldact) {	...	@@ -915,24 +947,24 @@ int internal_sigaction_norestorer(int signum, const void act, void oldact) {
915	internal_memcpy(&u_oldact->sa_mask, &k_oldact.sa_mask,	947	internal_memcpy(&u_oldact->sa_mask, &k_oldact.sa_mask,
916	sizeof(__sanitizer_kernel_sigset_t));	948	sizeof(__sanitizer_kernel_sigset_t));
917	u_oldact->sa_flags = k_oldact.sa_flags;	949	u_oldact->sa_flags = k_oldact.sa_flags;
918	#if !SANITIZER_ANDROID \|\| !SANITIZER_MIPS32	950	# if !SANITIZER_ANDROID \|\| !SANITIZER_MIPS32
919	u_oldact->sa_restorer = k_oldact.sa_restorer;	951	u_oldact->sa_restorer = k_oldact.sa_restorer;
920	#endif	952	# endif
921	}	953	}
922	return result;	954	return result;
923	}	955	}
924	#endif // SANITIZER_LINUX	956	# endif // SANITIZER_LINUX
925		957
926	uptr internal_sigprocmask(int how, __sanitizer_sigset_t *set,	958	uptr internal_sigprocmask(int how, __sanitizer_sigset_t *set,
927	__sanitizer_sigset_t *oldset) {	959	__sanitizer_sigset_t *oldset) {
928	#if SANITIZER_FREEBSD	960	# if SANITIZER_FREEBSD
929	return internal_syscall(SYSCALL(sigprocmask), how, set, oldset);	961	return internal_syscall(SYSCALL(sigprocmask), how, set, oldset);
930	#else	962	# else
931	__sanitizer_kernel_sigset_t k_set = (__sanitizer_kernel_sigset_t )set;	963	__sanitizer_kernel_sigset_t k_set = (__sanitizer_kernel_sigset_t )set;
932	__sanitizer_kernel_sigset_t k_oldset = (__sanitizer_kernel_sigset_t )oldset;	964	__sanitizer_kernel_sigset_t k_oldset = (__sanitizer_kernel_sigset_t )oldset;
933	return internal_syscall(SYSCALL(rt_sigprocmask), (uptr)how, (uptr)k_set,	965	return internal_syscall(SYSCALL(rt_sigprocmask), (uptr)how, (uptr)k_set,
934	(uptr)k_oldset, sizeof(__sanitizer_kernel_sigset_t));	966	(uptr)k_oldset, sizeof(__sanitizer_kernel_sigset_t));
935	#endif	967	# endif
936	}	968	}
937		969
938	void internal_sigfillset(__sanitizer_sigset_t *set) {	970	void internal_sigfillset(__sanitizer_sigset_t *set) {
...	@@ -943,7 +975,7 @@ void internal_sigemptyset(__sanitizer_sigset_t *set) {	...	@@ -943,7 +975,7 @@ void internal_sigemptyset(__sanitizer_sigset_t *set) {
943	internal_memset(set, 0, sizeof(*set));	975	internal_memset(set, 0, sizeof(*set));
944	}	976	}
945		977
946	#if SANITIZER_LINUX	978	# if SANITIZER_LINUX
947	void internal_sigdelset(__sanitizer_sigset_t *set, int signum) {	979	void internal_sigdelset(__sanitizer_sigset_t *set, int signum) {
948	signum -= 1;	980	signum -= 1;
949	CHECK_GE(signum, 0);	981	CHECK_GE(signum, 0);
...	@@ -963,7 +995,7 @@ bool internal_sigismember(__sanitizer_sigset_t *set, int signum) {	...	@@ -963,7 +995,7 @@ bool internal_sigismember(__sanitizer_sigset_t *set, int signum) {
963	const uptr bit = signum % (sizeof(k_set->sig[0]) * 8);	995	const uptr bit = signum % (sizeof(k_set->sig[0]) * 8);
964	return k_set->sig[idx] & ((uptr)1 << bit);	996	return k_set->sig[idx] & ((uptr)1 << bit);
965	}	997	}
966	#elif SANITIZER_FREEBSD	998	# elif SANITIZER_FREEBSD
967	uptr internal_procctl(int type, int id, int cmd, void *data) {	999	uptr internal_procctl(int type, int id, int cmd, void *data) {
968	return internal_syscall(SYSCALL(procctl), type, id, cmd, data);	1000	return internal_syscall(SYSCALL(procctl), type, id, cmd, data);
969	}	1001	}
...	@@ -977,10 +1009,10 @@ bool internal_sigismember(__sanitizer_sigset_t *set, int signum) {	...	@@ -977,10 +1009,10 @@ bool internal_sigismember(__sanitizer_sigset_t *set, int signum) {
977	sigset_t rset = reinterpret_cast<sigset_t >(set);	1009	sigset_t rset = reinterpret_cast<sigset_t >(set);
978	return sigismember(rset, signum);	1010	return sigismember(rset, signum);
979	}	1011	}
980	#endif	1012	# endif
981	#endif // !SANITIZER_SOLARIS	1013	# endif // !SANITIZER_SOLARIS
982		1014
983	#if !SANITIZER_NETBSD	1015	# if !SANITIZER_NETBSD
984	// ThreadLister implementation.	1016	// ThreadLister implementation.
985	ThreadLister::ThreadLister(pid_t pid) : pid_(pid), buffer_(4096) {	1017	ThreadLister::ThreadLister(pid_t pid) : pid_(pid), buffer_(4096) {
986	char task_directory_path[80];	1018	char task_directory_path[80];
...	@@ -1067,25 +1099,26 @@ ThreadLister::~ThreadLister() {	...	@@ -1067,25 +1099,26 @@ ThreadLister::~ThreadLister() {
1067	if (!internal_iserror(descriptor_))	1099	if (!internal_iserror(descriptor_))
1068	internal_close(descriptor_);	1100	internal_close(descriptor_);
1069	}	1101	}
1070	#endif	1102	# endif
1071		1103
1072	#if SANITIZER_WORDSIZE == 32	1104	# if SANITIZER_WORDSIZE == 32
1073	// Take care of unusable kernel area in top gigabyte.	1105	// Take care of unusable kernel area in top gigabyte.
1074	static uptr GetKernelAreaSize() {	1106	static uptr GetKernelAreaSize() {
1075	#if SANITIZER_LINUX && !SANITIZER_X32	1107	# if SANITIZER_LINUX && !SANITIZER_X32
1076	const uptr gbyte = 1UL << 30;	1108	const uptr gbyte = 1UL << 30;
1077		1109
1078	// Firstly check if there are writable segments	1110	// Firstly check if there are writable segments
1079	// mapped to top gigabyte (e.g. stack).	1111	// mapped to top gigabyte (e.g. stack).
1080	MemoryMappingLayout proc_maps(/cache_enabled/true);	1112	MemoryMappingLayout proc_maps(/cache_enabled/ true);
1081	if (proc_maps.Error())	1113	if (proc_maps.Error())
1082	return 0;	1114	return 0;
1083	MemoryMappedSegment segment;	1115	MemoryMappedSegment segment;
1084	while (proc_maps.Next(&segment)) {	1116	while (proc_maps.Next(&segment)) {
1085	if ((segment.end >= 3 * gbyte) && segment.IsWritable()) return 0;	1117	if ((segment.end >= 3 * gbyte) && segment.IsWritable())
		1118	return 0;
1086	}	1119	}
1087		1120
1088	#if !SANITIZER_ANDROID	1121	# if !SANITIZER_ANDROID
1089	// Even if nothing is mapped, top Gb may still be accessible	1122	// Even if nothing is mapped, top Gb may still be accessible
1090	// if we are running on 64-bit kernel.	1123	// if we are running on 64-bit kernel.
1091	// Uname may report misleading results if personality type	1124	// Uname may report misleading results if personality type
...	@@ -1095,21 +1128,22 @@ static uptr GetKernelAreaSize() {	...	@@ -1095,21 +1128,22 @@ static uptr GetKernelAreaSize() {
1095	if (!(pers & PER_MASK) && internal_uname(&uname_info) == 0 &&	1128	if (!(pers & PER_MASK) && internal_uname(&uname_info) == 0 &&
1096	internal_strstr(uname_info.machine, "64"))	1129	internal_strstr(uname_info.machine, "64"))
1097	return 0;	1130	return 0;
1098	#endif // SANITIZER_ANDROID	1131	# endif // SANITIZER_ANDROID
1099		1132
1100	// Top gigabyte is reserved for kernel.	1133	// Top gigabyte is reserved for kernel.
1101	return gbyte;	1134	return gbyte;
1102	#else	1135	# else
1103	return 0;	1136	return 0;
1104	#endif // SANITIZER_LINUX && !SANITIZER_X32	1137	# endif // SANITIZER_LINUX && !SANITIZER_X32
1105	}	1138	}
1106	#endif // SANITIZER_WORDSIZE == 32	1139	# endif // SANITIZER_WORDSIZE == 32
1107		1140
1108	uptr GetMaxVirtualAddress() {	1141	uptr GetMaxVirtualAddress() {
1109	#if SANITIZER_NETBSD && defined(__x86_64__)	1142	# if SANITIZER_NETBSD && defined(__x86_64__)
1110	return 0x7f7ffffff000ULL; // (0x00007f8000000000 - PAGE_SIZE)	1143	return 0x7f7ffffff000ULL; // (0x00007f8000000000 - PAGE_SIZE)
1111	#elif SANITIZER_WORDSIZE == 64	1144	# elif SANITIZER_WORDSIZE == 64
1112	# if defined(__powerpc64__) \|\| defined(__aarch64__) \|\| defined(__loongarch__)	1145	# if defined(__powerpc64__) \|\| defined(__aarch64__) \|\| \
		1146	defined(__loongarch__) \|\| SANITIZER_RISCV64
1113	// On PowerPC64 we have two different address space layouts: 44- and 46-bit.	1147	// On PowerPC64 we have two different address space layouts: 44- and 46-bit.
1114	// We somehow need to figure out which one we are using now and choose	1148	// We somehow need to figure out which one we are using now and choose
1115	// one of 0x00000fffffffffffUL and 0x00003fffffffffffUL.	1149	// one of 0x00000fffffffffffUL and 0x00003fffffffffffUL.
...	@@ -1118,97 +1152,97 @@ uptr GetMaxVirtualAddress() {	...	@@ -1118,97 +1152,97 @@ uptr GetMaxVirtualAddress() {
1118	// This should (does) work for both PowerPC64 Endian modes.	1152	// This should (does) work for both PowerPC64 Endian modes.
1119	// Similarly, aarch64 has multiple address space layouts: 39, 42 and 47-bit.	1153	// Similarly, aarch64 has multiple address space layouts: 39, 42 and 47-bit.
1120	// loongarch64 also has multiple address space layouts: default is 47-bit.	1154	// loongarch64 also has multiple address space layouts: default is 47-bit.
		1155	// RISC-V 64 also has multiple address space layouts: 39, 48 and 57-bit.
1121	return (1ULL << (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1)) - 1;	1156	return (1ULL << (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1)) - 1;
1122	#elif SANITIZER_RISCV64	1157	# elif SANITIZER_MIPS64
1123	return (1ULL << 38) - 1;
1124	# elif SANITIZER_MIPS64
1125	return (1ULL << 40) - 1; // 0x000000ffffffffffUL;	1158	return (1ULL << 40) - 1; // 0x000000ffffffffffUL;
1126	# elif defined(__s390x__)	1159	# elif defined(__s390x__)
1127	return (1ULL << 53) - 1; // 0x001fffffffffffffUL;	1160	return (1ULL << 53) - 1; // 0x001fffffffffffffUL;
1128	#elif defined(__sparc__)	1161	# elif defined(__sparc__)
1129	return ~(uptr)0;	1162	return ~(uptr)0;
1130	# else	1163	# else
1131	return (1ULL << 47) - 1; // 0x00007fffffffffffUL;	1164	return (1ULL << 47) - 1; // 0x00007fffffffffffUL;
1132	# endif	1165	# endif
1133	#else // SANITIZER_WORDSIZE == 32	1166	# else // SANITIZER_WORDSIZE == 32
1134	# if defined(__s390__)	1167	# if defined(__s390__)
1135	return (1ULL << 31) - 1; // 0x7fffffff;	1168	return (1ULL << 31) - 1; // 0x7fffffff;
1136	# else	1169	# else
1137	return (1ULL << 32) - 1; // 0xffffffff;	1170	return (1ULL << 32) - 1; // 0xffffffff;
1138	# endif	1171	# endif
1139	#endif // SANITIZER_WORDSIZE	1172	# endif // SANITIZER_WORDSIZE
1140	}	1173	}
1141		1174
1142	uptr GetMaxUserVirtualAddress() {	1175	uptr GetMaxUserVirtualAddress() {
1143	uptr addr = GetMaxVirtualAddress();	1176	uptr addr = GetMaxVirtualAddress();
1144	#if SANITIZER_WORDSIZE == 32 && !defined(__s390__)	1177	# if SANITIZER_WORDSIZE == 32 && !defined(__s390__)
1145	if (!common_flags()->full_address_space)	1178	if (!common_flags()->full_address_space)
1146	addr -= GetKernelAreaSize();	1179	addr -= GetKernelAreaSize();
1147	CHECK_LT(reinterpret_cast<uptr>(&addr), addr);	1180	CHECK_LT(reinterpret_cast<uptr>(&addr), addr);
1148	#endif	1181	# endif
1149	return addr;	1182	return addr;
1150	}	1183	}
1151		1184
1152	#if !SANITIZER_ANDROID	1185	# if !SANITIZER_ANDROID \|\| defined(__aarch64__)
1153	uptr GetPageSize() {	1186	uptr GetPageSize() {
1154	#if SANITIZER_LINUX && (defined(__x86_64__) \|\| defined(__i386__)) && \	1187	# if SANITIZER_LINUX && (defined(__x86_64__) \|\| defined(__i386__)) && \
1155	defined(EXEC_PAGESIZE)	1188	defined(EXEC_PAGESIZE)
1156	return EXEC_PAGESIZE;	1189	return EXEC_PAGESIZE;
1157	#elif SANITIZER_FREEBSD \|\| SANITIZER_NETBSD	1190	# elif SANITIZER_FREEBSD \|\| SANITIZER_NETBSD
1158	// Use sysctl as sysconf can trigger interceptors internally.	1191	// Use sysctl as sysconf can trigger interceptors internally.
1159	int pz = 0;	1192	int pz = 0;
1160	uptr pzl = sizeof(pz);	1193	uptr pzl = sizeof(pz);
1161	int mib[2] = {CTL_HW, HW_PAGESIZE};	1194	int mib[2] = {CTL_HW, HW_PAGESIZE};
1162	int rv = internal_sysctl(mib, 2, &pz, &pzl, nullptr, 0);	1195	int rv = internal_sysctl(mib, 2, &pz, &pzl, nullptr, 0);
1163	CHECK_EQ(rv, 0);	1196	CHECK_EQ(rv, 0);
1164	return (uptr)pz;	1197	return (uptr)pz;
1165	#elif SANITIZER_USE_GETAUXVAL	1198	# elif SANITIZER_USE_GETAUXVAL
1166	return getauxval(AT_PAGESZ);	1199	return getauxval(AT_PAGESZ);
1167	#else	1200	# else
1168	return sysconf(_SC_PAGESIZE); // EXEC_PAGESIZE may not be trustworthy.	1201	return sysconf(_SC_PAGESIZE); // EXEC_PAGESIZE may not be trustworthy.
1169	#endif	1202	# endif
1170	}	1203	}
1171	#endif // !SANITIZER_ANDROID	1204	# endif
1172		1205
1173	uptr ReadBinaryName(/out/char *buf, uptr buf_len) {	1206	uptr ReadBinaryName(/out/ char *buf, uptr buf_len) {
1174	#if SANITIZER_SOLARIS	1207	# if SANITIZER_SOLARIS
1175	const char *default_module_name = getexecname();	1208	const char *default_module_name = getexecname();
1176	CHECK_NE(default_module_name, NULL);	1209	CHECK_NE(default_module_name, NULL);
1177	return internal_snprintf(buf, buf_len, "%s", default_module_name);	1210	return internal_snprintf(buf, buf_len, "%s", default_module_name);
1178	#else	1211	# else
1179	#if SANITIZER_FREEBSD \|\| SANITIZER_NETBSD	1212	# if SANITIZER_FREEBSD \|\| SANITIZER_NETBSD
1180	#if SANITIZER_FREEBSD	1213	# if SANITIZER_FREEBSD
1181	const int Mib[4] = {CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, -1};	1214	const int Mib[4] = {CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, -1};
1182	#else	1215	# else
1183	const int Mib[4] = {CTL_KERN, KERN_PROC_ARGS, -1, KERN_PROC_PATHNAME};	1216	const int Mib[4] = {CTL_KERN, KERN_PROC_ARGS, -1, KERN_PROC_PATHNAME};
1184	#endif	1217	# endif
1185	const char *default_module_name = "kern.proc.pathname";	1218	const char *default_module_name = "kern.proc.pathname";
1186	uptr Size = buf_len;	1219	uptr Size = buf_len;
1187	bool IsErr =	1220	bool IsErr =
1188	(internal_sysctl(Mib, ARRAY_SIZE(Mib), buf, &Size, NULL, 0) != 0);	1221	(internal_sysctl(Mib, ARRAY_SIZE(Mib), buf, &Size, NULL, 0) != 0);
1189	int readlink_error = IsErr ? errno : 0;	1222	int readlink_error = IsErr ? errno : 0;
1190	uptr module_name_len = Size;	1223	uptr module_name_len = Size;
1191	#else	1224	# else
1192	const char *default_module_name = "/proc/self/exe";	1225	const char *default_module_name = "/proc/self/exe";
1193	uptr module_name_len = internal_readlink(	1226	uptr module_name_len = internal_readlink(default_module_name, buf, buf_len);
1194	default_module_name, buf, buf_len);
1195	int readlink_error;	1227	int readlink_error;
1196	bool IsErr = internal_iserror(module_name_len, &readlink_error);	1228	bool IsErr = internal_iserror(module_name_len, &readlink_error);
1197	#endif // SANITIZER_SOLARIS	1229	# endif
1198	if (IsErr) {	1230	if (IsErr) {
1199	// We can't read binary name for some reason, assume it's unknown.	1231	// We can't read binary name for some reason, assume it's unknown.
1200	Report("WARNING: reading executable name failed with errno %d, "	1232	Report(
1201	"some stack frames may not be symbolized\n", readlink_error);	1233	"WARNING: reading executable name failed with errno %d, "
1202	module_name_len = internal_snprintf(buf, buf_len, "%s",	1234	"some stack frames may not be symbolized\n",
1203	default_module_name);	1235	readlink_error);
		1236	module_name_len =
		1237	internal_snprintf(buf, buf_len, "%s", default_module_name);
1204	CHECK_LT(module_name_len, buf_len);	1238	CHECK_LT(module_name_len, buf_len);
1205	}	1239	}
1206	return module_name_len;	1240	return module_name_len;
1207	#endif	1241	# endif
1208	}	1242	}
1209		1243
1210	uptr ReadLongProcessName(/out/ char *buf, uptr buf_len) {	1244	uptr ReadLongProcessName(/out/ char *buf, uptr buf_len) {
1211	#if SANITIZER_LINUX	1245	# if SANITIZER_LINUX
1212	char *tmpbuf;	1246	char *tmpbuf;
1213	uptr tmpsize;	1247	uptr tmpsize;
1214	uptr tmplen;	1248	uptr tmplen;
...	@@ -1218,7 +1252,7 @@ uptr ReadLongProcessName(/out/ char *buf, uptr buf_len) {	...	@@ -1218,7 +1252,7 @@ uptr ReadLongProcessName(/out/ char *buf, uptr buf_len) {
1218	UnmapOrDie(tmpbuf, tmpsize);	1252	UnmapOrDie(tmpbuf, tmpsize);
1219	return internal_strlen(buf);	1253	return internal_strlen(buf);
1220	}	1254	}
1221	#endif	1255	# endif
1222	return ReadBinaryName(buf, buf_len);	1256	return ReadBinaryName(buf, buf_len);
1223	}	1257	}
1224		1258
...	@@ -1228,20 +1262,22 @@ bool LibraryNameIs(const char full_name, const char base_name) {	...	@@ -1228,20 +1262,22 @@ bool LibraryNameIs(const char full_name, const char base_name) {
1228	// Strip path.	1262	// Strip path.
1229	while (*name != '\0') name++;	1263	while (*name != '\0') name++;
1230	while (name > full_name && *name != '/') name--;	1264	while (name > full_name && *name != '/') name--;
1231	if (*name == '/') name++;	1265	if (*name == '/')
		1266	name++;
1232	uptr base_name_length = internal_strlen(base_name);	1267	uptr base_name_length = internal_strlen(base_name);
1233	if (internal_strncmp(name, base_name, base_name_length)) return false;	1268	if (internal_strncmp(name, base_name, base_name_length))
		1269	return false;
1234	return (name[base_name_length] == '-' \|\| name[base_name_length] == '.');	1270	return (name[base_name_length] == '-' \|\| name[base_name_length] == '.');
1235	}	1271	}
1236		1272
1237	#if !SANITIZER_ANDROID	1273	# if !SANITIZER_ANDROID
1238	// Call cb for each region mapped by map.	1274	// Call cb for each region mapped by map.
1239	void ForEachMappedRegion(link_map map, void (cb)(const void *, uptr)) {	1275	void ForEachMappedRegion(link_map map, void (cb)(const void *, uptr)) {
1240	CHECK_NE(map, nullptr);	1276	CHECK_NE(map, nullptr);
1241	#if !SANITIZER_FREEBSD	1277	# if !SANITIZER_FREEBSD
1242	typedef ElfW(Phdr) Elf_Phdr;	1278	typedef ElfW(Phdr) Elf_Phdr;
1243	typedef ElfW(Ehdr) Elf_Ehdr;	1279	typedef ElfW(Ehdr) Elf_Ehdr;
1244	#endif // !SANITIZER_FREEBSD	1280	# endif // !SANITIZER_FREEBSD
1245	char base = (char )map->l_addr;	1281	char base = (char )map->l_addr;
1246	Elf_Ehdr ehdr = (Elf_Ehdr )base;	1282	Elf_Ehdr ehdr = (Elf_Ehdr )base;
1247	char *phdrs = base + ehdr->e_phoff;	1283	char *phdrs = base + ehdr->e_phoff;
...	@@ -1273,10 +1309,10 @@ void ForEachMappedRegion(link_map map, void (cb)(const void *, uptr)) {	...	@@ -1273,10 +1309,10 @@ void ForEachMappedRegion(link_map map, void (cb)(const void *, uptr)) {
1273	}	1309	}
1274	}	1310	}
1275	}	1311	}
1276	#endif	1312	# endif
1277		1313
1278	#if SANITIZER_LINUX	1314	# if SANITIZER_LINUX
1279	#if defined(__x86_64__)	1315	# if defined(__x86_64__)
1280	// We cannot use glibc's clone wrapper, because it messes with the child	1316	// We cannot use glibc's clone wrapper, because it messes with the child
1281	// task's TLS. It writes the PID and TID of the child task to its thread	1317	// task's TLS. It writes the PID and TID of the child task to its thread
1282	// descriptor, but in our case the child task shares the thread descriptor with	1318	// descriptor, but in our case the child task shares the thread descriptor with
...	@@ -1295,50 +1331,46 @@ uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,	...	@@ -1295,50 +1331,46 @@ uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
1295	register void *r8 __asm__("r8") = newtls;	1331	register void *r8 __asm__("r8") = newtls;
1296	register int *r10 __asm__("r10") = child_tidptr;	1332	register int *r10 __asm__("r10") = child_tidptr;
1297	__asm__ __volatile__(	1333	__asm__ __volatile__(
1298	/* %rax = syscall(%rax = SYSCALL(clone),	1334	/* %rax = syscall(%rax = SYSCALL(clone),
1299	* %rdi = flags,	1335	* %rdi = flags,
1300	* %rsi = child_stack,	1336	* %rsi = child_stack,
1301	* %rdx = parent_tidptr,	1337	* %rdx = parent_tidptr,
1302	* %r8 = new_tls,	1338	* %r8 = new_tls,
1303	* %r10 = child_tidptr)	1339	* %r10 = child_tidptr)
1304	*/	1340	*/
1305	"syscall\n"	1341	"syscall\n"
1306		1342
1307	/* if (%rax != 0)	1343	/* if (%rax != 0)
1308	* return;	1344	* return;
1309	*/	1345	*/
1310	"testq %%rax,%%rax\n"	1346	"testq %%rax,%%rax\n"
1311	"jnz 1f\n"	1347	"jnz 1f\n"
1312		1348
1313	/* In the child. Terminate unwind chain. */	1349	/* In the child. Terminate unwind chain. */
1314	// XXX: We should also terminate the CFI unwind chain	1350	// XXX: We should also terminate the CFI unwind chain
1315	// here. Unfortunately clang 3.2 doesn't support the	1351	// here. Unfortunately clang 3.2 doesn't support the
1316	// necessary CFI directives, so we skip that part.	1352	// necessary CFI directives, so we skip that part.
1317	"xorq %%rbp,%%rbp\n"	1353	"xorq %%rbp,%%rbp\n"
1318		1354
1319	/* Call "fn(arg)". */	1355	/* Call "fn(arg)". */
1320	"popq %%rax\n"	1356	"popq %%rax\n"
1321	"popq %%rdi\n"	1357	"popq %%rdi\n"
1322	"call *%%rax\n"	1358	"call *%%rax\n"
1323		1359
1324	/* Call _exit(%rax). */	1360	/* Call _exit(%rax). */
1325	"movq %%rax,%%rdi\n"	1361	"movq %%rax,%%rdi\n"
1326	"movq %2,%%rax\n"	1362	"movq %2,%%rax\n"
1327	"syscall\n"	1363	"syscall\n"
1328		1364
1329	/* Return to parent. */	1365	/* Return to parent. */
1330	"1:\n"	1366	"1:\n"
1331	: "=a" (res)	1367	: "=a"(res)
1332	: "a"(SYSCALL(clone)), "i"(SYSCALL(exit)),	1368	: "a"(SYSCALL(clone)), "i"(SYSCALL(exit)), "S"(child_stack), "D"(flags),
1333	"S"(child_stack),	1369	"d"(parent_tidptr), "r"(r8), "r"(r10)
1334	"D"(flags),	1370	: "memory", "r11", "rcx");
1335	"d"(parent_tidptr),
1336	"r"(r8),
1337	"r"(r10)
1338	: "memory", "r11", "rcx");
1339	return res;	1371	return res;
1340	}	1372	}
1341	#elif defined(__mips__)	1373	# elif defined(__mips__)
1342	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,	1374	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
1343	int parent_tidptr, void newtls, int *child_tidptr) {	1375	int parent_tidptr, void newtls, int *child_tidptr) {
1344	long long res;	1376	long long res;
...	@@ -1353,68 +1385,63 @@ uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,	...	@@ -1353,68 +1385,63 @@ uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
1353	// We don't have proper CFI directives here because it requires alot of code	1385	// We don't have proper CFI directives here because it requires alot of code
1354	// for very marginal benefits.	1386	// for very marginal benefits.
1355	__asm__ __volatile__(	1387	__asm__ __volatile__(
1356	/* $v0 = syscall($v0 = __NR_clone,	1388	/* $v0 = syscall($v0 = __NR_clone,
1357	* $a0 = flags,	1389	* $a0 = flags,
1358	* $a1 = child_stack,	1390	* $a1 = child_stack,
1359	* $a2 = parent_tidptr,	1391	* $a2 = parent_tidptr,
1360	* $a3 = new_tls,	1392	* $a3 = new_tls,
1361	* $a4 = child_tidptr)	1393	* $a4 = child_tidptr)
1362	*/	1394	*/
1363	".cprestore 16;\n"	1395	".cprestore 16;\n"
1364	"move $4,%1;\n"	1396	"move $4,%1;\n"
1365	"move $5,%2;\n"	1397	"move $5,%2;\n"
1366	"move $6,%3;\n"	1398	"move $6,%3;\n"
1367	"move $7,%4;\n"	1399	"move $7,%4;\n"
1368	/* Store the fifth argument on stack	1400	/* Store the fifth argument on stack
1369	* if we are using 32-bit abi.	1401	* if we are using 32-bit abi.
1370	*/	1402	*/
1371	#if SANITIZER_WORDSIZE == 32	1403	# if SANITIZER_WORDSIZE == 32
1372	"lw %5,16($29);\n"	1404	"lw %5,16($29);\n"
1373	#else	1405	# else
1374	"move $8,%5;\n"	1406	"move $8,%5;\n"
1375	#endif	1407	# endif
1376	"li $2,%6;\n"	1408	"li $2,%6;\n"
1377	"syscall;\n"	1409	"syscall;\n"
1378		1410
1379	/* if ($v0 != 0)	1411	/* if ($v0 != 0)
1380	* return;	1412	* return;
1381	*/	1413	*/
1382	"bnez $2,1f;\n"	1414	"bnez $2,1f;\n"
1383		1415
1384	/* Call "fn(arg)". */	1416	/* Call "fn(arg)". */
1385	#if SANITIZER_WORDSIZE == 32	1417	# if SANITIZER_WORDSIZE == 32
1386	#ifdef __BIG_ENDIAN__	1418	# ifdef __BIG_ENDIAN__
1387	"lw $25,4($29);\n"	1419	"lw $25,4($29);\n"
1388	"lw $4,12($29);\n"	1420	"lw $4,12($29);\n"
1389	#else	1421	# else
1390	"lw $25,0($29);\n"	1422	"lw $25,0($29);\n"
1391	"lw $4,8($29);\n"	1423	"lw $4,8($29);\n"
1392	#endif	1424	# endif
1393	#else	1425	# else
1394	"ld $25,0($29);\n"	1426	"ld $25,0($29);\n"
1395	"ld $4,8($29);\n"	1427	"ld $4,8($29);\n"
1396	#endif	1428	# endif
1397	"jal $25;\n"	1429	"jal $25;\n"
1398		1430
1399	/* Call _exit($v0). */	1431	/* Call _exit($v0). */
1400	"move $4,$2;\n"	1432	"move $4,$2;\n"
1401	"li $2,%7;\n"	1433	"li $2,%7;\n"
1402	"syscall;\n"	1434	"syscall;\n"
1403		1435
1404	/* Return to parent. */	1436	/* Return to parent. */
1405	"1:\n"	1437	"1:\n"
1406	: "=r" (res)	1438	: "=r"(res)
1407	: "r"(flags),	1439	: "r"(flags), "r"(child_stack), "r"(parent_tidptr), "r"(a3), "r"(a4),
1408	"r"(child_stack),	1440	"i"(__NR_clone), "i"(__NR_exit)
1409	"r"(parent_tidptr),	1441	: "memory", "$29");
1410	"r"(a3),
1411	"r"(a4),
1412	"i"(__NR_clone),
1413	"i"(__NR_exit)
1414	: "memory", "$29" );
1415	return res;	1442	return res;
1416	}	1443	}
1417	#elif SANITIZER_RISCV64	1444	# elif SANITIZER_RISCV64
1418	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,	1445	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
1419	int parent_tidptr, void newtls, int *child_tidptr) {	1446	int parent_tidptr, void newtls, int *child_tidptr) {
1420	if (!fn \|\| !child_stack)	1447	if (!fn \|\| !child_stack)
...	@@ -1455,7 +1482,7 @@ uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,	...	@@ -1455,7 +1482,7 @@ uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
1455	: "memory");	1482	: "memory");
1456	return res;	1483	return res;
1457	}	1484	}
1458	#elif defined(__aarch64__)	1485	# elif defined(__aarch64__)
1459	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,	1486	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
1460	int parent_tidptr, void newtls, int *child_tidptr) {	1487	int parent_tidptr, void newtls, int *child_tidptr) {
1461	register long long res __asm__("x0");	1488	register long long res __asm__("x0");
...	@@ -1466,47 +1493,45 @@ uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,	...	@@ -1466,47 +1493,45 @@ uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
1466	((unsigned long long *)child_stack)[0] = (uptr)fn;	1493	((unsigned long long *)child_stack)[0] = (uptr)fn;
1467	((unsigned long long *)child_stack)[1] = (uptr)arg;	1494	((unsigned long long *)child_stack)[1] = (uptr)arg;
1468		1495
1469	register int (__fn)(void ) __asm__("x0") = fn;	1496	register int (__fn)(void ) __asm__("x0") = fn;
1470	register void *__stack __asm__("x1") = child_stack;	1497	register void *__stack __asm__("x1") = child_stack;
1471	register int __flags __asm__("x2") = flags;	1498	register int __flags __asm__("x2") = flags;
1472	register void *__arg __asm__("x3") = arg;	1499	register void *__arg __asm__("x3") = arg;
1473	register int *__ptid __asm__("x4") = parent_tidptr;	1500	register int *__ptid __asm__("x4") = parent_tidptr;
1474	register void *__tls __asm__("x5") = newtls;	1501	register void *__tls __asm__("x5") = newtls;
1475	register int *__ctid __asm__("x6") = child_tidptr;	1502	register int *__ctid __asm__("x6") = child_tidptr;
1476		1503
1477	__asm__ __volatile__(	1504	__asm__ __volatile__(
1478	"mov x0,x2\n" /* flags */	1505	"mov x0,x2\n" /* flags */
1479	"mov x2,x4\n" /* ptid */	1506	"mov x2,x4\n" /* ptid */
1480	"mov x3,x5\n" /* tls */	1507	"mov x3,x5\n" /* tls */
1481	"mov x4,x6\n" /* ctid */	1508	"mov x4,x6\n" /* ctid */
1482	"mov x8,%9\n" /* clone */	1509	"mov x8,%9\n" /* clone */
1483		1510
1484	"svc 0x0\n"	1511	"svc 0x0\n"
1485		1512
1486	/* if (%r0 != 0)	1513	/* if (%r0 != 0)
1487	* return %r0;	1514	* return %r0;
1488	*/	1515	*/
1489	"cmp x0, #0\n"	1516	"cmp x0, #0\n"
1490	"bne 1f\n"	1517	"bne 1f\n"
1491		1518
1492	/* In the child, now. Call "fn(arg)". */	1519	/* In the child, now. Call "fn(arg)". */
1493	"ldp x1, x0, [sp], #16\n"	1520	"ldp x1, x0, [sp], #16\n"
1494	"blr x1\n"	1521	"blr x1\n"
1495		1522
1496	/* Call _exit(%r0). */	1523	/* Call _exit(%r0). */
1497	"mov x8, %10\n"	1524	"mov x8, %10\n"
1498	"svc 0x0\n"	1525	"svc 0x0\n"
1499	"1:\n"	1526	"1:\n"
1500		1527
1501	: "=r" (res)	1528	: "=r"(res)
1502	: "i"(-EINVAL),	1529	: "i"(-EINVAL), "r"(__fn), "r"(__stack), "r"(__flags), "r"(__arg),
1503	"r"(__fn), "r"(__stack), "r"(__flags), "r"(__arg),	1530	"r"(__ptid), "r"(__tls), "r"(__ctid), "i"(__NR_clone), "i"(__NR_exit)
1504	"r"(__ptid), "r"(__tls), "r"(__ctid),	1531	: "x30", "memory");
1505	"i"(__NR_clone), "i"(__NR_exit)
1506	: "x30", "memory");
1507	return res;	1532	return res;
1508	}	1533	}
1509	#elif SANITIZER_LOONGARCH64	1534	# elif SANITIZER_LOONGARCH64
1510	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,	1535	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
1511	int parent_tidptr, void newtls, int *child_tidptr) {	1536	int parent_tidptr, void newtls, int *child_tidptr) {
1512	if (!fn \|\| !child_stack)	1537	if (!fn \|\| !child_stack)
...	@@ -1544,119 +1569,110 @@ uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,	...	@@ -1544,119 +1569,110 @@ uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
1544	: "=r"(res)	1569	: "=r"(res)
1545	: "0"(__flags), "r"(__stack), "r"(__ptid), "r"(__ctid), "r"(__tls),	1570	: "0"(__flags), "r"(__stack), "r"(__ptid), "r"(__ctid), "r"(__tls),
1546	"r"(__fn), "r"(__arg), "r"(nr_clone), "i"(__NR_exit)	1571	"r"(__fn), "r"(__arg), "r"(nr_clone), "i"(__NR_exit)
1547	: "memory", "$t0", "$t1", "$t2", "$t3", "$t4", "$t5", "$t6", "$t7", "$t8");	1572	: "memory", "$t0", "$t1", "$t2", "$t3", "$t4", "$t5", "$t6", "$t7",
		1573	"$t8");
1548	return res;	1574	return res;
1549	}	1575	}
1550	#elif defined(__powerpc64__)	1576	# elif defined(__powerpc64__)
1551	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,	1577	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
1552	int parent_tidptr, void newtls, int *child_tidptr) {	1578	int parent_tidptr, void newtls, int *child_tidptr) {
1553	long long res;	1579	long long res;
1554	// Stack frame structure.	1580	// Stack frame structure.
1555	#if SANITIZER_PPC64V1	1581	# if SANITIZER_PPC64V1
1556	// Back chain == 0 (SP + 112)	1582	// Back chain == 0 (SP + 112)
1557	// Frame (112 bytes):	1583	// Frame (112 bytes):
1558	// Parameter save area (SP + 48), 8 doublewords	1584	// Parameter save area (SP + 48), 8 doublewords
1559	// TOC save area (SP + 40)	1585	// TOC save area (SP + 40)
1560	// Link editor doubleword (SP + 32)	1586	// Link editor doubleword (SP + 32)
1561	// Compiler doubleword (SP + 24)	1587	// Compiler doubleword (SP + 24)
1562	// LR save area (SP + 16)	1588	// LR save area (SP + 16)
1563	// CR save area (SP + 8)	1589	// CR save area (SP + 8)
1564	// Back chain (SP + 0)	1590	// Back chain (SP + 0)
1565	# define FRAME_SIZE 112	1591	# define FRAME_SIZE 112
1566	# define FRAME_TOC_SAVE_OFFSET 40	1592	# define FRAME_TOC_SAVE_OFFSET 40
1567	#elif SANITIZER_PPC64V2	1593	# elif SANITIZER_PPC64V2
1568	// Back chain == 0 (SP + 32)	1594	// Back chain == 0 (SP + 32)
1569	// Frame (32 bytes):	1595	// Frame (32 bytes):
1570	// TOC save area (SP + 24)	1596	// TOC save area (SP + 24)
1571	// LR save area (SP + 16)	1597	// LR save area (SP + 16)
1572	// CR save area (SP + 8)	1598	// CR save area (SP + 8)
1573	// Back chain (SP + 0)	1599	// Back chain (SP + 0)
1574	# define FRAME_SIZE 32	1600	# define FRAME_SIZE 32
1575	# define FRAME_TOC_SAVE_OFFSET 24	1601	# define FRAME_TOC_SAVE_OFFSET 24
1576	#else	1602	# else
1577	# error "Unsupported PPC64 ABI"	1603	# error "Unsupported PPC64 ABI"
1578	#endif	1604	# endif
1579	if (!fn \|\| !child_stack)	1605	if (!fn \|\| !child_stack)
1580	return -EINVAL;	1606	return -EINVAL;
1581	CHECK_EQ(0, (uptr)child_stack % 16);	1607	CHECK_EQ(0, (uptr)child_stack % 16);
1582		1608
1583	register int (__fn)(void ) __asm__("r3") = fn;	1609	register int (__fn)(void ) __asm__("r3") = fn;
1584	register void *__cstack __asm__("r4") = child_stack;	1610	register void *__cstack __asm__("r4") = child_stack;
1585	register int __flags __asm__("r5") = flags;	1611	register int __flags __asm__("r5") = flags;
1586	register void *__arg __asm__("r6") = arg;	1612	register void *__arg __asm__("r6") = arg;
1587	register int *__ptidptr __asm__("r7") = parent_tidptr;	1613	register int *__ptidptr __asm__("r7") = parent_tidptr;
1588	register void *__newtls __asm__("r8") = newtls;	1614	register void *__newtls __asm__("r8") = newtls;
1589	register int *__ctidptr __asm__("r9") = child_tidptr;	1615	register int *__ctidptr __asm__("r9") = child_tidptr;
1590		1616
1591	__asm__ __volatile__(	1617	__asm__ __volatile__(
1592	/* fn and arg are saved across the syscall */	1618	/* fn and arg are saved across the syscall */
1593	"mr 28, %5\n\t"	1619	"mr 28, %5\n\t"
1594	"mr 27, %8\n\t"	1620	"mr 27, %8\n\t"
1595		1621
1596	/* syscall	1622	/* syscall
1597	r0 == __NR_clone	1623	r0 == __NR_clone
1598	r3 == flags	1624	r3 == flags
1599	r4 == child_stack	1625	r4 == child_stack
1600	r5 == parent_tidptr	1626	r5 == parent_tidptr
1601	r6 == newtls	1627	r6 == newtls
1602	r7 == child_tidptr */	1628	r7 == child_tidptr */
1603	"mr 3, %7\n\t"	1629	"mr 3, %7\n\t"
1604	"mr 5, %9\n\t"	1630	"mr 5, %9\n\t"
1605	"mr 6, %10\n\t"	1631	"mr 6, %10\n\t"
1606	"mr 7, %11\n\t"	1632	"mr 7, %11\n\t"
1607	"li 0, %3\n\t"	1633	"li 0, %3\n\t"
1608	"sc\n\t"	1634	"sc\n\t"
1609		1635
1610	/* Test if syscall was successful */	1636	/* Test if syscall was successful */
1611	"cmpdi cr1, 3, 0\n\t"	1637	"cmpdi cr1, 3, 0\n\t"
1612	"crandc cr14+eq, cr14+eq, cr0*4+so\n\t"	1638	"crandc cr14+eq, cr14+eq, cr0*4+so\n\t"
1613	"bne- cr1, 1f\n\t"	1639	"bne- cr1, 1f\n\t"
1614		1640
1615	/* Set up stack frame */	1641	/* Set up stack frame */
1616	"li 29, 0\n\t"	1642	"li 29, 0\n\t"
1617	"stdu 29, -8(1)\n\t"	1643	"stdu 29, -8(1)\n\t"
1618	"stdu 1, -%12(1)\n\t"	1644	"stdu 1, -%12(1)\n\t"
1619	/* Do the function call */	1645	/* Do the function call */
1620	"std 2, %13(1)\n\t"	1646	"std 2, %13(1)\n\t"
1621	#if SANITIZER_PPC64V1	1647	# if SANITIZER_PPC64V1
1622	"ld 0, 0(28)\n\t"	1648	"ld 0, 0(28)\n\t"
1623	"ld 2, 8(28)\n\t"	1649	"ld 2, 8(28)\n\t"
1624	"mtctr 0\n\t"	1650	"mtctr 0\n\t"
1625	#elif SANITIZER_PPC64V2	1651	# elif SANITIZER_PPC64V2
1626	"mr 12, 28\n\t"	1652	"mr 12, 28\n\t"
1627	"mtctr 12\n\t"	1653	"mtctr 12\n\t"
1628	#else	1654	# else
1629	# error "Unsupported PPC64 ABI"	1655	# error "Unsupported PPC64 ABI"
1630	#endif	1656	# endif
1631	"mr 3, 27\n\t"	1657	"mr 3, 27\n\t"
1632	"bctrl\n\t"	1658	"bctrl\n\t"
1633	"ld 2, %13(1)\n\t"	1659	"ld 2, %13(1)\n\t"
1634		1660
1635	/* Call _exit(r3) */	1661	/* Call _exit(r3) */
1636	"li 0, %4\n\t"	1662	"li 0, %4\n\t"
1637	"sc\n\t"	1663	"sc\n\t"
1638		1664
1639	/* Return to parent */	1665	/* Return to parent */
1640	"1:\n\t"	1666	"1:\n\t"
1641	"mr %0, 3\n\t"	1667	"mr %0, 3\n\t"
1642	: "=r" (res)	1668	: "=r"(res)
1643	: "0" (-1),	1669	: "0"(-1), "i"(EINVAL), "i"(__NR_clone), "i"(__NR_exit), "r"(__fn),
1644	"i" (EINVAL),	1670	"r"(__cstack), "r"(__flags), "r"(__arg), "r"(__ptidptr), "r"(__newtls),
1645	"i" (__NR_clone),	1671	"r"(__ctidptr), "i"(FRAME_SIZE), "i"(FRAME_TOC_SAVE_OFFSET)
1646	"i" (__NR_exit),	1672	: "cr0", "cr1", "memory", "ctr", "r0", "r27", "r28", "r29");
1647	"r" (__fn),
1648	"r" (__cstack),
1649	"r" (__flags),
1650	"r" (__arg),
1651	"r" (__ptidptr),
1652	"r" (__newtls),
1653	"r" (__ctidptr),
1654	"i" (FRAME_SIZE),
1655	"i" (FRAME_TOC_SAVE_OFFSET)
1656	: "cr0", "cr1", "memory", "ctr", "r0", "r27", "r28", "r29");
1657	return res;	1673	return res;
1658	}	1674	}
1659	#elif defined(__i386__)	1675	# elif defined(__i386__)
1660	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,	1676	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
1661	int parent_tidptr, void newtls, int *child_tidptr) {	1677	int parent_tidptr, void newtls, int *child_tidptr) {
1662	int res;	1678	int res;
...	@@ -1669,59 +1685,56 @@ uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,	...	@@ -1669,59 +1685,56 @@ uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
1669	((unsigned int *)child_stack)[2] = (uptr)fn;	1685	((unsigned int *)child_stack)[2] = (uptr)fn;
1670	((unsigned int *)child_stack)[3] = (uptr)arg;	1686	((unsigned int *)child_stack)[3] = (uptr)arg;
1671	__asm__ __volatile__(	1687	__asm__ __volatile__(
1672	/* %eax = syscall(%eax = SYSCALL(clone),	1688	/* %eax = syscall(%eax = SYSCALL(clone),
1673	* %ebx = flags,	1689	* %ebx = flags,
1674	* %ecx = child_stack,	1690	* %ecx = child_stack,
1675	* %edx = parent_tidptr,	1691	* %edx = parent_tidptr,
1676	* %esi = new_tls,	1692	* %esi = new_tls,
1677	* %edi = child_tidptr)	1693	* %edi = child_tidptr)
1678	*/	1694	*/
1679		1695
1680	/* Obtain flags */	1696	/* Obtain flags */
1681	"movl (%%ecx), %%ebx\n"	1697	"movl (%%ecx), %%ebx\n"
1682	/* Do the system call */	1698	/* Do the system call */
1683	"pushl %%ebx\n"	1699	"pushl %%ebx\n"
1684	"pushl %%esi\n"	1700	"pushl %%esi\n"
1685	"pushl %%edi\n"	1701	"pushl %%edi\n"
1686	/* Remember the flag value. */	1702	/* Remember the flag value. */
1687	"movl %%ebx, (%%ecx)\n"	1703	"movl %%ebx, (%%ecx)\n"
1688	"int $0x80\n"	1704	"int $0x80\n"
1689	"popl %%edi\n"	1705	"popl %%edi\n"
1690	"popl %%esi\n"	1706	"popl %%esi\n"
1691	"popl %%ebx\n"	1707	"popl %%ebx\n"
1692		1708
1693	/* if (%eax != 0)	1709	/* if (%eax != 0)
1694	* return;	1710	* return;
1695	*/	1711	*/
1696		1712
1697	"test %%eax,%%eax\n"	1713	"test %%eax,%%eax\n"
1698	"jnz 1f\n"	1714	"jnz 1f\n"
1699		1715
1700	/* terminate the stack frame */	1716	/* terminate the stack frame */
1701	"xorl %%ebp,%%ebp\n"	1717	"xorl %%ebp,%%ebp\n"
1702	/* Call FN. */	1718	/* Call FN. */
1703	"call *%%ebx\n"	1719	"call *%%ebx\n"
1704	#ifdef PIC	1720	# ifdef PIC
1705	"call here\n"	1721	"call here\n"
1706	"here:\n"	1722	"here:\n"
1707	"popl %%ebx\n"	1723	"popl %%ebx\n"
1708	"addl $_GLOBAL_OFFSET_TABLE_+[.-here], %%ebx\n"	1724	"addl $_GLOBAL_OFFSET_TABLE_+[.-here], %%ebx\n"
1709	#endif	1725	# endif
1710	/* Call exit */	1726	/* Call exit */
1711	"movl %%eax, %%ebx\n"	1727	"movl %%eax, %%ebx\n"
1712	"movl %2, %%eax\n"	1728	"movl %2, %%eax\n"
1713	"int $0x80\n"	1729	"int $0x80\n"
1714	"1:\n"	1730	"1:\n"
1715	: "=a" (res)	1731	: "=a"(res)
1716	: "a"(SYSCALL(clone)), "i"(SYSCALL(exit)),	1732	: "a"(SYSCALL(clone)), "i"(SYSCALL(exit)), "c"(child_stack),
1717	"c"(child_stack),	1733	"d"(parent_tidptr), "S"(newtls), "D"(child_tidptr)
1718	"d"(parent_tidptr),	1734	: "memory");
1719	"S"(newtls),
1720	"D"(child_tidptr)
1721	: "memory");
1722	return res;	1735	return res;
1723	}	1736	}
1724	#elif defined(__arm__)	1737	# elif defined(__arm__)
1725	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,	1738	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
1726	int parent_tidptr, void newtls, int *child_tidptr) {	1739	int parent_tidptr, void newtls, int *child_tidptr) {
1727	unsigned int res;	1740	unsigned int res;
...	@@ -1737,70 +1750,68 @@ uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,	...	@@ -1737,70 +1750,68 @@ uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
1737	register int *r4 __asm__("r4") = child_tidptr;	1750	register int *r4 __asm__("r4") = child_tidptr;
1738	register int r7 __asm__("r7") = __NR_clone;	1751	register int r7 __asm__("r7") = __NR_clone;
1739		1752
1740	#if __ARM_ARCH > 4 \|\| defined (__ARM_ARCH_4T__)	1753	# if __ARM_ARCH > 4 \|\| defined(__ARM_ARCH_4T__)
1741	# define ARCH_HAS_BX	1754	# define ARCH_HAS_BX
1742	#endif	1755	# endif
1743	#if __ARM_ARCH > 4	1756	# if __ARM_ARCH > 4
1744	# define ARCH_HAS_BLX	1757	# define ARCH_HAS_BLX
1745	#endif	1758	# endif
1746		1759
1747	#ifdef ARCH_HAS_BX	1760	# ifdef ARCH_HAS_BX
1748	# ifdef ARCH_HAS_BLX	1761	# ifdef ARCH_HAS_BLX
1749	# define BLX(R) "blx " #R "\n"	1762	# define BLX(R) "blx " #R "\n"
1750	# else	1763	# else
1751	# define BLX(R) "mov lr, pc; bx " #R "\n"	1764	# define BLX(R) "mov lr, pc; bx " #R "\n"
1752	# endif	1765	# endif
1753	#else	1766	# else
1754	# define BLX(R) "mov lr, pc; mov pc," #R "\n"	1767	# define BLX(R) "mov lr, pc; mov pc," #R "\n"
1755	#endif	1768	# endif
1756		1769
1757	__asm__ __volatile__(	1770	__asm__ __volatile__(
1758	/* %r0 = syscall(%r7 = SYSCALL(clone),	1771	/* %r0 = syscall(%r7 = SYSCALL(clone),
1759	* %r0 = flags,	1772	* %r0 = flags,
1760	* %r1 = child_stack,	1773	* %r1 = child_stack,
1761	* %r2 = parent_tidptr,	1774	* %r2 = parent_tidptr,
1762	* %r3 = new_tls,	1775	* %r3 = new_tls,
1763	* %r4 = child_tidptr)	1776	* %r4 = child_tidptr)
1764	*/	1777	*/
1765		1778
1766	/* Do the system call */	1779	/* Do the system call */
1767	"swi 0x0\n"	1780	"swi 0x0\n"
1768		1781
1769	/* if (%r0 != 0)	1782	/* if (%r0 != 0)
1770	* return %r0;	1783	* return %r0;
1771	*/	1784	*/
1772	"cmp r0, #0\n"	1785	"cmp r0, #0\n"
1773	"bne 1f\n"	1786	"bne 1f\n"
1774		1787
1775	/* In the child, now. Call "fn(arg)". */	1788	/* In the child, now. Call "fn(arg)". */
1776	"ldr r0, [sp, #4]\n"	1789	"ldr r0, [sp, #4]\n"
1777	"ldr ip, [sp], #8\n"	1790	"ldr ip, [sp], #8\n" BLX(ip)
1778	BLX(ip)	1791	/* Call _exit(%r0). */
1779	/* Call _exit(%r0). */	1792	"mov r7, %7\n"
1780	"mov r7, %7\n"	1793	"swi 0x0\n"
1781	"swi 0x0\n"	1794	"1:\n"
1782	"1:\n"	1795	"mov %0, r0\n"
1783	"mov %0, r0\n"	1796	: "=r"(res)
1784	: "=r"(res)	1797	: "r"(r0), "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r7), "i"(__NR_exit)
1785	: "r"(r0), "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r7),	1798	: "memory");
1786	"i"(__NR_exit)
1787	: "memory");
1788	return res;	1799	return res;
1789	}	1800	}
1790	#endif	1801	# endif
1791	#endif // SANITIZER_LINUX	1802	# endif // SANITIZER_LINUX
1792		1803
1793	#if SANITIZER_LINUX	1804	# if SANITIZER_LINUX
1794	int internal_uname(struct utsname *buf) {	1805	int internal_uname(struct utsname *buf) {
1795	return internal_syscall(SYSCALL(uname), buf);	1806	return internal_syscall(SYSCALL(uname), buf);
1796	}	1807	}
1797	#endif	1808	# endif
1798		1809
1799	#if SANITIZER_ANDROID	1810	# if SANITIZER_ANDROID
1800	#if __ANDROID_API__ < 21	1811	# if __ANDROID_API__ < 21
1801	extern "C" __attribute__((weak)) int dl_iterate_phdr(	1812	extern "C" __attribute__((weak)) int dl_iterate_phdr(
1802	int ()(struct dl_phdr_info , size_t, void ), void );	1813	int ()(struct dl_phdr_info , size_t, void ), void );
1803	#endif	1814	# endif
1804		1815
1805	static int dl_iterate_phdr_test_cb(struct dl_phdr_info *info, size_t size,	1816	static int dl_iterate_phdr_test_cb(struct dl_phdr_info *info, size_t size,
1806	void *data) {	1817	void *data) {
...	@@ -1817,40 +1828,41 @@ static int dl_iterate_phdr_test_cb(struct dl_phdr_info *info, size_t size,	...	@@ -1817,40 +1828,41 @@ static int dl_iterate_phdr_test_cb(struct dl_phdr_info *info, size_t size,
1817	static atomic_uint32_t android_api_level;	1828	static atomic_uint32_t android_api_level;
1818		1829
1819	static AndroidApiLevel AndroidDetectApiLevelStatic() {	1830	static AndroidApiLevel AndroidDetectApiLevelStatic() {
1820	#if __ANDROID_API__ <= 19	1831	# if __ANDROID_API__ <= 19
1821	return ANDROID_KITKAT;	1832	return ANDROID_KITKAT;
1822	#elif __ANDROID_API__ <= 22	1833	# elif __ANDROID_API__ <= 22
1823	return ANDROID_LOLLIPOP_MR1;	1834	return ANDROID_LOLLIPOP_MR1;
1824	#else	1835	# else
1825	return ANDROID_POST_LOLLIPOP;	1836	return ANDROID_POST_LOLLIPOP;
1826	#endif	1837	# endif
1827	}	1838	}
1828		1839
1829	static AndroidApiLevel AndroidDetectApiLevel() {	1840	static AndroidApiLevel AndroidDetectApiLevel() {
1830	if (!&dl_iterate_phdr)	1841	if (!&dl_iterate_phdr)
1831	return ANDROID_KITKAT; // K or lower	1842	return ANDROID_KITKAT; // K or lower
1832	bool base_name_seen = false;	1843	bool base_name_seen = false;
1833	dl_iterate_phdr(dl_iterate_phdr_test_cb, &base_name_seen);	1844	dl_iterate_phdr(dl_iterate_phdr_test_cb, &base_name_seen);
1834	if (base_name_seen)	1845	if (base_name_seen)
1835	return ANDROID_LOLLIPOP_MR1; // L MR1	1846	return ANDROID_LOLLIPOP_MR1; // L MR1
1836	return ANDROID_POST_LOLLIPOP; // post-L	1847	return ANDROID_POST_LOLLIPOP; // post-L
1837	// Plain L (API level 21) is completely broken wrt ASan and not very	1848	// Plain L (API level 21) is completely broken wrt ASan and not very
1838	// interesting to detect.	1849	// interesting to detect.
1839	}	1850	}
1840		1851
1841	extern "C" __attribute__((weak)) void* _DYNAMIC;	1852	extern "C" __attribute__((weak)) void *_DYNAMIC;
1842		1853
1843	AndroidApiLevel AndroidGetApiLevel() {	1854	AndroidApiLevel AndroidGetApiLevel() {
1844	AndroidApiLevel level =	1855	AndroidApiLevel level =
1845	(AndroidApiLevel)atomic_load(&android_api_level, memory_order_relaxed);	1856	(AndroidApiLevel)atomic_load(&android_api_level, memory_order_relaxed);
1846	if (level) return level;	1857	if (level)
		1858	return level;
1847	level = &_DYNAMIC == nullptr ? AndroidDetectApiLevelStatic()	1859	level = &_DYNAMIC == nullptr ? AndroidDetectApiLevelStatic()
1848	: AndroidDetectApiLevel();	1860	: AndroidDetectApiLevel();
1849	atomic_store(&android_api_level, level, memory_order_relaxed);	1861	atomic_store(&android_api_level, level, memory_order_relaxed);
1850	return level;	1862	return level;
1851	}	1863	}
1852		1864
1853	#endif	1865	# endif
1854		1866
1855	static HandleSignalMode GetHandleSignalModeImpl(int signum) {	1867	static HandleSignalMode GetHandleSignalModeImpl(int signum) {
1856	switch (signum) {	1868	switch (signum) {
...	@@ -1877,28 +1889,28 @@ HandleSignalMode GetHandleSignalMode(int signum) {	...	@@ -1877,28 +1889,28 @@ HandleSignalMode GetHandleSignalMode(int signum) {
1877	return result;	1889	return result;
1878	}	1890	}
1879		1891
1880	#if !SANITIZER_GO	1892	# if !SANITIZER_GO
1881	void internal_start_thread(void (func)(void arg), void *arg) {	1893	void internal_start_thread(void (func)(void arg), void *arg) {
1882	if (&real_pthread_create == 0)	1894	if (&internal_pthread_create == 0)
1883	return nullptr;	1895	return nullptr;
1884	// Start the thread with signals blocked, otherwise it can steal user signals.	1896	// Start the thread with signals blocked, otherwise it can steal user signals.
1885	ScopedBlockSignals block(nullptr);	1897	ScopedBlockSignals block(nullptr);
1886	void *th;	1898	void *th;
1887	real_pthread_create(&th, nullptr, func, arg);	1899	internal_pthread_create(&th, nullptr, func, arg);
1888	return th;	1900	return th;
1889	}	1901	}
1890		1902
1891	void internal_join_thread(void *th) {	1903	void internal_join_thread(void *th) {
1892	if (&real_pthread_join)	1904	if (&internal_pthread_join)
1893	real_pthread_join(th, nullptr);	1905	internal_pthread_join(th, nullptr);
1894	}	1906	}
1895	#else	1907	# else
1896	void internal_start_thread(void (func)(void ), void *arg) { return 0; }	1908	void internal_start_thread(void (func)(void ), void *arg) { return 0; }
1897		1909
1898	void internal_join_thread(void *th) {}	1910	void internal_join_thread(void *th) {}
1899	#endif	1911	# endif
1900		1912
1901	#if SANITIZER_LINUX && defined(__aarch64__)	1913	# if SANITIZER_LINUX && defined(__aarch64__)
1902	// Android headers in the older NDK releases miss this definition.	1914	// Android headers in the older NDK releases miss this definition.
1903	struct __sanitizer_esr_context {	1915	struct __sanitizer_esr_context {
1904	struct _aarch64_ctx head;	1916	struct _aarch64_ctx head;
...	@@ -1910,7 +1922,8 @@ static bool Aarch64GetESR(ucontext_t ucontext, u64 esr) {	...	@@ -1910,7 +1922,8 @@ static bool Aarch64GetESR(ucontext_t ucontext, u64 esr) {
1910	u8 aux = reinterpret_cast<u8 >(ucontext->uc_mcontext.__reserved);	1922	u8 aux = reinterpret_cast<u8 >(ucontext->uc_mcontext.__reserved);
1911	while (true) {	1923	while (true) {
1912	_aarch64_ctx ctx = (_aarch64_ctx )aux;	1924	_aarch64_ctx ctx = (_aarch64_ctx )aux;
1913	if (ctx->size == 0) break;	1925	if (ctx->size == 0)
		1926	break;
1914	if (ctx->magic == kEsrMagic) {	1927	if (ctx->magic == kEsrMagic) {
1915	esr = ((__sanitizer_esr_context )ctx)->esr;	1928	esr = ((__sanitizer_esr_context )ctx)->esr;
1916	return true;	1929	return true;
...	@@ -1919,31 +1932,29 @@ static bool Aarch64GetESR(ucontext_t ucontext, u64 esr) {	...	@@ -1919,31 +1932,29 @@ static bool Aarch64GetESR(ucontext_t ucontext, u64 esr) {
1919	}	1932	}
1920	return false;	1933	return false;
1921	}	1934	}
1922	#elif SANITIZER_FREEBSD && defined(__aarch64__)	1935	# elif SANITIZER_FREEBSD && defined(__aarch64__)
1923	// FreeBSD doesn't provide ESR in the ucontext.	1936	// FreeBSD doesn't provide ESR in the ucontext.
1924	static bool Aarch64GetESR(ucontext_t ucontext, u64 esr) {	1937	static bool Aarch64GetESR(ucontext_t ucontext, u64 esr) { return false; }
1925	return false;	1938	# endif
1926	}
1927	#endif
1928		1939
1929	using Context = ucontext_t;	1940	using Context = ucontext_t;
1930		1941
1931	SignalContext::WriteFlag SignalContext::GetWriteFlag() const {	1942	SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
1932	Context ucontext = (Context )context;	1943	Context ucontext = (Context )context;
1933	#if defined(__x86_64__) \|\| defined(__i386__)	1944	# if defined(__x86_64__) \|\| defined(__i386__)
1934	static const uptr PF_WRITE = 1U << 1;	1945	static const uptr PF_WRITE = 1U << 1;
1935	#if SANITIZER_FREEBSD	1946	# if SANITIZER_FREEBSD
1936	uptr err = ucontext->uc_mcontext.mc_err;	1947	uptr err = ucontext->uc_mcontext.mc_err;
1937	#elif SANITIZER_NETBSD	1948	# elif SANITIZER_NETBSD
1938	uptr err = ucontext->uc_mcontext.__gregs[_REG_ERR];	1949	uptr err = ucontext->uc_mcontext.__gregs[_REG_ERR];
1939	#elif SANITIZER_SOLARIS && defined(__i386__)	1950	# elif SANITIZER_SOLARIS && defined(__i386__)
1940	const int Err = 13;	1951	const int Err = 13;
1941	uptr err = ucontext->uc_mcontext.gregs[Err];	1952	uptr err = ucontext->uc_mcontext.gregs[Err];
1942	#else	1953	# else
1943	uptr err = ucontext->uc_mcontext.gregs[REG_ERR];	1954	uptr err = ucontext->uc_mcontext.gregs[REG_ERR];
1944	#endif // SANITIZER_FREEBSD	1955	# endif // SANITIZER_FREEBSD
1945	return err & PF_WRITE ? Write : Read;	1956	return err & PF_WRITE ? Write : Read;
1946	#elif defined(__mips__)	1957	# elif defined(__mips__)
1947	uint32_t *exception_source;	1958	uint32_t *exception_source;
1948	uint32_t faulty_instruction;	1959	uint32_t faulty_instruction;
1949	uint32_t op_code;	1960	uint32_t op_code;
...	@@ -1959,12 +1970,12 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {	...	@@ -1959,12 +1970,12 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
1959	case 0x29: // sh	1970	case 0x29: // sh
1960	case 0x2b: // sw	1971	case 0x2b: // sw
1961	case 0x3f: // sd	1972	case 0x3f: // sd
1962	#if __mips_isa_rev < 6	1973	# if __mips_isa_rev < 6
1963	case 0x2c: // sdl	1974	case 0x2c: // sdl
1964	case 0x2d: // sdr	1975	case 0x2d: // sdr
1965	case 0x2a: // swl	1976	case 0x2a: // swl
1966	case 0x2e: // swr	1977	case 0x2e: // swr
1967	#endif	1978	# endif
1968	return SignalContext::Write;	1979	return SignalContext::Write;
1969		1980
1970	case 0x20: // lb	1981	case 0x20: // lb
...	@@ -1974,14 +1985,14 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {	...	@@ -1974,14 +1985,14 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
1974	case 0x23: // lw	1985	case 0x23: // lw
1975	case 0x27: // lwu	1986	case 0x27: // lwu
1976	case 0x37: // ld	1987	case 0x37: // ld
1977	#if __mips_isa_rev < 6	1988	# if __mips_isa_rev < 6
1978	case 0x1a: // ldl	1989	case 0x1a: // ldl
1979	case 0x1b: // ldr	1990	case 0x1b: // ldr
1980	case 0x22: // lwl	1991	case 0x22: // lwl
1981	case 0x26: // lwr	1992	case 0x26: // lwr
1982	#endif	1993	# endif
1983	return SignalContext::Read;	1994	return SignalContext::Read;
1984	#if __mips_isa_rev == 6	1995	# if __mips_isa_rev == 6
1985	case 0x3b: // pcrel	1996	case 0x3b: // pcrel
1986	op_code = (faulty_instruction >> 19) & 0x3;	1997	op_code = (faulty_instruction >> 19) & 0x3;
1987	switch (op_code) {	1998	switch (op_code) {
...	@@ -1989,50 +2000,51 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {	...	@@ -1989,50 +2000,51 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
1989	case 0x2: // lwupc	2000	case 0x2: // lwupc
1990	return SignalContext::Read;	2001	return SignalContext::Read;
1991	}	2002	}
1992	#endif	2003	# endif
1993	}	2004	}
1994	return SignalContext::Unknown;	2005	return SignalContext::Unknown;
1995	#elif defined(__arm__)	2006	# elif defined(__arm__)
1996	static const uptr FSR_WRITE = 1U << 11;	2007	static const uptr FSR_WRITE = 1U << 11;
1997	uptr fsr = ucontext->uc_mcontext.error_code;	2008	uptr fsr = ucontext->uc_mcontext.error_code;
1998	return fsr & FSR_WRITE ? Write : Read;	2009	return fsr & FSR_WRITE ? Write : Read;
1999	#elif defined(__aarch64__)	2010	# elif defined(__aarch64__)
2000	static const u64 ESR_ELx_WNR = 1U << 6;	2011	static const u64 ESR_ELx_WNR = 1U << 6;
2001	u64 esr;	2012	u64 esr;
2002	if (!Aarch64GetESR(ucontext, &esr)) return Unknown;	2013	if (!Aarch64GetESR(ucontext, &esr))
		2014	return Unknown;
2003	return esr & ESR_ELx_WNR ? Write : Read;	2015	return esr & ESR_ELx_WNR ? Write : Read;
2004	#elif defined(__loongarch__)	2016	# elif defined(__loongarch__)
2005	u32 flags = ucontext->uc_mcontext.__flags;	2017	u32 flags = ucontext->uc_mcontext.__flags;
2006	if (flags & SC_ADDRERR_RD)	2018	if (flags & SC_ADDRERR_RD)
2007	return SignalContext::Read;	2019	return SignalContext::Read;
2008	if (flags & SC_ADDRERR_WR)	2020	if (flags & SC_ADDRERR_WR)
2009	return SignalContext::Write;	2021	return SignalContext::Write;
2010	return SignalContext::Unknown;	2022	return SignalContext::Unknown;
2011	#elif defined(__sparc__)	2023	# elif defined(__sparc__)
2012	// Decode the instruction to determine the access type.	2024	// Decode the instruction to determine the access type.
2013	// From OpenSolaris $SRC/uts/sun4/os/trap.c (get_accesstype).	2025	// From OpenSolaris $SRC/uts/sun4/os/trap.c (get_accesstype).
2014	#if SANITIZER_SOLARIS	2026	# if SANITIZER_SOLARIS
2015	uptr pc = ucontext->uc_mcontext.gregs[REG_PC];	2027	uptr pc = ucontext->uc_mcontext.gregs[REG_PC];
2016	#else	2028	# else
2017	// Historical BSDism here.	2029	// Historical BSDism here.
2018	struct sigcontext scontext = (struct sigcontext )context;	2030	struct sigcontext scontext = (struct sigcontext )context;
2019	#if defined(__arch64__)	2031	# if defined(__arch64__)
2020	uptr pc = scontext->sigc_regs.tpc;	2032	uptr pc = scontext->sigc_regs.tpc;
2021	#else	2033	# else
2022	uptr pc = scontext->si_regs.pc;	2034	uptr pc = scontext->si_regs.pc;
2023	#endif	2035	# endif
2024	#endif	2036	# endif
2025	u32 instr = (u32 )pc;	2037	u32 instr = (u32 )pc;
2026	return (instr >> 21) & 1 ? Write: Read;	2038	return (instr >> 21) & 1 ? Write : Read;
2027	#elif defined(__riscv)	2039	# elif defined(__riscv)
2028	#if SANITIZER_FREEBSD	2040	# if SANITIZER_FREEBSD
2029	unsigned long pc = ucontext->uc_mcontext.mc_gpregs.gp_sepc;	2041	unsigned long pc = ucontext->uc_mcontext.mc_gpregs.gp_sepc;
2030	#else	2042	# else
2031	unsigned long pc = ucontext->uc_mcontext.__gregs[REG_PC];	2043	unsigned long pc = ucontext->uc_mcontext.__gregs[REG_PC];
2032	#endif	2044	# endif
2033	unsigned faulty_instruction = (uint16_t )pc;	2045	unsigned faulty_instruction = (uint16_t )pc;
2034		2046
2035	#if defined(__riscv_compressed)	2047	# if defined(__riscv_compressed)
2036	if ((faulty_instruction & 0x3) != 0x3) { // it's a compressed instruction	2048	if ((faulty_instruction & 0x3) != 0x3) { // it's a compressed instruction
2037	// set op_bits to the instruction bits [1, 0, 15, 14, 13]	2049	// set op_bits to the instruction bits [1, 0, 15, 14, 13]
2038	unsigned op_bits =	2050	unsigned op_bits =
...	@@ -2040,38 +2052,38 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {	...	@@ -2040,38 +2052,38 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
2040	unsigned rd = faulty_instruction & 0xF80; // bits 7-11, inclusive	2052	unsigned rd = faulty_instruction & 0xF80; // bits 7-11, inclusive
2041	switch (op_bits) {	2053	switch (op_bits) {
2042	case 0b10'010: // c.lwsp (rd != x0)	2054	case 0b10'010: // c.lwsp (rd != x0)
2043	#if __riscv_xlen == 64	2055	# if __riscv_xlen == 64
2044	case 0b10'011: // c.ldsp (rd != x0)	2056	case 0b10'011: // c.ldsp (rd != x0)
2045	#endif	2057	# endif
2046	return rd ? SignalContext::Read : SignalContext::Unknown;	2058	return rd ? SignalContext::Read : SignalContext::Unknown;
2047	case 0b00'010: // c.lw	2059	case 0b00'010: // c.lw
2048	#if __riscv_flen >= 32 && __riscv_xlen == 32	2060	# if __riscv_flen >= 32 && __riscv_xlen == 32
2049	case 0b10'011: // c.flwsp	2061	case 0b10'011: // c.flwsp
2050	#endif	2062	# endif
2051	#if __riscv_flen >= 32 \|\| __riscv_xlen == 64	2063	# if __riscv_flen >= 32 \|\| __riscv_xlen == 64
2052	case 0b00'011: // c.flw / c.ld	2064	case 0b00'011: // c.flw / c.ld
2053	#endif	2065	# endif
2054	#if __riscv_flen == 64	2066	# if __riscv_flen == 64
2055	case 0b00'001: // c.fld	2067	case 0b00'001: // c.fld
2056	case 0b10'001: // c.fldsp	2068	case 0b10'001: // c.fldsp
2057	#endif	2069	# endif
2058	return SignalContext::Read;	2070	return SignalContext::Read;
2059	case 0b00'110: // c.sw	2071	case 0b00'110: // c.sw
2060	case 0b10'110: // c.swsp	2072	case 0b10'110: // c.swsp
2061	#if __riscv_flen >= 32 \|\| __riscv_xlen == 64	2073	# if __riscv_flen >= 32 \|\| __riscv_xlen == 64
2062	case 0b00'111: // c.fsw / c.sd	2074	case 0b00'111: // c.fsw / c.sd
2063	case 0b10'111: // c.fswsp / c.sdsp	2075	case 0b10'111: // c.fswsp / c.sdsp
2064	#endif	2076	# endif
2065	#if __riscv_flen == 64	2077	# if __riscv_flen == 64
2066	case 0b00'101: // c.fsd	2078	case 0b00'101: // c.fsd
2067	case 0b10'101: // c.fsdsp	2079	case 0b10'101: // c.fsdsp
2068	#endif	2080	# endif
2069	return SignalContext::Write;	2081	return SignalContext::Write;
2070	default:	2082	default:
2071	return SignalContext::Unknown;	2083	return SignalContext::Unknown;
2072	}	2084	}
2073	}	2085	}
2074	#endif	2086	# endif
2075		2087
2076	unsigned opcode = faulty_instruction & 0x7f; // lower 7 bits	2088	unsigned opcode = faulty_instruction & 0x7f; // lower 7 bits
2077	unsigned funct3 = (faulty_instruction >> 12) & 0x7; // bits 12-14, inclusive	2089	unsigned funct3 = (faulty_instruction >> 12) & 0x7; // bits 12-14, inclusive
...	@@ -2081,9 +2093,9 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {	...	@@ -2081,9 +2093,9 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
2081	case 0b000: // lb	2093	case 0b000: // lb
2082	case 0b001: // lh	2094	case 0b001: // lh
2083	case 0b010: // lw	2095	case 0b010: // lw
2084	#if __riscv_xlen == 64	2096	# if __riscv_xlen == 64
2085	case 0b011: // ld	2097	case 0b011: // ld
2086	#endif	2098	# endif
2087	case 0b100: // lbu	2099	case 0b100: // lbu
2088	case 0b101: // lhu	2100	case 0b101: // lhu
2089	return SignalContext::Read;	2101	return SignalContext::Read;
...	@@ -2095,20 +2107,20 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {	...	@@ -2095,20 +2107,20 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
2095	case 0b000: // sb	2107	case 0b000: // sb
2096	case 0b001: // sh	2108	case 0b001: // sh
2097	case 0b010: // sw	2109	case 0b010: // sw
2098	#if __riscv_xlen == 64	2110	# if __riscv_xlen == 64
2099	case 0b011: // sd	2111	case 0b011: // sd
2100	#endif	2112	# endif
2101	return SignalContext::Write;	2113	return SignalContext::Write;
2102	default:	2114	default:
2103	return SignalContext::Unknown;	2115	return SignalContext::Unknown;
2104	}	2116	}
2105	#if __riscv_flen >= 32	2117	# if __riscv_flen >= 32
2106	case 0b0000111: // floating-point loads	2118	case 0b0000111: // floating-point loads
2107	switch (funct3) {	2119	switch (funct3) {
2108	case 0b010: // flw	2120	case 0b010: // flw
2109	#if __riscv_flen == 64	2121	# if __riscv_flen == 64
2110	case 0b011: // fld	2122	case 0b011: // fld
2111	#endif	2123	# endif
2112	return SignalContext::Read;	2124	return SignalContext::Read;
2113	default:	2125	default:
2114	return SignalContext::Unknown;	2126	return SignalContext::Unknown;
...	@@ -2116,21 +2128,21 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {	...	@@ -2116,21 +2128,21 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
2116	case 0b0100111: // floating-point stores	2128	case 0b0100111: // floating-point stores
2117	switch (funct3) {	2129	switch (funct3) {
2118	case 0b010: // fsw	2130	case 0b010: // fsw
2119	#if __riscv_flen == 64	2131	# if __riscv_flen == 64
2120	case 0b011: // fsd	2132	case 0b011: // fsd
2121	#endif	2133	# endif
2122	return SignalContext::Write;	2134	return SignalContext::Write;
2123	default:	2135	default:
2124	return SignalContext::Unknown;	2136	return SignalContext::Unknown;
2125	}	2137	}
2126	#endif	2138	# endif
2127	default:	2139	default:
2128	return SignalContext::Unknown;	2140	return SignalContext::Unknown;
2129	}	2141	}
2130	#else	2142	# else
2131	(void)ucontext;	2143	(void)ucontext;
2132	return Unknown; // FIXME: Implement.	2144	return Unknown; // FIXME: Implement.
2133	#endif	2145	# endif
2134	}	2146	}
2135		2147
2136	bool SignalContext::IsTrueFaultingAddress() const {	2148	bool SignalContext::IsTrueFaultingAddress() const {
...	@@ -2139,129 +2151,288 @@ bool SignalContext::IsTrueFaultingAddress() const {	...	@@ -2139,129 +2151,288 @@ bool SignalContext::IsTrueFaultingAddress() const {
2139	return si->si_signo == SIGSEGV && si->si_code != 128;	2151	return si->si_signo == SIGSEGV && si->si_code != 128;
2140	}	2152	}
2141		2153
		2154	UNUSED
		2155	static const char *RegNumToRegName(int reg) {
		2156	switch (reg) {
		2157	# if SANITIZER_LINUX
		2158	# if defined(__x86_64__)
		2159	case REG_RAX:
		2160	return "rax";
		2161	case REG_RBX:
		2162	return "rbx";
		2163	case REG_RCX:
		2164	return "rcx";
		2165	case REG_RDX:
		2166	return "rdx";
		2167	case REG_RDI:
		2168	return "rdi";
		2169	case REG_RSI:
		2170	return "rsi";
		2171	case REG_RBP:
		2172	return "rbp";
		2173	case REG_RSP:
		2174	return "rsp";
		2175	case REG_R8:
		2176	return "r8";
		2177	case REG_R9:
		2178	return "r9";
		2179	case REG_R10:
		2180	return "r10";
		2181	case REG_R11:
		2182	return "r11";
		2183	case REG_R12:
		2184	return "r12";
		2185	case REG_R13:
		2186	return "r13";
		2187	case REG_R14:
		2188	return "r14";
		2189	case REG_R15:
		2190	return "r15";
		2191	# elif defined(__i386__)
		2192	case REG_EAX:
		2193	return "eax";
		2194	case REG_EBX:
		2195	return "ebx";
		2196	case REG_ECX:
		2197	return "ecx";
		2198	case REG_EDX:
		2199	return "edx";
		2200	case REG_EDI:
		2201	return "edi";
		2202	case REG_ESI:
		2203	return "esi";
		2204	case REG_EBP:
		2205	return "ebp";
		2206	case REG_ESP:
		2207	return "esp";
		2208	# endif
		2209	# endif
		2210	default:
		2211	return NULL;
		2212	}
		2213	return NULL;
		2214	}
		2215
		2216	# if SANITIZER_LINUX
		2217	UNUSED
		2218	static void DumpSingleReg(ucontext_t *ctx, int RegNum) {
		2219	const char *RegName = RegNumToRegName(RegNum);
		2220	# if defined(__x86_64__)
		2221	Printf("%s%s = 0x%016llx ", internal_strlen(RegName) == 2 ? " " : "",
		2222	RegName, ctx->uc_mcontext.gregs[RegNum]);
		2223	# elif defined(__i386__)
		2224	Printf("%s = 0x%08x ", RegName, ctx->uc_mcontext.gregs[RegNum]);
		2225	# else
		2226	(void)RegName;
		2227	# endif
		2228	}
		2229	# endif
		2230
2142	void SignalContext::DumpAllRegisters(void *context) {	2231	void SignalContext::DumpAllRegisters(void *context) {
2143	// FIXME: Implement this.	2232	ucontext_t ucontext = (ucontext_t )context;
		2233	# if SANITIZER_LINUX
		2234	# if defined(__x86_64__)
		2235	Report("Register values:\n");
		2236	DumpSingleReg(ucontext, REG_RAX);
		2237	DumpSingleReg(ucontext, REG_RBX);
		2238	DumpSingleReg(ucontext, REG_RCX);
		2239	DumpSingleReg(ucontext, REG_RDX);
		2240	Printf("\n");
		2241	DumpSingleReg(ucontext, REG_RDI);
		2242	DumpSingleReg(ucontext, REG_RSI);
		2243	DumpSingleReg(ucontext, REG_RBP);
		2244	DumpSingleReg(ucontext, REG_RSP);
		2245	Printf("\n");
		2246	DumpSingleReg(ucontext, REG_R8);
		2247	DumpSingleReg(ucontext, REG_R9);
		2248	DumpSingleReg(ucontext, REG_R10);
		2249	DumpSingleReg(ucontext, REG_R11);
		2250	Printf("\n");
		2251	DumpSingleReg(ucontext, REG_R12);
		2252	DumpSingleReg(ucontext, REG_R13);
		2253	DumpSingleReg(ucontext, REG_R14);
		2254	DumpSingleReg(ucontext, REG_R15);
		2255	Printf("\n");
		2256	# elif defined(__i386__)
		2257	// Duplication of this report print is caused by partial support
		2258	// of register values dumping. In case of unsupported yet architecture let's
		2259	// avoid printing 'Register values:' without actual values in the following
		2260	// output.
		2261	Report("Register values:\n");
		2262	DumpSingleReg(ucontext, REG_EAX);
		2263	DumpSingleReg(ucontext, REG_EBX);
		2264	DumpSingleReg(ucontext, REG_ECX);
		2265	DumpSingleReg(ucontext, REG_EDX);
		2266	Printf("\n");
		2267	DumpSingleReg(ucontext, REG_EDI);
		2268	DumpSingleReg(ucontext, REG_ESI);
		2269	DumpSingleReg(ucontext, REG_EBP);
		2270	DumpSingleReg(ucontext, REG_ESP);
		2271	Printf("\n");
		2272	# else
		2273	(void)ucontext;
		2274	# endif
		2275	# elif SANITIZER_FREEBSD
		2276	# if defined(__x86_64__)
		2277	Report("Register values:\n");
		2278	Printf("rax = 0x%016lx ", ucontext->uc_mcontext.mc_rax);
		2279	Printf("rbx = 0x%016lx ", ucontext->uc_mcontext.mc_rbx);
		2280	Printf("rcx = 0x%016lx ", ucontext->uc_mcontext.mc_rcx);
		2281	Printf("rdx = 0x%016lx ", ucontext->uc_mcontext.mc_rdx);
		2282	Printf("\n");
		2283	Printf("rdi = 0x%016lx ", ucontext->uc_mcontext.mc_rdi);
		2284	Printf("rsi = 0x%016lx ", ucontext->uc_mcontext.mc_rsi);
		2285	Printf("rbp = 0x%016lx ", ucontext->uc_mcontext.mc_rbp);
		2286	Printf("rsp = 0x%016lx ", ucontext->uc_mcontext.mc_rsp);
		2287	Printf("\n");
		2288	Printf(" r8 = 0x%016lx ", ucontext->uc_mcontext.mc_r8);
		2289	Printf(" r9 = 0x%016lx ", ucontext->uc_mcontext.mc_r9);
		2290	Printf("r10 = 0x%016lx ", ucontext->uc_mcontext.mc_r10);
		2291	Printf("r11 = 0x%016lx ", ucontext->uc_mcontext.mc_r11);
		2292	Printf("\n");
		2293	Printf("r12 = 0x%016lx ", ucontext->uc_mcontext.mc_r12);
		2294	Printf("r13 = 0x%016lx ", ucontext->uc_mcontext.mc_r13);
		2295	Printf("r14 = 0x%016lx ", ucontext->uc_mcontext.mc_r14);
		2296	Printf("r15 = 0x%016lx ", ucontext->uc_mcontext.mc_r15);
		2297	Printf("\n");
		2298	# elif defined(__i386__)
		2299	Report("Register values:\n");
		2300	Printf("eax = 0x%08x ", ucontext->uc_mcontext.mc_eax);
		2301	Printf("ebx = 0x%08x ", ucontext->uc_mcontext.mc_ebx);
		2302	Printf("ecx = 0x%08x ", ucontext->uc_mcontext.mc_ecx);
		2303	Printf("edx = 0x%08x ", ucontext->uc_mcontext.mc_edx);
		2304	Printf("\n");
		2305	Printf("edi = 0x%08x ", ucontext->uc_mcontext.mc_edi);
		2306	Printf("esi = 0x%08x ", ucontext->uc_mcontext.mc_esi);
		2307	Printf("ebp = 0x%08x ", ucontext->uc_mcontext.mc_ebp);
		2308	Printf("esp = 0x%08x ", ucontext->uc_mcontext.mc_esp);
		2309	Printf("\n");
		2310	# else
		2311	(void)ucontext;
		2312	# endif
		2313	# endif
		2314	// FIXME: Implement this for other OSes and architectures.
2144	}	2315	}
2145		2316
2146	static void GetPcSpBp(void context, uptr pc, uptr sp, uptr bp) {	2317	static void GetPcSpBp(void context, uptr pc, uptr sp, uptr bp) {
2147	#if SANITIZER_NETBSD	2318	# if SANITIZER_NETBSD
2148	// This covers all NetBSD architectures	2319	// This covers all NetBSD architectures
2149	ucontext_t ucontext = (ucontext_t )context;	2320	ucontext_t ucontext = (ucontext_t )context;
2150	*pc = _UC_MACHINE_PC(ucontext);	2321	*pc = _UC_MACHINE_PC(ucontext);
2151	*bp = _UC_MACHINE_FP(ucontext);	2322	*bp = _UC_MACHINE_FP(ucontext);
2152	*sp = _UC_MACHINE_SP(ucontext);	2323	*sp = _UC_MACHINE_SP(ucontext);
2153	#elif defined(__arm__)	2324	# elif defined(__arm__)
2154	ucontext_t ucontext = (ucontext_t)context;	2325	ucontext_t ucontext = (ucontext_t )context;
2155	*pc = ucontext->uc_mcontext.arm_pc;	2326	*pc = ucontext->uc_mcontext.arm_pc;
2156	*bp = ucontext->uc_mcontext.arm_fp;	2327	*bp = ucontext->uc_mcontext.arm_fp;
2157	*sp = ucontext->uc_mcontext.arm_sp;	2328	*sp = ucontext->uc_mcontext.arm_sp;
2158	#elif defined(__aarch64__)	2329	# elif defined(__aarch64__)
2159	# if SANITIZER_FREEBSD	2330	# if SANITIZER_FREEBSD
2160	ucontext_t ucontext = (ucontext_t)context;	2331	ucontext_t ucontext = (ucontext_t )context;
2161	*pc = ucontext->uc_mcontext.mc_gpregs.gp_elr;	2332	*pc = ucontext->uc_mcontext.mc_gpregs.gp_elr;
2162	*bp = ucontext->uc_mcontext.mc_gpregs.gp_x[29];	2333	*bp = ucontext->uc_mcontext.mc_gpregs.gp_x[29];
2163	*sp = ucontext->uc_mcontext.mc_gpregs.gp_sp;	2334	*sp = ucontext->uc_mcontext.mc_gpregs.gp_sp;
2164	# else	2335	# else
2165	ucontext_t ucontext = (ucontext_t)context;	2336	ucontext_t ucontext = (ucontext_t )context;
2166	*pc = ucontext->uc_mcontext.pc;	2337	*pc = ucontext->uc_mcontext.pc;
2167	*bp = ucontext->uc_mcontext.regs[29];	2338	*bp = ucontext->uc_mcontext.regs[29];
2168	*sp = ucontext->uc_mcontext.sp;	2339	*sp = ucontext->uc_mcontext.sp;
2169	# endif	2340	# endif
2170	#elif defined(__hppa__)	2341	# elif defined(__hppa__)
2171	ucontext_t ucontext = (ucontext_t)context;	2342	ucontext_t ucontext = (ucontext_t )context;
2172	*pc = ucontext->uc_mcontext.sc_iaoq[0];	2343	*pc = ucontext->uc_mcontext.sc_iaoq[0];
2173	/* GCC uses %r3 whenever a frame pointer is needed. */	2344	/* GCC uses %r3 whenever a frame pointer is needed. */
2174	*bp = ucontext->uc_mcontext.sc_gr[3];	2345	*bp = ucontext->uc_mcontext.sc_gr[3];
2175	*sp = ucontext->uc_mcontext.sc_gr[30];	2346	*sp = ucontext->uc_mcontext.sc_gr[30];
2176	#elif defined(__x86_64__)	2347	# elif defined(__x86_64__)
2177	# if SANITIZER_FREEBSD	2348	# if SANITIZER_FREEBSD
2178	ucontext_t ucontext = (ucontext_t)context;	2349	ucontext_t ucontext = (ucontext_t )context;
2179	*pc = ucontext->uc_mcontext.mc_rip;	2350	*pc = ucontext->uc_mcontext.mc_rip;
2180	*bp = ucontext->uc_mcontext.mc_rbp;	2351	*bp = ucontext->uc_mcontext.mc_rbp;
2181	*sp = ucontext->uc_mcontext.mc_rsp;	2352	*sp = ucontext->uc_mcontext.mc_rsp;
2182	# else	2353	# else
2183	ucontext_t ucontext = (ucontext_t)context;	2354	ucontext_t ucontext = (ucontext_t )context;
2184	*pc = ucontext->uc_mcontext.gregs[REG_RIP];	2355	*pc = ucontext->uc_mcontext.gregs[REG_RIP];
2185	*bp = ucontext->uc_mcontext.gregs[REG_RBP];	2356	*bp = ucontext->uc_mcontext.gregs[REG_RBP];
2186	*sp = ucontext->uc_mcontext.gregs[REG_RSP];	2357	*sp = ucontext->uc_mcontext.gregs[REG_RSP];
2187	# endif	2358	# endif
2188	#elif defined(__i386__)	2359	# elif defined(__i386__)
2189	# if SANITIZER_FREEBSD	2360	# if SANITIZER_FREEBSD
2190	ucontext_t ucontext = (ucontext_t)context;	2361	ucontext_t ucontext = (ucontext_t )context;
2191	*pc = ucontext->uc_mcontext.mc_eip;	2362	*pc = ucontext->uc_mcontext.mc_eip;
2192	*bp = ucontext->uc_mcontext.mc_ebp;	2363	*bp = ucontext->uc_mcontext.mc_ebp;
2193	*sp = ucontext->uc_mcontext.mc_esp;	2364	*sp = ucontext->uc_mcontext.mc_esp;
2194	# else	2365	# else
2195	ucontext_t ucontext = (ucontext_t)context;	2366	ucontext_t ucontext = (ucontext_t )context;
2196	# if SANITIZER_SOLARIS	2367	# if SANITIZER_SOLARIS
2197	/* Use the numeric values: the symbolic ones are undefined by llvm	2368	/* Use the numeric values: the symbolic ones are undefined by llvm
2198	include/llvm/Support/Solaris.h. */	2369	include/llvm/Support/Solaris.h. */
2199	# ifndef REG_EIP	2370	# ifndef REG_EIP
2200	# define REG_EIP 14 // REG_PC	2371	# define REG_EIP 14 // REG_PC
2201	# endif	2372	# endif
2202	# ifndef REG_EBP	2373	# ifndef REG_EBP
2203	# define REG_EBP 6 // REG_FP	2374	# define REG_EBP 6 // REG_FP
2204	# endif	2375	# endif
2205	# ifndef REG_UESP	2376	# ifndef REG_UESP
2206	# define REG_UESP 17 // REG_SP	2377	# define REG_UESP 17 // REG_SP
2207	# endif	2378	# endif
2208	# endif	2379	# endif
2209	*pc = ucontext->uc_mcontext.gregs[REG_EIP];	2380	*pc = ucontext->uc_mcontext.gregs[REG_EIP];
2210	*bp = ucontext->uc_mcontext.gregs[REG_EBP];	2381	*bp = ucontext->uc_mcontext.gregs[REG_EBP];
2211	*sp = ucontext->uc_mcontext.gregs[REG_UESP];	2382	*sp = ucontext->uc_mcontext.gregs[REG_UESP];
2212	# endif	2383	# endif
2213	#elif defined(__powerpc__) \|\| defined(__powerpc64__)	2384	# elif defined(__powerpc__) \|\| defined(__powerpc64__)
2214	# if SANITIZER_FREEBSD	2385	# if SANITIZER_FREEBSD
2215	ucontext_t ucontext = (ucontext_t )context;	2386	ucontext_t ucontext = (ucontext_t )context;
2216	*pc = ucontext->uc_mcontext.mc_srr0;	2387	*pc = ucontext->uc_mcontext.mc_srr0;
2217	*sp = ucontext->uc_mcontext.mc_frame[1];	2388	*sp = ucontext->uc_mcontext.mc_frame[1];
2218	*bp = ucontext->uc_mcontext.mc_frame[31];	2389	*bp = ucontext->uc_mcontext.mc_frame[31];
2219	# else	2390	# else
2220	ucontext_t ucontext = (ucontext_t)context;	2391	ucontext_t ucontext = (ucontext_t )context;
2221	*pc = ucontext->uc_mcontext.regs->nip;	2392	*pc = ucontext->uc_mcontext.regs->nip;
2222	*sp = ucontext->uc_mcontext.regs->gpr[PT_R1];	2393	*sp = ucontext->uc_mcontext.regs->gpr[PT_R1];
2223	// The powerpc{,64}-linux ABIs do not specify r31 as the frame	2394	// The powerpc{,64}-linux ABIs do not specify r31 as the frame
2224	// pointer, but GCC always uses r31 when we need a frame pointer.	2395	// pointer, but GCC always uses r31 when we need a frame pointer.
2225	*bp = ucontext->uc_mcontext.regs->gpr[PT_R31];	2396	*bp = ucontext->uc_mcontext.regs->gpr[PT_R31];
2226	# endif	2397	# endif
2227	#elif defined(__sparc__)	2398	# elif defined(__sparc__)
2228	#if defined(__arch64__) \|\| defined(__sparcv9)	2399	# if defined(__arch64__) \|\| defined(__sparcv9)
2229	#define STACK_BIAS 2047	2400	# define STACK_BIAS 2047
2230	#else	2401	# else
2231	#define STACK_BIAS 0	2402	# define STACK_BIAS 0
2232	# endif	2403	# endif
2233	# if SANITIZER_SOLARIS	2404	# if SANITIZER_SOLARIS
2234	ucontext_t ucontext = (ucontext_t )context;	2405	ucontext_t ucontext = (ucontext_t )context;
2235	*pc = ucontext->uc_mcontext.gregs[REG_PC];	2406	*pc = ucontext->uc_mcontext.gregs[REG_PC];
2236	*sp = ucontext->uc_mcontext.gregs[REG_O6] + STACK_BIAS;	2407	*sp = ucontext->uc_mcontext.gregs[REG_O6] + STACK_BIAS;
2237	#else	2408	# else
2238	// Historical BSDism here.	2409	// Historical BSDism here.
2239	struct sigcontext scontext = (struct sigcontext )context;	2410	struct sigcontext scontext = (struct sigcontext )context;
2240	#if defined(__arch64__)	2411	# if defined(__arch64__)
2241	*pc = scontext->sigc_regs.tpc;	2412	*pc = scontext->sigc_regs.tpc;
2242	*sp = scontext->sigc_regs.u_regs[14] + STACK_BIAS;	2413	*sp = scontext->sigc_regs.u_regs[14] + STACK_BIAS;
2243	#else	2414	# else
2244	*pc = scontext->si_regs.pc;	2415	*pc = scontext->si_regs.pc;
2245	*sp = scontext->si_regs.u_regs[14];	2416	*sp = scontext->si_regs.u_regs[14];
2246	#endif	2417	# endif
2247	# endif	2418	# endif
2248	bp = (uptr)((uhwptr )*sp)[14] + STACK_BIAS;	2419	bp = (uptr)((uhwptr )*sp)[14] + STACK_BIAS;
2249	#elif defined(__mips__)	2420	# elif defined(__mips__)
2250	ucontext_t ucontext = (ucontext_t)context;	2421	ucontext_t ucontext = (ucontext_t )context;
2251	*pc = ucontext->uc_mcontext.pc;	2422	*pc = ucontext->uc_mcontext.pc;
2252	*bp = ucontext->uc_mcontext.gregs[30];	2423	*bp = ucontext->uc_mcontext.gregs[30];
2253	*sp = ucontext->uc_mcontext.gregs[29];	2424	*sp = ucontext->uc_mcontext.gregs[29];
2254	#elif defined(__s390__)	2425	# elif defined(__s390__)
2255	ucontext_t ucontext = (ucontext_t)context;	2426	ucontext_t ucontext = (ucontext_t )context;
2256	# if defined(__s390x__)	2427	# if defined(__s390x__)
2257	*pc = ucontext->uc_mcontext.psw.addr;	2428	*pc = ucontext->uc_mcontext.psw.addr;
2258	# else	2429	# else
2259	*pc = ucontext->uc_mcontext.psw.addr & 0x7fffffff;	2430	*pc = ucontext->uc_mcontext.psw.addr & 0x7fffffff;
2260	# endif	2431	# endif
2261	*bp = ucontext->uc_mcontext.gregs[11];	2432	*bp = ucontext->uc_mcontext.gregs[11];
2262	*sp = ucontext->uc_mcontext.gregs[15];	2433	*sp = ucontext->uc_mcontext.gregs[15];
2263	#elif defined(__riscv)	2434	# elif defined(__riscv)
2264	ucontext_t ucontext = (ucontext_t)context;	2435	ucontext_t ucontext = (ucontext_t )context;
2265	# if SANITIZER_FREEBSD	2436	# if SANITIZER_FREEBSD
2266	*pc = ucontext->uc_mcontext.mc_gpregs.gp_sepc;	2437	*pc = ucontext->uc_mcontext.mc_gpregs.gp_sepc;
2267	*bp = ucontext->uc_mcontext.mc_gpregs.gp_s[0];	2438	*bp = ucontext->uc_mcontext.mc_gpregs.gp_s[0];
...	@@ -2293,7 +2464,7 @@ void InitializePlatformEarly() {	...	@@ -2293,7 +2464,7 @@ void InitializePlatformEarly() {
2293	}	2464	}
2294		2465
2295	void CheckASLR() {	2466	void CheckASLR() {
2296	#if SANITIZER_NETBSD	2467	# if SANITIZER_NETBSD
2297	int mib[3];	2468	int mib[3];
2298	int paxflags;	2469	int paxflags;
2299	uptr len = sizeof(paxflags);	2470	uptr len = sizeof(paxflags);
...	@@ -2308,12 +2479,13 @@ void CheckASLR() {	...	@@ -2308,12 +2479,13 @@ void CheckASLR() {
2308	}	2479	}
2309		2480
2310	if (UNLIKELY(paxflags & CTL_PROC_PAXFLAGS_ASLR)) {	2481	if (UNLIKELY(paxflags & CTL_PROC_PAXFLAGS_ASLR)) {
2311	Printf("This sanitizer is not compatible with enabled ASLR.\n"	2482	Printf(
2312	"To disable ASLR, please run \"paxctl +a %s\" and try again.\n",	2483	"This sanitizer is not compatible with enabled ASLR.\n"
2313	GetArgv()[0]);	2484	"To disable ASLR, please run \"paxctl +a %s\" and try again.\n",
		2485	GetArgv()[0]);
2314	Die();	2486	Die();
2315	}	2487	}
2316	#elif SANITIZER_FREEBSD	2488	# elif SANITIZER_FREEBSD
2317	int aslr_status;	2489	int aslr_status;
2318	int r = internal_procctl(P_PID, 0, PROC_ASLR_STATUS, &aslr_status);	2490	int r = internal_procctl(P_PID, 0, PROC_ASLR_STATUS, &aslr_status);
2319	if (UNLIKELY(r == -1)) {	2491	if (UNLIKELY(r == -1)) {
...	@@ -2323,9 +2495,13 @@ void CheckASLR() {	...	@@ -2323,9 +2495,13 @@ void CheckASLR() {
2323	return;	2495	return;
2324	}	2496	}
2325	if ((aslr_status & PROC_ASLR_ACTIVE) != 0) {	2497	if ((aslr_status & PROC_ASLR_ACTIVE) != 0) {
2326	Printf("This sanitizer is not compatible with enabled ASLR "	2498	VReport(1,
2327	"and binaries compiled with PIE\n");	2499	"This sanitizer is not compatible with enabled ASLR "
2328	Die();	2500	"and binaries compiled with PIE\n"
		2501	"ASLR will be disabled and the program re-executed.\n");
		2502	int aslr_ctl = PROC_ASLR_FORCE_DISABLE;
		2503	CHECK_NE(internal_procctl(P_PID, 0, PROC_ASLR_CTL, &aslr_ctl), -1);
		2504	ReExec();
2329	}	2505	}
2330	# elif SANITIZER_PPC64V2	2506	# elif SANITIZER_PPC64V2
2331	// Disable ASLR for Linux PPC64LE.	2507	// Disable ASLR for Linux PPC64LE.
...	@@ -2345,7 +2521,7 @@ void CheckASLR() {	...	@@ -2345,7 +2521,7 @@ void CheckASLR() {
2345	}	2521	}
2346		2522
2347	void CheckMPROTECT() {	2523	void CheckMPROTECT() {
2348	#if SANITIZER_NETBSD	2524	# if SANITIZER_NETBSD
2349	int mib[3];	2525	int mib[3];
2350	int paxflags;	2526	int paxflags;
2351	uptr len = sizeof(paxflags);	2527	uptr len = sizeof(paxflags);
...	@@ -2363,13 +2539,13 @@ void CheckMPROTECT() {	...	@@ -2363,13 +2539,13 @@ void CheckMPROTECT() {
2363	Printf("This sanitizer is not compatible with enabled MPROTECT\n");	2539	Printf("This sanitizer is not compatible with enabled MPROTECT\n");
2364	Die();	2540	Die();
2365	}	2541	}
2366	#else	2542	# else
2367	// Do nothing	2543	// Do nothing
2368	#endif	2544	# endif
2369	}	2545	}
2370		2546
2371	void CheckNoDeepBind(const char *filename, int flag) {	2547	void CheckNoDeepBind(const char *filename, int flag) {
2372	#ifdef RTLD_DEEPBIND	2548	# ifdef RTLD_DEEPBIND
2373	if (flag & RTLD_DEEPBIND) {	2549	if (flag & RTLD_DEEPBIND) {
2374	Report(	2550	Report(
2375	"You are trying to dlopen a %s shared library with RTLD_DEEPBIND flag"	2551	"You are trying to dlopen a %s shared library with RTLD_DEEPBIND flag"
...	@@ -2380,7 +2556,7 @@ void CheckNoDeepBind(const char *filename, int flag) {	...	@@ -2380,7 +2556,7 @@ void CheckNoDeepBind(const char *filename, int flag) {
2380	filename, filename);	2556	filename, filename);
2381	Die();	2557	Die();
2382	}	2558	}
2383	#endif	2559	# endif
2384	}	2560	}
2385		2561
2386	uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,	2562	uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,
...	@@ -2393,16 +2569,16 @@ uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,	...	@@ -2393,16 +2569,16 @@ uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,
2393	bool GetRandom(void *buffer, uptr length, bool blocking) {	2569	bool GetRandom(void *buffer, uptr length, bool blocking) {
2394	if (!buffer \|\| !length \|\| length > 256)	2570	if (!buffer \|\| !length \|\| length > 256)
2395	return false;	2571	return false;
2396	#if SANITIZER_USE_GETENTROPY	2572	# if SANITIZER_USE_GETENTROPY
2397	uptr rnd = getentropy(buffer, length);	2573	uptr rnd = getentropy(buffer, length);
2398	int rverrno = 0;	2574	int rverrno = 0;
2399	if (internal_iserror(rnd, &rverrno) && rverrno == EFAULT)	2575	if (internal_iserror(rnd, &rverrno) && rverrno == EFAULT)
2400	return false;	2576	return false;
2401	else if (rnd == 0)	2577	else if (rnd == 0)
2402	return true;	2578	return true;
2403	#endif // SANITIZER_USE_GETENTROPY	2579	# endif // SANITIZER_USE_GETENTROPY
2404		2580
2405	#if SANITIZER_USE_GETRANDOM	2581	# if SANITIZER_USE_GETRANDOM
2406	static atomic_uint8_t skip_getrandom_syscall;	2582	static atomic_uint8_t skip_getrandom_syscall;
2407	if (!atomic_load_relaxed(&skip_getrandom_syscall)) {	2583	if (!atomic_load_relaxed(&skip_getrandom_syscall)) {
2408	// Up to 256 bytes, getrandom will not be interrupted.	2584	// Up to 256 bytes, getrandom will not be interrupted.
...	@@ -2414,7 +2590,7 @@ bool GetRandom(void *buffer, uptr length, bool blocking) {	...	@@ -2414,7 +2590,7 @@ bool GetRandom(void *buffer, uptr length, bool blocking) {
2414	else if (res == length)	2590	else if (res == length)
2415	return true;	2591	return true;
2416	}	2592	}
2417	#endif // SANITIZER_USE_GETRANDOM	2593	# endif // SANITIZER_USE_GETRANDOM
2418	// Up to 256 bytes, a read off /dev/urandom will not be interrupted.	2594	// Up to 256 bytes, a read off /dev/urandom will not be interrupted.
2419	// blocking is moot here, O_NONBLOCK has no effect when opening /dev/urandom.	2595	// blocking is moot here, O_NONBLOCK has no effect when opening /dev/urandom.
2420	uptr fd = internal_open("/dev/urandom", O_RDONLY);	2596	uptr fd = internal_open("/dev/urandom", O_RDONLY);
...	@@ -2427,6 +2603,6 @@ bool GetRandom(void *buffer, uptr length, bool blocking) {	...	@@ -2427,6 +2603,6 @@ bool GetRandom(void *buffer, uptr length, bool blocking) {
2427	return true;	2603	return true;
2428	}	2604	}
2429		2605
2430	} // namespace __sanitizer	2606	} // namespace __sanitizer
2431		2607
2432	#endif	2608	#endif

lib/tsan/sanitizer_common/sanitizer_linux.h+69-45

...	@@ -13,15 +13,15 @@	...	@@ -13,15 +13,15 @@
13	#define SANITIZER_LINUX_H	13	#define SANITIZER_LINUX_H
14		14
15	#include "sanitizer_platform.h"	15	#include "sanitizer_platform.h"
16	#if SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_NETBSD \|\| \	16	#if SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_NETBSD \|\| \
17	SANITIZER_SOLARIS	17	SANITIZER_SOLARIS
18	#include "sanitizer_common.h"	18	# include "sanitizer_common.h"
19	#include "sanitizer_internal_defs.h"	19	# include "sanitizer_internal_defs.h"
20	#include "sanitizer_platform_limits_freebsd.h"	20	# include "sanitizer_platform_limits_freebsd.h"
21	#include "sanitizer_platform_limits_netbsd.h"	21	# include "sanitizer_platform_limits_netbsd.h"
22	#include "sanitizer_platform_limits_posix.h"	22	# include "sanitizer_platform_limits_posix.h"
23	#include "sanitizer_platform_limits_solaris.h"	23	# include "sanitizer_platform_limits_solaris.h"
24	#include "sanitizer_posix.h"	24	# include "sanitizer_posix.h"
25		25
26	struct link_map; // Opaque type returned by dlopen().	26	struct link_map; // Opaque type returned by dlopen().
27	struct utsname;	27	struct utsname;
...	@@ -46,9 +46,9 @@ void ReadProcMaps(ProcSelfMapsBuff *proc_maps);	...	@@ -46,9 +46,9 @@ void ReadProcMaps(ProcSelfMapsBuff *proc_maps);
46		46
47	// Syscall wrappers.	47	// Syscall wrappers.
48	uptr internal_getdents(fd_t fd, struct linux_dirent *dirp, unsigned int count);	48	uptr internal_getdents(fd_t fd, struct linux_dirent *dirp, unsigned int count);
49	uptr internal_sigaltstack(const void* ss, void* oss);	49	uptr internal_sigaltstack(const void ss, void oss);
50	uptr internal_sigprocmask(int how, __sanitizer_sigset_t *set,	50	uptr internal_sigprocmask(int how, __sanitizer_sigset_t *set,
51	__sanitizer_sigset_t *oldset);	51	__sanitizer_sigset_t *oldset);
52		52
53	void SetSigProcMask(__sanitizer_sigset_t set, __sanitizer_sigset_t oldset);	53	void SetSigProcMask(__sanitizer_sigset_t set, __sanitizer_sigset_t oldset);
54	void BlockSignals(__sanitizer_sigset_t *oldset = nullptr);	54	void BlockSignals(__sanitizer_sigset_t *oldset = nullptr);
...	@@ -65,10 +65,10 @@ struct ScopedBlockSignals {	...	@@ -65,10 +65,10 @@ struct ScopedBlockSignals {
65		65
66	# if SANITIZER_GLIBC	66	# if SANITIZER_GLIBC
67	uptr internal_clock_gettime(__sanitizer_clockid_t clk_id, void *tp);	67	uptr internal_clock_gettime(__sanitizer_clockid_t clk_id, void *tp);
68	#endif	68	# endif
69		69
70	// Linux-only syscalls.	70	// Linux-only syscalls.
71	#if SANITIZER_LINUX	71	# if SANITIZER_LINUX
72	uptr internal_prctl(int option, uptr arg2, uptr arg3, uptr arg4, uptr arg5);	72	uptr internal_prctl(int option, uptr arg2, uptr arg3, uptr arg4, uptr arg5);
73	# if defined(__x86_64__)	73	# if defined(__x86_64__)
74	uptr internal_arch_prctl(int option, uptr arg2);	74	uptr internal_arch_prctl(int option, uptr arg2);
...	@@ -83,15 +83,15 @@ void internal_sigdelset(__sanitizer_sigset_t *set, int signum);	...	@@ -83,15 +83,15 @@ void internal_sigdelset(__sanitizer_sigset_t *set, int signum);
83	defined(__arm__) \|\| SANITIZER_RISCV64 \|\| SANITIZER_LOONGARCH64	83	defined(__arm__) \|\| SANITIZER_RISCV64 \|\| SANITIZER_LOONGARCH64
84	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,	84	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
85	int parent_tidptr, void newtls, int *child_tidptr);	85	int parent_tidptr, void newtls, int *child_tidptr);
86	#endif	86	# endif
87	int internal_uname(struct utsname *buf);	87	int internal_uname(struct utsname *buf);
88	#elif SANITIZER_FREEBSD	88	# elif SANITIZER_FREEBSD
89	uptr internal_procctl(int type, int id, int cmd, void *data);	89	uptr internal_procctl(int type, int id, int cmd, void *data);
90	void internal_sigdelset(__sanitizer_sigset_t *set, int signum);	90	void internal_sigdelset(__sanitizer_sigset_t *set, int signum);
91	#elif SANITIZER_NETBSD	91	# elif SANITIZER_NETBSD
92	void internal_sigdelset(__sanitizer_sigset_t *set, int signum);	92	void internal_sigdelset(__sanitizer_sigset_t *set, int signum);
93	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg);	93	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg);
94	#endif // SANITIZER_LINUX	94	# endif // SANITIZER_LINUX
95		95
96	// This class reads thread IDs from /proc/<pid>/task using only syscalls.	96	// This class reads thread IDs from /proc/<pid>/task using only syscalls.
97	class ThreadLister {	97	class ThreadLister {
...	@@ -135,36 +135,60 @@ inline void ReleaseMemoryPagesToOSAndZeroFill(uptr beg, uptr end) {	...	@@ -135,36 +135,60 @@ inline void ReleaseMemoryPagesToOSAndZeroFill(uptr beg, uptr end) {
135	ReleaseMemoryPagesToOS(beg, end);	135	ReleaseMemoryPagesToOS(beg, end);
136	}	136	}
137		137
138	#if SANITIZER_ANDROID	138	# if SANITIZER_ANDROID
139		139
140	#if defined(__aarch64__)	140	# if defined(__aarch64__)
141	# define __get_tls() \	141	# define __get_tls() \
142	({ void** __v; __asm__("mrs %0, tpidr_el0" : "=r"(__v)); __v; })	142	({ \
143	#elif defined(__arm__)	143	void **__v; \
144	# define __get_tls() \	144	__asm__("mrs %0, tpidr_el0" : "=r"(__v)); \
145	({ void** __v; __asm__("mrc p15, 0, %0, c13, c0, 3" : "=r"(__v)); __v; })	145	__v; \
146	#elif defined(__mips__)	146	})
		147	# elif defined(__arm__)
		148	# define __get_tls() \
		149	({ \
		150	void **__v; \
		151	__asm__("mrc p15, 0, %0, c13, c0, 3" : "=r"(__v)); \
		152	__v; \
		153	})
		154	# elif defined(__mips__)
147	// On mips32r1, this goes via a kernel illegal instruction trap that's	155	// On mips32r1, this goes via a kernel illegal instruction trap that's
148	// optimized for v1.	156	// optimized for v1.
149	# define __get_tls() \	157	# define __get_tls() \
150	({ register void** __v asm("v1"); \	158	({ \
151	__asm__(".set push\n" \	159	register void **__v asm("v1"); \
152	".set mips32r2\n" \	160	__asm__( \
153	"rdhwr %0,$29\n" \	161	".set push\n" \
154	".set pop\n" : "=r"(__v)); \	162	".set mips32r2\n" \
155	__v; })	163	"rdhwr %0,$29\n" \
156	#elif defined (__riscv)	164	".set pop\n" \
157	# define __get_tls() \	165	: "=r"(__v)); \
158	({ void** __v; __asm__("mv %0, tp" : "=r"(__v)); __v; })	166	__v; \
159	#elif defined(__i386__)	167	})
160	# define __get_tls() \	168	# elif defined(__riscv)
161	({ void** __v; __asm__("movl %%gs:0, %0" : "=r"(__v)); __v; })	169	# define __get_tls() \
162	#elif defined(__x86_64__)	170	({ \
163	# define __get_tls() \	171	void **__v; \
164	({ void** __v; __asm__("mov %%fs:0, %0" : "=r"(__v)); __v; })	172	__asm__("mv %0, tp" : "=r"(__v)); \
165	#else	173	__v; \
166	#error "Unsupported architecture."	174	})
167	#endif	175	# elif defined(__i386__)
		176	# define __get_tls() \
		177	({ \
		178	void **__v; \
		179	__asm__("movl %%gs:0, %0" : "=r"(__v)); \
		180	__v; \
		181	})
		182	# elif defined(__x86_64__)
		183	# define __get_tls() \
		184	({ \
		185	void **__v; \
		186	__asm__("mov %%fs:0, %0" : "=r"(__v)); \
		187	__v; \
		188	})
		189	# else
		190	# error "Unsupported architecture."
		191	# endif
168		192
169	// The Android Bionic team has allocated a TLS slot for sanitizers starting	193	// The Android Bionic team has allocated a TLS slot for sanitizers starting
170	// with Q, given that Android currently doesn't support ELF TLS. It is used to	194	// with Q, given that Android currently doesn't support ELF TLS. It is used to
...	@@ -175,7 +199,7 @@ ALWAYS_INLINE uptr *get_android_tls_ptr() {	...	@@ -175,7 +199,7 @@ ALWAYS_INLINE uptr *get_android_tls_ptr() {
175	return reinterpret_cast<uptr *>(&__get_tls()[TLS_SLOT_SANITIZER]);	199	return reinterpret_cast<uptr *>(&__get_tls()[TLS_SLOT_SANITIZER]);
176	}	200	}
177		201
178	#endif // SANITIZER_ANDROID	202	# endif // SANITIZER_ANDROID
179		203
180	} // namespace __sanitizer	204	} // namespace __sanitizer
181		205

lib/tsan/sanitizer_common/sanitizer_linux_libcdep.cpp+252-231

...	@@ -16,89 +16,101 @@	...	@@ -16,89 +16,101 @@
16	#if SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_NETBSD \|\| \	16	#if SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_NETBSD \|\| \
17	SANITIZER_SOLARIS	17	SANITIZER_SOLARIS
18		18
19	#include "sanitizer_allocator_internal.h"	19	# include "sanitizer_allocator_internal.h"
20	#include "sanitizer_atomic.h"	20	# include "sanitizer_atomic.h"
21	#include "sanitizer_common.h"	21	# include "sanitizer_common.h"
22	#include "sanitizer_file.h"	22	# include "sanitizer_file.h"
23	#include "sanitizer_flags.h"	23	# include "sanitizer_flags.h"
24	#include "sanitizer_freebsd.h"	24	# include "sanitizer_getauxval.h"
25	#include "sanitizer_getauxval.h"	25	# include "sanitizer_glibc_version.h"
26	#include "sanitizer_glibc_version.h"	26	# include "sanitizer_linux.h"
27	#include "sanitizer_linux.h"	27	# include "sanitizer_placement_new.h"
28	#include "sanitizer_placement_new.h"	28	# include "sanitizer_procmaps.h"
29	#include "sanitizer_procmaps.h"	29	# include "sanitizer_solaris.h"
30	#include "sanitizer_solaris.h"	30
31		31	# if SANITIZER_NETBSD
32	#if SANITIZER_NETBSD	32	# define _RTLD_SOURCE // for __lwp_gettcb_fast() / __lwp_getprivate_fast()
33	#define _RTLD_SOURCE // for __lwp_gettcb_fast() / __lwp_getprivate_fast()	33	# endif
34	#endif
35		34
36	#include <dlfcn.h> // for dlsym()	35	# include <dlfcn.h> // for dlsym()
37	#include <link.h>	36	# include <link.h>
38	#include <pthread.h>	37	# include <pthread.h>
39	#include <signal.h>	38	# include <signal.h>
40	#include <sys/mman.h>	39	# include <sys/mman.h>
41	#include <sys/resource.h>	40	# include <sys/resource.h>
42	#include <syslog.h>	41	# include <syslog.h>
43		42
44	#if !defined(ElfW)	43	# if !defined(ElfW)
45	#define ElfW(type) Elf_##type	44	# define ElfW(type) Elf_##type
46	#endif	45	# endif
47		46
48	#if SANITIZER_FREEBSD	47	# if SANITIZER_FREEBSD
49	#include <pthread_np.h>	48	# include <pthread_np.h>
50	#include <osreldate.h>	49	# include <sys/auxv.h>
51	#include <sys/sysctl.h>	50	# include <sys/sysctl.h>
52	#define pthread_getattr_np pthread_attr_get_np	51	# define pthread_getattr_np pthread_attr_get_np
53	// The MAP_NORESERVE define has been removed in FreeBSD 11.x, and even before	52	// The MAP_NORESERVE define has been removed in FreeBSD 11.x, and even before
54	// that, it was never implemented. So just define it to zero.	53	// that, it was never implemented. So just define it to zero.
55	#undef MAP_NORESERVE	54	# undef MAP_NORESERVE
56	#define MAP_NORESERVE 0	55	# define MAP_NORESERVE 0
57	#endif	56	extern const Elf_Auxinfo *__elf_aux_vector;
		57	extern "C" int __sys_sigaction(int signum, const struct sigaction *act,
		58	struct sigaction *oldact);
		59	# endif
58		60
59	#if SANITIZER_NETBSD	61	# if SANITIZER_NETBSD
60	#include <sys/sysctl.h>	62	# include <lwp.h>
61	#include <sys/tls.h>	63	# include <sys/sysctl.h>
62	#include <lwp.h>	64	# include <sys/tls.h>
63	#endif	65	# endif
64		66
65	#if SANITIZER_SOLARIS	67	# if SANITIZER_SOLARIS
66	#include <stddef.h>	68	# include <stddef.h>
67	#include <stdlib.h>	69	# include <stdlib.h>
68	#include <thread.h>	70	# include <thread.h>
69	#endif	71	# endif
70		72
71	#if SANITIZER_ANDROID	73	# if SANITIZER_ANDROID
72	#include <android/api-level.h>	74	# include <android/api-level.h>
73	#if !defined(CPU_COUNT) && !defined(__aarch64__)	75	# if !defined(CPU_COUNT) && !defined(__aarch64__)
74	#include <dirent.h>	76	# include <dirent.h>
75	#include <fcntl.h>	77	# include <fcntl.h>
76	struct __sanitizer::linux_dirent {	78	struct __sanitizer::linux_dirent {
77	long d_ino;	79	long d_ino;
78	off_t d_off;	80	off_t d_off;
79	unsigned short d_reclen;	81	unsigned short d_reclen;
80	char d_name[];	82	char d_name[];
81	};	83	};
82	#endif	84	# endif
83	#endif	85	# endif
84		86
85	#if !SANITIZER_ANDROID	87	# if !SANITIZER_ANDROID
86	#include <elf.h>	88	# include <elf.h>
87	#include <unistd.h>	89	# include <unistd.h>
88	#endif	90	# endif
89		91
90	namespace __sanitizer {	92	namespace __sanitizer {
91		93
92	SANITIZER_WEAK_ATTRIBUTE int	94	SANITIZER_WEAK_ATTRIBUTE int real_sigaction(int signum, const void *act,
93	real_sigaction(int signum, const void act, void oldact);	95	void *oldact);
94		96
95	int internal_sigaction(int signum, const void act, void oldact) {	97	int internal_sigaction(int signum, const void act, void oldact) {
96	#if !SANITIZER_GO	98	# if SANITIZER_FREEBSD
		99	// On FreeBSD, call the sigaction syscall directly (part of libsys in FreeBSD
		100	// 15) since the libc version goes via a global interposing table. Due to
		101	// library initialization order the table can be relocated after the call to
		102	// InitializeDeadlySignals() which then crashes when dereferencing the
		103	// uninitialized pointer in libc.
		104	return __sys_sigaction(signum, (const struct sigaction *)act,
		105	(struct sigaction *)oldact);
		106	# else
		107	# if !SANITIZER_GO
97	if (&real_sigaction)	108	if (&real_sigaction)
98	return real_sigaction(signum, act, oldact);	109	return real_sigaction(signum, act, oldact);
99	#endif	110	# endif
100	return sigaction(signum, (const struct sigaction *)act,	111	return sigaction(signum, (const struct sigaction *)act,
101	(struct sigaction *)oldact);	112	(struct sigaction *)oldact);
		113	# endif
102	}	114	}
103		115
104	void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top,	116	void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top,
...	@@ -111,7 +123,7 @@ void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top,	...	@@ -111,7 +123,7 @@ void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top,
111	CHECK_EQ(getrlimit(RLIMIT_STACK, &rl), 0);	123	CHECK_EQ(getrlimit(RLIMIT_STACK, &rl), 0);
112		124
113	// Find the mapping that contains a stack variable.	125	// Find the mapping that contains a stack variable.
114	MemoryMappingLayout proc_maps(/cache_enabled/true);	126	MemoryMappingLayout proc_maps(/cache_enabled/ true);
115	if (proc_maps.Error()) {	127	if (proc_maps.Error()) {
116	stack_top = stack_bottom = 0;	128	stack_top = stack_bottom = 0;
117	return;	129	return;
...	@@ -119,7 +131,8 @@ void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top,	...	@@ -119,7 +131,8 @@ void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top,
119	MemoryMappedSegment segment;	131	MemoryMappedSegment segment;
120	uptr prev_end = 0;	132	uptr prev_end = 0;
121	while (proc_maps.Next(&segment)) {	133	while (proc_maps.Next(&segment)) {
122	if ((uptr)&rl < segment.end) break;	134	if ((uptr)&rl < segment.end)
		135	break;
123	prev_end = segment.end;	136	prev_end = segment.end;
124	}	137	}
125	CHECK((uptr)&rl >= segment.start && (uptr)&rl < segment.end);	138	CHECK((uptr)&rl >= segment.start && (uptr)&rl < segment.end);
...	@@ -127,7 +140,8 @@ void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top,	...	@@ -127,7 +140,8 @@ void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top,
127	// Get stacksize from rlimit, but clip it so that it does not overlap	140	// Get stacksize from rlimit, but clip it so that it does not overlap
128	// with other mappings.	141	// with other mappings.
129	uptr stacksize = rl.rlim_cur;	142	uptr stacksize = rl.rlim_cur;
130	if (stacksize > segment.end - prev_end) stacksize = segment.end - prev_end;	143	if (stacksize > segment.end - prev_end)
		144	stacksize = segment.end - prev_end;
131	// When running with unlimited stack size, we still want to set some limit.	145	// When running with unlimited stack size, we still want to set some limit.
132	// The unlimited stack size is caused by 'ulimit -s unlimited'.	146	// The unlimited stack size is caused by 'ulimit -s unlimited'.
133	// Also, for some reason, GNU make spawns subprocesses with unlimited stack.	147	// Also, for some reason, GNU make spawns subprocesses with unlimited stack.
...	@@ -135,43 +149,56 @@ void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top,	...	@@ -135,43 +149,56 @@ void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top,
135	stacksize = kMaxThreadStackSize;	149	stacksize = kMaxThreadStackSize;
136	*stack_top = segment.end;	150	*stack_top = segment.end;
137	*stack_bottom = segment.end - stacksize;	151	*stack_bottom = segment.end - stacksize;
		152
		153	uptr maxAddr = GetMaxUserVirtualAddress();
		154	// Edge case: the stack mapping on some systems may be off-by-one e.g.,
		155	// fffffffdf000-1000000000000 rw-p 00000000 00:00 0 [stack]
		156	// instead of:
		157	// fffffffdf000- ffffffffffff
		158	// The out-of-range stack_top can result in an invalid shadow address
		159	// calculation, since those usually assume the parameters are in range.
		160	if (*stack_top == maxAddr + 1)
		161	*stack_top = maxAddr;
		162	else
		163	CHECK_LE(*stack_top, maxAddr);
		164
138	return;	165	return;
139	}	166	}
140	uptr stacksize = 0;	167	uptr stacksize = 0;
141	void *stackaddr = nullptr;	168	void *stackaddr = nullptr;
142	#if SANITIZER_SOLARIS	169	# if SANITIZER_SOLARIS
143	stack_t ss;	170	stack_t ss;
144	CHECK_EQ(thr_stksegment(&ss), 0);	171	CHECK_EQ(thr_stksegment(&ss), 0);
145	stacksize = ss.ss_size;	172	stacksize = ss.ss_size;
146	stackaddr = (char *)ss.ss_sp - stacksize;	173	stackaddr = (char *)ss.ss_sp - stacksize;
147	#else // !SANITIZER_SOLARIS	174	# else // !SANITIZER_SOLARIS
148	pthread_attr_t attr;	175	pthread_attr_t attr;
149	pthread_attr_init(&attr);	176	pthread_attr_init(&attr);
150	CHECK_EQ(pthread_getattr_np(pthread_self(), &attr), 0);	177	CHECK_EQ(pthread_getattr_np(pthread_self(), &attr), 0);
151	internal_pthread_attr_getstack(&attr, &stackaddr, &stacksize);	178	internal_pthread_attr_getstack(&attr, &stackaddr, &stacksize);
152	pthread_attr_destroy(&attr);	179	pthread_attr_destroy(&attr);
153	#endif // SANITIZER_SOLARIS	180	# endif // SANITIZER_SOLARIS
154		181
155	*stack_top = (uptr)stackaddr + stacksize;	182	*stack_top = (uptr)stackaddr + stacksize;
156	*stack_bottom = (uptr)stackaddr;	183	*stack_bottom = (uptr)stackaddr;
157	}	184	}
158		185
159	#if !SANITIZER_GO	186	# if !SANITIZER_GO
160	bool SetEnv(const char name, const char value) {	187	bool SetEnv(const char name, const char value) {
161	void *f = dlsym(RTLD_NEXT, "setenv");	188	void *f = dlsym(RTLD_NEXT, "setenv");
162	if (!f)	189	if (!f)
163	return false;	190	return false;
164	typedef int(setenv_ft)(const char name, const char *value, int overwrite);	191	typedef int (setenv_ft)(const char name, const char *value, int overwrite);
165	setenv_ft setenv_f;	192	setenv_ft setenv_f;
166	CHECK_EQ(sizeof(setenv_f), sizeof(f));	193	CHECK_EQ(sizeof(setenv_f), sizeof(f));
167	internal_memcpy(&setenv_f, &f, sizeof(f));	194	internal_memcpy(&setenv_f, &f, sizeof(f));
168	return setenv_f(name, value, 1) == 0;	195	return setenv_f(name, value, 1) == 0;
169	}	196	}
170	#endif	197	# endif
171		198
172	__attribute__((unused)) static bool GetLibcVersion(int major, int minor,	199	__attribute__((unused)) static bool GetLibcVersion(int major, int minor,
173	int *patch) {	200	int *patch) {
174	#ifdef _CS_GNU_LIBC_VERSION	201	# ifdef _CS_GNU_LIBC_VERSION
175	char buf[64];	202	char buf[64];
176	uptr len = confstr(_CS_GNU_LIBC_VERSION, buf, sizeof(buf));	203	uptr len = confstr(_CS_GNU_LIBC_VERSION, buf, sizeof(buf));
177	if (len >= sizeof(buf))	204	if (len >= sizeof(buf))
...	@@ -185,9 +212,9 @@ __attribute__((unused)) static bool GetLibcVersion(int major, int minor,	...	@@ -185,9 +212,9 @@ __attribute__((unused)) static bool GetLibcVersion(int major, int minor,
185	minor = (p == '.') ? internal_simple_strtoll(p + 1, &p, 10) : 0;	212	minor = (p == '.') ? internal_simple_strtoll(p + 1, &p, 10) : 0;
186	patch = (p == '.') ? internal_simple_strtoll(p + 1, &p, 10) : 0;	213	patch = (p == '.') ? internal_simple_strtoll(p + 1, &p, 10) : 0;
187	return true;	214	return true;
188	#else	215	# else
189	return false;	216	return false;
190	#endif	217	# endif
191	}	218	}
192		219
193	// True if we can use dlpi_tls_data. glibc before 2.25 may leave NULL (BZ	220	// True if we can use dlpi_tls_data. glibc before 2.25 may leave NULL (BZ
...	@@ -198,42 +225,42 @@ __attribute__((unused)) static bool GetLibcVersion(int major, int minor,	...	@@ -198,42 +225,42 @@ __attribute__((unused)) static bool GetLibcVersion(int major, int minor,
198	// https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=254774	225	// https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=254774
199	__attribute__((unused)) static int g_use_dlpi_tls_data;	226	__attribute__((unused)) static int g_use_dlpi_tls_data;
200		227
201	#if SANITIZER_GLIBC && !SANITIZER_GO	228	# if SANITIZER_GLIBC && !SANITIZER_GO
202	__attribute__((unused)) static size_t g_tls_size;	229	__attribute__((unused)) static size_t g_tls_size;
203	void InitTlsSize() {	230	void InitTlsSize() {
204	int major, minor, patch;	231	int major, minor, patch;
205	g_use_dlpi_tls_data =	232	g_use_dlpi_tls_data =
206	GetLibcVersion(&major, &minor, &patch) && major == 2 && minor >= 25;	233	GetLibcVersion(&major, &minor, &patch) && major == 2 && minor >= 25;
207		234
208	#if defined(__aarch64__) \|\| defined(__x86_64__) \|\| defined(__powerpc64__) \|\| \	235	# if defined(__aarch64__) \|\| defined(__x86_64__) \|\| \
209	defined(__loongarch__)	236	defined(__powerpc64__) \|\| defined(__loongarch__)
210	void *get_tls_static_info = dlsym(RTLD_NEXT, "_dl_get_tls_static_info");	237	void *get_tls_static_info = dlsym(RTLD_NEXT, "_dl_get_tls_static_info");
211	size_t tls_align;	238	size_t tls_align;
212	((void ()(size_t , size_t *))get_tls_static_info)(&g_tls_size, &tls_align);	239	((void ()(size_t , size_t *))get_tls_static_info)(&g_tls_size, &tls_align);
213	#endif	240	# endif
214	}	241	}
215	#else	242	# else
216	void InitTlsSize() { }	243	void InitTlsSize() {}
217	#endif // SANITIZER_GLIBC && !SANITIZER_GO	244	# endif // SANITIZER_GLIBC && !SANITIZER_GO
218		245
219	// On glibc x86_64, ThreadDescriptorSize() needs to be precise due to the usage	246	// On glibc x86_64, ThreadDescriptorSize() needs to be precise due to the usage
220	// of g_tls_size. On other targets, ThreadDescriptorSize() is only used by lsan	247	// of g_tls_size. On other targets, ThreadDescriptorSize() is only used by lsan
221	// to get the pointer to thread-specific data keys in the thread control block.	248	// to get the pointer to thread-specific data keys in the thread control block.
222	#if (SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_SOLARIS) && \	249	# if (SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_SOLARIS) && \
223	!SANITIZER_ANDROID && !SANITIZER_GO	250	!SANITIZER_ANDROID && !SANITIZER_GO
224	// sizeof(struct pthread) from glibc.	251	// sizeof(struct pthread) from glibc.
225	static atomic_uintptr_t thread_descriptor_size;	252	static atomic_uintptr_t thread_descriptor_size;
226		253
227	static uptr ThreadDescriptorSizeFallback() {	254	static uptr ThreadDescriptorSizeFallback() {
228	uptr val = 0;	255	uptr val = 0;
229	#if defined(__x86_64__) \|\| defined(__i386__) \|\| defined(__arm__)	256	# if defined(__x86_64__) \|\| defined(__i386__) \|\| defined(__arm__)
230	int major;	257	int major;
231	int minor;	258	int minor;
232	int patch;	259	int patch;
233	if (GetLibcVersion(&major, &minor, &patch) && major == 2) {	260	if (GetLibcVersion(&major, &minor, &patch) && major == 2) {
234	/* sizeof(struct pthread) values from various glibc versions. */	261	/* sizeof(struct pthread) values from various glibc versions. */
235	if (SANITIZER_X32)	262	if (SANITIZER_X32)
236	val = 1728; // Assume only one particular version for x32.	263	val = 1728; // Assume only one particular version for x32.
237	// For ARM sizeof(struct pthread) changed in Glibc 2.23.	264	// For ARM sizeof(struct pthread) changed in Glibc 2.23.
238	else if (SANITIZER_ARM)	265	else if (SANITIZER_ARM)
239	val = minor <= 22 ? 1120 : 1216;	266	val = minor <= 22 ? 1120 : 1216;
...	@@ -256,19 +283,19 @@ static uptr ThreadDescriptorSizeFallback() {	...	@@ -256,19 +283,19 @@ static uptr ThreadDescriptorSizeFallback() {
256	else // minor == 32	283	else // minor == 32
257	val = FIRST_32_SECOND_64(1344, 2496);	284	val = FIRST_32_SECOND_64(1344, 2496);
258	}	285	}
259	#elif defined(__s390__) \|\| defined(__sparc__)	286	# elif defined(__s390__) \|\| defined(__sparc__)
260	// The size of a prefix of TCB including pthread::{specific_1stblock,specific}	287	// The size of a prefix of TCB including pthread::{specific_1stblock,specific}
261	// suffices. Just return offsetof(struct pthread, specific_used), which hasn't	288	// suffices. Just return offsetof(struct pthread, specific_used), which hasn't
262	// changed since 2007-05. Technically this applies to i386/x86_64 as well but	289	// changed since 2007-05. Technically this applies to i386/x86_64 as well but
263	// we call _dl_get_tls_static_info and need the precise size of struct	290	// we call _dl_get_tls_static_info and need the precise size of struct
264	// pthread.	291	// pthread.
265	return FIRST_32_SECOND_64(524, 1552);	292	return FIRST_32_SECOND_64(524, 1552);
266	#elif defined(__mips__)	293	# elif defined(__mips__)
267	// TODO(sagarthakur): add more values as per different glibc versions.	294	// TODO(sagarthakur): add more values as per different glibc versions.
268	val = FIRST_32_SECOND_64(1152, 1776);	295	val = FIRST_32_SECOND_64(1152, 1776);
269	#elif SANITIZER_LOONGARCH64	296	# elif SANITIZER_LOONGARCH64
270	val = 1856; // from glibc 2.36	297	val = 1856; // from glibc 2.36
271	#elif SANITIZER_RISCV64	298	# elif SANITIZER_RISCV64
272	int major;	299	int major;
273	int minor;	300	int minor;
274	int patch;	301	int patch;
...	@@ -283,12 +310,12 @@ static uptr ThreadDescriptorSizeFallback() {	...	@@ -283,12 +310,12 @@ static uptr ThreadDescriptorSizeFallback() {
283	val = 1936; // tested against glibc 2.32	310	val = 1936; // tested against glibc 2.32
284	}	311	}
285		312
286	#elif defined(__aarch64__)	313	# elif defined(__aarch64__)
287	// The sizeof (struct pthread) is the same from GLIBC 2.17 to 2.22.	314	// The sizeof (struct pthread) is the same from GLIBC 2.17 to 2.22.
288	val = 1776;	315	val = 1776;
289	#elif defined(__powerpc64__)	316	# elif defined(__powerpc64__)
290	val = 1776; // from glibc.ppc64le 2.20-8.fc21	317	val = 1776; // from glibc.ppc64le 2.20-8.fc21
291	#endif	318	# endif
292	return val;	319	return val;
293	}	320	}
294		321
...	@@ -307,26 +334,26 @@ uptr ThreadDescriptorSize() {	...	@@ -307,26 +334,26 @@ uptr ThreadDescriptorSize() {
307	return val;	334	return val;
308	}	335	}
309		336
310	#if defined(__mips__) \|\| defined(__powerpc64__) \|\| SANITIZER_RISCV64 \|\| \	337	# if defined(__mips__) \|\| defined(__powerpc64__) \|\| SANITIZER_RISCV64 \|\| \
311	SANITIZER_LOONGARCH64	338	SANITIZER_LOONGARCH64
312	// TlsPreTcbSize includes size of struct pthread_descr and size of tcb	339	// TlsPreTcbSize includes size of struct pthread_descr and size of tcb
313	// head structure. It lies before the static tls blocks.	340	// head structure. It lies before the static tls blocks.
314	static uptr TlsPreTcbSize() {	341	static uptr TlsPreTcbSize() {
315	#if defined(__mips__)	342	# if defined(__mips__)
316	const uptr kTcbHead = 16; // sizeof (tcbhead_t)
317	#elif defined(__powerpc64__)
318	const uptr kTcbHead = 88; // sizeof (tcbhead_t)
319	#elif SANITIZER_RISCV64
320	const uptr kTcbHead = 16; // sizeof (tcbhead_t)	343	const uptr kTcbHead = 16; // sizeof (tcbhead_t)
321	#elif SANITIZER_LOONGARCH64	344	# elif defined(__powerpc64__)
		345	const uptr kTcbHead = 88; // sizeof (tcbhead_t)
		346	# elif SANITIZER_RISCV64
322	const uptr kTcbHead = 16; // sizeof (tcbhead_t)	347	const uptr kTcbHead = 16; // sizeof (tcbhead_t)
323	#endif	348	# elif SANITIZER_LOONGARCH64
		349	const uptr kTcbHead = 16; // sizeof (tcbhead_t)
		350	# endif
324	const uptr kTlsAlign = 16;	351	const uptr kTlsAlign = 16;
325	const uptr kTlsPreTcbSize =	352	const uptr kTlsPreTcbSize =
326	RoundUpTo(ThreadDescriptorSize() + kTcbHead, kTlsAlign);	353	RoundUpTo(ThreadDescriptorSize() + kTcbHead, kTlsAlign);
327	return kTlsPreTcbSize;	354	return kTlsPreTcbSize;
328	}	355	}
329	#endif	356	# endif
330		357
331	namespace {	358	namespace {
332	struct TlsBlock {	359	struct TlsBlock {
...	@@ -336,7 +363,7 @@ struct TlsBlock {	...	@@ -336,7 +363,7 @@ struct TlsBlock {
336	};	363	};
337	} // namespace	364	} // namespace
338		365
339	#ifdef __s390__	366	# ifdef __s390__
340	extern "C" uptr __tls_get_offset(void *arg);	367	extern "C" uptr __tls_get_offset(void *arg);
341		368
342	static uptr TlsGetOffset(uptr ti_module, uptr ti_offset) {	369	static uptr TlsGetOffset(uptr ti_module, uptr ti_offset) {
...	@@ -354,16 +381,16 @@ static uptr TlsGetOffset(uptr ti_module, uptr ti_offset) {	...	@@ -354,16 +381,16 @@ static uptr TlsGetOffset(uptr ti_module, uptr ti_offset) {
354	: "memory", "cc", "0", "1", "3", "4", "5", "14");	381	: "memory", "cc", "0", "1", "3", "4", "5", "14");
355	return r2;	382	return r2;
356	}	383	}
357	#else	384	# else
358	extern "C" void __tls_get_addr(size_t );	385	extern "C" void __tls_get_addr(size_t );
359	#endif	386	# endif
360		387
361	static size_t main_tls_modid;	388	static size_t main_tls_modid;
362		389
363	static int CollectStaticTlsBlocks(struct dl_phdr_info *info, size_t size,	390	static int CollectStaticTlsBlocks(struct dl_phdr_info *info, size_t size,
364	void *data) {	391	void *data) {
365	size_t tls_modid;	392	size_t tls_modid;
366	#if SANITIZER_SOLARIS	393	# if SANITIZER_SOLARIS
367	// dlpi_tls_modid is only available since Solaris 11.4 SRU 10. Use	394	// dlpi_tls_modid is only available since Solaris 11.4 SRU 10. Use
368	// dlinfo(RTLD_DI_LINKMAP) instead which works on all of Solaris 11.3,	395	// dlinfo(RTLD_DI_LINKMAP) instead which works on all of Solaris 11.3,
369	// 11.4, and Illumos. The tlsmodid of the executable was changed to 1 in	396	// 11.4, and Illumos. The tlsmodid of the executable was changed to 1 in
...	@@ -376,27 +403,26 @@ static int CollectStaticTlsBlocks(struct dl_phdr_info *info, size_t size,	...	@@ -376,27 +403,26 @@ static int CollectStaticTlsBlocks(struct dl_phdr_info *info, size_t size,
376	Rt_map *map;	403	Rt_map *map;
377	dlinfo(RTLD_SELF, RTLD_DI_LINKMAP, &map);	404	dlinfo(RTLD_SELF, RTLD_DI_LINKMAP, &map);
378	tls_modid = map->rt_tlsmodid;	405	tls_modid = map->rt_tlsmodid;
379	#else	406	# else
380	main_tls_modid = 1;	407	main_tls_modid = 1;
381	tls_modid = info->dlpi_tls_modid;	408	tls_modid = info->dlpi_tls_modid;
382	#endif	409	# endif
383		410
384	if (tls_modid < main_tls_modid)	411	if (tls_modid < main_tls_modid)
385	return 0;	412	return 0;
386	uptr begin;	413	uptr begin;
387	#if !SANITIZER_SOLARIS	414	# if !SANITIZER_SOLARIS
388	begin = (uptr)info->dlpi_tls_data;	415	begin = (uptr)info->dlpi_tls_data;
389	#endif	416	# endif
390	if (!g_use_dlpi_tls_data) {	417	if (!g_use_dlpi_tls_data) {
391	// Call __tls_get_addr as a fallback. This forces TLS allocation on glibc	418	// Call __tls_get_addr as a fallback. This forces TLS allocation on glibc
392	// and FreeBSD.	419	// and FreeBSD.
393	#ifdef __s390__	420	# ifdef __s390__
394	begin = (uptr)__builtin_thread_pointer() +	421	begin = (uptr)__builtin_thread_pointer() + TlsGetOffset(tls_modid, 0);
395	TlsGetOffset(tls_modid, 0);	422	# else
396	#else
397	size_t mod_and_off[2] = {tls_modid, 0};	423	size_t mod_and_off[2] = {tls_modid, 0};
398	begin = (uptr)__tls_get_addr(mod_and_off);	424	begin = (uptr)__tls_get_addr(mod_and_off);
399	#endif	425	# endif
400	}	426	}
401	for (unsigned i = 0; i != info->dlpi_phnum; ++i)	427	for (unsigned i = 0; i != info->dlpi_phnum; ++i)
402	if (info->dlpi_phdr[i].p_type == PT_TLS) {	428	if (info->dlpi_phdr[i].p_type == PT_TLS) {
...	@@ -439,23 +465,21 @@ __attribute__((unused)) static void GetStaticTlsBoundary(uptr addr, uptr size,	...	@@ -439,23 +465,21 @@ __attribute__((unused)) static void GetStaticTlsBoundary(uptr addr, uptr size,
439	*addr = ranges[l].begin;	465	*addr = ranges[l].begin;
440	*size = ranges[r - 1].end - ranges[l].begin;	466	*size = ranges[r - 1].end - ranges[l].begin;
441	}	467	}
442	#endif // (x86_64 \|\| i386 \|\| mips \|\| ...) && (SANITIZER_FREEBSD \|\|	468	# endif // (x86_64 \|\| i386 \|\| mips \|\| ...) && (SANITIZER_FREEBSD \|\|
443	// SANITIZER_LINUX) && !SANITIZER_ANDROID && !SANITIZER_GO	469	// SANITIZER_LINUX) && !SANITIZER_ANDROID && !SANITIZER_GO
444		470
445	#if SANITIZER_NETBSD	471	# if SANITIZER_NETBSD
446	static struct tls_tcb * ThreadSelfTlsTcb() {	472	static struct tls_tcb *ThreadSelfTlsTcb() {
447	struct tls_tcb *tcb = nullptr;	473	struct tls_tcb *tcb = nullptr;
448	#ifdef __HAVE___LWP_GETTCB_FAST	474	# ifdef __HAVE___LWP_GETTCB_FAST
449	tcb = (struct tls_tcb *)__lwp_gettcb_fast();	475	tcb = (struct tls_tcb *)__lwp_gettcb_fast();
450	#elif defined(__HAVE___LWP_GETPRIVATE_FAST)	476	# elif defined(__HAVE___LWP_GETPRIVATE_FAST)
451	tcb = (struct tls_tcb *)__lwp_getprivate_fast();	477	tcb = (struct tls_tcb *)__lwp_getprivate_fast();
452	#endif	478	# endif
453	return tcb;	479	return tcb;
454	}	480	}
455		481
456	uptr ThreadSelf() {	482	uptr ThreadSelf() { return (uptr)ThreadSelfTlsTcb()->tcb_pthread; }
457	return (uptr)ThreadSelfTlsTcb()->tcb_pthread;
458	}
459		483
460	int GetSizeFromHdr(struct dl_phdr_info info, size_t size, void data) {	484	int GetSizeFromHdr(struct dl_phdr_info info, size_t size, void data) {
461	const Elf_Phdr *hdr = info->dlpi_phdr;	485	const Elf_Phdr *hdr = info->dlpi_phdr;
...	@@ -463,23 +487,23 @@ int GetSizeFromHdr(struct dl_phdr_info info, size_t size, void data) {	...	@@ -463,23 +487,23 @@ int GetSizeFromHdr(struct dl_phdr_info info, size_t size, void data) {
463		487
464	for (; hdr != last_hdr; ++hdr) {	488	for (; hdr != last_hdr; ++hdr) {
465	if (hdr->p_type == PT_TLS && info->dlpi_tls_modid == 1) {	489	if (hdr->p_type == PT_TLS && info->dlpi_tls_modid == 1) {
466	(uptr)data = hdr->p_memsz;	490	(uptr )data = hdr->p_memsz;
467	break;	491	break;
468	}	492	}
469	}	493	}
470	return 0;	494	return 0;
471	}	495	}
472	#endif // SANITIZER_NETBSD	496	# endif // SANITIZER_NETBSD
473		497
474	#if SANITIZER_ANDROID	498	# if SANITIZER_ANDROID
475	// Bionic provides this API since S.	499	// Bionic provides this API since S.
476	extern "C" SANITIZER_WEAK_ATTRIBUTE void __libc_get_static_tls_bounds(void **,	500	extern "C" SANITIZER_WEAK_ATTRIBUTE void __libc_get_static_tls_bounds(void **,
477	void **);	501	void **);
478	#endif	502	# endif
479		503
480	#if !SANITIZER_GO	504	# if !SANITIZER_GO
481	static void GetTls(uptr addr, uptr size) {	505	static void GetTls(uptr addr, uptr size) {
482	#if SANITIZER_ANDROID	506	# if SANITIZER_ANDROID
483	if (&__libc_get_static_tls_bounds) {	507	if (&__libc_get_static_tls_bounds) {
484	void *start_addr;	508	void *start_addr;
485	void *end_addr;	509	void *end_addr;
...	@@ -491,48 +515,48 @@ static void GetTls(uptr addr, uptr size) {	...	@@ -491,48 +515,48 @@ static void GetTls(uptr addr, uptr size) {
491	*addr = 0;	515	*addr = 0;
492	*size = 0;	516	*size = 0;
493	}	517	}
494	#elif SANITIZER_GLIBC && defined(__x86_64__)	518	# elif SANITIZER_GLIBC && defined(__x86_64__)
495	// For aarch64 and x86-64, use an O(1) approach which requires relatively	519	// For aarch64 and x86-64, use an O(1) approach which requires relatively
496	// precise ThreadDescriptorSize. g_tls_size was initialized in InitTlsSize.	520	// precise ThreadDescriptorSize. g_tls_size was initialized in InitTlsSize.
497	# if SANITIZER_X32	521	# if SANITIZER_X32
498	asm("mov %%fs:8,%0" : "=r"(*addr));	522	asm("mov %%fs:8,%0" : "=r"(*addr));
499	# else	523	# else
500	asm("mov %%fs:16,%0" : "=r"(*addr));	524	asm("mov %%fs:16,%0" : "=r"(*addr));
501	# endif	525	# endif
502	*size = g_tls_size;	526	*size = g_tls_size;
503	addr -= size;	527	addr -= size;
504	*addr += ThreadDescriptorSize();	528	*addr += ThreadDescriptorSize();
505	#elif SANITIZER_GLIBC && defined(__aarch64__)	529	# elif SANITIZER_GLIBC && defined(__aarch64__)
506	*addr = reinterpret_cast<uptr>(__builtin_thread_pointer()) -	530	*addr = reinterpret_cast<uptr>(__builtin_thread_pointer()) -
507	ThreadDescriptorSize();	531	ThreadDescriptorSize();
508	*size = g_tls_size + ThreadDescriptorSize();	532	*size = g_tls_size + ThreadDescriptorSize();
509	#elif SANITIZER_GLIBC && defined(__loongarch__)	533	# elif SANITIZER_GLIBC && defined(__loongarch__)
510	# ifdef __clang__	534	# ifdef __clang__
511	*addr = reinterpret_cast<uptr>(__builtin_thread_pointer()) -	535	*addr = reinterpret_cast<uptr>(__builtin_thread_pointer()) -
512	ThreadDescriptorSize();	536	ThreadDescriptorSize();
513	# else	537	# else
514	asm("or %0,$tp,$zero" : "=r"(*addr));	538	asm("or %0,$tp,$zero" : "=r"(*addr));
515	*addr -= ThreadDescriptorSize();	539	*addr -= ThreadDescriptorSize();
516	# endif	540	# endif
517	*size = g_tls_size + ThreadDescriptorSize();	541	*size = g_tls_size + ThreadDescriptorSize();
518	#elif SANITIZER_GLIBC && defined(__powerpc64__)	542	# elif SANITIZER_GLIBC && defined(__powerpc64__)
519	// Workaround for glibc<2.25(?). 2.27 is known to not need this.	543	// Workaround for glibc<2.25(?). 2.27 is known to not need this.
520	uptr tp;	544	uptr tp;
521	asm("addi %0,13,-0x7000" : "=r"(tp));	545	asm("addi %0,13,-0x7000" : "=r"(tp));
522	const uptr pre_tcb_size = TlsPreTcbSize();	546	const uptr pre_tcb_size = TlsPreTcbSize();
523	*addr = tp - pre_tcb_size;	547	*addr = tp - pre_tcb_size;
524	*size = g_tls_size + pre_tcb_size;	548	*size = g_tls_size + pre_tcb_size;
525	#elif SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_SOLARIS	549	# elif SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_SOLARIS
526	uptr align;	550	uptr align;
527	GetStaticTlsBoundary(addr, size, &align);	551	GetStaticTlsBoundary(addr, size, &align);
528	#if defined(__x86_64__) \|\| defined(__i386__) \|\| defined(__s390__) \|\| \	552	# if defined(__x86_64__) \|\| defined(__i386__) \|\| defined(__s390__) \|\| \
529	defined(__sparc__)	553	defined(__sparc__)
530	if (SANITIZER_GLIBC) {	554	if (SANITIZER_GLIBC) {
531	#if defined(__x86_64__) \|\| defined(__i386__)	555	# if defined(__x86_64__) \|\| defined(__i386__)
532	align = Max<uptr>(align, 64);	556	align = Max<uptr>(align, 64);
533	#else	557	# else
534	align = Max<uptr>(align, 16);	558	align = Max<uptr>(align, 16);
535	#endif	559	# endif
536	}	560	}
537	const uptr tp = RoundUpTo(addr + size, align);	561	const uptr tp = RoundUpTo(addr + size, align);
538		562
...	@@ -551,26 +575,26 @@ static void GetTls(uptr addr, uptr size) {	...	@@ -551,26 +575,26 @@ static void GetTls(uptr addr, uptr size) {
551	// because the number of bytes after pthread::specific is larger.	575	// because the number of bytes after pthread::specific is larger.
552	addr = tp - RoundUpTo(size, align);	576	addr = tp - RoundUpTo(size, align);
553	size = tp - addr + ThreadDescriptorSize();	577	size = tp - addr + ThreadDescriptorSize();
554	#else	578	# else
555	if (SANITIZER_GLIBC)	579	if (SANITIZER_GLIBC)
556	*size += 1664;	580	*size += 1664;
557	else if (SANITIZER_FREEBSD)	581	else if (SANITIZER_FREEBSD)
558	*size += 128; // RTLD_STATIC_TLS_EXTRA	582	*size += 128; // RTLD_STATIC_TLS_EXTRA
559	#if defined(__mips__) \|\| defined(__powerpc64__) \|\| SANITIZER_RISCV64	583	# if defined(__mips__) \|\| defined(__powerpc64__) \|\| SANITIZER_RISCV64
560	const uptr pre_tcb_size = TlsPreTcbSize();	584	const uptr pre_tcb_size = TlsPreTcbSize();
561	*addr -= pre_tcb_size;	585	*addr -= pre_tcb_size;
562	*size += pre_tcb_size;	586	*size += pre_tcb_size;
563	#else	587	# else
564	// arm and aarch64 reserve two words at TP, so this underestimates the range.	588	// arm and aarch64 reserve two words at TP, so this underestimates the range.
565	// However, this is sufficient for the purpose of finding the pointers to	589	// However, this is sufficient for the purpose of finding the pointers to
566	// thread-specific data keys.	590	// thread-specific data keys.
567	const uptr tcb_size = ThreadDescriptorSize();	591	const uptr tcb_size = ThreadDescriptorSize();
568	*addr -= tcb_size;	592	*addr -= tcb_size;
569	*size += tcb_size;	593	*size += tcb_size;
570	#endif	594	# endif
571	#endif	595	# endif
572	#elif SANITIZER_NETBSD	596	# elif SANITIZER_NETBSD
573	struct tls_tcb * const tcb = ThreadSelfTlsTcb();	597	struct tls_tcb *const tcb = ThreadSelfTlsTcb();
574	*addr = 0;	598	*addr = 0;
575	*size = 0;	599	*size = 0;
576	if (tcb != 0) {	600	if (tcb != 0) {
...	@@ -583,31 +607,31 @@ static void GetTls(uptr addr, uptr size) {	...	@@ -583,31 +607,31 @@ static void GetTls(uptr addr, uptr size) {
583	*addr = (uptr)tcb->tcb_dtv[1];	607	*addr = (uptr)tcb->tcb_dtv[1];
584	}	608	}
585	}	609	}
586	#else	610	# else
587	#error "Unknown OS"	611	# error "Unknown OS"
588	#endif	612	# endif
589	}	613	}
590	#endif	614	# endif
591		615
592	#if !SANITIZER_GO	616	# if !SANITIZER_GO
593	uptr GetTlsSize() {	617	uptr GetTlsSize() {
594	#if SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_NETBSD \|\| \	618	# if SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_NETBSD \|\| \
595	SANITIZER_SOLARIS	619	SANITIZER_SOLARIS
596	uptr addr, size;	620	uptr addr, size;
597	GetTls(&addr, &size);	621	GetTls(&addr, &size);
598	return size;	622	return size;
599	#else	623	# else
600	return 0;	624	return 0;
601	#endif	625	# endif
602	}	626	}
603	#endif	627	# endif
604		628
605	void GetThreadStackAndTls(bool main, uptr stk_addr, uptr stk_size,	629	void GetThreadStackAndTls(bool main, uptr stk_addr, uptr stk_size,
606	uptr tls_addr, uptr tls_size) {	630	uptr tls_addr, uptr tls_size) {
607	#if SANITIZER_GO	631	# if SANITIZER_GO
608	// Stub implementation for Go.	632	// Stub implementation for Go.
609	stk_addr = stk_size = tls_addr = tls_size = 0;	633	stk_addr = stk_size = tls_addr = tls_size = 0;
610	#else	634	# else
611	GetTls(tls_addr, tls_size);	635	GetTls(tls_addr, tls_size);
612		636
613	uptr stack_top, stack_bottom;	637	uptr stack_top, stack_bottom;
...	@@ -623,16 +647,12 @@ void GetThreadStackAndTls(bool main, uptr stk_addr, uptr stk_size,	...	@@ -623,16 +647,12 @@ void GetThreadStackAndTls(bool main, uptr stk_addr, uptr stk_size,
623	stk_size = tls_addr - *stk_addr;	647	stk_size = tls_addr - *stk_addr;
624	}	648	}
625	}	649	}
626	#endif	650	# endif
627	}	651	}
628		652
629	#if !SANITIZER_FREEBSD	653	# if !SANITIZER_FREEBSD
630	typedef ElfW(Phdr) Elf_Phdr;	654	typedef ElfW(Phdr) Elf_Phdr;
631	#elif SANITIZER_WORDSIZE == 32 && __FreeBSD_version <= 902001 // v9.2	655	# endif
632	#define Elf_Phdr XElf32_Phdr
633	#define dl_phdr_info xdl_phdr_info
634	#define dl_iterate_phdr(c, b) xdl_iterate_phdr((c), (b))
635	#endif // !SANITIZER_FREEBSD
636		656
637	struct DlIteratePhdrData {	657	struct DlIteratePhdrData {
638	InternalMmapVectorNoCtor<LoadedModule> *modules;	658	InternalMmapVectorNoCtor<LoadedModule> *modules;
...	@@ -652,8 +672,7 @@ static int AddModuleSegments(const char module_name, dl_phdr_info info,	...	@@ -652,8 +672,7 @@ static int AddModuleSegments(const char module_name, dl_phdr_info info,
652	uptr cur_end = cur_beg + phdr->p_memsz;	672	uptr cur_end = cur_beg + phdr->p_memsz;
653	bool executable = phdr->p_flags & PF_X;	673	bool executable = phdr->p_flags & PF_X;
654	bool writable = phdr->p_flags & PF_W;	674	bool writable = phdr->p_flags & PF_W;
655	cur_module.addAddressRange(cur_beg, cur_end, executable,	675	cur_module.addAddressRange(cur_beg, cur_end, executable, writable);
656	writable);
657	} else if (phdr->p_type == PT_NOTE) {	676	} else if (phdr->p_type == PT_NOTE) {
658	# ifdef NT_GNU_BUILD_ID	677	# ifdef NT_GNU_BUILD_ID
659	uptr off = 0;	678	uptr off = 0;
...	@@ -698,33 +717,30 @@ static int dl_iterate_phdr_cb(dl_phdr_info info, size_t size, void arg) {	...	@@ -698,33 +717,30 @@ static int dl_iterate_phdr_cb(dl_phdr_info info, size_t size, void arg) {
698	return AddModuleSegments(module_name.data(), info, data->modules);	717	return AddModuleSegments(module_name.data(), info, data->modules);
699	}	718	}
700		719
701	if (info->dlpi_name) {	720	if (info->dlpi_name)
702	InternalScopedString module_name;	721	return AddModuleSegments(info->dlpi_name, info, data->modules);
703	module_name.append("%s", info->dlpi_name);
704	return AddModuleSegments(module_name.data(), info, data->modules);
705	}
706		722
707	return 0;	723	return 0;
708	}	724	}
709		725
710	#if SANITIZER_ANDROID && __ANDROID_API__ < 21	726	# if SANITIZER_ANDROID && __ANDROID_API__ < 21
711	extern "C" __attribute__((weak)) int dl_iterate_phdr(	727	extern "C" __attribute__((weak)) int dl_iterate_phdr(
712	int ()(struct dl_phdr_info , size_t, void ), void );	728	int ()(struct dl_phdr_info , size_t, void ), void );
713	#endif	729	# endif
714		730
715	static bool requiresProcmaps() {	731	static bool requiresProcmaps() {
716	#if SANITIZER_ANDROID && __ANDROID_API__ <= 22	732	# if SANITIZER_ANDROID && __ANDROID_API__ <= 22
717	// Fall back to /proc/maps if dl_iterate_phdr is unavailable or broken.	733	// Fall back to /proc/maps if dl_iterate_phdr is unavailable or broken.
718	// The runtime check allows the same library to work with	734	// The runtime check allows the same library to work with
719	// both K and L (and future) Android releases.	735	// both K and L (and future) Android releases.
720	return AndroidGetApiLevel() <= ANDROID_LOLLIPOP_MR1;	736	return AndroidGetApiLevel() <= ANDROID_LOLLIPOP_MR1;
721	#else	737	# else
722	return false;	738	return false;
723	#endif	739	# endif
724	}	740	}
725		741
726	static void procmapsInit(InternalMmapVectorNoCtor<LoadedModule> *modules) {	742	static void procmapsInit(InternalMmapVectorNoCtor<LoadedModule> *modules) {
727	MemoryMappingLayout memory_mapping(/cache_enabled/true);	743	MemoryMappingLayout memory_mapping(/cache_enabled/ true);
728	memory_mapping.DumpListOfModules(modules);	744	memory_mapping.DumpListOfModules(modules);
729	}	745	}
730		746
...	@@ -776,22 +792,19 @@ uptr GetRSS() {	...	@@ -776,22 +792,19 @@ uptr GetRSS() {
776	// We need the second number which is RSS in pages.	792	// We need the second number which is RSS in pages.
777	char *pos = buf;	793	char *pos = buf;
778	// Skip the first number.	794	// Skip the first number.
779	while (pos >= '0' && pos <= '9')	795	while (pos >= '0' && pos <= '9') pos++;
780	pos++;
781	// Skip whitespaces.	796	// Skip whitespaces.
782	while (!(pos >= '0' && pos <= '9') && *pos != 0)	797	while (!(pos >= '0' && pos <= '9') && *pos != 0) pos++;
783	pos++;
784	// Read the number.	798	// Read the number.
785	uptr rss = 0;	799	uptr rss = 0;
786	while (pos >= '0' && pos <= '9')	800	while (pos >= '0' && pos <= '9') rss = rss * 10 + *pos++ - '0';
787	rss = rss * 10 + *pos++ - '0';
788	return rss * GetPageSizeCached();	801	return rss * GetPageSizeCached();
789	}	802	}
790		803
791	// sysconf(_SC_NPROCESSORS_{CONF,ONLN}) cannot be used on most platforms as	804	// sysconf(_SC_NPROCESSORS_{CONF,ONLN}) cannot be used on most platforms as
792	// they allocate memory.	805	// they allocate memory.
793	u32 GetNumberOfCPUs() {	806	u32 GetNumberOfCPUs() {
794	#if SANITIZER_FREEBSD \|\| SANITIZER_NETBSD	807	# if SANITIZER_FREEBSD \|\| SANITIZER_NETBSD
795	u32 ncpu;	808	u32 ncpu;
796	int req[2];	809	int req[2];
797	uptr len = sizeof(ncpu);	810	uptr len = sizeof(ncpu);
...	@@ -799,7 +812,7 @@ u32 GetNumberOfCPUs() {	...	@@ -799,7 +812,7 @@ u32 GetNumberOfCPUs() {
799	req[1] = HW_NCPU;	812	req[1] = HW_NCPU;
800	CHECK_EQ(internal_sysctl(req, 2, &ncpu, &len, NULL, 0), 0);	813	CHECK_EQ(internal_sysctl(req, 2, &ncpu, &len, NULL, 0), 0);
801	return ncpu;	814	return ncpu;
802	#elif SANITIZER_ANDROID && !defined(CPU_COUNT) && !defined(__aarch64__)	815	# elif SANITIZER_ANDROID && !defined(CPU_COUNT) && !defined(__aarch64__)
803	// Fall back to /sys/devices/system/cpu on Android when cpu_set_t doesn't	816	// Fall back to /sys/devices/system/cpu on Android when cpu_set_t doesn't
804	// exist in sched.h. That is the case for toolchains generated with older	817	// exist in sched.h. That is the case for toolchains generated with older
805	// NDKs.	818	// NDKs.
...	@@ -827,26 +840,26 @@ u32 GetNumberOfCPUs() {	...	@@ -827,26 +840,26 @@ u32 GetNumberOfCPUs() {
827	break;	840	break;
828	if (entry->d_ino != 0 && *d_type == DT_DIR) {	841	if (entry->d_ino != 0 && *d_type == DT_DIR) {
829	if (entry->d_name[0] == 'c' && entry->d_name[1] == 'p' &&	842	if (entry->d_name[0] == 'c' && entry->d_name[1] == 'p' &&
830	entry->d_name[2] == 'u' &&	843	entry->d_name[2] == 'u' && entry->d_name[3] >= '0' &&
831	entry->d_name[3] >= '0' && entry->d_name[3] <= '9')	844	entry->d_name[3] <= '9')
832	n_cpus++;	845	n_cpus++;
833	}	846	}
834	entry = (struct linux_dirent )(((u8 )entry) + entry->d_reclen);	847	entry = (struct linux_dirent )(((u8 )entry) + entry->d_reclen);
835	}	848	}
836	internal_close(fd);	849	internal_close(fd);
837	return n_cpus;	850	return n_cpus;
838	#elif SANITIZER_SOLARIS	851	# elif SANITIZER_SOLARIS
839	return sysconf(_SC_NPROCESSORS_ONLN);	852	return sysconf(_SC_NPROCESSORS_ONLN);
840	#else	853	# else
841	cpu_set_t CPUs;	854	cpu_set_t CPUs;
842	CHECK_EQ(sched_getaffinity(0, sizeof(cpu_set_t), &CPUs), 0);	855	CHECK_EQ(sched_getaffinity(0, sizeof(cpu_set_t), &CPUs), 0);
843	return CPU_COUNT(&CPUs);	856	return CPU_COUNT(&CPUs);
844	#endif	857	# endif
845	}	858	}
846		859
847	#if SANITIZER_LINUX	860	# if SANITIZER_LINUX
848		861
849	#if SANITIZER_ANDROID	862	# if SANITIZER_ANDROID
850	static atomic_uint8_t android_log_initialized;	863	static atomic_uint8_t android_log_initialized;
851		864
852	void AndroidLogInit() {	865	void AndroidLogInit() {
...	@@ -858,13 +871,15 @@ static bool ShouldLogAfterPrintf() {	...	@@ -858,13 +871,15 @@ static bool ShouldLogAfterPrintf() {
858	return atomic_load(&android_log_initialized, memory_order_acquire);	871	return atomic_load(&android_log_initialized, memory_order_acquire);
859	}	872	}
860		873
861	extern "C" SANITIZER_WEAK_ATTRIBUTE	874	extern "C" SANITIZER_WEAK_ATTRIBUTE int async_safe_write_log(int pri,
862	int async_safe_write_log(int pri, const char* tag, const char* msg);	875	const char *tag,
863	extern "C" SANITIZER_WEAK_ATTRIBUTE	876	const char *msg);
864	int __android_log_write(int prio, const char* tag, const char* msg);	877	extern "C" SANITIZER_WEAK_ATTRIBUTE int __android_log_write(int prio,
		878	const char *tag,
		879	const char *msg);
865		880
866	// ANDROID_LOG_INFO is 4, but can't be resolved at runtime.	881	// ANDROID_LOG_INFO is 4, but can't be resolved at runtime.
867	#define SANITIZER_ANDROID_LOG_INFO 4	882	# define SANITIZER_ANDROID_LOG_INFO 4
868		883
869	// async_safe_write_log is a new public version of __libc_write_log that is	884	// async_safe_write_log is a new public version of __libc_write_log that is
870	// used behind syslog. It is preferable to syslog as it will not do any dynamic	885	// used behind syslog. It is preferable to syslog as it will not do any dynamic
...	@@ -883,14 +898,14 @@ void WriteOneLineToSyslog(const char *s) {	...	@@ -883,14 +898,14 @@ void WriteOneLineToSyslog(const char *s) {
883	}	898	}
884	}	899	}
885		900
886	extern "C" SANITIZER_WEAK_ATTRIBUTE	901	extern "C" SANITIZER_WEAK_ATTRIBUTE void android_set_abort_message(
887	void android_set_abort_message(const char *);	902	const char *);
888		903
889	void SetAbortMessage(const char *str) {	904	void SetAbortMessage(const char *str) {
890	if (&android_set_abort_message)	905	if (&android_set_abort_message)
891	android_set_abort_message(str);	906	android_set_abort_message(str);
892	}	907	}
893	#else	908	# else
894	void AndroidLogInit() {}	909	void AndroidLogInit() {}
895		910
896	static bool ShouldLogAfterPrintf() { return true; }	911	static bool ShouldLogAfterPrintf() { return true; }
...	@@ -898,16 +913,16 @@ static bool ShouldLogAfterPrintf() { return true; }	...	@@ -898,16 +913,16 @@ static bool ShouldLogAfterPrintf() { return true; }
898	void WriteOneLineToSyslog(const char *s) { syslog(LOG_INFO, "%s", s); }	913	void WriteOneLineToSyslog(const char *s) { syslog(LOG_INFO, "%s", s); }
899		914
900	void SetAbortMessage(const char *str) {}	915	void SetAbortMessage(const char *str) {}
901	#endif // SANITIZER_ANDROID	916	# endif // SANITIZER_ANDROID
902		917
903	void LogMessageOnPrintf(const char *str) {	918	void LogMessageOnPrintf(const char *str) {
904	if (common_flags()->log_to_syslog && ShouldLogAfterPrintf())	919	if (common_flags()->log_to_syslog && ShouldLogAfterPrintf())
905	WriteToSyslog(str);	920	WriteToSyslog(str);
906	}	921	}
907		922
908	#endif // SANITIZER_LINUX	923	# endif // SANITIZER_LINUX
909		924
910	#if SANITIZER_GLIBC && !SANITIZER_GO	925	# if SANITIZER_GLIBC && !SANITIZER_GO
911	// glibc crashes when using clock_gettime from a preinit_array function as the	926	// glibc crashes when using clock_gettime from a preinit_array function as the
912	// vDSO function pointers haven't been initialized yet. __progname is	927	// vDSO function pointers haven't been initialized yet. __progname is
913	// initialized after the vDSO function pointers, so if it exists, is not null	928	// initialized after the vDSO function pointers, so if it exists, is not null
...	@@ -918,8 +933,8 @@ inline bool CanUseVDSO() { return &__progname && __progname && *__progname; }	...	@@ -918,8 +933,8 @@ inline bool CanUseVDSO() { return &__progname && __progname && *__progname; }
918	// MonotonicNanoTime is a timing function that can leverage the vDSO by calling	933	// MonotonicNanoTime is a timing function that can leverage the vDSO by calling
919	// clock_gettime. real_clock_gettime only exists if clock_gettime is	934	// clock_gettime. real_clock_gettime only exists if clock_gettime is
920	// intercepted, so define it weakly and use it if available.	935	// intercepted, so define it weakly and use it if available.
921	extern "C" SANITIZER_WEAK_ATTRIBUTE	936	extern "C" SANITIZER_WEAK_ATTRIBUTE int real_clock_gettime(u32 clk_id,
922	int real_clock_gettime(u32 clk_id, void *tp);	937	void *tp);
923	u64 MonotonicNanoTime() {	938	u64 MonotonicNanoTime() {
924	timespec ts;	939	timespec ts;
925	if (CanUseVDSO()) {	940	if (CanUseVDSO()) {
...	@@ -932,19 +947,26 @@ u64 MonotonicNanoTime() {	...	@@ -932,19 +947,26 @@ u64 MonotonicNanoTime() {
932	}	947	}
933	return (u64)ts.tv_sec * (1000ULL * 1000 * 1000) + ts.tv_nsec;	948	return (u64)ts.tv_sec * (1000ULL * 1000 * 1000) + ts.tv_nsec;
934	}	949	}
935	#else	950	# else
936	// Non-glibc & Go always use the regular function.	951	// Non-glibc & Go always use the regular function.
937	u64 MonotonicNanoTime() {	952	u64 MonotonicNanoTime() {
938	timespec ts;	953	timespec ts;
939	clock_gettime(CLOCK_MONOTONIC, &ts);	954	clock_gettime(CLOCK_MONOTONIC, &ts);
940	return (u64)ts.tv_sec * (1000ULL * 1000 * 1000) + ts.tv_nsec;	955	return (u64)ts.tv_sec * (1000ULL * 1000 * 1000) + ts.tv_nsec;
941	}	956	}
942	#endif // SANITIZER_GLIBC && !SANITIZER_GO	957	# endif // SANITIZER_GLIBC && !SANITIZER_GO
943		958
944	void ReExec() {	959	void ReExec() {
945	const char *pathname = "/proc/self/exe";	960	const char *pathname = "/proc/self/exe";
946		961
947	#if SANITIZER_NETBSD	962	# if SANITIZER_FREEBSD
		963	for (const auto *aux = __elf_aux_vector; aux->a_type != AT_NULL; aux++) {
		964	if (aux->a_type == AT_EXECPATH) {
		965	pathname = static_cast<const char *>(aux->a_un.a_ptr);
		966	break;
		967	}
		968	}
		969	# elif SANITIZER_NETBSD
948	static const int name[] = {	970	static const int name[] = {
949	CTL_KERN,	971	CTL_KERN,
950	KERN_PROC_ARGS,	972	KERN_PROC_ARGS,
...	@@ -957,14 +979,14 @@ void ReExec() {	...	@@ -957,14 +979,14 @@ void ReExec() {
957	len = sizeof(path);	979	len = sizeof(path);
958	if (internal_sysctl(name, ARRAY_SIZE(name), path, &len, NULL, 0) != -1)	980	if (internal_sysctl(name, ARRAY_SIZE(name), path, &len, NULL, 0) != -1)
959	pathname = path;	981	pathname = path;
960	#elif SANITIZER_SOLARIS	982	# elif SANITIZER_SOLARIS
961	pathname = getexecname();	983	pathname = getexecname();
962	CHECK_NE(pathname, NULL);	984	CHECK_NE(pathname, NULL);
963	#elif SANITIZER_USE_GETAUXVAL	985	# elif SANITIZER_USE_GETAUXVAL
964	// Calling execve with /proc/self/exe sets that as $EXEC_ORIGIN. Binaries that	986	// Calling execve with /proc/self/exe sets that as $EXEC_ORIGIN. Binaries that
965	// rely on that will fail to load shared libraries. Query AT_EXECFN instead.	987	// rely on that will fail to load shared libraries. Query AT_EXECFN instead.
966	pathname = reinterpret_cast<const char *>(getauxval(AT_EXECFN));	988	pathname = reinterpret_cast<const char *>(getauxval(AT_EXECFN));
967	#endif	989	# endif
968		990
969	uptr rv = internal_execve(pathname, GetArgv(), GetEnviron());	991	uptr rv = internal_execve(pathname, GetArgv(), GetEnviron());
970	int rverrno;	992	int rverrno;
...	@@ -986,9 +1008,8 @@ void UnmapFromTo(uptr from, uptr to) {	...	@@ -986,9 +1008,8 @@ void UnmapFromTo(uptr from, uptr to) {
986	}	1008	}
987		1009
988	uptr MapDynamicShadow(uptr shadow_size_bytes, uptr shadow_scale,	1010	uptr MapDynamicShadow(uptr shadow_size_bytes, uptr shadow_scale,
989	uptr min_shadow_base_alignment,	1011	uptr min_shadow_base_alignment, UNUSED uptr &high_mem_end,
990	UNUSED uptr &high_mem_end) {	1012	uptr granularity) {
991	const uptr granularity = GetMmapGranularity();
992	const uptr alignment =	1013	const uptr alignment =
993	Max<uptr>(granularity << shadow_scale, 1ULL << min_shadow_base_alignment);	1014	Max<uptr>(granularity << shadow_scale, 1ULL << min_shadow_base_alignment);
994	const uptr left_padding =	1015	const uptr left_padding =
...	@@ -1016,14 +1037,14 @@ static uptr MmapSharedNoReserve(uptr addr, uptr size) {	...	@@ -1016,14 +1037,14 @@ static uptr MmapSharedNoReserve(uptr addr, uptr size) {
1016		1037
1017	static uptr MremapCreateAlias(uptr base_addr, uptr alias_addr,	1038	static uptr MremapCreateAlias(uptr base_addr, uptr alias_addr,
1018	uptr alias_size) {	1039	uptr alias_size) {
1019	#if SANITIZER_LINUX	1040	# if SANITIZER_LINUX
1020	return internal_mremap(reinterpret_cast<void *>(base_addr), 0, alias_size,	1041	return internal_mremap(reinterpret_cast<void *>(base_addr), 0, alias_size,
1021	MREMAP_MAYMOVE \| MREMAP_FIXED,	1042	MREMAP_MAYMOVE \| MREMAP_FIXED,
1022	reinterpret_cast<void *>(alias_addr));	1043	reinterpret_cast<void *>(alias_addr));
1023	#else	1044	# else
1024	CHECK(false && "mremap is not supported outside of Linux");	1045	CHECK(false && "mremap is not supported outside of Linux");
1025	return 0;	1046	return 0;
1026	#endif	1047	# endif
1027	}	1048	}
1028		1049
1029	static void CreateAliases(uptr start_addr, uptr alias_size, uptr num_aliases) {	1050	static void CreateAliases(uptr start_addr, uptr alias_size, uptr num_aliases) {
...	@@ -1068,12 +1089,12 @@ uptr MapDynamicShadowAndAliases(uptr shadow_size, uptr alias_size,	...	@@ -1068,12 +1089,12 @@ uptr MapDynamicShadowAndAliases(uptr shadow_size, uptr alias_size,
1068	}	1089	}
1069		1090
1070	void InitializePlatformCommonFlags(CommonFlags *cf) {	1091	void InitializePlatformCommonFlags(CommonFlags *cf) {
1071	#if SANITIZER_ANDROID	1092	# if SANITIZER_ANDROID
1072	if (&__libc_get_static_tls_bounds == nullptr)	1093	if (&__libc_get_static_tls_bounds == nullptr)
1073	cf->detect_leaks = false;	1094	cf->detect_leaks = false;
1074	#endif	1095	# endif
1075	}	1096	}
1076		1097
1077	} // namespace __sanitizer	1098	} // namespace __sanitizer
1078		1099
1079	#endif	1100	#endif

lib/tsan/sanitizer_common/sanitizer_linux_s390.cpp+79-85

...	@@ -15,14 +15,14 @@	...	@@ -15,14 +15,14 @@
15		15
16	#if SANITIZER_LINUX && SANITIZER_S390	16	#if SANITIZER_LINUX && SANITIZER_S390
17		17
18	#include <dlfcn.h>	18	# include <dlfcn.h>
19	#include <errno.h>	19	# include <errno.h>
20	#include <sys/syscall.h>	20	# include <sys/syscall.h>
21	#include <sys/utsname.h>	21	# include <sys/utsname.h>
22	#include <unistd.h>	22	# include <unistd.h>
23		23
24	#include "sanitizer_libc.h"	24	# include "sanitizer_libc.h"
25	#include "sanitizer_linux.h"	25	# include "sanitizer_linux.h"
26		26
27	namespace __sanitizer {	27	namespace __sanitizer {
28		28
...	@@ -37,22 +37,19 @@ uptr internal_mmap(void *addr, uptr length, int prot, int flags, int fd,	...	@@ -37,22 +37,19 @@ uptr internal_mmap(void *addr, uptr length, int prot, int flags, int fd,
37	unsigned long fd;	37	unsigned long fd;
38	unsigned long offset;	38	unsigned long offset;
39	} params = {	39	} params = {
40	(unsigned long)addr,	40	(unsigned long)addr, (unsigned long)length, (unsigned long)prot,
41	(unsigned long)length,	41	(unsigned long)flags, (unsigned long)fd,
42	(unsigned long)prot,	42	# ifdef __s390x__
43	(unsigned long)flags,	43	(unsigned long)offset,
44	(unsigned long)fd,	44	# else
45	# ifdef __s390x__
46	(unsigned long)offset,
47	# else
48	(unsigned long)(offset / 4096),	45	(unsigned long)(offset / 4096),
49	# endif	46	# endif
50	};	47	};
51	# ifdef __s390x__	48	# ifdef __s390x__
52	return syscall(__NR_mmap, &params);	49	return syscall(__NR_mmap, &params);
53	# else	50	# else
54	return syscall(__NR_mmap2, &params);	51	return syscall(__NR_mmap2, &params);
55	# endif	52	# endif
56	}	53	}
57		54
58	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,	55	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
...	@@ -63,58 +60,54 @@ uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,	...	@@ -63,58 +60,54 @@ uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
63	}	60	}
64	CHECK_EQ(0, (uptr)child_stack % 16);	61	CHECK_EQ(0, (uptr)child_stack % 16);
65	// Minimum frame size.	62	// Minimum frame size.
66	#ifdef __s390x__	63	# ifdef __s390x__
67	child_stack = (char *)child_stack - 160;	64	child_stack = (char *)child_stack - 160;
68	#else	65	# else
69	child_stack = (char *)child_stack - 96;	66	child_stack = (char *)child_stack - 96;
70	#endif	67	# endif
71	// Terminate unwind chain.	68	// Terminate unwind chain.
72	((unsigned long *)child_stack)[0] = 0;	69	((unsigned long *)child_stack)[0] = 0;
73	// And pass parameters.	70	// And pass parameters.
74	((unsigned long *)child_stack)[1] = (uptr)fn;	71	((unsigned long *)child_stack)[1] = (uptr)fn;
75	((unsigned long *)child_stack)[2] = (uptr)arg;	72	((unsigned long *)child_stack)[2] = (uptr)arg;
76	register uptr res __asm__("r2");	73	register uptr res __asm__("r2");
77	register void *__cstack __asm__("r2") = child_stack;	74	register void *__cstack __asm__("r2") = child_stack;
78	register long __flags __asm__("r3") = flags;	75	register long __flags __asm__("r3") = flags;
79	register int * __ptidptr __asm__("r4") = parent_tidptr;	76	register int *__ptidptr __asm__("r4") = parent_tidptr;
80	register int * __ctidptr __asm__("r5") = child_tidptr;	77	register int *__ctidptr __asm__("r5") = child_tidptr;
81	register void * __newtls __asm__("r6") = newtls;	78	register void *__newtls __asm__("r6") = newtls;
82		79
83	__asm__ __volatile__(	80	__asm__ __volatile__(
84	/* Clone. */	81	/* Clone. */
85	"svc %1\n"	82	"svc %1\n"
86		83
87	/* if (%r2 != 0)	84	/* if (%r2 != 0)
88	* return;	85	* return;
89	*/	86	*/
90	#ifdef __s390x__	87	# ifdef __s390x__
91	"cghi %%r2, 0\n"	88	"cghi %%r2, 0\n"
92	#else	89	# else
93	"chi %%r2, 0\n"	90	"chi %%r2, 0\n"
94	#endif	91	# endif
95	"jne 1f\n"	92	"jne 1f\n"
96		93
97	/* Call "fn(arg)". */	94	/* Call "fn(arg)". */
98	#ifdef __s390x__	95	# ifdef __s390x__
99	"lmg %%r1, %%r2, 8(%%r15)\n"	96	"lmg %%r1, %%r2, 8(%%r15)\n"
100	#else	97	# else
101	"lm %%r1, %%r2, 4(%%r15)\n"	98	"lm %%r1, %%r2, 4(%%r15)\n"
102	#endif	99	# endif
103	"basr %%r14, %%r1\n"	100	"basr %%r14, %%r1\n"
104		101
105	/* Call _exit(%r2). */	102	/* Call _exit(%r2). */
106	"svc %2\n"	103	"svc %2\n"
107		104
108	/* Return to parent. */	105	/* Return to parent. */
109	"1:\n"	106	"1:\n"
110	: "=r" (res)	107	: "=r"(res)
111	: "i"(__NR_clone), "i"(__NR_exit),	108	: "i"(__NR_clone), "i"(__NR_exit), "r"(__cstack), "r"(__flags),
112	"r"(__cstack),	109	"r"(__ptidptr), "r"(__ctidptr), "r"(__newtls)
113	"r"(__flags),	110	: "memory", "cc");
114	"r"(__ptidptr),
115	"r"(__ctidptr),
116	"r"(__newtls)
117	: "memory", "cc");
118	if (res >= (uptr)-4095) {	111	if (res >= (uptr)-4095) {
119	errno = -res;	112	errno = -res;
120	return -1;	113	return -1;
...	@@ -122,7 +115,7 @@ uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,	...	@@ -122,7 +115,7 @@ uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
122	return res;	115	return res;
123	}	116	}
124		117
125	#if SANITIZER_S390_64	118	# if SANITIZER_S390_64
126	static bool FixedCVE_2016_2143() {	119	static bool FixedCVE_2016_2143() {
127	// Try to determine if the running kernel has a fix for CVE-2016-2143,	120	// Try to determine if the running kernel has a fix for CVE-2016-2143,
128	// return false if in doubt (better safe than sorry). Distros may want to	121	// return false if in doubt (better safe than sorry). Distros may want to
...	@@ -137,20 +130,20 @@ static bool FixedCVE_2016_2143() {	...	@@ -137,20 +130,20 @@ static bool FixedCVE_2016_2143() {
137	// At least first 2 should be matched.	130	// At least first 2 should be matched.
138	if (ptr[0] != '.')	131	if (ptr[0] != '.')
139	return false;	132	return false;
140	minor = internal_simple_strtoll(ptr+1, &ptr, 10);	133	minor = internal_simple_strtoll(ptr + 1, &ptr, 10);
141	// Third is optional.	134	// Third is optional.
142	if (ptr[0] == '.')	135	if (ptr[0] == '.')
143	patch = internal_simple_strtoll(ptr+1, &ptr, 10);	136	patch = internal_simple_strtoll(ptr + 1, &ptr, 10);
144	if (major < 3) {	137	if (major < 3) {
145	if (major == 2 && minor == 6 && patch == 32 && ptr[0] == '-' &&	138	if (major == 2 && minor == 6 && patch == 32 && ptr[0] == '-' &&
146	internal_strstr(ptr, ".el6")) {	139	internal_strstr(ptr, ".el6")) {
147	// Check RHEL6	140	// Check RHEL6
148	int r1 = internal_simple_strtoll(ptr+1, &ptr, 10);	141	int r1 = internal_simple_strtoll(ptr + 1, &ptr, 10);
149	if (r1 >= 657) // 2.6.32-657.el6 or later	142	if (r1 >= 657) // 2.6.32-657.el6 or later
150	return true;	143	return true;
151	if (r1 == 642 && ptr[0] == '.') {	144	if (r1 == 642 && ptr[0] == '.') {
152	int r2 = internal_simple_strtoll(ptr+1, &ptr, 10);	145	int r2 = internal_simple_strtoll(ptr + 1, &ptr, 10);
153	if (r2 >= 9) // 2.6.32-642.9.1.el6 or later	146	if (r2 >= 9) // 2.6.32-642.9.1.el6 or later
154	return true;	147	return true;
155	}	148	}
156	}	149	}
...	@@ -166,12 +159,12 @@ static bool FixedCVE_2016_2143() {	...	@@ -166,12 +159,12 @@ static bool FixedCVE_2016_2143() {
166	if (minor == 10 && patch == 0 && ptr[0] == '-' &&	159	if (minor == 10 && patch == 0 && ptr[0] == '-' &&
167	internal_strstr(ptr, ".el7")) {	160	internal_strstr(ptr, ".el7")) {
168	// Check RHEL7	161	// Check RHEL7
169	int r1 = internal_simple_strtoll(ptr+1, &ptr, 10);	162	int r1 = internal_simple_strtoll(ptr + 1, &ptr, 10);
170	if (r1 >= 426) // 3.10.0-426.el7 or later	163	if (r1 >= 426) // 3.10.0-426.el7 or later
171	return true;	164	return true;
172	if (r1 == 327 && ptr[0] == '.') {	165	if (r1 == 327 && ptr[0] == '.') {
173	int r2 = internal_simple_strtoll(ptr+1, &ptr, 10);	166	int r2 = internal_simple_strtoll(ptr + 1, &ptr, 10);
174	if (r2 >= 27) // 3.10.0-327.27.1.el7 or later	167	if (r2 >= 27) // 3.10.0-327.27.1.el7 or later
175	return true;	168	return true;
176	}	169	}
177	}	170	}
...	@@ -187,8 +180,8 @@ static bool FixedCVE_2016_2143() {	...	@@ -187,8 +180,8 @@ static bool FixedCVE_2016_2143() {
187	if (minor == 4 && patch == 0 && ptr[0] == '-' &&	180	if (minor == 4 && patch == 0 && ptr[0] == '-' &&
188	internal_strstr(buf.version, "Ubuntu")) {	181	internal_strstr(buf.version, "Ubuntu")) {
189	// Check Ubuntu 16.04	182	// Check Ubuntu 16.04
190	int r1 = internal_simple_strtoll(ptr+1, &ptr, 10);	183	int r1 = internal_simple_strtoll(ptr + 1, &ptr, 10);
191	if (r1 >= 13) // 4.4.0-13 or later	184	if (r1 >= 13) // 4.4.0-13 or later
192	return true;	185	return true;
193	}	186	}
194	// Otherwise, OK if 4.5+.	187	// Otherwise, OK if 4.5+.
...	@@ -211,18 +204,19 @@ void AvoidCVE_2016_2143() {	...	@@ -211,18 +204,19 @@ void AvoidCVE_2016_2143() {
211	if (GetEnv("SANITIZER_IGNORE_CVE_2016_2143"))	204	if (GetEnv("SANITIZER_IGNORE_CVE_2016_2143"))
212	return;	205	return;
213	Report(	206	Report(
214	"ERROR: Your kernel seems to be vulnerable to CVE-2016-2143. Using ASan,\n"	207	"ERROR: Your kernel seems to be vulnerable to CVE-2016-2143. Using "
215	"MSan, TSan, DFSan or LSan with such kernel can and will crash your\n"	208	"ASan,\n"
216	"machine, or worse.\n"	209	"MSan, TSan, DFSan or LSan with such kernel can and will crash your\n"
217	"\n"	210	"machine, or worse.\n"
218	"If you are certain your kernel is not vulnerable (you have compiled it\n"	211	"\n"
219	"yourself, or are using an unrecognized distribution kernel), you can\n"	212	"If you are certain your kernel is not vulnerable (you have compiled it\n"
220	"override this safety check by exporting SANITIZER_IGNORE_CVE_2016_2143\n"	213	"yourself, or are using an unrecognized distribution kernel), you can\n"
221	"with any value.\n");	214	"override this safety check by exporting SANITIZER_IGNORE_CVE_2016_2143\n"
		215	"with any value.\n");
222	Die();	216	Die();
223	}	217	}
224	#endif	218	# endif
225		219
226	} // namespace __sanitizer	220	} // namespace __sanitizer
227		221
228	#endif // SANITIZER_LINUX && SANITIZER_S390	222	#endif // SANITIZER_LINUX && SANITIZER_S390

lib/tsan/sanitizer_common/sanitizer_mac.cpp+4-4

lib/tsan/sanitizer_common/sanitizer_mallinfo.h+4

 @@ -31,6 +31,10 @@ struct __sanitizer_struct_mallinfo {
   int v[10];
 };
+struct __sanitizer_struct_mallinfo2 {
+  uptr v[10];
+};
 #endif
 }  // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_malloc_mac.inc+1-1

...	@@ -123,7 +123,7 @@ INTERCEPTOR(void, malloc_set_zone_name, malloc_zone_t zone, const char name) {	...	@@ -123,7 +123,7 @@ INTERCEPTOR(void, malloc_set_zone_name, malloc_zone_t zone, const char name) {
123	COMMON_MALLOC_ENTER();	123	COMMON_MALLOC_ENTER();
124	InternalScopedString new_name;	124	InternalScopedString new_name;
125	if (name && zone->introspect == sanitizer_zone.introspect) {	125	if (name && zone->introspect == sanitizer_zone.introspect) {
126	new_name.append(COMMON_MALLOC_ZONE_NAME "-%s", name);	126	new_name.AppendF(COMMON_MALLOC_ZONE_NAME "-%s", name);
127	name = new_name.data();	127	name = new_name.data();
128	}	128	}
129		129

lib/tsan/sanitizer_common/sanitizer_mutex.cpp+4-2

...	@@ -212,8 +212,10 @@ struct InternalDeadlockDetector {	...	@@ -212,8 +212,10 @@ struct InternalDeadlockDetector {
212	return initialized > 0;	212	return initialized > 0;
213	}	213	}
214	};	214	};
215		215	// This variable is used by the __tls_get_addr interceptor, so cannot use the
216	static THREADLOCAL InternalDeadlockDetector deadlock_detector;	216	// global-dynamic TLS model, as that would result in crashes.
		217	__attribute__((tls_model("initial-exec"))) static THREADLOCAL
		218	InternalDeadlockDetector deadlock_detector;
217		219
218	void CheckedMutex::LockImpl(uptr pc) { deadlock_detector.Lock(type_, pc); }	220	void CheckedMutex::LockImpl(uptr pc) { deadlock_detector.Lock(type_, pc); }
219		221

lib/tsan/sanitizer_common/sanitizer_placement_new.h+1-3

...	@@ -17,8 +17,6 @@	...	@@ -17,8 +17,6 @@
17		17
18	#include "sanitizer_internal_defs.h"	18	#include "sanitizer_internal_defs.h"
19		19
20	inline void operator new(__sanitizer::operator_new_size_type sz, void p) {	20	inline void operator new(__sanitizer::usize sz, void p) { return p; }
21	return p;
22	}
23		21
24	#endif // SANITIZER_PLACEMENT_NEW_H	22	#endif // SANITIZER_PLACEMENT_NEW_H

lib/tsan/sanitizer_common/sanitizer_platform.h+22-2

...	@@ -260,6 +260,17 @@	...	@@ -260,6 +260,17 @@
260	# define SANITIZER_ARM64 0	260	# define SANITIZER_ARM64 0
261	#endif	261	#endif
262		262
		263	#if SANITIZER_WINDOWS64 && SANITIZER_ARM64
		264	# define SANITIZER_WINDOWS_ARM64 1
		265	# define SANITIZER_WINDOWS_x64 0
		266	#elif SANITIZER_WINDOWS64 && !SANITIZER_ARM64
		267	# define SANITIZER_WINDOWS_ARM64 0
		268	# define SANITIZER_WINDOWS_x64 1
		269	#else
		270	# define SANITIZER_WINDOWS_ARM64 0
		271	# define SANITIZER_WINDOWS_x64 0
		272	#endif
		273
263	#if SANITIZER_SOLARIS && SANITIZER_WORDSIZE == 32	274	#if SANITIZER_SOLARIS && SANITIZER_WORDSIZE == 32
264	# define SANITIZER_SOLARIS32 1	275	# define SANITIZER_SOLARIS32 1
265	#else	276	#else
...	@@ -284,7 +295,8 @@	...	@@ -284,7 +295,8 @@
284	// For such platforms build this code with -DSANITIZER_CAN_USE_ALLOCATOR64=0 or	295	// For such platforms build this code with -DSANITIZER_CAN_USE_ALLOCATOR64=0 or
285	// change the definition of SANITIZER_CAN_USE_ALLOCATOR64 here.	296	// change the definition of SANITIZER_CAN_USE_ALLOCATOR64 here.
286	#ifndef SANITIZER_CAN_USE_ALLOCATOR64	297	#ifndef SANITIZER_CAN_USE_ALLOCATOR64
287	# if SANITIZER_RISCV64 \|\| SANITIZER_IOS	298	# if (SANITIZER_RISCV64 && !SANITIZER_FUCHSIA && !SANITIZER_LINUX) \|\| \
		299	SANITIZER_IOS \|\| SANITIZER_DRIVERKIT
288	# define SANITIZER_CAN_USE_ALLOCATOR64 0	300	# define SANITIZER_CAN_USE_ALLOCATOR64 0
289	# elif defined(__mips64) \|\| defined(__hexagon__)	301	# elif defined(__mips64) \|\| defined(__hexagon__)
290	# define SANITIZER_CAN_USE_ALLOCATOR64 0	302	# define SANITIZER_CAN_USE_ALLOCATOR64 0
...	@@ -303,7 +315,15 @@	...	@@ -303,7 +315,15 @@
303	# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 40)	315	# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 40)
304	# endif	316	# endif
305	#elif SANITIZER_RISCV64	317	#elif SANITIZER_RISCV64
306	# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 38)	318	// FIXME: Rather than hardcoding the VMA here, we should rely on
		319	// GetMaxUserVirtualAddress(). This will require some refactoring though since
		320	// many places either hardcode some value or SANITIZER_MMAP_RANGE_SIZE is
		321	// assumed to be some constant integer.
		322	# if SANITIZER_FUCHSIA
		323	# define SANITIZER_MMAP_RANGE_SIZE (1ULL << 38)
		324	# else
		325	# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 56)
		326	# endif
307	#elif defined(__aarch64__)	327	#elif defined(__aarch64__)
308	# if SANITIZER_APPLE	328	# if SANITIZER_APPLE
309	# if SANITIZER_OSX \|\| SANITIZER_IOSSIM	329	# if SANITIZER_OSX \|\| SANITIZER_IOSSIM

lib/tsan/sanitizer_common/sanitizer_platform_interceptors.h+10-6

 @@ -191,7 +191,8 @@
 #define SANITIZER_INTERCEPT_PREADV \
   (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID)
-#define SANITIZER_INTERCEPT_PWRITEV SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_PWRITEV \
+  (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID)
 #define SANITIZER_INTERCEPT_PREADV64 SI_GLIBC
 #define SANITIZER_INTERCEPT_PWRITEV64 SI_GLIBC
 @@ -301,7 +302,8 @@
 #define SANITIZER_INTERCEPT_CANONICALIZE_FILE_NAME (SI_GLIBC || SI_SOLARIS)
 #define SANITIZER_INTERCEPT_CONFSTR \
   (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
-#define SANITIZER_INTERCEPT_SCHED_GETAFFINITY SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_SCHED_GETAFFINITY \
+  (SI_LINUX_NOT_ANDROID || SI_FREEBSD)
 #define SANITIZER_INTERCEPT_SCHED_GETPARAM SI_LINUX_NOT_ANDROID || SI_SOLARIS
 #define SANITIZER_INTERCEPT_STRERROR SI_POSIX
 #define SANITIZER_INTERCEPT_STRERROR_R SI_POSIX
 @@ -462,7 +464,7 @@
   (SI_LINUX || SI_MAC || SI_WINDOWS || SI_FREEBSD || SI_NETBSD || SI_SOLARIS)
 #define SANITIZER_INTERCEPT_RECV_RECVFROM SI_POSIX
 #define SANITIZER_INTERCEPT_SEND_SENDTO SI_POSIX
-#define SANITIZER_INTERCEPT_EVENTFD_READ_WRITE SI_LINUX
+#define SANITIZER_INTERCEPT_EVENTFD_READ_WRITE (SI_LINUX || SI_FREEBSD)
 #define SI_STAT_LINUX (SI_LINUX && __GLIBC_PREREQ(2, 33))
 #define SANITIZER_INTERCEPT_STAT                                        \
 @@ -575,12 +577,12 @@
 #define SANITIZER_INTERCEPT_SL_INIT (SI_FREEBSD || SI_NETBSD)
 #define SANITIZER_INTERCEPT_GETRANDOM \
-  ((SI_LINUX && __GLIBC_PREREQ(2, 25)) || SI_FREEBSD)
+  ((SI_LINUX && __GLIBC_PREREQ(2, 25)) || SI_FREEBSD || SI_SOLARIS)
 #define SANITIZER_INTERCEPT___CXA_ATEXIT SI_NETBSD
 #define SANITIZER_INTERCEPT_ATEXIT SI_NETBSD
 #define SANITIZER_INTERCEPT_PTHREAD_ATFORK SI_NETBSD
 #define SANITIZER_INTERCEPT_GETENTROPY \
-  ((SI_LINUX && __GLIBC_PREREQ(2, 25)) || SI_FREEBSD)
+  ((SI_LINUX && __GLIBC_PREREQ(2, 25)) || SI_FREEBSD || SI_SOLARIS)
 #define SANITIZER_INTERCEPT_QSORT \
   (SI_POSIX && !SI_IOSSIM && !SI_WATCHOS && !SI_TVOS && !SI_ANDROID)
 #define SANITIZER_INTERCEPT_QSORT_R SI_GLIBC
 @@ -594,9 +596,11 @@
 #define SANITIZER_INTERCEPT___XUNAME SI_FREEBSD
 #define SANITIZER_INTERCEPT_FLOPEN SI_FREEBSD
 #define SANITIZER_INTERCEPT_PROCCTL SI_FREEBSD
-#define SANITIZER_INTERCEPT_HEXDUMP SI_FREEBSD
 #define SANITIZER_INTERCEPT_ARGP_PARSE SI_GLIBC
 #define SANITIZER_INTERCEPT_CPUSET_GETAFFINITY SI_FREEBSD
+// FIXME: also available from musl 1.2.5
+#define SANITIZER_INTERCEPT_PREADV2 (SI_LINUX && __GLIBC_PREREQ(2, 26))
+#define SANITIZER_INTERCEPT_PWRITEV2 (SI_LINUX && __GLIBC_PREREQ(2, 26))
 // This macro gives a way for downstream users to override the above
 // interceptor macros irrespective of the platform they are on. They have

lib/tsan/sanitizer_common/sanitizer_platform_limits_freebsd.cpp+2

...	@@ -475,6 +475,8 @@ CHECK_TYPE_SIZE(nfds_t);	...	@@ -475,6 +475,8 @@ CHECK_TYPE_SIZE(nfds_t);
475	CHECK_TYPE_SIZE(sigset_t);	475	CHECK_TYPE_SIZE(sigset_t);
476		476
477	COMPILER_CHECK(sizeof(__sanitizer_sigaction) == sizeof(struct sigaction));	477	COMPILER_CHECK(sizeof(__sanitizer_sigaction) == sizeof(struct sigaction));
		478	COMPILER_CHECK(sizeof(__sanitizer_siginfo) == sizeof(siginfo_t));
		479	CHECK_SIZE_AND_OFFSET(siginfo_t, si_value);
478	// Can't write checks for sa_handler and sa_sigaction due to them being	480	// Can't write checks for sa_handler and sa_sigaction due to them being
479	// preprocessor macros.	481	// preprocessor macros.
480	CHECK_STRUCT_SIZE_AND_OFFSET(sigaction, sa_mask);	482	CHECK_STRUCT_SIZE_AND_OFFSET(sigaction, sa_mask);

lib/tsan/sanitizer_common/sanitizer_platform_limits_freebsd.h+22-2

...	@@ -301,11 +301,29 @@ struct __sanitizer_sigset_t {	...	@@ -301,11 +301,29 @@ struct __sanitizer_sigset_t {
301		301
302	typedef __sanitizer_sigset_t __sanitizer_kernel_sigset_t;	302	typedef __sanitizer_sigset_t __sanitizer_kernel_sigset_t;
303		303
		304	union __sanitizer_sigval {
		305	int sival_int;
		306	void *sival_ptr;
		307	};
		308
304	struct __sanitizer_siginfo {	309	struct __sanitizer_siginfo {
305	// The size is determined by looking at sizeof of real siginfo_t on linux.	310	int si_signo;
306	u64 opaque[128 / sizeof(u64)];	311	int si_errno;
		312	int si_code;
		313	pid_t si_pid;
		314	u32 si_uid;
		315	int si_status;
		316	void *si_addr;
		317	union __sanitizer_sigval si_value;
		318	# if SANITIZER_WORDSIZE == 64
		319	char data[40];
		320	# else
		321	char data[32];
		322	# endif
307	};	323	};
308		324
		325	typedef __sanitizer_siginfo __sanitizer_siginfo_t;
		326
309	using __sanitizer_sighandler_ptr = void (*)(int sig);	327	using __sanitizer_sighandler_ptr = void (*)(int sig);
310	using __sanitizer_sigactionhandler_ptr = void (*)(int sig,	328	using __sanitizer_sigactionhandler_ptr = void (*)(int sig,
311	__sanitizer_siginfo *siginfo,	329	__sanitizer_siginfo *siginfo,
...	@@ -726,6 +744,8 @@ struct __sanitizer_cpuset {	...	@@ -726,6 +744,8 @@ struct __sanitizer_cpuset {
726		744
727	typedef struct __sanitizer_cpuset __sanitizer_cpuset_t;	745	typedef struct __sanitizer_cpuset __sanitizer_cpuset_t;
728	extern unsigned struct_cpuset_sz;	746	extern unsigned struct_cpuset_sz;
		747
		748	typedef unsigned long long __sanitizer_eventfd_t;
729	} // namespace __sanitizer	749	} // namespace __sanitizer
730		750
731	# define CHECK_TYPE_SIZE(TYPE) \	751	# define CHECK_TYPE_SIZE(TYPE) \

lib/tsan/sanitizer_common/sanitizer_platform_limits_openbsd.cpp deleted

lib/tsan/sanitizer_common/sanitizer_platform_limits_openbsd.h deleted

lib/tsan/sanitizer_common/sanitizer_platform_limits_posix.h+1

...	@@ -523,6 +523,7 @@ typedef long __sanitizer_clock_t;	...	@@ -523,6 +523,7 @@ typedef long __sanitizer_clock_t;
523		523
524	#if SANITIZER_LINUX	524	#if SANITIZER_LINUX
525	typedef int __sanitizer_clockid_t;	525	typedef int __sanitizer_clockid_t;
		526	typedef unsigned long long __sanitizer_eventfd_t;
526	#endif	527	#endif
527		528
528	#if SANITIZER_LINUX	529	#if SANITIZER_LINUX

lib/tsan/sanitizer_common/sanitizer_posix.cpp+6-6

...	@@ -54,12 +54,12 @@ void MmapOrDie(uptr size, const char mem_type, bool raw_report) {	...	@@ -54,12 +54,12 @@ void MmapOrDie(uptr size, const char mem_type, bool raw_report) {
54	return (void *)res;	54	return (void *)res;
55	}	55	}
56		56
57	void UnmapOrDie(void *addr, uptr size) {	57	void UnmapOrDie(void *addr, uptr size, bool raw_report) {
58	if (!addr \|\| !size) return;	58	if (!addr \|\| !size) return;
59	uptr res = internal_munmap(addr, size);	59	uptr res = internal_munmap(addr, size);
60	int reserrno;	60	int reserrno;
61	if (UNLIKELY(internal_iserror(res, &reserrno)))	61	if (UNLIKELY(internal_iserror(res, &reserrno)))
62	ReportMunmapFailureAndDie(addr, size, reserrno);	62	ReportMunmapFailureAndDie(addr, size, reserrno, raw_report);
63	DecreaseTotalMmap(size);	63	DecreaseTotalMmap(size);
64	}	64	}
65		65
...	@@ -85,8 +85,8 @@ void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment,	...	@@ -85,8 +85,8 @@ void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment,
85	CHECK(IsPowerOfTwo(size));	85	CHECK(IsPowerOfTwo(size));
86	CHECK(IsPowerOfTwo(alignment));	86	CHECK(IsPowerOfTwo(alignment));
87	uptr map_size = size + alignment;	87	uptr map_size = size + alignment;
88	// mmap maps entire pages and rounds up map_size needs to be a an integral	88	// mmap maps entire pages and rounds up map_size needs to be a an integral
89	// number of pages.	89	// number of pages.
90	// We need to be aware of this size for calculating end and for unmapping	90	// We need to be aware of this size for calculating end and for unmapping
91	// fragments before and after the alignment region.	91	// fragments before and after the alignment region.
92	map_size = RoundUpTo(map_size, GetPageSizeCached());	92	map_size = RoundUpTo(map_size, GetPageSizeCached());
...	@@ -130,8 +130,8 @@ static void *MmapFixedImpl(uptr fixed_addr, uptr size, bool tolerate_enomem,	...	@@ -130,8 +130,8 @@ static void *MmapFixedImpl(uptr fixed_addr, uptr size, bool tolerate_enomem,
130	if (tolerate_enomem && reserrno == ENOMEM)	130	if (tolerate_enomem && reserrno == ENOMEM)
131	return nullptr;	131	return nullptr;
132	char mem_type[40];	132	char mem_type[40];
133	internal_snprintf(mem_type, sizeof(mem_type), "memory at address 0x%zx",	133	internal_snprintf(mem_type, sizeof(mem_type), "memory at address %p",
134	fixed_addr);	134	(void *)fixed_addr);
135	ReportMmapFailureAndDie(size, mem_type, "allocate", reserrno);	135	ReportMmapFailureAndDie(size, mem_type, "allocate", reserrno);
136	}	136	}
137	IncreaseTotalMmap(size);	137	IncreaseTotalMmap(size);

lib/tsan/sanitizer_common/sanitizer_posix.h+14-14

...	@@ -74,21 +74,21 @@ int internal_sysctlbyname(const char sname, void oldp, uptr *oldlenp,	...	@@ -74,21 +74,21 @@ int internal_sysctlbyname(const char sname, void oldp, uptr *oldlenp,
74	// These functions call appropriate pthread_ functions directly, bypassing	74	// These functions call appropriate pthread_ functions directly, bypassing
75	// the interceptor. They are weak and may not be present in some tools.	75	// the interceptor. They are weak and may not be present in some tools.
76	SANITIZER_WEAK_ATTRIBUTE	76	SANITIZER_WEAK_ATTRIBUTE
77	int real_pthread_create(void th, void attr, void (callback)(void *),	77	int internal_pthread_create(void th, void attr, void (callback)(void *),
78	void *param);	78	void *param);
79	SANITIZER_WEAK_ATTRIBUTE	79	SANITIZER_WEAK_ATTRIBUTE
80	int real_pthread_join(void th, void *ret);	80	int internal_pthread_join(void th, void *ret);
81		81
82	#define DEFINE_REAL_PTHREAD_FUNCTIONS \	82	# define DEFINE_INTERNAL_PTHREAD_FUNCTIONS \
83	namespace __sanitizer { \	83	namespace __sanitizer { \
84	int real_pthread_create(void th, void attr, void (callback)(void *), \	84	int internal_pthread_create(void th, void attr, \
85	void *param) { \	85	void (callback)(void ), void param) { \
86	return REAL(pthread_create)(th, attr, callback, param); \	86	return REAL(pthread_create)(th, attr, callback, param); \
87	} \	87	} \
88	int real_pthread_join(void th, void *ret) { \	88	int internal_pthread_join(void th, void *ret) { \
89	return REAL(pthread_join(th, ret)); \	89	return REAL(pthread_join(th, ret)); \
90	} \	90	} \
91	} // namespace __sanitizer	91	} // namespace __sanitizer
92		92
93	int internal_pthread_attr_getstack(void attr, void addr, uptr size);	93	int internal_pthread_attr_getstack(void attr, void addr, uptr size);
94		94

lib/tsan/sanitizer_common/sanitizer_posix_libcdep.cpp+27-6

 @@ -91,12 +91,12 @@ static rlim_t getlim(int res) {
 static void setlim(int res, rlim_t lim) {
   struct rlimit rlim;
-  if (getrlimit(res, const_cast<struct rlimit *>(&rlim))) {
+  if (getrlimit(res, &rlim)) {
     Report("ERROR: %s getrlimit() failed %d\n", SanitizerToolName, errno);
     Die();
   }
   rlim.rlim_cur = lim;
-  if (setrlimit(res, const_cast<struct rlimit *>(&rlim))) {
+  if (setrlimit(res, &rlim)) {
     Report("ERROR: %s setrlimit() failed %d\n", SanitizerToolName, errno);
     Die();
   }
 @@ -104,7 +104,27 @@ static void setlim(int res, rlim_t lim) {
 void DisableCoreDumperIfNecessary() {
   if (common_flags()->disable_coredump) {
-    setlim(RLIMIT_CORE, 0);
+    rlimit rlim;
+    CHECK_EQ(0, getrlimit(RLIMIT_CORE, &rlim));
+    // On Linux, if the kernel.core_pattern sysctl starts with a '|' (i.e. it
+    // is being piped to a coredump handler such as systemd-coredumpd), the
+    // kernel ignores RLIMIT_CORE (since we aren't creating a file in the file
+    // system) except for the magic value of 1, which disables coredumps when
+    // piping. 1 byte is too small for any kind of valid core dump, so it
+    // also disables coredumps if kernel.core_pattern creates files directly.
+    // While most piped coredump handlers do respect the crashing processes'
+    // RLIMIT_CORE, this is notable not the case for Debian's systemd-coredump
+    // due to a local patch that changes sysctl.d/50-coredump.conf to ignore
+    // the specified limit and instead use RLIM_INFINITY.
+    //
+    // The alternative to using RLIMIT_CORE=1 would be to use prctl() with the
+    // PR_SET_DUMPABLE flag, however that also prevents ptrace(), so makes it
+    // impossible to attach a debugger.
+    //
+    // Note: we use rlim_max in the Min() call here since that is the upper
+    // limit for what can be set without getting an EINVAL error.
+    rlim.rlim_cur = Min<rlim_t>(SANITIZER_LINUX ? 1 : 0, rlim.rlim_max);
+    CHECK_EQ(0, setrlimit(RLIMIT_CORE, &rlim));
   }
 }
 @@ -307,9 +327,10 @@ static bool MmapFixed(uptr fixed_addr, uptr size, int additional_flags,
                 MAP_PRIVATE | MAP_FIXED | additional_flags | MAP_ANON, name);
   int reserrno;
   if (internal_iserror(p, &reserrno)) {
-    Report("ERROR: %s failed to "
+    Report(
-           "allocate 0x%zx (%zd) bytes at address %zx (errno: %d)\n",
+        "ERROR: %s failed to "
-           SanitizerToolName, size, size, fixed_addr, reserrno);
+        "allocate 0x%zx (%zd) bytes at address %p (errno: %d)\n",
+        SanitizerToolName, size, size, (void *)fixed_addr, reserrno);
     return false;
   }
   IncreaseTotalMmap(size);

lib/tsan/sanitizer_common/sanitizer_printf.cpp+9-2

 @@ -54,7 +54,7 @@ static int AppendNumber(char **buff, const char *buff_end, u64 absolute_value,
   uptr num_buffer[kMaxLen];
   int pos = 0;
   do {
-    RAW_CHECK_MSG((uptr)pos < kMaxLen, "AppendNumber buffer overflow");
+    RAW_CHECK_MSG((uptr)pos < kMaxLen, "AppendNumber buffer overflow",);
     num_buffer[pos++] = absolute_value % base;
     absolute_value /= base;
   } while (absolute_value > 0);
 @@ -337,7 +337,14 @@ int internal_snprintf(char *buffer, uptr length, const char *format, ...) {
   return needed_length;
 }
-void InternalScopedString::append(const char *format, ...) {
+void InternalScopedString::Append(const char *str) {
+  uptr prev_len = length();
+  uptr str_len = internal_strlen(str);
+  buffer_.resize(prev_len + str_len + 1);
+  internal_memcpy(buffer_.data() + prev_len, str, str_len + 1);
+}
+void InternalScopedString::AppendF(const char *format, ...) {
   uptr prev_len = length();
   while (true) {

lib/tsan/sanitizer_common/sanitizer_procmaps_bsd.cpp+25-22

...	@@ -13,9 +13,6 @@	...	@@ -13,9 +13,6 @@
13	#include "sanitizer_platform.h"	13	#include "sanitizer_platform.h"
14	#if SANITIZER_FREEBSD \|\| SANITIZER_NETBSD	14	#if SANITIZER_FREEBSD \|\| SANITIZER_NETBSD
15	#include "sanitizer_common.h"	15	#include "sanitizer_common.h"
16	#if SANITIZER_FREEBSD
17	#include "sanitizer_freebsd.h"
18	#endif
19	#include "sanitizer_procmaps.h"	16	#include "sanitizer_procmaps.h"
20		17
21	// clang-format off	18	// clang-format off
...	@@ -29,29 +26,35 @@	...	@@ -29,29 +26,35 @@
29		26
30	#include <limits.h>	27	#include <limits.h>
31		28
32	// Fix 'kinfo_vmentry' definition on FreeBSD prior v9.2 in 32-bit mode.
33	#if SANITIZER_FREEBSD && (SANITIZER_WORDSIZE == 32)
34	#include <osreldate.h>
35	#if __FreeBSD_version <= 902001 // v9.2
36	#define kinfo_vmentry xkinfo_vmentry
37	#endif
38	#endif
39
40	namespace __sanitizer {	29	namespace __sanitizer {
41		30
42	#if SANITIZER_FREEBSD	31	#if SANITIZER_FREEBSD
43	void GetMemoryProfile(fill_profile_f cb, uptr *stats) {	32	void GetMemoryProfile(fill_profile_f cb, uptr *stats) {
44	const int Mib[] = {	33	const int Mib[] = {CTL_KERN, KERN_PROC, KERN_PROC_PID, getpid()};
45	CTL_KERN,	34
46	KERN_PROC,	35	struct kinfo_proc *InfoProc;
47	KERN_PROC_PID,	36	uptr Len = sizeof(*InfoProc);
48	getpid()	37	uptr Size = Len;
49	};	38	InfoProc = (struct kinfo_proc *)MmapOrDie(Size, "GetMemoryProfile()");
50		39	CHECK_EQ(
51	struct kinfo_proc InfoProc;	40	internal_sysctl(Mib, ARRAY_SIZE(Mib), nullptr, (uptr *)InfoProc, &Len, 0),
52	uptr Len = sizeof(InfoProc);	41	0);
53	CHECK_EQ(internal_sysctl(Mib, ARRAY_SIZE(Mib), nullptr, (uptr *)&InfoProc, &Len, 0), 0);	42	cb(0, InfoProc->ki_rssize * GetPageSizeCached(), false, stats);
54	cb(0, InfoProc.ki_rssize * GetPageSizeCached(), false, stats);	43	UnmapOrDie(InfoProc, Size, true);
		44	}
		45	#elif SANITIZER_NETBSD
		46	void GetMemoryProfile(fill_profile_f cb, uptr *stats) {
		47	struct kinfo_proc2 *InfoProc;
		48	uptr Len = sizeof(*InfoProc);
		49	uptr Size = Len;
		50	const int Mib[] = {CTL_KERN, KERN_PROC2, KERN_PROC_PID,
		51	getpid(), (int)Size, 1};
		52	InfoProc = (struct kinfo_proc2 *)MmapOrDie(Size, "GetMemoryProfile()");
		53	CHECK_EQ(
		54	internal_sysctl(Mib, ARRAY_SIZE(Mib), nullptr, (uptr *)InfoProc, &Len, 0),
		55	0);
		56	cb(0, InfoProc->p_vm_rssize * GetPageSizeCached(), false, stats);
		57	UnmapOrDie(InfoProc, Size, true);
55	}	58	}
56	#endif	59	#endif
57		60

lib/tsan/sanitizer_common/sanitizer_procmaps_common.cpp+1-1

...	@@ -145,7 +145,7 @@ void MemoryMappingLayout::DumpListOfModules(	...	@@ -145,7 +145,7 @@ void MemoryMappingLayout::DumpListOfModules(
145	}	145	}
146	}	146	}
147		147
148	#if SANITIZER_LINUX \|\| SANITIZER_ANDROID \|\| SANITIZER_SOLARIS \|\| SANITIZER_NETBSD	148	#if SANITIZER_LINUX \|\| SANITIZER_ANDROID \|\| SANITIZER_SOLARIS
149	void GetMemoryProfile(fill_profile_f cb, uptr *stats) {	149	void GetMemoryProfile(fill_profile_f cb, uptr *stats) {
150	char *smaps = nullptr;	150	char *smaps = nullptr;
151	uptr smaps_cap = 0;	151	uptr smaps_cap = 0;

lib/tsan/sanitizer_common/sanitizer_ptrauth.h+24-22

...	@@ -9,31 +9,33 @@	...	@@ -9,31 +9,33 @@
9	#ifndef SANITIZER_PTRAUTH_H	9	#ifndef SANITIZER_PTRAUTH_H
10	#define SANITIZER_PTRAUTH_H	10	#define SANITIZER_PTRAUTH_H
11		11
12	#if __has_feature(ptrauth_calls)	12	#if __has_feature(ptrauth_intrinsics)
13	#include <ptrauth.h>	13	# include <ptrauth.h>
14	#elif defined(__ARM_FEATURE_PAC_DEFAULT) && !defined(__APPLE__)	14	#elif defined(__ARM_FEATURE_PAC_DEFAULT) && !defined(__APPLE__)
15	inline unsigned long ptrauth_strip(void* __value, unsigned int __key) {	15	// On the stack the link register is protected with Pointer
16	// On the stack the link register is protected with Pointer	16	// Authentication Code when compiled with -mbranch-protection.
17	// Authentication Code when compiled with -mbranch-protection.	17	// Let's stripping the PAC unconditionally because xpaclri is in
18	// Let's stripping the PAC unconditionally because xpaclri is in	18	// the NOP space so will do nothing when it is not enabled or not available.
19	// the NOP space so will do nothing when it is not enabled or not available.	19	# define ptrauth_strip(__value, __key) \
20	unsigned long ret;	20	({ \
21	asm volatile(	21	__typeof(__value) ret; \
22	"mov x30, %1\n\t"	22	asm volatile( \
23	"hint #7\n\t" // xpaclri	23	"mov x30, %1\n\t" \
24	"mov %0, x30\n\t"	24	"hint #7\n\t" \
25	: "=r"(ret)	25	"mov %0, x30\n\t" \
26	: "r"(__value)	26	"mov x30, xzr\n\t" \
27	: "x30");	27	: "=r"(ret) \
28	return ret;	28	: "r"(__value) \
29	}	29	: "x30"); \
30	#define ptrauth_auth_data(__value, __old_key, __old_data) __value	30	ret; \
31	#define ptrauth_string_discriminator(__string) ((int)0)	31	})
		32	# define ptrauth_auth_data(__value, __old_key, __old_data) __value
		33	# define ptrauth_string_discriminator(__string) ((int)0)
32	#else	34	#else
33	// Copied from <ptrauth.h>	35	// Copied from <ptrauth.h>
34	#define ptrauth_strip(__value, __key) __value	36	# define ptrauth_strip(__value, __key) __value
35	#define ptrauth_auth_data(__value, __old_key, __old_data) __value	37	# define ptrauth_auth_data(__value, __old_key, __old_data) __value
36	#define ptrauth_string_discriminator(__string) ((int)0)	38	# define ptrauth_string_discriminator(__string) ((int)0)
37	#endif	39	#endif
38		40
39	#define STRIP_PAC_PC(pc) ((uptr)ptrauth_strip(pc, 0))	41	#define STRIP_PAC_PC(pc) ((uptr)ptrauth_strip(pc, 0))

lib/tsan/sanitizer_common/sanitizer_redefine_builtins.h+10-6

 @@ -11,16 +11,19 @@
 //
 //===----------------------------------------------------------------------===//
 #ifndef SANITIZER_COMMON_NO_REDEFINE_BUILTINS
-#ifndef SANITIZER_REDEFINE_BUILTINS_H
+#  ifndef SANITIZER_REDEFINE_BUILTINS_H
-#define SANITIZER_REDEFINE_BUILTINS_H
+#    define SANITIZER_REDEFINE_BUILTINS_H
 // The asm hack only works with GCC and Clang.
-#if !defined(_WIN32)
+#    if !defined(_WIN32)
 asm("memcpy = __sanitizer_internal_memcpy");
 asm("memmove = __sanitizer_internal_memmove");
 asm("memset = __sanitizer_internal_memset");
+#      if defined(__cplusplus) && \
+          !defined(SANITIZER_COMMON_REDEFINE_BUILTINS_IN_STD)
 // The builtins should not be redefined in source files that make use of C++
 // standard libraries, in particular where C++STL headers with inline functions
 // are used. The redefinition in such cases would lead to ODR violations.
 @@ -46,7 +49,8 @@ using unordered_set = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file;
 using vector = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file;
 }  // namespace std
-#endif  // !_WIN32
+#      endif  // __cpluplus
+#    endif    // !_WIN32
-#endif  // SANITIZER_REDEFINE_BUILTINS_H
+#  endif  // SANITIZER_REDEFINE_BUILTINS_H
-#endif  // SANITIZER_COMMON_NO_REDEFINE_BUILTINS
+#endif    // SANITIZER_COMMON_NO_REDEFINE_BUILTINS

lib/tsan/sanitizer_common/sanitizer_ring_buffer.h+3-1

 @@ -47,7 +47,9 @@ class RingBuffer {
   void push(T t) {
     *next_ = t;
     next_--;
-    // The condition below works only if sizeof(T) is divisible by sizeof(T*).
+    static_assert((sizeof(T) % sizeof(T *)) == 0,
+                  "The condition below works only if sizeof(T) is divisible by "
+                  "sizeof(T*).");
     if (next_ <= reinterpret_cast<T*>(&next_))
       next_ = last_;
   }

lib/tsan/sanitizer_common/sanitizer_stack_store.cpp+7-2

 @@ -44,6 +44,9 @@ StackStore::Id StackStore::Store(const StackTrace &trace, uptr *pack) {
   uptr idx = 0;
   *pack = 0;
   uptr *stack_trace = Alloc(h.size + 1, &idx, pack);
+  // No more space.
+  if (stack_trace == nullptr)
+    return 0;
   *stack_trace = h.ToUptr();
   internal_memcpy(stack_trace + 1, trace.trace, h.size * sizeof(uptr));
   *pack += blocks_[GetBlockIdx(idx)].Stored(h.size + 1);
 @@ -76,8 +79,10 @@ uptr *StackStore::Alloc(uptr count, uptr *idx, uptr *pack) {
     uptr block_idx = GetBlockIdx(start);
     uptr last_idx = GetBlockIdx(start + count - 1);
     if (LIKELY(block_idx == last_idx)) {
-      // Fits into the a single block.
+      // Fits into a single block.
-      CHECK_LT(block_idx, ARRAY_SIZE(blocks_));
+      // No more available blocks.  Indicate inability to allocate more memory.
+      if (block_idx >= ARRAY_SIZE(blocks_))
+        return nullptr;
       *idx = start;
       return blocks_[block_idx].GetOrCreate(this) + GetInBlockIdx(start);
     }

lib/tsan/sanitizer_common/sanitizer_stackdepot.cpp+4-4

...	@@ -215,16 +215,16 @@ StackTrace StackDepotGet(u32 id) {	...	@@ -215,16 +215,16 @@ StackTrace StackDepotGet(u32 id) {
215	return theDepot.Get(id);	215	return theDepot.Get(id);
216	}	216	}
217		217
218	void StackDepotLockAll() {	218	void StackDepotLockBeforeFork() {
219	theDepot.LockAll();	219	theDepot.LockBeforeFork();
220	compress_thread.LockAndStop();	220	compress_thread.LockAndStop();
221	stackStore.LockAll();	221	stackStore.LockAll();
222	}	222	}
223		223
224	void StackDepotUnlockAll() {	224	void StackDepotUnlockAfterFork(bool fork_child) {
225	stackStore.UnlockAll();	225	stackStore.UnlockAll();
226	compress_thread.Unlock();	226	compress_thread.Unlock();
227	theDepot.UnlockAll();	227	theDepot.UnlockAfterFork(fork_child);
228	}	228	}
229		229
230	void StackDepotPrintAll() {	230	void StackDepotPrintAll() {

lib/tsan/sanitizer_common/sanitizer_stackdepot.h+2-2

...	@@ -39,8 +39,8 @@ StackDepotHandle StackDepotPut_WithHandle(StackTrace stack);	...	@@ -39,8 +39,8 @@ StackDepotHandle StackDepotPut_WithHandle(StackTrace stack);
39	// Retrieves a stored stack trace by the id.	39	// Retrieves a stored stack trace by the id.
40	StackTrace StackDepotGet(u32 id);	40	StackTrace StackDepotGet(u32 id);
41		41
42	void StackDepotLockAll();	42	void StackDepotLockBeforeFork();
43	void StackDepotUnlockAll();	43	void StackDepotUnlockAfterFork(bool fork_child);
44	void StackDepotPrintAll();	44	void StackDepotPrintAll();
45	void StackDepotStopBackgroundThread();	45	void StackDepotStopBackgroundThread();
46		46

lib/tsan/sanitizer_common/sanitizer_stackdepotbase.h+23-8

 @@ -52,8 +52,8 @@ class StackDepotBase {
     };
   }
-  void LockAll();
+  void LockBeforeFork();
-  void UnlockAll();
+  void UnlockAfterFork(bool fork_child);
   void PrintAll();
   void TestOnlyUnmap() {
 @@ -160,18 +160,33 @@ StackDepotBase<Node, kReservedBits, kTabSizeLog>::Get(u32 id) {
 }
 template <class Node, int kReservedBits, int kTabSizeLog>
-void StackDepotBase<Node, kReservedBits, kTabSizeLog>::LockAll() {
+void StackDepotBase<Node, kReservedBits, kTabSizeLog>::LockBeforeFork() {
-  for (int i = 0; i < kTabSize; ++i) {
+  // Do not lock hash table. It's very expensive, but it's not rely needed. The
-    lock(&tab[i]);
+  // parent process will neither lock nor unlock. Child process risks to be
-  }
+  // deadlocked on already locked buckets. To avoid deadlock we will unlock
+  // every locked buckets in `UnlockAfterFork`. This may affect consistency of
+  // the hash table, but the only issue is a few items inserted by parent
+  // process will be not found by child, and the child may insert them again,
+  // wasting some space in `stackStore`.
+  // We still need to lock nodes.
+  nodes.Lock();
 }
 template <class Node, int kReservedBits, int kTabSizeLog>
-void StackDepotBase<Node, kReservedBits, kTabSizeLog>::UnlockAll() {
+void StackDepotBase<Node, kReservedBits, kTabSizeLog>::UnlockAfterFork(
+    bool fork_child) {
+  nodes.Unlock();
+  // Only unlock in child process to avoid deadlock. See `LockBeforeFork`.
+  if (!fork_child)
+    return;
   for (int i = 0; i < kTabSize; ++i) {
     atomic_uint32_t *p = &tab[i];
     uptr s = atomic_load(p, memory_order_relaxed);
-    unlock(p, s & kUnlockMask);
+    if (s & kLockMask)
+      unlock(p, s & kUnlockMask);
   }
 }

lib/tsan/sanitizer_common/sanitizer_stacktrace_libcdep.cpp+22-20

 @@ -29,42 +29,43 @@ class StackTraceTextPrinter {
         frame_delimiter_(frame_delimiter),
         output_(output),
         dedup_token_(dedup_token),
-        symbolize_(RenderNeedsSymbolization(stack_trace_fmt)) {}
+        symbolize_(StackTracePrinter::GetOrInit()->RenderNeedsSymbolization(
+            stack_trace_fmt)) {}
   bool ProcessAddressFrames(uptr pc) {
-    SymbolizedStack *frames = symbolize_
+    SymbolizedStackHolder symbolized_stack(
-                                  ? Symbolizer::GetOrInit()->SymbolizePC(pc)
+        symbolize_ ? Symbolizer::GetOrInit()->SymbolizePC(pc)
-                                  : SymbolizedStack::New(pc);
+                   : SymbolizedStack::New(pc));
+    const SymbolizedStack *frames = symbolized_stack.get();
     if (!frames)
       return false;
-    for (SymbolizedStack *cur = frames; cur; cur = cur->next) {
+    for (const SymbolizedStack *cur = frames; cur; cur = cur->next) {
       uptr prev_len = output_->length();
-      RenderFrame(output_, stack_trace_fmt_, frame_num_++, cur->info.address,
+      StackTracePrinter::GetOrInit()->RenderFrame(
-                  symbolize_ ? &cur->info : nullptr,
+          output_, stack_trace_fmt_, frame_num_++, cur->info.address,
-                  common_flags()->symbolize_vs_style,
+          symbolize_ ? &cur->info : nullptr, common_flags()->symbolize_vs_style,
-                  common_flags()->strip_path_prefix);
+          common_flags()->strip_path_prefix);
       if (prev_len != output_->length())
-        output_->append("%c", frame_delimiter_);
+        output_->AppendF("%c", frame_delimiter_);
       ExtendDedupToken(cur);
     }
-    frames->ClearAll();
     return true;
   }
  private:
   // Extend the dedup token by appending a new frame.
-  void ExtendDedupToken(SymbolizedStack *stack) {
+  void ExtendDedupToken(const SymbolizedStack *stack) {
     if (!dedup_token_)
       return;
     if (dedup_frames_-- > 0) {
       if (dedup_token_->length())
-        dedup_token_->append("--");
+        dedup_token_->Append("--");
-      if (stack->info.function != nullptr)
+      if (stack->info.function)
-        dedup_token_->append("%s", stack->info.function);
+        dedup_token_->Append(stack->info.function);
     }
   }
 @@ -98,7 +99,7 @@ void StackTrace::PrintTo(InternalScopedString *output) const {
                                 output, &dedup_token);
   if (trace == nullptr || size == 0) {
-    output->append("    <empty stack>\n\n");
+    output->Append("    <empty stack>\n\n");
     return;
   }
 @@ -110,11 +111,11 @@ void StackTrace::PrintTo(InternalScopedString *output) const {
   }
   // Always add a trailing empty line after stack trace.
-  output->append("\n");
+  output->Append("\n");
   // Append deduplication token, if non-empty.
   if (dedup_token.length())
-    output->append("DEDUP_TOKEN: %s\n", dedup_token.data());
+    output->AppendF("DEDUP_TOKEN: %s\n", dedup_token.data());
 }
 uptr StackTrace::PrintTo(char *out_buf, uptr out_buf_size) const {
 @@ -197,7 +198,7 @@ void __sanitizer_symbolize_pc(uptr pc, const char *fmt, char *out_buf,
   StackTraceTextPrinter printer(fmt, '\0', &output, nullptr);
   if (!printer.ProcessAddressFrames(pc)) {
     output.clear();
-    output.append("<can't symbolize>");
+    output.Append("<can't symbolize>");
   }
   CopyStringToBuffer(output, out_buf, out_buf_size);
 }
 @@ -210,7 +211,8 @@ void __sanitizer_symbolize_global(uptr data_addr, const char *fmt,
   DataInfo DI;
   if (!Symbolizer::GetOrInit()->SymbolizeData(data_addr, &DI)) return;
   InternalScopedString data_desc;
-  RenderData(&data_desc, fmt, &DI, common_flags()->strip_path_prefix);
+  StackTracePrinter::GetOrInit()->RenderData(&data_desc, fmt, &DI,
+                                             common_flags()->strip_path_prefix);
   internal_strncpy(out_buf, data_desc.data(), out_buf_size);
   out_buf[out_buf_size - 1] = 0;
 }

lib/tsan/sanitizer_common/sanitizer_stacktrace_printer.cpp+81-52

 @@ -12,13 +12,28 @@
 #include "sanitizer_stacktrace_printer.h"
+#include "sanitizer_common.h"
 #include "sanitizer_file.h"
 #include "sanitizer_flags.h"
 #include "sanitizer_fuchsia.h"
+#include "sanitizer_symbolizer_markup.h"
 namespace __sanitizer {
-const char *StripFunctionName(const char *function) {
+StackTracePrinter *StackTracePrinter::GetOrInit() {
+  static StackTracePrinter *stacktrace_printer;
+  static StaticSpinMutex init_mu;
+  SpinMutexLock l(&init_mu);
+  if (stacktrace_printer)
+    return stacktrace_printer;
+  stacktrace_printer = StackTracePrinter::NewStackTracePrinter();
+  CHECK(stacktrace_printer);
+  return stacktrace_printer;
+}
+const char *StackTracePrinter::StripFunctionName(const char *function) {
   if (!common_flags()->demangle)
     return function;
   if (!function)
 @@ -47,6 +62,13 @@ const char *StripFunctionName(const char *function) {
 // sanitizer_symbolizer_markup.cpp implements these differently.
 #if !SANITIZER_SYMBOLIZER_MARKUP
+StackTracePrinter *StackTracePrinter::NewStackTracePrinter() {
+  if (common_flags()->enable_symbolizer_markup)
+    return new (GetGlobalLowLevelAllocator()) MarkupStackTracePrinter();
+  return new (GetGlobalLowLevelAllocator()) FormattedStackTracePrinter();
+}
 static const char *DemangleFunctionName(const char *function) {
   if (!common_flags()->demangle)
     return function;
 @@ -130,20 +152,23 @@ static void MaybeBuildIdToBuffer(const AddressInfo &info, bool PrefixSpace,
                                  InternalScopedString *buffer) {
   if (info.uuid_size) {
     if (PrefixSpace)
-      buffer->append(" ");
+      buffer->Append(" ");
-    buffer->append("(BuildId: ");
+    buffer->Append("(BuildId: ");
     for (uptr i = 0; i < info.uuid_size; ++i) {
-      buffer->append("%02x", info.uuid[i]);
+      buffer->AppendF("%02x", info.uuid[i]);
     }
-    buffer->append(")");
+    buffer->Append(")");
   }
 }
 static const char kDefaultFormat[] = "    #%n %p %F %L";
-void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no,
+void FormattedStackTracePrinter::RenderFrame(InternalScopedString *buffer,
-                 uptr address, const AddressInfo *info, bool vs_style,
+                                             const char *format, int frame_no,
-                 const char *strip_path_prefix) {
+                                             uptr address,
+                                             const AddressInfo *info,
+                                             bool vs_style,
+                                             const char *strip_path_prefix) {
   // info will be null in the case where symbolization is not needed for the
   // given format. This ensures that the code below will get a hard failure
   // rather than print incorrect information in case RenderNeedsSymbolization
 @@ -154,56 +179,56 @@ void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no,
     format = kDefaultFormat;
   for (const char *p = format; *p != '\0'; p++) {
     if (*p != '%') {
-      buffer->append("%c", *p);
+      buffer->AppendF("%c", *p);
       continue;
     }
     p++;
     switch (*p) {
     case '%':
-      buffer->append("%%");
+      buffer->Append("%");
       break;
     // Frame number and all fields of AddressInfo structure.
     case 'n':
-      buffer->append("%u", frame_no);
+      buffer->AppendF("%u", frame_no);
       break;
     case 'p':
-      buffer->append("0x%zx", address);
+      buffer->AppendF("%p", (void *)address);
       break;
     case 'm':
-      buffer->append("%s", StripPathPrefix(info->module, strip_path_prefix));
+      buffer->AppendF("%s", StripPathPrefix(info->module, strip_path_prefix));
       break;
     case 'o':
-      buffer->append("0x%zx", info->module_offset);
+      buffer->AppendF("0x%zx", info->module_offset);
       break;
     case 'b':
       MaybeBuildIdToBuffer(*info, /*PrefixSpace=*/false, buffer);
       break;
     case 'f':
-      buffer->append("%s",
+      buffer->AppendF("%s",
-                     DemangleFunctionName(StripFunctionName(info->function)));
+                      DemangleFunctionName(StripFunctionName(info->function)));
       break;
     case 'q':
-      buffer->append("0x%zx", info->function_offset != AddressInfo::kUnknown
+      buffer->AppendF("0x%zx", info->function_offset != AddressInfo::kUnknown
-                                  ? info->function_offset
+                                   ? info->function_offset
-                                  : 0x0);
+                                   : 0x0);
       break;
     case 's':
-      buffer->append("%s", StripPathPrefix(info->file, strip_path_prefix));
+      buffer->AppendF("%s", StripPathPrefix(info->file, strip_path_prefix));
       break;
     case 'l':
-      buffer->append("%d", info->line);
+      buffer->AppendF("%d", info->line);
       break;
     case 'c':
-      buffer->append("%d", info->column);
+      buffer->AppendF("%d", info->column);
       break;
     // Smarter special cases.
     case 'F':
       // Function name and offset, if file is unknown.
       if (info->function) {
-        buffer->append("in %s",
+        buffer->AppendF(
-                       DemangleFunctionName(StripFunctionName(info->function)));
+            "in %s", DemangleFunctionName(StripFunctionName(info->function)));
         if (!info->file && info->function_offset != AddressInfo::kUnknown)
-          buffer->append("+0x%zx", info->function_offset);
+          buffer->AppendF("+0x%zx", info->function_offset);
       }
       break;
     case 'S':
 @@ -224,7 +249,7 @@ void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no,
         MaybeBuildIdToBuffer(*info, /*PrefixSpace=*/true, buffer);
 #endif
       } else {
-        buffer->append("(<unknown module>)");
+        buffer->Append("(<unknown module>)");
       }
       break;
     case 'M':
 @@ -239,18 +264,18 @@ void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no,
         MaybeBuildIdToBuffer(*info, /*PrefixSpace=*/true, buffer);
 #endif
       } else {
-        buffer->append("(%p)", (void *)address);
+        buffer->AppendF("(%p)", (void *)address);
       }
       break;
     default:
       Report("Unsupported specifier in stack frame format: %c (%p)!\n", *p,
-             (void *)p);
+             (const void *)p);
       Die();
     }
   }
 }
-bool RenderNeedsSymbolization(const char *format) {
+bool FormattedStackTracePrinter::RenderNeedsSymbolization(const char *format) {
   if (0 == internal_strcmp(format, "DEFAULT"))
     format = kDefaultFormat;
   for (const char *p = format; *p != '\0'; p++) {
 @@ -273,30 +298,32 @@ bool RenderNeedsSymbolization(const char *format) {
   return false;
 }
-void RenderData(InternalScopedString *buffer, const char *format,
+void FormattedStackTracePrinter::RenderData(InternalScopedString *buffer,
-                const DataInfo *DI, const char *strip_path_prefix) {
+                                            const char *format,
+                                            const DataInfo *DI,
+                                            const char *strip_path_prefix) {
   for (const char *p = format; *p != '\0'; p++) {
     if (*p != '%') {
-      buffer->append("%c", *p);
+      buffer->AppendF("%c", *p);
       continue;
     }
     p++;
     switch (*p) {
       case '%':
-        buffer->append("%%");
+        buffer->Append("%");
         break;
       case 's':
-        buffer->append("%s", StripPathPrefix(DI->file, strip_path_prefix));
+        buffer->AppendF("%s", StripPathPrefix(DI->file, strip_path_prefix));
         break;
       case 'l':
-        buffer->append("%zu", DI->line);
+        buffer->AppendF("%zu", DI->line);
         break;
       case 'g':
-        buffer->append("%s", DI->name);
+        buffer->AppendF("%s", DI->name);
         break;
       default:
         Report("Unsupported specifier in stack frame format: %c (%p)!\n", *p,
-               (void *)p);
+               (const void *)p);
         Die();
     }
   }
 @@ -304,33 +331,35 @@ void RenderData(InternalScopedString *buffer, const char *format,
 #endif  // !SANITIZER_SYMBOLIZER_MARKUP
-void RenderSourceLocation(InternalScopedString *buffer, const char *file,
+void StackTracePrinter::RenderSourceLocation(InternalScopedString *buffer,
-                          int line, int column, bool vs_style,
+                                             const char *file, int line,
-                          const char *strip_path_prefix) {
+                                             int column, bool vs_style,
+                                             const char *strip_path_prefix) {
   if (vs_style && line > 0) {
-    buffer->append("%s(%d", StripPathPrefix(file, strip_path_prefix), line);
+    buffer->AppendF("%s(%d", StripPathPrefix(file, strip_path_prefix), line);
     if (column > 0)
-      buffer->append(",%d", column);
+      buffer->AppendF(",%d", column);
-    buffer->append(")");
+    buffer->Append(")");
     return;
   }
-  buffer->append("%s", StripPathPrefix(file, strip_path_prefix));
+  buffer->AppendF("%s", StripPathPrefix(file, strip_path_prefix));
   if (line > 0) {
-    buffer->append(":%d", line);
+    buffer->AppendF(":%d", line);
     if (column > 0)
-      buffer->append(":%d", column);
+      buffer->AppendF(":%d", column);
   }
 }
-void RenderModuleLocation(InternalScopedString *buffer, const char *module,
+void StackTracePrinter::RenderModuleLocation(InternalScopedString *buffer,
-                          uptr offset, ModuleArch arch,
+                                             const char *module, uptr offset,
-                          const char *strip_path_prefix) {
+                                             ModuleArch arch,
-  buffer->append("(%s", StripPathPrefix(module, strip_path_prefix));
+                                             const char *strip_path_prefix) {
+  buffer->AppendF("(%s", StripPathPrefix(module, strip_path_prefix));
   if (arch != kModuleArchUnknown) {
-    buffer->append(":%s", ModuleArchToString(arch));
+    buffer->AppendF(":%s", ModuleArchToString(arch));
   }
-  buffer->append("+0x%zx)", offset);
+  buffer->AppendF("+0x%zx)", offset);
 }
 } // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_stacktrace_printer.h+92-51

...	@@ -13,61 +13,102 @@	...	@@ -13,61 +13,102 @@
13	#define SANITIZER_STACKTRACE_PRINTER_H	13	#define SANITIZER_STACKTRACE_PRINTER_H
14		14
15	#include "sanitizer_common.h"	15	#include "sanitizer_common.h"
		16	#include "sanitizer_internal_defs.h"
16	#include "sanitizer_symbolizer.h"	17	#include "sanitizer_symbolizer.h"
17		18
18	namespace __sanitizer {	19	namespace __sanitizer {
19		20
20	// Strip interceptor prefixes from function name.	21	// StacktracePrinter is an interface that is implemented by
21	const char StripFunctionName(const char function);	22	// classes that can perform rendering of the different parts
22		23	// of a stacktrace.
23	// Render the contents of "info" structure, which represents the contents of	24	class StackTracePrinter {
24	// stack frame "frame_no" and appends it to the "buffer". "format" is a	25	public:
25	// string with placeholders, which is copied to the output with	26	static StackTracePrinter *GetOrInit();
26	// placeholders substituted with the contents of "info". For example,	27
27	// format string	28	// Strip interceptor prefixes from function name.
28	// " frame %n: function %F at %S"	29	const char StripFunctionName(const char function);
29	// will be turned into	30
30	// " frame 10: function foo::bar() at my/file.cc:10"	31	virtual void RenderFrame(InternalScopedString buffer, const char format,
31	// You may additionally pass "strip_path_prefix" to strip prefixes of paths to	32	int frame_no, uptr address, const AddressInfo *info,
32	// source files and modules.	33	bool vs_style, const char *strip_path_prefix = "") {
33	// Here's the full list of available placeholders:	34	// Should be pure virtual, but we can't depend on __cxa_pure_virtual.
34	// %% - represents a '%' character;	35	UNIMPLEMENTED();
35	// %n - frame number (copy of frame_no);	36	}
36	// %p - PC in hex format;	37
37	// %m - path to module (binary or shared object);	38	virtual bool RenderNeedsSymbolization(const char *format) {
38	// %o - offset in the module in hex format;	39	// Should be pure virtual, but we can't depend on __cxa_pure_virtual.
39	// %f - function name;	40	UNIMPLEMENTED();
40	// %q - offset in the function in hex format (if available);	41	}
41	// %s - path to source file;	42
42	// %l - line in the source file;	43	void RenderSourceLocation(InternalScopedString buffer, const char file,
43	// %c - column in the source file;	44	int line, int column, bool vs_style,
44	// %F - if function is known to be <foo>, prints "in <foo>", possibly	45	const char *strip_path_prefix);
45	// followed by the offset in this function, but only if source file	46
46	// is unknown;	47	void RenderModuleLocation(InternalScopedString buffer, const char module,
47	// %S - prints file/line/column information;	48	uptr offset, ModuleArch arch,
48	// %L - prints location information: file/line/column, if it is known, or	49	const char *strip_path_prefix);
49	// module+offset if it is known, or (<unknown module>) string.	50	virtual void RenderData(InternalScopedString buffer, const char format,
50	// %M - prints module basename and offset, if it is known, or PC.	51	const DataInfo *DI,
51	void RenderFrame(InternalScopedString buffer, const char format, int frame_no,	52	const char *strip_path_prefix = "") {
52	uptr address, const AddressInfo *info, bool vs_style,	53	// Should be pure virtual, but we can't depend on __cxa_pure_virtual.
53	const char *strip_path_prefix = "");	54	UNIMPLEMENTED();
54		55	}
55	bool RenderNeedsSymbolization(const char *format);	56
56		57	private:
57	void RenderSourceLocation(InternalScopedString buffer, const char file,	58	// To be called from StackTracePrinter::GetOrInit
58	int line, int column, bool vs_style,	59	static StackTracePrinter *NewStackTracePrinter();
59	const char *strip_path_prefix);	60
60		61	protected:
61	void RenderModuleLocation(InternalScopedString buffer, const char module,	62	~StackTracePrinter() {}
62	uptr offset, ModuleArch arch,	63	};
63	const char *strip_path_prefix);	64
64		65	class FormattedStackTracePrinter : public StackTracePrinter {
65	// Same as RenderFrame, but for data section (global variables).	66	public:
66	// Accepts %s, %l from above.	67	// Render the contents of "info" structure, which represents the contents of
67	// Also accepts:	68	// stack frame "frame_no" and appends it to the "buffer". "format" is a
68	// %g - name of the global variable.	69	// string with placeholders, which is copied to the output with
69	void RenderData(InternalScopedString buffer, const char format,	70	// placeholders substituted with the contents of "info". For example,
70	const DataInfo DI, const char strip_path_prefix = "");	71	// format string
		72	// " frame %n: function %F at %S"
		73	// will be turned into
		74	// " frame 10: function foo::bar() at my/file.cc:10"
		75	// You may additionally pass "strip_path_prefix" to strip prefixes of paths to
		76	// source files and modules.
		77	// Here's the full list of available placeholders:
		78	// %% - represents a '%' character;
		79	// %n - frame number (copy of frame_no);
		80	// %p - PC in hex format;
		81	// %m - path to module (binary or shared object);
		82	// %o - offset in the module in hex format;
		83	// %f - function name;
		84	// %q - offset in the function in hex format (if available);
		85	// %s - path to source file;
		86	// %l - line in the source file;
		87	// %c - column in the source file;
		88	// %F - if function is known to be <foo>, prints "in <foo>", possibly
		89	// followed by the offset in this function, but only if source file
		90	// is unknown;
		91	// %S - prints file/line/column information;
		92	// %L - prints location information: file/line/column, if it is known, or
		93	// module+offset if it is known, or (<unknown module>) string.
		94	// %M - prints module basename and offset, if it is known, or PC.
		95	void RenderFrame(InternalScopedString buffer, const char format,
		96	int frame_no, uptr address, const AddressInfo *info,
		97	bool vs_style, const char *strip_path_prefix = "") override;
		98
		99	bool RenderNeedsSymbolization(const char *format) override;
		100
		101	// Same as RenderFrame, but for data section (global variables).
		102	// Accepts %s, %l from above.
		103	// Also accepts:
		104	// %g - name of the global variable.
		105	void RenderData(InternalScopedString buffer, const char format,
		106	const DataInfo *DI,
		107	const char *strip_path_prefix = "") override;
		108
		109	protected:
		110	~FormattedStackTracePrinter() {}
		111	};
71		112
72	} // namespace __sanitizer	113	} // namespace __sanitizer
73		114

lib/tsan/sanitizer_common/sanitizer_stacktrace_sparc.cpp+5-6

 @@ -58,17 +58,16 @@ void BufferedStackTrace::UnwindFast(uptr pc, uptr bp, uptr stack_top,
   // Avoid infinite loop when frame == frame[0] by using frame > prev_frame.
   while (IsValidFrame(bp, stack_top, bottom) && IsAligned(bp, sizeof(uhwptr)) &&
          size < max_depth) {
-    uhwptr pc1 = ((uhwptr *)bp)[15];
+    // %o7 contains the address of the call instruction and not the
+    // return address, so we need to compensate.
+    uhwptr pc1 = GetNextInstructionPc(((uhwptr *)bp)[15]);
     // Let's assume that any pointer in the 0th page is invalid and
     // stop unwinding here.  If we're adding support for a platform
     // where this isn't true, we need to reconsider this check.
     if (pc1 < kPageSize)
       break;
-    if (pc1 != pc) {
+    if (pc1 != pc)
-      // %o7 contains the address of the call instruction and not the
+      trace_buffer[size++] = pc1;
-      // return address, so we need to compensate.
-      trace_buffer[size++] = GetNextInstructionPc((uptr)pc1);
-    }
     bottom = bp;
     bp = (uptr)((uhwptr *)bp)[14] + STACK_BIAS;
   }

lib/tsan/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cpp+5-5

...	@@ -257,8 +257,8 @@ static void TracerThreadDieCallback() {	...	@@ -257,8 +257,8 @@ static void TracerThreadDieCallback() {
257	static void TracerThreadSignalHandler(int signum, __sanitizer_siginfo *siginfo,	257	static void TracerThreadSignalHandler(int signum, __sanitizer_siginfo *siginfo,
258	void *uctx) {	258	void *uctx) {
259	SignalContext ctx(siginfo, uctx);	259	SignalContext ctx(siginfo, uctx);
260	Printf("Tracer caught signal %d: addr=0x%zx pc=0x%zx sp=0x%zx\n", signum,	260	Printf("Tracer caught signal %d: addr=%p pc=%p sp=%p\n", signum,
261	ctx.addr, ctx.pc, ctx.sp);	261	(void )ctx.addr, (void )ctx.pc, (void *)ctx.sp);
262	ThreadSuspender *inst = thread_suspender_instance;	262	ThreadSuspender *inst = thread_suspender_instance;
263	if (inst) {	263	if (inst) {
264	if (signum == SIGABRT)	264	if (signum == SIGABRT)
...	@@ -565,7 +565,7 @@ PtraceRegistersStatus SuspendedThreadsListLinux::GetRegistersAndSP(	...	@@ -565,7 +565,7 @@ PtraceRegistersStatus SuspendedThreadsListLinux::GetRegistersAndSP(
565	constexpr uptr uptr_sz = sizeof(uptr);	565	constexpr uptr uptr_sz = sizeof(uptr);
566	int pterrno;	566	int pterrno;
567	#ifdef ARCH_IOVEC_FOR_GETREGSET	567	#ifdef ARCH_IOVEC_FOR_GETREGSET
568	auto append = [&](uptr regset) {	568	auto AppendF = [&](uptr regset) {
569	uptr size = buffer->size();	569	uptr size = buffer->size();
570	// NT_X86_XSTATE requires 64bit alignment.	570	// NT_X86_XSTATE requires 64bit alignment.
571	uptr size_up = RoundUpTo(size, 8 / uptr_sz);	571	uptr size_up = RoundUpTo(size, 8 / uptr_sz);
...	@@ -596,11 +596,11 @@ PtraceRegistersStatus SuspendedThreadsListLinux::GetRegistersAndSP(	...	@@ -596,11 +596,11 @@ PtraceRegistersStatus SuspendedThreadsListLinux::GetRegistersAndSP(
596	};	596	};
597		597
598	buffer->clear();	598	buffer->clear();
599	bool fail = !append(NT_PRSTATUS);	599	bool fail = !AppendF(NT_PRSTATUS);
600	if (!fail) {	600	if (!fail) {
601	// Accept the first available and do not report errors.	601	// Accept the first available and do not report errors.
602	for (uptr regs : kExtraRegs)	602	for (uptr regs : kExtraRegs)
603	if (regs && append(regs))	603	if (regs && AppendF(regs))
604	break;	604	break;
605	}	605	}
606	#else	606	#else

lib/tsan/sanitizer_common/sanitizer_stoptheworld_netbsd_libcdep.cpp+2-2

...	@@ -158,8 +158,8 @@ static void TracerThreadDieCallback() {	...	@@ -158,8 +158,8 @@ static void TracerThreadDieCallback() {
158	static void TracerThreadSignalHandler(int signum, __sanitizer_siginfo *siginfo,	158	static void TracerThreadSignalHandler(int signum, __sanitizer_siginfo *siginfo,
159	void *uctx) {	159	void *uctx) {
160	SignalContext ctx(siginfo, uctx);	160	SignalContext ctx(siginfo, uctx);
161	Printf("Tracer caught signal %d: addr=0x%zx pc=0x%zx sp=0x%zx\n", signum,	161	Printf("Tracer caught signal %d: addr=%p pc=%p sp=%p\n", signum,
162	ctx.addr, ctx.pc, ctx.sp);	162	(void )ctx.addr, (void )ctx.pc, (void *)ctx.sp);
163	ThreadSuspender *inst = thread_suspender_instance;	163	ThreadSuspender *inst = thread_suspender_instance;
164	if (inst) {	164	if (inst) {
165	if (signum == SIGABRT)	165	if (signum == SIGABRT)

lib/tsan/sanitizer_common/sanitizer_suppressions.cpp+5-2

...	@@ -86,7 +86,7 @@ void SuppressionContext::ParseFromFile(const char *filename) {	...	@@ -86,7 +86,7 @@ void SuppressionContext::ParseFromFile(const char *filename) {
86	}	86	}
87		87
88	Parse(file_contents);	88	Parse(file_contents);
89	UnmapOrDie(file_contents, contents_size);	89	UnmapOrDie(file_contents, buffer_size);
90	}	90	}
91		91
92	bool SuppressionContext::Match(const char str, const char type,	92	bool SuppressionContext::Match(const char str, const char type,
...	@@ -138,7 +138,10 @@ void SuppressionContext::Parse(const char *str) {	...	@@ -138,7 +138,10 @@ void SuppressionContext::Parse(const char *str) {
138	}	138	}
139	}	139	}
140	if (type == suppression_types_num_) {	140	if (type == suppression_types_num_) {
141	Printf("%s: failed to parse suppressions\n", SanitizerToolName);	141	Printf("%s: failed to parse suppressions.\n", SanitizerToolName);
		142	Printf("Supported suppression types are:\n");
		143	for (type = 0; type < suppression_types_num_; type++)
		144	Printf("- %s\n", suppression_types_[type]);
142	Die();	145	Die();
143	}	146	}
144	Suppression s;	147	Suppression s;

lib/tsan/sanitizer_common/sanitizer_symbolizer.cpp+4-1

...	@@ -10,6 +10,8 @@	...	@@ -10,6 +10,8 @@
10	// run-time libraries.	10	// run-time libraries.
11	//===----------------------------------------------------------------------===//	11	//===----------------------------------------------------------------------===//
12		12
		13	#include <errno.h>
		14
13	#include "sanitizer_allocator_internal.h"	15	#include "sanitizer_allocator_internal.h"
14	#include "sanitizer_common.h"	16	#include "sanitizer_common.h"
15	#include "sanitizer_internal_defs.h"	17	#include "sanitizer_internal_defs.h"
...	@@ -128,7 +130,7 @@ Symbolizer::Symbolizer(IntrusiveList<SymbolizerTool> tools)	...	@@ -128,7 +130,7 @@ Symbolizer::Symbolizer(IntrusiveList<SymbolizerTool> tools)
128	start_hook_(0), end_hook_(0) {}	130	start_hook_(0), end_hook_(0) {}
129		131
130	Symbolizer::SymbolizerScope::SymbolizerScope(const Symbolizer *sym)	132	Symbolizer::SymbolizerScope::SymbolizerScope(const Symbolizer *sym)
131	: sym_(sym) {	133	: sym_(sym), errno_(errno) {
132	if (sym_->start_hook_)	134	if (sym_->start_hook_)
133	sym_->start_hook_();	135	sym_->start_hook_();
134	}	136	}
...	@@ -136,6 +138,7 @@ Symbolizer::SymbolizerScope::SymbolizerScope(const Symbolizer *sym)	...	@@ -136,6 +138,7 @@ Symbolizer::SymbolizerScope::SymbolizerScope(const Symbolizer *sym)
136	Symbolizer::SymbolizerScope::~SymbolizerScope() {	138	Symbolizer::SymbolizerScope::~SymbolizerScope() {
137	if (sym_->end_hook_)	139	if (sym_->end_hook_)
138	sym_->end_hook_();	140	sym_->end_hook_();
		141	errno = errno_;
139	}	142	}
140		143
141	} // namespace __sanitizer	144	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_symbolizer.h+25-2

 @@ -64,6 +64,26 @@ struct SymbolizedStack {
   SymbolizedStack();
 };
+class SymbolizedStackHolder {
+  SymbolizedStack *Stack;
+  void clear() {
+    if (Stack)
+      Stack->ClearAll();
+  }
+ public:
+  explicit SymbolizedStackHolder(SymbolizedStack *Stack = nullptr)
+      : Stack(Stack) {}
+  ~SymbolizedStackHolder() { clear(); }
+  void reset(SymbolizedStack *S = nullptr) {
+    if (Stack != S)
+      clear();
+    Stack = S;
+  }
+  const SymbolizedStack *get() const { return Stack; }
+};
 // For now, DataInfo is used to describe global variable.
 struct DataInfo {
   // Owns all the string members. Storage for them is
 @@ -136,7 +156,7 @@ class Symbolizer final {
   // Release internal caches (if any).
   void Flush();
-  // Attempts to demangle the provided C++ mangled name.
+  // Attempts to demangle the provided C++ mangled name. Never returns nullptr.
   const char *Demangle(const char *name);
   // Allow user to install hooks that would be called before/after Symbolizer
 @@ -154,6 +174,8 @@ class Symbolizer final {
   void InvalidateModuleList();
+  const ListOfModules &GetRefreshedListOfModules();
  private:
   // GetModuleNameAndOffsetForPC has to return a string to the caller.
   // Since the corresponding module might get unloaded later, we should create
 @@ -187,7 +209,7 @@ class Symbolizer final {
   // If stale, need to reload the modules before looking up addresses.
   bool modules_fresh_;
-  // Platform-specific default demangler, must not return nullptr.
+  // Platform-specific default demangler, returns nullptr on failure.
   const char *PlatformDemangle(const char *name);
   static Symbolizer *symbolizer_;
 @@ -212,6 +234,7 @@ class Symbolizer final {
     ~SymbolizerScope();
    private:
     const Symbolizer *sym_;
+    int errno_;  // Backup errno in case symbolizer change the value.
   };
 };

lib/tsan/sanitizer_common/sanitizer_symbolizer_fuchsia.h deleted-42

...	@@ -1,42 +0,0 @@
1	//===-- sanitizer_symbolizer_fuchsia.h -----------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is shared between various sanitizers' runtime libraries.
10	//
11	// Define Fuchsia's string formats and limits for the markup symbolizer.
12	//===----------------------------------------------------------------------===//
13	#ifndef SANITIZER_SYMBOLIZER_FUCHSIA_H
14	#define SANITIZER_SYMBOLIZER_FUCHSIA_H
15
16	#include "sanitizer_internal_defs.h"
17
18	namespace __sanitizer {
19
20	// See the spec at:
21	// https://fuchsia.googlesource.com/zircon/+/master/docs/symbolizer_markup.md
22
23	// This is used by UBSan for type names, and by ASan for global variable names.
24	constexpr const char *kFormatDemangle = "{{{symbol:%s}}}";
25	constexpr uptr kFormatDemangleMax = 1024; // Arbitrary.
26
27	// Function name or equivalent from PC location.
28	constexpr const char *kFormatFunction = "{{{pc:%p}}}";
29	constexpr uptr kFormatFunctionMax = 64; // More than big enough for 64-bit hex.
30
31	// Global variable name or equivalent from data memory address.
32	constexpr const char *kFormatData = "{{{data:%p}}}";
33
34	// One frame in a backtrace (printed on a line by itself).
35	constexpr const char *kFormatFrame = "{{{bt:%u:%p}}}";
36
37	// Dump trigger element.
38	#define FORMAT_DUMPFILE "{{{dumpfile:%s:%s}}}"
39
40	} // namespace __sanitizer
41
42	#endif // SANITIZER_SYMBOLIZER_FUCHSIA_H

lib/tsan/sanitizer_common/sanitizer_symbolizer_internal.h+9

...	@@ -160,6 +160,15 @@ void ParseSymbolizePCOutput(const char str, SymbolizedStack res);	...	@@ -160,6 +160,15 @@ void ParseSymbolizePCOutput(const char str, SymbolizedStack res);
160	// Used by LLVMSymbolizer and InternalSymbolizer.	160	// Used by LLVMSymbolizer and InternalSymbolizer.
161	void ParseSymbolizeDataOutput(const char str, DataInfo info);	161	void ParseSymbolizeDataOutput(const char str, DataInfo info);
162		162
		163	// Parses repeated strings in the following format:
		164	// <function_name>
		165	// <var_name>
		166	// <file_name>:<line_number>[:<column_number>]
		167	// [<frame_offset>\|??] [<size>\|??] [<tag_offset>\|??]
		168	// Used by LLVMSymbolizer and InternalSymbolizer.
		169	void ParseSymbolizeFrameOutput(const char *str,
		170	InternalMmapVector<LocalInfo> *locals);
		171
163	} // namespace __sanitizer	172	} // namespace __sanitizer
164		173
165	#endif // SANITIZER_SYMBOLIZER_INTERNAL_H	174	#endif // SANITIZER_SYMBOLIZER_INTERNAL_H

lib/tsan/sanitizer_common/sanitizer_symbolizer_libbacktrace.cpp+1-1

...	@@ -199,7 +199,7 @@ static char DemangleAlloc(const char name, bool always_alloc) {	...	@@ -199,7 +199,7 @@ static char DemangleAlloc(const char name, bool always_alloc) {
199	#endif	199	#endif
200	if (always_alloc)	200	if (always_alloc)
201	return internal_strdup(name);	201	return internal_strdup(name);
202	return 0;	202	return nullptr;
203	}	203	}
204		204
205	const char LibbacktraceSymbolizer::Demangle(const char name) {	205	const char LibbacktraceSymbolizer::Demangle(const char name) {

lib/tsan/sanitizer_common/sanitizer_symbolizer_libcdep.cpp+15-5

 @@ -117,7 +117,7 @@ bool Symbolizer::SymbolizeData(uptr addr, DataInfo *info) {
       return true;
     }
   }
-  return true;
+  return false;
 }
 bool Symbolizer::SymbolizeFrame(uptr addr, FrameInfo *info) {
 @@ -133,7 +133,7 @@ bool Symbolizer::SymbolizeFrame(uptr addr, FrameInfo *info) {
       return true;
     }
   }
-  return true;
+  return false;
 }
 bool Symbolizer::GetModuleNameAndOffsetForPC(uptr pc, const char **module_name,
 @@ -159,13 +159,16 @@ void Symbolizer::Flush() {
 }
 const char *Symbolizer::Demangle(const char *name) {
+  CHECK(name);
   Lock l(&mu_);
   for (auto &tool : tools_) {
     SymbolizerScope sym_scope(this);
     if (const char *demangled = tool.Demangle(name))
       return demangled;
   }
-  return PlatformDemangle(name);
+  if (const char *demangled = PlatformDemangle(name))
+    return demangled;
+  return name;
 }
 bool Symbolizer::FindModuleNameAndOffsetForAddress(uptr address,
 @@ -188,6 +191,13 @@ void Symbolizer::RefreshModules() {
   modules_fresh_ = true;
 }
+const ListOfModules &Symbolizer::GetRefreshedListOfModules() {
+  if (!modules_fresh_)
+    RefreshModules();
+  return modules_;
+}
 static const LoadedModule *SearchForModule(const ListOfModules &modules,
                                            uptr address) {
   for (uptr i = 0; i < modules.size(); i++) {
 @@ -382,8 +392,8 @@ void ParseSymbolizeDataOutput(const char *str, DataInfo *info) {
   str = ExtractUptr(str, "\n", &info->line);
 }
-static void ParseSymbolizeFrameOutput(const char *str,
+void ParseSymbolizeFrameOutput(const char *str,
-                                      InternalMmapVector<LocalInfo> *locals) {
+                               InternalMmapVector<LocalInfo> *locals) {
   if (internal_strncmp(str, "??", 2) == 0)
     return;

lib/tsan/sanitizer_common/sanitizer_symbolizer_mac.cpp+4-1

...	@@ -42,7 +42,8 @@ bool DlAddrSymbolizer::SymbolizePC(uptr addr, SymbolizedStack *stack) {	...	@@ -42,7 +42,8 @@ bool DlAddrSymbolizer::SymbolizePC(uptr addr, SymbolizedStack *stack) {
42	}	42	}
43		43
44	const char *demangled = DemangleSwiftAndCXX(info.dli_sname);	44	const char *demangled = DemangleSwiftAndCXX(info.dli_sname);
45	if (!demangled) return false;	45	if (!demangled)
		46	demangled = info.dli_sname;
46	stack->info.function = internal_strdup(demangled);	47	stack->info.function = internal_strdup(demangled);
47	return true;	48	return true;
48	}	49	}
...	@@ -52,6 +53,8 @@ bool DlAddrSymbolizer::SymbolizeData(uptr addr, DataInfo *datainfo) {	...	@@ -52,6 +53,8 @@ bool DlAddrSymbolizer::SymbolizeData(uptr addr, DataInfo *datainfo) {
52	int result = dladdr((const void *)addr, &info);	53	int result = dladdr((const void *)addr, &info);
53	if (!result) return false;	54	if (!result) return false;
54	const char *demangled = DemangleSwiftAndCXX(info.dli_sname);	55	const char *demangled = DemangleSwiftAndCXX(info.dli_sname);
		56	if (!demangled)
		57	demangled = info.dli_sname;
55	datainfo->name = internal_strdup(demangled);	58	datainfo->name = internal_strdup(demangled);
56	datainfo->start = (uptr)info.dli_saddr;	59	datainfo->start = (uptr)info.dli_saddr;
57	return true;	60	return true;

lib/tsan/sanitizer_common/sanitizer_symbolizer_markup.cpp+120-108

...	@@ -8,143 +8,155 @@	...	@@ -8,143 +8,155 @@
8	//	8	//
9	// This file is shared between various sanitizers' runtime libraries.	9	// This file is shared between various sanitizers' runtime libraries.
10	//	10	//
11	// Implementation of offline markup symbolizer.	11	// This generic support for offline symbolizing is based on the
		12	// Fuchsia port. We don't do any actual symbolization per se.
		13	// Instead, we emit text containing raw addresses and raw linkage
		14	// symbol names, embedded in Fuchsia's symbolization markup format.
		15	// See the spec at:
		16	// https://llvm.org/docs/SymbolizerMarkupFormat.html
12	//===----------------------------------------------------------------------===//	17	//===----------------------------------------------------------------------===//
13		18
14	#include "sanitizer_platform.h"	19	#include "sanitizer_symbolizer_markup.h"
15	#if SANITIZER_SYMBOLIZER_MARKUP
16
17	#if SANITIZER_FUCHSIA
18	#include "sanitizer_symbolizer_fuchsia.h"
19	# endif
20		20
21	# include <limits.h>	21	#include "sanitizer_common.h"
22	# include <unwind.h>	22	#include "sanitizer_symbolizer.h"
23		23	#include "sanitizer_symbolizer_markup_constants.h"
24	# include "sanitizer_stacktrace.h"
25	# include "sanitizer_symbolizer.h"
26		24
27	namespace __sanitizer {	25	namespace __sanitizer {
28		26
29	// This generic support for offline symbolizing is based on the	27	void MarkupStackTracePrinter::RenderData(InternalScopedString *buffer,
30	// Fuchsia port. We don't do any actual symbolization per se.	28	const char format, const DataInfo DI,
31	// Instead, we emit text containing raw addresses and raw linkage	29	const char *strip_path_prefix) {
32	// symbol names, embedded in Fuchsia's symbolization markup format.	30	RenderContext(buffer);
33	// Fuchsia's logging infrastructure emits enough information about	31	buffer->AppendF(kFormatData, reinterpret_cast<void *>(DI->start));
34	// process memory layout that a post-processing filter can do the
35	// symbolization and pretty-print the markup. See the spec at:
36	// https://fuchsia.googlesource.com/zircon/+/master/docs/symbolizer_markup.md
37
38	// This is used by UBSan for type names, and by ASan for global variable names.
39	// It's expected to return a static buffer that will be reused on each call.
40	const char Symbolizer::Demangle(const char name) {
41	static char buffer[kFormatDemangleMax];
42	internal_snprintf(buffer, sizeof(buffer), kFormatDemangle, name);
43	return buffer;
44	}	32	}
45		33
46	// This is used mostly for suppression matching. Making it work	34	bool MarkupStackTracePrinter::RenderNeedsSymbolization(const char *format) {
47	// would enable "interceptor_via_lib" suppressions. It's also used
48	// once in UBSan to say "in module ..." in a message that also
49	// includes an address in the module, so post-processing can already
50	// pretty-print that so as to indicate the module.
51	bool Symbolizer::GetModuleNameAndOffsetForPC(uptr pc, const char **module_name,
52	uptr *module_address) {
53	return false;	35	return false;
54	}	36	}
55		37
56	// This is mainly used by hwasan for online symbolization. This isn't needed	38	// We don't support the stack_trace_format flag at all.
57	// since hwasan can always just dump stack frames for offline symbolization.	39	void MarkupStackTracePrinter::RenderFrame(InternalScopedString *buffer,
58	bool Symbolizer::SymbolizeFrame(uptr addr, FrameInfo *info) { return false; }	40	const char *format, int frame_no,
59		41	uptr address, const AddressInfo *info,
60	// This is used in some places for suppression checking, which we	42	bool vs_style,
61	// don't really support for Fuchsia. It's also used in UBSan to	43	const char *strip_path_prefix) {
62	// identify a PC location to a function name, so we always fill in	44	CHECK(!RenderNeedsSymbolization(format));
63	// the function member with a string containing markup around the PC	45	RenderContext(buffer);
64	// value.	46	buffer->AppendF(kFormatFrame, frame_no, reinterpret_cast<void *>(address));
65	// TODO(mcgrathr): Under SANITIZER_GO, it's currently used by TSan	47	}
66	// to render stack frames, but that should be changed to use	48
67	// RenderStackFrame.	49	bool MarkupSymbolizerTool::SymbolizePC(uptr addr, SymbolizedStack *stack) {
68	SymbolizedStack *Symbolizer::SymbolizePC(uptr addr) {
69	SymbolizedStack *s = SymbolizedStack::New(addr);
70	char buffer[kFormatFunctionMax];	50	char buffer[kFormatFunctionMax];
71	internal_snprintf(buffer, sizeof(buffer), kFormatFunction, addr);	51	internal_snprintf(buffer, sizeof(buffer), kFormatFunction,
72	s->info.function = internal_strdup(buffer);	52	reinterpret_cast<void *>(addr));
73	return s;	53	stack->info.function = internal_strdup(buffer);
		54	return true;
74	}	55	}
75		56
76	// Always claim we succeeded, so that RenderDataInfo will be called.	57	bool MarkupSymbolizerTool::SymbolizeData(uptr addr, DataInfo *info) {
77	bool Symbolizer::SymbolizeData(uptr addr, DataInfo *info) {
78	info->Clear();	58	info->Clear();
79	info->start = addr;	59	info->start = addr;
80	return true;	60	return true;
81	}	61	}
82		62
83	// We ignore the format argument to __sanitizer_symbolize_global.	63	const char MarkupSymbolizerTool::Demangle(const char name) {
84	void RenderData(InternalScopedString buffer, const char format,	64	static char buffer[kFormatDemangleMax];
85	const DataInfo DI, const char strip_path_prefix) {	65	internal_snprintf(buffer, sizeof(buffer), kFormatDemangle, name);
86	buffer->append(kFormatData, DI->start);	66	return buffer;
87	}	67	}
88		68
89	bool RenderNeedsSymbolization(const char *format) { return false; }	69	// Fuchsia's implementation of symbolizer markup doesn't need to emit contextual
90		70	// elements at this point.
91	// We don't support the stack_trace_format flag at all.	71	// Fuchsia's logging infrastructure emits enough information about
92	void RenderFrame(InternalScopedString buffer, const char format, int frame_no,	72	// process memory layout that a post-processing filter can do the
93	uptr address, const AddressInfo *info, bool vs_style,	73	// symbolization and pretty-print the markup.
94	const char *strip_path_prefix) {	74	#if !SANITIZER_FUCHSIA
95	CHECK(!RenderNeedsSymbolization(format));	75
96	buffer->append(kFormatFrame, frame_no, address);	76	static bool ModulesEq(const LoadedModule &module,
		77	const RenderedModule &renderedModule) {
		78	return module.base_address() == renderedModule.base_address &&
		79	internal_memcmp(module.uuid(), renderedModule.uuid,
		80	module.uuid_size()) == 0 &&
		81	internal_strcmp(module.full_name(), renderedModule.full_name) == 0;
97	}	82	}
98		83
99	Symbolizer *Symbolizer::PlatformInit() {	84	static bool ModuleHasBeenRendered(
100	return new (symbolizer_allocator_) Symbolizer({});	85	const LoadedModule &module,
		86	const InternalMmapVectorNoCtor<RenderedModule> &renderedModules) {
		87	for (const auto &renderedModule : renderedModules)
		88	if (ModulesEq(module, renderedModule))
		89	return true;
		90
		91	return false;
101	}	92	}
102		93
103	void Symbolizer::LateInitialize() { Symbolizer::GetOrInit(); }	94	static void RenderModule(InternalScopedString *buffer,
104		95	const LoadedModule &module, uptr moduleId) {
105	void StartReportDeadlySignal() {}	96	InternalScopedString buildIdBuffer;
106	void ReportDeadlySignal(const SignalContext &sig, u32 tid,	97	for (uptr i = 0; i < module.uuid_size(); i++)
107	UnwindSignalStackCallbackType unwind,	98	buildIdBuffer.AppendF("%02x", module.uuid()[i]);
108	const void *unwind_context) {}	99
109		100	buffer->AppendF(kFormatModule, moduleId, module.full_name(),
110	#if SANITIZER_CAN_SLOW_UNWIND	101	buildIdBuffer.data());
111	struct UnwindTraceArg {	102	buffer->Append("\n");
112	BufferedStackTrace *stack;
113	u32 max_depth;
114	};
115
116	_Unwind_Reason_Code Unwind_Trace(struct _Unwind_Context ctx, void param) {
117	UnwindTraceArg arg = static_cast<UnwindTraceArg >(param);
118	CHECK_LT(arg->stack->size, arg->max_depth);
119	uptr pc = _Unwind_GetIP(ctx);
120	if (pc < PAGE_SIZE) return _URC_NORMAL_STOP;
121	arg->stack->trace_buffer[arg->stack->size++] = pc;
122	return (arg->stack->size == arg->max_depth ? _URC_NORMAL_STOP
123	: _URC_NO_REASON);
124	}	103	}
125		104
126	void BufferedStackTrace::UnwindSlow(uptr pc, u32 max_depth) {	105	static void RenderMmaps(InternalScopedString *buffer,
127	CHECK_GE(max_depth, 2);	106	const LoadedModule &module, uptr moduleId) {
128	size = 0;	107	InternalScopedString accessBuffer;
129	UnwindTraceArg arg = {this, Min(max_depth + 1, kStackTraceMax)};	108
130	_Unwind_Backtrace(Unwind_Trace, &arg);	109	// All module mmaps are readable at least
131	CHECK_GT(size, 0);	110	for (const auto &range : module.ranges()) {
132	// We need to pop a few frames so that pc is on top.	111	accessBuffer.Append("r");
133	uptr to_pop = LocatePcInTrace(pc);	112	if (range.writable)
134	// trace_buffer[0] belongs to the current function so we always pop it,	113	accessBuffer.Append("w");
135	// unless there is only 1 frame in the stack trace (1 frame is always better	114	if (range.executable)
136	// than 0!).	115	accessBuffer.Append("x");
137	PopStackFrames(Min(to_pop, static_cast<uptr>(1)));	116
138	trace_buffer[0] = pc;	117	//{{{mmap:%starting_addr:%size_in_hex:load:%moduleId:r%(w\|x):%relative_addr}}}
		118
		119	// module.base_address == dlpi_addr
		120	// range.beg == dlpi_addr + p_vaddr
		121	// relative address == p_vaddr == range.beg - module.base_address
		122	buffer->AppendF(kFormatMmap, reinterpret_cast<void *>(range.beg),
		123	range.end - range.beg, static_cast<int>(moduleId),
		124	accessBuffer.data(), range.beg - module.base_address());
		125
		126	buffer->Append("\n");
		127	accessBuffer.clear();
		128	}
139	}	129	}
140		130
141	void BufferedStackTrace::UnwindSlow(uptr pc, void *context, u32 max_depth) {	131	void MarkupStackTracePrinter::RenderContext(InternalScopedString *buffer) {
142	CHECK(context);	132	if (renderedModules_.size() == 0)
143	CHECK_GE(max_depth, 2);	133	buffer->Append("{{{reset}}}\n");
144	UNREACHABLE("signal context doesn't exist");	134
		135	const auto &modules = Symbolizer::GetOrInit()->GetRefreshedListOfModules();
		136
		137	for (const auto &module : modules) {
		138	if (ModuleHasBeenRendered(module, renderedModules_))
		139	continue;
		140
		141	// symbolizer markup id, used to refer to this modules from other contextual
		142	// elements
		143	uptr moduleId = renderedModules_.size();
		144
		145	RenderModule(buffer, module, moduleId);
		146	RenderMmaps(buffer, module, moduleId);
		147
		148	renderedModules_.push_back({
		149	internal_strdup(module.full_name()),
		150	module.base_address(),
		151	{},
		152	});
		153
		154	// kModuleUUIDSize is the size of curModule.uuid
		155	CHECK_GE(kModuleUUIDSize, module.uuid_size());
		156	internal_memcpy(renderedModules_.back().uuid, module.uuid(),
		157	module.uuid_size());
		158	}
145	}	159	}
146	#endif // SANITIZER_CAN_SLOW_UNWIND	160	#endif // !SANITIZER_FUCHSIA
147		161
148	} // namespace __sanitizer	162	} // namespace __sanitizer
149
150	#endif // SANITIZER_SYMBOLIZER_MARKUP

lib/tsan/sanitizer_common/sanitizer_symbolizer_markup.h created+79

...	@@ -0,0 +1,79 @@
1	//===-- sanitizer_symbolizer_markup.h -----------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is shared between various sanitizers' runtime libraries.
10	//
11	// Header for the offline markup symbolizer.
12	//===----------------------------------------------------------------------===//
13	#ifndef SANITIZER_SYMBOLIZER_MARKUP_H
14	#define SANITIZER_SYMBOLIZER_MARKUP_H
15
16	#include "sanitizer_common.h"
17	#include "sanitizer_stacktrace_printer.h"
18	#include "sanitizer_symbolizer.h"
19	#include "sanitizer_symbolizer_internal.h"
20
21	namespace __sanitizer {
22
23	// Simplier view of a LoadedModule. It only holds information necessary to
24	// identify unique modules.
25	struct RenderedModule {
26	char *full_name;
27	uptr base_address;
28	u8 uuid[kModuleUUIDSize]; // BuildId
29	};
30
31	class MarkupStackTracePrinter : public StackTracePrinter {
32	public:
33	// We don't support the stack_trace_format flag at all.
34	void RenderFrame(InternalScopedString buffer, const char format,
35	int frame_no, uptr address, const AddressInfo *info,
36	bool vs_style, const char *strip_path_prefix = "") override;
37
38	bool RenderNeedsSymbolization(const char *format) override;
39
40	// We ignore the format argument to __sanitizer_symbolize_global.
41	void RenderData(InternalScopedString buffer, const char format,
42	const DataInfo *DI,
43	const char *strip_path_prefix = "") override;
44
45	private:
46	// Keeps track of the modules that have been rendered to avoid re-rendering
47	// them
48	InternalMmapVector<RenderedModule> renderedModules_;
49	void RenderContext(InternalScopedString *buffer);
50
51	protected:
52	~MarkupStackTracePrinter() {}
53	};
54
55	class MarkupSymbolizerTool final : public SymbolizerTool {
56	public:
57	// This is used in some places for suppression checking, which we
58	// don't really support for Fuchsia. It's also used in UBSan to
59	// identify a PC location to a function name, so we always fill in
60	// the function member with a string containing markup around the PC
61	// value.
62	// TODO(mcgrathr): Under SANITIZER_GO, it's currently used by TSan
63	// to render stack frames, but that should be changed to use
64	// RenderStackFrame.
65	bool SymbolizePC(uptr addr, SymbolizedStack *stack) override;
66
67	// Always claim we succeeded, so that RenderDataInfo will be called.
68	bool SymbolizeData(uptr addr, DataInfo *info) override;
69
70	// May return NULL if demangling failed.
71	// This is used by UBSan for type names, and by ASan for global variable
72	// names. It's expected to return a static buffer that will be reused on each
73	// call.
74	const char Demangle(const char name) override;
75	};
76
77	} // namespace __sanitizer
78
79	#endif // SANITIZER_SYMBOLIZER_MARKUP_H

lib/tsan/sanitizer_common/sanitizer_symbolizer_markup_constants.h created+49

...	@@ -0,0 +1,49 @@
1	//===-- sanitizer_symbolizer_markup_constants.h
2	//-----------------------------------===//
3	//
4	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5	// See https://llvm.org/LICENSE.txt for license information.
6	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7	//
8	//===----------------------------------------------------------------------===//
9	//
10	// This file is shared between various sanitizers' runtime libraries.
11	//
12	// Define string formats and limits for the markup symbolizer.
13	//===----------------------------------------------------------------------===//
14	#ifndef SANITIZER_SYMBOLIZER_MARKUP_CONSTANTS_H
15	#define SANITIZER_SYMBOLIZER_MARKUP_CONSTANTS_H
16
17	#include "sanitizer_internal_defs.h"
18
19	namespace __sanitizer {
20
21	// See the spec at:
22	// https://fuchsia.googlesource.com/zircon/+/master/docs/symbolizer_markup.md
23
24	// This is used by UBSan for type names, and by ASan for global variable names.
25	constexpr const char *kFormatDemangle = "{{{symbol:%s}}}";
26	constexpr uptr kFormatDemangleMax = 1024; // Arbitrary.
27
28	// Function name or equivalent from PC location.
29	constexpr const char *kFormatFunction = "{{{pc:%p}}}";
30	constexpr uptr kFormatFunctionMax = 64; // More than big enough for 64-bit hex.
31
32	// Global variable name or equivalent from data memory address.
33	constexpr const char *kFormatData = "{{{data:%p}}}";
34
35	// One frame in a backtrace (printed on a line by itself).
36	constexpr const char *kFormatFrame = "{{{bt:%d:%p}}}";
37
38	// Module contextual element.
39	constexpr const char *kFormatModule = "{{{module:%zu:%s:elf:%s}}}";
40
41	// mmap for a module segment.
42	constexpr const char *kFormatMmap = "{{{mmap:%p:0x%zx:load:%d:%s:0x%zx}}}";
43
44	// Dump trigger element.
45	#define FORMAT_DUMPFILE "{{{dumpfile:%s:%s}}}"
46
47	} // namespace __sanitizer
48
49	#endif // SANITIZER_SYMBOLIZER_MARKUP_CONSTANTS_H

lib/tsan/sanitizer_common/sanitizer_symbolizer_markup_fuchsia.cpp created+85

...	@@ -0,0 +1,85 @@
1	//===-- sanitizer_symbolizer_markup_fuchsia.cpp ---------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is shared between various sanitizers' runtime libraries.
10	//
11	// Fuchsia specific implementation of offline markup symbolizer.
12	//===----------------------------------------------------------------------===//
13	#include "sanitizer_platform.h"
14
15	#if SANITIZER_SYMBOLIZER_MARKUP
16
17	# include "sanitizer_common.h"
18	# include "sanitizer_stacktrace_printer.h"
19	# include "sanitizer_symbolizer.h"
20	# include "sanitizer_symbolizer_markup.h"
21	# include "sanitizer_symbolizer_markup_constants.h"
22
23	namespace __sanitizer {
24
25	// This is used by UBSan for type names, and by ASan for global variable names.
26	// It's expected to return a static buffer that will be reused on each call.
27	const char Symbolizer::Demangle(const char name) {
28	static char buffer[kFormatDemangleMax];
29	internal_snprintf(buffer, sizeof(buffer), kFormatDemangle, name);
30	return buffer;
31	}
32
33	// This is used mostly for suppression matching. Making it work
34	// would enable "interceptor_via_lib" suppressions. It's also used
35	// once in UBSan to say "in module ..." in a message that also
36	// includes an address in the module, so post-processing can already
37	// pretty-print that so as to indicate the module.
38	bool Symbolizer::GetModuleNameAndOffsetForPC(uptr pc, const char **module_name,
39	uptr *module_address) {
40	return false;
41	}
42
43	// This is mainly used by hwasan for online symbolization. This isn't needed
44	// since hwasan can always just dump stack frames for offline symbolization.
45	bool Symbolizer::SymbolizeFrame(uptr addr, FrameInfo *info) { return false; }
46
47	// This is used in some places for suppression checking, which we
48	// don't really support for Fuchsia. It's also used in UBSan to
49	// identify a PC location to a function name, so we always fill in
50	// the function member with a string containing markup around the PC
51	// value.
52	// TODO(mcgrathr): Under SANITIZER_GO, it's currently used by TSan
53	// to render stack frames, but that should be changed to use
54	// RenderStackFrame.
55	SymbolizedStack *Symbolizer::SymbolizePC(uptr addr) {
56	SymbolizedStack *s = SymbolizedStack::New(addr);
57	char buffer[kFormatFunctionMax];
58	internal_snprintf(buffer, sizeof(buffer), kFormatFunction, addr);
59	s->info.function = internal_strdup(buffer);
60	return s;
61	}
62
63	// Always claim we succeeded, so that RenderDataInfo will be called.
64	bool Symbolizer::SymbolizeData(uptr addr, DataInfo *info) {
65	info->Clear();
66	info->start = addr;
67	return true;
68	}
69
70	// Fuchsia only uses MarkupStackTracePrinter
71	StackTracePrinter *StackTracePrinter::NewStackTracePrinter() {
72	return new (GetGlobalLowLevelAllocator()) MarkupStackTracePrinter();
73	}
74
75	void MarkupStackTracePrinter::RenderContext(InternalScopedString *) {}
76
77	Symbolizer *Symbolizer::PlatformInit() {
78	return new (symbolizer_allocator_) Symbolizer({});
79	}
80
81	void Symbolizer::LateInitialize() { Symbolizer::GetOrInit(); }
82
83	} // namespace __sanitizer
84
85	#endif // SANITIZER_SYMBOLIZER_MARKUP

lib/tsan/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cpp+36-31

 @@ -12,6 +12,7 @@
 //===----------------------------------------------------------------------===//
 #include "sanitizer_platform.h"
+#include "sanitizer_symbolizer_markup.h"
 #if SANITIZER_POSIX
 #  include <dlfcn.h>  // for dlsym()
 #  include <errno.h>
 @@ -56,7 +57,7 @@ const char *DemangleCXXABI(const char *name) {
           __cxxabiv1::__cxa_demangle(name, 0, 0, 0))
       return demangled_name;
-  return name;
+  return nullptr;
 }
 // As of now, there are no headers for the Swift runtime. Once they are
 @@ -324,9 +325,12 @@ __sanitizer_symbolize_code(const char *ModuleName, u64 ModuleOffset,
 SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE bool
 __sanitizer_symbolize_data(const char *ModuleName, u64 ModuleOffset,
                            char *Buffer, int MaxLength);
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE bool
+__sanitizer_symbolize_frame(const char *ModuleName, u64 ModuleOffset,
+                            char *Buffer, int MaxLength);
 SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
 __sanitizer_symbolize_flush();
-SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE int
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE bool
 __sanitizer_symbolize_demangle(const char *Name, char *Buffer, int MaxLength);
 SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE bool
 __sanitizer_symbolize_set_demangle(bool Demangle);
 @@ -337,19 +341,19 @@ __sanitizer_symbolize_set_inline_frames(bool InlineFrames);
 class InternalSymbolizer final : public SymbolizerTool {
  public:
   static InternalSymbolizer *get(LowLevelAllocator *alloc) {
-    if (__sanitizer_symbolize_set_demangle)
+    // These one is the most used one, so we will use it to detect a presence of
-      CHECK(__sanitizer_symbolize_set_demangle(common_flags()->demangle));
+    // internal symbolizer.
-    if (__sanitizer_symbolize_set_inline_frames)
+    if (&__sanitizer_symbolize_code == nullptr)
-      CHECK(__sanitizer_symbolize_set_inline_frames(
+      return nullptr;
-          common_flags()->symbolize_inline_frames));
+    CHECK(__sanitizer_symbolize_set_demangle(common_flags()->demangle));
-    if (__sanitizer_symbolize_code && __sanitizer_symbolize_data)
+    CHECK(__sanitizer_symbolize_set_inline_frames(
-      return new (*alloc) InternalSymbolizer();
+        common_flags()->symbolize_inline_frames));
-    return 0;
+    return new (*alloc) InternalSymbolizer();
   }
   bool SymbolizePC(uptr addr, SymbolizedStack *stack) override {
     bool result = __sanitizer_symbolize_code(
-        stack->info.module, stack->info.module_offset, buffer_, kBufferSize);
+        stack->info.module, stack->info.module_offset, buffer_, sizeof(buffer_));
     if (result)
       ParseSymbolizePCOutput(buffer_, stack);
     return result;
 @@ -357,7 +361,7 @@ class InternalSymbolizer final : public SymbolizerTool {
   bool SymbolizeData(uptr addr, DataInfo *info) override {
     bool result = __sanitizer_symbolize_data(info->module, info->module_offset,
-                                             buffer_, kBufferSize);
+                                             buffer_, sizeof(buffer_));
     if (result) {
       ParseSymbolizeDataOutput(buffer_, info);
       info->start += (addr - info->module_offset);  // Add the base address.
 @@ -365,34 +369,29 @@ class InternalSymbolizer final : public SymbolizerTool {
     return result;
   }
-  void Flush() override {
+  bool SymbolizeFrame(uptr addr, FrameInfo *info) override {
-    if (__sanitizer_symbolize_flush)
+    bool result = __sanitizer_symbolize_frame(info->module, info->module_offset,
-      __sanitizer_symbolize_flush();
+                                              buffer_, sizeof(buffer_));
+    if (result)
+      ParseSymbolizeFrameOutput(buffer_, &info->locals);
+    return result;
   }
+  void Flush() override { __sanitizer_symbolize_flush(); }
   const char *Demangle(const char *name) override {
-    if (__sanitizer_symbolize_demangle) {
+    if (__sanitizer_symbolize_demangle(name, buffer_, sizeof(buffer_))) {
-      for (uptr res_length = 1024;
+      char *res_buff = nullptr;
-           res_length <= InternalSizeClassMap::kMaxSize;) {
+      ExtractToken(buffer_, "", &res_buff);
-        char *res_buff = static_cast<char *>(InternalAlloc(res_length));
+      return res_buff;
-        uptr req_length =
-            __sanitizer_symbolize_demangle(name, res_buff, res_length);
-        if (req_length > res_length) {
-          res_length = req_length + 1;
-          InternalFree(res_buff);
-          continue;
-        }
-        return res_buff;
-      }
     }
-    return name;
+    return nullptr;
   }
  private:
   InternalSymbolizer() {}
-  static const int kBufferSize = 16 * 1024;
+  char buffer_[16 * 1024];
-  char buffer_[kBufferSize];
 };
 #  else  // SANITIZER_SUPPORTS_WEAK_HOOKS
 @@ -470,6 +469,12 @@ static void ChooseSymbolizerTools(IntrusiveList<SymbolizerTool> *list,
     VReport(2, "Symbolizer is disabled.\n");
     return;
   }
+  if (common_flags()->enable_symbolizer_markup) {
+    VReport(2, "Using symbolizer markup");
+    SymbolizerTool *tool = new (*allocator) MarkupSymbolizerTool();
+    CHECK(tool);
+    list->push_back(tool);
+  }
   if (IsAllocatorOutOfMemory()) {
     VReport(2, "Cannot use internal symbolizer: out of memory\n");
   } else if (SymbolizerTool *tool = InternalSymbolizer::get(allocator)) {

lib/tsan/sanitizer_common/sanitizer_symbolizer_report.cpp+62-18

...	@@ -28,14 +28,41 @@	...	@@ -28,14 +28,41 @@
28	namespace __sanitizer {	28	namespace __sanitizer {
29		29
30	#if !SANITIZER_GO	30	#if !SANITIZER_GO
		31
		32	static bool FrameIsInternal(const SymbolizedStack *frame) {
		33	if (!frame)
		34	return true;
		35	const char *file = frame->info.file;
		36	const char *module = frame->info.module;
		37	// On Gentoo, the path is g++-, so there's not* a missing /.
		38	if (file && (internal_strstr(file, "/compiler-rt/lib/") \|\|
		39	internal_strstr(file, "/include/c++/") \|\|
		40	internal_strstr(file, "/include/g++")))
		41	return true;
		42	if (file && internal_strstr(file, "\\compiler-rt\\lib\\"))
		43	return true;
		44	if (module && (internal_strstr(module, "libclang_rt.")))
		45	return true;
		46	if (module && (internal_strstr(module, "clang_rt.")))
		47	return true;
		48	return false;
		49	}
		50
		51	const SymbolizedStack SkipInternalFrames(const SymbolizedStack frames) {
		52	for (const SymbolizedStack *f = frames; f; f = f->next)
		53	if (!FrameIsInternal(f))
		54	return f;
		55	return nullptr;
		56	}
		57
31	void ReportErrorSummary(const char *error_type, const AddressInfo &info,	58	void ReportErrorSummary(const char *error_type, const AddressInfo &info,
32	const char *alt_tool_name) {	59	const char *alt_tool_name) {
33	if (!common_flags()->print_summary) return;	60	if (!common_flags()->print_summary) return;
34	InternalScopedString buff;	61	InternalScopedString buff;
35	buff.append("%s ", error_type);	62	buff.AppendF("%s ", error_type);
36	RenderFrame(&buff, "%L %F", 0, info.address, &info,	63	StackTracePrinter::GetOrInit()->RenderFrame(
37	common_flags()->symbolize_vs_style,	64	&buff, "%L %F", 0, info.address, &info,
38	common_flags()->strip_path_prefix);	65	common_flags()->symbolize_vs_style, common_flags()->strip_path_prefix);
39	ReportErrorSummary(buff.data(), alt_tool_name);	66	ReportErrorSummary(buff.data(), alt_tool_name);
40	}	67	}
41	#endif	68	#endif
...	@@ -75,16 +102,33 @@ void ReportErrorSummary(const char error_type, const StackTrace stack,	...	@@ -75,16 +102,33 @@ void ReportErrorSummary(const char error_type, const StackTrace stack,
75	#if !SANITIZER_GO	102	#if !SANITIZER_GO
76	if (!common_flags()->print_summary)	103	if (!common_flags()->print_summary)
77	return;	104	return;
78	if (stack->size == 0) {	105
79	ReportErrorSummary(error_type);	106	// Find first non-internal stack frame.
80	return;	107	for (uptr i = 0; i < stack->size; ++i) {
		108	uptr pc = StackTrace::GetPreviousInstructionPc(stack->trace[i]);
		109	SymbolizedStackHolder symbolized_stack(
		110	Symbolizer::GetOrInit()->SymbolizePC(pc));
		111	if (const SymbolizedStack *frame = symbolized_stack.get()) {
		112	if (const SymbolizedStack *summary_frame = SkipInternalFrames(frame)) {
		113	ReportErrorSummary(error_type, summary_frame->info, alt_tool_name);
		114	return;
		115	}
		116	}
		117	}
		118
		119	// Fallback to the top one.
		120	if (stack->size) {
		121	uptr pc = StackTrace::GetPreviousInstructionPc(stack->trace[0]);
		122	SymbolizedStackHolder symbolized_stack(
		123	Symbolizer::GetOrInit()->SymbolizePC(pc));
		124	if (const SymbolizedStack *frame = symbolized_stack.get()) {
		125	ReportErrorSummary(error_type, frame->info, alt_tool_name);
		126	return;
		127	}
81	}	128	}
82	// Currently, we include the first stack frame into the report summary.	129
83	// Maybe sometimes we need to choose another frame (e.g. skip memcpy/etc).	130	// Fallback to a summary without location.
84	uptr pc = StackTrace::GetPreviousInstructionPc(stack->trace[0]);	131	ReportErrorSummary(error_type);
85	SymbolizedStack *frame = Symbolizer::GetOrInit()->SymbolizePC(pc);
86	ReportErrorSummary(error_type, frame->info, alt_tool_name);
87	frame->ClearAll();
88	#endif	132	#endif
89	}	133	}
90		134
...	@@ -148,22 +192,22 @@ static void MaybeReportNonExecRegion(uptr pc) {	...	@@ -148,22 +192,22 @@ static void MaybeReportNonExecRegion(uptr pc) {
148	static void PrintMemoryByte(InternalScopedString str, const char before,	192	static void PrintMemoryByte(InternalScopedString str, const char before,
149	u8 byte) {	193	u8 byte) {
150	SanitizerCommonDecorator d;	194	SanitizerCommonDecorator d;
151	str->append("%s%s%x%x%s ", before, d.MemoryByte(), byte >> 4, byte & 15,	195	str->AppendF("%s%s%x%x%s ", before, d.MemoryByte(), byte >> 4, byte & 15,
152	d.Default());	196	d.Default());
153	}	197	}
154		198
155	static void MaybeDumpInstructionBytes(uptr pc) {	199	static void MaybeDumpInstructionBytes(uptr pc) {
156	if (!common_flags()->dump_instruction_bytes \|\| (pc < GetPageSizeCached()))	200	if (!common_flags()->dump_instruction_bytes \|\| (pc < GetPageSizeCached()))
157	return;	201	return;
158	InternalScopedString str;	202	InternalScopedString str;
159	str.append("First 16 instruction bytes at pc: ");	203	str.AppendF("First 16 instruction bytes at pc: ");
160	if (IsAccessibleMemoryRange(pc, 16)) {	204	if (IsAccessibleMemoryRange(pc, 16)) {
161	for (int i = 0; i < 16; ++i) {	205	for (int i = 0; i < 16; ++i) {
162	PrintMemoryByte(&str, "", ((u8 *)pc)[i]);	206	PrintMemoryByte(&str, "", ((u8 *)pc)[i]);
163	}	207	}
164	str.append("\n");	208	str.AppendF("\n");
165	} else {	209	} else {
166	str.append("unaccessible\n");	210	str.AppendF("unaccessible\n");
167	}	211	}
168	Report("%s", str.data());	212	Report("%s", str.data());
169	}	213	}

lib/tsan/sanitizer_common/sanitizer_symbolizer_report_fuchsia.cpp created+33

...	@@ -0,0 +1,33 @@
1	//===-- sanitizer_symbolizer_report_fuchsia.cpp
2	//-----------------------------------===//
3	//
4	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5	// See https://llvm.org/LICENSE.txt for license information.
6	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7	//
8	//===----------------------------------------------------------------------===//
9	//
10	// Implementation of the report functions for fuchsia.
11	//
12	//===----------------------------------------------------------------------===//
13
14	#include "sanitizer_platform.h"
15
16	#if SANITIZER_SYMBOLIZER_MARKUP
17
18	# include "sanitizer_common.h"
19
20	namespace __sanitizer {
21	void StartReportDeadlySignal() {}
22
23	void ReportDeadlySignal(const SignalContext &sig, u32 tid,
24	UnwindSignalStackCallbackType unwind,
25	const void *unwind_context) {}
26
27	void HandleDeadlySignal(void siginfo, void context, u32 tid,
28	UnwindSignalStackCallbackType unwind,
29	const void *unwind_context) {}
30
31	} // namespace __sanitizer
32
33	#endif // SANITIZER_SYMBOLIZER_MARKUP

lib/tsan/sanitizer_common/sanitizer_symbolizer_win.cpp+2-4

...	@@ -175,9 +175,7 @@ const char WinSymbolizerTool::Demangle(const char name) {	...	@@ -175,9 +175,7 @@ const char WinSymbolizerTool::Demangle(const char name) {
175	return name;	175	return name;
176	}	176	}
177		177
178	const char Symbolizer::PlatformDemangle(const char name) {	178	const char Symbolizer::PlatformDemangle(const char name) { return nullptr; }
179	return name;
180	}
181		179
182	namespace {	180	namespace {
183	struct ScopedHandle {	181	struct ScopedHandle {
...	@@ -233,7 +231,7 @@ bool SymbolizerProcess::StartSymbolizerSubprocess() {	...	@@ -233,7 +231,7 @@ bool SymbolizerProcess::StartSymbolizerSubprocess() {
233	CHECK(!internal_strchr(arg, '"') && "quotes in args unsupported");	231	CHECK(!internal_strchr(arg, '"') && "quotes in args unsupported");
234	CHECK(arglen > 0 && arg[arglen - 1] != '\\' &&	232	CHECK(arglen > 0 && arg[arglen - 1] != '\\' &&
235	"args ending in backslash and empty args unsupported");	233	"args ending in backslash and empty args unsupported");
236	command_line.append("\"%s\" ", arg);	234	command_line.AppendF("\"%s\" ", arg);
237	}	235	}
238	VReport(3, "Launching symbolizer command: %s\n", command_line.data());	236	VReport(3, "Launching symbolizer command: %s\n", command_line.data());
239		237

lib/tsan/sanitizer_common/sanitizer_syscall_linux_hexagon.inc created+131

...	@@ -0,0 +1,131 @@
1	//===-- sanitizer_syscall_linux_hexagon.inc ---------------------- C++ --===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// Implementations of internal_syscall and internal_iserror for Linux/hexagon.
10	//
11	//===----------------------------------------------------------------------===//
12
13	#define SYSCALL(name) __NR_##name
14
15	#define __internal_syscall_LL_E(x) \
16	((union { \
17	long long ll; \
18	long l[2]; \
19	}){.ll = x}) \
20	.l[0], \
21	((union { \
22	long long ll; \
23	long l[2]; \
24	}){.ll = x}) \
25	.l[1]
26	#define __internal_syscall_LL_O(x) 0, __SYSCALL_LL_E((x))
27
28	#define __asm_syscall(...) \
29	do { \
30	__asm__ __volatile__("trap0(#1)" : "=r"(r0) : __VA_ARGS__ : "memory"); \
31	return r0; \
32	} while (0)
33
34	#define __internal_syscall0(n) (__internal_syscall)(n)
35
36	static uptr __internal_syscall(long n) {
37	register u32 r6 __asm__("r6") = n;
38	register u32 r0 __asm__("r0");
39	__asm_syscall("r"(r6));
40	}
41
42	#define __internal_syscall1(n, a1) (__internal_syscall)(n, (long)(a1))
43
44	static uptr __internal_syscall(long n, long a) {
45	register u32 r6 __asm__("r6") = n;
46	register u32 r0 __asm__("r0") = a;
47	__asm_syscall("r"(r6), "0"(r0));
48	}
49
50	#define __internal_syscall2(n, a1, a2) \
51	(__internal_syscall)(n, (long)(a1), (long)(a2))
52
53	static uptr __internal_syscall(long n, long a, long b) {
54	register u32 r6 __asm__("r6") = n;
55	register u32 r0 __asm__("r0") = a;
56	register u32 r1 __asm__("r1") = b;
57	__asm_syscall("r"(r6), "0"(r0), "r"(r1));
58	}
59
60	#define __internal_syscall3(n, a1, a2, a3) \
61	(__internal_syscall)(n, (long)(a1), (long)(a2), (long)(a3))
62
63	static uptr __internal_syscall(long n, long a, long b, long c) {
64	register u32 r6 __asm__("r6") = n;
65	register u32 r0 __asm__("r0") = a;
66	register u32 r1 __asm__("r1") = b;
67	register u32 r2 __asm__("r2") = c;
68	__asm_syscall("r"(r6), "0"(r0), "r"(r1), "r"(r2));
69	}
70
71	#define __internal_syscall4(n, a1, a2, a3, a4) \
72	(__internal_syscall)(n, (long)(a1), (long)(a2), (long)(a3), (long)(a4))
73
74	static uptr __internal_syscall(long n, long a, long b, long c, long d) {
75	register u32 r6 __asm__("r6") = n;
76	register u32 r0 __asm__("r0") = a;
77	register u32 r1 __asm__("r1") = b;
78	register u32 r2 __asm__("r2") = c;
79	register u32 r3 __asm__("r3") = d;
80	__asm_syscall("r"(r6), "0"(r0), "r"(r1), "r"(r2), "r"(r3));
81	}
82
83	#define __internal_syscall5(n, a1, a2, a3, a4, a5) \
84	(__internal_syscall)(n, (long)(a1), (long)(a2), (long)(a3), (long)(a4), \
85	(long)(a5))
86
87	static uptr __internal_syscall(long n, long a, long b, long c, long d, long e) {
88	register u32 r6 __asm__("r6") = n;
89	register u32 r0 __asm__("r0") = a;
90	register u32 r1 __asm__("r1") = b;
91	register u32 r2 __asm__("r2") = c;
92	register u32 r3 __asm__("r3") = d;
93	register u32 r4 __asm__("r4") = e;
94	__asm_syscall("r"(r6), "0"(r0), "r"(r1), "r"(r2), "r"(r3), "r"(r4));
95	}
96
97	#define __internal_syscall6(n, a1, a2, a3, a4, a5, a6) \
98	(__internal_syscall)(n, (long)(a1), (long)(a2), (long)(a3), (long)(a4), \
99	(long)(a5), (long)(a6))
100
101	static uptr __internal_syscall(long n, long a, long b, long c, long d, long e,
102	long f) {
103	register u32 r6 __asm__("r6") = n;
104	register u32 r0 __asm__("r0") = a;
105	register u32 r1 __asm__("r1") = b;
106	register u32 r2 __asm__("r2") = c;
107	register u32 r3 __asm__("r3") = d;
108	register u32 r4 __asm__("r4") = e;
109	register u32 r5 __asm__("r5") = f;
110	__asm_syscall("r"(r6), "0"(r0), "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5));
111	}
112
113	#define __SYSCALL_NARGS_X(a1, a2, a3, a4, a5, a6, a7, a8, n, ...) n
114	#define __SYSCALL_NARGS(...) \
115	__SYSCALL_NARGS_X(__VA_ARGS__, 7, 6, 5, 4, 3, 2, 1, 0, )
116	#define __SYSCALL_CONCAT_X(a, b) a##b
117	#define __SYSCALL_CONCAT(a, b) __SYSCALL_CONCAT_X(a, b)
118	#define __SYSCALL_DISP(b, ...) \
119	__SYSCALL_CONCAT(b, __SYSCALL_NARGS(__VA_ARGS__))(__VA_ARGS__)
120
121	#define internal_syscall(...) __SYSCALL_DISP(__internal_syscall, __VA_ARGS__)
122
123	// Helper function used to avoid clobbering of errno.
124	bool internal_iserror(uptr retval, int *rverrno) {
125	if (retval >= (uptr)-4095) {
126	if (rverrno)
127	*rverrno = -retval;
128	return true;
129	}
130	return false;
131	}

lib/tsan/sanitizer_common/sanitizer_syscall_linux_loongarch64.inc created+171

...	@@ -0,0 +1,171 @@
1	//===-- sanitizer_syscall_linux_loongarch64.inc ------------------ C++ --===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// Implementations of internal_syscall and internal_iserror for
10	// Linux/loongarch64.
11	//
12	//===----------------------------------------------------------------------===//
13
14	// About local register variables:
15	// https://gcc.gnu.org/onlinedocs/gcc/Local-Register-Variables.html#Local-Register-Variables
16	//
17	// Kernel ABI:
18	// https://lore.kernel.org/loongarch/1f353678-3398-e30b-1c87-6edb278f74db@xen0n.name/T/#m1613bc86c2d7bf5f6da92bd62984302bfd699a2f
19	// syscall number is placed in a7
20	// parameters, if present, are placed in a0-a6
21	// upon return:
22	// the return value is placed in a0
23	// t0-t8 should be considered clobbered
24	// all other registers are preserved
25	#define SYSCALL(name) __NR_##name
26
27	#define INTERNAL_SYSCALL_CLOBBERS \
28	"memory", "$t0", "$t1", "$t2", "$t3", "$t4", "$t5", "$t6", "$t7", "$t8"
29
30	static uptr __internal_syscall(u64 nr) {
31	register u64 a7 asm("$a7") = nr;
32	register u64 a0 asm("$a0");
33	__asm__ volatile("syscall 0\n\t"
34	: "=r"(a0)
35	: "r"(a7)
36	: INTERNAL_SYSCALL_CLOBBERS);
37	return a0;
38	}
39	#define __internal_syscall0(n) (__internal_syscall)(n)
40
41	static uptr __internal_syscall(u64 nr, u64 arg1) {
42	register u64 a7 asm("$a7") = nr;
43	register u64 a0 asm("$a0") = arg1;
44	__asm__ volatile("syscall 0\n\t"
45	: "+r"(a0)
46	: "r"(a7)
47	: INTERNAL_SYSCALL_CLOBBERS);
48	return a0;
49	}
50	#define __internal_syscall1(n, a1) (__internal_syscall)(n, (u64)(a1))
51
52	static uptr __internal_syscall(u64 nr, u64 arg1, long arg2) {
53	register u64 a7 asm("$a7") = nr;
54	register u64 a0 asm("$a0") = arg1;
55	register u64 a1 asm("$a1") = arg2;
56	__asm__ volatile("syscall 0\n\t"
57	: "+r"(a0)
58	: "r"(a7), "r"(a1)
59	: INTERNAL_SYSCALL_CLOBBERS);
60	return a0;
61	}
62	#define __internal_syscall2(n, a1, a2) \
63	(__internal_syscall)(n, (u64)(a1), (long)(a2))
64
65	static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3) {
66	register u64 a7 asm("$a7") = nr;
67	register u64 a0 asm("$a0") = arg1;
68	register u64 a1 asm("$a1") = arg2;
69	register u64 a2 asm("$a2") = arg3;
70	__asm__ volatile("syscall 0\n\t"
71	: "+r"(a0)
72	: "r"(a7), "r"(a1), "r"(a2)
73	: INTERNAL_SYSCALL_CLOBBERS);
74	return a0;
75	}
76	#define __internal_syscall3(n, a1, a2, a3) \
77	(__internal_syscall)(n, (u64)(a1), (long)(a2), (long)(a3))
78
79	static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3,
80	u64 arg4) {
81	register u64 a7 asm("$a7") = nr;
82	register u64 a0 asm("$a0") = arg1;
83	register u64 a1 asm("$a1") = arg2;
84	register u64 a2 asm("$a2") = arg3;
85	register u64 a3 asm("$a3") = arg4;
86	__asm__ volatile("syscall 0\n\t"
87	: "+r"(a0)
88	: "r"(a7), "r"(a1), "r"(a2), "r"(a3)
89	: INTERNAL_SYSCALL_CLOBBERS);
90	return a0;
91	}
92	#define __internal_syscall4(n, a1, a2, a3, a4) \
93	(__internal_syscall)(n, (u64)(a1), (long)(a2), (long)(a3), (long)(a4))
94
95	static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3, u64 arg4,
96	long arg5) {
97	register u64 a7 asm("$a7") = nr;
98	register u64 a0 asm("$a0") = arg1;
99	register u64 a1 asm("$a1") = arg2;
100	register u64 a2 asm("$a2") = arg3;
101	register u64 a3 asm("$a3") = arg4;
102	register u64 a4 asm("$a4") = arg5;
103	__asm__ volatile("syscall 0\n\t"
104	: "+r"(a0)
105	: "r"(a7), "r"(a1), "r"(a2), "r"(a3), "r"(a4)
106	: INTERNAL_SYSCALL_CLOBBERS);
107	return a0;
108	}
109	#define __internal_syscall5(n, a1, a2, a3, a4, a5) \
110	(__internal_syscall)(n, (u64)(a1), (long)(a2), (long)(a3), (long)(a4), \
111	(u64)(a5))
112
113	static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3, u64 arg4,
114	long arg5, long arg6) {
115	register u64 a7 asm("$a7") = nr;
116	register u64 a0 asm("$a0") = arg1;
117	register u64 a1 asm("$a1") = arg2;
118	register u64 a2 asm("$a2") = arg3;
119	register u64 a3 asm("$a3") = arg4;
120	register u64 a4 asm("$a4") = arg5;
121	register u64 a5 asm("$a5") = arg6;
122	__asm__ volatile("syscall 0\n\t"
123	: "+r"(a0)
124	: "r"(a7), "r"(a1), "r"(a2), "r"(a3), "r"(a4), "r"(a5)
125	: INTERNAL_SYSCALL_CLOBBERS);
126	return a0;
127	}
128	#define __internal_syscall6(n, a1, a2, a3, a4, a5, a6) \
129	(__internal_syscall)(n, (u64)(a1), (long)(a2), (long)(a3), (long)(a4), \
130	(u64)(a5), (long)(a6))
131
132	static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3, u64 arg4,
133	long arg5, long arg6, long arg7) {
134	register u64 a7 asm("$a7") = nr;
135	register u64 a0 asm("$a0") = arg1;
136	register u64 a1 asm("$a1") = arg2;
137	register u64 a2 asm("$a2") = arg3;
138	register u64 a3 asm("$a3") = arg4;
139	register u64 a4 asm("$a4") = arg5;
140	register u64 a5 asm("$a5") = arg6;
141	register u64 a6 asm("$a6") = arg7;
142	__asm__ volatile("syscall 0\n\t"
143	: "+r"(a0)
144	: "r"(a7), "r"(a1), "r"(a2), "r"(a3), "r"(a4), "r"(a5),
145	"r"(a6)
146	: INTERNAL_SYSCALL_CLOBBERS);
147	return a0;
148	}
149	#define __internal_syscall7(n, a1, a2, a3, a4, a5, a6, a7) \
150	(__internal_syscall)(n, (u64)(a1), (long)(a2), (long)(a3), (long)(a4), \
151	(u64)(a5), (long)(a6), (long)(a7))
152
153	#define __SYSCALL_NARGS_X(a1, a2, a3, a4, a5, a6, a7, a8, n, ...) n
154	#define __SYSCALL_NARGS(...) \
155	__SYSCALL_NARGS_X(__VA_ARGS__, 7, 6, 5, 4, 3, 2, 1, 0, )
156	#define __SYSCALL_CONCAT_X(a, b) a##b
157	#define __SYSCALL_CONCAT(a, b) __SYSCALL_CONCAT_X(a, b)
158	#define __SYSCALL_DISP(b, ...) \
159	__SYSCALL_CONCAT(b, __SYSCALL_NARGS(__VA_ARGS__))(__VA_ARGS__)
160
161	#define internal_syscall(...) __SYSCALL_DISP(__internal_syscall, __VA_ARGS__)
162
163	// Helper function used to avoid clobbering of errno.
164	bool internal_iserror(uptr retval, int *internal_errno) {
165	if (retval >= (uptr)-4095) {
166	if (internal_errno)
167	*internal_errno = -retval;
168	return true;
169	}
170	return false;
171	}

lib/tsan/sanitizer_common/sanitizer_syscall_linux_riscv64.inc created+174

...	@@ -0,0 +1,174 @@
1	//===-- sanitizer_syscall_linux_riscv64.inc ---------------------- C++ --===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// Implementations of internal_syscall and internal_iserror for Linux/riscv64.
10	//
11	//===----------------------------------------------------------------------===//
12
13	// About local register variables:
14	// https://gcc.gnu.org/onlinedocs/gcc/Local-Register-Variables.html#Local-Register-Variables
15	//
16	// Kernel ABI...
17	// To my surprise I haven't found much information regarding it.
18	// Kernel source and internet browsing shows that:
19	// syscall number is passed in a7
20	// (http://man7.org/linux/man-pages/man2/syscall.2.html) results are return in
21	// a0 and a1 (http://man7.org/linux/man-pages/man2/syscall.2.html) arguments
22	// are passed in: a0-a7 (see below)
23	//
24	// Regarding the arguments. The only "documentation" I could find is
25	// this comment (!!!) by Bruce Hold on google forums (!!!):
26	// https://groups.google.com/a/groups.riscv.org/forum/#!topic/sw-dev/exbrzM3GZDQ
27	// Confirmed by inspecting glibc sources.
28	// Great way to document things.
29	#define SYSCALL(name) __NR_##name
30
31	#define INTERNAL_SYSCALL_CLOBBERS "memory"
32
33	static uptr __internal_syscall(u64 nr) {
34	register u64 a7 asm("a7") = nr;
35	register u64 a0 asm("a0");
36	__asm__ volatile("ecall\n\t"
37	: "=r"(a0)
38	: "r"(a7)
39	: INTERNAL_SYSCALL_CLOBBERS);
40	return a0;
41	}
42	#define __internal_syscall0(n) (__internal_syscall)(n)
43
44	static uptr __internal_syscall(u64 nr, u64 arg1) {
45	register u64 a7 asm("a7") = nr;
46	register u64 a0 asm("a0") = arg1;
47	__asm__ volatile("ecall\n\t"
48	: "+r"(a0)
49	: "r"(a7)
50	: INTERNAL_SYSCALL_CLOBBERS);
51	return a0;
52	}
53	#define __internal_syscall1(n, a1) (__internal_syscall)(n, (u64)(a1))
54
55	static uptr __internal_syscall(u64 nr, u64 arg1, long arg2) {
56	register u64 a7 asm("a7") = nr;
57	register u64 a0 asm("a0") = arg1;
58	register u64 a1 asm("a1") = arg2;
59	__asm__ volatile("ecall\n\t"
60	: "+r"(a0)
61	: "r"(a7), "r"(a1)
62	: INTERNAL_SYSCALL_CLOBBERS);
63	return a0;
64	}
65	#define __internal_syscall2(n, a1, a2) \
66	(__internal_syscall)(n, (u64)(a1), (long)(a2))
67
68	static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3) {
69	register u64 a7 asm("a7") = nr;
70	register u64 a0 asm("a0") = arg1;
71	register u64 a1 asm("a1") = arg2;
72	register u64 a2 asm("a2") = arg3;
73	__asm__ volatile("ecall\n\t"
74	: "+r"(a0)
75	: "r"(a7), "r"(a1), "r"(a2)
76	: INTERNAL_SYSCALL_CLOBBERS);
77	return a0;
78	}
79	#define __internal_syscall3(n, a1, a2, a3) \
80	(__internal_syscall)(n, (u64)(a1), (long)(a2), (long)(a3))
81
82	static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3,
83	u64 arg4) {
84	register u64 a7 asm("a7") = nr;
85	register u64 a0 asm("a0") = arg1;
86	register u64 a1 asm("a1") = arg2;
87	register u64 a2 asm("a2") = arg3;
88	register u64 a3 asm("a3") = arg4;
89	__asm__ volatile("ecall\n\t"
90	: "+r"(a0)
91	: "r"(a7), "r"(a1), "r"(a2), "r"(a3)
92	: INTERNAL_SYSCALL_CLOBBERS);
93	return a0;
94	}
95	#define __internal_syscall4(n, a1, a2, a3, a4) \
96	(__internal_syscall)(n, (u64)(a1), (long)(a2), (long)(a3), (long)(a4))
97
98	static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3, u64 arg4,
99	long arg5) {
100	register u64 a7 asm("a7") = nr;
101	register u64 a0 asm("a0") = arg1;
102	register u64 a1 asm("a1") = arg2;
103	register u64 a2 asm("a2") = arg3;
104	register u64 a3 asm("a3") = arg4;
105	register u64 a4 asm("a4") = arg5;
106	__asm__ volatile("ecall\n\t"
107	: "+r"(a0)
108	: "r"(a7), "r"(a1), "r"(a2), "r"(a3), "r"(a4)
109	: INTERNAL_SYSCALL_CLOBBERS);
110	return a0;
111	}
112	#define __internal_syscall5(n, a1, a2, a3, a4, a5) \
113	(__internal_syscall)(n, (u64)(a1), (long)(a2), (long)(a3), (long)(a4), \
114	(u64)(a5))
115
116	static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3, u64 arg4,
117	long arg5, long arg6) {
118	register u64 a7 asm("a7") = nr;
119	register u64 a0 asm("a0") = arg1;
120	register u64 a1 asm("a1") = arg2;
121	register u64 a2 asm("a2") = arg3;
122	register u64 a3 asm("a3") = arg4;
123	register u64 a4 asm("a4") = arg5;
124	register u64 a5 asm("a5") = arg6;
125	__asm__ volatile("ecall\n\t"
126	: "+r"(a0)
127	: "r"(a7), "r"(a1), "r"(a2), "r"(a3), "r"(a4), "r"(a5)
128	: INTERNAL_SYSCALL_CLOBBERS);
129	return a0;
130	}
131	#define __internal_syscall6(n, a1, a2, a3, a4, a5, a6) \
132	(__internal_syscall)(n, (u64)(a1), (long)(a2), (long)(a3), (long)(a4), \
133	(u64)(a5), (long)(a6))
134
135	static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3, u64 arg4,
136	long arg5, long arg6, long arg7) {
137	register u64 a7 asm("a7") = nr;
138	register u64 a0 asm("a0") = arg1;
139	register u64 a1 asm("a1") = arg2;
140	register u64 a2 asm("a2") = arg3;
141	register u64 a3 asm("a3") = arg4;
142	register u64 a4 asm("a4") = arg5;
143	register u64 a5 asm("a5") = arg6;
144	register u64 a6 asm("a6") = arg7;
145	__asm__ volatile("ecall\n\t"
146	: "+r"(a0)
147	: "r"(a7), "r"(a1), "r"(a2), "r"(a3), "r"(a4), "r"(a5),
148	"r"(a6)
149	: INTERNAL_SYSCALL_CLOBBERS);
150	return a0;
151	}
152	#define __internal_syscall7(n, a1, a2, a3, a4, a5, a6, a7) \
153	(__internal_syscall)(n, (u64)(a1), (long)(a2), (long)(a3), (long)(a4), \
154	(u64)(a5), (long)(a6), (long)(a7))
155
156	#define __SYSCALL_NARGS_X(a1, a2, a3, a4, a5, a6, a7, a8, n, ...) n
157	#define __SYSCALL_NARGS(...) \
158	__SYSCALL_NARGS_X(__VA_ARGS__, 7, 6, 5, 4, 3, 2, 1, 0, )
159	#define __SYSCALL_CONCAT_X(a, b) a##b
160	#define __SYSCALL_CONCAT(a, b) __SYSCALL_CONCAT_X(a, b)
161	#define __SYSCALL_DISP(b, ...) \
162	__SYSCALL_CONCAT(b, __SYSCALL_NARGS(__VA_ARGS__))(__VA_ARGS__)
163
164	#define internal_syscall(...) __SYSCALL_DISP(__internal_syscall, __VA_ARGS__)
165
166	// Helper function used to avoid clobbering of errno.
167	bool internal_iserror(uptr retval, int *rverrno) {
168	if (retval >= (uptr)-4095) {
169	if (rverrno)
170	*rverrno = -retval;
171	return true;
172	}
173	return false;
174	}

lib/tsan/sanitizer_common/sanitizer_thread_arg_retval.cpp+19-4

 @@ -23,6 +23,9 @@ void ThreadArgRetval::CreateLocked(uptr thread, bool detached,
   Data& t = data_[thread];
   t = {};
   t.gen = gen_++;
+  static_assert(sizeof(gen_) == sizeof(u32) && kInvalidGen == UINT32_MAX);
+  if (gen_ == kInvalidGen)
+    gen_ = 0;
   t.detached = detached;
   t.args = args;
 }
 @@ -53,16 +56,28 @@ void ThreadArgRetval::Finish(uptr thread, void* retval) {
 u32 ThreadArgRetval::BeforeJoin(uptr thread) const {
   __sanitizer::Lock lock(&mtx_);
   auto t = data_.find(thread);
-  CHECK(t);
+  if (t && !t->second.detached) {
-  CHECK(!t->second.detached);
+    return t->second.gen;
-  return t->second.gen;
+  }
+  if (!common_flags()->detect_invalid_join)
+    return kInvalidGen;
+  const char* reason = "unknown";
+  if (!t) {
+    reason = "already joined";
+  } else if (t->second.detached) {
+    reason = "detached";
+  }
+  Report("ERROR: %s: Joining %s thread, aborting.\n", SanitizerToolName,
+         reason);
+  Die();
 }
 void ThreadArgRetval::AfterJoin(uptr thread, u32 gen) {
   __sanitizer::Lock lock(&mtx_);
   auto t = data_.find(thread);
   if (!t || gen != t->second.gen) {
-    // Thread was reused and erased by any other event.
+    // Thread was reused and erased by any other event, or we had an invalid
+    // join.
     return;
   }
   CHECK(!t->second.detached);

lib/tsan/sanitizer_common/sanitizer_thread_arg_retval.h+1

...	@@ -93,6 +93,7 @@ class SANITIZER_MUTEX ThreadArgRetval {	...	@@ -93,6 +93,7 @@ class SANITIZER_MUTEX ThreadArgRetval {
93	// will keep pointers alive forever, missing leaks caused by cancelation.	93	// will keep pointers alive forever, missing leaks caused by cancelation.
94		94
95	private:	95	private:
		96	static const u32 kInvalidGen = UINT32_MAX;
96	struct Data {	97	struct Data {
97	Args args;	98	Args args;
98	u32 gen; // Avoid collision if thread id re-used.	99	u32 gen; // Avoid collision if thread id re-used.

lib/tsan/sanitizer_common/sanitizer_tls_get_addr.cpp+8-7

 @@ -121,25 +121,26 @@ DTLS::DTV *DTLS_on_tls_get_addr(void *arg_void, void *res,
   uptr tls_size = 0;
   uptr tls_beg = reinterpret_cast<uptr>(res) - arg->offset - kDtvOffset;
   VReport(2,
-          "__tls_get_addr: %p {0x%zx,0x%zx} => %p; tls_beg: 0x%zx; sp: %p "
+          "__tls_get_addr: %p {0x%zx,0x%zx} => %p; tls_beg: %p; sp: %p "
           "num_live_dtls %zd\n",
-          (void *)arg, arg->dso_id, arg->offset, res, tls_beg, (void *)&tls_beg,
+          (void *)arg, arg->dso_id, arg->offset, res, (void *)tls_beg,
+          (void *)&tls_beg,
           atomic_load(&number_of_live_dtls, memory_order_relaxed));
   if (dtls.last_memalign_ptr == tls_beg) {
     tls_size = dtls.last_memalign_size;
-    VReport(2, "__tls_get_addr: glibc <=2.24 suspected; tls={0x%zx,0x%zx}\n",
+    VReport(2, "__tls_get_addr: glibc <=2.24 suspected; tls={%p,0x%zx}\n",
-            tls_beg, tls_size);
+            (void *)tls_beg, tls_size);
   } else if (tls_beg >= static_tls_begin && tls_beg < static_tls_end) {
     // This is the static TLS block which was initialized / unpoisoned at thread
     // creation.
-    VReport(2, "__tls_get_addr: static tls: 0x%zx\n", tls_beg);
+    VReport(2, "__tls_get_addr: static tls: %p\n", (void *)tls_beg);
     tls_size = 0;
   } else if (const void *start =
                  __sanitizer_get_allocated_begin((void *)tls_beg)) {
     tls_beg = (uptr)start;
     tls_size = __sanitizer_get_allocated_size(start);
-    VReport(2, "__tls_get_addr: glibc >=2.25 suspected; tls={0x%zx,0x%zx}\n",
+    VReport(2, "__tls_get_addr: glibc >=2.25 suspected; tls={%p,0x%zx}\n",
-            tls_beg, tls_size);
+            (void *)tls_beg, tls_size);
   } else {
     VReport(2, "__tls_get_addr: Can't guess glibc version\n");
     // This may happen inside the DTOR of main thread, so just ignore it.

lib/tsan/sanitizer_common/sanitizer_unwind_fuchsia.cpp created+66

...	@@ -0,0 +1,66 @@
1	//===------------------ sanitizer_unwind_fuchsia.cpp
2	//---------------------------===//
3	//
4	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5	// See https://llvm.org/LICENSE.txt for license information.
6	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7	//
8	//===----------------------------------------------------------------------===//
9	//
10	/// Sanitizer unwind Fuchsia specific functions.
11	//
12	//===----------------------------------------------------------------------===//
13
14	#include "sanitizer_platform.h"
15	#if SANITIZER_FUCHSIA
16
17	# include <limits.h>
18	# include <unwind.h>
19
20	# include "sanitizer_common.h"
21	# include "sanitizer_stacktrace.h"
22
23	namespace __sanitizer {
24
25	# if SANITIZER_CAN_SLOW_UNWIND
26	struct UnwindTraceArg {
27	BufferedStackTrace *stack;
28	u32 max_depth;
29	};
30
31	_Unwind_Reason_Code Unwind_Trace(struct _Unwind_Context ctx, void param) {
32	UnwindTraceArg arg = static_cast<UnwindTraceArg >(param);
33	CHECK_LT(arg->stack->size, arg->max_depth);
34	uptr pc = _Unwind_GetIP(ctx);
35	if (pc < GetPageSizeCached())
36	return _URC_NORMAL_STOP;
37	arg->stack->trace_buffer[arg->stack->size++] = pc;
38	return (arg->stack->size == arg->max_depth ? _URC_NORMAL_STOP
39	: _URC_NO_REASON);
40	}
41
42	void BufferedStackTrace::UnwindSlow(uptr pc, u32 max_depth) {
43	CHECK_GE(max_depth, 2);
44	size = 0;
45	UnwindTraceArg arg = {this, Min(max_depth + 1, kStackTraceMax)};
46	_Unwind_Backtrace(Unwind_Trace, &arg);
47	CHECK_GT(size, 0);
48	// We need to pop a few frames so that pc is on top.
49	uptr to_pop = LocatePcInTrace(pc);
50	// trace_buffer[0] belongs to the current function so we always pop it,
51	// unless there is only 1 frame in the stack trace (1 frame is always better
52	// than 0!).
53	PopStackFrames(Min(to_pop, static_cast<uptr>(1)));
54	trace_buffer[0] = pc;
55	}
56
57	void BufferedStackTrace::UnwindSlow(uptr pc, void *context, u32 max_depth) {
58	CHECK(context);
59	CHECK_GE(max_depth, 2);
60	UNREACHABLE("signal context doesn't exist");
61	}
62	# endif // SANITIZER_CAN_SLOW_UNWIND
63
64	} // namespace __sanitizer
65
66	#endif // SANITIZER_FUCHSIA

lib/tsan/sanitizer_common/sanitizer_unwind_win.cpp+7

 @@ -70,10 +70,17 @@ void BufferedStackTrace::UnwindSlow(uptr pc, void *context, u32 max_depth) {
   stack_frame.AddrStack.Offset = ctx.Rsp;
 #      endif
 #    else
+#      if SANITIZER_ARM
+  int machine_type = IMAGE_FILE_MACHINE_ARM;
+  stack_frame.AddrPC.Offset = ctx.Pc;
+  stack_frame.AddrFrame.Offset = ctx.R11;
+  stack_frame.AddrStack.Offset = ctx.Sp;
+#      else
   int machine_type = IMAGE_FILE_MACHINE_I386;
   stack_frame.AddrPC.Offset = ctx.Eip;
   stack_frame.AddrFrame.Offset = ctx.Ebp;
   stack_frame.AddrStack.Offset = ctx.Esp;
+#      endif
 #    endif
   stack_frame.AddrPC.Mode = AddrModeFlat;
   stack_frame.AddrFrame.Mode = AddrModeFlat;

lib/tsan/sanitizer_common/sanitizer_win.cpp+13-12

 @@ -144,7 +144,7 @@ void *MmapOrDie(uptr size, const char *mem_type, bool raw_report) {
   return rv;
 }
-void UnmapOrDie(void *addr, uptr size) {
+void UnmapOrDie(void *addr, uptr size, bool raw_report) {
   if (!size || !addr)
     return;
 @@ -156,10 +156,7 @@ void UnmapOrDie(void *addr, uptr size) {
   // fails try MEM_DECOMMIT.
   if (VirtualFree(addr, 0, MEM_RELEASE) == 0) {
     if (VirtualFree(addr, size, MEM_DECOMMIT) == 0) {
-      Report("ERROR: %s failed to "
+      ReportMunmapFailureAndDie(addr, size, GetLastError(), raw_report);
-             "deallocate 0x%zx (%zd) bytes at address %p (error code: %d)\n",
-             SanitizerToolName, size, size, addr, GetLastError());
-      CHECK("unable to unmap" && 0);
     }
   }
 }
 @@ -279,8 +276,8 @@ void *MmapFixedOrDie(uptr fixed_addr, uptr size, const char *name) {
       MEM_COMMIT, PAGE_READWRITE);
   if (p == 0) {
     char mem_type[30];
-    internal_snprintf(mem_type, sizeof(mem_type), "memory at address 0x%zx",
+    internal_snprintf(mem_type, sizeof(mem_type), "memory at address %p",
-                      fixed_addr);
+                      (void *)fixed_addr);
     ReportMmapFailureAndDie(size, mem_type, "allocate", GetLastError());
   }
   return p;
 @@ -311,8 +308,8 @@ void *MmapFixedOrDieOnFatalError(uptr fixed_addr, uptr size, const char *name) {
       MEM_COMMIT, PAGE_READWRITE);
   if (p == 0) {
     char mem_type[30];
-    internal_snprintf(mem_type, sizeof(mem_type), "memory at address 0x%zx",
+    internal_snprintf(mem_type, sizeof(mem_type), "memory at address %p",
-                      fixed_addr);
+                      (void *)fixed_addr);
     return ReturnNullptrOnOOMOrDie(size, mem_type, "allocate");
   }
   return p;
 @@ -387,9 +384,8 @@ bool DontDumpShadowMemory(uptr addr, uptr length) {
 }
 uptr MapDynamicShadow(uptr shadow_size_bytes, uptr shadow_scale,
-                      uptr min_shadow_base_alignment,
+                      uptr min_shadow_base_alignment, UNUSED uptr &high_mem_end,
-                      UNUSED uptr &high_mem_end) {
+                      uptr granularity) {
-  const uptr granularity = GetMmapGranularity();
   const uptr alignment =
       Max<uptr>(granularity << shadow_scale, 1ULL << min_shadow_base_alignment);
   const uptr left_padding =
 @@ -996,8 +992,13 @@ void SignalContext::InitPcSpBp() {
   sp = (uptr)context_record->Rsp;
 #    endif
 #  else
+#    if SANITIZER_ARM
+  bp = (uptr)context_record->R11;
+  sp = (uptr)context_record->Sp;
+#    else
   bp = (uptr)context_record->Ebp;
   sp = (uptr)context_record->Esp;
+#    endif
 #  endif
 }

lib/tsan/tsan_debugging.cpp+3-1

 @@ -35,7 +35,9 @@ static const char *ReportTypeDescription(ReportType typ) {
     case ReportTypeSignalUnsafe: return "signal-unsafe-call";
     case ReportTypeErrnoInSignal: return "errno-in-signal-handler";
     case ReportTypeDeadlock: return "lock-order-inversion";
-    // No default case so compiler warns us if we miss one
+    case ReportTypeMutexHeldWrongContext:
+      return "mutex-held-in-wrong-context";
+      // No default case so compiler warns us if we miss one
   }
   UNREACHABLE("missing case");
 }

lib/tsan/tsan_defs.h+1-1

...	@@ -30,7 +30,7 @@	...	@@ -30,7 +30,7 @@
30	# define __MM_MALLOC_H	30	# define __MM_MALLOC_H
31	# include <emmintrin.h>	31	# include <emmintrin.h>
32	# include <smmintrin.h>	32	# include <smmintrin.h>
33	# define VECTOR_ALIGNED ALIGNED(16)	33	# define VECTOR_ALIGNED alignas(16)
34	typedef __m128i m128;	34	typedef __m128i m128;
35	#else	35	#else
36	# define VECTOR_ALIGNED	36	# define VECTOR_ALIGNED

lib/tsan/tsan_dispatch_defs.h-7

...	@@ -56,13 +56,6 @@ extern const dispatch_block_t _dispatch_data_destructor_munmap;	...	@@ -56,13 +56,6 @@ extern const dispatch_block_t _dispatch_data_destructor_munmap;
56	# define DISPATCH_NOESCAPE	56	# define DISPATCH_NOESCAPE
57	#endif	57	#endif
58		58
59	#if SANITIZER_APPLE
60	# define SANITIZER_WEAK_IMPORT extern "C" __attribute((weak_import))
61	#else
62	# define SANITIZER_WEAK_IMPORT extern "C" __attribute((weak))
63	#endif
64
65
66	// Data types used in dispatch APIs	59	// Data types used in dispatch APIs
67	typedef unsigned long size_t;	60	typedef unsigned long size_t;
68	typedef unsigned long uintptr_t;	61	typedef unsigned long uintptr_t;

lib/tsan/tsan_interceptors_posix.cpp+93-42

...	@@ -14,6 +14,7 @@	...	@@ -14,6 +14,7 @@
14		14
15	#include "sanitizer_common/sanitizer_atomic.h"	15	#include "sanitizer_common/sanitizer_atomic.h"
16	#include "sanitizer_common/sanitizer_errno.h"	16	#include "sanitizer_common/sanitizer_errno.h"
		17	#include "sanitizer_common/sanitizer_glibc_version.h"
17	#include "sanitizer_common/sanitizer_libc.h"	18	#include "sanitizer_common/sanitizer_libc.h"
18	#include "sanitizer_common/sanitizer_linux.h"	19	#include "sanitizer_common/sanitizer_linux.h"
19	#include "sanitizer_common/sanitizer_platform_limits_netbsd.h"	20	#include "sanitizer_common/sanitizer_platform_limits_netbsd.h"
...	@@ -81,6 +82,8 @@ struct ucontext_t {	...	@@ -81,6 +82,8 @@ struct ucontext_t {
81	#define PTHREAD_ABI_BASE "GLIBC_2.17"	82	#define PTHREAD_ABI_BASE "GLIBC_2.17"
82	#elif SANITIZER_LOONGARCH64	83	#elif SANITIZER_LOONGARCH64
83	#define PTHREAD_ABI_BASE "GLIBC_2.36"	84	#define PTHREAD_ABI_BASE "GLIBC_2.36"
		85	#elif SANITIZER_RISCV64
		86	# define PTHREAD_ABI_BASE "GLIBC_2.27"
84	#endif	87	#endif
85		88
86	extern "C" int pthread_attr_init(void *attr);	89	extern "C" int pthread_attr_init(void *attr);
...	@@ -205,7 +208,7 @@ struct AtExitCtx {	...	@@ -205,7 +208,7 @@ struct AtExitCtx {
205	struct InterceptorContext {	208	struct InterceptorContext {
206	// The object is 64-byte aligned, because we want hot data to be located	209	// The object is 64-byte aligned, because we want hot data to be located
207	// in a single cache line if possible (it's accessed in every interceptor).	210	// in a single cache line if possible (it's accessed in every interceptor).
208	ALIGNED(64) LibIgnore libignore;	211	alignas(64) LibIgnore libignore;
209	__sanitizer_sigaction sigactions[kSigCount];	212	__sanitizer_sigaction sigactions[kSigCount];
210	#if !SANITIZER_APPLE && !SANITIZER_NETBSD	213	#if !SANITIZER_APPLE && !SANITIZER_NETBSD
211	unsigned finalize_key;	214	unsigned finalize_key;
...	@@ -217,7 +220,7 @@ struct InterceptorContext {	...	@@ -217,7 +220,7 @@ struct InterceptorContext {
217	InterceptorContext() : libignore(LINKER_INITIALIZED), atexit_mu(MutexTypeAtExit), AtExitStack() {}	220	InterceptorContext() : libignore(LINKER_INITIALIZED), atexit_mu(MutexTypeAtExit), AtExitStack() {}
218	};	221	};
219		222
220	static ALIGNED(64) char interceptor_placeholder[sizeof(InterceptorContext)];	223	alignas(64) static char interceptor_placeholder[sizeof(InterceptorContext)];
221	InterceptorContext *interceptor_ctx() {	224	InterceptorContext *interceptor_ctx() {
222	return reinterpret_cast<InterceptorContext*>(&interceptor_placeholder[0]);	225	return reinterpret_cast<InterceptorContext*>(&interceptor_placeholder[0]);
223	}	226	}
...	@@ -1085,7 +1088,18 @@ TSAN_INTERCEPTOR(int, pthread_join, void th, void *ret) {	...	@@ -1085,7 +1088,18 @@ TSAN_INTERCEPTOR(int, pthread_join, void th, void *ret) {
1085	return res;	1088	return res;
1086	}	1089	}
1087		1090
1088	DEFINE_REAL_PTHREAD_FUNCTIONS	1091	// DEFINE_INTERNAL_PTHREAD_FUNCTIONS
		1092	namespace __sanitizer {
		1093	int internal_pthread_create(void th, void attr, void (callback)(void *),
		1094	void *param) {
		1095	ScopedIgnoreInterceptors ignore;
		1096	return REAL(pthread_create)(th, attr, callback, param);
		1097	}
		1098	int internal_pthread_join(void th, void *ret) {
		1099	ScopedIgnoreInterceptors ignore;
		1100	return REAL(pthread_join(th, ret));
		1101	}
		1102	} // namespace __sanitizer
1089		1103
1090	TSAN_INTERCEPTOR(int, pthread_detach, void *th) {	1104	TSAN_INTERCEPTOR(int, pthread_detach, void *th) {
1091	SCOPED_INTERCEPTOR_RAW(pthread_detach, th);	1105	SCOPED_INTERCEPTOR_RAW(pthread_detach, th);
...	@@ -1338,7 +1352,7 @@ TSAN_INTERCEPTOR(int, pthread_mutex_destroy, void *m) {	...	@@ -1338,7 +1352,7 @@ TSAN_INTERCEPTOR(int, pthread_mutex_destroy, void *m) {
1338	TSAN_INTERCEPTOR(int, pthread_mutex_lock, void *m) {	1352	TSAN_INTERCEPTOR(int, pthread_mutex_lock, void *m) {
1339	SCOPED_TSAN_INTERCEPTOR(pthread_mutex_lock, m);	1353	SCOPED_TSAN_INTERCEPTOR(pthread_mutex_lock, m);
1340	MutexPreLock(thr, pc, (uptr)m);	1354	MutexPreLock(thr, pc, (uptr)m);
1341	int res = REAL(pthread_mutex_lock)(m);	1355	int res = BLOCK_REAL(pthread_mutex_lock)(m);
1342	if (res == errno_EOWNERDEAD)	1356	if (res == errno_EOWNERDEAD)
1343	MutexRepair(thr, pc, (uptr)m);	1357	MutexRepair(thr, pc, (uptr)m);
1344	if (res == 0 \|\| res == errno_EOWNERDEAD)	1358	if (res == 0 \|\| res == errno_EOWNERDEAD)
...	@@ -1378,6 +1392,22 @@ TSAN_INTERCEPTOR(int, pthread_mutex_unlock, void *m) {	...	@@ -1378,6 +1392,22 @@ TSAN_INTERCEPTOR(int, pthread_mutex_unlock, void *m) {
1378	return res;	1392	return res;
1379	}	1393	}
1380		1394
		1395	#if SANITIZER_LINUX
		1396	TSAN_INTERCEPTOR(int, pthread_mutex_clocklock, void *m,
		1397	__sanitizer_clockid_t clock, void *abstime) {
		1398	SCOPED_TSAN_INTERCEPTOR(pthread_mutex_clocklock, m, clock, abstime);
		1399	MutexPreLock(thr, pc, (uptr)m);
		1400	int res = BLOCK_REAL(pthread_mutex_clocklock)(m, clock, abstime);
		1401	if (res == errno_EOWNERDEAD)
		1402	MutexRepair(thr, pc, (uptr)m);
		1403	if (res == 0 \|\| res == errno_EOWNERDEAD)
		1404	MutexPostLock(thr, pc, (uptr)m);
		1405	if (res == errno_EINVAL)
		1406	MutexInvalidAccess(thr, pc, (uptr)m);
		1407	return res;
		1408	}
		1409	#endif
		1410
1381	#if SANITIZER_GLIBC	1411	#if SANITIZER_GLIBC
1382	# if !__GLIBC_PREREQ(2, 34)	1412	# if !__GLIBC_PREREQ(2, 34)
1383	// glibc 2.34 applies a non-default version for the two functions. They are no	1413	// glibc 2.34 applies a non-default version for the two functions. They are no
...	@@ -1385,7 +1415,7 @@ TSAN_INTERCEPTOR(int, pthread_mutex_unlock, void *m) {	...	@@ -1385,7 +1415,7 @@ TSAN_INTERCEPTOR(int, pthread_mutex_unlock, void *m) {
1385	TSAN_INTERCEPTOR(int, __pthread_mutex_lock, void *m) {	1415	TSAN_INTERCEPTOR(int, __pthread_mutex_lock, void *m) {
1386	SCOPED_TSAN_INTERCEPTOR(__pthread_mutex_lock, m);	1416	SCOPED_TSAN_INTERCEPTOR(__pthread_mutex_lock, m);
1387	MutexPreLock(thr, pc, (uptr)m);	1417	MutexPreLock(thr, pc, (uptr)m);
1388	int res = REAL(__pthread_mutex_lock)(m);	1418	int res = BLOCK_REAL(__pthread_mutex_lock)(m);
1389	if (res == errno_EOWNERDEAD)	1419	if (res == errno_EOWNERDEAD)
1390	MutexRepair(thr, pc, (uptr)m);	1420	MutexRepair(thr, pc, (uptr)m);
1391	if (res == 0 \|\| res == errno_EOWNERDEAD)	1421	if (res == 0 \|\| res == errno_EOWNERDEAD)
...	@@ -1428,7 +1458,7 @@ TSAN_INTERCEPTOR(int, pthread_spin_destroy, void *m) {	...	@@ -1428,7 +1458,7 @@ TSAN_INTERCEPTOR(int, pthread_spin_destroy, void *m) {
1428	TSAN_INTERCEPTOR(int, pthread_spin_lock, void *m) {	1458	TSAN_INTERCEPTOR(int, pthread_spin_lock, void *m) {
1429	SCOPED_TSAN_INTERCEPTOR(pthread_spin_lock, m);	1459	SCOPED_TSAN_INTERCEPTOR(pthread_spin_lock, m);
1430	MutexPreLock(thr, pc, (uptr)m);	1460	MutexPreLock(thr, pc, (uptr)m);
1431	int res = REAL(pthread_spin_lock)(m);	1461	int res = BLOCK_REAL(pthread_spin_lock)(m);
1432	if (res == 0) {	1462	if (res == 0) {
1433	MutexPostLock(thr, pc, (uptr)m);	1463	MutexPostLock(thr, pc, (uptr)m);
1434	}	1464	}
...	@@ -1503,7 +1533,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_timedrdlock, void m, void abstime) {	...	@@ -1503,7 +1533,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_timedrdlock, void m, void abstime) {
1503	TSAN_INTERCEPTOR(int, pthread_rwlock_wrlock, void *m) {	1533	TSAN_INTERCEPTOR(int, pthread_rwlock_wrlock, void *m) {
1504	SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_wrlock, m);	1534	SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_wrlock, m);
1505	MutexPreLock(thr, pc, (uptr)m);	1535	MutexPreLock(thr, pc, (uptr)m);
1506	int res = REAL(pthread_rwlock_wrlock)(m);	1536	int res = BLOCK_REAL(pthread_rwlock_wrlock)(m);
1507	if (res == 0) {	1537	if (res == 0) {
1508	MutexPostLock(thr, pc, (uptr)m);	1538	MutexPostLock(thr, pc, (uptr)m);
1509	}	1539	}
...	@@ -1595,47 +1625,40 @@ TSAN_INTERCEPTOR(int, __fxstat, int version, int fd, void *buf) {	...	@@ -1595,47 +1625,40 @@ TSAN_INTERCEPTOR(int, __fxstat, int version, int fd, void *buf) {
1595	FdAccess(thr, pc, fd);	1625	FdAccess(thr, pc, fd);
1596	return REAL(__fxstat)(version, fd, buf);	1626	return REAL(__fxstat)(version, fd, buf);
1597	}	1627	}
1598	#define TSAN_MAYBE_INTERCEPT___FXSTAT TSAN_INTERCEPT(__fxstat)	1628
		1629	TSAN_INTERCEPTOR(int, __fxstat64, int version, int fd, void *buf) {
		1630	SCOPED_TSAN_INTERCEPTOR(__fxstat64, version, fd, buf);
		1631	if (fd > 0)
		1632	FdAccess(thr, pc, fd);
		1633	return REAL(__fxstat64)(version, fd, buf);
		1634	}
		1635	#define TSAN_MAYBE_INTERCEPT___FXSTAT TSAN_INTERCEPT(__fxstat); TSAN_INTERCEPT(__fxstat64)
1599	#else	1636	#else
1600	#define TSAN_MAYBE_INTERCEPT___FXSTAT	1637	#define TSAN_MAYBE_INTERCEPT___FXSTAT
1601	#endif	1638	#endif
1602		1639
		1640	#if !SANITIZER_GLIBC \|\| __GLIBC_PREREQ(2, 33)
1603	TSAN_INTERCEPTOR(int, fstat, int fd, void *buf) {	1641	TSAN_INTERCEPTOR(int, fstat, int fd, void *buf) {
1604	#if SANITIZER_GLIBC
1605	SCOPED_TSAN_INTERCEPTOR(__fxstat, 0, fd, buf);
1606	if (fd > 0)
1607	FdAccess(thr, pc, fd);
1608	return REAL(__fxstat)(0, fd, buf);
1609	#else
1610	SCOPED_TSAN_INTERCEPTOR(fstat, fd, buf);	1642	SCOPED_TSAN_INTERCEPTOR(fstat, fd, buf);
1611	if (fd > 0)	1643	if (fd > 0)
1612	FdAccess(thr, pc, fd);	1644	FdAccess(thr, pc, fd);
1613	return REAL(fstat)(fd, buf);	1645	return REAL(fstat)(fd, buf);
1614	#endif
1615	}
1616
1617	#if SANITIZER_GLIBC
1618	TSAN_INTERCEPTOR(int, __fxstat64, int version, int fd, void *buf) {
1619	SCOPED_TSAN_INTERCEPTOR(__fxstat64, version, fd, buf);
1620	if (fd > 0)
1621	FdAccess(thr, pc, fd);
1622	return REAL(__fxstat64)(version, fd, buf);
1623	}	1646	}
1624	#define TSAN_MAYBE_INTERCEPT___FXSTAT64 TSAN_INTERCEPT(__fxstat64)	1647	# define TSAN_MAYBE_INTERCEPT_FSTAT TSAN_INTERCEPT(fstat)
1625	#else	1648	#else
1626	#define TSAN_MAYBE_INTERCEPT___FXSTAT64	1649	# define TSAN_MAYBE_INTERCEPT_FSTAT
1627	#endif	1650	#endif
1628		1651
1629	#if SANITIZER_GLIBC	1652	#if __GLIBC_PREREQ(2, 33)
1630	TSAN_INTERCEPTOR(int, fstat64, int fd, void *buf) {	1653	TSAN_INTERCEPTOR(int, fstat64, int fd, void *buf) {
1631	SCOPED_TSAN_INTERCEPTOR(__fxstat64, 0, fd, buf);	1654	SCOPED_TSAN_INTERCEPTOR(fstat64, fd, buf);
1632	if (fd > 0)	1655	if (fd > 0)
1633	FdAccess(thr, pc, fd);	1656	FdAccess(thr, pc, fd);
1634	return REAL(__fxstat64)(0, fd, buf);	1657	return REAL(fstat64)(fd, buf);
1635	}	1658	}
1636	#define TSAN_MAYBE_INTERCEPT_FSTAT64 TSAN_INTERCEPT(fstat64)	1659	# define TSAN_MAYBE_INTERCEPT_FSTAT64 TSAN_INTERCEPT(fstat64)
1637	#else	1660	#else
1638	#define TSAN_MAYBE_INTERCEPT_FSTAT64	1661	# define TSAN_MAYBE_INTERCEPT_FSTAT64
1639	#endif	1662	#endif
1640		1663
1641	TSAN_INTERCEPTOR(int, open, const char *name, int oflag, ...) {	1664	TSAN_INTERCEPTOR(int, open, const char *name, int oflag, ...) {
...	@@ -2565,7 +2588,7 @@ int sigaction_impl(int sig, const __sanitizer_sigaction *act,	...	@@ -2565,7 +2588,7 @@ int sigaction_impl(int sig, const __sanitizer_sigaction *act,
2565	// Copy act into sigactions[sig].	2588	// Copy act into sigactions[sig].
2566	// Can't use struct copy, because compiler can emit call to memcpy.	2589	// Can't use struct copy, because compiler can emit call to memcpy.
2567	// Can't use internal_memcpy, because it copies byte-by-byte,	2590	// Can't use internal_memcpy, because it copies byte-by-byte,
2568	// and signal handler reads the handler concurrently. It it can read	2591	// and signal handler reads the handler concurrently. It can read
2569	// some bytes from old value and some bytes from new value.	2592	// some bytes from old value and some bytes from new value.
2570	// Use volatile to prevent insertion of memcpy.	2593	// Use volatile to prevent insertion of memcpy.
2571	sigactions[sig].handler =	2594	sigactions[sig].handler =
...	@@ -2655,6 +2678,25 @@ static USED void syscall_fd_release(uptr pc, int fd) {	...	@@ -2655,6 +2678,25 @@ static USED void syscall_fd_release(uptr pc, int fd) {
2655	FdRelease(thr, pc, fd);	2678	FdRelease(thr, pc, fd);
2656	}	2679	}
2657		2680
		2681	static USED void sycall_blocking_start() {
		2682	DPrintf("sycall_blocking_start()\n");
		2683	ThreadState *thr = cur_thread();
		2684	EnterBlockingFunc(thr);
		2685	// When we are in a "blocking call", we process signals asynchronously
		2686	// (right when they arrive). In this context we do not expect to be
		2687	// executing any user/runtime code. The known interceptor sequence when
		2688	// this is not true is: pthread_join -> munmap(stack). It's fine
		2689	// to ignore munmap in this case -- we handle stack shadow separately.
		2690	thr->ignore_interceptors++;
		2691	}
		2692
		2693	static USED void sycall_blocking_end() {
		2694	DPrintf("sycall_blocking_end()\n");
		2695	ThreadState *thr = cur_thread();
		2696	thr->ignore_interceptors--;
		2697	atomic_store(&thr->in_blocking_func, 0, memory_order_relaxed);
		2698	}
		2699
2658	static void syscall_pre_fork(uptr pc) { ForkBefore(cur_thread(), pc); }	2700	static void syscall_pre_fork(uptr pc) { ForkBefore(cur_thread(), pc); }
2659		2701
2660	static void syscall_post_fork(uptr pc, int pid) {	2702	static void syscall_post_fork(uptr pc, int pid) {
...	@@ -2709,6 +2751,9 @@ static void syscall_post_fork(uptr pc, int pid) {	...	@@ -2709,6 +2751,9 @@ static void syscall_post_fork(uptr pc, int pid) {
2709	#define COMMON_SYSCALL_POST_FORK(res) \	2751	#define COMMON_SYSCALL_POST_FORK(res) \
2710	syscall_post_fork(GET_CALLER_PC(), res)	2752	syscall_post_fork(GET_CALLER_PC(), res)
2711		2753
		2754	#define COMMON_SYSCALL_BLOCKING_START() sycall_blocking_start()
		2755	#define COMMON_SYSCALL_BLOCKING_END() sycall_blocking_end()
		2756
2712	#include "sanitizer_common/sanitizer_common_syscalls.inc"	2757	#include "sanitizer_common/sanitizer_common_syscalls.inc"
2713	#include "sanitizer_common/sanitizer_syscalls_netbsd.inc"	2758	#include "sanitizer_common/sanitizer_syscalls_netbsd.inc"
2714		2759
...	@@ -2843,8 +2888,21 @@ void InitializeInterceptors() {	...	@@ -2843,8 +2888,21 @@ void InitializeInterceptors() {
2843	REAL(memcpy) = internal_memcpy;	2888	REAL(memcpy) = internal_memcpy;
2844	#endif	2889	#endif
2845		2890
		2891	__interception::DoesNotSupportStaticLinking();
		2892
2846	new(interceptor_ctx()) InterceptorContext();	2893	new(interceptor_ctx()) InterceptorContext();
2847		2894
		2895	// Interpose __tls_get_addr before the common interposers. This is needed
		2896	// because dlsym() may call malloc on failure which could result in other
		2897	// interposed functions being called that could eventually make use of TLS.
		2898	#ifdef NEED_TLS_GET_ADDR
		2899	# if !SANITIZER_S390
		2900	TSAN_INTERCEPT(__tls_get_addr);
		2901	# else
		2902	TSAN_INTERCEPT(__tls_get_addr_internal);
		2903	TSAN_INTERCEPT(__tls_get_offset);
		2904	# endif
		2905	#endif
2848	InitializeCommonInterceptors();	2906	InitializeCommonInterceptors();
2849	InitializeSignalInterceptors();	2907	InitializeSignalInterceptors();
2850	InitializeLibdispatchInterceptors();	2908	InitializeLibdispatchInterceptors();
...	@@ -2900,6 +2958,9 @@ void InitializeInterceptors() {	...	@@ -2900,6 +2958,9 @@ void InitializeInterceptors() {
2900	TSAN_INTERCEPT(pthread_mutex_trylock);	2958	TSAN_INTERCEPT(pthread_mutex_trylock);
2901	TSAN_INTERCEPT(pthread_mutex_timedlock);	2959	TSAN_INTERCEPT(pthread_mutex_timedlock);
2902	TSAN_INTERCEPT(pthread_mutex_unlock);	2960	TSAN_INTERCEPT(pthread_mutex_unlock);
		2961	#if SANITIZER_LINUX
		2962	TSAN_INTERCEPT(pthread_mutex_clocklock);
		2963	#endif
2903	#if SANITIZER_GLIBC	2964	#if SANITIZER_GLIBC
2904	# if !__GLIBC_PREREQ(2, 34)	2965	# if !__GLIBC_PREREQ(2, 34)
2905	TSAN_INTERCEPT(__pthread_mutex_lock);	2966	TSAN_INTERCEPT(__pthread_mutex_lock);
...	@@ -2929,10 +2990,9 @@ void InitializeInterceptors() {	...	@@ -2929,10 +2990,9 @@ void InitializeInterceptors() {
2929		2990
2930	TSAN_INTERCEPT(pthread_once);	2991	TSAN_INTERCEPT(pthread_once);
2931		2992
2932	TSAN_INTERCEPT(fstat);
2933	TSAN_MAYBE_INTERCEPT___FXSTAT;	2993	TSAN_MAYBE_INTERCEPT___FXSTAT;
		2994	TSAN_MAYBE_INTERCEPT_FSTAT;
2934	TSAN_MAYBE_INTERCEPT_FSTAT64;	2995	TSAN_MAYBE_INTERCEPT_FSTAT64;
2935	TSAN_MAYBE_INTERCEPT___FXSTAT64;
2936	TSAN_INTERCEPT(open);	2996	TSAN_INTERCEPT(open);
2937	TSAN_MAYBE_INTERCEPT_OPEN64;	2997	TSAN_MAYBE_INTERCEPT_OPEN64;
2938	TSAN_INTERCEPT(creat);	2998	TSAN_INTERCEPT(creat);
...	@@ -2989,15 +3049,6 @@ void InitializeInterceptors() {	...	@@ -2989,15 +3049,6 @@ void InitializeInterceptors() {
2989	TSAN_INTERCEPT(__cxa_atexit);	3049	TSAN_INTERCEPT(__cxa_atexit);
2990	TSAN_INTERCEPT(_exit);	3050	TSAN_INTERCEPT(_exit);
2991		3051
2992	#ifdef NEED_TLS_GET_ADDR
2993	#if !SANITIZER_S390
2994	TSAN_INTERCEPT(__tls_get_addr);
2995	#else
2996	TSAN_INTERCEPT(__tls_get_addr_internal);
2997	TSAN_INTERCEPT(__tls_get_offset);
2998	#endif
2999	#endif
3000
3001	TSAN_MAYBE_INTERCEPT__LWP_EXIT;	3052	TSAN_MAYBE_INTERCEPT__LWP_EXIT;
3002	TSAN_MAYBE_INTERCEPT_THR_EXIT;	3053	TSAN_MAYBE_INTERCEPT_THR_EXIT;
3003		3054

lib/tsan/tsan_interface.h+8

...	@@ -419,6 +419,14 @@ void __tsan_go_atomic32_fetch_add(ThreadState thr, uptr cpc, uptr pc, u8 a);	...	@@ -419,6 +419,14 @@ void __tsan_go_atomic32_fetch_add(ThreadState thr, uptr cpc, uptr pc, u8 a);
419	SANITIZER_INTERFACE_ATTRIBUTE	419	SANITIZER_INTERFACE_ATTRIBUTE
420	void __tsan_go_atomic64_fetch_add(ThreadState thr, uptr cpc, uptr pc, u8 a);	420	void __tsan_go_atomic64_fetch_add(ThreadState thr, uptr cpc, uptr pc, u8 a);
421	SANITIZER_INTERFACE_ATTRIBUTE	421	SANITIZER_INTERFACE_ATTRIBUTE
		422	void __tsan_go_atomic32_fetch_and(ThreadState thr, uptr cpc, uptr pc, u8 a);
		423	SANITIZER_INTERFACE_ATTRIBUTE
		424	void __tsan_go_atomic64_fetch_and(ThreadState thr, uptr cpc, uptr pc, u8 a);
		425	SANITIZER_INTERFACE_ATTRIBUTE
		426	void __tsan_go_atomic32_fetch_or(ThreadState thr, uptr cpc, uptr pc, u8 a);
		427	SANITIZER_INTERFACE_ATTRIBUTE
		428	void __tsan_go_atomic64_fetch_or(ThreadState thr, uptr cpc, uptr pc, u8 a);
		429	SANITIZER_INTERFACE_ATTRIBUTE
422	void __tsan_go_atomic32_exchange(ThreadState thr, uptr cpc, uptr pc, u8 a);	430	void __tsan_go_atomic32_exchange(ThreadState thr, uptr cpc, uptr pc, u8 a);
423	SANITIZER_INTERFACE_ATTRIBUTE	431	SANITIZER_INTERFACE_ATTRIBUTE
424	void __tsan_go_atomic64_exchange(ThreadState thr, uptr cpc, uptr pc, u8 a);	432	void __tsan_go_atomic64_exchange(ThreadState thr, uptr cpc, uptr pc, u8 a);

lib/tsan/tsan_interface_ann.cpp+23-1

...	@@ -76,7 +76,7 @@ struct DynamicAnnContext {	...	@@ -76,7 +76,7 @@ struct DynamicAnnContext {
76	};	76	};
77		77
78	static DynamicAnnContext *dyn_ann_ctx;	78	static DynamicAnnContext *dyn_ann_ctx;
79	static char dyn_ann_ctx_placeholder[sizeof(DynamicAnnContext)] ALIGNED(64);	79	alignas(64) static char dyn_ann_ctx_placeholder[sizeof(DynamicAnnContext)];
80		80
81	static void AddExpectRace(ExpectRace *list,	81	static void AddExpectRace(ExpectRace *list,
82	char f, int l, uptr addr, uptr size, char desc) {	82	char f, int l, uptr addr, uptr size, char desc) {
...	@@ -435,4 +435,26 @@ void __tsan_mutex_post_divert(void *addr, unsigned flagz) {	...	@@ -435,4 +435,26 @@ void __tsan_mutex_post_divert(void *addr, unsigned flagz) {
435	ThreadIgnoreBegin(thr, 0);	435	ThreadIgnoreBegin(thr, 0);
436	ThreadIgnoreSyncBegin(thr, 0);	436	ThreadIgnoreSyncBegin(thr, 0);
437	}	437	}
		438
		439	static void ReportMutexHeldWrongContext(ThreadState *thr, uptr pc) {
		440	ThreadRegistryLock l(&ctx->thread_registry);
		441	ScopedReport rep(ReportTypeMutexHeldWrongContext);
		442	for (uptr i = 0; i < thr->mset.Size(); ++i) {
		443	MutexSet::Desc desc = thr->mset.Get(i);
		444	rep.AddMutex(desc.addr, desc.stack_id);
		445	}
		446	VarSizeStackTrace trace;
		447	ObtainCurrentStack(thr, pc, &trace);
		448	rep.AddStack(trace, true);
		449	OutputReport(thr, rep);
		450	}
		451
		452	INTERFACE_ATTRIBUTE
		453	void __tsan_check_no_mutexes_held() {
		454	SCOPED_ANNOTATION(__tsan_check_no_mutexes_held);
		455	if (thr->mset.Size() == 0) {
		456	return;
		457	}
		458	ReportMutexHeldWrongContext(thr, pc);
		459	}
438	} // extern "C"	460	} // extern "C"

lib/tsan/tsan_interface_atomic.cpp+24

...	@@ -894,6 +894,30 @@ void __tsan_go_atomic64_fetch_add(ThreadState thr, uptr cpc, uptr pc, u8 a) {	...	@@ -894,6 +894,30 @@ void __tsan_go_atomic64_fetch_add(ThreadState thr, uptr cpc, uptr pc, u8 a) {
894	ATOMIC_RET(FetchAdd, (a64)(a+16), (a64)a, (a64*)(a+8), mo_acq_rel);	894	ATOMIC_RET(FetchAdd, (a64)(a+16), (a64)a, (a64*)(a+8), mo_acq_rel);
895	}	895	}
896		896
		897	SANITIZER_INTERFACE_ATTRIBUTE
		898	void __tsan_go_atomic32_fetch_and(ThreadState thr, uptr cpc, uptr pc, u8 a) {
		899	ATOMIC_RET(FetchAnd, (a32 )(a + 16), (a32 )a, (a32 *)(a + 8),
		900	mo_acq_rel);
		901	}
		902
		903	SANITIZER_INTERFACE_ATTRIBUTE
		904	void __tsan_go_atomic64_fetch_and(ThreadState thr, uptr cpc, uptr pc, u8 a) {
		905	ATOMIC_RET(FetchAnd, (a64 )(a + 16), (a64 )a, (a64 *)(a + 8),
		906	mo_acq_rel);
		907	}
		908
		909	SANITIZER_INTERFACE_ATTRIBUTE
		910	void __tsan_go_atomic32_fetch_or(ThreadState thr, uptr cpc, uptr pc, u8 a) {
		911	ATOMIC_RET(FetchOr, (a32 )(a + 16), (a32 )a, (a32 *)(a + 8),
		912	mo_acq_rel);
		913	}
		914
		915	SANITIZER_INTERFACE_ATTRIBUTE
		916	void __tsan_go_atomic64_fetch_or(ThreadState thr, uptr cpc, uptr pc, u8 a) {
		917	ATOMIC_RET(FetchOr, (a64 )(a + 16), (a64 )a, (a64 *)(a + 8),
		918	mo_acq_rel);
		919	}
		920
897	SANITIZER_INTERFACE_ATTRIBUTE	921	SANITIZER_INTERFACE_ATTRIBUTE
898	void __tsan_go_atomic32_exchange(ThreadState thr, uptr cpc, uptr pc, u8 a) {	922	void __tsan_go_atomic32_exchange(ThreadState thr, uptr cpc, uptr pc, u8 a) {
899	ATOMIC_RET(Exchange, (a32)(a+16), (a32)a, (a32*)(a+8), mo_acq_rel);	923	ATOMIC_RET(Exchange, (a32)(a+16), (a32)a, (a32*)(a+8), mo_acq_rel);

lib/tsan/tsan_mman.cpp+20-9

...	@@ -9,17 +9,19 @@	...	@@ -9,17 +9,19 @@
9	// This file is a part of ThreadSanitizer (TSan), a race detector.	9	// This file is a part of ThreadSanitizer (TSan), a race detector.
10	//	10	//
11	//===----------------------------------------------------------------------===//	11	//===----------------------------------------------------------------------===//
		12	#include "tsan_mman.h"
		13
12	#include "sanitizer_common/sanitizer_allocator_checks.h"	14	#include "sanitizer_common/sanitizer_allocator_checks.h"
13	#include "sanitizer_common/sanitizer_allocator_interface.h"	15	#include "sanitizer_common/sanitizer_allocator_interface.h"
14	#include "sanitizer_common/sanitizer_allocator_report.h"	16	#include "sanitizer_common/sanitizer_allocator_report.h"
15	#include "sanitizer_common/sanitizer_common.h"	17	#include "sanitizer_common/sanitizer_common.h"
16	#include "sanitizer_common/sanitizer_errno.h"	18	#include "sanitizer_common/sanitizer_errno.h"
17	#include "sanitizer_common/sanitizer_placement_new.h"	19	#include "sanitizer_common/sanitizer_placement_new.h"
		20	#include "sanitizer_common/sanitizer_stackdepot.h"
		21	#include "tsan_flags.h"
18	#include "tsan_interface.h"	22	#include "tsan_interface.h"
19	#include "tsan_mman.h"
20	#include "tsan_rtl.h"
21	#include "tsan_report.h"	23	#include "tsan_report.h"
22	#include "tsan_flags.h"	24	#include "tsan_rtl.h"
23		25
24	namespace __tsan {	26	namespace __tsan {
25		27
...	@@ -52,7 +54,7 @@ struct MapUnmapCallback {	...	@@ -52,7 +54,7 @@ struct MapUnmapCallback {
52	}	54	}
53	};	55	};
54		56
55	static char allocator_placeholder[sizeof(Allocator)] ALIGNED(64);	57	alignas(64) static char allocator_placeholder[sizeof(Allocator)];
56	Allocator *allocator() {	58	Allocator *allocator() {
57	return reinterpret_cast<Allocator*>(&allocator_placeholder);	59	return reinterpret_cast<Allocator*>(&allocator_placeholder);
58	}	60	}
...	@@ -73,7 +75,7 @@ struct GlobalProc {	...	@@ -73,7 +75,7 @@ struct GlobalProc {
73	internal_alloc_mtx(MutexTypeInternalAlloc) {}	75	internal_alloc_mtx(MutexTypeInternalAlloc) {}
74	};	76	};
75		77
76	static char global_proc_placeholder[sizeof(GlobalProc)] ALIGNED(64);	78	alignas(64) static char global_proc_placeholder[sizeof(GlobalProc)];
77	GlobalProc *global_proc() {	79	GlobalProc *global_proc() {
78	return reinterpret_cast<GlobalProc*>(&global_proc_placeholder);	80	return reinterpret_cast<GlobalProc*>(&global_proc_placeholder);
79	}	81	}
...	@@ -115,12 +117,21 @@ ScopedGlobalProcessor::~ScopedGlobalProcessor() {	...	@@ -115,12 +117,21 @@ ScopedGlobalProcessor::~ScopedGlobalProcessor() {
115	gp->mtx.Unlock();	117	gp->mtx.Unlock();
116	}	118	}
117		119
118	void AllocatorLock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {	120	void AllocatorLockBeforeFork() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
119	global_proc()->internal_alloc_mtx.Lock();	121	global_proc()->internal_alloc_mtx.Lock();
120	InternalAllocatorLock();	122	InternalAllocatorLock();
121	}	123	#if !SANITIZER_APPLE
122		124	// OS X allocates from hooks, see 6a3958247a.
123	void AllocatorUnlock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {	125	allocator()->ForceLock();
		126	StackDepotLockBeforeFork();
		127	#endif
		128	}
		129
		130	void AllocatorUnlockAfterFork(bool child) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
		131	#if !SANITIZER_APPLE
		132	StackDepotUnlockAfterFork(child);
		133	allocator()->ForceUnlock();
		134	#endif
124	InternalAllocatorUnlock();	135	InternalAllocatorUnlock();
125	global_proc()->internal_alloc_mtx.Unlock();	136	global_proc()->internal_alloc_mtx.Unlock();
126	}	137	}

lib/tsan/tsan_mman.h+2-2

...	@@ -24,8 +24,8 @@ void ReplaceSystemMalloc();	...	@@ -24,8 +24,8 @@ void ReplaceSystemMalloc();
24	void AllocatorProcStart(Processor *proc);	24	void AllocatorProcStart(Processor *proc);
25	void AllocatorProcFinish(Processor *proc);	25	void AllocatorProcFinish(Processor *proc);
26	void AllocatorPrintStats();	26	void AllocatorPrintStats();
27	void AllocatorLock();	27	void AllocatorLockBeforeFork();
28	void AllocatorUnlock();	28	void AllocatorUnlockAfterFork(bool child);
29	void GlobalProcessorLock();	29	void GlobalProcessorLock();
30	void GlobalProcessorUnlock();	30	void GlobalProcessorUnlock();
31		31

lib/tsan/tsan_platform.h+128-23

...	@@ -46,17 +46,16 @@ enum {	...	@@ -46,17 +46,16 @@ enum {
46		46
47	/*	47	/*
48	C/C++ on linux/x86_64 and freebsd/x86_64	48	C/C++ on linux/x86_64 and freebsd/x86_64
49	0000 0000 1000 - 0080 0000 0000: main binary and/or MAP_32BIT mappings (512GB)	49	0000 0000 1000 - 0200 0000 0000: main binary and/or MAP_32BIT mappings (2TB)
50	0040 0000 0000 - 0100 0000 0000: -	50	0200 0000 0000 - 1000 0000 0000: -
51	0100 0000 0000 - 1000 0000 0000: shadow	51	1000 0000 0000 - 3000 0000 0000: shadow (32TB)
52	1000 0000 0000 - 3000 0000 0000: -	52	3000 0000 0000 - 3800 0000 0000: metainfo (memory blocks and sync objects; 8TB)
53	3000 0000 0000 - 3400 0000 0000: metainfo (memory blocks and sync objects)	53	3800 0000 0000 - 5500 0000 0000: -
54	3400 0000 0000 - 5500 0000 0000: -	54	5500 0000 0000 - 5a00 0000 0000: pie binaries without ASLR or on 4.1+ kernels
55	5500 0000 0000 - 5680 0000 0000: pie binaries without ASLR or on 4.1+ kernels	55	5a00 0000 0000 - 7200 0000 0000: -
56	5680 0000 0000 - 7d00 0000 0000: -	56	7200 0000 0000 - 7300 0000 0000: heap (1TB)
57	7b00 0000 0000 - 7c00 0000 0000: heap	57	7300 0000 0000 - 7a00 0000 0000: -
58	7c00 0000 0000 - 7e80 0000 0000: -	58	7a00 0000 0000 - 8000 0000 0000: modules and main thread stack (6TB)
59	7e80 0000 0000 - 8000 0000 0000: modules and main thread stack
60		59
61	C/C++ on netbsd/amd64 can reuse the same mapping:	60	C/C++ on netbsd/amd64 can reuse the same mapping:
62	* The address space starts from 0x1000 (option with 0x0) and ends with	61	* The address space starts from 0x1000 (option with 0x0) and ends with
...	@@ -72,20 +71,20 @@ C/C++ on netbsd/amd64 can reuse the same mapping:	...	@@ -72,20 +71,20 @@ C/C++ on netbsd/amd64 can reuse the same mapping:
72	*/	71	*/
73	struct Mapping48AddressSpace {	72	struct Mapping48AddressSpace {
74	static const uptr kMetaShadowBeg = 0x300000000000ull;	73	static const uptr kMetaShadowBeg = 0x300000000000ull;
75	static const uptr kMetaShadowEnd = 0x340000000000ull;	74	static const uptr kMetaShadowEnd = 0x380000000000ull;
76	static const uptr kShadowBeg = 0x010000000000ull;	75	static const uptr kShadowBeg = 0x100000000000ull;
77	static const uptr kShadowEnd = 0x100000000000ull;	76	static const uptr kShadowEnd = 0x300000000000ull;
78	static const uptr kHeapMemBeg = 0x7b0000000000ull;	77	static const uptr kHeapMemBeg = 0x720000000000ull;
79	static const uptr kHeapMemEnd = 0x7c0000000000ull;	78	static const uptr kHeapMemEnd = 0x730000000000ull;
80	static const uptr kLoAppMemBeg = 0x000000001000ull;	79	static const uptr kLoAppMemBeg = 0x000000001000ull;
81	static const uptr kLoAppMemEnd = 0x008000000000ull;	80	static const uptr kLoAppMemEnd = 0x020000000000ull;
82	static const uptr kMidAppMemBeg = 0x550000000000ull;	81	static const uptr kMidAppMemBeg = 0x550000000000ull;
83	static const uptr kMidAppMemEnd = 0x568000000000ull;	82	static const uptr kMidAppMemEnd = 0x5a0000000000ull;
84	static const uptr kHiAppMemBeg = 0x7e8000000000ull;	83	static const uptr kHiAppMemBeg = 0x7a0000000000ull;
85	static const uptr kHiAppMemEnd = 0x800000000000ull;	84	static const uptr kHiAppMemEnd = 0x800000000000ull;
86	static const uptr kShadowMsk = 0x780000000000ull;	85	static const uptr kShadowMsk = 0x700000000000ull;
87	static const uptr kShadowXor = 0x040000000000ull;	86	static const uptr kShadowXor = 0x000000000000ull;
88	static const uptr kShadowAdd = 0x000000000000ull;	87	static const uptr kShadowAdd = 0x100000000000ull;
89	static const uptr kVdsoBeg = 0xf000000000000000ull;	88	static const uptr kVdsoBeg = 0xf000000000000000ull;
90	};	89	};
91		90
...	@@ -377,6 +376,71 @@ struct MappingPPC64_47 {	...	@@ -377,6 +376,71 @@ struct MappingPPC64_47 {
377	static const uptr kMidAppMemEnd = 0;	376	static const uptr kMidAppMemEnd = 0;
378	};	377	};
379		378
		379	/*
		380	C/C++ on linux/riscv64 (39-bit VMA)
		381	0000 0010 00 - 0200 0000 00: main binary ( 8 GB)
		382	0200 0000 00 - 1000 0000 00: -
		383	1000 0000 00 - 4000 0000 00: shadow memory (64 GB)
		384	4000 0000 00 - 4800 0000 00: metainfo (16 GB)
		385	4800 0000 00 - 5500 0000 00: -
		386	5500 0000 00 - 5a00 0000 00: main binary (PIE) (~8 GB)
		387	5600 0000 00 - 7c00 0000 00: -
		388	7d00 0000 00 - 7fff ffff ff: libraries and main thread stack ( 8 GB)
		389
		390	mmap by default allocates from top downwards
		391	VDSO sits below loader and above dynamic libraries, within HiApp region.
		392	Heap starts after program region whose position depends on pie or non-pie.
		393	Disable tracking them since their locations are not fixed.
		394	*/
		395	struct MappingRiscv64_39 {
		396	static const uptr kLoAppMemBeg = 0x0000001000ull;
		397	static const uptr kLoAppMemEnd = 0x0200000000ull;
		398	static const uptr kShadowBeg = 0x1000000000ull;
		399	static const uptr kShadowEnd = 0x2000000000ull;
		400	static const uptr kMetaShadowBeg = 0x2000000000ull;
		401	static const uptr kMetaShadowEnd = 0x2400000000ull;
		402	static const uptr kMidAppMemBeg = 0x2aaaaaa000ull;
		403	static const uptr kMidAppMemEnd = 0x2c00000000ull;
		404	static const uptr kHeapMemBeg = 0x2c00000000ull;
		405	static const uptr kHeapMemEnd = 0x2c00000000ull;
		406	static const uptr kHiAppMemBeg = 0x3c00000000ull;
		407	static const uptr kHiAppMemEnd = 0x3fffffffffull;
		408	static const uptr kShadowMsk = 0x3800000000ull;
		409	static const uptr kShadowXor = 0x0800000000ull;
		410	static const uptr kShadowAdd = 0x0000000000ull;
		411	static const uptr kVdsoBeg = 0x4000000000ull;
		412	};
		413
		414	/*
		415	C/C++ on linux/riscv64 (48-bit VMA)
		416	0000 0000 1000 - 0400 0000 0000: main binary ( 4 TB)
		417	0500 0000 0000 - 2000 0000 0000: -
		418	2000 0000 0000 - 4000 0000 0000: shadow memory (32 TB)
		419	4000 0000 0000 - 4800 0000 0000: metainfo ( 8 TB)
		420	4800 0000 0000 - 5555 5555 5000: -
		421	5555 5555 5000 - 5a00 0000 0000: main binary (PIE) (~5 TB)
		422	5a00 0000 0000 - 7a00 0000 0000: -
		423	7a00 0000 0000 - 7fff ffff ffff: libraries and main thread stack ( 6 TB)
		424	*/
		425	struct MappingRiscv64_48 {
		426	static const uptr kLoAppMemBeg = 0x000000001000ull;
		427	static const uptr kLoAppMemEnd = 0x040000000000ull;
		428	static const uptr kShadowBeg = 0x200000000000ull;
		429	static const uptr kShadowEnd = 0x400000000000ull;
		430	static const uptr kMetaShadowBeg = 0x400000000000ull;
		431	static const uptr kMetaShadowEnd = 0x480000000000ull;
		432	static const uptr kMidAppMemBeg = 0x555555555000ull;
		433	static const uptr kMidAppMemEnd = 0x5a0000000000ull;
		434	static const uptr kHeapMemBeg = 0x5a0000000000ull;
		435	static const uptr kHeapMemEnd = 0x5a0000000000ull;
		436	static const uptr kHiAppMemBeg = 0x7a0000000000ull;
		437	static const uptr kHiAppMemEnd = 0x7fffffffffffull;
		438	static const uptr kShadowMsk = 0x700000000000ull;
		439	static const uptr kShadowXor = 0x100000000000ull;
		440	static const uptr kShadowAdd = 0x000000000000ull;
		441	static const uptr kVdsoBeg = 0x800000000000ull;
		442	};
		443
380	/*	444	/*
381	C/C++ on linux/s390x	445	C/C++ on linux/s390x
382	While the kernel provides a 64-bit address space, we have to restrict ourselves	446	While the kernel provides a 64-bit address space, we have to restrict ourselves
...	@@ -558,6 +622,35 @@ struct MappingGoAarch64 {	...	@@ -558,6 +622,35 @@ struct MappingGoAarch64 {
558	static const uptr kShadowAdd = 0x200000000000ull;	622	static const uptr kShadowAdd = 0x200000000000ull;
559	};	623	};
560		624
		625	/* Go on linux/loongarch64 (47-bit VMA)
		626	0000 0000 1000 - 0000 1000 0000: executable
		627	0000 1000 0000 - 00c0 0000 0000: -
		628	00c0 0000 0000 - 00e0 0000 0000: heap
		629	00e0 0000 0000 - 2000 0000 0000: -
		630	2000 0000 0000 - 2800 0000 0000: shadow
		631	2800 0000 0000 - 3000 0000 0000: -
		632	3000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects)
		633	3200 0000 0000 - 8000 0000 0000: -
		634	*/
		635	struct MappingGoLoongArch64_47 {
		636	static const uptr kMetaShadowBeg = 0x300000000000ull;
		637	static const uptr kMetaShadowEnd = 0x320000000000ull;
		638	static const uptr kShadowBeg = 0x200000000000ull;
		639	static const uptr kShadowEnd = 0x280000000000ull;
		640	static const uptr kLoAppMemBeg = 0x000000001000ull;
		641	static const uptr kLoAppMemEnd = 0x00e000000000ull;
		642	static const uptr kMidAppMemBeg = 0;
		643	static const uptr kMidAppMemEnd = 0;
		644	static const uptr kHiAppMemBeg = 0;
		645	static const uptr kHiAppMemEnd = 0;
		646	static const uptr kHeapMemBeg = 0;
		647	static const uptr kHeapMemEnd = 0;
		648	static const uptr kVdsoBeg = 0;
		649	static const uptr kShadowMsk = 0;
		650	static const uptr kShadowXor = 0;
		651	static const uptr kShadowAdd = 0x200000000000ull;
		652	};
		653
561	/*	654	/*
562	Go on linux/mips64 (47-bit VMA)	655	Go on linux/mips64 (47-bit VMA)
563	0000 0000 1000 - 0000 1000 0000: executable	656	0000 0000 1000 - 0000 1000 0000: executable
...	@@ -633,6 +726,8 @@ ALWAYS_INLINE auto SelectMapping(Arg arg) {	...	@@ -633,6 +726,8 @@ ALWAYS_INLINE auto SelectMapping(Arg arg) {
633	return Func::template Apply<MappingGoS390x>(arg);	726	return Func::template Apply<MappingGoS390x>(arg);
634	# elif defined(__aarch64__)	727	# elif defined(__aarch64__)
635	return Func::template Apply<MappingGoAarch64>(arg);	728	return Func::template Apply<MappingGoAarch64>(arg);
		729	# elif defined(__loongarch_lp64)
		730	return Func::template Apply<MappingGoLoongArch64_47>(arg);
636	# elif SANITIZER_WINDOWS	731	# elif SANITIZER_WINDOWS
637	return Func::template Apply<MappingGoWindows>(arg);	732	return Func::template Apply<MappingGoWindows>(arg);
638	# else	733	# else
...	@@ -665,6 +760,13 @@ ALWAYS_INLINE auto SelectMapping(Arg arg) {	...	@@ -665,6 +760,13 @@ ALWAYS_INLINE auto SelectMapping(Arg arg) {
665	}	760	}
666	# elif defined(__mips64)	761	# elif defined(__mips64)
667	return Func::template Apply<MappingMips64_40>(arg);	762	return Func::template Apply<MappingMips64_40>(arg);
		763	# elif SANITIZER_RISCV64
		764	switch (vmaSize) {
		765	case 39:
		766	return Func::template Apply<MappingRiscv64_39>(arg);
		767	case 48:
		768	return Func::template Apply<MappingRiscv64_48>(arg);
		769	}
668	# elif defined(__s390x__)	770	# elif defined(__s390x__)
669	return Func::template Apply<MappingS390x>(arg);	771	return Func::template Apply<MappingS390x>(arg);
670	# else	772	# else
...	@@ -686,12 +788,15 @@ void ForEachMapping() {	...	@@ -686,12 +788,15 @@ void ForEachMapping() {
686	Func::template Apply<MappingPPC64_44>();	788	Func::template Apply<MappingPPC64_44>();
687	Func::template Apply<MappingPPC64_46>();	789	Func::template Apply<MappingPPC64_46>();
688	Func::template Apply<MappingPPC64_47>();	790	Func::template Apply<MappingPPC64_47>();
		791	Func::template Apply<MappingRiscv64_39>();
		792	Func::template Apply<MappingRiscv64_48>();
689	Func::template Apply<MappingS390x>();	793	Func::template Apply<MappingS390x>();
690	Func::template Apply<MappingGo48>();	794	Func::template Apply<MappingGo48>();
691	Func::template Apply<MappingGoWindows>();	795	Func::template Apply<MappingGoWindows>();
692	Func::template Apply<MappingGoPPC64_46>();	796	Func::template Apply<MappingGoPPC64_46>();
693	Func::template Apply<MappingGoPPC64_47>();	797	Func::template Apply<MappingGoPPC64_47>();
694	Func::template Apply<MappingGoAarch64>();	798	Func::template Apply<MappingGoAarch64>();
		799	Func::template Apply<MappingGoLoongArch64_47>();
695	Func::template Apply<MappingGoMips64_47>();	800	Func::template Apply<MappingGoMips64_47>();
696	Func::template Apply<MappingGoS390x>();	801	Func::template Apply<MappingGoS390x>();
697	}	802	}
...	@@ -919,7 +1024,7 @@ inline uptr RestoreAddr(uptr addr) {	...	@@ -919,7 +1024,7 @@ inline uptr RestoreAddr(uptr addr) {
919		1024
920	void InitializePlatform();	1025	void InitializePlatform();
921	void InitializePlatformEarly();	1026	void InitializePlatformEarly();
922	void CheckAndProtect();	1027	bool CheckAndProtect(bool protect, bool ignore_heap, bool print_warnings);
923	void InitializeShadowMemoryPlatform();	1028	void InitializeShadowMemoryPlatform();
924	void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns);	1029	void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns);
925	int ExtractResolvFDs(void state, int fds, int nfd);	1030	int ExtractResolvFDs(void state, int fds, int nfd);

lib/tsan/tsan_platform_linux.cpp+159-61

...	@@ -152,7 +152,7 @@ void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns) {	...	@@ -152,7 +152,7 @@ void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns) {
152	#if !SANITIZER_GO	152	#if !SANITIZER_GO
153	// Mark shadow for .rodata sections with the special Shadow::kRodata marker.	153	// Mark shadow for .rodata sections with the special Shadow::kRodata marker.
154	// Accesses to .rodata can't race, so this saves time, memory and trace space.	154	// Accesses to .rodata can't race, so this saves time, memory and trace space.
155	static void MapRodata() {	155	static NOINLINE void MapRodata(char* buffer, uptr size) {
156	// First create temp file.	156	// First create temp file.
157	const char *tmpdir = GetEnv("TMPDIR");	157	const char *tmpdir = GetEnv("TMPDIR");
158	if (tmpdir == 0)	158	if (tmpdir == 0)
...	@@ -163,13 +163,12 @@ static void MapRodata() {	...	@@ -163,13 +163,12 @@ static void MapRodata() {
163	#endif	163	#endif
164	if (tmpdir == 0)	164	if (tmpdir == 0)
165	return;	165	return;
166	char name[256];	166	internal_snprintf(buffer, size, "%s/tsan.rodata.%d",
167	internal_snprintf(name, sizeof(name), "%s/tsan.rodata.%d",
168	tmpdir, (int)internal_getpid());	167	tmpdir, (int)internal_getpid());
169	uptr openrv = internal_open(name, O_RDWR \| O_CREAT \| O_EXCL, 0600);	168	uptr openrv = internal_open(buffer, O_RDWR \| O_CREAT \| O_EXCL, 0600);
170	if (internal_iserror(openrv))	169	if (internal_iserror(openrv))
171	return;	170	return;
172	internal_unlink(name); // Unlink it now, so that we can reuse the buffer.	171	internal_unlink(buffer); // Unlink it now, so that we can reuse the buffer.
173	fd_t fd = openrv;	172	fd_t fd = openrv;
174	// Fill the file with Shadow::kRodata.	173	// Fill the file with Shadow::kRodata.
175	const uptr kMarkerSize = 512 * 1024 / sizeof(RawShadow);	174	const uptr kMarkerSize = 512 * 1024 / sizeof(RawShadow);
...	@@ -188,8 +187,8 @@ static void MapRodata() {	...	@@ -188,8 +187,8 @@ static void MapRodata() {
188	}	187	}
189	// Map the file into shadow of .rodata sections.	188	// Map the file into shadow of .rodata sections.
190	MemoryMappingLayout proc_maps(/cache_enabled/true);	189	MemoryMappingLayout proc_maps(/cache_enabled/true);
191	// Reusing the buffer 'name'.	190	// Reusing the buffer 'buffer'.
192	MemoryMappedSegment segment(name, ARRAY_SIZE(name));	191	MemoryMappedSegment segment(buffer, size);
193	while (proc_maps.Next(&segment)) {	192	while (proc_maps.Next(&segment)) {
194	if (segment.filename[0] != 0 && segment.filename[0] != '[' &&	193	if (segment.filename[0] != 0 && segment.filename[0] != '[' &&
195	segment.IsReadable() && segment.IsExecutable() &&	194	segment.IsReadable() && segment.IsExecutable() &&
...	@@ -209,11 +208,103 @@ static void MapRodata() {	...	@@ -209,11 +208,103 @@ static void MapRodata() {
209	}	208	}
210		209
211	void InitializeShadowMemoryPlatform() {	210	void InitializeShadowMemoryPlatform() {
212	MapRodata();	211	char buffer[256]; // Keep in a different frame.
		212	MapRodata(buffer, sizeof(buffer));
213	}	213	}
214		214
215	#endif // #if !SANITIZER_GO	215	#endif // #if !SANITIZER_GO
216		216
		217	# if !SANITIZER_GO
		218	static void ReExecIfNeeded(bool ignore_heap) {
		219	// Go maps shadow memory lazily and works fine with limited address space.
		220	// Unlimited stack is not a problem as well, because the executable
		221	// is not compiled with -pie.
		222	bool reexec = false;
		223	// TSan doesn't play well with unlimited stack size (as stack
		224	// overlaps with shadow memory). If we detect unlimited stack size,
		225	// we re-exec the program with limited stack size as a best effort.
		226	if (StackSizeIsUnlimited()) {
		227	const uptr kMaxStackSize = 32 * 1024 * 1024;
		228	VReport(1,
		229	"Program is run with unlimited stack size, which wouldn't "
		230	"work with ThreadSanitizer.\n"
		231	"Re-execing with stack size limited to %zd bytes.\n",
		232	kMaxStackSize);
		233	SetStackSizeLimitInBytes(kMaxStackSize);
		234	reexec = true;
		235	}
		236
		237	if (!AddressSpaceIsUnlimited()) {
		238	Report(
		239	"WARNING: Program is run with limited virtual address space,"
		240	" which wouldn't work with ThreadSanitizer.\n");
		241	Report("Re-execing with unlimited virtual address space.\n");
		242	SetAddressSpaceUnlimited();
		243	reexec = true;
		244	}
		245
		246	# if SANITIZER_LINUX
		247	# if SANITIZER_ANDROID && (defined(__aarch64__) \|\| defined(__x86_64__))
		248	// ASLR personality check.
		249	int old_personality = personality(0xffffffff);
		250	bool aslr_on =
		251	(old_personality != -1) && ((old_personality & ADDR_NO_RANDOMIZE) == 0);
		252
		253	// After patch "arm64: mm: support ARCH_MMAP_RND_BITS." is introduced in
		254	// linux kernel, the random gap between stack and mapped area is increased
		255	// from 128M to 36G on 39-bit aarch64. As it is almost impossible to cover
		256	// this big range, we should disable randomized virtual space on aarch64.
		257	if (aslr_on) {
		258	VReport(1,
		259	"WARNING: Program is run with randomized virtual address "
		260	"space, which wouldn't work with ThreadSanitizer on Android.\n"
		261	"Re-execing with fixed virtual address space.\n");
		262	CHECK_NE(personality(old_personality \| ADDR_NO_RANDOMIZE), -1);
		263	reexec = true;
		264	}
		265	# endif
		266
		267	if (reexec) {
		268	// Don't check the address space since we're going to re-exec anyway.
		269	} else if (!CheckAndProtect(false, ignore_heap, false)) {
		270	// ASLR personality check.
		271	// N.B. 'personality' is sometimes forbidden by sandboxes, so we only call
		272	// this as a last resort (when the memory mapping is incompatible and TSan
		273	// would fail anyway).
		274	int old_personality = personality(0xffffffff);
		275	bool aslr_on =
		276	(old_personality != -1) && ((old_personality & ADDR_NO_RANDOMIZE) == 0);
		277
		278	if (aslr_on) {
		279	// Disable ASLR if the memory layout was incompatible.
		280	// Alternatively, we could just keep re-execing until we get lucky
		281	// with a compatible randomized layout, but the risk is that if it's
		282	// not an ASLR-related issue, we will be stuck in an infinite loop of
		283	// re-execing (unless we change ReExec to pass a parameter of the
		284	// number of retries allowed.)
		285	VReport(1,
		286	"WARNING: ThreadSanitizer: memory layout is incompatible, "
		287	"possibly due to high-entropy ASLR.\n"
		288	"Re-execing with fixed virtual address space.\n"
		289	"N.B. reducing ASLR entropy is preferable.\n");
		290	CHECK_NE(personality(old_personality \| ADDR_NO_RANDOMIZE), -1);
		291	reexec = true;
		292	} else {
		293	Printf(
		294	"FATAL: ThreadSanitizer: memory layout is incompatible, "
		295	"even though ASLR is disabled.\n"
		296	"Please file a bug.\n");
		297	DumpProcessMap();
		298	Die();
		299	}
		300	}
		301	# endif // SANITIZER_LINUX
		302
		303	if (reexec)
		304	ReExec();
		305	}
		306	# endif
		307
217	void InitializePlatformEarly() {	308	void InitializePlatformEarly() {
218	vmaSize =	309	vmaSize =
219	(MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1);	310	(MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1);
...	@@ -238,7 +329,13 @@ void InitializePlatformEarly() {	...	@@ -238,7 +329,13 @@ void InitializePlatformEarly() {
238	Printf("FATAL: Found %zd - Supported 47\n", vmaSize);	329	Printf("FATAL: Found %zd - Supported 47\n", vmaSize);
239	Die();	330	Die();
240	}	331	}
241	# endif	332	# else
		333	if (vmaSize != 47) {
		334	Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
		335	Printf("FATAL: Found %zd - Supported 47\n", vmaSize);
		336	Die();
		337	}
		338	# endif
242	#elif defined(__powerpc64__)	339	#elif defined(__powerpc64__)
243	# if !SANITIZER_GO	340	# if !SANITIZER_GO
244	if (vmaSize != 44 && vmaSize != 46 && vmaSize != 47) {	341	if (vmaSize != 44 && vmaSize != 46 && vmaSize != 47) {
...	@@ -267,7 +364,22 @@ void InitializePlatformEarly() {	...	@@ -267,7 +364,22 @@ void InitializePlatformEarly() {
267	Die();	364	Die();
268	}	365	}
269	# endif	366	# endif
270	#endif	367	# elif SANITIZER_RISCV64
		368	// the bottom half of vma is allocated for userspace
		369	vmaSize = vmaSize + 1;
		370	# if !SANITIZER_GO
		371	if (vmaSize != 39 && vmaSize != 48) {
		372	Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
		373	Printf("FATAL: Found %zd - Supported 39 and 48\n", vmaSize);
		374	Die();
		375	}
		376	# endif
		377	# endif
		378
		379	# if !SANITIZER_GO
		380	// Heap has not been allocated yet
		381	ReExecIfNeeded(false);
		382	# endif
271	}	383	}
272		384
273	void InitializePlatform() {	385	void InitializePlatform() {
...	@@ -278,52 +390,34 @@ void InitializePlatform() {	...	@@ -278,52 +390,34 @@ void InitializePlatform() {
278	// is not compiled with -pie.	390	// is not compiled with -pie.
279	#if !SANITIZER_GO	391	#if !SANITIZER_GO
280	{	392	{
281	bool reexec = false;	393	# if SANITIZER_LINUX && (defined(__aarch64__) \|\| defined(__loongarch_lp64))
282	// TSan doesn't play well with unlimited stack size (as stack
283	// overlaps with shadow memory). If we detect unlimited stack size,
284	// we re-exec the program with limited stack size as a best effort.
285	if (StackSizeIsUnlimited()) {
286	const uptr kMaxStackSize = 32 * 1024 * 1024;
287	VReport(1, "Program is run with unlimited stack size, which wouldn't "
288	"work with ThreadSanitizer.\n"
289	"Re-execing with stack size limited to %zd bytes.\n",
290	kMaxStackSize);
291	SetStackSizeLimitInBytes(kMaxStackSize);
292	reexec = true;
293	}
294
295	if (!AddressSpaceIsUnlimited()) {
296	Report("WARNING: Program is run with limited virtual address space,"
297	" which wouldn't work with ThreadSanitizer.\n");
298	Report("Re-execing with unlimited virtual address space.\n");
299	SetAddressSpaceUnlimited();
300	reexec = true;
301	}
302	#if SANITIZER_ANDROID && (defined(__aarch64__) \|\| defined(__x86_64__))
303	// After patch "arm64: mm: support ARCH_MMAP_RND_BITS." is introduced in
304	// linux kernel, the random gap between stack and mapped area is increased
305	// from 128M to 36G on 39-bit aarch64. As it is almost impossible to cover
306	// this big range, we should disable randomized virtual space on aarch64.
307	// ASLR personality check.
308	int old_personality = personality(0xffffffff);
309	if (old_personality != -1 && (old_personality & ADDR_NO_RANDOMIZE) == 0) {
310	VReport(1, "WARNING: Program is run with randomized virtual address "
311	"space, which wouldn't work with ThreadSanitizer.\n"
312	"Re-execing with fixed virtual address space.\n");
313	CHECK_NE(personality(old_personality \| ADDR_NO_RANDOMIZE), -1);
314	reexec = true;
315	}
316
317	#endif
318	#if SANITIZER_LINUX && (defined(__aarch64__) \|\| defined(__loongarch_lp64))
319	// Initialize the xor key used in {sig}{set,long}jump.	394	// Initialize the xor key used in {sig}{set,long}jump.
320	InitializeLongjmpXorKey();	395	InitializeLongjmpXorKey();
321	#endif	396	# endif
322	if (reexec)	397	}
323	ReExec();	398
		399	// We called ReExecIfNeeded() in InitializePlatformEarly(), but there are
		400	// intervening allocations that result in an edge case:
		401	// 1) InitializePlatformEarly(): memory layout is compatible
		402	// 2) Intervening allocations happen
		403	// 3) InitializePlatform(): memory layout is incompatible and fails
		404	// CheckAndProtect()
		405	# if !SANITIZER_GO
		406	// Heap has already been allocated
		407	ReExecIfNeeded(true);
		408	# endif
		409
		410	// Earlier initialization steps already re-exec'ed until we got a compatible
		411	// memory layout, so we don't expect any more issues here.
		412	if (!CheckAndProtect(true, true, true)) {
		413	Printf(
		414	"FATAL: ThreadSanitizer: unexpectedly found incompatible memory "
		415	"layout.\n");
		416	Printf("FATAL: Please file a bug.\n");
		417	DumpProcessMap();
		418	Die();
324	}	419	}
325		420
326	CheckAndProtect();
327	InitTlsSize();	421	InitTlsSize();
328	#endif // !SANITIZER_GO	422	#endif // !SANITIZER_GO
329	}	423	}
...	@@ -399,13 +493,15 @@ static uptr UnmangleLongJmpSp(uptr mangled_sp) {	...	@@ -399,13 +493,15 @@ static uptr UnmangleLongJmpSp(uptr mangled_sp) {
399	return mangled_sp ^ xor_key;	493	return mangled_sp ^ xor_key;
400	#elif defined(__mips__)	494	#elif defined(__mips__)
401	return mangled_sp;	495	return mangled_sp;
402	#elif defined(__s390x__)	496	# elif SANITIZER_RISCV64
		497	return mangled_sp;
		498	# elif defined(__s390x__)
403	// tcbhead_t.stack_guard	499	// tcbhead_t.stack_guard
404	uptr xor_key = ((uptr *)__builtin_thread_pointer())[5];	500	uptr xor_key = ((uptr *)__builtin_thread_pointer())[5];
405	return mangled_sp ^ xor_key;	501	return mangled_sp ^ xor_key;
406	#else	502	# else
407	#error "Unknown platform"	503	# error "Unknown platform"
408	#endif	504	# endif
409	}	505	}
410		506
411	#if SANITIZER_NETBSD	507	#if SANITIZER_NETBSD
...	@@ -429,11 +525,13 @@ static uptr UnmangleLongJmpSp(uptr mangled_sp) {	...	@@ -429,11 +525,13 @@ static uptr UnmangleLongJmpSp(uptr mangled_sp) {
429	# define LONG_JMP_SP_ENV_SLOT 1	525	# define LONG_JMP_SP_ENV_SLOT 1
430	# elif defined(__mips64)	526	# elif defined(__mips64)
431	# define LONG_JMP_SP_ENV_SLOT 1	527	# define LONG_JMP_SP_ENV_SLOT 1
432	# elif defined(__s390x__)	528	# elif SANITIZER_RISCV64
433	# define LONG_JMP_SP_ENV_SLOT 9	529	# define LONG_JMP_SP_ENV_SLOT 13
434	# else	530	# elif defined(__s390x__)
435	# define LONG_JMP_SP_ENV_SLOT 6	531	# define LONG_JMP_SP_ENV_SLOT 9
436	# endif	532	# else
		533	# define LONG_JMP_SP_ENV_SLOT 6
		534	# endif
437	#endif	535	#endif
438		536
439	uptr ExtractLongJmpSp(uptr *env) {	537	uptr ExtractLongJmpSp(uptr *env) {

lib/tsan/tsan_platform_mac.cpp+6-3

 @@ -46,8 +46,8 @@
 namespace __tsan {
 #if !SANITIZER_GO
-static char main_thread_state[sizeof(ThreadState)] ALIGNED(
+alignas(SANITIZER_CACHE_LINE_SIZE) static char main_thread_state[sizeof(
-    SANITIZER_CACHE_LINE_SIZE);
+    ThreadState)];
 static ThreadState *dead_thread_state;
 static pthread_key_t thread_state_key;
 @@ -239,7 +239,10 @@ static uptr longjmp_xor_key = 0;
 void InitializePlatform() {
   DisableCoreDumperIfNecessary();
 #if !SANITIZER_GO
-  CheckAndProtect();
+  if (!CheckAndProtect(true, true, true)) {
+    Printf("FATAL: ThreadSanitizer: found incompatible memory layout.\n");
+    Die();
+  }
   InitializeThreadStateStorage();

lib/tsan/tsan_platform_posix.cpp+37-6

...	@@ -94,22 +94,51 @@ static void ProtectRange(uptr beg, uptr end) {	...	@@ -94,22 +94,51 @@ static void ProtectRange(uptr beg, uptr end) {
94	}	94	}
95	}	95	}
96		96
97	void CheckAndProtect() {	97	// CheckAndProtect will check if the memory layout is compatible with TSan.
		98	// Optionally (if 'protect' is true), it will set the memory regions between
		99	// app memory to be inaccessible.
		100	// 'ignore_heap' means it will not consider heap memory allocations to be a
		101	// conflict. Set this based on whether we are calling CheckAndProtect before
		102	// or after the allocator has initialized the heap.
		103	bool CheckAndProtect(bool protect, bool ignore_heap, bool print_warnings) {
98	// Ensure that the binary is indeed compiled with -pie.	104	// Ensure that the binary is indeed compiled with -pie.
99	MemoryMappingLayout proc_maps(true);	105	MemoryMappingLayout proc_maps(true);
100	MemoryMappedSegment segment;	106	MemoryMappedSegment segment;
101	while (proc_maps.Next(&segment)) {	107	while (proc_maps.Next(&segment)) {
102	if (IsAppMem(segment.start)) continue;	108	if (segment.start >= HeapMemBeg() && segment.end <= HeapEnd()) {
		109	if (ignore_heap) {
		110	continue;
		111	} else {
		112	return false;
		113	}
		114	}
		115
		116	// Note: IsAppMem includes if it is heap memory, hence we must
		117	// put this check after the heap bounds check.
		118	if (IsAppMem(segment.start) && IsAppMem(segment.end - 1))
		119	continue;
		120
		121	// Guard page after the heap end
103	if (segment.start >= HeapMemEnd() && segment.start < HeapEnd()) continue;	122	if (segment.start >= HeapMemEnd() && segment.start < HeapEnd()) continue;
		123
104	if (segment.protection == 0) // Zero page or mprotected.	124	if (segment.protection == 0) // Zero page or mprotected.
105	continue;	125	continue;
		126
106	if (segment.start >= VdsoBeg()) // vdso	127	if (segment.start >= VdsoBeg()) // vdso
107	break;	128	break;
108	Printf("FATAL: ThreadSanitizer: unexpected memory mapping 0x%zx-0x%zx\n",	129
109	segment.start, segment.end);	130	// Debug output can break tests. Suppress this message in most cases.
110	Die();	131	if (print_warnings)
		132	Printf(
		133	"WARNING: ThreadSanitizer: unexpected memory mapping 0x%zx-0x%zx\n",
		134	segment.start, segment.end);
		135
		136	return false;
111	}	137	}
112		138
		139	if (!protect)
		140	return true;
		141
113	# if SANITIZER_IOS && !SANITIZER_IOSSIM	142	# if SANITIZER_IOS && !SANITIZER_IOSSIM
114	ProtectRange(HeapMemEnd(), ShadowBeg());	143	ProtectRange(HeapMemEnd(), ShadowBeg());
115	ProtectRange(ShadowEnd(), MetaShadowBeg());	144	ProtectRange(ShadowEnd(), MetaShadowBeg());
...	@@ -135,8 +164,10 @@ void CheckAndProtect() {	...	@@ -135,8 +164,10 @@ void CheckAndProtect() {
135	// Older s390x kernels may not support 5-level page tables.	164	// Older s390x kernels may not support 5-level page tables.
136	TryProtectRange(user_addr_max_l4, user_addr_max_l5);	165	TryProtectRange(user_addr_max_l4, user_addr_max_l5);
137	#endif	166	#endif
		167
		168	return true;
138	}	169	}
139	#endif	170	# endif
140		171
141	} // namespace __tsan	172	} // namespace __tsan
142		173

lib/tsan/tsan_preinit.cpp+4-6

...	@@ -16,11 +16,9 @@	...	@@ -16,11 +16,9 @@
16		16
17	#if SANITIZER_CAN_USE_PREINIT_ARRAY	17	#if SANITIZER_CAN_USE_PREINIT_ARRAY
18		18
19	// The symbol is called __local_tsan_preinit, because it's not intended to be	19	// This section is linked into the main executable when -fsanitize=thread is
20	// exported.	20	// specified to perform initialization at a very early stage.
21	// This code linked into the main executable when -fsanitize=thread is in	21	__attribute__((section(".preinit_array"), used)) static auto preinit =
22	// the link flags. It can only use exported interface functions.	22	__tsan_init;
23	__attribute__((section(".preinit_array"), used))
24	void (*__local_tsan_preinit)(void) = __tsan_init;
25		23
26	#endif	24	#endif

lib/tsan/tsan_report.cpp+12-26

 @@ -93,7 +93,9 @@ static const char *ReportTypeString(ReportType typ, uptr tag) {
       return "signal handler spoils errno";
     case ReportTypeDeadlock:
       return "lock-order-inversion (potential deadlock)";
-    // No default case so compiler warns us if we miss one
+    case ReportTypeMutexHeldWrongContext:
+      return "mutex held in the wrong context";
+      // No default case so compiler warns us if we miss one
   }
   UNREACHABLE("missing case");
 }
 @@ -106,10 +108,10 @@ void PrintStack(const ReportStack *ent) {
   SymbolizedStack *frame = ent->frames;
   for (int i = 0; frame && frame->info.address; frame = frame->next, i++) {
     InternalScopedString res;
-    RenderFrame(&res, common_flags()->stack_trace_format, i,
+    StackTracePrinter::GetOrInit()->RenderFrame(
-                frame->info.address, &frame->info,
+        &res, common_flags()->stack_trace_format, i, frame->info.address,
-                common_flags()->symbolize_vs_style,
+        &frame->info, common_flags()->symbolize_vs_style,
-                common_flags()->strip_path_prefix);
+        common_flags()->strip_path_prefix);
     Printf("%s\n", res.data());
   }
   Printf("\n");
 @@ -271,26 +273,10 @@ static ReportStack *ChooseSummaryStack(const ReportDesc *rep) {
   return 0;
 }
-static bool FrameIsInternal(const SymbolizedStack *frame) {
+static const SymbolizedStack *SkipTsanInternalFrames(SymbolizedStack *frames) {
-  if (frame == 0)
+  if (const SymbolizedStack *f = SkipInternalFrames(frames))
-    return false;
+    return f;
-  const char *file = frame->info.file;
+  return frames;  // Fallback to the top frame.
-  const char *module = frame->info.module;
-  if (file != 0 &&
-      (internal_strstr(file, "tsan_interceptors_posix.cpp") ||
-       internal_strstr(file, "tsan_interceptors_memintrinsics.cpp") ||
-       internal_strstr(file, "sanitizer_common_interceptors.inc") ||
-       internal_strstr(file, "tsan_interface_")))
-    return true;
-  if (module != 0 && (internal_strstr(module, "libclang_rt.tsan_")))
-    return true;
-  return false;
-}
-static SymbolizedStack *SkipTsanInternalFrames(SymbolizedStack *frames) {
-  while (FrameIsInternal(frames) && frames->next)
-    frames = frames->next;
-  return frames;
 }
 void PrintReport(const ReportDesc *rep) {
 @@ -364,7 +350,7 @@ void PrintReport(const ReportDesc *rep) {
     Printf("  And %d more similar thread leaks.\n\n", rep->count - 1);
   if (ReportStack *stack = ChooseSummaryStack(rep)) {
-    if (SymbolizedStack *frame = SkipTsanInternalFrames(stack->frames))
+    if (const SymbolizedStack *frame = SkipTsanInternalFrames(stack->frames))
       ReportErrorSummary(rep_typ_str, frame->info);
   }

lib/tsan/tsan_report.h+2-1

...	@@ -34,7 +34,8 @@ enum ReportType {	...	@@ -34,7 +34,8 @@ enum ReportType {
34	ReportTypeMutexBadReadUnlock,	34	ReportTypeMutexBadReadUnlock,
35	ReportTypeSignalUnsafe,	35	ReportTypeSignalUnsafe,
36	ReportTypeErrnoInSignal,	36	ReportTypeErrnoInSignal,
37	ReportTypeDeadlock	37	ReportTypeDeadlock,
		38	ReportTypeMutexHeldWrongContext
38	};	39	};
39		40
40	struct ReportStack {	41	struct ReportStack {

lib/tsan/tsan_rtl.cpp+14-10

 @@ -35,8 +35,10 @@ extern "C" void __tsan_resume() {
   __tsan_resumed = 1;
 }
+#if SANITIZER_APPLE
 SANITIZER_WEAK_DEFAULT_IMPL
 void __tsan_test_only_on_fork() {}
+#endif
 namespace __tsan {
 @@ -46,11 +48,10 @@ int (*on_finalize)(int);
 #endif
 #if !SANITIZER_GO && !SANITIZER_APPLE
-__attribute__((tls_model("initial-exec")))
+alignas(SANITIZER_CACHE_LINE_SIZE) THREADLOCAL __attribute__((tls_model(
-THREADLOCAL char cur_thread_placeholder[sizeof(ThreadState)] ALIGNED(
+    "initial-exec"))) char cur_thread_placeholder[sizeof(ThreadState)];
-    SANITIZER_CACHE_LINE_SIZE);
 #endif
-static char ctx_placeholder[sizeof(Context)] ALIGNED(SANITIZER_CACHE_LINE_SIZE);
+alignas(SANITIZER_CACHE_LINE_SIZE) static char ctx_placeholder[sizeof(Context)];
 Context *ctx;
 // Can be overriden by a front-end.
 @@ -446,7 +447,7 @@ static bool InitializeMemoryProfiler() {
     ctx->memprof_fd = 2;
   } else {
     InternalScopedString filename;
-    filename.append("%s.%d", fname, (int)internal_getpid());
+    filename.AppendF("%s.%d", fname, (int)internal_getpid());
     ctx->memprof_fd = OpenFile(filename.data(), WrOnly);
     if (ctx->memprof_fd == kInvalidFd) {
       Printf("ThreadSanitizer: failed to open memory profile file '%s'\n",
 @@ -813,7 +814,7 @@ void ForkBefore(ThreadState* thr, uptr pc) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
   ctx->thread_registry.Lock();
   ctx->slot_mtx.Lock();
   ScopedErrorReportLock::Lock();
-  AllocatorLock();
+  AllocatorLockBeforeFork();
   // Suppress all reports in the pthread_atfork callbacks.
   // Reports will deadlock on the report_mtx.
   // We could ignore sync operations as well,
 @@ -828,14 +829,17 @@ void ForkBefore(ThreadState* thr, uptr pc) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
   // Disables memory write in OnUserAlloc/Free.
   thr->ignore_reads_and_writes++;
+#  if SANITIZER_APPLE
   __tsan_test_only_on_fork();
+#  endif
 }
-static void ForkAfter(ThreadState* thr) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
+static void ForkAfter(ThreadState* thr,
+                      bool child) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
   thr->suppress_reports--;  // Enabled in ForkBefore.
   thr->ignore_interceptors--;
   thr->ignore_reads_and_writes--;
-  AllocatorUnlock();
+  AllocatorUnlockAfterFork(child);
   ScopedErrorReportLock::Unlock();
   ctx->slot_mtx.Unlock();
   ctx->thread_registry.Unlock();
 @@ -845,10 +849,10 @@ static void ForkAfter(ThreadState* thr) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
   GlobalProcessorUnlock();
 }
-void ForkParentAfter(ThreadState* thr, uptr pc) { ForkAfter(thr); }
+void ForkParentAfter(ThreadState* thr, uptr pc) { ForkAfter(thr, false); }
 void ForkChildAfter(ThreadState* thr, uptr pc, bool start_thread) {
-  ForkAfter(thr);
+  ForkAfter(thr, true);
   u32 nthread = ctx->thread_registry.OnFork(thr->tid);
   VPrintf(1,
           "ThreadSanitizer: forked new process with pid %d,"

lib/tsan/tsan_rtl.h+6-6

...	@@ -56,8 +56,8 @@ namespace __tsan {	...	@@ -56,8 +56,8 @@ namespace __tsan {
56		56
57	#if !SANITIZER_GO	57	#if !SANITIZER_GO
58	struct MapUnmapCallback;	58	struct MapUnmapCallback;
59	#if defined(__mips64) \|\| defined(__aarch64__) \|\| defined(__loongarch__) \|\| \	59	# if defined(__mips64) \|\| defined(__aarch64__) \|\| defined(__loongarch__) \|\| \
60	defined(__powerpc__)	60	defined(__powerpc__) \|\| SANITIZER_RISCV64
61		61
62	struct AP32 {	62	struct AP32 {
63	static const uptr kSpaceBeg = 0;	63	static const uptr kSpaceBeg = 0;
...	@@ -136,7 +136,7 @@ struct TidEpoch {	...	@@ -136,7 +136,7 @@ struct TidEpoch {
136	Epoch epoch;	136	Epoch epoch;
137	};	137	};
138		138
139	struct TidSlot {	139	struct alignas(SANITIZER_CACHE_LINE_SIZE) TidSlot {
140	Mutex mtx;	140	Mutex mtx;
141	Sid sid;	141	Sid sid;
142	atomic_uint32_t raw_epoch;	142	atomic_uint32_t raw_epoch;
...	@@ -153,10 +153,10 @@ struct TidSlot {	...	@@ -153,10 +153,10 @@ struct TidSlot {
153	}	153	}
154		154
155	TidSlot();	155	TidSlot();
156	} ALIGNED(SANITIZER_CACHE_LINE_SIZE);	156	};
157		157
158	// This struct is stored in TLS.	158	// This struct is stored in TLS.
159	struct ThreadState {	159	struct alignas(SANITIZER_CACHE_LINE_SIZE) ThreadState {
160	FastState fast_state;	160	FastState fast_state;
161	int ignore_sync;	161	int ignore_sync;
162	#if !SANITIZER_GO	162	#if !SANITIZER_GO
...	@@ -234,7 +234,7 @@ struct ThreadState {	...	@@ -234,7 +234,7 @@ struct ThreadState {
234	const ReportDesc *current_report;	234	const ReportDesc *current_report;
235		235
236	explicit ThreadState(Tid tid);	236	explicit ThreadState(Tid tid);
237	} ALIGNED(SANITIZER_CACHE_LINE_SIZE);	237	};
238		238
239	#if !SANITIZER_GO	239	#if !SANITIZER_GO
240	#if SANITIZER_APPLE \|\| SANITIZER_ANDROID	240	#if SANITIZER_APPLE \|\| SANITIZER_ANDROID

lib/tsan/tsan_rtl_aarch64.S+7

...	@@ -2,6 +2,7 @@	...	@@ -2,6 +2,7 @@
2	#if defined(__aarch64__)	2	#if defined(__aarch64__)
3		3
4	#include "sanitizer_common/sanitizer_asm.h"	4	#include "sanitizer_common/sanitizer_asm.h"
		5	#include "builtins/assembly.h"
5		6
6	#if !defined(__APPLE__)	7	#if !defined(__APPLE__)
7	.section .text	8	.section .text
...	@@ -16,6 +17,7 @@ ASM_HIDDEN(__tsan_setjmp)	...	@@ -16,6 +17,7 @@ ASM_HIDDEN(__tsan_setjmp)
16	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(setjmp))	17	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(setjmp))
17	ASM_SYMBOL_INTERCEPTOR(setjmp):	18	ASM_SYMBOL_INTERCEPTOR(setjmp):
18	CFI_STARTPROC	19	CFI_STARTPROC
		20	BTI_C
19		21
20	// Save frame/link register	22	// Save frame/link register
21	stp x29, x30, [sp, -32]!	23	stp x29, x30, [sp, -32]!
...	@@ -66,6 +68,7 @@ ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(setjmp))	...	@@ -66,6 +68,7 @@ ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(setjmp))
66	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(_setjmp))	68	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(_setjmp))
67	ASM_SYMBOL_INTERCEPTOR(_setjmp):	69	ASM_SYMBOL_INTERCEPTOR(_setjmp):
68	CFI_STARTPROC	70	CFI_STARTPROC
		71	BTI_C
69		72
70	// Save frame/link register	73	// Save frame/link register
71	stp x29, x30, [sp, -32]!	74	stp x29, x30, [sp, -32]!
...	@@ -116,6 +119,7 @@ ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(_setjmp))	...	@@ -116,6 +119,7 @@ ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(_setjmp))
116	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))	119	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))
117	ASM_SYMBOL_INTERCEPTOR(sigsetjmp):	120	ASM_SYMBOL_INTERCEPTOR(sigsetjmp):
118	CFI_STARTPROC	121	CFI_STARTPROC
		122	BTI_C
119		123
120	// Save frame/link register	124	// Save frame/link register
121	stp x29, x30, [sp, -32]!	125	stp x29, x30, [sp, -32]!
...	@@ -168,6 +172,7 @@ ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))	...	@@ -168,6 +172,7 @@ ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))
168	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))	172	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))
169	ASM_SYMBOL_INTERCEPTOR(__sigsetjmp):	173	ASM_SYMBOL_INTERCEPTOR(__sigsetjmp):
170	CFI_STARTPROC	174	CFI_STARTPROC
		175	BTI_C
171		176
172	// Save frame/link register	177	// Save frame/link register
173	stp x29, x30, [sp, -32]!	178	stp x29, x30, [sp, -32]!
...	@@ -217,4 +222,6 @@ ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))	...	@@ -217,4 +222,6 @@ ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))
217		222
218	NO_EXEC_STACK_DIRECTIVE	223	NO_EXEC_STACK_DIRECTIVE
219		224
		225	GNU_PROPERTY_BTI_PAC
		226
220	#endif	227	#endif

lib/tsan/tsan_rtl_access.cpp+14-8

...	@@ -672,22 +672,28 @@ void MemoryAccessRangeT(ThreadState* thr, uptr pc, uptr addr, uptr size) {	...	@@ -672,22 +672,28 @@ void MemoryAccessRangeT(ThreadState* thr, uptr pc, uptr addr, uptr size) {
672		672
673	#if SANITIZER_DEBUG	673	#if SANITIZER_DEBUG
674	if (!IsAppMem(addr)) {	674	if (!IsAppMem(addr)) {
675	Printf("Access to non app mem %zx\n", addr);	675	Printf("Access to non app mem start: %p\n", (void*)addr);
676	DCHECK(IsAppMem(addr));	676	DCHECK(IsAppMem(addr));
677	}	677	}
678	if (!IsAppMem(addr + size - 1)) {	678	if (!IsAppMem(addr + size - 1)) {
679	Printf("Access to non app mem %zx\n", addr + size - 1);	679	Printf("Access to non app mem end: %p\n", (void*)(addr + size - 1));
680	DCHECK(IsAppMem(addr + size - 1));	680	DCHECK(IsAppMem(addr + size - 1));
681	}	681	}
682	if (!IsShadowMem(shadow_mem)) {	682	if (!IsShadowMem(shadow_mem)) {
683	Printf("Bad shadow addr %p (%zx)\n", static_cast<void*>(shadow_mem), addr);	683	Printf("Bad shadow start addr: %p (%p)\n", shadow_mem, (void*)addr);
684	DCHECK(IsShadowMem(shadow_mem));	684	DCHECK(IsShadowMem(shadow_mem));
685	}	685	}
686	if (!IsShadowMem(shadow_mem + size * kShadowCnt - 1)) {	686
687	Printf("Bad shadow addr %p (%zx)\n",	687	RawShadow* shadow_mem_end = reinterpret_cast<RawShadow*>(
688	static_cast<void>(shadow_mem + size kShadowCnt - 1),	688	reinterpret_cast<uptr>(shadow_mem) + size * kShadowMultiplier - 1);
689	addr + size - 1);	689	if (!IsShadowMem(shadow_mem_end)) {
690	DCHECK(IsShadowMem(shadow_mem + size * kShadowCnt - 1));	690	Printf("Bad shadow end addr: %p (%p)\n", shadow_mem_end,
		691	(void*)(addr + size - 1));
		692	Printf(
		693	"Shadow start addr (ok): %p (%p); size: 0x%zx; kShadowMultiplier: "
		694	"%zx\n",
		695	shadow_mem, (void*)addr, size, kShadowMultiplier);
		696	DCHECK(IsShadowMem(shadow_mem_end));
691	}	697	}
692	#endif	698	#endif
693		699

lib/tsan/tsan_rtl_loongarch64.S created+196

...	@@ -0,0 +1,196 @@
1	#include "sanitizer_common/sanitizer_asm.h"
2
3	.section .text
4
5	ASM_HIDDEN(__tsan_setjmp)
6	.comm _ZN14__interception11real_setjmpE,8,8
7	.globl ASM_SYMBOL_INTERCEPTOR(setjmp)
8	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(setjmp))
9	ASM_SYMBOL_INTERCEPTOR(setjmp):
10	CFI_STARTPROC
11
12	// Save frame pointer and return address register
13	addi.d $sp, $sp, -32
14	st.d $ra, $sp, 24
15	st.d $fp, $sp, 16
16	CFI_DEF_CFA_OFFSET (32)
17	CFI_OFFSET (1, -8)
18	CFI_OFFSET (22, -16)
19
20	// Adjust the SP for previous frame
21	addi.d $fp, $sp, 32
22	CFI_DEF_CFA_REGISTER (22)
23
24	// Save env parameter
25	st.d $a0, $sp, 8
26	CFI_OFFSET (4, -24)
27
28	// Obtain SP, first argument to `void __tsan_setjmp(uptr sp)`
29	addi.d $a0, $fp, 0
30
31	// call tsan interceptor
32	bl ASM_SYMBOL(__tsan_setjmp)
33
34	// Restore env parameter
35	ld.d $a0, $sp, 8
36	CFI_RESTORE (4)
37
38	// Restore frame/link register
39	ld.d $fp, $sp, 16
40	ld.d $ra, $sp, 24
41	addi.d $sp, $sp, 32
42	CFI_RESTORE (22)
43	CFI_RESTORE (1)
44	CFI_DEF_CFA (3, 0)
45
46	// tail jump to libc setjmp
47	la.local $a1, _ZN14__interception11real_setjmpE
48	ld.d $a1, $a1, 0
49	jr $a1
50
51	CFI_ENDPROC
52	ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(setjmp))
53
54	.comm _ZN14__interception12real__setjmpE,8,8
55	.globl ASM_SYMBOL_INTERCEPTOR(_setjmp)
56	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(_setjmp))
57	ASM_SYMBOL_INTERCEPTOR(_setjmp):
58	CFI_STARTPROC
59
60	// Save frame pointer and return address register
61	addi.d $sp, $sp, -32
62	st.d $ra, $sp, 24
63	st.d $fp, $sp, 16
64	CFI_DEF_CFA_OFFSET (32)
65	CFI_OFFSET (1, -8)
66	CFI_OFFSET (22, -16)
67
68	// Adjust the SP for previous frame
69	addi.d $fp, $sp, 32
70	CFI_DEF_CFA_REGISTER (22)
71
72	// Save env parameter
73	st.d $a0, $sp, 8
74	CFI_OFFSET (4, -24)
75
76	// Obtain SP, first argument to `void __tsan_setjmp(uptr sp)`
77	addi.d $a0, $fp, 0
78
79	// call tsan interceptor
80	bl ASM_SYMBOL(__tsan_setjmp)
81
82	// Restore env parameter
83	ld.d $a0, $sp, 8
84	CFI_RESTORE (4)
85
86	// Restore frame/link register
87	ld.d $fp, $sp, 16
88	ld.d $ra, $sp, 24
89	addi.d $sp, $sp, 32
90	CFI_RESTORE (22)
91	CFI_RESTORE (1)
92	CFI_DEF_CFA (3, 0)
93
94	// tail jump to libc setjmp
95	la.local $a1, _ZN14__interception12real__setjmpE
96	ld.d $a1, $a1, 0
97	jr $a1
98
99	CFI_ENDPROC
100	ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(_setjmp))
101
102	.comm _ZN14__interception14real_sigsetjmpE,8,8
103	.globl ASM_SYMBOL_INTERCEPTOR(sigsetjmp)
104	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))
105	ASM_SYMBOL_INTERCEPTOR(sigsetjmp):
106	CFI_STARTPROC
107
108	// Save frame pointer and return address register
109	addi.d $sp, $sp, -32
110	st.d $ra, $sp, 24
111	st.d $fp, $sp, 16
112	CFI_DEF_CFA_OFFSET (32)
113	CFI_OFFSET (1, -8)
114	CFI_OFFSET (22, -16)
115
116	// Adjust the SP for previous frame
117	addi.d $fp, $sp, 32
118	CFI_DEF_CFA_REGISTER (22)
119
120	// Save env parameter
121	st.d $a0, $sp, 8
122	CFI_OFFSET (4, -24)
123
124	// Obtain SP, first argument to `void __tsan_setjmp(uptr sp)`
125	addi.d $a0, $fp, 0
126
127	// call tsan interceptor
128	bl ASM_SYMBOL(__tsan_setjmp)
129
130	// Restore env parameter
131	ld.d $a0, $sp, 8
132	CFI_RESTORE (4)
133
134	// Restore frame/link register
135	ld.d $fp, $sp, 16
136	ld.d $ra, $sp, 24
137	addi.d $sp, $sp, 32
138	CFI_RESTORE (22)
139	CFI_RESTORE (1)
140	CFI_DEF_CFA (3, 0)
141
142	// tail jump to libc setjmp
143	la.local $a1, _ZN14__interception14real_sigsetjmpE
144	ld.d $a1, $a1, 0
145	jr $a1
146
147	CFI_ENDPROC
148	ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))
149
150	.comm _ZN14__interception16real___sigsetjmpE,8,8
151	.globl ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)
152	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))
153	ASM_SYMBOL_INTERCEPTOR(__sigsetjmp):
154	CFI_STARTPROC
155
156	// Save frame pointer and return address register
157	addi.d $sp, $sp, -32
158	st.d $ra, $sp, 24
159	st.d $fp, $sp, 16
160	CFI_DEF_CFA_OFFSET (32)
161	CFI_OFFSET (1, -8)
162	CFI_OFFSET (22, -16)
163
164	// Adjust the SP for previous frame
165	addi.d $fp, $sp, 32
166	CFI_DEF_CFA_REGISTER (22)
167
168	// Save env parameter
169	st.d $a0, $sp, 8
170	CFI_OFFSET (4, -24)
171
172	// Obtain SP, first argument to `void __tsan_setjmp(uptr sp)`
173	addi.d $a0, $fp, 0
174
175	// call tsan interceptor
176	bl ASM_SYMBOL(__tsan_setjmp)
177
178	// Restore env parameter
179	ld.d $a0, $sp, 8
180	CFI_RESTORE (4)
181
182	// Restore frame/link register
183	ld.d $fp, $sp, 16
184	ld.d $ra, $sp, 24
185	addi.d $sp, $sp, 32
186	CFI_RESTORE (22)
187	CFI_RESTORE (1)
188	CFI_DEF_CFA (3, 0)
189
190	// tail jump to libc setjmp
191	la.local $a1, _ZN14__interception16real___sigsetjmpE
192	ld.d $a1, $a1, 0
193	jr $a1
194
195	CFI_ENDPROC
196	ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))

lib/tsan/tsan_rtl_mutex.cpp+1-1

 @@ -446,9 +446,9 @@ void Acquire(ThreadState *thr, uptr pc, uptr addr) {
   if (!s)
     return;
   SlotLocker locker(thr);
+  ReadLock lock(&s->mtx);
   if (!s->clock)
     return;
-  ReadLock lock(&s->mtx);
   thr->clock.Acquire(s->clock);
 }

lib/tsan/tsan_rtl_riscv64.S created+203

...	@@ -0,0 +1,203 @@
1	#include "sanitizer_common/sanitizer_asm.h"
2
3	.section .text
4
5	.comm _ZN14__interception11real_setjmpE,8,8
6	.globl ASM_SYMBOL_INTERCEPTOR(setjmp)
7	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(setjmp))
8	ASM_SYMBOL_INTERCEPTOR(setjmp):
9	CFI_STARTPROC
10
11	// Save frame pointer and return address register
12	addi sp, sp, -32
13	sd ra, 24(sp)
14	sd s0, 16(sp)
15	CFI_DEF_CFA_OFFSET (32)
16	CFI_OFFSET (1, -8)
17	CFI_OFFSET (8, -16)
18
19	// Adjust the SP for previous frame
20	addi s0, sp, 32
21	CFI_DEF_CFA_REGISTER (8)
22
23	// Save env parameter
24	sd a0, 8(sp)
25	CFI_OFFSET (10, -24)
26
27	// Obtain SP, first argument to `void __tsan_setjmp(uptr sp)`
28	addi a0, s0, 0
29
30	// call tsan interceptor
31	call ASM_SYMBOL(__tsan_setjmp)
32
33	// Restore env parameter
34	ld a0, 8(sp)
35	CFI_RESTORE (10)
36
37	// Restore frame/link register
38	ld s0, 16(sp)
39	ld ra, 24(sp)
40	addi sp, sp, 32
41	CFI_RESTORE (8)
42	CFI_RESTORE (1)
43	CFI_DEF_CFA (2, 0)
44
45	// tail jump to libc setjmp
46	la t1, _ZN14__interception11real_setjmpE
47	ld t1, 0(t1)
48	jr t1
49
50	CFI_ENDPROC
51	ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(setjmp))
52
53	.comm _ZN14__interception12real__setjmpE,8,8
54	.globl ASM_SYMBOL_INTERCEPTOR(_setjmp)
55	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(_setjmp))
56	ASM_SYMBOL_INTERCEPTOR(_setjmp):
57	CFI_STARTPROC
58
59	// Save frame pointer and return address register
60	addi sp, sp, -32
61	sd ra, 24(sp)
62	sd s0, 16(sp)
63	CFI_DEF_CFA_OFFSET (32)
64	CFI_OFFSET (1, -8)
65	CFI_OFFSET (8, -16)
66
67	// Adjust the SP for previous frame
68	addi s0, sp, 32
69	CFI_DEF_CFA_REGISTER (8)
70
71	// Save env parameter
72	sd a0, 8(sp)
73	CFI_OFFSET (10, -24)
74
75	// Obtain SP, first argument to `void __tsan_setjmp(uptr sp)`
76	addi a0, s0, 0
77
78	// call tsan interceptor
79	call ASM_SYMBOL(__tsan_setjmp)
80
81	// Restore env parameter
82	ld a0, 8(sp)
83	CFI_RESTORE (10)
84
85	// Restore frame/link register
86	ld s0, 16(sp)
87	ld ra, 24(sp)
88	addi sp, sp, 32
89	CFI_RESTORE (8)
90	CFI_RESTORE (1)
91	CFI_DEF_CFA (2, 0)
92
93	// tail jump to libc setjmp
94	la t1, _ZN14__interception12real__setjmpE
95	ld t1, 0(t1)
96	jr t1
97
98	CFI_ENDPROC
99	ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(_setjmp))
100
101	.comm _ZN14__interception14real_sigsetjmpE,8,8
102	.globl ASM_SYMBOL_INTERCEPTOR(sigsetjmp)
103	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))
104	ASM_SYMBOL_INTERCEPTOR(sigsetjmp):
105	CFI_STARTPROC
106
107	// Save frame pointer and return address register
108	addi sp, sp, -32
109	sd ra, 24(sp)
110	sd s0, 16(sp)
111	CFI_DEF_CFA_OFFSET (32)
112	CFI_OFFSET (1, -8)
113	CFI_OFFSET (8, -16)
114
115	// Adjust the SP for previous frame
116	addi s0, sp, 32
117	CFI_DEF_CFA_REGISTER (8)
118
119	// Save env parameter
120	sd a0, 8(sp)
121	sd a1, 0(sp)
122	CFI_OFFSET (10, -24)
123	CFI_OFFSET (11, -32)
124
125	// Obtain SP, first argument to `void __tsan_setjmp(uptr sp)`
126	addi a0, s0, 0
127
128	// call tsan interceptor
129	call ASM_SYMBOL(__tsan_setjmp)
130
131	// Restore env parameter
132	ld a0, 8(sp)
133	ld a1, 0(sp)
134	CFI_RESTORE (10)
135	CFI_RESTORE (11)
136
137	// Restore frame/link register
138	ld s0, 16(sp)
139	ld ra, 24(sp)
140	addi sp, sp, 32
141	CFI_RESTORE (8)
142	CFI_RESTORE (1)
143	CFI_DEF_CFA (2, 0)
144
145	// tail jump to libc setjmp
146	la t1, _ZN14__interception14real_sigsetjmpE
147	ld t1, 0(t1)
148	jr t1
149
150	CFI_ENDPROC
151	ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))
152
153	.comm _ZN14__interception16real___sigsetjmpE,8,8
154	.globl ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)
155	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))
156	ASM_SYMBOL_INTERCEPTOR(__sigsetjmp):
157	CFI_STARTPROC
158
159	// Save frame pointer and return address register
160	addi sp, sp, -32
161	sd ra, 24(sp)
162	sd s0, 16(sp)
163	CFI_DEF_CFA_OFFSET (32)
164	CFI_OFFSET (1, -8)
165	CFI_OFFSET (8, -16)
166
167	// Adjust the SP for previous frame
168	addi s0, sp, 32
169	CFI_DEF_CFA_REGISTER (8)
170
171	// Save env parameter
172	sd a0, 8(sp)
173	sd a1, 0(sp)
174	CFI_OFFSET (10, -24)
175	CFI_OFFSET (11, -32)
176
177	// Obtain SP, first argument to `void __tsan_setjmp(uptr sp)`
178	addi a0, s0, 0
179
180	// call tsan interceptor
181	call ASM_SYMBOL(__tsan_setjmp)
182
183	// Restore env parameter
184	ld a0, 8(sp)
185	ld a1, 0(sp)
186	CFI_RESTORE (10)
187	CFI_RESTORE (11)
188
189	// Restore frame/link register
190	ld s0, 16(sp)
191	ld ra, 24(sp)
192	addi sp, sp, 32
193	CFI_RESTORE (8)
194	CFI_RESTORE (1)
195	CFI_DEF_CFA (2, 0)
196
197	// tail jump to libc setjmp
198	la t1, _ZN14__interception16real___sigsetjmpE
199	ld t1, 0(t1)
200	jr t1
201
202	CFI_ENDPROC
203	ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))

lib/tsan/tsan_rtl_s390x.S created+49

...	@@ -0,0 +1,49 @@
1	#include "sanitizer_common/sanitizer_asm.h"
2
3	#define CFA_OFFSET 160
4	#define R2_REL_OFFSET 16
5	#define R3_REL_OFFSET 24
6	#define R14_REL_OFFSET 112
7	#define R15_REL_OFFSET 120
8	#define FRAME_SIZE 160
9
10	.text
11
12	ASM_HIDDEN(__tsan_setjmp)
13
14	.macro intercept symbol, real
15	.comm \real, 8, 8
16	.globl ASM_SYMBOL_INTERCEPTOR(\symbol)
17	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(\symbol))
18	ASM_SYMBOL_INTERCEPTOR(\symbol):
19	CFI_STARTPROC
20	stmg %r2, %r3, R2_REL_OFFSET(%r15)
21	CFI_REL_OFFSET(%r2, R2_REL_OFFSET)
22	CFI_REL_OFFSET(%r3, R3_REL_OFFSET)
23	stmg %r14, %r15, R14_REL_OFFSET(%r15)
24	CFI_REL_OFFSET(%r14, R14_REL_OFFSET)
25	CFI_REL_OFFSET(%r15, R15_REL_OFFSET)
26	aghi %r15, -FRAME_SIZE
27	CFI_ADJUST_CFA_OFFSET(FRAME_SIZE)
28	la %r2, FRAME_SIZE(%r15)
29	brasl %r14, ASM_SYMBOL(__tsan_setjmp)
30	lmg %r14, %r15, FRAME_SIZE + R14_REL_OFFSET(%r15)
31	CFI_RESTORE(%r14)
32	CFI_RESTORE(%r15)
33	CFI_DEF_CFA_OFFSET(CFA_OFFSET)
34	lmg %r2, %r3, R2_REL_OFFSET(%r15)
35	CFI_RESTORE(%r2)
36	CFI_RESTORE(%r3)
37	larl %r1, \real
38	lg %r1, 0(%r1)
39	br %r1
40	CFI_ENDPROC
41	ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(\symbol))
42	.endm
43
44	intercept setjmp, _ZN14__interception11real_setjmpE
45	intercept _setjmp, _ZN14__interception12real__setjmpE
46	intercept sigsetjmp, _ZN14__interception14real_sigsetjmpE
47	intercept __sigsetjmp, _ZN14__interception16real___sigsetjmpE
48
49	NO_EXEC_STACK_DIRECTIVE

lib/tsan/tsan_rtl_thread.cpp+5-5

 @@ -160,6 +160,10 @@ void ThreadStart(ThreadState *thr, Tid tid, tid_t os_id,
   }
   Free(thr->tctx->sync);
+#if !SANITIZER_GO
+  thr->is_inited = true;
+#endif
   uptr stk_addr = 0;
   uptr stk_size = 0;
   uptr tls_addr = 0;
 @@ -200,15 +204,11 @@ void ThreadStart(ThreadState *thr, Tid tid, tid_t os_id,
 }
 void ThreadContext::OnStarted(void *arg) {
-  thr = static_cast<ThreadState *>(arg);
   DPrintf("#%d: ThreadStart\n", tid);
-  new (thr) ThreadState(tid);
+  thr = new (arg) ThreadState(tid);
   if (common_flags()->detect_deadlocks)
     thr->dd_lt = ctx->dd->CreateLogicalThread(tid);
   thr->tctx = this;
-#if !SANITIZER_GO
-  thr->is_inited = true;
-#endif
 }
 void ThreadFinish(ThreadState *thr) {

lib/tsan/tsan_suppressions.cpp+2-1

...	@@ -42,7 +42,7 @@ const char *__tsan_default_suppressions() {	...	@@ -42,7 +42,7 @@ const char *__tsan_default_suppressions() {
42		42
43	namespace __tsan {	43	namespace __tsan {
44		44
45	ALIGNED(64) static char suppression_placeholder[sizeof(SuppressionContext)];	45	alignas(64) static char suppression_placeholder[sizeof(SuppressionContext)];
46	static SuppressionContext *suppression_ctx = nullptr;	46	static SuppressionContext *suppression_ctx = nullptr;
47	static const char *kSuppressionTypes[] = {	47	static const char *kSuppressionTypes[] = {
48	kSuppressionRace, kSuppressionRaceTop, kSuppressionMutex,	48	kSuppressionRace, kSuppressionRaceTop, kSuppressionMutex,
...	@@ -81,6 +81,7 @@ static const char *conv(ReportType typ) {	...	@@ -81,6 +81,7 @@ static const char *conv(ReportType typ) {
81	case ReportTypeMutexBadUnlock:	81	case ReportTypeMutexBadUnlock:
82	case ReportTypeMutexBadReadLock:	82	case ReportTypeMutexBadReadLock:
83	case ReportTypeMutexBadReadUnlock:	83	case ReportTypeMutexBadReadUnlock:
		84	case ReportTypeMutexHeldWrongContext:
84	return kSuppressionMutex;	85	return kSuppressionMutex;
85	case ReportTypeSignalUnsafe:	86	case ReportTypeSignalUnsafe:
86	case ReportTypeErrnoInSignal:	87	case ReportTypeErrnoInSignal:

lib/tsan/tsan_vector_clock.h+1-1

...	@@ -34,7 +34,7 @@ class VectorClock {	...	@@ -34,7 +34,7 @@ class VectorClock {
34	VectorClock& operator=(const VectorClock& other);	34	VectorClock& operator=(const VectorClock& other);
35		35
36	private:	36	private:
37	Epoch clk_[kThreadSlotCount] VECTOR_ALIGNED;	37	VECTOR_ALIGNED Epoch clk_[kThreadSlotCount];
38	};	38	};
39		39
40	ALWAYS_INLINE Epoch VectorClock::Get(Sid sid) const {	40	ALWAYS_INLINE Epoch VectorClock::Get(Sid sid) const {

src/libtsan.zig+11-5

 @@ -160,10 +160,13 @@ pub fn buildTsan(comp: *Compilation, prog_node: std.Progress.Node) BuildError!vo
     }
     {
         const asm_source = switch (target.cpu.arch) {
-            .aarch64 => "tsan_rtl_aarch64.S",
+            .aarch64, .aarch64_be => "tsan_rtl_aarch64.S",
+            .loongarch64 => "tsan_rtl_loongarch64.S",
+            .mips64, .mips64el => "tsan_rtl_mips64.S",
+            .powerpc64, .powerpc64le => "tsan_rtl_ppc64.S",
+            .riscv64 => "tsan_rtl_riscv64.S",
+            .s390x => "tsan_rtl_s390x.S",
             .x86_64 => "tsan_rtl_amd64.S",
-            .mips64 => "tsan_rtl_mips64.S",
-            .powerpc64 => "tsan_rtl_ppc64.S",
             else => return error.TSANUnsupportedCPUArchitecture,
         };
         var cflags = std.ArrayList([]const u8).init(arena);
 @@ -416,7 +419,6 @@ const sanitizer_common_sources = [_][]const u8{
     "sanitizer_platform_limits_freebsd.cpp",
     "sanitizer_platform_limits_linux.cpp",
     "sanitizer_platform_limits_netbsd.cpp",
-    "sanitizer_platform_limits_openbsd.cpp",
     "sanitizer_platform_limits_posix.cpp",
     "sanitizer_platform_limits_solaris.cpp",
     "sanitizer_posix.cpp",
 @@ -429,7 +431,6 @@ const sanitizer_common_sources = [_][]const u8{
     "sanitizer_procmaps_solaris.cpp",
     "sanitizer_range.cpp",
     "sanitizer_solaris.cpp",
-    "sanitizer_stack_store.cpp",
     "sanitizer_stoptheworld_fuchsia.cpp",
     "sanitizer_stoptheworld_mac.cpp",
     "sanitizer_stoptheworld_win.cpp",
 @@ -452,6 +453,7 @@ const sanitizer_nolibc_sources = [_][]const u8{
 const sanitizer_libcdep_sources = [_][]const u8{
     "sanitizer_common_libcdep.cpp",
     "sanitizer_allocator_checks.cpp",
+    "sanitizer_dl.cpp",
     "sanitizer_linux_libcdep.cpp",
     "sanitizer_mac_libcdep.cpp",
     "sanitizer_posix_libcdep.cpp",
 @@ -461,6 +463,7 @@ const sanitizer_libcdep_sources = [_][]const u8{
 const sanitizer_symbolizer_sources = [_][]const u8{
     "sanitizer_allocator_report.cpp",
+    "sanitizer_stack_store.cpp",
     "sanitizer_stackdepot.cpp",
     "sanitizer_stacktrace.cpp",
     "sanitizer_stacktrace_libcdep.cpp",
 @@ -471,10 +474,13 @@ const sanitizer_symbolizer_sources = [_][]const u8{
     "sanitizer_symbolizer_libcdep.cpp",
     "sanitizer_symbolizer_mac.cpp",
     "sanitizer_symbolizer_markup.cpp",
+    "sanitizer_symbolizer_markup_fuchsia.cpp",
     "sanitizer_symbolizer_posix_libcdep.cpp",
     "sanitizer_symbolizer_report.cpp",
+    "sanitizer_symbolizer_report_fuchsia.cpp",
     "sanitizer_symbolizer_win.cpp",
     "sanitizer_unwind_linux_libcdep.cpp",
+    "sanitizer_unwind_fuchsia.cpp",
     "sanitizer_unwind_win.cpp",
 };

author	Alex Rønne Petersen <alex@alexrp.com> 2024-09-24 13:47:29+02:00
committer	Andrew Kelley <andrew@ziglang.org> 2024-09-24 23:58:21-07:00
log	a40cdad18c06fd377622c47aa34564df7ea959b5
tree	dbcfabc2191cdbf2acc6c356c5ea25237b29d8f1
parent	7f6b7c56089eaa5b147e71b3d98328498c9025c8

...	@@ -69,7 +69,7 @@ void ReportFile::ReopenIfNecessary() {	...	@@ -69,7 +69,7 @@ void ReportFile::ReopenIfNecessary() {
69	WriteToFile(kStderrFd, ErrorMsgPrefix, internal_strlen(ErrorMsgPrefix));	69	WriteToFile(kStderrFd, ErrorMsgPrefix, internal_strlen(ErrorMsgPrefix));
70	WriteToFile(kStderrFd, full_path, internal_strlen(full_path));	70	WriteToFile(kStderrFd, full_path, internal_strlen(full_path));
71	char errmsg[100];	71	char errmsg[100];
72	internal_snprintf(errmsg, sizeof(errmsg), " (reason: %d)", err);	72	internal_snprintf(errmsg, sizeof(errmsg), " (reason: %d)\n", err);
73	WriteToFile(kStderrFd, errmsg, internal_strlen(errmsg));	73	WriteToFile(kStderrFd, errmsg, internal_strlen(errmsg));
74	Die();	74	Die();
75	}	75	}
...	@@ -88,6 +88,8 @@ static void RecursiveCreateParentDirs(char *path) {	...	@@ -88,6 +88,8 @@ static void RecursiveCreateParentDirs(char *path) {
88	const char *ErrorMsgPrefix = "ERROR: Can't create directory: ";	88	const char *ErrorMsgPrefix = "ERROR: Can't create directory: ";
89	WriteToFile(kStderrFd, ErrorMsgPrefix, internal_strlen(ErrorMsgPrefix));	89	WriteToFile(kStderrFd, ErrorMsgPrefix, internal_strlen(ErrorMsgPrefix));
90	WriteToFile(kStderrFd, path, internal_strlen(path));	90	WriteToFile(kStderrFd, path, internal_strlen(path));
		91	const char *ErrorMsgSuffix = "\n";
		92	WriteToFile(kStderrFd, ErrorMsgSuffix, internal_strlen(ErrorMsgSuffix));
91	Die();	93	Die();
92	}	94	}
93	path[i] = save;	95	path[i] = save;

...	@@ -1188,8 +1188,8 @@ uptr GetMaxVirtualAddress() {	...	@@ -1188,8 +1188,8 @@ uptr GetMaxVirtualAddress() {
1188	}	1188	}
1189		1189
1190	uptr MapDynamicShadow(uptr shadow_size_bytes, uptr shadow_scale,	1190	uptr MapDynamicShadow(uptr shadow_size_bytes, uptr shadow_scale,
1191	uptr min_shadow_base_alignment, uptr &high_mem_end) {	1191	uptr min_shadow_base_alignment, uptr &high_mem_end,
1192	const uptr granularity = GetMmapGranularity();	1192	uptr granularity) {
1193	const uptr alignment =	1193	const uptr alignment =
1194	Max<uptr>(granularity << shadow_scale, 1ULL << min_shadow_base_alignment);	1194	Max<uptr>(granularity << shadow_scale, 1ULL << min_shadow_base_alignment);
1195	const uptr left_padding =	1195	const uptr left_padding =
...	@@ -1372,8 +1372,8 @@ void DumpProcessMap() {	...	@@ -1372,8 +1372,8 @@ void DumpProcessMap() {
1372	for (uptr i = 0; i < modules.size(); ++i) {	1372	for (uptr i = 0; i < modules.size(); ++i) {
1373	char uuid_str[128];	1373	char uuid_str[128];
1374	FormatUUID(uuid_str, sizeof(uuid_str), modules[i].uuid());	1374	FormatUUID(uuid_str, sizeof(uuid_str), modules[i].uuid());
1375	Printf("0x%zx-0x%zx %s (%s) %s\n", modules[i].base_address(),	1375	Printf("%p-%p %s (%s) %s\n", (void *)modules[i].base_address(),
1376	modules[i].max_address(), modules[i].full_name(),	1376	(void *)modules[i].max_address(), modules[i].full_name(),
1377	ModuleArchToString(modules[i].arch()), uuid_str);	1377	ModuleArchToString(modules[i].arch()), uuid_str);
1378	}	1378	}
1379	Printf("End of module map.\n");	1379	Printf("End of module map.\n");

tsan: Update to LLVM 19.1.0.