Commit a40cdad18c06fd377622c47aa34564df7ea959b5 | zig

lib/tsan/builtins/assembly.h created+293

...	...	@@ -0,0 +1,293 @@
	1	//===-- assembly.h - compiler-rt assembler support macros -----------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file defines macros for use in compiler-rt assembler source.
	10	// This file is not part of the interface of this library.
	11	//
	12	//===----------------------------------------------------------------------===//
	13
	14	#ifndef COMPILERRT_ASSEMBLY_H
	15	#define COMPILERRT_ASSEMBLY_H
	16
	17	#if defined(__linux__) && defined(__CET__)
	18	#if __has_include(<cet.h>)
	19	#include <cet.h>
	20	#endif
	21	#endif
	22
	23	#if defined(__APPLE__) && defined(__aarch64__)
	24	#define SEPARATOR %%
	25	#else
	26	#define SEPARATOR ;
	27	#endif
	28
	29	#if defined(__APPLE__)
	30	#define HIDDEN(name) .private_extern name
	31	#define LOCAL_LABEL(name) L_##name
	32	// tell linker it can break up file at label boundaries
	33	#define FILE_LEVEL_DIRECTIVE .subsections_via_symbols
	34	#define SYMBOL_IS_FUNC(name)
	35	#define CONST_SECTION .const
	36
	37	#define NO_EXEC_STACK_DIRECTIVE
	38
	39	#elif defined(__ELF__)
	40
	41	#define HIDDEN(name) .hidden name
	42	#define LOCAL_LABEL(name) .L_##name
	43	#define FILE_LEVEL_DIRECTIVE
	44	#if defined(__arm__) \|\| defined(__aarch64__)
	45	#define SYMBOL_IS_FUNC(name) .type name,%function
	46	#else
	47	#define SYMBOL_IS_FUNC(name) .type name,@function
	48	#endif
	49	#define CONST_SECTION .section .rodata
	50
	51	#if defined(__GNU__) \|\| defined(__FreeBSD__) \|\| defined(__Fuchsia__) \|\| \
	52	defined(__linux__)
	53	#define NO_EXEC_STACK_DIRECTIVE .section .note.GNU-stack,"",%progbits
	54	#else
	55	#define NO_EXEC_STACK_DIRECTIVE
	56	#endif
	57
	58	#else // !__APPLE__ && !__ELF__
	59
	60	#define HIDDEN(name)
	61	#define LOCAL_LABEL(name) .L ## name
	62	#define FILE_LEVEL_DIRECTIVE
	63	#define SYMBOL_IS_FUNC(name) \
	64	.def name SEPARATOR \
	65	.scl 2 SEPARATOR \
	66	.type 32 SEPARATOR \
	67	.endef
	68	#define CONST_SECTION .section .rdata,"rd"
	69
	70	#define NO_EXEC_STACK_DIRECTIVE
	71
	72	#endif
	73
	74	#if defined(__arm__) \|\| defined(__aarch64__)
	75	#define FUNC_ALIGN \
	76	.text SEPARATOR \
	77	.balign 16 SEPARATOR
	78	#else
	79	#define FUNC_ALIGN
	80	#endif
	81
	82	// BTI and PAC gnu property note
	83	#define NT_GNU_PROPERTY_TYPE_0 5
	84	#define GNU_PROPERTY_AARCH64_FEATURE_1_AND 0xc0000000
	85	#define GNU_PROPERTY_AARCH64_FEATURE_1_BTI 1
	86	#define GNU_PROPERTY_AARCH64_FEATURE_1_PAC 2
	87
	88	#if defined(__ARM_FEATURE_BTI_DEFAULT)
	89	#define BTI_FLAG GNU_PROPERTY_AARCH64_FEATURE_1_BTI
	90	#else
	91	#define BTI_FLAG 0
	92	#endif
	93
	94	#if __ARM_FEATURE_PAC_DEFAULT & 3
	95	#define PAC_FLAG GNU_PROPERTY_AARCH64_FEATURE_1_PAC
	96	#else
	97	#define PAC_FLAG 0
	98	#endif
	99
	100	#define GNU_PROPERTY(type, value) \
	101	.pushsection .note.gnu.property, "a" SEPARATOR \
	102	.p2align 3 SEPARATOR \
	103	.word 4 SEPARATOR \
	104	.word 16 SEPARATOR \
	105	.word NT_GNU_PROPERTY_TYPE_0 SEPARATOR \
	106	.asciz "GNU" SEPARATOR \
	107	.word type SEPARATOR \
	108	.word 4 SEPARATOR \
	109	.word value SEPARATOR \
	110	.word 0 SEPARATOR \
	111	.popsection
	112
	113	#if BTI_FLAG != 0
	114	#define BTI_C hint #34
	115	#define BTI_J hint #36
	116	#else
	117	#define BTI_C
	118	#define BTI_J
	119	#endif
	120
	121	#if (BTI_FLAG \| PAC_FLAG) != 0
	122	#define GNU_PROPERTY_BTI_PAC \
	123	GNU_PROPERTY(GNU_PROPERTY_AARCH64_FEATURE_1_AND, BTI_FLAG \| PAC_FLAG)
	124	#else
	125	#define GNU_PROPERTY_BTI_PAC
	126	#endif
	127
	128	#if defined(__clang__) \|\| defined(__GCC_HAVE_DWARF2_CFI_ASM)
	129	#define CFI_START .cfi_startproc
	130	#define CFI_END .cfi_endproc
	131	#else
	132	#define CFI_START
	133	#define CFI_END
	134	#endif
	135
	136	#if defined(__arm__)
	137
	138	// Determine actual [ARM][THUMB[1][2]] ISA using compiler predefined macros:
	139	// - for '-mthumb -march=armv6' compiler defines '__thumb__'
	140	// - for '-mthumb -march=armv7' compiler defines '__thumb__' and '__thumb2__'
	141	#if defined(__thumb2__) \|\| defined(__thumb__)
	142	#define DEFINE_CODE_STATE .thumb SEPARATOR
	143	#define DECLARE_FUNC_ENCODING .thumb_func SEPARATOR
	144	#if defined(__thumb2__)
	145	#define USE_THUMB_2
	146	#define IT(cond) it cond
	147	#define ITT(cond) itt cond
	148	#define ITE(cond) ite cond
	149	#else
	150	#define USE_THUMB_1
	151	#define IT(cond)
	152	#define ITT(cond)
	153	#define ITE(cond)
	154	#endif // defined(__thumb__2)
	155	#else // !defined(__thumb2__) && !defined(__thumb__)
	156	#define DEFINE_CODE_STATE .arm SEPARATOR
	157	#define DECLARE_FUNC_ENCODING
	158	#define IT(cond)
	159	#define ITT(cond)
	160	#define ITE(cond)
	161	#endif
	162
	163	#if defined(USE_THUMB_1) && defined(USE_THUMB_2)
	164	#error "USE_THUMB_1 and USE_THUMB_2 can't be defined together."
	165	#endif
	166
	167	#if defined(__ARM_ARCH_4T__) \|\| __ARM_ARCH >= 5
	168	#define ARM_HAS_BX
	169	#endif
	170	#if !defined(__ARM_FEATURE_CLZ) && !defined(USE_THUMB_1) && \
	171	(__ARM_ARCH >= 6 \|\| (__ARM_ARCH == 5 && !defined(__ARM_ARCH_5__)))
	172	#define __ARM_FEATURE_CLZ
	173	#endif
	174
	175	#ifdef ARM_HAS_BX
	176	#define JMP(r) bx r
	177	#define JMPc(r, c) bx##c r
	178	#else
	179	#define JMP(r) mov pc, r
	180	#define JMPc(r, c) mov##c pc, r
	181	#endif
	182
	183	// pop {pc} can't switch Thumb mode on ARMv4T
	184	#if __ARM_ARCH >= 5
	185	#define POP_PC() pop {pc}
	186	#else
	187	#define POP_PC() \
	188	pop {ip}; \
	189	JMP(ip)
	190	#endif
	191
	192	#if defined(USE_THUMB_2)
	193	#define WIDE(op) op.w
	194	#else
	195	#define WIDE(op) op
	196	#endif
	197	#else // !defined(__arm)
	198	#define DECLARE_FUNC_ENCODING
	199	#define DEFINE_CODE_STATE
	200	#endif
	201
	202	#define GLUE2_(a, b) a##b
	203	#define GLUE(a, b) GLUE2_(a, b)
	204	#define GLUE2(a, b) GLUE2_(a, b)
	205	#define GLUE3_(a, b, c) a##b##c
	206	#define GLUE3(a, b, c) GLUE3_(a, b, c)
	207	#define GLUE4_(a, b, c, d) a##b##c##d
	208	#define GLUE4(a, b, c, d) GLUE4_(a, b, c, d)
	209
	210	#define SYMBOL_NAME(name) GLUE(__USER_LABEL_PREFIX__, name)
	211
	212	#ifdef VISIBILITY_HIDDEN
	213	#define DECLARE_SYMBOL_VISIBILITY(name) \
	214	HIDDEN(SYMBOL_NAME(name)) SEPARATOR
	215	#define DECLARE_SYMBOL_VISIBILITY_UNMANGLED(name) \
	216	HIDDEN(name) SEPARATOR
	217	#else
	218	#define DECLARE_SYMBOL_VISIBILITY(name)
	219	#define DECLARE_SYMBOL_VISIBILITY_UNMANGLED(name)
	220	#endif
	221
	222	#define DEFINE_COMPILERRT_FUNCTION(name) \
	223	DEFINE_CODE_STATE \
	224	FILE_LEVEL_DIRECTIVE SEPARATOR \
	225	.globl SYMBOL_NAME(name) SEPARATOR \
	226	SYMBOL_IS_FUNC(SYMBOL_NAME(name)) SEPARATOR \
	227	DECLARE_SYMBOL_VISIBILITY(name) \
	228	DECLARE_FUNC_ENCODING \
	229	SYMBOL_NAME(name):
	230
	231	#define DEFINE_COMPILERRT_THUMB_FUNCTION(name) \
	232	DEFINE_CODE_STATE \
	233	FILE_LEVEL_DIRECTIVE SEPARATOR \
	234	.globl SYMBOL_NAME(name) SEPARATOR \
	235	SYMBOL_IS_FUNC(SYMBOL_NAME(name)) SEPARATOR \
	236	DECLARE_SYMBOL_VISIBILITY(name) SEPARATOR \
	237	.thumb_func SEPARATOR \
	238	SYMBOL_NAME(name):
	239
	240	#define DEFINE_COMPILERRT_PRIVATE_FUNCTION(name) \
	241	DEFINE_CODE_STATE \
	242	FILE_LEVEL_DIRECTIVE SEPARATOR \
	243	.globl SYMBOL_NAME(name) SEPARATOR \
	244	SYMBOL_IS_FUNC(SYMBOL_NAME(name)) SEPARATOR \
	245	HIDDEN(SYMBOL_NAME(name)) SEPARATOR \
	246	DECLARE_FUNC_ENCODING \
	247	SYMBOL_NAME(name):
	248
	249	#define DEFINE_COMPILERRT_PRIVATE_FUNCTION_UNMANGLED(name) \
	250	DEFINE_CODE_STATE \
	251	.globl name SEPARATOR \
	252	SYMBOL_IS_FUNC(name) SEPARATOR \
	253	HIDDEN(name) SEPARATOR \
	254	DECLARE_FUNC_ENCODING \
	255	name:
	256
	257	#define DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(name) \
	258	DEFINE_CODE_STATE \
	259	FUNC_ALIGN \
	260	.globl name SEPARATOR \
	261	SYMBOL_IS_FUNC(name) SEPARATOR \
	262	DECLARE_SYMBOL_VISIBILITY_UNMANGLED(name) SEPARATOR \
	263	DECLARE_FUNC_ENCODING \
	264	name: \
	265	SEPARATOR CFI_START \
	266	SEPARATOR BTI_C
	267
	268	#define DEFINE_COMPILERRT_FUNCTION_ALIAS(name, target) \
	269	.globl SYMBOL_NAME(name) SEPARATOR \
	270	SYMBOL_IS_FUNC(SYMBOL_NAME(name)) SEPARATOR \
	271	DECLARE_SYMBOL_VISIBILITY(name) SEPARATOR \
	272	.set SYMBOL_NAME(name), SYMBOL_NAME(target) SEPARATOR
	273
	274	#if defined(__ARM_EABI__)
	275	#define DEFINE_AEABI_FUNCTION_ALIAS(aeabi_name, name) \
	276	DEFINE_COMPILERRT_FUNCTION_ALIAS(aeabi_name, name)
	277	#else
	278	#define DEFINE_AEABI_FUNCTION_ALIAS(aeabi_name, name)
	279	#endif
	280
	281	#ifdef __ELF__
	282	#define END_COMPILERRT_FUNCTION(name) \
	283	.size SYMBOL_NAME(name), . - SYMBOL_NAME(name)
	284	#define END_COMPILERRT_OUTLINE_FUNCTION(name) \
	285	CFI_END SEPARATOR \
	286	.size SYMBOL_NAME(name), . - SYMBOL_NAME(name)
	287	#else
	288	#define END_COMPILERRT_FUNCTION(name)
	289	#define END_COMPILERRT_OUTLINE_FUNCTION(name) \
	290	CFI_END
	291	#endif
	292
	293	#endif // COMPILERRT_ASSEMBLY_H

lib/tsan/interception/interception.h+24-5

...	...	@@ -185,6 +185,11 @@ const interpose_substitution substitution_##func_name[] \
185	185	# else
186	186	# define __ASM_WEAK_WRAPPER(func) ".weak " #func "\n"
187	187	# endif // SANITIZER_FREEBSD \|\| SANITIZER_NETBSD
	188	# if defined(__arm__) \|\| defined(__aarch64__)
	189	# define ASM_TYPE_FUNCTION_STR "%function"
	190	# else
	191	# define ASM_TYPE_FUNCTION_STR "@function"
	192	# endif
188	193	// Keep trampoline implementation in sync with sanitizer_common/sanitizer_asm.h
189	194	# define DECLARE_WRAPPER(ret_type, func, ...) \
190	195	extern "C" ret_type func(__VA_ARGS__); \
...	...	@@ -196,12 +201,14 @@ const interpose_substitution substitution_##func_name[] \
196	201	__ASM_WEAK_WRAPPER(func) \
197	202	".set " #func ", " SANITIZER_STRINGIFY(TRAMPOLINE(func)) "\n" \
198	203	".globl " SANITIZER_STRINGIFY(TRAMPOLINE(func)) "\n" \
199		".type " SANITIZER_STRINGIFY(TRAMPOLINE(func)) ", %function\n" \
	204	".type " SANITIZER_STRINGIFY(TRAMPOLINE(func)) ", " \
	205	ASM_TYPE_FUNCTION_STR "\n" \
200	206	SANITIZER_STRINGIFY(TRAMPOLINE(func)) ":\n" \
201		SANITIZER_STRINGIFY(CFI_STARTPROC) "\n" \
202		SANITIZER_STRINGIFY(ASM_TAIL_CALL) " __interceptor_" \
203		SANITIZER_STRINGIFY(ASM_PREEMPTIBLE_SYM(func)) "\n" \
204		SANITIZER_STRINGIFY(CFI_ENDPROC) "\n" \
	207	C_ASM_STARTPROC "\n" \
	208	C_ASM_TAIL_CALL(SANITIZER_STRINGIFY(TRAMPOLINE(func)), \
	209	"__interceptor_" \
	210	SANITIZER_STRINGIFY(ASM_PREEMPTIBLE_SYM(func))) "\n" \
	211	C_ASM_ENDPROC "\n" \
205	212	".size " SANITIZER_STRINGIFY(TRAMPOLINE(func)) ", " \
206	213	".-" SANITIZER_STRINGIFY(TRAMPOLINE(func)) "\n" \
207	214	);
...	...	@@ -341,6 +348,18 @@ typedef unsigned long long uptr;
341	348	#else
342	349	typedef unsigned long uptr;
343	350	#endif // _WIN64
	351
	352	#if defined(__ELF__) && !SANITIZER_FUCHSIA
	353	// The use of interceptors makes many sanitizers unusable for static linking.
	354	// Define a function, if called, will cause a linker error (undefined _DYNAMIC).
	355	// However, -static-pie (which is not common) cannot be detected at link time.
	356	extern uptr kDynamic[] asm("_DYNAMIC");
	357	inline void DoesNotSupportStaticLinking() {
	358	[[maybe_unused]] volatile auto x = &kDynamic;
	359	}
	360	#else
	361	inline void DoesNotSupportStaticLinking() {}
	362	#endif
344	363	} // namespace __interception
345	364
346	365	#define INCLUDED_FROM_INTERCEPTION_LIB

lib/tsan/interception/interception_linux.h+9-7

...	...	@@ -28,12 +28,14 @@ bool InterceptFunction(const char name, const char ver, uptr *ptr_to_real,
28	28	uptr func, uptr trampoline);
29	29	} // namespace __interception
30	30
31		#define INTERCEPT_FUNCTION_LINUX_OR_FREEBSD(func) \
32		::__interception::InterceptFunction( \
33		#func, \
34		(::__interception::uptr *)&REAL(func), \
35		(::__interception::uptr)&(func), \
36		(::__interception::uptr)&TRAMPOLINE(func))
	31	// Cast func to type of REAL(func) before casting to uptr in case it is an
	32	// overloaded function, which is the case for some glibc functions when
	33	// _FORTIFY_SOURCE is used. This disambiguates which overload to use.
	34	#define INTERCEPT_FUNCTION_LINUX_OR_FREEBSD(func) \
	35	::__interception::InterceptFunction( \
	36	#func, (::__interception::uptr *)&REAL(func), \
	37	(::__interception::uptr)(decltype(REAL(func)))&(func), \
	38	(::__interception::uptr) &TRAMPOLINE(func))
37	39
38	40	// dlvsym is a GNU extension supported by some other platforms.
39	41	#if SANITIZER_GLIBC \|\| SANITIZER_FREEBSD \|\| SANITIZER_NETBSD
...	...	@@ -41,7 +43,7 @@ bool InterceptFunction(const char name, const char ver, uptr *ptr_to_real,
41	43	::__interception::InterceptFunction( \
42	44	#func, symver, \
43	45	(::__interception::uptr *)&REAL(func), \
44		(::__interception::uptr)&(func), \
	46	(::__interception::uptr)(decltype(REAL(func)))&(func), \
45	47	(::__interception::uptr)&TRAMPOLINE(func))
46	48	#else
47	49	#define INTERCEPT_FUNCTION_VER_LINUX_OR_FREEBSD(func, symver) \

lib/tsan/interception/interception_win.cpp+56-27

...	...	@@ -1,4 +1,4 @@
1		//===-- interception_linux.cpp ----------------------------------- C++ --===//
	1	//===-- interception_win.cpp ------------------------------------- C++ --===//
2	2	//
3	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	4	// See https://llvm.org/LICENSE.txt for license information.
...	...	@@ -339,7 +339,7 @@ struct TrampolineMemoryRegion {
339	339	uptr max_size;
340	340	};
341	341
342		UNUSED static const uptr kTrampolineScanLimitRange = 1 << 31; // 2 gig
	342	UNUSED static const uptr kTrampolineScanLimitRange = 1ull << 31; // 2 gig
343	343	static const int kMaxTrampolineRegion = 1024;
344	344	static TrampolineMemoryRegion TrampolineRegions[kMaxTrampolineRegion];
345	345
...	...	@@ -431,7 +431,8 @@ static uptr AllocateMemoryForTrampoline(uptr image_address, size_t size) {
431	431	// The following prologues cannot be patched because of the short jump
432	432	// jumping to the patching region.
433	433
434		#if SANITIZER_WINDOWS64
	434	// Short jump patterns below are only for x86_64.
	435	# if SANITIZER_WINDOWS_x64
435	436	// ntdll!wcslen in Win11
436	437	// 488bc1 mov rax,rcx
437	438	// 0fb710 movzx edx,word ptr [rax]
...	...	@@ -457,7 +458,12 @@ static const u8 kPrologueWithShortJump2[] = {
457	458
458	459	// Returns 0 on error.
459	460	static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
460		#if SANITIZER_WINDOWS64
	461	#if SANITIZER_ARM64
	462	// An ARM64 instruction is 4 bytes long.
	463	return 4;
	464	#endif
	465
	466	# if SANITIZER_WINDOWS_x64
461	467	if (memcmp((u8*)address, kPrologueWithShortJump1,
462	468	sizeof(kPrologueWithShortJump1)) == 0 \|\|
463	469	memcmp((u8*)address, kPrologueWithShortJump2,
...	...	@@ -473,6 +479,8 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
473	479
474	480	switch ((u8)address) {
475	481	case 0x90: // 90 : nop
	482	case 0xC3: // C3 : ret (for small/empty function interception
	483	case 0xCC: // CC : int 3 i.e. registering weak functions)
476	484	return 1;
477	485
478	486	case 0x50: // push eax / rax
...	...	@@ -496,7 +504,6 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
496	504	// Cannot overwrite control-instruction. Return 0 to indicate failure.
497	505	case 0xE9: // E9 XX XX XX XX : jmp <label>
498	506	case 0xE8: // E8 XX XX XX XX : call <func>
499		case 0xC3: // C3 : ret
500	507	case 0xEB: // EB XX : jmp XX (short jump)
501	508	case 0x70: // 7Y YY : jy XX (short conditional jump)
502	509	case 0x71:
...	...	@@ -539,7 +546,12 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
539	546	return 7;
540	547	}
541	548
542		#if SANITIZER_WINDOWS64
	549	switch (0x000000FF & (u32 )address) {
	550	case 0xc2: // C2 XX XX : ret XX (needed for registering weak functions)
	551	return 3;
	552	}
	553
	554	# if SANITIZER_WINDOWS_x64
543	555	switch ((u8)address) {
544	556	case 0xA1: // A1 XX XX XX XX XX XX XX XX :
545	557	// movabs eax, dword ptr ds:[XXXXXXXX]
...	...	@@ -572,6 +584,7 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
572	584	case 0x018a: // mov al, byte ptr [rcx]
573	585	return 2;
574	586
	587	case 0x058A: // 8A 05 XX XX XX XX : mov al, byte ptr [XX XX XX XX]
575	588	case 0x058B: // 8B 05 XX XX XX XX : mov eax, dword ptr [XX XX XX XX]
576	589	if (rel_offset)
577	590	*rel_offset = 2;
...	...	@@ -598,6 +611,7 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
598	611	case 0xc18b4c: // 4C 8B C1 : mov r8, rcx
599	612	case 0xd2b60f: // 0f b6 d2 : movzx edx, dl
600	613	case 0xca2b48: // 48 2b ca : sub rcx, rdx
	614	case 0xca3b48: // 48 3b ca : cmp rcx, rdx
601	615	case 0x10b70f: // 0f b7 10 : movzx edx, WORD PTR [rax]
602	616	case 0xc00b4d: // 3d 0b c0 : or r8, r8
603	617	case 0xc08b41: // 41 8b c0 : mov eax, r8d
...	...	@@ -617,9 +631,11 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
617	631
618	632	case 0x058b48: // 48 8b 05 XX XX XX XX :
619	633	// mov rax, QWORD PTR [rip + XXXXXXXX]
	634	case 0x058d48: // 48 8d 05 XX XX XX XX :
	635	// lea rax, QWORD PTR [rip + XXXXXXXX]
620	636	case 0x25ff48: // 48 ff 25 XX XX XX XX :
621	637	// rex.W jmp QWORD PTR [rip + XXXXXXXX]
622
	638	case 0x158D4C: // 4c 8d 15 XX XX XX XX : lea r10, [rip + XX]
623	639	// Instructions having offset relative to 'rip' need offset adjustment.
624	640	if (rel_offset)
625	641	*rel_offset = 3;
...	...	@@ -721,16 +737,22 @@ static bool CopyInstructions(uptr to, uptr from, size_t size) {
721	737	size_t instruction_size = GetInstructionSize(from + cursor, &rel_offset);
722	738	if (!instruction_size)
723	739	return false;
724		_memcpy((void)(to + cursor), (void)(from + cursor),
	740	_memcpy((void )(to + cursor), (void )(from + cursor),
725	741	(size_t)instruction_size);
726	742	if (rel_offset) {
727		uptr delta = to - from;
728		uptr relocated_offset = (u32)(to + cursor + rel_offset) - delta;
729		#if SANITIZER_WINDOWS64
730		if (relocated_offset + 0x80000000U >= 0xFFFFFFFFU)
	743	# if SANITIZER_WINDOWS64
	744	// we want to make sure that the new relative offset still fits in 32-bits
	745	// this will be untrue if relocated_offset \notin [-231, 231)
	746	s64 delta = to - from;
	747	s64 relocated_offset = (s32 )(to + cursor + rel_offset) - delta;
	748	if (-0x8000'0000ll > relocated_offset \|\| relocated_offset > 0x7FFF'FFFFll)
731	749	return false;
732		#endif
733		(u32)(to + cursor + rel_offset) = relocated_offset;
	750	# else
	751	// on 32-bit, the relative offset will always be correct
	752	s32 delta = to - from;
	753	s32 relocated_offset = (s32 )(to + cursor + rel_offset) - delta;
	754	# endif
	755	(s32 )(to + cursor + rel_offset) = relocated_offset;
734	756	}
735	757	cursor += instruction_size;
736	758	}
...	...	@@ -932,19 +954,26 @@ bool OverrideFunction(
932	954
933	955	static void **InterestingDLLsAvailable() {
934	956	static const char *InterestingDLLs[] = {
935		"kernel32.dll",
936		"msvcr100.dll", // VS2010
937		"msvcr110.dll", // VS2012
938		"msvcr120.dll", // VS2013
939		"vcruntime140.dll", // VS2015
940		"ucrtbase.dll", // Universal CRT
941		#if (defined(__MINGW32__) && defined(__i386__))
942		"libc++.dll", // libc++
943		"libunwind.dll", // libunwind
944		#endif
945		// NTDLL should go last as it exports some functions that we should
946		// override in the CRT [presumably only used internally].
947		"ntdll.dll", NULL};
	957	"kernel32.dll",
	958	"msvcr100d.dll", // VS2010
	959	"msvcr110d.dll", // VS2012
	960	"msvcr120d.dll", // VS2013
	961	"vcruntime140d.dll", // VS2015
	962	"ucrtbased.dll", // Universal CRT
	963	"msvcr100.dll", // VS2010
	964	"msvcr110.dll", // VS2012
	965	"msvcr120.dll", // VS2013
	966	"vcruntime140.dll", // VS2015
	967	"ucrtbase.dll", // Universal CRT
	968	# if (defined(__MINGW32__) && defined(__i386__))
	969	"libc++.dll", // libc++
	970	"libunwind.dll", // libunwind
	971	# endif
	972	// NTDLL should go last as it exports some functions that we should
	973	// override in the CRT [presumably only used internally].
	974	"ntdll.dll",
	975	NULL
	976	};
948	977	static void *result[ARRAY_SIZE(InterestingDLLs)] = { 0 };
949	978	if (!result[0]) {
950	979	for (size_t i = 0, j = 0; InterestingDLLs[i]; ++i) {

lib/tsan/sanitizer_common/sanitizer_allocator.cpp+8-2

...	...	@@ -25,7 +25,7 @@ namespace __sanitizer {
25	25	const char *PrimaryAllocatorName = "SizeClassAllocator";
26	26	const char *SecondaryAllocatorName = "LargeMmapAllocator";
27	27
28		static ALIGNED(64) char internal_alloc_placeholder[sizeof(InternalAllocator)];
	28	alignas(64) static char internal_alloc_placeholder[sizeof(InternalAllocator)];
29	29	static atomic_uint8_t internal_allocator_initialized;
30	30	static StaticSpinMutex internal_alloc_init_mu;
31	31
...	...	@@ -138,14 +138,20 @@ void InternalAllocatorUnlock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
138	138
139	139	// LowLevelAllocator
140	140	constexpr uptr kLowLevelAllocatorDefaultAlignment = 8;
	141	constexpr uptr kMinNumPagesRounded = 16;
	142	constexpr uptr kMinRoundedSize = 65536;
141	143	static uptr low_level_alloc_min_alignment = kLowLevelAllocatorDefaultAlignment;
142	144	static LowLevelAllocateCallback low_level_alloc_callback;
143	145
	146	static LowLevelAllocator Alloc;
	147	LowLevelAllocator &GetGlobalLowLevelAllocator() { return Alloc; }
	148
144	149	void *LowLevelAllocator::Allocate(uptr size) {
145	150	// Align allocation size.
146	151	size = RoundUpTo(size, low_level_alloc_min_alignment);
147	152	if (allocated_end_ - allocated_current_ < (sptr)size) {
148		uptr size_to_allocate = RoundUpTo(size, GetPageSizeCached());
	153	uptr size_to_allocate = RoundUpTo(
	154	size, Min(GetPageSizeCached() * kMinNumPagesRounded, kMinRoundedSize));
149	155	allocated_current_ = (char *)MmapOrDie(size_to_allocate, __func__);
150	156	allocated_end_ = allocated_current_ + size_to_allocate;
151	157	if (low_level_alloc_callback) {

lib/tsan/sanitizer_common/sanitizer_allocator_interface.h+2

...	...	@@ -40,6 +40,8 @@ SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
40	40	void __sanitizer_malloc_hook(void *ptr, uptr size);
41	41	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
42	42	void __sanitizer_free_hook(void *ptr);
	43	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE int
	44	__sanitizer_ignore_free_hook(void *ptr);
43	45
44	46	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
45	47	__sanitizer_purge_allocator();

lib/tsan/sanitizer_common/sanitizer_allocator_primary32.h+1-1

...	...	@@ -278,7 +278,7 @@ class SizeClassAllocator32 {
278	278	static const uptr kRegionSize = 1 << kRegionSizeLog;
279	279	static const uptr kNumPossibleRegions = kSpaceSize / kRegionSize;
280	280
281		struct ALIGNED(SANITIZER_CACHE_LINE_SIZE) SizeClassInfo {
	281	struct alignas(SANITIZER_CACHE_LINE_SIZE) SizeClassInfo {
282	282	StaticSpinMutex mutex;
283	283	IntrusiveList<TransferBatch> free_list;
284	284	u32 rand_state;

lib/tsan/sanitizer_common/sanitizer_allocator_primary64.h+9-7

...	...	@@ -316,13 +316,13 @@ class SizeClassAllocator64 {
316	316	Printf(
317	317	"%s %02zd (%6zd): mapped: %6zdK allocs: %7zd frees: %7zd inuse: %6zd "
318	318	"num_freed_chunks %7zd avail: %6zd rss: %6zdK releases: %6zd "
319		"last released: %6lldK region: 0x%zx\n",
	319	"last released: %6lldK region: %p\n",
320	320	region->exhausted ? "F" : " ", class_id, ClassIdToSize(class_id),
321	321	region->mapped_user >> 10, region->stats.n_allocated,
322	322	region->stats.n_freed, in_use, region->num_freed_chunks, avail_chunks,
323	323	rss >> 10, region->rtoi.num_releases,
324	324	region->rtoi.last_released_bytes >> 10,
325		SpaceBeg() + kRegionSize * class_id);
	325	(void )(SpaceBeg() + kRegionSize class_id));
326	326	}
327	327
328	328	void PrintStats() {
...	...	@@ -636,15 +636,17 @@ class SizeClassAllocator64 {
636	636	}
637	637	uptr SpaceEnd() const { return SpaceBeg() + kSpaceSize; }
638	638	// kRegionSize should be able to satisfy the largest size class.
639		static_assert(kRegionSize >= SizeClassMap::kMaxSize);
	639	static_assert(kRegionSize >= SizeClassMap::kMaxSize,
	640	"Region size exceed largest size");
640	641	// kRegionSize must be <= 2^36, see CompactPtrT.
641		COMPILER_CHECK((kRegionSize) <= (1ULL << (SANITIZER_WORDSIZE / 2 + 4)));
	642	COMPILER_CHECK((kRegionSize) <=
	643	(1ULL << (sizeof(CompactPtrT) * 8 + kCompactPtrScale)));
642	644	// Call mmap for user memory with at least this size.
643		static const uptr kUserMapSize = 1 << 16;
	645	static const uptr kUserMapSize = 1 << 18;
644	646	// Call mmap for metadata memory with at least this size.
645	647	static const uptr kMetaMapSize = 1 << 16;
646	648	// Call mmap for free array memory with at least this size.
647		static const uptr kFreeArrayMapSize = 1 << 16;
	649	static const uptr kFreeArrayMapSize = 1 << 18;
648	650
649	651	atomic_sint32_t release_to_os_interval_ms_;
650	652
...	...	@@ -665,7 +667,7 @@ class SizeClassAllocator64 {
665	667	u64 last_released_bytes;
666	668	};
667	669
668		struct ALIGNED(SANITIZER_CACHE_LINE_SIZE) RegionInfo {
	670	struct alignas(SANITIZER_CACHE_LINE_SIZE) RegionInfo {
669	671	Mutex mutex;
670	672	uptr num_freed_chunks; // Number of elements in the freearray.
671	673	uptr mapped_free_array; // Bytes mapped for freearray.

lib/tsan/sanitizer_common/sanitizer_asm.h+40-3

...	...	@@ -42,6 +42,16 @@
42	42	# define CFI_RESTORE(reg)
43	43	#endif
44	44
	45	#if defined(__aarch64__) && defined(__ARM_FEATURE_BTI_DEFAULT)
	46	# define ASM_STARTPROC CFI_STARTPROC; hint #34
	47	# define C_ASM_STARTPROC SANITIZER_STRINGIFY(CFI_STARTPROC) "\nhint #34"
	48	#else
	49	# define ASM_STARTPROC CFI_STARTPROC
	50	# define C_ASM_STARTPROC SANITIZER_STRINGIFY(CFI_STARTPROC)
	51	#endif
	52	#define ASM_ENDPROC CFI_ENDPROC
	53	#define C_ASM_ENDPROC SANITIZER_STRINGIFY(CFI_ENDPROC)
	54
45	55	#if defined(__x86_64__) \|\| defined(__i386__) \|\| defined(__sparc__)
46	56	# define ASM_TAIL_CALL jmp
47	57	#elif defined(__arm__) \|\| defined(__aarch64__) \|\| defined(__mips__) \|\| \
...	...	@@ -53,6 +63,29 @@
53	63	# define ASM_TAIL_CALL tail
54	64	#endif
55	65
	66	// Currently, almost all of the shared libraries rely on the value of
	67	// $t9 to get the address of current function, instead of PCREL, even
	68	// on MIPSr6. To be compatiable with them, we have to set $t9 properly.
	69	// MIPS uses GOT to get the address of preemptible functions.
	70	#if defined(__mips64)
	71	# define C_ASM_TAIL_CALL(t_func, i_func) \
	72	"lui $t8, %hi(%neg(%gp_rel(" t_func ")))\n" \
	73	"daddu $t8, $t8, $t9\n" \
	74	"daddiu $t8, $t8, %lo(%neg(%gp_rel(" t_func ")))\n" \
	75	"ld $t9, %got_disp(" i_func ")($t8)\n" \
	76	"jr $t9\n"
	77	#elif defined(__mips__)
	78	# define C_ASM_TAIL_CALL(t_func, i_func) \
	79	".set noreorder\n" \
	80	".cpload $t9\n" \
	81	".set reorder\n" \
	82	"lw $t9, %got(" i_func ")($gp)\n" \
	83	"jr $t9\n"
	84	#elif defined(ASM_TAIL_CALL)
	85	# define C_ASM_TAIL_CALL(t_func, i_func) \
	86	SANITIZER_STRINGIFY(ASM_TAIL_CALL) " " i_func
	87	#endif
	88
56	89	#if defined(__ELF__) && defined(__x86_64__) \|\| defined(__i386__) \|\| \
57	90	defined(__riscv)
58	91	# define ASM_PREEMPTIBLE_SYM(sym) sym@plt
...	...	@@ -62,7 +95,11 @@
62	95
63	96	#if !defined(__APPLE__)
64	97	# define ASM_HIDDEN(symbol) .hidden symbol
65		# define ASM_TYPE_FUNCTION(symbol) .type symbol, %function
	98	# if defined(__arm__) \|\| defined(__aarch64__)
	99	# define ASM_TYPE_FUNCTION(symbol) .type symbol, %function
	100	# else
	101	# define ASM_TYPE_FUNCTION(symbol) .type symbol, @function
	102	# endif
66	103	# define ASM_SIZE(symbol) .size symbol, .-symbol
67	104	# define ASM_SYMBOL(symbol) symbol
68	105	# define ASM_SYMBOL_INTERCEPTOR(symbol) symbol
...	...	@@ -87,9 +124,9 @@
87	124	.globl __interceptor_trampoline_##name; \
88	125	ASM_TYPE_FUNCTION(__interceptor_trampoline_##name); \
89	126	__interceptor_trampoline_##name: \
90		CFI_STARTPROC; \
	127	ASM_STARTPROC; \
91	128	ASM_TAIL_CALL ASM_PREEMPTIBLE_SYM(__interceptor_##name); \
92		CFI_ENDPROC; \
	129	ASM_ENDPROC; \
93	130	ASM_SIZE(__interceptor_trampoline_##name)
94	131	# define ASM_INTERCEPTOR_TRAMPOLINE_SUPPORT 1
95	132	# endif // Architecture supports interceptor trampoline

lib/tsan/sanitizer_common/sanitizer_atomic.h+13-1

...	...	@@ -18,12 +18,24 @@
18	18	namespace __sanitizer {
19	19
20	20	enum memory_order {
	21	// If the __atomic atomic builtins are supported (Clang/GCC), use the
	22	// compiler provided macro values so that we can map the atomic operations
	23	// to __atomic_* directly.
	24	#ifdef __ATOMIC_SEQ_CST
	25	memory_order_relaxed = __ATOMIC_RELAXED,
	26	memory_order_consume = __ATOMIC_CONSUME,
	27	memory_order_acquire = __ATOMIC_ACQUIRE,
	28	memory_order_release = __ATOMIC_RELEASE,
	29	memory_order_acq_rel = __ATOMIC_ACQ_REL,
	30	memory_order_seq_cst = __ATOMIC_SEQ_CST
	31	#else
21	32	memory_order_relaxed = 1 << 0,
22	33	memory_order_consume = 1 << 1,
23	34	memory_order_acquire = 1 << 2,
24	35	memory_order_release = 1 << 3,
25	36	memory_order_acq_rel = 1 << 4,
26	37	memory_order_seq_cst = 1 << 5
	38	#endif
27	39	};
28	40
29	41	struct atomic_uint8_t {
...	...	@@ -49,7 +61,7 @@ struct atomic_uint32_t {
49	61	struct atomic_uint64_t {
50	62	typedef u64 Type;
51	63	// On 32-bit platforms u64 is not necessary aligned on 8 bytes.
52		volatile ALIGNED(8) Type val_dont_use;
	64	alignas(8) volatile Type val_dont_use;
53	65	};
54	66
55	67	struct atomic_uintptr_t {

lib/tsan/sanitizer_common/sanitizer_atomic_clang.h+40-45

...	...	@@ -14,60 +14,63 @@
14	14	#ifndef SANITIZER_ATOMIC_CLANG_H
15	15	#define SANITIZER_ATOMIC_CLANG_H
16	16
17		#if defined(__i386__) \|\| defined(__x86_64__)
18		# include "sanitizer_atomic_clang_x86.h"
19		#else
20		# include "sanitizer_atomic_clang_other.h"
21		#endif
22
23	17	namespace __sanitizer {
24	18
25		// We would like to just use compiler builtin atomic operations
26		// for loads and stores, but they are mostly broken in clang:
27		// - they lead to vastly inefficient code generation
28		// (http://llvm.org/bugs/show_bug.cgi?id=17281)
29		// - 64-bit atomic operations are not implemented on x86_32
30		// (http://llvm.org/bugs/show_bug.cgi?id=15034)
31		// - they are not implemented on ARM
32		// error: undefined reference to '__atomic_load_4'
	19	// We use the compiler builtin atomic operations for loads and stores, which
	20	// generates correct code for all architectures, but may require libatomic
	21	// on platforms where e.g. 64-bit atomics are not supported natively.
33	22
34	23	// See http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html
35	24	// for mappings of the memory model to different processors.
36	25
37		inline void atomic_signal_fence(memory_order) {
	26	inline void atomic_signal_fence(memory_order mo) { __atomic_signal_fence(mo); }
	27
	28	inline void atomic_thread_fence(memory_order mo) { __atomic_thread_fence(mo); }
	29
	30	inline void proc_yield(int cnt) {
	31	__asm__ __volatile__("" ::: "memory");
	32	#if defined(__i386__) \|\| defined(__x86_64__)
	33	for (int i = 0; i < cnt; i++) __asm__ __volatile__("pause");
38	34	__asm__ __volatile__("" ::: "memory");
	35	#endif
39	36	}
40	37
41		inline void atomic_thread_fence(memory_order) {
42		__sync_synchronize();
	38	template <typename T>
	39	inline typename T::Type atomic_load(const volatile T *a, memory_order mo) {
	40	DCHECK(mo == memory_order_relaxed \|\| mo == memory_order_consume \|\|
	41	mo == memory_order_acquire \|\| mo == memory_order_seq_cst);
	42	DCHECK(!((uptr)a % sizeof(*a)));
	43	return __atomic_load_n(&a->val_dont_use, mo);
43	44	}
44	45
45		template<typename T>
46		inline typename T::Type atomic_fetch_add(volatile T *a,
47		typename T::Type v, memory_order mo) {
48		(void)mo;
	46	template <typename T>
	47	inline void atomic_store(volatile T *a, typename T::Type v, memory_order mo) {
	48	DCHECK(mo == memory_order_relaxed \|\| mo == memory_order_release \|\|
	49	mo == memory_order_seq_cst);
49	50	DCHECK(!((uptr)a % sizeof(*a)));
50		return __sync_fetch_and_add(&a->val_dont_use, v);
	51	__atomic_store_n(&a->val_dont_use, v, mo);
51	52	}
52	53
53		template<typename T>
54		inline typename T::Type atomic_fetch_sub(volatile T *a,
55		typename T::Type v, memory_order mo) {
	54	template <typename T>
	55	inline typename T::Type atomic_fetch_add(volatile T *a, typename T::Type v,
	56	memory_order mo) {
	57	DCHECK(!((uptr)a % sizeof(*a)));
	58	return __atomic_fetch_add(&a->val_dont_use, v, mo);
	59	}
	60
	61	template <typename T>
	62	inline typename T::Type atomic_fetch_sub(volatile T *a, typename T::Type v,
	63	memory_order mo) {
56	64	(void)mo;
57	65	DCHECK(!((uptr)a % sizeof(*a)));
58		return __sync_fetch_and_add(&a->val_dont_use, -v);
	66	return __atomic_fetch_sub(&a->val_dont_use, v, mo);
59	67	}
60	68
61		template<typename T>
62		inline typename T::Type atomic_exchange(volatile T *a,
63		typename T::Type v, memory_order mo) {
	69	template <typename T>
	70	inline typename T::Type atomic_exchange(volatile T *a, typename T::Type v,
	71	memory_order mo) {
64	72	DCHECK(!((uptr)a % sizeof(*a)));
65		if (mo & (memory_order_release \| memory_order_acq_rel \| memory_order_seq_cst))
66		__sync_synchronize();
67		v = __sync_lock_test_and_set(&a->val_dont_use, v);
68		if (mo == memory_order_seq_cst)
69		__sync_synchronize();
70		return v;
	73	return __atomic_exchange_n(&a->val_dont_use, v, mo);
71	74	}
72	75
73	76	template <typename T>
...	...	@@ -82,9 +85,8 @@ inline bool atomic_compare_exchange_strong(volatile T a, typename T::Type cmp,
82	85	__ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
83	86	}
84	87
85		template<typename T>
86		inline bool atomic_compare_exchange_weak(volatile T *a,
87		typename T::Type *cmp,
	88	template <typename T>
	89	inline bool atomic_compare_exchange_weak(volatile T a, typename T::Type cmp,
88	90	typename T::Type xchg,
89	91	memory_order mo) {
90	92	return atomic_compare_exchange_strong(a, cmp, xchg, mo);
...	...	@@ -92,13 +94,6 @@ inline bool atomic_compare_exchange_weak(volatile T *a,
92	94
93	95	} // namespace __sanitizer
94	96
95		// This include provides explicit template instantiations for atomic_uint64_t
96		// on MIPS32, which does not directly support 8 byte atomics. It has to
97		// proceed the template definitions above.
98		#if defined(_MIPS_SIM) && defined(_ABIO32) && _MIPS_SIM == _ABIO32
99		# include "sanitizer_atomic_clang_mips.h"
100		#endif
101
102	97	#undef ATOMIC_ORDER
103	98
104	99	#endif // SANITIZER_ATOMIC_CLANG_H

lib/tsan/sanitizer_common/sanitizer_atomic_clang_mips.h deleted-117

...	...	@@ -1,117 +0,0 @@
1		//===-- sanitizer_atomic_clang_mips.h ---------------------------- C++ --===//
2		//
3		// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4		// See https://llvm.org/LICENSE.txt for license information.
5		// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6		//
7		//===----------------------------------------------------------------------===//
8		//
9		// This file is a part of ThreadSanitizer/AddressSanitizer runtime.
10		// Not intended for direct inclusion. Include sanitizer_atomic.h.
11		//
12		//===----------------------------------------------------------------------===//
13
14		#ifndef SANITIZER_ATOMIC_CLANG_MIPS_H
15		#define SANITIZER_ATOMIC_CLANG_MIPS_H
16
17		namespace __sanitizer {
18
19		// MIPS32 does not support atomics > 4 bytes. To address this lack of
20		// functionality, the sanitizer library provides helper methods which use an
21		// internal spin lock mechanism to emulate atomic operations when the size is
22		// 8 bytes.
23		static void __spin_lock(volatile int *lock) {
24		while (__sync_lock_test_and_set(lock, 1))
25		while (*lock) {
26		}
27		}
28
29		static void __spin_unlock(volatile int *lock) { __sync_lock_release(lock); }
30
31		// Make sure the lock is on its own cache line to prevent false sharing.
32		// Put it inside a struct that is aligned and padded to the typical MIPS
33		// cacheline which is 32 bytes.
34		static struct {
35		int lock;
36		char pad[32 - sizeof(int)];
37		} __attribute__((aligned(32))) lock = {0, {0}};
38
39		template <>
40		inline atomic_uint64_t::Type atomic_fetch_add(volatile atomic_uint64_t *ptr,
41		atomic_uint64_t::Type val,
42		memory_order mo) {
43		DCHECK(mo &
44		(memory_order_relaxed \| memory_order_release \| memory_order_seq_cst));
45		DCHECK(!((uptr)ptr % sizeof(*ptr)));
46
47		atomic_uint64_t::Type ret;
48
49		__spin_lock(&lock.lock);
50		ret = (const_cast<atomic_uint64_t::Type volatile >(&ptr->val_dont_use));
51		ptr->val_dont_use = ret + val;
52		__spin_unlock(&lock.lock);
53
54		return ret;
55		}
56
57		template <>
58		inline atomic_uint64_t::Type atomic_fetch_sub(volatile atomic_uint64_t *ptr,
59		atomic_uint64_t::Type val,
60		memory_order mo) {
61		return atomic_fetch_add(ptr, -val, mo);
62		}
63
64		template <>
65		inline bool atomic_compare_exchange_strong(volatile atomic_uint64_t *ptr,
66		atomic_uint64_t::Type *cmp,
67		atomic_uint64_t::Type xchg,
68		memory_order mo) {
69		DCHECK(mo &
70		(memory_order_relaxed \| memory_order_release \| memory_order_seq_cst));
71		DCHECK(!((uptr)ptr % sizeof(*ptr)));
72
73		typedef atomic_uint64_t::Type Type;
74		Type cmpv = *cmp;
75		Type prev;
76		bool ret = false;
77
78		__spin_lock(&lock.lock);
79		prev = (const_cast<Type volatile >(&ptr->val_dont_use));
80		if (prev == cmpv) {
81		ret = true;
82		ptr->val_dont_use = xchg;
83		}
84		__spin_unlock(&lock.lock);
85
86		return ret;
87		}
88
89		template <>
90		inline atomic_uint64_t::Type atomic_load(const volatile atomic_uint64_t *ptr,
91		memory_order mo) {
92		DCHECK(mo &
93		(memory_order_relaxed \| memory_order_release \| memory_order_seq_cst));
94		DCHECK(!((uptr)ptr % sizeof(*ptr)));
95
96		atomic_uint64_t::Type zero = 0;
97		volatile atomic_uint64_t *Newptr =
98		const_cast<volatile atomic_uint64_t *>(ptr);
99		return atomic_fetch_add(Newptr, zero, mo);
100		}
101
102		template <>
103		inline void atomic_store(volatile atomic_uint64_t *ptr, atomic_uint64_t::Type v,
104		memory_order mo) {
105		DCHECK(mo &
106		(memory_order_relaxed \| memory_order_release \| memory_order_seq_cst));
107		DCHECK(!((uptr)ptr % sizeof(*ptr)));
108
109		__spin_lock(&lock.lock);
110		ptr->val_dont_use = v;
111		__spin_unlock(&lock.lock);
112		}
113
114		} // namespace __sanitizer
115
116		#endif // SANITIZER_ATOMIC_CLANG_MIPS_H
117

lib/tsan/sanitizer_common/sanitizer_atomic_clang_other.h deleted-85

...	...	@@ -1,85 +0,0 @@
1		//===-- sanitizer_atomic_clang_other.h --------------------------- C++ --===//
2		//
3		// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4		// See https://llvm.org/LICENSE.txt for license information.
5		// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6		//
7		//===----------------------------------------------------------------------===//
8		//
9		// This file is a part of ThreadSanitizer/AddressSanitizer runtime.
10		// Not intended for direct inclusion. Include sanitizer_atomic.h.
11		//
12		//===----------------------------------------------------------------------===//
13
14		#ifndef SANITIZER_ATOMIC_CLANG_OTHER_H
15		#define SANITIZER_ATOMIC_CLANG_OTHER_H
16
17		namespace __sanitizer {
18
19
20		inline void proc_yield(int cnt) {
21		__asm__ __volatile__("" ::: "memory");
22		}
23
24		template<typename T>
25		inline typename T::Type atomic_load(
26		const volatile T *a, memory_order mo) {
27		DCHECK(mo & (memory_order_relaxed \| memory_order_consume
28		\| memory_order_acquire \| memory_order_seq_cst));
29		DCHECK(!((uptr)a % sizeof(*a)));
30		typename T::Type v;
31
32		if (sizeof(a) < 8 \|\| sizeof(void) == 8) {
33		// Assume that aligned loads are atomic.
34		if (mo == memory_order_relaxed) {
35		v = a->val_dont_use;
36		} else if (mo == memory_order_consume) {
37		// Assume that processor respects data dependencies
38		// (and that compiler won't break them).
39		__asm__ __volatile__("" ::: "memory");
40		v = a->val_dont_use;
41		__asm__ __volatile__("" ::: "memory");
42		} else if (mo == memory_order_acquire) {
43		__asm__ __volatile__("" ::: "memory");
44		v = a->val_dont_use;
45		__sync_synchronize();
46		} else { // seq_cst
47		// E.g. on POWER we need a hw fence even before the store.
48		__sync_synchronize();
49		v = a->val_dont_use;
50		__sync_synchronize();
51		}
52		} else {
53		__atomic_load(const_cast<typename T::Type volatile *>(&a->val_dont_use), &v,
54		__ATOMIC_SEQ_CST);
55		}
56		return v;
57		}
58
59		template<typename T>
60		inline void atomic_store(volatile T *a, typename T::Type v, memory_order mo) {
61		DCHECK(mo & (memory_order_relaxed \| memory_order_release
62		\| memory_order_seq_cst));
63		DCHECK(!((uptr)a % sizeof(*a)));
64
65		if (sizeof(a) < 8 \|\| sizeof(void) == 8) {
66		// Assume that aligned loads are atomic.
67		if (mo == memory_order_relaxed) {
68		a->val_dont_use = v;
69		} else if (mo == memory_order_release) {
70		__sync_synchronize();
71		a->val_dont_use = v;
72		__asm__ __volatile__("" ::: "memory");
73		} else { // seq_cst
74		__sync_synchronize();
75		a->val_dont_use = v;
76		__sync_synchronize();
77		}
78		} else {
79		__atomic_store(&a->val_dont_use, &v, __ATOMIC_SEQ_CST);
80		}
81		}
82
83		} // namespace __sanitizer
84
85		#endif // #ifndef SANITIZER_ATOMIC_CLANG_OTHER_H

lib/tsan/sanitizer_common/sanitizer_atomic_clang_x86.h deleted-113

...	...	@@ -1,113 +0,0 @@
1		//===-- sanitizer_atomic_clang_x86.h ----------------------------- C++ --===//
2		//
3		// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4		// See https://llvm.org/LICENSE.txt for license information.
5		// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6		//
7		//===----------------------------------------------------------------------===//
8		//
9		// This file is a part of ThreadSanitizer/AddressSanitizer runtime.
10		// Not intended for direct inclusion. Include sanitizer_atomic.h.
11		//
12		//===----------------------------------------------------------------------===//
13
14		#ifndef SANITIZER_ATOMIC_CLANG_X86_H
15		#define SANITIZER_ATOMIC_CLANG_X86_H
16
17		namespace __sanitizer {
18
19		inline void proc_yield(int cnt) {
20		__asm__ __volatile__("" ::: "memory");
21		for (int i = 0; i < cnt; i++)
22		__asm__ __volatile__("pause");
23		__asm__ __volatile__("" ::: "memory");
24		}
25
26		template<typename T>
27		inline typename T::Type atomic_load(
28		const volatile T *a, memory_order mo) {
29		DCHECK(mo & (memory_order_relaxed \| memory_order_consume
30		\| memory_order_acquire \| memory_order_seq_cst));
31		DCHECK(!((uptr)a % sizeof(*a)));
32		typename T::Type v;
33
34		if (sizeof(a) < 8 \|\| sizeof(void) == 8) {
35		// Assume that aligned loads are atomic.
36		if (mo == memory_order_relaxed) {
37		v = a->val_dont_use;
38		} else if (mo == memory_order_consume) {
39		// Assume that processor respects data dependencies
40		// (and that compiler won't break them).
41		__asm__ __volatile__("" ::: "memory");
42		v = a->val_dont_use;
43		__asm__ __volatile__("" ::: "memory");
44		} else if (mo == memory_order_acquire) {
45		__asm__ __volatile__("" ::: "memory");
46		v = a->val_dont_use;
47		// On x86 loads are implicitly acquire.
48		__asm__ __volatile__("" ::: "memory");
49		} else { // seq_cst
50		// On x86 plain MOV is enough for seq_cst store.
51		__asm__ __volatile__("" ::: "memory");
52		v = a->val_dont_use;
53		__asm__ __volatile__("" ::: "memory");
54		}
55		} else {
56		// 64-bit load on 32-bit platform.
57		__asm__ __volatile__(
58		"movq %1, %%mm0;" // Use mmx reg for 64-bit atomic moves
59		"movq %%mm0, %0;" // (ptr could be read-only)
60		"emms;" // Empty mmx state/Reset FP regs
61		: "=m" (v)
62		: "m" (a->val_dont_use)
63		: // mark the mmx registers as clobbered
64		#ifdef __MMX__
65		"mm0", "mm1", "mm2", "mm3", "mm4", "mm5", "mm6", "mm7",
66		#endif // #ifdef __MMX__
67		"memory");
68		}
69		return v;
70		}
71
72		template<typename T>
73		inline void atomic_store(volatile T *a, typename T::Type v, memory_order mo) {
74		DCHECK(mo & (memory_order_relaxed \| memory_order_release
75		\| memory_order_seq_cst));
76		DCHECK(!((uptr)a % sizeof(*a)));
77
78		if (sizeof(a) < 8 \|\| sizeof(void) == 8) {
79		// Assume that aligned loads are atomic.
80		if (mo == memory_order_relaxed) {
81		a->val_dont_use = v;
82		} else if (mo == memory_order_release) {
83		// On x86 stores are implicitly release.
84		__asm__ __volatile__("" ::: "memory");
85		a->val_dont_use = v;
86		__asm__ __volatile__("" ::: "memory");
87		} else { // seq_cst
88		// On x86 stores are implicitly release.
89		__asm__ __volatile__("" ::: "memory");
90		a->val_dont_use = v;
91		__sync_synchronize();
92		}
93		} else {
94		// 64-bit store on 32-bit platform.
95		__asm__ __volatile__(
96		"movq %1, %%mm0;" // Use mmx reg for 64-bit atomic moves
97		"movq %%mm0, %0;"
98		"emms;" // Empty mmx state/Reset FP regs
99		: "=m" (a->val_dont_use)
100		: "m" (v)
101		: // mark the mmx registers as clobbered
102		#ifdef __MMX__
103		"mm0", "mm1", "mm2", "mm3", "mm4", "mm5", "mm6", "mm7",
104		#endif // #ifdef __MMX__
105		"memory");
106		if (mo == memory_order_seq_cst)
107		__sync_synchronize();
108		}
109		}
110
111		} // namespace __sanitizer
112
113		#endif // #ifndef SANITIZER_ATOMIC_CLANG_X86_H

lib/tsan/sanitizer_common/sanitizer_atomic_msvc.h+4-4

...	...	@@ -70,8 +70,8 @@ inline void proc_yield(int cnt) {
70	70	template<typename T>
71	71	inline typename T::Type atomic_load(
72	72	const volatile T *a, memory_order mo) {
73		DCHECK(mo & (memory_order_relaxed \| memory_order_consume
74		\| memory_order_acquire \| memory_order_seq_cst));
	73	DCHECK(mo == memory_order_relaxed \|\| mo == memory_order_consume \|\|
	74	mo == memory_order_acquire \|\| mo == memory_order_seq_cst);
75	75	DCHECK(!((uptr)a % sizeof(*a)));
76	76	typename T::Type v;
77	77	// FIXME(dvyukov): 64-bit load is not atomic on 32-bits.
...	...	@@ -87,8 +87,8 @@ inline typename T::Type atomic_load(
87	87
88	88	template<typename T>
89	89	inline void atomic_store(volatile T *a, typename T::Type v, memory_order mo) {
90		DCHECK(mo & (memory_order_relaxed \| memory_order_release
91		\| memory_order_seq_cst));
	90	DCHECK(mo == memory_order_relaxed \|\| mo == memory_order_release \|\|
	91	mo == memory_order_seq_cst);
92	92	DCHECK(!((uptr)a % sizeof(*a)));
93	93	// FIXME(dvyukov): 64-bit store is not atomic on 32-bits.
94	94	if (mo == memory_order_relaxed) {

lib/tsan/sanitizer_common/sanitizer_bitvector.h+4-4

...	...	@@ -321,23 +321,23 @@ class TwoLevelBitVector {
321	321	};
322	322
323	323	private:
324		void check(uptr idx) const { CHECK_LE(idx, size()); }
	324	void check(uptr idx) const { CHECK_LT(idx, size()); }
325	325
326	326	uptr idx0(uptr idx) const {
327	327	uptr res = idx / (BV::kSize * BV::kSize);
328		CHECK_LE(res, kLevel1Size);
	328	CHECK_LT(res, kLevel1Size);
329	329	return res;
330	330	}
331	331
332	332	uptr idx1(uptr idx) const {
333	333	uptr res = (idx / BV::kSize) % BV::kSize;
334		CHECK_LE(res, BV::kSize);
	334	CHECK_LT(res, BV::kSize);
335	335	return res;
336	336	}
337	337
338	338	uptr idx2(uptr idx) const {
339	339	uptr res = idx % BV::kSize;
340		CHECK_LE(res, BV::kSize);
	340	CHECK_LT(res, BV::kSize);
341	341	return res;
342	342	}
343	343

lib/tsan/sanitizer_common/sanitizer_chained_origin_depot.cpp+4-2

...	...	@@ -139,9 +139,11 @@ u32 ChainedOriginDepot::Get(u32 id, u32 *other) {
139	139	return desc.here_id;
140	140	}
141	141
142		void ChainedOriginDepot::LockAll() { depot.LockAll(); }
	142	void ChainedOriginDepot::LockBeforeFork() { depot.LockBeforeFork(); }
143	143
144		void ChainedOriginDepot::UnlockAll() { depot.UnlockAll(); }
	144	void ChainedOriginDepot::UnlockAfterFork(bool fork_child) {
	145	depot.UnlockAfterFork(fork_child);
	146	}
145	147
146	148	void ChainedOriginDepot::TestOnlyUnmap() { depot.TestOnlyUnmap(); }
147	149

lib/tsan/sanitizer_common/sanitizer_chained_origin_depot.h+2-2

...	...	@@ -32,8 +32,8 @@ class ChainedOriginDepot {
32	32	// Retrieves the stored StackDepot ID for the given origin ID.
33	33	u32 Get(u32 id, u32 *other);
34	34
35		void LockAll();
36		void UnlockAll();
	35	void LockBeforeFork();
	36	void UnlockAfterFork(bool fork_child);
37	37	void TestOnlyUnmap();
38	38
39	39	private:

lib/tsan/sanitizer_common/sanitizer_common.cpp+17-3

...	...	@@ -115,8 +115,9 @@ void ReportErrorSummary(const char error_message, const char alt_tool_name) {
115	115	if (!common_flags()->print_summary)
116	116	return;
117	117	InternalScopedString buff;
118		buff.append("SUMMARY: %s: %s",
119		alt_tool_name ? alt_tool_name : SanitizerToolName, error_message);
	118	buff.AppendF("SUMMARY: %s: %s",
	119	alt_tool_name ? alt_tool_name : SanitizerToolName,
	120	error_message);
120	121	__sanitizer_report_error_summary(buff.data());
121	122	}
122	123
...	...	@@ -346,7 +347,13 @@ void RunMallocHooks(void *ptr, uptr size) {
346	347	}
347	348	}
348	349
349		void RunFreeHooks(void *ptr) {
	350	// Returns '1' if the call to free() should be ignored (based on
	351	// __sanitizer_ignore_free_hook), or '0' otherwise.
	352	int RunFreeHooks(void *ptr) {
	353	if (__sanitizer_ignore_free_hook(ptr)) {
	354	return 1;
	355	}
	356
350	357	__sanitizer_free_hook(ptr);
351	358	for (int i = 0; i < kMaxMallocFreeHooks; i++) {
352	359	auto hook = MFHooks[i].free_hook;
...	...	@@ -354,6 +361,8 @@ void RunFreeHooks(void *ptr) {
354	361	break;
355	362	hook(ptr);
356	363	}
	364
	365	return 0;
357	366	}
358	367
359	368	static int InstallMallocFreeHooks(void (malloc_hook)(const void , uptr),
...	...	@@ -418,4 +427,9 @@ SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_free_hook, void *ptr) {
418	427	(void)ptr;
419	428	}
420	429
	430	SANITIZER_INTERFACE_WEAK_DEF(int, __sanitizer_ignore_free_hook, void *ptr) {
	431	(void)ptr;
	432	return 0;
	433	}
	434
421	435	} // extern "C"

lib/tsan/sanitizer_common/sanitizer_common.h+28-17

...	...	@@ -32,6 +32,7 @@ struct AddressInfo;
32	32	struct BufferedStackTrace;
33	33	struct SignalContext;
34	34	struct StackTrace;
	35	struct SymbolizedStack;
35	36
36	37	// Constants.
37	38	const uptr kWordSize = SANITIZER_WORDSIZE / 8;
...	...	@@ -59,14 +60,10 @@ inline int Verbosity() {
59	60	return atomic_load(&current_verbosity, memory_order_relaxed);
60	61	}
61	62
62		#if SANITIZER_ANDROID
63		inline uptr GetPageSize() {
64		// Android post-M sysconf(_SC_PAGESIZE) crashes if called from .preinit_array.
65		return 4096;
66		}
67		inline uptr GetPageSizeCached() {
68		return 4096;
69		}
	63	#if SANITIZER_ANDROID && !defined(__aarch64__)
	64	// 32-bit Android only has 4k pages.
	65	inline uptr GetPageSize() { return 4096; }
	66	inline uptr GetPageSizeCached() { return 4096; }
70	67	#else
71	68	uptr GetPageSize();
72	69	extern uptr PageSizeCached;
...	...	@@ -76,6 +73,7 @@ inline uptr GetPageSizeCached() {
76	73	return PageSizeCached;
77	74	}
78	75	#endif
	76
79	77	uptr GetMmapGranularity();
80	78	uptr GetMaxVirtualAddress();
81	79	uptr GetMaxUserVirtualAddress();
...	...	@@ -90,10 +88,11 @@ void GetThreadStackAndTls(bool main, uptr stk_addr, uptr stk_size,
90	88
91	89	// Memory management
92	90	void MmapOrDie(uptr size, const char mem_type, bool raw_report = false);
	91
93	92	inline void MmapOrDieQuietly(uptr size, const char mem_type) {
94	93	return MmapOrDie(size, mem_type, /raw_report/ true);
95	94	}
96		void UnmapOrDie(void *addr, uptr size);
	95	void UnmapOrDie(void *addr, uptr size, bool raw_report = false);
97	96	// Behaves just like MmapOrDie, but tolerates out of memory condition, in that
98	97	// case returns nullptr.
99	98	void MmapOrDieOnFatalError(uptr size, const char mem_type);
...	...	@@ -138,7 +137,8 @@ void UnmapFromTo(uptr from, uptr to);
138	137	// shadow_size_bytes bytes on the right, which on linux is mapped no access.
139	138	// The high_mem_end may be updated if the original shadow size doesn't fit.
140	139	uptr MapDynamicShadow(uptr shadow_size_bytes, uptr shadow_scale,
141		uptr min_shadow_base_alignment, uptr &high_mem_end);
	140	uptr min_shadow_base_alignment, uptr &high_mem_end,
	141	uptr granularity);
142	142
143	143	// Let S = max(shadow_size, num_aliases * alias_size, ring_buffer_size).
144	144	// Reserves 2*S bytes of address space to the right of the returned address and
...	...	@@ -177,7 +177,7 @@ bool DontDumpShadowMemory(uptr addr, uptr length);
177	177	// Check if the built VMA size matches the runtime one.
178	178	void CheckVMASize();
179	179	void RunMallocHooks(void *ptr, uptr size);
180		void RunFreeHooks(void *ptr);
	180	int RunFreeHooks(void *ptr);
181	181
182	182	class ReservedAddressRange {
183	183	public:
...	...	@@ -208,6 +208,11 @@ void ParseUnixMemoryProfile(fill_profile_f cb, uptr stats, char smaps,
208	208	// Simple low-level (mmap-based) allocator for internal use. Doesn't have
209	209	// constructor, so all instances of LowLevelAllocator should be
210	210	// linker initialized.
	211	//
	212	// NOTE: Users should instead use the singleton provided via
	213	// `GetGlobalLowLevelAllocator()` rather than create a new one. This way, the
	214	// number of mmap fragments can be reduced and use the same contiguous mmap
	215	// provided by this singleton.
211	216	class LowLevelAllocator {
212	217	public:
213	218	// Requires an external lock.
...	...	@@ -224,6 +229,8 @@ typedef void (*LowLevelAllocateCallback)(uptr ptr, uptr size);
224	229	// Passing NULL removes the callback.
225	230	void SetLowLevelAllocateCallback(LowLevelAllocateCallback callback);
226	231
	232	LowLevelAllocator &GetGlobalLowLevelAllocator();
	233
227	234	// IO
228	235	void CatastrophicErrorWrite(const char *buffer, uptr length);
229	236	void RawWrite(const char *buffer);
...	...	@@ -386,6 +393,8 @@ void ReportErrorSummary(const char *error_type, const AddressInfo &info,
386	393	// Same as above, but obtains AddressInfo by symbolizing top stack trace frame.
387	394	void ReportErrorSummary(const char error_type, const StackTrace trace,
388	395	const char *alt_tool_name = nullptr);
	396	// Skips frames which we consider internal and not usefull to the users.
	397	const SymbolizedStack SkipInternalFrames(const SymbolizedStack frames);
389	398
390	399	void ReportMmapWriteExec(int prot, int mflags);
391	400
...	...	@@ -500,7 +509,7 @@ inline int ToLower(int c) {
500	509	// A low-level vector based on mmap. May incur a significant memory overhead for
501	510	// small vectors.
502	511	// WARNING: The current implementation supports only POD types.
503		template<typename T>
	512	template <typename T, bool raw_report = false>
504	513	class InternalMmapVectorNoCtor {
505	514	public:
506	515	using value_type = T;
...	...	@@ -510,7 +519,7 @@ class InternalMmapVectorNoCtor {
510	519	data_ = 0;
511	520	reserve(initial_capacity);
512	521	}
513		void Destroy() { UnmapOrDie(data_, capacity_bytes_); }
	522	void Destroy() { UnmapOrDie(data_, capacity_bytes_, raw_report); }
514	523	T &operator[](uptr i) {
515	524	CHECK_LT(i, size_);
516	525	return data_[i];
...	...	@@ -586,9 +595,10 @@ class InternalMmapVectorNoCtor {
586	595	CHECK_LE(size_, new_capacity);
587	596	uptr new_capacity_bytes =
588	597	RoundUpTo(new_capacity * sizeof(T), GetPageSizeCached());
589		T new_data = (T )MmapOrDie(new_capacity_bytes, "InternalMmapVector");
	598	T *new_data =
	599	(T *)MmapOrDie(new_capacity_bytes, "InternalMmapVector", raw_report);
590	600	internal_memcpy(new_data, data_, size_ * sizeof(T));
591		UnmapOrDie(data_, capacity_bytes_);
	601	UnmapOrDie(data_, capacity_bytes_, raw_report);
592	602	data_ = new_data;
593	603	capacity_bytes_ = new_capacity_bytes;
594	604	}
...	...	@@ -636,7 +646,8 @@ class InternalScopedString {
636	646	buffer_.resize(1);
637	647	buffer_[0] = '\0';
638	648	}
639		void append(const char *format, ...) FORMAT(2, 3);
	649	void Append(const char *str);
	650	void AppendF(const char *format, ...) FORMAT(2, 3);
640	651	const char *data() const { return buffer_.data(); }
641	652	char *data() { return buffer_.data(); }
642	653
...	...	@@ -1086,7 +1097,7 @@ inline u32 GetNumberOfCPUsCached() {
1086	1097
1087	1098	} // namespace __sanitizer
1088	1099
1089		inline void *operator new(__sanitizer::operator_new_size_type size,
	1100	inline void *operator new(__sanitizer::usize size,
1090	1101	__sanitizer::LowLevelAllocator &alloc) {
1091	1102	return alloc.Allocate(size);
1092	1103	}

lib/tsan/sanitizer_common/sanitizer_common_interceptors.inc+137-66

...	...	@@ -33,16 +33,17 @@
33	33	// COMMON_INTERCEPTOR_STRERROR
34	34	//===----------------------------------------------------------------------===//
35	35
	36	#include <stdarg.h>
	37
36	38	#include "interception/interception.h"
37	39	#include "sanitizer_addrhashmap.h"
	40	#include "sanitizer_dl.h"
38	41	#include "sanitizer_errno.h"
39	42	#include "sanitizer_placement_new.h"
40	43	#include "sanitizer_platform_interceptors.h"
41	44	#include "sanitizer_symbolizer.h"
42	45	#include "sanitizer_tls_get_addr.h"
43	46
44		#include <stdarg.h>
45
46	47	#if SANITIZER_INTERCEPTOR_HOOKS
47	48	#define CALL_WEAK_INTERCEPTOR_HOOK(f, ...) f(__VA_ARGS__);
48	49	#define DECLARE_WEAK_INTERCEPTOR_HOOK(f, ...) \
...	...	@@ -445,11 +446,13 @@ INTERCEPTOR(char, textdomain, const char domainname) {
445	446	#define INIT_TEXTDOMAIN
446	447	#endif
447	448
448		#if SANITIZER_INTERCEPT_STRCMP
	449	#if SANITIZER_INTERCEPT_STRCMP \|\| SANITIZER_INTERCEPT_MEMCMP
449	450	static inline int CharCmpX(unsigned char c1, unsigned char c2) {
450	451	return (c1 == c2) ? 0 : (c1 < c2) ? -1 : 1;
451	452	}
	453	#endif
452	454
	455	#if SANITIZER_INTERCEPT_STRCMP
453	456	DECLARE_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_strcmp, uptr called_pc,
454	457	const char s1, const char s2, int result)
455	458
...	...	@@ -971,7 +974,7 @@ INTERCEPTOR(SSIZE_T, read, int fd, void *ptr, SIZE_T count) {
971	974	// FIXME: under ASan the call below may write to freed memory and corrupt
972	975	// its metadata. See
973	976	// https://github.com/google/sanitizers/issues/321.
974		SSIZE_T res = REAL(read)(fd, ptr, count);
	977	SSIZE_T res = COMMON_INTERCEPTOR_BLOCK_REAL(read)(fd, ptr, count);
975	978	if (res > 0) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, res);
976	979	if (res >= 0 && fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);
977	980	return res;
...	...	@@ -1006,7 +1009,7 @@ INTERCEPTOR(SSIZE_T, pread, int fd, void *ptr, SIZE_T count, OFF_T offset) {
1006	1009	// FIXME: under ASan the call below may write to freed memory and corrupt
1007	1010	// its metadata. See
1008	1011	// https://github.com/google/sanitizers/issues/321.
1009		SSIZE_T res = REAL(pread)(fd, ptr, count, offset);
	1012	SSIZE_T res = COMMON_INTERCEPTOR_BLOCK_REAL(pread)(fd, ptr, count, offset);
1010	1013	if (res > 0) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, res);
1011	1014	if (res >= 0 && fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);
1012	1015	return res;
...	...	@@ -1024,7 +1027,7 @@ INTERCEPTOR(SSIZE_T, pread64, int fd, void *ptr, SIZE_T count, OFF64_T offset) {
1024	1027	// FIXME: under ASan the call below may write to freed memory and corrupt
1025	1028	// its metadata. See
1026	1029	// https://github.com/google/sanitizers/issues/321.
1027		SSIZE_T res = REAL(pread64)(fd, ptr, count, offset);
	1030	SSIZE_T res = COMMON_INTERCEPTOR_BLOCK_REAL(pread64)(fd, ptr, count, offset);
1028	1031	if (res > 0) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, res);
1029	1032	if (res >= 0 && fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);
1030	1033	return res;
...	...	@@ -1040,7 +1043,7 @@ INTERCEPTOR_WITH_SUFFIX(SSIZE_T, readv, int fd, __sanitizer_iovec *iov,
1040	1043	void *ctx;
1041	1044	COMMON_INTERCEPTOR_ENTER(ctx, readv, fd, iov, iovcnt);
1042	1045	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
1043		SSIZE_T res = REAL(readv)(fd, iov, iovcnt);
	1046	SSIZE_T res = COMMON_INTERCEPTOR_BLOCK_REAL(readv)(fd, iov, iovcnt);
1044	1047	if (res > 0) write_iovec(ctx, iov, iovcnt, res);
1045	1048	if (res >= 0 && fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);
1046	1049	return res;
...	...	@@ -1056,7 +1059,7 @@ INTERCEPTOR(SSIZE_T, preadv, int fd, __sanitizer_iovec *iov, int iovcnt,
1056	1059	void *ctx;
1057	1060	COMMON_INTERCEPTOR_ENTER(ctx, preadv, fd, iov, iovcnt, offset);
1058	1061	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
1059		SSIZE_T res = REAL(preadv)(fd, iov, iovcnt, offset);
	1062	SSIZE_T res = COMMON_INTERCEPTOR_BLOCK_REAL(preadv)(fd, iov, iovcnt, offset);
1060	1063	if (res > 0) write_iovec(ctx, iov, iovcnt, res);
1061	1064	if (res >= 0 && fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);
1062	1065	return res;
...	...	@@ -1072,7 +1075,8 @@ INTERCEPTOR(SSIZE_T, preadv64, int fd, __sanitizer_iovec *iov, int iovcnt,
1072	1075	void *ctx;
1073	1076	COMMON_INTERCEPTOR_ENTER(ctx, preadv64, fd, iov, iovcnt, offset);
1074	1077	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
1075		SSIZE_T res = REAL(preadv64)(fd, iov, iovcnt, offset);
	1078	SSIZE_T res =
	1079	COMMON_INTERCEPTOR_BLOCK_REAL(preadv64)(fd, iov, iovcnt, offset);
1076	1080	if (res > 0) write_iovec(ctx, iov, iovcnt, res);
1077	1081	if (res >= 0 && fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);
1078	1082	return res;
...	...	@@ -1088,8 +1092,9 @@ INTERCEPTOR(SSIZE_T, write, int fd, void *ptr, SIZE_T count) {
1088	1092	COMMON_INTERCEPTOR_ENTER(ctx, write, fd, ptr, count);
1089	1093	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
1090	1094	if (fd >= 0) COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd);
1091		SSIZE_T res = REAL(write)(fd, ptr, count);
1092		// FIXME: this check should be _before_ the call to REAL(write), not after
	1095	SSIZE_T res = COMMON_INTERCEPTOR_BLOCK_REAL(write)(fd, ptr, count);
	1096	// FIXME: this check should be _before_ the call to
	1097	// COMMON_INTERCEPTOR_BLOCK_REAL(write), not after
1093	1098	if (res > 0) COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, res);
1094	1099	return res;
1095	1100	}
...	...	@@ -1118,7 +1123,7 @@ INTERCEPTOR(SSIZE_T, pwrite, int fd, void *ptr, SIZE_T count, OFF_T offset) {
1118	1123	COMMON_INTERCEPTOR_ENTER(ctx, pwrite, fd, ptr, count, offset);
1119	1124	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
1120	1125	if (fd >= 0) COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd);
1121		SSIZE_T res = REAL(pwrite)(fd, ptr, count, offset);
	1126	SSIZE_T res = COMMON_INTERCEPTOR_BLOCK_REAL(pwrite)(fd, ptr, count, offset);
1122	1127	if (res > 0) COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, res);
1123	1128	return res;
1124	1129	}
...	...	@@ -1134,7 +1139,7 @@ INTERCEPTOR(SSIZE_T, pwrite64, int fd, void *ptr, OFF64_T count,
1134	1139	COMMON_INTERCEPTOR_ENTER(ctx, pwrite64, fd, ptr, count, offset);
1135	1140	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
1136	1141	if (fd >= 0) COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd);
1137		SSIZE_T res = REAL(pwrite64)(fd, ptr, count, offset);
	1142	SSIZE_T res = COMMON_INTERCEPTOR_BLOCK_REAL(pwrite64)(fd, ptr, count, offset);
1138	1143	if (res > 0) COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, res);
1139	1144	return res;
1140	1145	}
...	...	@@ -1150,7 +1155,7 @@ INTERCEPTOR_WITH_SUFFIX(SSIZE_T, writev, int fd, __sanitizer_iovec *iov,
1150	1155	COMMON_INTERCEPTOR_ENTER(ctx, writev, fd, iov, iovcnt);
1151	1156	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
1152	1157	if (fd >= 0) COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd);
1153		SSIZE_T res = REAL(writev)(fd, iov, iovcnt);
	1158	SSIZE_T res = COMMON_INTERCEPTOR_BLOCK_REAL(writev)(fd, iov, iovcnt);
1154	1159	if (res > 0) read_iovec(ctx, iov, iovcnt, res);
1155	1160	return res;
1156	1161	}
...	...	@@ -1166,7 +1171,7 @@ INTERCEPTOR(SSIZE_T, pwritev, int fd, __sanitizer_iovec *iov, int iovcnt,
1166	1171	COMMON_INTERCEPTOR_ENTER(ctx, pwritev, fd, iov, iovcnt, offset);
1167	1172	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
1168	1173	if (fd >= 0) COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd);
1169		SSIZE_T res = REAL(pwritev)(fd, iov, iovcnt, offset);
	1174	SSIZE_T res = COMMON_INTERCEPTOR_BLOCK_REAL(pwritev)(fd, iov, iovcnt, offset);
1170	1175	if (res > 0) read_iovec(ctx, iov, iovcnt, res);
1171	1176	return res;
1172	1177	}
...	...	@@ -1182,7 +1187,8 @@ INTERCEPTOR(SSIZE_T, pwritev64, int fd, __sanitizer_iovec *iov, int iovcnt,
1182	1187	COMMON_INTERCEPTOR_ENTER(ctx, pwritev64, fd, iov, iovcnt, offset);
1183	1188	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
1184	1189	if (fd >= 0) COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd);
1185		SSIZE_T res = REAL(pwritev64)(fd, iov, iovcnt, offset);
	1190	SSIZE_T res =
	1191	COMMON_INTERCEPTOR_BLOCK_REAL(pwritev64)(fd, iov, iovcnt, offset);
1186	1192	if (res > 0) read_iovec(ctx, iov, iovcnt, res);
1187	1193	return res;
1188	1194	}
...	...	@@ -1245,6 +1251,7 @@ INTERCEPTOR(int, prctl, int option, unsigned long arg2, unsigned long arg3,
1245	1251	void *ctx;
1246	1252	COMMON_INTERCEPTOR_ENTER(ctx, prctl, option, arg2, arg3, arg4, arg5);
1247	1253	static const int PR_SET_NAME = 15;
	1254	static const int PR_GET_NAME = 16;
1248	1255	static const int PR_SET_VMA = 0x53564d41;
1249	1256	static const int PR_SCHED_CORE = 62;
1250	1257	static const int PR_SCHED_CORE_GET = 0;
...	...	@@ -1258,7 +1265,11 @@ INTERCEPTOR(int, prctl, int option, unsigned long arg2, unsigned long arg3,
1258	1265	internal_strncpy(buff, (char *)arg2, 15);
1259	1266	buff[15] = 0;
1260	1267	COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, buff);
1261		} else if (res != -1 && option == PR_SCHED_CORE && arg2 == PR_SCHED_CORE_GET) {
	1268	} else if (res == 0 && option == PR_GET_NAME) {
	1269	char name = (char )arg2;
	1270	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, name, internal_strlen(name) + 1);
	1271	} else if (res != -1 && option == PR_SCHED_CORE &&
	1272	arg2 == PR_SCHED_CORE_GET) {
1262	1273	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, (u64*)(arg5), sizeof(u64));
1263	1274	}
1264	1275	return res;
...	...	@@ -2546,7 +2557,7 @@ INTERCEPTOR_WITH_SUFFIX(int, wait, int *status) {
2546	2557	// FIXME: under ASan the call below may write to freed memory and corrupt
2547	2558	// its metadata. See
2548	2559	// https://github.com/google/sanitizers/issues/321.
2549		int res = REAL(wait)(status);
	2560	int res = COMMON_INTERCEPTOR_BLOCK_REAL(wait)(status);
2550	2561	if (res != -1 && status)
2551	2562	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, status, sizeof(*status));
2552	2563	return res;
...	...	@@ -2564,7 +2575,7 @@ INTERCEPTOR_WITH_SUFFIX(int, waitid, int idtype, int id, void *infop,
2564	2575	// FIXME: under ASan the call below may write to freed memory and corrupt
2565	2576	// its metadata. See
2566	2577	// https://github.com/google/sanitizers/issues/321.
2567		int res = REAL(waitid)(idtype, id, infop, options);
	2578	int res = COMMON_INTERCEPTOR_BLOCK_REAL(waitid)(idtype, id, infop, options);
2568	2579	if (res != -1 && infop)
2569	2580	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, infop, siginfo_t_sz);
2570	2581	return res;
...	...	@@ -2575,7 +2586,7 @@ INTERCEPTOR_WITH_SUFFIX(int, waitpid, int pid, int *status, int options) {
2575	2586	// FIXME: under ASan the call below may write to freed memory and corrupt
2576	2587	// its metadata. See
2577	2588	// https://github.com/google/sanitizers/issues/321.
2578		int res = REAL(waitpid)(pid, status, options);
	2589	int res = COMMON_INTERCEPTOR_BLOCK_REAL(waitpid)(pid, status, options);
2579	2590	if (res != -1 && status)
2580	2591	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, status, sizeof(*status));
2581	2592	return res;
...	...	@@ -2586,7 +2597,7 @@ INTERCEPTOR(int, wait3, int status, int options, void rusage) {
2586	2597	// FIXME: under ASan the call below may write to freed memory and corrupt
2587	2598	// its metadata. See
2588	2599	// https://github.com/google/sanitizers/issues/321.
2589		int res = REAL(wait3)(status, options, rusage);
	2600	int res = COMMON_INTERCEPTOR_BLOCK_REAL(wait3)(status, options, rusage);
2590	2601	if (res != -1) {
2591	2602	if (status) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, status, sizeof(*status));
2592	2603	if (rusage) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, rusage, struct_rusage_sz);
...	...	@@ -2600,7 +2611,8 @@ INTERCEPTOR(int, __wait4, int pid, int status, int options, void rusage) {
2600	2611	// FIXME: under ASan the call below may write to freed memory and corrupt
2601	2612	// its metadata. See
2602	2613	// https://github.com/google/sanitizers/issues/321.
2603		int res = REAL(__wait4)(pid, status, options, rusage);
	2614	int res =
	2615	COMMON_INTERCEPTOR_BLOCK_REAL(__wait4)(pid, status, options, rusage);
2604	2616	if (res != -1) {
2605	2617	if (status) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, status, sizeof(*status));
2606	2618	if (rusage) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, rusage, struct_rusage_sz);
...	...	@@ -2615,7 +2627,7 @@ INTERCEPTOR(int, wait4, int pid, int status, int options, void rusage) {
2615	2627	// FIXME: under ASan the call below may write to freed memory and corrupt
2616	2628	// its metadata. See
2617	2629	// https://github.com/google/sanitizers/issues/321.
2618		int res = REAL(wait4)(pid, status, options, rusage);
	2630	int res = COMMON_INTERCEPTOR_BLOCK_REAL(wait4)(pid, status, options, rusage);
2619	2631	if (res != -1) {
2620	2632	if (status) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, status, sizeof(*status));
2621	2633	if (rusage) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, rusage, struct_rusage_sz);
...	...	@@ -2993,7 +3005,7 @@ INTERCEPTOR(int, accept, int fd, void addr, unsigned addrlen) {
2993	3005	COMMON_INTERCEPTOR_READ_RANGE(ctx, addrlen, sizeof(*addrlen));
2994	3006	addrlen0 = *addrlen;
2995	3007	}
2996		int fd2 = REAL(accept)(fd, addr, addrlen);
	3008	int fd2 = COMMON_INTERCEPTOR_BLOCK_REAL(accept)(fd, addr, addrlen);
2997	3009	if (fd2 >= 0) {
2998	3010	if (fd >= 0) COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, fd2);
2999	3011	if (addr && addrlen)
...	...	@@ -3018,7 +3030,7 @@ INTERCEPTOR(int, accept4, int fd, void addr, unsigned addrlen, int f) {
3018	3030	// FIXME: under ASan the call below may write to freed memory and corrupt
3019	3031	// its metadata. See
3020	3032	// https://github.com/google/sanitizers/issues/321.
3021		int fd2 = REAL(accept4)(fd, addr, addrlen, f);
	3033	int fd2 = COMMON_INTERCEPTOR_BLOCK_REAL(accept4)(fd, addr, addrlen, f);
3022	3034	if (fd2 >= 0) {
3023	3035	if (fd >= 0) COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, fd2);
3024	3036	if (addr && addrlen)
...	...	@@ -3042,7 +3054,7 @@ INTERCEPTOR(int, paccept, int fd, void addr, unsigned addrlen,
3042	3054	addrlen0 = *addrlen;
3043	3055	}
3044	3056	if (set) COMMON_INTERCEPTOR_READ_RANGE(ctx, set, sizeof(*set));
3045		int fd2 = REAL(paccept)(fd, addr, addrlen, set, f);
	3057	int fd2 = COMMON_INTERCEPTOR_BLOCK_REAL(paccept)(fd, addr, addrlen, set, f);
3046	3058	if (fd2 >= 0) {
3047	3059	if (fd >= 0) COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, fd2);
3048	3060	if (addr && addrlen)
...	...	@@ -3123,7 +3135,7 @@ INTERCEPTOR(SSIZE_T, recvmsg, int fd, struct __sanitizer_msghdr *msg,
3123	3135	// FIXME: under ASan the call below may write to freed memory and corrupt
3124	3136	// its metadata. See
3125	3137	// https://github.com/google/sanitizers/issues/321.
3126		SSIZE_T res = REAL(recvmsg)(fd, msg, flags);
	3138	SSIZE_T res = COMMON_INTERCEPTOR_BLOCK_REAL(recvmsg)(fd, msg, flags);
3127	3139	if (res >= 0) {
3128	3140	if (fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);
3129	3141	if (msg) {
...	...	@@ -3144,7 +3156,8 @@ INTERCEPTOR(int, recvmmsg, int fd, struct __sanitizer_mmsghdr *msgvec,
3144	3156	void *ctx;
3145	3157	COMMON_INTERCEPTOR_ENTER(ctx, recvmmsg, fd, msgvec, vlen, flags, timeout);
3146	3158	if (timeout) COMMON_INTERCEPTOR_READ_RANGE(ctx, timeout, struct_timespec_sz);
3147		int res = REAL(recvmmsg)(fd, msgvec, vlen, flags, timeout);
	3159	int res =
	3160	COMMON_INTERCEPTOR_BLOCK_REAL(recvmmsg)(fd, msgvec, vlen, flags, timeout);
3148	3161	if (res >= 0) {
3149	3162	if (fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);
3150	3163	for (int i = 0; i < res; ++i) {
...	...	@@ -3222,7 +3235,7 @@ INTERCEPTOR(SSIZE_T, sendmsg, int fd, struct __sanitizer_msghdr *msg,
3222	3235	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
3223	3236	COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd);
3224	3237	}
3225		SSIZE_T res = REAL(sendmsg)(fd, msg, flags);
	3238	SSIZE_T res = COMMON_INTERCEPTOR_BLOCK_REAL(sendmsg)(fd, msg, flags);
3226	3239	if (common_flags()->intercept_send && res >= 0 && msg)
3227	3240	read_msghdr(ctx, msg, res);
3228	3241	return res;
...	...	@@ -3241,7 +3254,7 @@ INTERCEPTOR(int, sendmmsg, int fd, struct __sanitizer_mmsghdr *msgvec,
3241	3254	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
3242	3255	COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd);
3243	3256	}
3244		int res = REAL(sendmmsg)(fd, msgvec, vlen, flags);
	3257	int res = COMMON_INTERCEPTOR_BLOCK_REAL(sendmmsg)(fd, msgvec, vlen, flags);
3245	3258	if (res >= 0 && msgvec) {
3246	3259	for (int i = 0; i < res; ++i) {
3247	3260	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, &msgvec[i].msg_len,
...	...	@@ -3264,7 +3277,7 @@ INTERCEPTOR(int, msgsnd, int msqid, const void *msgp, SIZE_T msgsz,
3264	3277	COMMON_INTERCEPTOR_ENTER(ctx, msgsnd, msqid, msgp, msgsz, msgflg);
3265	3278	if (msgp)
3266	3279	COMMON_INTERCEPTOR_READ_RANGE(ctx, msgp, sizeof(long) + msgsz);
3267		int res = REAL(msgsnd)(msqid, msgp, msgsz, msgflg);
	3280	int res = COMMON_INTERCEPTOR_BLOCK_REAL(msgsnd)(msqid, msgp, msgsz, msgflg);
3268	3281	return res;
3269	3282	}
3270	3283
...	...	@@ -3272,7 +3285,8 @@ INTERCEPTOR(SSIZE_T, msgrcv, int msqid, void *msgp, SIZE_T msgsz,
3272	3285	long msgtyp, int msgflg) {
3273	3286	void *ctx;
3274	3287	COMMON_INTERCEPTOR_ENTER(ctx, msgrcv, msqid, msgp, msgsz, msgtyp, msgflg);
3275		SSIZE_T len = REAL(msgrcv)(msqid, msgp, msgsz, msgtyp, msgflg);
	3288	SSIZE_T len =
	3289	COMMON_INTERCEPTOR_BLOCK_REAL(msgrcv)(msqid, msgp, msgsz, msgtyp, msgflg);
3276	3290	if (len != -1)
3277	3291	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, msgp, sizeof(long) + len);
3278	3292	return len;
...	...	@@ -6116,7 +6130,7 @@ INTERCEPTOR(int, flopen, const char *path, int flags, ...) {
6116	6130	if (path) {
6117	6131	COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
6118	6132	}
6119		return REAL(flopen)(path, flags, mode);
	6133	return COMMON_INTERCEPTOR_BLOCK_REAL(flopen)(path, flags, mode);
6120	6134	}
6121	6135
6122	6136	INTERCEPTOR(int, flopenat, int dirfd, const char *path, int flags, ...) {
...	...	@@ -6129,7 +6143,7 @@ INTERCEPTOR(int, flopenat, int dirfd, const char *path, int flags, ...) {
6129	6143	if (path) {
6130	6144	COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
6131	6145	}
6132		return REAL(flopenat)(dirfd, path, flags, mode);
	6146	return COMMON_INTERCEPTOR_BLOCK_REAL(flopenat)(dirfd, path, flags, mode);
6133	6147	}
6134	6148
6135	6149	#define INIT_FLOPEN \
...	...	@@ -6305,7 +6319,36 @@ INTERCEPTOR(int, fclose, __sanitizer_FILE *fp) {
6305	6319	INTERCEPTOR(void, dlopen, const char filename, int flag) {
6306	6320	void *ctx;
6307	6321	COMMON_INTERCEPTOR_ENTER_NOIGNORE(ctx, dlopen, filename, flag);
6308		if (filename) COMMON_INTERCEPTOR_READ_STRING(ctx, filename, 0);
	6322
	6323	if (filename) {
	6324	COMMON_INTERCEPTOR_READ_STRING(ctx, filename, 0);
	6325
	6326	# if !SANITIZER_DYNAMIC
	6327	// We care about a very specific use-case: dladdr on
	6328	// statically-linked ASan may return <main program>
	6329	// instead of the library.
	6330	// We therefore only take effect if the sanitizer is statically
	6331	// linked, and we don't bother canonicalizing paths because
	6332	// dladdr should return the same address both times (we assume
	6333	// the user did not canonicalize the result from dladdr).
	6334	if (common_flags()->test_only_replace_dlopen_main_program) {
	6335	VPrintf(1, "dlopen interceptor: filename: %s\n", filename);
	6336
	6337	const char *SelfFName = DladdrSelfFName();
	6338	VPrintf(1, "dlopen interceptor: DladdrSelfFName: %p %s\n",
	6339	(const void *)SelfFName, SelfFName);
	6340
	6341	if (SelfFName && internal_strcmp(SelfFName, filename) == 0) {
	6342	// It's possible they copied the string from dladdr, so
	6343	// we do a string comparison rather than pointer comparison.
	6344	VPrintf(1, "dlopen interceptor: replacing %s because it matches %s\n",
	6345	filename, SelfFName);
	6346	filename = (char *)0; // RTLD_DEFAULT
	6347	}
	6348	}
	6349	# endif // !SANITIZER_DYNAMIC
	6350	}
	6351
6309	6352	void *res = COMMON_INTERCEPTOR_DLOPEN(filename, flag);
6310	6353	Symbolizer::GetOrInit()->InvalidateModuleList();
6311	6354	COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, res);
...	...	@@ -6685,7 +6728,7 @@ INTERCEPTOR(SSIZE_T, recv, int fd, void *buf, SIZE_T len, int flags) {
6685	6728	void *ctx;
6686	6729	COMMON_INTERCEPTOR_ENTER(ctx, recv, fd, buf, len, flags);
6687	6730	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
6688		SSIZE_T res = REAL(recv)(fd, buf, len, flags);
	6731	SSIZE_T res = COMMON_INTERCEPTOR_BLOCK_REAL(recv)(fd, buf, len, flags);
6689	6732	if (res > 0) {
6690	6733	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, Min((SIZE_T)res, len));
6691	6734	}
...	...	@@ -6702,7 +6745,8 @@ INTERCEPTOR(SSIZE_T, recvfrom, int fd, void *buf, SIZE_T len, int flags,
6702	6745	SIZE_T srcaddr_sz;
6703	6746	if (srcaddr) srcaddr_sz = *addrlen;
6704	6747	(void)srcaddr_sz; // prevent "set but not used" warning
6705		SSIZE_T res = REAL(recvfrom)(fd, buf, len, flags, srcaddr, addrlen);
	6748	SSIZE_T res = COMMON_INTERCEPTOR_BLOCK_REAL(recvfrom)(fd, buf, len, flags,
	6749	srcaddr, addrlen);
6706	6750	if (res > 0)
6707	6751	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, Min((SIZE_T)res, len));
6708	6752	if (res >= 0 && srcaddr)
...	...	@@ -6725,7 +6769,7 @@ INTERCEPTOR(SSIZE_T, send, int fd, void *buf, SIZE_T len, int flags) {
6725	6769	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
6726	6770	COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd);
6727	6771	}
6728		SSIZE_T res = REAL(send)(fd, buf, len, flags);
	6772	SSIZE_T res = COMMON_INTERCEPTOR_BLOCK_REAL(send)(fd, buf, len, flags);
6729	6773	if (common_flags()->intercept_send && res > 0)
6730	6774	COMMON_INTERCEPTOR_READ_RANGE(ctx, buf, Min((SIZE_T)res, len));
6731	6775	return res;
...	...	@@ -6740,7 +6784,8 @@ INTERCEPTOR(SSIZE_T, sendto, int fd, void *buf, SIZE_T len, int flags,
6740	6784	COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd);
6741	6785	}
6742	6786	// Can't check dstaddr as it may have uninitialized padding at the end.
6743		SSIZE_T res = REAL(sendto)(fd, buf, len, flags, dstaddr, addrlen);
	6787	SSIZE_T res = COMMON_INTERCEPTOR_BLOCK_REAL(sendto)(fd, buf, len, flags,
	6788	dstaddr, addrlen);
6744	6789	if (common_flags()->intercept_send && res > 0)
6745	6790	COMMON_INTERCEPTOR_READ_RANGE(ctx, buf, Min((SIZE_T)res, len));
6746	6791	return res;
...	...	@@ -6753,25 +6798,25 @@ INTERCEPTOR(SSIZE_T, sendto, int fd, void *buf, SIZE_T len, int flags,
6753	6798	#endif
6754	6799
6755	6800	#if SANITIZER_INTERCEPT_EVENTFD_READ_WRITE
6756		INTERCEPTOR(int, eventfd_read, int fd, u64 *value) {
	6801	INTERCEPTOR(int, eventfd_read, int fd, __sanitizer_eventfd_t *value) {
6757	6802	void *ctx;
6758	6803	COMMON_INTERCEPTOR_ENTER(ctx, eventfd_read, fd, value);
6759	6804	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
6760		int res = REAL(eventfd_read)(fd, value);
	6805	int res = COMMON_INTERCEPTOR_BLOCK_REAL(eventfd_read)(fd, value);
6761	6806	if (res == 0) {
6762	6807	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, value, sizeof(*value));
6763	6808	if (fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);
6764	6809	}
6765	6810	return res;
6766	6811	}
6767		INTERCEPTOR(int, eventfd_write, int fd, u64 value) {
	6812	INTERCEPTOR(int, eventfd_write, int fd, __sanitizer_eventfd_t value) {
6768	6813	void *ctx;
6769	6814	COMMON_INTERCEPTOR_ENTER(ctx, eventfd_write, fd, value);
6770	6815	if (fd >= 0) {
6771	6816	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
6772	6817	COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd);
6773	6818	}
6774		int res = REAL(eventfd_write)(fd, value);
	6819	int res = COMMON_INTERCEPTOR_BLOCK_REAL(eventfd_write)(fd, value);
6775	6820	return res;
6776	6821	}
6777	6822	#define INIT_EVENTFD_READ_WRITE \
...	...	@@ -7394,7 +7439,8 @@ INTERCEPTOR(int, open_by_handle_at, int mount_fd, struct file_handle* handle,
7394	7439	COMMON_INTERCEPTOR_READ_RANGE(
7395	7440	ctx, &sanitizer_handle->f_handle, sanitizer_handle->handle_bytes);
7396	7441
7397		return REAL(open_by_handle_at)(mount_fd, handle, flags);
	7442	return COMMON_INTERCEPTOR_BLOCK_REAL(open_by_handle_at)(mount_fd, handle,
	7443	flags);
7398	7444	}
7399	7445
7400	7446	#define INIT_OPEN_BY_HANDLE_AT COMMON_INTERCEPT_FUNCTION(open_by_handle_at)
...	...	@@ -7609,9 +7655,9 @@ static void write_protoent(void ctx, struct __sanitizer_protoent p) {
7609	7655	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->p_aliases, pp_size * sizeof(char *));
7610	7656	}
7611	7657
7612		INTERCEPTOR(struct __sanitizer_protoent *, getprotoent) {
	7658	INTERCEPTOR(struct __sanitizer_protoent *, getprotoent,) {
7613	7659	void *ctx;
7614		COMMON_INTERCEPTOR_ENTER(ctx, getprotoent);
	7660	COMMON_INTERCEPTOR_ENTER(ctx, getprotoent,);
7615	7661	struct __sanitizer_protoent *p = REAL(getprotoent)();
7616	7662	if (p)
7617	7663	write_protoent(ctx, p);
...	...	@@ -7698,9 +7744,9 @@ INTERCEPTOR(int, getprotobynumber_r, int num,
7698	7744	#endif
7699	7745
7700	7746	#if SANITIZER_INTERCEPT_NETENT
7701		INTERCEPTOR(struct __sanitizer_netent *, getnetent) {
	7747	INTERCEPTOR(struct __sanitizer_netent *, getnetent,) {
7702	7748	void *ctx;
7703		COMMON_INTERCEPTOR_ENTER(ctx, getnetent);
	7749	COMMON_INTERCEPTOR_ENTER(ctx, getnetent,);
7704	7750	struct __sanitizer_netent *n = REAL(getnetent)();
7705	7751	if (n) {
7706	7752	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n, sizeof(*n));
...	...	@@ -9862,9 +9908,9 @@ INTERCEPTOR(char , fdevname_r, int fd, char buf, SIZE_T len) {
9862	9908	#endif
9863	9909
9864	9910	#if SANITIZER_INTERCEPT_GETUSERSHELL
9865		INTERCEPTOR(char *, getusershell) {
	9911	INTERCEPTOR(char *, getusershell,) {
9866	9912	void *ctx;
9867		COMMON_INTERCEPTOR_ENTER(ctx, getusershell);
	9913	COMMON_INTERCEPTOR_ENTER(ctx, getusershell,);
9868	9914	char *res = REAL(getusershell)();
9869	9915	if (res)
9870	9916	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, internal_strlen(res) + 1);
...	...	@@ -9933,7 +9979,13 @@ INTERCEPTOR(void, sl_free, void *sl, int freeall) {
9933	9979	INTERCEPTOR(SSIZE_T, getrandom, void *buf, SIZE_T buflen, unsigned int flags) {
9934	9980	void *ctx;
9935	9981	COMMON_INTERCEPTOR_ENTER(ctx, getrandom, buf, buflen, flags);
9936		SSIZE_T n = REAL(getrandom)(buf, buflen, flags);
	9982	// If GRND_NONBLOCK is set in the flags, it is non blocking.
	9983	static const int grnd_nonblock = 1;
	9984	SSIZE_T n;
	9985	if ((flags & grnd_nonblock))
	9986	n = REAL(getrandom)(buf, buflen, flags);
	9987	else
	9988	n = COMMON_INTERCEPTOR_BLOCK_REAL(getrandom)(buf, buflen, flags);
9937	9989	if (n > 0) {
9938	9990	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, n);
9939	9991	}
...	...	@@ -10180,20 +10232,6 @@ INTERCEPTOR(int, __xuname, int size, void *utsname) {
10180	10232	#define INIT___XUNAME
10181	10233	#endif
10182	10234
10183		#if SANITIZER_INTERCEPT_HEXDUMP
10184		INTERCEPTOR(void, hexdump, const void ptr, int length, const char header, int flags) {
10185		void *ctx;
10186		COMMON_INTERCEPTOR_ENTER(ctx, hexdump, ptr, length, header, flags);
10187		COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, length);
10188		COMMON_INTERCEPTOR_READ_RANGE(ctx, header, internal_strlen(header) + 1);
10189		REAL(hexdump)(ptr, length, header, flags);
10190		}
10191
10192		#define INIT_HEXDUMP COMMON_INTERCEPT_FUNCTION(hexdump);
10193		#else
10194		#define INIT_HEXDUMP
10195		#endif
10196
10197	10235	#if SANITIZER_INTERCEPT_ARGP_PARSE
10198	10236	INTERCEPTOR(int, argp_parse, const struct argp argp, int argc, char *argv,
10199	10237	unsigned flags, int arg_index, void input) {
...	...	@@ -10226,6 +10264,38 @@ INTERCEPTOR(int, cpuset_getaffinity, int level, int which, __int64_t id, SIZE_T
10226	10264	#define INIT_CPUSET_GETAFFINITY
10227	10265	#endif
10228	10266
	10267	#if SANITIZER_INTERCEPT_PREADV2
	10268	INTERCEPTOR(SSIZE_T, preadv2, int fd, __sanitizer_iovec *iov, int iovcnt,
	10269	OFF_T offset, int flags) {
	10270	void *ctx;
	10271	COMMON_INTERCEPTOR_ENTER(ctx, preadv2, fd, iov, iovcnt, offset, flags);
	10272	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
	10273	SSIZE_T res = REAL(preadv2)(fd, iov, iovcnt, offset, flags);
	10274	if (res > 0) write_iovec(ctx, iov, iovcnt, res);
	10275	if (res >= 0 && fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);
	10276	return res;
	10277	}
	10278	#define INIT_PREADV2 COMMON_INTERCEPT_FUNCTION(preadv2)
	10279	#else
	10280	#define INIT_PREADV2
	10281	#endif
	10282
	10283	#if SANITIZER_INTERCEPT_PWRITEV2
	10284	INTERCEPTOR(SSIZE_T, pwritev2, int fd, __sanitizer_iovec *iov, int iovcnt,
	10285	OFF_T offset, int flags) {
	10286	void *ctx;
	10287	COMMON_INTERCEPTOR_ENTER(ctx, pwritev2, fd, iov, iovcnt, offset, flags);
	10288	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
	10289	if (fd >= 0) COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd);
	10290	SSIZE_T res = REAL(pwritev2)(fd, iov, iovcnt, offset, flags);
	10291	if (res > 0) read_iovec(ctx, iov, iovcnt, res);
	10292	return res;
	10293	}
	10294	#define INIT_PWRITEV2 COMMON_INTERCEPT_FUNCTION(pwritev2)
	10295	#else
	10296	#define INIT_PWRITEV2
	10297	#endif
	10298
10229	10299	#include "sanitizer_common_interceptors_netbsd_compat.inc"
10230	10300
10231	10301	namespace __sanitizer {
...	...	@@ -10543,9 +10613,10 @@ static void InitializeCommonInterceptors() {
10543	10613	INIT_PROCCTL
10544	10614	INIT_UNAME;
10545	10615	INIT___XUNAME;
10546		INIT_HEXDUMP;
10547	10616	INIT_ARGP_PARSE;
10548	10617	INIT_CPUSET_GETAFFINITY;
	10618	INIT_PREADV2;
	10619	INIT_PWRITEV2;
10549	10620
10550	10621	INIT___PRINTF_CHK;
10551	10622	}

lib/tsan/sanitizer_common/sanitizer_common_interceptors_format.inc+8-7

...	...	@@ -547,24 +547,25 @@ static void printf_common(void ctx, const char format, va_list aq) {
547	547	continue;
548	548	} else if (size == FSS_STRLEN) {
549	549	if (void argp = va_arg(aq, void )) {
	550	uptr len;
550	551	if (dir.starredPrecision) {
551	552	// FIXME: properly support starred precision for strings.
552		size = 0;
	553	len = 0;
553	554	} else if (dir.fieldPrecision > 0) {
554	555	// Won't read more than "precision" symbols.
555		size = internal_strnlen((const char *)argp, dir.fieldPrecision);
556		if (size < dir.fieldPrecision) size++;
	556	len = internal_strnlen((const char *)argp, dir.fieldPrecision);
	557	if (len < (uptr)dir.fieldPrecision)
	558	len++;
557	559	} else {
558	560	// Whole string will be accessed.
559		size = internal_strlen((const char *)argp) + 1;
	561	len = internal_strlen((const char *)argp) + 1;
560	562	}
561		COMMON_INTERCEPTOR_READ_RANGE(ctx, argp, size);
	563	COMMON_INTERCEPTOR_READ_RANGE(ctx, argp, len);
562	564	}
563	565	} else if (size == FSS_WCSLEN) {
564	566	if (void argp = va_arg(aq, void )) {
565	567	// FIXME: Properly support wide-character strings (via wcsrtombs).
566		size = 0;
567		COMMON_INTERCEPTOR_READ_RANGE(ctx, argp, size);
	568	COMMON_INTERCEPTOR_READ_RANGE(ctx, argp, 0);
568	569	}
569	570	} else {
570	571	// Skip non-pointer args

lib/tsan/sanitizer_common/sanitizer_common_interface.inc+1

...	...	@@ -46,6 +46,7 @@ INTERFACE_FUNCTION(__sanitizer_purge_allocator)
46	46	INTERFACE_FUNCTION(__sanitizer_print_memory_profile)
47	47	INTERFACE_WEAK_FUNCTION(__sanitizer_free_hook)
48	48	INTERFACE_WEAK_FUNCTION(__sanitizer_malloc_hook)
	49	INTERFACE_WEAK_FUNCTION(__sanitizer_ignore_free_hook)
49	50	// Memintrinsic functions.
50	51	INTERFACE_FUNCTION(__sanitizer_internal_memcpy)
51	52	INTERFACE_FUNCTION(__sanitizer_internal_memmove)

lib/tsan/sanitizer_common/sanitizer_common_interface_posix.inc+1

...	...	@@ -9,6 +9,7 @@
9	9	//===----------------------------------------------------------------------===//
10	10	INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_code)
11	11	INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_data)
	12	INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_frame)
12	13	INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_demangle)
13	14	INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_flush)
14	15	INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_set_demangle)

lib/tsan/sanitizer_common/sanitizer_common_libcdep.cpp+6-4

...	...	@@ -87,8 +87,8 @@ void MaybeStartBackgroudThread() {
87	87	if (!common_flags()->hard_rss_limit_mb &&
88	88	!common_flags()->soft_rss_limit_mb &&
89	89	!common_flags()->heap_profile) return;
90		if (!&real_pthread_create) {
91		VPrintf(1, "%s: real_pthread_create undefined\n", SanitizerToolName);
	90	if (!&internal_pthread_create) {
	91	VPrintf(1, "%s: internal_pthread_create undefined\n", SanitizerToolName);
92	92	return; // Can't spawn the thread anyway.
93	93	}
94	94
...	...	@@ -119,8 +119,10 @@ void MaybeStartBackgroudThread() {}
119	119	#endif
120	120
121	121	void WriteToSyslog(const char *msg) {
	122	if (!msg)
	123	return;
122	124	InternalScopedString msg_copy;
123		msg_copy.append("%s", msg);
	125	msg_copy.Append(msg);
124	126	const char *p = msg_copy.data();
125	127
126	128	// Print one line at a time.
...	...	@@ -167,7 +169,7 @@ void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name,
167	169	: !MmapFixedNoReserve(beg, size, name)) {
168	170	Report(
169	171	"ReserveShadowMemoryRange failed while trying to map 0x%zx bytes. "
170		"Perhaps you're using ulimit -v\n",
	172	"Perhaps you're using ulimit -v or ulimit -d\n",
171	173	size);
172	174	Abort();
173	175	}

lib/tsan/sanitizer_common/sanitizer_common_syscalls.inc+35

...	...	@@ -38,6 +38,10 @@
38	38	// Called before fork syscall.
39	39	// COMMON_SYSCALL_POST_FORK(long res)
40	40	// Called after fork syscall.
	41	// COMMON_SYSCALL_BLOCKING_START()
	42	// Called before blocking syscall.
	43	// COMMON_SYSCALL_BLOCKING_END()
	44	// Called after blocking syscall.
41	45	//===----------------------------------------------------------------------===//
42	46
43	47	#include "sanitizer_platform.h"
...	...	@@ -85,6 +89,16 @@
85	89	{}
86	90	# endif
87	91
	92	# ifndef COMMON_SYSCALL_BLOCKING_START
	93	# define COMMON_SYSCALL_BLOCKING_START() \
	94	{}
	95	# endif
	96
	97	# ifndef COMMON_SYSCALL_BLOCKING_END
	98	# define COMMON_SYSCALL_BLOCKING_END() \
	99	{}
	100	# endif
	101
88	102	// FIXME: do some kind of PRE_READ for all syscall arguments (int(s) and such).
89	103
90	104	extern "C" {
...	...	@@ -2808,6 +2822,15 @@ PRE_SYSCALL(fchownat)
2808	2822	POST_SYSCALL(fchownat)
2809	2823	(long res, long dfd, const void *filename, long user, long group, long flag) {}
2810	2824
	2825	PRE_SYSCALL(fchmodat2)(long dfd, const void *filename, long mode, long flag) {
	2826	if (filename)
	2827	PRE_READ(filename,
	2828	__sanitizer::internal_strlen((const char *)filename) + 1);
	2829	}
	2830
	2831	POST_SYSCALL(fchmodat2)
	2832	(long res, long dfd, const void *filename, long mode, long flag) {}
	2833
2811	2834	PRE_SYSCALL(openat)(long dfd, const void *filename, long flags, long mode) {
2812	2835	if (filename)
2813	2836	PRE_READ(filename,
...	...	@@ -3167,6 +3190,18 @@ POST_SYSCALL(sigaltstack)(long res, void ss, void oss) {
3167	3190	}
3168	3191	}
3169	3192	}
	3193
	3194	PRE_SYSCALL(futex)
	3195	(void uaddr, long futex_op, long val, void timeout, void *uaddr2, long val3) {
	3196	COMMON_SYSCALL_BLOCKING_START();
	3197	}
	3198
	3199	POST_SYSCALL(futex)
	3200	(long res, void uaddr, long futex_op, long val, void timeout, void *uaddr2,
	3201	long val3) {
	3202	COMMON_SYSCALL_BLOCKING_END();
	3203	}
	3204
3170	3205	} // extern "C"
3171	3206
3172	3207	# undef PRE_SYSCALL

lib/tsan/sanitizer_common/sanitizer_dl.cpp created+37

...	...	@@ -0,0 +1,37 @@
	1	//===-- sanitizer_dl.cpp --------------------------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file has helper functions that depend on libc's dynamic loading
	10	// introspection.
	11	//
	12	//===----------------------------------------------------------------------===//
	13
	14	#include "sanitizer_dl.h"
	15
	16	#include "sanitizer_common/sanitizer_platform.h"
	17
	18	#if SANITIZER_GLIBC
	19	# include <dlfcn.h>
	20	#endif
	21
	22	namespace __sanitizer {
	23	extern const char *SanitizerToolName;
	24
	25	const char *DladdrSelfFName(void) {
	26	#if SANITIZER_GLIBC
	27	Dl_info info;
	28	int ret = dladdr((void *)&SanitizerToolName, &info);
	29	if (ret) {
	30	return info.dli_fname;
	31	}
	32	#endif
	33
	34	return nullptr;
	35	}
	36
	37	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_dl.h created+26

...	...	@@ -0,0 +1,26 @@
	1	//===-- sanitizer_dl.h ----------------------------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file has helper functions that depend on libc's dynamic loading
	10	// introspection.
	11	//
	12	//===----------------------------------------------------------------------===//
	13
	14	#ifndef SANITIZER_DL_H
	15	#define SANITIZER_DL_H
	16
	17	namespace __sanitizer {
	18
	19	// Returns the path to the shared object or - in the case of statically linked
	20	// sanitizers
	21	// - the main program itself, that contains the sanitizer.
	22	const char* DladdrSelfFName(void);
	23
	24	} // namespace __sanitizer
	25
	26	#endif // SANITIZER_DL_H

lib/tsan/sanitizer_common/sanitizer_file.cpp+3-1

...	...	@@ -69,7 +69,7 @@ void ReportFile::ReopenIfNecessary() {
69	69	WriteToFile(kStderrFd, ErrorMsgPrefix, internal_strlen(ErrorMsgPrefix));
70	70	WriteToFile(kStderrFd, full_path, internal_strlen(full_path));
71	71	char errmsg[100];
72		internal_snprintf(errmsg, sizeof(errmsg), " (reason: %d)", err);
	72	internal_snprintf(errmsg, sizeof(errmsg), " (reason: %d)\n", err);
73	73	WriteToFile(kStderrFd, errmsg, internal_strlen(errmsg));
74	74	Die();
75	75	}
...	...	@@ -88,6 +88,8 @@ static void RecursiveCreateParentDirs(char *path) {
88	88	const char *ErrorMsgPrefix = "ERROR: Can't create directory: ";
89	89	WriteToFile(kStderrFd, ErrorMsgPrefix, internal_strlen(ErrorMsgPrefix));
90	90	WriteToFile(kStderrFd, path, internal_strlen(path));
	91	const char *ErrorMsgSuffix = "\n";
	92	WriteToFile(kStderrFd, ErrorMsgSuffix, internal_strlen(ErrorMsgSuffix));
91	93	Die();
92	94	}
93	95	path[i] = save;

lib/tsan/sanitizer_common/sanitizer_file.h+1-1

...	...	@@ -84,7 +84,7 @@ bool IsPathSeparator(const char c);
84	84	bool IsAbsolutePath(const char *path);
85	85	// Returns true on success, false on failure.
86	86	bool CreateDir(const char *pathname);
87		// Starts a subprocess and returs its pid.
	87	// Starts a subprocess and returns its pid.
88	88	// If *_fd parameters are not kInvalidFd their corresponding input/output
89	89	// streams will be redirect to the file. The files will always be closed
90	90	// in parent process even in case of an error.

lib/tsan/sanitizer_common/sanitizer_flag_parser.cpp+3-4

...	...	@@ -19,8 +19,6 @@
19	19
20	20	namespace __sanitizer {
21	21
22		LowLevelAllocator FlagParser::Alloc;
23
24	22	class UnknownFlags {
25	23	static const int kMaxUnknownFlags = 20;
26	24	const char *unknown_flags_[kMaxUnknownFlags];
...	...	@@ -49,7 +47,7 @@ void ReportUnrecognizedFlags() {
49	47
50	48	char FlagParser::ll_strndup(const char s, uptr n) {
51	49	uptr len = internal_strnlen(s, n);
52		char s2 = (char)Alloc.Allocate(len + 1);
	50	char s2 = (char )GetGlobalLowLevelAllocator().Allocate(len + 1);
53	51	internal_memcpy(s2, s, len);
54	52	s2[len] = 0;
55	53	return s2;
...	...	@@ -185,7 +183,8 @@ void FlagParser::RegisterHandler(const char name, FlagHandlerBase handler,
185	183	}
186	184
187	185	FlagParser::FlagParser() : n_flags_(0), buf_(nullptr), pos_(0) {
188		flags_ = (Flag )Alloc.Allocate(sizeof(Flag) kMaxFlags);
	186	flags_ =
	187	(Flag )GetGlobalLowLevelAllocator().Allocate(sizeof(Flag) kMaxFlags);
189	188	}
190	189
191	190	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_flag_parser.h+1-3

...	...	@@ -178,8 +178,6 @@ class FlagParser {
178	178	bool ParseFile(const char *path, bool ignore_missing);
179	179	void PrintFlagDescriptions();
180	180
181		static LowLevelAllocator Alloc;
182
183	181	private:
184	182	void fatal_error(const char *err);
185	183	bool is_space(char c);
...	...	@@ -193,7 +191,7 @@ class FlagParser {
193	191	template <typename T>
194	192	static void RegisterFlag(FlagParser parser, const char name, const char *desc,
195	193	T *var) {
196		FlagHandler<T> *fh = new (FlagParser::Alloc) FlagHandler<T>(var);
	194	FlagHandler<T> *fh = new (GetGlobalLowLevelAllocator()) FlagHandler<T>(var);
197	195	parser->RegisterHandler(name, fh, desc);
198	196	}
199	197

lib/tsan/sanitizer_common/sanitizer_flags.cpp+2-2

...	...	@@ -108,11 +108,11 @@ class FlagHandlerInclude final : public FlagHandlerBase {
108	108	};
109	109
110	110	void RegisterIncludeFlags(FlagParser parser, CommonFlags cf) {
111		FlagHandlerInclude *fh_include = new (FlagParser::Alloc)
	111	FlagHandlerInclude *fh_include = new (GetGlobalLowLevelAllocator())
112	112	FlagHandlerInclude(parser, /ignore_missing/ false);
113	113	parser->RegisterHandler("include", fh_include,
114	114	"read more options from the given file");
115		FlagHandlerInclude *fh_include_if_exists = new (FlagParser::Alloc)
	115	FlagHandlerInclude *fh_include_if_exists = new (GetGlobalLowLevelAllocator())
116	116	FlagHandlerInclude(parser, /ignore_missing/ true);
117	117	parser->RegisterHandler(
118	118	"include_if_exists", fh_include_if_exists,

lib/tsan/sanitizer_common/sanitizer_flags.inc+13

...	...	@@ -269,3 +269,16 @@ COMMON_FLAG(bool, detect_write_exec, false,
269	269	COMMON_FLAG(bool, test_only_emulate_no_memorymap, false,
270	270	"TEST ONLY fail to read memory mappings to emulate sanitized "
271	271	"\"init\"")
	272	// With static linking, dladdr((void*)pthread_join) or similar will return the
	273	// path to the main program. This flag will replace dlopen(<main program,...>
	274	// with dlopen(NULL,...), which is the correct way to get a handle to the main
	275	// program.
	276	COMMON_FLAG(bool, test_only_replace_dlopen_main_program, false,
	277	"TEST ONLY replace dlopen(<main program>,...) with dlopen(NULL)")
	278
	279	COMMON_FLAG(bool, enable_symbolizer_markup, SANITIZER_FUCHSIA,
	280	"Use sanitizer symbolizer markup, available on Linux "
	281	"and always set true for Fuchsia.")
	282
	283	COMMON_FLAG(bool, detect_invalid_join, true,
	284	"If set, check invalid joins of threads.")

lib/tsan/sanitizer_common/sanitizer_flat_map.h+4

...	...	@@ -109,6 +109,10 @@ class TwoLevelMap {
109	109	return *AddressSpaceView::LoadWritable(&map2[idx % kSize2]);
110	110	}
111	111
	112	void Lock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS { mu_.Lock(); }
	113
	114	void Unlock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS { mu_.Unlock(); }
	115
112	116	private:
113	117	constexpr uptr MmapSize() const {
114	118	return RoundUpTo(kSize2 * sizeof(T), GetPageSizeCached());

lib/tsan/sanitizer_common/sanitizer_freebsd.h deleted-137

...	...	@@ -1,137 +0,0 @@
1		//===-- sanitizer_freebsd.h -------------------------------------- C++ --===//
2		//
3		// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4		// See https://llvm.org/LICENSE.txt for license information.
5		// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6		//
7		//===----------------------------------------------------------------------===//
8		//
9		// This file is a part of Sanitizer runtime. It contains FreeBSD-specific
10		// definitions.
11		//
12		//===----------------------------------------------------------------------===//
13
14		#ifndef SANITIZER_FREEBSD_H
15		#define SANITIZER_FREEBSD_H
16
17		#include "sanitizer_internal_defs.h"
18
19		// x86-64 FreeBSD 9.2 and older define 'ucontext_t' incorrectly in
20		// 32-bit mode.
21		#if SANITIZER_FREEBSD && (SANITIZER_WORDSIZE == 32)
22		#include <osreldate.h>
23		#if __FreeBSD_version <= 902001 // v9.2
24		#include <link.h>
25		#include <sys/param.h>
26		#include <ucontext.h>
27
28		namespace __sanitizer {
29
30		typedef unsigned long long __xuint64_t;
31
32		typedef __int32_t __xregister_t;
33
34		typedef struct __xmcontext {
35		__xregister_t mc_onstack;
36		__xregister_t mc_gs;
37		__xregister_t mc_fs;
38		__xregister_t mc_es;
39		__xregister_t mc_ds;
40		__xregister_t mc_edi;
41		__xregister_t mc_esi;
42		__xregister_t mc_ebp;
43		__xregister_t mc_isp;
44		__xregister_t mc_ebx;
45		__xregister_t mc_edx;
46		__xregister_t mc_ecx;
47		__xregister_t mc_eax;
48		__xregister_t mc_trapno;
49		__xregister_t mc_err;
50		__xregister_t mc_eip;
51		__xregister_t mc_cs;
52		__xregister_t mc_eflags;
53		__xregister_t mc_esp;
54		__xregister_t mc_ss;
55
56		int mc_len;
57		int mc_fpformat;
58		int mc_ownedfp;
59		__xregister_t mc_flags;
60
61		int mc_fpstate[128] __aligned(16);
62		__xregister_t mc_fsbase;
63		__xregister_t mc_gsbase;
64		__xregister_t mc_xfpustate;
65		__xregister_t mc_xfpustate_len;
66
67		int mc_spare2[4];
68		} xmcontext_t;
69
70		typedef struct __xucontext {
71		sigset_t uc_sigmask;
72		xmcontext_t uc_mcontext;
73
74		struct __ucontext *uc_link;
75		stack_t uc_stack;
76		int uc_flags;
77		int __spare__[4];
78		} xucontext_t;
79
80		struct xkinfo_vmentry {
81		int kve_structsize;
82		int kve_type;
83		__xuint64_t kve_start;
84		__xuint64_t kve_end;
85		__xuint64_t kve_offset;
86		__xuint64_t kve_vn_fileid;
87		__uint32_t kve_vn_fsid;
88		int kve_flags;
89		int kve_resident;
90		int kve_private_resident;
91		int kve_protection;
92		int kve_ref_count;
93		int kve_shadow_count;
94		int kve_vn_type;
95		__xuint64_t kve_vn_size;
96		__uint32_t kve_vn_rdev;
97		__uint16_t kve_vn_mode;
98		__uint16_t kve_status;
99		int _kve_ispare[12];
100		char kve_path[PATH_MAX];
101		};
102
103		typedef struct {
104		__uint32_t p_type;
105		__uint32_t p_offset;
106		__uint32_t p_vaddr;
107		__uint32_t p_paddr;
108		__uint32_t p_filesz;
109		__uint32_t p_memsz;
110		__uint32_t p_flags;
111		__uint32_t p_align;
112		} XElf32_Phdr;
113
114		struct xdl_phdr_info {
115		Elf_Addr dlpi_addr;
116		const char *dlpi_name;
117		const XElf32_Phdr *dlpi_phdr;
118		Elf_Half dlpi_phnum;
119		unsigned long long int dlpi_adds;
120		unsigned long long int dlpi_subs;
121		size_t dlpi_tls_modid;
122		void *dlpi_tls_data;
123		};
124
125		typedef int (__xdl_iterate_hdr_callback)(struct xdl_phdr_info , size_t,
126		void *);
127		typedef int xdl_iterate_phdr_t(__xdl_iterate_hdr_callback, void *);
128
129		#define xdl_iterate_phdr(callback, param) \
130		(((xdl_iterate_phdr_t *)dl_iterate_phdr)((callback), (param)))
131
132		} // namespace __sanitizer
133
134		#endif // __FreeBSD_version <= 902001
135		#endif // SANITIZER_FREEBSD && (SANITIZER_WORDSIZE == 32)
136
137		#endif // SANITIZER_FREEBSD_H

lib/tsan/sanitizer_common/sanitizer_fuchsia.cpp+82-25

...	...	@@ -129,6 +129,60 @@ uptr GetMaxVirtualAddress() { return GetMaxUserVirtualAddress(); }
129	129
130	130	bool ErrorIsOOM(error_t err) { return err == ZX_ERR_NO_MEMORY; }
131	131
	132	// For any sanitizer internal that needs to map something which can be unmapped
	133	// later, first attempt to map to a pre-allocated VMAR. This helps reduce
	134	// fragmentation from many small anonymous mmap calls. A good value for this
	135	// VMAR size would be the total size of your typical sanitizer internal objects
	136	// allocated in an "average" process lifetime. Examples of this include:
	137	// FakeStack, LowLevelAllocator mappings, TwoLevelMap, InternalMmapVector,
	138	// StackStore, CreateAsanThread, etc.
	139	//
	140	// This is roughly equal to the total sum of sanitizer internal mappings for a
	141	// large test case.
	142	constexpr size_t kSanitizerHeapVmarSize = 13ULL << 20;
	143	static zx_handle_t gSanitizerHeapVmar = ZX_HANDLE_INVALID;
	144
	145	static zx_status_t GetSanitizerHeapVmar(zx_handle_t *vmar) {
	146	zx_status_t status = ZX_OK;
	147	if (gSanitizerHeapVmar == ZX_HANDLE_INVALID) {
	148	CHECK_EQ(kSanitizerHeapVmarSize % GetPageSizeCached(), 0);
	149	uintptr_t base;
	150	status = _zx_vmar_allocate(
	151	_zx_vmar_root_self(),
	152	ZX_VM_CAN_MAP_READ \| ZX_VM_CAN_MAP_WRITE \| ZX_VM_CAN_MAP_SPECIFIC, 0,
	153	kSanitizerHeapVmarSize, &gSanitizerHeapVmar, &base);
	154	}
	155	*vmar = gSanitizerHeapVmar;
	156	if (status == ZX_OK)
	157	CHECK_NE(gSanitizerHeapVmar, ZX_HANDLE_INVALID);
	158	return status;
	159	}
	160
	161	static zx_status_t TryVmoMapSanitizerVmar(zx_vm_option_t options,
	162	size_t vmar_offset, zx_handle_t vmo,
	163	size_t size, uintptr_t *addr,
	164	zx_handle_t *vmar_used = nullptr) {
	165	zx_handle_t vmar;
	166	zx_status_t status = GetSanitizerHeapVmar(&vmar);
	167	if (status != ZX_OK)
	168	return status;
	169
	170	status = _zx_vmar_map(gSanitizerHeapVmar, options, vmar_offset, vmo,
	171	/vmo_offset=/0, size, addr);
	172	if (vmar_used)
	173	*vmar_used = gSanitizerHeapVmar;
	174	if (status == ZX_ERR_NO_RESOURCES \|\| status == ZX_ERR_INVALID_ARGS) {
	175	// This means there's no space in the heap VMAR, so fallback to the root
	176	// VMAR.
	177	status = _zx_vmar_map(_zx_vmar_root_self(), options, vmar_offset, vmo,
	178	/vmo_offset=/0, size, addr);
	179	if (vmar_used)
	180	*vmar_used = _zx_vmar_root_self();
	181	}
	182
	183	return status;
	184	}
	185
132	186	static void DoAnonymousMmapOrDie(uptr size, const char mem_type,
133	187	bool raw_report, bool die_for_nomem) {
134	188	size = RoundUpTo(size, GetPageSize());
...	...	@@ -144,11 +198,9 @@ static void DoAnonymousMmapOrDie(uptr size, const char mem_type,
144	198	_zx_object_set_property(vmo, ZX_PROP_NAME, mem_type,
145	199	internal_strlen(mem_type));
146	200
147		// TODO(mcgrathr): Maybe allocate a VMAR for all sanitizer heap and use that?
148	201	uintptr_t addr;
149		status =
150		_zx_vmar_map(_zx_vmar_root_self(), ZX_VM_PERM_READ \| ZX_VM_PERM_WRITE, 0,
151		vmo, 0, size, &addr);
	202	status = TryVmoMapSanitizerVmar(ZX_VM_PERM_READ \| ZX_VM_PERM_WRITE,
	203	/vmar_offset=/0, vmo, size, &addr);
152	204	_zx_handle_close(vmo);
153	205
154	206	if (status != ZX_OK) {
...	...	@@ -226,27 +278,32 @@ static uptr DoMmapFixedOrDie(zx_handle_t vmar, uptr fixed_addr, uptr map_size,
226	278
227	279	uptr ReservedAddressRange::Map(uptr fixed_addr, uptr map_size,
228	280	const char *name) {
229		return DoMmapFixedOrDie(os_handle_, fixed_addr, map_size, base_, name_,
230		false);
	281	return DoMmapFixedOrDie(os_handle_, fixed_addr, map_size, base_,
	282	name ? name : name_, false);
231	283	}
232	284
233	285	uptr ReservedAddressRange::MapOrDie(uptr fixed_addr, uptr map_size,
234	286	const char *name) {
235		return DoMmapFixedOrDie(os_handle_, fixed_addr, map_size, base_, name_, true);
	287	return DoMmapFixedOrDie(os_handle_, fixed_addr, map_size, base_,
	288	name ? name : name_, true);
236	289	}
237	290
238		void UnmapOrDieVmar(void *addr, uptr size, zx_handle_t target_vmar) {
	291	void UnmapOrDieVmar(void *addr, uptr size, zx_handle_t target_vmar,
	292	bool raw_report) {
239	293	if (!addr \|\| !size)
240	294	return;
241	295	size = RoundUpTo(size, GetPageSize());
242	296
243	297	zx_status_t status =
244	298	_zx_vmar_unmap(target_vmar, reinterpret_cast<uintptr_t>(addr), size);
245		if (status != ZX_OK) {
246		Report("ERROR: %s failed to deallocate 0x%zx (%zd) bytes at address %p\n",
247		SanitizerToolName, size, size, addr);
248		CHECK("unable to unmap" && 0);
	299	if (status == ZX_ERR_INVALID_ARGS && target_vmar == gSanitizerHeapVmar) {
	300	// If there wasn't any space in the heap vmar, the fallback was the root
	301	// vmar.
	302	status = _zx_vmar_unmap(_zx_vmar_root_self(),
	303	reinterpret_cast<uintptr_t>(addr), size);
249	304	}
	305	if (status != ZX_OK)
	306	ReportMunmapFailureAndDie(addr, size, status, raw_report);
250	307
251	308	DecreaseTotalMmap(size);
252	309	}
...	...	@@ -268,7 +325,8 @@ void ReservedAddressRange::Unmap(uptr addr, uptr size) {
268	325	}
269	326	// Partial unmapping does not affect the fact that the initial range is still
270	327	// reserved, and the resulting unmapped memory can't be reused.
271		UnmapOrDieVmar(reinterpret_cast<void *>(addr), size, vmar);
	328	UnmapOrDieVmar(reinterpret_cast<void *>(addr), size, vmar,
	329	/raw_report=/false);
272	330	}
273	331
274	332	// This should never be called.
...	...	@@ -307,17 +365,16 @@ void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment,
307	365	_zx_object_set_property(vmo, ZX_PROP_NAME, mem_type,
308	366	internal_strlen(mem_type));
309	367
310		// TODO(mcgrathr): Maybe allocate a VMAR for all sanitizer heap and use that?
311
312	368	// Map a larger size to get a chunk of address space big enough that
313	369	// it surely contains an aligned region of the requested size. Then
314	370	// overwrite the aligned middle portion with a mapping from the
315	371	// beginning of the VMO, and unmap the excess before and after.
316	372	size_t map_size = size + alignment;
317	373	uintptr_t addr;
318		status =
319		_zx_vmar_map(_zx_vmar_root_self(), ZX_VM_PERM_READ \| ZX_VM_PERM_WRITE, 0,
320		vmo, 0, map_size, &addr);
	374	zx_handle_t vmar_used;
	375	status = TryVmoMapSanitizerVmar(ZX_VM_PERM_READ \| ZX_VM_PERM_WRITE,
	376	/vmar_offset=/0, vmo, map_size, &addr,
	377	&vmar_used);
321	378	if (status == ZX_OK) {
322	379	uintptr_t map_addr = addr;
323	380	uintptr_t map_end = map_addr + map_size;
...	...	@@ -325,12 +382,12 @@ void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment,
325	382	uintptr_t end = addr + size;
326	383	if (addr != map_addr) {
327	384	zx_info_vmar_t info;
328		status = _zx_object_get_info(_zx_vmar_root_self(), ZX_INFO_VMAR, &info,
329		sizeof(info), NULL, NULL);
	385	status = _zx_object_get_info(vmar_used, ZX_INFO_VMAR, &info, sizeof(info),
	386	NULL, NULL);
330	387	if (status == ZX_OK) {
331	388	uintptr_t new_addr;
332	389	status = _zx_vmar_map(
333		_zx_vmar_root_self(),
	390	vmar_used,
334	391	ZX_VM_PERM_READ \| ZX_VM_PERM_WRITE \| ZX_VM_SPECIFIC_OVERWRITE,
335	392	addr - info.base, vmo, 0, size, &new_addr);
336	393	if (status == ZX_OK)
...	...	@@ -338,9 +395,9 @@ void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment,
338	395	}
339	396	}
340	397	if (status == ZX_OK && addr != map_addr)
341		status = _zx_vmar_unmap(_zx_vmar_root_self(), map_addr, addr - map_addr);
	398	status = _zx_vmar_unmap(vmar_used, map_addr, addr - map_addr);
342	399	if (status == ZX_OK && end != map_end)
343		status = _zx_vmar_unmap(_zx_vmar_root_self(), end, map_end - end);
	400	status = _zx_vmar_unmap(vmar_used, end, map_end - end);
344	401	}
345	402	_zx_handle_close(vmo);
346	403
...	...	@@ -355,8 +412,8 @@ void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment,
355	412	return reinterpret_cast<void *>(addr);
356	413	}
357	414
358		void UnmapOrDie(void *addr, uptr size) {
359		UnmapOrDieVmar(addr, size, _zx_vmar_root_self());
	415	void UnmapOrDie(void *addr, uptr size, bool raw_report) {
	416	UnmapOrDieVmar(addr, size, gSanitizerHeapVmar, raw_report);
360	417	}
361	418
362	419	void ReleaseMemoryPagesToOS(uptr beg, uptr end) {

lib/tsan/sanitizer_common/sanitizer_hash.h+1-1

...	...	@@ -62,6 +62,6 @@ class MurMur2Hash64Builder {
62	62	return x;
63	63	}
64	64	};
65		} //namespace __sanitizer
	65	} // namespace __sanitizer
66	66
67	67	#endif // SANITIZER_HASH_H

lib/tsan/sanitizer_common/sanitizer_internal_defs.h+25-9

...	...	@@ -15,6 +15,11 @@
15	15	#include "sanitizer_platform.h"
16	16	#include "sanitizer_redefine_builtins.h"
17	17
	18	// GCC does not understand __has_feature.
	19	#if !defined(__has_feature)
	20	#define __has_feature(x) 0
	21	#endif
	22
18	23	#ifndef SANITIZER_DEBUG
19	24	# define SANITIZER_DEBUG 0
20	25	#endif
...	...	@@ -30,13 +35,20 @@
30	35	# define SANITIZER_INTERFACE_ATTRIBUTE __declspec(dllexport)
31	36	#endif
32	37	# define SANITIZER_WEAK_ATTRIBUTE
	38	# define SANITIZER_WEAK_IMPORT
33	39	#elif SANITIZER_GO
34	40	# define SANITIZER_INTERFACE_ATTRIBUTE
35	41	# define SANITIZER_WEAK_ATTRIBUTE
	42	# define SANITIZER_WEAK_IMPORT
36	43	#else
37	44	# define SANITIZER_INTERFACE_ATTRIBUTE __attribute__((visibility("default")))
38	45	# define SANITIZER_WEAK_ATTRIBUTE __attribute__((weak))
39		#endif
	46	# if SANITIZER_APPLE
	47	# define SANITIZER_WEAK_IMPORT extern "C" __attribute((weak_import))
	48	# else
	49	# define SANITIZER_WEAK_IMPORT extern "C" SANITIZER_WEAK_ATTRIBUTE
	50	# endif // SANITIZER_APPLE
	51	#endif // SANITIZER_WINDOWS
40	52
41	53	//--------------------------- WEAK FUNCTIONS ---------------------------------//
42	54	// When working with weak functions, to simplify the code and make it more
...	...	@@ -179,15 +191,19 @@ typedef uptr OFF_T;
179	191	#endif
180	192	typedef u64 OFF64_T;
181	193
182		#if (SANITIZER_WORDSIZE == 64) \|\| SANITIZER_APPLE
183		typedef uptr operator_new_size_type;
	194	#ifdef __SIZE_TYPE__
	195	typedef __SIZE_TYPE__ usize;
184	196	#else
185		# if defined(__s390__) && !defined(__s390x__)
186		// Special case: 31-bit s390 has unsigned long as size_t.
187		typedef unsigned long operator_new_size_type;
188		# else
189		typedef u32 operator_new_size_type;
190		# endif
	197	// Since we use this for operator new, usize must match the real size_t, but on
	198	// 32-bit Windows the definition of uptr does not actually match uintptr_t or
	199	// size_t because we are working around typedef mismatches for the (S)SIZE_T
	200	// types used in interception.h.
	201	// Until the definition of uptr has been fixed we have to special case Win32.
	202	# if SANITIZER_WINDOWS && SANITIZER_WORDSIZE == 32
	203	typedef unsigned int usize;
	204	# else
	205	typedef uptr usize;
	206	# endif
191	207	#endif
192	208
193	209	typedef u64 tid_t;

lib/tsan/sanitizer_common/sanitizer_libc.cpp+16

...	...	@@ -199,6 +199,14 @@ char internal_strncat(char dst, const char *src, uptr n) {
199	199	return dst;
200	200	}
201	201
	202	wchar_t internal_wcscpy(wchar_t dst, const wchar_t *src) {
	203	wchar_t *dst_it = dst;
	204	do {
	205	dst_it++ = src++;
	206	} while (*src);
	207	return dst;
	208	}
	209
202	210	uptr internal_strlcpy(char dst, const char src, uptr maxlen) {
203	211	const uptr srclen = internal_strlen(src);
204	212	if (srclen < maxlen) {
...	...	@@ -218,6 +226,14 @@ char internal_strncpy(char dst, const char *src, uptr n) {
218	226	return dst;
219	227	}
220	228
	229	wchar_t internal_wcsncpy(wchar_t dst, const wchar_t *src, uptr n) {
	230	uptr i;
	231	for (i = 0; i < n && src[i]; ++i)
	232	dst[i] = src[i];
	233	internal_memset(dst + i, 0, (n - i) * sizeof(wchar_t));
	234	return dst;
	235	}
	236
221	237	uptr internal_strnlen(const char *s, uptr maxlen) {
222	238	uptr i = 0;
223	239	while (i < maxlen && s[i]) i++;

lib/tsan/sanitizer_common/sanitizer_libc.h+2-1

...	...	@@ -71,7 +71,8 @@ int internal_snprintf(char buffer, uptr length, const char format, ...)
71	71	FORMAT(3, 4);
72	72	uptr internal_wcslen(const wchar_t *s);
73	73	uptr internal_wcsnlen(const wchar_t *s, uptr maxlen);
74
	74	wchar_t internal_wcscpy(wchar_t dst, const wchar_t *src);
	75	wchar_t internal_wcsncpy(wchar_t dst, const wchar_t *src, uptr maxlen);
75	76	// Return true if all bytes in [mem, mem+size) are zero.
76	77	// Optimized for the case when the result is true.
77	78	bool mem_is_zero(const char *mem, uptr size);

lib/tsan/sanitizer_common/sanitizer_libignore.cpp+2-2

...	...	@@ -105,8 +105,8 @@ void LibIgnore::OnLibraryLoaded(const char *name) {
105	105	continue;
106	106	if (IsPcInstrumented(range.beg) && IsPcInstrumented(range.end - 1))
107	107	continue;
108		VReport(1, "Adding instrumented range 0x%zx-0x%zx from library '%s'\n",
109		range.beg, range.end, mod.full_name());
	108	VReport(1, "Adding instrumented range %p-%p from library '%s'\n",
	109	(void )range.beg, (void )range.end, mod.full_name());
110	110	const uptr idx =
111	111	atomic_load(&instrumented_ranges_count_, memory_order_relaxed);
112	112	CHECK_LT(idx, ARRAY_SIZE(instrumented_code_ranges_));

lib/tsan/sanitizer_common/sanitizer_linux.cpp+1009-833

...	...	@@ -16,101 +16,105 @@
16	16	#if SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_NETBSD \|\| \
17	17	SANITIZER_SOLARIS
18	18
19		#include "sanitizer_common.h"
20		#include "sanitizer_flags.h"
21		#include "sanitizer_getauxval.h"
22		#include "sanitizer_internal_defs.h"
23		#include "sanitizer_libc.h"
24		#include "sanitizer_linux.h"
25		#include "sanitizer_mutex.h"
26		#include "sanitizer_placement_new.h"
27		#include "sanitizer_procmaps.h"
28
29		#if SANITIZER_LINUX && !SANITIZER_GO
30		#include <asm/param.h>
31		#endif
	19	# include "sanitizer_common.h"
	20	# include "sanitizer_flags.h"
	21	# include "sanitizer_getauxval.h"
	22	# include "sanitizer_internal_defs.h"
	23	# include "sanitizer_libc.h"
	24	# include "sanitizer_linux.h"
	25	# include "sanitizer_mutex.h"
	26	# include "sanitizer_placement_new.h"
	27	# include "sanitizer_procmaps.h"
	28
	29	# if SANITIZER_LINUX && !SANITIZER_GO
	30	# include <asm/param.h>
	31	# endif
32	32
33	33	// For mips64, syscall(__NR_stat) fills the buffer in the 'struct kernel_stat'
34	34	// format. Struct kernel_stat is defined as 'struct stat' in asm/stat.h. To
35	35	// access stat from asm/stat.h, without conflicting with definition in
36		// sys/stat.h, we use this trick.
37		#if SANITIZER_MIPS64
38		#include <asm/unistd.h>
39		#include <sys/types.h>
40		#define stat kernel_stat
41		#if SANITIZER_GO
42		#undef st_atime
43		#undef st_mtime
44		#undef st_ctime
45		#define st_atime st_atim
46		#define st_mtime st_mtim
47		#define st_ctime st_ctim
48		#endif
49		#include <asm/stat.h>
50		#undef stat
51		#endif
	36	// sys/stat.h, we use this trick. sparc64 is similar, using
	37	// syscall(__NR_stat64) and struct kernel_stat64.
	38	# if SANITIZER_LINUX && (SANITIZER_MIPS64 \|\| SANITIZER_SPARC64)
	39	# include <asm/unistd.h>
	40	# include <sys/types.h>
	41	# define stat kernel_stat
	42	# if SANITIZER_SPARC64
	43	# define stat64 kernel_stat64
	44	# endif
	45	# if SANITIZER_GO
	46	# undef st_atime
	47	# undef st_mtime
	48	# undef st_ctime
	49	# define st_atime st_atim
	50	# define st_mtime st_mtim
	51	# define st_ctime st_ctim
	52	# endif
	53	# include <asm/stat.h>
	54	# undef stat
	55	# undef stat64
	56	# endif
52	57
53		#include <dlfcn.h>
54		#include <errno.h>
55		#include <fcntl.h>
56		#include <link.h>
57		#include <pthread.h>
58		#include <sched.h>
59		#include <signal.h>
60		#include <sys/mman.h>
61		#include <sys/param.h>
62		#if !SANITIZER_SOLARIS
63		#include <sys/ptrace.h>
64		#endif
65		#include <sys/resource.h>
66		#include <sys/stat.h>
67		#include <sys/syscall.h>
68		#include <sys/time.h>
69		#include <sys/types.h>
70		#include <ucontext.h>
71		#include <unistd.h>
72
73		#if SANITIZER_LINUX
74		#include <sys/utsname.h>
75		#endif
	58	# include <dlfcn.h>
	59	# include <errno.h>
	60	# include <fcntl.h>
	61	# include <link.h>
	62	# include <pthread.h>
	63	# include <sched.h>
	64	# include <signal.h>
	65	# include <sys/mman.h>
	66	# if !SANITIZER_SOLARIS
	67	# include <sys/ptrace.h>
	68	# endif
	69	# include <sys/resource.h>
	70	# include <sys/stat.h>
	71	# include <sys/syscall.h>
	72	# include <sys/time.h>
	73	# include <sys/types.h>
	74	# include <ucontext.h>
	75	# include <unistd.h>
76	76
77		#if SANITIZER_LINUX && !SANITIZER_ANDROID
78		#include <sys/personality.h>
79		#endif
	77	# if SANITIZER_LINUX
	78	# include <sys/utsname.h>
	79	# endif
80	80
81		#if SANITIZER_LINUX && defined(__loongarch__)
82		# include <sys/sysmacros.h>
83		#endif
	81	# if SANITIZER_LINUX && !SANITIZER_ANDROID
	82	# include <sys/personality.h>
	83	# endif
84	84
85		#if SANITIZER_FREEBSD
86		#include <sys/exec.h>
87		#include <sys/procctl.h>
88		#include <sys/sysctl.h>
89		#include <machine/atomic.h>
	85	# if SANITIZER_LINUX && defined(__loongarch__)
	86	# include <sys/sysmacros.h>
	87	# endif
	88
	89	# if SANITIZER_FREEBSD
	90	# include <machine/atomic.h>
	91	# include <sys/exec.h>
	92	# include <sys/procctl.h>
	93	# include <sys/sysctl.h>
90	94	extern "C" {
91	95	// <sys/umtx.h> must be included after <errno.h> and <sys/types.h> on
92	96	// FreeBSD 9.2 and 10.0.
93		#include <sys/umtx.h>
	97	# include <sys/umtx.h>
94	98	}
95		#include <sys/thr.h>
96		#endif // SANITIZER_FREEBSD
	99	# include <sys/thr.h>
	100	# endif // SANITIZER_FREEBSD
97	101
98		#if SANITIZER_NETBSD
99		#include <limits.h> // For NAME_MAX
100		#include <sys/sysctl.h>
101		#include <sys/exec.h>
	102	# if SANITIZER_NETBSD
	103	# include <limits.h> // For NAME_MAX
	104	# include <sys/exec.h>
	105	# include <sys/sysctl.h>
102	106	extern struct ps_strings *__ps_strings;
103		#endif // SANITIZER_NETBSD
	107	# endif // SANITIZER_NETBSD
104	108
105		#if SANITIZER_SOLARIS
106		#include <stdlib.h>
107		#include <thread.h>
108		#define environ _environ
109		#endif
	109	# if SANITIZER_SOLARIS
	110	# include <stdlib.h>
	111	# include <thread.h>
	112	# define environ _environ
	113	# endif
110	114
111	115	extern char **environ;
112	116
113		#if SANITIZER_LINUX
	117	# if SANITIZER_LINUX
114	118	// <linux/time.h>
115	119	struct kernel_timeval {
116	120	long tv_sec;
...	...	@@ -123,36 +127,32 @@ const int FUTEX_WAKE = 1;
123	127	const int FUTEX_PRIVATE_FLAG = 128;
124	128	const int FUTEX_WAIT_PRIVATE = FUTEX_WAIT \| FUTEX_PRIVATE_FLAG;
125	129	const int FUTEX_WAKE_PRIVATE = FUTEX_WAKE \| FUTEX_PRIVATE_FLAG;
126		#endif // SANITIZER_LINUX
	130	# endif // SANITIZER_LINUX
127	131
128	132	// Are we using 32-bit or 64-bit Linux syscalls?
129	133	// x32 (which defines __x86_64__) has SANITIZER_WORDSIZE == 32
130	134	// but it still needs to use 64-bit syscalls.
131		#if SANITIZER_LINUX && (defined(__x86_64__) \|\| defined(__powerpc64__) \|\| \
132		SANITIZER_WORDSIZE == 64 \|\| \
133		(defined(__mips__) && _MIPS_SIM == _ABIN32))
134		# define SANITIZER_LINUX_USES_64BIT_SYSCALLS 1
135		#else
136		# define SANITIZER_LINUX_USES_64BIT_SYSCALLS 0
137		#endif
	135	# if SANITIZER_LINUX && (defined(__x86_64__) \|\| defined(__powerpc64__) \|\| \
	136	SANITIZER_WORDSIZE == 64 \|\| \
	137	(defined(__mips__) && _MIPS_SIM == _ABIN32))
	138	# define SANITIZER_LINUX_USES_64BIT_SYSCALLS 1
	139	# else
	140	# define SANITIZER_LINUX_USES_64BIT_SYSCALLS 0
	141	# endif
138	142
139		// Note : FreeBSD had implemented both
140		// Linux apis, available from
141		// future 12.x version most likely
142		#if SANITIZER_LINUX && defined(__NR_getrandom)
143		# if !defined(GRND_NONBLOCK)
144		# define GRND_NONBLOCK 1
145		# endif
146		# define SANITIZER_USE_GETRANDOM 1
147		#else
148		# define SANITIZER_USE_GETRANDOM 0
149		#endif // SANITIZER_LINUX && defined(__NR_getrandom)
150
151		#if SANITIZER_FREEBSD && __FreeBSD_version >= 1200000
152		# define SANITIZER_USE_GETENTROPY 1
153		#else
154		# define SANITIZER_USE_GETENTROPY 0
155		#endif
	143	// Note : FreeBSD implemented both Linux and OpenBSD apis.
	144	# if SANITIZER_LINUX && defined(__NR_getrandom)
	145	# if !defined(GRND_NONBLOCK)
	146	# define GRND_NONBLOCK 1
	147	# endif
	148	# define SANITIZER_USE_GETRANDOM 1
	149	# else
	150	# define SANITIZER_USE_GETRANDOM 0
	151	# endif // SANITIZER_LINUX && defined(__NR_getrandom)
	152
	153	# if SANITIZER_FREEBSD
	154	# define SANITIZER_USE_GETENTROPY 1
	155	# endif
156	156
157	157	namespace __sanitizer {
158	158
...	...	@@ -160,6 +160,7 @@ void SetSigProcMask(__sanitizer_sigset_t set, __sanitizer_sigset_t oldset) {
160	160	CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, set, oldset));
161	161	}
162	162
	163	// Block asynchronous signals
163	164	void BlockSignals(__sanitizer_sigset_t *oldset) {
164	165	__sanitizer_sigset_t set;
165	166	internal_sigfillset(&set);
...	...	@@ -174,7 +175,17 @@ void BlockSignals(__sanitizer_sigset_t *oldset) {
174	175	// If this signal is blocked, such calls cannot be handled and the process may
175	176	// hang.
176	177	internal_sigdelset(&set, 31);
	178
	179	// Don't block synchronous signals
	180	internal_sigdelset(&set, SIGSEGV);
	181	internal_sigdelset(&set, SIGBUS);
	182	internal_sigdelset(&set, SIGILL);
	183	internal_sigdelset(&set, SIGTRAP);
	184	internal_sigdelset(&set, SIGABRT);
	185	internal_sigdelset(&set, SIGFPE);
	186	internal_sigdelset(&set, SIGPIPE);
177	187	# endif
	188
178	189	SetSigProcMask(&set, oldset);
179	190	}
180	191
...	...	@@ -203,33 +214,33 @@ ScopedBlockSignals::~ScopedBlockSignals() { SetSigProcMask(&saved_, nullptr); }
203	214	# endif
204	215
205	216	// --------------- sanitizer_libc.h
206		#if !SANITIZER_SOLARIS && !SANITIZER_NETBSD
207		#if !SANITIZER_S390
	217	# if !SANITIZER_SOLARIS && !SANITIZER_NETBSD
	218	# if !SANITIZER_S390
208	219	uptr internal_mmap(void *addr, uptr length, int prot, int flags, int fd,
209	220	u64 offset) {
210		#if SANITIZER_FREEBSD \|\| SANITIZER_LINUX_USES_64BIT_SYSCALLS
	221	# if SANITIZER_FREEBSD \|\| SANITIZER_LINUX_USES_64BIT_SYSCALLS
211	222	return internal_syscall(SYSCALL(mmap), (uptr)addr, length, prot, flags, fd,
212	223	offset);
213		#else
	224	# else
214	225	// mmap2 specifies file offset in 4096-byte units.
215	226	CHECK(IsAligned(offset, 4096));
216	227	return internal_syscall(SYSCALL(mmap2), addr, length, prot, flags, fd,
217		offset / 4096);
218		#endif
	228	(OFF_T)(offset / 4096));
	229	# endif
219	230	}
220		#endif // !SANITIZER_S390
	231	# endif // !SANITIZER_S390
221	232
222	233	uptr internal_munmap(void *addr, uptr length) {
223	234	return internal_syscall(SYSCALL(munmap), (uptr)addr, length);
224	235	}
225	236
226		#if SANITIZER_LINUX
	237	# if SANITIZER_LINUX
227	238	uptr internal_mremap(void *old_address, uptr old_size, uptr new_size, int flags,
228	239	void *new_address) {
229	240	return internal_syscall(SYSCALL(mremap), (uptr)old_address, old_size,
230	241	new_size, flags, (uptr)new_address);
231	242	}
232		#endif
	243	# endif
233	244
234	245	int internal_mprotect(void *addr, uptr length, int prot) {
235	246	return internal_syscall(SYSCALL(mprotect), (uptr)addr, length, prot);
...	...	@@ -239,25 +250,23 @@ int internal_madvise(uptr addr, uptr length, int advice) {
239	250	return internal_syscall(SYSCALL(madvise), addr, length, advice);
240	251	}
241	252
242		uptr internal_close(fd_t fd) {
243		return internal_syscall(SYSCALL(close), fd);
244		}
	253	uptr internal_close(fd_t fd) { return internal_syscall(SYSCALL(close), fd); }
245	254
246	255	uptr internal_open(const char *filename, int flags) {
247	256	# if SANITIZER_LINUX
248	257	return internal_syscall(SYSCALL(openat), AT_FDCWD, (uptr)filename, flags);
249		#else
	258	# else
250	259	return internal_syscall(SYSCALL(open), (uptr)filename, flags);
251		#endif
	260	# endif
252	261	}
253	262
254	263	uptr internal_open(const char *filename, int flags, u32 mode) {
255	264	# if SANITIZER_LINUX
256	265	return internal_syscall(SYSCALL(openat), AT_FDCWD, (uptr)filename, flags,
257	266	mode);
258		#else
	267	# else
259	268	return internal_syscall(SYSCALL(open), (uptr)filename, flags, mode);
260		#endif
	269	# endif
261	270	}
262	271
263	272	uptr internal_read(fd_t fd, void *buf, uptr count) {
...	...	@@ -276,12 +285,12 @@ uptr internal_write(fd_t fd, const void *buf, uptr count) {
276	285
277	286	uptr internal_ftruncate(fd_t fd, uptr size) {
278	287	sptr res;
279		HANDLE_EINTR(res, (sptr)internal_syscall(SYSCALL(ftruncate), fd,
280		(OFF_T)size));
	288	HANDLE_EINTR(res,
	289	(sptr)internal_syscall(SYSCALL(ftruncate), fd, (OFF_T)size));
281	290	return res;
282	291	}
283	292
284		#if (!SANITIZER_LINUX_USES_64BIT_SYSCALLS \|\| SANITIZER_SPARC) && SANITIZER_LINUX
	293	# if !SANITIZER_LINUX_USES_64BIT_SYSCALLS && SANITIZER_LINUX
285	294	static void stat64_to_stat(struct stat64 in, struct stat out) {
286	295	internal_memset(out, 0, sizeof(*out));
287	296	out->st_dev = in->st_dev;
...	...	@@ -298,9 +307,9 @@ static void stat64_to_stat(struct stat64 in, struct stat out) {
298	307	out->st_mtime = in->st_mtime;
299	308	out->st_ctime = in->st_ctime;
300	309	}
301		#endif
	310	# endif
302	311
303		#if SANITIZER_LINUX && defined(__loongarch__)
	312	# if SANITIZER_LINUX && defined(__loongarch__)
304	313	static void statx_to_stat(struct statx in, struct stat out) {
305	314	internal_memset(out, 0, sizeof(*out));
306	315	out->st_dev = makedev(in->stx_dev_major, in->stx_dev_minor);
...	...	@@ -320,27 +329,32 @@ static void statx_to_stat(struct statx in, struct stat out) {
320	329	out->st_ctime = in->stx_ctime.tv_sec;
321	330	out->st_ctim.tv_nsec = in->stx_ctime.tv_nsec;
322	331	}
323		#endif
	332	# endif
324	333
325		#if SANITIZER_MIPS64
	334	# if SANITIZER_MIPS64 \|\| SANITIZER_SPARC64
	335	# if SANITIZER_MIPS64
	336	typedef struct kernel_stat kstat_t;
	337	# else
	338	typedef struct kernel_stat64 kstat_t;
	339	# endif
326	340	// Undefine compatibility macros from <sys/stat.h>
327	341	// so that they would not clash with the kernel_stat
328	342	// st_[a\|m\|c]time fields
329		#if !SANITIZER_GO
330		#undef st_atime
331		#undef st_mtime
332		#undef st_ctime
333		#endif
334		#if defined(SANITIZER_ANDROID)
	343	# if !SANITIZER_GO
	344	# undef st_atime
	345	# undef st_mtime
	346	# undef st_ctime
	347	# endif
	348	# if defined(SANITIZER_ANDROID)
335	349	// Bionic sys/stat.h defines additional macros
336	350	// for compatibility with the old NDKs and
337	351	// they clash with the kernel_stat structure
338	352	// st_[a\|m\|c]time_nsec fields.
339		#undef st_atime_nsec
340		#undef st_mtime_nsec
341		#undef st_ctime_nsec
342		#endif
343		static void kernel_stat_to_stat(struct kernel_stat in, struct stat out) {
	353	# undef st_atime_nsec
	354	# undef st_mtime_nsec
	355	# undef st_ctime_nsec
	356	# endif
	357	static void kernel_stat_to_stat(kstat_t in, struct stat out) {
344	358	internal_memset(out, 0, sizeof(*out));
345	359	out->st_dev = in->st_dev;
346	360	out->st_ino = in->st_ino;
...	...	@@ -352,96 +366,113 @@ static void kernel_stat_to_stat(struct kernel_stat in, struct stat out) {
352	366	out->st_size = in->st_size;
353	367	out->st_blksize = in->st_blksize;
354	368	out->st_blocks = in->st_blocks;
355		#if defined(__USE_MISC) \|\| \
356		defined(__USE_XOPEN2K8) \|\| \
357		defined(SANITIZER_ANDROID)
	369	# if defined(__USE_MISC) \|\| defined(__USE_XOPEN2K8) \|\| \
	370	defined(SANITIZER_ANDROID)
358	371	out->st_atim.tv_sec = in->st_atime;
359	372	out->st_atim.tv_nsec = in->st_atime_nsec;
360	373	out->st_mtim.tv_sec = in->st_mtime;
361	374	out->st_mtim.tv_nsec = in->st_mtime_nsec;
362	375	out->st_ctim.tv_sec = in->st_ctime;
363	376	out->st_ctim.tv_nsec = in->st_ctime_nsec;
364		#else
	377	# else
365	378	out->st_atime = in->st_atime;
366	379	out->st_atimensec = in->st_atime_nsec;
367	380	out->st_mtime = in->st_mtime;
368	381	out->st_mtimensec = in->st_mtime_nsec;
369	382	out->st_ctime = in->st_ctime;
370	383	out->st_atimensec = in->st_ctime_nsec;
371		#endif
	384	# endif
372	385	}
373		#endif
	386	# endif
374	387
375	388	uptr internal_stat(const char path, void buf) {
376		# if SANITIZER_FREEBSD
	389	# if SANITIZER_FREEBSD
377	390	return internal_syscall(SYSCALL(fstatat), AT_FDCWD, (uptr)path, (uptr)buf, 0);
378		# elif SANITIZER_LINUX
379		# if defined(__loongarch__)
	391	# elif SANITIZER_LINUX
	392	# if defined(__loongarch__)
380	393	struct statx bufx;
381	394	int res = internal_syscall(SYSCALL(statx), AT_FDCWD, (uptr)path,
382	395	AT_NO_AUTOMOUNT, STATX_BASIC_STATS, (uptr)&bufx);
383	396	statx_to_stat(&bufx, (struct stat *)buf);
384	397	return res;
385		# elif (SANITIZER_WORDSIZE == 64 \|\| SANITIZER_X32 \|\| \
386		(defined(__mips__) && _MIPS_SIM == _ABIN32)) && \
387		!SANITIZER_SPARC
	398	# elif (SANITIZER_WORDSIZE == 64 \|\| SANITIZER_X32 \|\| \
	399	(defined(__mips__) && _MIPS_SIM == _ABIN32)) && \
	400	!SANITIZER_SPARC
388	401	return internal_syscall(SYSCALL(newfstatat), AT_FDCWD, (uptr)path, (uptr)buf,
389	402	0);
390		# else
	403	# elif SANITIZER_SPARC64
	404	kstat_t buf64;
	405	int res = internal_syscall(SYSCALL(fstatat64), AT_FDCWD, (uptr)path,
	406	(uptr)&buf64, 0);
	407	kernel_stat_to_stat(&buf64, (struct stat *)buf);
	408	return res;
	409	# else
391	410	struct stat64 buf64;
392	411	int res = internal_syscall(SYSCALL(fstatat64), AT_FDCWD, (uptr)path,
393	412	(uptr)&buf64, 0);
394	413	stat64_to_stat(&buf64, (struct stat *)buf);
395	414	return res;
396		# endif
397		# else
	415	# endif
	416	# else
398	417	struct stat64 buf64;
399	418	int res = internal_syscall(SYSCALL(stat64), path, &buf64);
400	419	stat64_to_stat(&buf64, (struct stat *)buf);
401	420	return res;
402		# endif
	421	# endif
403	422	}
404	423
405	424	uptr internal_lstat(const char path, void buf) {
406		# if SANITIZER_FREEBSD
	425	# if SANITIZER_FREEBSD
407	426	return internal_syscall(SYSCALL(fstatat), AT_FDCWD, (uptr)path, (uptr)buf,
408	427	AT_SYMLINK_NOFOLLOW);
409		# elif SANITIZER_LINUX
410		# if defined(__loongarch__)
	428	# elif SANITIZER_LINUX
	429	# if defined(__loongarch__)
411	430	struct statx bufx;
412	431	int res = internal_syscall(SYSCALL(statx), AT_FDCWD, (uptr)path,
413	432	AT_SYMLINK_NOFOLLOW \| AT_NO_AUTOMOUNT,
414	433	STATX_BASIC_STATS, (uptr)&bufx);
415	434	statx_to_stat(&bufx, (struct stat *)buf);
416	435	return res;
417		# elif (defined(_LP64) \|\| SANITIZER_X32 \|\| \
418		(defined(__mips__) && _MIPS_SIM == _ABIN32)) && \
419		!SANITIZER_SPARC
	436	# elif (defined(_LP64) \|\| SANITIZER_X32 \|\| \
	437	(defined(__mips__) && _MIPS_SIM == _ABIN32)) && \
	438	!SANITIZER_SPARC
420	439	return internal_syscall(SYSCALL(newfstatat), AT_FDCWD, (uptr)path, (uptr)buf,
421	440	AT_SYMLINK_NOFOLLOW);
422		# else
	441	# elif SANITIZER_SPARC64
	442	kstat_t buf64;
	443	int res = internal_syscall(SYSCALL(fstatat64), AT_FDCWD, (uptr)path,
	444	(uptr)&buf64, AT_SYMLINK_NOFOLLOW);
	445	kernel_stat_to_stat(&buf64, (struct stat *)buf);
	446	return res;
	447	# else
423	448	struct stat64 buf64;
424	449	int res = internal_syscall(SYSCALL(fstatat64), AT_FDCWD, (uptr)path,
425	450	(uptr)&buf64, AT_SYMLINK_NOFOLLOW);
426	451	stat64_to_stat(&buf64, (struct stat *)buf);
427	452	return res;
428		# endif
429		# else
	453	# endif
	454	# else
430	455	struct stat64 buf64;
431	456	int res = internal_syscall(SYSCALL(lstat64), path, &buf64);
432	457	stat64_to_stat(&buf64, (struct stat *)buf);
433	458	return res;
434		# endif
	459	# endif
435	460	}
436	461
437	462	uptr internal_fstat(fd_t fd, void *buf) {
438		#if SANITIZER_FREEBSD \|\| SANITIZER_LINUX_USES_64BIT_SYSCALLS
439		#if SANITIZER_MIPS64
	463	# if SANITIZER_FREEBSD \|\| SANITIZER_LINUX_USES_64BIT_SYSCALLS
	464	# if SANITIZER_MIPS64
440	465	// For mips64, fstat syscall fills buffer in the format of kernel_stat
441		struct kernel_stat kbuf;
	466	kstat_t kbuf;
442	467	int res = internal_syscall(SYSCALL(fstat), fd, &kbuf);
443	468	kernel_stat_to_stat(&kbuf, (struct stat *)buf);
444	469	return res;
	470	# elif SANITIZER_LINUX && SANITIZER_SPARC64
	471	// For sparc64, fstat64 syscall fills buffer in the format of kernel_stat64
	472	kstat_t kbuf;
	473	int res = internal_syscall(SYSCALL(fstat64), fd, &kbuf);
	474	kernel_stat_to_stat(&kbuf, (struct stat *)buf);
	475	return res;
445	476	# elif SANITIZER_LINUX && defined(__loongarch__)
446	477	struct statx bufx;
447	478	int res = internal_syscall(SYSCALL(statx), fd, "", AT_EMPTY_PATH,
...	...	@@ -451,12 +482,12 @@ uptr internal_fstat(fd_t fd, void *buf) {
451	482	# else
452	483	return internal_syscall(SYSCALL(fstat), fd, (uptr)buf);
453	484	# endif
454		#else
	485	# else
455	486	struct stat64 buf64;
456	487	int res = internal_syscall(SYSCALL(fstat64), fd, &buf64);
457	488	stat64_to_stat(&buf64, (struct stat *)buf);
458	489	return res;
459		#endif
	490	# endif
460	491	}
461	492
462	493	uptr internal_filesize(fd_t fd) {
...	...	@@ -466,50 +497,46 @@ uptr internal_filesize(fd_t fd) {
466	497	return (uptr)st.st_size;
467	498	}
468	499
469		uptr internal_dup(int oldfd) {
470		return internal_syscall(SYSCALL(dup), oldfd);
471		}
	500	uptr internal_dup(int oldfd) { return internal_syscall(SYSCALL(dup), oldfd); }
472	501
473	502	uptr internal_dup2(int oldfd, int newfd) {
474	503	# if SANITIZER_LINUX
475	504	return internal_syscall(SYSCALL(dup3), oldfd, newfd, 0);
476		#else
	505	# else
477	506	return internal_syscall(SYSCALL(dup2), oldfd, newfd);
478		#endif
	507	# endif
479	508	}
480	509
481	510	uptr internal_readlink(const char path, char buf, uptr bufsize) {
482	511	# if SANITIZER_LINUX
483	512	return internal_syscall(SYSCALL(readlinkat), AT_FDCWD, (uptr)path, (uptr)buf,
484	513	bufsize);
485		#else
	514	# else
486	515	return internal_syscall(SYSCALL(readlink), (uptr)path, (uptr)buf, bufsize);
487		#endif
	516	# endif
488	517	}
489	518
490	519	uptr internal_unlink(const char *path) {
491	520	# if SANITIZER_LINUX
492	521	return internal_syscall(SYSCALL(unlinkat), AT_FDCWD, (uptr)path, 0);
493		#else
	522	# else
494	523	return internal_syscall(SYSCALL(unlink), (uptr)path);
495		#endif
	524	# endif
496	525	}
497	526
498	527	uptr internal_rename(const char oldpath, const char newpath) {
499		# if (defined(__riscv) \|\| defined(__loongarch__)) && defined(__linux__)
	528	# if (defined(__riscv) \|\| defined(__loongarch__)) && defined(__linux__)
500	529	return internal_syscall(SYSCALL(renameat2), AT_FDCWD, (uptr)oldpath, AT_FDCWD,
501	530	(uptr)newpath, 0);
502		# elif SANITIZER_LINUX
	531	# elif SANITIZER_LINUX
503	532	return internal_syscall(SYSCALL(renameat), AT_FDCWD, (uptr)oldpath, AT_FDCWD,
504	533	(uptr)newpath);
505		# else
	534	# else
506	535	return internal_syscall(SYSCALL(rename), (uptr)oldpath, (uptr)newpath);
507		# endif
	536	# endif
508	537	}
509	538
510		uptr internal_sched_yield() {
511		return internal_syscall(SYSCALL(sched_yield));
512		}
	539	uptr internal_sched_yield() { return internal_syscall(SYSCALL(sched_yield)); }
513	540
514	541	void internal_usleep(u64 useconds) {
515	542	struct timespec ts;
...	...	@@ -523,18 +550,18 @@ uptr internal_execve(const char filename, char const argv[],
523	550	return internal_syscall(SYSCALL(execve), (uptr)filename, (uptr)argv,
524	551	(uptr)envp);
525	552	}
526		#endif // !SANITIZER_SOLARIS && !SANITIZER_NETBSD
	553	# endif // !SANITIZER_SOLARIS && !SANITIZER_NETBSD
527	554
528		#if !SANITIZER_NETBSD
	555	# if !SANITIZER_NETBSD
529	556	void internal__exit(int exitcode) {
530		#if SANITIZER_FREEBSD \|\| SANITIZER_SOLARIS
	557	# if SANITIZER_FREEBSD \|\| SANITIZER_SOLARIS
531	558	internal_syscall(SYSCALL(exit), exitcode);
532		#else
	559	# else
533	560	internal_syscall(SYSCALL(exit_group), exitcode);
534		#endif
	561	# endif
535	562	Die(); // Unreachable.
536	563	}
537		#endif // !SANITIZER_NETBSD
	564	# endif // !SANITIZER_NETBSD
538	565
539	566	// ----------------- sanitizer_common.h
540	567	bool FileExists(const char *filename) {
...	...	@@ -556,30 +583,32 @@ bool DirExists(const char *path) {
556	583
557	584	# if !SANITIZER_NETBSD
558	585	tid_t GetTid() {
559		#if SANITIZER_FREEBSD
	586	# if SANITIZER_FREEBSD
560	587	long Tid;
561	588	thr_self(&Tid);
562	589	return Tid;
563		#elif SANITIZER_SOLARIS
	590	# elif SANITIZER_SOLARIS
564	591	return thr_self();
565		#else
	592	# else
566	593	return internal_syscall(SYSCALL(gettid));
567		#endif
	594	# endif
568	595	}
569	596
570	597	int TgKill(pid_t pid, tid_t tid, int sig) {
571		#if SANITIZER_LINUX
	598	# if SANITIZER_LINUX
572	599	return internal_syscall(SYSCALL(tgkill), pid, tid, sig);
573		#elif SANITIZER_FREEBSD
	600	# elif SANITIZER_FREEBSD
574	601	return internal_syscall(SYSCALL(thr_kill2), pid, tid, sig);
575		#elif SANITIZER_SOLARIS
	602	# elif SANITIZER_SOLARIS
576	603	(void)pid;
577		return thr_kill(tid, sig);
578		#endif
	604	errno = thr_kill(tid, sig);
	605	// TgKill is expected to return -1 on error, not an errno.
	606	return errno != 0 ? -1 : 0;
	607	# endif
579	608	}
580		#endif
	609	# endif
581	610
582		#if SANITIZER_GLIBC
	611	# if SANITIZER_GLIBC
583	612	u64 NanoTime() {
584	613	kernel_timeval tv;
585	614	internal_memset(&tv, 0, sizeof(tv));
...	...	@@ -590,19 +619,19 @@ u64 NanoTime() {
590	619	uptr internal_clock_gettime(__sanitizer_clockid_t clk_id, void *tp) {
591	620	return internal_syscall(SYSCALL(clock_gettime), clk_id, tp);
592	621	}
593		#elif !SANITIZER_SOLARIS && !SANITIZER_NETBSD
	622	# elif !SANITIZER_SOLARIS && !SANITIZER_NETBSD
594	623	u64 NanoTime() {
595	624	struct timespec ts;
596	625	clock_gettime(CLOCK_REALTIME, &ts);
597	626	return (u64)ts.tv_sec * 1000 * 1000 * 1000 + ts.tv_nsec;
598	627	}
599		#endif
	628	# endif
600	629
601	630	// Like getenv, but reads env directly from /proc (on Linux) or parses the
602	631	// 'environ' array (on some others) and does not use libc. This function
603	632	// should be called first inside __asan_init.
604	633	const char GetEnv(const char name) {
605		#if SANITIZER_FREEBSD \|\| SANITIZER_NETBSD \|\| SANITIZER_SOLARIS
	634	# if SANITIZER_FREEBSD \|\| SANITIZER_NETBSD \|\| SANITIZER_SOLARIS
606	635	if (::environ != 0) {
607	636	uptr NameLen = internal_strlen(name);
608	637	for (char *Env = ::environ; Env != 0; Env++) {
...	...	@@ -611,7 +640,7 @@ const char GetEnv(const char name) {
611	640	}
612	641	}
613	642	return 0; // Not found.
614		#elif SANITIZER_LINUX
	643	# elif SANITIZER_LINUX
615	644	static char *environ;
616	645	static uptr len;
617	646	static bool inited;
...	...	@@ -621,13 +650,13 @@ const char GetEnv(const char name) {
621	650	if (!ReadFileToBuffer("/proc/self/environ", &environ, &environ_size, &len))
622	651	environ = nullptr;
623	652	}
624		if (!environ \|\| len == 0) return nullptr;
	653	if (!environ \|\| len == 0)
	654	return nullptr;
625	655	uptr namelen = internal_strlen(name);
626	656	const char *p = environ;
627	657	while (*p != '\0') { // will happen at the \0\0 that terminates the buffer
628	658	// proc file has the format NAME=value\0NAME=value\0NAME=value\0...
629		const char* endp =
630		(char*)internal_memchr(p, '\0', len - (p - environ));
	659	const char endp = (char )internal_memchr(p, '\0', len - (p - environ));
631	660	if (!endp) // this entry isn't NUL terminated
632	661	return nullptr;
633	662	else if (!internal_memcmp(p, name, namelen) && p[namelen] == '=') // Match.
...	...	@@ -635,18 +664,18 @@ const char GetEnv(const char name) {
635	664	p = endp + 1;
636	665	}
637	666	return nullptr; // Not found.
638		#else
639		#error "Unsupported platform"
640		#endif
	667	# else
	668	# error "Unsupported platform"
	669	# endif
641	670	}
642	671
643		#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD && !SANITIZER_GO
	672	# if !SANITIZER_FREEBSD && !SANITIZER_NETBSD && !SANITIZER_GO
644	673	extern "C" {
645	674	SANITIZER_WEAK_ATTRIBUTE extern void *__libc_stack_end;
646	675	}
647		#endif
	676	# endif
648	677
649		#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
	678	# if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
650	679	static void ReadNullSepFileToArray(const char path, char **arr,
651	680	int arr_size) {
652	681	char *buff;
...	...	@@ -659,20 +688,21 @@ static void ReadNullSepFileToArray(const char path, char **arr,
659	688	}
660	689	(*arr)[0] = buff;
661	690	int count, i;
662		for (count = 1, i = 1; ; i++) {
	691	for (count = 1, i = 1;; i++) {
663	692	if (buff[i] == 0) {
664		if (buff[i+1] == 0) break;
665		(*arr)[count] = &buff[i+1];
	693	if (buff[i + 1] == 0)
	694	break;
	695	(*arr)[count] = &buff[i + 1];
666	696	CHECK_LE(count, arr_size - 1); // FIXME: make this more flexible.
667	697	count++;
668	698	}
669	699	}
670	700	(*arr)[count] = nullptr;
671	701	}
672		#endif
	702	# endif
673	703
674	704	static void GetArgsAndEnv(char *argv, char *envp) {
675		#if SANITIZER_FREEBSD
	705	# if SANITIZER_FREEBSD
676	706	// On FreeBSD, retrieving the argument and environment arrays is done via the
677	707	// kern.ps_strings sysctl, which returns a pointer to a structure containing
678	708	// this information. See also <sys/exec.h>.
...	...	@@ -684,30 +714,30 @@ static void GetArgsAndEnv(char *argv, char *envp) {
684	714	}
685	715	*argv = pss->ps_argvstr;
686	716	*envp = pss->ps_envstr;
687		#elif SANITIZER_NETBSD
	717	# elif SANITIZER_NETBSD
688	718	*argv = __ps_strings->ps_argvstr;
689	719	*envp = __ps_strings->ps_envstr;
690		#else // SANITIZER_FREEBSD
691		#if !SANITIZER_GO
	720	# else // SANITIZER_FREEBSD
	721	# if !SANITIZER_GO
692	722	if (&__libc_stack_end) {
693		uptr* stack_end = (uptr*)__libc_stack_end;
	723	uptr stack_end = (uptr )__libc_stack_end;
694	724	// Normally argc can be obtained from *stack_end, however, on ARM glibc's
695	725	// _start clobbers it:
696	726	// https://sourceware.org/git/?p=glibc.git;a=blob;f=sysdeps/arm/start.S;hb=refs/heads/release/2.31/master#l75
697	727	// Do not special-case ARM and infer argc from argv everywhere.
698	728	int argc = 0;
699	729	while (stack_end[argc + 1]) argc++;
700		argv = (char*)(stack_end + 1);
701		envp = (char*)(stack_end + argc + 2);
	730	argv = (char *)(stack_end + 1);
	731	envp = (char *)(stack_end + argc + 2);
702	732	} else {
703		#endif // !SANITIZER_GO
	733	# endif // !SANITIZER_GO
704	734	static const int kMaxArgv = 2000, kMaxEnvp = 2000;
705	735	ReadNullSepFileToArray("/proc/self/cmdline", argv, kMaxArgv);
706	736	ReadNullSepFileToArray("/proc/self/environ", envp, kMaxEnvp);
707		#if !SANITIZER_GO
	737	# if !SANITIZER_GO
708	738	}
709		#endif // !SANITIZER_GO
710		#endif // SANITIZER_FREEBSD
	739	# endif // !SANITIZER_GO
	740	# endif // SANITIZER_FREEBSD
711	741	}
712	742
713	743	char **GetArgv() {
...	...	@@ -722,12 +752,12 @@ char **GetEnviron() {
722	752	return envp;
723	753	}
724	754
725		#if !SANITIZER_SOLARIS
	755	# if !SANITIZER_SOLARIS
726	756	void FutexWait(atomic_uint32_t *p, u32 cmp) {
727	757	# if SANITIZER_FREEBSD
728	758	_umtx_op(p, UMTX_OP_WAIT_UINT, cmp, 0, 0);
729	759	# elif SANITIZER_NETBSD
730		sched_yield(); /* No userspace futex-like synchronization */
	760	sched_yield(); /* No userspace futex-like synchronization */
731	761	# else
732	762	internal_syscall(SYSCALL(futex), (uptr)p, FUTEX_WAIT_PRIVATE, cmp, 0, 0, 0);
733	763	# endif
...	...	@@ -737,7 +767,7 @@ void FutexWake(atomic_uint32_t *p, u32 count) {
737	767	# if SANITIZER_FREEBSD
738	768	_umtx_op(p, UMTX_OP_WAKE, count, 0, 0);
739	769	# elif SANITIZER_NETBSD
740		/* No userspace futex-like synchronization */
	770	/* No userspace futex-like synchronization */
741	771	# else
742	772	internal_syscall(SYSCALL(futex), (uptr)p, FUTEX_WAKE_PRIVATE, count, 0, 0, 0);
743	773	# endif
...	...	@@ -749,26 +779,26 @@ void FutexWake(atomic_uint32_t *p, u32 count) {
749	779	// The actual size of this structure is specified by d_reclen.
750	780	// Note that getdents64 uses a different structure format. We only provide the
751	781	// 32-bit syscall here.
752		#if SANITIZER_NETBSD
	782	# if SANITIZER_NETBSD
753	783	// Not used
754		#else
	784	# else
755	785	struct linux_dirent {
756	786	# if SANITIZER_X32 \|\| SANITIZER_LINUX
757	787	u64 d_ino;
758	788	u64 d_off;
759	789	# else
760		unsigned long d_ino;
761		unsigned long d_off;
	790	unsigned long d_ino;
	791	unsigned long d_off;
762	792	# endif
763		unsigned short d_reclen;
	793	unsigned short d_reclen;
764	794	# if SANITIZER_LINUX
765		unsigned char d_type;
	795	unsigned char d_type;
766	796	# endif
767		char d_name[256];
	797	char d_name[256];
768	798	};
769		#endif
	799	# endif
770	800
771		#if !SANITIZER_SOLARIS && !SANITIZER_NETBSD
	801	# if !SANITIZER_SOLARIS && !SANITIZER_NETBSD
772	802	// Syscall wrappers.
773	803	uptr internal_ptrace(int request, int pid, void addr, void data) {
774	804	return internal_syscall(SYSCALL(ptrace), request, pid, (uptr)addr,
...	...	@@ -780,24 +810,20 @@ uptr internal_waitpid(int pid, int *status, int options) {
780	810	0 /* rusage */);
781	811	}
782	812
783		uptr internal_getpid() {
784		return internal_syscall(SYSCALL(getpid));
785		}
	813	uptr internal_getpid() { return internal_syscall(SYSCALL(getpid)); }
786	814
787		uptr internal_getppid() {
788		return internal_syscall(SYSCALL(getppid));
789		}
	815	uptr internal_getppid() { return internal_syscall(SYSCALL(getppid)); }
790	816
791	817	int internal_dlinfo(void handle, int request, void p) {
792		#if SANITIZER_FREEBSD
	818	# if SANITIZER_FREEBSD
793	819	return dlinfo(handle, request, p);
794		#else
	820	# else
795	821	UNIMPLEMENTED();
796		#endif
	822	# endif
797	823	}
798	824
799	825	uptr internal_getdents(fd_t fd, struct linux_dirent *dirp, unsigned int count) {
800		#if SANITIZER_FREEBSD
	826	# if SANITIZER_FREEBSD
801	827	return internal_syscall(SYSCALL(getdirentries), fd, (uptr)dirp, count, NULL);
802	828	# elif SANITIZER_LINUX
803	829	return internal_syscall(SYSCALL(getdents64), fd, (uptr)dirp, count);
...	...	@@ -810,7 +836,7 @@ uptr internal_lseek(fd_t fd, OFF_T offset, int whence) {
810	836	return internal_syscall(SYSCALL(lseek), fd, offset, whence);
811	837	}
812	838
813		#if SANITIZER_LINUX
	839	# if SANITIZER_LINUX
814	840	uptr internal_prctl(int option, uptr arg2, uptr arg3, uptr arg4, uptr arg5) {
815	841	return internal_syscall(SYSCALL(prctl), option, arg2, arg3, arg4, arg5);
816	842	}
...	...	@@ -827,10 +853,16 @@ uptr internal_sigaltstack(const void ss, void oss) {
827	853	return internal_syscall(SYSCALL(sigaltstack), (uptr)ss, (uptr)oss);
828	854	}
829	855
	856	extern "C" pid_t __fork(void);
	857
830	858	int internal_fork() {
831	859	# if SANITIZER_LINUX
832	860	# if SANITIZER_S390
833	861	return internal_syscall(SYSCALL(clone), 0, SIGCHLD);
	862	# elif SANITIZER_SPARC
	863	// The clone syscall interface on SPARC differs massively from the rest,
	864	// so fall back to __fork.
	865	return __fork();
834	866	# else
835	867	return internal_syscall(SYSCALL(clone), SIGCHLD, 0);
836	868	# endif
...	...	@@ -839,7 +871,7 @@ int internal_fork() {
839	871	# endif
840	872	}
841	873
842		#if SANITIZER_FREEBSD
	874	# if SANITIZER_FREEBSD
843	875	int internal_sysctl(const int name, unsigned int namelen, void oldp,
844	876	uptr oldlenp, const void newp, uptr newlen) {
845	877	return internal_syscall(SYSCALL(__sysctl), name, namelen, oldp,
...	...	@@ -854,11 +886,11 @@ int internal_sysctlbyname(const char sname, void oldp, uptr *oldlenp,
854	886	// followed by sysctl(). To avoid calling the intercepted version and
855	887	// asserting if this happens during startup, call the real sysctlnametomib()
856	888	// followed by internal_sysctl() if the syscall is not available.
857		#ifdef SYS___sysctlbyname
	889	# ifdef SYS___sysctlbyname
858	890	return internal_syscall(SYSCALL(__sysctlbyname), sname,
859	891	internal_strlen(sname), oldp, (size_t *)oldlenp, newp,
860	892	(size_t)newlen);
861		#else
	893	# else
862	894	static decltype(sysctlnametomib) *real_sysctlnametomib = nullptr;
863	895	if (!real_sysctlnametomib)
864	896	real_sysctlnametomib =
...	...	@@ -870,12 +902,12 @@ int internal_sysctlbyname(const char sname, void oldp, uptr *oldlenp,
870	902	if (real_sysctlnametomib(sname, oid, &len) == -1)
871	903	return (-1);
872	904	return internal_sysctl(oid, len, oldp, oldlenp, newp, newlen);
873		#endif
	905	# endif
874	906	}
875		#endif
	907	# endif
876	908
877		#if SANITIZER_LINUX
878		#define SA_RESTORER 0x04000000
	909	# if SANITIZER_LINUX
	910	# define SA_RESTORER 0x04000000
879	911	// Doesn't set sa_restorer if the caller did not set it, so use with caution
880	912	//(see below).
881	913	int internal_sigaction_norestorer(int signum, const void act, void oldact) {
...	...	@@ -899,15 +931,15 @@ int internal_sigaction_norestorer(int signum, const void act, void oldact) {
899	931	// rt_sigaction, so we need to do the same (we'll need to reimplement the
900	932	// restorers; for x86_64 the restorer address can be obtained from
901	933	// oldact->sa_restorer upon a call to sigaction(xxx, NULL, oldact).
902		#if !SANITIZER_ANDROID \|\| !SANITIZER_MIPS32
	934	# if !SANITIZER_ANDROID \|\| !SANITIZER_MIPS32
903	935	k_act.sa_restorer = u_act->sa_restorer;
904		#endif
	936	# endif
905	937	}
906	938
907	939	uptr result = internal_syscall(SYSCALL(rt_sigaction), (uptr)signum,
908		(uptr)(u_act ? &k_act : nullptr),
909		(uptr)(u_oldact ? &k_oldact : nullptr),
910		(uptr)sizeof(__sanitizer_kernel_sigset_t));
	940	(uptr)(u_act ? &k_act : nullptr),
	941	(uptr)(u_oldact ? &k_oldact : nullptr),
	942	(uptr)sizeof(__sanitizer_kernel_sigset_t));
911	943
912	944	if ((result == 0) && u_oldact) {
913	945	u_oldact->handler = k_oldact.handler;
...	...	@@ -915,24 +947,24 @@ int internal_sigaction_norestorer(int signum, const void act, void oldact) {
915	947	internal_memcpy(&u_oldact->sa_mask, &k_oldact.sa_mask,
916	948	sizeof(__sanitizer_kernel_sigset_t));
917	949	u_oldact->sa_flags = k_oldact.sa_flags;
918		#if !SANITIZER_ANDROID \|\| !SANITIZER_MIPS32
	950	# if !SANITIZER_ANDROID \|\| !SANITIZER_MIPS32
919	951	u_oldact->sa_restorer = k_oldact.sa_restorer;
920		#endif
	952	# endif
921	953	}
922	954	return result;
923	955	}
924		#endif // SANITIZER_LINUX
	956	# endif // SANITIZER_LINUX
925	957
926	958	uptr internal_sigprocmask(int how, __sanitizer_sigset_t *set,
927	959	__sanitizer_sigset_t *oldset) {
928		#if SANITIZER_FREEBSD
	960	# if SANITIZER_FREEBSD
929	961	return internal_syscall(SYSCALL(sigprocmask), how, set, oldset);
930		#else
	962	# else
931	963	__sanitizer_kernel_sigset_t k_set = (__sanitizer_kernel_sigset_t )set;
932	964	__sanitizer_kernel_sigset_t k_oldset = (__sanitizer_kernel_sigset_t )oldset;
933	965	return internal_syscall(SYSCALL(rt_sigprocmask), (uptr)how, (uptr)k_set,
934	966	(uptr)k_oldset, sizeof(__sanitizer_kernel_sigset_t));
935		#endif
	967	# endif
936	968	}
937	969
938	970	void internal_sigfillset(__sanitizer_sigset_t *set) {
...	...	@@ -943,7 +975,7 @@ void internal_sigemptyset(__sanitizer_sigset_t *set) {
943	975	internal_memset(set, 0, sizeof(*set));
944	976	}
945	977
946		#if SANITIZER_LINUX
	978	# if SANITIZER_LINUX
947	979	void internal_sigdelset(__sanitizer_sigset_t *set, int signum) {
948	980	signum -= 1;
949	981	CHECK_GE(signum, 0);
...	...	@@ -963,7 +995,7 @@ bool internal_sigismember(__sanitizer_sigset_t *set, int signum) {
963	995	const uptr bit = signum % (sizeof(k_set->sig[0]) * 8);
964	996	return k_set->sig[idx] & ((uptr)1 << bit);
965	997	}
966		#elif SANITIZER_FREEBSD
	998	# elif SANITIZER_FREEBSD
967	999	uptr internal_procctl(int type, int id, int cmd, void *data) {
968	1000	return internal_syscall(SYSCALL(procctl), type, id, cmd, data);
969	1001	}
...	...	@@ -977,10 +1009,10 @@ bool internal_sigismember(__sanitizer_sigset_t *set, int signum) {
977	1009	sigset_t rset = reinterpret_cast<sigset_t >(set);
978	1010	return sigismember(rset, signum);
979	1011	}
980		#endif
981		#endif // !SANITIZER_SOLARIS
	1012	# endif
	1013	# endif // !SANITIZER_SOLARIS
982	1014
983		#if !SANITIZER_NETBSD
	1015	# if !SANITIZER_NETBSD
984	1016	// ThreadLister implementation.
985	1017	ThreadLister::ThreadLister(pid_t pid) : pid_(pid), buffer_(4096) {
986	1018	char task_directory_path[80];
...	...	@@ -1067,25 +1099,26 @@ ThreadLister::~ThreadLister() {
1067	1099	if (!internal_iserror(descriptor_))
1068	1100	internal_close(descriptor_);
1069	1101	}
1070		#endif
	1102	# endif
1071	1103
1072		#if SANITIZER_WORDSIZE == 32
	1104	# if SANITIZER_WORDSIZE == 32
1073	1105	// Take care of unusable kernel area in top gigabyte.
1074	1106	static uptr GetKernelAreaSize() {
1075		#if SANITIZER_LINUX && !SANITIZER_X32
	1107	# if SANITIZER_LINUX && !SANITIZER_X32
1076	1108	const uptr gbyte = 1UL << 30;
1077	1109
1078	1110	// Firstly check if there are writable segments
1079	1111	// mapped to top gigabyte (e.g. stack).
1080		MemoryMappingLayout proc_maps(/cache_enabled/true);
	1112	MemoryMappingLayout proc_maps(/cache_enabled/ true);
1081	1113	if (proc_maps.Error())
1082	1114	return 0;
1083	1115	MemoryMappedSegment segment;
1084	1116	while (proc_maps.Next(&segment)) {
1085		if ((segment.end >= 3 * gbyte) && segment.IsWritable()) return 0;
	1117	if ((segment.end >= 3 * gbyte) && segment.IsWritable())
	1118	return 0;
1086	1119	}
1087	1120
1088		#if !SANITIZER_ANDROID
	1121	# if !SANITIZER_ANDROID
1089	1122	// Even if nothing is mapped, top Gb may still be accessible
1090	1123	// if we are running on 64-bit kernel.
1091	1124	// Uname may report misleading results if personality type
...	...	@@ -1095,21 +1128,22 @@ static uptr GetKernelAreaSize() {
1095	1128	if (!(pers & PER_MASK) && internal_uname(&uname_info) == 0 &&
1096	1129	internal_strstr(uname_info.machine, "64"))
1097	1130	return 0;
1098		#endif // SANITIZER_ANDROID
	1131	# endif // SANITIZER_ANDROID
1099	1132
1100	1133	// Top gigabyte is reserved for kernel.
1101	1134	return gbyte;
1102		#else
	1135	# else
1103	1136	return 0;
1104		#endif // SANITIZER_LINUX && !SANITIZER_X32
	1137	# endif // SANITIZER_LINUX && !SANITIZER_X32
1105	1138	}
1106		#endif // SANITIZER_WORDSIZE == 32
	1139	# endif // SANITIZER_WORDSIZE == 32
1107	1140
1108	1141	uptr GetMaxVirtualAddress() {
1109		#if SANITIZER_NETBSD && defined(__x86_64__)
	1142	# if SANITIZER_NETBSD && defined(__x86_64__)
1110	1143	return 0x7f7ffffff000ULL; // (0x00007f8000000000 - PAGE_SIZE)
1111		#elif SANITIZER_WORDSIZE == 64
1112		# if defined(__powerpc64__) \|\| defined(__aarch64__) \|\| defined(__loongarch__)
	1144	# elif SANITIZER_WORDSIZE == 64
	1145	# if defined(__powerpc64__) \|\| defined(__aarch64__) \|\| \
	1146	defined(__loongarch__) \|\| SANITIZER_RISCV64
1113	1147	// On PowerPC64 we have two different address space layouts: 44- and 46-bit.
1114	1148	// We somehow need to figure out which one we are using now and choose
1115	1149	// one of 0x00000fffffffffffUL and 0x00003fffffffffffUL.
...	...	@@ -1118,97 +1152,97 @@ uptr GetMaxVirtualAddress() {
1118	1152	// This should (does) work for both PowerPC64 Endian modes.
1119	1153	// Similarly, aarch64 has multiple address space layouts: 39, 42 and 47-bit.
1120	1154	// loongarch64 also has multiple address space layouts: default is 47-bit.
	1155	// RISC-V 64 also has multiple address space layouts: 39, 48 and 57-bit.
1121	1156	return (1ULL << (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1)) - 1;
1122		#elif SANITIZER_RISCV64
1123		return (1ULL << 38) - 1;
1124		# elif SANITIZER_MIPS64
	1157	# elif SANITIZER_MIPS64
1125	1158	return (1ULL << 40) - 1; // 0x000000ffffffffffUL;
1126		# elif defined(__s390x__)
	1159	# elif defined(__s390x__)
1127	1160	return (1ULL << 53) - 1; // 0x001fffffffffffffUL;
1128		#elif defined(__sparc__)
	1161	# elif defined(__sparc__)
1129	1162	return ~(uptr)0;
1130		# else
	1163	# else
1131	1164	return (1ULL << 47) - 1; // 0x00007fffffffffffUL;
1132		# endif
1133		#else // SANITIZER_WORDSIZE == 32
1134		# if defined(__s390__)
	1165	# endif
	1166	# else // SANITIZER_WORDSIZE == 32
	1167	# if defined(__s390__)
1135	1168	return (1ULL << 31) - 1; // 0x7fffffff;
1136		# else
	1169	# else
1137	1170	return (1ULL << 32) - 1; // 0xffffffff;
1138		# endif
1139		#endif // SANITIZER_WORDSIZE
	1171	# endif
	1172	# endif // SANITIZER_WORDSIZE
1140	1173	}
1141	1174
1142	1175	uptr GetMaxUserVirtualAddress() {
1143	1176	uptr addr = GetMaxVirtualAddress();
1144		#if SANITIZER_WORDSIZE == 32 && !defined(__s390__)
	1177	# if SANITIZER_WORDSIZE == 32 && !defined(__s390__)
1145	1178	if (!common_flags()->full_address_space)
1146	1179	addr -= GetKernelAreaSize();
1147	1180	CHECK_LT(reinterpret_cast<uptr>(&addr), addr);
1148		#endif
	1181	# endif
1149	1182	return addr;
1150	1183	}
1151	1184
1152		#if !SANITIZER_ANDROID
	1185	# if !SANITIZER_ANDROID \|\| defined(__aarch64__)
1153	1186	uptr GetPageSize() {
1154		#if SANITIZER_LINUX && (defined(__x86_64__) \|\| defined(__i386__)) && \
1155		defined(EXEC_PAGESIZE)
	1187	# if SANITIZER_LINUX && (defined(__x86_64__) \|\| defined(__i386__)) && \
	1188	defined(EXEC_PAGESIZE)
1156	1189	return EXEC_PAGESIZE;
1157		#elif SANITIZER_FREEBSD \|\| SANITIZER_NETBSD
1158		// Use sysctl as sysconf can trigger interceptors internally.
	1190	# elif SANITIZER_FREEBSD \|\| SANITIZER_NETBSD
	1191	// Use sysctl as sysconf can trigger interceptors internally.
1159	1192	int pz = 0;
1160	1193	uptr pzl = sizeof(pz);
1161	1194	int mib[2] = {CTL_HW, HW_PAGESIZE};
1162	1195	int rv = internal_sysctl(mib, 2, &pz, &pzl, nullptr, 0);
1163	1196	CHECK_EQ(rv, 0);
1164	1197	return (uptr)pz;
1165		#elif SANITIZER_USE_GETAUXVAL
	1198	# elif SANITIZER_USE_GETAUXVAL
1166	1199	return getauxval(AT_PAGESZ);
1167		#else
	1200	# else
1168	1201	return sysconf(_SC_PAGESIZE); // EXEC_PAGESIZE may not be trustworthy.
1169		#endif
	1202	# endif
1170	1203	}
1171		#endif // !SANITIZER_ANDROID
	1204	# endif
1172	1205
1173		uptr ReadBinaryName(/out/char *buf, uptr buf_len) {
1174		#if SANITIZER_SOLARIS
	1206	uptr ReadBinaryName(/out/ char *buf, uptr buf_len) {
	1207	# if SANITIZER_SOLARIS
1175	1208	const char *default_module_name = getexecname();
1176	1209	CHECK_NE(default_module_name, NULL);
1177	1210	return internal_snprintf(buf, buf_len, "%s", default_module_name);
1178		#else
1179		#if SANITIZER_FREEBSD \|\| SANITIZER_NETBSD
1180		#if SANITIZER_FREEBSD
	1211	# else
	1212	# if SANITIZER_FREEBSD \|\| SANITIZER_NETBSD
	1213	# if SANITIZER_FREEBSD
1181	1214	const int Mib[4] = {CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, -1};
1182		#else
	1215	# else
1183	1216	const int Mib[4] = {CTL_KERN, KERN_PROC_ARGS, -1, KERN_PROC_PATHNAME};
1184		#endif
	1217	# endif
1185	1218	const char *default_module_name = "kern.proc.pathname";
1186	1219	uptr Size = buf_len;
1187	1220	bool IsErr =
1188	1221	(internal_sysctl(Mib, ARRAY_SIZE(Mib), buf, &Size, NULL, 0) != 0);
1189	1222	int readlink_error = IsErr ? errno : 0;
1190	1223	uptr module_name_len = Size;
1191		#else
	1224	# else
1192	1225	const char *default_module_name = "/proc/self/exe";
1193		uptr module_name_len = internal_readlink(
1194		default_module_name, buf, buf_len);
	1226	uptr module_name_len = internal_readlink(default_module_name, buf, buf_len);
1195	1227	int readlink_error;
1196	1228	bool IsErr = internal_iserror(module_name_len, &readlink_error);
1197		#endif // SANITIZER_SOLARIS
	1229	# endif
1198	1230	if (IsErr) {
1199	1231	// We can't read binary name for some reason, assume it's unknown.
1200		Report("WARNING: reading executable name failed with errno %d, "
1201		"some stack frames may not be symbolized\n", readlink_error);
1202		module_name_len = internal_snprintf(buf, buf_len, "%s",
1203		default_module_name);
	1232	Report(
	1233	"WARNING: reading executable name failed with errno %d, "
	1234	"some stack frames may not be symbolized\n",
	1235	readlink_error);
	1236	module_name_len =
	1237	internal_snprintf(buf, buf_len, "%s", default_module_name);
1204	1238	CHECK_LT(module_name_len, buf_len);
1205	1239	}
1206	1240	return module_name_len;
1207		#endif
	1241	# endif
1208	1242	}
1209	1243
1210	1244	uptr ReadLongProcessName(/out/ char *buf, uptr buf_len) {
1211		#if SANITIZER_LINUX
	1245	# if SANITIZER_LINUX
1212	1246	char *tmpbuf;
1213	1247	uptr tmpsize;
1214	1248	uptr tmplen;
...	...	@@ -1218,7 +1252,7 @@ uptr ReadLongProcessName(/out/ char *buf, uptr buf_len) {
1218	1252	UnmapOrDie(tmpbuf, tmpsize);
1219	1253	return internal_strlen(buf);
1220	1254	}
1221		#endif
	1255	# endif
1222	1256	return ReadBinaryName(buf, buf_len);
1223	1257	}
1224	1258
...	...	@@ -1228,20 +1262,22 @@ bool LibraryNameIs(const char full_name, const char base_name) {
1228	1262	// Strip path.
1229	1263	while (*name != '\0') name++;
1230	1264	while (name > full_name && *name != '/') name--;
1231		if (*name == '/') name++;
	1265	if (*name == '/')
	1266	name++;
1232	1267	uptr base_name_length = internal_strlen(base_name);
1233		if (internal_strncmp(name, base_name, base_name_length)) return false;
	1268	if (internal_strncmp(name, base_name, base_name_length))
	1269	return false;
1234	1270	return (name[base_name_length] == '-' \|\| name[base_name_length] == '.');
1235	1271	}
1236	1272
1237		#if !SANITIZER_ANDROID
	1273	# if !SANITIZER_ANDROID
1238	1274	// Call cb for each region mapped by map.
1239	1275	void ForEachMappedRegion(link_map map, void (cb)(const void *, uptr)) {
1240	1276	CHECK_NE(map, nullptr);
1241		#if !SANITIZER_FREEBSD
	1277	# if !SANITIZER_FREEBSD
1242	1278	typedef ElfW(Phdr) Elf_Phdr;
1243	1279	typedef ElfW(Ehdr) Elf_Ehdr;
1244		#endif // !SANITIZER_FREEBSD
	1280	# endif // !SANITIZER_FREEBSD
1245	1281	char base = (char )map->l_addr;
1246	1282	Elf_Ehdr ehdr = (Elf_Ehdr )base;
1247	1283	char *phdrs = base + ehdr->e_phoff;
...	...	@@ -1273,10 +1309,10 @@ void ForEachMappedRegion(link_map map, void (cb)(const void *, uptr)) {
1273	1309	}
1274	1310	}
1275	1311	}
1276		#endif
	1312	# endif
1277	1313
1278		#if SANITIZER_LINUX
1279		#if defined(__x86_64__)
	1314	# if SANITIZER_LINUX
	1315	# if defined(__x86_64__)
1280	1316	// We cannot use glibc's clone wrapper, because it messes with the child
1281	1317	// task's TLS. It writes the PID and TID of the child task to its thread
1282	1318	// descriptor, but in our case the child task shares the thread descriptor with
...	...	@@ -1295,50 +1331,46 @@ uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
1295	1331	register void *r8 __asm__("r8") = newtls;
1296	1332	register int *r10 __asm__("r10") = child_tidptr;
1297	1333	__asm__ __volatile__(
1298		/* %rax = syscall(%rax = SYSCALL(clone),
1299		* %rdi = flags,
1300		* %rsi = child_stack,
1301		* %rdx = parent_tidptr,
1302		* %r8 = new_tls,
1303		* %r10 = child_tidptr)
1304		*/
1305		"syscall\n"
1306
1307		/* if (%rax != 0)
1308		* return;
1309		*/
1310		"testq %%rax,%%rax\n"
1311		"jnz 1f\n"
1312
1313		/* In the child. Terminate unwind chain. */
1314		// XXX: We should also terminate the CFI unwind chain
1315		// here. Unfortunately clang 3.2 doesn't support the
1316		// necessary CFI directives, so we skip that part.
1317		"xorq %%rbp,%%rbp\n"
1318
1319		/* Call "fn(arg)". */
1320		"popq %%rax\n"
1321		"popq %%rdi\n"
1322		"call *%%rax\n"
1323
1324		/* Call _exit(%rax). */
1325		"movq %%rax,%%rdi\n"
1326		"movq %2,%%rax\n"
1327		"syscall\n"
1328
1329		/* Return to parent. */
1330		"1:\n"
1331		: "=a" (res)
1332		: "a"(SYSCALL(clone)), "i"(SYSCALL(exit)),
1333		"S"(child_stack),
1334		"D"(flags),
1335		"d"(parent_tidptr),
1336		"r"(r8),
1337		"r"(r10)
1338		: "memory", "r11", "rcx");
	1334	/* %rax = syscall(%rax = SYSCALL(clone),
	1335	* %rdi = flags,
	1336	* %rsi = child_stack,
	1337	* %rdx = parent_tidptr,
	1338	* %r8 = new_tls,
	1339	* %r10 = child_tidptr)
	1340	*/
	1341	"syscall\n"
	1342
	1343	/* if (%rax != 0)
	1344	* return;
	1345	*/
	1346	"testq %%rax,%%rax\n"
	1347	"jnz 1f\n"
	1348
	1349	/* In the child. Terminate unwind chain. */
	1350	// XXX: We should also terminate the CFI unwind chain
	1351	// here. Unfortunately clang 3.2 doesn't support the
	1352	// necessary CFI directives, so we skip that part.
	1353	"xorq %%rbp,%%rbp\n"
	1354
	1355	/* Call "fn(arg)". */
	1356	"popq %%rax\n"
	1357	"popq %%rdi\n"
	1358	"call *%%rax\n"
	1359
	1360	/* Call _exit(%rax). */
	1361	"movq %%rax,%%rdi\n"
	1362	"movq %2,%%rax\n"
	1363	"syscall\n"
	1364
	1365	/* Return to parent. */
	1366	"1:\n"
	1367	: "=a"(res)
	1368	: "a"(SYSCALL(clone)), "i"(SYSCALL(exit)), "S"(child_stack), "D"(flags),
	1369	"d"(parent_tidptr), "r"(r8), "r"(r10)
	1370	: "memory", "r11", "rcx");
1339	1371	return res;
1340	1372	}
1341		#elif defined(__mips__)
	1373	# elif defined(__mips__)
1342	1374	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
1343	1375	int parent_tidptr, void newtls, int *child_tidptr) {
1344	1376	long long res;
...	...	@@ -1353,68 +1385,63 @@ uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
1353	1385	// We don't have proper CFI directives here because it requires alot of code
1354	1386	// for very marginal benefits.
1355	1387	__asm__ __volatile__(
1356		/* $v0 = syscall($v0 = __NR_clone,
1357		* $a0 = flags,
1358		* $a1 = child_stack,
1359		* $a2 = parent_tidptr,
1360		* $a3 = new_tls,
1361		* $a4 = child_tidptr)
1362		*/
1363		".cprestore 16;\n"
1364		"move $4,%1;\n"
1365		"move $5,%2;\n"
1366		"move $6,%3;\n"
1367		"move $7,%4;\n"
1368		/* Store the fifth argument on stack
1369		* if we are using 32-bit abi.
1370		*/
1371		#if SANITIZER_WORDSIZE == 32
1372		"lw %5,16($29);\n"
1373		#else
1374		"move $8,%5;\n"
1375		#endif
1376		"li $2,%6;\n"
1377		"syscall;\n"
1378
1379		/* if ($v0 != 0)
1380		* return;
1381		*/
1382		"bnez $2,1f;\n"
1383
1384		/* Call "fn(arg)". */
1385		#if SANITIZER_WORDSIZE == 32
1386		#ifdef __BIG_ENDIAN__
1387		"lw $25,4($29);\n"
1388		"lw $4,12($29);\n"
1389		#else
1390		"lw $25,0($29);\n"
1391		"lw $4,8($29);\n"
1392		#endif
1393		#else
1394		"ld $25,0($29);\n"
1395		"ld $4,8($29);\n"
1396		#endif
1397		"jal $25;\n"
1398
1399		/* Call _exit($v0). */
1400		"move $4,$2;\n"
1401		"li $2,%7;\n"
1402		"syscall;\n"
1403
1404		/* Return to parent. */
1405		"1:\n"
1406		: "=r" (res)
1407		: "r"(flags),
1408		"r"(child_stack),
1409		"r"(parent_tidptr),
1410		"r"(a3),
1411		"r"(a4),
1412		"i"(__NR_clone),
1413		"i"(__NR_exit)
1414		: "memory", "$29" );
	1388	/* $v0 = syscall($v0 = __NR_clone,
	1389	* $a0 = flags,
	1390	* $a1 = child_stack,
	1391	* $a2 = parent_tidptr,
	1392	* $a3 = new_tls,
	1393	* $a4 = child_tidptr)
	1394	*/
	1395	".cprestore 16;\n"
	1396	"move $4,%1;\n"
	1397	"move $5,%2;\n"
	1398	"move $6,%3;\n"
	1399	"move $7,%4;\n"
	1400	/* Store the fifth argument on stack
	1401	* if we are using 32-bit abi.
	1402	*/
	1403	# if SANITIZER_WORDSIZE == 32
	1404	"lw %5,16($29);\n"
	1405	# else
	1406	"move $8,%5;\n"
	1407	# endif
	1408	"li $2,%6;\n"
	1409	"syscall;\n"
	1410
	1411	/* if ($v0 != 0)
	1412	* return;
	1413	*/
	1414	"bnez $2,1f;\n"
	1415
	1416	/* Call "fn(arg)". */
	1417	# if SANITIZER_WORDSIZE == 32
	1418	# ifdef __BIG_ENDIAN__
	1419	"lw $25,4($29);\n"
	1420	"lw $4,12($29);\n"
	1421	# else
	1422	"lw $25,0($29);\n"
	1423	"lw $4,8($29);\n"
	1424	# endif
	1425	# else
	1426	"ld $25,0($29);\n"
	1427	"ld $4,8($29);\n"
	1428	# endif
	1429	"jal $25;\n"
	1430
	1431	/* Call _exit($v0). */
	1432	"move $4,$2;\n"
	1433	"li $2,%7;\n"
	1434	"syscall;\n"
	1435
	1436	/* Return to parent. */
	1437	"1:\n"
	1438	: "=r"(res)
	1439	: "r"(flags), "r"(child_stack), "r"(parent_tidptr), "r"(a3), "r"(a4),
	1440	"i"(__NR_clone), "i"(__NR_exit)
	1441	: "memory", "$29");
1415	1442	return res;
1416	1443	}
1417		#elif SANITIZER_RISCV64
	1444	# elif SANITIZER_RISCV64
1418	1445	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
1419	1446	int parent_tidptr, void newtls, int *child_tidptr) {
1420	1447	if (!fn \|\| !child_stack)
...	...	@@ -1455,7 +1482,7 @@ uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
1455	1482	: "memory");
1456	1483	return res;
1457	1484	}
1458		#elif defined(__aarch64__)
	1485	# elif defined(__aarch64__)
1459	1486	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
1460	1487	int parent_tidptr, void newtls, int *child_tidptr) {
1461	1488	register long long res __asm__("x0");
...	...	@@ -1466,47 +1493,45 @@ uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
1466	1493	((unsigned long long *)child_stack)[0] = (uptr)fn;
1467	1494	((unsigned long long *)child_stack)[1] = (uptr)arg;
1468	1495
1469		register int (__fn)(void ) __asm__("x0") = fn;
	1496	register int (__fn)(void ) __asm__("x0") = fn;
1470	1497	register void *__stack __asm__("x1") = child_stack;
1471		register int __flags __asm__("x2") = flags;
1472		register void *__arg __asm__("x3") = arg;
1473		register int *__ptid __asm__("x4") = parent_tidptr;
1474		register void *__tls __asm__("x5") = newtls;
1475		register int *__ctid __asm__("x6") = child_tidptr;
	1498	register int __flags __asm__("x2") = flags;
	1499	register void *__arg __asm__("x3") = arg;
	1500	register int *__ptid __asm__("x4") = parent_tidptr;
	1501	register void *__tls __asm__("x5") = newtls;
	1502	register int *__ctid __asm__("x6") = child_tidptr;
1476	1503
1477	1504	__asm__ __volatile__(
1478		"mov x0,x2\n" /* flags */
1479		"mov x2,x4\n" /* ptid */
1480		"mov x3,x5\n" /* tls */
1481		"mov x4,x6\n" /* ctid */
1482		"mov x8,%9\n" /* clone */
1483
1484		"svc 0x0\n"
1485
1486		/* if (%r0 != 0)
1487		* return %r0;
1488		*/
1489		"cmp x0, #0\n"
1490		"bne 1f\n"
1491
1492		/* In the child, now. Call "fn(arg)". */
1493		"ldp x1, x0, [sp], #16\n"
1494		"blr x1\n"
1495
1496		/* Call _exit(%r0). */
1497		"mov x8, %10\n"
1498		"svc 0x0\n"
1499		"1:\n"
1500
1501		: "=r" (res)
1502		: "i"(-EINVAL),
1503		"r"(__fn), "r"(__stack), "r"(__flags), "r"(__arg),
1504		"r"(__ptid), "r"(__tls), "r"(__ctid),
1505		"i"(__NR_clone), "i"(__NR_exit)
1506		: "x30", "memory");
	1505	"mov x0,x2\n" /* flags */
	1506	"mov x2,x4\n" /* ptid */
	1507	"mov x3,x5\n" /* tls */
	1508	"mov x4,x6\n" /* ctid */
	1509	"mov x8,%9\n" /* clone */
	1510
	1511	"svc 0x0\n"
	1512
	1513	/* if (%r0 != 0)
	1514	* return %r0;
	1515	*/
	1516	"cmp x0, #0\n"
	1517	"bne 1f\n"
	1518
	1519	/* In the child, now. Call "fn(arg)". */
	1520	"ldp x1, x0, [sp], #16\n"
	1521	"blr x1\n"
	1522
	1523	/* Call _exit(%r0). */
	1524	"mov x8, %10\n"
	1525	"svc 0x0\n"
	1526	"1:\n"
	1527
	1528	: "=r"(res)
	1529	: "i"(-EINVAL), "r"(__fn), "r"(__stack), "r"(__flags), "r"(__arg),
	1530	"r"(__ptid), "r"(__tls), "r"(__ctid), "i"(__NR_clone), "i"(__NR_exit)
	1531	: "x30", "memory");
1507	1532	return res;
1508	1533	}
1509		#elif SANITIZER_LOONGARCH64
	1534	# elif SANITIZER_LOONGARCH64
1510	1535	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
1511	1536	int parent_tidptr, void newtls, int *child_tidptr) {
1512	1537	if (!fn \|\| !child_stack)
...	...	@@ -1544,119 +1569,110 @@ uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
1544	1569	: "=r"(res)
1545	1570	: "0"(__flags), "r"(__stack), "r"(__ptid), "r"(__ctid), "r"(__tls),
1546	1571	"r"(__fn), "r"(__arg), "r"(nr_clone), "i"(__NR_exit)
1547		: "memory", "$t0", "$t1", "$t2", "$t3", "$t4", "$t5", "$t6", "$t7", "$t8");
	1572	: "memory", "$t0", "$t1", "$t2", "$t3", "$t4", "$t5", "$t6", "$t7",
	1573	"$t8");
1548	1574	return res;
1549	1575	}
1550		#elif defined(__powerpc64__)
	1576	# elif defined(__powerpc64__)
1551	1577	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
1552		int parent_tidptr, void newtls, int *child_tidptr) {
	1578	int parent_tidptr, void newtls, int *child_tidptr) {
1553	1579	long long res;
1554	1580	// Stack frame structure.
1555		#if SANITIZER_PPC64V1
1556		// Back chain == 0 (SP + 112)
1557		// Frame (112 bytes):
1558		// Parameter save area (SP + 48), 8 doublewords
1559		// TOC save area (SP + 40)
1560		// Link editor doubleword (SP + 32)
1561		// Compiler doubleword (SP + 24)
1562		// LR save area (SP + 16)
1563		// CR save area (SP + 8)
1564		// Back chain (SP + 0)
1565		# define FRAME_SIZE 112
1566		# define FRAME_TOC_SAVE_OFFSET 40
1567		#elif SANITIZER_PPC64V2
1568		// Back chain == 0 (SP + 32)
1569		// Frame (32 bytes):
1570		// TOC save area (SP + 24)
1571		// LR save area (SP + 16)
1572		// CR save area (SP + 8)
1573		// Back chain (SP + 0)
1574		# define FRAME_SIZE 32
1575		# define FRAME_TOC_SAVE_OFFSET 24
1576		#else
1577		# error "Unsupported PPC64 ABI"
1578		#endif
	1581	# if SANITIZER_PPC64V1
	1582	// Back chain == 0 (SP + 112)
	1583	// Frame (112 bytes):
	1584	// Parameter save area (SP + 48), 8 doublewords
	1585	// TOC save area (SP + 40)
	1586	// Link editor doubleword (SP + 32)
	1587	// Compiler doubleword (SP + 24)
	1588	// LR save area (SP + 16)
	1589	// CR save area (SP + 8)
	1590	// Back chain (SP + 0)
	1591	# define FRAME_SIZE 112
	1592	# define FRAME_TOC_SAVE_OFFSET 40
	1593	# elif SANITIZER_PPC64V2
	1594	// Back chain == 0 (SP + 32)
	1595	// Frame (32 bytes):
	1596	// TOC save area (SP + 24)
	1597	// LR save area (SP + 16)
	1598	// CR save area (SP + 8)
	1599	// Back chain (SP + 0)
	1600	# define FRAME_SIZE 32
	1601	# define FRAME_TOC_SAVE_OFFSET 24
	1602	# else
	1603	# error "Unsupported PPC64 ABI"
	1604	# endif
1579	1605	if (!fn \|\| !child_stack)
1580	1606	return -EINVAL;
1581	1607	CHECK_EQ(0, (uptr)child_stack % 16);
1582	1608
1583	1609	register int (__fn)(void ) __asm__("r3") = fn;
1584		register void *__cstack __asm__("r4") = child_stack;
1585		register int __flags __asm__("r5") = flags;
1586		register void *__arg __asm__("r6") = arg;
1587		register int *__ptidptr __asm__("r7") = parent_tidptr;
1588		register void *__newtls __asm__("r8") = newtls;
1589		register int *__ctidptr __asm__("r9") = child_tidptr;
1590
1591		__asm__ __volatile__(
1592		/* fn and arg are saved across the syscall */
1593		"mr 28, %5\n\t"
1594		"mr 27, %8\n\t"
1595
1596		/* syscall
1597		r0 == __NR_clone
1598		r3 == flags
1599		r4 == child_stack
1600		r5 == parent_tidptr
1601		r6 == newtls
1602		r7 == child_tidptr */
1603		"mr 3, %7\n\t"
1604		"mr 5, %9\n\t"
1605		"mr 6, %10\n\t"
1606		"mr 7, %11\n\t"
1607		"li 0, %3\n\t"
1608		"sc\n\t"
1609
1610		/* Test if syscall was successful */
1611		"cmpdi cr1, 3, 0\n\t"
1612		"crandc cr14+eq, cr14+eq, cr0*4+so\n\t"
1613		"bne- cr1, 1f\n\t"
1614
1615		/* Set up stack frame */
1616		"li 29, 0\n\t"
1617		"stdu 29, -8(1)\n\t"
1618		"stdu 1, -%12(1)\n\t"
1619		/* Do the function call */
1620		"std 2, %13(1)\n\t"
1621		#if SANITIZER_PPC64V1
1622		"ld 0, 0(28)\n\t"
1623		"ld 2, 8(28)\n\t"
1624		"mtctr 0\n\t"
1625		#elif SANITIZER_PPC64V2
1626		"mr 12, 28\n\t"
1627		"mtctr 12\n\t"
1628		#else
1629		# error "Unsupported PPC64 ABI"
1630		#endif
1631		"mr 3, 27\n\t"
1632		"bctrl\n\t"
1633		"ld 2, %13(1)\n\t"
1634
1635		/* Call _exit(r3) */
1636		"li 0, %4\n\t"
1637		"sc\n\t"
1638
1639		/* Return to parent */
1640		"1:\n\t"
1641		"mr %0, 3\n\t"
1642		: "=r" (res)
1643		: "0" (-1),
1644		"i" (EINVAL),
1645		"i" (__NR_clone),
1646		"i" (__NR_exit),
1647		"r" (__fn),
1648		"r" (__cstack),
1649		"r" (__flags),
1650		"r" (__arg),
1651		"r" (__ptidptr),
1652		"r" (__newtls),
1653		"r" (__ctidptr),
1654		"i" (FRAME_SIZE),
1655		"i" (FRAME_TOC_SAVE_OFFSET)
1656		: "cr0", "cr1", "memory", "ctr", "r0", "r27", "r28", "r29");
	1610	register void *__cstack __asm__("r4") = child_stack;
	1611	register int __flags __asm__("r5") = flags;
	1612	register void *__arg __asm__("r6") = arg;
	1613	register int *__ptidptr __asm__("r7") = parent_tidptr;
	1614	register void *__newtls __asm__("r8") = newtls;
	1615	register int *__ctidptr __asm__("r9") = child_tidptr;
	1616
	1617	__asm__ __volatile__(
	1618	/* fn and arg are saved across the syscall */
	1619	"mr 28, %5\n\t"
	1620	"mr 27, %8\n\t"
	1621
	1622	/* syscall
	1623	r0 == __NR_clone
	1624	r3 == flags
	1625	r4 == child_stack
	1626	r5 == parent_tidptr
	1627	r6 == newtls
	1628	r7 == child_tidptr */
	1629	"mr 3, %7\n\t"
	1630	"mr 5, %9\n\t"
	1631	"mr 6, %10\n\t"
	1632	"mr 7, %11\n\t"
	1633	"li 0, %3\n\t"
	1634	"sc\n\t"
	1635
	1636	/* Test if syscall was successful */
	1637	"cmpdi cr1, 3, 0\n\t"
	1638	"crandc cr14+eq, cr14+eq, cr0*4+so\n\t"
	1639	"bne- cr1, 1f\n\t"
	1640
	1641	/* Set up stack frame */
	1642	"li 29, 0\n\t"
	1643	"stdu 29, -8(1)\n\t"
	1644	"stdu 1, -%12(1)\n\t"
	1645	/* Do the function call */
	1646	"std 2, %13(1)\n\t"
	1647	# if SANITIZER_PPC64V1
	1648	"ld 0, 0(28)\n\t"
	1649	"ld 2, 8(28)\n\t"
	1650	"mtctr 0\n\t"
	1651	# elif SANITIZER_PPC64V2
	1652	"mr 12, 28\n\t"
	1653	"mtctr 12\n\t"
	1654	# else
	1655	# error "Unsupported PPC64 ABI"
	1656	# endif
	1657	"mr 3, 27\n\t"
	1658	"bctrl\n\t"
	1659	"ld 2, %13(1)\n\t"
	1660
	1661	/* Call _exit(r3) */
	1662	"li 0, %4\n\t"
	1663	"sc\n\t"
	1664
	1665	/* Return to parent */
	1666	"1:\n\t"
	1667	"mr %0, 3\n\t"
	1668	: "=r"(res)
	1669	: "0"(-1), "i"(EINVAL), "i"(__NR_clone), "i"(__NR_exit), "r"(__fn),
	1670	"r"(__cstack), "r"(__flags), "r"(__arg), "r"(__ptidptr), "r"(__newtls),
	1671	"r"(__ctidptr), "i"(FRAME_SIZE), "i"(FRAME_TOC_SAVE_OFFSET)
	1672	: "cr0", "cr1", "memory", "ctr", "r0", "r27", "r28", "r29");
1657	1673	return res;
1658	1674	}
1659		#elif defined(__i386__)
	1675	# elif defined(__i386__)
1660	1676	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
1661	1677	int parent_tidptr, void newtls, int *child_tidptr) {
1662	1678	int res;
...	...	@@ -1669,59 +1685,56 @@ uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
1669	1685	((unsigned int *)child_stack)[2] = (uptr)fn;
1670	1686	((unsigned int *)child_stack)[3] = (uptr)arg;
1671	1687	__asm__ __volatile__(
1672		/* %eax = syscall(%eax = SYSCALL(clone),
1673		* %ebx = flags,
1674		* %ecx = child_stack,
1675		* %edx = parent_tidptr,
1676		* %esi = new_tls,
1677		* %edi = child_tidptr)
1678		*/
1679
1680		/* Obtain flags */
1681		"movl (%%ecx), %%ebx\n"
1682		/* Do the system call */
1683		"pushl %%ebx\n"
1684		"pushl %%esi\n"
1685		"pushl %%edi\n"
1686		/* Remember the flag value. */
1687		"movl %%ebx, (%%ecx)\n"
1688		"int $0x80\n"
1689		"popl %%edi\n"
1690		"popl %%esi\n"
1691		"popl %%ebx\n"
1692
1693		/* if (%eax != 0)
1694		* return;
1695		*/
1696
1697		"test %%eax,%%eax\n"
1698		"jnz 1f\n"
1699
1700		/* terminate the stack frame */
1701		"xorl %%ebp,%%ebp\n"
1702		/* Call FN. */
1703		"call *%%ebx\n"
1704		#ifdef PIC
1705		"call here\n"
1706		"here:\n"
1707		"popl %%ebx\n"
1708		"addl $_GLOBAL_OFFSET_TABLE_+[.-here], %%ebx\n"
1709		#endif
1710		/* Call exit */
1711		"movl %%eax, %%ebx\n"
1712		"movl %2, %%eax\n"
1713		"int $0x80\n"
1714		"1:\n"
1715		: "=a" (res)
1716		: "a"(SYSCALL(clone)), "i"(SYSCALL(exit)),
1717		"c"(child_stack),
1718		"d"(parent_tidptr),
1719		"S"(newtls),
1720		"D"(child_tidptr)
1721		: "memory");
	1688	/* %eax = syscall(%eax = SYSCALL(clone),
	1689	* %ebx = flags,
	1690	* %ecx = child_stack,
	1691	* %edx = parent_tidptr,
	1692	* %esi = new_tls,
	1693	* %edi = child_tidptr)
	1694	*/
	1695
	1696	/* Obtain flags */
	1697	"movl (%%ecx), %%ebx\n"
	1698	/* Do the system call */
	1699	"pushl %%ebx\n"
	1700	"pushl %%esi\n"
	1701	"pushl %%edi\n"
	1702	/* Remember the flag value. */
	1703	"movl %%ebx, (%%ecx)\n"
	1704	"int $0x80\n"
	1705	"popl %%edi\n"
	1706	"popl %%esi\n"
	1707	"popl %%ebx\n"
	1708
	1709	/* if (%eax != 0)
	1710	* return;
	1711	*/
	1712
	1713	"test %%eax,%%eax\n"
	1714	"jnz 1f\n"
	1715
	1716	/* terminate the stack frame */
	1717	"xorl %%ebp,%%ebp\n"
	1718	/* Call FN. */
	1719	"call *%%ebx\n"
	1720	# ifdef PIC
	1721	"call here\n"
	1722	"here:\n"
	1723	"popl %%ebx\n"
	1724	"addl $_GLOBAL_OFFSET_TABLE_+[.-here], %%ebx\n"
	1725	# endif
	1726	/* Call exit */
	1727	"movl %%eax, %%ebx\n"
	1728	"movl %2, %%eax\n"
	1729	"int $0x80\n"
	1730	"1:\n"
	1731	: "=a"(res)
	1732	: "a"(SYSCALL(clone)), "i"(SYSCALL(exit)), "c"(child_stack),
	1733	"d"(parent_tidptr), "S"(newtls), "D"(child_tidptr)
	1734	: "memory");
1722	1735	return res;
1723	1736	}
1724		#elif defined(__arm__)
	1737	# elif defined(__arm__)
1725	1738	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
1726	1739	int parent_tidptr, void newtls, int *child_tidptr) {
1727	1740	unsigned int res;
...	...	@@ -1737,70 +1750,68 @@ uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
1737	1750	register int *r4 __asm__("r4") = child_tidptr;
1738	1751	register int r7 __asm__("r7") = __NR_clone;
1739	1752
1740		#if __ARM_ARCH > 4 \|\| defined (__ARM_ARCH_4T__)
1741		# define ARCH_HAS_BX
1742		#endif
1743		#if __ARM_ARCH > 4
1744		# define ARCH_HAS_BLX
1745		#endif
	1753	# if __ARM_ARCH > 4 \|\| defined(__ARM_ARCH_4T__)
	1754	# define ARCH_HAS_BX
	1755	# endif
	1756	# if __ARM_ARCH > 4
	1757	# define ARCH_HAS_BLX
	1758	# endif
1746	1759
1747		#ifdef ARCH_HAS_BX
1748		# ifdef ARCH_HAS_BLX
1749		# define BLX(R) "blx " #R "\n"
1750		# else
1751		# define BLX(R) "mov lr, pc; bx " #R "\n"
1752		# endif
1753		#else
1754		# define BLX(R) "mov lr, pc; mov pc," #R "\n"
1755		#endif
	1760	# ifdef ARCH_HAS_BX
	1761	# ifdef ARCH_HAS_BLX
	1762	# define BLX(R) "blx " #R "\n"
	1763	# else
	1764	# define BLX(R) "mov lr, pc; bx " #R "\n"
	1765	# endif
	1766	# else
	1767	# define BLX(R) "mov lr, pc; mov pc," #R "\n"
	1768	# endif
1756	1769
1757	1770	__asm__ __volatile__(
1758		/* %r0 = syscall(%r7 = SYSCALL(clone),
1759		* %r0 = flags,
1760		* %r1 = child_stack,
1761		* %r2 = parent_tidptr,
1762		* %r3 = new_tls,
1763		* %r4 = child_tidptr)
1764		*/
1765
1766		/* Do the system call */
1767		"swi 0x0\n"
1768
1769		/* if (%r0 != 0)
1770		* return %r0;
1771		*/
1772		"cmp r0, #0\n"
1773		"bne 1f\n"
1774
1775		/* In the child, now. Call "fn(arg)". */
1776		"ldr r0, [sp, #4]\n"
1777		"ldr ip, [sp], #8\n"
1778		BLX(ip)
1779		/* Call _exit(%r0). */
1780		"mov r7, %7\n"
1781		"swi 0x0\n"
1782		"1:\n"
1783		"mov %0, r0\n"
1784		: "=r"(res)
1785		: "r"(r0), "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r7),
1786		"i"(__NR_exit)
1787		: "memory");
	1771	/* %r0 = syscall(%r7 = SYSCALL(clone),
	1772	* %r0 = flags,
	1773	* %r1 = child_stack,
	1774	* %r2 = parent_tidptr,
	1775	* %r3 = new_tls,
	1776	* %r4 = child_tidptr)
	1777	*/
	1778
	1779	/* Do the system call */
	1780	"swi 0x0\n"
	1781
	1782	/* if (%r0 != 0)
	1783	* return %r0;
	1784	*/
	1785	"cmp r0, #0\n"
	1786	"bne 1f\n"
	1787
	1788	/* In the child, now. Call "fn(arg)". */
	1789	"ldr r0, [sp, #4]\n"
	1790	"ldr ip, [sp], #8\n" BLX(ip)
	1791	/* Call _exit(%r0). */
	1792	"mov r7, %7\n"
	1793	"swi 0x0\n"
	1794	"1:\n"
	1795	"mov %0, r0\n"
	1796	: "=r"(res)
	1797	: "r"(r0), "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r7), "i"(__NR_exit)
	1798	: "memory");
1788	1799	return res;
1789	1800	}
1790		#endif
1791		#endif // SANITIZER_LINUX
	1801	# endif
	1802	# endif // SANITIZER_LINUX
1792	1803
1793		#if SANITIZER_LINUX
	1804	# if SANITIZER_LINUX
1794	1805	int internal_uname(struct utsname *buf) {
1795	1806	return internal_syscall(SYSCALL(uname), buf);
1796	1807	}
1797		#endif
	1808	# endif
1798	1809
1799		#if SANITIZER_ANDROID
1800		#if __ANDROID_API__ < 21
	1810	# if SANITIZER_ANDROID
	1811	# if __ANDROID_API__ < 21
1801	1812	extern "C" __attribute__((weak)) int dl_iterate_phdr(
1802	1813	int ()(struct dl_phdr_info , size_t, void ), void );
1803		#endif
	1814	# endif
1804	1815
1805	1816	static int dl_iterate_phdr_test_cb(struct dl_phdr_info *info, size_t size,
1806	1817	void *data) {
...	...	@@ -1817,40 +1828,41 @@ static int dl_iterate_phdr_test_cb(struct dl_phdr_info *info, size_t size,
1817	1828	static atomic_uint32_t android_api_level;
1818	1829
1819	1830	static AndroidApiLevel AndroidDetectApiLevelStatic() {
1820		#if __ANDROID_API__ <= 19
	1831	# if __ANDROID_API__ <= 19
1821	1832	return ANDROID_KITKAT;
1822		#elif __ANDROID_API__ <= 22
	1833	# elif __ANDROID_API__ <= 22
1823	1834	return ANDROID_LOLLIPOP_MR1;
1824		#else
	1835	# else
1825	1836	return ANDROID_POST_LOLLIPOP;
1826		#endif
	1837	# endif
1827	1838	}
1828	1839
1829	1840	static AndroidApiLevel AndroidDetectApiLevel() {
1830	1841	if (!&dl_iterate_phdr)
1831		return ANDROID_KITKAT; // K or lower
	1842	return ANDROID_KITKAT; // K or lower
1832	1843	bool base_name_seen = false;
1833	1844	dl_iterate_phdr(dl_iterate_phdr_test_cb, &base_name_seen);
1834	1845	if (base_name_seen)
1835		return ANDROID_LOLLIPOP_MR1; // L MR1
	1846	return ANDROID_LOLLIPOP_MR1; // L MR1
1836	1847	return ANDROID_POST_LOLLIPOP; // post-L
1837	1848	// Plain L (API level 21) is completely broken wrt ASan and not very
1838	1849	// interesting to detect.
1839	1850	}
1840	1851
1841		extern "C" __attribute__((weak)) void* _DYNAMIC;
	1852	extern "C" __attribute__((weak)) void *_DYNAMIC;
1842	1853
1843	1854	AndroidApiLevel AndroidGetApiLevel() {
1844	1855	AndroidApiLevel level =
1845	1856	(AndroidApiLevel)atomic_load(&android_api_level, memory_order_relaxed);
1846		if (level) return level;
	1857	if (level)
	1858	return level;
1847	1859	level = &_DYNAMIC == nullptr ? AndroidDetectApiLevelStatic()
1848	1860	: AndroidDetectApiLevel();
1849	1861	atomic_store(&android_api_level, level, memory_order_relaxed);
1850	1862	return level;
1851	1863	}
1852	1864
1853		#endif
	1865	# endif
1854	1866
1855	1867	static HandleSignalMode GetHandleSignalModeImpl(int signum) {
1856	1868	switch (signum) {
...	...	@@ -1877,28 +1889,28 @@ HandleSignalMode GetHandleSignalMode(int signum) {
1877	1889	return result;
1878	1890	}
1879	1891
1880		#if !SANITIZER_GO
	1892	# if !SANITIZER_GO
1881	1893	void internal_start_thread(void (func)(void arg), void *arg) {
1882		if (&real_pthread_create == 0)
	1894	if (&internal_pthread_create == 0)
1883	1895	return nullptr;
1884	1896	// Start the thread with signals blocked, otherwise it can steal user signals.
1885	1897	ScopedBlockSignals block(nullptr);
1886	1898	void *th;
1887		real_pthread_create(&th, nullptr, func, arg);
	1899	internal_pthread_create(&th, nullptr, func, arg);
1888	1900	return th;
1889	1901	}
1890	1902
1891	1903	void internal_join_thread(void *th) {
1892		if (&real_pthread_join)
1893		real_pthread_join(th, nullptr);
	1904	if (&internal_pthread_join)
	1905	internal_pthread_join(th, nullptr);
1894	1906	}
1895		#else
	1907	# else
1896	1908	void internal_start_thread(void (func)(void ), void *arg) { return 0; }
1897	1909
1898	1910	void internal_join_thread(void *th) {}
1899		#endif
	1911	# endif
1900	1912
1901		#if SANITIZER_LINUX && defined(__aarch64__)
	1913	# if SANITIZER_LINUX && defined(__aarch64__)
1902	1914	// Android headers in the older NDK releases miss this definition.
1903	1915	struct __sanitizer_esr_context {
1904	1916	struct _aarch64_ctx head;
...	...	@@ -1910,7 +1922,8 @@ static bool Aarch64GetESR(ucontext_t ucontext, u64 esr) {
1910	1922	u8 aux = reinterpret_cast<u8 >(ucontext->uc_mcontext.__reserved);
1911	1923	while (true) {
1912	1924	_aarch64_ctx ctx = (_aarch64_ctx )aux;
1913		if (ctx->size == 0) break;
	1925	if (ctx->size == 0)
	1926	break;
1914	1927	if (ctx->magic == kEsrMagic) {
1915	1928	esr = ((__sanitizer_esr_context )ctx)->esr;
1916	1929	return true;
...	...	@@ -1919,31 +1932,29 @@ static bool Aarch64GetESR(ucontext_t ucontext, u64 esr) {
1919	1932	}
1920	1933	return false;
1921	1934	}
1922		#elif SANITIZER_FREEBSD && defined(__aarch64__)
	1935	# elif SANITIZER_FREEBSD && defined(__aarch64__)
1923	1936	// FreeBSD doesn't provide ESR in the ucontext.
1924		static bool Aarch64GetESR(ucontext_t ucontext, u64 esr) {
1925		return false;
1926		}
1927		#endif
	1937	static bool Aarch64GetESR(ucontext_t ucontext, u64 esr) { return false; }
	1938	# endif
1928	1939
1929	1940	using Context = ucontext_t;
1930	1941
1931	1942	SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
1932	1943	Context ucontext = (Context )context;
1933		#if defined(__x86_64__) \|\| defined(__i386__)
	1944	# if defined(__x86_64__) \|\| defined(__i386__)
1934	1945	static const uptr PF_WRITE = 1U << 1;
1935		#if SANITIZER_FREEBSD
	1946	# if SANITIZER_FREEBSD
1936	1947	uptr err = ucontext->uc_mcontext.mc_err;
1937		#elif SANITIZER_NETBSD
	1948	# elif SANITIZER_NETBSD
1938	1949	uptr err = ucontext->uc_mcontext.__gregs[_REG_ERR];
1939		#elif SANITIZER_SOLARIS && defined(__i386__)
	1950	# elif SANITIZER_SOLARIS && defined(__i386__)
1940	1951	const int Err = 13;
1941	1952	uptr err = ucontext->uc_mcontext.gregs[Err];
1942		#else
	1953	# else
1943	1954	uptr err = ucontext->uc_mcontext.gregs[REG_ERR];
1944		#endif // SANITIZER_FREEBSD
	1955	# endif // SANITIZER_FREEBSD
1945	1956	return err & PF_WRITE ? Write : Read;
1946		#elif defined(__mips__)
	1957	# elif defined(__mips__)
1947	1958	uint32_t *exception_source;
1948	1959	uint32_t faulty_instruction;
1949	1960	uint32_t op_code;
...	...	@@ -1959,12 +1970,12 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
1959	1970	case 0x29: // sh
1960	1971	case 0x2b: // sw
1961	1972	case 0x3f: // sd
1962		#if __mips_isa_rev < 6
	1973	# if __mips_isa_rev < 6
1963	1974	case 0x2c: // sdl
1964	1975	case 0x2d: // sdr
1965	1976	case 0x2a: // swl
1966	1977	case 0x2e: // swr
1967		#endif
	1978	# endif
1968	1979	return SignalContext::Write;
1969	1980
1970	1981	case 0x20: // lb
...	...	@@ -1974,14 +1985,14 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
1974	1985	case 0x23: // lw
1975	1986	case 0x27: // lwu
1976	1987	case 0x37: // ld
1977		#if __mips_isa_rev < 6
	1988	# if __mips_isa_rev < 6
1978	1989	case 0x1a: // ldl
1979	1990	case 0x1b: // ldr
1980	1991	case 0x22: // lwl
1981	1992	case 0x26: // lwr
1982		#endif
	1993	# endif
1983	1994	return SignalContext::Read;
1984		#if __mips_isa_rev == 6
	1995	# if __mips_isa_rev == 6
1985	1996	case 0x3b: // pcrel
1986	1997	op_code = (faulty_instruction >> 19) & 0x3;
1987	1998	switch (op_code) {
...	...	@@ -1989,50 +2000,51 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
1989	2000	case 0x2: // lwupc
1990	2001	return SignalContext::Read;
1991	2002	}
1992		#endif
	2003	# endif
1993	2004	}
1994	2005	return SignalContext::Unknown;
1995		#elif defined(__arm__)
	2006	# elif defined(__arm__)
1996	2007	static const uptr FSR_WRITE = 1U << 11;
1997	2008	uptr fsr = ucontext->uc_mcontext.error_code;
1998	2009	return fsr & FSR_WRITE ? Write : Read;
1999		#elif defined(__aarch64__)
	2010	# elif defined(__aarch64__)
2000	2011	static const u64 ESR_ELx_WNR = 1U << 6;
2001	2012	u64 esr;
2002		if (!Aarch64GetESR(ucontext, &esr)) return Unknown;
	2013	if (!Aarch64GetESR(ucontext, &esr))
	2014	return Unknown;
2003	2015	return esr & ESR_ELx_WNR ? Write : Read;
2004		#elif defined(__loongarch__)
	2016	# elif defined(__loongarch__)
2005	2017	u32 flags = ucontext->uc_mcontext.__flags;
2006	2018	if (flags & SC_ADDRERR_RD)
2007	2019	return SignalContext::Read;
2008	2020	if (flags & SC_ADDRERR_WR)
2009	2021	return SignalContext::Write;
2010	2022	return SignalContext::Unknown;
2011		#elif defined(__sparc__)
	2023	# elif defined(__sparc__)
2012	2024	// Decode the instruction to determine the access type.
2013	2025	// From OpenSolaris $SRC/uts/sun4/os/trap.c (get_accesstype).
2014		#if SANITIZER_SOLARIS
	2026	# if SANITIZER_SOLARIS
2015	2027	uptr pc = ucontext->uc_mcontext.gregs[REG_PC];
2016		#else
	2028	# else
2017	2029	// Historical BSDism here.
2018	2030	struct sigcontext scontext = (struct sigcontext )context;
2019		#if defined(__arch64__)
	2031	# if defined(__arch64__)
2020	2032	uptr pc = scontext->sigc_regs.tpc;
2021		#else
	2033	# else
2022	2034	uptr pc = scontext->si_regs.pc;
2023		#endif
2024		#endif
	2035	# endif
	2036	# endif
2025	2037	u32 instr = (u32 )pc;
2026		return (instr >> 21) & 1 ? Write: Read;
2027		#elif defined(__riscv)
2028		#if SANITIZER_FREEBSD
	2038	return (instr >> 21) & 1 ? Write : Read;
	2039	# elif defined(__riscv)
	2040	# if SANITIZER_FREEBSD
2029	2041	unsigned long pc = ucontext->uc_mcontext.mc_gpregs.gp_sepc;
2030		#else
	2042	# else
2031	2043	unsigned long pc = ucontext->uc_mcontext.__gregs[REG_PC];
2032		#endif
	2044	# endif
2033	2045	unsigned faulty_instruction = (uint16_t )pc;
2034	2046
2035		#if defined(__riscv_compressed)
	2047	# if defined(__riscv_compressed)
2036	2048	if ((faulty_instruction & 0x3) != 0x3) { // it's a compressed instruction
2037	2049	// set op_bits to the instruction bits [1, 0, 15, 14, 13]
2038	2050	unsigned op_bits =
...	...	@@ -2040,38 +2052,38 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
2040	2052	unsigned rd = faulty_instruction & 0xF80; // bits 7-11, inclusive
2041	2053	switch (op_bits) {
2042	2054	case 0b10'010: // c.lwsp (rd != x0)
2043		#if __riscv_xlen == 64
	2055	# if __riscv_xlen == 64
2044	2056	case 0b10'011: // c.ldsp (rd != x0)
2045		#endif
	2057	# endif
2046	2058	return rd ? SignalContext::Read : SignalContext::Unknown;
2047	2059	case 0b00'010: // c.lw
2048		#if __riscv_flen >= 32 && __riscv_xlen == 32
	2060	# if __riscv_flen >= 32 && __riscv_xlen == 32
2049	2061	case 0b10'011: // c.flwsp
2050		#endif
2051		#if __riscv_flen >= 32 \|\| __riscv_xlen == 64
	2062	# endif
	2063	# if __riscv_flen >= 32 \|\| __riscv_xlen == 64
2052	2064	case 0b00'011: // c.flw / c.ld
2053		#endif
2054		#if __riscv_flen == 64
	2065	# endif
	2066	# if __riscv_flen == 64
2055	2067	case 0b00'001: // c.fld
2056	2068	case 0b10'001: // c.fldsp
2057		#endif
	2069	# endif
2058	2070	return SignalContext::Read;
2059	2071	case 0b00'110: // c.sw
2060	2072	case 0b10'110: // c.swsp
2061		#if __riscv_flen >= 32 \|\| __riscv_xlen == 64
	2073	# if __riscv_flen >= 32 \|\| __riscv_xlen == 64
2062	2074	case 0b00'111: // c.fsw / c.sd
2063	2075	case 0b10'111: // c.fswsp / c.sdsp
2064		#endif
2065		#if __riscv_flen == 64
	2076	# endif
	2077	# if __riscv_flen == 64
2066	2078	case 0b00'101: // c.fsd
2067	2079	case 0b10'101: // c.fsdsp
2068		#endif
	2080	# endif
2069	2081	return SignalContext::Write;
2070	2082	default:
2071	2083	return SignalContext::Unknown;
2072	2084	}
2073	2085	}
2074		#endif
	2086	# endif
2075	2087
2076	2088	unsigned opcode = faulty_instruction & 0x7f; // lower 7 bits
2077	2089	unsigned funct3 = (faulty_instruction >> 12) & 0x7; // bits 12-14, inclusive
...	...	@@ -2081,9 +2093,9 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
2081	2093	case 0b000: // lb
2082	2094	case 0b001: // lh
2083	2095	case 0b010: // lw
2084		#if __riscv_xlen == 64
	2096	# if __riscv_xlen == 64
2085	2097	case 0b011: // ld
2086		#endif
	2098	# endif
2087	2099	case 0b100: // lbu
2088	2100	case 0b101: // lhu
2089	2101	return SignalContext::Read;
...	...	@@ -2095,20 +2107,20 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
2095	2107	case 0b000: // sb
2096	2108	case 0b001: // sh
2097	2109	case 0b010: // sw
2098		#if __riscv_xlen == 64
	2110	# if __riscv_xlen == 64
2099	2111	case 0b011: // sd
2100		#endif
	2112	# endif
2101	2113	return SignalContext::Write;
2102	2114	default:
2103	2115	return SignalContext::Unknown;
2104	2116	}
2105		#if __riscv_flen >= 32
	2117	# if __riscv_flen >= 32
2106	2118	case 0b0000111: // floating-point loads
2107	2119	switch (funct3) {
2108	2120	case 0b010: // flw
2109		#if __riscv_flen == 64
	2121	# if __riscv_flen == 64
2110	2122	case 0b011: // fld
2111		#endif
	2123	# endif
2112	2124	return SignalContext::Read;
2113	2125	default:
2114	2126	return SignalContext::Unknown;
...	...	@@ -2116,21 +2128,21 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
2116	2128	case 0b0100111: // floating-point stores
2117	2129	switch (funct3) {
2118	2130	case 0b010: // fsw
2119		#if __riscv_flen == 64
	2131	# if __riscv_flen == 64
2120	2132	case 0b011: // fsd
2121		#endif
	2133	# endif
2122	2134	return SignalContext::Write;
2123	2135	default:
2124	2136	return SignalContext::Unknown;
2125	2137	}
2126		#endif
	2138	# endif
2127	2139	default:
2128	2140	return SignalContext::Unknown;
2129	2141	}
2130		#else
	2142	# else
2131	2143	(void)ucontext;
2132	2144	return Unknown; // FIXME: Implement.
2133		#endif
	2145	# endif
2134	2146	}
2135	2147
2136	2148	bool SignalContext::IsTrueFaultingAddress() const {
...	...	@@ -2139,129 +2151,288 @@ bool SignalContext::IsTrueFaultingAddress() const {
2139	2151	return si->si_signo == SIGSEGV && si->si_code != 128;
2140	2152	}
2141	2153
	2154	UNUSED
	2155	static const char *RegNumToRegName(int reg) {
	2156	switch (reg) {
	2157	# if SANITIZER_LINUX
	2158	# if defined(__x86_64__)
	2159	case REG_RAX:
	2160	return "rax";
	2161	case REG_RBX:
	2162	return "rbx";
	2163	case REG_RCX:
	2164	return "rcx";
	2165	case REG_RDX:
	2166	return "rdx";
	2167	case REG_RDI:
	2168	return "rdi";
	2169	case REG_RSI:
	2170	return "rsi";
	2171	case REG_RBP:
	2172	return "rbp";
	2173	case REG_RSP:
	2174	return "rsp";
	2175	case REG_R8:
	2176	return "r8";
	2177	case REG_R9:
	2178	return "r9";
	2179	case REG_R10:
	2180	return "r10";
	2181	case REG_R11:
	2182	return "r11";
	2183	case REG_R12:
	2184	return "r12";
	2185	case REG_R13:
	2186	return "r13";
	2187	case REG_R14:
	2188	return "r14";
	2189	case REG_R15:
	2190	return "r15";
	2191	# elif defined(__i386__)
	2192	case REG_EAX:
	2193	return "eax";
	2194	case REG_EBX:
	2195	return "ebx";
	2196	case REG_ECX:
	2197	return "ecx";
	2198	case REG_EDX:
	2199	return "edx";
	2200	case REG_EDI:
	2201	return "edi";
	2202	case REG_ESI:
	2203	return "esi";
	2204	case REG_EBP:
	2205	return "ebp";
	2206	case REG_ESP:
	2207	return "esp";
	2208	# endif
	2209	# endif
	2210	default:
	2211	return NULL;
	2212	}
	2213	return NULL;
	2214	}
	2215
	2216	# if SANITIZER_LINUX
	2217	UNUSED
	2218	static void DumpSingleReg(ucontext_t *ctx, int RegNum) {
	2219	const char *RegName = RegNumToRegName(RegNum);
	2220	# if defined(__x86_64__)
	2221	Printf("%s%s = 0x%016llx ", internal_strlen(RegName) == 2 ? " " : "",
	2222	RegName, ctx->uc_mcontext.gregs[RegNum]);
	2223	# elif defined(__i386__)
	2224	Printf("%s = 0x%08x ", RegName, ctx->uc_mcontext.gregs[RegNum]);
	2225	# else
	2226	(void)RegName;
	2227	# endif
	2228	}
	2229	# endif
	2230
2142	2231	void SignalContext::DumpAllRegisters(void *context) {
2143		// FIXME: Implement this.
	2232	ucontext_t ucontext = (ucontext_t )context;
	2233	# if SANITIZER_LINUX
	2234	# if defined(__x86_64__)
	2235	Report("Register values:\n");
	2236	DumpSingleReg(ucontext, REG_RAX);
	2237	DumpSingleReg(ucontext, REG_RBX);
	2238	DumpSingleReg(ucontext, REG_RCX);
	2239	DumpSingleReg(ucontext, REG_RDX);
	2240	Printf("\n");
	2241	DumpSingleReg(ucontext, REG_RDI);
	2242	DumpSingleReg(ucontext, REG_RSI);
	2243	DumpSingleReg(ucontext, REG_RBP);
	2244	DumpSingleReg(ucontext, REG_RSP);
	2245	Printf("\n");
	2246	DumpSingleReg(ucontext, REG_R8);
	2247	DumpSingleReg(ucontext, REG_R9);
	2248	DumpSingleReg(ucontext, REG_R10);
	2249	DumpSingleReg(ucontext, REG_R11);
	2250	Printf("\n");
	2251	DumpSingleReg(ucontext, REG_R12);
	2252	DumpSingleReg(ucontext, REG_R13);
	2253	DumpSingleReg(ucontext, REG_R14);
	2254	DumpSingleReg(ucontext, REG_R15);
	2255	Printf("\n");
	2256	# elif defined(__i386__)
	2257	// Duplication of this report print is caused by partial support
	2258	// of register values dumping. In case of unsupported yet architecture let's
	2259	// avoid printing 'Register values:' without actual values in the following
	2260	// output.
	2261	Report("Register values:\n");
	2262	DumpSingleReg(ucontext, REG_EAX);
	2263	DumpSingleReg(ucontext, REG_EBX);
	2264	DumpSingleReg(ucontext, REG_ECX);
	2265	DumpSingleReg(ucontext, REG_EDX);
	2266	Printf("\n");
	2267	DumpSingleReg(ucontext, REG_EDI);
	2268	DumpSingleReg(ucontext, REG_ESI);
	2269	DumpSingleReg(ucontext, REG_EBP);
	2270	DumpSingleReg(ucontext, REG_ESP);
	2271	Printf("\n");
	2272	# else
	2273	(void)ucontext;
	2274	# endif
	2275	# elif SANITIZER_FREEBSD
	2276	# if defined(__x86_64__)
	2277	Report("Register values:\n");
	2278	Printf("rax = 0x%016lx ", ucontext->uc_mcontext.mc_rax);
	2279	Printf("rbx = 0x%016lx ", ucontext->uc_mcontext.mc_rbx);
	2280	Printf("rcx = 0x%016lx ", ucontext->uc_mcontext.mc_rcx);
	2281	Printf("rdx = 0x%016lx ", ucontext->uc_mcontext.mc_rdx);
	2282	Printf("\n");
	2283	Printf("rdi = 0x%016lx ", ucontext->uc_mcontext.mc_rdi);
	2284	Printf("rsi = 0x%016lx ", ucontext->uc_mcontext.mc_rsi);
	2285	Printf("rbp = 0x%016lx ", ucontext->uc_mcontext.mc_rbp);
	2286	Printf("rsp = 0x%016lx ", ucontext->uc_mcontext.mc_rsp);
	2287	Printf("\n");
	2288	Printf(" r8 = 0x%016lx ", ucontext->uc_mcontext.mc_r8);
	2289	Printf(" r9 = 0x%016lx ", ucontext->uc_mcontext.mc_r9);
	2290	Printf("r10 = 0x%016lx ", ucontext->uc_mcontext.mc_r10);
	2291	Printf("r11 = 0x%016lx ", ucontext->uc_mcontext.mc_r11);
	2292	Printf("\n");
	2293	Printf("r12 = 0x%016lx ", ucontext->uc_mcontext.mc_r12);
	2294	Printf("r13 = 0x%016lx ", ucontext->uc_mcontext.mc_r13);
	2295	Printf("r14 = 0x%016lx ", ucontext->uc_mcontext.mc_r14);
	2296	Printf("r15 = 0x%016lx ", ucontext->uc_mcontext.mc_r15);
	2297	Printf("\n");
	2298	# elif defined(__i386__)
	2299	Report("Register values:\n");
	2300	Printf("eax = 0x%08x ", ucontext->uc_mcontext.mc_eax);
	2301	Printf("ebx = 0x%08x ", ucontext->uc_mcontext.mc_ebx);
	2302	Printf("ecx = 0x%08x ", ucontext->uc_mcontext.mc_ecx);
	2303	Printf("edx = 0x%08x ", ucontext->uc_mcontext.mc_edx);
	2304	Printf("\n");
	2305	Printf("edi = 0x%08x ", ucontext->uc_mcontext.mc_edi);
	2306	Printf("esi = 0x%08x ", ucontext->uc_mcontext.mc_esi);
	2307	Printf("ebp = 0x%08x ", ucontext->uc_mcontext.mc_ebp);
	2308	Printf("esp = 0x%08x ", ucontext->uc_mcontext.mc_esp);
	2309	Printf("\n");
	2310	# else
	2311	(void)ucontext;
	2312	# endif
	2313	# endif
	2314	// FIXME: Implement this for other OSes and architectures.
2144	2315	}
2145	2316
2146	2317	static void GetPcSpBp(void context, uptr pc, uptr sp, uptr bp) {
2147		#if SANITIZER_NETBSD
	2318	# if SANITIZER_NETBSD
2148	2319	// This covers all NetBSD architectures
2149	2320	ucontext_t ucontext = (ucontext_t )context;
2150	2321	*pc = _UC_MACHINE_PC(ucontext);
2151	2322	*bp = _UC_MACHINE_FP(ucontext);
2152	2323	*sp = _UC_MACHINE_SP(ucontext);
2153		#elif defined(__arm__)
2154		ucontext_t ucontext = (ucontext_t)context;
	2324	# elif defined(__arm__)
	2325	ucontext_t ucontext = (ucontext_t )context;
2155	2326	*pc = ucontext->uc_mcontext.arm_pc;
2156	2327	*bp = ucontext->uc_mcontext.arm_fp;
2157	2328	*sp = ucontext->uc_mcontext.arm_sp;
2158		#elif defined(__aarch64__)
2159		# if SANITIZER_FREEBSD
2160		ucontext_t ucontext = (ucontext_t)context;
	2329	# elif defined(__aarch64__)
	2330	# if SANITIZER_FREEBSD
	2331	ucontext_t ucontext = (ucontext_t )context;
2161	2332	*pc = ucontext->uc_mcontext.mc_gpregs.gp_elr;
2162	2333	*bp = ucontext->uc_mcontext.mc_gpregs.gp_x[29];
2163	2334	*sp = ucontext->uc_mcontext.mc_gpregs.gp_sp;
2164		# else
2165		ucontext_t ucontext = (ucontext_t)context;
	2335	# else
	2336	ucontext_t ucontext = (ucontext_t )context;
2166	2337	*pc = ucontext->uc_mcontext.pc;
2167	2338	*bp = ucontext->uc_mcontext.regs[29];
2168	2339	*sp = ucontext->uc_mcontext.sp;
2169		# endif
2170		#elif defined(__hppa__)
2171		ucontext_t ucontext = (ucontext_t)context;
	2340	# endif
	2341	# elif defined(__hppa__)
	2342	ucontext_t ucontext = (ucontext_t )context;
2172	2343	*pc = ucontext->uc_mcontext.sc_iaoq[0];
2173	2344	/* GCC uses %r3 whenever a frame pointer is needed. */
2174	2345	*bp = ucontext->uc_mcontext.sc_gr[3];
2175	2346	*sp = ucontext->uc_mcontext.sc_gr[30];
2176		#elif defined(__x86_64__)
2177		# if SANITIZER_FREEBSD
2178		ucontext_t ucontext = (ucontext_t)context;
	2347	# elif defined(__x86_64__)
	2348	# if SANITIZER_FREEBSD
	2349	ucontext_t ucontext = (ucontext_t )context;
2179	2350	*pc = ucontext->uc_mcontext.mc_rip;
2180	2351	*bp = ucontext->uc_mcontext.mc_rbp;
2181	2352	*sp = ucontext->uc_mcontext.mc_rsp;
2182		# else
2183		ucontext_t ucontext = (ucontext_t)context;
	2353	# else
	2354	ucontext_t ucontext = (ucontext_t )context;
2184	2355	*pc = ucontext->uc_mcontext.gregs[REG_RIP];
2185	2356	*bp = ucontext->uc_mcontext.gregs[REG_RBP];
2186	2357	*sp = ucontext->uc_mcontext.gregs[REG_RSP];
2187		# endif
2188		#elif defined(__i386__)
2189		# if SANITIZER_FREEBSD
2190		ucontext_t ucontext = (ucontext_t)context;
	2358	# endif
	2359	# elif defined(__i386__)
	2360	# if SANITIZER_FREEBSD
	2361	ucontext_t ucontext = (ucontext_t )context;
2191	2362	*pc = ucontext->uc_mcontext.mc_eip;
2192	2363	*bp = ucontext->uc_mcontext.mc_ebp;
2193	2364	*sp = ucontext->uc_mcontext.mc_esp;
2194		# else
2195		ucontext_t ucontext = (ucontext_t)context;
2196		# if SANITIZER_SOLARIS
	2365	# else
	2366	ucontext_t ucontext = (ucontext_t )context;
	2367	# if SANITIZER_SOLARIS
2197	2368	/* Use the numeric values: the symbolic ones are undefined by llvm
2198	2369	include/llvm/Support/Solaris.h. */
2199		# ifndef REG_EIP
2200		# define REG_EIP 14 // REG_PC
2201		# endif
2202		# ifndef REG_EBP
2203		# define REG_EBP 6 // REG_FP
2204		# endif
2205		# ifndef REG_UESP
2206		# define REG_UESP 17 // REG_SP
2207		# endif
2208		# endif
	2370	# ifndef REG_EIP
	2371	# define REG_EIP 14 // REG_PC
	2372	# endif
	2373	# ifndef REG_EBP
	2374	# define REG_EBP 6 // REG_FP
	2375	# endif
	2376	# ifndef REG_UESP
	2377	# define REG_UESP 17 // REG_SP
	2378	# endif
	2379	# endif
2209	2380	*pc = ucontext->uc_mcontext.gregs[REG_EIP];
2210	2381	*bp = ucontext->uc_mcontext.gregs[REG_EBP];
2211	2382	*sp = ucontext->uc_mcontext.gregs[REG_UESP];
2212		# endif
2213		#elif defined(__powerpc__) \|\| defined(__powerpc64__)
	2383	# endif
	2384	# elif defined(__powerpc__) \|\| defined(__powerpc64__)
2214	2385	# if SANITIZER_FREEBSD
2215	2386	ucontext_t ucontext = (ucontext_t )context;
2216	2387	*pc = ucontext->uc_mcontext.mc_srr0;
2217	2388	*sp = ucontext->uc_mcontext.mc_frame[1];
2218	2389	*bp = ucontext->uc_mcontext.mc_frame[31];
2219	2390	# else
2220		ucontext_t ucontext = (ucontext_t)context;
	2391	ucontext_t ucontext = (ucontext_t )context;
2221	2392	*pc = ucontext->uc_mcontext.regs->nip;
2222	2393	*sp = ucontext->uc_mcontext.regs->gpr[PT_R1];
2223	2394	// The powerpc{,64}-linux ABIs do not specify r31 as the frame
2224	2395	// pointer, but GCC always uses r31 when we need a frame pointer.
2225	2396	*bp = ucontext->uc_mcontext.regs->gpr[PT_R31];
2226	2397	# endif
2227		#elif defined(__sparc__)
2228		#if defined(__arch64__) \|\| defined(__sparcv9)
2229		#define STACK_BIAS 2047
2230		#else
2231		#define STACK_BIAS 0
2232		# endif
2233		# if SANITIZER_SOLARIS
	2398	# elif defined(__sparc__)
	2399	# if defined(__arch64__) \|\| defined(__sparcv9)
	2400	# define STACK_BIAS 2047
	2401	# else
	2402	# define STACK_BIAS 0
	2403	# endif
	2404	# if SANITIZER_SOLARIS
2234	2405	ucontext_t ucontext = (ucontext_t )context;
2235	2406	*pc = ucontext->uc_mcontext.gregs[REG_PC];
2236	2407	*sp = ucontext->uc_mcontext.gregs[REG_O6] + STACK_BIAS;
2237		#else
	2408	# else
2238	2409	// Historical BSDism here.
2239	2410	struct sigcontext scontext = (struct sigcontext )context;
2240		#if defined(__arch64__)
	2411	# if defined(__arch64__)
2241	2412	*pc = scontext->sigc_regs.tpc;
2242	2413	*sp = scontext->sigc_regs.u_regs[14] + STACK_BIAS;
2243		#else
	2414	# else
2244	2415	*pc = scontext->si_regs.pc;
2245	2416	*sp = scontext->si_regs.u_regs[14];
2246		#endif
2247		# endif
	2417	# endif
	2418	# endif
2248	2419	bp = (uptr)((uhwptr )*sp)[14] + STACK_BIAS;
2249		#elif defined(__mips__)
2250		ucontext_t ucontext = (ucontext_t)context;
	2420	# elif defined(__mips__)
	2421	ucontext_t ucontext = (ucontext_t )context;
2251	2422	*pc = ucontext->uc_mcontext.pc;
2252	2423	*bp = ucontext->uc_mcontext.gregs[30];
2253	2424	*sp = ucontext->uc_mcontext.gregs[29];
2254		#elif defined(__s390__)
2255		ucontext_t ucontext = (ucontext_t)context;
2256		# if defined(__s390x__)
	2425	# elif defined(__s390__)
	2426	ucontext_t ucontext = (ucontext_t )context;
	2427	# if defined(__s390x__)
2257	2428	*pc = ucontext->uc_mcontext.psw.addr;
2258		# else
	2429	# else
2259	2430	*pc = ucontext->uc_mcontext.psw.addr & 0x7fffffff;
2260		# endif
	2431	# endif
2261	2432	*bp = ucontext->uc_mcontext.gregs[11];
2262	2433	*sp = ucontext->uc_mcontext.gregs[15];
2263		#elif defined(__riscv)
2264		ucontext_t ucontext = (ucontext_t)context;
	2434	# elif defined(__riscv)
	2435	ucontext_t ucontext = (ucontext_t )context;
2265	2436	# if SANITIZER_FREEBSD
2266	2437	*pc = ucontext->uc_mcontext.mc_gpregs.gp_sepc;
2267	2438	*bp = ucontext->uc_mcontext.mc_gpregs.gp_s[0];
...	...	@@ -2293,7 +2464,7 @@ void InitializePlatformEarly() {
2293	2464	}
2294	2465
2295	2466	void CheckASLR() {
2296		#if SANITIZER_NETBSD
	2467	# if SANITIZER_NETBSD
2297	2468	int mib[3];
2298	2469	int paxflags;
2299	2470	uptr len = sizeof(paxflags);
...	...	@@ -2308,12 +2479,13 @@ void CheckASLR() {
2308	2479	}
2309	2480
2310	2481	if (UNLIKELY(paxflags & CTL_PROC_PAXFLAGS_ASLR)) {
2311		Printf("This sanitizer is not compatible with enabled ASLR.\n"
2312		"To disable ASLR, please run \"paxctl +a %s\" and try again.\n",
2313		GetArgv()[0]);
	2482	Printf(
	2483	"This sanitizer is not compatible with enabled ASLR.\n"
	2484	"To disable ASLR, please run \"paxctl +a %s\" and try again.\n",
	2485	GetArgv()[0]);
2314	2486	Die();
2315	2487	}
2316		#elif SANITIZER_FREEBSD
	2488	# elif SANITIZER_FREEBSD
2317	2489	int aslr_status;
2318	2490	int r = internal_procctl(P_PID, 0, PROC_ASLR_STATUS, &aslr_status);
2319	2491	if (UNLIKELY(r == -1)) {
...	...	@@ -2323,9 +2495,13 @@ void CheckASLR() {
2323	2495	return;
2324	2496	}
2325	2497	if ((aslr_status & PROC_ASLR_ACTIVE) != 0) {
2326		Printf("This sanitizer is not compatible with enabled ASLR "
2327		"and binaries compiled with PIE\n");
2328		Die();
	2498	VReport(1,
	2499	"This sanitizer is not compatible with enabled ASLR "
	2500	"and binaries compiled with PIE\n"
	2501	"ASLR will be disabled and the program re-executed.\n");
	2502	int aslr_ctl = PROC_ASLR_FORCE_DISABLE;
	2503	CHECK_NE(internal_procctl(P_PID, 0, PROC_ASLR_CTL, &aslr_ctl), -1);
	2504	ReExec();
2329	2505	}
2330	2506	# elif SANITIZER_PPC64V2
2331	2507	// Disable ASLR for Linux PPC64LE.
...	...	@@ -2345,7 +2521,7 @@ void CheckASLR() {
2345	2521	}
2346	2522
2347	2523	void CheckMPROTECT() {
2348		#if SANITIZER_NETBSD
	2524	# if SANITIZER_NETBSD
2349	2525	int mib[3];
2350	2526	int paxflags;
2351	2527	uptr len = sizeof(paxflags);
...	...	@@ -2363,13 +2539,13 @@ void CheckMPROTECT() {
2363	2539	Printf("This sanitizer is not compatible with enabled MPROTECT\n");
2364	2540	Die();
2365	2541	}
2366		#else
	2542	# else
2367	2543	// Do nothing
2368		#endif
	2544	# endif
2369	2545	}
2370	2546
2371	2547	void CheckNoDeepBind(const char *filename, int flag) {
2372		#ifdef RTLD_DEEPBIND
	2548	# ifdef RTLD_DEEPBIND
2373	2549	if (flag & RTLD_DEEPBIND) {
2374	2550	Report(
2375	2551	"You are trying to dlopen a %s shared library with RTLD_DEEPBIND flag"
...	...	@@ -2380,7 +2556,7 @@ void CheckNoDeepBind(const char *filename, int flag) {
2380	2556	filename, filename);
2381	2557	Die();
2382	2558	}
2383		#endif
	2559	# endif
2384	2560	}
2385	2561
2386	2562	uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,
...	...	@@ -2393,16 +2569,16 @@ uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,
2393	2569	bool GetRandom(void *buffer, uptr length, bool blocking) {
2394	2570	if (!buffer \|\| !length \|\| length > 256)
2395	2571	return false;
2396		#if SANITIZER_USE_GETENTROPY
	2572	# if SANITIZER_USE_GETENTROPY
2397	2573	uptr rnd = getentropy(buffer, length);
2398	2574	int rverrno = 0;
2399	2575	if (internal_iserror(rnd, &rverrno) && rverrno == EFAULT)
2400	2576	return false;
2401	2577	else if (rnd == 0)
2402	2578	return true;
2403		#endif // SANITIZER_USE_GETENTROPY
	2579	# endif // SANITIZER_USE_GETENTROPY
2404	2580
2405		#if SANITIZER_USE_GETRANDOM
	2581	# if SANITIZER_USE_GETRANDOM
2406	2582	static atomic_uint8_t skip_getrandom_syscall;
2407	2583	if (!atomic_load_relaxed(&skip_getrandom_syscall)) {
2408	2584	// Up to 256 bytes, getrandom will not be interrupted.
...	...	@@ -2414,7 +2590,7 @@ bool GetRandom(void *buffer, uptr length, bool blocking) {
2414	2590	else if (res == length)
2415	2591	return true;
2416	2592	}
2417		#endif // SANITIZER_USE_GETRANDOM
	2593	# endif // SANITIZER_USE_GETRANDOM
2418	2594	// Up to 256 bytes, a read off /dev/urandom will not be interrupted.
2419	2595	// blocking is moot here, O_NONBLOCK has no effect when opening /dev/urandom.
2420	2596	uptr fd = internal_open("/dev/urandom", O_RDONLY);
...	...	@@ -2427,6 +2603,6 @@ bool GetRandom(void *buffer, uptr length, bool blocking) {
2427	2603	return true;
2428	2604	}
2429	2605
2430		} // namespace __sanitizer
	2606	} // namespace __sanitizer
2431	2607
2432	2608	#endif

lib/tsan/sanitizer_common/sanitizer_linux.h+69-45

...	...	@@ -13,15 +13,15 @@
13	13	#define SANITIZER_LINUX_H
14	14
15	15	#include "sanitizer_platform.h"
16		#if SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_NETBSD \|\| \
	16	#if SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_NETBSD \|\| \
17	17	SANITIZER_SOLARIS
18		#include "sanitizer_common.h"
19		#include "sanitizer_internal_defs.h"
20		#include "sanitizer_platform_limits_freebsd.h"
21		#include "sanitizer_platform_limits_netbsd.h"
22		#include "sanitizer_platform_limits_posix.h"
23		#include "sanitizer_platform_limits_solaris.h"
24		#include "sanitizer_posix.h"
	18	# include "sanitizer_common.h"
	19	# include "sanitizer_internal_defs.h"
	20	# include "sanitizer_platform_limits_freebsd.h"
	21	# include "sanitizer_platform_limits_netbsd.h"
	22	# include "sanitizer_platform_limits_posix.h"
	23	# include "sanitizer_platform_limits_solaris.h"
	24	# include "sanitizer_posix.h"
25	25
26	26	struct link_map; // Opaque type returned by dlopen().
27	27	struct utsname;
...	...	@@ -46,9 +46,9 @@ void ReadProcMaps(ProcSelfMapsBuff *proc_maps);
46	46
47	47	// Syscall wrappers.
48	48	uptr internal_getdents(fd_t fd, struct linux_dirent *dirp, unsigned int count);
49		uptr internal_sigaltstack(const void* ss, void* oss);
	49	uptr internal_sigaltstack(const void ss, void oss);
50	50	uptr internal_sigprocmask(int how, __sanitizer_sigset_t *set,
51		__sanitizer_sigset_t *oldset);
	51	__sanitizer_sigset_t *oldset);
52	52
53	53	void SetSigProcMask(__sanitizer_sigset_t set, __sanitizer_sigset_t oldset);
54	54	void BlockSignals(__sanitizer_sigset_t *oldset = nullptr);
...	...	@@ -65,10 +65,10 @@ struct ScopedBlockSignals {
65	65
66	66	# if SANITIZER_GLIBC
67	67	uptr internal_clock_gettime(__sanitizer_clockid_t clk_id, void *tp);
68		#endif
	68	# endif
69	69
70	70	// Linux-only syscalls.
71		#if SANITIZER_LINUX
	71	# if SANITIZER_LINUX
72	72	uptr internal_prctl(int option, uptr arg2, uptr arg3, uptr arg4, uptr arg5);
73	73	# if defined(__x86_64__)
74	74	uptr internal_arch_prctl(int option, uptr arg2);
...	...	@@ -83,15 +83,15 @@ void internal_sigdelset(__sanitizer_sigset_t *set, int signum);
83	83	defined(__arm__) \|\| SANITIZER_RISCV64 \|\| SANITIZER_LOONGARCH64
84	84	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
85	85	int parent_tidptr, void newtls, int *child_tidptr);
86		#endif
	86	# endif
87	87	int internal_uname(struct utsname *buf);
88		#elif SANITIZER_FREEBSD
	88	# elif SANITIZER_FREEBSD
89	89	uptr internal_procctl(int type, int id, int cmd, void *data);
90	90	void internal_sigdelset(__sanitizer_sigset_t *set, int signum);
91		#elif SANITIZER_NETBSD
	91	# elif SANITIZER_NETBSD
92	92	void internal_sigdelset(__sanitizer_sigset_t *set, int signum);
93	93	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg);
94		#endif // SANITIZER_LINUX
	94	# endif // SANITIZER_LINUX
95	95
96	96	// This class reads thread IDs from /proc/<pid>/task using only syscalls.
97	97	class ThreadLister {
...	...	@@ -135,36 +135,60 @@ inline void ReleaseMemoryPagesToOSAndZeroFill(uptr beg, uptr end) {
135	135	ReleaseMemoryPagesToOS(beg, end);
136	136	}
137	137
138		#if SANITIZER_ANDROID
139
140		#if defined(__aarch64__)
141		# define __get_tls() \
142		({ void** __v; __asm__("mrs %0, tpidr_el0" : "=r"(__v)); __v; })
143		#elif defined(__arm__)
144		# define __get_tls() \
145		({ void** __v; __asm__("mrc p15, 0, %0, c13, c0, 3" : "=r"(__v)); __v; })
146		#elif defined(__mips__)
	138	# if SANITIZER_ANDROID
	139
	140	# if defined(__aarch64__)
	141	# define __get_tls() \
	142	({ \
	143	void **__v; \
	144	__asm__("mrs %0, tpidr_el0" : "=r"(__v)); \
	145	__v; \
	146	})
	147	# elif defined(__arm__)
	148	# define __get_tls() \
	149	({ \
	150	void **__v; \
	151	__asm__("mrc p15, 0, %0, c13, c0, 3" : "=r"(__v)); \
	152	__v; \
	153	})
	154	# elif defined(__mips__)
147	155	// On mips32r1, this goes via a kernel illegal instruction trap that's
148	156	// optimized for v1.
149		# define __get_tls() \
150		({ register void** __v asm("v1"); \
151		__asm__(".set push\n" \
152		".set mips32r2\n" \
153		"rdhwr %0,$29\n" \
154		".set pop\n" : "=r"(__v)); \
155		__v; })
156		#elif defined (__riscv)
157		# define __get_tls() \
158		({ void** __v; __asm__("mv %0, tp" : "=r"(__v)); __v; })
159		#elif defined(__i386__)
160		# define __get_tls() \
161		({ void** __v; __asm__("movl %%gs:0, %0" : "=r"(__v)); __v; })
162		#elif defined(__x86_64__)
163		# define __get_tls() \
164		({ void** __v; __asm__("mov %%fs:0, %0" : "=r"(__v)); __v; })
165		#else
166		#error "Unsupported architecture."
167		#endif
	157	# define __get_tls() \
	158	({ \
	159	register void **__v asm("v1"); \
	160	__asm__( \
	161	".set push\n" \
	162	".set mips32r2\n" \
	163	"rdhwr %0,$29\n" \
	164	".set pop\n" \
	165	: "=r"(__v)); \
	166	__v; \
	167	})
	168	# elif defined(__riscv)
	169	# define __get_tls() \
	170	({ \
	171	void **__v; \
	172	__asm__("mv %0, tp" : "=r"(__v)); \
	173	__v; \
	174	})
	175	# elif defined(__i386__)
	176	# define __get_tls() \
	177	({ \
	178	void **__v; \
	179	__asm__("movl %%gs:0, %0" : "=r"(__v)); \
	180	__v; \
	181	})
	182	# elif defined(__x86_64__)
	183	# define __get_tls() \
	184	({ \
	185	void **__v; \
	186	__asm__("mov %%fs:0, %0" : "=r"(__v)); \
	187	__v; \
	188	})
	189	# else
	190	# error "Unsupported architecture."
	191	# endif
168	192
169	193	// The Android Bionic team has allocated a TLS slot for sanitizers starting
170	194	// with Q, given that Android currently doesn't support ELF TLS. It is used to
...	...	@@ -175,7 +199,7 @@ ALWAYS_INLINE uptr *get_android_tls_ptr() {
175	199	return reinterpret_cast<uptr *>(&__get_tls()[TLS_SLOT_SANITIZER]);
176	200	}
177	201
178		#endif // SANITIZER_ANDROID
	202	# endif // SANITIZER_ANDROID
179	203
180	204	} // namespace __sanitizer
181	205

lib/tsan/sanitizer_common/sanitizer_linux_libcdep.cpp+252-231

...	...	@@ -16,89 +16,101 @@
16	16	#if SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_NETBSD \|\| \
17	17	SANITIZER_SOLARIS
18	18
19		#include "sanitizer_allocator_internal.h"
20		#include "sanitizer_atomic.h"
21		#include "sanitizer_common.h"
22		#include "sanitizer_file.h"
23		#include "sanitizer_flags.h"
24		#include "sanitizer_freebsd.h"
25		#include "sanitizer_getauxval.h"
26		#include "sanitizer_glibc_version.h"
27		#include "sanitizer_linux.h"
28		#include "sanitizer_placement_new.h"
29		#include "sanitizer_procmaps.h"
30		#include "sanitizer_solaris.h"
31
32		#if SANITIZER_NETBSD
33		#define _RTLD_SOURCE // for __lwp_gettcb_fast() / __lwp_getprivate_fast()
34		#endif
	19	# include "sanitizer_allocator_internal.h"
	20	# include "sanitizer_atomic.h"
	21	# include "sanitizer_common.h"
	22	# include "sanitizer_file.h"
	23	# include "sanitizer_flags.h"
	24	# include "sanitizer_getauxval.h"
	25	# include "sanitizer_glibc_version.h"
	26	# include "sanitizer_linux.h"
	27	# include "sanitizer_placement_new.h"
	28	# include "sanitizer_procmaps.h"
	29	# include "sanitizer_solaris.h"
	30
	31	# if SANITIZER_NETBSD
	32	# define _RTLD_SOURCE // for __lwp_gettcb_fast() / __lwp_getprivate_fast()
	33	# endif
35	34
36		#include <dlfcn.h> // for dlsym()
37		#include <link.h>
38		#include <pthread.h>
39		#include <signal.h>
40		#include <sys/mman.h>
41		#include <sys/resource.h>
42		#include <syslog.h>
	35	# include <dlfcn.h> // for dlsym()
	36	# include <link.h>
	37	# include <pthread.h>
	38	# include <signal.h>
	39	# include <sys/mman.h>
	40	# include <sys/resource.h>
	41	# include <syslog.h>
43	42
44		#if !defined(ElfW)
45		#define ElfW(type) Elf_##type
46		#endif
	43	# if !defined(ElfW)
	44	# define ElfW(type) Elf_##type
	45	# endif
47	46
48		#if SANITIZER_FREEBSD
49		#include <pthread_np.h>
50		#include <osreldate.h>
51		#include <sys/sysctl.h>
52		#define pthread_getattr_np pthread_attr_get_np
	47	# if SANITIZER_FREEBSD
	48	# include <pthread_np.h>
	49	# include <sys/auxv.h>
	50	# include <sys/sysctl.h>
	51	# define pthread_getattr_np pthread_attr_get_np
53	52	// The MAP_NORESERVE define has been removed in FreeBSD 11.x, and even before
54	53	// that, it was never implemented. So just define it to zero.
55		#undef MAP_NORESERVE
56		#define MAP_NORESERVE 0
57		#endif
	54	# undef MAP_NORESERVE
	55	# define MAP_NORESERVE 0
	56	extern const Elf_Auxinfo *__elf_aux_vector;
	57	extern "C" int __sys_sigaction(int signum, const struct sigaction *act,
	58	struct sigaction *oldact);
	59	# endif
58	60
59		#if SANITIZER_NETBSD
60		#include <sys/sysctl.h>
61		#include <sys/tls.h>
62		#include <lwp.h>
63		#endif
	61	# if SANITIZER_NETBSD
	62	# include <lwp.h>
	63	# include <sys/sysctl.h>
	64	# include <sys/tls.h>
	65	# endif
64	66
65		#if SANITIZER_SOLARIS
66		#include <stddef.h>
67		#include <stdlib.h>
68		#include <thread.h>
69		#endif
	67	# if SANITIZER_SOLARIS
	68	# include <stddef.h>
	69	# include <stdlib.h>
	70	# include <thread.h>
	71	# endif
70	72
71		#if SANITIZER_ANDROID
72		#include <android/api-level.h>
73		#if !defined(CPU_COUNT) && !defined(__aarch64__)
74		#include <dirent.h>
75		#include <fcntl.h>
	73	# if SANITIZER_ANDROID
	74	# include <android/api-level.h>
	75	# if !defined(CPU_COUNT) && !defined(__aarch64__)
	76	# include <dirent.h>
	77	# include <fcntl.h>
76	78	struct __sanitizer::linux_dirent {
77		long d_ino;
78		off_t d_off;
	79	long d_ino;
	80	off_t d_off;
79	81	unsigned short d_reclen;
80		char d_name[];
	82	char d_name[];
81	83	};
82		#endif
83		#endif
	84	# endif
	85	# endif
84	86
85		#if !SANITIZER_ANDROID
86		#include <elf.h>
87		#include <unistd.h>
88		#endif
	87	# if !SANITIZER_ANDROID
	88	# include <elf.h>
	89	# include <unistd.h>
	90	# endif
89	91
90	92	namespace __sanitizer {
91	93
92		SANITIZER_WEAK_ATTRIBUTE int
93		real_sigaction(int signum, const void act, void oldact);
	94	SANITIZER_WEAK_ATTRIBUTE int real_sigaction(int signum, const void *act,
	95	void *oldact);
94	96
95	97	int internal_sigaction(int signum, const void act, void oldact) {
96		#if !SANITIZER_GO
	98	# if SANITIZER_FREEBSD
	99	// On FreeBSD, call the sigaction syscall directly (part of libsys in FreeBSD
	100	// 15) since the libc version goes via a global interposing table. Due to
	101	// library initialization order the table can be relocated after the call to
	102	// InitializeDeadlySignals() which then crashes when dereferencing the
	103	// uninitialized pointer in libc.
	104	return __sys_sigaction(signum, (const struct sigaction *)act,
	105	(struct sigaction *)oldact);
	106	# else
	107	# if !SANITIZER_GO
97	108	if (&real_sigaction)
98	109	return real_sigaction(signum, act, oldact);
99		#endif
	110	# endif
100	111	return sigaction(signum, (const struct sigaction *)act,
101	112	(struct sigaction *)oldact);
	113	# endif
102	114	}
103	115
104	116	void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top,
...	...	@@ -111,7 +123,7 @@ void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top,
111	123	CHECK_EQ(getrlimit(RLIMIT_STACK, &rl), 0);
112	124
113	125	// Find the mapping that contains a stack variable.
114		MemoryMappingLayout proc_maps(/cache_enabled/true);
	126	MemoryMappingLayout proc_maps(/cache_enabled/ true);
115	127	if (proc_maps.Error()) {
116	128	stack_top = stack_bottom = 0;
117	129	return;
...	...	@@ -119,7 +131,8 @@ void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top,
119	131	MemoryMappedSegment segment;
120	132	uptr prev_end = 0;
121	133	while (proc_maps.Next(&segment)) {
122		if ((uptr)&rl < segment.end) break;
	134	if ((uptr)&rl < segment.end)
	135	break;
123	136	prev_end = segment.end;
124	137	}
125	138	CHECK((uptr)&rl >= segment.start && (uptr)&rl < segment.end);
...	...	@@ -127,7 +140,8 @@ void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top,
127	140	// Get stacksize from rlimit, but clip it so that it does not overlap
128	141	// with other mappings.
129	142	uptr stacksize = rl.rlim_cur;
130		if (stacksize > segment.end - prev_end) stacksize = segment.end - prev_end;
	143	if (stacksize > segment.end - prev_end)
	144	stacksize = segment.end - prev_end;
131	145	// When running with unlimited stack size, we still want to set some limit.
132	146	// The unlimited stack size is caused by 'ulimit -s unlimited'.
133	147	// Also, for some reason, GNU make spawns subprocesses with unlimited stack.
...	...	@@ -135,43 +149,56 @@ void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top,
135	149	stacksize = kMaxThreadStackSize;
136	150	*stack_top = segment.end;
137	151	*stack_bottom = segment.end - stacksize;
	152
	153	uptr maxAddr = GetMaxUserVirtualAddress();
	154	// Edge case: the stack mapping on some systems may be off-by-one e.g.,
	155	// fffffffdf000-1000000000000 rw-p 00000000 00:00 0 [stack]
	156	// instead of:
	157	// fffffffdf000- ffffffffffff
	158	// The out-of-range stack_top can result in an invalid shadow address
	159	// calculation, since those usually assume the parameters are in range.
	160	if (*stack_top == maxAddr + 1)
	161	*stack_top = maxAddr;
	162	else
	163	CHECK_LE(*stack_top, maxAddr);
	164
138	165	return;
139	166	}
140	167	uptr stacksize = 0;
141	168	void *stackaddr = nullptr;
142		#if SANITIZER_SOLARIS
	169	# if SANITIZER_SOLARIS
143	170	stack_t ss;
144	171	CHECK_EQ(thr_stksegment(&ss), 0);
145	172	stacksize = ss.ss_size;
146	173	stackaddr = (char *)ss.ss_sp - stacksize;
147		#else // !SANITIZER_SOLARIS
	174	# else // !SANITIZER_SOLARIS
148	175	pthread_attr_t attr;
149	176	pthread_attr_init(&attr);
150	177	CHECK_EQ(pthread_getattr_np(pthread_self(), &attr), 0);
151	178	internal_pthread_attr_getstack(&attr, &stackaddr, &stacksize);
152	179	pthread_attr_destroy(&attr);
153		#endif // SANITIZER_SOLARIS
	180	# endif // SANITIZER_SOLARIS
154	181
155	182	*stack_top = (uptr)stackaddr + stacksize;
156	183	*stack_bottom = (uptr)stackaddr;
157	184	}
158	185
159		#if !SANITIZER_GO
	186	# if !SANITIZER_GO
160	187	bool SetEnv(const char name, const char value) {
161	188	void *f = dlsym(RTLD_NEXT, "setenv");
162	189	if (!f)
163	190	return false;
164		typedef int(setenv_ft)(const char name, const char *value, int overwrite);
	191	typedef int (setenv_ft)(const char name, const char *value, int overwrite);
165	192	setenv_ft setenv_f;
166	193	CHECK_EQ(sizeof(setenv_f), sizeof(f));
167	194	internal_memcpy(&setenv_f, &f, sizeof(f));
168	195	return setenv_f(name, value, 1) == 0;
169	196	}
170		#endif
	197	# endif
171	198
172	199	__attribute__((unused)) static bool GetLibcVersion(int major, int minor,
173	200	int *patch) {
174		#ifdef _CS_GNU_LIBC_VERSION
	201	# ifdef _CS_GNU_LIBC_VERSION
175	202	char buf[64];
176	203	uptr len = confstr(_CS_GNU_LIBC_VERSION, buf, sizeof(buf));
177	204	if (len >= sizeof(buf))
...	...	@@ -185,9 +212,9 @@ __attribute__((unused)) static bool GetLibcVersion(int major, int minor,
185	212	minor = (p == '.') ? internal_simple_strtoll(p + 1, &p, 10) : 0;
186	213	patch = (p == '.') ? internal_simple_strtoll(p + 1, &p, 10) : 0;
187	214	return true;
188		#else
	215	# else
189	216	return false;
190		#endif
	217	# endif
191	218	}
192	219
193	220	// True if we can use dlpi_tls_data. glibc before 2.25 may leave NULL (BZ
...	...	@@ -198,42 +225,42 @@ __attribute__((unused)) static bool GetLibcVersion(int major, int minor,
198	225	// https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=254774
199	226	__attribute__((unused)) static int g_use_dlpi_tls_data;
200	227
201		#if SANITIZER_GLIBC && !SANITIZER_GO
	228	# if SANITIZER_GLIBC && !SANITIZER_GO
202	229	__attribute__((unused)) static size_t g_tls_size;
203	230	void InitTlsSize() {
204	231	int major, minor, patch;
205	232	g_use_dlpi_tls_data =
206	233	GetLibcVersion(&major, &minor, &patch) && major == 2 && minor >= 25;
207	234
208		#if defined(__aarch64__) \|\| defined(__x86_64__) \|\| defined(__powerpc64__) \|\| \
209		defined(__loongarch__)
	235	# if defined(__aarch64__) \|\| defined(__x86_64__) \|\| \
	236	defined(__powerpc64__) \|\| defined(__loongarch__)
210	237	void *get_tls_static_info = dlsym(RTLD_NEXT, "_dl_get_tls_static_info");
211	238	size_t tls_align;
212	239	((void ()(size_t , size_t *))get_tls_static_info)(&g_tls_size, &tls_align);
213		#endif
	240	# endif
214	241	}
215		#else
216		void InitTlsSize() { }
217		#endif // SANITIZER_GLIBC && !SANITIZER_GO
	242	# else
	243	void InitTlsSize() {}
	244	# endif // SANITIZER_GLIBC && !SANITIZER_GO
218	245
219	246	// On glibc x86_64, ThreadDescriptorSize() needs to be precise due to the usage
220	247	// of g_tls_size. On other targets, ThreadDescriptorSize() is only used by lsan
221	248	// to get the pointer to thread-specific data keys in the thread control block.
222		#if (SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_SOLARIS) && \
223		!SANITIZER_ANDROID && !SANITIZER_GO
	249	# if (SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_SOLARIS) && \
	250	!SANITIZER_ANDROID && !SANITIZER_GO
224	251	// sizeof(struct pthread) from glibc.
225	252	static atomic_uintptr_t thread_descriptor_size;
226	253
227	254	static uptr ThreadDescriptorSizeFallback() {
228	255	uptr val = 0;
229		#if defined(__x86_64__) \|\| defined(__i386__) \|\| defined(__arm__)
	256	# if defined(__x86_64__) \|\| defined(__i386__) \|\| defined(__arm__)
230	257	int major;
231	258	int minor;
232	259	int patch;
233	260	if (GetLibcVersion(&major, &minor, &patch) && major == 2) {
234	261	/* sizeof(struct pthread) values from various glibc versions. */
235	262	if (SANITIZER_X32)
236		val = 1728; // Assume only one particular version for x32.
	263	val = 1728; // Assume only one particular version for x32.
237	264	// For ARM sizeof(struct pthread) changed in Glibc 2.23.
238	265	else if (SANITIZER_ARM)
239	266	val = minor <= 22 ? 1120 : 1216;
...	...	@@ -256,19 +283,19 @@ static uptr ThreadDescriptorSizeFallback() {
256	283	else // minor == 32
257	284	val = FIRST_32_SECOND_64(1344, 2496);
258	285	}
259		#elif defined(__s390__) \|\| defined(__sparc__)
	286	# elif defined(__s390__) \|\| defined(__sparc__)
260	287	// The size of a prefix of TCB including pthread::{specific_1stblock,specific}
261	288	// suffices. Just return offsetof(struct pthread, specific_used), which hasn't
262	289	// changed since 2007-05. Technically this applies to i386/x86_64 as well but
263	290	// we call _dl_get_tls_static_info and need the precise size of struct
264	291	// pthread.
265	292	return FIRST_32_SECOND_64(524, 1552);
266		#elif defined(__mips__)
	293	# elif defined(__mips__)
267	294	// TODO(sagarthakur): add more values as per different glibc versions.
268	295	val = FIRST_32_SECOND_64(1152, 1776);
269		#elif SANITIZER_LOONGARCH64
270		val = 1856; // from glibc 2.36
271		#elif SANITIZER_RISCV64
	296	# elif SANITIZER_LOONGARCH64
	297	val = 1856; // from glibc 2.36
	298	# elif SANITIZER_RISCV64
272	299	int major;
273	300	int minor;
274	301	int patch;
...	...	@@ -283,12 +310,12 @@ static uptr ThreadDescriptorSizeFallback() {
283	310	val = 1936; // tested against glibc 2.32
284	311	}
285	312
286		#elif defined(__aarch64__)
	313	# elif defined(__aarch64__)
287	314	// The sizeof (struct pthread) is the same from GLIBC 2.17 to 2.22.
288	315	val = 1776;
289		#elif defined(__powerpc64__)
290		val = 1776; // from glibc.ppc64le 2.20-8.fc21
291		#endif
	316	# elif defined(__powerpc64__)
	317	val = 1776; // from glibc.ppc64le 2.20-8.fc21
	318	# endif
292	319	return val;
293	320	}
294	321
...	...	@@ -307,26 +334,26 @@ uptr ThreadDescriptorSize() {
307	334	return val;
308	335	}
309	336
310		#if defined(__mips__) \|\| defined(__powerpc64__) \|\| SANITIZER_RISCV64 \|\| \
311		SANITIZER_LOONGARCH64
	337	# if defined(__mips__) \|\| defined(__powerpc64__) \|\| SANITIZER_RISCV64 \|\| \
	338	SANITIZER_LOONGARCH64
312	339	// TlsPreTcbSize includes size of struct pthread_descr and size of tcb
313	340	// head structure. It lies before the static tls blocks.
314	341	static uptr TlsPreTcbSize() {
315		#if defined(__mips__)
316		const uptr kTcbHead = 16; // sizeof (tcbhead_t)
317		#elif defined(__powerpc64__)
318		const uptr kTcbHead = 88; // sizeof (tcbhead_t)
319		#elif SANITIZER_RISCV64
	342	# if defined(__mips__)
320	343	const uptr kTcbHead = 16; // sizeof (tcbhead_t)
321		#elif SANITIZER_LOONGARCH64
	344	# elif defined(__powerpc64__)
	345	const uptr kTcbHead = 88; // sizeof (tcbhead_t)
	346	# elif SANITIZER_RISCV64
322	347	const uptr kTcbHead = 16; // sizeof (tcbhead_t)
323		#endif
	348	# elif SANITIZER_LOONGARCH64
	349	const uptr kTcbHead = 16; // sizeof (tcbhead_t)
	350	# endif
324	351	const uptr kTlsAlign = 16;
325	352	const uptr kTlsPreTcbSize =
326	353	RoundUpTo(ThreadDescriptorSize() + kTcbHead, kTlsAlign);
327	354	return kTlsPreTcbSize;
328	355	}
329		#endif
	356	# endif
330	357
331	358	namespace {
332	359	struct TlsBlock {
...	...	@@ -336,7 +363,7 @@ struct TlsBlock {
336	363	};
337	364	} // namespace
338	365
339		#ifdef __s390__
	366	# ifdef __s390__
340	367	extern "C" uptr __tls_get_offset(void *arg);
341	368
342	369	static uptr TlsGetOffset(uptr ti_module, uptr ti_offset) {
...	...	@@ -354,16 +381,16 @@ static uptr TlsGetOffset(uptr ti_module, uptr ti_offset) {
354	381	: "memory", "cc", "0", "1", "3", "4", "5", "14");
355	382	return r2;
356	383	}
357		#else
	384	# else
358	385	extern "C" void __tls_get_addr(size_t );
359		#endif
	386	# endif
360	387
361	388	static size_t main_tls_modid;
362	389
363	390	static int CollectStaticTlsBlocks(struct dl_phdr_info *info, size_t size,
364	391	void *data) {
365	392	size_t tls_modid;
366		#if SANITIZER_SOLARIS
	393	# if SANITIZER_SOLARIS
367	394	// dlpi_tls_modid is only available since Solaris 11.4 SRU 10. Use
368	395	// dlinfo(RTLD_DI_LINKMAP) instead which works on all of Solaris 11.3,
369	396	// 11.4, and Illumos. The tlsmodid of the executable was changed to 1 in
...	...	@@ -376,27 +403,26 @@ static int CollectStaticTlsBlocks(struct dl_phdr_info *info, size_t size,
376	403	Rt_map *map;
377	404	dlinfo(RTLD_SELF, RTLD_DI_LINKMAP, &map);
378	405	tls_modid = map->rt_tlsmodid;
379		#else
	406	# else
380	407	main_tls_modid = 1;
381	408	tls_modid = info->dlpi_tls_modid;
382		#endif
	409	# endif
383	410
384	411	if (tls_modid < main_tls_modid)
385	412	return 0;
386	413	uptr begin;
387		#if !SANITIZER_SOLARIS
	414	# if !SANITIZER_SOLARIS
388	415	begin = (uptr)info->dlpi_tls_data;
389		#endif
	416	# endif
390	417	if (!g_use_dlpi_tls_data) {
391	418	// Call __tls_get_addr as a fallback. This forces TLS allocation on glibc
392	419	// and FreeBSD.
393		#ifdef __s390__
394		begin = (uptr)__builtin_thread_pointer() +
395		TlsGetOffset(tls_modid, 0);
396		#else
	420	# ifdef __s390__
	421	begin = (uptr)__builtin_thread_pointer() + TlsGetOffset(tls_modid, 0);
	422	# else
397	423	size_t mod_and_off[2] = {tls_modid, 0};
398	424	begin = (uptr)__tls_get_addr(mod_and_off);
399		#endif
	425	# endif
400	426	}
401	427	for (unsigned i = 0; i != info->dlpi_phnum; ++i)
402	428	if (info->dlpi_phdr[i].p_type == PT_TLS) {
...	...	@@ -439,23 +465,21 @@ __attribute__((unused)) static void GetStaticTlsBoundary(uptr addr, uptr size,
439	465	*addr = ranges[l].begin;
440	466	*size = ranges[r - 1].end - ranges[l].begin;
441	467	}
442		#endif // (x86_64 \|\| i386 \|\| mips \|\| ...) && (SANITIZER_FREEBSD \|\|
443		// SANITIZER_LINUX) && !SANITIZER_ANDROID && !SANITIZER_GO
	468	# endif // (x86_64 \|\| i386 \|\| mips \|\| ...) && (SANITIZER_FREEBSD \|\|
	469	// SANITIZER_LINUX) && !SANITIZER_ANDROID && !SANITIZER_GO
444	470
445		#if SANITIZER_NETBSD
446		static struct tls_tcb * ThreadSelfTlsTcb() {
	471	# if SANITIZER_NETBSD
	472	static struct tls_tcb *ThreadSelfTlsTcb() {
447	473	struct tls_tcb *tcb = nullptr;
448		#ifdef __HAVE___LWP_GETTCB_FAST
	474	# ifdef __HAVE___LWP_GETTCB_FAST
449	475	tcb = (struct tls_tcb *)__lwp_gettcb_fast();
450		#elif defined(__HAVE___LWP_GETPRIVATE_FAST)
	476	# elif defined(__HAVE___LWP_GETPRIVATE_FAST)
451	477	tcb = (struct tls_tcb *)__lwp_getprivate_fast();
452		#endif
	478	# endif
453	479	return tcb;
454	480	}
455	481
456		uptr ThreadSelf() {
457		return (uptr)ThreadSelfTlsTcb()->tcb_pthread;
458		}
	482	uptr ThreadSelf() { return (uptr)ThreadSelfTlsTcb()->tcb_pthread; }
459	483
460	484	int GetSizeFromHdr(struct dl_phdr_info info, size_t size, void data) {
461	485	const Elf_Phdr *hdr = info->dlpi_phdr;
...	...	@@ -463,23 +487,23 @@ int GetSizeFromHdr(struct dl_phdr_info info, size_t size, void data) {
463	487
464	488	for (; hdr != last_hdr; ++hdr) {
465	489	if (hdr->p_type == PT_TLS && info->dlpi_tls_modid == 1) {
466		(uptr)data = hdr->p_memsz;
	490	(uptr )data = hdr->p_memsz;
467	491	break;
468	492	}
469	493	}
470	494	return 0;
471	495	}
472		#endif // SANITIZER_NETBSD
	496	# endif // SANITIZER_NETBSD
473	497
474		#if SANITIZER_ANDROID
	498	# if SANITIZER_ANDROID
475	499	// Bionic provides this API since S.
476	500	extern "C" SANITIZER_WEAK_ATTRIBUTE void __libc_get_static_tls_bounds(void **,
477	501	void **);
478		#endif
	502	# endif
479	503
480		#if !SANITIZER_GO
	504	# if !SANITIZER_GO
481	505	static void GetTls(uptr addr, uptr size) {
482		#if SANITIZER_ANDROID
	506	# if SANITIZER_ANDROID
483	507	if (&__libc_get_static_tls_bounds) {
484	508	void *start_addr;
485	509	void *end_addr;
...	...	@@ -491,48 +515,48 @@ static void GetTls(uptr addr, uptr size) {
491	515	*addr = 0;
492	516	*size = 0;
493	517	}
494		#elif SANITIZER_GLIBC && defined(__x86_64__)
	518	# elif SANITIZER_GLIBC && defined(__x86_64__)
495	519	// For aarch64 and x86-64, use an O(1) approach which requires relatively
496	520	// precise ThreadDescriptorSize. g_tls_size was initialized in InitTlsSize.
497		# if SANITIZER_X32
	521	# if SANITIZER_X32
498	522	asm("mov %%fs:8,%0" : "=r"(*addr));
499		# else
	523	# else
500	524	asm("mov %%fs:16,%0" : "=r"(*addr));
501		# endif
	525	# endif
502	526	*size = g_tls_size;
503	527	addr -= size;
504	528	*addr += ThreadDescriptorSize();
505		#elif SANITIZER_GLIBC && defined(__aarch64__)
	529	# elif SANITIZER_GLIBC && defined(__aarch64__)
506	530	*addr = reinterpret_cast<uptr>(__builtin_thread_pointer()) -
507	531	ThreadDescriptorSize();
508	532	*size = g_tls_size + ThreadDescriptorSize();
509		#elif SANITIZER_GLIBC && defined(__loongarch__)
510		# ifdef __clang__
	533	# elif SANITIZER_GLIBC && defined(__loongarch__)
	534	# ifdef __clang__
511	535	*addr = reinterpret_cast<uptr>(__builtin_thread_pointer()) -
512	536	ThreadDescriptorSize();
513		# else
	537	# else
514	538	asm("or %0,$tp,$zero" : "=r"(*addr));
515	539	*addr -= ThreadDescriptorSize();
516		# endif
	540	# endif
517	541	*size = g_tls_size + ThreadDescriptorSize();
518		#elif SANITIZER_GLIBC && defined(__powerpc64__)
	542	# elif SANITIZER_GLIBC && defined(__powerpc64__)
519	543	// Workaround for glibc<2.25(?). 2.27 is known to not need this.
520	544	uptr tp;
521	545	asm("addi %0,13,-0x7000" : "=r"(tp));
522	546	const uptr pre_tcb_size = TlsPreTcbSize();
523	547	*addr = tp - pre_tcb_size;
524	548	*size = g_tls_size + pre_tcb_size;
525		#elif SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_SOLARIS
	549	# elif SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_SOLARIS
526	550	uptr align;
527	551	GetStaticTlsBoundary(addr, size, &align);
528		#if defined(__x86_64__) \|\| defined(__i386__) \|\| defined(__s390__) \|\| \
529		defined(__sparc__)
	552	# if defined(__x86_64__) \|\| defined(__i386__) \|\| defined(__s390__) \|\| \
	553	defined(__sparc__)
530	554	if (SANITIZER_GLIBC) {
531		#if defined(__x86_64__) \|\| defined(__i386__)
	555	# if defined(__x86_64__) \|\| defined(__i386__)
532	556	align = Max<uptr>(align, 64);
533		#else
	557	# else
534	558	align = Max<uptr>(align, 16);
535		#endif
	559	# endif
536	560	}
537	561	const uptr tp = RoundUpTo(addr + size, align);
538	562
...	...	@@ -551,26 +575,26 @@ static void GetTls(uptr addr, uptr size) {
551	575	// because the number of bytes after pthread::specific is larger.
552	576	addr = tp - RoundUpTo(size, align);
553	577	size = tp - addr + ThreadDescriptorSize();
554		#else
	578	# else
555	579	if (SANITIZER_GLIBC)
556	580	*size += 1664;
557	581	else if (SANITIZER_FREEBSD)
558	582	*size += 128; // RTLD_STATIC_TLS_EXTRA
559		#if defined(__mips__) \|\| defined(__powerpc64__) \|\| SANITIZER_RISCV64
	583	# if defined(__mips__) \|\| defined(__powerpc64__) \|\| SANITIZER_RISCV64
560	584	const uptr pre_tcb_size = TlsPreTcbSize();
561	585	*addr -= pre_tcb_size;
562	586	*size += pre_tcb_size;
563		#else
	587	# else
564	588	// arm and aarch64 reserve two words at TP, so this underestimates the range.
565	589	// However, this is sufficient for the purpose of finding the pointers to
566	590	// thread-specific data keys.
567	591	const uptr tcb_size = ThreadDescriptorSize();
568	592	*addr -= tcb_size;
569	593	*size += tcb_size;
570		#endif
571		#endif
572		#elif SANITIZER_NETBSD
573		struct tls_tcb * const tcb = ThreadSelfTlsTcb();
	594	# endif
	595	# endif
	596	# elif SANITIZER_NETBSD
	597	struct tls_tcb *const tcb = ThreadSelfTlsTcb();
574	598	*addr = 0;
575	599	*size = 0;
576	600	if (tcb != 0) {
...	...	@@ -583,31 +607,31 @@ static void GetTls(uptr addr, uptr size) {
583	607	*addr = (uptr)tcb->tcb_dtv[1];
584	608	}
585	609	}
586		#else
587		#error "Unknown OS"
588		#endif
	610	# else
	611	# error "Unknown OS"
	612	# endif
589	613	}
590		#endif
	614	# endif
591	615
592		#if !SANITIZER_GO
	616	# if !SANITIZER_GO
593	617	uptr GetTlsSize() {
594		#if SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_NETBSD \|\| \
595		SANITIZER_SOLARIS
	618	# if SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_NETBSD \|\| \
	619	SANITIZER_SOLARIS
596	620	uptr addr, size;
597	621	GetTls(&addr, &size);
598	622	return size;
599		#else
	623	# else
600	624	return 0;
601		#endif
	625	# endif
602	626	}
603		#endif
	627	# endif
604	628
605	629	void GetThreadStackAndTls(bool main, uptr stk_addr, uptr stk_size,
606	630	uptr tls_addr, uptr tls_size) {
607		#if SANITIZER_GO
	631	# if SANITIZER_GO
608	632	// Stub implementation for Go.
609	633	stk_addr = stk_size = tls_addr = tls_size = 0;
610		#else
	634	# else
611	635	GetTls(tls_addr, tls_size);
612	636
613	637	uptr stack_top, stack_bottom;
...	...	@@ -623,16 +647,12 @@ void GetThreadStackAndTls(bool main, uptr stk_addr, uptr stk_size,
623	647	stk_size = tls_addr - *stk_addr;
624	648	}
625	649	}
626		#endif
	650	# endif
627	651	}
628	652
629		#if !SANITIZER_FREEBSD
	653	# if !SANITIZER_FREEBSD
630	654	typedef ElfW(Phdr) Elf_Phdr;
631		#elif SANITIZER_WORDSIZE == 32 && __FreeBSD_version <= 902001 // v9.2
632		#define Elf_Phdr XElf32_Phdr
633		#define dl_phdr_info xdl_phdr_info
634		#define dl_iterate_phdr(c, b) xdl_iterate_phdr((c), (b))
635		#endif // !SANITIZER_FREEBSD
	655	# endif
636	656
637	657	struct DlIteratePhdrData {
638	658	InternalMmapVectorNoCtor<LoadedModule> *modules;
...	...	@@ -652,8 +672,7 @@ static int AddModuleSegments(const char module_name, dl_phdr_info info,
652	672	uptr cur_end = cur_beg + phdr->p_memsz;
653	673	bool executable = phdr->p_flags & PF_X;
654	674	bool writable = phdr->p_flags & PF_W;
655		cur_module.addAddressRange(cur_beg, cur_end, executable,
656		writable);
	675	cur_module.addAddressRange(cur_beg, cur_end, executable, writable);
657	676	} else if (phdr->p_type == PT_NOTE) {
658	677	# ifdef NT_GNU_BUILD_ID
659	678	uptr off = 0;
...	...	@@ -698,33 +717,30 @@ static int dl_iterate_phdr_cb(dl_phdr_info info, size_t size, void arg) {
698	717	return AddModuleSegments(module_name.data(), info, data->modules);
699	718	}
700	719
701		if (info->dlpi_name) {
702		InternalScopedString module_name;
703		module_name.append("%s", info->dlpi_name);
704		return AddModuleSegments(module_name.data(), info, data->modules);
705		}
	720	if (info->dlpi_name)
	721	return AddModuleSegments(info->dlpi_name, info, data->modules);
706	722
707	723	return 0;
708	724	}
709	725
710		#if SANITIZER_ANDROID && __ANDROID_API__ < 21
	726	# if SANITIZER_ANDROID && __ANDROID_API__ < 21
711	727	extern "C" __attribute__((weak)) int dl_iterate_phdr(
712	728	int ()(struct dl_phdr_info , size_t, void ), void );
713		#endif
	729	# endif
714	730
715	731	static bool requiresProcmaps() {
716		#if SANITIZER_ANDROID && __ANDROID_API__ <= 22
	732	# if SANITIZER_ANDROID && __ANDROID_API__ <= 22
717	733	// Fall back to /proc/maps if dl_iterate_phdr is unavailable or broken.
718	734	// The runtime check allows the same library to work with
719	735	// both K and L (and future) Android releases.
720	736	return AndroidGetApiLevel() <= ANDROID_LOLLIPOP_MR1;
721		#else
	737	# else
722	738	return false;
723		#endif
	739	# endif
724	740	}
725	741
726	742	static void procmapsInit(InternalMmapVectorNoCtor<LoadedModule> *modules) {
727		MemoryMappingLayout memory_mapping(/cache_enabled/true);
	743	MemoryMappingLayout memory_mapping(/cache_enabled/ true);
728	744	memory_mapping.DumpListOfModules(modules);
729	745	}
730	746
...	...	@@ -776,22 +792,19 @@ uptr GetRSS() {
776	792	// We need the second number which is RSS in pages.
777	793	char *pos = buf;
778	794	// Skip the first number.
779		while (pos >= '0' && pos <= '9')
780		pos++;
	795	while (pos >= '0' && pos <= '9') pos++;
781	796	// Skip whitespaces.
782		while (!(pos >= '0' && pos <= '9') && *pos != 0)
783		pos++;
	797	while (!(pos >= '0' && pos <= '9') && *pos != 0) pos++;
784	798	// Read the number.
785	799	uptr rss = 0;
786		while (pos >= '0' && pos <= '9')
787		rss = rss * 10 + *pos++ - '0';
	800	while (pos >= '0' && pos <= '9') rss = rss * 10 + *pos++ - '0';
788	801	return rss * GetPageSizeCached();
789	802	}
790	803
791	804	// sysconf(_SC_NPROCESSORS_{CONF,ONLN}) cannot be used on most platforms as
792	805	// they allocate memory.
793	806	u32 GetNumberOfCPUs() {
794		#if SANITIZER_FREEBSD \|\| SANITIZER_NETBSD
	807	# if SANITIZER_FREEBSD \|\| SANITIZER_NETBSD
795	808	u32 ncpu;
796	809	int req[2];
797	810	uptr len = sizeof(ncpu);
...	...	@@ -799,7 +812,7 @@ u32 GetNumberOfCPUs() {
799	812	req[1] = HW_NCPU;
800	813	CHECK_EQ(internal_sysctl(req, 2, &ncpu, &len, NULL, 0), 0);
801	814	return ncpu;
802		#elif SANITIZER_ANDROID && !defined(CPU_COUNT) && !defined(__aarch64__)
	815	# elif SANITIZER_ANDROID && !defined(CPU_COUNT) && !defined(__aarch64__)
803	816	// Fall back to /sys/devices/system/cpu on Android when cpu_set_t doesn't
804	817	// exist in sched.h. That is the case for toolchains generated with older
805	818	// NDKs.
...	...	@@ -827,26 +840,26 @@ u32 GetNumberOfCPUs() {
827	840	break;
828	841	if (entry->d_ino != 0 && *d_type == DT_DIR) {
829	842	if (entry->d_name[0] == 'c' && entry->d_name[1] == 'p' &&
830		entry->d_name[2] == 'u' &&
831		entry->d_name[3] >= '0' && entry->d_name[3] <= '9')
	843	entry->d_name[2] == 'u' && entry->d_name[3] >= '0' &&
	844	entry->d_name[3] <= '9')
832	845	n_cpus++;
833	846	}
834	847	entry = (struct linux_dirent )(((u8 )entry) + entry->d_reclen);
835	848	}
836	849	internal_close(fd);
837	850	return n_cpus;
838		#elif SANITIZER_SOLARIS
	851	# elif SANITIZER_SOLARIS
839	852	return sysconf(_SC_NPROCESSORS_ONLN);
840		#else
	853	# else
841	854	cpu_set_t CPUs;
842	855	CHECK_EQ(sched_getaffinity(0, sizeof(cpu_set_t), &CPUs), 0);
843	856	return CPU_COUNT(&CPUs);
844		#endif
	857	# endif
845	858	}
846	859
847		#if SANITIZER_LINUX
	860	# if SANITIZER_LINUX
848	861
849		#if SANITIZER_ANDROID
	862	# if SANITIZER_ANDROID
850	863	static atomic_uint8_t android_log_initialized;
851	864
852	865	void AndroidLogInit() {
...	...	@@ -858,13 +871,15 @@ static bool ShouldLogAfterPrintf() {
858	871	return atomic_load(&android_log_initialized, memory_order_acquire);
859	872	}
860	873
861		extern "C" SANITIZER_WEAK_ATTRIBUTE
862		int async_safe_write_log(int pri, const char* tag, const char* msg);
863		extern "C" SANITIZER_WEAK_ATTRIBUTE
864		int __android_log_write(int prio, const char* tag, const char* msg);
	874	extern "C" SANITIZER_WEAK_ATTRIBUTE int async_safe_write_log(int pri,
	875	const char *tag,
	876	const char *msg);
	877	extern "C" SANITIZER_WEAK_ATTRIBUTE int __android_log_write(int prio,
	878	const char *tag,
	879	const char *msg);
865	880
866	881	// ANDROID_LOG_INFO is 4, but can't be resolved at runtime.
867		#define SANITIZER_ANDROID_LOG_INFO 4
	882	# define SANITIZER_ANDROID_LOG_INFO 4
868	883
869	884	// async_safe_write_log is a new public version of __libc_write_log that is
870	885	// used behind syslog. It is preferable to syslog as it will not do any dynamic
...	...	@@ -883,14 +898,14 @@ void WriteOneLineToSyslog(const char *s) {
883	898	}
884	899	}
885	900
886		extern "C" SANITIZER_WEAK_ATTRIBUTE
887		void android_set_abort_message(const char *);
	901	extern "C" SANITIZER_WEAK_ATTRIBUTE void android_set_abort_message(
	902	const char *);
888	903
889	904	void SetAbortMessage(const char *str) {
890	905	if (&android_set_abort_message)
891	906	android_set_abort_message(str);
892	907	}
893		#else
	908	# else
894	909	void AndroidLogInit() {}
895	910
896	911	static bool ShouldLogAfterPrintf() { return true; }
...	...	@@ -898,16 +913,16 @@ static bool ShouldLogAfterPrintf() { return true; }
898	913	void WriteOneLineToSyslog(const char *s) { syslog(LOG_INFO, "%s", s); }
899	914
900	915	void SetAbortMessage(const char *str) {}
901		#endif // SANITIZER_ANDROID
	916	# endif // SANITIZER_ANDROID
902	917
903	918	void LogMessageOnPrintf(const char *str) {
904	919	if (common_flags()->log_to_syslog && ShouldLogAfterPrintf())
905	920	WriteToSyslog(str);
906	921	}
907	922
908		#endif // SANITIZER_LINUX
	923	# endif // SANITIZER_LINUX
909	924
910		#if SANITIZER_GLIBC && !SANITIZER_GO
	925	# if SANITIZER_GLIBC && !SANITIZER_GO
911	926	// glibc crashes when using clock_gettime from a preinit_array function as the
912	927	// vDSO function pointers haven't been initialized yet. __progname is
913	928	// initialized after the vDSO function pointers, so if it exists, is not null
...	...	@@ -918,8 +933,8 @@ inline bool CanUseVDSO() { return &__progname && __progname && *__progname; }
918	933	// MonotonicNanoTime is a timing function that can leverage the vDSO by calling
919	934	// clock_gettime. real_clock_gettime only exists if clock_gettime is
920	935	// intercepted, so define it weakly and use it if available.
921		extern "C" SANITIZER_WEAK_ATTRIBUTE
922		int real_clock_gettime(u32 clk_id, void *tp);
	936	extern "C" SANITIZER_WEAK_ATTRIBUTE int real_clock_gettime(u32 clk_id,
	937	void *tp);
923	938	u64 MonotonicNanoTime() {
924	939	timespec ts;
925	940	if (CanUseVDSO()) {
...	...	@@ -932,19 +947,26 @@ u64 MonotonicNanoTime() {
932	947	}
933	948	return (u64)ts.tv_sec * (1000ULL * 1000 * 1000) + ts.tv_nsec;
934	949	}
935		#else
	950	# else
936	951	// Non-glibc & Go always use the regular function.
937	952	u64 MonotonicNanoTime() {
938	953	timespec ts;
939	954	clock_gettime(CLOCK_MONOTONIC, &ts);
940	955	return (u64)ts.tv_sec * (1000ULL * 1000 * 1000) + ts.tv_nsec;
941	956	}
942		#endif // SANITIZER_GLIBC && !SANITIZER_GO
	957	# endif // SANITIZER_GLIBC && !SANITIZER_GO
943	958
944	959	void ReExec() {
945	960	const char *pathname = "/proc/self/exe";
946	961
947		#if SANITIZER_NETBSD
	962	# if SANITIZER_FREEBSD
	963	for (const auto *aux = __elf_aux_vector; aux->a_type != AT_NULL; aux++) {
	964	if (aux->a_type == AT_EXECPATH) {
	965	pathname = static_cast<const char *>(aux->a_un.a_ptr);
	966	break;
	967	}
	968	}
	969	# elif SANITIZER_NETBSD
948	970	static const int name[] = {
949	971	CTL_KERN,
950	972	KERN_PROC_ARGS,
...	...	@@ -957,14 +979,14 @@ void ReExec() {
957	979	len = sizeof(path);
958	980	if (internal_sysctl(name, ARRAY_SIZE(name), path, &len, NULL, 0) != -1)
959	981	pathname = path;
960		#elif SANITIZER_SOLARIS
	982	# elif SANITIZER_SOLARIS
961	983	pathname = getexecname();
962	984	CHECK_NE(pathname, NULL);
963		#elif SANITIZER_USE_GETAUXVAL
	985	# elif SANITIZER_USE_GETAUXVAL
964	986	// Calling execve with /proc/self/exe sets that as $EXEC_ORIGIN. Binaries that
965	987	// rely on that will fail to load shared libraries. Query AT_EXECFN instead.
966	988	pathname = reinterpret_cast<const char *>(getauxval(AT_EXECFN));
967		#endif
	989	# endif
968	990
969	991	uptr rv = internal_execve(pathname, GetArgv(), GetEnviron());
970	992	int rverrno;
...	...	@@ -986,9 +1008,8 @@ void UnmapFromTo(uptr from, uptr to) {
986	1008	}
987	1009
988	1010	uptr MapDynamicShadow(uptr shadow_size_bytes, uptr shadow_scale,
989		uptr min_shadow_base_alignment,
990		UNUSED uptr &high_mem_end) {
991		const uptr granularity = GetMmapGranularity();
	1011	uptr min_shadow_base_alignment, UNUSED uptr &high_mem_end,
	1012	uptr granularity) {
992	1013	const uptr alignment =
993	1014	Max<uptr>(granularity << shadow_scale, 1ULL << min_shadow_base_alignment);
994	1015	const uptr left_padding =
...	...	@@ -1016,14 +1037,14 @@ static uptr MmapSharedNoReserve(uptr addr, uptr size) {
1016	1037
1017	1038	static uptr MremapCreateAlias(uptr base_addr, uptr alias_addr,
1018	1039	uptr alias_size) {
1019		#if SANITIZER_LINUX
	1040	# if SANITIZER_LINUX
1020	1041	return internal_mremap(reinterpret_cast<void *>(base_addr), 0, alias_size,
1021	1042	MREMAP_MAYMOVE \| MREMAP_FIXED,
1022	1043	reinterpret_cast<void *>(alias_addr));
1023		#else
	1044	# else
1024	1045	CHECK(false && "mremap is not supported outside of Linux");
1025	1046	return 0;
1026		#endif
	1047	# endif
1027	1048	}
1028	1049
1029	1050	static void CreateAliases(uptr start_addr, uptr alias_size, uptr num_aliases) {
...	...	@@ -1068,12 +1089,12 @@ uptr MapDynamicShadowAndAliases(uptr shadow_size, uptr alias_size,
1068	1089	}
1069	1090
1070	1091	void InitializePlatformCommonFlags(CommonFlags *cf) {
1071		#if SANITIZER_ANDROID
	1092	# if SANITIZER_ANDROID
1072	1093	if (&__libc_get_static_tls_bounds == nullptr)
1073	1094	cf->detect_leaks = false;
1074		#endif
	1095	# endif
1075	1096	}
1076	1097
1077		} // namespace __sanitizer
	1098	} // namespace __sanitizer
1078	1099
1079	1100	#endif

lib/tsan/sanitizer_common/sanitizer_linux_s390.cpp+79-85

...	...	@@ -15,14 +15,14 @@
15	15
16	16	#if SANITIZER_LINUX && SANITIZER_S390
17	17
18		#include <dlfcn.h>
19		#include <errno.h>
20		#include <sys/syscall.h>
21		#include <sys/utsname.h>
22		#include <unistd.h>
	18	# include <dlfcn.h>
	19	# include <errno.h>
	20	# include <sys/syscall.h>
	21	# include <sys/utsname.h>
	22	# include <unistd.h>
23	23
24		#include "sanitizer_libc.h"
25		#include "sanitizer_linux.h"
	24	# include "sanitizer_libc.h"
	25	# include "sanitizer_linux.h"
26	26
27	27	namespace __sanitizer {
28	28
...	...	@@ -37,22 +37,19 @@ uptr internal_mmap(void *addr, uptr length, int prot, int flags, int fd,
37	37	unsigned long fd;
38	38	unsigned long offset;
39	39	} params = {
40		(unsigned long)addr,
41		(unsigned long)length,
42		(unsigned long)prot,
43		(unsigned long)flags,
44		(unsigned long)fd,
45		# ifdef __s390x__
46		(unsigned long)offset,
47		# else
	40	(unsigned long)addr, (unsigned long)length, (unsigned long)prot,
	41	(unsigned long)flags, (unsigned long)fd,
	42	# ifdef __s390x__
	43	(unsigned long)offset,
	44	# else
48	45	(unsigned long)(offset / 4096),
49		# endif
	46	# endif
50	47	};
51		# ifdef __s390x__
	48	# ifdef __s390x__
52	49	return syscall(__NR_mmap, &params);
53		# else
	50	# else
54	51	return syscall(__NR_mmap2, &params);
55		# endif
	52	# endif
56	53	}
57	54
58	55	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
...	...	@@ -63,58 +60,54 @@ uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
63	60	}
64	61	CHECK_EQ(0, (uptr)child_stack % 16);
65	62	// Minimum frame size.
66		#ifdef __s390x__
	63	# ifdef __s390x__
67	64	child_stack = (char *)child_stack - 160;
68		#else
	65	# else
69	66	child_stack = (char *)child_stack - 96;
70		#endif
	67	# endif
71	68	// Terminate unwind chain.
72	69	((unsigned long *)child_stack)[0] = 0;
73	70	// And pass parameters.
74	71	((unsigned long *)child_stack)[1] = (uptr)fn;
75	72	((unsigned long *)child_stack)[2] = (uptr)arg;
76	73	register uptr res __asm__("r2");
77		register void *__cstack __asm__("r2") = child_stack;
78		register long __flags __asm__("r3") = flags;
79		register int * __ptidptr __asm__("r4") = parent_tidptr;
80		register int * __ctidptr __asm__("r5") = child_tidptr;
81		register void * __newtls __asm__("r6") = newtls;
	74	register void *__cstack __asm__("r2") = child_stack;
	75	register long __flags __asm__("r3") = flags;
	76	register int *__ptidptr __asm__("r4") = parent_tidptr;
	77	register int *__ctidptr __asm__("r5") = child_tidptr;
	78	register void *__newtls __asm__("r6") = newtls;
82	79
83	80	__asm__ __volatile__(
84		/* Clone. */
85		"svc %1\n"
86
87		/* if (%r2 != 0)
88		* return;
89		*/
90		#ifdef __s390x__
91		"cghi %%r2, 0\n"
92		#else
93		"chi %%r2, 0\n"
94		#endif
95		"jne 1f\n"
96
97		/* Call "fn(arg)". */
98		#ifdef __s390x__
99		"lmg %%r1, %%r2, 8(%%r15)\n"
100		#else
101		"lm %%r1, %%r2, 4(%%r15)\n"
102		#endif
103		"basr %%r14, %%r1\n"
104
105		/* Call _exit(%r2). */
106		"svc %2\n"
107
108		/* Return to parent. */
109		"1:\n"
110		: "=r" (res)
111		: "i"(__NR_clone), "i"(__NR_exit),
112		"r"(__cstack),
113		"r"(__flags),
114		"r"(__ptidptr),
115		"r"(__ctidptr),
116		"r"(__newtls)
117		: "memory", "cc");
	81	/* Clone. */
	82	"svc %1\n"
	83
	84	/* if (%r2 != 0)
	85	* return;
	86	*/
	87	# ifdef __s390x__
	88	"cghi %%r2, 0\n"
	89	# else
	90	"chi %%r2, 0\n"
	91	# endif
	92	"jne 1f\n"
	93
	94	/* Call "fn(arg)". */
	95	# ifdef __s390x__
	96	"lmg %%r1, %%r2, 8(%%r15)\n"
	97	# else
	98	"lm %%r1, %%r2, 4(%%r15)\n"
	99	# endif
	100	"basr %%r14, %%r1\n"
	101
	102	/* Call _exit(%r2). */
	103	"svc %2\n"
	104
	105	/* Return to parent. */
	106	"1:\n"
	107	: "=r"(res)
	108	: "i"(__NR_clone), "i"(__NR_exit), "r"(__cstack), "r"(__flags),
	109	"r"(__ptidptr), "r"(__ctidptr), "r"(__newtls)
	110	: "memory", "cc");
118	111	if (res >= (uptr)-4095) {
119	112	errno = -res;
120	113	return -1;
...	...	@@ -122,7 +115,7 @@ uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
122	115	return res;
123	116	}
124	117
125		#if SANITIZER_S390_64
	118	# if SANITIZER_S390_64
126	119	static bool FixedCVE_2016_2143() {
127	120	// Try to determine if the running kernel has a fix for CVE-2016-2143,
128	121	// return false if in doubt (better safe than sorry). Distros may want to
...	...	@@ -137,20 +130,20 @@ static bool FixedCVE_2016_2143() {
137	130	// At least first 2 should be matched.
138	131	if (ptr[0] != '.')
139	132	return false;
140		minor = internal_simple_strtoll(ptr+1, &ptr, 10);
	133	minor = internal_simple_strtoll(ptr + 1, &ptr, 10);
141	134	// Third is optional.
142	135	if (ptr[0] == '.')
143		patch = internal_simple_strtoll(ptr+1, &ptr, 10);
	136	patch = internal_simple_strtoll(ptr + 1, &ptr, 10);
144	137	if (major < 3) {
145	138	if (major == 2 && minor == 6 && patch == 32 && ptr[0] == '-' &&
146	139	internal_strstr(ptr, ".el6")) {
147	140	// Check RHEL6
148		int r1 = internal_simple_strtoll(ptr+1, &ptr, 10);
149		if (r1 >= 657) // 2.6.32-657.el6 or later
	141	int r1 = internal_simple_strtoll(ptr + 1, &ptr, 10);
	142	if (r1 >= 657) // 2.6.32-657.el6 or later
150	143	return true;
151	144	if (r1 == 642 && ptr[0] == '.') {
152		int r2 = internal_simple_strtoll(ptr+1, &ptr, 10);
153		if (r2 >= 9) // 2.6.32-642.9.1.el6 or later
	145	int r2 = internal_simple_strtoll(ptr + 1, &ptr, 10);
	146	if (r2 >= 9) // 2.6.32-642.9.1.el6 or later
154	147	return true;
155	148	}
156	149	}
...	...	@@ -166,12 +159,12 @@ static bool FixedCVE_2016_2143() {
166	159	if (minor == 10 && patch == 0 && ptr[0] == '-' &&
167	160	internal_strstr(ptr, ".el7")) {
168	161	// Check RHEL7
169		int r1 = internal_simple_strtoll(ptr+1, &ptr, 10);
170		if (r1 >= 426) // 3.10.0-426.el7 or later
	162	int r1 = internal_simple_strtoll(ptr + 1, &ptr, 10);
	163	if (r1 >= 426) // 3.10.0-426.el7 or later
171	164	return true;
172	165	if (r1 == 327 && ptr[0] == '.') {
173		int r2 = internal_simple_strtoll(ptr+1, &ptr, 10);
174		if (r2 >= 27) // 3.10.0-327.27.1.el7 or later
	166	int r2 = internal_simple_strtoll(ptr + 1, &ptr, 10);
	167	if (r2 >= 27) // 3.10.0-327.27.1.el7 or later
175	168	return true;
176	169	}
177	170	}
...	...	@@ -187,8 +180,8 @@ static bool FixedCVE_2016_2143() {
187	180	if (minor == 4 && patch == 0 && ptr[0] == '-' &&
188	181	internal_strstr(buf.version, "Ubuntu")) {
189	182	// Check Ubuntu 16.04
190		int r1 = internal_simple_strtoll(ptr+1, &ptr, 10);
191		if (r1 >= 13) // 4.4.0-13 or later
	183	int r1 = internal_simple_strtoll(ptr + 1, &ptr, 10);
	184	if (r1 >= 13) // 4.4.0-13 or later
192	185	return true;
193	186	}
194	187	// Otherwise, OK if 4.5+.
...	...	@@ -211,18 +204,19 @@ void AvoidCVE_2016_2143() {
211	204	if (GetEnv("SANITIZER_IGNORE_CVE_2016_2143"))
212	205	return;
213	206	Report(
214		"ERROR: Your kernel seems to be vulnerable to CVE-2016-2143. Using ASan,\n"
215		"MSan, TSan, DFSan or LSan with such kernel can and will crash your\n"
216		"machine, or worse.\n"
217		"\n"
218		"If you are certain your kernel is not vulnerable (you have compiled it\n"
219		"yourself, or are using an unrecognized distribution kernel), you can\n"
220		"override this safety check by exporting SANITIZER_IGNORE_CVE_2016_2143\n"
221		"with any value.\n");
	207	"ERROR: Your kernel seems to be vulnerable to CVE-2016-2143. Using "
	208	"ASan,\n"
	209	"MSan, TSan, DFSan or LSan with such kernel can and will crash your\n"
	210	"machine, or worse.\n"
	211	"\n"
	212	"If you are certain your kernel is not vulnerable (you have compiled it\n"
	213	"yourself, or are using an unrecognized distribution kernel), you can\n"
	214	"override this safety check by exporting SANITIZER_IGNORE_CVE_2016_2143\n"
	215	"with any value.\n");
222	216	Die();
223	217	}
224		#endif
	218	# endif
225	219
226		} // namespace __sanitizer
	220	} // namespace __sanitizer
227	221
228		#endif // SANITIZER_LINUX && SANITIZER_S390
	222	#endif // SANITIZER_LINUX && SANITIZER_S390

lib/tsan/sanitizer_common/sanitizer_mac.cpp+4-4

...	...	@@ -1188,8 +1188,8 @@ uptr GetMaxVirtualAddress() {
1188	1188	}
1189	1189
1190	1190	uptr MapDynamicShadow(uptr shadow_size_bytes, uptr shadow_scale,
1191		uptr min_shadow_base_alignment, uptr &high_mem_end) {
1192		const uptr granularity = GetMmapGranularity();
	1191	uptr min_shadow_base_alignment, uptr &high_mem_end,
	1192	uptr granularity) {
1193	1193	const uptr alignment =
1194	1194	Max<uptr>(granularity << shadow_scale, 1ULL << min_shadow_base_alignment);
1195	1195	const uptr left_padding =
...	...	@@ -1372,8 +1372,8 @@ void DumpProcessMap() {
1372	1372	for (uptr i = 0; i < modules.size(); ++i) {
1373	1373	char uuid_str[128];
1374	1374	FormatUUID(uuid_str, sizeof(uuid_str), modules[i].uuid());
1375		Printf("0x%zx-0x%zx %s (%s) %s\n", modules[i].base_address(),
1376		modules[i].max_address(), modules[i].full_name(),
	1375	Printf("%p-%p %s (%s) %s\n", (void *)modules[i].base_address(),
	1376	(void *)modules[i].max_address(), modules[i].full_name(),
1377	1377	ModuleArchToString(modules[i].arch()), uuid_str);
1378	1378	}
1379	1379	Printf("End of module map.\n");

lib/tsan/sanitizer_common/sanitizer_mallinfo.h+4

...	...	@@ -31,6 +31,10 @@ struct __sanitizer_struct_mallinfo {
31	31	int v[10];
32	32	};
33	33
	34	struct __sanitizer_struct_mallinfo2 {
	35	uptr v[10];
	36	};
	37
34	38	#endif
35	39
36	40	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_malloc_mac.inc+1-1

...	...	@@ -123,7 +123,7 @@ INTERCEPTOR(void, malloc_set_zone_name, malloc_zone_t zone, const char name) {
123	123	COMMON_MALLOC_ENTER();
124	124	InternalScopedString new_name;
125	125	if (name && zone->introspect == sanitizer_zone.introspect) {
126		new_name.append(COMMON_MALLOC_ZONE_NAME "-%s", name);
	126	new_name.AppendF(COMMON_MALLOC_ZONE_NAME "-%s", name);
127	127	name = new_name.data();
128	128	}
129	129

lib/tsan/sanitizer_common/sanitizer_mutex.cpp+4-2

...	...	@@ -212,8 +212,10 @@ struct InternalDeadlockDetector {
212	212	return initialized > 0;
213	213	}
214	214	};
215
216		static THREADLOCAL InternalDeadlockDetector deadlock_detector;
	215	// This variable is used by the __tls_get_addr interceptor, so cannot use the
	216	// global-dynamic TLS model, as that would result in crashes.
	217	__attribute__((tls_model("initial-exec"))) static THREADLOCAL
	218	InternalDeadlockDetector deadlock_detector;
217	219
218	220	void CheckedMutex::LockImpl(uptr pc) { deadlock_detector.Lock(type_, pc); }
219	221

lib/tsan/sanitizer_common/sanitizer_placement_new.h+1-3

...	...	@@ -17,8 +17,6 @@
17	17
18	18	#include "sanitizer_internal_defs.h"
19	19
20		inline void operator new(__sanitizer::operator_new_size_type sz, void p) {
21		return p;
22		}
	20	inline void operator new(__sanitizer::usize sz, void p) { return p; }
23	21
24	22	#endif // SANITIZER_PLACEMENT_NEW_H

lib/tsan/sanitizer_common/sanitizer_platform.h+22-2

...	...	@@ -260,6 +260,17 @@
260	260	# define SANITIZER_ARM64 0
261	261	#endif
262	262
	263	#if SANITIZER_WINDOWS64 && SANITIZER_ARM64
	264	# define SANITIZER_WINDOWS_ARM64 1
	265	# define SANITIZER_WINDOWS_x64 0
	266	#elif SANITIZER_WINDOWS64 && !SANITIZER_ARM64
	267	# define SANITIZER_WINDOWS_ARM64 0
	268	# define SANITIZER_WINDOWS_x64 1
	269	#else
	270	# define SANITIZER_WINDOWS_ARM64 0
	271	# define SANITIZER_WINDOWS_x64 0
	272	#endif
	273
263	274	#if SANITIZER_SOLARIS && SANITIZER_WORDSIZE == 32
264	275	# define SANITIZER_SOLARIS32 1
265	276	#else
...	...	@@ -284,7 +295,8 @@
284	295	// For such platforms build this code with -DSANITIZER_CAN_USE_ALLOCATOR64=0 or
285	296	// change the definition of SANITIZER_CAN_USE_ALLOCATOR64 here.
286	297	#ifndef SANITIZER_CAN_USE_ALLOCATOR64
287		# if SANITIZER_RISCV64 \|\| SANITIZER_IOS
	298	# if (SANITIZER_RISCV64 && !SANITIZER_FUCHSIA && !SANITIZER_LINUX) \|\| \
	299	SANITIZER_IOS \|\| SANITIZER_DRIVERKIT
288	300	# define SANITIZER_CAN_USE_ALLOCATOR64 0
289	301	# elif defined(__mips64) \|\| defined(__hexagon__)
290	302	# define SANITIZER_CAN_USE_ALLOCATOR64 0
...	...	@@ -303,7 +315,15 @@
303	315	# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 40)
304	316	# endif
305	317	#elif SANITIZER_RISCV64
306		# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 38)
	318	// FIXME: Rather than hardcoding the VMA here, we should rely on
	319	// GetMaxUserVirtualAddress(). This will require some refactoring though since
	320	// many places either hardcode some value or SANITIZER_MMAP_RANGE_SIZE is
	321	// assumed to be some constant integer.
	322	# if SANITIZER_FUCHSIA
	323	# define SANITIZER_MMAP_RANGE_SIZE (1ULL << 38)
	324	# else
	325	# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 56)
	326	# endif
307	327	#elif defined(__aarch64__)
308	328	# if SANITIZER_APPLE
309	329	# if SANITIZER_OSX \|\| SANITIZER_IOSSIM

lib/tsan/sanitizer_common/sanitizer_platform_interceptors.h+10-6

...	...	@@ -191,7 +191,8 @@
191	191
192	192	#define SANITIZER_INTERCEPT_PREADV \
193	193	(SI_FREEBSD \|\| SI_NETBSD \|\| SI_LINUX_NOT_ANDROID)
194		#define SANITIZER_INTERCEPT_PWRITEV SI_LINUX_NOT_ANDROID
	194	#define SANITIZER_INTERCEPT_PWRITEV \
	195	(SI_FREEBSD \|\| SI_NETBSD \|\| SI_LINUX_NOT_ANDROID)
195	196	#define SANITIZER_INTERCEPT_PREADV64 SI_GLIBC
196	197	#define SANITIZER_INTERCEPT_PWRITEV64 SI_GLIBC
197	198
...	...	@@ -301,7 +302,8 @@
301	302	#define SANITIZER_INTERCEPT_CANONICALIZE_FILE_NAME (SI_GLIBC \|\| SI_SOLARIS)
302	303	#define SANITIZER_INTERCEPT_CONFSTR \
303	304	(SI_FREEBSD \|\| SI_NETBSD \|\| SI_MAC \|\| SI_LINUX_NOT_ANDROID \|\| SI_SOLARIS)
304		#define SANITIZER_INTERCEPT_SCHED_GETAFFINITY SI_LINUX_NOT_ANDROID
	305	#define SANITIZER_INTERCEPT_SCHED_GETAFFINITY \
	306	(SI_LINUX_NOT_ANDROID \|\| SI_FREEBSD)
305	307	#define SANITIZER_INTERCEPT_SCHED_GETPARAM SI_LINUX_NOT_ANDROID \|\| SI_SOLARIS
306	308	#define SANITIZER_INTERCEPT_STRERROR SI_POSIX
307	309	#define SANITIZER_INTERCEPT_STRERROR_R SI_POSIX
...	...	@@ -462,7 +464,7 @@
462	464	(SI_LINUX \|\| SI_MAC \|\| SI_WINDOWS \|\| SI_FREEBSD \|\| SI_NETBSD \|\| SI_SOLARIS)
463	465	#define SANITIZER_INTERCEPT_RECV_RECVFROM SI_POSIX
464	466	#define SANITIZER_INTERCEPT_SEND_SENDTO SI_POSIX
465		#define SANITIZER_INTERCEPT_EVENTFD_READ_WRITE SI_LINUX
	467	#define SANITIZER_INTERCEPT_EVENTFD_READ_WRITE (SI_LINUX \|\| SI_FREEBSD)
466	468
467	469	#define SI_STAT_LINUX (SI_LINUX && __GLIBC_PREREQ(2, 33))
468	470	#define SANITIZER_INTERCEPT_STAT \
...	...	@@ -575,12 +577,12 @@
575	577	#define SANITIZER_INTERCEPT_SL_INIT (SI_FREEBSD \|\| SI_NETBSD)
576	578
577	579	#define SANITIZER_INTERCEPT_GETRANDOM \
578		((SI_LINUX && __GLIBC_PREREQ(2, 25)) \|\| SI_FREEBSD)
	580	((SI_LINUX && __GLIBC_PREREQ(2, 25)) \|\| SI_FREEBSD \|\| SI_SOLARIS)
579	581	#define SANITIZER_INTERCEPT___CXA_ATEXIT SI_NETBSD
580	582	#define SANITIZER_INTERCEPT_ATEXIT SI_NETBSD
581	583	#define SANITIZER_INTERCEPT_PTHREAD_ATFORK SI_NETBSD
582	584	#define SANITIZER_INTERCEPT_GETENTROPY \
583		((SI_LINUX && __GLIBC_PREREQ(2, 25)) \|\| SI_FREEBSD)
	585	((SI_LINUX && __GLIBC_PREREQ(2, 25)) \|\| SI_FREEBSD \|\| SI_SOLARIS)
584	586	#define SANITIZER_INTERCEPT_QSORT \
585	587	(SI_POSIX && !SI_IOSSIM && !SI_WATCHOS && !SI_TVOS && !SI_ANDROID)
586	588	#define SANITIZER_INTERCEPT_QSORT_R SI_GLIBC
...	...	@@ -594,9 +596,11 @@
594	596	#define SANITIZER_INTERCEPT___XUNAME SI_FREEBSD
595	597	#define SANITIZER_INTERCEPT_FLOPEN SI_FREEBSD
596	598	#define SANITIZER_INTERCEPT_PROCCTL SI_FREEBSD
597		#define SANITIZER_INTERCEPT_HEXDUMP SI_FREEBSD
598	599	#define SANITIZER_INTERCEPT_ARGP_PARSE SI_GLIBC
599	600	#define SANITIZER_INTERCEPT_CPUSET_GETAFFINITY SI_FREEBSD
	601	// FIXME: also available from musl 1.2.5
	602	#define SANITIZER_INTERCEPT_PREADV2 (SI_LINUX && __GLIBC_PREREQ(2, 26))
	603	#define SANITIZER_INTERCEPT_PWRITEV2 (SI_LINUX && __GLIBC_PREREQ(2, 26))
600	604
601	605	// This macro gives a way for downstream users to override the above
602	606	// interceptor macros irrespective of the platform they are on. They have

lib/tsan/sanitizer_common/sanitizer_platform_limits_freebsd.cpp+2

...	...	@@ -475,6 +475,8 @@ CHECK_TYPE_SIZE(nfds_t);
475	475	CHECK_TYPE_SIZE(sigset_t);
476	476
477	477	COMPILER_CHECK(sizeof(__sanitizer_sigaction) == sizeof(struct sigaction));
	478	COMPILER_CHECK(sizeof(__sanitizer_siginfo) == sizeof(siginfo_t));
	479	CHECK_SIZE_AND_OFFSET(siginfo_t, si_value);
478	480	// Can't write checks for sa_handler and sa_sigaction due to them being
479	481	// preprocessor macros.
480	482	CHECK_STRUCT_SIZE_AND_OFFSET(sigaction, sa_mask);

lib/tsan/sanitizer_common/sanitizer_platform_limits_freebsd.h+22-2

...	...	@@ -301,11 +301,29 @@ struct __sanitizer_sigset_t {
301	301
302	302	typedef __sanitizer_sigset_t __sanitizer_kernel_sigset_t;
303	303
	304	union __sanitizer_sigval {
	305	int sival_int;
	306	void *sival_ptr;
	307	};
	308
304	309	struct __sanitizer_siginfo {
305		// The size is determined by looking at sizeof of real siginfo_t on linux.
306		u64 opaque[128 / sizeof(u64)];
	310	int si_signo;
	311	int si_errno;
	312	int si_code;
	313	pid_t si_pid;
	314	u32 si_uid;
	315	int si_status;
	316	void *si_addr;
	317	union __sanitizer_sigval si_value;
	318	# if SANITIZER_WORDSIZE == 64
	319	char data[40];
	320	# else
	321	char data[32];
	322	# endif
307	323	};
308	324
	325	typedef __sanitizer_siginfo __sanitizer_siginfo_t;
	326
309	327	using __sanitizer_sighandler_ptr = void (*)(int sig);
310	328	using __sanitizer_sigactionhandler_ptr = void (*)(int sig,
311	329	__sanitizer_siginfo *siginfo,
...	...	@@ -726,6 +744,8 @@ struct __sanitizer_cpuset {
726	744
727	745	typedef struct __sanitizer_cpuset __sanitizer_cpuset_t;
728	746	extern unsigned struct_cpuset_sz;
	747
	748	typedef unsigned long long __sanitizer_eventfd_t;
729	749	} // namespace __sanitizer
730	750
731	751	# define CHECK_TYPE_SIZE(TYPE) \

lib/tsan/sanitizer_common/sanitizer_platform_limits_openbsd.cpp deleted

lib/tsan/sanitizer_common/sanitizer_platform_limits_openbsd.h deleted

lib/tsan/sanitizer_common/sanitizer_platform_limits_posix.h+1

...	...	@@ -523,6 +523,7 @@ typedef long __sanitizer_clock_t;
523	523
524	524	#if SANITIZER_LINUX
525	525	typedef int __sanitizer_clockid_t;
	526	typedef unsigned long long __sanitizer_eventfd_t;
526	527	#endif
527	528
528	529	#if SANITIZER_LINUX

lib/tsan/sanitizer_common/sanitizer_posix.cpp+6-6

...	...	@@ -54,12 +54,12 @@ void MmapOrDie(uptr size, const char mem_type, bool raw_report) {
54	54	return (void *)res;
55	55	}
56	56
57		void UnmapOrDie(void *addr, uptr size) {
	57	void UnmapOrDie(void *addr, uptr size, bool raw_report) {
58	58	if (!addr \|\| !size) return;
59	59	uptr res = internal_munmap(addr, size);
60	60	int reserrno;
61	61	if (UNLIKELY(internal_iserror(res, &reserrno)))
62		ReportMunmapFailureAndDie(addr, size, reserrno);
	62	ReportMunmapFailureAndDie(addr, size, reserrno, raw_report);
63	63	DecreaseTotalMmap(size);
64	64	}
65	65
...	...	@@ -85,8 +85,8 @@ void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment,
85	85	CHECK(IsPowerOfTwo(size));
86	86	CHECK(IsPowerOfTwo(alignment));
87	87	uptr map_size = size + alignment;
88		// mmap maps entire pages and rounds up map_size needs to be a an integral
89		// number of pages.
	88	// mmap maps entire pages and rounds up map_size needs to be a an integral
	89	// number of pages.
90	90	// We need to be aware of this size for calculating end and for unmapping
91	91	// fragments before and after the alignment region.
92	92	map_size = RoundUpTo(map_size, GetPageSizeCached());
...	...	@@ -130,8 +130,8 @@ static void *MmapFixedImpl(uptr fixed_addr, uptr size, bool tolerate_enomem,
130	130	if (tolerate_enomem && reserrno == ENOMEM)
131	131	return nullptr;
132	132	char mem_type[40];
133		internal_snprintf(mem_type, sizeof(mem_type), "memory at address 0x%zx",
134		fixed_addr);
	133	internal_snprintf(mem_type, sizeof(mem_type), "memory at address %p",
	134	(void *)fixed_addr);
135	135	ReportMmapFailureAndDie(size, mem_type, "allocate", reserrno);
136	136	}
137	137	IncreaseTotalMmap(size);

lib/tsan/sanitizer_common/sanitizer_posix.h+14-14

...	...	@@ -74,21 +74,21 @@ int internal_sysctlbyname(const char sname, void oldp, uptr *oldlenp,
74	74	// These functions call appropriate pthread_ functions directly, bypassing
75	75	// the interceptor. They are weak and may not be present in some tools.
76	76	SANITIZER_WEAK_ATTRIBUTE
77		int real_pthread_create(void th, void attr, void (callback)(void *),
78		void *param);
	77	int internal_pthread_create(void th, void attr, void (callback)(void *),
	78	void *param);
79	79	SANITIZER_WEAK_ATTRIBUTE
80		int real_pthread_join(void th, void *ret);
81
82		#define DEFINE_REAL_PTHREAD_FUNCTIONS \
83		namespace __sanitizer { \
84		int real_pthread_create(void th, void attr, void (callback)(void *), \
85		void *param) { \
86		return REAL(pthread_create)(th, attr, callback, param); \
87		} \
88		int real_pthread_join(void th, void *ret) { \
89		return REAL(pthread_join(th, ret)); \
90		} \
91		} // namespace __sanitizer
	80	int internal_pthread_join(void th, void *ret);
	81
	82	# define DEFINE_INTERNAL_PTHREAD_FUNCTIONS \
	83	namespace __sanitizer { \
	84	int internal_pthread_create(void th, void attr, \
	85	void (callback)(void ), void param) { \
	86	return REAL(pthread_create)(th, attr, callback, param); \
	87	} \
	88	int internal_pthread_join(void th, void *ret) { \
	89	return REAL(pthread_join(th, ret)); \
	90	} \
	91	} // namespace __sanitizer
92	92
93	93	int internal_pthread_attr_getstack(void attr, void addr, uptr size);
94	94

lib/tsan/sanitizer_common/sanitizer_posix_libcdep.cpp+27-6

...	...	@@ -91,12 +91,12 @@ static rlim_t getlim(int res) {
91	91
92	92	static void setlim(int res, rlim_t lim) {
93	93	struct rlimit rlim;
94		if (getrlimit(res, const_cast<struct rlimit *>(&rlim))) {
	94	if (getrlimit(res, &rlim)) {
95	95	Report("ERROR: %s getrlimit() failed %d\n", SanitizerToolName, errno);
96	96	Die();
97	97	}
98	98	rlim.rlim_cur = lim;
99		if (setrlimit(res, const_cast<struct rlimit *>(&rlim))) {
	99	if (setrlimit(res, &rlim)) {
100	100	Report("ERROR: %s setrlimit() failed %d\n", SanitizerToolName, errno);
101	101	Die();
102	102	}
...	...	@@ -104,7 +104,27 @@ static void setlim(int res, rlim_t lim) {
104	104
105	105	void DisableCoreDumperIfNecessary() {
106	106	if (common_flags()->disable_coredump) {
107		setlim(RLIMIT_CORE, 0);
	107	rlimit rlim;
	108	CHECK_EQ(0, getrlimit(RLIMIT_CORE, &rlim));
	109	// On Linux, if the kernel.core_pattern sysctl starts with a '\|' (i.e. it
	110	// is being piped to a coredump handler such as systemd-coredumpd), the
	111	// kernel ignores RLIMIT_CORE (since we aren't creating a file in the file
	112	// system) except for the magic value of 1, which disables coredumps when
	113	// piping. 1 byte is too small for any kind of valid core dump, so it
	114	// also disables coredumps if kernel.core_pattern creates files directly.
	115	// While most piped coredump handlers do respect the crashing processes'
	116	// RLIMIT_CORE, this is notable not the case for Debian's systemd-coredump
	117	// due to a local patch that changes sysctl.d/50-coredump.conf to ignore
	118	// the specified limit and instead use RLIM_INFINITY.
	119	//
	120	// The alternative to using RLIMIT_CORE=1 would be to use prctl() with the
	121	// PR_SET_DUMPABLE flag, however that also prevents ptrace(), so makes it
	122	// impossible to attach a debugger.
	123	//
	124	// Note: we use rlim_max in the Min() call here since that is the upper
	125	// limit for what can be set without getting an EINVAL error.
	126	rlim.rlim_cur = Min<rlim_t>(SANITIZER_LINUX ? 1 : 0, rlim.rlim_max);
	127	CHECK_EQ(0, setrlimit(RLIMIT_CORE, &rlim));
108	128	}
109	129	}
110	130
...	...	@@ -307,9 +327,10 @@ static bool MmapFixed(uptr fixed_addr, uptr size, int additional_flags,
307	327	MAP_PRIVATE \| MAP_FIXED \| additional_flags \| MAP_ANON, name);
308	328	int reserrno;
309	329	if (internal_iserror(p, &reserrno)) {
310		Report("ERROR: %s failed to "
311		"allocate 0x%zx (%zd) bytes at address %zx (errno: %d)\n",
312		SanitizerToolName, size, size, fixed_addr, reserrno);
	330	Report(
	331	"ERROR: %s failed to "
	332	"allocate 0x%zx (%zd) bytes at address %p (errno: %d)\n",
	333	SanitizerToolName, size, size, (void *)fixed_addr, reserrno);
313	334	return false;
314	335	}
315	336	IncreaseTotalMmap(size);

lib/tsan/sanitizer_common/sanitizer_printf.cpp+9-2

...	...	@@ -54,7 +54,7 @@ static int AppendNumber(char *buff, const char buff_end, u64 absolute_value,
54	54	uptr num_buffer[kMaxLen];
55	55	int pos = 0;
56	56	do {
57		RAW_CHECK_MSG((uptr)pos < kMaxLen, "AppendNumber buffer overflow");
	57	RAW_CHECK_MSG((uptr)pos < kMaxLen, "AppendNumber buffer overflow",);
58	58	num_buffer[pos++] = absolute_value % base;
59	59	absolute_value /= base;
60	60	} while (absolute_value > 0);
...	...	@@ -337,7 +337,14 @@ int internal_snprintf(char buffer, uptr length, const char format, ...) {
337	337	return needed_length;
338	338	}
339	339
340		void InternalScopedString::append(const char *format, ...) {
	340	void InternalScopedString::Append(const char *str) {
	341	uptr prev_len = length();
	342	uptr str_len = internal_strlen(str);
	343	buffer_.resize(prev_len + str_len + 1);
	344	internal_memcpy(buffer_.data() + prev_len, str, str_len + 1);
	345	}
	346
	347	void InternalScopedString::AppendF(const char *format, ...) {
341	348	uptr prev_len = length();
342	349
343	350	while (true) {

lib/tsan/sanitizer_common/sanitizer_procmaps_bsd.cpp+25-22

...	...	@@ -13,9 +13,6 @@
13	13	#include "sanitizer_platform.h"
14	14	#if SANITIZER_FREEBSD \|\| SANITIZER_NETBSD
15	15	#include "sanitizer_common.h"
16		#if SANITIZER_FREEBSD
17		#include "sanitizer_freebsd.h"
18		#endif
19	16	#include "sanitizer_procmaps.h"
20	17
21	18	// clang-format off
...	...	@@ -29,29 +26,35 @@
29	26
30	27	#include <limits.h>
31	28
32		// Fix 'kinfo_vmentry' definition on FreeBSD prior v9.2 in 32-bit mode.
33		#if SANITIZER_FREEBSD && (SANITIZER_WORDSIZE == 32)
34		#include <osreldate.h>
35		#if __FreeBSD_version <= 902001 // v9.2
36		#define kinfo_vmentry xkinfo_vmentry
37		#endif
38		#endif
39
40	29	namespace __sanitizer {
41	30
42	31	#if SANITIZER_FREEBSD
43	32	void GetMemoryProfile(fill_profile_f cb, uptr *stats) {
44		const int Mib[] = {
45		CTL_KERN,
46		KERN_PROC,
47		KERN_PROC_PID,
48		getpid()
49		};
50
51		struct kinfo_proc InfoProc;
52		uptr Len = sizeof(InfoProc);
53		CHECK_EQ(internal_sysctl(Mib, ARRAY_SIZE(Mib), nullptr, (uptr *)&InfoProc, &Len, 0), 0);
54		cb(0, InfoProc.ki_rssize * GetPageSizeCached(), false, stats);
	33	const int Mib[] = {CTL_KERN, KERN_PROC, KERN_PROC_PID, getpid()};
	34
	35	struct kinfo_proc *InfoProc;
	36	uptr Len = sizeof(*InfoProc);
	37	uptr Size = Len;
	38	InfoProc = (struct kinfo_proc *)MmapOrDie(Size, "GetMemoryProfile()");
	39	CHECK_EQ(
	40	internal_sysctl(Mib, ARRAY_SIZE(Mib), nullptr, (uptr *)InfoProc, &Len, 0),
	41	0);
	42	cb(0, InfoProc->ki_rssize * GetPageSizeCached(), false, stats);
	43	UnmapOrDie(InfoProc, Size, true);
	44	}
	45	#elif SANITIZER_NETBSD
	46	void GetMemoryProfile(fill_profile_f cb, uptr *stats) {
	47	struct kinfo_proc2 *InfoProc;
	48	uptr Len = sizeof(*InfoProc);
	49	uptr Size = Len;
	50	const int Mib[] = {CTL_KERN, KERN_PROC2, KERN_PROC_PID,
	51	getpid(), (int)Size, 1};
	52	InfoProc = (struct kinfo_proc2 *)MmapOrDie(Size, "GetMemoryProfile()");
	53	CHECK_EQ(
	54	internal_sysctl(Mib, ARRAY_SIZE(Mib), nullptr, (uptr *)InfoProc, &Len, 0),
	55	0);
	56	cb(0, InfoProc->p_vm_rssize * GetPageSizeCached(), false, stats);
	57	UnmapOrDie(InfoProc, Size, true);
55	58	}
56	59	#endif
57	60

lib/tsan/sanitizer_common/sanitizer_procmaps_common.cpp+1-1

...	...	@@ -145,7 +145,7 @@ void MemoryMappingLayout::DumpListOfModules(
145	145	}
146	146	}
147	147
148		#if SANITIZER_LINUX \|\| SANITIZER_ANDROID \|\| SANITIZER_SOLARIS \|\| SANITIZER_NETBSD
	148	#if SANITIZER_LINUX \|\| SANITIZER_ANDROID \|\| SANITIZER_SOLARIS
149	149	void GetMemoryProfile(fill_profile_f cb, uptr *stats) {
150	150	char *smaps = nullptr;
151	151	uptr smaps_cap = 0;

lib/tsan/sanitizer_common/sanitizer_ptrauth.h+24-22

...	...	@@ -9,31 +9,33 @@
9	9	#ifndef SANITIZER_PTRAUTH_H
10	10	#define SANITIZER_PTRAUTH_H
11	11
12		#if __has_feature(ptrauth_calls)
13		#include <ptrauth.h>
	12	#if __has_feature(ptrauth_intrinsics)
	13	# include <ptrauth.h>
14	14	#elif defined(__ARM_FEATURE_PAC_DEFAULT) && !defined(__APPLE__)
15		inline unsigned long ptrauth_strip(void* __value, unsigned int __key) {
16		// On the stack the link register is protected with Pointer
17		// Authentication Code when compiled with -mbranch-protection.
18		// Let's stripping the PAC unconditionally because xpaclri is in
19		// the NOP space so will do nothing when it is not enabled or not available.
20		unsigned long ret;
21		asm volatile(
22		"mov x30, %1\n\t"
23		"hint #7\n\t" // xpaclri
24		"mov %0, x30\n\t"
25		: "=r"(ret)
26		: "r"(__value)
27		: "x30");
28		return ret;
29		}
30		#define ptrauth_auth_data(__value, __old_key, __old_data) __value
31		#define ptrauth_string_discriminator(__string) ((int)0)
	15	// On the stack the link register is protected with Pointer
	16	// Authentication Code when compiled with -mbranch-protection.
	17	// Let's stripping the PAC unconditionally because xpaclri is in
	18	// the NOP space so will do nothing when it is not enabled or not available.
	19	# define ptrauth_strip(__value, __key) \
	20	({ \
	21	__typeof(__value) ret; \
	22	asm volatile( \
	23	"mov x30, %1\n\t" \
	24	"hint #7\n\t" \
	25	"mov %0, x30\n\t" \
	26	"mov x30, xzr\n\t" \
	27	: "=r"(ret) \
	28	: "r"(__value) \
	29	: "x30"); \
	30	ret; \
	31	})
	32	# define ptrauth_auth_data(__value, __old_key, __old_data) __value
	33	# define ptrauth_string_discriminator(__string) ((int)0)
32	34	#else
33	35	// Copied from <ptrauth.h>
34		#define ptrauth_strip(__value, __key) __value
35		#define ptrauth_auth_data(__value, __old_key, __old_data) __value
36		#define ptrauth_string_discriminator(__string) ((int)0)
	36	# define ptrauth_strip(__value, __key) __value
	37	# define ptrauth_auth_data(__value, __old_key, __old_data) __value
	38	# define ptrauth_string_discriminator(__string) ((int)0)
37	39	#endif
38	40
39	41	#define STRIP_PAC_PC(pc) ((uptr)ptrauth_strip(pc, 0))

lib/tsan/sanitizer_common/sanitizer_redefine_builtins.h+10-6

...	...	@@ -11,16 +11,19 @@
11	11	//
12	12	//===----------------------------------------------------------------------===//
13	13	#ifndef SANITIZER_COMMON_NO_REDEFINE_BUILTINS
14		#ifndef SANITIZER_REDEFINE_BUILTINS_H
15		#define SANITIZER_REDEFINE_BUILTINS_H
	14	# ifndef SANITIZER_REDEFINE_BUILTINS_H
	15	# define SANITIZER_REDEFINE_BUILTINS_H
16	16
17	17	// The asm hack only works with GCC and Clang.
18		#if !defined(_WIN32)
	18	# if !defined(_WIN32)
19	19
20	20	asm("memcpy = __sanitizer_internal_memcpy");
21	21	asm("memmove = __sanitizer_internal_memmove");
22	22	asm("memset = __sanitizer_internal_memset");
23	23
	24	# if defined(__cplusplus) && \
	25	!defined(SANITIZER_COMMON_REDEFINE_BUILTINS_IN_STD)
	26
24	27	// The builtins should not be redefined in source files that make use of C++
25	28	// standard libraries, in particular where C++STL headers with inline functions
26	29	// are used. The redefinition in such cases would lead to ODR violations.
...	...	@@ -46,7 +49,8 @@ using unordered_set = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file;
46	49	using vector = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file;
47	50	} // namespace std
48	51
49		#endif // !_WIN32
	52	# endif // __cpluplus
	53	# endif // !_WIN32
50	54
51		#endif // SANITIZER_REDEFINE_BUILTINS_H
52		#endif // SANITIZER_COMMON_NO_REDEFINE_BUILTINS
	55	# endif // SANITIZER_REDEFINE_BUILTINS_H
	56	#endif // SANITIZER_COMMON_NO_REDEFINE_BUILTINS

lib/tsan/sanitizer_common/sanitizer_ring_buffer.h+3-1

...	...	@@ -47,7 +47,9 @@ class RingBuffer {
47	47	void push(T t) {
48	48	*next_ = t;
49	49	next_--;
50		// The condition below works only if sizeof(T) is divisible by sizeof(T*).
	50	static_assert((sizeof(T) % sizeof(T *)) == 0,
	51	"The condition below works only if sizeof(T) is divisible by "
	52	"sizeof(T*).");
51	53	if (next_ <= reinterpret_cast<T*>(&next_))
52	54	next_ = last_;
53	55	}

lib/tsan/sanitizer_common/sanitizer_stack_store.cpp+7-2

...	...	@@ -44,6 +44,9 @@ StackStore::Id StackStore::Store(const StackTrace &trace, uptr *pack) {
44	44	uptr idx = 0;
45	45	*pack = 0;
46	46	uptr *stack_trace = Alloc(h.size + 1, &idx, pack);
	47	// No more space.
	48	if (stack_trace == nullptr)
	49	return 0;
47	50	*stack_trace = h.ToUptr();
48	51	internal_memcpy(stack_trace + 1, trace.trace, h.size * sizeof(uptr));
49	52	*pack += blocks_[GetBlockIdx(idx)].Stored(h.size + 1);
...	...	@@ -76,8 +79,10 @@ uptr StackStore::Alloc(uptr count, uptr idx, uptr *pack) {
76	79	uptr block_idx = GetBlockIdx(start);
77	80	uptr last_idx = GetBlockIdx(start + count - 1);
78	81	if (LIKELY(block_idx == last_idx)) {
79		// Fits into the a single block.
80		CHECK_LT(block_idx, ARRAY_SIZE(blocks_));
	82	// Fits into a single block.
	83	// No more available blocks. Indicate inability to allocate more memory.
	84	if (block_idx >= ARRAY_SIZE(blocks_))
	85	return nullptr;
81	86	*idx = start;
82	87	return blocks_[block_idx].GetOrCreate(this) + GetInBlockIdx(start);
83	88	}

lib/tsan/sanitizer_common/sanitizer_stackdepot.cpp+4-4

...	...	@@ -215,16 +215,16 @@ StackTrace StackDepotGet(u32 id) {
215	215	return theDepot.Get(id);
216	216	}
217	217
218		void StackDepotLockAll() {
219		theDepot.LockAll();
	218	void StackDepotLockBeforeFork() {
	219	theDepot.LockBeforeFork();
220	220	compress_thread.LockAndStop();
221	221	stackStore.LockAll();
222	222	}
223	223
224		void StackDepotUnlockAll() {
	224	void StackDepotUnlockAfterFork(bool fork_child) {
225	225	stackStore.UnlockAll();
226	226	compress_thread.Unlock();
227		theDepot.UnlockAll();
	227	theDepot.UnlockAfterFork(fork_child);
228	228	}
229	229
230	230	void StackDepotPrintAll() {

lib/tsan/sanitizer_common/sanitizer_stackdepot.h+2-2

...	...	@@ -39,8 +39,8 @@ StackDepotHandle StackDepotPut_WithHandle(StackTrace stack);
39	39	// Retrieves a stored stack trace by the id.
40	40	StackTrace StackDepotGet(u32 id);
41	41
42		void StackDepotLockAll();
43		void StackDepotUnlockAll();
	42	void StackDepotLockBeforeFork();
	43	void StackDepotUnlockAfterFork(bool fork_child);
44	44	void StackDepotPrintAll();
45	45	void StackDepotStopBackgroundThread();
46	46

lib/tsan/sanitizer_common/sanitizer_stackdepotbase.h+23-8

...	...	@@ -52,8 +52,8 @@ class StackDepotBase {
52	52	};
53	53	}
54	54
55		void LockAll();
56		void UnlockAll();
	55	void LockBeforeFork();
	56	void UnlockAfterFork(bool fork_child);
57	57	void PrintAll();
58	58
59	59	void TestOnlyUnmap() {
...	...	@@ -160,18 +160,33 @@ StackDepotBase<Node, kReservedBits, kTabSizeLog>::Get(u32 id) {
160	160	}
161	161
162	162	template <class Node, int kReservedBits, int kTabSizeLog>
163		void StackDepotBase<Node, kReservedBits, kTabSizeLog>::LockAll() {
164		for (int i = 0; i < kTabSize; ++i) {
165		lock(&tab[i]);
166		}
	163	void StackDepotBase<Node, kReservedBits, kTabSizeLog>::LockBeforeFork() {
	164	// Do not lock hash table. It's very expensive, but it's not rely needed. The
	165	// parent process will neither lock nor unlock. Child process risks to be
	166	// deadlocked on already locked buckets. To avoid deadlock we will unlock
	167	// every locked buckets in `UnlockAfterFork`. This may affect consistency of
	168	// the hash table, but the only issue is a few items inserted by parent
	169	// process will be not found by child, and the child may insert them again,
	170	// wasting some space in `stackStore`.
	171
	172	// We still need to lock nodes.
	173	nodes.Lock();
167	174	}
168	175
169	176	template <class Node, int kReservedBits, int kTabSizeLog>
170		void StackDepotBase<Node, kReservedBits, kTabSizeLog>::UnlockAll() {
	177	void StackDepotBase<Node, kReservedBits, kTabSizeLog>::UnlockAfterFork(
	178	bool fork_child) {
	179	nodes.Unlock();
	180
	181	// Only unlock in child process to avoid deadlock. See `LockBeforeFork`.
	182	if (!fork_child)
	183	return;
	184
171	185	for (int i = 0; i < kTabSize; ++i) {
172	186	atomic_uint32_t *p = &tab[i];
173	187	uptr s = atomic_load(p, memory_order_relaxed);
174		unlock(p, s & kUnlockMask);
	188	if (s & kLockMask)
	189	unlock(p, s & kUnlockMask);
175	190	}
176	191	}
177	192

lib/tsan/sanitizer_common/sanitizer_stacktrace_libcdep.cpp+22-20

...	...	@@ -29,42 +29,43 @@ class StackTraceTextPrinter {
29	29	frame_delimiter_(frame_delimiter),
30	30	output_(output),
31	31	dedup_token_(dedup_token),
32		symbolize_(RenderNeedsSymbolization(stack_trace_fmt)) {}
	32	symbolize_(StackTracePrinter::GetOrInit()->RenderNeedsSymbolization(
	33	stack_trace_fmt)) {}
33	34
34	35	bool ProcessAddressFrames(uptr pc) {
35		SymbolizedStack *frames = symbolize_
36		? Symbolizer::GetOrInit()->SymbolizePC(pc)
37		: SymbolizedStack::New(pc);
	36	SymbolizedStackHolder symbolized_stack(
	37	symbolize_ ? Symbolizer::GetOrInit()->SymbolizePC(pc)
	38	: SymbolizedStack::New(pc));
	39	const SymbolizedStack *frames = symbolized_stack.get();
38	40	if (!frames)
39	41	return false;
40	42
41		for (SymbolizedStack *cur = frames; cur; cur = cur->next) {
	43	for (const SymbolizedStack *cur = frames; cur; cur = cur->next) {
42	44	uptr prev_len = output_->length();
43		RenderFrame(output_, stack_trace_fmt_, frame_num_++, cur->info.address,
44		symbolize_ ? &cur->info : nullptr,
45		common_flags()->symbolize_vs_style,
46		common_flags()->strip_path_prefix);
	45	StackTracePrinter::GetOrInit()->RenderFrame(
	46	output_, stack_trace_fmt_, frame_num_++, cur->info.address,
	47	symbolize_ ? &cur->info : nullptr, common_flags()->symbolize_vs_style,
	48	common_flags()->strip_path_prefix);
47	49
48	50	if (prev_len != output_->length())
49		output_->append("%c", frame_delimiter_);
	51	output_->AppendF("%c", frame_delimiter_);
50	52
51	53	ExtendDedupToken(cur);
52	54	}
53		frames->ClearAll();
54	55	return true;
55	56	}
56	57
57	58	private:
58	59	// Extend the dedup token by appending a new frame.
59		void ExtendDedupToken(SymbolizedStack *stack) {
	60	void ExtendDedupToken(const SymbolizedStack *stack) {
60	61	if (!dedup_token_)
61	62	return;
62	63
63	64	if (dedup_frames_-- > 0) {
64	65	if (dedup_token_->length())
65		dedup_token_->append("--");
66		if (stack->info.function != nullptr)
67		dedup_token_->append("%s", stack->info.function);
	66	dedup_token_->Append("--");
	67	if (stack->info.function)
	68	dedup_token_->Append(stack->info.function);
68	69	}
69	70	}
70	71
...	...	@@ -98,7 +99,7 @@ void StackTrace::PrintTo(InternalScopedString *output) const {
98	99	output, &dedup_token);
99	100
100	101	if (trace == nullptr \|\| size == 0) {
101		output->append(" <empty stack>\n\n");
	102	output->Append(" <empty stack>\n\n");
102	103	return;
103	104	}
104	105
...	...	@@ -110,11 +111,11 @@ void StackTrace::PrintTo(InternalScopedString *output) const {
110	111	}
111	112
112	113	// Always add a trailing empty line after stack trace.
113		output->append("\n");
	114	output->Append("\n");
114	115
115	116	// Append deduplication token, if non-empty.
116	117	if (dedup_token.length())
117		output->append("DEDUP_TOKEN: %s\n", dedup_token.data());
	118	output->AppendF("DEDUP_TOKEN: %s\n", dedup_token.data());
118	119	}
119	120
120	121	uptr StackTrace::PrintTo(char *out_buf, uptr out_buf_size) const {
...	...	@@ -197,7 +198,7 @@ void __sanitizer_symbolize_pc(uptr pc, const char fmt, char out_buf,
197	198	StackTraceTextPrinter printer(fmt, '\0', &output, nullptr);
198	199	if (!printer.ProcessAddressFrames(pc)) {
199	200	output.clear();
200		output.append("<can't symbolize>");
	201	output.Append("<can't symbolize>");
201	202	}
202	203	CopyStringToBuffer(output, out_buf, out_buf_size);
203	204	}
...	...	@@ -210,7 +211,8 @@ void __sanitizer_symbolize_global(uptr data_addr, const char *fmt,
210	211	DataInfo DI;
211	212	if (!Symbolizer::GetOrInit()->SymbolizeData(data_addr, &DI)) return;
212	213	InternalScopedString data_desc;
213		RenderData(&data_desc, fmt, &DI, common_flags()->strip_path_prefix);
	214	StackTracePrinter::GetOrInit()->RenderData(&data_desc, fmt, &DI,
	215	common_flags()->strip_path_prefix);
214	216	internal_strncpy(out_buf, data_desc.data(), out_buf_size);
215	217	out_buf[out_buf_size - 1] = 0;
216	218	}

lib/tsan/sanitizer_common/sanitizer_stacktrace_printer.cpp+81-52

...	...	@@ -12,13 +12,28 @@
12	12
13	13	#include "sanitizer_stacktrace_printer.h"
14	14
	15	#include "sanitizer_common.h"
15	16	#include "sanitizer_file.h"
16	17	#include "sanitizer_flags.h"
17	18	#include "sanitizer_fuchsia.h"
	19	#include "sanitizer_symbolizer_markup.h"
18	20
19	21	namespace __sanitizer {
20	22
21		const char StripFunctionName(const char function) {
	23	StackTracePrinter *StackTracePrinter::GetOrInit() {
	24	static StackTracePrinter *stacktrace_printer;
	25	static StaticSpinMutex init_mu;
	26	SpinMutexLock l(&init_mu);
	27	if (stacktrace_printer)
	28	return stacktrace_printer;
	29
	30	stacktrace_printer = StackTracePrinter::NewStackTracePrinter();
	31
	32	CHECK(stacktrace_printer);
	33	return stacktrace_printer;
	34	}
	35
	36	const char StackTracePrinter::StripFunctionName(const char function) {
22	37	if (!common_flags()->demangle)
23	38	return function;
24	39	if (!function)
...	...	@@ -47,6 +62,13 @@ const char StripFunctionName(const char function) {
47	62	// sanitizer_symbolizer_markup.cpp implements these differently.
48	63	#if !SANITIZER_SYMBOLIZER_MARKUP
49	64
	65	StackTracePrinter *StackTracePrinter::NewStackTracePrinter() {
	66	if (common_flags()->enable_symbolizer_markup)
	67	return new (GetGlobalLowLevelAllocator()) MarkupStackTracePrinter();
	68
	69	return new (GetGlobalLowLevelAllocator()) FormattedStackTracePrinter();
	70	}
	71
50	72	static const char DemangleFunctionName(const char function) {
51	73	if (!common_flags()->demangle)
52	74	return function;
...	...	@@ -130,20 +152,23 @@ static void MaybeBuildIdToBuffer(const AddressInfo &info, bool PrefixSpace,
130	152	InternalScopedString *buffer) {
131	153	if (info.uuid_size) {
132	154	if (PrefixSpace)
133		buffer->append(" ");
134		buffer->append("(BuildId: ");
	155	buffer->Append(" ");
	156	buffer->Append("(BuildId: ");
135	157	for (uptr i = 0; i < info.uuid_size; ++i) {
136		buffer->append("%02x", info.uuid[i]);
	158	buffer->AppendF("%02x", info.uuid[i]);
137	159	}
138		buffer->append(")");
	160	buffer->Append(")");
139	161	}
140	162	}
141	163
142	164	static const char kDefaultFormat[] = " #%n %p %F %L";
143	165
144		void RenderFrame(InternalScopedString buffer, const char format, int frame_no,
145		uptr address, const AddressInfo *info, bool vs_style,
146		const char *strip_path_prefix) {
	166	void FormattedStackTracePrinter::RenderFrame(InternalScopedString *buffer,
	167	const char *format, int frame_no,
	168	uptr address,
	169	const AddressInfo *info,
	170	bool vs_style,
	171	const char *strip_path_prefix) {
147	172	// info will be null in the case where symbolization is not needed for the
148	173	// given format. This ensures that the code below will get a hard failure
149	174	// rather than print incorrect information in case RenderNeedsSymbolization
...	...	@@ -154,56 +179,56 @@ void RenderFrame(InternalScopedString buffer, const char format, int frame_no,
154	179	format = kDefaultFormat;
155	180	for (const char p = format; p != '\0'; p++) {
156	181	if (*p != '%') {
157		buffer->append("%c", *p);
	182	buffer->AppendF("%c", *p);
158	183	continue;
159	184	}
160	185	p++;
161	186	switch (*p) {
162	187	case '%':
163		buffer->append("%%");
	188	buffer->Append("%");
164	189	break;
165	190	// Frame number and all fields of AddressInfo structure.
166	191	case 'n':
167		buffer->append("%u", frame_no);
	192	buffer->AppendF("%u", frame_no);
168	193	break;
169	194	case 'p':
170		buffer->append("0x%zx", address);
	195	buffer->AppendF("%p", (void *)address);
171	196	break;
172	197	case 'm':
173		buffer->append("%s", StripPathPrefix(info->module, strip_path_prefix));
	198	buffer->AppendF("%s", StripPathPrefix(info->module, strip_path_prefix));
174	199	break;
175	200	case 'o':
176		buffer->append("0x%zx", info->module_offset);
	201	buffer->AppendF("0x%zx", info->module_offset);
177	202	break;
178	203	case 'b':
179	204	MaybeBuildIdToBuffer(info, /PrefixSpace=*/false, buffer);
180	205	break;
181	206	case 'f':
182		buffer->append("%s",
183		DemangleFunctionName(StripFunctionName(info->function)));
	207	buffer->AppendF("%s",
	208	DemangleFunctionName(StripFunctionName(info->function)));
184	209	break;
185	210	case 'q':
186		buffer->append("0x%zx", info->function_offset != AddressInfo::kUnknown
187		? info->function_offset
188		: 0x0);
	211	buffer->AppendF("0x%zx", info->function_offset != AddressInfo::kUnknown
	212	? info->function_offset
	213	: 0x0);
189	214	break;
190	215	case 's':
191		buffer->append("%s", StripPathPrefix(info->file, strip_path_prefix));
	216	buffer->AppendF("%s", StripPathPrefix(info->file, strip_path_prefix));
192	217	break;
193	218	case 'l':
194		buffer->append("%d", info->line);
	219	buffer->AppendF("%d", info->line);
195	220	break;
196	221	case 'c':
197		buffer->append("%d", info->column);
	222	buffer->AppendF("%d", info->column);
198	223	break;
199	224	// Smarter special cases.
200	225	case 'F':
201	226	// Function name and offset, if file is unknown.
202	227	if (info->function) {
203		buffer->append("in %s",
204		DemangleFunctionName(StripFunctionName(info->function)));
	228	buffer->AppendF(
	229	"in %s", DemangleFunctionName(StripFunctionName(info->function)));
205	230	if (!info->file && info->function_offset != AddressInfo::kUnknown)
206		buffer->append("+0x%zx", info->function_offset);
	231	buffer->AppendF("+0x%zx", info->function_offset);
207	232	}
208	233	break;
209	234	case 'S':
...	...	@@ -224,7 +249,7 @@ void RenderFrame(InternalScopedString buffer, const char format, int frame_no,
224	249	MaybeBuildIdToBuffer(info, /PrefixSpace=*/true, buffer);
225	250	#endif
226	251	} else {
227		buffer->append("(<unknown module>)");
	252	buffer->Append("(<unknown module>)");
228	253	}
229	254	break;
230	255	case 'M':
...	...	@@ -239,18 +264,18 @@ void RenderFrame(InternalScopedString buffer, const char format, int frame_no,
239	264	MaybeBuildIdToBuffer(info, /PrefixSpace=*/true, buffer);
240	265	#endif
241	266	} else {
242		buffer->append("(%p)", (void *)address);
	267	buffer->AppendF("(%p)", (void *)address);
243	268	}
244	269	break;
245	270	default:
246	271	Report("Unsupported specifier in stack frame format: %c (%p)!\n", *p,
247		(void *)p);
	272	(const void *)p);
248	273	Die();
249	274	}
250	275	}
251	276	}
252	277
253		bool RenderNeedsSymbolization(const char *format) {
	278	bool FormattedStackTracePrinter::RenderNeedsSymbolization(const char *format) {
254	279	if (0 == internal_strcmp(format, "DEFAULT"))
255	280	format = kDefaultFormat;
256	281	for (const char p = format; p != '\0'; p++) {
...	...	@@ -273,30 +298,32 @@ bool RenderNeedsSymbolization(const char *format) {
273	298	return false;
274	299	}
275	300
276		void RenderData(InternalScopedString buffer, const char format,
277		const DataInfo DI, const char strip_path_prefix) {
	301	void FormattedStackTracePrinter::RenderData(InternalScopedString *buffer,
	302	const char *format,
	303	const DataInfo *DI,
	304	const char *strip_path_prefix) {
278	305	for (const char p = format; p != '\0'; p++) {
279	306	if (*p != '%') {
280		buffer->append("%c", *p);
	307	buffer->AppendF("%c", *p);
281	308	continue;
282	309	}
283	310	p++;
284	311	switch (*p) {
285	312	case '%':
286		buffer->append("%%");
	313	buffer->Append("%");
287	314	break;
288	315	case 's':
289		buffer->append("%s", StripPathPrefix(DI->file, strip_path_prefix));
	316	buffer->AppendF("%s", StripPathPrefix(DI->file, strip_path_prefix));
290	317	break;
291	318	case 'l':
292		buffer->append("%zu", DI->line);
	319	buffer->AppendF("%zu", DI->line);
293	320	break;
294	321	case 'g':
295		buffer->append("%s", DI->name);
	322	buffer->AppendF("%s", DI->name);
296	323	break;
297	324	default:
298	325	Report("Unsupported specifier in stack frame format: %c (%p)!\n", *p,
299		(void *)p);
	326	(const void *)p);
300	327	Die();
301	328	}
302	329	}
...	...	@@ -304,33 +331,35 @@ void RenderData(InternalScopedString buffer, const char format,
304	331
305	332	#endif // !SANITIZER_SYMBOLIZER_MARKUP
306	333
307		void RenderSourceLocation(InternalScopedString buffer, const char file,
308		int line, int column, bool vs_style,
309		const char *strip_path_prefix) {
	334	void StackTracePrinter::RenderSourceLocation(InternalScopedString *buffer,
	335	const char *file, int line,
	336	int column, bool vs_style,
	337	const char *strip_path_prefix) {
310	338	if (vs_style && line > 0) {
311		buffer->append("%s(%d", StripPathPrefix(file, strip_path_prefix), line);
	339	buffer->AppendF("%s(%d", StripPathPrefix(file, strip_path_prefix), line);
312	340	if (column > 0)
313		buffer->append(",%d", column);
314		buffer->append(")");
	341	buffer->AppendF(",%d", column);
	342	buffer->Append(")");
315	343	return;
316	344	}
317	345
318		buffer->append("%s", StripPathPrefix(file, strip_path_prefix));
	346	buffer->AppendF("%s", StripPathPrefix(file, strip_path_prefix));
319	347	if (line > 0) {
320		buffer->append(":%d", line);
	348	buffer->AppendF(":%d", line);
321	349	if (column > 0)
322		buffer->append(":%d", column);
	350	buffer->AppendF(":%d", column);
323	351	}
324	352	}
325	353
326		void RenderModuleLocation(InternalScopedString buffer, const char module,
327		uptr offset, ModuleArch arch,
328		const char *strip_path_prefix) {
329		buffer->append("(%s", StripPathPrefix(module, strip_path_prefix));
	354	void StackTracePrinter::RenderModuleLocation(InternalScopedString *buffer,
	355	const char *module, uptr offset,
	356	ModuleArch arch,
	357	const char *strip_path_prefix) {
	358	buffer->AppendF("(%s", StripPathPrefix(module, strip_path_prefix));
330	359	if (arch != kModuleArchUnknown) {
331		buffer->append(":%s", ModuleArchToString(arch));
	360	buffer->AppendF(":%s", ModuleArchToString(arch));
332	361	}
333		buffer->append("+0x%zx)", offset);
	362	buffer->AppendF("+0x%zx)", offset);
334	363	}
335	364
336	365	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_stacktrace_printer.h+92-51

...	...	@@ -13,61 +13,102 @@
13	13	#define SANITIZER_STACKTRACE_PRINTER_H
14	14
15	15	#include "sanitizer_common.h"
	16	#include "sanitizer_internal_defs.h"
16	17	#include "sanitizer_symbolizer.h"
17	18
18	19	namespace __sanitizer {
19	20
20		// Strip interceptor prefixes from function name.
21		const char StripFunctionName(const char function);
22
23		// Render the contents of "info" structure, which represents the contents of
24		// stack frame "frame_no" and appends it to the "buffer". "format" is a
25		// string with placeholders, which is copied to the output with
26		// placeholders substituted with the contents of "info". For example,
27		// format string
28		// " frame %n: function %F at %S"
29		// will be turned into
30		// " frame 10: function foo::bar() at my/file.cc:10"
31		// You may additionally pass "strip_path_prefix" to strip prefixes of paths to
32		// source files and modules.
33		// Here's the full list of available placeholders:
34		// %% - represents a '%' character;
35		// %n - frame number (copy of frame_no);
36		// %p - PC in hex format;
37		// %m - path to module (binary or shared object);
38		// %o - offset in the module in hex format;
39		// %f - function name;
40		// %q - offset in the function in hex format (if available);
41		// %s - path to source file;
42		// %l - line in the source file;
43		// %c - column in the source file;
44		// %F - if function is known to be <foo>, prints "in <foo>", possibly
45		// followed by the offset in this function, but only if source file
46		// is unknown;
47		// %S - prints file/line/column information;
48		// %L - prints location information: file/line/column, if it is known, or
49		// module+offset if it is known, or (<unknown module>) string.
50		// %M - prints module basename and offset, if it is known, or PC.
51		void RenderFrame(InternalScopedString buffer, const char format, int frame_no,
52		uptr address, const AddressInfo *info, bool vs_style,
53		const char *strip_path_prefix = "");
54
55		bool RenderNeedsSymbolization(const char *format);
56
57		void RenderSourceLocation(InternalScopedString buffer, const char file,
58		int line, int column, bool vs_style,
59		const char *strip_path_prefix);
60
61		void RenderModuleLocation(InternalScopedString buffer, const char module,
62		uptr offset, ModuleArch arch,
63		const char *strip_path_prefix);
64
65		// Same as RenderFrame, but for data section (global variables).
66		// Accepts %s, %l from above.
67		// Also accepts:
68		// %g - name of the global variable.
69		void RenderData(InternalScopedString buffer, const char format,
70		const DataInfo DI, const char strip_path_prefix = "");
	21	// StacktracePrinter is an interface that is implemented by
	22	// classes that can perform rendering of the different parts
	23	// of a stacktrace.
	24	class StackTracePrinter {
	25	public:
	26	static StackTracePrinter *GetOrInit();
	27
	28	// Strip interceptor prefixes from function name.
	29	const char StripFunctionName(const char function);
	30
	31	virtual void RenderFrame(InternalScopedString buffer, const char format,
	32	int frame_no, uptr address, const AddressInfo *info,
	33	bool vs_style, const char *strip_path_prefix = "") {
	34	// Should be pure virtual, but we can't depend on __cxa_pure_virtual.
	35	UNIMPLEMENTED();
	36	}
	37
	38	virtual bool RenderNeedsSymbolization(const char *format) {
	39	// Should be pure virtual, but we can't depend on __cxa_pure_virtual.
	40	UNIMPLEMENTED();
	41	}
	42
	43	void RenderSourceLocation(InternalScopedString buffer, const char file,
	44	int line, int column, bool vs_style,
	45	const char *strip_path_prefix);
	46
	47	void RenderModuleLocation(InternalScopedString buffer, const char module,
	48	uptr offset, ModuleArch arch,
	49	const char *strip_path_prefix);
	50	virtual void RenderData(InternalScopedString buffer, const char format,
	51	const DataInfo *DI,
	52	const char *strip_path_prefix = "") {
	53	// Should be pure virtual, but we can't depend on __cxa_pure_virtual.
	54	UNIMPLEMENTED();
	55	}
	56
	57	private:
	58	// To be called from StackTracePrinter::GetOrInit
	59	static StackTracePrinter *NewStackTracePrinter();
	60
	61	protected:
	62	~StackTracePrinter() {}
	63	};
	64
	65	class FormattedStackTracePrinter : public StackTracePrinter {
	66	public:
	67	// Render the contents of "info" structure, which represents the contents of
	68	// stack frame "frame_no" and appends it to the "buffer". "format" is a
	69	// string with placeholders, which is copied to the output with
	70	// placeholders substituted with the contents of "info". For example,
	71	// format string
	72	// " frame %n: function %F at %S"
	73	// will be turned into
	74	// " frame 10: function foo::bar() at my/file.cc:10"
	75	// You may additionally pass "strip_path_prefix" to strip prefixes of paths to
	76	// source files and modules.
	77	// Here's the full list of available placeholders:
	78	// %% - represents a '%' character;
	79	// %n - frame number (copy of frame_no);
	80	// %p - PC in hex format;
	81	// %m - path to module (binary or shared object);
	82	// %o - offset in the module in hex format;
	83	// %f - function name;
	84	// %q - offset in the function in hex format (if available);
	85	// %s - path to source file;
	86	// %l - line in the source file;
	87	// %c - column in the source file;
	88	// %F - if function is known to be <foo>, prints "in <foo>", possibly
	89	// followed by the offset in this function, but only if source file
	90	// is unknown;
	91	// %S - prints file/line/column information;
	92	// %L - prints location information: file/line/column, if it is known, or
	93	// module+offset if it is known, or (<unknown module>) string.
	94	// %M - prints module basename and offset, if it is known, or PC.
	95	void RenderFrame(InternalScopedString buffer, const char format,
	96	int frame_no, uptr address, const AddressInfo *info,
	97	bool vs_style, const char *strip_path_prefix = "") override;
	98
	99	bool RenderNeedsSymbolization(const char *format) override;
	100
	101	// Same as RenderFrame, but for data section (global variables).
	102	// Accepts %s, %l from above.
	103	// Also accepts:
	104	// %g - name of the global variable.
	105	void RenderData(InternalScopedString buffer, const char format,
	106	const DataInfo *DI,
	107	const char *strip_path_prefix = "") override;
	108
	109	protected:
	110	~FormattedStackTracePrinter() {}
	111	};
71	112
72	113	} // namespace __sanitizer
73	114

lib/tsan/sanitizer_common/sanitizer_stacktrace_sparc.cpp+5-6

...	...	@@ -58,17 +58,16 @@ void BufferedStackTrace::UnwindFast(uptr pc, uptr bp, uptr stack_top,
58	58	// Avoid infinite loop when frame == frame[0] by using frame > prev_frame.
59	59	while (IsValidFrame(bp, stack_top, bottom) && IsAligned(bp, sizeof(uhwptr)) &&
60	60	size < max_depth) {
61		uhwptr pc1 = ((uhwptr *)bp)[15];
	61	// %o7 contains the address of the call instruction and not the
	62	// return address, so we need to compensate.
	63	uhwptr pc1 = GetNextInstructionPc(((uhwptr *)bp)[15]);
62	64	// Let's assume that any pointer in the 0th page is invalid and
63	65	// stop unwinding here. If we're adding support for a platform
64	66	// where this isn't true, we need to reconsider this check.
65	67	if (pc1 < kPageSize)
66	68	break;
67		if (pc1 != pc) {
68		// %o7 contains the address of the call instruction and not the
69		// return address, so we need to compensate.
70		trace_buffer[size++] = GetNextInstructionPc((uptr)pc1);
71		}
	69	if (pc1 != pc)
	70	trace_buffer[size++] = pc1;
72	71	bottom = bp;
73	72	bp = (uptr)((uhwptr *)bp)[14] + STACK_BIAS;
74	73	}

lib/tsan/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cpp+5-5

...	...	@@ -257,8 +257,8 @@ static void TracerThreadDieCallback() {
257	257	static void TracerThreadSignalHandler(int signum, __sanitizer_siginfo *siginfo,
258	258	void *uctx) {
259	259	SignalContext ctx(siginfo, uctx);
260		Printf("Tracer caught signal %d: addr=0x%zx pc=0x%zx sp=0x%zx\n", signum,
261		ctx.addr, ctx.pc, ctx.sp);
	260	Printf("Tracer caught signal %d: addr=%p pc=%p sp=%p\n", signum,
	261	(void )ctx.addr, (void )ctx.pc, (void *)ctx.sp);
262	262	ThreadSuspender *inst = thread_suspender_instance;
263	263	if (inst) {
264	264	if (signum == SIGABRT)
...	...	@@ -565,7 +565,7 @@ PtraceRegistersStatus SuspendedThreadsListLinux::GetRegistersAndSP(
565	565	constexpr uptr uptr_sz = sizeof(uptr);
566	566	int pterrno;
567	567	#ifdef ARCH_IOVEC_FOR_GETREGSET
568		auto append = [&](uptr regset) {
	568	auto AppendF = [&](uptr regset) {
569	569	uptr size = buffer->size();
570	570	// NT_X86_XSTATE requires 64bit alignment.
571	571	uptr size_up = RoundUpTo(size, 8 / uptr_sz);
...	...	@@ -596,11 +596,11 @@ PtraceRegistersStatus SuspendedThreadsListLinux::GetRegistersAndSP(
596	596	};
597	597
598	598	buffer->clear();
599		bool fail = !append(NT_PRSTATUS);
	599	bool fail = !AppendF(NT_PRSTATUS);
600	600	if (!fail) {
601	601	// Accept the first available and do not report errors.
602	602	for (uptr regs : kExtraRegs)
603		if (regs && append(regs))
	603	if (regs && AppendF(regs))
604	604	break;
605	605	}
606	606	#else

lib/tsan/sanitizer_common/sanitizer_stoptheworld_netbsd_libcdep.cpp+2-2

...	...	@@ -158,8 +158,8 @@ static void TracerThreadDieCallback() {
158	158	static void TracerThreadSignalHandler(int signum, __sanitizer_siginfo *siginfo,
159	159	void *uctx) {
160	160	SignalContext ctx(siginfo, uctx);
161		Printf("Tracer caught signal %d: addr=0x%zx pc=0x%zx sp=0x%zx\n", signum,
162		ctx.addr, ctx.pc, ctx.sp);
	161	Printf("Tracer caught signal %d: addr=%p pc=%p sp=%p\n", signum,
	162	(void )ctx.addr, (void )ctx.pc, (void *)ctx.sp);
163	163	ThreadSuspender *inst = thread_suspender_instance;
164	164	if (inst) {
165	165	if (signum == SIGABRT)

lib/tsan/sanitizer_common/sanitizer_suppressions.cpp+5-2

...	...	@@ -86,7 +86,7 @@ void SuppressionContext::ParseFromFile(const char *filename) {
86	86	}
87	87
88	88	Parse(file_contents);
89		UnmapOrDie(file_contents, contents_size);
	89	UnmapOrDie(file_contents, buffer_size);
90	90	}
91	91
92	92	bool SuppressionContext::Match(const char str, const char type,
...	...	@@ -138,7 +138,10 @@ void SuppressionContext::Parse(const char *str) {
138	138	}
139	139	}
140	140	if (type == suppression_types_num_) {
141		Printf("%s: failed to parse suppressions\n", SanitizerToolName);
	141	Printf("%s: failed to parse suppressions.\n", SanitizerToolName);
	142	Printf("Supported suppression types are:\n");
	143	for (type = 0; type < suppression_types_num_; type++)
	144	Printf("- %s\n", suppression_types_[type]);
142	145	Die();
143	146	}
144	147	Suppression s;

lib/tsan/sanitizer_common/sanitizer_symbolizer.cpp+4-1

...	...	@@ -10,6 +10,8 @@
10	10	// run-time libraries.
11	11	//===----------------------------------------------------------------------===//
12	12
	13	#include <errno.h>
	14
13	15	#include "sanitizer_allocator_internal.h"
14	16	#include "sanitizer_common.h"
15	17	#include "sanitizer_internal_defs.h"
...	...	@@ -128,7 +130,7 @@ Symbolizer::Symbolizer(IntrusiveList<SymbolizerTool> tools)
128	130	start_hook_(0), end_hook_(0) {}
129	131
130	132	Symbolizer::SymbolizerScope::SymbolizerScope(const Symbolizer *sym)
131		: sym_(sym) {
	133	: sym_(sym), errno_(errno) {
132	134	if (sym_->start_hook_)
133	135	sym_->start_hook_();
134	136	}
...	...	@@ -136,6 +138,7 @@ Symbolizer::SymbolizerScope::SymbolizerScope(const Symbolizer *sym)
136	138	Symbolizer::SymbolizerScope::~SymbolizerScope() {
137	139	if (sym_->end_hook_)
138	140	sym_->end_hook_();
	141	errno = errno_;
139	142	}
140	143
141	144	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_symbolizer.h+25-2

...	...	@@ -64,6 +64,26 @@ struct SymbolizedStack {
64	64	SymbolizedStack();
65	65	};
66	66
	67	class SymbolizedStackHolder {
	68	SymbolizedStack *Stack;
	69
	70	void clear() {
	71	if (Stack)
	72	Stack->ClearAll();
	73	}
	74
	75	public:
	76	explicit SymbolizedStackHolder(SymbolizedStack *Stack = nullptr)
	77	: Stack(Stack) {}
	78	~SymbolizedStackHolder() { clear(); }
	79	void reset(SymbolizedStack *S = nullptr) {
	80	if (Stack != S)
	81	clear();
	82	Stack = S;
	83	}
	84	const SymbolizedStack *get() const { return Stack; }
	85	};
	86
67	87	// For now, DataInfo is used to describe global variable.
68	88	struct DataInfo {
69	89	// Owns all the string members. Storage for them is
...	...	@@ -136,7 +156,7 @@ class Symbolizer final {
136	156
137	157	// Release internal caches (if any).
138	158	void Flush();
139		// Attempts to demangle the provided C++ mangled name.
	159	// Attempts to demangle the provided C++ mangled name. Never returns nullptr.
140	160	const char Demangle(const char name);
141	161
142	162	// Allow user to install hooks that would be called before/after Symbolizer
...	...	@@ -154,6 +174,8 @@ class Symbolizer final {
154	174
155	175	void InvalidateModuleList();
156	176
	177	const ListOfModules &GetRefreshedListOfModules();
	178
157	179	private:
158	180	// GetModuleNameAndOffsetForPC has to return a string to the caller.
159	181	// Since the corresponding module might get unloaded later, we should create
...	...	@@ -187,7 +209,7 @@ class Symbolizer final {
187	209	// If stale, need to reload the modules before looking up addresses.
188	210	bool modules_fresh_;
189	211
190		// Platform-specific default demangler, must not return nullptr.
	212	// Platform-specific default demangler, returns nullptr on failure.
191	213	const char PlatformDemangle(const char name);
192	214
193	215	static Symbolizer *symbolizer_;
...	...	@@ -212,6 +234,7 @@ class Symbolizer final {
212	234	~SymbolizerScope();
213	235	private:
214	236	const Symbolizer *sym_;
	237	int errno_; // Backup errno in case symbolizer change the value.
215	238	};
216	239	};
217	240

lib/tsan/sanitizer_common/sanitizer_symbolizer_fuchsia.h deleted-42

...	...	@@ -1,42 +0,0 @@
1		//===-- sanitizer_symbolizer_fuchsia.h -----------------------------------===//
2		//
3		// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4		// See https://llvm.org/LICENSE.txt for license information.
5		// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6		//
7		//===----------------------------------------------------------------------===//
8		//
9		// This file is shared between various sanitizers' runtime libraries.
10		//
11		// Define Fuchsia's string formats and limits for the markup symbolizer.
12		//===----------------------------------------------------------------------===//
13		#ifndef SANITIZER_SYMBOLIZER_FUCHSIA_H
14		#define SANITIZER_SYMBOLIZER_FUCHSIA_H
15
16		#include "sanitizer_internal_defs.h"
17
18		namespace __sanitizer {
19
20		// See the spec at:
21		// https://fuchsia.googlesource.com/zircon/+/master/docs/symbolizer_markup.md
22
23		// This is used by UBSan for type names, and by ASan for global variable names.
24		constexpr const char *kFormatDemangle = "{{{symbol:%s}}}";
25		constexpr uptr kFormatDemangleMax = 1024; // Arbitrary.
26
27		// Function name or equivalent from PC location.
28		constexpr const char *kFormatFunction = "{{{pc:%p}}}";
29		constexpr uptr kFormatFunctionMax = 64; // More than big enough for 64-bit hex.
30
31		// Global variable name or equivalent from data memory address.
32		constexpr const char *kFormatData = "{{{data:%p}}}";
33
34		// One frame in a backtrace (printed on a line by itself).
35		constexpr const char *kFormatFrame = "{{{bt:%u:%p}}}";
36
37		// Dump trigger element.
38		#define FORMAT_DUMPFILE "{{{dumpfile:%s:%s}}}"
39
40		} // namespace __sanitizer
41
42		#endif // SANITIZER_SYMBOLIZER_FUCHSIA_H

lib/tsan/sanitizer_common/sanitizer_symbolizer_internal.h+9

...	...	@@ -160,6 +160,15 @@ void ParseSymbolizePCOutput(const char str, SymbolizedStack res);
160	160	// Used by LLVMSymbolizer and InternalSymbolizer.
161	161	void ParseSymbolizeDataOutput(const char str, DataInfo info);
162	162
	163	// Parses repeated strings in the following format:
	164	// <function_name>
	165	// <var_name>
	166	// <file_name>:<line_number>[:<column_number>]
	167	// [<frame_offset>\|??] [<size>\|??] [<tag_offset>\|??]
	168	// Used by LLVMSymbolizer and InternalSymbolizer.
	169	void ParseSymbolizeFrameOutput(const char *str,
	170	InternalMmapVector<LocalInfo> *locals);
	171
163	172	} // namespace __sanitizer
164	173
165	174	#endif // SANITIZER_SYMBOLIZER_INTERNAL_H

lib/tsan/sanitizer_common/sanitizer_symbolizer_libbacktrace.cpp+1-1

...	...	@@ -199,7 +199,7 @@ static char DemangleAlloc(const char name, bool always_alloc) {
199	199	#endif
200	200	if (always_alloc)
201	201	return internal_strdup(name);
202		return 0;
	202	return nullptr;
203	203	}
204	204
205	205	const char LibbacktraceSymbolizer::Demangle(const char name) {

lib/tsan/sanitizer_common/sanitizer_symbolizer_libcdep.cpp+15-5

...	...	@@ -117,7 +117,7 @@ bool Symbolizer::SymbolizeData(uptr addr, DataInfo *info) {
117	117	return true;
118	118	}
119	119	}
120		return true;
	120	return false;
121	121	}
122	122
123	123	bool Symbolizer::SymbolizeFrame(uptr addr, FrameInfo *info) {
...	...	@@ -133,7 +133,7 @@ bool Symbolizer::SymbolizeFrame(uptr addr, FrameInfo *info) {
133	133	return true;
134	134	}
135	135	}
136		return true;
	136	return false;
137	137	}
138	138
139	139	bool Symbolizer::GetModuleNameAndOffsetForPC(uptr pc, const char **module_name,
...	...	@@ -159,13 +159,16 @@ void Symbolizer::Flush() {
159	159	}
160	160
161	161	const char Symbolizer::Demangle(const char name) {
	162	CHECK(name);
162	163	Lock l(&mu_);
163	164	for (auto &tool : tools_) {
164	165	SymbolizerScope sym_scope(this);
165	166	if (const char *demangled = tool.Demangle(name))
166	167	return demangled;
167	168	}
168		return PlatformDemangle(name);
	169	if (const char *demangled = PlatformDemangle(name))
	170	return demangled;
	171	return name;
169	172	}
170	173
171	174	bool Symbolizer::FindModuleNameAndOffsetForAddress(uptr address,
...	...	@@ -188,6 +191,13 @@ void Symbolizer::RefreshModules() {
188	191	modules_fresh_ = true;
189	192	}
190	193
	194	const ListOfModules &Symbolizer::GetRefreshedListOfModules() {
	195	if (!modules_fresh_)
	196	RefreshModules();
	197
	198	return modules_;
	199	}
	200
191	201	static const LoadedModule *SearchForModule(const ListOfModules &modules,
192	202	uptr address) {
193	203	for (uptr i = 0; i < modules.size(); i++) {
...	...	@@ -382,8 +392,8 @@ void ParseSymbolizeDataOutput(const char str, DataInfo info) {
382	392	str = ExtractUptr(str, "\n", &info->line);
383	393	}
384	394
385		static void ParseSymbolizeFrameOutput(const char *str,
386		InternalMmapVector<LocalInfo> *locals) {
	395	void ParseSymbolizeFrameOutput(const char *str,
	396	InternalMmapVector<LocalInfo> *locals) {
387	397	if (internal_strncmp(str, "??", 2) == 0)
388	398	return;
389	399

lib/tsan/sanitizer_common/sanitizer_symbolizer_mac.cpp+4-1

...	...	@@ -42,7 +42,8 @@ bool DlAddrSymbolizer::SymbolizePC(uptr addr, SymbolizedStack *stack) {
42	42	}
43	43
44	44	const char *demangled = DemangleSwiftAndCXX(info.dli_sname);
45		if (!demangled) return false;
	45	if (!demangled)
	46	demangled = info.dli_sname;
46	47	stack->info.function = internal_strdup(demangled);
47	48	return true;
48	49	}
...	...	@@ -52,6 +53,8 @@ bool DlAddrSymbolizer::SymbolizeData(uptr addr, DataInfo *datainfo) {
52	53	int result = dladdr((const void *)addr, &info);
53	54	if (!result) return false;
54	55	const char *demangled = DemangleSwiftAndCXX(info.dli_sname);
	56	if (!demangled)
	57	demangled = info.dli_sname;
55	58	datainfo->name = internal_strdup(demangled);
56	59	datainfo->start = (uptr)info.dli_saddr;
57	60	return true;

lib/tsan/sanitizer_common/sanitizer_symbolizer_markup.cpp+120-108

...	...	@@ -8,143 +8,155 @@
8	8	//
9	9	// This file is shared between various sanitizers' runtime libraries.
10	10	//
11		// Implementation of offline markup symbolizer.
	11	// This generic support for offline symbolizing is based on the
	12	// Fuchsia port. We don't do any actual symbolization per se.
	13	// Instead, we emit text containing raw addresses and raw linkage
	14	// symbol names, embedded in Fuchsia's symbolization markup format.
	15	// See the spec at:
	16	// https://llvm.org/docs/SymbolizerMarkupFormat.html
12	17	//===----------------------------------------------------------------------===//
13	18
14		#include "sanitizer_platform.h"
15		#if SANITIZER_SYMBOLIZER_MARKUP
16
17		#if SANITIZER_FUCHSIA
18		#include "sanitizer_symbolizer_fuchsia.h"
19		# endif
	19	#include "sanitizer_symbolizer_markup.h"
20	20
21		# include <limits.h>
22		# include <unwind.h>
23
24		# include "sanitizer_stacktrace.h"
25		# include "sanitizer_symbolizer.h"
	21	#include "sanitizer_common.h"
	22	#include "sanitizer_symbolizer.h"
	23	#include "sanitizer_symbolizer_markup_constants.h"
26	24
27	25	namespace __sanitizer {
28	26
29		// This generic support for offline symbolizing is based on the
30		// Fuchsia port. We don't do any actual symbolization per se.
31		// Instead, we emit text containing raw addresses and raw linkage
32		// symbol names, embedded in Fuchsia's symbolization markup format.
33		// Fuchsia's logging infrastructure emits enough information about
34		// process memory layout that a post-processing filter can do the
35		// symbolization and pretty-print the markup. See the spec at:
36		// https://fuchsia.googlesource.com/zircon/+/master/docs/symbolizer_markup.md
37
38		// This is used by UBSan for type names, and by ASan for global variable names.
39		// It's expected to return a static buffer that will be reused on each call.
40		const char Symbolizer::Demangle(const char name) {
41		static char buffer[kFormatDemangleMax];
42		internal_snprintf(buffer, sizeof(buffer), kFormatDemangle, name);
43		return buffer;
	27	void MarkupStackTracePrinter::RenderData(InternalScopedString *buffer,
	28	const char format, const DataInfo DI,
	29	const char *strip_path_prefix) {
	30	RenderContext(buffer);
	31	buffer->AppendF(kFormatData, reinterpret_cast<void *>(DI->start));
44	32	}
45	33
46		// This is used mostly for suppression matching. Making it work
47		// would enable "interceptor_via_lib" suppressions. It's also used
48		// once in UBSan to say "in module ..." in a message that also
49		// includes an address in the module, so post-processing can already
50		// pretty-print that so as to indicate the module.
51		bool Symbolizer::GetModuleNameAndOffsetForPC(uptr pc, const char **module_name,
52		uptr *module_address) {
	34	bool MarkupStackTracePrinter::RenderNeedsSymbolization(const char *format) {
53	35	return false;
54	36	}
55	37
56		// This is mainly used by hwasan for online symbolization. This isn't needed
57		// since hwasan can always just dump stack frames for offline symbolization.
58		bool Symbolizer::SymbolizeFrame(uptr addr, FrameInfo *info) { return false; }
59
60		// This is used in some places for suppression checking, which we
61		// don't really support for Fuchsia. It's also used in UBSan to
62		// identify a PC location to a function name, so we always fill in
63		// the function member with a string containing markup around the PC
64		// value.
65		// TODO(mcgrathr): Under SANITIZER_GO, it's currently used by TSan
66		// to render stack frames, but that should be changed to use
67		// RenderStackFrame.
68		SymbolizedStack *Symbolizer::SymbolizePC(uptr addr) {
69		SymbolizedStack *s = SymbolizedStack::New(addr);
	38	// We don't support the stack_trace_format flag at all.
	39	void MarkupStackTracePrinter::RenderFrame(InternalScopedString *buffer,
	40	const char *format, int frame_no,
	41	uptr address, const AddressInfo *info,
	42	bool vs_style,
	43	const char *strip_path_prefix) {
	44	CHECK(!RenderNeedsSymbolization(format));
	45	RenderContext(buffer);
	46	buffer->AppendF(kFormatFrame, frame_no, reinterpret_cast<void *>(address));
	47	}
	48
	49	bool MarkupSymbolizerTool::SymbolizePC(uptr addr, SymbolizedStack *stack) {
70	50	char buffer[kFormatFunctionMax];
71		internal_snprintf(buffer, sizeof(buffer), kFormatFunction, addr);
72		s->info.function = internal_strdup(buffer);
73		return s;
	51	internal_snprintf(buffer, sizeof(buffer), kFormatFunction,
	52	reinterpret_cast<void *>(addr));
	53	stack->info.function = internal_strdup(buffer);
	54	return true;
74	55	}
75	56
76		// Always claim we succeeded, so that RenderDataInfo will be called.
77		bool Symbolizer::SymbolizeData(uptr addr, DataInfo *info) {
	57	bool MarkupSymbolizerTool::SymbolizeData(uptr addr, DataInfo *info) {
78	58	info->Clear();
79	59	info->start = addr;
80	60	return true;
81	61	}
82	62
83		// We ignore the format argument to __sanitizer_symbolize_global.
84		void RenderData(InternalScopedString buffer, const char format,
85		const DataInfo DI, const char strip_path_prefix) {
86		buffer->append(kFormatData, DI->start);
	63	const char MarkupSymbolizerTool::Demangle(const char name) {
	64	static char buffer[kFormatDemangleMax];
	65	internal_snprintf(buffer, sizeof(buffer), kFormatDemangle, name);
	66	return buffer;
87	67	}
88	68
89		bool RenderNeedsSymbolization(const char *format) { return false; }
90
91		// We don't support the stack_trace_format flag at all.
92		void RenderFrame(InternalScopedString buffer, const char format, int frame_no,
93		uptr address, const AddressInfo *info, bool vs_style,
94		const char *strip_path_prefix) {
95		CHECK(!RenderNeedsSymbolization(format));
96		buffer->append(kFormatFrame, frame_no, address);
	69	// Fuchsia's implementation of symbolizer markup doesn't need to emit contextual
	70	// elements at this point.
	71	// Fuchsia's logging infrastructure emits enough information about
	72	// process memory layout that a post-processing filter can do the
	73	// symbolization and pretty-print the markup.
	74	#if !SANITIZER_FUCHSIA
	75
	76	static bool ModulesEq(const LoadedModule &module,
	77	const RenderedModule &renderedModule) {
	78	return module.base_address() == renderedModule.base_address &&
	79	internal_memcmp(module.uuid(), renderedModule.uuid,
	80	module.uuid_size()) == 0 &&
	81	internal_strcmp(module.full_name(), renderedModule.full_name) == 0;
97	82	}
98	83
99		Symbolizer *Symbolizer::PlatformInit() {
100		return new (symbolizer_allocator_) Symbolizer({});
	84	static bool ModuleHasBeenRendered(
	85	const LoadedModule &module,
	86	const InternalMmapVectorNoCtor<RenderedModule> &renderedModules) {
	87	for (const auto &renderedModule : renderedModules)
	88	if (ModulesEq(module, renderedModule))
	89	return true;
	90
	91	return false;
101	92	}
102	93
103		void Symbolizer::LateInitialize() { Symbolizer::GetOrInit(); }
104
105		void StartReportDeadlySignal() {}
106		void ReportDeadlySignal(const SignalContext &sig, u32 tid,
107		UnwindSignalStackCallbackType unwind,
108		const void *unwind_context) {}
109
110		#if SANITIZER_CAN_SLOW_UNWIND
111		struct UnwindTraceArg {
112		BufferedStackTrace *stack;
113		u32 max_depth;
114		};
115
116		_Unwind_Reason_Code Unwind_Trace(struct _Unwind_Context ctx, void param) {
117		UnwindTraceArg arg = static_cast<UnwindTraceArg >(param);
118		CHECK_LT(arg->stack->size, arg->max_depth);
119		uptr pc = _Unwind_GetIP(ctx);
120		if (pc < PAGE_SIZE) return _URC_NORMAL_STOP;
121		arg->stack->trace_buffer[arg->stack->size++] = pc;
122		return (arg->stack->size == arg->max_depth ? _URC_NORMAL_STOP
123		: _URC_NO_REASON);
	94	static void RenderModule(InternalScopedString *buffer,
	95	const LoadedModule &module, uptr moduleId) {
	96	InternalScopedString buildIdBuffer;
	97	for (uptr i = 0; i < module.uuid_size(); i++)
	98	buildIdBuffer.AppendF("%02x", module.uuid()[i]);
	99
	100	buffer->AppendF(kFormatModule, moduleId, module.full_name(),
	101	buildIdBuffer.data());
	102	buffer->Append("\n");
124	103	}
125	104
126		void BufferedStackTrace::UnwindSlow(uptr pc, u32 max_depth) {
127		CHECK_GE(max_depth, 2);
128		size = 0;
129		UnwindTraceArg arg = {this, Min(max_depth + 1, kStackTraceMax)};
130		_Unwind_Backtrace(Unwind_Trace, &arg);
131		CHECK_GT(size, 0);
132		// We need to pop a few frames so that pc is on top.
133		uptr to_pop = LocatePcInTrace(pc);
134		// trace_buffer[0] belongs to the current function so we always pop it,
135		// unless there is only 1 frame in the stack trace (1 frame is always better
136		// than 0!).
137		PopStackFrames(Min(to_pop, static_cast<uptr>(1)));
138		trace_buffer[0] = pc;
	105	static void RenderMmaps(InternalScopedString *buffer,
	106	const LoadedModule &module, uptr moduleId) {
	107	InternalScopedString accessBuffer;
	108
	109	// All module mmaps are readable at least
	110	for (const auto &range : module.ranges()) {
	111	accessBuffer.Append("r");
	112	if (range.writable)
	113	accessBuffer.Append("w");
	114	if (range.executable)
	115	accessBuffer.Append("x");
	116
	117	//{{{mmap:%starting_addr:%size_in_hex:load:%moduleId:r%(w\|x):%relative_addr}}}
	118
	119	// module.base_address == dlpi_addr
	120	// range.beg == dlpi_addr + p_vaddr
	121	// relative address == p_vaddr == range.beg - module.base_address
	122	buffer->AppendF(kFormatMmap, reinterpret_cast<void *>(range.beg),
	123	range.end - range.beg, static_cast<int>(moduleId),
	124	accessBuffer.data(), range.beg - module.base_address());
	125
	126	buffer->Append("\n");
	127	accessBuffer.clear();
	128	}
139	129	}
140	130
141		void BufferedStackTrace::UnwindSlow(uptr pc, void *context, u32 max_depth) {
142		CHECK(context);
143		CHECK_GE(max_depth, 2);
144		UNREACHABLE("signal context doesn't exist");
	131	void MarkupStackTracePrinter::RenderContext(InternalScopedString *buffer) {
	132	if (renderedModules_.size() == 0)
	133	buffer->Append("{{{reset}}}\n");
	134
	135	const auto &modules = Symbolizer::GetOrInit()->GetRefreshedListOfModules();
	136
	137	for (const auto &module : modules) {
	138	if (ModuleHasBeenRendered(module, renderedModules_))
	139	continue;
	140
	141	// symbolizer markup id, used to refer to this modules from other contextual
	142	// elements
	143	uptr moduleId = renderedModules_.size();
	144
	145	RenderModule(buffer, module, moduleId);
	146	RenderMmaps(buffer, module, moduleId);
	147
	148	renderedModules_.push_back({
	149	internal_strdup(module.full_name()),
	150	module.base_address(),
	151	{},
	152	});
	153
	154	// kModuleUUIDSize is the size of curModule.uuid
	155	CHECK_GE(kModuleUUIDSize, module.uuid_size());
	156	internal_memcpy(renderedModules_.back().uuid, module.uuid(),
	157	module.uuid_size());
	158	}
145	159	}
146		#endif // SANITIZER_CAN_SLOW_UNWIND
	160	#endif // !SANITIZER_FUCHSIA
147	161
148	162	} // namespace __sanitizer
149
150		#endif // SANITIZER_SYMBOLIZER_MARKUP

lib/tsan/sanitizer_common/sanitizer_symbolizer_markup.h created+79

...	...	@@ -0,0 +1,79 @@
	1	//===-- sanitizer_symbolizer_markup.h -----------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file is shared between various sanitizers' runtime libraries.
	10	//
	11	// Header for the offline markup symbolizer.
	12	//===----------------------------------------------------------------------===//
	13	#ifndef SANITIZER_SYMBOLIZER_MARKUP_H
	14	#define SANITIZER_SYMBOLIZER_MARKUP_H
	15
	16	#include "sanitizer_common.h"
	17	#include "sanitizer_stacktrace_printer.h"
	18	#include "sanitizer_symbolizer.h"
	19	#include "sanitizer_symbolizer_internal.h"
	20
	21	namespace __sanitizer {
	22
	23	// Simplier view of a LoadedModule. It only holds information necessary to
	24	// identify unique modules.
	25	struct RenderedModule {
	26	char *full_name;
	27	uptr base_address;
	28	u8 uuid[kModuleUUIDSize]; // BuildId
	29	};
	30
	31	class MarkupStackTracePrinter : public StackTracePrinter {
	32	public:
	33	// We don't support the stack_trace_format flag at all.
	34	void RenderFrame(InternalScopedString buffer, const char format,
	35	int frame_no, uptr address, const AddressInfo *info,
	36	bool vs_style, const char *strip_path_prefix = "") override;
	37
	38	bool RenderNeedsSymbolization(const char *format) override;
	39
	40	// We ignore the format argument to __sanitizer_symbolize_global.
	41	void RenderData(InternalScopedString buffer, const char format,
	42	const DataInfo *DI,
	43	const char *strip_path_prefix = "") override;
	44
	45	private:
	46	// Keeps track of the modules that have been rendered to avoid re-rendering
	47	// them
	48	InternalMmapVector<RenderedModule> renderedModules_;
	49	void RenderContext(InternalScopedString *buffer);
	50
	51	protected:
	52	~MarkupStackTracePrinter() {}
	53	};
	54
	55	class MarkupSymbolizerTool final : public SymbolizerTool {
	56	public:
	57	// This is used in some places for suppression checking, which we
	58	// don't really support for Fuchsia. It's also used in UBSan to
	59	// identify a PC location to a function name, so we always fill in
	60	// the function member with a string containing markup around the PC
	61	// value.
	62	// TODO(mcgrathr): Under SANITIZER_GO, it's currently used by TSan
	63	// to render stack frames, but that should be changed to use
	64	// RenderStackFrame.
	65	bool SymbolizePC(uptr addr, SymbolizedStack *stack) override;
	66
	67	// Always claim we succeeded, so that RenderDataInfo will be called.
	68	bool SymbolizeData(uptr addr, DataInfo *info) override;
	69
	70	// May return NULL if demangling failed.
	71	// This is used by UBSan for type names, and by ASan for global variable
	72	// names. It's expected to return a static buffer that will be reused on each
	73	// call.
	74	const char Demangle(const char name) override;
	75	};
	76
	77	} // namespace __sanitizer
	78
	79	#endif // SANITIZER_SYMBOLIZER_MARKUP_H

lib/tsan/sanitizer_common/sanitizer_symbolizer_markup_constants.h created+49

...	...	@@ -0,0 +1,49 @@
	1	//===-- sanitizer_symbolizer_markup_constants.h
	2	//-----------------------------------===//
	3	//
	4	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	5	// See https://llvm.org/LICENSE.txt for license information.
	6	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	7	//
	8	//===----------------------------------------------------------------------===//
	9	//
	10	// This file is shared between various sanitizers' runtime libraries.
	11	//
	12	// Define string formats and limits for the markup symbolizer.
	13	//===----------------------------------------------------------------------===//
	14	#ifndef SANITIZER_SYMBOLIZER_MARKUP_CONSTANTS_H
	15	#define SANITIZER_SYMBOLIZER_MARKUP_CONSTANTS_H
	16
	17	#include "sanitizer_internal_defs.h"
	18
	19	namespace __sanitizer {
	20
	21	// See the spec at:
	22	// https://fuchsia.googlesource.com/zircon/+/master/docs/symbolizer_markup.md
	23
	24	// This is used by UBSan for type names, and by ASan for global variable names.
	25	constexpr const char *kFormatDemangle = "{{{symbol:%s}}}";
	26	constexpr uptr kFormatDemangleMax = 1024; // Arbitrary.
	27
	28	// Function name or equivalent from PC location.
	29	constexpr const char *kFormatFunction = "{{{pc:%p}}}";
	30	constexpr uptr kFormatFunctionMax = 64; // More than big enough for 64-bit hex.
	31
	32	// Global variable name or equivalent from data memory address.
	33	constexpr const char *kFormatData = "{{{data:%p}}}";
	34
	35	// One frame in a backtrace (printed on a line by itself).
	36	constexpr const char *kFormatFrame = "{{{bt:%d:%p}}}";
	37
	38	// Module contextual element.
	39	constexpr const char *kFormatModule = "{{{module:%zu:%s:elf:%s}}}";
	40
	41	// mmap for a module segment.
	42	constexpr const char *kFormatMmap = "{{{mmap:%p:0x%zx:load:%d:%s:0x%zx}}}";
	43
	44	// Dump trigger element.
	45	#define FORMAT_DUMPFILE "{{{dumpfile:%s:%s}}}"
	46
	47	} // namespace __sanitizer
	48
	49	#endif // SANITIZER_SYMBOLIZER_MARKUP_CONSTANTS_H

lib/tsan/sanitizer_common/sanitizer_symbolizer_markup_fuchsia.cpp created+85

...	...	@@ -0,0 +1,85 @@
	1	//===-- sanitizer_symbolizer_markup_fuchsia.cpp ---------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file is shared between various sanitizers' runtime libraries.
	10	//
	11	// Fuchsia specific implementation of offline markup symbolizer.
	12	//===----------------------------------------------------------------------===//
	13	#include "sanitizer_platform.h"
	14
	15	#if SANITIZER_SYMBOLIZER_MARKUP
	16
	17	# include "sanitizer_common.h"
	18	# include "sanitizer_stacktrace_printer.h"
	19	# include "sanitizer_symbolizer.h"
	20	# include "sanitizer_symbolizer_markup.h"
	21	# include "sanitizer_symbolizer_markup_constants.h"
	22
	23	namespace __sanitizer {
	24
	25	// This is used by UBSan for type names, and by ASan for global variable names.
	26	// It's expected to return a static buffer that will be reused on each call.
	27	const char Symbolizer::Demangle(const char name) {
	28	static char buffer[kFormatDemangleMax];
	29	internal_snprintf(buffer, sizeof(buffer), kFormatDemangle, name);
	30	return buffer;
	31	}
	32
	33	// This is used mostly for suppression matching. Making it work
	34	// would enable "interceptor_via_lib" suppressions. It's also used
	35	// once in UBSan to say "in module ..." in a message that also
	36	// includes an address in the module, so post-processing can already
	37	// pretty-print that so as to indicate the module.
	38	bool Symbolizer::GetModuleNameAndOffsetForPC(uptr pc, const char **module_name,
	39	uptr *module_address) {
	40	return false;
	41	}
	42
	43	// This is mainly used by hwasan for online symbolization. This isn't needed
	44	// since hwasan can always just dump stack frames for offline symbolization.
	45	bool Symbolizer::SymbolizeFrame(uptr addr, FrameInfo *info) { return false; }
	46
	47	// This is used in some places for suppression checking, which we
	48	// don't really support for Fuchsia. It's also used in UBSan to
	49	// identify a PC location to a function name, so we always fill in
	50	// the function member with a string containing markup around the PC
	51	// value.
	52	// TODO(mcgrathr): Under SANITIZER_GO, it's currently used by TSan
	53	// to render stack frames, but that should be changed to use
	54	// RenderStackFrame.
	55	SymbolizedStack *Symbolizer::SymbolizePC(uptr addr) {
	56	SymbolizedStack *s = SymbolizedStack::New(addr);
	57	char buffer[kFormatFunctionMax];
	58	internal_snprintf(buffer, sizeof(buffer), kFormatFunction, addr);
	59	s->info.function = internal_strdup(buffer);
	60	return s;
	61	}
	62
	63	// Always claim we succeeded, so that RenderDataInfo will be called.
	64	bool Symbolizer::SymbolizeData(uptr addr, DataInfo *info) {
	65	info->Clear();
	66	info->start = addr;
	67	return true;
	68	}
	69
	70	// Fuchsia only uses MarkupStackTracePrinter
	71	StackTracePrinter *StackTracePrinter::NewStackTracePrinter() {
	72	return new (GetGlobalLowLevelAllocator()) MarkupStackTracePrinter();
	73	}
	74
	75	void MarkupStackTracePrinter::RenderContext(InternalScopedString *) {}
	76
	77	Symbolizer *Symbolizer::PlatformInit() {
	78	return new (symbolizer_allocator_) Symbolizer({});
	79	}
	80
	81	void Symbolizer::LateInitialize() { Symbolizer::GetOrInit(); }
	82
	83	} // namespace __sanitizer
	84
	85	#endif // SANITIZER_SYMBOLIZER_MARKUP

lib/tsan/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cpp+36-31

...	...	@@ -12,6 +12,7 @@
12	12	//===----------------------------------------------------------------------===//
13	13
14	14	#include "sanitizer_platform.h"
	15	#include "sanitizer_symbolizer_markup.h"
15	16	#if SANITIZER_POSIX
16	17	# include <dlfcn.h> // for dlsym()
17	18	# include <errno.h>
...	...	@@ -56,7 +57,7 @@ const char DemangleCXXABI(const char name) {
56	57	__cxxabiv1::__cxa_demangle(name, 0, 0, 0))
57	58	return demangled_name;
58	59
59		return name;
	60	return nullptr;
60	61	}
61	62
62	63	// As of now, there are no headers for the Swift runtime. Once they are
...	...	@@ -324,9 +325,12 @@ __sanitizer_symbolize_code(const char *ModuleName, u64 ModuleOffset,
324	325	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE bool
325	326	__sanitizer_symbolize_data(const char *ModuleName, u64 ModuleOffset,
326	327	char *Buffer, int MaxLength);
	328	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE bool
	329	__sanitizer_symbolize_frame(const char *ModuleName, u64 ModuleOffset,
	330	char *Buffer, int MaxLength);
327	331	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
328	332	__sanitizer_symbolize_flush();
329		SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE int
	333	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE bool
330	334	__sanitizer_symbolize_demangle(const char Name, char Buffer, int MaxLength);
331	335	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE bool
332	336	__sanitizer_symbolize_set_demangle(bool Demangle);
...	...	@@ -337,19 +341,19 @@ __sanitizer_symbolize_set_inline_frames(bool InlineFrames);
337	341	class InternalSymbolizer final : public SymbolizerTool {
338	342	public:
339	343	static InternalSymbolizer get(LowLevelAllocator alloc) {
340		if (__sanitizer_symbolize_set_demangle)
341		CHECK(__sanitizer_symbolize_set_demangle(common_flags()->demangle));
342		if (__sanitizer_symbolize_set_inline_frames)
343		CHECK(__sanitizer_symbolize_set_inline_frames(
344		common_flags()->symbolize_inline_frames));
345		if (__sanitizer_symbolize_code && __sanitizer_symbolize_data)
346		return new (*alloc) InternalSymbolizer();
347		return 0;
	344	// These one is the most used one, so we will use it to detect a presence of
	345	// internal symbolizer.
	346	if (&__sanitizer_symbolize_code == nullptr)
	347	return nullptr;
	348	CHECK(__sanitizer_symbolize_set_demangle(common_flags()->demangle));
	349	CHECK(__sanitizer_symbolize_set_inline_frames(
	350	common_flags()->symbolize_inline_frames));
	351	return new (*alloc) InternalSymbolizer();
348	352	}
349	353
350	354	bool SymbolizePC(uptr addr, SymbolizedStack *stack) override {
351	355	bool result = __sanitizer_symbolize_code(
352		stack->info.module, stack->info.module_offset, buffer_, kBufferSize);
	356	stack->info.module, stack->info.module_offset, buffer_, sizeof(buffer_));
353	357	if (result)
354	358	ParseSymbolizePCOutput(buffer_, stack);
355	359	return result;
...	...	@@ -357,7 +361,7 @@ class InternalSymbolizer final : public SymbolizerTool {
357	361
358	362	bool SymbolizeData(uptr addr, DataInfo *info) override {
359	363	bool result = __sanitizer_symbolize_data(info->module, info->module_offset,
360		buffer_, kBufferSize);
	364	buffer_, sizeof(buffer_));
361	365	if (result) {
362	366	ParseSymbolizeDataOutput(buffer_, info);
363	367	info->start += (addr - info->module_offset); // Add the base address.
...	...	@@ -365,34 +369,29 @@ class InternalSymbolizer final : public SymbolizerTool {
365	369	return result;
366	370	}
367	371
368		void Flush() override {
369		if (__sanitizer_symbolize_flush)
370		__sanitizer_symbolize_flush();
	372	bool SymbolizeFrame(uptr addr, FrameInfo *info) override {
	373	bool result = __sanitizer_symbolize_frame(info->module, info->module_offset,
	374	buffer_, sizeof(buffer_));
	375	if (result)
	376	ParseSymbolizeFrameOutput(buffer_, &info->locals);
	377	return result;
371	378	}
372	379
	380	void Flush() override { __sanitizer_symbolize_flush(); }
	381
373	382	const char Demangle(const char name) override {
374		if (__sanitizer_symbolize_demangle) {
375		for (uptr res_length = 1024;
376		res_length <= InternalSizeClassMap::kMaxSize;) {
377		char res_buff = static_cast<char >(InternalAlloc(res_length));
378		uptr req_length =
379		__sanitizer_symbolize_demangle(name, res_buff, res_length);
380		if (req_length > res_length) {
381		res_length = req_length + 1;
382		InternalFree(res_buff);
383		continue;
384		}
385		return res_buff;
386		}
	383	if (__sanitizer_symbolize_demangle(name, buffer_, sizeof(buffer_))) {
	384	char *res_buff = nullptr;
	385	ExtractToken(buffer_, "", &res_buff);
	386	return res_buff;
387	387	}
388		return name;
	388	return nullptr;
389	389	}
390	390
391	391	private:
392	392	InternalSymbolizer() {}
393	393
394		static const int kBufferSize = 16 * 1024;
395		char buffer_[kBufferSize];
	394	char buffer_[16 * 1024];
396	395	};
397	396	# else // SANITIZER_SUPPORTS_WEAK_HOOKS
398	397
...	...	@@ -470,6 +469,12 @@ static void ChooseSymbolizerTools(IntrusiveList<SymbolizerTool> *list,
470	469	VReport(2, "Symbolizer is disabled.\n");
471	470	return;
472	471	}
	472	if (common_flags()->enable_symbolizer_markup) {
	473	VReport(2, "Using symbolizer markup");
	474	SymbolizerTool tool = new (allocator) MarkupSymbolizerTool();
	475	CHECK(tool);
	476	list->push_back(tool);
	477	}
473	478	if (IsAllocatorOutOfMemory()) {
474	479	VReport(2, "Cannot use internal symbolizer: out of memory\n");
475	480	} else if (SymbolizerTool *tool = InternalSymbolizer::get(allocator)) {

lib/tsan/sanitizer_common/sanitizer_symbolizer_report.cpp+62-18

...	...	@@ -28,14 +28,41 @@
28	28	namespace __sanitizer {
29	29
30	30	#if !SANITIZER_GO
	31
	32	static bool FrameIsInternal(const SymbolizedStack *frame) {
	33	if (!frame)
	34	return true;
	35	const char *file = frame->info.file;
	36	const char *module = frame->info.module;
	37	// On Gentoo, the path is g++-, so there's not* a missing /.
	38	if (file && (internal_strstr(file, "/compiler-rt/lib/") \|\|
	39	internal_strstr(file, "/include/c++/") \|\|
	40	internal_strstr(file, "/include/g++")))
	41	return true;
	42	if (file && internal_strstr(file, "\\compiler-rt\\lib\\"))
	43	return true;
	44	if (module && (internal_strstr(module, "libclang_rt.")))
	45	return true;
	46	if (module && (internal_strstr(module, "clang_rt.")))
	47	return true;
	48	return false;
	49	}
	50
	51	const SymbolizedStack SkipInternalFrames(const SymbolizedStack frames) {
	52	for (const SymbolizedStack *f = frames; f; f = f->next)
	53	if (!FrameIsInternal(f))
	54	return f;
	55	return nullptr;
	56	}
	57
31	58	void ReportErrorSummary(const char *error_type, const AddressInfo &info,
32	59	const char *alt_tool_name) {
33	60	if (!common_flags()->print_summary) return;
34	61	InternalScopedString buff;
35		buff.append("%s ", error_type);
36		RenderFrame(&buff, "%L %F", 0, info.address, &info,
37		common_flags()->symbolize_vs_style,
38		common_flags()->strip_path_prefix);
	62	buff.AppendF("%s ", error_type);
	63	StackTracePrinter::GetOrInit()->RenderFrame(
	64	&buff, "%L %F", 0, info.address, &info,
	65	common_flags()->symbolize_vs_style, common_flags()->strip_path_prefix);
39	66	ReportErrorSummary(buff.data(), alt_tool_name);
40	67	}
41	68	#endif
...	...	@@ -75,16 +102,33 @@ void ReportErrorSummary(const char error_type, const StackTrace stack,
75	102	#if !SANITIZER_GO
76	103	if (!common_flags()->print_summary)
77	104	return;
78		if (stack->size == 0) {
79		ReportErrorSummary(error_type);
80		return;
	105
	106	// Find first non-internal stack frame.
	107	for (uptr i = 0; i < stack->size; ++i) {
	108	uptr pc = StackTrace::GetPreviousInstructionPc(stack->trace[i]);
	109	SymbolizedStackHolder symbolized_stack(
	110	Symbolizer::GetOrInit()->SymbolizePC(pc));
	111	if (const SymbolizedStack *frame = symbolized_stack.get()) {
	112	if (const SymbolizedStack *summary_frame = SkipInternalFrames(frame)) {
	113	ReportErrorSummary(error_type, summary_frame->info, alt_tool_name);
	114	return;
	115	}
	116	}
	117	}
	118
	119	// Fallback to the top one.
	120	if (stack->size) {
	121	uptr pc = StackTrace::GetPreviousInstructionPc(stack->trace[0]);
	122	SymbolizedStackHolder symbolized_stack(
	123	Symbolizer::GetOrInit()->SymbolizePC(pc));
	124	if (const SymbolizedStack *frame = symbolized_stack.get()) {
	125	ReportErrorSummary(error_type, frame->info, alt_tool_name);
	126	return;
	127	}
81	128	}
82		// Currently, we include the first stack frame into the report summary.
83		// Maybe sometimes we need to choose another frame (e.g. skip memcpy/etc).
84		uptr pc = StackTrace::GetPreviousInstructionPc(stack->trace[0]);
85		SymbolizedStack *frame = Symbolizer::GetOrInit()->SymbolizePC(pc);
86		ReportErrorSummary(error_type, frame->info, alt_tool_name);
87		frame->ClearAll();
	129
	130	// Fallback to a summary without location.
	131	ReportErrorSummary(error_type);
88	132	#endif
89	133	}
90	134
...	...	@@ -148,22 +192,22 @@ static void MaybeReportNonExecRegion(uptr pc) {
148	192	static void PrintMemoryByte(InternalScopedString str, const char before,
149	193	u8 byte) {
150	194	SanitizerCommonDecorator d;
151		str->append("%s%s%x%x%s ", before, d.MemoryByte(), byte >> 4, byte & 15,
152		d.Default());
	195	str->AppendF("%s%s%x%x%s ", before, d.MemoryByte(), byte >> 4, byte & 15,
	196	d.Default());
153	197	}
154	198
155	199	static void MaybeDumpInstructionBytes(uptr pc) {
156	200	if (!common_flags()->dump_instruction_bytes \|\| (pc < GetPageSizeCached()))
157	201	return;
158	202	InternalScopedString str;
159		str.append("First 16 instruction bytes at pc: ");
	203	str.AppendF("First 16 instruction bytes at pc: ");
160	204	if (IsAccessibleMemoryRange(pc, 16)) {
161	205	for (int i = 0; i < 16; ++i) {
162	206	PrintMemoryByte(&str, "", ((u8 *)pc)[i]);
163	207	}
164		str.append("\n");
	208	str.AppendF("\n");
165	209	} else {
166		str.append("unaccessible\n");
	210	str.AppendF("unaccessible\n");
167	211	}
168	212	Report("%s", str.data());
169	213	}

lib/tsan/sanitizer_common/sanitizer_symbolizer_report_fuchsia.cpp created+33

...	...	@@ -0,0 +1,33 @@
	1	//===-- sanitizer_symbolizer_report_fuchsia.cpp
	2	//-----------------------------------===//
	3	//
	4	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	5	// See https://llvm.org/LICENSE.txt for license information.
	6	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	7	//
	8	//===----------------------------------------------------------------------===//
	9	//
	10	// Implementation of the report functions for fuchsia.
	11	//
	12	//===----------------------------------------------------------------------===//
	13
	14	#include "sanitizer_platform.h"
	15
	16	#if SANITIZER_SYMBOLIZER_MARKUP
	17
	18	# include "sanitizer_common.h"
	19
	20	namespace __sanitizer {
	21	void StartReportDeadlySignal() {}
	22
	23	void ReportDeadlySignal(const SignalContext &sig, u32 tid,
	24	UnwindSignalStackCallbackType unwind,
	25	const void *unwind_context) {}
	26
	27	void HandleDeadlySignal(void siginfo, void context, u32 tid,
	28	UnwindSignalStackCallbackType unwind,
	29	const void *unwind_context) {}
	30
	31	} // namespace __sanitizer
	32
	33	#endif // SANITIZER_SYMBOLIZER_MARKUP

lib/tsan/sanitizer_common/sanitizer_symbolizer_win.cpp+2-4

...	...	@@ -175,9 +175,7 @@ const char WinSymbolizerTool::Demangle(const char name) {
175	175	return name;
176	176	}
177	177
178		const char Symbolizer::PlatformDemangle(const char name) {
179		return name;
180		}
	178	const char Symbolizer::PlatformDemangle(const char name) { return nullptr; }
181	179
182	180	namespace {
183	181	struct ScopedHandle {
...	...	@@ -233,7 +231,7 @@ bool SymbolizerProcess::StartSymbolizerSubprocess() {
233	231	CHECK(!internal_strchr(arg, '"') && "quotes in args unsupported");
234	232	CHECK(arglen > 0 && arg[arglen - 1] != '\\' &&
235	233	"args ending in backslash and empty args unsupported");
236		command_line.append("\"%s\" ", arg);
	234	command_line.AppendF("\"%s\" ", arg);
237	235	}
238	236	VReport(3, "Launching symbolizer command: %s\n", command_line.data());
239	237

lib/tsan/sanitizer_common/sanitizer_syscall_linux_hexagon.inc created+131

...	...	@@ -0,0 +1,131 @@
	1	//===-- sanitizer_syscall_linux_hexagon.inc ---------------------- C++ --===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// Implementations of internal_syscall and internal_iserror for Linux/hexagon.
	10	//
	11	//===----------------------------------------------------------------------===//
	12
	13	#define SYSCALL(name) __NR_##name
	14
	15	#define __internal_syscall_LL_E(x) \
	16	((union { \
	17	long long ll; \
	18	long l[2]; \
	19	}){.ll = x}) \
	20	.l[0], \
	21	((union { \
	22	long long ll; \
	23	long l[2]; \
	24	}){.ll = x}) \
	25	.l[1]
	26	#define __internal_syscall_LL_O(x) 0, __SYSCALL_LL_E((x))
	27
	28	#define __asm_syscall(...) \
	29	do { \
	30	__asm__ __volatile__("trap0(#1)" : "=r"(r0) : __VA_ARGS__ : "memory"); \
	31	return r0; \
	32	} while (0)
	33
	34	#define __internal_syscall0(n) (__internal_syscall)(n)
	35
	36	static uptr __internal_syscall(long n) {
	37	register u32 r6 __asm__("r6") = n;
	38	register u32 r0 __asm__("r0");
	39	__asm_syscall("r"(r6));
	40	}
	41
	42	#define __internal_syscall1(n, a1) (__internal_syscall)(n, (long)(a1))
	43
	44	static uptr __internal_syscall(long n, long a) {
	45	register u32 r6 __asm__("r6") = n;
	46	register u32 r0 __asm__("r0") = a;
	47	__asm_syscall("r"(r6), "0"(r0));
	48	}
	49
	50	#define __internal_syscall2(n, a1, a2) \
	51	(__internal_syscall)(n, (long)(a1), (long)(a2))
	52
	53	static uptr __internal_syscall(long n, long a, long b) {
	54	register u32 r6 __asm__("r6") = n;
	55	register u32 r0 __asm__("r0") = a;
	56	register u32 r1 __asm__("r1") = b;
	57	__asm_syscall("r"(r6), "0"(r0), "r"(r1));
	58	}
	59
	60	#define __internal_syscall3(n, a1, a2, a3) \
	61	(__internal_syscall)(n, (long)(a1), (long)(a2), (long)(a3))
	62
	63	static uptr __internal_syscall(long n, long a, long b, long c) {
	64	register u32 r6 __asm__("r6") = n;
	65	register u32 r0 __asm__("r0") = a;
	66	register u32 r1 __asm__("r1") = b;
	67	register u32 r2 __asm__("r2") = c;
	68	__asm_syscall("r"(r6), "0"(r0), "r"(r1), "r"(r2));
	69	}
	70
	71	#define __internal_syscall4(n, a1, a2, a3, a4) \
	72	(__internal_syscall)(n, (long)(a1), (long)(a2), (long)(a3), (long)(a4))
	73
	74	static uptr __internal_syscall(long n, long a, long b, long c, long d) {
	75	register u32 r6 __asm__("r6") = n;
	76	register u32 r0 __asm__("r0") = a;
	77	register u32 r1 __asm__("r1") = b;
	78	register u32 r2 __asm__("r2") = c;
	79	register u32 r3 __asm__("r3") = d;
	80	__asm_syscall("r"(r6), "0"(r0), "r"(r1), "r"(r2), "r"(r3));
	81	}
	82
	83	#define __internal_syscall5(n, a1, a2, a3, a4, a5) \
	84	(__internal_syscall)(n, (long)(a1), (long)(a2), (long)(a3), (long)(a4), \
	85	(long)(a5))
	86
	87	static uptr __internal_syscall(long n, long a, long b, long c, long d, long e) {
	88	register u32 r6 __asm__("r6") = n;
	89	register u32 r0 __asm__("r0") = a;
	90	register u32 r1 __asm__("r1") = b;
	91	register u32 r2 __asm__("r2") = c;
	92	register u32 r3 __asm__("r3") = d;
	93	register u32 r4 __asm__("r4") = e;
	94	__asm_syscall("r"(r6), "0"(r0), "r"(r1), "r"(r2), "r"(r3), "r"(r4));
	95	}
	96
	97	#define __internal_syscall6(n, a1, a2, a3, a4, a5, a6) \
	98	(__internal_syscall)(n, (long)(a1), (long)(a2), (long)(a3), (long)(a4), \
	99	(long)(a5), (long)(a6))
	100
	101	static uptr __internal_syscall(long n, long a, long b, long c, long d, long e,
	102	long f) {
	103	register u32 r6 __asm__("r6") = n;
	104	register u32 r0 __asm__("r0") = a;
	105	register u32 r1 __asm__("r1") = b;
	106	register u32 r2 __asm__("r2") = c;
	107	register u32 r3 __asm__("r3") = d;
	108	register u32 r4 __asm__("r4") = e;
	109	register u32 r5 __asm__("r5") = f;
	110	__asm_syscall("r"(r6), "0"(r0), "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5));
	111	}
	112
	113	#define __SYSCALL_NARGS_X(a1, a2, a3, a4, a5, a6, a7, a8, n, ...) n
	114	#define __SYSCALL_NARGS(...) \
	115	__SYSCALL_NARGS_X(__VA_ARGS__, 7, 6, 5, 4, 3, 2, 1, 0, )
	116	#define __SYSCALL_CONCAT_X(a, b) a##b
	117	#define __SYSCALL_CONCAT(a, b) __SYSCALL_CONCAT_X(a, b)
	118	#define __SYSCALL_DISP(b, ...) \
	119	__SYSCALL_CONCAT(b, __SYSCALL_NARGS(__VA_ARGS__))(__VA_ARGS__)
	120
	121	#define internal_syscall(...) __SYSCALL_DISP(__internal_syscall, __VA_ARGS__)
	122
	123	// Helper function used to avoid clobbering of errno.
	124	bool internal_iserror(uptr retval, int *rverrno) {
	125	if (retval >= (uptr)-4095) {
	126	if (rverrno)
	127	*rverrno = -retval;
	128	return true;
	129	}
	130	return false;
	131	}

lib/tsan/sanitizer_common/sanitizer_syscall_linux_loongarch64.inc created+171

...	...	@@ -0,0 +1,171 @@
	1	//===-- sanitizer_syscall_linux_loongarch64.inc ------------------ C++ --===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// Implementations of internal_syscall and internal_iserror for
	10	// Linux/loongarch64.
	11	//
	12	//===----------------------------------------------------------------------===//
	13
	14	// About local register variables:
	15	// https://gcc.gnu.org/onlinedocs/gcc/Local-Register-Variables.html#Local-Register-Variables
	16	//
	17	// Kernel ABI:
	18	// https://lore.kernel.org/loongarch/1f353678-3398-e30b-1c87-6edb278f74db@xen0n.name/T/#m1613bc86c2d7bf5f6da92bd62984302bfd699a2f
	19	// syscall number is placed in a7
	20	// parameters, if present, are placed in a0-a6
	21	// upon return:
	22	// the return value is placed in a0
	23	// t0-t8 should be considered clobbered
	24	// all other registers are preserved
	25	#define SYSCALL(name) __NR_##name
	26
	27	#define INTERNAL_SYSCALL_CLOBBERS \
	28	"memory", "$t0", "$t1", "$t2", "$t3", "$t4", "$t5", "$t6", "$t7", "$t8"
	29
	30	static uptr __internal_syscall(u64 nr) {
	31	register u64 a7 asm("$a7") = nr;
	32	register u64 a0 asm("$a0");
	33	__asm__ volatile("syscall 0\n\t"
	34	: "=r"(a0)
	35	: "r"(a7)
	36	: INTERNAL_SYSCALL_CLOBBERS);
	37	return a0;
	38	}
	39	#define __internal_syscall0(n) (__internal_syscall)(n)
	40
	41	static uptr __internal_syscall(u64 nr, u64 arg1) {
	42	register u64 a7 asm("$a7") = nr;
	43	register u64 a0 asm("$a0") = arg1;
	44	__asm__ volatile("syscall 0\n\t"
	45	: "+r"(a0)
	46	: "r"(a7)
	47	: INTERNAL_SYSCALL_CLOBBERS);
	48	return a0;
	49	}
	50	#define __internal_syscall1(n, a1) (__internal_syscall)(n, (u64)(a1))
	51
	52	static uptr __internal_syscall(u64 nr, u64 arg1, long arg2) {
	53	register u64 a7 asm("$a7") = nr;
	54	register u64 a0 asm("$a0") = arg1;
	55	register u64 a1 asm("$a1") = arg2;
	56	__asm__ volatile("syscall 0\n\t"
	57	: "+r"(a0)
	58	: "r"(a7), "r"(a1)
	59	: INTERNAL_SYSCALL_CLOBBERS);
	60	return a0;
	61	}
	62	#define __internal_syscall2(n, a1, a2) \
	63	(__internal_syscall)(n, (u64)(a1), (long)(a2))
	64
	65	static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3) {
	66	register u64 a7 asm("$a7") = nr;
	67	register u64 a0 asm("$a0") = arg1;
	68	register u64 a1 asm("$a1") = arg2;
	69	register u64 a2 asm("$a2") = arg3;
	70	__asm__ volatile("syscall 0\n\t"
	71	: "+r"(a0)
	72	: "r"(a7), "r"(a1), "r"(a2)
	73	: INTERNAL_SYSCALL_CLOBBERS);
	74	return a0;
	75	}
	76	#define __internal_syscall3(n, a1, a2, a3) \
	77	(__internal_syscall)(n, (u64)(a1), (long)(a2), (long)(a3))
	78
	79	static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3,
	80	u64 arg4) {
	81	register u64 a7 asm("$a7") = nr;
	82	register u64 a0 asm("$a0") = arg1;
	83	register u64 a1 asm("$a1") = arg2;
	84	register u64 a2 asm("$a2") = arg3;
	85	register u64 a3 asm("$a3") = arg4;
	86	__asm__ volatile("syscall 0\n\t"
	87	: "+r"(a0)
	88	: "r"(a7), "r"(a1), "r"(a2), "r"(a3)
	89	: INTERNAL_SYSCALL_CLOBBERS);
	90	return a0;
	91	}
	92	#define __internal_syscall4(n, a1, a2, a3, a4) \
	93	(__internal_syscall)(n, (u64)(a1), (long)(a2), (long)(a3), (long)(a4))
	94
	95	static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3, u64 arg4,
	96	long arg5) {
	97	register u64 a7 asm("$a7") = nr;
	98	register u64 a0 asm("$a0") = arg1;
	99	register u64 a1 asm("$a1") = arg2;
	100	register u64 a2 asm("$a2") = arg3;
	101	register u64 a3 asm("$a3") = arg4;
	102	register u64 a4 asm("$a4") = arg5;
	103	__asm__ volatile("syscall 0\n\t"
	104	: "+r"(a0)
	105	: "r"(a7), "r"(a1), "r"(a2), "r"(a3), "r"(a4)
	106	: INTERNAL_SYSCALL_CLOBBERS);
	107	return a0;
	108	}
	109	#define __internal_syscall5(n, a1, a2, a3, a4, a5) \
	110	(__internal_syscall)(n, (u64)(a1), (long)(a2), (long)(a3), (long)(a4), \
	111	(u64)(a5))
	112
	113	static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3, u64 arg4,
	114	long arg5, long arg6) {
	115	register u64 a7 asm("$a7") = nr;
	116	register u64 a0 asm("$a0") = arg1;
	117	register u64 a1 asm("$a1") = arg2;
	118	register u64 a2 asm("$a2") = arg3;
	119	register u64 a3 asm("$a3") = arg4;
	120	register u64 a4 asm("$a4") = arg5;
	121	register u64 a5 asm("$a5") = arg6;
	122	__asm__ volatile("syscall 0\n\t"
	123	: "+r"(a0)
	124	: "r"(a7), "r"(a1), "r"(a2), "r"(a3), "r"(a4), "r"(a5)
	125	: INTERNAL_SYSCALL_CLOBBERS);
	126	return a0;
	127	}
	128	#define __internal_syscall6(n, a1, a2, a3, a4, a5, a6) \
	129	(__internal_syscall)(n, (u64)(a1), (long)(a2), (long)(a3), (long)(a4), \
	130	(u64)(a5), (long)(a6))
	131
	132	static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3, u64 arg4,
	133	long arg5, long arg6, long arg7) {
	134	register u64 a7 asm("$a7") = nr;
	135	register u64 a0 asm("$a0") = arg1;
	136	register u64 a1 asm("$a1") = arg2;
	137	register u64 a2 asm("$a2") = arg3;
	138	register u64 a3 asm("$a3") = arg4;
	139	register u64 a4 asm("$a4") = arg5;
	140	register u64 a5 asm("$a5") = arg6;
	141	register u64 a6 asm("$a6") = arg7;
	142	__asm__ volatile("syscall 0\n\t"
	143	: "+r"(a0)
	144	: "r"(a7), "r"(a1), "r"(a2), "r"(a3), "r"(a4), "r"(a5),
	145	"r"(a6)
	146	: INTERNAL_SYSCALL_CLOBBERS);
	147	return a0;
	148	}
	149	#define __internal_syscall7(n, a1, a2, a3, a4, a5, a6, a7) \
	150	(__internal_syscall)(n, (u64)(a1), (long)(a2), (long)(a3), (long)(a4), \
	151	(u64)(a5), (long)(a6), (long)(a7))
	152
	153	#define __SYSCALL_NARGS_X(a1, a2, a3, a4, a5, a6, a7, a8, n, ...) n
	154	#define __SYSCALL_NARGS(...) \
	155	__SYSCALL_NARGS_X(__VA_ARGS__, 7, 6, 5, 4, 3, 2, 1, 0, )
	156	#define __SYSCALL_CONCAT_X(a, b) a##b
	157	#define __SYSCALL_CONCAT(a, b) __SYSCALL_CONCAT_X(a, b)
	158	#define __SYSCALL_DISP(b, ...) \
	159	__SYSCALL_CONCAT(b, __SYSCALL_NARGS(__VA_ARGS__))(__VA_ARGS__)
	160
	161	#define internal_syscall(...) __SYSCALL_DISP(__internal_syscall, __VA_ARGS__)
	162
	163	// Helper function used to avoid clobbering of errno.
	164	bool internal_iserror(uptr retval, int *internal_errno) {
	165	if (retval >= (uptr)-4095) {
	166	if (internal_errno)
	167	*internal_errno = -retval;
	168	return true;
	169	}
	170	return false;
	171	}

lib/tsan/sanitizer_common/sanitizer_syscall_linux_riscv64.inc created+174

...	...	@@ -0,0 +1,174 @@
	1	//===-- sanitizer_syscall_linux_riscv64.inc ---------------------- C++ --===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// Implementations of internal_syscall and internal_iserror for Linux/riscv64.
	10	//
	11	//===----------------------------------------------------------------------===//
	12
	13	// About local register variables:
	14	// https://gcc.gnu.org/onlinedocs/gcc/Local-Register-Variables.html#Local-Register-Variables
	15	//
	16	// Kernel ABI...
	17	// To my surprise I haven't found much information regarding it.
	18	// Kernel source and internet browsing shows that:
	19	// syscall number is passed in a7
	20	// (http://man7.org/linux/man-pages/man2/syscall.2.html) results are return in
	21	// a0 and a1 (http://man7.org/linux/man-pages/man2/syscall.2.html) arguments
	22	// are passed in: a0-a7 (see below)
	23	//
	24	// Regarding the arguments. The only "documentation" I could find is
	25	// this comment (!!!) by Bruce Hold on google forums (!!!):
	26	// https://groups.google.com/a/groups.riscv.org/forum/#!topic/sw-dev/exbrzM3GZDQ
	27	// Confirmed by inspecting glibc sources.
	28	// Great way to document things.
	29	#define SYSCALL(name) __NR_##name
	30
	31	#define INTERNAL_SYSCALL_CLOBBERS "memory"
	32
	33	static uptr __internal_syscall(u64 nr) {
	34	register u64 a7 asm("a7") = nr;
	35	register u64 a0 asm("a0");
	36	__asm__ volatile("ecall\n\t"
	37	: "=r"(a0)
	38	: "r"(a7)
	39	: INTERNAL_SYSCALL_CLOBBERS);
	40	return a0;
	41	}
	42	#define __internal_syscall0(n) (__internal_syscall)(n)
	43
	44	static uptr __internal_syscall(u64 nr, u64 arg1) {
	45	register u64 a7 asm("a7") = nr;
	46	register u64 a0 asm("a0") = arg1;
	47	__asm__ volatile("ecall\n\t"
	48	: "+r"(a0)
	49	: "r"(a7)
	50	: INTERNAL_SYSCALL_CLOBBERS);
	51	return a0;
	52	}
	53	#define __internal_syscall1(n, a1) (__internal_syscall)(n, (u64)(a1))
	54
	55	static uptr __internal_syscall(u64 nr, u64 arg1, long arg2) {
	56	register u64 a7 asm("a7") = nr;
	57	register u64 a0 asm("a0") = arg1;
	58	register u64 a1 asm("a1") = arg2;
	59	__asm__ volatile("ecall\n\t"
	60	: "+r"(a0)
	61	: "r"(a7), "r"(a1)
	62	: INTERNAL_SYSCALL_CLOBBERS);
	63	return a0;
	64	}
	65	#define __internal_syscall2(n, a1, a2) \
	66	(__internal_syscall)(n, (u64)(a1), (long)(a2))
	67
	68	static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3) {
	69	register u64 a7 asm("a7") = nr;
	70	register u64 a0 asm("a0") = arg1;
	71	register u64 a1 asm("a1") = arg2;
	72	register u64 a2 asm("a2") = arg3;
	73	__asm__ volatile("ecall\n\t"
	74	: "+r"(a0)
	75	: "r"(a7), "r"(a1), "r"(a2)
	76	: INTERNAL_SYSCALL_CLOBBERS);
	77	return a0;
	78	}
	79	#define __internal_syscall3(n, a1, a2, a3) \
	80	(__internal_syscall)(n, (u64)(a1), (long)(a2), (long)(a3))
	81
	82	static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3,
	83	u64 arg4) {
	84	register u64 a7 asm("a7") = nr;
	85	register u64 a0 asm("a0") = arg1;
	86	register u64 a1 asm("a1") = arg2;
	87	register u64 a2 asm("a2") = arg3;
	88	register u64 a3 asm("a3") = arg4;
	89	__asm__ volatile("ecall\n\t"
	90	: "+r"(a0)
	91	: "r"(a7), "r"(a1), "r"(a2), "r"(a3)
	92	: INTERNAL_SYSCALL_CLOBBERS);
	93	return a0;
	94	}
	95	#define __internal_syscall4(n, a1, a2, a3, a4) \
	96	(__internal_syscall)(n, (u64)(a1), (long)(a2), (long)(a3), (long)(a4))
	97
	98	static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3, u64 arg4,
	99	long arg5) {
	100	register u64 a7 asm("a7") = nr;
	101	register u64 a0 asm("a0") = arg1;
	102	register u64 a1 asm("a1") = arg2;
	103	register u64 a2 asm("a2") = arg3;
	104	register u64 a3 asm("a3") = arg4;
	105	register u64 a4 asm("a4") = arg5;
	106	__asm__ volatile("ecall\n\t"
	107	: "+r"(a0)
	108	: "r"(a7), "r"(a1), "r"(a2), "r"(a3), "r"(a4)
	109	: INTERNAL_SYSCALL_CLOBBERS);
	110	return a0;
	111	}
	112	#define __internal_syscall5(n, a1, a2, a3, a4, a5) \
	113	(__internal_syscall)(n, (u64)(a1), (long)(a2), (long)(a3), (long)(a4), \
	114	(u64)(a5))
	115
	116	static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3, u64 arg4,
	117	long arg5, long arg6) {
	118	register u64 a7 asm("a7") = nr;
	119	register u64 a0 asm("a0") = arg1;
	120	register u64 a1 asm("a1") = arg2;
	121	register u64 a2 asm("a2") = arg3;
	122	register u64 a3 asm("a3") = arg4;
	123	register u64 a4 asm("a4") = arg5;
	124	register u64 a5 asm("a5") = arg6;
	125	__asm__ volatile("ecall\n\t"
	126	: "+r"(a0)
	127	: "r"(a7), "r"(a1), "r"(a2), "r"(a3), "r"(a4), "r"(a5)
	128	: INTERNAL_SYSCALL_CLOBBERS);
	129	return a0;
	130	}
	131	#define __internal_syscall6(n, a1, a2, a3, a4, a5, a6) \
	132	(__internal_syscall)(n, (u64)(a1), (long)(a2), (long)(a3), (long)(a4), \
	133	(u64)(a5), (long)(a6))
	134
	135	static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3, u64 arg4,
	136	long arg5, long arg6, long arg7) {
	137	register u64 a7 asm("a7") = nr;
	138	register u64 a0 asm("a0") = arg1;
	139	register u64 a1 asm("a1") = arg2;
	140	register u64 a2 asm("a2") = arg3;
	141	register u64 a3 asm("a3") = arg4;
	142	register u64 a4 asm("a4") = arg5;
	143	register u64 a5 asm("a5") = arg6;
	144	register u64 a6 asm("a6") = arg7;
	145	__asm__ volatile("ecall\n\t"
	146	: "+r"(a0)
	147	: "r"(a7), "r"(a1), "r"(a2), "r"(a3), "r"(a4), "r"(a5),
	148	"r"(a6)
	149	: INTERNAL_SYSCALL_CLOBBERS);
	150	return a0;
	151	}
	152	#define __internal_syscall7(n, a1, a2, a3, a4, a5, a6, a7) \
	153	(__internal_syscall)(n, (u64)(a1), (long)(a2), (long)(a3), (long)(a4), \
	154	(u64)(a5), (long)(a6), (long)(a7))
	155
	156	#define __SYSCALL_NARGS_X(a1, a2, a3, a4, a5, a6, a7, a8, n, ...) n
	157	#define __SYSCALL_NARGS(...) \
	158	__SYSCALL_NARGS_X(__VA_ARGS__, 7, 6, 5, 4, 3, 2, 1, 0, )
	159	#define __SYSCALL_CONCAT_X(a, b) a##b
	160	#define __SYSCALL_CONCAT(a, b) __SYSCALL_CONCAT_X(a, b)
	161	#define __SYSCALL_DISP(b, ...) \
	162	__SYSCALL_CONCAT(b, __SYSCALL_NARGS(__VA_ARGS__))(__VA_ARGS__)
	163
	164	#define internal_syscall(...) __SYSCALL_DISP(__internal_syscall, __VA_ARGS__)
	165
	166	// Helper function used to avoid clobbering of errno.
	167	bool internal_iserror(uptr retval, int *rverrno) {
	168	if (retval >= (uptr)-4095) {
	169	if (rverrno)
	170	*rverrno = -retval;
	171	return true;
	172	}
	173	return false;
	174	}

lib/tsan/sanitizer_common/sanitizer_thread_arg_retval.cpp+19-4

...	...	@@ -23,6 +23,9 @@ void ThreadArgRetval::CreateLocked(uptr thread, bool detached,
23	23	Data& t = data_[thread];
24	24	t = {};
25	25	t.gen = gen_++;
	26	static_assert(sizeof(gen_) == sizeof(u32) && kInvalidGen == UINT32_MAX);
	27	if (gen_ == kInvalidGen)
	28	gen_ = 0;
26	29	t.detached = detached;
27	30	t.args = args;
28	31	}
...	...	@@ -53,16 +56,28 @@ void ThreadArgRetval::Finish(uptr thread, void* retval) {
53	56	u32 ThreadArgRetval::BeforeJoin(uptr thread) const {
54	57	__sanitizer::Lock lock(&mtx_);
55	58	auto t = data_.find(thread);
56		CHECK(t);
57		CHECK(!t->second.detached);
58		return t->second.gen;
	59	if (t && !t->second.detached) {
	60	return t->second.gen;
	61	}
	62	if (!common_flags()->detect_invalid_join)
	63	return kInvalidGen;
	64	const char* reason = "unknown";
	65	if (!t) {
	66	reason = "already joined";
	67	} else if (t->second.detached) {
	68	reason = "detached";
	69	}
	70	Report("ERROR: %s: Joining %s thread, aborting.\n", SanitizerToolName,
	71	reason);
	72	Die();
59	73	}
60	74
61	75	void ThreadArgRetval::AfterJoin(uptr thread, u32 gen) {
62	76	__sanitizer::Lock lock(&mtx_);
63	77	auto t = data_.find(thread);
64	78	if (!t \|\| gen != t->second.gen) {
65		// Thread was reused and erased by any other event.
	79	// Thread was reused and erased by any other event, or we had an invalid
	80	// join.
66	81	return;
67	82	}
68	83	CHECK(!t->second.detached);

lib/tsan/sanitizer_common/sanitizer_thread_arg_retval.h+1

...	...	@@ -93,6 +93,7 @@ class SANITIZER_MUTEX ThreadArgRetval {
93	93	// will keep pointers alive forever, missing leaks caused by cancelation.
94	94
95	95	private:
	96	static const u32 kInvalidGen = UINT32_MAX;
96	97	struct Data {
97	98	Args args;
98	99	u32 gen; // Avoid collision if thread id re-used.

lib/tsan/sanitizer_common/sanitizer_tls_get_addr.cpp+8-7

...	...	@@ -121,25 +121,26 @@ DTLS::DTV DTLS_on_tls_get_addr(void arg_void, void *res,
121	121	uptr tls_size = 0;
122	122	uptr tls_beg = reinterpret_cast<uptr>(res) - arg->offset - kDtvOffset;
123	123	VReport(2,
124		"__tls_get_addr: %p {0x%zx,0x%zx} => %p; tls_beg: 0x%zx; sp: %p "
	124	"__tls_get_addr: %p {0x%zx,0x%zx} => %p; tls_beg: %p; sp: %p "
125	125	"num_live_dtls %zd\n",
126		(void )arg, arg->dso_id, arg->offset, res, tls_beg, (void )&tls_beg,
	126	(void )arg, arg->dso_id, arg->offset, res, (void )tls_beg,
	127	(void *)&tls_beg,
127	128	atomic_load(&number_of_live_dtls, memory_order_relaxed));
128	129	if (dtls.last_memalign_ptr == tls_beg) {
129	130	tls_size = dtls.last_memalign_size;
130		VReport(2, "__tls_get_addr: glibc <=2.24 suspected; tls={0x%zx,0x%zx}\n",
131		tls_beg, tls_size);
	131	VReport(2, "__tls_get_addr: glibc <=2.24 suspected; tls={%p,0x%zx}\n",
	132	(void *)tls_beg, tls_size);
132	133	} else if (tls_beg >= static_tls_begin && tls_beg < static_tls_end) {
133	134	// This is the static TLS block which was initialized / unpoisoned at thread
134	135	// creation.
135		VReport(2, "__tls_get_addr: static tls: 0x%zx\n", tls_beg);
	136	VReport(2, "__tls_get_addr: static tls: %p\n", (void *)tls_beg);
136	137	tls_size = 0;
137	138	} else if (const void *start =
138	139	__sanitizer_get_allocated_begin((void *)tls_beg)) {
139	140	tls_beg = (uptr)start;
140	141	tls_size = __sanitizer_get_allocated_size(start);
141		VReport(2, "__tls_get_addr: glibc >=2.25 suspected; tls={0x%zx,0x%zx}\n",
142		tls_beg, tls_size);
	142	VReport(2, "__tls_get_addr: glibc >=2.25 suspected; tls={%p,0x%zx}\n",
	143	(void *)tls_beg, tls_size);
143	144	} else {
144	145	VReport(2, "__tls_get_addr: Can't guess glibc version\n");
145	146	// This may happen inside the DTOR of main thread, so just ignore it.

lib/tsan/sanitizer_common/sanitizer_unwind_fuchsia.cpp created+66

...	...	@@ -0,0 +1,66 @@
	1	//===------------------ sanitizer_unwind_fuchsia.cpp
	2	//---------------------------===//
	3	//
	4	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	5	// See https://llvm.org/LICENSE.txt for license information.
	6	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	7	//
	8	//===----------------------------------------------------------------------===//
	9	//
	10	/// Sanitizer unwind Fuchsia specific functions.
	11	//
	12	//===----------------------------------------------------------------------===//
	13
	14	#include "sanitizer_platform.h"
	15	#if SANITIZER_FUCHSIA
	16
	17	# include <limits.h>
	18	# include <unwind.h>
	19
	20	# include "sanitizer_common.h"
	21	# include "sanitizer_stacktrace.h"
	22
	23	namespace __sanitizer {
	24
	25	# if SANITIZER_CAN_SLOW_UNWIND
	26	struct UnwindTraceArg {
	27	BufferedStackTrace *stack;
	28	u32 max_depth;
	29	};
	30
	31	_Unwind_Reason_Code Unwind_Trace(struct _Unwind_Context ctx, void param) {
	32	UnwindTraceArg arg = static_cast<UnwindTraceArg >(param);
	33	CHECK_LT(arg->stack->size, arg->max_depth);
	34	uptr pc = _Unwind_GetIP(ctx);
	35	if (pc < GetPageSizeCached())
	36	return _URC_NORMAL_STOP;
	37	arg->stack->trace_buffer[arg->stack->size++] = pc;
	38	return (arg->stack->size == arg->max_depth ? _URC_NORMAL_STOP
	39	: _URC_NO_REASON);
	40	}
	41
	42	void BufferedStackTrace::UnwindSlow(uptr pc, u32 max_depth) {
	43	CHECK_GE(max_depth, 2);
	44	size = 0;
	45	UnwindTraceArg arg = {this, Min(max_depth + 1, kStackTraceMax)};
	46	_Unwind_Backtrace(Unwind_Trace, &arg);
	47	CHECK_GT(size, 0);
	48	// We need to pop a few frames so that pc is on top.
	49	uptr to_pop = LocatePcInTrace(pc);
	50	// trace_buffer[0] belongs to the current function so we always pop it,
	51	// unless there is only 1 frame in the stack trace (1 frame is always better
	52	// than 0!).
	53	PopStackFrames(Min(to_pop, static_cast<uptr>(1)));
	54	trace_buffer[0] = pc;
	55	}
	56
	57	void BufferedStackTrace::UnwindSlow(uptr pc, void *context, u32 max_depth) {
	58	CHECK(context);
	59	CHECK_GE(max_depth, 2);
	60	UNREACHABLE("signal context doesn't exist");
	61	}
	62	# endif // SANITIZER_CAN_SLOW_UNWIND
	63
	64	} // namespace __sanitizer
	65
	66	#endif // SANITIZER_FUCHSIA

lib/tsan/sanitizer_common/sanitizer_unwind_win.cpp+7

...	...	@@ -70,10 +70,17 @@ void BufferedStackTrace::UnwindSlow(uptr pc, void *context, u32 max_depth) {
70	70	stack_frame.AddrStack.Offset = ctx.Rsp;
71	71	# endif
72	72	# else
	73	# if SANITIZER_ARM
	74	int machine_type = IMAGE_FILE_MACHINE_ARM;
	75	stack_frame.AddrPC.Offset = ctx.Pc;
	76	stack_frame.AddrFrame.Offset = ctx.R11;
	77	stack_frame.AddrStack.Offset = ctx.Sp;
	78	# else
73	79	int machine_type = IMAGE_FILE_MACHINE_I386;
74	80	stack_frame.AddrPC.Offset = ctx.Eip;
75	81	stack_frame.AddrFrame.Offset = ctx.Ebp;
76	82	stack_frame.AddrStack.Offset = ctx.Esp;
	83	# endif
77	84	# endif
78	85	stack_frame.AddrPC.Mode = AddrModeFlat;
79	86	stack_frame.AddrFrame.Mode = AddrModeFlat;

lib/tsan/sanitizer_common/sanitizer_win.cpp+13-12

...	...	@@ -144,7 +144,7 @@ void MmapOrDie(uptr size, const char mem_type, bool raw_report) {
144	144	return rv;
145	145	}
146	146
147		void UnmapOrDie(void *addr, uptr size) {
	147	void UnmapOrDie(void *addr, uptr size, bool raw_report) {
148	148	if (!size \|\| !addr)
149	149	return;
150	150
...	...	@@ -156,10 +156,7 @@ void UnmapOrDie(void *addr, uptr size) {
156	156	// fails try MEM_DECOMMIT.
157	157	if (VirtualFree(addr, 0, MEM_RELEASE) == 0) {
158	158	if (VirtualFree(addr, size, MEM_DECOMMIT) == 0) {
159		Report("ERROR: %s failed to "
160		"deallocate 0x%zx (%zd) bytes at address %p (error code: %d)\n",
161		SanitizerToolName, size, size, addr, GetLastError());
162		CHECK("unable to unmap" && 0);
	159	ReportMunmapFailureAndDie(addr, size, GetLastError(), raw_report);
163	160	}
164	161	}
165	162	}
...	...	@@ -279,8 +276,8 @@ void MmapFixedOrDie(uptr fixed_addr, uptr size, const char name) {
279	276	MEM_COMMIT, PAGE_READWRITE);
280	277	if (p == 0) {
281	278	char mem_type[30];
282		internal_snprintf(mem_type, sizeof(mem_type), "memory at address 0x%zx",
283		fixed_addr);
	279	internal_snprintf(mem_type, sizeof(mem_type), "memory at address %p",
	280	(void *)fixed_addr);
284	281	ReportMmapFailureAndDie(size, mem_type, "allocate", GetLastError());
285	282	}
286	283	return p;
...	...	@@ -311,8 +308,8 @@ void MmapFixedOrDieOnFatalError(uptr fixed_addr, uptr size, const char name) {
311	308	MEM_COMMIT, PAGE_READWRITE);
312	309	if (p == 0) {
313	310	char mem_type[30];
314		internal_snprintf(mem_type, sizeof(mem_type), "memory at address 0x%zx",
315		fixed_addr);
	311	internal_snprintf(mem_type, sizeof(mem_type), "memory at address %p",
	312	(void *)fixed_addr);
316	313	return ReturnNullptrOnOOMOrDie(size, mem_type, "allocate");
317	314	}
318	315	return p;
...	...	@@ -387,9 +384,8 @@ bool DontDumpShadowMemory(uptr addr, uptr length) {
387	384	}
388	385
389	386	uptr MapDynamicShadow(uptr shadow_size_bytes, uptr shadow_scale,
390		uptr min_shadow_base_alignment,
391		UNUSED uptr &high_mem_end) {
392		const uptr granularity = GetMmapGranularity();
	387	uptr min_shadow_base_alignment, UNUSED uptr &high_mem_end,
	388	uptr granularity) {
393	389	const uptr alignment =
394	390	Max<uptr>(granularity << shadow_scale, 1ULL << min_shadow_base_alignment);
395	391	const uptr left_padding =
...	...	@@ -996,8 +992,13 @@ void SignalContext::InitPcSpBp() {
996	992	sp = (uptr)context_record->Rsp;
997	993	# endif
998	994	# else
	995	# if SANITIZER_ARM
	996	bp = (uptr)context_record->R11;
	997	sp = (uptr)context_record->Sp;
	998	# else
999	999	bp = (uptr)context_record->Ebp;
1000	1000	sp = (uptr)context_record->Esp;
	1001	# endif
1001	1002	# endif
1002	1003	}
1003	1004

lib/tsan/tsan_debugging.cpp+3-1

...	...	@@ -35,7 +35,9 @@ static const char *ReportTypeDescription(ReportType typ) {
35	35	case ReportTypeSignalUnsafe: return "signal-unsafe-call";
36	36	case ReportTypeErrnoInSignal: return "errno-in-signal-handler";
37	37	case ReportTypeDeadlock: return "lock-order-inversion";
38		// No default case so compiler warns us if we miss one
	38	case ReportTypeMutexHeldWrongContext:
	39	return "mutex-held-in-wrong-context";
	40	// No default case so compiler warns us if we miss one
39	41	}
40	42	UNREACHABLE("missing case");
41	43	}

lib/tsan/tsan_defs.h+1-1

...	...	@@ -30,7 +30,7 @@
30	30	# define __MM_MALLOC_H
31	31	# include <emmintrin.h>
32	32	# include <smmintrin.h>
33		# define VECTOR_ALIGNED ALIGNED(16)
	33	# define VECTOR_ALIGNED alignas(16)
34	34	typedef __m128i m128;
35	35	#else
36	36	# define VECTOR_ALIGNED

lib/tsan/tsan_dispatch_defs.h-7

...	...	@@ -56,13 +56,6 @@ extern const dispatch_block_t _dispatch_data_destructor_munmap;
56	56	# define DISPATCH_NOESCAPE
57	57	#endif
58	58
59		#if SANITIZER_APPLE
60		# define SANITIZER_WEAK_IMPORT extern "C" __attribute((weak_import))
61		#else
62		# define SANITIZER_WEAK_IMPORT extern "C" __attribute((weak))
63		#endif
64
65
66	59	// Data types used in dispatch APIs
67	60	typedef unsigned long size_t;
68	61	typedef unsigned long uintptr_t;

lib/tsan/tsan_interceptors_posix.cpp+93-42

...	...	@@ -14,6 +14,7 @@
14	14
15	15	#include "sanitizer_common/sanitizer_atomic.h"
16	16	#include "sanitizer_common/sanitizer_errno.h"
	17	#include "sanitizer_common/sanitizer_glibc_version.h"
17	18	#include "sanitizer_common/sanitizer_libc.h"
18	19	#include "sanitizer_common/sanitizer_linux.h"
19	20	#include "sanitizer_common/sanitizer_platform_limits_netbsd.h"
...	...	@@ -81,6 +82,8 @@ struct ucontext_t {
81	82	#define PTHREAD_ABI_BASE "GLIBC_2.17"
82	83	#elif SANITIZER_LOONGARCH64
83	84	#define PTHREAD_ABI_BASE "GLIBC_2.36"
	85	#elif SANITIZER_RISCV64
	86	# define PTHREAD_ABI_BASE "GLIBC_2.27"
84	87	#endif
85	88
86	89	extern "C" int pthread_attr_init(void *attr);
...	...	@@ -205,7 +208,7 @@ struct AtExitCtx {
205	208	struct InterceptorContext {
206	209	// The object is 64-byte aligned, because we want hot data to be located
207	210	// in a single cache line if possible (it's accessed in every interceptor).
208		ALIGNED(64) LibIgnore libignore;
	211	alignas(64) LibIgnore libignore;
209	212	__sanitizer_sigaction sigactions[kSigCount];
210	213	#if !SANITIZER_APPLE && !SANITIZER_NETBSD
211	214	unsigned finalize_key;
...	...	@@ -217,7 +220,7 @@ struct InterceptorContext {
217	220	InterceptorContext() : libignore(LINKER_INITIALIZED), atexit_mu(MutexTypeAtExit), AtExitStack() {}
218	221	};
219	222
220		static ALIGNED(64) char interceptor_placeholder[sizeof(InterceptorContext)];
	223	alignas(64) static char interceptor_placeholder[sizeof(InterceptorContext)];
221	224	InterceptorContext *interceptor_ctx() {
222	225	return reinterpret_cast<InterceptorContext*>(&interceptor_placeholder[0]);
223	226	}
...	...	@@ -1085,7 +1088,18 @@ TSAN_INTERCEPTOR(int, pthread_join, void th, void *ret) {
1085	1088	return res;
1086	1089	}
1087	1090
1088		DEFINE_REAL_PTHREAD_FUNCTIONS
	1091	// DEFINE_INTERNAL_PTHREAD_FUNCTIONS
	1092	namespace __sanitizer {
	1093	int internal_pthread_create(void th, void attr, void (callback)(void *),
	1094	void *param) {
	1095	ScopedIgnoreInterceptors ignore;
	1096	return REAL(pthread_create)(th, attr, callback, param);
	1097	}
	1098	int internal_pthread_join(void th, void *ret) {
	1099	ScopedIgnoreInterceptors ignore;
	1100	return REAL(pthread_join(th, ret));
	1101	}
	1102	} // namespace __sanitizer
1089	1103
1090	1104	TSAN_INTERCEPTOR(int, pthread_detach, void *th) {
1091	1105	SCOPED_INTERCEPTOR_RAW(pthread_detach, th);
...	...	@@ -1338,7 +1352,7 @@ TSAN_INTERCEPTOR(int, pthread_mutex_destroy, void *m) {
1338	1352	TSAN_INTERCEPTOR(int, pthread_mutex_lock, void *m) {
1339	1353	SCOPED_TSAN_INTERCEPTOR(pthread_mutex_lock, m);
1340	1354	MutexPreLock(thr, pc, (uptr)m);
1341		int res = REAL(pthread_mutex_lock)(m);
	1355	int res = BLOCK_REAL(pthread_mutex_lock)(m);
1342	1356	if (res == errno_EOWNERDEAD)
1343	1357	MutexRepair(thr, pc, (uptr)m);
1344	1358	if (res == 0 \|\| res == errno_EOWNERDEAD)
...	...	@@ -1378,6 +1392,22 @@ TSAN_INTERCEPTOR(int, pthread_mutex_unlock, void *m) {
1378	1392	return res;
1379	1393	}
1380	1394
	1395	#if SANITIZER_LINUX
	1396	TSAN_INTERCEPTOR(int, pthread_mutex_clocklock, void *m,
	1397	__sanitizer_clockid_t clock, void *abstime) {
	1398	SCOPED_TSAN_INTERCEPTOR(pthread_mutex_clocklock, m, clock, abstime);
	1399	MutexPreLock(thr, pc, (uptr)m);
	1400	int res = BLOCK_REAL(pthread_mutex_clocklock)(m, clock, abstime);
	1401	if (res == errno_EOWNERDEAD)
	1402	MutexRepair(thr, pc, (uptr)m);
	1403	if (res == 0 \|\| res == errno_EOWNERDEAD)
	1404	MutexPostLock(thr, pc, (uptr)m);
	1405	if (res == errno_EINVAL)
	1406	MutexInvalidAccess(thr, pc, (uptr)m);
	1407	return res;
	1408	}
	1409	#endif
	1410
1381	1411	#if SANITIZER_GLIBC
1382	1412	# if !__GLIBC_PREREQ(2, 34)
1383	1413	// glibc 2.34 applies a non-default version for the two functions. They are no
...	...	@@ -1385,7 +1415,7 @@ TSAN_INTERCEPTOR(int, pthread_mutex_unlock, void *m) {
1385	1415	TSAN_INTERCEPTOR(int, __pthread_mutex_lock, void *m) {
1386	1416	SCOPED_TSAN_INTERCEPTOR(__pthread_mutex_lock, m);
1387	1417	MutexPreLock(thr, pc, (uptr)m);
1388		int res = REAL(__pthread_mutex_lock)(m);
	1418	int res = BLOCK_REAL(__pthread_mutex_lock)(m);
1389	1419	if (res == errno_EOWNERDEAD)
1390	1420	MutexRepair(thr, pc, (uptr)m);
1391	1421	if (res == 0 \|\| res == errno_EOWNERDEAD)
...	...	@@ -1428,7 +1458,7 @@ TSAN_INTERCEPTOR(int, pthread_spin_destroy, void *m) {
1428	1458	TSAN_INTERCEPTOR(int, pthread_spin_lock, void *m) {
1429	1459	SCOPED_TSAN_INTERCEPTOR(pthread_spin_lock, m);
1430	1460	MutexPreLock(thr, pc, (uptr)m);
1431		int res = REAL(pthread_spin_lock)(m);
	1461	int res = BLOCK_REAL(pthread_spin_lock)(m);
1432	1462	if (res == 0) {
1433	1463	MutexPostLock(thr, pc, (uptr)m);
1434	1464	}
...	...	@@ -1503,7 +1533,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_timedrdlock, void m, void abstime) {
1503	1533	TSAN_INTERCEPTOR(int, pthread_rwlock_wrlock, void *m) {
1504	1534	SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_wrlock, m);
1505	1535	MutexPreLock(thr, pc, (uptr)m);
1506		int res = REAL(pthread_rwlock_wrlock)(m);
	1536	int res = BLOCK_REAL(pthread_rwlock_wrlock)(m);
1507	1537	if (res == 0) {
1508	1538	MutexPostLock(thr, pc, (uptr)m);
1509	1539	}
...	...	@@ -1595,47 +1625,40 @@ TSAN_INTERCEPTOR(int, __fxstat, int version, int fd, void *buf) {
1595	1625	FdAccess(thr, pc, fd);
1596	1626	return REAL(__fxstat)(version, fd, buf);
1597	1627	}
1598		#define TSAN_MAYBE_INTERCEPT___FXSTAT TSAN_INTERCEPT(__fxstat)
	1628
	1629	TSAN_INTERCEPTOR(int, __fxstat64, int version, int fd, void *buf) {
	1630	SCOPED_TSAN_INTERCEPTOR(__fxstat64, version, fd, buf);
	1631	if (fd > 0)
	1632	FdAccess(thr, pc, fd);
	1633	return REAL(__fxstat64)(version, fd, buf);
	1634	}
	1635	#define TSAN_MAYBE_INTERCEPT___FXSTAT TSAN_INTERCEPT(__fxstat); TSAN_INTERCEPT(__fxstat64)
1599	1636	#else
1600	1637	#define TSAN_MAYBE_INTERCEPT___FXSTAT
1601	1638	#endif
1602	1639
	1640	#if !SANITIZER_GLIBC \|\| __GLIBC_PREREQ(2, 33)
1603	1641	TSAN_INTERCEPTOR(int, fstat, int fd, void *buf) {
1604		#if SANITIZER_GLIBC
1605		SCOPED_TSAN_INTERCEPTOR(__fxstat, 0, fd, buf);
1606		if (fd > 0)
1607		FdAccess(thr, pc, fd);
1608		return REAL(__fxstat)(0, fd, buf);
1609		#else
1610	1642	SCOPED_TSAN_INTERCEPTOR(fstat, fd, buf);
1611	1643	if (fd > 0)
1612	1644	FdAccess(thr, pc, fd);
1613	1645	return REAL(fstat)(fd, buf);
1614		#endif
1615		}
1616
1617		#if SANITIZER_GLIBC
1618		TSAN_INTERCEPTOR(int, __fxstat64, int version, int fd, void *buf) {
1619		SCOPED_TSAN_INTERCEPTOR(__fxstat64, version, fd, buf);
1620		if (fd > 0)
1621		FdAccess(thr, pc, fd);
1622		return REAL(__fxstat64)(version, fd, buf);
1623	1646	}
1624		#define TSAN_MAYBE_INTERCEPT___FXSTAT64 TSAN_INTERCEPT(__fxstat64)
	1647	# define TSAN_MAYBE_INTERCEPT_FSTAT TSAN_INTERCEPT(fstat)
1625	1648	#else
1626		#define TSAN_MAYBE_INTERCEPT___FXSTAT64
	1649	# define TSAN_MAYBE_INTERCEPT_FSTAT
1627	1650	#endif
1628	1651
1629		#if SANITIZER_GLIBC
	1652	#if __GLIBC_PREREQ(2, 33)
1630	1653	TSAN_INTERCEPTOR(int, fstat64, int fd, void *buf) {
1631		SCOPED_TSAN_INTERCEPTOR(__fxstat64, 0, fd, buf);
	1654	SCOPED_TSAN_INTERCEPTOR(fstat64, fd, buf);
1632	1655	if (fd > 0)
1633	1656	FdAccess(thr, pc, fd);
1634		return REAL(__fxstat64)(0, fd, buf);
	1657	return REAL(fstat64)(fd, buf);
1635	1658	}
1636		#define TSAN_MAYBE_INTERCEPT_FSTAT64 TSAN_INTERCEPT(fstat64)
	1659	# define TSAN_MAYBE_INTERCEPT_FSTAT64 TSAN_INTERCEPT(fstat64)
1637	1660	#else
1638		#define TSAN_MAYBE_INTERCEPT_FSTAT64
	1661	# define TSAN_MAYBE_INTERCEPT_FSTAT64
1639	1662	#endif
1640	1663
1641	1664	TSAN_INTERCEPTOR(int, open, const char *name, int oflag, ...) {
...	...	@@ -2565,7 +2588,7 @@ int sigaction_impl(int sig, const __sanitizer_sigaction *act,
2565	2588	// Copy act into sigactions[sig].
2566	2589	// Can't use struct copy, because compiler can emit call to memcpy.
2567	2590	// Can't use internal_memcpy, because it copies byte-by-byte,
2568		// and signal handler reads the handler concurrently. It it can read
	2591	// and signal handler reads the handler concurrently. It can read
2569	2592	// some bytes from old value and some bytes from new value.
2570	2593	// Use volatile to prevent insertion of memcpy.
2571	2594	sigactions[sig].handler =
...	...	@@ -2655,6 +2678,25 @@ static USED void syscall_fd_release(uptr pc, int fd) {
2655	2678	FdRelease(thr, pc, fd);
2656	2679	}
2657	2680
	2681	static USED void sycall_blocking_start() {
	2682	DPrintf("sycall_blocking_start()\n");
	2683	ThreadState *thr = cur_thread();
	2684	EnterBlockingFunc(thr);
	2685	// When we are in a "blocking call", we process signals asynchronously
	2686	// (right when they arrive). In this context we do not expect to be
	2687	// executing any user/runtime code. The known interceptor sequence when
	2688	// this is not true is: pthread_join -> munmap(stack). It's fine
	2689	// to ignore munmap in this case -- we handle stack shadow separately.
	2690	thr->ignore_interceptors++;
	2691	}
	2692
	2693	static USED void sycall_blocking_end() {
	2694	DPrintf("sycall_blocking_end()\n");
	2695	ThreadState *thr = cur_thread();
	2696	thr->ignore_interceptors--;
	2697	atomic_store(&thr->in_blocking_func, 0, memory_order_relaxed);
	2698	}
	2699
2658	2700	static void syscall_pre_fork(uptr pc) { ForkBefore(cur_thread(), pc); }
2659	2701
2660	2702	static void syscall_post_fork(uptr pc, int pid) {
...	...	@@ -2709,6 +2751,9 @@ static void syscall_post_fork(uptr pc, int pid) {
2709	2751	#define COMMON_SYSCALL_POST_FORK(res) \
2710	2752	syscall_post_fork(GET_CALLER_PC(), res)
2711	2753
	2754	#define COMMON_SYSCALL_BLOCKING_START() sycall_blocking_start()
	2755	#define COMMON_SYSCALL_BLOCKING_END() sycall_blocking_end()
	2756
2712	2757	#include "sanitizer_common/sanitizer_common_syscalls.inc"
2713	2758	#include "sanitizer_common/sanitizer_syscalls_netbsd.inc"
2714	2759
...	...	@@ -2843,8 +2888,21 @@ void InitializeInterceptors() {
2843	2888	REAL(memcpy) = internal_memcpy;
2844	2889	#endif
2845	2890
	2891	__interception::DoesNotSupportStaticLinking();
	2892
2846	2893	new(interceptor_ctx()) InterceptorContext();
2847	2894
	2895	// Interpose __tls_get_addr before the common interposers. This is needed
	2896	// because dlsym() may call malloc on failure which could result in other
	2897	// interposed functions being called that could eventually make use of TLS.
	2898	#ifdef NEED_TLS_GET_ADDR
	2899	# if !SANITIZER_S390
	2900	TSAN_INTERCEPT(__tls_get_addr);
	2901	# else
	2902	TSAN_INTERCEPT(__tls_get_addr_internal);
	2903	TSAN_INTERCEPT(__tls_get_offset);
	2904	# endif
	2905	#endif
2848	2906	InitializeCommonInterceptors();
2849	2907	InitializeSignalInterceptors();
2850	2908	InitializeLibdispatchInterceptors();
...	...	@@ -2900,6 +2958,9 @@ void InitializeInterceptors() {
2900	2958	TSAN_INTERCEPT(pthread_mutex_trylock);
2901	2959	TSAN_INTERCEPT(pthread_mutex_timedlock);
2902	2960	TSAN_INTERCEPT(pthread_mutex_unlock);
	2961	#if SANITIZER_LINUX
	2962	TSAN_INTERCEPT(pthread_mutex_clocklock);
	2963	#endif
2903	2964	#if SANITIZER_GLIBC
2904	2965	# if !__GLIBC_PREREQ(2, 34)
2905	2966	TSAN_INTERCEPT(__pthread_mutex_lock);
...	...	@@ -2929,10 +2990,9 @@ void InitializeInterceptors() {
2929	2990
2930	2991	TSAN_INTERCEPT(pthread_once);
2931	2992
2932		TSAN_INTERCEPT(fstat);
2933	2993	TSAN_MAYBE_INTERCEPT___FXSTAT;
	2994	TSAN_MAYBE_INTERCEPT_FSTAT;
2934	2995	TSAN_MAYBE_INTERCEPT_FSTAT64;
2935		TSAN_MAYBE_INTERCEPT___FXSTAT64;
2936	2996	TSAN_INTERCEPT(open);
2937	2997	TSAN_MAYBE_INTERCEPT_OPEN64;
2938	2998	TSAN_INTERCEPT(creat);
...	...	@@ -2989,15 +3049,6 @@ void InitializeInterceptors() {
2989	3049	TSAN_INTERCEPT(__cxa_atexit);
2990	3050	TSAN_INTERCEPT(_exit);
2991	3051
2992		#ifdef NEED_TLS_GET_ADDR
2993		#if !SANITIZER_S390
2994		TSAN_INTERCEPT(__tls_get_addr);
2995		#else
2996		TSAN_INTERCEPT(__tls_get_addr_internal);
2997		TSAN_INTERCEPT(__tls_get_offset);
2998		#endif
2999		#endif
3000
3001	3052	TSAN_MAYBE_INTERCEPT__LWP_EXIT;
3002	3053	TSAN_MAYBE_INTERCEPT_THR_EXIT;
3003	3054

lib/tsan/tsan_interface.h+8

...	...	@@ -419,6 +419,14 @@ void __tsan_go_atomic32_fetch_add(ThreadState thr, uptr cpc, uptr pc, u8 a);
419	419	SANITIZER_INTERFACE_ATTRIBUTE
420	420	void __tsan_go_atomic64_fetch_add(ThreadState thr, uptr cpc, uptr pc, u8 a);
421	421	SANITIZER_INTERFACE_ATTRIBUTE
	422	void __tsan_go_atomic32_fetch_and(ThreadState thr, uptr cpc, uptr pc, u8 a);
	423	SANITIZER_INTERFACE_ATTRIBUTE
	424	void __tsan_go_atomic64_fetch_and(ThreadState thr, uptr cpc, uptr pc, u8 a);
	425	SANITIZER_INTERFACE_ATTRIBUTE
	426	void __tsan_go_atomic32_fetch_or(ThreadState thr, uptr cpc, uptr pc, u8 a);
	427	SANITIZER_INTERFACE_ATTRIBUTE
	428	void __tsan_go_atomic64_fetch_or(ThreadState thr, uptr cpc, uptr pc, u8 a);
	429	SANITIZER_INTERFACE_ATTRIBUTE
422	430	void __tsan_go_atomic32_exchange(ThreadState thr, uptr cpc, uptr pc, u8 a);
423	431	SANITIZER_INTERFACE_ATTRIBUTE
424	432	void __tsan_go_atomic64_exchange(ThreadState thr, uptr cpc, uptr pc, u8 a);

lib/tsan/tsan_interface_ann.cpp+23-1

...	...	@@ -76,7 +76,7 @@ struct DynamicAnnContext {
76	76	};
77	77
78	78	static DynamicAnnContext *dyn_ann_ctx;
79		static char dyn_ann_ctx_placeholder[sizeof(DynamicAnnContext)] ALIGNED(64);
	79	alignas(64) static char dyn_ann_ctx_placeholder[sizeof(DynamicAnnContext)];
80	80
81	81	static void AddExpectRace(ExpectRace *list,
82	82	char f, int l, uptr addr, uptr size, char desc) {
...	...	@@ -435,4 +435,26 @@ void __tsan_mutex_post_divert(void *addr, unsigned flagz) {
435	435	ThreadIgnoreBegin(thr, 0);
436	436	ThreadIgnoreSyncBegin(thr, 0);
437	437	}
	438
	439	static void ReportMutexHeldWrongContext(ThreadState *thr, uptr pc) {
	440	ThreadRegistryLock l(&ctx->thread_registry);
	441	ScopedReport rep(ReportTypeMutexHeldWrongContext);
	442	for (uptr i = 0; i < thr->mset.Size(); ++i) {
	443	MutexSet::Desc desc = thr->mset.Get(i);
	444	rep.AddMutex(desc.addr, desc.stack_id);
	445	}
	446	VarSizeStackTrace trace;
	447	ObtainCurrentStack(thr, pc, &trace);
	448	rep.AddStack(trace, true);
	449	OutputReport(thr, rep);
	450	}
	451
	452	INTERFACE_ATTRIBUTE
	453	void __tsan_check_no_mutexes_held() {
	454	SCOPED_ANNOTATION(__tsan_check_no_mutexes_held);
	455	if (thr->mset.Size() == 0) {
	456	return;
	457	}
	458	ReportMutexHeldWrongContext(thr, pc);
	459	}
438	460	} // extern "C"

lib/tsan/tsan_interface_atomic.cpp+24

...	...	@@ -894,6 +894,30 @@ void __tsan_go_atomic64_fetch_add(ThreadState thr, uptr cpc, uptr pc, u8 a) {
894	894	ATOMIC_RET(FetchAdd, (a64)(a+16), (a64)a, (a64*)(a+8), mo_acq_rel);
895	895	}
896	896
	897	SANITIZER_INTERFACE_ATTRIBUTE
	898	void __tsan_go_atomic32_fetch_and(ThreadState thr, uptr cpc, uptr pc, u8 a) {
	899	ATOMIC_RET(FetchAnd, (a32 )(a + 16), (a32 )a, (a32 *)(a + 8),
	900	mo_acq_rel);
	901	}
	902
	903	SANITIZER_INTERFACE_ATTRIBUTE
	904	void __tsan_go_atomic64_fetch_and(ThreadState thr, uptr cpc, uptr pc, u8 a) {
	905	ATOMIC_RET(FetchAnd, (a64 )(a + 16), (a64 )a, (a64 *)(a + 8),
	906	mo_acq_rel);
	907	}
	908
	909	SANITIZER_INTERFACE_ATTRIBUTE
	910	void __tsan_go_atomic32_fetch_or(ThreadState thr, uptr cpc, uptr pc, u8 a) {
	911	ATOMIC_RET(FetchOr, (a32 )(a + 16), (a32 )a, (a32 *)(a + 8),
	912	mo_acq_rel);
	913	}
	914
	915	SANITIZER_INTERFACE_ATTRIBUTE
	916	void __tsan_go_atomic64_fetch_or(ThreadState thr, uptr cpc, uptr pc, u8 a) {
	917	ATOMIC_RET(FetchOr, (a64 )(a + 16), (a64 )a, (a64 *)(a + 8),
	918	mo_acq_rel);
	919	}
	920
897	921	SANITIZER_INTERFACE_ATTRIBUTE
898	922	void __tsan_go_atomic32_exchange(ThreadState thr, uptr cpc, uptr pc, u8 a) {
899	923	ATOMIC_RET(Exchange, (a32)(a+16), (a32)a, (a32*)(a+8), mo_acq_rel);

lib/tsan/tsan_mman.cpp+20-9

...	...	@@ -9,17 +9,19 @@
9	9	// This file is a part of ThreadSanitizer (TSan), a race detector.
10	10	//
11	11	//===----------------------------------------------------------------------===//
	12	#include "tsan_mman.h"
	13
12	14	#include "sanitizer_common/sanitizer_allocator_checks.h"
13	15	#include "sanitizer_common/sanitizer_allocator_interface.h"
14	16	#include "sanitizer_common/sanitizer_allocator_report.h"
15	17	#include "sanitizer_common/sanitizer_common.h"
16	18	#include "sanitizer_common/sanitizer_errno.h"
17	19	#include "sanitizer_common/sanitizer_placement_new.h"
	20	#include "sanitizer_common/sanitizer_stackdepot.h"
	21	#include "tsan_flags.h"
18	22	#include "tsan_interface.h"
19		#include "tsan_mman.h"
20		#include "tsan_rtl.h"
21	23	#include "tsan_report.h"
22		#include "tsan_flags.h"
	24	#include "tsan_rtl.h"
23	25
24	26	namespace __tsan {
25	27
...	...	@@ -52,7 +54,7 @@ struct MapUnmapCallback {
52	54	}
53	55	};
54	56
55		static char allocator_placeholder[sizeof(Allocator)] ALIGNED(64);
	57	alignas(64) static char allocator_placeholder[sizeof(Allocator)];
56	58	Allocator *allocator() {
57	59	return reinterpret_cast<Allocator*>(&allocator_placeholder);
58	60	}
...	...	@@ -73,7 +75,7 @@ struct GlobalProc {
73	75	internal_alloc_mtx(MutexTypeInternalAlloc) {}
74	76	};
75	77
76		static char global_proc_placeholder[sizeof(GlobalProc)] ALIGNED(64);
	78	alignas(64) static char global_proc_placeholder[sizeof(GlobalProc)];
77	79	GlobalProc *global_proc() {
78	80	return reinterpret_cast<GlobalProc*>(&global_proc_placeholder);
79	81	}
...	...	@@ -115,12 +117,21 @@ ScopedGlobalProcessor::~ScopedGlobalProcessor() {
115	117	gp->mtx.Unlock();
116	118	}
117	119
118		void AllocatorLock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
	120	void AllocatorLockBeforeFork() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
119	121	global_proc()->internal_alloc_mtx.Lock();
120	122	InternalAllocatorLock();
121		}
122
123		void AllocatorUnlock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
	123	#if !SANITIZER_APPLE
	124	// OS X allocates from hooks, see 6a3958247a.
	125	allocator()->ForceLock();
	126	StackDepotLockBeforeFork();
	127	#endif
	128	}
	129
	130	void AllocatorUnlockAfterFork(bool child) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
	131	#if !SANITIZER_APPLE
	132	StackDepotUnlockAfterFork(child);
	133	allocator()->ForceUnlock();
	134	#endif
124	135	InternalAllocatorUnlock();
125	136	global_proc()->internal_alloc_mtx.Unlock();
126	137	}

lib/tsan/tsan_mman.h+2-2

...	...	@@ -24,8 +24,8 @@ void ReplaceSystemMalloc();
24	24	void AllocatorProcStart(Processor *proc);
25	25	void AllocatorProcFinish(Processor *proc);
26	26	void AllocatorPrintStats();
27		void AllocatorLock();
28		void AllocatorUnlock();
	27	void AllocatorLockBeforeFork();
	28	void AllocatorUnlockAfterFork(bool child);
29	29	void GlobalProcessorLock();
30	30	void GlobalProcessorUnlock();
31	31

lib/tsan/tsan_platform.h+128-23

...	...	@@ -46,17 +46,16 @@ enum {
46	46
47	47	/*
48	48	C/C++ on linux/x86_64 and freebsd/x86_64
49		0000 0000 1000 - 0080 0000 0000: main binary and/or MAP_32BIT mappings (512GB)
50		0040 0000 0000 - 0100 0000 0000: -
51		0100 0000 0000 - 1000 0000 0000: shadow
52		1000 0000 0000 - 3000 0000 0000: -
53		3000 0000 0000 - 3400 0000 0000: metainfo (memory blocks and sync objects)
54		3400 0000 0000 - 5500 0000 0000: -
55		5500 0000 0000 - 5680 0000 0000: pie binaries without ASLR or on 4.1+ kernels
56		5680 0000 0000 - 7d00 0000 0000: -
57		7b00 0000 0000 - 7c00 0000 0000: heap
58		7c00 0000 0000 - 7e80 0000 0000: -
59		7e80 0000 0000 - 8000 0000 0000: modules and main thread stack
	49	0000 0000 1000 - 0200 0000 0000: main binary and/or MAP_32BIT mappings (2TB)
	50	0200 0000 0000 - 1000 0000 0000: -
	51	1000 0000 0000 - 3000 0000 0000: shadow (32TB)
	52	3000 0000 0000 - 3800 0000 0000: metainfo (memory blocks and sync objects; 8TB)
	53	3800 0000 0000 - 5500 0000 0000: -
	54	5500 0000 0000 - 5a00 0000 0000: pie binaries without ASLR or on 4.1+ kernels
	55	5a00 0000 0000 - 7200 0000 0000: -
	56	7200 0000 0000 - 7300 0000 0000: heap (1TB)
	57	7300 0000 0000 - 7a00 0000 0000: -
	58	7a00 0000 0000 - 8000 0000 0000: modules and main thread stack (6TB)
60	59
61	60	C/C++ on netbsd/amd64 can reuse the same mapping:
62	61	* The address space starts from 0x1000 (option with 0x0) and ends with
...	...	@@ -72,20 +71,20 @@ C/C++ on netbsd/amd64 can reuse the same mapping:
72	71	*/
73	72	struct Mapping48AddressSpace {
74	73	static const uptr kMetaShadowBeg = 0x300000000000ull;
75		static const uptr kMetaShadowEnd = 0x340000000000ull;
76		static const uptr kShadowBeg = 0x010000000000ull;
77		static const uptr kShadowEnd = 0x100000000000ull;
78		static const uptr kHeapMemBeg = 0x7b0000000000ull;
79		static const uptr kHeapMemEnd = 0x7c0000000000ull;
	74	static const uptr kMetaShadowEnd = 0x380000000000ull;
	75	static const uptr kShadowBeg = 0x100000000000ull;
	76	static const uptr kShadowEnd = 0x300000000000ull;
	77	static const uptr kHeapMemBeg = 0x720000000000ull;
	78	static const uptr kHeapMemEnd = 0x730000000000ull;
80	79	static const uptr kLoAppMemBeg = 0x000000001000ull;
81		static const uptr kLoAppMemEnd = 0x008000000000ull;
	80	static const uptr kLoAppMemEnd = 0x020000000000ull;
82	81	static const uptr kMidAppMemBeg = 0x550000000000ull;
83		static const uptr kMidAppMemEnd = 0x568000000000ull;
84		static const uptr kHiAppMemBeg = 0x7e8000000000ull;
	82	static const uptr kMidAppMemEnd = 0x5a0000000000ull;
	83	static const uptr kHiAppMemBeg = 0x7a0000000000ull;
85	84	static const uptr kHiAppMemEnd = 0x800000000000ull;
86		static const uptr kShadowMsk = 0x780000000000ull;
87		static const uptr kShadowXor = 0x040000000000ull;
88		static const uptr kShadowAdd = 0x000000000000ull;
	85	static const uptr kShadowMsk = 0x700000000000ull;
	86	static const uptr kShadowXor = 0x000000000000ull;
	87	static const uptr kShadowAdd = 0x100000000000ull;
89	88	static const uptr kVdsoBeg = 0xf000000000000000ull;
90	89	};
91	90
...	...	@@ -377,6 +376,71 @@ struct MappingPPC64_47 {
377	376	static const uptr kMidAppMemEnd = 0;
378	377	};
379	378
	379	/*
	380	C/C++ on linux/riscv64 (39-bit VMA)
	381	0000 0010 00 - 0200 0000 00: main binary ( 8 GB)
	382	0200 0000 00 - 1000 0000 00: -
	383	1000 0000 00 - 4000 0000 00: shadow memory (64 GB)
	384	4000 0000 00 - 4800 0000 00: metainfo (16 GB)
	385	4800 0000 00 - 5500 0000 00: -
	386	5500 0000 00 - 5a00 0000 00: main binary (PIE) (~8 GB)
	387	5600 0000 00 - 7c00 0000 00: -
	388	7d00 0000 00 - 7fff ffff ff: libraries and main thread stack ( 8 GB)
	389
	390	mmap by default allocates from top downwards
	391	VDSO sits below loader and above dynamic libraries, within HiApp region.
	392	Heap starts after program region whose position depends on pie or non-pie.
	393	Disable tracking them since their locations are not fixed.
	394	*/
	395	struct MappingRiscv64_39 {
	396	static const uptr kLoAppMemBeg = 0x0000001000ull;
	397	static const uptr kLoAppMemEnd = 0x0200000000ull;
	398	static const uptr kShadowBeg = 0x1000000000ull;
	399	static const uptr kShadowEnd = 0x2000000000ull;
	400	static const uptr kMetaShadowBeg = 0x2000000000ull;
	401	static const uptr kMetaShadowEnd = 0x2400000000ull;
	402	static const uptr kMidAppMemBeg = 0x2aaaaaa000ull;
	403	static const uptr kMidAppMemEnd = 0x2c00000000ull;
	404	static const uptr kHeapMemBeg = 0x2c00000000ull;
	405	static const uptr kHeapMemEnd = 0x2c00000000ull;
	406	static const uptr kHiAppMemBeg = 0x3c00000000ull;
	407	static const uptr kHiAppMemEnd = 0x3fffffffffull;
	408	static const uptr kShadowMsk = 0x3800000000ull;
	409	static const uptr kShadowXor = 0x0800000000ull;
	410	static const uptr kShadowAdd = 0x0000000000ull;
	411	static const uptr kVdsoBeg = 0x4000000000ull;
	412	};
	413
	414	/*
	415	C/C++ on linux/riscv64 (48-bit VMA)
	416	0000 0000 1000 - 0400 0000 0000: main binary ( 4 TB)
	417	0500 0000 0000 - 2000 0000 0000: -
	418	2000 0000 0000 - 4000 0000 0000: shadow memory (32 TB)
	419	4000 0000 0000 - 4800 0000 0000: metainfo ( 8 TB)
	420	4800 0000 0000 - 5555 5555 5000: -
	421	5555 5555 5000 - 5a00 0000 0000: main binary (PIE) (~5 TB)
	422	5a00 0000 0000 - 7a00 0000 0000: -
	423	7a00 0000 0000 - 7fff ffff ffff: libraries and main thread stack ( 6 TB)
	424	*/
	425	struct MappingRiscv64_48 {
	426	static const uptr kLoAppMemBeg = 0x000000001000ull;
	427	static const uptr kLoAppMemEnd = 0x040000000000ull;
	428	static const uptr kShadowBeg = 0x200000000000ull;
	429	static const uptr kShadowEnd = 0x400000000000ull;
	430	static const uptr kMetaShadowBeg = 0x400000000000ull;
	431	static const uptr kMetaShadowEnd = 0x480000000000ull;
	432	static const uptr kMidAppMemBeg = 0x555555555000ull;
	433	static const uptr kMidAppMemEnd = 0x5a0000000000ull;
	434	static const uptr kHeapMemBeg = 0x5a0000000000ull;
	435	static const uptr kHeapMemEnd = 0x5a0000000000ull;
	436	static const uptr kHiAppMemBeg = 0x7a0000000000ull;
	437	static const uptr kHiAppMemEnd = 0x7fffffffffffull;
	438	static const uptr kShadowMsk = 0x700000000000ull;
	439	static const uptr kShadowXor = 0x100000000000ull;
	440	static const uptr kShadowAdd = 0x000000000000ull;
	441	static const uptr kVdsoBeg = 0x800000000000ull;
	442	};
	443
380	444	/*
381	445	C/C++ on linux/s390x
382	446	While the kernel provides a 64-bit address space, we have to restrict ourselves
...	...	@@ -558,6 +622,35 @@ struct MappingGoAarch64 {
558	622	static const uptr kShadowAdd = 0x200000000000ull;
559	623	};
560	624
	625	/* Go on linux/loongarch64 (47-bit VMA)
	626	0000 0000 1000 - 0000 1000 0000: executable
	627	0000 1000 0000 - 00c0 0000 0000: -
	628	00c0 0000 0000 - 00e0 0000 0000: heap
	629	00e0 0000 0000 - 2000 0000 0000: -
	630	2000 0000 0000 - 2800 0000 0000: shadow
	631	2800 0000 0000 - 3000 0000 0000: -
	632	3000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects)
	633	3200 0000 0000 - 8000 0000 0000: -
	634	*/
	635	struct MappingGoLoongArch64_47 {
	636	static const uptr kMetaShadowBeg = 0x300000000000ull;
	637	static const uptr kMetaShadowEnd = 0x320000000000ull;
	638	static const uptr kShadowBeg = 0x200000000000ull;
	639	static const uptr kShadowEnd = 0x280000000000ull;
	640	static const uptr kLoAppMemBeg = 0x000000001000ull;
	641	static const uptr kLoAppMemEnd = 0x00e000000000ull;
	642	static const uptr kMidAppMemBeg = 0;
	643	static const uptr kMidAppMemEnd = 0;
	644	static const uptr kHiAppMemBeg = 0;
	645	static const uptr kHiAppMemEnd = 0;
	646	static const uptr kHeapMemBeg = 0;
	647	static const uptr kHeapMemEnd = 0;
	648	static const uptr kVdsoBeg = 0;
	649	static const uptr kShadowMsk = 0;
	650	static const uptr kShadowXor = 0;
	651	static const uptr kShadowAdd = 0x200000000000ull;
	652	};
	653
561	654	/*
562	655	Go on linux/mips64 (47-bit VMA)
563	656	0000 0000 1000 - 0000 1000 0000: executable
...	...	@@ -633,6 +726,8 @@ ALWAYS_INLINE auto SelectMapping(Arg arg) {
633	726	return Func::template Apply<MappingGoS390x>(arg);
634	727	# elif defined(__aarch64__)
635	728	return Func::template Apply<MappingGoAarch64>(arg);
	729	# elif defined(__loongarch_lp64)
	730	return Func::template Apply<MappingGoLoongArch64_47>(arg);
636	731	# elif SANITIZER_WINDOWS
637	732	return Func::template Apply<MappingGoWindows>(arg);
638	733	# else
...	...	@@ -665,6 +760,13 @@ ALWAYS_INLINE auto SelectMapping(Arg arg) {
665	760	}
666	761	# elif defined(__mips64)
667	762	return Func::template Apply<MappingMips64_40>(arg);
	763	# elif SANITIZER_RISCV64
	764	switch (vmaSize) {
	765	case 39:
	766	return Func::template Apply<MappingRiscv64_39>(arg);
	767	case 48:
	768	return Func::template Apply<MappingRiscv64_48>(arg);
	769	}
668	770	# elif defined(__s390x__)
669	771	return Func::template Apply<MappingS390x>(arg);
670	772	# else
...	...	@@ -686,12 +788,15 @@ void ForEachMapping() {
686	788	Func::template Apply<MappingPPC64_44>();
687	789	Func::template Apply<MappingPPC64_46>();
688	790	Func::template Apply<MappingPPC64_47>();
	791	Func::template Apply<MappingRiscv64_39>();
	792	Func::template Apply<MappingRiscv64_48>();
689	793	Func::template Apply<MappingS390x>();
690	794	Func::template Apply<MappingGo48>();
691	795	Func::template Apply<MappingGoWindows>();
692	796	Func::template Apply<MappingGoPPC64_46>();
693	797	Func::template Apply<MappingGoPPC64_47>();
694	798	Func::template Apply<MappingGoAarch64>();
	799	Func::template Apply<MappingGoLoongArch64_47>();
695	800	Func::template Apply<MappingGoMips64_47>();
696	801	Func::template Apply<MappingGoS390x>();
697	802	}
...	...	@@ -919,7 +1024,7 @@ inline uptr RestoreAddr(uptr addr) {
919	1024
920	1025	void InitializePlatform();
921	1026	void InitializePlatformEarly();
922		void CheckAndProtect();
	1027	bool CheckAndProtect(bool protect, bool ignore_heap, bool print_warnings);
923	1028	void InitializeShadowMemoryPlatform();
924	1029	void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns);
925	1030	int ExtractResolvFDs(void state, int fds, int nfd);

lib/tsan/tsan_platform_linux.cpp+159-61

...	...	@@ -152,7 +152,7 @@ void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns) {
152	152	#if !SANITIZER_GO
153	153	// Mark shadow for .rodata sections with the special Shadow::kRodata marker.
154	154	// Accesses to .rodata can't race, so this saves time, memory and trace space.
155		static void MapRodata() {
	155	static NOINLINE void MapRodata(char* buffer, uptr size) {
156	156	// First create temp file.
157	157	const char *tmpdir = GetEnv("TMPDIR");
158	158	if (tmpdir == 0)
...	...	@@ -163,13 +163,12 @@ static void MapRodata() {
163	163	#endif
164	164	if (tmpdir == 0)
165	165	return;
166		char name[256];
167		internal_snprintf(name, sizeof(name), "%s/tsan.rodata.%d",
	166	internal_snprintf(buffer, size, "%s/tsan.rodata.%d",
168	167	tmpdir, (int)internal_getpid());
169		uptr openrv = internal_open(name, O_RDWR \| O_CREAT \| O_EXCL, 0600);
	168	uptr openrv = internal_open(buffer, O_RDWR \| O_CREAT \| O_EXCL, 0600);
170	169	if (internal_iserror(openrv))
171	170	return;
172		internal_unlink(name); // Unlink it now, so that we can reuse the buffer.
	171	internal_unlink(buffer); // Unlink it now, so that we can reuse the buffer.
173	172	fd_t fd = openrv;
174	173	// Fill the file with Shadow::kRodata.
175	174	const uptr kMarkerSize = 512 * 1024 / sizeof(RawShadow);
...	...	@@ -188,8 +187,8 @@ static void MapRodata() {
188	187	}
189	188	// Map the file into shadow of .rodata sections.
190	189	MemoryMappingLayout proc_maps(/cache_enabled/true);
191		// Reusing the buffer 'name'.
192		MemoryMappedSegment segment(name, ARRAY_SIZE(name));
	190	// Reusing the buffer 'buffer'.
	191	MemoryMappedSegment segment(buffer, size);
193	192	while (proc_maps.Next(&segment)) {
194	193	if (segment.filename[0] != 0 && segment.filename[0] != '[' &&
195	194	segment.IsReadable() && segment.IsExecutable() &&
...	...	@@ -209,11 +208,103 @@ static void MapRodata() {
209	208	}
210	209
211	210	void InitializeShadowMemoryPlatform() {
212		MapRodata();
	211	char buffer[256]; // Keep in a different frame.
	212	MapRodata(buffer, sizeof(buffer));
213	213	}
214	214
215	215	#endif // #if !SANITIZER_GO
216	216
	217	# if !SANITIZER_GO
	218	static void ReExecIfNeeded(bool ignore_heap) {
	219	// Go maps shadow memory lazily and works fine with limited address space.
	220	// Unlimited stack is not a problem as well, because the executable
	221	// is not compiled with -pie.
	222	bool reexec = false;
	223	// TSan doesn't play well with unlimited stack size (as stack
	224	// overlaps with shadow memory). If we detect unlimited stack size,
	225	// we re-exec the program with limited stack size as a best effort.
	226	if (StackSizeIsUnlimited()) {
	227	const uptr kMaxStackSize = 32 * 1024 * 1024;
	228	VReport(1,
	229	"Program is run with unlimited stack size, which wouldn't "
	230	"work with ThreadSanitizer.\n"
	231	"Re-execing with stack size limited to %zd bytes.\n",
	232	kMaxStackSize);
	233	SetStackSizeLimitInBytes(kMaxStackSize);
	234	reexec = true;
	235	}
	236
	237	if (!AddressSpaceIsUnlimited()) {
	238	Report(
	239	"WARNING: Program is run with limited virtual address space,"
	240	" which wouldn't work with ThreadSanitizer.\n");
	241	Report("Re-execing with unlimited virtual address space.\n");
	242	SetAddressSpaceUnlimited();
	243	reexec = true;
	244	}
	245
	246	# if SANITIZER_LINUX
	247	# if SANITIZER_ANDROID && (defined(__aarch64__) \|\| defined(__x86_64__))
	248	// ASLR personality check.
	249	int old_personality = personality(0xffffffff);
	250	bool aslr_on =
	251	(old_personality != -1) && ((old_personality & ADDR_NO_RANDOMIZE) == 0);
	252
	253	// After patch "arm64: mm: support ARCH_MMAP_RND_BITS." is introduced in
	254	// linux kernel, the random gap between stack and mapped area is increased
	255	// from 128M to 36G on 39-bit aarch64. As it is almost impossible to cover
	256	// this big range, we should disable randomized virtual space on aarch64.
	257	if (aslr_on) {
	258	VReport(1,
	259	"WARNING: Program is run with randomized virtual address "
	260	"space, which wouldn't work with ThreadSanitizer on Android.\n"
	261	"Re-execing with fixed virtual address space.\n");
	262	CHECK_NE(personality(old_personality \| ADDR_NO_RANDOMIZE), -1);
	263	reexec = true;
	264	}
	265	# endif
	266
	267	if (reexec) {
	268	// Don't check the address space since we're going to re-exec anyway.
	269	} else if (!CheckAndProtect(false, ignore_heap, false)) {
	270	// ASLR personality check.
	271	// N.B. 'personality' is sometimes forbidden by sandboxes, so we only call
	272	// this as a last resort (when the memory mapping is incompatible and TSan
	273	// would fail anyway).
	274	int old_personality = personality(0xffffffff);
	275	bool aslr_on =
	276	(old_personality != -1) && ((old_personality & ADDR_NO_RANDOMIZE) == 0);
	277
	278	if (aslr_on) {
	279	// Disable ASLR if the memory layout was incompatible.
	280	// Alternatively, we could just keep re-execing until we get lucky
	281	// with a compatible randomized layout, but the risk is that if it's
	282	// not an ASLR-related issue, we will be stuck in an infinite loop of
	283	// re-execing (unless we change ReExec to pass a parameter of the
	284	// number of retries allowed.)
	285	VReport(1,
	286	"WARNING: ThreadSanitizer: memory layout is incompatible, "
	287	"possibly due to high-entropy ASLR.\n"
	288	"Re-execing with fixed virtual address space.\n"
	289	"N.B. reducing ASLR entropy is preferable.\n");
	290	CHECK_NE(personality(old_personality \| ADDR_NO_RANDOMIZE), -1);
	291	reexec = true;
	292	} else {
	293	Printf(
	294	"FATAL: ThreadSanitizer: memory layout is incompatible, "
	295	"even though ASLR is disabled.\n"
	296	"Please file a bug.\n");
	297	DumpProcessMap();
	298	Die();
	299	}
	300	}
	301	# endif // SANITIZER_LINUX
	302
	303	if (reexec)
	304	ReExec();
	305	}
	306	# endif
	307
217	308	void InitializePlatformEarly() {
218	309	vmaSize =
219	310	(MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1);
...	...	@@ -238,7 +329,13 @@ void InitializePlatformEarly() {
238	329	Printf("FATAL: Found %zd - Supported 47\n", vmaSize);
239	330	Die();
240	331	}
241		# endif
	332	# else
	333	if (vmaSize != 47) {
	334	Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
	335	Printf("FATAL: Found %zd - Supported 47\n", vmaSize);
	336	Die();
	337	}
	338	# endif
242	339	#elif defined(__powerpc64__)
243	340	# if !SANITIZER_GO
244	341	if (vmaSize != 44 && vmaSize != 46 && vmaSize != 47) {
...	...	@@ -267,7 +364,22 @@ void InitializePlatformEarly() {
267	364	Die();
268	365	}
269	366	# endif
270		#endif
	367	# elif SANITIZER_RISCV64
	368	// the bottom half of vma is allocated for userspace
	369	vmaSize = vmaSize + 1;
	370	# if !SANITIZER_GO
	371	if (vmaSize != 39 && vmaSize != 48) {
	372	Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
	373	Printf("FATAL: Found %zd - Supported 39 and 48\n", vmaSize);
	374	Die();
	375	}
	376	# endif
	377	# endif
	378
	379	# if !SANITIZER_GO
	380	// Heap has not been allocated yet
	381	ReExecIfNeeded(false);
	382	# endif
271	383	}
272	384
273	385	void InitializePlatform() {
...	...	@@ -278,52 +390,34 @@ void InitializePlatform() {
278	390	// is not compiled with -pie.
279	391	#if !SANITIZER_GO
280	392	{
281		bool reexec = false;
282		// TSan doesn't play well with unlimited stack size (as stack
283		// overlaps with shadow memory). If we detect unlimited stack size,
284		// we re-exec the program with limited stack size as a best effort.
285		if (StackSizeIsUnlimited()) {
286		const uptr kMaxStackSize = 32 * 1024 * 1024;
287		VReport(1, "Program is run with unlimited stack size, which wouldn't "
288		"work with ThreadSanitizer.\n"
289		"Re-execing with stack size limited to %zd bytes.\n",
290		kMaxStackSize);
291		SetStackSizeLimitInBytes(kMaxStackSize);
292		reexec = true;
293		}
294
295		if (!AddressSpaceIsUnlimited()) {
296		Report("WARNING: Program is run with limited virtual address space,"
297		" which wouldn't work with ThreadSanitizer.\n");
298		Report("Re-execing with unlimited virtual address space.\n");
299		SetAddressSpaceUnlimited();
300		reexec = true;
301		}
302		#if SANITIZER_ANDROID && (defined(__aarch64__) \|\| defined(__x86_64__))
303		// After patch "arm64: mm: support ARCH_MMAP_RND_BITS." is introduced in
304		// linux kernel, the random gap between stack and mapped area is increased
305		// from 128M to 36G on 39-bit aarch64. As it is almost impossible to cover
306		// this big range, we should disable randomized virtual space on aarch64.
307		// ASLR personality check.
308		int old_personality = personality(0xffffffff);
309		if (old_personality != -1 && (old_personality & ADDR_NO_RANDOMIZE) == 0) {
310		VReport(1, "WARNING: Program is run with randomized virtual address "
311		"space, which wouldn't work with ThreadSanitizer.\n"
312		"Re-execing with fixed virtual address space.\n");
313		CHECK_NE(personality(old_personality \| ADDR_NO_RANDOMIZE), -1);
314		reexec = true;
315		}
316
317		#endif
318		#if SANITIZER_LINUX && (defined(__aarch64__) \|\| defined(__loongarch_lp64))
	393	# if SANITIZER_LINUX && (defined(__aarch64__) \|\| defined(__loongarch_lp64))
319	394	// Initialize the xor key used in {sig}{set,long}jump.
320	395	InitializeLongjmpXorKey();
321		#endif
322		if (reexec)
323		ReExec();
	396	# endif
	397	}
	398
	399	// We called ReExecIfNeeded() in InitializePlatformEarly(), but there are
	400	// intervening allocations that result in an edge case:
	401	// 1) InitializePlatformEarly(): memory layout is compatible
	402	// 2) Intervening allocations happen
	403	// 3) InitializePlatform(): memory layout is incompatible and fails
	404	// CheckAndProtect()
	405	# if !SANITIZER_GO
	406	// Heap has already been allocated
	407	ReExecIfNeeded(true);
	408	# endif
	409
	410	// Earlier initialization steps already re-exec'ed until we got a compatible
	411	// memory layout, so we don't expect any more issues here.
	412	if (!CheckAndProtect(true, true, true)) {
	413	Printf(
	414	"FATAL: ThreadSanitizer: unexpectedly found incompatible memory "
	415	"layout.\n");
	416	Printf("FATAL: Please file a bug.\n");
	417	DumpProcessMap();
	418	Die();
324	419	}
325	420
326		CheckAndProtect();
327	421	InitTlsSize();
328	422	#endif // !SANITIZER_GO
329	423	}
...	...	@@ -399,13 +493,15 @@ static uptr UnmangleLongJmpSp(uptr mangled_sp) {
399	493	return mangled_sp ^ xor_key;
400	494	#elif defined(__mips__)
401	495	return mangled_sp;
402		#elif defined(__s390x__)
	496	# elif SANITIZER_RISCV64
	497	return mangled_sp;
	498	# elif defined(__s390x__)
403	499	// tcbhead_t.stack_guard
404	500	uptr xor_key = ((uptr *)__builtin_thread_pointer())[5];
405	501	return mangled_sp ^ xor_key;
406		#else
407		#error "Unknown platform"
408		#endif
	502	# else
	503	# error "Unknown platform"
	504	# endif
409	505	}
410	506
411	507	#if SANITIZER_NETBSD
...	...	@@ -429,11 +525,13 @@ static uptr UnmangleLongJmpSp(uptr mangled_sp) {
429	525	# define LONG_JMP_SP_ENV_SLOT 1
430	526	# elif defined(__mips64)
431	527	# define LONG_JMP_SP_ENV_SLOT 1
432		# elif defined(__s390x__)
433		# define LONG_JMP_SP_ENV_SLOT 9
434		# else
435		# define LONG_JMP_SP_ENV_SLOT 6
436		# endif
	528	# elif SANITIZER_RISCV64
	529	# define LONG_JMP_SP_ENV_SLOT 13
	530	# elif defined(__s390x__)
	531	# define LONG_JMP_SP_ENV_SLOT 9
	532	# else
	533	# define LONG_JMP_SP_ENV_SLOT 6
	534	# endif
437	535	#endif
438	536
439	537	uptr ExtractLongJmpSp(uptr *env) {

lib/tsan/tsan_platform_mac.cpp+6-3

...	...	@@ -46,8 +46,8 @@
46	46	namespace __tsan {
47	47
48	48	#if !SANITIZER_GO
49		static char main_thread_state[sizeof(ThreadState)] ALIGNED(
50		SANITIZER_CACHE_LINE_SIZE);
	49	alignas(SANITIZER_CACHE_LINE_SIZE) static char main_thread_state[sizeof(
	50	ThreadState)];
51	51	static ThreadState *dead_thread_state;
52	52	static pthread_key_t thread_state_key;
53	53
...	...	@@ -239,7 +239,10 @@ static uptr longjmp_xor_key = 0;
239	239	void InitializePlatform() {
240	240	DisableCoreDumperIfNecessary();
241	241	#if !SANITIZER_GO
242		CheckAndProtect();
	242	if (!CheckAndProtect(true, true, true)) {
	243	Printf("FATAL: ThreadSanitizer: found incompatible memory layout.\n");
	244	Die();
	245	}
243	246
244	247	InitializeThreadStateStorage();
245	248

lib/tsan/tsan_platform_posix.cpp+37-6

...	...	@@ -94,22 +94,51 @@ static void ProtectRange(uptr beg, uptr end) {
94	94	}
95	95	}
96	96
97		void CheckAndProtect() {
	97	// CheckAndProtect will check if the memory layout is compatible with TSan.
	98	// Optionally (if 'protect' is true), it will set the memory regions between
	99	// app memory to be inaccessible.
	100	// 'ignore_heap' means it will not consider heap memory allocations to be a
	101	// conflict. Set this based on whether we are calling CheckAndProtect before
	102	// or after the allocator has initialized the heap.
	103	bool CheckAndProtect(bool protect, bool ignore_heap, bool print_warnings) {
98	104	// Ensure that the binary is indeed compiled with -pie.
99	105	MemoryMappingLayout proc_maps(true);
100	106	MemoryMappedSegment segment;
101	107	while (proc_maps.Next(&segment)) {
102		if (IsAppMem(segment.start)) continue;
	108	if (segment.start >= HeapMemBeg() && segment.end <= HeapEnd()) {
	109	if (ignore_heap) {
	110	continue;
	111	} else {
	112	return false;
	113	}
	114	}
	115
	116	// Note: IsAppMem includes if it is heap memory, hence we must
	117	// put this check after the heap bounds check.
	118	if (IsAppMem(segment.start) && IsAppMem(segment.end - 1))
	119	continue;
	120
	121	// Guard page after the heap end
103	122	if (segment.start >= HeapMemEnd() && segment.start < HeapEnd()) continue;
	123
104	124	if (segment.protection == 0) // Zero page or mprotected.
105	125	continue;
	126
106	127	if (segment.start >= VdsoBeg()) // vdso
107	128	break;
108		Printf("FATAL: ThreadSanitizer: unexpected memory mapping 0x%zx-0x%zx\n",
109		segment.start, segment.end);
110		Die();
	129
	130	// Debug output can break tests. Suppress this message in most cases.
	131	if (print_warnings)
	132	Printf(
	133	"WARNING: ThreadSanitizer: unexpected memory mapping 0x%zx-0x%zx\n",
	134	segment.start, segment.end);
	135
	136	return false;
111	137	}
112	138
	139	if (!protect)
	140	return true;
	141
113	142	# if SANITIZER_IOS && !SANITIZER_IOSSIM
114	143	ProtectRange(HeapMemEnd(), ShadowBeg());
115	144	ProtectRange(ShadowEnd(), MetaShadowBeg());
...	...	@@ -135,8 +164,10 @@ void CheckAndProtect() {
135	164	// Older s390x kernels may not support 5-level page tables.
136	165	TryProtectRange(user_addr_max_l4, user_addr_max_l5);
137	166	#endif
	167
	168	return true;
138	169	}
139		#endif
	170	# endif
140	171
141	172	} // namespace __tsan
142	173

lib/tsan/tsan_preinit.cpp+4-6

...	...	@@ -16,11 +16,9 @@
16	16
17	17	#if SANITIZER_CAN_USE_PREINIT_ARRAY
18	18
19		// The symbol is called __local_tsan_preinit, because it's not intended to be
20		// exported.
21		// This code linked into the main executable when -fsanitize=thread is in
22		// the link flags. It can only use exported interface functions.
23		__attribute__((section(".preinit_array"), used))
24		void (*__local_tsan_preinit)(void) = __tsan_init;
	19	// This section is linked into the main executable when -fsanitize=thread is
	20	// specified to perform initialization at a very early stage.
	21	__attribute__((section(".preinit_array"), used)) static auto preinit =
	22	__tsan_init;
25	23
26	24	#endif

lib/tsan/tsan_report.cpp+12-26

...	...	@@ -93,7 +93,9 @@ static const char *ReportTypeString(ReportType typ, uptr tag) {
93	93	return "signal handler spoils errno";
94	94	case ReportTypeDeadlock:
95	95	return "lock-order-inversion (potential deadlock)";
96		// No default case so compiler warns us if we miss one
	96	case ReportTypeMutexHeldWrongContext:
	97	return "mutex held in the wrong context";
	98	// No default case so compiler warns us if we miss one
97	99	}
98	100	UNREACHABLE("missing case");
99	101	}
...	...	@@ -106,10 +108,10 @@ void PrintStack(const ReportStack *ent) {
106	108	SymbolizedStack *frame = ent->frames;
107	109	for (int i = 0; frame && frame->info.address; frame = frame->next, i++) {
108	110	InternalScopedString res;
109		RenderFrame(&res, common_flags()->stack_trace_format, i,
110		frame->info.address, &frame->info,
111		common_flags()->symbolize_vs_style,
112		common_flags()->strip_path_prefix);
	111	StackTracePrinter::GetOrInit()->RenderFrame(
	112	&res, common_flags()->stack_trace_format, i, frame->info.address,
	113	&frame->info, common_flags()->symbolize_vs_style,
	114	common_flags()->strip_path_prefix);
113	115	Printf("%s\n", res.data());
114	116	}
115	117	Printf("\n");
...	...	@@ -271,26 +273,10 @@ static ReportStack ChooseSummaryStack(const ReportDesc rep) {
271	273	return 0;
272	274	}
273	275
274		static bool FrameIsInternal(const SymbolizedStack *frame) {
275		if (frame == 0)
276		return false;
277		const char *file = frame->info.file;
278		const char *module = frame->info.module;
279		if (file != 0 &&
280		(internal_strstr(file, "tsan_interceptors_posix.cpp") \|\|
281		internal_strstr(file, "tsan_interceptors_memintrinsics.cpp") \|\|
282		internal_strstr(file, "sanitizer_common_interceptors.inc") \|\|
283		internal_strstr(file, "tsan_interface_")))
284		return true;
285		if (module != 0 && (internal_strstr(module, "libclang_rt.tsan_")))
286		return true;
287		return false;
288		}
289
290		static SymbolizedStack SkipTsanInternalFrames(SymbolizedStack frames) {
291		while (FrameIsInternal(frames) && frames->next)
292		frames = frames->next;
293		return frames;
	276	static const SymbolizedStack SkipTsanInternalFrames(SymbolizedStack frames) {
	277	if (const SymbolizedStack *f = SkipInternalFrames(frames))
	278	return f;
	279	return frames; // Fallback to the top frame.
294	280	}
295	281
296	282	void PrintReport(const ReportDesc *rep) {
...	...	@@ -364,7 +350,7 @@ void PrintReport(const ReportDesc *rep) {
364	350	Printf(" And %d more similar thread leaks.\n\n", rep->count - 1);
365	351
366	352	if (ReportStack *stack = ChooseSummaryStack(rep)) {
367		if (SymbolizedStack *frame = SkipTsanInternalFrames(stack->frames))
	353	if (const SymbolizedStack *frame = SkipTsanInternalFrames(stack->frames))
368	354	ReportErrorSummary(rep_typ_str, frame->info);
369	355	}
370	356

lib/tsan/tsan_report.h+2-1

...	...	@@ -34,7 +34,8 @@ enum ReportType {
34	34	ReportTypeMutexBadReadUnlock,
35	35	ReportTypeSignalUnsafe,
36	36	ReportTypeErrnoInSignal,
37		ReportTypeDeadlock
	37	ReportTypeDeadlock,
	38	ReportTypeMutexHeldWrongContext
38	39	};
39	40
40	41	struct ReportStack {

lib/tsan/tsan_rtl.cpp+14-10

...	...	@@ -35,8 +35,10 @@ extern "C" void __tsan_resume() {
35	35	__tsan_resumed = 1;
36	36	}
37	37
	38	#if SANITIZER_APPLE
38	39	SANITIZER_WEAK_DEFAULT_IMPL
39	40	void __tsan_test_only_on_fork() {}
	41	#endif
40	42
41	43	namespace __tsan {
42	44
...	...	@@ -46,11 +48,10 @@ int (*on_finalize)(int);
46	48	#endif
47	49
48	50	#if !SANITIZER_GO && !SANITIZER_APPLE
49		__attribute__((tls_model("initial-exec")))
50		THREADLOCAL char cur_thread_placeholder[sizeof(ThreadState)] ALIGNED(
51		SANITIZER_CACHE_LINE_SIZE);
	51	alignas(SANITIZER_CACHE_LINE_SIZE) THREADLOCAL __attribute__((tls_model(
	52	"initial-exec"))) char cur_thread_placeholder[sizeof(ThreadState)];
52	53	#endif
53		static char ctx_placeholder[sizeof(Context)] ALIGNED(SANITIZER_CACHE_LINE_SIZE);
	54	alignas(SANITIZER_CACHE_LINE_SIZE) static char ctx_placeholder[sizeof(Context)];
54	55	Context *ctx;
55	56
56	57	// Can be overriden by a front-end.
...	...	@@ -446,7 +447,7 @@ static bool InitializeMemoryProfiler() {
446	447	ctx->memprof_fd = 2;
447	448	} else {
448	449	InternalScopedString filename;
449		filename.append("%s.%d", fname, (int)internal_getpid());
	450	filename.AppendF("%s.%d", fname, (int)internal_getpid());
450	451	ctx->memprof_fd = OpenFile(filename.data(), WrOnly);
451	452	if (ctx->memprof_fd == kInvalidFd) {
452	453	Printf("ThreadSanitizer: failed to open memory profile file '%s'\n",
...	...	@@ -813,7 +814,7 @@ void ForkBefore(ThreadState* thr, uptr pc) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
813	814	ctx->thread_registry.Lock();
814	815	ctx->slot_mtx.Lock();
815	816	ScopedErrorReportLock::Lock();
816		AllocatorLock();
	817	AllocatorLockBeforeFork();
817	818	// Suppress all reports in the pthread_atfork callbacks.
818	819	// Reports will deadlock on the report_mtx.
819	820	// We could ignore sync operations as well,
...	...	@@ -828,14 +829,17 @@ void ForkBefore(ThreadState* thr, uptr pc) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
828	829	// Disables memory write in OnUserAlloc/Free.
829	830	thr->ignore_reads_and_writes++;
830	831
	832	# if SANITIZER_APPLE
831	833	__tsan_test_only_on_fork();
	834	# endif
832	835	}
833	836
834		static void ForkAfter(ThreadState* thr) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
	837	static void ForkAfter(ThreadState* thr,
	838	bool child) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
835	839	thr->suppress_reports--; // Enabled in ForkBefore.
836	840	thr->ignore_interceptors--;
837	841	thr->ignore_reads_and_writes--;
838		AllocatorUnlock();
	842	AllocatorUnlockAfterFork(child);
839	843	ScopedErrorReportLock::Unlock();
840	844	ctx->slot_mtx.Unlock();
841	845	ctx->thread_registry.Unlock();
...	...	@@ -845,10 +849,10 @@ static void ForkAfter(ThreadState* thr) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
845	849	GlobalProcessorUnlock();
846	850	}
847	851
848		void ForkParentAfter(ThreadState* thr, uptr pc) { ForkAfter(thr); }
	852	void ForkParentAfter(ThreadState* thr, uptr pc) { ForkAfter(thr, false); }
849	853
850	854	void ForkChildAfter(ThreadState* thr, uptr pc, bool start_thread) {
851		ForkAfter(thr);
	855	ForkAfter(thr, true);
852	856	u32 nthread = ctx->thread_registry.OnFork(thr->tid);
853	857	VPrintf(1,
854	858	"ThreadSanitizer: forked new process with pid %d,"

lib/tsan/tsan_rtl.h+6-6

...	...	@@ -56,8 +56,8 @@ namespace __tsan {
56	56
57	57	#if !SANITIZER_GO
58	58	struct MapUnmapCallback;
59		#if defined(__mips64) \|\| defined(__aarch64__) \|\| defined(__loongarch__) \|\| \
60		defined(__powerpc__)
	59	# if defined(__mips64) \|\| defined(__aarch64__) \|\| defined(__loongarch__) \|\| \
	60	defined(__powerpc__) \|\| SANITIZER_RISCV64
61	61
62	62	struct AP32 {
63	63	static const uptr kSpaceBeg = 0;
...	...	@@ -136,7 +136,7 @@ struct TidEpoch {
136	136	Epoch epoch;
137	137	};
138	138
139		struct TidSlot {
	139	struct alignas(SANITIZER_CACHE_LINE_SIZE) TidSlot {
140	140	Mutex mtx;
141	141	Sid sid;
142	142	atomic_uint32_t raw_epoch;
...	...	@@ -153,10 +153,10 @@ struct TidSlot {
153	153	}
154	154
155	155	TidSlot();
156		} ALIGNED(SANITIZER_CACHE_LINE_SIZE);
	156	};
157	157
158	158	// This struct is stored in TLS.
159		struct ThreadState {
	159	struct alignas(SANITIZER_CACHE_LINE_SIZE) ThreadState {
160	160	FastState fast_state;
161	161	int ignore_sync;
162	162	#if !SANITIZER_GO
...	...	@@ -234,7 +234,7 @@ struct ThreadState {
234	234	const ReportDesc *current_report;
235	235
236	236	explicit ThreadState(Tid tid);
237		} ALIGNED(SANITIZER_CACHE_LINE_SIZE);
	237	};
238	238
239	239	#if !SANITIZER_GO
240	240	#if SANITIZER_APPLE \|\| SANITIZER_ANDROID

lib/tsan/tsan_rtl_aarch64.S+7

...	...	@@ -2,6 +2,7 @@
2	2	#if defined(__aarch64__)
3	3
4	4	#include "sanitizer_common/sanitizer_asm.h"
	5	#include "builtins/assembly.h"
5	6
6	7	#if !defined(__APPLE__)
7	8	.section .text
...	...	@@ -16,6 +17,7 @@ ASM_HIDDEN(__tsan_setjmp)
16	17	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(setjmp))
17	18	ASM_SYMBOL_INTERCEPTOR(setjmp):
18	19	CFI_STARTPROC
	20	BTI_C
19	21
20	22	// Save frame/link register
21	23	stp x29, x30, [sp, -32]!
...	...	@@ -66,6 +68,7 @@ ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(setjmp))
66	68	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(_setjmp))
67	69	ASM_SYMBOL_INTERCEPTOR(_setjmp):
68	70	CFI_STARTPROC
	71	BTI_C
69	72
70	73	// Save frame/link register
71	74	stp x29, x30, [sp, -32]!
...	...	@@ -116,6 +119,7 @@ ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(_setjmp))
116	119	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))
117	120	ASM_SYMBOL_INTERCEPTOR(sigsetjmp):
118	121	CFI_STARTPROC
	122	BTI_C
119	123
120	124	// Save frame/link register
121	125	stp x29, x30, [sp, -32]!
...	...	@@ -168,6 +172,7 @@ ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))
168	172	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))
169	173	ASM_SYMBOL_INTERCEPTOR(__sigsetjmp):
170	174	CFI_STARTPROC
	175	BTI_C
171	176
172	177	// Save frame/link register
173	178	stp x29, x30, [sp, -32]!
...	...	@@ -217,4 +222,6 @@ ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))
217	222
218	223	NO_EXEC_STACK_DIRECTIVE
219	224
	225	GNU_PROPERTY_BTI_PAC
	226
220	227	#endif

lib/tsan/tsan_rtl_access.cpp+14-8

...	...	@@ -672,22 +672,28 @@ void MemoryAccessRangeT(ThreadState* thr, uptr pc, uptr addr, uptr size) {
672	672
673	673	#if SANITIZER_DEBUG
674	674	if (!IsAppMem(addr)) {
675		Printf("Access to non app mem %zx\n", addr);
	675	Printf("Access to non app mem start: %p\n", (void*)addr);
676	676	DCHECK(IsAppMem(addr));
677	677	}
678	678	if (!IsAppMem(addr + size - 1)) {
679		Printf("Access to non app mem %zx\n", addr + size - 1);
	679	Printf("Access to non app mem end: %p\n", (void*)(addr + size - 1));
680	680	DCHECK(IsAppMem(addr + size - 1));
681	681	}
682	682	if (!IsShadowMem(shadow_mem)) {
683		Printf("Bad shadow addr %p (%zx)\n", static_cast<void*>(shadow_mem), addr);
	683	Printf("Bad shadow start addr: %p (%p)\n", shadow_mem, (void*)addr);
684	684	DCHECK(IsShadowMem(shadow_mem));
685	685	}
686		if (!IsShadowMem(shadow_mem + size * kShadowCnt - 1)) {
687		Printf("Bad shadow addr %p (%zx)\n",
688		static_cast<void>(shadow_mem + size kShadowCnt - 1),
689		addr + size - 1);
690		DCHECK(IsShadowMem(shadow_mem + size * kShadowCnt - 1));
	686
	687	RawShadow* shadow_mem_end = reinterpret_cast<RawShadow*>(
	688	reinterpret_cast<uptr>(shadow_mem) + size * kShadowMultiplier - 1);
	689	if (!IsShadowMem(shadow_mem_end)) {
	690	Printf("Bad shadow end addr: %p (%p)\n", shadow_mem_end,
	691	(void*)(addr + size - 1));
	692	Printf(
	693	"Shadow start addr (ok): %p (%p); size: 0x%zx; kShadowMultiplier: "
	694	"%zx\n",
	695	shadow_mem, (void*)addr, size, kShadowMultiplier);
	696	DCHECK(IsShadowMem(shadow_mem_end));
691	697	}
692	698	#endif
693	699

lib/tsan/tsan_rtl_loongarch64.S created+196

...	...	@@ -0,0 +1,196 @@
	1	#include "sanitizer_common/sanitizer_asm.h"
	2
	3	.section .text
	4
	5	ASM_HIDDEN(__tsan_setjmp)
	6	.comm _ZN14__interception11real_setjmpE,8,8
	7	.globl ASM_SYMBOL_INTERCEPTOR(setjmp)
	8	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(setjmp))
	9	ASM_SYMBOL_INTERCEPTOR(setjmp):
	10	CFI_STARTPROC
	11
	12	// Save frame pointer and return address register
	13	addi.d $sp, $sp, -32
	14	st.d $ra, $sp, 24
	15	st.d $fp, $sp, 16
	16	CFI_DEF_CFA_OFFSET (32)
	17	CFI_OFFSET (1, -8)
	18	CFI_OFFSET (22, -16)
	19
	20	// Adjust the SP for previous frame
	21	addi.d $fp, $sp, 32
	22	CFI_DEF_CFA_REGISTER (22)
	23
	24	// Save env parameter
	25	st.d $a0, $sp, 8
	26	CFI_OFFSET (4, -24)
	27
	28	// Obtain SP, first argument to `void __tsan_setjmp(uptr sp)`
	29	addi.d $a0, $fp, 0
	30
	31	// call tsan interceptor
	32	bl ASM_SYMBOL(__tsan_setjmp)
	33
	34	// Restore env parameter
	35	ld.d $a0, $sp, 8
	36	CFI_RESTORE (4)
	37
	38	// Restore frame/link register
	39	ld.d $fp, $sp, 16
	40	ld.d $ra, $sp, 24
	41	addi.d $sp, $sp, 32
	42	CFI_RESTORE (22)
	43	CFI_RESTORE (1)
	44	CFI_DEF_CFA (3, 0)
	45
	46	// tail jump to libc setjmp
	47	la.local $a1, _ZN14__interception11real_setjmpE
	48	ld.d $a1, $a1, 0
	49	jr $a1
	50
	51	CFI_ENDPROC
	52	ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(setjmp))
	53
	54	.comm _ZN14__interception12real__setjmpE,8,8
	55	.globl ASM_SYMBOL_INTERCEPTOR(_setjmp)
	56	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(_setjmp))
	57	ASM_SYMBOL_INTERCEPTOR(_setjmp):
	58	CFI_STARTPROC
	59
	60	// Save frame pointer and return address register
	61	addi.d $sp, $sp, -32
	62	st.d $ra, $sp, 24
	63	st.d $fp, $sp, 16
	64	CFI_DEF_CFA_OFFSET (32)
	65	CFI_OFFSET (1, -8)
	66	CFI_OFFSET (22, -16)
	67
	68	// Adjust the SP for previous frame
	69	addi.d $fp, $sp, 32
	70	CFI_DEF_CFA_REGISTER (22)
	71
	72	// Save env parameter
	73	st.d $a0, $sp, 8
	74	CFI_OFFSET (4, -24)
	75
	76	// Obtain SP, first argument to `void __tsan_setjmp(uptr sp)`
	77	addi.d $a0, $fp, 0
	78
	79	// call tsan interceptor
	80	bl ASM_SYMBOL(__tsan_setjmp)
	81
	82	// Restore env parameter
	83	ld.d $a0, $sp, 8
	84	CFI_RESTORE (4)
	85
	86	// Restore frame/link register
	87	ld.d $fp, $sp, 16
	88	ld.d $ra, $sp, 24
	89	addi.d $sp, $sp, 32
	90	CFI_RESTORE (22)
	91	CFI_RESTORE (1)
	92	CFI_DEF_CFA (3, 0)
	93
	94	// tail jump to libc setjmp
	95	la.local $a1, _ZN14__interception12real__setjmpE
	96	ld.d $a1, $a1, 0
	97	jr $a1
	98
	99	CFI_ENDPROC
	100	ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(_setjmp))
	101
	102	.comm _ZN14__interception14real_sigsetjmpE,8,8
	103	.globl ASM_SYMBOL_INTERCEPTOR(sigsetjmp)
	104	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))
	105	ASM_SYMBOL_INTERCEPTOR(sigsetjmp):
	106	CFI_STARTPROC
	107
	108	// Save frame pointer and return address register
	109	addi.d $sp, $sp, -32
	110	st.d $ra, $sp, 24
	111	st.d $fp, $sp, 16
	112	CFI_DEF_CFA_OFFSET (32)
	113	CFI_OFFSET (1, -8)
	114	CFI_OFFSET (22, -16)
	115
	116	// Adjust the SP for previous frame
	117	addi.d $fp, $sp, 32
	118	CFI_DEF_CFA_REGISTER (22)
	119
	120	// Save env parameter
	121	st.d $a0, $sp, 8
	122	CFI_OFFSET (4, -24)
	123
	124	// Obtain SP, first argument to `void __tsan_setjmp(uptr sp)`
	125	addi.d $a0, $fp, 0
	126
	127	// call tsan interceptor
	128	bl ASM_SYMBOL(__tsan_setjmp)
	129
	130	// Restore env parameter
	131	ld.d $a0, $sp, 8
	132	CFI_RESTORE (4)
	133
	134	// Restore frame/link register
	135	ld.d $fp, $sp, 16
	136	ld.d $ra, $sp, 24
	137	addi.d $sp, $sp, 32
	138	CFI_RESTORE (22)
	139	CFI_RESTORE (1)
	140	CFI_DEF_CFA (3, 0)
	141
	142	// tail jump to libc setjmp
	143	la.local $a1, _ZN14__interception14real_sigsetjmpE
	144	ld.d $a1, $a1, 0
	145	jr $a1
	146
	147	CFI_ENDPROC
	148	ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))
	149
	150	.comm _ZN14__interception16real___sigsetjmpE,8,8
	151	.globl ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)
	152	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))
	153	ASM_SYMBOL_INTERCEPTOR(__sigsetjmp):
	154	CFI_STARTPROC
	155
	156	// Save frame pointer and return address register
	157	addi.d $sp, $sp, -32
	158	st.d $ra, $sp, 24
	159	st.d $fp, $sp, 16
	160	CFI_DEF_CFA_OFFSET (32)
	161	CFI_OFFSET (1, -8)
	162	CFI_OFFSET (22, -16)
	163
	164	// Adjust the SP for previous frame
	165	addi.d $fp, $sp, 32
	166	CFI_DEF_CFA_REGISTER (22)
	167
	168	// Save env parameter
	169	st.d $a0, $sp, 8
	170	CFI_OFFSET (4, -24)
	171
	172	// Obtain SP, first argument to `void __tsan_setjmp(uptr sp)`
	173	addi.d $a0, $fp, 0
	174
	175	// call tsan interceptor
	176	bl ASM_SYMBOL(__tsan_setjmp)
	177
	178	// Restore env parameter
	179	ld.d $a0, $sp, 8
	180	CFI_RESTORE (4)
	181
	182	// Restore frame/link register
	183	ld.d $fp, $sp, 16
	184	ld.d $ra, $sp, 24
	185	addi.d $sp, $sp, 32
	186	CFI_RESTORE (22)
	187	CFI_RESTORE (1)
	188	CFI_DEF_CFA (3, 0)
	189
	190	// tail jump to libc setjmp
	191	la.local $a1, _ZN14__interception16real___sigsetjmpE
	192	ld.d $a1, $a1, 0
	193	jr $a1
	194
	195	CFI_ENDPROC
	196	ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))

lib/tsan/tsan_rtl_mutex.cpp+1-1

...	...	@@ -446,9 +446,9 @@ void Acquire(ThreadState *thr, uptr pc, uptr addr) {
446	446	if (!s)
447	447	return;
448	448	SlotLocker locker(thr);
	449	ReadLock lock(&s->mtx);
449	450	if (!s->clock)
450	451	return;
451		ReadLock lock(&s->mtx);
452	452	thr->clock.Acquire(s->clock);
453	453	}
454	454

lib/tsan/tsan_rtl_riscv64.S created+203

...	...	@@ -0,0 +1,203 @@
	1	#include "sanitizer_common/sanitizer_asm.h"
	2
	3	.section .text
	4
	5	.comm _ZN14__interception11real_setjmpE,8,8
	6	.globl ASM_SYMBOL_INTERCEPTOR(setjmp)
	7	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(setjmp))
	8	ASM_SYMBOL_INTERCEPTOR(setjmp):
	9	CFI_STARTPROC
	10
	11	// Save frame pointer and return address register
	12	addi sp, sp, -32
	13	sd ra, 24(sp)
	14	sd s0, 16(sp)
	15	CFI_DEF_CFA_OFFSET (32)
	16	CFI_OFFSET (1, -8)
	17	CFI_OFFSET (8, -16)
	18
	19	// Adjust the SP for previous frame
	20	addi s0, sp, 32
	21	CFI_DEF_CFA_REGISTER (8)
	22
	23	// Save env parameter
	24	sd a0, 8(sp)
	25	CFI_OFFSET (10, -24)
	26
	27	// Obtain SP, first argument to `void __tsan_setjmp(uptr sp)`
	28	addi a0, s0, 0
	29
	30	// call tsan interceptor
	31	call ASM_SYMBOL(__tsan_setjmp)
	32
	33	// Restore env parameter
	34	ld a0, 8(sp)
	35	CFI_RESTORE (10)
	36
	37	// Restore frame/link register
	38	ld s0, 16(sp)
	39	ld ra, 24(sp)
	40	addi sp, sp, 32
	41	CFI_RESTORE (8)
	42	CFI_RESTORE (1)
	43	CFI_DEF_CFA (2, 0)
	44
	45	// tail jump to libc setjmp
	46	la t1, _ZN14__interception11real_setjmpE
	47	ld t1, 0(t1)
	48	jr t1
	49
	50	CFI_ENDPROC
	51	ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(setjmp))
	52
	53	.comm _ZN14__interception12real__setjmpE,8,8
	54	.globl ASM_SYMBOL_INTERCEPTOR(_setjmp)
	55	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(_setjmp))
	56	ASM_SYMBOL_INTERCEPTOR(_setjmp):
	57	CFI_STARTPROC
	58
	59	// Save frame pointer and return address register
	60	addi sp, sp, -32
	61	sd ra, 24(sp)
	62	sd s0, 16(sp)
	63	CFI_DEF_CFA_OFFSET (32)
	64	CFI_OFFSET (1, -8)
	65	CFI_OFFSET (8, -16)
	66
	67	// Adjust the SP for previous frame
	68	addi s0, sp, 32
	69	CFI_DEF_CFA_REGISTER (8)
	70
	71	// Save env parameter
	72	sd a0, 8(sp)
	73	CFI_OFFSET (10, -24)
	74
	75	// Obtain SP, first argument to `void __tsan_setjmp(uptr sp)`
	76	addi a0, s0, 0
	77
	78	// call tsan interceptor
	79	call ASM_SYMBOL(__tsan_setjmp)
	80
	81	// Restore env parameter
	82	ld a0, 8(sp)
	83	CFI_RESTORE (10)
	84
	85	// Restore frame/link register
	86	ld s0, 16(sp)
	87	ld ra, 24(sp)
	88	addi sp, sp, 32
	89	CFI_RESTORE (8)
	90	CFI_RESTORE (1)
	91	CFI_DEF_CFA (2, 0)
	92
	93	// tail jump to libc setjmp
	94	la t1, _ZN14__interception12real__setjmpE
	95	ld t1, 0(t1)
	96	jr t1
	97
	98	CFI_ENDPROC
	99	ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(_setjmp))
	100
	101	.comm _ZN14__interception14real_sigsetjmpE,8,8
	102	.globl ASM_SYMBOL_INTERCEPTOR(sigsetjmp)
	103	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))
	104	ASM_SYMBOL_INTERCEPTOR(sigsetjmp):
	105	CFI_STARTPROC
	106
	107	// Save frame pointer and return address register
	108	addi sp, sp, -32
	109	sd ra, 24(sp)
	110	sd s0, 16(sp)
	111	CFI_DEF_CFA_OFFSET (32)
	112	CFI_OFFSET (1, -8)
	113	CFI_OFFSET (8, -16)
	114
	115	// Adjust the SP for previous frame
	116	addi s0, sp, 32
	117	CFI_DEF_CFA_REGISTER (8)
	118
	119	// Save env parameter
	120	sd a0, 8(sp)
	121	sd a1, 0(sp)
	122	CFI_OFFSET (10, -24)
	123	CFI_OFFSET (11, -32)
	124
	125	// Obtain SP, first argument to `void __tsan_setjmp(uptr sp)`
	126	addi a0, s0, 0
	127
	128	// call tsan interceptor
	129	call ASM_SYMBOL(__tsan_setjmp)
	130
	131	// Restore env parameter
	132	ld a0, 8(sp)
	133	ld a1, 0(sp)
	134	CFI_RESTORE (10)
	135	CFI_RESTORE (11)
	136
	137	// Restore frame/link register
	138	ld s0, 16(sp)
	139	ld ra, 24(sp)
	140	addi sp, sp, 32
	141	CFI_RESTORE (8)
	142	CFI_RESTORE (1)
	143	CFI_DEF_CFA (2, 0)
	144
	145	// tail jump to libc setjmp
	146	la t1, _ZN14__interception14real_sigsetjmpE
	147	ld t1, 0(t1)
	148	jr t1
	149
	150	CFI_ENDPROC
	151	ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))
	152
	153	.comm _ZN14__interception16real___sigsetjmpE,8,8
	154	.globl ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)
	155	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))
	156	ASM_SYMBOL_INTERCEPTOR(__sigsetjmp):
	157	CFI_STARTPROC
	158
	159	// Save frame pointer and return address register
	160	addi sp, sp, -32
	161	sd ra, 24(sp)
	162	sd s0, 16(sp)
	163	CFI_DEF_CFA_OFFSET (32)
	164	CFI_OFFSET (1, -8)
	165	CFI_OFFSET (8, -16)
	166
	167	// Adjust the SP for previous frame
	168	addi s0, sp, 32
	169	CFI_DEF_CFA_REGISTER (8)
	170
	171	// Save env parameter
	172	sd a0, 8(sp)
	173	sd a1, 0(sp)
	174	CFI_OFFSET (10, -24)
	175	CFI_OFFSET (11, -32)
	176
	177	// Obtain SP, first argument to `void __tsan_setjmp(uptr sp)`
	178	addi a0, s0, 0
	179
	180	// call tsan interceptor
	181	call ASM_SYMBOL(__tsan_setjmp)
	182
	183	// Restore env parameter
	184	ld a0, 8(sp)
	185	ld a1, 0(sp)
	186	CFI_RESTORE (10)
	187	CFI_RESTORE (11)
	188
	189	// Restore frame/link register
	190	ld s0, 16(sp)
	191	ld ra, 24(sp)
	192	addi sp, sp, 32
	193	CFI_RESTORE (8)
	194	CFI_RESTORE (1)
	195	CFI_DEF_CFA (2, 0)
	196
	197	// tail jump to libc setjmp
	198	la t1, _ZN14__interception16real___sigsetjmpE
	199	ld t1, 0(t1)
	200	jr t1
	201
	202	CFI_ENDPROC
	203	ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))

lib/tsan/tsan_rtl_s390x.S created+49

...	...	@@ -0,0 +1,49 @@
	1	#include "sanitizer_common/sanitizer_asm.h"
	2
	3	#define CFA_OFFSET 160
	4	#define R2_REL_OFFSET 16
	5	#define R3_REL_OFFSET 24
	6	#define R14_REL_OFFSET 112
	7	#define R15_REL_OFFSET 120
	8	#define FRAME_SIZE 160
	9
	10	.text
	11
	12	ASM_HIDDEN(__tsan_setjmp)
	13
	14	.macro intercept symbol, real
	15	.comm \real, 8, 8
	16	.globl ASM_SYMBOL_INTERCEPTOR(\symbol)
	17	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(\symbol))
	18	ASM_SYMBOL_INTERCEPTOR(\symbol):
	19	CFI_STARTPROC
	20	stmg %r2, %r3, R2_REL_OFFSET(%r15)
	21	CFI_REL_OFFSET(%r2, R2_REL_OFFSET)
	22	CFI_REL_OFFSET(%r3, R3_REL_OFFSET)
	23	stmg %r14, %r15, R14_REL_OFFSET(%r15)
	24	CFI_REL_OFFSET(%r14, R14_REL_OFFSET)
	25	CFI_REL_OFFSET(%r15, R15_REL_OFFSET)
	26	aghi %r15, -FRAME_SIZE
	27	CFI_ADJUST_CFA_OFFSET(FRAME_SIZE)
	28	la %r2, FRAME_SIZE(%r15)
	29	brasl %r14, ASM_SYMBOL(__tsan_setjmp)
	30	lmg %r14, %r15, FRAME_SIZE + R14_REL_OFFSET(%r15)
	31	CFI_RESTORE(%r14)
	32	CFI_RESTORE(%r15)
	33	CFI_DEF_CFA_OFFSET(CFA_OFFSET)
	34	lmg %r2, %r3, R2_REL_OFFSET(%r15)
	35	CFI_RESTORE(%r2)
	36	CFI_RESTORE(%r3)
	37	larl %r1, \real
	38	lg %r1, 0(%r1)
	39	br %r1
	40	CFI_ENDPROC
	41	ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(\symbol))
	42	.endm
	43
	44	intercept setjmp, _ZN14__interception11real_setjmpE
	45	intercept _setjmp, _ZN14__interception12real__setjmpE
	46	intercept sigsetjmp, _ZN14__interception14real_sigsetjmpE
	47	intercept __sigsetjmp, _ZN14__interception16real___sigsetjmpE
	48
	49	NO_EXEC_STACK_DIRECTIVE

lib/tsan/tsan_rtl_thread.cpp+5-5

...	...	@@ -160,6 +160,10 @@ void ThreadStart(ThreadState *thr, Tid tid, tid_t os_id,
160	160	}
161	161	Free(thr->tctx->sync);
162	162
	163	#if !SANITIZER_GO
	164	thr->is_inited = true;
	165	#endif
	166
163	167	uptr stk_addr = 0;
164	168	uptr stk_size = 0;
165	169	uptr tls_addr = 0;
...	...	@@ -200,15 +204,11 @@ void ThreadStart(ThreadState *thr, Tid tid, tid_t os_id,
200	204	}
201	205
202	206	void ThreadContext::OnStarted(void *arg) {
203		thr = static_cast<ThreadState *>(arg);
204	207	DPrintf("#%d: ThreadStart\n", tid);
205		new (thr) ThreadState(tid);
	208	thr = new (arg) ThreadState(tid);
206	209	if (common_flags()->detect_deadlocks)
207	210	thr->dd_lt = ctx->dd->CreateLogicalThread(tid);
208	211	thr->tctx = this;
209		#if !SANITIZER_GO
210		thr->is_inited = true;
211		#endif
212	212	}
213	213
214	214	void ThreadFinish(ThreadState *thr) {

lib/tsan/tsan_suppressions.cpp+2-1

...	...	@@ -42,7 +42,7 @@ const char *__tsan_default_suppressions() {
42	42
43	43	namespace __tsan {
44	44
45		ALIGNED(64) static char suppression_placeholder[sizeof(SuppressionContext)];
	45	alignas(64) static char suppression_placeholder[sizeof(SuppressionContext)];
46	46	static SuppressionContext *suppression_ctx = nullptr;
47	47	static const char *kSuppressionTypes[] = {
48	48	kSuppressionRace, kSuppressionRaceTop, kSuppressionMutex,
...	...	@@ -81,6 +81,7 @@ static const char *conv(ReportType typ) {
81	81	case ReportTypeMutexBadUnlock:
82	82	case ReportTypeMutexBadReadLock:
83	83	case ReportTypeMutexBadReadUnlock:
	84	case ReportTypeMutexHeldWrongContext:
84	85	return kSuppressionMutex;
85	86	case ReportTypeSignalUnsafe:
86	87	case ReportTypeErrnoInSignal:

lib/tsan/tsan_vector_clock.h+1-1

...	...	@@ -34,7 +34,7 @@ class VectorClock {
34	34	VectorClock& operator=(const VectorClock& other);
35	35
36	36	private:
37		Epoch clk_[kThreadSlotCount] VECTOR_ALIGNED;
	37	VECTOR_ALIGNED Epoch clk_[kThreadSlotCount];
38	38	};
39	39
40	40	ALWAYS_INLINE Epoch VectorClock::Get(Sid sid) const {

src/libtsan.zig+11-5

...	...	@@ -160,10 +160,13 @@ pub fn buildTsan(comp: *Compilation, prog_node: std.Progress.Node) BuildError!vo
160	160	}
161	161	{
162	162	const asm_source = switch (target.cpu.arch) {
163		.aarch64 => "tsan_rtl_aarch64.S",
	163	.aarch64, .aarch64_be => "tsan_rtl_aarch64.S",
	164	.loongarch64 => "tsan_rtl_loongarch64.S",
	165	.mips64, .mips64el => "tsan_rtl_mips64.S",
	166	.powerpc64, .powerpc64le => "tsan_rtl_ppc64.S",
	167	.riscv64 => "tsan_rtl_riscv64.S",
	168	.s390x => "tsan_rtl_s390x.S",
164	169	.x86_64 => "tsan_rtl_amd64.S",
165		.mips64 => "tsan_rtl_mips64.S",
166		.powerpc64 => "tsan_rtl_ppc64.S",
167	170	else => return error.TSANUnsupportedCPUArchitecture,
168	171	};
169	172	var cflags = std.ArrayList([]const u8).init(arena);
...	...	@@ -416,7 +419,6 @@ const sanitizer_common_sources = [_][]const u8{
416	419	"sanitizer_platform_limits_freebsd.cpp",
417	420	"sanitizer_platform_limits_linux.cpp",
418	421	"sanitizer_platform_limits_netbsd.cpp",
419		"sanitizer_platform_limits_openbsd.cpp",
420	422	"sanitizer_platform_limits_posix.cpp",
421	423	"sanitizer_platform_limits_solaris.cpp",
422	424	"sanitizer_posix.cpp",
...	...	@@ -429,7 +431,6 @@ const sanitizer_common_sources = [_][]const u8{
429	431	"sanitizer_procmaps_solaris.cpp",
430	432	"sanitizer_range.cpp",
431	433	"sanitizer_solaris.cpp",
432		"sanitizer_stack_store.cpp",
433	434	"sanitizer_stoptheworld_fuchsia.cpp",
434	435	"sanitizer_stoptheworld_mac.cpp",
435	436	"sanitizer_stoptheworld_win.cpp",
...	...	@@ -452,6 +453,7 @@ const sanitizer_nolibc_sources = [_][]const u8{
452	453	const sanitizer_libcdep_sources = [_][]const u8{
453	454	"sanitizer_common_libcdep.cpp",
454	455	"sanitizer_allocator_checks.cpp",
	456	"sanitizer_dl.cpp",
455	457	"sanitizer_linux_libcdep.cpp",
456	458	"sanitizer_mac_libcdep.cpp",
457	459	"sanitizer_posix_libcdep.cpp",
...	...	@@ -461,6 +463,7 @@ const sanitizer_libcdep_sources = [_][]const u8{
461	463
462	464	const sanitizer_symbolizer_sources = [_][]const u8{
463	465	"sanitizer_allocator_report.cpp",
	466	"sanitizer_stack_store.cpp",
464	467	"sanitizer_stackdepot.cpp",
465	468	"sanitizer_stacktrace.cpp",
466	469	"sanitizer_stacktrace_libcdep.cpp",
...	...	@@ -471,10 +474,13 @@ const sanitizer_symbolizer_sources = [_][]const u8{
471	474	"sanitizer_symbolizer_libcdep.cpp",
472	475	"sanitizer_symbolizer_mac.cpp",
473	476	"sanitizer_symbolizer_markup.cpp",
	477	"sanitizer_symbolizer_markup_fuchsia.cpp",
474	478	"sanitizer_symbolizer_posix_libcdep.cpp",
475	479	"sanitizer_symbolizer_report.cpp",
	480	"sanitizer_symbolizer_report_fuchsia.cpp",
476	481	"sanitizer_symbolizer_win.cpp",
477	482	"sanitizer_unwind_linux_libcdep.cpp",
	483	"sanitizer_unwind_fuchsia.cpp",
478	484	"sanitizer_unwind_win.cpp",
479	485	};
480	486

author	Alex Rønne Petersen <alex@alexrp.com> 2024-09-24 13:47:29+02:00
committer	Andrew Kelley <andrew@ziglang.org> 2024-09-24 23:58:21-07:00
log	a40cdad18c06fd377622c47aa34564df7ea959b5
tree	dbcfabc2191cdbf2acc6c356c5ea25237b29d8f1
parent	7f6b7c56089eaa5b147e71b3d98328498c9025c8

tsan: Update to LLVM 19.1.0.