Commit 8219d92987c7d9641d6b24c4d5be29e3a80fd6b9 | zig

author	Andrew Kelley <andrew@ziglang.org> 2020-12-22 19:25:24-07:00
committer	Andrew Kelley <andrew@ziglang.org> 2020-12-24 01:18:48-07:00
log	8219d92987c7d9641d6b24c4d5be29e3a80fd6b9
tree	026578eeb7d11f5813d081025763f064401536e1
parent	42b4a48bc96ce22562230cd1a266f93a013c76dd

stage2: fix Cache deadlock and build more of TSAN

 * rename is_compiler_rt_or_libc to skip_linker_dependencies
   and set it to `true` for all sub-Compilations. I believe
   this resolves the deadlock we were experiencing on Drone
   CI and on some users' computers. I will remove the CI workaround in
   a follow-up commit.
 * enabling TSAN automatically causes the Compilation to link against
   libc++ even if not requested, because TSAN depends on libc++.
 * add -fno-rtti flags where appropriate when building TSAN objects.
   Thanks Firefox317 for pointing this out.
 * TSAN support: resolve all the undefined symbols. We are still seeing
   a dependency on __gcc_personality_v0 but will resolve this one in a
   follow-up commit.
 * static libs do not try to build libc++ or libc++abi.

48 files changed, 10172 insertions(+), 41 deletions(-)

lib
- tsan
src

lib/tsan/interception/interception_linux.cpp created+83

...	...	@@ -0,0 +1,83 @@
	1	//===-- interception_linux.cpp ----------------------------------- C++ --===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file is a part of AddressSanitizer, an address sanity checker.
	10	//
	11	// Linux-specific interception methods.
	12	//===----------------------------------------------------------------------===//
	13
	14	#include "interception.h"
	15
	16	#if SANITIZER_LINUX \|\| SANITIZER_FREEBSD \|\| SANITIZER_NETBSD \|\| \
	17	SANITIZER_OPENBSD \|\| SANITIZER_SOLARIS
	18
	19	#include <dlfcn.h> // for dlsym() and dlvsym()
	20
	21	namespace __interception {
	22
	23	#if SANITIZER_NETBSD
	24	static int StrCmp(const char s1, const char s2) {
	25	while (true) {
	26	if (s1 != s2)
	27	return false;
	28	if (*s1 == 0)
	29	return true;
	30	s1++;
	31	s2++;
	32	}
	33	}
	34	#endif
	35
	36	static void GetFuncAddr(const char name, uptr wrapper_addr) {
	37	#if SANITIZER_NETBSD
	38	// FIXME: Find a better way to handle renames
	39	if (StrCmp(name, "sigaction"))
	40	name = "__sigaction14";
	41	#endif
	42	void *addr = dlsym(RTLD_NEXT, name);
	43	if (!addr) {
	44	// If the lookup using RTLD_NEXT failed, the sanitizer runtime library is
	45	// later in the library search order than the DSO that we are trying to
	46	// intercept, which means that we cannot intercept this function. We still
	47	// want the address of the real definition, though, so look it up using
	48	// RTLD_DEFAULT.
	49	addr = dlsym(RTLD_DEFAULT, name);
	50
	51	// In case `name' is not loaded, dlsym ends up finding the actual wrapper.
	52	// We don't want to intercept the wrapper and have it point to itself.
	53	if ((uptr)addr == wrapper_addr)
	54	addr = nullptr;
	55	}
	56	return addr;
	57	}
	58
	59	bool InterceptFunction(const char name, uptr ptr_to_real, uptr func,
	60	uptr wrapper) {
	61	void *addr = GetFuncAddr(name, wrapper);
	62	*ptr_to_real = (uptr)addr;
	63	return addr && (func == wrapper);
	64	}
	65
	66	// Android and Solaris do not have dlvsym
	67	#if !SANITIZER_ANDROID && !SANITIZER_SOLARIS && !SANITIZER_OPENBSD
	68	static void GetFuncAddr(const char name, const char *ver) {
	69	return dlvsym(RTLD_NEXT, name, ver);
	70	}
	71
	72	bool InterceptFunction(const char name, const char ver, uptr *ptr_to_real,
	73	uptr func, uptr wrapper) {
	74	void *addr = GetFuncAddr(name, ver);
	75	*ptr_to_real = (uptr)addr;
	76	return addr && (func == wrapper);
	77	}
	78	#endif // !SANITIZER_ANDROID
	79
	80	} // namespace __interception
	81
	82	#endif // SANITIZER_LINUX \|\| SANITIZER_FREEBSD \|\| SANITIZER_NETBSD \|\|
	83	// SANITIZER_OPENBSD \|\| SANITIZER_SOLARIS

lib/tsan/interception/interception_mac.cpp created+18

...	...	@@ -0,0 +1,18 @@
	1	//===-- interception_mac.cpp ------------------------------------- C++ --===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file is a part of AddressSanitizer, an address sanity checker.
	10	//
	11	// Mac-specific interception methods.
	12	//===----------------------------------------------------------------------===//
	13
	14	#include "interception.h"
	15
	16	#if SANITIZER_MAC
	17
	18	#endif // SANITIZER_MAC

lib/tsan/interception/interception_type_test.cpp created+39

...	...	@@ -0,0 +1,39 @@
	1	//===-- interception_type_test.cpp ------------------------------- C++ --===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file is a part of AddressSanitizer, an address sanity checker.
	10	//
	11	// Compile-time tests of the internal type definitions.
	12	//===----------------------------------------------------------------------===//
	13
	14	#include "interception.h"
	15
	16	#if SANITIZER_LINUX \|\| SANITIZER_MAC
	17
	18	#include <sys/types.h>
	19	#include <stddef.h>
	20	#include <stdint.h>
	21
	22	COMPILER_CHECK(sizeof(::SIZE_T) == sizeof(size_t));
	23	COMPILER_CHECK(sizeof(::SSIZE_T) == sizeof(ssize_t));
	24	COMPILER_CHECK(sizeof(::PTRDIFF_T) == sizeof(ptrdiff_t));
	25	COMPILER_CHECK(sizeof(::INTMAX_T) == sizeof(intmax_t));
	26
	27	#if !SANITIZER_MAC
	28	COMPILER_CHECK(sizeof(::OFF64_T) == sizeof(off64_t));
	29	#endif
	30
	31	// The following are the cases when pread (and friends) is used instead of
	32	// pread64. In those cases we need OFF_T to match off_t. We don't care about the
	33	// rest (they depend on _FILE_OFFSET_BITS setting when building an application).
	34	# if SANITIZER_ANDROID \|\| !defined _FILE_OFFSET_BITS \|\| \
	35	_FILE_OFFSET_BITS != 64
	36	COMPILER_CHECK(sizeof(::OFF_T) == sizeof(off_t));
	37	# endif
	38
	39	#endif

lib/tsan/interception/interception_win.cpp created+1022

...	...	@@ -0,0 +1,1022 @@
	1	//===-- interception_linux.cpp ----------------------------------- C++ --===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file is a part of AddressSanitizer, an address sanity checker.
	10	//
	11	// Windows-specific interception methods.
	12	//
	13	// This file is implementing several hooking techniques to intercept calls
	14	// to functions. The hooks are dynamically installed by modifying the assembly
	15	// code.
	16	//
	17	// The hooking techniques are making assumptions on the way the code is
	18	// generated and are safe under these assumptions.
	19	//
	20	// On 64-bit architecture, there is no direct 64-bit jump instruction. To allow
	21	// arbitrary branching on the whole memory space, the notion of trampoline
	22	// region is used. A trampoline region is a memory space withing 2G boundary
	23	// where it is safe to add custom assembly code to build 64-bit jumps.
	24	//
	25	// Hooking techniques
	26	// ==================
	27	//
	28	// 1) Detour
	29	//
	30	// The Detour hooking technique is assuming the presence of an header with
	31	// padding and an overridable 2-bytes nop instruction (mov edi, edi). The
	32	// nop instruction can safely be replaced by a 2-bytes jump without any need
	33	// to save the instruction. A jump to the target is encoded in the function
	34	// header and the nop instruction is replaced by a short jump to the header.
	35	//
	36	// head: 5 x nop head: jmp <hook>
	37	// func: mov edi, edi --> func: jmp short <head>
	38	// [...] real: [...]
	39	//
	40	// This technique is only implemented on 32-bit architecture.
	41	// Most of the time, Windows API are hookable with the detour technique.
	42	//
	43	// 2) Redirect Jump
	44	//
	45	// The redirect jump is applicable when the first instruction is a direct
	46	// jump. The instruction is replaced by jump to the hook.
	47	//
	48	// func: jmp <label> --> func: jmp <hook>
	49	//
	50	// On an 64-bit architecture, a trampoline is inserted.
	51	//
	52	// func: jmp <label> --> func: jmp <tramp>
	53	// [...]
	54	//
	55	// [trampoline]
	56	// tramp: jmp QWORD [addr]
	57	// addr: .bytes <hook>
	58	//
	59	// Note: <real> is equilavent to <label>.
	60	//
	61	// 3) HotPatch
	62	//
	63	// The HotPatch hooking is assuming the presence of an header with padding
	64	// and a first instruction with at least 2-bytes.
	65	//
	66	// The reason to enforce the 2-bytes limitation is to provide the minimal
	67	// space to encode a short jump. HotPatch technique is only rewriting one
	68	// instruction to avoid breaking a sequence of instructions containing a
	69	// branching target.
	70	//
	71	// Assumptions are enforced by MSVC compiler by using the /HOTPATCH flag.
	72	// see: https://msdn.microsoft.com/en-us/library/ms173507.aspx
	73	// Default padding length is 5 bytes in 32-bits and 6 bytes in 64-bits.
	74	//
	75	// head: 5 x nop head: jmp <hook>
	76	// func: <instr> --> func: jmp short <head>
	77	// [...] body: [...]
	78	//
	79	// [trampoline]
	80	// real: <instr>
	81	// jmp <body>
	82	//
	83	// On an 64-bit architecture:
	84	//
	85	// head: 6 x nop head: jmp QWORD [addr1]
	86	// func: <instr> --> func: jmp short <head>
	87	// [...] body: [...]
	88	//
	89	// [trampoline]
	90	// addr1: .bytes <hook>
	91	// real: <instr>
	92	// jmp QWORD [addr2]
	93	// addr2: .bytes <body>
	94	//
	95	// 4) Trampoline
	96	//
	97	// The Trampoline hooking technique is the most aggressive one. It is
	98	// assuming that there is a sequence of instructions that can be safely
	99	// replaced by a jump (enough room and no incoming branches).
	100	//
	101	// Unfortunately, these assumptions can't be safely presumed and code may
	102	// be broken after hooking.
	103	//
	104	// func: <instr> --> func: jmp <hook>
	105	// <instr>
	106	// [...] body: [...]
	107	//
	108	// [trampoline]
	109	// real: <instr>
	110	// <instr>
	111	// jmp <body>
	112	//
	113	// On an 64-bit architecture:
	114	//
	115	// func: <instr> --> func: jmp QWORD [addr1]
	116	// <instr>
	117	// [...] body: [...]
	118	//
	119	// [trampoline]
	120	// addr1: .bytes <hook>
	121	// real: <instr>
	122	// <instr>
	123	// jmp QWORD [addr2]
	124	// addr2: .bytes <body>
	125	//===----------------------------------------------------------------------===//
	126
	127	#include "interception.h"
	128
	129	#if SANITIZER_WINDOWS
	130	#include "sanitizer_common/sanitizer_platform.h"
	131	#define WIN32_LEAN_AND_MEAN
	132	#include <windows.h>
	133
	134	namespace __interception {
	135
	136	static const int kAddressLength = FIRST_32_SECOND_64(4, 8);
	137	static const int kJumpInstructionLength = 5;
	138	static const int kShortJumpInstructionLength = 2;
	139	static const int kIndirectJumpInstructionLength = 6;
	140	static const int kBranchLength =
	141	FIRST_32_SECOND_64(kJumpInstructionLength, kIndirectJumpInstructionLength);
	142	static const int kDirectBranchLength = kBranchLength + kAddressLength;
	143
	144	static void InterceptionFailed() {
	145	// Do we have a good way to abort with an error message here?
	146	__debugbreak();
	147	}
	148
	149	static bool DistanceIsWithin2Gig(uptr from, uptr target) {
	150	#if SANITIZER_WINDOWS64
	151	if (from < target)
	152	return target - from <= (uptr)0x7FFFFFFFU;
	153	else
	154	return from - target <= (uptr)0x80000000U;
	155	#else
	156	// In a 32-bit address space, the address calculation will wrap, so this check
	157	// is unnecessary.
	158	return true;
	159	#endif
	160	}
	161
	162	static uptr GetMmapGranularity() {
	163	SYSTEM_INFO si;
	164	GetSystemInfo(&si);
	165	return si.dwAllocationGranularity;
	166	}
	167
	168	static uptr RoundUpTo(uptr size, uptr boundary) {
	169	return (size + boundary - 1) & ~(boundary - 1);
	170	}
	171
	172	// FIXME: internal_str* and internal_mem* functions should be moved from the
	173	// ASan sources into interception/.
	174
	175	static size_t _strlen(const char *str) {
	176	const char* p = str;
	177	while (*p != '\0') ++p;
	178	return p - str;
	179	}
	180
	181	static char* _strchr(char* str, char c) {
	182	while (*str) {
	183	if (*str == c)
	184	return str;
	185	++str;
	186	}
	187	return nullptr;
	188	}
	189
	190	static void _memset(void *p, int value, size_t sz) {
	191	for (size_t i = 0; i < sz; ++i)
	192	((char*)p)[i] = (char)value;
	193	}
	194
	195	static void _memcpy(void dst, void src, size_t sz) {
	196	char dst_c = (char)dst,
	197	src_c = (char)src;
	198	for (size_t i = 0; i < sz; ++i)
	199	dst_c[i] = src_c[i];
	200	}
	201
	202	static bool ChangeMemoryProtection(
	203	uptr address, uptr size, DWORD *old_protection) {
	204	return ::VirtualProtect((void*)address, size,
	205	PAGE_EXECUTE_READWRITE,
	206	old_protection) != FALSE;
	207	}
	208
	209	static bool RestoreMemoryProtection(
	210	uptr address, uptr size, DWORD old_protection) {
	211	DWORD unused;
	212	return ::VirtualProtect((void*)address, size,
	213	old_protection,
	214	&unused) != FALSE;
	215	}
	216
	217	static bool IsMemoryPadding(uptr address, uptr size) {
	218	u8* function = (u8*)address;
	219	for (size_t i = 0; i < size; ++i)
	220	if (function[i] != 0x90 && function[i] != 0xCC)
	221	return false;
	222	return true;
	223	}
	224
	225	static const u8 kHintNop8Bytes[] = {
	226	0x0F, 0x1F, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00
	227	};
	228
	229	template<class T>
	230	static bool FunctionHasPrefix(uptr address, const T &pattern) {
	231	u8* function = (u8*)address - sizeof(pattern);
	232	for (size_t i = 0; i < sizeof(pattern); ++i)
	233	if (function[i] != pattern[i])
	234	return false;
	235	return true;
	236	}
	237
	238	static bool FunctionHasPadding(uptr address, uptr size) {
	239	if (IsMemoryPadding(address - size, size))
	240	return true;
	241	if (size <= sizeof(kHintNop8Bytes) &&
	242	FunctionHasPrefix(address, kHintNop8Bytes))
	243	return true;
	244	return false;
	245	}
	246
	247	static void WritePadding(uptr from, uptr size) {
	248	_memset((void*)from, 0xCC, (size_t)size);
	249	}
	250
	251	static void WriteJumpInstruction(uptr from, uptr target) {
	252	if (!DistanceIsWithin2Gig(from + kJumpInstructionLength, target))
	253	InterceptionFailed();
	254	ptrdiff_t offset = target - from - kJumpInstructionLength;
	255	(u8)from = 0xE9;
	256	(u32)(from + 1) = offset;
	257	}
	258
	259	static void WriteShortJumpInstruction(uptr from, uptr target) {
	260	sptr offset = target - from - kShortJumpInstructionLength;
	261	if (offset < -128 \|\| offset > 127)
	262	InterceptionFailed();
	263	(u8)from = 0xEB;
	264	(u8)(from + 1) = (u8)offset;
	265	}
	266
	267	#if SANITIZER_WINDOWS64
	268	static void WriteIndirectJumpInstruction(uptr from, uptr indirect_target) {
	269	// jmp [rip + <offset>] = FF 25 <offset> where <offset> is a relative
	270	// offset.
	271	// The offset is the distance from then end of the jump instruction to the
	272	// memory location containing the targeted address. The displacement is still
	273	// 32-bit in x64, so indirect_target must be located within +/- 2GB range.
	274	int offset = indirect_target - from - kIndirectJumpInstructionLength;
	275	if (!DistanceIsWithin2Gig(from + kIndirectJumpInstructionLength,
	276	indirect_target)) {
	277	InterceptionFailed();
	278	}
	279	(u16)from = 0x25FF;
	280	(u32)(from + 2) = offset;
	281	}
	282	#endif
	283
	284	static void WriteBranch(
	285	uptr from, uptr indirect_target, uptr target) {
	286	#if SANITIZER_WINDOWS64
	287	WriteIndirectJumpInstruction(from, indirect_target);
	288	(u64)indirect_target = target;
	289	#else
	290	(void)indirect_target;
	291	WriteJumpInstruction(from, target);
	292	#endif
	293	}
	294
	295	static void WriteDirectBranch(uptr from, uptr target) {
	296	#if SANITIZER_WINDOWS64
	297	// Emit an indirect jump through immediately following bytes:
	298	// jmp [rip + kBranchLength]
	299	// .quad <target>
	300	WriteBranch(from, from + kBranchLength, target);
	301	#else
	302	WriteJumpInstruction(from, target);
	303	#endif
	304	}
	305
	306	struct TrampolineMemoryRegion {
	307	uptr content;
	308	uptr allocated_size;
	309	uptr max_size;
	310	};
	311
	312	static const uptr kTrampolineScanLimitRange = 1 << 31; // 2 gig
	313	static const int kMaxTrampolineRegion = 1024;
	314	static TrampolineMemoryRegion TrampolineRegions[kMaxTrampolineRegion];
	315
	316	static void *AllocateTrampolineRegion(uptr image_address, size_t granularity) {
	317	#if SANITIZER_WINDOWS64
	318	uptr address = image_address;
	319	uptr scanned = 0;
	320	while (scanned < kTrampolineScanLimitRange) {
	321	MEMORY_BASIC_INFORMATION info;
	322	if (!::VirtualQuery((void*)address, &info, sizeof(info)))
	323	return nullptr;
	324
	325	// Check whether a region can be allocated at \|address\|.
	326	if (info.State == MEM_FREE && info.RegionSize >= granularity) {
	327	void page = ::VirtualAlloc((void)RoundUpTo(address, granularity),
	328	granularity,
	329	MEM_RESERVE \| MEM_COMMIT,
	330	PAGE_EXECUTE_READWRITE);
	331	return page;
	332	}
	333
	334	// Move to the next region.
	335	address = (uptr)info.BaseAddress + info.RegionSize;
	336	scanned += info.RegionSize;
	337	}
	338	return nullptr;
	339	#else
	340	return ::VirtualAlloc(nullptr,
	341	granularity,
	342	MEM_RESERVE \| MEM_COMMIT,
	343	PAGE_EXECUTE_READWRITE);
	344	#endif
	345	}
	346
	347	// Used by unittests to release mapped memory space.
	348	void TestOnlyReleaseTrampolineRegions() {
	349	for (size_t bucket = 0; bucket < kMaxTrampolineRegion; ++bucket) {
	350	TrampolineMemoryRegion *current = &TrampolineRegions[bucket];
	351	if (current->content == 0)
	352	return;
	353	::VirtualFree((void*)current->content, 0, MEM_RELEASE);
	354	current->content = 0;
	355	}
	356	}
	357
	358	static uptr AllocateMemoryForTrampoline(uptr image_address, size_t size) {
	359	// Find a region within 2G with enough space to allocate \|size\| bytes.
	360	TrampolineMemoryRegion *region = nullptr;
	361	for (size_t bucket = 0; bucket < kMaxTrampolineRegion; ++bucket) {
	362	TrampolineMemoryRegion* current = &TrampolineRegions[bucket];
	363	if (current->content == 0) {
	364	// No valid region found, allocate a new region.
	365	size_t bucket_size = GetMmapGranularity();
	366	void *content = AllocateTrampolineRegion(image_address, bucket_size);
	367	if (content == nullptr)
	368	return 0U;
	369
	370	current->content = (uptr)content;
	371	current->allocated_size = 0;
	372	current->max_size = bucket_size;
	373	region = current;
	374	break;
	375	} else if (current->max_size - current->allocated_size > size) {
	376	#if SANITIZER_WINDOWS64
	377	// In 64-bits, the memory space must be allocated within 2G boundary.
	378	uptr next_address = current->content + current->allocated_size;
	379	if (next_address < image_address \|\|
	380	next_address - image_address >= 0x7FFF0000)
	381	continue;
	382	#endif
	383	// The space can be allocated in the current region.
	384	region = current;
	385	break;
	386	}
	387	}
	388
	389	// Failed to find a region.
	390	if (region == nullptr)
	391	return 0U;
	392
	393	// Allocate the space in the current region.
	394	uptr allocated_space = region->content + region->allocated_size;
	395	region->allocated_size += size;
	396	WritePadding(allocated_space, size);
	397
	398	return allocated_space;
	399	}
	400
	401	// Returns 0 on error.
	402	static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
	403	switch ((u64)address) {
	404	case 0x90909090909006EB: // stub: jmp over 6 x nop.
	405	return 8;
	406	}
	407
	408	switch ((u8)address) {
	409	case 0x90: // 90 : nop
	410	return 1;
	411
	412	case 0x50: // push eax / rax
	413	case 0x51: // push ecx / rcx
	414	case 0x52: // push edx / rdx
	415	case 0x53: // push ebx / rbx
	416	case 0x54: // push esp / rsp
	417	case 0x55: // push ebp / rbp
	418	case 0x56: // push esi / rsi
	419	case 0x57: // push edi / rdi
	420	case 0x5D: // pop ebp / rbp
	421	return 1;
	422
	423	case 0x6A: // 6A XX = push XX
	424	return 2;
	425
	426	case 0xb8: // b8 XX XX XX XX : mov eax, XX XX XX XX
	427	case 0xB9: // b9 XX XX XX XX : mov ecx, XX XX XX XX
	428	return 5;
	429
	430	// Cannot overwrite control-instruction. Return 0 to indicate failure.
	431	case 0xE9: // E9 XX XX XX XX : jmp <label>
	432	case 0xE8: // E8 XX XX XX XX : call <func>
	433	case 0xC3: // C3 : ret
	434	case 0xEB: // EB XX : jmp XX (short jump)
	435	case 0x70: // 7Y YY : jy XX (short conditional jump)
	436	case 0x71:
	437	case 0x72:
	438	case 0x73:
	439	case 0x74:
	440	case 0x75:
	441	case 0x76:
	442	case 0x77:
	443	case 0x78:
	444	case 0x79:
	445	case 0x7A:
	446	case 0x7B:
	447	case 0x7C:
	448	case 0x7D:
	449	case 0x7E:
	450	case 0x7F:
	451	return 0;
	452	}
	453
	454	switch ((u16)(address)) {
	455	case 0x018A: // 8A 01 : mov al, byte ptr [ecx]
	456	case 0xFF8B: // 8B FF : mov edi, edi
	457	case 0xEC8B: // 8B EC : mov ebp, esp
	458	case 0xc889: // 89 C8 : mov eax, ecx
	459	case 0xC18B: // 8B C1 : mov eax, ecx
	460	case 0xC033: // 33 C0 : xor eax, eax
	461	case 0xC933: // 33 C9 : xor ecx, ecx
	462	case 0xD233: // 33 D2 : xor edx, edx
	463	return 2;
	464
	465	// Cannot overwrite control-instruction. Return 0 to indicate failure.
	466	case 0x25FF: // FF 25 XX XX XX XX : jmp [XXXXXXXX]
	467	return 0;
	468	}
	469
	470	switch (0x00FFFFFF & (u32)address) {
	471	case 0x24A48D: // 8D A4 24 XX XX XX XX : lea esp, [esp + XX XX XX XX]
	472	return 7;
	473	}
	474
	475	#if SANITIZER_WINDOWS64
	476	switch ((u8)address) {
	477	case 0xA1: // A1 XX XX XX XX XX XX XX XX :
	478	// movabs eax, dword ptr ds:[XXXXXXXX]
	479	return 9;
	480	}
	481
	482	switch ((u16)address) {
	483	case 0x5040: // push rax
	484	case 0x5140: // push rcx
	485	case 0x5240: // push rdx
	486	case 0x5340: // push rbx
	487	case 0x5440: // push rsp
	488	case 0x5540: // push rbp
	489	case 0x5640: // push rsi
	490	case 0x5740: // push rdi
	491	case 0x5441: // push r12
	492	case 0x5541: // push r13
	493	case 0x5641: // push r14
	494	case 0x5741: // push r15
	495	case 0x9066: // Two-byte NOP
	496	return 2;
	497
	498	case 0x058B: // 8B 05 XX XX XX XX : mov eax, dword ptr [XX XX XX XX]
	499	if (rel_offset)
	500	*rel_offset = 2;
	501	return 6;
	502	}
	503
	504	switch (0x00FFFFFF & (u32)address) {
	505	case 0xe58948: // 48 8b c4 : mov rbp, rsp
	506	case 0xc18b48: // 48 8b c1 : mov rax, rcx
	507	case 0xc48b48: // 48 8b c4 : mov rax, rsp
	508	case 0xd9f748: // 48 f7 d9 : neg rcx
	509	case 0xd12b48: // 48 2b d1 : sub rdx, rcx
	510	case 0x07c1f6: // f6 c1 07 : test cl, 0x7
	511	case 0xc98548: // 48 85 C9 : test rcx, rcx
	512	case 0xc0854d: // 4d 85 c0 : test r8, r8
	513	case 0xc2b60f: // 0f b6 c2 : movzx eax, dl
	514	case 0xc03345: // 45 33 c0 : xor r8d, r8d
	515	case 0xc93345: // 45 33 c9 : xor r9d, r9d
	516	case 0xdb3345: // 45 33 DB : xor r11d, r11d
	517	case 0xd98b4c: // 4c 8b d9 : mov r11, rcx
	518	case 0xd28b4c: // 4c 8b d2 : mov r10, rdx
	519	case 0xc98b4c: // 4C 8B C9 : mov r9, rcx
	520	case 0xc18b4c: // 4C 8B C1 : mov r8, rcx
	521	case 0xd2b60f: // 0f b6 d2 : movzx edx, dl
	522	case 0xca2b48: // 48 2b ca : sub rcx, rdx
	523	case 0x10b70f: // 0f b7 10 : movzx edx, WORD PTR [rax]
	524	case 0xc00b4d: // 3d 0b c0 : or r8, r8
	525	case 0xd18b48: // 48 8b d1 : mov rdx, rcx
	526	case 0xdc8b4c: // 4c 8b dc : mov r11, rsp
	527	case 0xd18b4c: // 4c 8b d1 : mov r10, rcx
	528	case 0xE0E483: // 83 E4 E0 : and esp, 0xFFFFFFE0
	529	return 3;
	530
	531	case 0xec8348: // 48 83 ec XX : sub rsp, XX
	532	case 0xf88349: // 49 83 f8 XX : cmp r8, XX
	533	case 0x588948: // 48 89 58 XX : mov QWORD PTR[rax + XX], rbx
	534	return 4;
	535
	536	case 0xec8148: // 48 81 EC XX XX XX XX : sub rsp, XXXXXXXX
	537	return 7;
	538
	539	case 0x058b48: // 48 8b 05 XX XX XX XX :
	540	// mov rax, QWORD PTR [rip + XXXXXXXX]
	541	case 0x25ff48: // 48 ff 25 XX XX XX XX :
	542	// rex.W jmp QWORD PTR [rip + XXXXXXXX]
	543
	544	// Instructions having offset relative to 'rip' need offset adjustment.
	545	if (rel_offset)
	546	*rel_offset = 3;
	547	return 7;
	548
	549	case 0x2444c7: // C7 44 24 XX YY YY YY YY
	550	// mov dword ptr [rsp + XX], YYYYYYYY
	551	return 8;
	552	}
	553
	554	switch ((u32)(address)) {
	555	case 0x24448b48: // 48 8b 44 24 XX : mov rax, QWORD ptr [rsp + XX]
	556	case 0x246c8948: // 48 89 6C 24 XX : mov QWORD ptr [rsp + XX], rbp
	557	case 0x245c8948: // 48 89 5c 24 XX : mov QWORD PTR [rsp + XX], rbx
	558	case 0x24748948: // 48 89 74 24 XX : mov QWORD PTR [rsp + XX], rsi
	559	case 0x244C8948: // 48 89 4C 24 XX : mov QWORD PTR [rsp + XX], rcx
	560	case 0x24548948: // 48 89 54 24 XX : mov QWORD PTR [rsp + XX], rdx
	561	case 0x244c894c: // 4c 89 4c 24 XX : mov QWORD PTR [rsp + XX], r9
	562	case 0x2444894c: // 4c 89 44 24 XX : mov QWORD PTR [rsp + XX], r8
	563	return 5;
	564	case 0x24648348: // 48 83 64 24 XX : and QWORD PTR [rsp + XX], YY
	565	return 6;
	566	}
	567
	568	#else
	569
	570	switch ((u8)address) {
	571	case 0xA1: // A1 XX XX XX XX : mov eax, dword ptr ds:[XXXXXXXX]
	572	return 5;
	573	}
	574	switch ((u16)address) {
	575	case 0x458B: // 8B 45 XX : mov eax, dword ptr [ebp + XX]
	576	case 0x5D8B: // 8B 5D XX : mov ebx, dword ptr [ebp + XX]
	577	case 0x7D8B: // 8B 7D XX : mov edi, dword ptr [ebp + XX]
	578	case 0xEC83: // 83 EC XX : sub esp, XX
	579	case 0x75FF: // FF 75 XX : push dword ptr [ebp + XX]
	580	return 3;
	581	case 0xC1F7: // F7 C1 XX YY ZZ WW : test ecx, WWZZYYXX
	582	case 0x25FF: // FF 25 XX YY ZZ WW : jmp dword ptr ds:[WWZZYYXX]
	583	return 6;
	584	case 0x3D83: // 83 3D XX YY ZZ WW TT : cmp TT, WWZZYYXX
	585	return 7;
	586	case 0x7D83: // 83 7D XX YY : cmp dword ptr [ebp + XX], YY
	587	return 4;
	588	}
	589
	590	switch (0x00FFFFFF & (u32)address) {
	591	case 0x24448A: // 8A 44 24 XX : mov eal, dword ptr [esp + XX]
	592	case 0x24448B: // 8B 44 24 XX : mov eax, dword ptr [esp + XX]
	593	case 0x244C8B: // 8B 4C 24 XX : mov ecx, dword ptr [esp + XX]
	594	case 0x24548B: // 8B 54 24 XX : mov edx, dword ptr [esp + XX]
	595	case 0x24748B: // 8B 74 24 XX : mov esi, dword ptr [esp + XX]
	596	case 0x247C8B: // 8B 7C 24 XX : mov edi, dword ptr [esp + XX]
	597	return 4;
	598	}
	599
	600	switch ((u32)address) {
	601	case 0x2444B60F: // 0F B6 44 24 XX : movzx eax, byte ptr [esp + XX]
	602	return 5;
	603	}
	604	#endif
	605
	606	// Unknown instruction!
	607	// FIXME: Unknown instruction failures might happen when we add a new
	608	// interceptor or a new compiler version. In either case, they should result
	609	// in visible and readable error messages. However, merely calling abort()
	610	// leads to an infinite recursion in CheckFailed.
	611	InterceptionFailed();
	612	return 0;
	613	}
	614
	615	// Returns 0 on error.
	616	static size_t RoundUpToInstrBoundary(size_t size, uptr address) {
	617	size_t cursor = 0;
	618	while (cursor < size) {
	619	size_t instruction_size = GetInstructionSize(address + cursor);
	620	if (!instruction_size)
	621	return 0;
	622	cursor += instruction_size;
	623	}
	624	return cursor;
	625	}
	626
	627	static bool CopyInstructions(uptr to, uptr from, size_t size) {
	628	size_t cursor = 0;
	629	while (cursor != size) {
	630	size_t rel_offset = 0;
	631	size_t instruction_size = GetInstructionSize(from + cursor, &rel_offset);
	632	_memcpy((void)(to + cursor), (void)(from + cursor),
	633	(size_t)instruction_size);
	634	if (rel_offset) {
	635	uptr delta = to - from;
	636	uptr relocated_offset = (u32)(to + cursor + rel_offset) - delta;
	637	#if SANITIZER_WINDOWS64
	638	if (relocated_offset + 0x80000000U >= 0xFFFFFFFFU)
	639	return false;
	640	#endif
	641	(u32)(to + cursor + rel_offset) = relocated_offset;
	642	}
	643	cursor += instruction_size;
	644	}
	645	return true;
	646	}
	647
	648
	649	#if !SANITIZER_WINDOWS64
	650	bool OverrideFunctionWithDetour(
	651	uptr old_func, uptr new_func, uptr *orig_old_func) {
	652	const int kDetourHeaderLen = 5;
	653	const u16 kDetourInstruction = 0xFF8B;
	654
	655	uptr header = (uptr)old_func - kDetourHeaderLen;
	656	uptr patch_length = kDetourHeaderLen + kShortJumpInstructionLength;
	657
	658	// Validate that the function is hookable.
	659	if ((u16)old_func != kDetourInstruction \|\|
	660	!IsMemoryPadding(header, kDetourHeaderLen))
	661	return false;
	662
	663	// Change memory protection to writable.
	664	DWORD protection = 0;
	665	if (!ChangeMemoryProtection(header, patch_length, &protection))
	666	return false;
	667
	668	// Write a relative jump to the redirected function.
	669	WriteJumpInstruction(header, new_func);
	670
	671	// Write the short jump to the function prefix.
	672	WriteShortJumpInstruction(old_func, header);
	673
	674	// Restore previous memory protection.
	675	if (!RestoreMemoryProtection(header, patch_length, protection))
	676	return false;
	677
	678	if (orig_old_func)
	679	*orig_old_func = old_func + kShortJumpInstructionLength;
	680
	681	return true;
	682	}
	683	#endif
	684
	685	bool OverrideFunctionWithRedirectJump(
	686	uptr old_func, uptr new_func, uptr *orig_old_func) {
	687	// Check whether the first instruction is a relative jump.
	688	if ((u8)old_func != 0xE9)
	689	return false;
	690
	691	if (orig_old_func) {
	692	uptr relative_offset = (u32)(old_func + 1);
	693	uptr absolute_target = old_func + relative_offset + kJumpInstructionLength;
	694	*orig_old_func = absolute_target;
	695	}
	696
	697	#if SANITIZER_WINDOWS64
	698	// If needed, get memory space for a trampoline jump.
	699	uptr trampoline = AllocateMemoryForTrampoline(old_func, kDirectBranchLength);
	700	if (!trampoline)
	701	return false;
	702	WriteDirectBranch(trampoline, new_func);
	703	#endif
	704
	705	// Change memory protection to writable.
	706	DWORD protection = 0;
	707	if (!ChangeMemoryProtection(old_func, kJumpInstructionLength, &protection))
	708	return false;
	709
	710	// Write a relative jump to the redirected function.
	711	WriteJumpInstruction(old_func, FIRST_32_SECOND_64(new_func, trampoline));
	712
	713	// Restore previous memory protection.
	714	if (!RestoreMemoryProtection(old_func, kJumpInstructionLength, protection))
	715	return false;
	716
	717	return true;
	718	}
	719
	720	bool OverrideFunctionWithHotPatch(
	721	uptr old_func, uptr new_func, uptr *orig_old_func) {
	722	const int kHotPatchHeaderLen = kBranchLength;
	723
	724	uptr header = (uptr)old_func - kHotPatchHeaderLen;
	725	uptr patch_length = kHotPatchHeaderLen + kShortJumpInstructionLength;
	726
	727	// Validate that the function is hot patchable.
	728	size_t instruction_size = GetInstructionSize(old_func);
	729	if (instruction_size < kShortJumpInstructionLength \|\|
	730	!FunctionHasPadding(old_func, kHotPatchHeaderLen))
	731	return false;
	732
	733	if (orig_old_func) {
	734	// Put the needed instructions into the trampoline bytes.
	735	uptr trampoline_length = instruction_size + kDirectBranchLength;
	736	uptr trampoline = AllocateMemoryForTrampoline(old_func, trampoline_length);
	737	if (!trampoline)
	738	return false;
	739	if (!CopyInstructions(trampoline, old_func, instruction_size))
	740	return false;
	741	WriteDirectBranch(trampoline + instruction_size,
	742	old_func + instruction_size);
	743	*orig_old_func = trampoline;
	744	}
	745
	746	// If needed, get memory space for indirect address.
	747	uptr indirect_address = 0;
	748	#if SANITIZER_WINDOWS64
	749	indirect_address = AllocateMemoryForTrampoline(old_func, kAddressLength);
	750	if (!indirect_address)
	751	return false;
	752	#endif
	753
	754	// Change memory protection to writable.
	755	DWORD protection = 0;
	756	if (!ChangeMemoryProtection(header, patch_length, &protection))
	757	return false;
	758
	759	// Write jumps to the redirected function.
	760	WriteBranch(header, indirect_address, new_func);
	761	WriteShortJumpInstruction(old_func, header);
	762
	763	// Restore previous memory protection.
	764	if (!RestoreMemoryProtection(header, patch_length, protection))
	765	return false;
	766
	767	return true;
	768	}
	769
	770	bool OverrideFunctionWithTrampoline(
	771	uptr old_func, uptr new_func, uptr *orig_old_func) {
	772
	773	size_t instructions_length = kBranchLength;
	774	size_t padding_length = 0;
	775	uptr indirect_address = 0;
	776
	777	if (orig_old_func) {
	778	// Find out the number of bytes of the instructions we need to copy
	779	// to the trampoline.
	780	instructions_length = RoundUpToInstrBoundary(kBranchLength, old_func);
	781	if (!instructions_length)
	782	return false;
	783
	784	// Put the needed instructions into the trampoline bytes.
	785	uptr trampoline_length = instructions_length + kDirectBranchLength;
	786	uptr trampoline = AllocateMemoryForTrampoline(old_func, trampoline_length);
	787	if (!trampoline)
	788	return false;
	789	if (!CopyInstructions(trampoline, old_func, instructions_length))
	790	return false;
	791	WriteDirectBranch(trampoline + instructions_length,
	792	old_func + instructions_length);
	793	*orig_old_func = trampoline;
	794	}
	795
	796	#if SANITIZER_WINDOWS64
	797	// Check if the targeted address can be encoded in the function padding.
	798	// Otherwise, allocate it in the trampoline region.
	799	if (IsMemoryPadding(old_func - kAddressLength, kAddressLength)) {
	800	indirect_address = old_func - kAddressLength;
	801	padding_length = kAddressLength;
	802	} else {
	803	indirect_address = AllocateMemoryForTrampoline(old_func, kAddressLength);
	804	if (!indirect_address)
	805	return false;
	806	}
	807	#endif
	808
	809	// Change memory protection to writable.
	810	uptr patch_address = old_func - padding_length;
	811	uptr patch_length = instructions_length + padding_length;
	812	DWORD protection = 0;
	813	if (!ChangeMemoryProtection(patch_address, patch_length, &protection))
	814	return false;
	815
	816	// Patch the original function.
	817	WriteBranch(old_func, indirect_address, new_func);
	818
	819	// Restore previous memory protection.
	820	if (!RestoreMemoryProtection(patch_address, patch_length, protection))
	821	return false;
	822
	823	return true;
	824	}
	825
	826	bool OverrideFunction(
	827	uptr old_func, uptr new_func, uptr *orig_old_func) {
	828	#if !SANITIZER_WINDOWS64
	829	if (OverrideFunctionWithDetour(old_func, new_func, orig_old_func))
	830	return true;
	831	#endif
	832	if (OverrideFunctionWithRedirectJump(old_func, new_func, orig_old_func))
	833	return true;
	834	if (OverrideFunctionWithHotPatch(old_func, new_func, orig_old_func))
	835	return true;
	836	if (OverrideFunctionWithTrampoline(old_func, new_func, orig_old_func))
	837	return true;
	838	return false;
	839	}
	840
	841	static void **InterestingDLLsAvailable() {
	842	static const char *InterestingDLLs[] = {
	843	"kernel32.dll",
	844	"msvcr100.dll", // VS2010
	845	"msvcr110.dll", // VS2012
	846	"msvcr120.dll", // VS2013
	847	"vcruntime140.dll", // VS2015
	848	"ucrtbase.dll", // Universal CRT
	849	// NTDLL should go last as it exports some functions that we should
	850	// override in the CRT [presumably only used internally].
	851	"ntdll.dll", NULL};
	852	static void *result[ARRAY_SIZE(InterestingDLLs)] = { 0 };
	853	if (!result[0]) {
	854	for (size_t i = 0, j = 0; InterestingDLLs[i]; ++i) {
	855	if (HMODULE h = GetModuleHandleA(InterestingDLLs[i]))
	856	result[j++] = (void *)h;
	857	}
	858	}
	859	return &result[0];
	860	}
	861
	862	namespace {
	863	// Utility for reading loaded PE images.
	864	template <typename T> class RVAPtr {
	865	public:
	866	RVAPtr(void *module, uptr rva)
	867	: ptr_(reinterpret_cast<T >(reinterpret_cast<char >(module) + rva)) {}
	868	operator T *() { return ptr_; }
	869	T *operator->() { return ptr_; }
	870	T *operator++() { return ++ptr_; }
	871
	872	private:
	873	T *ptr_;
	874	};
	875	} // namespace
	876
	877	// Internal implementation of GetProcAddress. At least since Windows 8,
	878	// GetProcAddress appears to initialize DLLs before returning function pointers
	879	// into them. This is problematic for the sanitizers, because they typically
	880	// want to intercept malloc before MSVCRT initializes. Our internal
	881	// implementation walks the export list manually without doing initialization.
	882	uptr InternalGetProcAddress(void module, const char func_name) {
	883	// Check that the module header is full and present.
	884	RVAPtr<IMAGE_DOS_HEADER> dos_stub(module, 0);
	885	RVAPtr<IMAGE_NT_HEADERS> headers(module, dos_stub->e_lfanew);
	886	if (!module \|\| dos_stub->e_magic != IMAGE_DOS_SIGNATURE \|\| // "MZ"
	887	headers->Signature != IMAGE_NT_SIGNATURE \|\| // "PE\0\0"
	888	headers->FileHeader.SizeOfOptionalHeader <
	889	sizeof(IMAGE_OPTIONAL_HEADER)) {
	890	return 0;
	891	}
	892
	893	IMAGE_DATA_DIRECTORY *export_directory =
	894	&headers->OptionalHeader.DataDirectory[IMAGE_DIRECTORY_ENTRY_EXPORT];
	895	if (export_directory->Size == 0)
	896	return 0;
	897	RVAPtr<IMAGE_EXPORT_DIRECTORY> exports(module,
	898	export_directory->VirtualAddress);
	899	RVAPtr<DWORD> functions(module, exports->AddressOfFunctions);
	900	RVAPtr<DWORD> names(module, exports->AddressOfNames);
	901	RVAPtr<WORD> ordinals(module, exports->AddressOfNameOrdinals);
	902
	903	for (DWORD i = 0; i < exports->NumberOfNames; i++) {
	904	RVAPtr<char> name(module, names[i]);
	905	if (!strcmp(func_name, name)) {
	906	DWORD index = ordinals[i];
	907	RVAPtr<char> func(module, functions[index]);
	908
	909	// Handle forwarded functions.
	910	DWORD offset = functions[index];
	911	if (offset >= export_directory->VirtualAddress &&
	912	offset < export_directory->VirtualAddress + export_directory->Size) {
	913	// An entry for a forwarded function is a string with the following
	914	// format: "<module> . <function_name>" that is stored into the
	915	// exported directory.
	916	char function_name[256];
	917	size_t funtion_name_length = _strlen(func);
	918	if (funtion_name_length >= sizeof(function_name) - 1)
	919	InterceptionFailed();
	920
	921	_memcpy(function_name, func, funtion_name_length);
	922	function_name[funtion_name_length] = '\0';
	923	char* separator = _strchr(function_name, '.');
	924	if (!separator)
	925	InterceptionFailed();
	926	*separator = '\0';
	927
	928	void* redirected_module = GetModuleHandleA(function_name);
	929	if (!redirected_module)
	930	InterceptionFailed();
	931	return InternalGetProcAddress(redirected_module, separator + 1);
	932	}
	933
	934	return (uptr)(char *)func;
	935	}
	936	}
	937
	938	return 0;
	939	}
	940
	941	bool OverrideFunction(
	942	const char func_name, uptr new_func, uptr orig_old_func) {
	943	bool hooked = false;
	944	void **DLLs = InterestingDLLsAvailable();
	945	for (size_t i = 0; DLLs[i]; ++i) {
	946	uptr func_addr = InternalGetProcAddress(DLLs[i], func_name);
	947	if (func_addr &&
	948	OverrideFunction(func_addr, new_func, orig_old_func)) {
	949	hooked = true;
	950	}
	951	}
	952	return hooked;
	953	}
	954
	955	bool OverrideImportedFunction(const char *module_to_patch,
	956	const char *imported_module,
	957	const char *function_name, uptr new_function,
	958	uptr *orig_old_func) {
	959	HMODULE module = GetModuleHandleA(module_to_patch);
	960	if (!module)
	961	return false;
	962
	963	// Check that the module header is full and present.
	964	RVAPtr<IMAGE_DOS_HEADER> dos_stub(module, 0);
	965	RVAPtr<IMAGE_NT_HEADERS> headers(module, dos_stub->e_lfanew);
	966	if (!module \|\| dos_stub->e_magic != IMAGE_DOS_SIGNATURE \|\| // "MZ"
	967	headers->Signature != IMAGE_NT_SIGNATURE \|\| // "PE\0\0"
	968	headers->FileHeader.SizeOfOptionalHeader <
	969	sizeof(IMAGE_OPTIONAL_HEADER)) {
	970	return false;
	971	}
	972
	973	IMAGE_DATA_DIRECTORY *import_directory =
	974	&headers->OptionalHeader.DataDirectory[IMAGE_DIRECTORY_ENTRY_IMPORT];
	975
	976	// Iterate the list of imported DLLs. FirstThunk will be null for the last
	977	// entry.
	978	RVAPtr<IMAGE_IMPORT_DESCRIPTOR> imports(module,
	979	import_directory->VirtualAddress);
	980	for (; imports->FirstThunk != 0; ++imports) {
	981	RVAPtr<const char> modname(module, imports->Name);
	982	if (_stricmp(&*modname, imported_module) == 0)
	983	break;
	984	}
	985	if (imports->FirstThunk == 0)
	986	return false;
	987
	988	// We have two parallel arrays: the import address table (IAT) and the table
	989	// of names. They start out containing the same data, but the loader rewrites
	990	// the IAT to hold imported addresses and leaves the name table in
	991	// OriginalFirstThunk alone.
	992	RVAPtr<IMAGE_THUNK_DATA> name_table(module, imports->OriginalFirstThunk);
	993	RVAPtr<IMAGE_THUNK_DATA> iat(module, imports->FirstThunk);
	994	for (; name_table->u1.Ordinal != 0; ++name_table, ++iat) {
	995	if (!IMAGE_SNAP_BY_ORDINAL(name_table->u1.Ordinal)) {
	996	RVAPtr<IMAGE_IMPORT_BY_NAME> import_by_name(
	997	module, name_table->u1.ForwarderString);
	998	const char *funcname = &import_by_name->Name[0];
	999	if (strcmp(funcname, function_name) == 0)
	1000	break;
	1001	}
	1002	}
	1003	if (name_table->u1.Ordinal == 0)
	1004	return false;
	1005
	1006	// Now we have the correct IAT entry. Do the swap. We have to make the page
	1007	// read/write first.
	1008	if (orig_old_func)
	1009	*orig_old_func = iat->u1.AddressOfData;
	1010	DWORD old_prot, unused_prot;
	1011	if (!VirtualProtect(&iat->u1.AddressOfData, 4, PAGE_EXECUTE_READWRITE,
	1012	&old_prot))
	1013	return false;
	1014	iat->u1.AddressOfData = new_function;
	1015	if (!VirtualProtect(&iat->u1.AddressOfData, 4, old_prot, &unused_prot))
	1016	return false; // Not clear if this failure bothers us.
	1017	return true;
	1018	}
	1019
	1020	} // namespace __interception
	1021
	1022	#endif // SANITIZER_MAC

lib/tsan/sanitizer_common/sanitizer_allocator_checks.cpp created+22

...	...	@@ -0,0 +1,22 @@
	1	//===-- sanitizer_allocator_checks.cpp --------------------------- C++ --===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// Various checks shared between ThreadSanitizer, MemorySanitizer, etc. memory
	10	// allocators.
	11	//
	12	//===----------------------------------------------------------------------===//
	13
	14	#include "sanitizer_errno.h"
	15
	16	namespace __sanitizer {
	17
	18	void SetErrnoToENOMEM() {
	19	errno = errno_ENOMEM;
	20	}
	21
	22	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_allocator_report.cpp created+137

...	...	@@ -0,0 +1,137 @@
	1	//===-- sanitizer_allocator_report.cpp --------------------------- C++ --===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	///
	9	/// \file
	10	/// Shared allocator error reporting for ThreadSanitizer, MemorySanitizer, etc.
	11	///
	12	//===----------------------------------------------------------------------===//
	13
	14	#include "sanitizer_allocator.h"
	15	#include "sanitizer_allocator_report.h"
	16	#include "sanitizer_common.h"
	17	#include "sanitizer_report_decorator.h"
	18
	19	namespace __sanitizer {
	20
	21	class ScopedAllocatorErrorReport {
	22	public:
	23	ScopedAllocatorErrorReport(const char *error_summary_,
	24	const StackTrace *stack_)
	25	: error_summary(error_summary_),
	26	stack(stack_) {
	27	Printf("%s", d.Error());
	28	}
	29	~ScopedAllocatorErrorReport() {
	30	Printf("%s", d.Default());
	31	stack->Print();
	32	PrintHintAllocatorCannotReturnNull();
	33	ReportErrorSummary(error_summary, stack);
	34	}
	35
	36	private:
	37	ScopedErrorReportLock lock;
	38	const char *error_summary;
	39	const StackTrace* const stack;
	40	const SanitizerCommonDecorator d;
	41	};
	42
	43	void NORETURN ReportCallocOverflow(uptr count, uptr size,
	44	const StackTrace *stack) {
	45	{
	46	ScopedAllocatorErrorReport report("calloc-overflow", stack);
	47	Report("ERROR: %s: calloc parameters overflow: count * size (%zd * %zd) "
	48	"cannot be represented in type size_t\n", SanitizerToolName, count,
	49	size);
	50	}
	51	Die();
	52	}
	53
	54	void NORETURN ReportReallocArrayOverflow(uptr count, uptr size,
	55	const StackTrace *stack) {
	56	{
	57	ScopedAllocatorErrorReport report("reallocarray-overflow", stack);
	58	Report(
	59	"ERROR: %s: reallocarray parameters overflow: count * size (%zd * %zd) "
	60	"cannot be represented in type size_t\n",
	61	SanitizerToolName, count, size);
	62	}
	63	Die();
	64	}
	65
	66	void NORETURN ReportPvallocOverflow(uptr size, const StackTrace *stack) {
	67	{
	68	ScopedAllocatorErrorReport report("pvalloc-overflow", stack);
	69	Report("ERROR: %s: pvalloc parameters overflow: size 0x%zx rounded up to "
	70	"system page size 0x%zx cannot be represented in type size_t\n",
	71	SanitizerToolName, size, GetPageSizeCached());
	72	}
	73	Die();
	74	}
	75
	76	void NORETURN ReportInvalidAllocationAlignment(uptr alignment,
	77	const StackTrace *stack) {
	78	{
	79	ScopedAllocatorErrorReport report("invalid-allocation-alignment", stack);
	80	Report("ERROR: %s: invalid allocation alignment: %zd, alignment must be a "
	81	"power of two\n", SanitizerToolName, alignment);
	82	}
	83	Die();
	84	}
	85
	86	void NORETURN ReportInvalidAlignedAllocAlignment(uptr size, uptr alignment,
	87	const StackTrace *stack) {
	88	{
	89	ScopedAllocatorErrorReport report("invalid-aligned-alloc-alignment", stack);
	90	#if SANITIZER_POSIX
	91	Report("ERROR: %s: invalid alignment requested in "
	92	"aligned_alloc: %zd, alignment must be a power of two and the "
	93	"requested size 0x%zx must be a multiple of alignment\n",
	94	SanitizerToolName, alignment, size);
	95	#else
	96	Report("ERROR: %s: invalid alignment requested in aligned_alloc: %zd, "
	97	"the requested size 0x%zx must be a multiple of alignment\n",
	98	SanitizerToolName, alignment, size);
	99	#endif
	100	}
	101	Die();
	102	}
	103
	104	void NORETURN ReportInvalidPosixMemalignAlignment(uptr alignment,
	105	const StackTrace *stack) {
	106	{
	107	ScopedAllocatorErrorReport report("invalid-posix-memalign-alignment",
	108	stack);
	109	Report(
	110	"ERROR: %s: invalid alignment requested in "
	111	"posix_memalign: %zd, alignment must be a power of two and a "
	112	"multiple of sizeof(void*) == %zd\n",
	113	SanitizerToolName, alignment, sizeof(void *));
	114	}
	115	Die();
	116	}
	117
	118	void NORETURN ReportAllocationSizeTooBig(uptr user_size, uptr max_size,
	119	const StackTrace *stack) {
	120	{
	121	ScopedAllocatorErrorReport report("allocation-size-too-big", stack);
	122	Report("ERROR: %s: requested allocation size 0x%zx exceeds maximum "
	123	"supported size of 0x%zx\n", SanitizerToolName, user_size, max_size);
	124	}
	125	Die();
	126	}
	127
	128	void NORETURN ReportOutOfMemory(uptr requested_size, const StackTrace *stack) {
	129	{
	130	ScopedAllocatorErrorReport report("out-of-memory", stack);
	131	Report("ERROR: %s: allocator is out of memory trying to allocate 0x%zx "
	132	"bytes\n", SanitizerToolName, requested_size);
	133	}
	134	Die();
	135	}
	136
	137	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_common_libcdep.cpp created+149

...	...	@@ -0,0 +1,149 @@
	1	//===-- sanitizer_common_libcdep.cpp --------------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file is shared between AddressSanitizer and ThreadSanitizer
	10	// run-time libraries.
	11	//===----------------------------------------------------------------------===//
	12
	13	#include "sanitizer_allocator_interface.h"
	14	#include "sanitizer_common.h"
	15	#include "sanitizer_flags.h"
	16	#include "sanitizer_procmaps.h"
	17
	18
	19	namespace __sanitizer {
	20
	21	static void (*SoftRssLimitExceededCallback)(bool exceeded);
	22	void SetSoftRssLimitExceededCallback(void (*Callback)(bool exceeded)) {
	23	CHECK_EQ(SoftRssLimitExceededCallback, nullptr);
	24	SoftRssLimitExceededCallback = Callback;
	25	}
	26
	27	#if (SANITIZER_LINUX \|\| SANITIZER_NETBSD) && !SANITIZER_GO
	28	// Weak default implementation for when sanitizer_stackdepot is not linked in.
	29	SANITIZER_WEAK_ATTRIBUTE StackDepotStats *StackDepotGetStats() {
	30	return nullptr;
	31	}
	32
	33	void BackgroundThread(void arg) {
	34	const uptr hard_rss_limit_mb = common_flags()->hard_rss_limit_mb;
	35	const uptr soft_rss_limit_mb = common_flags()->soft_rss_limit_mb;
	36	const bool heap_profile = common_flags()->heap_profile;
	37	uptr prev_reported_rss = 0;
	38	uptr prev_reported_stack_depot_size = 0;
	39	bool reached_soft_rss_limit = false;
	40	uptr rss_during_last_reported_profile = 0;
	41	while (true) {
	42	SleepForMillis(100);
	43	const uptr current_rss_mb = GetRSS() >> 20;
	44	if (Verbosity()) {
	45	// If RSS has grown 10% since last time, print some information.
	46	if (prev_reported_rss * 11 / 10 < current_rss_mb) {
	47	Printf("%s: RSS: %zdMb\n", SanitizerToolName, current_rss_mb);
	48	prev_reported_rss = current_rss_mb;
	49	}
	50	// If stack depot has grown 10% since last time, print it too.
	51	StackDepotStats *stack_depot_stats = StackDepotGetStats();
	52	if (stack_depot_stats) {
	53	if (prev_reported_stack_depot_size * 11 / 10 <
	54	stack_depot_stats->allocated) {
	55	Printf("%s: StackDepot: %zd ids; %zdM allocated\n",
	56	SanitizerToolName,
	57	stack_depot_stats->n_uniq_ids,
	58	stack_depot_stats->allocated >> 20);
	59	prev_reported_stack_depot_size = stack_depot_stats->allocated;
	60	}
	61	}
	62	}
	63	// Check RSS against the limit.
	64	if (hard_rss_limit_mb && hard_rss_limit_mb < current_rss_mb) {
	65	Report("%s: hard rss limit exhausted (%zdMb vs %zdMb)\n",
	66	SanitizerToolName, hard_rss_limit_mb, current_rss_mb);
	67	DumpProcessMap();
	68	Die();
	69	}
	70	if (soft_rss_limit_mb) {
	71	if (soft_rss_limit_mb < current_rss_mb && !reached_soft_rss_limit) {
	72	reached_soft_rss_limit = true;
	73	Report("%s: soft rss limit exhausted (%zdMb vs %zdMb)\n",
	74	SanitizerToolName, soft_rss_limit_mb, current_rss_mb);
	75	if (SoftRssLimitExceededCallback)
	76	SoftRssLimitExceededCallback(true);
	77	} else if (soft_rss_limit_mb >= current_rss_mb &&
	78	reached_soft_rss_limit) {
	79	reached_soft_rss_limit = false;
	80	if (SoftRssLimitExceededCallback)
	81	SoftRssLimitExceededCallback(false);
	82	}
	83	}
	84	if (heap_profile &&
	85	current_rss_mb > rss_during_last_reported_profile * 1.1) {
	86	Printf("\n\nHEAP PROFILE at RSS %zdMb\n", current_rss_mb);
	87	__sanitizer_print_memory_profile(90, 20);
	88	rss_during_last_reported_profile = current_rss_mb;
	89	}
	90	}
	91	}
	92	#endif
	93
	94	void WriteToSyslog(const char *msg) {
	95	InternalScopedString msg_copy(kErrorMessageBufferSize);
	96	msg_copy.append("%s", msg);
	97	char *p = msg_copy.data();
	98	char *q;
	99
	100	// Print one line at a time.
	101	// syslog, at least on Android, has an implicit message length limit.
	102	while ((q = internal_strchr(p, '\n'))) {
	103	*q = '\0';
	104	WriteOneLineToSyslog(p);
	105	p = q + 1;
	106	}
	107	// Print remaining characters, if there are any.
	108	// Note that this will add an extra newline at the end.
	109	// FIXME: buffer extra output. This would need a thread-local buffer, which
	110	// on Android requires plugging into the tools (ex. ASan's) Thread class.
	111	if (*p)
	112	WriteOneLineToSyslog(p);
	113	}
	114
	115	void MaybeStartBackgroudThread() {
	116	#if (SANITIZER_LINUX \|\| SANITIZER_NETBSD) && \
	117	!SANITIZER_GO // Need to implement/test on other platforms.
	118	// Start the background thread if one of the rss limits is given.
	119	if (!common_flags()->hard_rss_limit_mb &&
	120	!common_flags()->soft_rss_limit_mb &&
	121	!common_flags()->heap_profile) return;
	122	if (!&real_pthread_create) return; // Can't spawn the thread anyway.
	123	internal_start_thread(BackgroundThread, nullptr);
	124	#endif
	125	}
	126
	127	static void (*sandboxing_callback)();
	128	void SetSandboxingCallback(void (*f)()) {
	129	sandboxing_callback = f;
	130	}
	131
	132	uptr ReservedAddressRange::InitAligned(uptr size, uptr align,
	133	const char *name) {
	134	CHECK(IsPowerOfTwo(align));
	135	if (align <= GetPageSizeCached())
	136	return Init(size, name);
	137	uptr start = Init(size + align, name);
	138	start += align - (start & (align - 1));
	139	return start;
	140	}
	141
	142	} // namespace __sanitizer
	143
	144	SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_sandbox_on_notify,
	145	__sanitizer_sandbox_arguments *args) {
	146	__sanitizer::PlatformPrepareForSandboxing(args);
	147	if (__sanitizer::sandboxing_callback)
	148	__sanitizer::sandboxing_callback();
	149	}

lib/tsan/sanitizer_common/sanitizer_common_nolibc.cpp created+34

...	...	@@ -0,0 +1,34 @@
	1	//===-- sanitizer_common_nolibc.cpp ---------------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file contains stubs for libc function to facilitate optional use of
	10	// libc in no-libcdep sources.
	11	//===----------------------------------------------------------------------===//
	12
	13	#include "sanitizer_platform.h"
	14	#include "sanitizer_common.h"
	15	#include "sanitizer_libc.h"
	16
	17	namespace __sanitizer {
	18
	19	// The Windows implementations of these functions use the win32 API directly,
	20	// bypassing libc.
	21	#if !SANITIZER_WINDOWS
	22	#if SANITIZER_LINUX
	23	void LogMessageOnPrintf(const char *str) {}
	24	#endif
	25	void WriteToSyslog(const char *buffer) {}
	26	void Abort() { internal__exit(1); }
	27	void SleepForSeconds(int seconds) { internal_sleep(seconds); }
	28	#endif // !SANITIZER_WINDOWS
	29
	30	#if !SANITIZER_WINDOWS && !SANITIZER_MAC
	31	void ListOfModules::init() {}
	32	#endif
	33
	34	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_linux_libcdep.cpp created+846

...	...	@@ -0,0 +1,846 @@
	1	//===-- sanitizer_linux_libcdep.cpp ---------------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file is shared between AddressSanitizer and ThreadSanitizer
	10	// run-time libraries and implements linux-specific functions from
	11	// sanitizer_libc.h.
	12	//===----------------------------------------------------------------------===//
	13
	14	#include "sanitizer_platform.h"
	15
	16	#if SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_NETBSD \|\| \
	17	SANITIZER_OPENBSD \|\| SANITIZER_SOLARIS
	18
	19	#include "sanitizer_allocator_internal.h"
	20	#include "sanitizer_atomic.h"
	21	#include "sanitizer_common.h"
	22	#include "sanitizer_file.h"
	23	#include "sanitizer_flags.h"
	24	#include "sanitizer_freebsd.h"
	25	#include "sanitizer_getauxval.h"
	26	#include "sanitizer_glibc_version.h"
	27	#include "sanitizer_linux.h"
	28	#include "sanitizer_placement_new.h"
	29	#include "sanitizer_procmaps.h"
	30
	31	#include <dlfcn.h> // for dlsym()
	32	#include <link.h>
	33	#include <pthread.h>
	34	#include <signal.h>
	35	#include <sys/resource.h>
	36	#include <syslog.h>
	37
	38	#if !defined(ElfW)
	39	#define ElfW(type) Elf_##type
	40	#endif
	41
	42	#if SANITIZER_FREEBSD
	43	#include <pthread_np.h>
	44	#include <osreldate.h>
	45	#include <sys/sysctl.h>
	46	#define pthread_getattr_np pthread_attr_get_np
	47	#endif
	48
	49	#if SANITIZER_OPENBSD
	50	#include <pthread_np.h>
	51	#include <sys/sysctl.h>
	52	#endif
	53
	54	#if SANITIZER_NETBSD
	55	#include <sys/sysctl.h>
	56	#include <sys/tls.h>
	57	#include <lwp.h>
	58	#endif
	59
	60	#if SANITIZER_SOLARIS
	61	#include <stdlib.h>
	62	#include <thread.h>
	63	#endif
	64
	65	#if SANITIZER_ANDROID
	66	#include <android/api-level.h>
	67	#if !defined(CPU_COUNT) && !defined(__aarch64__)
	68	#include <dirent.h>
	69	#include <fcntl.h>
	70	struct __sanitizer::linux_dirent {
	71	long d_ino;
	72	off_t d_off;
	73	unsigned short d_reclen;
	74	char d_name[];
	75	};
	76	#endif
	77	#endif
	78
	79	#if !SANITIZER_ANDROID
	80	#include <elf.h>
	81	#include <unistd.h>
	82	#endif
	83
	84	namespace __sanitizer {
	85
	86	SANITIZER_WEAK_ATTRIBUTE int
	87	real_sigaction(int signum, const void act, void oldact);
	88
	89	int internal_sigaction(int signum, const void act, void oldact) {
	90	#if !SANITIZER_GO
	91	if (&real_sigaction)
	92	return real_sigaction(signum, act, oldact);
	93	#endif
	94	return sigaction(signum, (const struct sigaction *)act,
	95	(struct sigaction *)oldact);
	96	}
	97
	98	void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top,
	99	uptr *stack_bottom) {
	100	CHECK(stack_top);
	101	CHECK(stack_bottom);
	102	if (at_initialization) {
	103	// This is the main thread. Libpthread may not be initialized yet.
	104	struct rlimit rl;
	105	CHECK_EQ(getrlimit(RLIMIT_STACK, &rl), 0);
	106
	107	// Find the mapping that contains a stack variable.
	108	MemoryMappingLayout proc_maps(/cache_enabled/true);
	109	if (proc_maps.Error()) {
	110	stack_top = stack_bottom = 0;
	111	return;
	112	}
	113	MemoryMappedSegment segment;
	114	uptr prev_end = 0;
	115	while (proc_maps.Next(&segment)) {
	116	if ((uptr)&rl < segment.end) break;
	117	prev_end = segment.end;
	118	}
	119	CHECK((uptr)&rl >= segment.start && (uptr)&rl < segment.end);
	120
	121	// Get stacksize from rlimit, but clip it so that it does not overlap
	122	// with other mappings.
	123	uptr stacksize = rl.rlim_cur;
	124	if (stacksize > segment.end - prev_end) stacksize = segment.end - prev_end;
	125	// When running with unlimited stack size, we still want to set some limit.
	126	// The unlimited stack size is caused by 'ulimit -s unlimited'.
	127	// Also, for some reason, GNU make spawns subprocesses with unlimited stack.
	128	if (stacksize > kMaxThreadStackSize)
	129	stacksize = kMaxThreadStackSize;
	130	*stack_top = segment.end;
	131	*stack_bottom = segment.end - stacksize;
	132	return;
	133	}
	134	uptr stacksize = 0;
	135	void *stackaddr = nullptr;
	136	#if SANITIZER_SOLARIS
	137	stack_t ss;
	138	CHECK_EQ(thr_stksegment(&ss), 0);
	139	stacksize = ss.ss_size;
	140	stackaddr = (char *)ss.ss_sp - stacksize;
	141	#elif SANITIZER_OPENBSD
	142	stack_t sattr;
	143	CHECK_EQ(pthread_stackseg_np(pthread_self(), &sattr), 0);
	144	stackaddr = sattr.ss_sp;
	145	stacksize = sattr.ss_size;
	146	#else // !SANITIZER_SOLARIS
	147	pthread_attr_t attr;
	148	pthread_attr_init(&attr);
	149	CHECK_EQ(pthread_getattr_np(pthread_self(), &attr), 0);
	150	my_pthread_attr_getstack(&attr, &stackaddr, &stacksize);
	151	pthread_attr_destroy(&attr);
	152	#endif // SANITIZER_SOLARIS
	153
	154	*stack_top = (uptr)stackaddr + stacksize;
	155	*stack_bottom = (uptr)stackaddr;
	156	}
	157
	158	#if !SANITIZER_GO
	159	bool SetEnv(const char name, const char value) {
	160	void *f = dlsym(RTLD_NEXT, "setenv");
	161	if (!f)
	162	return false;
	163	typedef int(setenv_ft)(const char name, const char *value, int overwrite);
	164	setenv_ft setenv_f;
	165	CHECK_EQ(sizeof(setenv_f), sizeof(f));
	166	internal_memcpy(&setenv_f, &f, sizeof(f));
	167	return setenv_f(name, value, 1) == 0;
	168	}
	169	#endif
	170
	171	__attribute__((unused)) static bool GetLibcVersion(int major, int minor,
	172	int *patch) {
	173	#ifdef _CS_GNU_LIBC_VERSION
	174	char buf[64];
	175	uptr len = confstr(_CS_GNU_LIBC_VERSION, buf, sizeof(buf));
	176	if (len >= sizeof(buf))
	177	return false;
	178	buf[len] = 0;
	179	static const char kGLibC[] = "glibc ";
	180	if (internal_strncmp(buf, kGLibC, sizeof(kGLibC) - 1) != 0)
	181	return false;
	182	const char *p = buf + sizeof(kGLibC) - 1;
	183	*major = internal_simple_strtoll(p, &p, 10);
	184	minor = (p == '.') ? internal_simple_strtoll(p + 1, &p, 10) : 0;
	185	patch = (p == '.') ? internal_simple_strtoll(p + 1, &p, 10) : 0;
	186	return true;
	187	#else
	188	return false;
	189	#endif
	190	}
	191
	192	#if !SANITIZER_FREEBSD && !SANITIZER_ANDROID && !SANITIZER_GO && \
	193	!SANITIZER_NETBSD && !SANITIZER_OPENBSD && !SANITIZER_SOLARIS
	194	static uptr g_tls_size;
	195
	196	#ifdef __i386__
	197	# define CHECK_GET_TLS_STATIC_INFO_VERSION (!__GLIBC_PREREQ(2, 27))
	198	#else
	199	# define CHECK_GET_TLS_STATIC_INFO_VERSION 0
	200	#endif
	201
	202	#if CHECK_GET_TLS_STATIC_INFO_VERSION
	203	# define DL_INTERNAL_FUNCTION __attribute__((regparm(3), stdcall))
	204	#else
	205	# define DL_INTERNAL_FUNCTION
	206	#endif
	207
	208	namespace {
	209	struct GetTlsStaticInfoCall {
	210	typedef void (get_tls_func)(size_t, size_t*);
	211	};
	212	struct GetTlsStaticInfoRegparmCall {
	213	typedef void (get_tls_func)(size_t, size_t*) DL_INTERNAL_FUNCTION;
	214	};
	215
	216	template <typename T>
	217	void CallGetTls(void* ptr, size_t* size, size_t* align) {
	218	typename T::get_tls_func get_tls;
	219	CHECK_EQ(sizeof(get_tls), sizeof(ptr));
	220	internal_memcpy(&get_tls, &ptr, sizeof(ptr));
	221	CHECK_NE(get_tls, 0);
	222	get_tls(size, align);
	223	}
	224
	225	bool CmpLibcVersion(int major, int minor, int patch) {
	226	int ma;
	227	int mi;
	228	int pa;
	229	if (!GetLibcVersion(&ma, &mi, &pa))
	230	return false;
	231	if (ma > major)
	232	return true;
	233	if (ma < major)
	234	return false;
	235	if (mi > minor)
	236	return true;
	237	if (mi < minor)
	238	return false;
	239	return pa >= patch;
	240	}
	241
	242	} // namespace
	243
	244	void InitTlsSize() {
	245	// all current supported platforms have 16 bytes stack alignment
	246	const size_t kStackAlign = 16;
	247	void *get_tls_static_info_ptr = dlsym(RTLD_NEXT, "_dl_get_tls_static_info");
	248	size_t tls_size = 0;
	249	size_t tls_align = 0;
	250	// On i?86, _dl_get_tls_static_info used to be internal_function, i.e.
	251	// __attribute__((regparm(3), stdcall)) before glibc 2.27 and is normal
	252	// function in 2.27 and later.
	253	if (CHECK_GET_TLS_STATIC_INFO_VERSION && !CmpLibcVersion(2, 27, 0))
	254	CallGetTls<GetTlsStaticInfoRegparmCall>(get_tls_static_info_ptr,
	255	&tls_size, &tls_align);
	256	else
	257	CallGetTls<GetTlsStaticInfoCall>(get_tls_static_info_ptr,
	258	&tls_size, &tls_align);
	259	if (tls_align < kStackAlign)
	260	tls_align = kStackAlign;
	261	g_tls_size = RoundUpTo(tls_size, tls_align);
	262	}
	263	#else
	264	void InitTlsSize() { }
	265	#endif // !SANITIZER_FREEBSD && !SANITIZER_ANDROID && !SANITIZER_GO &&
	266	// !SANITIZER_NETBSD && !SANITIZER_SOLARIS
	267
	268	#if (defined(__x86_64__) \|\| defined(__i386__) \|\| defined(__mips__) \|\| \
	269	defined(__aarch64__) \|\| defined(__powerpc64__) \|\| defined(__s390__) \|\| \
	270	defined(__arm__)) && \
	271	SANITIZER_LINUX && !SANITIZER_ANDROID
	272	// sizeof(struct pthread) from glibc.
	273	static atomic_uintptr_t thread_descriptor_size;
	274
	275	uptr ThreadDescriptorSize() {
	276	uptr val = atomic_load_relaxed(&thread_descriptor_size);
	277	if (val)
	278	return val;
	279	#if defined(__x86_64__) \|\| defined(__i386__) \|\| defined(__arm__)
	280	int major;
	281	int minor;
	282	int patch;
	283	if (GetLibcVersion(&major, &minor, &patch) && major == 2) {
	284	/* sizeof(struct pthread) values from various glibc versions. */
	285	if (SANITIZER_X32)
	286	val = 1728; // Assume only one particular version for x32.
	287	// For ARM sizeof(struct pthread) changed in Glibc 2.23.
	288	else if (SANITIZER_ARM)
	289	val = minor <= 22 ? 1120 : 1216;
	290	else if (minor <= 3)
	291	val = FIRST_32_SECOND_64(1104, 1696);
	292	else if (minor == 4)
	293	val = FIRST_32_SECOND_64(1120, 1728);
	294	else if (minor == 5)
	295	val = FIRST_32_SECOND_64(1136, 1728);
	296	else if (minor <= 9)
	297	val = FIRST_32_SECOND_64(1136, 1712);
	298	else if (minor == 10)
	299	val = FIRST_32_SECOND_64(1168, 1776);
	300	else if (minor == 11 \|\| (minor == 12 && patch == 1))
	301	val = FIRST_32_SECOND_64(1168, 2288);
	302	else if (minor <= 14)
	303	val = FIRST_32_SECOND_64(1168, 2304);
	304	else
	305	val = FIRST_32_SECOND_64(1216, 2304);
	306	}
	307	#elif defined(__mips__)
	308	// TODO(sagarthakur): add more values as per different glibc versions.
	309	val = FIRST_32_SECOND_64(1152, 1776);
	310	#elif defined(__aarch64__)
	311	// The sizeof (struct pthread) is the same from GLIBC 2.17 to 2.22.
	312	val = 1776;
	313	#elif defined(__powerpc64__)
	314	val = 1776; // from glibc.ppc64le 2.20-8.fc21
	315	#elif defined(__s390__)
	316	val = FIRST_32_SECOND_64(1152, 1776); // valid for glibc 2.22
	317	#endif
	318	if (val)
	319	atomic_store_relaxed(&thread_descriptor_size, val);
	320	return val;
	321	}
	322
	323	// The offset at which pointer to self is located in the thread descriptor.
	324	const uptr kThreadSelfOffset = FIRST_32_SECOND_64(8, 16);
	325
	326	uptr ThreadSelfOffset() {
	327	return kThreadSelfOffset;
	328	}
	329
	330	#if defined(__mips__) \|\| defined(__powerpc64__)
	331	// TlsPreTcbSize includes size of struct pthread_descr and size of tcb
	332	// head structure. It lies before the static tls blocks.
	333	static uptr TlsPreTcbSize() {
	334	# if defined(__mips__)
	335	const uptr kTcbHead = 16; // sizeof (tcbhead_t)
	336	# elif defined(__powerpc64__)
	337	const uptr kTcbHead = 88; // sizeof (tcbhead_t)
	338	# endif
	339	const uptr kTlsAlign = 16;
	340	const uptr kTlsPreTcbSize =
	341	RoundUpTo(ThreadDescriptorSize() + kTcbHead, kTlsAlign);
	342	return kTlsPreTcbSize;
	343	}
	344	#endif
	345
	346	uptr ThreadSelf() {
	347	uptr descr_addr;
	348	# if defined(__i386__)
	349	asm("mov %%gs:%c1,%0" : "=r"(descr_addr) : "i"(kThreadSelfOffset));
	350	# elif defined(__x86_64__)
	351	asm("mov %%fs:%c1,%0" : "=r"(descr_addr) : "i"(kThreadSelfOffset));
	352	# elif defined(__mips__)
	353	// MIPS uses TLS variant I. The thread pointer (in hardware register $29)
	354	// points to the end of the TCB + 0x7000. The pthread_descr structure is
	355	// immediately in front of the TCB. TlsPreTcbSize() includes the size of the
	356	// TCB and the size of pthread_descr.
	357	const uptr kTlsTcbOffset = 0x7000;
	358	uptr thread_pointer;
	359	asm volatile(".set push;\
	360	.set mips64r2;\
	361	rdhwr %0,$29;\
	362	.set pop" : "=r" (thread_pointer));
	363	descr_addr = thread_pointer - kTlsTcbOffset - TlsPreTcbSize();
	364	# elif defined(__aarch64__) \|\| defined(__arm__)
	365	descr_addr = reinterpret_cast<uptr>(__builtin_thread_pointer()) -
	366	ThreadDescriptorSize();
	367	# elif defined(__s390__)
	368	descr_addr = reinterpret_cast<uptr>(__builtin_thread_pointer());
	369	# elif defined(__powerpc64__)
	370	// PPC64LE uses TLS variant I. The thread pointer (in GPR 13)
	371	// points to the end of the TCB + 0x7000. The pthread_descr structure is
	372	// immediately in front of the TCB. TlsPreTcbSize() includes the size of the
	373	// TCB and the size of pthread_descr.
	374	const uptr kTlsTcbOffset = 0x7000;
	375	uptr thread_pointer;
	376	asm("addi %0,13,%1" : "=r"(thread_pointer) : "I"(-kTlsTcbOffset));
	377	descr_addr = thread_pointer - TlsPreTcbSize();
	378	# else
	379	# error "unsupported CPU arch"
	380	# endif
	381	return descr_addr;
	382	}
	383	#endif // (x86_64 \|\| i386 \|\| MIPS) && SANITIZER_LINUX
	384
	385	#if SANITIZER_FREEBSD
	386	static void **ThreadSelfSegbase() {
	387	void **segbase = 0;
	388	# if defined(__i386__)
	389	// sysarch(I386_GET_GSBASE, segbase);
	390	__asm __volatile("mov %%gs:0, %0" : "=r" (segbase));
	391	# elif defined(__x86_64__)
	392	// sysarch(AMD64_GET_FSBASE, segbase);
	393	__asm __volatile("movq %%fs:0, %0" : "=r" (segbase));
	394	# else
	395	# error "unsupported CPU arch"
	396	# endif
	397	return segbase;
	398	}
	399
	400	uptr ThreadSelf() {
	401	return (uptr)ThreadSelfSegbase()[2];
	402	}
	403	#endif // SANITIZER_FREEBSD
	404
	405	#if SANITIZER_NETBSD
	406	static struct tls_tcb * ThreadSelfTlsTcb() {
	407	return (struct tls_tcb *)_lwp_getprivate();
	408	}
	409
	410	uptr ThreadSelf() {
	411	return (uptr)ThreadSelfTlsTcb()->tcb_pthread;
	412	}
	413
	414	int GetSizeFromHdr(struct dl_phdr_info info, size_t size, void data) {
	415	const Elf_Phdr *hdr = info->dlpi_phdr;
	416	const Elf_Phdr *last_hdr = hdr + info->dlpi_phnum;
	417
	418	for (; hdr != last_hdr; ++hdr) {
	419	if (hdr->p_type == PT_TLS && info->dlpi_tls_modid == 1) {
	420	(uptr)data = hdr->p_memsz;
	421	break;
	422	}
	423	}
	424	return 0;
	425	}
	426	#endif // SANITIZER_NETBSD
	427
	428	#if !SANITIZER_GO
	429	static void GetTls(uptr addr, uptr size) {
	430	#if SANITIZER_LINUX && !SANITIZER_ANDROID
	431	# if defined(__x86_64__) \|\| defined(__i386__) \|\| defined(__s390__)
	432	*addr = ThreadSelf();
	433	*size = GetTlsSize();
	434	addr -= size;
	435	*addr += ThreadDescriptorSize();
	436	# elif defined(__mips__) \|\| defined(__aarch64__) \|\| defined(__powerpc64__) \
	437	\|\| defined(__arm__)
	438	*addr = ThreadSelf();
	439	*size = GetTlsSize();
	440	# else
	441	*addr = 0;
	442	*size = 0;
	443	# endif
	444	#elif SANITIZER_FREEBSD
	445	void** segbase = ThreadSelfSegbase();
	446	*addr = 0;
	447	*size = 0;
	448	if (segbase != 0) {
	449	// tcbalign = 16
	450	// tls_size = round(tls_static_space, tcbalign);
	451	// dtv = segbase[1];
	452	// dtv[2] = segbase - tls_static_space;
	453	void dtv = (void) segbase[1];
	454	*addr = (uptr) dtv[2];
	455	size = (addr == 0) ? 0 : ((uptr) segbase[0] - (uptr) dtv[2]);
	456	}
	457	#elif SANITIZER_NETBSD
	458	struct tls_tcb * const tcb = ThreadSelfTlsTcb();
	459	*addr = 0;
	460	*size = 0;
	461	if (tcb != 0) {
	462	// Find size (p_memsz) of dlpi_tls_modid 1 (TLS block of the main program).
	463	// ld.elf_so hardcodes the index 1.
	464	dl_iterate_phdr(GetSizeFromHdr, size);
	465
	466	if (*size != 0) {
	467	// The block has been found and tcb_dtv[1] contains the base address
	468	*addr = (uptr)tcb->tcb_dtv[1];
	469	}
	470	}
	471	#elif SANITIZER_OPENBSD
	472	*addr = 0;
	473	*size = 0;
	474	#elif SANITIZER_ANDROID
	475	*addr = 0;
	476	*size = 0;
	477	#elif SANITIZER_SOLARIS
	478	// FIXME
	479	*addr = 0;
	480	*size = 0;
	481	#else
	482	# error "Unknown OS"
	483	#endif
	484	}
	485	#endif
	486
	487	#if !SANITIZER_GO
	488	uptr GetTlsSize() {
	489	#if SANITIZER_FREEBSD \|\| SANITIZER_ANDROID \|\| SANITIZER_NETBSD \|\| \
	490	SANITIZER_OPENBSD \|\| SANITIZER_SOLARIS
	491	uptr addr, size;
	492	GetTls(&addr, &size);
	493	return size;
	494	#elif defined(__mips__) \|\| defined(__powerpc64__)
	495	return RoundUpTo(g_tls_size + TlsPreTcbSize(), 16);
	496	#else
	497	return g_tls_size;
	498	#endif
	499	}
	500	#endif
	501
	502	void GetThreadStackAndTls(bool main, uptr stk_addr, uptr stk_size,
	503	uptr tls_addr, uptr tls_size) {
	504	#if SANITIZER_GO
	505	// Stub implementation for Go.
	506	stk_addr = stk_size = tls_addr = tls_size = 0;
	507	#else
	508	GetTls(tls_addr, tls_size);
	509
	510	uptr stack_top, stack_bottom;
	511	GetThreadStackTopAndBottom(main, &stack_top, &stack_bottom);
	512	*stk_addr = stack_bottom;
	513	*stk_size = stack_top - stack_bottom;
	514
	515	if (!main) {
	516	// If stack and tls intersect, make them non-intersecting.
	517	if (tls_addr > stk_addr && tls_addr < stk_addr + *stk_size) {
	518	CHECK_GT(tls_addr + tls_size, *stk_addr);
	519	CHECK_LE(tls_addr + tls_size, stk_addr + stk_size);
	520	stk_size -= tls_size;
	521	tls_addr = stk_addr + *stk_size;
	522	}
	523	}
	524	#endif
	525	}
	526
	527	#if !SANITIZER_FREEBSD && !SANITIZER_OPENBSD
	528	typedef ElfW(Phdr) Elf_Phdr;
	529	#elif SANITIZER_WORDSIZE == 32 && __FreeBSD_version <= 902001 // v9.2
	530	#define Elf_Phdr XElf32_Phdr
	531	#define dl_phdr_info xdl_phdr_info
	532	#define dl_iterate_phdr(c, b) xdl_iterate_phdr((c), (b))
	533	#endif // !SANITIZER_FREEBSD && !SANITIZER_OPENBSD
	534
	535	struct DlIteratePhdrData {
	536	InternalMmapVectorNoCtor<LoadedModule> *modules;
	537	bool first;
	538	};
	539
	540	static int dl_iterate_phdr_cb(dl_phdr_info info, size_t size, void arg) {
	541	DlIteratePhdrData data = (DlIteratePhdrData)arg;
	542	InternalScopedString module_name(kMaxPathLength);
	543	if (data->first) {
	544	data->first = false;
	545	// First module is the binary itself.
	546	ReadBinaryNameCached(module_name.data(), module_name.size());
	547	} else if (info->dlpi_name) {
	548	module_name.append("%s", info->dlpi_name);
	549	}
	550	if (module_name[0] == '\0')
	551	return 0;
	552	LoadedModule cur_module;
	553	cur_module.set(module_name.data(), info->dlpi_addr);
	554	for (int i = 0; i < (int)info->dlpi_phnum; i++) {
	555	const Elf_Phdr *phdr = &info->dlpi_phdr[i];
	556	if (phdr->p_type == PT_LOAD) {
	557	uptr cur_beg = info->dlpi_addr + phdr->p_vaddr;
	558	uptr cur_end = cur_beg + phdr->p_memsz;
	559	bool executable = phdr->p_flags & PF_X;
	560	bool writable = phdr->p_flags & PF_W;
	561	cur_module.addAddressRange(cur_beg, cur_end, executable,
	562	writable);
	563	}
	564	}
	565	data->modules->push_back(cur_module);
	566	return 0;
	567	}
	568
	569	#if SANITIZER_ANDROID && __ANDROID_API__ < 21
	570	extern "C" __attribute__((weak)) int dl_iterate_phdr(
	571	int ()(struct dl_phdr_info , size_t, void ), void );
	572	#endif
	573
	574	static bool requiresProcmaps() {
	575	#if SANITIZER_ANDROID && __ANDROID_API__ <= 22
	576	// Fall back to /proc/maps if dl_iterate_phdr is unavailable or broken.
	577	// The runtime check allows the same library to work with
	578	// both K and L (and future) Android releases.
	579	return AndroidGetApiLevel() <= ANDROID_LOLLIPOP_MR1;
	580	#else
	581	return false;
	582	#endif
	583	}
	584
	585	static void procmapsInit(InternalMmapVectorNoCtor<LoadedModule> *modules) {
	586	MemoryMappingLayout memory_mapping(/cache_enabled/true);
	587	memory_mapping.DumpListOfModules(modules);
	588	}
	589
	590	void ListOfModules::init() {
	591	clearOrInit();
	592	if (requiresProcmaps()) {
	593	procmapsInit(&modules_);
	594	} else {
	595	DlIteratePhdrData data = {&modules_, true};
	596	dl_iterate_phdr(dl_iterate_phdr_cb, &data);
	597	}
	598	}
	599
	600	// When a custom loader is used, dl_iterate_phdr may not contain the full
	601	// list of modules. Allow callers to fall back to using procmaps.
	602	void ListOfModules::fallbackInit() {
	603	if (!requiresProcmaps()) {
	604	clearOrInit();
	605	procmapsInit(&modules_);
	606	} else {
	607	clear();
	608	}
	609	}
	610
	611	// getrusage does not give us the current RSS, only the max RSS.
	612	// Still, this is better than nothing if /proc/self/statm is not available
	613	// for some reason, e.g. due to a sandbox.
	614	static uptr GetRSSFromGetrusage() {
	615	struct rusage usage;
	616	if (getrusage(RUSAGE_SELF, &usage)) // Failed, probably due to a sandbox.
	617	return 0;
	618	return usage.ru_maxrss << 10; // ru_maxrss is in Kb.
	619	}
	620
	621	uptr GetRSS() {
	622	if (!common_flags()->can_use_proc_maps_statm)
	623	return GetRSSFromGetrusage();
	624	fd_t fd = OpenFile("/proc/self/statm", RdOnly);
	625	if (fd == kInvalidFd)
	626	return GetRSSFromGetrusage();
	627	char buf[64];
	628	uptr len = internal_read(fd, buf, sizeof(buf) - 1);
	629	internal_close(fd);
	630	if ((sptr)len <= 0)
	631	return 0;
	632	buf[len] = 0;
	633	// The format of the file is:
	634	// 1084 89 69 11 0 79 0
	635	// We need the second number which is RSS in pages.
	636	char *pos = buf;
	637	// Skip the first number.
	638	while (pos >= '0' && pos <= '9')
	639	pos++;
	640	// Skip whitespaces.
	641	while (!(pos >= '0' && pos <= '9') && *pos != 0)
	642	pos++;
	643	// Read the number.
	644	uptr rss = 0;
	645	while (pos >= '0' && pos <= '9')
	646	rss = rss * 10 + *pos++ - '0';
	647	return rss * GetPageSizeCached();
	648	}
	649
	650	// sysconf(_SC_NPROCESSORS_{CONF,ONLN}) cannot be used on most platforms as
	651	// they allocate memory.
	652	u32 GetNumberOfCPUs() {
	653	#if SANITIZER_FREEBSD \|\| SANITIZER_NETBSD \|\| SANITIZER_OPENBSD
	654	u32 ncpu;
	655	int req[2];
	656	uptr len = sizeof(ncpu);
	657	req[0] = CTL_HW;
	658	req[1] = HW_NCPU;
	659	CHECK_EQ(internal_sysctl(req, 2, &ncpu, &len, NULL, 0), 0);
	660	return ncpu;
	661	#elif SANITIZER_ANDROID && !defined(CPU_COUNT) && !defined(__aarch64__)
	662	// Fall back to /sys/devices/system/cpu on Android when cpu_set_t doesn't
	663	// exist in sched.h. That is the case for toolchains generated with older
	664	// NDKs.
	665	// This code doesn't work on AArch64 because internal_getdents makes use of
	666	// the 64bit getdents syscall, but cpu_set_t seems to always exist on AArch64.
	667	uptr fd = internal_open("/sys/devices/system/cpu", O_RDONLY \| O_DIRECTORY);
	668	if (internal_iserror(fd))
	669	return 0;
	670	InternalMmapVector<u8> buffer(4096);
	671	uptr bytes_read = buffer.size();
	672	uptr n_cpus = 0;
	673	u8 *d_type;
	674	struct linux_dirent entry = (struct linux_dirent )&buffer[bytes_read];
	675	while (true) {
	676	if ((u8 *)entry >= &buffer[bytes_read]) {
	677	bytes_read = internal_getdents(fd, (struct linux_dirent *)buffer.data(),
	678	buffer.size());
	679	if (internal_iserror(bytes_read) \|\| !bytes_read)
	680	break;
	681	entry = (struct linux_dirent *)buffer.data();
	682	}
	683	d_type = (u8 *)entry + entry->d_reclen - 1;
	684	if (d_type >= &buffer[bytes_read] \|\|
	685	(u8 *)&entry->d_name[3] >= &buffer[bytes_read])
	686	break;
	687	if (entry->d_ino != 0 && *d_type == DT_DIR) {
	688	if (entry->d_name[0] == 'c' && entry->d_name[1] == 'p' &&
	689	entry->d_name[2] == 'u' &&
	690	entry->d_name[3] >= '0' && entry->d_name[3] <= '9')
	691	n_cpus++;
	692	}
	693	entry = (struct linux_dirent )(((u8 )entry) + entry->d_reclen);
	694	}
	695	internal_close(fd);
	696	return n_cpus;
	697	#elif SANITIZER_SOLARIS
	698	return sysconf(_SC_NPROCESSORS_ONLN);
	699	#else
	700	cpu_set_t CPUs;
	701	CHECK_EQ(sched_getaffinity(0, sizeof(cpu_set_t), &CPUs), 0);
	702	return CPU_COUNT(&CPUs);
	703	#endif
	704	}
	705
	706	#if SANITIZER_LINUX
	707
	708	# if SANITIZER_ANDROID
	709	static atomic_uint8_t android_log_initialized;
	710
	711	void AndroidLogInit() {
	712	openlog(GetProcessName(), 0, LOG_USER);
	713	atomic_store(&android_log_initialized, 1, memory_order_release);
	714	}
	715
	716	static bool ShouldLogAfterPrintf() {
	717	return atomic_load(&android_log_initialized, memory_order_acquire);
	718	}
	719
	720	extern "C" SANITIZER_WEAK_ATTRIBUTE
	721	int async_safe_write_log(int pri, const char* tag, const char* msg);
	722	extern "C" SANITIZER_WEAK_ATTRIBUTE
	723	int __android_log_write(int prio, const char* tag, const char* msg);
	724
	725	// ANDROID_LOG_INFO is 4, but can't be resolved at runtime.
	726	#define SANITIZER_ANDROID_LOG_INFO 4
	727
	728	// async_safe_write_log is a new public version of __libc_write_log that is
	729	// used behind syslog. It is preferable to syslog as it will not do any dynamic
	730	// memory allocation or formatting.
	731	// If the function is not available, syslog is preferred for L+ (it was broken
	732	// pre-L) as __android_log_write triggers a racey behavior with the strncpy
	733	// interceptor. Fallback to __android_log_write pre-L.
	734	void WriteOneLineToSyslog(const char *s) {
	735	if (&async_safe_write_log) {
	736	async_safe_write_log(SANITIZER_ANDROID_LOG_INFO, GetProcessName(), s);
	737	} else if (AndroidGetApiLevel() > ANDROID_KITKAT) {
	738	syslog(LOG_INFO, "%s", s);
	739	} else {
	740	CHECK(&__android_log_write);
	741	__android_log_write(SANITIZER_ANDROID_LOG_INFO, nullptr, s);
	742	}
	743	}
	744
	745	extern "C" SANITIZER_WEAK_ATTRIBUTE
	746	void android_set_abort_message(const char *);
	747
	748	void SetAbortMessage(const char *str) {
	749	if (&android_set_abort_message)
	750	android_set_abort_message(str);
	751	}
	752	# else
	753	void AndroidLogInit() {}
	754
	755	static bool ShouldLogAfterPrintf() { return true; }
	756
	757	void WriteOneLineToSyslog(const char *s) { syslog(LOG_INFO, "%s", s); }
	758
	759	void SetAbortMessage(const char *str) {}
	760	# endif // SANITIZER_ANDROID
	761
	762	void LogMessageOnPrintf(const char *str) {
	763	if (common_flags()->log_to_syslog && ShouldLogAfterPrintf())
	764	WriteToSyslog(str);
	765	}
	766
	767	#endif // SANITIZER_LINUX
	768
	769	#if SANITIZER_LINUX && !SANITIZER_GO
	770	// glibc crashes when using clock_gettime from a preinit_array function as the
	771	// vDSO function pointers haven't been initialized yet. __progname is
	772	// initialized after the vDSO function pointers, so if it exists, is not null
	773	// and is not empty, we can use clock_gettime.
	774	extern "C" SANITIZER_WEAK_ATTRIBUTE char *__progname;
	775	INLINE bool CanUseVDSO() {
	776	// Bionic is safe, it checks for the vDSO function pointers to be initialized.
	777	if (SANITIZER_ANDROID)
	778	return true;
	779	if (&__progname && __progname && *__progname)
	780	return true;
	781	return false;
	782	}
	783
	784	// MonotonicNanoTime is a timing function that can leverage the vDSO by calling
	785	// clock_gettime. real_clock_gettime only exists if clock_gettime is
	786	// intercepted, so define it weakly and use it if available.
	787	extern "C" SANITIZER_WEAK_ATTRIBUTE
	788	int real_clock_gettime(u32 clk_id, void *tp);
	789	u64 MonotonicNanoTime() {
	790	timespec ts;
	791	if (CanUseVDSO()) {
	792	if (&real_clock_gettime)
	793	real_clock_gettime(CLOCK_MONOTONIC, &ts);
	794	else
	795	clock_gettime(CLOCK_MONOTONIC, &ts);
	796	} else {
	797	internal_clock_gettime(CLOCK_MONOTONIC, &ts);
	798	}
	799	return (u64)ts.tv_sec * (1000ULL * 1000 * 1000) + ts.tv_nsec;
	800	}
	801	#else
	802	// Non-Linux & Go always use the syscall.
	803	u64 MonotonicNanoTime() {
	804	timespec ts;
	805	internal_clock_gettime(CLOCK_MONOTONIC, &ts);
	806	return (u64)ts.tv_sec * (1000ULL * 1000 * 1000) + ts.tv_nsec;
	807	}
	808	#endif // SANITIZER_LINUX && !SANITIZER_GO
	809
	810	#if !SANITIZER_OPENBSD
	811	void ReExec() {
	812	const char *pathname = "/proc/self/exe";
	813
	814	#if SANITIZER_NETBSD
	815	static const int name[] = {
	816	CTL_KERN,
	817	KERN_PROC_ARGS,
	818	-1,
	819	KERN_PROC_PATHNAME,
	820	};
	821	char path[400];
	822	uptr len;
	823
	824	len = sizeof(path);
	825	if (internal_sysctl(name, ARRAY_SIZE(name), path, &len, NULL, 0) != -1)
	826	pathname = path;
	827	#elif SANITIZER_SOLARIS
	828	pathname = getexecname();
	829	CHECK_NE(pathname, NULL);
	830	#elif SANITIZER_USE_GETAUXVAL
	831	// Calling execve with /proc/self/exe sets that as $EXEC_ORIGIN. Binaries that
	832	// rely on that will fail to load shared libraries. Query AT_EXECFN instead.
	833	pathname = reinterpret_cast<const char *>(getauxval(AT_EXECFN));
	834	#endif
	835
	836	uptr rv = internal_execve(pathname, GetArgv(), GetEnviron());
	837	int rverrno;
	838	CHECK_EQ(internal_iserror(rv, &rverrno), true);
	839	Printf("execve failed, errno %d\n", rverrno);
	840	Die();
	841	}
	842	#endif // !SANITIZER_OPENBSD
	843
	844	} // namespace __sanitizer
	845
	846	#endif

lib/tsan/sanitizer_common/sanitizer_mac_libcdep.cpp created+29

...	...	@@ -0,0 +1,29 @@
	1	//===-- sanitizer_mac_libcdep.cpp -----------------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file is shared between various sanitizers' runtime libraries and
	10	// implements OSX-specific functions.
	11	//===----------------------------------------------------------------------===//
	12
	13	#include "sanitizer_platform.h"
	14	#if SANITIZER_MAC
	15	#include "sanitizer_mac.h"
	16
	17	#include <sys/mman.h>
	18
	19	namespace __sanitizer {
	20
	21	void RestrictMemoryToMaxAddress(uptr max_address) {
	22	uptr size_to_mmap = GetMaxUserVirtualAddress() + 1 - max_address;
	23	void *res = MmapFixedNoAccess(max_address, size_to_mmap, "high gap");
	24	CHECK(res != MAP_FAILED);
	25	}
	26
	27	} // namespace __sanitizer
	28
	29	#endif // SANITIZER_MAC

lib/tsan/sanitizer_common/sanitizer_posix_libcdep.cpp created+509

...	...	@@ -0,0 +1,509 @@
	1	//===-- sanitizer_posix_libcdep.cpp ---------------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file is shared between AddressSanitizer and ThreadSanitizer
	10	// run-time libraries and implements libc-dependent POSIX-specific functions
	11	// from sanitizer_libc.h.
	12	//===----------------------------------------------------------------------===//
	13
	14	#include "sanitizer_platform.h"
	15
	16	#if SANITIZER_POSIX
	17
	18	#include "sanitizer_common.h"
	19	#include "sanitizer_flags.h"
	20	#include "sanitizer_platform_limits_netbsd.h"
	21	#include "sanitizer_platform_limits_openbsd.h"
	22	#include "sanitizer_platform_limits_posix.h"
	23	#include "sanitizer_platform_limits_solaris.h"
	24	#include "sanitizer_posix.h"
	25	#include "sanitizer_procmaps.h"
	26
	27	#include <errno.h>
	28	#include <fcntl.h>
	29	#include <pthread.h>
	30	#include <signal.h>
	31	#include <stdlib.h>
	32	#include <sys/mman.h>
	33	#include <sys/resource.h>
	34	#include <sys/stat.h>
	35	#include <sys/time.h>
	36	#include <sys/types.h>
	37	#include <sys/wait.h>
	38	#include <unistd.h>
	39
	40	#if SANITIZER_FREEBSD
	41	// The MAP_NORESERVE define has been removed in FreeBSD 11.x, and even before
	42	// that, it was never implemented. So just define it to zero.
	43	#undef MAP_NORESERVE
	44	#define MAP_NORESERVE 0
	45	#endif
	46
	47	typedef void (sa_sigaction_t)(int, siginfo_t , void *);
	48
	49	namespace __sanitizer {
	50
	51	u32 GetUid() {
	52	return getuid();
	53	}
	54
	55	uptr GetThreadSelf() {
	56	return (uptr)pthread_self();
	57	}
	58
	59	void ReleaseMemoryPagesToOS(uptr beg, uptr end) {
	60	uptr page_size = GetPageSizeCached();
	61	uptr beg_aligned = RoundUpTo(beg, page_size);
	62	uptr end_aligned = RoundDownTo(end, page_size);
	63	if (beg_aligned < end_aligned)
	64	// In the default Solaris compilation environment, madvise() is declared
	65	// to take a caddr_t arg; casting it to void * results in an invalid
	66	// conversion error, so use char * instead.
	67	madvise((char *)beg_aligned, end_aligned - beg_aligned,
	68	SANITIZER_MADVISE_DONTNEED);
	69	}
	70
	71	void SetShadowRegionHugePageMode(uptr addr, uptr size) {
	72	#ifdef MADV_NOHUGEPAGE // May not be defined on old systems.
	73	if (common_flags()->no_huge_pages_for_shadow)
	74	madvise((char *)addr, size, MADV_NOHUGEPAGE);
	75	else
	76	madvise((char *)addr, size, MADV_HUGEPAGE);
	77	#endif // MADV_NOHUGEPAGE
	78	}
	79
	80	bool DontDumpShadowMemory(uptr addr, uptr length) {
	81	#if defined(MADV_DONTDUMP)
	82	return madvise((char *)addr, length, MADV_DONTDUMP) == 0;
	83	#elif defined(MADV_NOCORE)
	84	return madvise((char *)addr, length, MADV_NOCORE) == 0;
	85	#else
	86	return true;
	87	#endif // MADV_DONTDUMP
	88	}
	89
	90	static rlim_t getlim(int res) {
	91	rlimit rlim;
	92	CHECK_EQ(0, getrlimit(res, &rlim));
	93	return rlim.rlim_cur;
	94	}
	95
	96	static void setlim(int res, rlim_t lim) {
	97	struct rlimit rlim;
	98	if (getrlimit(res, const_cast<struct rlimit *>(&rlim))) {
	99	Report("ERROR: %s getrlimit() failed %d\n", SanitizerToolName, errno);
	100	Die();
	101	}
	102	rlim.rlim_cur = lim;
	103	if (setrlimit(res, const_cast<struct rlimit *>(&rlim))) {
	104	Report("ERROR: %s setrlimit() failed %d\n", SanitizerToolName, errno);
	105	Die();
	106	}
	107	}
	108
	109	void DisableCoreDumperIfNecessary() {
	110	if (common_flags()->disable_coredump) {
	111	setlim(RLIMIT_CORE, 0);
	112	}
	113	}
	114
	115	bool StackSizeIsUnlimited() {
	116	rlim_t stack_size = getlim(RLIMIT_STACK);
	117	return (stack_size == RLIM_INFINITY);
	118	}
	119
	120	void SetStackSizeLimitInBytes(uptr limit) {
	121	setlim(RLIMIT_STACK, (rlim_t)limit);
	122	CHECK(!StackSizeIsUnlimited());
	123	}
	124
	125	bool AddressSpaceIsUnlimited() {
	126	rlim_t as_size = getlim(RLIMIT_AS);
	127	return (as_size == RLIM_INFINITY);
	128	}
	129
	130	void SetAddressSpaceUnlimited() {
	131	setlim(RLIMIT_AS, RLIM_INFINITY);
	132	CHECK(AddressSpaceIsUnlimited());
	133	}
	134
	135	void SleepForSeconds(int seconds) {
	136	sleep(seconds);
	137	}
	138
	139	void SleepForMillis(int millis) {
	140	usleep(millis * 1000);
	141	}
	142
	143	void Abort() {
	144	#if !SANITIZER_GO
	145	// If we are handling SIGABRT, unhandle it first.
	146	// TODO(vitalybuka): Check if handler belongs to sanitizer.
	147	if (GetHandleSignalMode(SIGABRT) != kHandleSignalNo) {
	148	struct sigaction sigact;
	149	internal_memset(&sigact, 0, sizeof(sigact));
	150	sigact.sa_sigaction = (sa_sigaction_t)SIG_DFL;
	151	internal_sigaction(SIGABRT, &sigact, nullptr);
	152	}
	153	#endif
	154
	155	abort();
	156	}
	157
	158	int Atexit(void (*function)(void)) {
	159	#if !SANITIZER_GO
	160	return atexit(function);
	161	#else
	162	return 0;
	163	#endif
	164	}
	165
	166	bool SupportsColoredOutput(fd_t fd) {
	167	return isatty(fd) != 0;
	168	}
	169
	170	#if !SANITIZER_GO
	171	// TODO(glider): different tools may require different altstack size.
	172	static const uptr kAltStackSize = SIGSTKSZ * 4; // SIGSTKSZ is not enough.
	173
	174	void SetAlternateSignalStack() {
	175	stack_t altstack, oldstack;
	176	CHECK_EQ(0, sigaltstack(nullptr, &oldstack));
	177	// If the alternate stack is already in place, do nothing.
	178	// Android always sets an alternate stack, but it's too small for us.
	179	if (!SANITIZER_ANDROID && !(oldstack.ss_flags & SS_DISABLE)) return;
	180	// TODO(glider): the mapped stack should have the MAP_STACK flag in the
	181	// future. It is not required by man 2 sigaltstack now (they're using
	182	// malloc()).
	183	void* base = MmapOrDie(kAltStackSize, __func__);
	184	altstack.ss_sp = (char*) base;
	185	altstack.ss_flags = 0;
	186	altstack.ss_size = kAltStackSize;
	187	CHECK_EQ(0, sigaltstack(&altstack, nullptr));
	188	}
	189
	190	void UnsetAlternateSignalStack() {
	191	stack_t altstack, oldstack;
	192	altstack.ss_sp = nullptr;
	193	altstack.ss_flags = SS_DISABLE;
	194	altstack.ss_size = kAltStackSize; // Some sane value required on Darwin.
	195	CHECK_EQ(0, sigaltstack(&altstack, &oldstack));
	196	UnmapOrDie(oldstack.ss_sp, oldstack.ss_size);
	197	}
	198
	199	static void MaybeInstallSigaction(int signum,
	200	SignalHandlerType handler) {
	201	if (GetHandleSignalMode(signum) == kHandleSignalNo) return;
	202
	203	struct sigaction sigact;
	204	internal_memset(&sigact, 0, sizeof(sigact));
	205	sigact.sa_sigaction = (sa_sigaction_t)handler;
	206	// Do not block the signal from being received in that signal's handler.
	207	// Clients are responsible for handling this correctly.
	208	sigact.sa_flags = SA_SIGINFO \| SA_NODEFER;
	209	if (common_flags()->use_sigaltstack) sigact.sa_flags \|= SA_ONSTACK;
	210	CHECK_EQ(0, internal_sigaction(signum, &sigact, nullptr));
	211	VReport(1, "Installed the sigaction for signal %d\n", signum);
	212	}
	213
	214	void InstallDeadlySignalHandlers(SignalHandlerType handler) {
	215	// Set the alternate signal stack for the main thread.
	216	// This will cause SetAlternateSignalStack to be called twice, but the stack
	217	// will be actually set only once.
	218	if (common_flags()->use_sigaltstack) SetAlternateSignalStack();
	219	MaybeInstallSigaction(SIGSEGV, handler);
	220	MaybeInstallSigaction(SIGBUS, handler);
	221	MaybeInstallSigaction(SIGABRT, handler);
	222	MaybeInstallSigaction(SIGFPE, handler);
	223	MaybeInstallSigaction(SIGILL, handler);
	224	MaybeInstallSigaction(SIGTRAP, handler);
	225	}
	226
	227	bool SignalContext::IsStackOverflow() const {
	228	// Access at a reasonable offset above SP, or slightly below it (to account
	229	// for x86_64 or PowerPC redzone, ARM push of multiple registers, etc) is
	230	// probably a stack overflow.
	231	#ifdef __s390__
	232	// On s390, the fault address in siginfo points to start of the page, not
	233	// to the precise word that was accessed. Mask off the low bits of sp to
	234	// take it into account.
	235	bool IsStackAccess = addr >= (sp & ~0xFFF) && addr < sp + 0xFFFF;
	236	#else
	237	// Let's accept up to a page size away from top of stack. Things like stack
	238	// probing can trigger accesses with such large offsets.
	239	bool IsStackAccess = addr + GetPageSizeCached() > sp && addr < sp + 0xFFFF;
	240	#endif
	241
	242	#if __powerpc__
	243	// Large stack frames can be allocated with e.g.
	244	// lis r0,-10000
	245	// stdux r1,r1,r0 # store sp to [sp-10000] and update sp by -10000
	246	// If the store faults then sp will not have been updated, so test above
	247	// will not work, because the fault address will be more than just "slightly"
	248	// below sp.
	249	if (!IsStackAccess && IsAccessibleMemoryRange(pc, 4)) {
	250	u32 inst = (unsigned )pc;
	251	u32 ra = (inst >> 16) & 0x1F;
	252	u32 opcd = inst >> 26;
	253	u32 xo = (inst >> 1) & 0x3FF;
	254	// Check for store-with-update to sp. The instructions we accept are:
	255	// stbu rs,d(ra) stbux rs,ra,rb
	256	// sthu rs,d(ra) sthux rs,ra,rb
	257	// stwu rs,d(ra) stwux rs,ra,rb
	258	// stdu rs,ds(ra) stdux rs,ra,rb
	259	// where ra is r1 (the stack pointer).
	260	if (ra == 1 &&
	261	(opcd == 39 \|\| opcd == 45 \|\| opcd == 37 \|\| opcd == 62 \|\|
	262	(opcd == 31 && (xo == 247 \|\| xo == 439 \|\| xo == 183 \|\| xo == 181))))
	263	IsStackAccess = true;
	264	}
	265	#endif // __powerpc__
	266
	267	// We also check si_code to filter out SEGV caused by something else other
	268	// then hitting the guard page or unmapped memory, like, for example,
	269	// unaligned memory access.
	270	auto si = static_cast<const siginfo_t *>(siginfo);
	271	return IsStackAccess &&
	272	(si->si_code == si_SEGV_MAPERR \|\| si->si_code == si_SEGV_ACCERR);
	273	}
	274
	275	#endif // SANITIZER_GO
	276
	277	bool IsAccessibleMemoryRange(uptr beg, uptr size) {
	278	uptr page_size = GetPageSizeCached();
	279	// Checking too large memory ranges is slow.
	280	CHECK_LT(size, page_size * 10);
	281	int sock_pair[2];
	282	if (pipe(sock_pair))
	283	return false;
	284	uptr bytes_written =
	285	internal_write(sock_pair[1], reinterpret_cast<void *>(beg), size);
	286	int write_errno;
	287	bool result;
	288	if (internal_iserror(bytes_written, &write_errno)) {
	289	CHECK_EQ(EFAULT, write_errno);
	290	result = false;
	291	} else {
	292	result = (bytes_written == size);
	293	}
	294	internal_close(sock_pair[0]);
	295	internal_close(sock_pair[1]);
	296	return result;
	297	}
	298
	299	void PlatformPrepareForSandboxing(__sanitizer_sandbox_arguments *args) {
	300	// Some kinds of sandboxes may forbid filesystem access, so we won't be able
	301	// to read the file mappings from /proc/self/maps. Luckily, neither the
	302	// process will be able to load additional libraries, so it's fine to use the
	303	// cached mappings.
	304	MemoryMappingLayout::CacheMemoryMappings();
	305	}
	306
	307	static bool MmapFixed(uptr fixed_addr, uptr size, int additional_flags,
	308	const char *name) {
	309	size = RoundUpTo(size, GetPageSizeCached());
	310	fixed_addr = RoundDownTo(fixed_addr, GetPageSizeCached());
	311	uptr p =
	312	MmapNamed((void *)fixed_addr, size, PROT_READ \| PROT_WRITE,
	313	MAP_PRIVATE \| MAP_FIXED \| additional_flags \| MAP_ANON, name);
	314	int reserrno;
	315	if (internal_iserror(p, &reserrno)) {
	316	Report("ERROR: %s failed to "
	317	"allocate 0x%zx (%zd) bytes at address %zx (errno: %d)\n",
	318	SanitizerToolName, size, size, fixed_addr, reserrno);
	319	return false;
	320	}
	321	IncreaseTotalMmap(size);
	322	return true;
	323	}
	324
	325	bool MmapFixedNoReserve(uptr fixed_addr, uptr size, const char *name) {
	326	return MmapFixed(fixed_addr, size, MAP_NORESERVE, name);
	327	}
	328
	329	bool MmapFixedSuperNoReserve(uptr fixed_addr, uptr size, const char *name) {
	330	#if SANITIZER_FREEBSD
	331	if (common_flags()->no_huge_pages_for_shadow)
	332	return MmapFixedNoReserve(fixed_addr, size, name);
	333	// MAP_NORESERVE is implicit with FreeBSD
	334	return MmapFixed(fixed_addr, size, MAP_ALIGNED_SUPER, name);
	335	#else
	336	bool r = MmapFixedNoReserve(fixed_addr, size, name);
	337	if (r)
	338	SetShadowRegionHugePageMode(fixed_addr, size);
	339	return r;
	340	#endif
	341	}
	342
	343	uptr ReservedAddressRange::Init(uptr size, const char *name, uptr fixed_addr) {
	344	base_ = fixed_addr ? MmapFixedNoAccess(fixed_addr, size, name)
	345	: MmapNoAccess(size);
	346	size_ = size;
	347	name_ = name;
	348	(void)os_handle_; // unsupported
	349	return reinterpret_cast<uptr>(base_);
	350	}
	351
	352	// Uses fixed_addr for now.
	353	// Will use offset instead once we've implemented this function for real.
	354	uptr ReservedAddressRange::Map(uptr fixed_addr, uptr size, const char *name) {
	355	return reinterpret_cast<uptr>(
	356	MmapFixedOrDieOnFatalError(fixed_addr, size, name));
	357	}
	358
	359	uptr ReservedAddressRange::MapOrDie(uptr fixed_addr, uptr size,
	360	const char *name) {
	361	return reinterpret_cast<uptr>(MmapFixedOrDie(fixed_addr, size, name));
	362	}
	363
	364	void ReservedAddressRange::Unmap(uptr addr, uptr size) {
	365	CHECK_LE(size, size_);
	366	if (addr == reinterpret_cast<uptr>(base_))
	367	// If we unmap the whole range, just null out the base.
	368	base_ = (size == size_) ? nullptr : reinterpret_cast<void*>(addr + size);
	369	else
	370	CHECK_EQ(addr + size, reinterpret_cast<uptr>(base_) + size_);
	371	size_ -= size;
	372	UnmapOrDie(reinterpret_cast<void*>(addr), size);
	373	}
	374
	375	void MmapFixedNoAccess(uptr fixed_addr, uptr size, const char name) {
	376	return (void )MmapNamed((void )fixed_addr, size, PROT_NONE,
	377	MAP_PRIVATE \| MAP_FIXED \| MAP_NORESERVE \| MAP_ANON,
	378	name);
	379	}
	380
	381	void *MmapNoAccess(uptr size) {
	382	unsigned flags = MAP_PRIVATE \| MAP_ANON \| MAP_NORESERVE;
	383	return (void *)internal_mmap(nullptr, size, PROT_NONE, flags, -1, 0);
	384	}
	385
	386	// This function is defined elsewhere if we intercepted pthread_attr_getstack.
	387	extern "C" {
	388	SANITIZER_WEAK_ATTRIBUTE int
	389	real_pthread_attr_getstack(void attr, void addr, size_t size);
	390	} // extern "C"
	391
	392	int my_pthread_attr_getstack(void attr, void addr, uptr size) {
	393	#if !SANITIZER_GO && !SANITIZER_MAC
	394	if (&real_pthread_attr_getstack)
	395	return real_pthread_attr_getstack((pthread_attr_t *)attr, addr,
	396	(size_t *)size);
	397	#endif
	398	return pthread_attr_getstack((pthread_attr_t )attr, addr, (size_t )size);
	399	}
	400
	401	#if !SANITIZER_GO
	402	void AdjustStackSize(void *attr_) {
	403	pthread_attr_t attr = (pthread_attr_t )attr_;
	404	uptr stackaddr = 0;
	405	uptr stacksize = 0;
	406	my_pthread_attr_getstack(attr, (void**)&stackaddr, &stacksize);
	407	// GLibC will return (0 - stacksize) as the stack address in the case when
	408	// stacksize is set, but stackaddr is not.
	409	bool stack_set = (stackaddr != 0) && (stackaddr + stacksize != 0);
	410	// We place a lot of tool data into TLS, account for that.
	411	const uptr minstacksize = GetTlsSize() + 128*1024;
	412	if (stacksize < minstacksize) {
	413	if (!stack_set) {
	414	if (stacksize != 0) {
	415	VPrintf(1, "Sanitizer: increasing stacksize %zu->%zu\n", stacksize,
	416	minstacksize);
	417	pthread_attr_setstacksize(attr, minstacksize);
	418	}
	419	} else {
	420	Printf("Sanitizer: pre-allocated stack size is insufficient: "
	421	"%zu < %zu\n", stacksize, minstacksize);
	422	Printf("Sanitizer: pthread_create is likely to fail.\n");
	423	}
	424	}
	425	}
	426	#endif // !SANITIZER_GO
	427
	428	pid_t StartSubprocess(const char program, const char const argv[],
	429	const char *const envp[], fd_t stdin_fd, fd_t stdout_fd,
	430	fd_t stderr_fd) {
	431	auto file_closer = at_scope_exit([&] {
	432	if (stdin_fd != kInvalidFd) {
	433	internal_close(stdin_fd);
	434	}
	435	if (stdout_fd != kInvalidFd) {
	436	internal_close(stdout_fd);
	437	}
	438	if (stderr_fd != kInvalidFd) {
	439	internal_close(stderr_fd);
	440	}
	441	});
	442
	443	int pid = internal_fork();
	444
	445	if (pid < 0) {
	446	int rverrno;
	447	if (internal_iserror(pid, &rverrno)) {
	448	Report("WARNING: failed to fork (errno %d)\n", rverrno);
	449	}
	450	return pid;
	451	}
	452
	453	if (pid == 0) {
	454	// Child subprocess
	455	if (stdin_fd != kInvalidFd) {
	456	internal_close(STDIN_FILENO);
	457	internal_dup2(stdin_fd, STDIN_FILENO);
	458	internal_close(stdin_fd);
	459	}
	460	if (stdout_fd != kInvalidFd) {
	461	internal_close(STDOUT_FILENO);
	462	internal_dup2(stdout_fd, STDOUT_FILENO);
	463	internal_close(stdout_fd);
	464	}
	465	if (stderr_fd != kInvalidFd) {
	466	internal_close(STDERR_FILENO);
	467	internal_dup2(stderr_fd, STDERR_FILENO);
	468	internal_close(stderr_fd);
	469	}
	470
	471	for (int fd = sysconf(_SC_OPEN_MAX); fd > 2; fd--) internal_close(fd);
	472
	473	internal_execve(program, const_cast<char **>(&argv[0]),
	474	const_cast<char const >(envp));
	475	internal__exit(1);
	476	}
	477
	478	return pid;
	479	}
	480
	481	bool IsProcessRunning(pid_t pid) {
	482	int process_status;
	483	uptr waitpid_status = internal_waitpid(pid, &process_status, WNOHANG);
	484	int local_errno;
	485	if (internal_iserror(waitpid_status, &local_errno)) {
	486	VReport(1, "Waiting on the process failed (errno %d).\n", local_errno);
	487	return false;
	488	}
	489	return waitpid_status == 0;
	490	}
	491
	492	int WaitForProcess(pid_t pid) {
	493	int process_status;
	494	uptr waitpid_status = internal_waitpid(pid, &process_status, 0);
	495	int local_errno;
	496	if (internal_iserror(waitpid_status, &local_errno)) {
	497	VReport(1, "Waiting on the process failed (errno %d).\n", local_errno);
	498	return -1;
	499	}
	500	return process_status;
	501	}
	502
	503	bool IsStateDetached(int state) {
	504	return state == PTHREAD_CREATE_DETACHED;
	505	}
	506
	507	} // namespace __sanitizer
	508
	509	#endif // SANITIZER_POSIX

lib/tsan/sanitizer_common/sanitizer_stackdepot.cpp created+149

...	...	@@ -0,0 +1,149 @@
	1	//===-- sanitizer_stackdepot.cpp ------------------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file is shared between AddressSanitizer and ThreadSanitizer
	10	// run-time libraries.
	11	//===----------------------------------------------------------------------===//
	12
	13	#include "sanitizer_stackdepot.h"
	14
	15	#include "sanitizer_common.h"
	16	#include "sanitizer_hash.h"
	17	#include "sanitizer_stackdepotbase.h"
	18
	19	namespace __sanitizer {
	20
	21	struct StackDepotNode {
	22	StackDepotNode *link;
	23	u32 id;
	24	atomic_uint32_t hash_and_use_count; // hash_bits : 12; use_count : 20;
	25	u32 size;
	26	u32 tag;
	27	uptr stack[1]; // [size]
	28
	29	static const u32 kTabSizeLog = SANITIZER_ANDROID ? 16 : 20;
	30	// Lower kTabSizeLog bits are equal for all items in one bucket.
	31	// We use these bits to store the per-stack use counter.
	32	static const u32 kUseCountBits = kTabSizeLog;
	33	static const u32 kMaxUseCount = 1 << kUseCountBits;
	34	static const u32 kUseCountMask = (1 << kUseCountBits) - 1;
	35	static const u32 kHashMask = ~kUseCountMask;
	36
	37	typedef StackTrace args_type;
	38	bool eq(u32 hash, const args_type &args) const {
	39	u32 hash_bits =
	40	atomic_load(&hash_and_use_count, memory_order_relaxed) & kHashMask;
	41	if ((hash & kHashMask) != hash_bits \|\| args.size != size \|\| args.tag != tag)
	42	return false;
	43	uptr i = 0;
	44	for (; i < size; i++) {
	45	if (stack[i] != args.trace[i]) return false;
	46	}
	47	return true;
	48	}
	49	static uptr storage_size(const args_type &args) {
	50	return sizeof(StackDepotNode) + (args.size - 1) * sizeof(uptr);
	51	}
	52	static u32 hash(const args_type &args) {
	53	MurMur2HashBuilder H(args.size * sizeof(uptr));
	54	for (uptr i = 0; i < args.size; i++) H.add(args.trace[i]);
	55	return H.get();
	56	}
	57	static bool is_valid(const args_type &args) {
	58	return args.size > 0 && args.trace;
	59	}
	60	void store(const args_type &args, u32 hash) {
	61	atomic_store(&hash_and_use_count, hash & kHashMask, memory_order_relaxed);
	62	size = args.size;
	63	tag = args.tag;
	64	internal_memcpy(stack, args.trace, size * sizeof(uptr));
	65	}
	66	args_type load() const {
	67	return args_type(&stack[0], size, tag);
	68	}
	69	StackDepotHandle get_handle() { return StackDepotHandle(this); }
	70
	71	typedef StackDepotHandle handle_type;
	72	};
	73
	74	COMPILER_CHECK(StackDepotNode::kMaxUseCount == (u32)kStackDepotMaxUseCount);
	75
	76	u32 StackDepotHandle::id() { return node_->id; }
	77	int StackDepotHandle::use_count() {
	78	return atomic_load(&node_->hash_and_use_count, memory_order_relaxed) &
	79	StackDepotNode::kUseCountMask;
	80	}
	81	void StackDepotHandle::inc_use_count_unsafe() {
	82	u32 prev =
	83	atomic_fetch_add(&node_->hash_and_use_count, 1, memory_order_relaxed) &
	84	StackDepotNode::kUseCountMask;
	85	CHECK_LT(prev + 1, StackDepotNode::kMaxUseCount);
	86	}
	87
	88	// FIXME(dvyukov): this single reserved bit is used in TSan.
	89	typedef StackDepotBase<StackDepotNode, 1, StackDepotNode::kTabSizeLog>
	90	StackDepot;
	91	static StackDepot theDepot;
	92
	93	StackDepotStats *StackDepotGetStats() {
	94	return theDepot.GetStats();
	95	}
	96
	97	u32 StackDepotPut(StackTrace stack) {
	98	StackDepotHandle h = theDepot.Put(stack);
	99	return h.valid() ? h.id() : 0;
	100	}
	101
	102	StackDepotHandle StackDepotPut_WithHandle(StackTrace stack) {
	103	return theDepot.Put(stack);
	104	}
	105
	106	StackTrace StackDepotGet(u32 id) {
	107	return theDepot.Get(id);
	108	}
	109
	110	void StackDepotLockAll() {
	111	theDepot.LockAll();
	112	}
	113
	114	void StackDepotUnlockAll() {
	115	theDepot.UnlockAll();
	116	}
	117
	118	bool StackDepotReverseMap::IdDescPair::IdComparator(
	119	const StackDepotReverseMap::IdDescPair &a,
	120	const StackDepotReverseMap::IdDescPair &b) {
	121	return a.id < b.id;
	122	}
	123
	124	StackDepotReverseMap::StackDepotReverseMap() {
	125	map_.reserve(StackDepotGetStats()->n_uniq_ids + 100);
	126	for (int idx = 0; idx < StackDepot::kTabSize; idx++) {
	127	atomic_uintptr_t *p = &theDepot.tab[idx];
	128	uptr v = atomic_load(p, memory_order_consume);
	129	StackDepotNode s = (StackDepotNode)(v & ~1);
	130	for (; s; s = s->link) {
	131	IdDescPair pair = {s->id, s};
	132	map_.push_back(pair);
	133	}
	134	}
	135	Sort(map_.data(), map_.size(), &IdDescPair::IdComparator);
	136	}
	137
	138	StackTrace StackDepotReverseMap::Get(u32 id) {
	139	if (!map_.size())
	140	return StackTrace();
	141	IdDescPair pair = {id, nullptr};
	142	uptr idx =
	143	InternalLowerBound(map_, 0, map_.size(), pair, IdDescPair::IdComparator);
	144	if (idx > map_.size() \|\| map_[idx].id != id)
	145	return StackTrace();
	146	return map_[idx].desc->load();
	147	}
	148
	149	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_stacktrace.cpp created+133

...	...	@@ -0,0 +1,133 @@
	1	//===-- sanitizer_stacktrace.cpp ------------------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file is shared between AddressSanitizer and ThreadSanitizer
	10	// run-time libraries.
	11	//===----------------------------------------------------------------------===//
	12
	13	#include "sanitizer_common.h"
	14	#include "sanitizer_flags.h"
	15	#include "sanitizer_stacktrace.h"
	16
	17	namespace __sanitizer {
	18
	19	uptr StackTrace::GetNextInstructionPc(uptr pc) {
	20	#if defined(__sparc__) \|\| defined(__mips__)
	21	return pc + 8;
	22	#elif defined(__powerpc__) \|\| defined(__arm__) \|\| defined(__aarch64__)
	23	return pc + 4;
	24	#else
	25	return pc + 1;
	26	#endif
	27	}
	28
	29	uptr StackTrace::GetCurrentPc() {
	30	return GET_CALLER_PC();
	31	}
	32
	33	void BufferedStackTrace::Init(const uptr *pcs, uptr cnt, uptr extra_top_pc) {
	34	size = cnt + !!extra_top_pc;
	35	CHECK_LE(size, kStackTraceMax);
	36	internal_memcpy(trace_buffer, pcs, cnt * sizeof(trace_buffer[0]));
	37	if (extra_top_pc)
	38	trace_buffer[cnt] = extra_top_pc;
	39	top_frame_bp = 0;
	40	}
	41
	42	// Sparc implemention is in its own file.
	43	#if !defined(__sparc__)
	44
	45	// In GCC on ARM bp points to saved lr, not fp, so we should check the next
	46	// cell in stack to be a saved frame pointer. GetCanonicFrame returns the
	47	// pointer to saved frame pointer in any case.
	48	static inline uhwptr *GetCanonicFrame(uptr bp,
	49	uptr stack_top,
	50	uptr stack_bottom) {
	51	CHECK_GT(stack_top, stack_bottom);
	52	#ifdef __arm__
	53	if (!IsValidFrame(bp, stack_top, stack_bottom)) return 0;
	54	uhwptr bp_prev = (uhwptr )bp;
	55	if (IsValidFrame((uptr)bp_prev[0], stack_top, stack_bottom)) return bp_prev;
	56	// The next frame pointer does not look right. This could be a GCC frame, step
	57	// back by 1 word and try again.
	58	if (IsValidFrame((uptr)bp_prev[-1], stack_top, stack_bottom))
	59	return bp_prev - 1;
	60	// Nope, this does not look right either. This means the frame after next does
	61	// not have a valid frame pointer, but we can still extract the caller PC.
	62	// Unfortunately, there is no way to decide between GCC and LLVM frame
	63	// layouts. Assume LLVM.
	64	return bp_prev;
	65	#else
	66	return (uhwptr*)bp;
	67	#endif
	68	}
	69
	70	void BufferedStackTrace::UnwindFast(uptr pc, uptr bp, uptr stack_top,
	71	uptr stack_bottom, u32 max_depth) {
	72	// TODO(yln): add arg sanity check for stack_top/stack_bottom
	73	CHECK_GE(max_depth, 2);
	74	const uptr kPageSize = GetPageSizeCached();
	75	trace_buffer[0] = pc;
	76	size = 1;
	77	if (stack_top < 4096) return; // Sanity check for stack top.
	78	uhwptr *frame = GetCanonicFrame(bp, stack_top, stack_bottom);
	79	// Lowest possible address that makes sense as the next frame pointer.
	80	// Goes up as we walk the stack.
	81	uptr bottom = stack_bottom;
	82	// Avoid infinite loop when frame == frame[0] by using frame > prev_frame.
	83	while (IsValidFrame((uptr)frame, stack_top, bottom) &&
	84	IsAligned((uptr)frame, sizeof(*frame)) &&
	85	size < max_depth) {
	86	#ifdef __powerpc__
	87	// PowerPC ABIs specify that the return address is saved at offset
	88	// 16 of the caller's stack frame. Thus we must dereference the
	89	// back chain to find the caller frame before extracting it.
	90	uhwptr caller_frame = (uhwptr)frame[0];
	91	if (!IsValidFrame((uptr)caller_frame, stack_top, bottom) \|\|
	92	!IsAligned((uptr)caller_frame, sizeof(uhwptr)))
	93	break;
	94	uhwptr pc1 = caller_frame[2];
	95	#elif defined(__s390__)
	96	uhwptr pc1 = frame[14];
	97	#else
	98	uhwptr pc1 = frame[1];
	99	#endif
	100	// Let's assume that any pointer in the 0th page (i.e. <0x1000 on i386 and
	101	// x86_64) is invalid and stop unwinding here. If we're adding support for
	102	// a platform where this isn't true, we need to reconsider this check.
	103	if (pc1 < kPageSize)
	104	break;
	105	if (pc1 != pc) {
	106	trace_buffer[size++] = (uptr) pc1;
	107	}
	108	bottom = (uptr)frame;
	109	frame = GetCanonicFrame((uptr)frame[0], stack_top, bottom);
	110	}
	111	}
	112
	113	#endif // !defined(__sparc__)
	114
	115	void BufferedStackTrace::PopStackFrames(uptr count) {
	116	CHECK_LT(count, size);
	117	size -= count;
	118	for (uptr i = 0; i < size; ++i) {
	119	trace_buffer[i] = trace_buffer[i + count];
	120	}
	121	}
	122
	123	static uptr Distance(uptr a, uptr b) { return a < b ? b - a : a - b; }
	124
	125	uptr BufferedStackTrace::LocatePcInTrace(uptr pc) {
	126	uptr best = 0;
	127	for (uptr i = 1; i < size; ++i) {
	128	if (Distance(trace[i], pc) < Distance(trace[best], pc)) best = i;
	129	}
	130	return best;
	131	}
	132
	133	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_stacktrace_libcdep.cpp created+159

...	...	@@ -0,0 +1,159 @@
	1	//===-- sanitizer_stacktrace_libcdep.cpp ----------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file is shared between AddressSanitizer and ThreadSanitizer
	10	// run-time libraries.
	11	//===----------------------------------------------------------------------===//
	12
	13	#include "sanitizer_common.h"
	14	#include "sanitizer_placement_new.h"
	15	#include "sanitizer_stacktrace.h"
	16	#include "sanitizer_stacktrace_printer.h"
	17	#include "sanitizer_symbolizer.h"
	18
	19	namespace __sanitizer {
	20
	21	void StackTrace::Print() const {
	22	if (trace == nullptr \|\| size == 0) {
	23	Printf(" <empty stack>\n\n");
	24	return;
	25	}
	26	InternalScopedString frame_desc(GetPageSizeCached() * 2);
	27	InternalScopedString dedup_token(GetPageSizeCached());
	28	int dedup_frames = common_flags()->dedup_token_length;
	29	uptr frame_num = 0;
	30	for (uptr i = 0; i < size && trace[i]; i++) {
	31	// PCs in stack traces are actually the return addresses, that is,
	32	// addresses of the next instructions after the call.
	33	uptr pc = GetPreviousInstructionPc(trace[i]);
	34	SymbolizedStack *frames = Symbolizer::GetOrInit()->SymbolizePC(pc);
	35	CHECK(frames);
	36	for (SymbolizedStack *cur = frames; cur; cur = cur->next) {
	37	frame_desc.clear();
	38	RenderFrame(&frame_desc, common_flags()->stack_trace_format, frame_num++,
	39	cur->info, common_flags()->symbolize_vs_style,
	40	common_flags()->strip_path_prefix);
	41	Printf("%s\n", frame_desc.data());
	42	if (dedup_frames-- > 0) {
	43	if (dedup_token.length())
	44	dedup_token.append("--");
	45	if (cur->info.function != nullptr)
	46	dedup_token.append(cur->info.function);
	47	}
	48	}
	49	frames->ClearAll();
	50	}
	51	// Always print a trailing empty line after stack trace.
	52	Printf("\n");
	53	if (dedup_token.length())
	54	Printf("DEDUP_TOKEN: %s\n", dedup_token.data());
	55	}
	56
	57	void BufferedStackTrace::Unwind(u32 max_depth, uptr pc, uptr bp, void *context,
	58	uptr stack_top, uptr stack_bottom,
	59	bool request_fast_unwind) {
	60	// Ensures all call sites get what they requested.
	61	CHECK_EQ(request_fast_unwind, WillUseFastUnwind(request_fast_unwind));
	62	top_frame_bp = (max_depth > 0) ? bp : 0;
	63	// Avoid doing any work for small max_depth.
	64	if (max_depth == 0) {
	65	size = 0;
	66	return;
	67	}
	68	if (max_depth == 1) {
	69	size = 1;
	70	trace_buffer[0] = pc;
	71	return;
	72	}
	73	if (!WillUseFastUnwind(request_fast_unwind)) {
	74	#if SANITIZER_CAN_SLOW_UNWIND
	75	if (context)
	76	UnwindSlow(pc, context, max_depth);
	77	else
	78	UnwindSlow(pc, max_depth);
	79	#else
	80	UNREACHABLE("slow unwind requested but not available");
	81	#endif
	82	} else {
	83	UnwindFast(pc, bp, stack_top, stack_bottom, max_depth);
	84	}
	85	}
	86
	87	static int GetModuleAndOffsetForPc(uptr pc, char *module_name,
	88	uptr module_name_len, uptr *pc_offset) {
	89	const char *found_module_name = nullptr;
	90	bool ok = Symbolizer::GetOrInit()->GetModuleNameAndOffsetForPC(
	91	pc, &found_module_name, pc_offset);
	92
	93	if (!ok) return false;
	94
	95	if (module_name && module_name_len) {
	96	internal_strncpy(module_name, found_module_name, module_name_len);
	97	module_name[module_name_len - 1] = '\x00';
	98	}
	99	return true;
	100	}
	101
	102	} // namespace __sanitizer
	103	using namespace __sanitizer;
	104
	105	extern "C" {
	106	SANITIZER_INTERFACE_ATTRIBUTE
	107	void __sanitizer_symbolize_pc(uptr pc, const char fmt, char out_buf,
	108	uptr out_buf_size) {
	109	if (!out_buf_size) return;
	110	pc = StackTrace::GetPreviousInstructionPc(pc);
	111	SymbolizedStack *frame = Symbolizer::GetOrInit()->SymbolizePC(pc);
	112	if (!frame) {
	113	internal_strncpy(out_buf, "<can't symbolize>", out_buf_size);
	114	out_buf[out_buf_size - 1] = 0;
	115	return;
	116	}
	117	InternalScopedString frame_desc(GetPageSizeCached());
	118	uptr frame_num = 0;
	119	// Reserve one byte for the final 0.
	120	char *out_end = out_buf + out_buf_size - 1;
	121	for (SymbolizedStack *cur = frame; cur && out_buf < out_end;
	122	cur = cur->next) {
	123	frame_desc.clear();
	124	RenderFrame(&frame_desc, fmt, frame_num++, cur->info,
	125	common_flags()->symbolize_vs_style,
	126	common_flags()->strip_path_prefix);
	127	if (!frame_desc.length())
	128	continue;
	129	// Reserve one byte for the terminating 0.
	130	uptr n = out_end - out_buf - 1;
	131	internal_strncpy(out_buf, frame_desc.data(), n);
	132	out_buf += __sanitizer::Min<uptr>(n, frame_desc.length());
	133	*out_buf++ = 0;
	134	}
	135	CHECK(out_buf <= out_end);
	136	*out_buf = 0;
	137	}
	138
	139	SANITIZER_INTERFACE_ATTRIBUTE
	140	void __sanitizer_symbolize_global(uptr data_addr, const char *fmt,
	141	char *out_buf, uptr out_buf_size) {
	142	if (!out_buf_size) return;
	143	out_buf[0] = 0;
	144	DataInfo DI;
	145	if (!Symbolizer::GetOrInit()->SymbolizeData(data_addr, &DI)) return;
	146	InternalScopedString data_desc(GetPageSizeCached());
	147	RenderData(&data_desc, fmt, &DI, common_flags()->strip_path_prefix);
	148	internal_strncpy(out_buf, data_desc.data(), out_buf_size);
	149	out_buf[out_buf_size - 1] = 0;
	150	}
	151
	152	SANITIZER_INTERFACE_ATTRIBUTE
	153	int __sanitizer_get_module_and_offset_for_pc(uptr pc, char *module_name,
	154	uptr module_name_len,
	155	uptr *pc_offset) {
	156	return __sanitizer::GetModuleAndOffsetForPc(pc, module_name, module_name_len,
	157	pc_offset);
	158	}
	159	} // extern "C"

lib/tsan/sanitizer_common/sanitizer_stacktrace_printer.cpp created+263

...	...	@@ -0,0 +1,263 @@
	1	//===-- sanitizer_common.cpp ----------------------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file is shared between sanitizers' run-time libraries.
	10	//
	11	//===----------------------------------------------------------------------===//
	12
	13	#include "sanitizer_stacktrace_printer.h"
	14	#include "sanitizer_file.h"
	15	#include "sanitizer_fuchsia.h"
	16
	17	namespace __sanitizer {
	18
	19	// sanitizer_symbolizer_markup.cpp implements these differently.
	20	#if !SANITIZER_SYMBOLIZER_MARKUP
	21
	22	static const char StripFunctionName(const char function, const char *prefix) {
	23	if (!function) return nullptr;
	24	if (!prefix) return function;
	25	uptr prefix_len = internal_strlen(prefix);
	26	if (0 == internal_strncmp(function, prefix, prefix_len))
	27	return function + prefix_len;
	28	return function;
	29	}
	30
	31	static const char DemangleFunctionName(const char function) {
	32	if (!function) return nullptr;
	33
	34	// NetBSD uses indirection for old threading functions for historical reasons
	35	// The mangled names are internal implementation detail and should not be
	36	// exposed even in backtraces.
	37	#if SANITIZER_NETBSD
	38	if (!internal_strcmp(function, "__libc_mutex_init"))
	39	return "pthread_mutex_init";
	40	if (!internal_strcmp(function, "__libc_mutex_lock"))
	41	return "pthread_mutex_lock";
	42	if (!internal_strcmp(function, "__libc_mutex_trylock"))
	43	return "pthread_mutex_trylock";
	44	if (!internal_strcmp(function, "__libc_mutex_unlock"))
	45	return "pthread_mutex_unlock";
	46	if (!internal_strcmp(function, "__libc_mutex_destroy"))
	47	return "pthread_mutex_destroy";
	48	if (!internal_strcmp(function, "__libc_mutexattr_init"))
	49	return "pthread_mutexattr_init";
	50	if (!internal_strcmp(function, "__libc_mutexattr_settype"))
	51	return "pthread_mutexattr_settype";
	52	if (!internal_strcmp(function, "__libc_mutexattr_destroy"))
	53	return "pthread_mutexattr_destroy";
	54	if (!internal_strcmp(function, "__libc_cond_init"))
	55	return "pthread_cond_init";
	56	if (!internal_strcmp(function, "__libc_cond_signal"))
	57	return "pthread_cond_signal";
	58	if (!internal_strcmp(function, "__libc_cond_broadcast"))
	59	return "pthread_cond_broadcast";
	60	if (!internal_strcmp(function, "__libc_cond_wait"))
	61	return "pthread_cond_wait";
	62	if (!internal_strcmp(function, "__libc_cond_timedwait"))
	63	return "pthread_cond_timedwait";
	64	if (!internal_strcmp(function, "__libc_cond_destroy"))
	65	return "pthread_cond_destroy";
	66	if (!internal_strcmp(function, "__libc_rwlock_init"))
	67	return "pthread_rwlock_init";
	68	if (!internal_strcmp(function, "__libc_rwlock_rdlock"))
	69	return "pthread_rwlock_rdlock";
	70	if (!internal_strcmp(function, "__libc_rwlock_wrlock"))
	71	return "pthread_rwlock_wrlock";
	72	if (!internal_strcmp(function, "__libc_rwlock_tryrdlock"))
	73	return "pthread_rwlock_tryrdlock";
	74	if (!internal_strcmp(function, "__libc_rwlock_trywrlock"))
	75	return "pthread_rwlock_trywrlock";
	76	if (!internal_strcmp(function, "__libc_rwlock_unlock"))
	77	return "pthread_rwlock_unlock";
	78	if (!internal_strcmp(function, "__libc_rwlock_destroy"))
	79	return "pthread_rwlock_destroy";
	80	if (!internal_strcmp(function, "__libc_thr_keycreate"))
	81	return "pthread_key_create";
	82	if (!internal_strcmp(function, "__libc_thr_setspecific"))
	83	return "pthread_setspecific";
	84	if (!internal_strcmp(function, "__libc_thr_getspecific"))
	85	return "pthread_getspecific";
	86	if (!internal_strcmp(function, "__libc_thr_keydelete"))
	87	return "pthread_key_delete";
	88	if (!internal_strcmp(function, "__libc_thr_once"))
	89	return "pthread_once";
	90	if (!internal_strcmp(function, "__libc_thr_self"))
	91	return "pthread_self";
	92	if (!internal_strcmp(function, "__libc_thr_exit"))
	93	return "pthread_exit";
	94	if (!internal_strcmp(function, "__libc_thr_setcancelstate"))
	95	return "pthread_setcancelstate";
	96	if (!internal_strcmp(function, "__libc_thr_equal"))
	97	return "pthread_equal";
	98	if (!internal_strcmp(function, "__libc_thr_curcpu"))
	99	return "pthread_curcpu_np";
	100	if (!internal_strcmp(function, "__libc_thr_sigsetmask"))
	101	return "pthread_sigmask";
	102	#endif
	103
	104	return function;
	105	}
	106
	107	static const char kDefaultFormat[] = " #%n %p %F %L";
	108
	109	void RenderFrame(InternalScopedString buffer, const char format, int frame_no,
	110	const AddressInfo &info, bool vs_style,
	111	const char strip_path_prefix, const char strip_func_prefix) {
	112	if (0 == internal_strcmp(format, "DEFAULT"))
	113	format = kDefaultFormat;
	114	for (const char p = format; p != '\0'; p++) {
	115	if (*p != '%') {
	116	buffer->append("%c", *p);
	117	continue;
	118	}
	119	p++;
	120	switch (*p) {
	121	case '%':
	122	buffer->append("%%");
	123	break;
	124	// Frame number and all fields of AddressInfo structure.
	125	case 'n':
	126	buffer->append("%zu", frame_no);
	127	break;
	128	case 'p':
	129	buffer->append("0x%zx", info.address);
	130	break;
	131	case 'm':
	132	buffer->append("%s", StripPathPrefix(info.module, strip_path_prefix));
	133	break;
	134	case 'o':
	135	buffer->append("0x%zx", info.module_offset);
	136	break;
	137	case 'f':
	138	buffer->append("%s",
	139	DemangleFunctionName(
	140	StripFunctionName(info.function, strip_func_prefix)));
	141	break;
	142	case 'q':
	143	buffer->append("0x%zx", info.function_offset != AddressInfo::kUnknown
	144	? info.function_offset
	145	: 0x0);
	146	break;
	147	case 's':
	148	buffer->append("%s", StripPathPrefix(info.file, strip_path_prefix));
	149	break;
	150	case 'l':
	151	buffer->append("%d", info.line);
	152	break;
	153	case 'c':
	154	buffer->append("%d", info.column);
	155	break;
	156	// Smarter special cases.
	157	case 'F':
	158	// Function name and offset, if file is unknown.
	159	if (info.function) {
	160	buffer->append("in %s",
	161	DemangleFunctionName(
	162	StripFunctionName(info.function, strip_func_prefix)));
	163	if (!info.file && info.function_offset != AddressInfo::kUnknown)
	164	buffer->append("+0x%zx", info.function_offset);
	165	}
	166	break;
	167	case 'S':
	168	// File/line information.
	169	RenderSourceLocation(buffer, info.file, info.line, info.column, vs_style,
	170	strip_path_prefix);
	171	break;
	172	case 'L':
	173	// Source location, or module location.
	174	if (info.file) {
	175	RenderSourceLocation(buffer, info.file, info.line, info.column,
	176	vs_style, strip_path_prefix);
	177	} else if (info.module) {
	178	RenderModuleLocation(buffer, info.module, info.module_offset,
	179	info.module_arch, strip_path_prefix);
	180	} else {
	181	buffer->append("(<unknown module>)");
	182	}
	183	break;
	184	case 'M':
	185	// Module basename and offset, or PC.
	186	if (info.address & kExternalPCBit)
	187	{} // There PCs are not meaningful.
	188	else if (info.module)
	189	// Always strip the module name for %M.
	190	RenderModuleLocation(buffer, StripModuleName(info.module),
	191	info.module_offset, info.module_arch, "");
	192	else
	193	buffer->append("(%p)", (void *)info.address);
	194	break;
	195	default:
	196	Report("Unsupported specifier in stack frame format: %c (0x%zx)!\n", *p,
	197	*p);
	198	Die();
	199	}
	200	}
	201	}
	202
	203	void RenderData(InternalScopedString buffer, const char format,
	204	const DataInfo DI, const char strip_path_prefix) {
	205	for (const char p = format; p != '\0'; p++) {
	206	if (*p != '%') {
	207	buffer->append("%c", *p);
	208	continue;
	209	}
	210	p++;
	211	switch (*p) {
	212	case '%':
	213	buffer->append("%%");
	214	break;
	215	case 's':
	216	buffer->append("%s", StripPathPrefix(DI->file, strip_path_prefix));
	217	break;
	218	case 'l':
	219	buffer->append("%d", DI->line);
	220	break;
	221	case 'g':
	222	buffer->append("%s", DI->name);
	223	break;
	224	default:
	225	Report("Unsupported specifier in stack frame format: %c (0x%zx)!\n", *p,
	226	*p);
	227	Die();
	228	}
	229	}
	230	}
	231
	232	#endif // !SANITIZER_SYMBOLIZER_MARKUP
	233
	234	void RenderSourceLocation(InternalScopedString buffer, const char file,
	235	int line, int column, bool vs_style,
	236	const char *strip_path_prefix) {
	237	if (vs_style && line > 0) {
	238	buffer->append("%s(%d", StripPathPrefix(file, strip_path_prefix), line);
	239	if (column > 0)
	240	buffer->append(",%d", column);
	241	buffer->append(")");
	242	return;
	243	}
	244
	245	buffer->append("%s", StripPathPrefix(file, strip_path_prefix));
	246	if (line > 0) {
	247	buffer->append(":%d", line);
	248	if (column > 0)
	249	buffer->append(":%d", column);
	250	}
	251	}
	252
	253	void RenderModuleLocation(InternalScopedString buffer, const char module,
	254	uptr offset, ModuleArch arch,
	255	const char *strip_path_prefix) {
	256	buffer->append("(%s", StripPathPrefix(module, strip_path_prefix));
	257	if (arch != kModuleArchUnknown) {
	258	buffer->append(":%s", ModuleArchToString(arch));
	259	}
	260	buffer->append("+0x%zx)", offset);
	261	}
	262
	263	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_stacktrace_sparc.cpp created+85

...	...	@@ -0,0 +1,85 @@
	1	//===-- sanitizer_stacktrace_sparc.cpp ------------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file is shared between AddressSanitizer and ThreadSanitizer
	10	// run-time libraries.
	11	//
	12	// Implemention of fast stack unwinding for Sparc.
	13	//===----------------------------------------------------------------------===//
	14
	15	#if defined(__sparc__)
	16
	17	#if defined(__arch64__) \|\| defined(__sparcv9)
	18	#define STACK_BIAS 2047
	19	#else
	20	#define STACK_BIAS 0
	21	#endif
	22
	23	#include "sanitizer_common.h"
	24	#include "sanitizer_stacktrace.h"
	25
	26	namespace __sanitizer {
	27
	28	void BufferedStackTrace::UnwindFast(uptr pc, uptr bp, uptr stack_top,
	29	uptr stack_bottom, u32 max_depth) {
	30	// TODO(yln): add arg sanity check for stack_top/stack_bottom
	31	CHECK_GE(max_depth, 2);
	32	const uptr kPageSize = GetPageSizeCached();
	33	#if defined(__GNUC__)
	34	// __builtin_return_address returns the address of the call instruction
	35	// on the SPARC and not the return address, so we need to compensate.
	36	trace_buffer[0] = GetNextInstructionPc(pc);
	37	#else
	38	trace_buffer[0] = pc;
	39	#endif
	40	size = 1;
	41	if (stack_top < 4096) return; // Sanity check for stack top.
	42	// Flush register windows to memory
	43	#if defined(__sparc_v9__) \|\| defined(__sparcv9__) \|\| defined(__sparcv9)
	44	asm volatile("flushw" ::: "memory");
	45	#else
	46	asm volatile("ta 3" ::: "memory");
	47	#endif
	48	// On the SPARC, the return address is not in the frame, it is in a
	49	// register. There is no way to access it off of the current frame
	50	// pointer, but it can be accessed off the previous frame pointer by
	51	// reading the value from the register window save area.
	52	uptr prev_bp = GET_CURRENT_FRAME();
	53	uptr next_bp = prev_bp;
	54	unsigned int i = 0;
	55	while (next_bp != bp && IsAligned(next_bp, sizeof(uhwptr)) && i++ < 8) {
	56	prev_bp = next_bp;
	57	next_bp = (uptr)((uhwptr *)next_bp)[14] + STACK_BIAS;
	58	}
	59	if (next_bp == bp)
	60	bp = prev_bp;
	61	// Lowest possible address that makes sense as the next frame pointer.
	62	// Goes up as we walk the stack.
	63	uptr bottom = stack_bottom;
	64	// Avoid infinite loop when frame == frame[0] by using frame > prev_frame.
	65	while (IsValidFrame(bp, stack_top, bottom) && IsAligned(bp, sizeof(uhwptr)) &&
	66	size < max_depth) {
	67	uhwptr pc1 = ((uhwptr *)bp)[15];
	68	// Let's assume that any pointer in the 0th page is invalid and
	69	// stop unwinding here. If we're adding support for a platform
	70	// where this isn't true, we need to reconsider this check.
	71	if (pc1 < kPageSize)
	72	break;
	73	if (pc1 != pc) {
	74	// %o7 contains the address of the call instruction and not the
	75	// return address, so we need to compensate.
	76	trace_buffer[size++] = GetNextInstructionPc((uptr)pc1);
	77	}
	78	bottom = bp;
	79	bp = (uptr)((uhwptr *)bp)[14] + STACK_BIAS;
	80	}
	81	}
	82
	83	} // namespace __sanitizer
	84
	85	#endif // !defined(__sparc__)

lib/tsan/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cpp created+573

...	...	@@ -0,0 +1,573 @@
	1	//===-- sanitizer_stoptheworld_linux_libcdep.cpp --------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// See sanitizer_stoptheworld.h for details.
	10	// This implementation was inspired by Markus Gutschke's linuxthreads.cc.
	11	//
	12	//===----------------------------------------------------------------------===//
	13
	14	#include "sanitizer_platform.h"
	15
	16	#if SANITIZER_LINUX && (defined(__x86_64__) \|\| defined(__mips__) \|\| \
	17	defined(__aarch64__) \|\| defined(__powerpc64__) \|\| \
	18	defined(__s390__) \|\| defined(__i386__) \|\| \
	19	defined(__arm__))
	20
	21	#include "sanitizer_stoptheworld.h"
	22
	23	#include "sanitizer_platform_limits_posix.h"
	24	#include "sanitizer_atomic.h"
	25
	26	#include <errno.h>
	27	#include <sched.h> // for CLONE_* definitions
	28	#include <stddef.h>
	29	#include <sys/prctl.h> // for PR_* definitions
	30	#include <sys/ptrace.h> // for PTRACE_* definitions
	31	#include <sys/types.h> // for pid_t
	32	#include <sys/uio.h> // for iovec
	33	#include <elf.h> // for NT_PRSTATUS
	34	#if defined(__aarch64__) && !SANITIZER_ANDROID
	35	// GLIBC 2.20+ sys/user does not include asm/ptrace.h
	36	# include <asm/ptrace.h>
	37	#endif
	38	#include <sys/user.h> // for user_regs_struct
	39	#if SANITIZER_ANDROID && SANITIZER_MIPS
	40	# include <asm/reg.h> // for mips SP register in sys/user.h
	41	#endif
	42	#include <sys/wait.h> // for signal-related stuff
	43
	44	#ifdef sa_handler
	45	# undef sa_handler
	46	#endif
	47
	48	#ifdef sa_sigaction
	49	# undef sa_sigaction
	50	#endif
	51
	52	#include "sanitizer_common.h"
	53	#include "sanitizer_flags.h"
	54	#include "sanitizer_libc.h"
	55	#include "sanitizer_linux.h"
	56	#include "sanitizer_mutex.h"
	57	#include "sanitizer_placement_new.h"
	58
	59	// Sufficiently old kernel headers don't provide this value, but we can still
	60	// call prctl with it. If the runtime kernel is new enough, the prctl call will
	61	// have the desired effect; if the kernel is too old, the call will error and we
	62	// can ignore said error.
	63	#ifndef PR_SET_PTRACER
	64	#define PR_SET_PTRACER 0x59616d61
	65	#endif
	66
	67	// This module works by spawning a Linux task which then attaches to every
	68	// thread in the caller process with ptrace. This suspends the threads, and
	69	// PTRACE_GETREGS can then be used to obtain their register state. The callback
	70	// supplied to StopTheWorld() is run in the tracer task while the threads are
	71	// suspended.
	72	// The tracer task must be placed in a different thread group for ptrace to
	73	// work, so it cannot be spawned as a pthread. Instead, we use the low-level
	74	// clone() interface (we want to share the address space with the caller
	75	// process, so we prefer clone() over fork()).
	76	//
	77	// We don't use any libc functions, relying instead on direct syscalls. There
	78	// are two reasons for this:
	79	// 1. calling a library function while threads are suspended could cause a
	80	// deadlock, if one of the treads happens to be holding a libc lock;
	81	// 2. it's generally not safe to call libc functions from the tracer task,
	82	// because clone() does not set up a thread-local storage for it. Any
	83	// thread-local variables used by libc will be shared between the tracer task
	84	// and the thread which spawned it.
	85
	86	namespace __sanitizer {
	87
	88	class SuspendedThreadsListLinux : public SuspendedThreadsList {
	89	public:
	90	SuspendedThreadsListLinux() { thread_ids_.reserve(1024); }
	91
	92	tid_t GetThreadID(uptr index) const;
	93	uptr ThreadCount() const;
	94	bool ContainsTid(tid_t thread_id) const;
	95	void Append(tid_t tid);
	96
	97	PtraceRegistersStatus GetRegistersAndSP(uptr index, uptr *buffer,
	98	uptr *sp) const;
	99	uptr RegisterCount() const;
	100
	101	private:
	102	InternalMmapVector<tid_t> thread_ids_;
	103	};
	104
	105	// Structure for passing arguments into the tracer thread.
	106	struct TracerThreadArgument {
	107	StopTheWorldCallback callback;
	108	void *callback_argument;
	109	// The tracer thread waits on this mutex while the parent finishes its
	110	// preparations.
	111	BlockingMutex mutex;
	112	// Tracer thread signals its completion by setting done.
	113	atomic_uintptr_t done;
	114	uptr parent_pid;
	115	};
	116
	117	// This class handles thread suspending/unsuspending in the tracer thread.
	118	class ThreadSuspender {
	119	public:
	120	explicit ThreadSuspender(pid_t pid, TracerThreadArgument *arg)
	121	: arg(arg)
	122	, pid_(pid) {
	123	CHECK_GE(pid, 0);
	124	}
	125	bool SuspendAllThreads();
	126	void ResumeAllThreads();
	127	void KillAllThreads();
	128	SuspendedThreadsListLinux &suspended_threads_list() {
	129	return suspended_threads_list_;
	130	}
	131	TracerThreadArgument *arg;
	132	private:
	133	SuspendedThreadsListLinux suspended_threads_list_;
	134	pid_t pid_;
	135	bool SuspendThread(tid_t thread_id);
	136	};
	137
	138	bool ThreadSuspender::SuspendThread(tid_t tid) {
	139	// Are we already attached to this thread?
	140	// Currently this check takes linear time, however the number of threads is
	141	// usually small.
	142	if (suspended_threads_list_.ContainsTid(tid)) return false;
	143	int pterrno;
	144	if (internal_iserror(internal_ptrace(PTRACE_ATTACH, tid, nullptr, nullptr),
	145	&pterrno)) {
	146	// Either the thread is dead, or something prevented us from attaching.
	147	// Log this event and move on.
	148	VReport(1, "Could not attach to thread %zu (errno %d).\n", (uptr)tid,
	149	pterrno);
	150	return false;
	151	} else {
	152	VReport(2, "Attached to thread %zu.\n", (uptr)tid);
	153	// The thread is not guaranteed to stop before ptrace returns, so we must
	154	// wait on it. Note: if the thread receives a signal concurrently,
	155	// we can get notification about the signal before notification about stop.
	156	// In such case we need to forward the signal to the thread, otherwise
	157	// the signal will be missed (as we do PTRACE_DETACH with arg=0) and
	158	// any logic relying on signals will break. After forwarding we need to
	159	// continue to wait for stopping, because the thread is not stopped yet.
	160	// We do ignore delivery of SIGSTOP, because we want to make stop-the-world
	161	// as invisible as possible.
	162	for (;;) {
	163	int status;
	164	uptr waitpid_status;
	165	HANDLE_EINTR(waitpid_status, internal_waitpid(tid, &status, __WALL));
	166	int wperrno;
	167	if (internal_iserror(waitpid_status, &wperrno)) {
	168	// Got a ECHILD error. I don't think this situation is possible, but it
	169	// doesn't hurt to report it.
	170	VReport(1, "Waiting on thread %zu failed, detaching (errno %d).\n",
	171	(uptr)tid, wperrno);
	172	internal_ptrace(PTRACE_DETACH, tid, nullptr, nullptr);
	173	return false;
	174	}
	175	if (WIFSTOPPED(status) && WSTOPSIG(status) != SIGSTOP) {
	176	internal_ptrace(PTRACE_CONT, tid, nullptr,
	177	(void*)(uptr)WSTOPSIG(status));
	178	continue;
	179	}
	180	break;
	181	}
	182	suspended_threads_list_.Append(tid);
	183	return true;
	184	}
	185	}
	186
	187	void ThreadSuspender::ResumeAllThreads() {
	188	for (uptr i = 0; i < suspended_threads_list_.ThreadCount(); i++) {
	189	pid_t tid = suspended_threads_list_.GetThreadID(i);
	190	int pterrno;
	191	if (!internal_iserror(internal_ptrace(PTRACE_DETACH, tid, nullptr, nullptr),
	192	&pterrno)) {
	193	VReport(2, "Detached from thread %d.\n", tid);
	194	} else {
	195	// Either the thread is dead, or we are already detached.
	196	// The latter case is possible, for instance, if this function was called
	197	// from a signal handler.
	198	VReport(1, "Could not detach from thread %d (errno %d).\n", tid, pterrno);
	199	}
	200	}
	201	}
	202
	203	void ThreadSuspender::KillAllThreads() {
	204	for (uptr i = 0; i < suspended_threads_list_.ThreadCount(); i++)
	205	internal_ptrace(PTRACE_KILL, suspended_threads_list_.GetThreadID(i),
	206	nullptr, nullptr);
	207	}
	208
	209	bool ThreadSuspender::SuspendAllThreads() {
	210	ThreadLister thread_lister(pid_);
	211	bool retry = true;
	212	InternalMmapVector<tid_t> threads;
	213	threads.reserve(128);
	214	for (int i = 0; i < 30 && retry; ++i) {
	215	retry = false;
	216	switch (thread_lister.ListThreads(&threads)) {
	217	case ThreadLister::Error:
	218	ResumeAllThreads();
	219	return false;
	220	case ThreadLister::Incomplete:
	221	retry = true;
	222	break;
	223	case ThreadLister::Ok:
	224	break;
	225	}
	226	for (tid_t tid : threads) {
	227	if (SuspendThread(tid))
	228	retry = true;
	229	}
	230	}
	231	return suspended_threads_list_.ThreadCount();
	232	}
	233
	234	// Pointer to the ThreadSuspender instance for use in signal handler.
	235	static ThreadSuspender *thread_suspender_instance = nullptr;
	236
	237	// Synchronous signals that should not be blocked.
	238	static const int kSyncSignals[] = { SIGABRT, SIGILL, SIGFPE, SIGSEGV, SIGBUS,
	239	SIGXCPU, SIGXFSZ };
	240
	241	static void TracerThreadDieCallback() {
	242	// Generally a call to Die() in the tracer thread should be fatal to the
	243	// parent process as well, because they share the address space.
	244	// This really only works correctly if all the threads are suspended at this
	245	// point. So we correctly handle calls to Die() from within the callback, but
	246	// not those that happen before or after the callback. Hopefully there aren't
	247	// a lot of opportunities for that to happen...
	248	ThreadSuspender *inst = thread_suspender_instance;
	249	if (inst && stoptheworld_tracer_pid == internal_getpid()) {
	250	inst->KillAllThreads();
	251	thread_suspender_instance = nullptr;
	252	}
	253	}
	254
	255	// Signal handler to wake up suspended threads when the tracer thread dies.
	256	static void TracerThreadSignalHandler(int signum, __sanitizer_siginfo *siginfo,
	257	void *uctx) {
	258	SignalContext ctx(siginfo, uctx);
	259	Printf("Tracer caught signal %d: addr=0x%zx pc=0x%zx sp=0x%zx\n", signum,
	260	ctx.addr, ctx.pc, ctx.sp);
	261	ThreadSuspender *inst = thread_suspender_instance;
	262	if (inst) {
	263	if (signum == SIGABRT)
	264	inst->KillAllThreads();
	265	else
	266	inst->ResumeAllThreads();
	267	RAW_CHECK(RemoveDieCallback(TracerThreadDieCallback));
	268	thread_suspender_instance = nullptr;
	269	atomic_store(&inst->arg->done, 1, memory_order_relaxed);
	270	}
	271	internal__exit((signum == SIGABRT) ? 1 : 2);
	272	}
	273
	274	// Size of alternative stack for signal handlers in the tracer thread.
	275	static const int kHandlerStackSize = 8192;
	276
	277	// This function will be run as a cloned task.
	278	static int TracerThread(void* argument) {
	279	TracerThreadArgument *tracer_thread_argument =
	280	(TracerThreadArgument *)argument;
	281
	282	internal_prctl(PR_SET_PDEATHSIG, SIGKILL, 0, 0, 0);
	283	// Check if parent is already dead.
	284	if (internal_getppid() != tracer_thread_argument->parent_pid)
	285	internal__exit(4);
	286
	287	// Wait for the parent thread to finish preparations.
	288	tracer_thread_argument->mutex.Lock();
	289	tracer_thread_argument->mutex.Unlock();
	290
	291	RAW_CHECK(AddDieCallback(TracerThreadDieCallback));
	292
	293	ThreadSuspender thread_suspender(internal_getppid(), tracer_thread_argument);
	294	// Global pointer for the signal handler.
	295	thread_suspender_instance = &thread_suspender;
	296
	297	// Alternate stack for signal handling.
	298	InternalMmapVector<char> handler_stack_memory(kHandlerStackSize);
	299	stack_t handler_stack;
	300	internal_memset(&handler_stack, 0, sizeof(handler_stack));
	301	handler_stack.ss_sp = handler_stack_memory.data();
	302	handler_stack.ss_size = kHandlerStackSize;
	303	internal_sigaltstack(&handler_stack, nullptr);
	304
	305	// Install our handler for synchronous signals. Other signals should be
	306	// blocked by the mask we inherited from the parent thread.
	307	for (uptr i = 0; i < ARRAY_SIZE(kSyncSignals); i++) {
	308	__sanitizer_sigaction act;
	309	internal_memset(&act, 0, sizeof(act));
	310	act.sigaction = TracerThreadSignalHandler;
	311	act.sa_flags = SA_ONSTACK \| SA_SIGINFO;
	312	internal_sigaction_norestorer(kSyncSignals[i], &act, 0);
	313	}
	314
	315	int exit_code = 0;
	316	if (!thread_suspender.SuspendAllThreads()) {
	317	VReport(1, "Failed suspending threads.\n");
	318	exit_code = 3;
	319	} else {
	320	tracer_thread_argument->callback(thread_suspender.suspended_threads_list(),
	321	tracer_thread_argument->callback_argument);
	322	thread_suspender.ResumeAllThreads();
	323	exit_code = 0;
	324	}
	325	RAW_CHECK(RemoveDieCallback(TracerThreadDieCallback));
	326	thread_suspender_instance = nullptr;
	327	atomic_store(&tracer_thread_argument->done, 1, memory_order_relaxed);
	328	return exit_code;
	329	}
	330
	331	class ScopedStackSpaceWithGuard {
	332	public:
	333	explicit ScopedStackSpaceWithGuard(uptr stack_size) {
	334	stack_size_ = stack_size;
	335	guard_size_ = GetPageSizeCached();
	336	// FIXME: Omitting MAP_STACK here works in current kernels but might break
	337	// in the future.
	338	guard_start_ = (uptr)MmapOrDie(stack_size_ + guard_size_,
	339	"ScopedStackWithGuard");
	340	CHECK(MprotectNoAccess((uptr)guard_start_, guard_size_));
	341	}
	342	~ScopedStackSpaceWithGuard() {
	343	UnmapOrDie((void *)guard_start_, stack_size_ + guard_size_);
	344	}
	345	void *Bottom() const {
	346	return (void *)(guard_start_ + stack_size_ + guard_size_);
	347	}
	348
	349	private:
	350	uptr stack_size_;
	351	uptr guard_size_;
	352	uptr guard_start_;
	353	};
	354
	355	// We have a limitation on the stack frame size, so some stuff had to be moved
	356	// into globals.
	357	static __sanitizer_sigset_t blocked_sigset;
	358	static __sanitizer_sigset_t old_sigset;
	359
	360	class StopTheWorldScope {
	361	public:
	362	StopTheWorldScope() {
	363	// Make this process dumpable. Processes that are not dumpable cannot be
	364	// attached to.
	365	process_was_dumpable_ = internal_prctl(PR_GET_DUMPABLE, 0, 0, 0, 0);
	366	if (!process_was_dumpable_)
	367	internal_prctl(PR_SET_DUMPABLE, 1, 0, 0, 0);
	368	}
	369
	370	~StopTheWorldScope() {
	371	// Restore the dumpable flag.
	372	if (!process_was_dumpable_)
	373	internal_prctl(PR_SET_DUMPABLE, 0, 0, 0, 0);
	374	}
	375
	376	private:
	377	int process_was_dumpable_;
	378	};
	379
	380	// When sanitizer output is being redirected to file (i.e. by using log_path),
	381	// the tracer should write to the parent's log instead of trying to open a new
	382	// file. Alert the logging code to the fact that we have a tracer.
	383	struct ScopedSetTracerPID {
	384	explicit ScopedSetTracerPID(uptr tracer_pid) {
	385	stoptheworld_tracer_pid = tracer_pid;
	386	stoptheworld_tracer_ppid = internal_getpid();
	387	}
	388	~ScopedSetTracerPID() {
	389	stoptheworld_tracer_pid = 0;
	390	stoptheworld_tracer_ppid = 0;
	391	}
	392	};
	393
	394	void StopTheWorld(StopTheWorldCallback callback, void *argument) {
	395	StopTheWorldScope in_stoptheworld;
	396	// Prepare the arguments for TracerThread.
	397	struct TracerThreadArgument tracer_thread_argument;
	398	tracer_thread_argument.callback = callback;
	399	tracer_thread_argument.callback_argument = argument;
	400	tracer_thread_argument.parent_pid = internal_getpid();
	401	atomic_store(&tracer_thread_argument.done, 0, memory_order_relaxed);
	402	const uptr kTracerStackSize = 2 * 1024 * 1024;
	403	ScopedStackSpaceWithGuard tracer_stack(kTracerStackSize);
	404	// Block the execution of TracerThread until after we have set ptrace
	405	// permissions.
	406	tracer_thread_argument.mutex.Lock();
	407	// Signal handling story.
	408	// We don't want async signals to be delivered to the tracer thread,
	409	// so we block all async signals before creating the thread. An async signal
	410	// handler can temporary modify errno, which is shared with this thread.
	411	// We ought to use pthread_sigmask here, because sigprocmask has undefined
	412	// behavior in multithreaded programs. However, on linux sigprocmask is
	413	// equivalent to pthread_sigmask with the exception that pthread_sigmask
	414	// does not allow to block some signals used internally in pthread
	415	// implementation. We are fine with blocking them here, we are really not
	416	// going to pthread_cancel the thread.
	417	// The tracer thread should not raise any synchronous signals. But in case it
	418	// does, we setup a special handler for sync signals that properly kills the
	419	// parent as well. Note: we don't pass CLONE_SIGHAND to clone, so handlers
	420	// in the tracer thread won't interfere with user program. Double note: if a
	421	// user does something along the lines of 'kill -11 pid', that can kill the
	422	// process even if user setup own handler for SEGV.
	423	// Thing to watch out for: this code should not change behavior of user code
	424	// in any observable way. In particular it should not override user signal
	425	// handlers.
	426	internal_sigfillset(&blocked_sigset);
	427	for (uptr i = 0; i < ARRAY_SIZE(kSyncSignals); i++)
	428	internal_sigdelset(&blocked_sigset, kSyncSignals[i]);
	429	int rv = internal_sigprocmask(SIG_BLOCK, &blocked_sigset, &old_sigset);
	430	CHECK_EQ(rv, 0);
	431	uptr tracer_pid = internal_clone(
	432	TracerThread, tracer_stack.Bottom(),
	433	CLONE_VM \| CLONE_FS \| CLONE_FILES \| CLONE_UNTRACED,
	434	&tracer_thread_argument, nullptr /* parent_tidptr */,
	435	nullptr /* newtls /, nullptr / child_tidptr */);
	436	internal_sigprocmask(SIG_SETMASK, &old_sigset, 0);
	437	int local_errno = 0;
	438	if (internal_iserror(tracer_pid, &local_errno)) {
	439	VReport(1, "Failed spawning a tracer thread (errno %d).\n", local_errno);
	440	tracer_thread_argument.mutex.Unlock();
	441	} else {
	442	ScopedSetTracerPID scoped_set_tracer_pid(tracer_pid);
	443	// On some systems we have to explicitly declare that we want to be traced
	444	// by the tracer thread.
	445	internal_prctl(PR_SET_PTRACER, tracer_pid, 0, 0, 0);
	446	// Allow the tracer thread to start.
	447	tracer_thread_argument.mutex.Unlock();
	448	// NOTE: errno is shared between this thread and the tracer thread.
	449	// internal_waitpid() may call syscall() which can access/spoil errno,
	450	// so we can't call it now. Instead we for the tracer thread to finish using
	451	// the spin loop below. Man page for sched_yield() says "In the Linux
	452	// implementation, sched_yield() always succeeds", so let's hope it does not
	453	// spoil errno. Note that this spin loop runs only for brief periods before
	454	// the tracer thread has suspended us and when it starts unblocking threads.
	455	while (atomic_load(&tracer_thread_argument.done, memory_order_relaxed) == 0)
	456	sched_yield();
	457	// Now the tracer thread is about to exit and does not touch errno,
	458	// wait for it.
	459	for (;;) {
	460	uptr waitpid_status = internal_waitpid(tracer_pid, nullptr, __WALL);
	461	if (!internal_iserror(waitpid_status, &local_errno))
	462	break;
	463	if (local_errno == EINTR)
	464	continue;
	465	VReport(1, "Waiting on the tracer thread failed (errno %d).\n",
	466	local_errno);
	467	break;
	468	}
	469	}
	470	}
	471
	472	// Platform-specific methods from SuspendedThreadsList.
	473	#if SANITIZER_ANDROID && defined(__arm__)
	474	typedef pt_regs regs_struct;
	475	#define REG_SP ARM_sp
	476
	477	#elif SANITIZER_LINUX && defined(__arm__)
	478	typedef user_regs regs_struct;
	479	#define REG_SP uregs[13]
	480
	481	#elif defined(__i386__) \|\| defined(__x86_64__)
	482	typedef user_regs_struct regs_struct;
	483	#if defined(__i386__)
	484	#define REG_SP esp
	485	#else
	486	#define REG_SP rsp
	487	#endif
	488
	489	#elif defined(__powerpc__) \|\| defined(__powerpc64__)
	490	typedef pt_regs regs_struct;
	491	#define REG_SP gpr[PT_R1]
	492
	493	#elif defined(__mips__)
	494	typedef struct user regs_struct;
	495	# if SANITIZER_ANDROID
	496	# define REG_SP regs[EF_R29]
	497	# else
	498	# define REG_SP regs[EF_REG29]
	499	# endif
	500
	501	#elif defined(__aarch64__)
	502	typedef struct user_pt_regs regs_struct;
	503	#define REG_SP sp
	504	#define ARCH_IOVEC_FOR_GETREGSET
	505
	506	#elif defined(__s390__)
	507	typedef _user_regs_struct regs_struct;
	508	#define REG_SP gprs[15]
	509	#define ARCH_IOVEC_FOR_GETREGSET
	510
	511	#else
	512	#error "Unsupported architecture"
	513	#endif // SANITIZER_ANDROID && defined(__arm__)
	514
	515	tid_t SuspendedThreadsListLinux::GetThreadID(uptr index) const {
	516	CHECK_LT(index, thread_ids_.size());
	517	return thread_ids_[index];
	518	}
	519
	520	uptr SuspendedThreadsListLinux::ThreadCount() const {
	521	return thread_ids_.size();
	522	}
	523
	524	bool SuspendedThreadsListLinux::ContainsTid(tid_t thread_id) const {
	525	for (uptr i = 0; i < thread_ids_.size(); i++) {
	526	if (thread_ids_[i] == thread_id) return true;
	527	}
	528	return false;
	529	}
	530
	531	void SuspendedThreadsListLinux::Append(tid_t tid) {
	532	thread_ids_.push_back(tid);
	533	}
	534
	535	PtraceRegistersStatus SuspendedThreadsListLinux::GetRegistersAndSP(
	536	uptr index, uptr buffer, uptr sp) const {
	537	pid_t tid = GetThreadID(index);
	538	regs_struct regs;
	539	int pterrno;
	540	#ifdef ARCH_IOVEC_FOR_GETREGSET
	541	struct iovec regset_io;
	542	regset_io.iov_base = &regs;
	543	regset_io.iov_len = sizeof(regs_struct);
	544	bool isErr = internal_iserror(internal_ptrace(PTRACE_GETREGSET, tid,
	545	(void)NT_PRSTATUS, (void)&regset_io),
	546	&pterrno);
	547	#else
	548	bool isErr = internal_iserror(internal_ptrace(PTRACE_GETREGS, tid, nullptr,
	549	&regs), &pterrno);
	550	#endif
	551	if (isErr) {
	552	VReport(1, "Could not get registers from thread %d (errno %d).\n", tid,
	553	pterrno);
	554	// ESRCH means that the given thread is not suspended or already dead.
	555	// Therefore it's unsafe to inspect its data (e.g. walk through stack) and
	556	// we should notify caller about this.
	557	return pterrno == ESRCH ? REGISTERS_UNAVAILABLE_FATAL
	558	: REGISTERS_UNAVAILABLE;
	559	}
	560
	561	*sp = regs.REG_SP;
	562	internal_memcpy(buffer, &regs, sizeof(regs));
	563	return REGISTERS_AVAILABLE;
	564	}
	565
	566	uptr SuspendedThreadsListLinux::RegisterCount() const {
	567	return sizeof(regs_struct) / sizeof(uptr);
	568	}
	569	} // namespace __sanitizer
	570
	571	#endif // SANITIZER_LINUX && (defined(__x86_64__) \|\| defined(__mips__)
	572	// \|\| defined(__aarch64__) \|\| defined(__powerpc64__)
	573	// \|\| defined(__s390__) \|\| defined(__i386__) \|\| defined(__arm__)

lib/tsan/sanitizer_common/sanitizer_stoptheworld_netbsd_libcdep.cpp created+364

...	...	@@ -0,0 +1,364 @@
	1	//===-- sanitizer_stoptheworld_netbsd_libcdep.cpp -------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// See sanitizer_stoptheworld.h for details.
	10	// This implementation was inspired by Markus Gutschke's linuxthreads.cc.
	11	//
	12	// This is a NetBSD variation of Linux stoptheworld implementation
	13	// See sanitizer_stoptheworld_linux_libcdep.cpp for code comments.
	14	//
	15	//===----------------------------------------------------------------------===//
	16
	17	#include "sanitizer_platform.h"
	18
	19	#if SANITIZER_NETBSD
	20
	21	#include "sanitizer_stoptheworld.h"
	22
	23	#include "sanitizer_atomic.h"
	24	#include "sanitizer_platform_limits_posix.h"
	25
	26	#include <sys/types.h>
	27
	28	#include <sys/ptrace.h>
	29	#include <sys/uio.h>
	30	#include <sys/wait.h>
	31
	32	#include <machine/reg.h>
	33
	34	#include <elf.h>
	35	#include <errno.h>
	36	#include <sched.h>
	37	#include <signal.h>
	38	#include <stddef.h>
	39
	40	#define internal_sigaction_norestorer internal_sigaction
	41
	42	#include "sanitizer_common.h"
	43	#include "sanitizer_flags.h"
	44	#include "sanitizer_libc.h"
	45	#include "sanitizer_linux.h"
	46	#include "sanitizer_mutex.h"
	47	#include "sanitizer_placement_new.h"
	48
	49	namespace __sanitizer {
	50
	51	class SuspendedThreadsListNetBSD : public SuspendedThreadsList {
	52	public:
	53	SuspendedThreadsListNetBSD() { thread_ids_.reserve(1024); }
	54
	55	tid_t GetThreadID(uptr index) const;
	56	uptr ThreadCount() const;
	57	bool ContainsTid(tid_t thread_id) const;
	58	void Append(tid_t tid);
	59
	60	PtraceRegistersStatus GetRegistersAndSP(uptr index, uptr *buffer,
	61	uptr *sp) const;
	62	uptr RegisterCount() const;
	63
	64	private:
	65	InternalMmapVector<tid_t> thread_ids_;
	66	};
	67
	68	struct TracerThreadArgument {
	69	StopTheWorldCallback callback;
	70	void *callback_argument;
	71	BlockingMutex mutex;
	72	atomic_uintptr_t done;
	73	uptr parent_pid;
	74	};
	75
	76	class ThreadSuspender {
	77	public:
	78	explicit ThreadSuspender(pid_t pid, TracerThreadArgument *arg)
	79	: arg(arg), pid_(pid) {
	80	CHECK_GE(pid, 0);
	81	}
	82	bool SuspendAllThreads();
	83	void ResumeAllThreads();
	84	void KillAllThreads();
	85	SuspendedThreadsListNetBSD &suspended_threads_list() {
	86	return suspended_threads_list_;
	87	}
	88	TracerThreadArgument *arg;
	89
	90	private:
	91	SuspendedThreadsListNetBSD suspended_threads_list_;
	92	pid_t pid_;
	93	};
	94
	95	void ThreadSuspender::ResumeAllThreads() {
	96	int pterrno;
	97	if (!internal_iserror(internal_ptrace(PT_DETACH, pid_, (void *)(uptr)1, 0),
	98	&pterrno)) {
	99	VReport(2, "Detached from process %d.\n", pid_);
	100	} else {
	101	VReport(1, "Could not detach from process %d (errno %d).\n", pid_, pterrno);
	102	}
	103	}
	104
	105	void ThreadSuspender::KillAllThreads() {
	106	internal_ptrace(PT_KILL, pid_, nullptr, 0);
	107	}
	108
	109	bool ThreadSuspender::SuspendAllThreads() {
	110	int pterrno;
	111	if (internal_iserror(internal_ptrace(PT_ATTACH, pid_, nullptr, 0),
	112	&pterrno)) {
	113	Printf("Could not attach to process %d (errno %d).\n", pid_, pterrno);
	114	return false;
	115	}
	116
	117	int status;
	118	uptr waitpid_status;
	119	HANDLE_EINTR(waitpid_status, internal_waitpid(pid_, &status, 0));
	120
	121	VReport(2, "Attached to process %d.\n", pid_);
	122
	123	#ifdef PT_LWPNEXT
	124	struct ptrace_lwpstatus pl;
	125	int op = PT_LWPNEXT;
	126	#else
	127	struct ptrace_lwpinfo pl;
	128	int op = PT_LWPINFO;
	129	#endif
	130
	131	pl.pl_lwpid = 0;
	132
	133	int val;
	134	while ((val = ptrace(op, pid_, (void *)&pl, sizeof(pl))) != -1 &&
	135	pl.pl_lwpid != 0) {
	136	suspended_threads_list_.Append(pl.pl_lwpid);
	137	VReport(2, "Appended thread %d in process %d.\n", pl.pl_lwpid, pid_);
	138	}
	139	return true;
	140	}
	141
	142	// Pointer to the ThreadSuspender instance for use in signal handler.
	143	static ThreadSuspender *thread_suspender_instance = nullptr;
	144
	145	// Synchronous signals that should not be blocked.
	146	static const int kSyncSignals[] = {SIGABRT, SIGILL, SIGFPE, SIGSEGV,
	147	SIGBUS, SIGXCPU, SIGXFSZ};
	148
	149	static void TracerThreadDieCallback() {
	150	ThreadSuspender *inst = thread_suspender_instance;
	151	if (inst && stoptheworld_tracer_pid == internal_getpid()) {
	152	inst->KillAllThreads();
	153	thread_suspender_instance = nullptr;
	154	}
	155	}
	156
	157	// Signal handler to wake up suspended threads when the tracer thread dies.
	158	static void TracerThreadSignalHandler(int signum, __sanitizer_siginfo *siginfo,
	159	void *uctx) {
	160	SignalContext ctx(siginfo, uctx);
	161	Printf("Tracer caught signal %d: addr=0x%zx pc=0x%zx sp=0x%zx\n", signum,
	162	ctx.addr, ctx.pc, ctx.sp);
	163	ThreadSuspender *inst = thread_suspender_instance;
	164	if (inst) {
	165	if (signum == SIGABRT)
	166	inst->KillAllThreads();
	167	else
	168	inst->ResumeAllThreads();
	169	RAW_CHECK(RemoveDieCallback(TracerThreadDieCallback));
	170	thread_suspender_instance = nullptr;
	171	atomic_store(&inst->arg->done, 1, memory_order_relaxed);
	172	}
	173	internal__exit((signum == SIGABRT) ? 1 : 2);
	174	}
	175
	176	// Size of alternative stack for signal handlers in the tracer thread.
	177	static const int kHandlerStackSize = 8192;
	178
	179	// This function will be run as a cloned task.
	180	static int TracerThread(void *argument) {
	181	TracerThreadArgument *tracer_thread_argument =
	182	(TracerThreadArgument *)argument;
	183
	184	// Check if parent is already dead.
	185	if (internal_getppid() != tracer_thread_argument->parent_pid)
	186	internal__exit(4);
	187
	188	// Wait for the parent thread to finish preparations.
	189	tracer_thread_argument->mutex.Lock();
	190	tracer_thread_argument->mutex.Unlock();
	191
	192	RAW_CHECK(AddDieCallback(TracerThreadDieCallback));
	193
	194	ThreadSuspender thread_suspender(internal_getppid(), tracer_thread_argument);
	195	// Global pointer for the signal handler.
	196	thread_suspender_instance = &thread_suspender;
	197
	198	// Alternate stack for signal handling.
	199	InternalMmapVector<char> handler_stack_memory(kHandlerStackSize);
	200	stack_t handler_stack;
	201	internal_memset(&handler_stack, 0, sizeof(handler_stack));
	202	handler_stack.ss_sp = handler_stack_memory.data();
	203	handler_stack.ss_size = kHandlerStackSize;
	204	internal_sigaltstack(&handler_stack, nullptr);
	205
	206	// Install our handler for synchronous signals. Other signals should be
	207	// blocked by the mask we inherited from the parent thread.
	208	for (uptr i = 0; i < ARRAY_SIZE(kSyncSignals); i++) {
	209	__sanitizer_sigaction act;
	210	internal_memset(&act, 0, sizeof(act));
	211	act.sigaction = TracerThreadSignalHandler;
	212	act.sa_flags = SA_ONSTACK \| SA_SIGINFO;
	213	internal_sigaction_norestorer(kSyncSignals[i], &act, 0);
	214	}
	215
	216	int exit_code = 0;
	217	if (!thread_suspender.SuspendAllThreads()) {
	218	VReport(1, "Failed suspending threads.\n");
	219	exit_code = 3;
	220	} else {
	221	tracer_thread_argument->callback(thread_suspender.suspended_threads_list(),
	222	tracer_thread_argument->callback_argument);
	223	thread_suspender.ResumeAllThreads();
	224	exit_code = 0;
	225	}
	226	RAW_CHECK(RemoveDieCallback(TracerThreadDieCallback));
	227	thread_suspender_instance = nullptr;
	228	atomic_store(&tracer_thread_argument->done, 1, memory_order_relaxed);
	229	return exit_code;
	230	}
	231
	232	class ScopedStackSpaceWithGuard {
	233	public:
	234	explicit ScopedStackSpaceWithGuard(uptr stack_size) {
	235	stack_size_ = stack_size;
	236	guard_size_ = GetPageSizeCached();
	237	// FIXME: Omitting MAP_STACK here works in current kernels but might break
	238	// in the future.
	239	guard_start_ =
	240	(uptr)MmapOrDie(stack_size_ + guard_size_, "ScopedStackWithGuard");
	241	CHECK(MprotectNoAccess((uptr)guard_start_, guard_size_));
	242	}
	243	~ScopedStackSpaceWithGuard() {
	244	UnmapOrDie((void *)guard_start_, stack_size_ + guard_size_);
	245	}
	246	void *Bottom() const {
	247	return (void *)(guard_start_ + stack_size_ + guard_size_);
	248	}
	249
	250	private:
	251	uptr stack_size_;
	252	uptr guard_size_;
	253	uptr guard_start_;
	254	};
	255
	256	static __sanitizer_sigset_t blocked_sigset;
	257	static __sanitizer_sigset_t old_sigset;
	258
	259	struct ScopedSetTracerPID {
	260	explicit ScopedSetTracerPID(uptr tracer_pid) {
	261	stoptheworld_tracer_pid = tracer_pid;
	262	stoptheworld_tracer_ppid = internal_getpid();
	263	}
	264	~ScopedSetTracerPID() {
	265	stoptheworld_tracer_pid = 0;
	266	stoptheworld_tracer_ppid = 0;
	267	}
	268	};
	269
	270	void StopTheWorld(StopTheWorldCallback callback, void *argument) {
	271	// Prepare the arguments for TracerThread.
	272	struct TracerThreadArgument tracer_thread_argument;
	273	tracer_thread_argument.callback = callback;
	274	tracer_thread_argument.callback_argument = argument;
	275	tracer_thread_argument.parent_pid = internal_getpid();
	276	atomic_store(&tracer_thread_argument.done, 0, memory_order_relaxed);
	277	const uptr kTracerStackSize = 2 * 1024 * 1024;
	278	ScopedStackSpaceWithGuard tracer_stack(kTracerStackSize);
	279
	280	tracer_thread_argument.mutex.Lock();
	281
	282	internal_sigfillset(&blocked_sigset);
	283	for (uptr i = 0; i < ARRAY_SIZE(kSyncSignals); i++)
	284	internal_sigdelset(&blocked_sigset, kSyncSignals[i]);
	285	int rv = internal_sigprocmask(SIG_BLOCK, &blocked_sigset, &old_sigset);
	286	CHECK_EQ(rv, 0);
	287	uptr tracer_pid = internal_clone(TracerThread, tracer_stack.Bottom(),
	288	CLONE_VM \| CLONE_FS \| CLONE_FILES,
	289	&tracer_thread_argument);
	290	internal_sigprocmask(SIG_SETMASK, &old_sigset, 0);
	291	int local_errno = 0;
	292	if (internal_iserror(tracer_pid, &local_errno)) {
	293	VReport(1, "Failed spawning a tracer thread (errno %d).\n", local_errno);
	294	tracer_thread_argument.mutex.Unlock();
	295	} else {
	296	ScopedSetTracerPID scoped_set_tracer_pid(tracer_pid);
	297
	298	tracer_thread_argument.mutex.Unlock();
	299
	300	while (atomic_load(&tracer_thread_argument.done, memory_order_relaxed) == 0)
	301	sched_yield();
	302
	303	for (;;) {
	304	uptr waitpid_status = internal_waitpid(tracer_pid, nullptr, __WALL);
	305	if (!internal_iserror(waitpid_status, &local_errno))
	306	break;
	307	if (local_errno == EINTR)
	308	continue;
	309	VReport(1, "Waiting on the tracer thread failed (errno %d).\n",
	310	local_errno);
	311	break;
	312	}
	313	}
	314	}
	315
	316	tid_t SuspendedThreadsListNetBSD::GetThreadID(uptr index) const {
	317	CHECK_LT(index, thread_ids_.size());
	318	return thread_ids_[index];
	319	}
	320
	321	uptr SuspendedThreadsListNetBSD::ThreadCount() const {
	322	return thread_ids_.size();
	323	}
	324
	325	bool SuspendedThreadsListNetBSD::ContainsTid(tid_t thread_id) const {
	326	for (uptr i = 0; i < thread_ids_.size(); i++) {
	327	if (thread_ids_[i] == thread_id)
	328	return true;
	329	}
	330	return false;
	331	}
	332
	333	void SuspendedThreadsListNetBSD::Append(tid_t tid) {
	334	thread_ids_.push_back(tid);
	335	}
	336
	337	PtraceRegistersStatus SuspendedThreadsListNetBSD::GetRegistersAndSP(
	338	uptr index, uptr buffer, uptr sp) const {
	339	lwpid_t tid = GetThreadID(index);
	340	pid_t ppid = internal_getppid();
	341	struct reg regs;
	342	int pterrno;
	343	bool isErr =
	344	internal_iserror(internal_ptrace(PT_GETREGS, ppid, &regs, tid), &pterrno);
	345	if (isErr) {
	346	VReport(1,
	347	"Could not get registers from process %d thread %d (errno %d).\n",
	348	ppid, tid, pterrno);
	349	return pterrno == ESRCH ? REGISTERS_UNAVAILABLE_FATAL
	350	: REGISTERS_UNAVAILABLE;
	351	}
	352
	353	*sp = PTRACE_REG_SP(&regs);
	354	internal_memcpy(buffer, &regs, sizeof(regs));
	355
	356	return REGISTERS_AVAILABLE;
	357	}
	358
	359	uptr SuspendedThreadsListNetBSD::RegisterCount() const {
	360	return sizeof(struct reg) / sizeof(uptr);
	361	}
	362	} // namespace __sanitizer
	363
	364	#endif

lib/tsan/sanitizer_common/sanitizer_symbolizer.cpp created+135

...	...	@@ -0,0 +1,135 @@
	1	//===-- sanitizer_symbolizer.cpp ------------------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file is shared between AddressSanitizer and ThreadSanitizer
	10	// run-time libraries.
	11	//===----------------------------------------------------------------------===//
	12
	13	#include "sanitizer_allocator_internal.h"
	14	#include "sanitizer_platform.h"
	15	#include "sanitizer_internal_defs.h"
	16	#include "sanitizer_libc.h"
	17	#include "sanitizer_placement_new.h"
	18	#include "sanitizer_symbolizer_internal.h"
	19
	20	namespace __sanitizer {
	21
	22	AddressInfo::AddressInfo() {
	23	internal_memset(this, 0, sizeof(AddressInfo));
	24	function_offset = kUnknown;
	25	}
	26
	27	void AddressInfo::Clear() {
	28	InternalFree(module);
	29	InternalFree(function);
	30	InternalFree(file);
	31	internal_memset(this, 0, sizeof(AddressInfo));
	32	function_offset = kUnknown;
	33	}
	34
	35	void AddressInfo::FillModuleInfo(const char *mod_name, uptr mod_offset,
	36	ModuleArch mod_arch) {
	37	module = internal_strdup(mod_name);
	38	module_offset = mod_offset;
	39	module_arch = mod_arch;
	40	}
	41
	42	SymbolizedStack::SymbolizedStack() : next(nullptr), info() {}
	43
	44	SymbolizedStack *SymbolizedStack::New(uptr addr) {
	45	void *mem = InternalAlloc(sizeof(SymbolizedStack));
	46	SymbolizedStack *res = new(mem) SymbolizedStack();
	47	res->info.address = addr;
	48	return res;
	49	}
	50
	51	void SymbolizedStack::ClearAll() {
	52	info.Clear();
	53	if (next)
	54	next->ClearAll();
	55	InternalFree(this);
	56	}
	57
	58	DataInfo::DataInfo() {
	59	internal_memset(this, 0, sizeof(DataInfo));
	60	}
	61
	62	void DataInfo::Clear() {
	63	InternalFree(module);
	64	InternalFree(file);
	65	InternalFree(name);
	66	internal_memset(this, 0, sizeof(DataInfo));
	67	}
	68
	69	void FrameInfo::Clear() {
	70	InternalFree(module);
	71	for (LocalInfo &local : locals) {
	72	InternalFree(local.function_name);
	73	InternalFree(local.name);
	74	InternalFree(local.decl_file);
	75	}
	76	locals.clear();
	77	}
	78
	79	Symbolizer *Symbolizer::symbolizer_;
	80	StaticSpinMutex Symbolizer::init_mu_;
	81	LowLevelAllocator Symbolizer::symbolizer_allocator_;
	82
	83	void Symbolizer::InvalidateModuleList() {
	84	modules_fresh_ = false;
	85	}
	86
	87	void Symbolizer::AddHooks(Symbolizer::StartSymbolizationHook start_hook,
	88	Symbolizer::EndSymbolizationHook end_hook) {
	89	CHECK(start_hook_ == 0 && end_hook_ == 0);
	90	start_hook_ = start_hook;
	91	end_hook_ = end_hook;
	92	}
	93
	94	const char Symbolizer::ModuleNameOwner::GetOwnedCopy(const char str) {
	95	mu_->CheckLocked();
	96
	97	// 'str' will be the same string multiple times in a row, optimize this case.
	98	if (last_match_ && !internal_strcmp(last_match_, str))
	99	return last_match_;
	100
	101	// FIXME: this is linear search.
	102	// We should optimize this further if this turns out to be a bottleneck later.
	103	for (uptr i = 0; i < storage_.size(); ++i) {
	104	if (!internal_strcmp(storage_[i], str)) {
	105	last_match_ = storage_[i];
	106	return last_match_;
	107	}
	108	}
	109	last_match_ = internal_strdup(str);
	110	storage_.push_back(last_match_);
	111	return last_match_;
	112	}
	113
	114	Symbolizer::Symbolizer(IntrusiveList<SymbolizerTool> tools)
	115	: module_names_(&mu_), modules_(), modules_fresh_(false), tools_(tools),
	116	start_hook_(0), end_hook_(0) {}
	117
	118	Symbolizer::SymbolizerScope::SymbolizerScope(const Symbolizer *sym)
	119	: sym_(sym) {
	120	if (sym_->start_hook_)
	121	sym_->start_hook_();
	122	}
	123
	124	Symbolizer::SymbolizerScope::~SymbolizerScope() {
	125	if (sym_->end_hook_)
	126	sym_->end_hook_();
	127	}
	128
	129	void Symbolizer::LateInitializeTools() {
	130	for (auto &tool : tools_) {
	131	tool.LateInitialize();
	132	}
	133	}
	134
	135	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_symbolizer_libbacktrace.cpp created+209

...	...	@@ -0,0 +1,209 @@
	1	//===-- sanitizer_symbolizer_libbacktrace.cpp -----------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file is shared between AddressSanitizer and ThreadSanitizer
	10	// run-time libraries.
	11	// Libbacktrace implementation of symbolizer parts.
	12	//===----------------------------------------------------------------------===//
	13
	14	#include "sanitizer_platform.h"
	15
	16	#include "sanitizer_internal_defs.h"
	17	#include "sanitizer_symbolizer.h"
	18	#include "sanitizer_symbolizer_libbacktrace.h"
	19
	20	#if SANITIZER_LIBBACKTRACE
	21	# include "backtrace-supported.h"
	22	# if SANITIZER_POSIX && BACKTRACE_SUPPORTED && !BACKTRACE_USES_MALLOC
	23	# include "backtrace.h"
	24	# if SANITIZER_CP_DEMANGLE
	25	# undef ARRAY_SIZE
	26	# include "demangle.h"
	27	# endif
	28	# else
	29	# define SANITIZER_LIBBACKTRACE 0
	30	# endif
	31	#endif
	32
	33	namespace __sanitizer {
	34
	35	static char DemangleAlloc(const char name, bool always_alloc);
	36
	37	#if SANITIZER_LIBBACKTRACE
	38
	39	namespace {
	40
	41	# if SANITIZER_CP_DEMANGLE
	42	struct CplusV3DemangleData {
	43	char *buf;
	44	uptr size, allocated;
	45	};
	46
	47	extern "C" {
	48	static void CplusV3DemangleCallback(const char s, size_t l, void vdata) {
	49	CplusV3DemangleData data = (CplusV3DemangleData )vdata;
	50	uptr needed = data->size + l + 1;
	51	if (needed > data->allocated) {
	52	data->allocated *= 2;
	53	if (needed > data->allocated)
	54	data->allocated = needed;
	55	char buf = (char )InternalAlloc(data->allocated);
	56	if (data->buf) {
	57	internal_memcpy(buf, data->buf, data->size);
	58	InternalFree(data->buf);
	59	}
	60	data->buf = buf;
	61	}
	62	internal_memcpy(data->buf + data->size, s, l);
	63	data->buf[data->size + l] = '\0';
	64	data->size += l;
	65	}
	66	} // extern "C"
	67
	68	char CplusV3Demangle(const char name) {
	69	CplusV3DemangleData data;
	70	data.buf = 0;
	71	data.size = 0;
	72	data.allocated = 0;
	73	if (cplus_demangle_v3_callback(name, DMGL_PARAMS \| DMGL_ANSI,
	74	CplusV3DemangleCallback, &data)) {
	75	if (data.size + 64 > data.allocated)
	76	return data.buf;
	77	char *buf = internal_strdup(data.buf);
	78	InternalFree(data.buf);
	79	return buf;
	80	}
	81	if (data.buf)
	82	InternalFree(data.buf);
	83	return 0;
	84	}
	85	# endif // SANITIZER_CP_DEMANGLE
	86
	87	struct SymbolizeCodeCallbackArg {
	88	SymbolizedStack *first;
	89	SymbolizedStack *last;
	90	uptr frames_symbolized;
	91
	92	AddressInfo *get_new_frame(uintptr_t addr) {
	93	CHECK(last);
	94	if (frames_symbolized > 0) {
	95	SymbolizedStack *cur = SymbolizedStack::New(addr);
	96	AddressInfo *info = &cur->info;
	97	info->FillModuleInfo(first->info.module, first->info.module_offset,
	98	first->info.module_arch);
	99	last->next = cur;
	100	last = cur;
	101	}
	102	CHECK_EQ(addr, first->info.address);
	103	CHECK_EQ(addr, last->info.address);
	104	return &last->info;
	105	}
	106	};
	107
	108	extern "C" {
	109	static int SymbolizeCodePCInfoCallback(void *vdata, uintptr_t addr,
	110	const char *filename, int lineno,
	111	const char *function) {
	112	SymbolizeCodeCallbackArg cdata = (SymbolizeCodeCallbackArg )vdata;
	113	if (function) {
	114	AddressInfo *info = cdata->get_new_frame(addr);
	115	info->function = DemangleAlloc(function, /always_alloc/ true);
	116	if (filename)
	117	info->file = internal_strdup(filename);
	118	info->line = lineno;
	119	cdata->frames_symbolized++;
	120	}
	121	return 0;
	122	}
	123
	124	static void SymbolizeCodeCallback(void *vdata, uintptr_t addr,
	125	const char *symname, uintptr_t, uintptr_t) {
	126	SymbolizeCodeCallbackArg cdata = (SymbolizeCodeCallbackArg )vdata;
	127	if (symname) {
	128	AddressInfo *info = cdata->get_new_frame(addr);
	129	info->function = DemangleAlloc(symname, /always_alloc/ true);
	130	cdata->frames_symbolized++;
	131	}
	132	}
	133
	134	static void SymbolizeDataCallback(void vdata, uintptr_t, const char symname,
	135	uintptr_t symval, uintptr_t symsize) {
	136	DataInfo info = (DataInfo )vdata;
	137	if (symname && symval) {
	138	info->name = DemangleAlloc(symname, /always_alloc/ true);
	139	info->start = symval;
	140	info->size = symsize;
	141	}
	142	}
	143
	144	static void ErrorCallback(void , const char , int) {}
	145	} // extern "C"
	146
	147	} // namespace
	148
	149	LibbacktraceSymbolizer LibbacktraceSymbolizer::get(LowLevelAllocator alloc) {
	150	// State created in backtrace_create_state is leaked.
	151	void state = (void )(backtrace_create_state("/proc/self/exe", 0,
	152	ErrorCallback, NULL));
	153	if (!state)
	154	return 0;
	155	return new(*alloc) LibbacktraceSymbolizer(state);
	156	}
	157
	158	bool LibbacktraceSymbolizer::SymbolizePC(uptr addr, SymbolizedStack *stack) {
	159	SymbolizeCodeCallbackArg data;
	160	data.first = stack;
	161	data.last = stack;
	162	data.frames_symbolized = 0;
	163	backtrace_pcinfo((backtrace_state *)state_, addr, SymbolizeCodePCInfoCallback,
	164	ErrorCallback, &data);
	165	if (data.frames_symbolized > 0)
	166	return true;
	167	backtrace_syminfo((backtrace_state *)state_, addr, SymbolizeCodeCallback,
	168	ErrorCallback, &data);
	169	return (data.frames_symbolized > 0);
	170	}
	171
	172	bool LibbacktraceSymbolizer::SymbolizeData(uptr addr, DataInfo *info) {
	173	backtrace_syminfo((backtrace_state *)state_, addr, SymbolizeDataCallback,
	174	ErrorCallback, info);
	175	return true;
	176	}
	177
	178	#else // SANITIZER_LIBBACKTRACE
	179
	180	LibbacktraceSymbolizer LibbacktraceSymbolizer::get(LowLevelAllocator alloc) {
	181	return 0;
	182	}
	183
	184	bool LibbacktraceSymbolizer::SymbolizePC(uptr addr, SymbolizedStack *stack) {
	185	(void)state_;
	186	return false;
	187	}
	188
	189	bool LibbacktraceSymbolizer::SymbolizeData(uptr addr, DataInfo *info) {
	190	return false;
	191	}
	192
	193	#endif // SANITIZER_LIBBACKTRACE
	194
	195	static char DemangleAlloc(const char name, bool always_alloc) {
	196	#if SANITIZER_LIBBACKTRACE && SANITIZER_CP_DEMANGLE
	197	if (char *demangled = CplusV3Demangle(name))
	198	return demangled;
	199	#endif
	200	if (always_alloc)
	201	return internal_strdup(name);
	202	return 0;
	203	}
	204
	205	const char LibbacktraceSymbolizer::Demangle(const char name) {
	206	return DemangleAlloc(name, /always_alloc/ false);
	207	}
	208
	209	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_symbolizer_libcdep.cpp created+554

...	...	@@ -0,0 +1,554 @@
	1	//===-- sanitizer_symbolizer_libcdep.cpp ----------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file is shared between AddressSanitizer and ThreadSanitizer
	10	// run-time libraries.
	11	//===----------------------------------------------------------------------===//
	12
	13	#include "sanitizer_allocator_internal.h"
	14	#include "sanitizer_internal_defs.h"
	15	#include "sanitizer_symbolizer_internal.h"
	16
	17	namespace __sanitizer {
	18
	19	Symbolizer *Symbolizer::GetOrInit() {
	20	SpinMutexLock l(&init_mu_);
	21	if (symbolizer_)
	22	return symbolizer_;
	23	symbolizer_ = PlatformInit();
	24	CHECK(symbolizer_);
	25	return symbolizer_;
	26	}
	27
	28	// See sanitizer_symbolizer_markup.cpp.
	29	#if !SANITIZER_SYMBOLIZER_MARKUP
	30
	31	const char ExtractToken(const char str, const char delims, char *result) {
	32	uptr prefix_len = internal_strcspn(str, delims);
	33	result = (char)InternalAlloc(prefix_len + 1);
	34	internal_memcpy(*result, str, prefix_len);
	35	(*result)[prefix_len] = '\0';
	36	const char *prefix_end = str + prefix_len;
	37	if (*prefix_end != '\0') prefix_end++;
	38	return prefix_end;
	39	}
	40
	41	const char ExtractInt(const char str, const char delims, int result) {
	42	char *buff = nullptr;
	43	const char *ret = ExtractToken(str, delims, &buff);
	44	if (buff) {
	45	*result = (int)internal_atoll(buff);
	46	}
	47	InternalFree(buff);
	48	return ret;
	49	}
	50
	51	const char ExtractUptr(const char str, const char delims, uptr result) {
	52	char *buff = nullptr;
	53	const char *ret = ExtractToken(str, delims, &buff);
	54	if (buff) {
	55	*result = (uptr)internal_atoll(buff);
	56	}
	57	InternalFree(buff);
	58	return ret;
	59	}
	60
	61	const char ExtractSptr(const char str, const char delims, sptr result) {
	62	char *buff = nullptr;
	63	const char *ret = ExtractToken(str, delims, &buff);
	64	if (buff) {
	65	*result = (sptr)internal_atoll(buff);
	66	}
	67	InternalFree(buff);
	68	return ret;
	69	}
	70
	71	const char ExtractTokenUpToDelimiter(const char str, const char *delimiter,
	72	char **result) {
	73	const char *found_delimiter = internal_strstr(str, delimiter);
	74	uptr prefix_len =
	75	found_delimiter ? found_delimiter - str : internal_strlen(str);
	76	result = (char )InternalAlloc(prefix_len + 1);
	77	internal_memcpy(*result, str, prefix_len);
	78	(*result)[prefix_len] = '\0';
	79	const char *prefix_end = str + prefix_len;
	80	if (*prefix_end != '\0') prefix_end += internal_strlen(delimiter);
	81	return prefix_end;
	82	}
	83
	84	SymbolizedStack *Symbolizer::SymbolizePC(uptr addr) {
	85	BlockingMutexLock l(&mu_);
	86	const char *module_name = nullptr;
	87	uptr module_offset;
	88	ModuleArch arch;
	89	SymbolizedStack *res = SymbolizedStack::New(addr);
	90	if (!FindModuleNameAndOffsetForAddress(addr, &module_name, &module_offset,
	91	&arch))
	92	return res;
	93	// Always fill data about module name and offset.
	94	res->info.FillModuleInfo(module_name, module_offset, arch);
	95	for (auto &tool : tools_) {
	96	SymbolizerScope sym_scope(this);
	97	if (tool.SymbolizePC(addr, res)) {
	98	return res;
	99	}
	100	}
	101	return res;
	102	}
	103
	104	bool Symbolizer::SymbolizeData(uptr addr, DataInfo *info) {
	105	BlockingMutexLock l(&mu_);
	106	const char *module_name = nullptr;
	107	uptr module_offset;
	108	ModuleArch arch;
	109	if (!FindModuleNameAndOffsetForAddress(addr, &module_name, &module_offset,
	110	&arch))
	111	return false;
	112	info->Clear();
	113	info->module = internal_strdup(module_name);
	114	info->module_offset = module_offset;
	115	info->module_arch = arch;
	116	for (auto &tool : tools_) {
	117	SymbolizerScope sym_scope(this);
	118	if (tool.SymbolizeData(addr, info)) {
	119	return true;
	120	}
	121	}
	122	return true;
	123	}
	124
	125	bool Symbolizer::SymbolizeFrame(uptr addr, FrameInfo *info) {
	126	BlockingMutexLock l(&mu_);
	127	const char *module_name = nullptr;
	128	if (!FindModuleNameAndOffsetForAddress(
	129	addr, &module_name, &info->module_offset, &info->module_arch))
	130	return false;
	131	info->module = internal_strdup(module_name);
	132	for (auto &tool : tools_) {
	133	SymbolizerScope sym_scope(this);
	134	if (tool.SymbolizeFrame(addr, info)) {
	135	return true;
	136	}
	137	}
	138	return true;
	139	}
	140
	141	bool Symbolizer::GetModuleNameAndOffsetForPC(uptr pc, const char **module_name,
	142	uptr *module_address) {
	143	BlockingMutexLock l(&mu_);
	144	const char *internal_module_name = nullptr;
	145	ModuleArch arch;
	146	if (!FindModuleNameAndOffsetForAddress(pc, &internal_module_name,
	147	module_address, &arch))
	148	return false;
	149
	150	if (module_name)
	151	*module_name = module_names_.GetOwnedCopy(internal_module_name);
	152	return true;
	153	}
	154
	155	void Symbolizer::Flush() {
	156	BlockingMutexLock l(&mu_);
	157	for (auto &tool : tools_) {
	158	SymbolizerScope sym_scope(this);
	159	tool.Flush();
	160	}
	161	}
	162
	163	const char Symbolizer::Demangle(const char name) {
	164	BlockingMutexLock l(&mu_);
	165	for (auto &tool : tools_) {
	166	SymbolizerScope sym_scope(this);
	167	if (const char *demangled = tool.Demangle(name))
	168	return demangled;
	169	}
	170	return PlatformDemangle(name);
	171	}
	172
	173	bool Symbolizer::FindModuleNameAndOffsetForAddress(uptr address,
	174	const char **module_name,
	175	uptr *module_offset,
	176	ModuleArch *module_arch) {
	177	const LoadedModule *module = FindModuleForAddress(address);
	178	if (!module)
	179	return false;
	180	*module_name = module->full_name();
	181	*module_offset = address - module->base_address();
	182	*module_arch = module->arch();
	183	return true;
	184	}
	185
	186	void Symbolizer::RefreshModules() {
	187	modules_.init();
	188	fallback_modules_.fallbackInit();
	189	RAW_CHECK(modules_.size() > 0);
	190	modules_fresh_ = true;
	191	}
	192
	193	static const LoadedModule *SearchForModule(const ListOfModules &modules,
	194	uptr address) {
	195	for (uptr i = 0; i < modules.size(); i++) {
	196	if (modules[i].containsAddress(address)) {
	197	return &modules[i];
	198	}
	199	}
	200	return nullptr;
	201	}
	202
	203	const LoadedModule *Symbolizer::FindModuleForAddress(uptr address) {
	204	bool modules_were_reloaded = false;
	205	if (!modules_fresh_) {
	206	RefreshModules();
	207	modules_were_reloaded = true;
	208	}
	209	const LoadedModule *module = SearchForModule(modules_, address);
	210	if (module) return module;
	211
	212	// dlopen/dlclose interceptors invalidate the module list, but when
	213	// interception is disabled, we need to retry if the lookup fails in
	214	// case the module list changed.
	215	#if !SANITIZER_INTERCEPT_DLOPEN_DLCLOSE
	216	if (!modules_were_reloaded) {
	217	RefreshModules();
	218	module = SearchForModule(modules_, address);
	219	if (module) return module;
	220	}
	221	#endif
	222
	223	if (fallback_modules_.size()) {
	224	module = SearchForModule(fallback_modules_, address);
	225	}
	226	return module;
	227	}
	228
	229	// For now we assume the following protocol:
	230	// For each request of the form
	231	// <module_name> <module_offset>
	232	// passed to STDIN, external symbolizer prints to STDOUT response:
	233	// <function_name>
	234	// <file_name>:<line_number>:<column_number>
	235	// <function_name>
	236	// <file_name>:<line_number>:<column_number>
	237	// ...
	238	// <empty line>
	239	class LLVMSymbolizerProcess : public SymbolizerProcess {
	240	public:
	241	explicit LLVMSymbolizerProcess(const char *path)
	242	: SymbolizerProcess(path, /use_posix_spawn=/SANITIZER_MAC) {}
	243
	244	private:
	245	bool ReachedEndOfOutput(const char *buffer, uptr length) const override {
	246	// Empty line marks the end of llvm-symbolizer output.
	247	return length >= 2 && buffer[length - 1] == '\n' &&
	248	buffer[length - 2] == '\n';
	249	}
	250
	251	// When adding a new architecture, don't forget to also update
	252	// script/asan_symbolize.py and sanitizer_common.h.
	253	void GetArgV(const char *path_to_binary,
	254	const char *(&argv)[kArgVMax]) const override {
	255	#if defined(__x86_64h__)
	256	const char* const kSymbolizerArch = "--default-arch=x86_64h";
	257	#elif defined(__x86_64__)
	258	const char* const kSymbolizerArch = "--default-arch=x86_64";
	259	#elif defined(__i386__)
	260	const char* const kSymbolizerArch = "--default-arch=i386";
	261	#elif defined(__aarch64__)
	262	const char* const kSymbolizerArch = "--default-arch=arm64";
	263	#elif defined(__arm__)
	264	const char* const kSymbolizerArch = "--default-arch=arm";
	265	#elif defined(__powerpc64__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
	266	const char* const kSymbolizerArch = "--default-arch=powerpc64";
	267	#elif defined(__powerpc64__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
	268	const char* const kSymbolizerArch = "--default-arch=powerpc64le";
	269	#elif defined(__s390x__)
	270	const char* const kSymbolizerArch = "--default-arch=s390x";
	271	#elif defined(__s390__)
	272	const char* const kSymbolizerArch = "--default-arch=s390";
	273	#else
	274	const char* const kSymbolizerArch = "--default-arch=unknown";
	275	#endif
	276
	277	const char *const inline_flag = common_flags()->symbolize_inline_frames
	278	? "--inlining=true"
	279	: "--inlining=false";
	280	int i = 0;
	281	argv[i++] = path_to_binary;
	282	argv[i++] = inline_flag;
	283	argv[i++] = kSymbolizerArch;
	284	argv[i++] = nullptr;
	285	}
	286	};
	287
	288	LLVMSymbolizer::LLVMSymbolizer(const char path, LowLevelAllocator allocator)
	289	: symbolizer_process_(new(*allocator) LLVMSymbolizerProcess(path)) {}
	290
	291	// Parse a <file>:<line>[:<column>] buffer. The file path may contain colons on
	292	// Windows, so extract tokens from the right hand side first. The column info is
	293	// also optional.
	294	static const char ParseFileLineInfo(AddressInfo info, const char *str) {
	295	char *file_line_info = nullptr;
	296	str = ExtractToken(str, "\n", &file_line_info);
	297	CHECK(file_line_info);
	298
	299	if (uptr size = internal_strlen(file_line_info)) {
	300	char *back = file_line_info + size - 1;
	301	for (int i = 0; i < 2; ++i) {
	302	while (back > file_line_info && IsDigit(*back)) --back;
	303	if (*back != ':' \|\| !IsDigit(back[1])) break;
	304	info->column = info->line;
	305	info->line = internal_atoll(back + 1);
	306	// Truncate the string at the colon to keep only filename.
	307	*back = '\0';
	308	--back;
	309	}
	310	ExtractToken(file_line_info, "", &info->file);
	311	}
	312
	313	InternalFree(file_line_info);
	314	return str;
	315	}
	316
	317	// Parses one or more two-line strings in the following format:
	318	// <function_name>
	319	// <file_name>:<line_number>[:<column_number>]
	320	// Used by LLVMSymbolizer, Addr2LinePool and InternalSymbolizer, since all of
	321	// them use the same output format.
	322	void ParseSymbolizePCOutput(const char str, SymbolizedStack res) {
	323	bool top_frame = true;
	324	SymbolizedStack *last = res;
	325	while (true) {
	326	char *function_name = nullptr;
	327	str = ExtractToken(str, "\n", &function_name);
	328	CHECK(function_name);
	329	if (function_name[0] == '\0') {
	330	// There are no more frames.
	331	InternalFree(function_name);
	332	break;
	333	}
	334	SymbolizedStack *cur;
	335	if (top_frame) {
	336	cur = res;
	337	top_frame = false;
	338	} else {
	339	cur = SymbolizedStack::New(res->info.address);
	340	cur->info.FillModuleInfo(res->info.module, res->info.module_offset,
	341	res->info.module_arch);
	342	last->next = cur;
	343	last = cur;
	344	}
	345
	346	AddressInfo *info = &cur->info;
	347	info->function = function_name;
	348	str = ParseFileLineInfo(info, str);
	349
	350	// Functions and filenames can be "??", in which case we write 0
	351	// to address info to mark that names are unknown.
	352	if (0 == internal_strcmp(info->function, "??")) {
	353	InternalFree(info->function);
	354	info->function = 0;
	355	}
	356	if (0 == internal_strcmp(info->file, "??")) {
	357	InternalFree(info->file);
	358	info->file = 0;
	359	}
	360	}
	361	}
	362
	363	// Parses a two-line string in the following format:
	364	// <symbol_name>
	365	// <start_address> <size>
	366	// Used by LLVMSymbolizer and InternalSymbolizer.
	367	void ParseSymbolizeDataOutput(const char str, DataInfo info) {
	368	str = ExtractToken(str, "\n", &info->name);
	369	str = ExtractUptr(str, " ", &info->start);
	370	str = ExtractUptr(str, "\n", &info->size);
	371	}
	372
	373	static void ParseSymbolizeFrameOutput(const char *str,
	374	InternalMmapVector<LocalInfo> *locals) {
	375	if (internal_strncmp(str, "??", 2) == 0)
	376	return;
	377
	378	while (*str) {
	379	LocalInfo local;
	380	str = ExtractToken(str, "\n", &local.function_name);
	381	str = ExtractToken(str, "\n", &local.name);
	382
	383	AddressInfo addr;
	384	str = ParseFileLineInfo(&addr, str);
	385	local.decl_file = addr.file;
	386	local.decl_line = addr.line;
	387
	388	local.has_frame_offset = internal_strncmp(str, "??", 2) != 0;
	389	str = ExtractSptr(str, " ", &local.frame_offset);
	390
	391	local.has_size = internal_strncmp(str, "??", 2) != 0;
	392	str = ExtractUptr(str, " ", &local.size);
	393
	394	local.has_tag_offset = internal_strncmp(str, "??", 2) != 0;
	395	str = ExtractUptr(str, "\n", &local.tag_offset);
	396
	397	locals->push_back(local);
	398	}
	399	}
	400
	401	bool LLVMSymbolizer::SymbolizePC(uptr addr, SymbolizedStack *stack) {
	402	AddressInfo *info = &stack->info;
	403	const char *buf = FormatAndSendCommand(
	404	"CODE", info->module, info->module_offset, info->module_arch);
	405	if (!buf)
	406	return false;
	407	ParseSymbolizePCOutput(buf, stack);
	408	return true;
	409	}
	410
	411	bool LLVMSymbolizer::SymbolizeData(uptr addr, DataInfo *info) {
	412	const char *buf = FormatAndSendCommand(
	413	"DATA", info->module, info->module_offset, info->module_arch);
	414	if (!buf)
	415	return false;
	416	ParseSymbolizeDataOutput(buf, info);
	417	info->start += (addr - info->module_offset); // Add the base address.
	418	return true;
	419	}
	420
	421	bool LLVMSymbolizer::SymbolizeFrame(uptr addr, FrameInfo *info) {
	422	const char *buf = FormatAndSendCommand(
	423	"FRAME", info->module, info->module_offset, info->module_arch);
	424	if (!buf)
	425	return false;
	426	ParseSymbolizeFrameOutput(buf, &info->locals);
	427	return true;
	428	}
	429
	430	const char LLVMSymbolizer::FormatAndSendCommand(const char command_prefix,
	431	const char *module_name,
	432	uptr module_offset,
	433	ModuleArch arch) {
	434	CHECK(module_name);
	435	int size_needed = 0;
	436	if (arch == kModuleArchUnknown)
	437	size_needed = internal_snprintf(buffer_, kBufferSize, "%s \"%s\" 0x%zx\n",
	438	command_prefix, module_name, module_offset);
	439	else
	440	size_needed = internal_snprintf(buffer_, kBufferSize,
	441	"%s \"%s:%s\" 0x%zx\n", command_prefix,
	442	module_name, ModuleArchToString(arch),
	443	module_offset);
	444
	445	if (size_needed >= static_cast<int>(kBufferSize)) {
	446	Report("WARNING: Command buffer too small");
	447	return nullptr;
	448	}
	449
	450	return symbolizer_process_->SendCommand(buffer_);
	451	}
	452
	453	SymbolizerProcess::SymbolizerProcess(const char *path, bool use_posix_spawn)
	454	: path_(path),
	455	input_fd_(kInvalidFd),
	456	output_fd_(kInvalidFd),
	457	times_restarted_(0),
	458	failed_to_start_(false),
	459	reported_invalid_path_(false),
	460	use_posix_spawn_(use_posix_spawn) {
	461	CHECK(path_);
	462	CHECK_NE(path_[0], '\0');
	463	}
	464
	465	static bool IsSameModule(const char* path) {
	466	if (const char* ProcessName = GetProcessName()) {
	467	if (const char* SymbolizerName = StripModuleName(path)) {
	468	return !internal_strcmp(ProcessName, SymbolizerName);
	469	}
	470	}
	471	return false;
	472	}
	473
	474	const char SymbolizerProcess::SendCommand(const char command) {
	475	if (failed_to_start_)
	476	return nullptr;
	477	if (IsSameModule(path_)) {
	478	Report("WARNING: Symbolizer was blocked from starting itself!\n");
	479	failed_to_start_ = true;
	480	return nullptr;
	481	}
	482	for (; times_restarted_ < kMaxTimesRestarted; times_restarted_++) {
	483	// Start or restart symbolizer if we failed to send command to it.
	484	if (const char *res = SendCommandImpl(command))
	485	return res;
	486	Restart();
	487	}
	488	if (!failed_to_start_) {
	489	Report("WARNING: Failed to use and restart external symbolizer!\n");
	490	failed_to_start_ = true;
	491	}
	492	return nullptr;
	493	}
	494
	495	const char SymbolizerProcess::SendCommandImpl(const char command) {
	496	if (input_fd_ == kInvalidFd \|\| output_fd_ == kInvalidFd)
	497	return nullptr;
	498	if (!WriteToSymbolizer(command, internal_strlen(command)))
	499	return nullptr;
	500	if (!ReadFromSymbolizer(buffer_, kBufferSize))
	501	return nullptr;
	502	return buffer_;
	503	}
	504
	505	bool SymbolizerProcess::Restart() {
	506	if (input_fd_ != kInvalidFd)
	507	CloseFile(input_fd_);
	508	if (output_fd_ != kInvalidFd)
	509	CloseFile(output_fd_);
	510	return StartSymbolizerSubprocess();
	511	}
	512
	513	bool SymbolizerProcess::ReadFromSymbolizer(char *buffer, uptr max_length) {
	514	if (max_length == 0)
	515	return true;
	516	uptr read_len = 0;
	517	while (true) {
	518	uptr just_read = 0;
	519	bool success = ReadFromFile(input_fd_, buffer + read_len,
	520	max_length - read_len - 1, &just_read);
	521	// We can't read 0 bytes, as we don't expect external symbolizer to close
	522	// its stdout.
	523	if (!success \|\| just_read == 0) {
	524	Report("WARNING: Can't read from symbolizer at fd %d\n", input_fd_);
	525	return false;
	526	}
	527	read_len += just_read;
	528	if (ReachedEndOfOutput(buffer, read_len))
	529	break;
	530	if (read_len + 1 == max_length) {
	531	Report("WARNING: Symbolizer buffer too small\n");
	532	read_len = 0;
	533	break;
	534	}
	535	}
	536	buffer[read_len] = '\0';
	537	return true;
	538	}
	539
	540	bool SymbolizerProcess::WriteToSymbolizer(const char *buffer, uptr length) {
	541	if (length == 0)
	542	return true;
	543	uptr write_len = 0;
	544	bool success = WriteToFile(output_fd_, buffer, length, &write_len);
	545	if (!success \|\| write_len != length) {
	546	Report("WARNING: Can't write to symbolizer at fd %d\n", output_fd_);
	547	return false;
	548	}
	549	return true;
	550	}
	551
	552	#endif // !SANITIZER_SYMBOLIZER_MARKUP
	553
	554	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_symbolizer_mac.cpp created+249

...	...	@@ -0,0 +1,249 @@
	1	//===-- sanitizer_symbolizer_mac.cpp --------------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file is shared between various sanitizers' runtime libraries.
	10	//
	11	// Implementation of Mac-specific "atos" symbolizer.
	12	//===----------------------------------------------------------------------===//
	13
	14	#include "sanitizer_platform.h"
	15	#if SANITIZER_MAC
	16
	17	#include "sanitizer_allocator_internal.h"
	18	#include "sanitizer_mac.h"
	19	#include "sanitizer_symbolizer_mac.h"
	20
	21	#include <dlfcn.h>
	22	#include <errno.h>
	23	#include <mach/mach.h>
	24	#include <stdlib.h>
	25	#include <sys/wait.h>
	26	#include <unistd.h>
	27	#include <util.h>
	28
	29	namespace __sanitizer {
	30
	31	bool DlAddrSymbolizer::SymbolizePC(uptr addr, SymbolizedStack *stack) {
	32	Dl_info info;
	33	int result = dladdr((const void *)addr, &info);
	34	if (!result) return false;
	35
	36	CHECK(addr >= reinterpret_cast<uptr>(info.dli_saddr));
	37	stack->info.function_offset = addr - reinterpret_cast<uptr>(info.dli_saddr);
	38	const char *demangled = DemangleSwiftAndCXX(info.dli_sname);
	39	if (!demangled) return false;
	40	stack->info.function = internal_strdup(demangled);
	41	return true;
	42	}
	43
	44	bool DlAddrSymbolizer::SymbolizeData(uptr addr, DataInfo *datainfo) {
	45	Dl_info info;
	46	int result = dladdr((const void *)addr, &info);
	47	if (!result) return false;
	48	const char *demangled = DemangleSwiftAndCXX(info.dli_sname);
	49	datainfo->name = internal_strdup(demangled);
	50	datainfo->start = (uptr)info.dli_saddr;
	51	return true;
	52	}
	53
	54	#define K_ATOS_ENV_VAR "__check_mach_ports_lookup"
	55
	56	// This cannot live in `AtosSymbolizerProcess` because instances of that object
	57	// are allocated by the internal allocator which under ASan is poisoned with
	58	// kAsanInternalHeapMagic.
	59	static char kAtosMachPortEnvEntry[] = K_ATOS_ENV_VAR "=000000000000000";
	60
	61	class AtosSymbolizerProcess : public SymbolizerProcess {
	62	public:
	63	explicit AtosSymbolizerProcess(const char *path)
	64	: SymbolizerProcess(path, /use_posix_spawn/ true) {
	65	pid_str_[0] = '\0';
	66	}
	67
	68	void LateInitialize() {
	69	if (SANITIZER_IOSSIM) {
	70	// `putenv()` may call malloc/realloc so it is only safe to do this
	71	// during LateInitialize() or later (i.e. we can't do this in the
	72	// constructor). We also can't do this in `StartSymbolizerSubprocess()`
	73	// because in TSan we switch allocators when we're symbolizing.
	74	// We use `putenv()` rather than `setenv()` so that we can later directly
	75	// write into the storage without LibC getting involved to change what the
	76	// variable is set to
	77	int result = putenv(kAtosMachPortEnvEntry);
	78	CHECK_EQ(result, 0);
	79	}
	80	}
	81
	82	private:
	83	bool StartSymbolizerSubprocess() override {
	84	// Configure sandbox before starting atos process.
	85
	86	// Put the string command line argument in the object so that it outlives
	87	// the call to GetArgV.
	88	internal_snprintf(pid_str_, sizeof(pid_str_), "%d", internal_getpid());
	89
	90	if (SANITIZER_IOSSIM) {
	91	// `atos` in the simulator is restricted in its ability to retrieve the
	92	// task port for the target process (us) so we need to do extra work
	93	// to pass our task port to it.
	94	mach_port_t ports[]{mach_task_self()};
	95	kern_return_t ret =
	96	mach_ports_register(mach_task_self(), ports, /count=/1);
	97	CHECK_EQ(ret, KERN_SUCCESS);
	98
	99	// Set environment variable that signals to `atos` that it should look
	100	// for our task port. We can't call `setenv()` here because it might call
	101	// malloc/realloc. To avoid that we instead update the
	102	// `mach_port_env_var_entry_` variable with our current PID.
	103	uptr count = internal_snprintf(kAtosMachPortEnvEntry,
	104	sizeof(kAtosMachPortEnvEntry),
	105	K_ATOS_ENV_VAR "=%s", pid_str_);
	106	CHECK_GE(count, sizeof(K_ATOS_ENV_VAR) + internal_strlen(pid_str_));
	107	// Document our assumption but without calling `getenv()` in normal
	108	// builds.
	109	DCHECK(getenv(K_ATOS_ENV_VAR));
	110	DCHECK_EQ(internal_strcmp(getenv(K_ATOS_ENV_VAR), pid_str_), 0);
	111	}
	112
	113	return SymbolizerProcess::StartSymbolizerSubprocess();
	114	}
	115
	116	bool ReachedEndOfOutput(const char *buffer, uptr length) const override {
	117	return (length >= 1 && buffer[length - 1] == '\n');
	118	}
	119
	120	void GetArgV(const char *path_to_binary,
	121	const char *(&argv)[kArgVMax]) const override {
	122	int i = 0;
	123	argv[i++] = path_to_binary;
	124	argv[i++] = "-p";
	125	argv[i++] = &pid_str_[0];
	126	if (GetMacosAlignedVersion() == MacosVersion(10, 9)) {
	127	// On Mavericks atos prints a deprecation warning which we suppress by
	128	// passing -d. The warning isn't present on other OSX versions, even the
	129	// newer ones.
	130	argv[i++] = "-d";
	131	}
	132	argv[i++] = nullptr;
	133	}
	134
	135	char pid_str_[16];
	136	// Space for `\0` in `K_ATOS_ENV_VAR` is reused for `=`.
	137	static_assert(sizeof(kAtosMachPortEnvEntry) ==
	138	(sizeof(K_ATOS_ENV_VAR) + sizeof(pid_str_)),
	139	"sizes should match");
	140	};
	141
	142	#undef K_ATOS_ENV_VAR
	143
	144	static bool ParseCommandOutput(const char str, uptr addr, char *out_name,
	145	char out_module, char out_file, uptr *line,
	146	uptr *start_address) {
	147	// Trim ending newlines.
	148	char *trim;
	149	ExtractTokenUpToDelimiter(str, "\n", &trim);
	150
	151	// The line from `atos` is in one of these formats:
	152	// myfunction (in library.dylib) (sourcefile.c:17)
	153	// myfunction (in library.dylib) + 0x1fe
	154	// myfunction (in library.dylib) + 15
	155	// 0xdeadbeef (in library.dylib) + 0x1fe
	156	// 0xdeadbeef (in library.dylib) + 15
	157	// 0xdeadbeef (in library.dylib)
	158	// 0xdeadbeef
	159
	160	const char *rest = trim;
	161	char *symbol_name;
	162	rest = ExtractTokenUpToDelimiter(rest, " (in ", &symbol_name);
	163	if (rest[0] == '\0') {
	164	InternalFree(symbol_name);
	165	InternalFree(trim);
	166	return false;
	167	}
	168
	169	if (internal_strncmp(symbol_name, "0x", 2) != 0)
	170	*out_name = symbol_name;
	171	else
	172	InternalFree(symbol_name);
	173	rest = ExtractTokenUpToDelimiter(rest, ") ", out_module);
	174
	175	if (rest[0] == '(') {
	176	if (out_file) {
	177	rest++;
	178	rest = ExtractTokenUpToDelimiter(rest, ":", out_file);
	179	char *extracted_line_number;
	180	rest = ExtractTokenUpToDelimiter(rest, ")", &extracted_line_number);
	181	if (line) *line = (uptr)internal_atoll(extracted_line_number);
	182	InternalFree(extracted_line_number);
	183	}
	184	} else if (rest[0] == '+') {
	185	rest += 2;
	186	uptr offset = internal_atoll(rest);
	187	if (start_address) *start_address = addr - offset;
	188	}
	189
	190	InternalFree(trim);
	191	return true;
	192	}
	193
	194	AtosSymbolizer::AtosSymbolizer(const char path, LowLevelAllocator allocator)
	195	: process_(new (*allocator) AtosSymbolizerProcess(path)) {}
	196
	197	bool AtosSymbolizer::SymbolizePC(uptr addr, SymbolizedStack *stack) {
	198	if (!process_) return false;
	199	if (addr == 0) return false;
	200	char command[32];
	201	internal_snprintf(command, sizeof(command), "0x%zx\n", addr);
	202	const char *buf = process_->SendCommand(command);
	203	if (!buf) return false;
	204	uptr line;
	205	uptr start_address = AddressInfo::kUnknown;
	206	if (!ParseCommandOutput(buf, addr, &stack->info.function, &stack->info.module,
	207	&stack->info.file, &line, &start_address)) {
	208	process_ = nullptr;
	209	return false;
	210	}
	211	stack->info.line = (int)line;
	212
	213	if (start_address == AddressInfo::kUnknown) {
	214	// Fallback to dladdr() to get function start address if atos doesn't report
	215	// it.
	216	Dl_info info;
	217	int result = dladdr((const void *)addr, &info);
	218	if (result)
	219	start_address = reinterpret_cast<uptr>(info.dli_saddr);
	220	}
	221
	222	// Only assig to `function_offset` if we were able to get the function's
	223	// start address.
	224	if (start_address != AddressInfo::kUnknown) {
	225	CHECK(addr >= start_address);
	226	stack->info.function_offset = addr - start_address;
	227	}
	228	return true;
	229	}
	230
	231	bool AtosSymbolizer::SymbolizeData(uptr addr, DataInfo *info) {
	232	if (!process_) return false;
	233	char command[32];
	234	internal_snprintf(command, sizeof(command), "0x%zx\n", addr);
	235	const char *buf = process_->SendCommand(command);
	236	if (!buf) return false;
	237	if (!ParseCommandOutput(buf, addr, &info->name, &info->module, nullptr,
	238	nullptr, &info->start)) {
	239	process_ = nullptr;
	240	return false;
	241	}
	242	return true;
	243	}
	244
	245	void AtosSymbolizer::LateInitialize() { process_->LateInitialize(); }
	246
	247	} // namespace __sanitizer
	248
	249	#endif // SANITIZER_MAC

lib/tsan/sanitizer_common/sanitizer_symbolizer_markup.cpp created+146

...	...	@@ -0,0 +1,146 @@
	1	//===-- sanitizer_symbolizer_markup.cpp -----------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file is shared between various sanitizers' runtime libraries.
	10	//
	11	// Implementation of offline markup symbolizer.
	12	//===----------------------------------------------------------------------===//
	13
	14	#include "sanitizer_platform.h"
	15	#if SANITIZER_SYMBOLIZER_MARKUP
	16
	17	#if SANITIZER_FUCHSIA
	18	#include "sanitizer_symbolizer_fuchsia.h"
	19	#elif SANITIZER_RTEMS
	20	#include "sanitizer_symbolizer_rtems.h"
	21	#endif
	22	#include "sanitizer_stacktrace.h"
	23	#include "sanitizer_symbolizer.h"
	24
	25	#include <limits.h>
	26	#include <unwind.h>
	27
	28	namespace __sanitizer {
	29
	30	// This generic support for offline symbolizing is based on the
	31	// Fuchsia port. We don't do any actual symbolization per se.
	32	// Instead, we emit text containing raw addresses and raw linkage
	33	// symbol names, embedded in Fuchsia's symbolization markup format.
	34	// Fuchsia's logging infrastructure emits enough information about
	35	// process memory layout that a post-processing filter can do the
	36	// symbolization and pretty-print the markup. See the spec at:
	37	// https://fuchsia.googlesource.com/zircon/+/master/docs/symbolizer_markup.md
	38
	39	// This is used by UBSan for type names, and by ASan for global variable names.
	40	// It's expected to return a static buffer that will be reused on each call.
	41	const char Symbolizer::Demangle(const char name) {
	42	static char buffer[kFormatDemangleMax];
	43	internal_snprintf(buffer, sizeof(buffer), kFormatDemangle, name);
	44	return buffer;
	45	}
	46
	47	// This is used mostly for suppression matching. Making it work
	48	// would enable "interceptor_via_lib" suppressions. It's also used
	49	// once in UBSan to say "in module ..." in a message that also
	50	// includes an address in the module, so post-processing can already
	51	// pretty-print that so as to indicate the module.
	52	bool Symbolizer::GetModuleNameAndOffsetForPC(uptr pc, const char **module_name,
	53	uptr *module_address) {
	54	return false;
	55	}
	56
	57	// This is used in some places for suppression checking, which we
	58	// don't really support for Fuchsia. It's also used in UBSan to
	59	// identify a PC location to a function name, so we always fill in
	60	// the function member with a string containing markup around the PC
	61	// value.
	62	// TODO(mcgrathr): Under SANITIZER_GO, it's currently used by TSan
	63	// to render stack frames, but that should be changed to use
	64	// RenderStackFrame.
	65	SymbolizedStack *Symbolizer::SymbolizePC(uptr addr) {
	66	SymbolizedStack *s = SymbolizedStack::New(addr);
	67	char buffer[kFormatFunctionMax];
	68	internal_snprintf(buffer, sizeof(buffer), kFormatFunction, addr);
	69	s->info.function = internal_strdup(buffer);
	70	return s;
	71	}
	72
	73	// Always claim we succeeded, so that RenderDataInfo will be called.
	74	bool Symbolizer::SymbolizeData(uptr addr, DataInfo *info) {
	75	info->Clear();
	76	info->start = addr;
	77	return true;
	78	}
	79
	80	// We ignore the format argument to __sanitizer_symbolize_global.
	81	void RenderData(InternalScopedString buffer, const char format,
	82	const DataInfo DI, const char strip_path_prefix) {
	83	buffer->append(kFormatData, DI->start);
	84	}
	85
	86	// We don't support the stack_trace_format flag at all.
	87	void RenderFrame(InternalScopedString buffer, const char format, int frame_no,
	88	const AddressInfo &info, bool vs_style,
	89	const char strip_path_prefix, const char strip_func_prefix) {
	90	buffer->append(kFormatFrame, frame_no, info.address);
	91	}
	92
	93	Symbolizer *Symbolizer::PlatformInit() {
	94	return new (symbolizer_allocator_) Symbolizer({});
	95	}
	96
	97	void Symbolizer::LateInitialize() {
	98	Symbolizer::GetOrInit()->LateInitializeTools();
	99	}
	100
	101	void StartReportDeadlySignal() {}
	102	void ReportDeadlySignal(const SignalContext &sig, u32 tid,
	103	UnwindSignalStackCallbackType unwind,
	104	const void *unwind_context) {}
	105
	106	#if SANITIZER_CAN_SLOW_UNWIND
	107	struct UnwindTraceArg {
	108	BufferedStackTrace *stack;
	109	u32 max_depth;
	110	};
	111
	112	_Unwind_Reason_Code Unwind_Trace(struct _Unwind_Context ctx, void param) {
	113	UnwindTraceArg arg = static_cast<UnwindTraceArg >(param);
	114	CHECK_LT(arg->stack->size, arg->max_depth);
	115	uptr pc = _Unwind_GetIP(ctx);
	116	if (pc < PAGE_SIZE) return _URC_NORMAL_STOP;
	117	arg->stack->trace_buffer[arg->stack->size++] = pc;
	118	return (arg->stack->size == arg->max_depth ? _URC_NORMAL_STOP
	119	: _URC_NO_REASON);
	120	}
	121
	122	void BufferedStackTrace::UnwindSlow(uptr pc, u32 max_depth) {
	123	CHECK_GE(max_depth, 2);
	124	size = 0;
	125	UnwindTraceArg arg = {this, Min(max_depth + 1, kStackTraceMax)};
	126	_Unwind_Backtrace(Unwind_Trace, &arg);
	127	CHECK_GT(size, 0);
	128	// We need to pop a few frames so that pc is on top.
	129	uptr to_pop = LocatePcInTrace(pc);
	130	// trace_buffer[0] belongs to the current function so we always pop it,
	131	// unless there is only 1 frame in the stack trace (1 frame is always better
	132	// than 0!).
	133	PopStackFrames(Min(to_pop, static_cast<uptr>(1)));
	134	trace_buffer[0] = pc;
	135	}
	136
	137	void BufferedStackTrace::UnwindSlow(uptr pc, void *context, u32 max_depth) {
	138	CHECK(context);
	139	CHECK_GE(max_depth, 2);
	140	UNREACHABLE("signal context doesn't exist");
	141	}
	142	#endif // SANITIZER_CAN_SLOW_UNWIND
	143
	144	} // namespace __sanitizer
	145
	146	#endif // SANITIZER_SYMBOLIZER_MARKUP

lib/tsan/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cpp created+492

...	...	@@ -0,0 +1,492 @@
	1	//===-- sanitizer_symbolizer_posix_libcdep.cpp ----------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file is shared between AddressSanitizer and ThreadSanitizer
	10	// run-time libraries.
	11	// POSIX-specific implementation of symbolizer parts.
	12	//===----------------------------------------------------------------------===//
	13
	14	#include "sanitizer_platform.h"
	15	#if SANITIZER_POSIX
	16	#include "sanitizer_allocator_internal.h"
	17	#include "sanitizer_common.h"
	18	#include "sanitizer_file.h"
	19	#include "sanitizer_flags.h"
	20	#include "sanitizer_internal_defs.h"
	21	#include "sanitizer_linux.h"
	22	#include "sanitizer_placement_new.h"
	23	#include "sanitizer_posix.h"
	24	#include "sanitizer_procmaps.h"
	25	#include "sanitizer_symbolizer_internal.h"
	26	#include "sanitizer_symbolizer_libbacktrace.h"
	27	#include "sanitizer_symbolizer_mac.h"
	28
	29	#include <dlfcn.h> // for dlsym()
	30	#include <errno.h>
	31	#include <stdint.h>
	32	#include <stdlib.h>
	33	#include <sys/wait.h>
	34	#include <unistd.h>
	35
	36	// C++ demangling function, as required by Itanium C++ ABI. This is weak,
	37	// because we do not require a C++ ABI library to be linked to a program
	38	// using sanitizers; if it's not present, we'll just use the mangled name.
	39	namespace __cxxabiv1 {
	40	extern "C" SANITIZER_WEAK_ATTRIBUTE
	41	char __cxa_demangle(const char mangled, char *buffer,
	42	size_t length, int status);
	43	}
	44
	45	namespace __sanitizer {
	46
	47	// Attempts to demangle the name via __cxa_demangle from __cxxabiv1.
	48	const char DemangleCXXABI(const char name) {
	49	// FIXME: __cxa_demangle aggressively insists on allocating memory.
	50	// There's not much we can do about that, short of providing our
	51	// own demangler (libc++abi's implementation could be adapted so that
	52	// it does not allocate). For now, we just call it anyway, and we leak
	53	// the returned value.
	54	if (&__cxxabiv1::__cxa_demangle)
	55	if (const char *demangled_name =
	56	__cxxabiv1::__cxa_demangle(name, 0, 0, 0))
	57	return demangled_name;
	58
	59	return name;
	60	}
	61
	62	// As of now, there are no headers for the Swift runtime. Once they are
	63	// present, we will weakly link since we do not require Swift runtime to be
	64	// linked.
	65	typedef char (swift_demangle_ft)(const char *mangledName,
	66	size_t mangledNameLength, char *outputBuffer,
	67	size_t *outputBufferSize, uint32_t flags);
	68	static swift_demangle_ft swift_demangle_f;
	69
	70	// This must not happen lazily at symbolication time, because dlsym uses
	71	// malloc and thread-local storage, which is not a good thing to do during
	72	// symbolication.
	73	static void InitializeSwiftDemangler() {
	74	swift_demangle_f = (swift_demangle_ft)dlsym(RTLD_DEFAULT, "swift_demangle");
	75	(void)dlerror(); // Cleanup error message in case of failure
	76	}
	77
	78	// Attempts to demangle a Swift name. The demangler will return nullptr if a
	79	// non-Swift name is passed in.
	80	const char DemangleSwift(const char name) {
	81	if (swift_demangle_f)
	82	return swift_demangle_f(name, internal_strlen(name), 0, 0, 0);
	83
	84	return nullptr;
	85	}
	86
	87	const char DemangleSwiftAndCXX(const char name) {
	88	if (!name) return nullptr;
	89	if (const char *swift_demangled_name = DemangleSwift(name))
	90	return swift_demangled_name;
	91	return DemangleCXXABI(name);
	92	}
	93
	94	static bool CreateTwoHighNumberedPipes(int infd_, int outfd_) {
	95	int *infd = NULL;
	96	int *outfd = NULL;
	97	// The client program may close its stdin and/or stdout and/or stderr
	98	// thus allowing socketpair to reuse file descriptors 0, 1 or 2.
	99	// In this case the communication between the forked processes may be
	100	// broken if either the parent or the child tries to close or duplicate
	101	// these descriptors. The loop below produces two pairs of file
	102	// descriptors, each greater than 2 (stderr).
	103	int sock_pair[5][2];
	104	for (int i = 0; i < 5; i++) {
	105	if (pipe(sock_pair[i]) == -1) {
	106	for (int j = 0; j < i; j++) {
	107	internal_close(sock_pair[j][0]);
	108	internal_close(sock_pair[j][1]);
	109	}
	110	return false;
	111	} else if (sock_pair[i][0] > 2 && sock_pair[i][1] > 2) {
	112	if (infd == NULL) {
	113	infd = sock_pair[i];
	114	} else {
	115	outfd = sock_pair[i];
	116	for (int j = 0; j < i; j++) {
	117	if (sock_pair[j] == infd) continue;
	118	internal_close(sock_pair[j][0]);
	119	internal_close(sock_pair[j][1]);
	120	}
	121	break;
	122	}
	123	}
	124	}
	125	CHECK(infd);
	126	CHECK(outfd);
	127	infd_[0] = infd[0];
	128	infd_[1] = infd[1];
	129	outfd_[0] = outfd[0];
	130	outfd_[1] = outfd[1];
	131	return true;
	132	}
	133
	134	bool SymbolizerProcess::StartSymbolizerSubprocess() {
	135	if (!FileExists(path_)) {
	136	if (!reported_invalid_path_) {
	137	Report("WARNING: invalid path to external symbolizer!\n");
	138	reported_invalid_path_ = true;
	139	}
	140	return false;
	141	}
	142
	143	const char *argv[kArgVMax];
	144	GetArgV(path_, argv);
	145	pid_t pid;
	146
	147	// Report how symbolizer is being launched for debugging purposes.
	148	if (Verbosity() >= 3) {
	149	// Only use `Report` for first line so subsequent prints don't get prefixed
	150	// with current PID.
	151	Report("Launching Symbolizer process: ");
	152	for (unsigned index = 0; index < kArgVMax && argv[index]; ++index)
	153	Printf("%s ", argv[index]);
	154	Printf("\n");
	155	}
	156
	157	if (use_posix_spawn_) {
	158	#if SANITIZER_MAC
	159	fd_t fd = internal_spawn(argv, const_cast<const char **>(GetEnvP()), &pid);
	160	if (fd == kInvalidFd) {
	161	Report("WARNING: failed to spawn external symbolizer (errno: %d)\n",
	162	errno);
	163	return false;
	164	}
	165
	166	input_fd_ = fd;
	167	output_fd_ = fd;
	168	#else // SANITIZER_MAC
	169	UNIMPLEMENTED();
	170	#endif // SANITIZER_MAC
	171	} else {
	172	fd_t infd[2] = {}, outfd[2] = {};
	173	if (!CreateTwoHighNumberedPipes(infd, outfd)) {
	174	Report("WARNING: Can't create a socket pair to start "
	175	"external symbolizer (errno: %d)\n", errno);
	176	return false;
	177	}
	178
	179	pid = StartSubprocess(path_, argv, GetEnvP(), /* stdin */ outfd[0],
	180	/* stdout */ infd[1]);
	181	if (pid < 0) {
	182	internal_close(infd[0]);
	183	internal_close(outfd[1]);
	184	return false;
	185	}
	186
	187	input_fd_ = infd[0];
	188	output_fd_ = outfd[1];
	189	}
	190
	191	CHECK_GT(pid, 0);
	192
	193	// Check that symbolizer subprocess started successfully.
	194	SleepForMillis(kSymbolizerStartupTimeMillis);
	195	if (!IsProcessRunning(pid)) {
	196	// Either waitpid failed, or child has already exited.
	197	Report("WARNING: external symbolizer didn't start up correctly!\n");
	198	return false;
	199	}
	200
	201	return true;
	202	}
	203
	204	class Addr2LineProcess : public SymbolizerProcess {
	205	public:
	206	Addr2LineProcess(const char path, const char module_name)
	207	: SymbolizerProcess(path), module_name_(internal_strdup(module_name)) {}
	208
	209	const char *module_name() const { return module_name_; }
	210
	211	private:
	212	void GetArgV(const char *path_to_binary,
	213	const char *(&argv)[kArgVMax]) const override {
	214	int i = 0;
	215	argv[i++] = path_to_binary;
	216	argv[i++] = "-iCfe";
	217	argv[i++] = module_name_;
	218	argv[i++] = nullptr;
	219	}
	220
	221	bool ReachedEndOfOutput(const char *buffer, uptr length) const override;
	222
	223	bool ReadFromSymbolizer(char *buffer, uptr max_length) override {
	224	if (!SymbolizerProcess::ReadFromSymbolizer(buffer, max_length))
	225	return false;
	226	// The returned buffer is empty when output is valid, but exceeds
	227	// max_length.
	228	if (*buffer == '\0')
	229	return true;
	230	// We should cut out output_terminator_ at the end of given buffer,
	231	// appended by addr2line to mark the end of its meaningful output.
	232	// We cannot scan buffer from it's beginning, because it is legal for it
	233	// to start with output_terminator_ in case given offset is invalid. So,
	234	// scanning from second character.
	235	char *garbage = internal_strstr(buffer + 1, output_terminator_);
	236	// This should never be NULL since buffer must end up with
	237	// output_terminator_.
	238	CHECK(garbage);
	239	// Trim the buffer.
	240	garbage[0] = '\0';
	241	return true;
	242	}
	243
	244	const char *module_name_; // Owned, leaked.
	245	static const char output_terminator_[];
	246	};
	247
	248	const char Addr2LineProcess::output_terminator_[] = "??\n??:0\n";
	249
	250	bool Addr2LineProcess::ReachedEndOfOutput(const char *buffer,
	251	uptr length) const {
	252	const size_t kTerminatorLen = sizeof(output_terminator_) - 1;
	253	// Skip, if we read just kTerminatorLen bytes, because Addr2Line output
	254	// should consist at least of two pairs of lines:
	255	// 1. First one, corresponding to given offset to be symbolized
	256	// (may be equal to output_terminator_, if offset is not valid).
	257	// 2. Second one for output_terminator_, itself to mark the end of output.
	258	if (length <= kTerminatorLen) return false;
	259	// Addr2Line output should end up with output_terminator_.
	260	return !internal_memcmp(buffer + length - kTerminatorLen,
	261	output_terminator_, kTerminatorLen);
	262	}
	263
	264	class Addr2LinePool : public SymbolizerTool {
	265	public:
	266	explicit Addr2LinePool(const char *addr2line_path,
	267	LowLevelAllocator *allocator)
	268	: addr2line_path_(addr2line_path), allocator_(allocator) {
	269	addr2line_pool_.reserve(16);
	270	}
	271
	272	bool SymbolizePC(uptr addr, SymbolizedStack *stack) override {
	273	if (const char *buf =
	274	SendCommand(stack->info.module, stack->info.module_offset)) {
	275	ParseSymbolizePCOutput(buf, stack);
	276	return true;
	277	}
	278	return false;
	279	}
	280
	281	bool SymbolizeData(uptr addr, DataInfo *info) override {
	282	return false;
	283	}
	284
	285	private:
	286	const char SendCommand(const char module_name, uptr module_offset) {
	287	Addr2LineProcess *addr2line = 0;
	288	for (uptr i = 0; i < addr2line_pool_.size(); ++i) {
	289	if (0 ==
	290	internal_strcmp(module_name, addr2line_pool_[i]->module_name())) {
	291	addr2line = addr2line_pool_[i];
	292	break;
	293	}
	294	}
	295	if (!addr2line) {
	296	addr2line =
	297	new(*allocator_) Addr2LineProcess(addr2line_path_, module_name);
	298	addr2line_pool_.push_back(addr2line);
	299	}
	300	CHECK_EQ(0, internal_strcmp(module_name, addr2line->module_name()));
	301	char buffer[kBufferSize];
	302	internal_snprintf(buffer, kBufferSize, "0x%zx\n0x%zx\n",
	303	module_offset, dummy_address_);
	304	return addr2line->SendCommand(buffer);
	305	}
	306
	307	static const uptr kBufferSize = 64;
	308	const char *addr2line_path_;
	309	LowLevelAllocator *allocator_;
	310	InternalMmapVector<Addr2LineProcess*> addr2line_pool_;
	311	static const uptr dummy_address_ =
	312	FIRST_32_SECOND_64(UINT32_MAX, UINT64_MAX);
	313	};
	314
	315	#if SANITIZER_SUPPORTS_WEAK_HOOKS
	316	extern "C" {
	317	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE bool
	318	__sanitizer_symbolize_code(const char *ModuleName, u64 ModuleOffset,
	319	char *Buffer, int MaxLength,
	320	bool SymbolizeInlineFrames);
	321	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
	322	bool __sanitizer_symbolize_data(const char *ModuleName, u64 ModuleOffset,
	323	char *Buffer, int MaxLength);
	324	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
	325	void __sanitizer_symbolize_flush();
	326	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
	327	int __sanitizer_symbolize_demangle(const char Name, char Buffer,
	328	int MaxLength);
	329	} // extern "C"
	330
	331	class InternalSymbolizer : public SymbolizerTool {
	332	public:
	333	static InternalSymbolizer get(LowLevelAllocator alloc) {
	334	if (__sanitizer_symbolize_code != 0 &&
	335	__sanitizer_symbolize_data != 0) {
	336	return new(*alloc) InternalSymbolizer();
	337	}
	338	return 0;
	339	}
	340
	341	bool SymbolizePC(uptr addr, SymbolizedStack *stack) override {
	342	bool result = __sanitizer_symbolize_code(
	343	stack->info.module, stack->info.module_offset, buffer_, kBufferSize,
	344	common_flags()->symbolize_inline_frames);
	345	if (result) ParseSymbolizePCOutput(buffer_, stack);
	346	return result;
	347	}
	348
	349	bool SymbolizeData(uptr addr, DataInfo *info) override {
	350	bool result = __sanitizer_symbolize_data(info->module, info->module_offset,
	351	buffer_, kBufferSize);
	352	if (result) {
	353	ParseSymbolizeDataOutput(buffer_, info);
	354	info->start += (addr - info->module_offset); // Add the base address.
	355	}
	356	return result;
	357	}
	358
	359	void Flush() override {
	360	if (__sanitizer_symbolize_flush)
	361	__sanitizer_symbolize_flush();
	362	}
	363
	364	const char Demangle(const char name) override {
	365	if (__sanitizer_symbolize_demangle) {
	366	for (uptr res_length = 1024;
	367	res_length <= InternalSizeClassMap::kMaxSize;) {
	368	char res_buff = static_cast<char>(InternalAlloc(res_length));
	369	uptr req_length =
	370	__sanitizer_symbolize_demangle(name, res_buff, res_length);
	371	if (req_length > res_length) {
	372	res_length = req_length + 1;
	373	InternalFree(res_buff);
	374	continue;
	375	}
	376	return res_buff;
	377	}
	378	}
	379	return name;
	380	}
	381
	382	private:
	383	InternalSymbolizer() { }
	384
	385	static const int kBufferSize = 16 * 1024;
	386	char buffer_[kBufferSize];
	387	};
	388	#else // SANITIZER_SUPPORTS_WEAK_HOOKS
	389
	390	class InternalSymbolizer : public SymbolizerTool {
	391	public:
	392	static InternalSymbolizer get(LowLevelAllocator alloc) { return 0; }
	393	};
	394
	395	#endif // SANITIZER_SUPPORTS_WEAK_HOOKS
	396
	397	const char Symbolizer::PlatformDemangle(const char name) {
	398	return DemangleSwiftAndCXX(name);
	399	}
	400
	401	static SymbolizerTool ChooseExternalSymbolizer(LowLevelAllocator allocator) {
	402	const char *path = common_flags()->external_symbolizer_path;
	403	const char *binary_name = path ? StripModuleName(path) : "";
	404	if (path && path[0] == '\0') {
	405	VReport(2, "External symbolizer is explicitly disabled.\n");
	406	return nullptr;
	407	} else if (!internal_strcmp(binary_name, "llvm-symbolizer")) {
	408	VReport(2, "Using llvm-symbolizer at user-specified path: %s\n", path);
	409	return new(*allocator) LLVMSymbolizer(path, allocator);
	410	} else if (!internal_strcmp(binary_name, "atos")) {
	411	#if SANITIZER_MAC
	412	VReport(2, "Using atos at user-specified path: %s\n", path);
	413	return new(*allocator) AtosSymbolizer(path, allocator);
	414	#else // SANITIZER_MAC
	415	Report("ERROR: Using `atos` is only supported on Darwin.\n");
	416	Die();
	417	#endif // SANITIZER_MAC
	418	} else if (!internal_strcmp(binary_name, "addr2line")) {
	419	VReport(2, "Using addr2line at user-specified path: %s\n", path);
	420	return new(*allocator) Addr2LinePool(path, allocator);
	421	} else if (path) {
	422	Report("ERROR: External symbolizer path is set to '%s' which isn't "
	423	"a known symbolizer. Please set the path to the llvm-symbolizer "
	424	"binary or other known tool.\n", path);
	425	Die();
	426	}
	427
	428	// Otherwise symbolizer program is unknown, let's search $PATH
	429	CHECK(path == nullptr);
	430	#if SANITIZER_MAC
	431	if (const char *found_path = FindPathToBinary("atos")) {
	432	VReport(2, "Using atos found at: %s\n", found_path);
	433	return new(*allocator) AtosSymbolizer(found_path, allocator);
	434	}
	435	#endif // SANITIZER_MAC
	436	if (const char *found_path = FindPathToBinary("llvm-symbolizer")) {
	437	VReport(2, "Using llvm-symbolizer found at: %s\n", found_path);
	438	return new(*allocator) LLVMSymbolizer(found_path, allocator);
	439	}
	440	if (common_flags()->allow_addr2line) {
	441	if (const char *found_path = FindPathToBinary("addr2line")) {
	442	VReport(2, "Using addr2line found at: %s\n", found_path);
	443	return new(*allocator) Addr2LinePool(found_path, allocator);
	444	}
	445	}
	446	return nullptr;
	447	}
	448
	449	static void ChooseSymbolizerTools(IntrusiveList<SymbolizerTool> *list,
	450	LowLevelAllocator *allocator) {
	451	if (!common_flags()->symbolize) {
	452	VReport(2, "Symbolizer is disabled.\n");
	453	return;
	454	}
	455	if (IsAllocatorOutOfMemory()) {
	456	VReport(2, "Cannot use internal symbolizer: out of memory\n");
	457	} else if (SymbolizerTool *tool = InternalSymbolizer::get(allocator)) {
	458	VReport(2, "Using internal symbolizer.\n");
	459	list->push_back(tool);
	460	return;
	461	}
	462	if (SymbolizerTool *tool = LibbacktraceSymbolizer::get(allocator)) {
	463	VReport(2, "Using libbacktrace symbolizer.\n");
	464	list->push_back(tool);
	465	return;
	466	}
	467
	468	if (SymbolizerTool *tool = ChooseExternalSymbolizer(allocator)) {
	469	list->push_back(tool);
	470	}
	471
	472	#if SANITIZER_MAC
	473	VReport(2, "Using dladdr symbolizer.\n");
	474	list->push_back(new(*allocator) DlAddrSymbolizer());
	475	#endif // SANITIZER_MAC
	476	}
	477
	478	Symbolizer *Symbolizer::PlatformInit() {
	479	IntrusiveList<SymbolizerTool> list;
	480	list.clear();
	481	ChooseSymbolizerTools(&list, &symbolizer_allocator_);
	482	return new(symbolizer_allocator_) Symbolizer(list);
	483	}
	484
	485	void Symbolizer::LateInitialize() {
	486	Symbolizer::GetOrInit()->LateInitializeTools();
	487	InitializeSwiftDemangler();
	488	}
	489
	490	} // namespace __sanitizer
	491
	492	#endif // SANITIZER_POSIX

lib/tsan/sanitizer_common/sanitizer_symbolizer_report.cpp created+293

...	...	@@ -0,0 +1,293 @@
	1	//===-- sanitizer_symbolizer_report.cpp -----------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	///
	9	/// This file is shared between AddressSanitizer and other sanitizer run-time
	10	/// libraries and implements symbolized reports related functions.
	11	///
	12	//===----------------------------------------------------------------------===//
	13
	14	#include "sanitizer_common.h"
	15	#include "sanitizer_file.h"
	16	#include "sanitizer_flags.h"
	17	#include "sanitizer_procmaps.h"
	18	#include "sanitizer_report_decorator.h"
	19	#include "sanitizer_stacktrace.h"
	20	#include "sanitizer_stacktrace_printer.h"
	21	#include "sanitizer_symbolizer.h"
	22
	23	#if SANITIZER_POSIX
	24	# include "sanitizer_posix.h"
	25	# include <sys/mman.h>
	26	#endif
	27
	28	namespace __sanitizer {
	29
	30	#if !SANITIZER_GO
	31	void ReportErrorSummary(const char *error_type, const AddressInfo &info,
	32	const char *alt_tool_name) {
	33	if (!common_flags()->print_summary) return;
	34	InternalScopedString buff(kMaxSummaryLength);
	35	buff.append("%s ", error_type);
	36	RenderFrame(&buff, "%L %F", 0, info, common_flags()->symbolize_vs_style,
	37	common_flags()->strip_path_prefix);
	38	ReportErrorSummary(buff.data(), alt_tool_name);
	39	}
	40	#endif
	41
	42	#if !SANITIZER_FUCHSIA
	43
	44	bool ReportFile::SupportsColors() {
	45	SpinMutexLock l(mu);
	46	ReopenIfNecessary();
	47	return SupportsColoredOutput(fd);
	48	}
	49
	50	static INLINE bool ReportSupportsColors() {
	51	return report_file.SupportsColors();
	52	}
	53
	54	#else // SANITIZER_FUCHSIA
	55
	56	// Fuchsia's logs always go through post-processing that handles colorization.
	57	static INLINE bool ReportSupportsColors() { return true; }
	58
	59	#endif // !SANITIZER_FUCHSIA
	60
	61	bool ColorizeReports() {
	62	// FIXME: Add proper Windows support to AnsiColorDecorator and re-enable color
	63	// printing on Windows.
	64	if (SANITIZER_WINDOWS)
	65	return false;
	66
	67	const char *flag = common_flags()->color;
	68	return internal_strcmp(flag, "always") == 0 \|\|
	69	(internal_strcmp(flag, "auto") == 0 && ReportSupportsColors());
	70	}
	71
	72	void ReportErrorSummary(const char error_type, const StackTrace stack,
	73	const char *alt_tool_name) {
	74	#if !SANITIZER_GO
	75	if (!common_flags()->print_summary)
	76	return;
	77	if (stack->size == 0) {
	78	ReportErrorSummary(error_type);
	79	return;
	80	}
	81	// Currently, we include the first stack frame into the report summary.
	82	// Maybe sometimes we need to choose another frame (e.g. skip memcpy/etc).
	83	uptr pc = StackTrace::GetPreviousInstructionPc(stack->trace[0]);
	84	SymbolizedStack *frame = Symbolizer::GetOrInit()->SymbolizePC(pc);
	85	ReportErrorSummary(error_type, frame->info, alt_tool_name);
	86	frame->ClearAll();
	87	#endif
	88	}
	89
	90	void ReportMmapWriteExec(int prot) {
	91	#if SANITIZER_POSIX && (!SANITIZER_GO && !SANITIZER_ANDROID)
	92	if ((prot & (PROT_WRITE \| PROT_EXEC)) != (PROT_WRITE \| PROT_EXEC))
	93	return;
	94
	95	ScopedErrorReportLock l;
	96	SanitizerCommonDecorator d;
	97
	98	InternalMmapVector<BufferedStackTrace> stack_buffer(1);
	99	BufferedStackTrace *stack = stack_buffer.data();
	100	stack->Reset();
	101	uptr top = 0;
	102	uptr bottom = 0;
	103	GET_CALLER_PC_BP_SP;
	104	(void)sp;
	105	bool fast = common_flags()->fast_unwind_on_fatal;
	106	if (StackTrace::WillUseFastUnwind(fast)) {
	107	GetThreadStackTopAndBottom(false, &top, &bottom);
	108	stack->Unwind(kStackTraceMax, pc, bp, nullptr, top, bottom, true);
	109	} else {
	110	stack->Unwind(kStackTraceMax, pc, 0, nullptr, 0, 0, false);
	111	}
	112
	113	Printf("%s", d.Warning());
	114	Report("WARNING: %s: writable-executable page usage\n", SanitizerToolName);
	115	Printf("%s", d.Default());
	116
	117	stack->Print();
	118	ReportErrorSummary("w-and-x-usage", stack);
	119	#endif
	120	}
	121
	122	#if !SANITIZER_FUCHSIA && !SANITIZER_RTEMS && !SANITIZER_GO
	123	void StartReportDeadlySignal() {
	124	// Write the first message using fd=2, just in case.
	125	// It may actually fail to write in case stderr is closed.
	126	CatastrophicErrorWrite(SanitizerToolName, internal_strlen(SanitizerToolName));
	127	static const char kDeadlySignal[] = ":DEADLYSIGNAL\n";
	128	CatastrophicErrorWrite(kDeadlySignal, sizeof(kDeadlySignal) - 1);
	129	}
	130
	131	static void MaybeReportNonExecRegion(uptr pc) {
	132	#if SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_NETBSD
	133	MemoryMappingLayout proc_maps(/cache_enabled/ true);
	134	MemoryMappedSegment segment;
	135	while (proc_maps.Next(&segment)) {
	136	if (pc >= segment.start && pc < segment.end && !segment.IsExecutable())
	137	Report("Hint: PC is at a non-executable region. Maybe a wild jump?\n");
	138	}
	139	#endif
	140	}
	141
	142	static void PrintMemoryByte(InternalScopedString str, const char before,
	143	u8 byte) {
	144	SanitizerCommonDecorator d;
	145	str->append("%s%s%x%x%s ", before, d.MemoryByte(), byte >> 4, byte & 15,
	146	d.Default());
	147	}
	148
	149	static void MaybeDumpInstructionBytes(uptr pc) {
	150	if (!common_flags()->dump_instruction_bytes \|\| (pc < GetPageSizeCached()))
	151	return;
	152	InternalScopedString str(1024);
	153	str.append("First 16 instruction bytes at pc: ");
	154	if (IsAccessibleMemoryRange(pc, 16)) {
	155	for (int i = 0; i < 16; ++i) {
	156	PrintMemoryByte(&str, "", ((u8 *)pc)[i]);
	157	}
	158	str.append("\n");
	159	} else {
	160	str.append("unaccessible\n");
	161	}
	162	Report("%s", str.data());
	163	}
	164
	165	static void MaybeDumpRegisters(void *context) {
	166	if (!common_flags()->dump_registers) return;
	167	SignalContext::DumpAllRegisters(context);
	168	}
	169
	170	static void ReportStackOverflowImpl(const SignalContext &sig, u32 tid,
	171	UnwindSignalStackCallbackType unwind,
	172	const void *unwind_context) {
	173	SanitizerCommonDecorator d;
	174	Printf("%s", d.Warning());
	175	static const char kDescription[] = "stack-overflow";
	176	Report("ERROR: %s: %s on address %p (pc %p bp %p sp %p T%d)\n",
	177	SanitizerToolName, kDescription, (void )sig.addr, (void )sig.pc,
	178	(void )sig.bp, (void )sig.sp, tid);
	179	Printf("%s", d.Default());
	180	InternalMmapVector<BufferedStackTrace> stack_buffer(1);
	181	BufferedStackTrace *stack = stack_buffer.data();
	182	stack->Reset();
	183	unwind(sig, unwind_context, stack);
	184	stack->Print();
	185	ReportErrorSummary(kDescription, stack);
	186	}
	187
	188	static void ReportDeadlySignalImpl(const SignalContext &sig, u32 tid,
	189	UnwindSignalStackCallbackType unwind,
	190	const void *unwind_context) {
	191	SanitizerCommonDecorator d;
	192	Printf("%s", d.Warning());
	193	const char *description = sig.Describe();
	194	if (sig.is_memory_access && !sig.is_true_faulting_addr)
	195	Report("ERROR: %s: %s on unknown address (pc %p bp %p sp %p T%d)\n",
	196	SanitizerToolName, description, (void )sig.pc, (void )sig.bp,
	197	(void *)sig.sp, tid);
	198	else
	199	Report("ERROR: %s: %s on unknown address %p (pc %p bp %p sp %p T%d)\n",
	200	SanitizerToolName, description, (void )sig.addr, (void )sig.pc,
	201	(void )sig.bp, (void )sig.sp, tid);
	202	Printf("%s", d.Default());
	203	if (sig.pc < GetPageSizeCached())
	204	Report("Hint: pc points to the zero page.\n");
	205	if (sig.is_memory_access) {
	206	const char *access_type =
	207	sig.write_flag == SignalContext::WRITE
	208	? "WRITE"
	209	: (sig.write_flag == SignalContext::READ ? "READ" : "UNKNOWN");
	210	Report("The signal is caused by a %s memory access.\n", access_type);
	211	if (!sig.is_true_faulting_addr)
	212	Report("Hint: this fault was caused by a dereference of a high value "
	213	"address (see register values below). Dissassemble the provided "
	214	"pc to learn which register was used.\n");
	215	else if (sig.addr < GetPageSizeCached())
	216	Report("Hint: address points to the zero page.\n");
	217	}
	218	MaybeReportNonExecRegion(sig.pc);
	219	InternalMmapVector<BufferedStackTrace> stack_buffer(1);
	220	BufferedStackTrace *stack = stack_buffer.data();
	221	stack->Reset();
	222	unwind(sig, unwind_context, stack);
	223	stack->Print();
	224	MaybeDumpInstructionBytes(sig.pc);
	225	MaybeDumpRegisters(sig.context);
	226	Printf("%s can not provide additional info.\n", SanitizerToolName);
	227	ReportErrorSummary(description, stack);
	228	}
	229
	230	void ReportDeadlySignal(const SignalContext &sig, u32 tid,
	231	UnwindSignalStackCallbackType unwind,
	232	const void *unwind_context) {
	233	if (sig.IsStackOverflow())
	234	ReportStackOverflowImpl(sig, tid, unwind, unwind_context);
	235	else
	236	ReportDeadlySignalImpl(sig, tid, unwind, unwind_context);
	237	}
	238
	239	void HandleDeadlySignal(void siginfo, void context, u32 tid,
	240	UnwindSignalStackCallbackType unwind,
	241	const void *unwind_context) {
	242	StartReportDeadlySignal();
	243	ScopedErrorReportLock rl;
	244	SignalContext sig(siginfo, context);
	245	ReportDeadlySignal(sig, tid, unwind, unwind_context);
	246	Report("ABORTING\n");
	247	Die();
	248	}
	249
	250	#endif // !SANITIZER_FUCHSIA && !SANITIZER_GO
	251
	252	static atomic_uintptr_t reporting_thread = {0};
	253	static StaticSpinMutex CommonSanitizerReportMutex;
	254
	255	ScopedErrorReportLock::ScopedErrorReportLock() {
	256	uptr current = GetThreadSelf();
	257	for (;;) {
	258	uptr expected = 0;
	259	if (atomic_compare_exchange_strong(&reporting_thread, &expected, current,
	260	memory_order_relaxed)) {
	261	// We've claimed reporting_thread so proceed.
	262	CommonSanitizerReportMutex.Lock();
	263	return;
	264	}
	265
	266	if (expected == current) {
	267	// This is either asynch signal or nested error during error reporting.
	268	// Fail simple to avoid deadlocks in Report().
	269
	270	// Can't use Report() here because of potential deadlocks in nested
	271	// signal handlers.
	272	CatastrophicErrorWrite(SanitizerToolName,
	273	internal_strlen(SanitizerToolName));
	274	static const char msg[] = ": nested bug in the same thread, aborting.\n";
	275	CatastrophicErrorWrite(msg, sizeof(msg) - 1);
	276
	277	internal__exit(common_flags()->exitcode);
	278	}
	279
	280	internal_sched_yield();
	281	}
	282	}
	283
	284	ScopedErrorReportLock::~ScopedErrorReportLock() {
	285	CommonSanitizerReportMutex.Unlock();
	286	atomic_store_relaxed(&reporting_thread, 0);
	287	}
	288
	289	void ScopedErrorReportLock::CheckLocked() {
	290	CommonSanitizerReportMutex.CheckLocked();
	291	}
	292
	293	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_symbolizer_win.cpp created+318

...	...	@@ -0,0 +1,318 @@
	1	//===-- sanitizer_symbolizer_win.cpp --------------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file is shared between AddressSanitizer and ThreadSanitizer
	10	// run-time libraries.
	11	// Windows-specific implementation of symbolizer parts.
	12	//===----------------------------------------------------------------------===//
	13
	14	#include "sanitizer_platform.h"
	15	#if SANITIZER_WINDOWS
	16
	17	#include "sanitizer_dbghelp.h"
	18	#include "sanitizer_symbolizer_internal.h"
	19
	20	namespace __sanitizer {
	21
	22	decltype(::StackWalk64) *StackWalk64;
	23	decltype(::SymCleanup) *SymCleanup;
	24	decltype(::SymFromAddr) *SymFromAddr;
	25	decltype(::SymFunctionTableAccess64) *SymFunctionTableAccess64;
	26	decltype(::SymGetLineFromAddr64) *SymGetLineFromAddr64;
	27	decltype(::SymGetModuleBase64) *SymGetModuleBase64;
	28	decltype(::SymGetSearchPathW) *SymGetSearchPathW;
	29	decltype(::SymInitialize) *SymInitialize;
	30	decltype(::SymSetOptions) *SymSetOptions;
	31	decltype(::SymSetSearchPathW) *SymSetSearchPathW;
	32	decltype(::UnDecorateSymbolName) *UnDecorateSymbolName;
	33
	34	namespace {
	35
	36	class WinSymbolizerTool : public SymbolizerTool {
	37	public:
	38	// The constructor is provided to avoid synthesized memsets.
	39	WinSymbolizerTool() {}
	40
	41	bool SymbolizePC(uptr addr, SymbolizedStack *stack) override;
	42	bool SymbolizeData(uptr addr, DataInfo *info) override {
	43	return false;
	44	}
	45	const char Demangle(const char name) override;
	46	};
	47
	48	bool is_dbghelp_initialized = false;
	49
	50	bool TrySymInitialize() {
	51	SymSetOptions(SYMOPT_DEFERRED_LOADS \| SYMOPT_UNDNAME \| SYMOPT_LOAD_LINES);
	52	return SymInitialize(GetCurrentProcess(), 0, TRUE);
	53	// FIXME: We don't call SymCleanup() on exit yet - should we?
	54	}
	55
	56	} // namespace
	57
	58	// Initializes DbgHelp library, if it's not yet initialized. Calls to this
	59	// function should be synchronized with respect to other calls to DbgHelp API
	60	// (e.g. from WinSymbolizerTool).
	61	void InitializeDbgHelpIfNeeded() {
	62	if (is_dbghelp_initialized)
	63	return;
	64
	65	HMODULE dbghelp = LoadLibraryA("dbghelp.dll");
	66	CHECK(dbghelp && "failed to load dbghelp.dll");
	67
	68	#define DBGHELP_IMPORT(name) \
	69	do { \
	70	name = \
	71	reinterpret_cast<decltype(::name) *>(GetProcAddress(dbghelp, #name)); \
	72	CHECK(name != nullptr); \
	73	} while (0)
	74	DBGHELP_IMPORT(StackWalk64);
	75	DBGHELP_IMPORT(SymCleanup);
	76	DBGHELP_IMPORT(SymFromAddr);
	77	DBGHELP_IMPORT(SymFunctionTableAccess64);
	78	DBGHELP_IMPORT(SymGetLineFromAddr64);
	79	DBGHELP_IMPORT(SymGetModuleBase64);
	80	DBGHELP_IMPORT(SymGetSearchPathW);
	81	DBGHELP_IMPORT(SymInitialize);
	82	DBGHELP_IMPORT(SymSetOptions);
	83	DBGHELP_IMPORT(SymSetSearchPathW);
	84	DBGHELP_IMPORT(UnDecorateSymbolName);
	85	#undef DBGHELP_IMPORT
	86
	87	if (!TrySymInitialize()) {
	88	// OK, maybe the client app has called SymInitialize already.
	89	// That's a bit unfortunate for us as all the DbgHelp functions are
	90	// single-threaded and we can't coordinate with the app.
	91	// FIXME: Can we stop the other threads at this point?
	92	// Anyways, we have to reconfigure stuff to make sure that SymInitialize
	93	// has all the appropriate options set.
	94	// Cross our fingers and reinitialize DbgHelp.
	95	Report("* WARNING: Failed to initialize DbgHelp! *\n");
	96	Report("* Most likely this means that the app is already *\n");
	97	Report("* using DbgHelp, possibly with incompatible flags. *\n");
	98	Report("* Due to technical reasons, symbolization might crash *\n");
	99	Report("* or produce wrong results. *\n");
	100	SymCleanup(GetCurrentProcess());
	101	TrySymInitialize();
	102	}
	103	is_dbghelp_initialized = true;
	104
	105	// When an executable is run from a location different from the one where it
	106	// was originally built, we may not see the nearby PDB files.
	107	// To work around this, let's append the directory of the main module
	108	// to the symbol search path. All the failures below are not fatal.
	109	const size_t kSymPathSize = 2048;
	110	static wchar_t path_buffer[kSymPathSize + 1 + MAX_PATH];
	111	if (!SymGetSearchPathW(GetCurrentProcess(), path_buffer, kSymPathSize)) {
	112	Report("* WARNING: Failed to SymGetSearchPathW *\n");
	113	return;
	114	}
	115	size_t sz = wcslen(path_buffer);
	116	if (sz) {
	117	CHECK_EQ(0, wcscat_s(path_buffer, L";"));
	118	sz++;
	119	}
	120	DWORD res = GetModuleFileNameW(NULL, path_buffer + sz, MAX_PATH);
	121	if (res == 0 \|\| res == MAX_PATH) {
	122	Report("* WARNING: Failed to getting the EXE directory *\n");
	123	return;
	124	}
	125	// Write the zero character in place of the last backslash to get the
	126	// directory of the main module at the end of path_buffer.
	127	wchar_t *last_bslash = wcsrchr(path_buffer + sz, L'\\');
	128	CHECK_NE(last_bslash, 0);
	129	*last_bslash = L'\0';
	130	if (!SymSetSearchPathW(GetCurrentProcess(), path_buffer)) {
	131	Report("*** WARNING: Failed to SymSetSearchPathW\n");
	132	return;
	133	}
	134	}
	135
	136	bool WinSymbolizerTool::SymbolizePC(uptr addr, SymbolizedStack *frame) {
	137	InitializeDbgHelpIfNeeded();
	138
	139	// See http://msdn.microsoft.com/en-us/library/ms680578(VS.85).aspx
	140	char buffer[sizeof(SYMBOL_INFO) + MAX_SYM_NAME * sizeof(CHAR)];
	141	PSYMBOL_INFO symbol = (PSYMBOL_INFO)buffer;
	142	symbol->SizeOfStruct = sizeof(SYMBOL_INFO);
	143	symbol->MaxNameLen = MAX_SYM_NAME;
	144	DWORD64 offset = 0;
	145	BOOL got_objname = SymFromAddr(GetCurrentProcess(),
	146	(DWORD64)addr, &offset, symbol);
	147	if (!got_objname)
	148	return false;
	149
	150	DWORD unused;
	151	IMAGEHLP_LINE64 line_info;
	152	line_info.SizeOfStruct = sizeof(IMAGEHLP_LINE64);
	153	BOOL got_fileline = SymGetLineFromAddr64(GetCurrentProcess(), (DWORD64)addr,
	154	&unused, &line_info);
	155	frame->info.function = internal_strdup(symbol->Name);
	156	frame->info.function_offset = (uptr)offset;
	157	if (got_fileline) {
	158	frame->info.file = internal_strdup(line_info.FileName);
	159	frame->info.line = line_info.LineNumber;
	160	}
	161	// Only consider this a successful symbolization attempt if we got file info.
	162	// Otherwise, try llvm-symbolizer.
	163	return got_fileline;
	164	}
	165
	166	const char WinSymbolizerTool::Demangle(const char name) {
	167	CHECK(is_dbghelp_initialized);
	168	static char demangle_buffer[1000];
	169	if (name[0] == '\01' &&
	170	UnDecorateSymbolName(name + 1, demangle_buffer, sizeof(demangle_buffer),
	171	UNDNAME_NAME_ONLY))
	172	return demangle_buffer;
	173	else
	174	return name;
	175	}
	176
	177	const char Symbolizer::PlatformDemangle(const char name) {
	178	return name;
	179	}
	180
	181	namespace {
	182	struct ScopedHandle {
	183	ScopedHandle() : h_(nullptr) {}
	184	explicit ScopedHandle(HANDLE h) : h_(h) {}
	185	~ScopedHandle() {
	186	if (h_)
	187	::CloseHandle(h_);
	188	}
	189	HANDLE get() { return h_; }
	190	HANDLE *receive() { return &h_; }
	191	HANDLE release() {
	192	HANDLE h = h_;
	193	h_ = nullptr;
	194	return h;
	195	}
	196	HANDLE h_;
	197	};
	198	} // namespace
	199
	200	bool SymbolizerProcess::StartSymbolizerSubprocess() {
	201	// Create inherited pipes for stdin and stdout.
	202	ScopedHandle stdin_read, stdin_write;
	203	ScopedHandle stdout_read, stdout_write;
	204	SECURITY_ATTRIBUTES attrs;
	205	attrs.nLength = sizeof(SECURITY_ATTRIBUTES);
	206	attrs.bInheritHandle = TRUE;
	207	attrs.lpSecurityDescriptor = nullptr;
	208	if (!::CreatePipe(stdin_read.receive(), stdin_write.receive(), &attrs, 0) \|\|
	209	!::CreatePipe(stdout_read.receive(), stdout_write.receive(), &attrs, 0)) {
	210	VReport(2, "WARNING: %s CreatePipe failed (error code: %d)\n",
	211	SanitizerToolName, path_, GetLastError());
	212	return false;
	213	}
	214
	215	// Don't inherit the writing end of stdin or the reading end of stdout.
	216	if (!SetHandleInformation(stdin_write.get(), HANDLE_FLAG_INHERIT, 0) \|\|
	217	!SetHandleInformation(stdout_read.get(), HANDLE_FLAG_INHERIT, 0)) {
	218	VReport(2, "WARNING: %s SetHandleInformation failed (error code: %d)\n",
	219	SanitizerToolName, path_, GetLastError());
	220	return false;
	221	}
	222
	223	// Compute the command line. Wrap double quotes around everything.
	224	const char *argv[kArgVMax];
	225	GetArgV(path_, argv);
	226	InternalScopedString command_line(kMaxPathLength * 3);
	227	for (int i = 0; argv[i]; i++) {
	228	const char *arg = argv[i];
	229	int arglen = internal_strlen(arg);
	230	// Check that tool command lines are simple and that complete escaping is
	231	// unnecessary.
	232	CHECK(!internal_strchr(arg, '"') && "quotes in args unsupported");
	233	CHECK(!internal_strstr(arg, "\\\\") &&
	234	"double backslashes in args unsupported");
	235	CHECK(arglen > 0 && arg[arglen - 1] != '\\' &&
	236	"args ending in backslash and empty args unsupported");
	237	command_line.append("\"%s\" ", arg);
	238	}
	239	VReport(3, "Launching symbolizer command: %s\n", command_line.data());
	240
	241	// Launch llvm-symbolizer with stdin and stdout redirected.
	242	STARTUPINFOA si;
	243	memset(&si, 0, sizeof(si));
	244	si.cb = sizeof(si);
	245	si.dwFlags \|= STARTF_USESTDHANDLES;
	246	si.hStdInput = stdin_read.get();
	247	si.hStdOutput = stdout_write.get();
	248	PROCESS_INFORMATION pi;
	249	memset(&pi, 0, sizeof(pi));
	250	if (!CreateProcessA(path_, // Executable
	251	command_line.data(), // Command line
	252	nullptr, // Process handle not inheritable
	253	nullptr, // Thread handle not inheritable
	254	TRUE, // Set handle inheritance to TRUE
	255	0, // Creation flags
	256	nullptr, // Use parent's environment block
	257	nullptr, // Use parent's starting directory
	258	&si, &pi)) {
	259	VReport(2, "WARNING: %s failed to create process for %s (error code: %d)\n",
	260	SanitizerToolName, path_, GetLastError());
	261	return false;
	262	}
	263
	264	// Process creation succeeded, so transfer handle ownership into the fields.
	265	input_fd_ = stdout_read.release();
	266	output_fd_ = stdin_write.release();
	267
	268	// The llvm-symbolizer process is responsible for quitting itself when the
	269	// stdin pipe is closed, so we don't need these handles. Close them to prevent
	270	// leaks. If we ever want to try to kill the symbolizer process from the
	271	// parent, we'll want to hang on to these handles.
	272	CloseHandle(pi.hProcess);
	273	CloseHandle(pi.hThread);
	274	return true;
	275	}
	276
	277	static void ChooseSymbolizerTools(IntrusiveList<SymbolizerTool> *list,
	278	LowLevelAllocator *allocator) {
	279	if (!common_flags()->symbolize) {
	280	VReport(2, "Symbolizer is disabled.\n");
	281	return;
	282	}
	283
	284	// Add llvm-symbolizer in case the binary has dwarf.
	285	const char *user_path = common_flags()->external_symbolizer_path;
	286	const char *path =
	287	user_path ? user_path : FindPathToBinary("llvm-symbolizer.exe");
	288	if (path) {
	289	VReport(2, "Using llvm-symbolizer at %spath: %s\n",
	290	user_path ? "user-specified " : "", path);
	291	list->push_back(new(*allocator) LLVMSymbolizer(path, allocator));
	292	} else {
	293	if (user_path && user_path[0] == '\0') {
	294	VReport(2, "External symbolizer is explicitly disabled.\n");
	295	} else {
	296	VReport(2, "External symbolizer is not present.\n");
	297	}
	298	}
	299
	300	// Add the dbghelp based symbolizer.
	301	list->push_back(new(*allocator) WinSymbolizerTool());
	302	}
	303
	304	Symbolizer *Symbolizer::PlatformInit() {
	305	IntrusiveList<SymbolizerTool> list;
	306	list.clear();
	307	ChooseSymbolizerTools(&list, &symbolizer_allocator_);
	308
	309	return new(symbolizer_allocator_) Symbolizer(list);
	310	}
	311
	312	void Symbolizer::LateInitialize() {
	313	Symbolizer::GetOrInit()->LateInitializeTools();
	314	}
	315
	316	} // namespace __sanitizer
	317
	318	#endif // _WIN32

lib/tsan/sanitizer_common/sanitizer_unwind_linux_libcdep.cpp created+180

...	...	@@ -0,0 +1,180 @@
	1	//===-- sanitizer_unwind_linux_libcdep.cpp --------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file contains the unwind.h-based (aka "slow") stack unwinding routines
	10	// available to the tools on Linux, Android, NetBSD, FreeBSD, and Solaris.
	11	//===----------------------------------------------------------------------===//
	12
	13	#include "sanitizer_platform.h"
	14	#if SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_NETBSD \|\| \
	15	SANITIZER_SOLARIS
	16	#include "sanitizer_common.h"
	17	#include "sanitizer_stacktrace.h"
	18
	19	#if SANITIZER_ANDROID
	20	#include <dlfcn.h> // for dlopen()
	21	#endif
	22
	23	#if SANITIZER_FREEBSD
	24	#define _GNU_SOURCE // to declare _Unwind_Backtrace() from <unwind.h>
	25	#endif
	26	#include <unwind.h>
	27
	28	namespace __sanitizer {
	29
	30	namespace {
	31
	32	//---------------------------- UnwindSlow --------------------------------------
	33
	34	typedef struct {
	35	uptr absolute_pc;
	36	uptr stack_top;
	37	uptr stack_size;
	38	} backtrace_frame_t;
	39
	40	extern "C" {
	41	typedef void (acquire_my_map_info_list_func)();
	42	typedef void (release_my_map_info_list_func)(void map);
	43	typedef sptr (*unwind_backtrace_signal_arch_func)(
	44	void siginfo, void sigcontext, void *map_info_list,
	45	backtrace_frame_t *backtrace, uptr ignore_depth, uptr max_depth);
	46	acquire_my_map_info_list_func acquire_my_map_info_list;
	47	release_my_map_info_list_func release_my_map_info_list;
	48	unwind_backtrace_signal_arch_func unwind_backtrace_signal_arch;
	49	} // extern "C"
	50
	51	#if defined(__arm__) && !SANITIZER_NETBSD
	52	// NetBSD uses dwarf EH
	53	#define UNWIND_STOP _URC_END_OF_STACK
	54	#define UNWIND_CONTINUE _URC_NO_REASON
	55	#else
	56	#define UNWIND_STOP _URC_NORMAL_STOP
	57	#define UNWIND_CONTINUE _URC_NO_REASON
	58	#endif
	59
	60	uptr Unwind_GetIP(struct _Unwind_Context *ctx) {
	61	#if defined(__arm__) && !SANITIZER_MAC
	62	uptr val;
	63	_Unwind_VRS_Result res = _Unwind_VRS_Get(ctx, _UVRSC_CORE,
	64	15 /* r15 = PC */, _UVRSD_UINT32, &val);
	65	CHECK(res == _UVRSR_OK && "_Unwind_VRS_Get failed");
	66	// Clear the Thumb bit.
	67	return val & ~(uptr)1;
	68	#else
	69	return (uptr)_Unwind_GetIP(ctx);
	70	#endif
	71	}
	72
	73	struct UnwindTraceArg {
	74	BufferedStackTrace *stack;
	75	u32 max_depth;
	76	};
	77
	78	_Unwind_Reason_Code Unwind_Trace(struct _Unwind_Context ctx, void param) {
	79	UnwindTraceArg arg = (UnwindTraceArg)param;
	80	CHECK_LT(arg->stack->size, arg->max_depth);
	81	uptr pc = Unwind_GetIP(ctx);
	82	const uptr kPageSize = GetPageSizeCached();
	83	// Let's assume that any pointer in the 0th page (i.e. <0x1000 on i386 and
	84	// x86_64) is invalid and stop unwinding here. If we're adding support for
	85	// a platform where this isn't true, we need to reconsider this check.
	86	if (pc < kPageSize) return UNWIND_STOP;
	87	arg->stack->trace_buffer[arg->stack->size++] = pc;
	88	if (arg->stack->size == arg->max_depth) return UNWIND_STOP;
	89	return UNWIND_CONTINUE;
	90	}
	91
	92	} // namespace
	93
	94	#if SANITIZER_ANDROID
	95	void SanitizerInitializeUnwinder() {
	96	if (AndroidGetApiLevel() >= ANDROID_LOLLIPOP_MR1) return;
	97
	98	// Pre-lollipop Android can not unwind through signal handler frames with
	99	// libgcc unwinder, but it has a libcorkscrew.so library with the necessary
	100	// workarounds.
	101	void *p = dlopen("libcorkscrew.so", RTLD_LAZY);
	102	if (!p) {
	103	VReport(1,
	104	"Failed to open libcorkscrew.so. You may see broken stack traces "
	105	"in SEGV reports.");
	106	return;
	107	}
	108	acquire_my_map_info_list =
	109	(acquire_my_map_info_list_func)(uptr)dlsym(p, "acquire_my_map_info_list");
	110	release_my_map_info_list =
	111	(release_my_map_info_list_func)(uptr)dlsym(p, "release_my_map_info_list");
	112	unwind_backtrace_signal_arch = (unwind_backtrace_signal_arch_func)(uptr)dlsym(
	113	p, "unwind_backtrace_signal_arch");
	114	if (!acquire_my_map_info_list \|\| !release_my_map_info_list \|\|
	115	!unwind_backtrace_signal_arch) {
	116	VReport(1,
	117	"Failed to find one of the required symbols in libcorkscrew.so. "
	118	"You may see broken stack traces in SEGV reports.");
	119	acquire_my_map_info_list = 0;
	120	unwind_backtrace_signal_arch = 0;
	121	release_my_map_info_list = 0;
	122	}
	123	}
	124	#endif
	125
	126	void BufferedStackTrace::UnwindSlow(uptr pc, u32 max_depth) {
	127	CHECK_GE(max_depth, 2);
	128	size = 0;
	129	UnwindTraceArg arg = {this, Min(max_depth + 1, kStackTraceMax)};
	130	_Unwind_Backtrace(Unwind_Trace, &arg);
	131	// We need to pop a few frames so that pc is on top.
	132	uptr to_pop = LocatePcInTrace(pc);
	133	// trace_buffer[0] belongs to the current function so we always pop it,
	134	// unless there is only 1 frame in the stack trace (1 frame is always better
	135	// than 0!).
	136	// 1-frame stacks don't normally happen, but this depends on the actual
	137	// unwinder implementation (libgcc, libunwind, etc) which is outside of our
	138	// control.
	139	if (to_pop == 0 && size > 1)
	140	to_pop = 1;
	141	PopStackFrames(to_pop);
	142	#if defined(__GNUC__) && defined(__sparc__)
	143	// __builtin_return_address returns the address of the call instruction
	144	// on the SPARC and not the return address, so we need to compensate.
	145	trace_buffer[0] = GetNextInstructionPc(pc);
	146	#else
	147	trace_buffer[0] = pc;
	148	#endif
	149	}
	150
	151	void BufferedStackTrace::UnwindSlow(uptr pc, void *context, u32 max_depth) {
	152	CHECK(context);
	153	CHECK_GE(max_depth, 2);
	154	if (!unwind_backtrace_signal_arch) {
	155	UnwindSlow(pc, max_depth);
	156	return;
	157	}
	158
	159	void *map = acquire_my_map_info_list();
	160	CHECK(map);
	161	InternalMmapVector<backtrace_frame_t> frames(kStackTraceMax);
	162	// siginfo argument appears to be unused.
	163	sptr res = unwind_backtrace_signal_arch(/* siginfo */ 0, context, map,
	164	frames.data(),
	165	/* ignore_depth */ 0, max_depth);
	166	release_my_map_info_list(map);
	167	if (res < 0) return;
	168	CHECK_LE((uptr)res, kStackTraceMax);
	169
	170	size = 0;
	171	// +2 compensate for libcorkscrew unwinder returning addresses of call
	172	// instructions instead of raw return addresses.
	173	for (sptr i = 0; i < res; ++i)
	174	trace_buffer[size++] = frames[i].absolute_pc + 2;
	175	}
	176
	177	} // namespace __sanitizer
	178
	179	#endif // SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_NETBSD \|\|
	180	// SANITIZER_SOLARIS

lib/tsan/sanitizer_common/sanitizer_unwind_win.cpp created+75

...	...	@@ -0,0 +1,75 @@
	1	//===-- sanitizer_unwind_win.cpp ------------------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	/// Sanitizer unwind Windows specific functions.
	10	//
	11	//===----------------------------------------------------------------------===//
	12
	13	#include "sanitizer_platform.h"
	14	#if SANITIZER_WINDOWS
	15
	16	#define WIN32_LEAN_AND_MEAN
	17	#define NOGDI
	18	#include <windows.h>
	19
	20	#include "sanitizer_dbghelp.h" // for StackWalk64
	21	#include "sanitizer_stacktrace.h"
	22	#include "sanitizer_symbolizer.h" // for InitializeDbgHelpIfNeeded
	23
	24	using namespace __sanitizer;
	25
	26	#if !SANITIZER_GO
	27	void BufferedStackTrace::UnwindSlow(uptr pc, u32 max_depth) {
	28	CHECK_GE(max_depth, 2);
	29	// FIXME: CaptureStackBackTrace might be too slow for us.
	30	// FIXME: Compare with StackWalk64.
	31	// FIXME: Look at LLVMUnhandledExceptionFilter in Signals.inc
	32	size = CaptureStackBackTrace(1, Min(max_depth, kStackTraceMax),
	33	(void **)&trace_buffer[0], 0);
	34	if (size == 0)
	35	return;
	36
	37	// Skip the RTL frames by searching for the PC in the stacktrace.
	38	uptr pc_location = LocatePcInTrace(pc);
	39	PopStackFrames(pc_location);
	40	}
	41
	42	void BufferedStackTrace::UnwindSlow(uptr pc, void *context, u32 max_depth) {
	43	CHECK(context);
	44	CHECK_GE(max_depth, 2);
	45	CONTEXT ctx = (CONTEXT )context;
	46	STACKFRAME64 stack_frame;
	47	memset(&stack_frame, 0, sizeof(stack_frame));
	48
	49	InitializeDbgHelpIfNeeded();
	50
	51	size = 0;
	52	#if defined(_WIN64)
	53	int machine_type = IMAGE_FILE_MACHINE_AMD64;
	54	stack_frame.AddrPC.Offset = ctx.Rip;
	55	stack_frame.AddrFrame.Offset = ctx.Rbp;
	56	stack_frame.AddrStack.Offset = ctx.Rsp;
	57	#else
	58	int machine_type = IMAGE_FILE_MACHINE_I386;
	59	stack_frame.AddrPC.Offset = ctx.Eip;
	60	stack_frame.AddrFrame.Offset = ctx.Ebp;
	61	stack_frame.AddrStack.Offset = ctx.Esp;
	62	#endif
	63	stack_frame.AddrPC.Mode = AddrModeFlat;
	64	stack_frame.AddrFrame.Mode = AddrModeFlat;
	65	stack_frame.AddrStack.Mode = AddrModeFlat;
	66	while (StackWalk64(machine_type, GetCurrentProcess(), GetCurrentThread(),
	67	&stack_frame, &ctx, NULL, SymFunctionTableAccess64,
	68	SymGetModuleBase64, NULL) &&
	69	size < Min(max_depth, kStackTraceMax)) {
	70	trace_buffer[size++] = (uptr)stack_frame.AddrPC.Offset;
	71	}
	72	}
	73	#endif // #if !SANITIZER_GO
	74
	75	#endif // SANITIZER_WINDOWS

lib/tsan/tsan_interceptors_mac.cpp created+519

...	...	@@ -0,0 +1,519 @@
	1	//===-- tsan_interceptors_mac.cpp -----------------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file is a part of ThreadSanitizer (TSan), a race detector.
	10	//
	11	// Mac-specific interceptors.
	12	//===----------------------------------------------------------------------===//
	13
	14	#include "sanitizer_common/sanitizer_platform.h"
	15	#if SANITIZER_MAC
	16
	17	#include "interception/interception.h"
	18	#include "tsan_interceptors.h"
	19	#include "tsan_interface.h"
	20	#include "tsan_interface_ann.h"
	21	#include "sanitizer_common/sanitizer_addrhashmap.h"
	22
	23	#include <errno.h>
	24	#include <libkern/OSAtomic.h>
	25	#include <objc/objc-sync.h>
	26	#include <os/lock.h>
	27	#include <sys/ucontext.h>
	28
	29	#if defined(__has_include) && __has_include(<xpc/xpc.h>)
	30	#include <xpc/xpc.h>
	31	#endif // #if defined(__has_include) && __has_include(<xpc/xpc.h>)
	32
	33	typedef long long_t;
	34
	35	extern "C" {
	36	int getcontext(ucontext_t *ucp) __attribute__((returns_twice));
	37	int setcontext(const ucontext_t *ucp);
	38	}
	39
	40	namespace __tsan {
	41
	42	// The non-barrier versions of OSAtomic* functions are semantically mo_relaxed,
	43	// but the two variants (e.g. OSAtomicAdd32 and OSAtomicAdd32Barrier) are
	44	// actually aliases of each other, and we cannot have different interceptors for
	45	// them, because they're actually the same function. Thus, we have to stay
	46	// conservative and treat the non-barrier versions as mo_acq_rel.
	47	static const morder kMacOrderBarrier = mo_acq_rel;
	48	static const morder kMacOrderNonBarrier = mo_acq_rel;
	49
	50	#define OSATOMIC_INTERCEPTOR(return_t, t, tsan_t, f, tsan_atomic_f, mo) \
	51	TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \
	52	SCOPED_TSAN_INTERCEPTOR(f, x, ptr); \
	53	return tsan_atomic_f((volatile tsan_t *)ptr, x, mo); \
	54	}
	55
	56	#define OSATOMIC_INTERCEPTOR_PLUS_X(return_t, t, tsan_t, f, tsan_atomic_f, mo) \
	57	TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \
	58	SCOPED_TSAN_INTERCEPTOR(f, x, ptr); \
	59	return tsan_atomic_f((volatile tsan_t *)ptr, x, mo) + x; \
	60	}
	61
	62	#define OSATOMIC_INTERCEPTOR_PLUS_1(return_t, t, tsan_t, f, tsan_atomic_f, mo) \
	63	TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) { \
	64	SCOPED_TSAN_INTERCEPTOR(f, ptr); \
	65	return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) + 1; \
	66	}
	67
	68	#define OSATOMIC_INTERCEPTOR_MINUS_1(return_t, t, tsan_t, f, tsan_atomic_f, \
	69	mo) \
	70	TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) { \
	71	SCOPED_TSAN_INTERCEPTOR(f, ptr); \
	72	return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) - 1; \
	73	}
	74
	75	#define OSATOMIC_INTERCEPTORS_ARITHMETIC(f, tsan_atomic_f, m) \
	76	m(int32_t, int32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f, \
	77	kMacOrderNonBarrier) \
	78	m(int32_t, int32_t, a32, f##32##Barrier, __tsan_atomic32_##tsan_atomic_f, \
	79	kMacOrderBarrier) \
	80	m(int64_t, int64_t, a64, f##64, __tsan_atomic64_##tsan_atomic_f, \
	81	kMacOrderNonBarrier) \
	82	m(int64_t, int64_t, a64, f##64##Barrier, __tsan_atomic64_##tsan_atomic_f, \
	83	kMacOrderBarrier)
	84
	85	#define OSATOMIC_INTERCEPTORS_BITWISE(f, tsan_atomic_f, m, m_orig) \
	86	m(int32_t, uint32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f, \
	87	kMacOrderNonBarrier) \
	88	m(int32_t, uint32_t, a32, f##32##Barrier, __tsan_atomic32_##tsan_atomic_f, \
	89	kMacOrderBarrier) \
	90	m_orig(int32_t, uint32_t, a32, f##32##Orig, __tsan_atomic32_##tsan_atomic_f, \
	91	kMacOrderNonBarrier) \
	92	m_orig(int32_t, uint32_t, a32, f##32##OrigBarrier, \
	93	__tsan_atomic32_##tsan_atomic_f, kMacOrderBarrier)
	94
	95	OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicAdd, fetch_add,
	96	OSATOMIC_INTERCEPTOR_PLUS_X)
	97	OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicIncrement, fetch_add,
	98	OSATOMIC_INTERCEPTOR_PLUS_1)
	99	OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicDecrement, fetch_sub,
	100	OSATOMIC_INTERCEPTOR_MINUS_1)
	101	OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicOr, fetch_or, OSATOMIC_INTERCEPTOR_PLUS_X,
	102	OSATOMIC_INTERCEPTOR)
	103	OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicAnd, fetch_and,
	104	OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR)
	105	OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicXor, fetch_xor,
	106	OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR)
	107
	108	#define OSATOMIC_INTERCEPTORS_CAS(f, tsan_atomic_f, tsan_t, t) \
	109	TSAN_INTERCEPTOR(bool, f, t old_value, t new_value, t volatile *ptr) { \
	110	SCOPED_TSAN_INTERCEPTOR(f, old_value, new_value, ptr); \
	111	return tsan_atomic_f##_compare_exchange_strong( \
	112	(volatile tsan_t )ptr, (tsan_t )&old_value, (tsan_t)new_value, \
	113	kMacOrderNonBarrier, kMacOrderNonBarrier); \
	114	} \
	115	\
	116	TSAN_INTERCEPTOR(bool, f##Barrier, t old_value, t new_value, \
	117	t volatile *ptr) { \
	118	SCOPED_TSAN_INTERCEPTOR(f##Barrier, old_value, new_value, ptr); \
	119	return tsan_atomic_f##_compare_exchange_strong( \
	120	(volatile tsan_t )ptr, (tsan_t )&old_value, (tsan_t)new_value, \
	121	kMacOrderBarrier, kMacOrderNonBarrier); \
	122	}
	123
	124	OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapInt, __tsan_atomic32, a32, int)
	125	OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapLong, __tsan_atomic64, a64,
	126	long_t)
	127	OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapPtr, __tsan_atomic64, a64,
	128	void *)
	129	OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap32, __tsan_atomic32, a32,
	130	int32_t)
	131	OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap64, __tsan_atomic64, a64,
	132	int64_t)
	133
	134	#define OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, mo) \
	135	TSAN_INTERCEPTOR(bool, f, uint32_t n, volatile void *ptr) { \
	136	SCOPED_TSAN_INTERCEPTOR(f, n, ptr); \
	137	volatile char byte_ptr = ((volatile char )ptr) + (n >> 3); \
	138	char bit = 0x80u >> (n & 7); \
	139	char mask = clear ? ~bit : bit; \
	140	char orig_byte = op((volatile a8 *)byte_ptr, mask, mo); \
	141	return orig_byte & bit; \
	142	}
	143
	144	#define OSATOMIC_INTERCEPTORS_BITOP(f, op, clear) \
	145	OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, kMacOrderNonBarrier) \
	146	OSATOMIC_INTERCEPTOR_BITOP(f##Barrier, op, clear, kMacOrderBarrier)
	147
	148	OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndSet, __tsan_atomic8_fetch_or, false)
	149	OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndClear, __tsan_atomic8_fetch_and,
	150	true)
	151
	152	TSAN_INTERCEPTOR(void, OSAtomicEnqueue, OSQueueHead list, void item,
	153	size_t offset) {
	154	SCOPED_TSAN_INTERCEPTOR(OSAtomicEnqueue, list, item, offset);
	155	__tsan_release(item);
	156	REAL(OSAtomicEnqueue)(list, item, offset);
	157	}
	158
	159	TSAN_INTERCEPTOR(void , OSAtomicDequeue, OSQueueHead list, size_t offset) {
	160	SCOPED_TSAN_INTERCEPTOR(OSAtomicDequeue, list, offset);
	161	void *item = REAL(OSAtomicDequeue)(list, offset);
	162	if (item) __tsan_acquire(item);
	163	return item;
	164	}
	165
	166	// OSAtomicFifoEnqueue and OSAtomicFifoDequeue are only on OS X.
	167	#if !SANITIZER_IOS
	168
	169	TSAN_INTERCEPTOR(void, OSAtomicFifoEnqueue, OSFifoQueueHead list, void item,
	170	size_t offset) {
	171	SCOPED_TSAN_INTERCEPTOR(OSAtomicFifoEnqueue, list, item, offset);
	172	__tsan_release(item);
	173	REAL(OSAtomicFifoEnqueue)(list, item, offset);
	174	}
	175
	176	TSAN_INTERCEPTOR(void , OSAtomicFifoDequeue, OSFifoQueueHead list,
	177	size_t offset) {
	178	SCOPED_TSAN_INTERCEPTOR(OSAtomicFifoDequeue, list, offset);
	179	void *item = REAL(OSAtomicFifoDequeue)(list, offset);
	180	if (item) __tsan_acquire(item);
	181	return item;
	182	}
	183
	184	#endif
	185
	186	TSAN_INTERCEPTOR(void, OSSpinLockLock, volatile OSSpinLock *lock) {
	187	CHECK(!cur_thread()->is_dead);
	188	if (!cur_thread()->is_inited) {
	189	return REAL(OSSpinLockLock)(lock);
	190	}
	191	SCOPED_TSAN_INTERCEPTOR(OSSpinLockLock, lock);
	192	REAL(OSSpinLockLock)(lock);
	193	Acquire(thr, pc, (uptr)lock);
	194	}
	195
	196	TSAN_INTERCEPTOR(bool, OSSpinLockTry, volatile OSSpinLock *lock) {
	197	CHECK(!cur_thread()->is_dead);
	198	if (!cur_thread()->is_inited) {
	199	return REAL(OSSpinLockTry)(lock);
	200	}
	201	SCOPED_TSAN_INTERCEPTOR(OSSpinLockTry, lock);
	202	bool result = REAL(OSSpinLockTry)(lock);
	203	if (result)
	204	Acquire(thr, pc, (uptr)lock);
	205	return result;
	206	}
	207
	208	TSAN_INTERCEPTOR(void, OSSpinLockUnlock, volatile OSSpinLock *lock) {
	209	CHECK(!cur_thread()->is_dead);
	210	if (!cur_thread()->is_inited) {
	211	return REAL(OSSpinLockUnlock)(lock);
	212	}
	213	SCOPED_TSAN_INTERCEPTOR(OSSpinLockUnlock, lock);
	214	Release(thr, pc, (uptr)lock);
	215	REAL(OSSpinLockUnlock)(lock);
	216	}
	217
	218	TSAN_INTERCEPTOR(void, os_lock_lock, void *lock) {
	219	CHECK(!cur_thread()->is_dead);
	220	if (!cur_thread()->is_inited) {
	221	return REAL(os_lock_lock)(lock);
	222	}
	223	SCOPED_TSAN_INTERCEPTOR(os_lock_lock, lock);
	224	REAL(os_lock_lock)(lock);
	225	Acquire(thr, pc, (uptr)lock);
	226	}
	227
	228	TSAN_INTERCEPTOR(bool, os_lock_trylock, void *lock) {
	229	CHECK(!cur_thread()->is_dead);
	230	if (!cur_thread()->is_inited) {
	231	return REAL(os_lock_trylock)(lock);
	232	}
	233	SCOPED_TSAN_INTERCEPTOR(os_lock_trylock, lock);
	234	bool result = REAL(os_lock_trylock)(lock);
	235	if (result)
	236	Acquire(thr, pc, (uptr)lock);
	237	return result;
	238	}
	239
	240	TSAN_INTERCEPTOR(void, os_lock_unlock, void *lock) {
	241	CHECK(!cur_thread()->is_dead);
	242	if (!cur_thread()->is_inited) {
	243	return REAL(os_lock_unlock)(lock);
	244	}
	245	SCOPED_TSAN_INTERCEPTOR(os_lock_unlock, lock);
	246	Release(thr, pc, (uptr)lock);
	247	REAL(os_lock_unlock)(lock);
	248	}
	249
	250	TSAN_INTERCEPTOR(void, os_unfair_lock_lock, os_unfair_lock_t lock) {
	251	if (!cur_thread()->is_inited \|\| cur_thread()->is_dead) {
	252	return REAL(os_unfair_lock_lock)(lock);
	253	}
	254	SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_lock, lock);
	255	REAL(os_unfair_lock_lock)(lock);
	256	Acquire(thr, pc, (uptr)lock);
	257	}
	258
	259	TSAN_INTERCEPTOR(void, os_unfair_lock_lock_with_options, os_unfair_lock_t lock,
	260	u32 options) {
	261	if (!cur_thread()->is_inited \|\| cur_thread()->is_dead) {
	262	return REAL(os_unfair_lock_lock_with_options)(lock, options);
	263	}
	264	SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_lock_with_options, lock, options);
	265	REAL(os_unfair_lock_lock_with_options)(lock, options);
	266	Acquire(thr, pc, (uptr)lock);
	267	}
	268
	269	TSAN_INTERCEPTOR(bool, os_unfair_lock_trylock, os_unfair_lock_t lock) {
	270	if (!cur_thread()->is_inited \|\| cur_thread()->is_dead) {
	271	return REAL(os_unfair_lock_trylock)(lock);
	272	}
	273	SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_trylock, lock);
	274	bool result = REAL(os_unfair_lock_trylock)(lock);
	275	if (result)
	276	Acquire(thr, pc, (uptr)lock);
	277	return result;
	278	}
	279
	280	TSAN_INTERCEPTOR(void, os_unfair_lock_unlock, os_unfair_lock_t lock) {
	281	if (!cur_thread()->is_inited \|\| cur_thread()->is_dead) {
	282	return REAL(os_unfair_lock_unlock)(lock);
	283	}
	284	SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_unlock, lock);
	285	Release(thr, pc, (uptr)lock);
	286	REAL(os_unfair_lock_unlock)(lock);
	287	}
	288
	289	#if defined(__has_include) && __has_include(<xpc/xpc.h>)
	290
	291	TSAN_INTERCEPTOR(void, xpc_connection_set_event_handler,
	292	xpc_connection_t connection, xpc_handler_t handler) {
	293	SCOPED_TSAN_INTERCEPTOR(xpc_connection_set_event_handler, connection,
	294	handler);
	295	Release(thr, pc, (uptr)connection);
	296	xpc_handler_t new_handler = ^(xpc_object_t object) {
	297	{
	298	SCOPED_INTERCEPTOR_RAW(xpc_connection_set_event_handler);
	299	Acquire(thr, pc, (uptr)connection);
	300	}
	301	handler(object);
	302	};
	303	REAL(xpc_connection_set_event_handler)(connection, new_handler);
	304	}
	305
	306	TSAN_INTERCEPTOR(void, xpc_connection_send_barrier, xpc_connection_t connection,
	307	dispatch_block_t barrier) {
	308	SCOPED_TSAN_INTERCEPTOR(xpc_connection_send_barrier, connection, barrier);
	309	Release(thr, pc, (uptr)connection);
	310	dispatch_block_t new_barrier = ^() {
	311	{
	312	SCOPED_INTERCEPTOR_RAW(xpc_connection_send_barrier);
	313	Acquire(thr, pc, (uptr)connection);
	314	}
	315	barrier();
	316	};
	317	REAL(xpc_connection_send_barrier)(connection, new_barrier);
	318	}
	319
	320	TSAN_INTERCEPTOR(void, xpc_connection_send_message_with_reply,
	321	xpc_connection_t connection, xpc_object_t message,
	322	dispatch_queue_t replyq, xpc_handler_t handler) {
	323	SCOPED_TSAN_INTERCEPTOR(xpc_connection_send_message_with_reply, connection,
	324	message, replyq, handler);
	325	Release(thr, pc, (uptr)connection);
	326	xpc_handler_t new_handler = ^(xpc_object_t object) {
	327	{
	328	SCOPED_INTERCEPTOR_RAW(xpc_connection_send_message_with_reply);
	329	Acquire(thr, pc, (uptr)connection);
	330	}
	331	handler(object);
	332	};
	333	REAL(xpc_connection_send_message_with_reply)
	334	(connection, message, replyq, new_handler);
	335	}
	336
	337	TSAN_INTERCEPTOR(void, xpc_connection_cancel, xpc_connection_t connection) {
	338	SCOPED_TSAN_INTERCEPTOR(xpc_connection_cancel, connection);
	339	Release(thr, pc, (uptr)connection);
	340	REAL(xpc_connection_cancel)(connection);
	341	}
	342
	343	#endif // #if defined(__has_include) && __has_include(<xpc/xpc.h>)
	344
	345	// Determines whether the Obj-C object pointer is a tagged pointer. Tagged
	346	// pointers encode the object data directly in their pointer bits and do not
	347	// have an associated memory allocation. The Obj-C runtime uses tagged pointers
	348	// to transparently optimize small objects.
	349	static bool IsTaggedObjCPointer(id obj) {
	350	const uptr kPossibleTaggedBits = 0x8000000000000001ull;
	351	return ((uptr)obj & kPossibleTaggedBits) != 0;
	352	}
	353
	354	// Returns an address which can be used to inform TSan about synchronization
	355	// points (MutexLock/Unlock). The TSan infrastructure expects this to be a valid
	356	// address in the process space. We do a small allocation here to obtain a
	357	// stable address (the array backing the hash map can change). The memory is
	358	// never free'd (leaked) and allocation and locking are slow, but this code only
	359	// runs for @synchronized with tagged pointers, which is very rare.
	360	static uptr GetOrCreateSyncAddress(uptr addr, ThreadState *thr, uptr pc) {
	361	typedef AddrHashMap<uptr, 5> Map;
	362	static Map Addresses;
	363	Map::Handle h(&Addresses, addr);
	364	if (h.created()) {
	365	ThreadIgnoreBegin(thr, pc);
	366	h = (uptr) user_alloc(thr, pc, /size=*/1);
	367	ThreadIgnoreEnd(thr, pc);
	368	}
	369	return *h;
	370	}
	371
	372	// Returns an address on which we can synchronize given an Obj-C object pointer.
	373	// For normal object pointers, this is just the address of the object in memory.
	374	// Tagged pointers are not backed by an actual memory allocation, so we need to
	375	// synthesize a valid address.
	376	static uptr SyncAddressForObjCObject(id obj, ThreadState *thr, uptr pc) {
	377	if (IsTaggedObjCPointer(obj))
	378	return GetOrCreateSyncAddress((uptr)obj, thr, pc);
	379	return (uptr)obj;
	380	}
	381
	382	TSAN_INTERCEPTOR(int, objc_sync_enter, id obj) {
	383	SCOPED_TSAN_INTERCEPTOR(objc_sync_enter, obj);
	384	if (!obj) return REAL(objc_sync_enter)(obj);
	385	uptr addr = SyncAddressForObjCObject(obj, thr, pc);
	386	MutexPreLock(thr, pc, addr, MutexFlagWriteReentrant);
	387	int result = REAL(objc_sync_enter)(obj);
	388	CHECK_EQ(result, OBJC_SYNC_SUCCESS);
	389	MutexPostLock(thr, pc, addr, MutexFlagWriteReentrant);
	390	return result;
	391	}
	392
	393	TSAN_INTERCEPTOR(int, objc_sync_exit, id obj) {
	394	SCOPED_TSAN_INTERCEPTOR(objc_sync_exit, obj);
	395	if (!obj) return REAL(objc_sync_exit)(obj);
	396	uptr addr = SyncAddressForObjCObject(obj, thr, pc);
	397	MutexUnlock(thr, pc, addr);
	398	int result = REAL(objc_sync_exit)(obj);
	399	if (result != OBJC_SYNC_SUCCESS) MutexInvalidAccess(thr, pc, addr);
	400	return result;
	401	}
	402
	403	TSAN_INTERCEPTOR(int, swapcontext, ucontext_t oucp, const ucontext_t ucp) {
	404	{
	405	SCOPED_INTERCEPTOR_RAW(swapcontext, oucp, ucp);
	406	}
	407	// Bacause of swapcontext() semantics we have no option but to copy its
	408	// impementation here
	409	if (!oucp \|\| !ucp) {
	410	errno = EINVAL;
	411	return -1;
	412	}
	413	ThreadState *thr = cur_thread();
	414	const int UCF_SWAPPED = 0x80000000;
	415	oucp->uc_onstack &= ~UCF_SWAPPED;
	416	thr->ignore_interceptors++;
	417	int ret = getcontext(oucp);
	418	if (!(oucp->uc_onstack & UCF_SWAPPED)) {
	419	thr->ignore_interceptors--;
	420	if (!ret) {
	421	oucp->uc_onstack \|= UCF_SWAPPED;
	422	ret = setcontext(ucp);
	423	}
	424	}
	425	return ret;
	426	}
	427
	428	// On macOS, libc++ is always linked dynamically, so intercepting works the
	429	// usual way.
	430	#define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR
	431
	432	namespace {
	433	struct fake_shared_weak_count {
	434	volatile a64 shared_owners;
	435	volatile a64 shared_weak_owners;
	436	virtual void _unused_0x0() = 0;
	437	virtual void _unused_0x8() = 0;
	438	virtual void on_zero_shared() = 0;
	439	virtual void _unused_0x18() = 0;
	440	virtual void on_zero_shared_weak() = 0;
	441	};
	442	} // namespace
	443
	444	// The following code adds libc++ interceptors for:
	445	// void __shared_weak_count::__release_shared() _NOEXCEPT;
	446	// bool __shared_count::__release_shared() _NOEXCEPT;
	447	// Shared and weak pointers in C++ maintain reference counts via atomics in
	448	// libc++.dylib, which are TSan-invisible, and this leads to false positives in
	449	// destructor code. These interceptors re-implements the whole functions so that
	450	// the mo_acq_rel semantics of the atomic decrement are visible.
	451	//
	452	// Unfortunately, the interceptors cannot simply Acquire/Release some sync
	453	// object and call the original function, because it would have a race between
	454	// the sync and the destruction of the object. Calling both under a lock will
	455	// not work because the destructor can invoke this interceptor again (and even
	456	// in a different thread, so recursive locks don't help).
	457
	458	STDCXX_INTERCEPTOR(void, _ZNSt3__119__shared_weak_count16__release_sharedEv,
	459	fake_shared_weak_count *o) {
	460	if (!flags()->shared_ptr_interceptor)
	461	return REAL(_ZNSt3__119__shared_weak_count16__release_sharedEv)(o);
	462
	463	SCOPED_TSAN_INTERCEPTOR(_ZNSt3__119__shared_weak_count16__release_sharedEv,
	464	o);
	465	if (__tsan_atomic64_fetch_add(&o->shared_owners, -1, mo_release) == 0) {
	466	Acquire(thr, pc, (uptr)&o->shared_owners);
	467	o->on_zero_shared();
	468	if (__tsan_atomic64_fetch_add(&o->shared_weak_owners, -1, mo_release) ==
	469	0) {
	470	Acquire(thr, pc, (uptr)&o->shared_weak_owners);
	471	o->on_zero_shared_weak();
	472	}
	473	}
	474	}
	475
	476	STDCXX_INTERCEPTOR(bool, _ZNSt3__114__shared_count16__release_sharedEv,
	477	fake_shared_weak_count *o) {
	478	if (!flags()->shared_ptr_interceptor)
	479	return REAL(_ZNSt3__114__shared_count16__release_sharedEv)(o);
	480
	481	SCOPED_TSAN_INTERCEPTOR(_ZNSt3__114__shared_count16__release_sharedEv, o);
	482	if (__tsan_atomic64_fetch_add(&o->shared_owners, -1, mo_release) == 0) {
	483	Acquire(thr, pc, (uptr)&o->shared_owners);
	484	o->on_zero_shared();
	485	return true;
	486	}
	487	return false;
	488	}
	489
	490	namespace {
	491	struct call_once_callback_args {
	492	void (orig_func)(void arg);
	493	void *orig_arg;
	494	void *flag;
	495	};
	496
	497	void call_once_callback_wrapper(void *arg) {
	498	call_once_callback_args new_args = (call_once_callback_args )arg;
	499	new_args->orig_func(new_args->orig_arg);
	500	__tsan_release(new_args->flag);
	501	}
	502	} // namespace
	503
	504	// This adds a libc++ interceptor for:
	505	// void __call_once(volatile unsigned long&, void, void()(void*));
	506	// C++11 call_once is implemented via an internal function __call_once which is
	507	// inside libc++.dylib, and the atomic release store inside it is thus
	508	// TSan-invisible. To avoid false positives, this interceptor wraps the callback
	509	// function and performs an explicit Release after the user code has run.
	510	STDCXX_INTERCEPTOR(void, _ZNSt3__111__call_onceERVmPvPFvS2_E, void *flag,
	511	void arg, void (func)(void *arg)) {
	512	call_once_callback_args new_args = {func, arg, flag};
	513	REAL(_ZNSt3__111__call_onceERVmPvPFvS2_E)(flag, &new_args,
	514	call_once_callback_wrapper);
	515	}
	516
	517	} // namespace __tsan
	518
	519	#endif // SANITIZER_MAC

lib/tsan/tsan_interceptors_mach_vm.cpp created+52

...	...	@@ -0,0 +1,52 @@
	1	//===-- tsan_interceptors_mach_vm.cpp -------------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file is a part of ThreadSanitizer (TSan), a race detector.
	10	//
	11	// Interceptors for mach_vm_* user space memory routines on Darwin.
	12	//===----------------------------------------------------------------------===//
	13
	14	#include "interception/interception.h"
	15	#include "tsan_interceptors.h"
	16	#include "tsan_platform.h"
	17
	18	#include <mach/mach.h>
	19
	20	namespace __tsan {
	21
	22	static bool intersects_with_shadow(mach_vm_address_t *address,
	23	mach_vm_size_t size, int flags) {
	24	// VM_FLAGS_FIXED is 0x0, so we have to test for VM_FLAGS_ANYWHERE.
	25	if (flags & VM_FLAGS_ANYWHERE) return false;
	26	uptr ptr = *address;
	27	return !IsAppMem(ptr) \|\| !IsAppMem(ptr + size - 1);
	28	}
	29
	30	TSAN_INTERCEPTOR(kern_return_t, mach_vm_allocate, vm_map_t target,
	31	mach_vm_address_t *address, mach_vm_size_t size, int flags) {
	32	SCOPED_TSAN_INTERCEPTOR(mach_vm_allocate, target, address, size, flags);
	33	if (target != mach_task_self())
	34	return REAL(mach_vm_allocate)(target, address, size, flags);
	35	if (intersects_with_shadow(address, size, flags))
	36	return KERN_NO_SPACE;
	37	kern_return_t res = REAL(mach_vm_allocate)(target, address, size, flags);
	38	if (res == KERN_SUCCESS)
	39	MemoryRangeImitateWriteOrResetRange(thr, pc, *address, size);
	40	return res;
	41	}
	42
	43	TSAN_INTERCEPTOR(kern_return_t, mach_vm_deallocate, vm_map_t target,
	44	mach_vm_address_t address, mach_vm_size_t size) {
	45	SCOPED_TSAN_INTERCEPTOR(mach_vm_deallocate, target, address, size);
	46	if (target != mach_task_self())
	47	return REAL(mach_vm_deallocate)(target, address, size);
	48	UnmapShadow(thr, address, size);
	49	return REAL(mach_vm_deallocate)(target, address, size);
	50	}
	51
	52	} // namespace __tsan

lib/tsan/tsan_platform_linux.cpp created+517

...	...	@@ -0,0 +1,517 @@
	1	//===-- tsan_platform_linux.cpp -------------------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file is a part of ThreadSanitizer (TSan), a race detector.
	10	//
	11	// Linux- and BSD-specific code.
	12	//===----------------------------------------------------------------------===//
	13
	14	#include "sanitizer_common/sanitizer_platform.h"
	15	#if SANITIZER_LINUX \|\| SANITIZER_FREEBSD \|\| SANITIZER_NETBSD \|\| \
	16	SANITIZER_OPENBSD
	17
	18	#include "sanitizer_common/sanitizer_common.h"
	19	#include "sanitizer_common/sanitizer_libc.h"
	20	#include "sanitizer_common/sanitizer_linux.h"
	21	#include "sanitizer_common/sanitizer_platform_limits_netbsd.h"
	22	#include "sanitizer_common/sanitizer_platform_limits_openbsd.h"
	23	#include "sanitizer_common/sanitizer_platform_limits_posix.h"
	24	#include "sanitizer_common/sanitizer_posix.h"
	25	#include "sanitizer_common/sanitizer_procmaps.h"
	26	#include "sanitizer_common/sanitizer_stackdepot.h"
	27	#include "sanitizer_common/sanitizer_stoptheworld.h"
	28	#include "tsan_flags.h"
	29	#include "tsan_platform.h"
	30	#include "tsan_rtl.h"
	31
	32	#include <fcntl.h>
	33	#include <pthread.h>
	34	#include <signal.h>
	35	#include <stdio.h>
	36	#include <stdlib.h>
	37	#include <string.h>
	38	#include <stdarg.h>
	39	#include <sys/mman.h>
	40	#if SANITIZER_LINUX
	41	#include <sys/personality.h>
	42	#include <setjmp.h>
	43	#endif
	44	#include <sys/syscall.h>
	45	#include <sys/socket.h>
	46	#include <sys/time.h>
	47	#include <sys/types.h>
	48	#include <sys/resource.h>
	49	#include <sys/stat.h>
	50	#include <unistd.h>
	51	#include <sched.h>
	52	#include <dlfcn.h>
	53	#if SANITIZER_LINUX
	54	#define __need_res_state
	55	#include <resolv.h>
	56	#endif
	57
	58	#ifdef sa_handler
	59	# undef sa_handler
	60	#endif
	61
	62	#ifdef sa_sigaction
	63	# undef sa_sigaction
	64	#endif
	65
	66	#if SANITIZER_FREEBSD
	67	extern "C" void *__libc_stack_end;
	68	void *__libc_stack_end = 0;
	69	#endif
	70
	71	#if SANITIZER_LINUX && defined(__aarch64__) && !SANITIZER_GO
	72	# define INIT_LONGJMP_XOR_KEY 1
	73	#else
	74	# define INIT_LONGJMP_XOR_KEY 0
	75	#endif
	76
	77	#if INIT_LONGJMP_XOR_KEY
	78	#include "interception/interception.h"
	79	// Must be declared outside of other namespaces.
	80	DECLARE_REAL(int, _setjmp, void *env)
	81	#endif
	82
	83	namespace __tsan {
	84
	85	#if INIT_LONGJMP_XOR_KEY
	86	static void InitializeLongjmpXorKey();
	87	static uptr longjmp_xor_key;
	88	#endif
	89
	90	#ifdef TSAN_RUNTIME_VMA
	91	// Runtime detected VMA size.
	92	uptr vmaSize;
	93	#endif
	94
	95	enum {
	96	MemTotal = 0,
	97	MemShadow = 1,
	98	MemMeta = 2,
	99	MemFile = 3,
	100	MemMmap = 4,
	101	MemTrace = 5,
	102	MemHeap = 6,
	103	MemOther = 7,
	104	MemCount = 8,
	105	};
	106
	107	void FillProfileCallback(uptr p, uptr rss, bool file,
	108	uptr *mem, uptr stats_size) {
	109	mem[MemTotal] += rss;
	110	if (p >= ShadowBeg() && p < ShadowEnd())
	111	mem[MemShadow] += rss;
	112	else if (p >= MetaShadowBeg() && p < MetaShadowEnd())
	113	mem[MemMeta] += rss;
	114	#if !SANITIZER_GO
	115	else if (p >= HeapMemBeg() && p < HeapMemEnd())
	116	mem[MemHeap] += rss;
	117	else if (p >= LoAppMemBeg() && p < LoAppMemEnd())
	118	mem[file ? MemFile : MemMmap] += rss;
	119	else if (p >= HiAppMemBeg() && p < HiAppMemEnd())
	120	mem[file ? MemFile : MemMmap] += rss;
	121	#else
	122	else if (p >= AppMemBeg() && p < AppMemEnd())
	123	mem[file ? MemFile : MemMmap] += rss;
	124	#endif
	125	else if (p >= TraceMemBeg() && p < TraceMemEnd())
	126	mem[MemTrace] += rss;
	127	else
	128	mem[MemOther] += rss;
	129	}
	130
	131	void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive) {
	132	uptr mem[MemCount];
	133	internal_memset(mem, 0, sizeof(mem[0]) * MemCount);
	134	__sanitizer::GetMemoryProfile(FillProfileCallback, mem, 7);
	135	StackDepotStats *stacks = StackDepotGetStats();
	136	internal_snprintf(buf, buf_size,
	137	"RSS %zd MB: shadow:%zd meta:%zd file:%zd mmap:%zd"
	138	" trace:%zd heap:%zd other:%zd stacks=%zd[%zd] nthr=%zd/%zd\n",
	139	mem[MemTotal] >> 20, mem[MemShadow] >> 20, mem[MemMeta] >> 20,
	140	mem[MemFile] >> 20, mem[MemMmap] >> 20, mem[MemTrace] >> 20,
	141	mem[MemHeap] >> 20, mem[MemOther] >> 20,
	142	stacks->allocated >> 20, stacks->n_uniq_ids,
	143	nlive, nthread);
	144	}
	145
	146	#if SANITIZER_LINUX
	147	void FlushShadowMemoryCallback(
	148	const SuspendedThreadsList &suspended_threads_list,
	149	void *argument) {
	150	ReleaseMemoryPagesToOS(ShadowBeg(), ShadowEnd());
	151	}
	152	#endif
	153
	154	void FlushShadowMemory() {
	155	#if SANITIZER_LINUX
	156	StopTheWorld(FlushShadowMemoryCallback, 0);
	157	#endif
	158	}
	159
	160	#if !SANITIZER_GO
	161	// Mark shadow for .rodata sections with the special kShadowRodata marker.
	162	// Accesses to .rodata can't race, so this saves time, memory and trace space.
	163	static void MapRodata() {
	164	// First create temp file.
	165	const char *tmpdir = GetEnv("TMPDIR");
	166	if (tmpdir == 0)
	167	tmpdir = GetEnv("TEST_TMPDIR");
	168	#ifdef P_tmpdir
	169	if (tmpdir == 0)
	170	tmpdir = P_tmpdir;
	171	#endif
	172	if (tmpdir == 0)
	173	return;
	174	char name[256];
	175	internal_snprintf(name, sizeof(name), "%s/tsan.rodata.%d",
	176	tmpdir, (int)internal_getpid());
	177	uptr openrv = internal_open(name, O_RDWR \| O_CREAT \| O_EXCL, 0600);
	178	if (internal_iserror(openrv))
	179	return;
	180	internal_unlink(name); // Unlink it now, so that we can reuse the buffer.
	181	fd_t fd = openrv;
	182	// Fill the file with kShadowRodata.
	183	const uptr kMarkerSize = 512 * 1024 / sizeof(u64);
	184	InternalMmapVector<u64> marker(kMarkerSize);
	185	// volatile to prevent insertion of memset
	186	for (volatile u64 *p = marker.data(); p < marker.data() + kMarkerSize; p++)
	187	*p = kShadowRodata;
	188	internal_write(fd, marker.data(), marker.size() * sizeof(u64));
	189	// Map the file into memory.
	190	uptr page = internal_mmap(0, GetPageSizeCached(), PROT_READ \| PROT_WRITE,
	191	MAP_PRIVATE \| MAP_ANONYMOUS, fd, 0);
	192	if (internal_iserror(page)) {
	193	internal_close(fd);
	194	return;
	195	}
	196	// Map the file into shadow of .rodata sections.
	197	MemoryMappingLayout proc_maps(/cache_enabled/true);
	198	// Reusing the buffer 'name'.
	199	MemoryMappedSegment segment(name, ARRAY_SIZE(name));
	200	while (proc_maps.Next(&segment)) {
	201	if (segment.filename[0] != 0 && segment.filename[0] != '[' &&
	202	segment.IsReadable() && segment.IsExecutable() &&
	203	!segment.IsWritable() && IsAppMem(segment.start)) {
	204	// Assume it's .rodata
	205	char shadow_start = (char )MemToShadow(segment.start);
	206	char shadow_end = (char )MemToShadow(segment.end);
	207	for (char *p = shadow_start; p < shadow_end;
	208	p += marker.size() * sizeof(u64)) {
	209	internal_mmap(p, Min<uptr>(marker.size() * sizeof(u64), shadow_end - p),
	210	PROT_READ, MAP_PRIVATE \| MAP_FIXED, fd, 0);
	211	}
	212	}
	213	}
	214	internal_close(fd);
	215	}
	216
	217	void InitializeShadowMemoryPlatform() {
	218	MapRodata();
	219	}
	220
	221	#endif // #if !SANITIZER_GO
	222
	223	void InitializePlatformEarly() {
	224	#ifdef TSAN_RUNTIME_VMA
	225	vmaSize =
	226	(MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1);
	227	#if defined(__aarch64__)
	228	# if !SANITIZER_GO
	229	if (vmaSize != 39 && vmaSize != 42 && vmaSize != 48) {
	230	Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
	231	Printf("FATAL: Found %zd - Supported 39, 42 and 48\n", vmaSize);
	232	Die();
	233	}
	234	#else
	235	if (vmaSize != 48) {
	236	Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
	237	Printf("FATAL: Found %zd - Supported 48\n", vmaSize);
	238	Die();
	239	}
	240	#endif
	241	#elif defined(__powerpc64__)
	242	# if !SANITIZER_GO
	243	if (vmaSize != 44 && vmaSize != 46 && vmaSize != 47) {
	244	Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
	245	Printf("FATAL: Found %zd - Supported 44, 46, and 47\n", vmaSize);
	246	Die();
	247	}
	248	# else
	249	if (vmaSize != 46 && vmaSize != 47) {
	250	Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
	251	Printf("FATAL: Found %zd - Supported 46, and 47\n", vmaSize);
	252	Die();
	253	}
	254	# endif
	255	#endif
	256	#endif
	257	}
	258
	259	void InitializePlatform() {
	260	DisableCoreDumperIfNecessary();
	261
	262	// Go maps shadow memory lazily and works fine with limited address space.
	263	// Unlimited stack is not a problem as well, because the executable
	264	// is not compiled with -pie.
	265	#if !SANITIZER_GO
	266	{
	267	bool reexec = false;
	268	// TSan doesn't play well with unlimited stack size (as stack
	269	// overlaps with shadow memory). If we detect unlimited stack size,
	270	// we re-exec the program with limited stack size as a best effort.
	271	if (StackSizeIsUnlimited()) {
	272	const uptr kMaxStackSize = 32 * 1024 * 1024;
	273	VReport(1, "Program is run with unlimited stack size, which wouldn't "
	274	"work with ThreadSanitizer.\n"
	275	"Re-execing with stack size limited to %zd bytes.\n",
	276	kMaxStackSize);
	277	SetStackSizeLimitInBytes(kMaxStackSize);
	278	reexec = true;
	279	}
	280
	281	if (!AddressSpaceIsUnlimited()) {
	282	Report("WARNING: Program is run with limited virtual address space,"
	283	" which wouldn't work with ThreadSanitizer.\n");
	284	Report("Re-execing with unlimited virtual address space.\n");
	285	SetAddressSpaceUnlimited();
	286	reexec = true;
	287	}
	288	#if SANITIZER_LINUX && defined(__aarch64__)
	289	// After patch "arm64: mm: support ARCH_MMAP_RND_BITS." is introduced in
	290	// linux kernel, the random gap between stack and mapped area is increased
	291	// from 128M to 36G on 39-bit aarch64. As it is almost impossible to cover
	292	// this big range, we should disable randomized virtual space on aarch64.
	293	int old_personality = personality(0xffffffff);
	294	if (old_personality != -1 && (old_personality & ADDR_NO_RANDOMIZE) == 0) {
	295	VReport(1, "WARNING: Program is run with randomized virtual address "
	296	"space, which wouldn't work with ThreadSanitizer.\n"
	297	"Re-execing with fixed virtual address space.\n");
	298	CHECK_NE(personality(old_personality \| ADDR_NO_RANDOMIZE), -1);
	299	reexec = true;
	300	}
	301	// Initialize the xor key used in {sig}{set,long}jump.
	302	InitializeLongjmpXorKey();
	303	#endif
	304	if (reexec)
	305	ReExec();
	306	}
	307
	308	CheckAndProtect();
	309	InitTlsSize();
	310	#endif // !SANITIZER_GO
	311	}
	312
	313	#if !SANITIZER_GO
	314	// Extract file descriptors passed to glibc internal __res_iclose function.
	315	// This is required to properly "close" the fds, because we do not see internal
	316	// closes within glibc. The code is a pure hack.
	317	int ExtractResolvFDs(void state, int fds, int nfd) {
	318	#if SANITIZER_LINUX && !SANITIZER_ANDROID
	319	int cnt = 0;
	320	struct __res_state statp = (struct __res_state)state;
	321	for (int i = 0; i < MAXNS && cnt < nfd; i++) {
	322	if (statp->_u._ext.nsaddrs[i] && statp->_u._ext.nssocks[i] != -1)
	323	fds[cnt++] = statp->_u._ext.nssocks[i];
	324	}
	325	return cnt;
	326	#else
	327	return 0;
	328	#endif
	329	}
	330
	331	// Extract file descriptors passed via UNIX domain sockets.
	332	// This is requried to properly handle "open" of these fds.
	333	// see 'man recvmsg' and 'man 3 cmsg'.
	334	int ExtractRecvmsgFDs(void msgp, int fds, int nfd) {
	335	int res = 0;
	336	msghdr msg = (msghdr)msgp;
	337	struct cmsghdr *cmsg = CMSG_FIRSTHDR(msg);
	338	for (; cmsg; cmsg = CMSG_NXTHDR(msg, cmsg)) {
	339	if (cmsg->cmsg_level != SOL_SOCKET \|\| cmsg->cmsg_type != SCM_RIGHTS)
	340	continue;
	341	int n = (cmsg->cmsg_len - CMSG_LEN(0)) / sizeof(fds[0]);
	342	for (int i = 0; i < n; i++) {
	343	fds[res++] = ((int*)CMSG_DATA(cmsg))[i];
	344	if (res == nfd)
	345	return res;
	346	}
	347	}
	348	return res;
	349	}
	350
	351	// Reverse operation of libc stack pointer mangling
	352	static uptr UnmangleLongJmpSp(uptr mangled_sp) {
	353	#if defined(__x86_64__)
	354	# if SANITIZER_LINUX
	355	// Reverse of:
	356	// xor %fs:0x30, %rsi
	357	// rol $0x11, %rsi
	358	uptr sp;
	359	asm("ror $0x11, %0 \n"
	360	"xor %%fs:0x30, %0 \n"
	361	: "=r" (sp)
	362	: "0" (mangled_sp));
	363	return sp;
	364	# else
	365	return mangled_sp;
	366	# endif
	367	#elif defined(__aarch64__)
	368	# if SANITIZER_LINUX
	369	return mangled_sp ^ longjmp_xor_key;
	370	# else
	371	return mangled_sp;
	372	# endif
	373	#elif defined(__powerpc64__)
	374	// Reverse of:
	375	// ld r4, -28696(r13)
	376	// xor r4, r3, r4
	377	uptr xor_key;
	378	asm("ld %0, -28696(%%r13)" : "=r" (xor_key));
	379	return mangled_sp ^ xor_key;
	380	#elif defined(__mips__)
	381	return mangled_sp;
	382	#else
	383	#error "Unknown platform"
	384	#endif
	385	}
	386
	387	#ifdef __powerpc__
	388	# define LONG_JMP_SP_ENV_SLOT 0
	389	#elif SANITIZER_FREEBSD
	390	# define LONG_JMP_SP_ENV_SLOT 2
	391	#elif SANITIZER_NETBSD
	392	# define LONG_JMP_SP_ENV_SLOT 6
	393	#elif SANITIZER_LINUX
	394	# ifdef __aarch64__
	395	# define LONG_JMP_SP_ENV_SLOT 13
	396	# elif defined(__mips64)
	397	# define LONG_JMP_SP_ENV_SLOT 1
	398	# else
	399	# define LONG_JMP_SP_ENV_SLOT 6
	400	# endif
	401	#endif
	402
	403	uptr ExtractLongJmpSp(uptr *env) {
	404	uptr mangled_sp = env[LONG_JMP_SP_ENV_SLOT];
	405	return UnmangleLongJmpSp(mangled_sp);
	406	}
	407
	408	#if INIT_LONGJMP_XOR_KEY
	409	// GLIBC mangles the function pointers in jmp_buf (used in {set,long}*jmp
	410	// functions) by XORing them with a random key. For AArch64 it is a global
	411	// variable rather than a TCB one (as for x86_64/powerpc). We obtain the key by
	412	// issuing a setjmp and XORing the SP pointer values to derive the key.
	413	static void InitializeLongjmpXorKey() {
	414	// 1. Call REAL(setjmp), which stores the mangled SP in env.
	415	jmp_buf env;
	416	REAL(_setjmp)(env);
	417
	418	// 2. Retrieve vanilla/mangled SP.
	419	uptr sp;
	420	asm("mov %0, sp" : "=r" (sp));
	421	uptr mangled_sp = ((uptr *)&env)[LONG_JMP_SP_ENV_SLOT];
	422
	423	// 3. xor SPs to obtain key.
	424	longjmp_xor_key = mangled_sp ^ sp;
	425	}
	426	#endif
	427
	428	void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size) {
	429	// Check that the thr object is in tls;
	430	const uptr thr_beg = (uptr)thr;
	431	const uptr thr_end = (uptr)thr + sizeof(*thr);
	432	CHECK_GE(thr_beg, tls_addr);
	433	CHECK_LE(thr_beg, tls_addr + tls_size);
	434	CHECK_GE(thr_end, tls_addr);
	435	CHECK_LE(thr_end, tls_addr + tls_size);
	436	// Since the thr object is huge, skip it.
	437	MemoryRangeImitateWrite(thr, /pc=/2, tls_addr, thr_beg - tls_addr);
	438	MemoryRangeImitateWrite(thr, /pc=/2, thr_end,
	439	tls_addr + tls_size - thr_end);
	440	}
	441
	442	// Note: this function runs with async signals enabled,
	443	// so it must not touch any tsan state.
	444	int call_pthread_cancel_with_cleanup(int(fn)(void c, void *m,
	445	void abstime), void c, void m, void abstime,
	446	void(cleanup)(void arg), void *arg) {
	447	// pthread_cleanup_push/pop are hardcore macros mess.
	448	// We can't intercept nor call them w/o including pthread.h.
	449	int res;
	450	pthread_cleanup_push(cleanup, arg);
	451	res = fn(c, m, abstime);
	452	pthread_cleanup_pop(0);
	453	return res;
	454	}
	455	#endif // !SANITIZER_GO
	456
	457	#if !SANITIZER_GO
	458	void ReplaceSystemMalloc() { }
	459	#endif
	460
	461	#if !SANITIZER_GO
	462	#if SANITIZER_ANDROID
	463	// On Android, one thread can call intercepted functions after
	464	// DestroyThreadState(), so add a fake thread state for "dead" threads.
	465	static ThreadState *dead_thread_state = nullptr;
	466
	467	ThreadState *cur_thread() {
	468	ThreadState* thr = reinterpret_cast<ThreadState>(get_android_tls_ptr());
	469	if (thr == nullptr) {
	470	__sanitizer_sigset_t emptyset;
	471	internal_sigfillset(&emptyset);
	472	__sanitizer_sigset_t oldset;
	473	CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, &emptyset, &oldset));
	474	thr = reinterpret_cast<ThreadState>(get_android_tls_ptr());
	475	if (thr == nullptr) {
	476	thr = reinterpret_cast<ThreadState*>(MmapOrDie(sizeof(ThreadState),
	477	"ThreadState"));
	478	*get_android_tls_ptr() = reinterpret_cast<uptr>(thr);
	479	if (dead_thread_state == nullptr) {
	480	dead_thread_state = reinterpret_cast<ThreadState*>(
	481	MmapOrDie(sizeof(ThreadState), "ThreadState"));
	482	dead_thread_state->fast_state.SetIgnoreBit();
	483	dead_thread_state->ignore_interceptors = 1;
	484	dead_thread_state->is_dead = true;
	485	const_cast<int>(&dead_thread_state->tid) = -1;
	486	CHECK_EQ(0, internal_mprotect(dead_thread_state, sizeof(ThreadState),
	487	PROT_READ));
	488	}
	489	}
	490	CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, &oldset, nullptr));
	491	}
	492	return thr;
	493	}
	494
	495	void set_cur_thread(ThreadState *thr) {
	496	*get_android_tls_ptr() = reinterpret_cast<uptr>(thr);
	497	}
	498
	499	void cur_thread_finalize() {
	500	__sanitizer_sigset_t emptyset;
	501	internal_sigfillset(&emptyset);
	502	__sanitizer_sigset_t oldset;
	503	CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, &emptyset, &oldset));
	504	ThreadState* thr = reinterpret_cast<ThreadState>(get_android_tls_ptr());
	505	if (thr != dead_thread_state) {
	506	*get_android_tls_ptr() = reinterpret_cast<uptr>(dead_thread_state);
	507	UnmapOrDie(thr, sizeof(ThreadState));
	508	}
	509	CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, &oldset, nullptr));
	510	}
	511	#endif // SANITIZER_ANDROID
	512	#endif // if !SANITIZER_GO
	513
	514	} // namespace __tsan
	515
	516	#endif // SANITIZER_LINUX \|\| SANITIZER_FREEBSD \|\| SANITIZER_NETBSD \|\|
	517	// SANITIZER_OPENBSD

lib/tsan/tsan_platform_mac.cpp created+324

...	...	@@ -0,0 +1,324 @@
	1	//===-- tsan_platform_mac.cpp ---------------------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file is a part of ThreadSanitizer (TSan), a race detector.
	10	//
	11	// Mac-specific code.
	12	//===----------------------------------------------------------------------===//
	13
	14	#include "sanitizer_common/sanitizer_platform.h"
	15	#if SANITIZER_MAC
	16
	17	#include "sanitizer_common/sanitizer_atomic.h"
	18	#include "sanitizer_common/sanitizer_common.h"
	19	#include "sanitizer_common/sanitizer_libc.h"
	20	#include "sanitizer_common/sanitizer_posix.h"
	21	#include "sanitizer_common/sanitizer_procmaps.h"
	22	#include "sanitizer_common/sanitizer_ptrauth.h"
	23	#include "sanitizer_common/sanitizer_stackdepot.h"
	24	#include "tsan_platform.h"
	25	#include "tsan_rtl.h"
	26	#include "tsan_flags.h"
	27
	28	#include <mach/mach.h>
	29	#include <pthread.h>
	30	#include <signal.h>
	31	#include <stdio.h>
	32	#include <stdlib.h>
	33	#include <string.h>
	34	#include <stdarg.h>
	35	#include <sys/mman.h>
	36	#include <sys/syscall.h>
	37	#include <sys/time.h>
	38	#include <sys/types.h>
	39	#include <sys/resource.h>
	40	#include <sys/stat.h>
	41	#include <unistd.h>
	42	#include <errno.h>
	43	#include <sched.h>
	44
	45	namespace __tsan {
	46
	47	#if !SANITIZER_GO
	48	static void SignalSafeGetOrAllocate(uptr dst, uptr size) {
	49	atomic_uintptr_t a = (atomic_uintptr_t )dst;
	50	void val = (void )atomic_load_relaxed(a);
	51	atomic_signal_fence(memory_order_acquire); // Turns the previous load into
	52	// acquire wrt signals.
	53	if (UNLIKELY(val == nullptr)) {
	54	val = (void *)internal_mmap(nullptr, size, PROT_READ \| PROT_WRITE,
	55	MAP_PRIVATE \| MAP_ANON, -1, 0);
	56	CHECK(val);
	57	void *cmp = nullptr;
	58	if (!atomic_compare_exchange_strong(a, (uintptr_t *)&cmp, (uintptr_t)val,
	59	memory_order_acq_rel)) {
	60	internal_munmap(val, size);
	61	val = cmp;
	62	}
	63	}
	64	return val;
	65	}
	66
	67	// On OS X, accessing TLVs via __thread or manually by using pthread_key_* is
	68	// problematic, because there are several places where interceptors are called
	69	// when TLVs are not accessible (early process startup, thread cleanup, ...).
	70	// The following provides a "poor man's TLV" implementation, where we use the
	71	// shadow memory of the pointer returned by pthread_self() to store a pointer to
	72	// the ThreadState object. The main thread's ThreadState is stored separately
	73	// in a static variable, because we need to access it even before the
	74	// shadow memory is set up.
	75	static uptr main_thread_identity = 0;
	76	ALIGNED(64) static char main_thread_state[sizeof(ThreadState)];
	77	static ThreadState main_thread_state_loc = (ThreadState )main_thread_state;
	78
	79	// We cannot use pthread_self() before libpthread has been initialized. Our
	80	// current heuristic for guarding this is checking `main_thread_identity` which
	81	// is only assigned in `__tsan::InitializePlatform`.
	82	static ThreadState **cur_thread_location() {
	83	if (main_thread_identity == 0)
	84	return &main_thread_state_loc;
	85	uptr thread_identity = (uptr)pthread_self();
	86	if (thread_identity == main_thread_identity)
	87	return &main_thread_state_loc;
	88	return (ThreadState **)MemToShadow(thread_identity);
	89	}
	90
	91	ThreadState *cur_thread() {
	92	return (ThreadState *)SignalSafeGetOrAllocate(
	93	(uptr *)cur_thread_location(), sizeof(ThreadState));
	94	}
	95
	96	void set_cur_thread(ThreadState *thr) {
	97	*cur_thread_location() = thr;
	98	}
	99
	100	// TODO(kuba.brecka): This is not async-signal-safe. In particular, we call
	101	// munmap first and then clear `fake_tls`; if we receive a signal in between,
	102	// handler will try to access the unmapped ThreadState.
	103	void cur_thread_finalize() {
	104	ThreadState **thr_state_loc = cur_thread_location();
	105	if (thr_state_loc == &main_thread_state_loc) {
	106	// Calling dispatch_main() or xpc_main() actually invokes pthread_exit to
	107	// exit the main thread. Let's keep the main thread's ThreadState.
	108	return;
	109	}
	110	internal_munmap(*thr_state_loc, sizeof(ThreadState));
	111	*thr_state_loc = nullptr;
	112	}
	113	#endif
	114
	115	void FlushShadowMemory() {
	116	}
	117
	118	static void RegionMemUsage(uptr start, uptr end, uptr res, uptr dirty) {
	119	vm_address_t address = start;
	120	vm_address_t end_address = end;
	121	uptr resident_pages = 0;
	122	uptr dirty_pages = 0;
	123	while (address < end_address) {
	124	vm_size_t vm_region_size;
	125	mach_msg_type_number_t count = VM_REGION_EXTENDED_INFO_COUNT;
	126	vm_region_extended_info_data_t vm_region_info;
	127	mach_port_t object_name;
	128	kern_return_t ret = vm_region_64(
	129	mach_task_self(), &address, &vm_region_size, VM_REGION_EXTENDED_INFO,
	130	(vm_region_info_t)&vm_region_info, &count, &object_name);
	131	if (ret != KERN_SUCCESS) break;
	132
	133	resident_pages += vm_region_info.pages_resident;
	134	dirty_pages += vm_region_info.pages_dirtied;
	135
	136	address += vm_region_size;
	137	}
	138	res = resident_pages GetPageSizeCached();
	139	dirty = dirty_pages GetPageSizeCached();
	140	}
	141
	142	void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive) {
	143	uptr shadow_res, shadow_dirty;
	144	uptr meta_res, meta_dirty;
	145	uptr trace_res, trace_dirty;
	146	RegionMemUsage(ShadowBeg(), ShadowEnd(), &shadow_res, &shadow_dirty);
	147	RegionMemUsage(MetaShadowBeg(), MetaShadowEnd(), &meta_res, &meta_dirty);
	148	RegionMemUsage(TraceMemBeg(), TraceMemEnd(), &trace_res, &trace_dirty);
	149
	150	#if !SANITIZER_GO
	151	uptr low_res, low_dirty;
	152	uptr high_res, high_dirty;
	153	uptr heap_res, heap_dirty;
	154	RegionMemUsage(LoAppMemBeg(), LoAppMemEnd(), &low_res, &low_dirty);
	155	RegionMemUsage(HiAppMemBeg(), HiAppMemEnd(), &high_res, &high_dirty);
	156	RegionMemUsage(HeapMemBeg(), HeapMemEnd(), &heap_res, &heap_dirty);
	157	#else // !SANITIZER_GO
	158	uptr app_res, app_dirty;
	159	RegionMemUsage(AppMemBeg(), AppMemEnd(), &app_res, &app_dirty);
	160	#endif
	161
	162	StackDepotStats *stacks = StackDepotGetStats();
	163	internal_snprintf(buf, buf_size,
	164	"shadow (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
	165	"meta (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
	166	"traces (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
	167	#if !SANITIZER_GO
	168	"low app (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
	169	"high app (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
	170	"heap (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
	171	#else // !SANITIZER_GO
	172	"app (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
	173	#endif
	174	"stacks: %zd unique IDs, %zd kB allocated\n"
	175	"threads: %zd total, %zd live\n"
	176	"------------------------------\n",
	177	ShadowBeg(), ShadowEnd(), shadow_res / 1024, shadow_dirty / 1024,
	178	MetaShadowBeg(), MetaShadowEnd(), meta_res / 1024, meta_dirty / 1024,
	179	TraceMemBeg(), TraceMemEnd(), trace_res / 1024, trace_dirty / 1024,
	180	#if !SANITIZER_GO
	181	LoAppMemBeg(), LoAppMemEnd(), low_res / 1024, low_dirty / 1024,
	182	HiAppMemBeg(), HiAppMemEnd(), high_res / 1024, high_dirty / 1024,
	183	HeapMemBeg(), HeapMemEnd(), heap_res / 1024, heap_dirty / 1024,
	184	#else // !SANITIZER_GO
	185	AppMemBeg(), AppMemEnd(), app_res / 1024, app_dirty / 1024,
	186	#endif
	187	stacks->n_uniq_ids, stacks->allocated / 1024,
	188	nthread, nlive);
	189	}
	190
	191	#if !SANITIZER_GO
	192	void InitializeShadowMemoryPlatform() { }
	193
	194	// On OS X, GCD worker threads are created without a call to pthread_create. We
	195	// need to properly register these threads with ThreadCreate and ThreadStart.
	196	// These threads don't have a parent thread, as they are created "spuriously".
	197	// We're using a libpthread API that notifies us about a newly created thread.
	198	// The `thread == pthread_self()` check indicates this is actually a worker
	199	// thread. If it's just a regular thread, this hook is called on the parent
	200	// thread.
	201	typedef void (*pthread_introspection_hook_t)(unsigned int event,
	202	pthread_t thread, void *addr,
	203	size_t size);
	204	extern "C" pthread_introspection_hook_t pthread_introspection_hook_install(
	205	pthread_introspection_hook_t hook);
	206	static const uptr PTHREAD_INTROSPECTION_THREAD_CREATE = 1;
	207	static const uptr PTHREAD_INTROSPECTION_THREAD_TERMINATE = 3;
	208	static pthread_introspection_hook_t prev_pthread_introspection_hook;
	209	static void my_pthread_introspection_hook(unsigned int event, pthread_t thread,
	210	void *addr, size_t size) {
	211	if (event == PTHREAD_INTROSPECTION_THREAD_CREATE) {
	212	if (thread == pthread_self()) {
	213	// The current thread is a newly created GCD worker thread.
	214	ThreadState *thr = cur_thread();
	215	Processor *proc = ProcCreate();
	216	ProcWire(proc, thr);
	217	ThreadState *parent_thread_state = nullptr; // No parent.
	218	int tid = ThreadCreate(parent_thread_state, 0, (uptr)thread, true);
	219	CHECK_NE(tid, 0);
	220	ThreadStart(thr, tid, GetTid(), ThreadType::Worker);
	221	}
	222	} else if (event == PTHREAD_INTROSPECTION_THREAD_TERMINATE) {
	223	if (thread == pthread_self()) {
	224	ThreadState *thr = cur_thread();
	225	if (thr->tctx) {
	226	DestroyThreadState();
	227	}
	228	}
	229	}
	230
	231	if (prev_pthread_introspection_hook != nullptr)
	232	prev_pthread_introspection_hook(event, thread, addr, size);
	233	}
	234	#endif
	235
	236	void InitializePlatformEarly() {
	237	#if defined(__aarch64__)
	238	uptr max_vm = GetMaxUserVirtualAddress() + 1;
	239	if (max_vm != Mapping::kHiAppMemEnd) {
	240	Printf("ThreadSanitizer: unsupported vm address limit %p, expected %p.\n",
	241	max_vm, Mapping::kHiAppMemEnd);
	242	Die();
	243	}
	244	#endif
	245	}
	246
	247	static uptr longjmp_xor_key = 0;
	248
	249	void InitializePlatform() {
	250	DisableCoreDumperIfNecessary();
	251	#if !SANITIZER_GO
	252	CheckAndProtect();
	253
	254	CHECK_EQ(main_thread_identity, 0);
	255	main_thread_identity = (uptr)pthread_self();
	256
	257	prev_pthread_introspection_hook =
	258	pthread_introspection_hook_install(&my_pthread_introspection_hook);
	259	#endif
	260
	261	if (GetMacosAlignedVersion() >= MacosVersion(10, 14)) {
	262	// Libsystem currently uses a process-global key; this might change.
	263	const unsigned kTLSLongjmpXorKeySlot = 0x7;
	264	longjmp_xor_key = (uptr)pthread_getspecific(kTLSLongjmpXorKeySlot);
	265	}
	266	}
	267
	268	#ifdef __aarch64__
	269	# define LONG_JMP_SP_ENV_SLOT \
	270	((GetMacosAlignedVersion() >= MacosVersion(10, 14)) ? 12 : 13)
	271	#else
	272	# define LONG_JMP_SP_ENV_SLOT 2
	273	#endif
	274
	275	uptr ExtractLongJmpSp(uptr *env) {
	276	uptr mangled_sp = env[LONG_JMP_SP_ENV_SLOT];
	277	uptr sp = mangled_sp ^ longjmp_xor_key;
	278	sp = (uptr)ptrauth_auth_data((void *)sp, ptrauth_key_asdb,
	279	ptrauth_string_discriminator("sp"));
	280	return sp;
	281	}
	282
	283	#if !SANITIZER_GO
	284	void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size) {
	285	// The pointer to the ThreadState object is stored in the shadow memory
	286	// of the tls.
	287	uptr tls_end = tls_addr + tls_size;
	288	uptr thread_identity = (uptr)pthread_self();
	289	if (thread_identity == main_thread_identity) {
	290	MemoryRangeImitateWrite(thr, /pc=/2, tls_addr, tls_size);
	291	} else {
	292	uptr thr_state_start = thread_identity;
	293	uptr thr_state_end = thr_state_start + sizeof(uptr);
	294	CHECK_GE(thr_state_start, tls_addr);
	295	CHECK_LE(thr_state_start, tls_addr + tls_size);
	296	CHECK_GE(thr_state_end, tls_addr);
	297	CHECK_LE(thr_state_end, tls_addr + tls_size);
	298	MemoryRangeImitateWrite(thr, /pc=/2, tls_addr,
	299	thr_state_start - tls_addr);
	300	MemoryRangeImitateWrite(thr, /pc=/2, thr_state_end,
	301	tls_end - thr_state_end);
	302	}
	303	}
	304	#endif
	305
	306	#if !SANITIZER_GO
	307	// Note: this function runs with async signals enabled,
	308	// so it must not touch any tsan state.
	309	int call_pthread_cancel_with_cleanup(int(fn)(void c, void *m,
	310	void abstime), void c, void m, void abstime,
	311	void(cleanup)(void arg), void *arg) {
	312	// pthread_cleanup_push/pop are hardcore macros mess.
	313	// We can't intercept nor call them w/o including pthread.h.
	314	int res;
	315	pthread_cleanup_push(cleanup, arg);
	316	res = fn(c, m, abstime);
	317	pthread_cleanup_pop(0);
	318	return res;
	319	}
	320	#endif
	321
	322	} // namespace __tsan
	323
	324	#endif // SANITIZER_MAC

lib/tsan/tsan_platform_posix.cpp created+167

...	...	@@ -0,0 +1,167 @@
	1	//===-- tsan_platform_posix.cpp -------------------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file is a part of ThreadSanitizer (TSan), a race detector.
	10	//
	11	// POSIX-specific code.
	12	//===----------------------------------------------------------------------===//
	13
	14	#include "sanitizer_common/sanitizer_platform.h"
	15	#if SANITIZER_POSIX
	16
	17	#include "sanitizer_common/sanitizer_common.h"
	18	#include "sanitizer_common/sanitizer_errno.h"
	19	#include "sanitizer_common/sanitizer_libc.h"
	20	#include "sanitizer_common/sanitizer_procmaps.h"
	21	#include "tsan_platform.h"
	22	#include "tsan_rtl.h"
	23
	24	namespace __tsan {
	25
	26	static const char kShadowMemoryMappingWarning[] =
	27	"FATAL: %s can not madvise shadow region [%zx, %zx] with %s (errno: %d)\n";
	28	static const char kShadowMemoryMappingHint[] =
	29	"HINT: if %s is not supported in your environment, you may set "
	30	"TSAN_OPTIONS=%s=0\n";
	31
	32	static void NoHugePagesInShadow(uptr addr, uptr size) {
	33	SetShadowRegionHugePageMode(addr, size);
	34	}
	35
	36	static void DontDumpShadow(uptr addr, uptr size) {
	37	if (common_flags()->use_madv_dontdump)
	38	if (!DontDumpShadowMemory(addr, size)) {
	39	Printf(kShadowMemoryMappingWarning, SanitizerToolName, addr, addr + size,
	40	"MADV_DONTDUMP", errno);
	41	Printf(kShadowMemoryMappingHint, "MADV_DONTDUMP", "use_madv_dontdump");
	42	Die();
	43	}
	44	}
	45
	46	#if !SANITIZER_GO
	47	void InitializeShadowMemory() {
	48	// Map memory shadow.
	49	if (!MmapFixedNoReserve(ShadowBeg(), ShadowEnd() - ShadowBeg(), "shadow")) {
	50	Printf("FATAL: ThreadSanitizer can not mmap the shadow memory\n");
	51	Printf("FATAL: Make sure to compile with -fPIE and to link with -pie.\n");
	52	Die();
	53	}
	54	// This memory range is used for thread stacks and large user mmaps.
	55	// Frequently a thread uses only a small part of stack and similarly
	56	// a program uses a small part of large mmap. On some programs
	57	// we see 20% memory usage reduction without huge pages for this range.
	58	// FIXME: don't use constants here.
	59	#if defined(__x86_64__)
	60	const uptr kMadviseRangeBeg = 0x7f0000000000ull;
	61	const uptr kMadviseRangeSize = 0x010000000000ull;
	62	#elif defined(__mips64)
	63	const uptr kMadviseRangeBeg = 0xff00000000ull;
	64	const uptr kMadviseRangeSize = 0x0100000000ull;
	65	#elif defined(__aarch64__) && defined(__APPLE__)
	66	uptr kMadviseRangeBeg = LoAppMemBeg();
	67	uptr kMadviseRangeSize = LoAppMemEnd() - LoAppMemBeg();
	68	#elif defined(__aarch64__)
	69	uptr kMadviseRangeBeg = 0;
	70	uptr kMadviseRangeSize = 0;
	71	if (vmaSize == 39) {
	72	kMadviseRangeBeg = 0x7d00000000ull;
	73	kMadviseRangeSize = 0x0300000000ull;
	74	} else if (vmaSize == 42) {
	75	kMadviseRangeBeg = 0x3f000000000ull;
	76	kMadviseRangeSize = 0x01000000000ull;
	77	} else {
	78	DCHECK(0);
	79	}
	80	#elif defined(__powerpc64__)
	81	uptr kMadviseRangeBeg = 0;
	82	uptr kMadviseRangeSize = 0;
	83	if (vmaSize == 44) {
	84	kMadviseRangeBeg = 0x0f60000000ull;
	85	kMadviseRangeSize = 0x0010000000ull;
	86	} else if (vmaSize == 46) {
	87	kMadviseRangeBeg = 0x3f0000000000ull;
	88	kMadviseRangeSize = 0x010000000000ull;
	89	} else {
	90	DCHECK(0);
	91	}
	92	#endif
	93	NoHugePagesInShadow(MemToShadow(kMadviseRangeBeg),
	94	kMadviseRangeSize * kShadowMultiplier);
	95	DontDumpShadow(ShadowBeg(), ShadowEnd() - ShadowBeg());
	96	DPrintf("memory shadow: %zx-%zx (%zuGB)\n",
	97	ShadowBeg(), ShadowEnd(),
	98	(ShadowEnd() - ShadowBeg()) >> 30);
	99
	100	// Map meta shadow.
	101	const uptr meta = MetaShadowBeg();
	102	const uptr meta_size = MetaShadowEnd() - meta;
	103	if (!MmapFixedNoReserve(meta, meta_size, "meta shadow")) {
	104	Printf("FATAL: ThreadSanitizer can not mmap the shadow memory\n");
	105	Printf("FATAL: Make sure to compile with -fPIE and to link with -pie.\n");
	106	Die();
	107	}
	108	NoHugePagesInShadow(meta, meta_size);
	109	DontDumpShadow(meta, meta_size);
	110	DPrintf("meta shadow: %zx-%zx (%zuGB)\n",
	111	meta, meta + meta_size, meta_size >> 30);
	112
	113	InitializeShadowMemoryPlatform();
	114	}
	115
	116	static void ProtectRange(uptr beg, uptr end) {
	117	CHECK_LE(beg, end);
	118	if (beg == end)
	119	return;
	120	if (beg != (uptr)MmapFixedNoAccess(beg, end - beg)) {
	121	Printf("FATAL: ThreadSanitizer can not protect [%zx,%zx]\n", beg, end);
	122	Printf("FATAL: Make sure you are not using unlimited stack\n");
	123	Die();
	124	}
	125	}
	126
	127	void CheckAndProtect() {
	128	// Ensure that the binary is indeed compiled with -pie.
	129	MemoryMappingLayout proc_maps(true);
	130	MemoryMappedSegment segment;
	131	while (proc_maps.Next(&segment)) {
	132	if (IsAppMem(segment.start)) continue;
	133	if (segment.start >= HeapMemEnd() && segment.start < HeapEnd()) continue;
	134	if (segment.protection == 0) // Zero page or mprotected.
	135	continue;
	136	if (segment.start >= VdsoBeg()) // vdso
	137	break;
	138	Printf("FATAL: ThreadSanitizer: unexpected memory mapping %p-%p\n",
	139	segment.start, segment.end);
	140	Die();
	141	}
	142
	143	#if defined(__aarch64__) && defined(__APPLE__)
	144	ProtectRange(HeapMemEnd(), ShadowBeg());
	145	ProtectRange(ShadowEnd(), MetaShadowBeg());
	146	ProtectRange(MetaShadowEnd(), TraceMemBeg());
	147	#else
	148	ProtectRange(LoAppMemEnd(), ShadowBeg());
	149	ProtectRange(ShadowEnd(), MetaShadowBeg());
	150	#ifdef TSAN_MID_APP_RANGE
	151	ProtectRange(MetaShadowEnd(), MidAppMemBeg());
	152	ProtectRange(MidAppMemEnd(), TraceMemBeg());
	153	#else
	154	ProtectRange(MetaShadowEnd(), TraceMemBeg());
	155	#endif
	156	// Memory for traces is mapped lazily in MapThreadTrace.
	157	// Protect the whole range for now, so that user does not map something here.
	158	ProtectRange(TraceMemBeg(), TraceMemEnd());
	159	ProtectRange(TraceMemEnd(), HeapMemBeg());
	160	ProtectRange(HeapEnd(), HiAppMemBeg());
	161	#endif
	162	}
	163	#endif
	164
	165	} // namespace __tsan
	166
	167	#endif // SANITIZER_POSIX

lib/tsan/tsan_rtl_aarch64.S created+245

...	...	@@ -0,0 +1,245 @@
	1	// The content of this file is AArch64-only:
	2	#if defined(__aarch64__)
	3
	4	#include "sanitizer_common/sanitizer_asm.h"
	5
	6	#if defined(__APPLE__)
	7	.align 2
	8
	9	.section __DATA,__nl_symbol_ptr,non_lazy_symbol_pointers
	10	.long _setjmp$non_lazy_ptr
	11	_setjmp$non_lazy_ptr:
	12	.indirect_symbol _setjmp
	13	.long 0
	14
	15	.section __DATA,__nl_symbol_ptr,non_lazy_symbol_pointers
	16	.long __setjmp$non_lazy_ptr
	17	__setjmp$non_lazy_ptr:
	18	.indirect_symbol __setjmp
	19	.long 0
	20
	21	.section __DATA,__nl_symbol_ptr,non_lazy_symbol_pointers
	22	.long _sigsetjmp$non_lazy_ptr
	23	_sigsetjmp$non_lazy_ptr:
	24	.indirect_symbol _sigsetjmp
	25	.long 0
	26	#endif
	27
	28	#if !defined(__APPLE__)
	29	.section .text
	30	#else
	31	.section __TEXT,__text
	32	.align 3
	33	#endif
	34
	35	ASM_HIDDEN(__tsan_setjmp)
	36	.comm _ZN14__interception11real_setjmpE,8,8
	37	.globl ASM_SYMBOL_INTERCEPTOR(setjmp)
	38	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(setjmp))
	39	ASM_SYMBOL_INTERCEPTOR(setjmp):
	40	CFI_STARTPROC
	41
	42	// Save frame/link register
	43	stp x29, x30, [sp, -32]!
	44	CFI_DEF_CFA_OFFSET (32)
	45	CFI_OFFSET (29, -32)
	46	CFI_OFFSET (30, -24)
	47
	48	// Adjust the SP for previous frame
	49	add x29, sp, 0
	50	CFI_DEF_CFA_REGISTER (29)
	51
	52	// Save env parameter
	53	str x0, [sp, 16]
	54	CFI_OFFSET (0, -16)
	55
	56	// Obtain SP, first argument to `void __tsan_setjmp(uptr sp)`
	57	add x0, x29, 32
	58
	59	// call tsan interceptor
	60	bl ASM_SYMBOL(__tsan_setjmp)
	61
	62	// Restore env parameter
	63	ldr x0, [sp, 16]
	64	CFI_RESTORE (0)
	65
	66	// Restore frame/link register
	67	ldp x29, x30, [sp], 32
	68	CFI_RESTORE (29)
	69	CFI_RESTORE (30)
	70	CFI_DEF_CFA (31, 0)
	71
	72	// tail jump to libc setjmp
	73	#if !defined(__APPLE__)
	74	adrp x1, :got:_ZN14__interception11real_setjmpE
	75	ldr x1, [x1, #:got_lo12:_ZN14__interception11real_setjmpE]
	76	ldr x1, [x1]
	77	#else
	78	adrp x1, _setjmp$non_lazy_ptr@page
	79	add x1, x1, _setjmp$non_lazy_ptr@pageoff
	80	ldr x1, [x1]
	81	#endif
	82	br x1
	83
	84	CFI_ENDPROC
	85	ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(setjmp))
	86
	87	.comm _ZN14__interception12real__setjmpE,8,8
	88	.globl ASM_SYMBOL_INTERCEPTOR(_setjmp)
	89	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(_setjmp))
	90	ASM_SYMBOL_INTERCEPTOR(_setjmp):
	91	CFI_STARTPROC
	92
	93	// Save frame/link register
	94	stp x29, x30, [sp, -32]!
	95	CFI_DEF_CFA_OFFSET (32)
	96	CFI_OFFSET (29, -32)
	97	CFI_OFFSET (30, -24)
	98
	99	// Adjust the SP for previous frame
	100	add x29, sp, 0
	101	CFI_DEF_CFA_REGISTER (29)
	102
	103	// Save env parameter
	104	str x0, [sp, 16]
	105	CFI_OFFSET (0, -16)
	106
	107	// Obtain SP, first argument to `void __tsan_setjmp(uptr sp)`
	108	add x0, x29, 32
	109
	110	// call tsan interceptor
	111	bl ASM_SYMBOL(__tsan_setjmp)
	112
	113	// Restore env parameter
	114	ldr x0, [sp, 16]
	115	CFI_RESTORE (0)
	116
	117	// Restore frame/link register
	118	ldp x29, x30, [sp], 32
	119	CFI_RESTORE (29)
	120	CFI_RESTORE (30)
	121	CFI_DEF_CFA (31, 0)
	122
	123	// tail jump to libc setjmp
	124	#if !defined(__APPLE__)
	125	adrp x1, :got:_ZN14__interception12real__setjmpE
	126	ldr x1, [x1, #:got_lo12:_ZN14__interception12real__setjmpE]
	127	ldr x1, [x1]
	128	#else
	129	adrp x1, __setjmp$non_lazy_ptr@page
	130	add x1, x1, __setjmp$non_lazy_ptr@pageoff
	131	ldr x1, [x1]
	132	#endif
	133	br x1
	134
	135	CFI_ENDPROC
	136	ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(_setjmp))
	137
	138	.comm _ZN14__interception14real_sigsetjmpE,8,8
	139	.globl ASM_SYMBOL_INTERCEPTOR(sigsetjmp)
	140	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))
	141	ASM_SYMBOL_INTERCEPTOR(sigsetjmp):
	142	CFI_STARTPROC
	143
	144	// Save frame/link register
	145	stp x29, x30, [sp, -32]!
	146	CFI_DEF_CFA_OFFSET (32)
	147	CFI_OFFSET (29, -32)
	148	CFI_OFFSET (30, -24)
	149
	150	// Adjust the SP for previous frame
	151	add x29, sp, 0
	152	CFI_DEF_CFA_REGISTER (29)
	153
	154	// Save env and savesigs parameter
	155	stp x0, x1, [sp, 16]
	156	CFI_OFFSET (0, -16)
	157	CFI_OFFSET (1, -8)
	158
	159	// Obtain SP, first argument to `void __tsan_setjmp(uptr sp)`
	160	add x0, x29, 32
	161
	162	// call tsan interceptor
	163	bl ASM_SYMBOL(__tsan_setjmp)
	164
	165	// Restore env and savesigs parameter
	166	ldp x0, x1, [sp, 16]
	167	CFI_RESTORE (0)
	168	CFI_RESTORE (1)
	169
	170	// Restore frame/link register
	171	ldp x29, x30, [sp], 32
	172	CFI_RESTORE (29)
	173	CFI_RESTORE (30)
	174	CFI_DEF_CFA (31, 0)
	175
	176	// tail jump to libc sigsetjmp
	177	#if !defined(__APPLE__)
	178	adrp x2, :got:_ZN14__interception14real_sigsetjmpE
	179	ldr x2, [x2, #:got_lo12:_ZN14__interception14real_sigsetjmpE]
	180	ldr x2, [x2]
	181	#else
	182	adrp x2, _sigsetjmp$non_lazy_ptr@page
	183	add x2, x2, _sigsetjmp$non_lazy_ptr@pageoff
	184	ldr x2, [x2]
	185	#endif
	186	br x2
	187	CFI_ENDPROC
	188	ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))
	189
	190	#if !defined(__APPLE__)
	191	.comm _ZN14__interception16real___sigsetjmpE,8,8
	192	.globl ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)
	193	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))
	194	ASM_SYMBOL_INTERCEPTOR(__sigsetjmp):
	195	CFI_STARTPROC
	196
	197	// Save frame/link register
	198	stp x29, x30, [sp, -32]!
	199	CFI_DEF_CFA_OFFSET (32)
	200	CFI_OFFSET (29, -32)
	201	CFI_OFFSET (30, -24)
	202
	203	// Adjust the SP for previous frame
	204	add x29, sp, 0
	205	CFI_DEF_CFA_REGISTER (29)
	206
	207	// Save env and savesigs parameter
	208	stp x0, x1, [sp, 16]
	209	CFI_OFFSET (0, -16)
	210	CFI_OFFSET (1, -8)
	211
	212	// Obtain SP, first argument to `void __tsan_setjmp(uptr sp)`
	213	add x0, x29, 32
	214
	215	// call tsan interceptor
	216	bl ASM_SYMBOL(__tsan_setjmp)
	217
	218	// Restore env and savesigs parameter
	219	ldp x0, x1, [sp, 16]
	220	CFI_RESTORE (0)
	221	CFI_RESTORE (1)
	222
	223	// Restore frame/link register
	224	ldp x29, x30, [sp], 32
	225	CFI_RESTORE (29)
	226	CFI_RESTORE (30)
	227	CFI_DEF_CFA (31, 0)
	228
	229	// tail jump to libc __sigsetjmp
	230	#if !defined(__APPLE__)
	231	adrp x2, :got:_ZN14__interception16real___sigsetjmpE
	232	ldr x2, [x2, #:got_lo12:_ZN14__interception16real___sigsetjmpE]
	233	ldr x2, [x2]
	234	#else
	235	adrp x2, ASM_SYMBOL(__sigsetjmp)@page
	236	add x2, x2, ASM_SYMBOL(__sigsetjmp)@pageoff
	237	#endif
	238	br x2
	239	CFI_ENDPROC
	240	ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))
	241	#endif
	242
	243	NO_EXEC_STACK_DIRECTIVE
	244
	245	#endif

lib/tsan/tsan_rtl_amd64.S created+366

...	...	@@ -0,0 +1,366 @@
	1	// The content of this file is x86_64-only:
	2	#if defined(__x86_64__)
	3
	4	#include "sanitizer_common/sanitizer_asm.h"
	5
	6	#if !defined(__APPLE__)
	7	.section .text
	8	#else
	9	.section __TEXT,__text
	10	#endif
	11
	12	ASM_HIDDEN(__tsan_trace_switch)
	13	.globl ASM_SYMBOL(__tsan_trace_switch_thunk)
	14	ASM_SYMBOL(__tsan_trace_switch_thunk):
	15	CFI_STARTPROC
	16	# Save scratch registers.
	17	push %rax
	18	CFI_ADJUST_CFA_OFFSET(8)
	19	CFI_REL_OFFSET(%rax, 0)
	20	push %rcx
	21	CFI_ADJUST_CFA_OFFSET(8)
	22	CFI_REL_OFFSET(%rcx, 0)
	23	push %rdx
	24	CFI_ADJUST_CFA_OFFSET(8)
	25	CFI_REL_OFFSET(%rdx, 0)
	26	push %rsi
	27	CFI_ADJUST_CFA_OFFSET(8)
	28	CFI_REL_OFFSET(%rsi, 0)
	29	push %rdi
	30	CFI_ADJUST_CFA_OFFSET(8)
	31	CFI_REL_OFFSET(%rdi, 0)
	32	push %r8
	33	CFI_ADJUST_CFA_OFFSET(8)
	34	CFI_REL_OFFSET(%r8, 0)
	35	push %r9
	36	CFI_ADJUST_CFA_OFFSET(8)
	37	CFI_REL_OFFSET(%r9, 0)
	38	push %r10
	39	CFI_ADJUST_CFA_OFFSET(8)
	40	CFI_REL_OFFSET(%r10, 0)
	41	push %r11
	42	CFI_ADJUST_CFA_OFFSET(8)
	43	CFI_REL_OFFSET(%r11, 0)
	44	# Align stack frame.
	45	push %rbx # non-scratch
	46	CFI_ADJUST_CFA_OFFSET(8)
	47	CFI_REL_OFFSET(%rbx, 0)
	48	mov %rsp, %rbx # save current rsp
	49	CFI_DEF_CFA_REGISTER(%rbx)
	50	shr $4, %rsp # clear 4 lsb, align to 16
	51	shl $4, %rsp
	52
	53	call ASM_SYMBOL(__tsan_trace_switch)
	54
	55	# Unalign stack frame back.
	56	mov %rbx, %rsp # restore the original rsp
	57	CFI_DEF_CFA_REGISTER(%rsp)
	58	pop %rbx
	59	CFI_ADJUST_CFA_OFFSET(-8)
	60	# Restore scratch registers.
	61	pop %r11
	62	CFI_ADJUST_CFA_OFFSET(-8)
	63	pop %r10
	64	CFI_ADJUST_CFA_OFFSET(-8)
	65	pop %r9
	66	CFI_ADJUST_CFA_OFFSET(-8)
	67	pop %r8
	68	CFI_ADJUST_CFA_OFFSET(-8)
	69	pop %rdi
	70	CFI_ADJUST_CFA_OFFSET(-8)
	71	pop %rsi
	72	CFI_ADJUST_CFA_OFFSET(-8)
	73	pop %rdx
	74	CFI_ADJUST_CFA_OFFSET(-8)
	75	pop %rcx
	76	CFI_ADJUST_CFA_OFFSET(-8)
	77	pop %rax
	78	CFI_ADJUST_CFA_OFFSET(-8)
	79	CFI_RESTORE(%rax)
	80	CFI_RESTORE(%rbx)
	81	CFI_RESTORE(%rcx)
	82	CFI_RESTORE(%rdx)
	83	CFI_RESTORE(%rsi)
	84	CFI_RESTORE(%rdi)
	85	CFI_RESTORE(%r8)
	86	CFI_RESTORE(%r9)
	87	CFI_RESTORE(%r10)
	88	CFI_RESTORE(%r11)
	89	ret
	90	CFI_ENDPROC
	91
	92	ASM_HIDDEN(__tsan_report_race)
	93	.globl ASM_SYMBOL(__tsan_report_race_thunk)
	94	ASM_SYMBOL(__tsan_report_race_thunk):
	95	CFI_STARTPROC
	96	# Save scratch registers.
	97	push %rax
	98	CFI_ADJUST_CFA_OFFSET(8)
	99	CFI_REL_OFFSET(%rax, 0)
	100	push %rcx
	101	CFI_ADJUST_CFA_OFFSET(8)
	102	CFI_REL_OFFSET(%rcx, 0)
	103	push %rdx
	104	CFI_ADJUST_CFA_OFFSET(8)
	105	CFI_REL_OFFSET(%rdx, 0)
	106	push %rsi
	107	CFI_ADJUST_CFA_OFFSET(8)
	108	CFI_REL_OFFSET(%rsi, 0)
	109	push %rdi
	110	CFI_ADJUST_CFA_OFFSET(8)
	111	CFI_REL_OFFSET(%rdi, 0)
	112	push %r8
	113	CFI_ADJUST_CFA_OFFSET(8)
	114	CFI_REL_OFFSET(%r8, 0)
	115	push %r9
	116	CFI_ADJUST_CFA_OFFSET(8)
	117	CFI_REL_OFFSET(%r9, 0)
	118	push %r10
	119	CFI_ADJUST_CFA_OFFSET(8)
	120	CFI_REL_OFFSET(%r10, 0)
	121	push %r11
	122	CFI_ADJUST_CFA_OFFSET(8)
	123	CFI_REL_OFFSET(%r11, 0)
	124	# Align stack frame.
	125	push %rbx # non-scratch
	126	CFI_ADJUST_CFA_OFFSET(8)
	127	CFI_REL_OFFSET(%rbx, 0)
	128	mov %rsp, %rbx # save current rsp
	129	CFI_DEF_CFA_REGISTER(%rbx)
	130	shr $4, %rsp # clear 4 lsb, align to 16
	131	shl $4, %rsp
	132
	133	call ASM_SYMBOL(__tsan_report_race)
	134
	135	# Unalign stack frame back.
	136	mov %rbx, %rsp # restore the original rsp
	137	CFI_DEF_CFA_REGISTER(%rsp)
	138	pop %rbx
	139	CFI_ADJUST_CFA_OFFSET(-8)
	140	# Restore scratch registers.
	141	pop %r11
	142	CFI_ADJUST_CFA_OFFSET(-8)
	143	pop %r10
	144	CFI_ADJUST_CFA_OFFSET(-8)
	145	pop %r9
	146	CFI_ADJUST_CFA_OFFSET(-8)
	147	pop %r8
	148	CFI_ADJUST_CFA_OFFSET(-8)
	149	pop %rdi
	150	CFI_ADJUST_CFA_OFFSET(-8)
	151	pop %rsi
	152	CFI_ADJUST_CFA_OFFSET(-8)
	153	pop %rdx
	154	CFI_ADJUST_CFA_OFFSET(-8)
	155	pop %rcx
	156	CFI_ADJUST_CFA_OFFSET(-8)
	157	pop %rax
	158	CFI_ADJUST_CFA_OFFSET(-8)
	159	CFI_RESTORE(%rax)
	160	CFI_RESTORE(%rbx)
	161	CFI_RESTORE(%rcx)
	162	CFI_RESTORE(%rdx)
	163	CFI_RESTORE(%rsi)
	164	CFI_RESTORE(%rdi)
	165	CFI_RESTORE(%r8)
	166	CFI_RESTORE(%r9)
	167	CFI_RESTORE(%r10)
	168	CFI_RESTORE(%r11)
	169	ret
	170	CFI_ENDPROC
	171
	172	ASM_HIDDEN(__tsan_setjmp)
	173	#if defined(__NetBSD__)
	174	.comm _ZN14__interception15real___setjmp14E,8,8
	175	#elif !defined(__APPLE__)
	176	.comm _ZN14__interception11real_setjmpE,8,8
	177	#endif
	178	#if defined(__NetBSD__)
	179	.globl ASM_SYMBOL_INTERCEPTOR(__setjmp14)
	180	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__setjmp14))
	181	ASM_SYMBOL_INTERCEPTOR(__setjmp14):
	182	#else
	183	.globl ASM_SYMBOL_INTERCEPTOR(setjmp)
	184	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(setjmp))
	185	ASM_SYMBOL_INTERCEPTOR(setjmp):
	186	#endif
	187	CFI_STARTPROC
	188	// save env parameter
	189	push %rdi
	190	CFI_ADJUST_CFA_OFFSET(8)
	191	CFI_REL_OFFSET(%rdi, 0)
	192	// obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)`
	193	#if defined(__FreeBSD__) \|\| defined(__NetBSD__)
	194	lea 8(%rsp), %rdi
	195	#elif defined(__linux__) \|\| defined(__APPLE__)
	196	lea 16(%rsp), %rdi
	197	#else
	198	# error "Unknown platform"
	199	#endif
	200	// call tsan interceptor
	201	call ASM_SYMBOL(__tsan_setjmp)
	202	// restore env parameter
	203	pop %rdi
	204	CFI_ADJUST_CFA_OFFSET(-8)
	205	CFI_RESTORE(%rdi)
	206	// tail jump to libc setjmp
	207	movl $0, %eax
	208	#if defined(__NetBSD__)
	209	movq _ZN14__interception15real___setjmp14E@GOTPCREL(%rip), %rdx
	210	jmp *(%rdx)
	211	#elif !defined(__APPLE__)
	212	movq _ZN14__interception11real_setjmpE@GOTPCREL(%rip), %rdx
	213	jmp *(%rdx)
	214	#else
	215	jmp ASM_SYMBOL(setjmp)
	216	#endif
	217	CFI_ENDPROC
	218	#if defined(__NetBSD__)
	219	ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__setjmp14))
	220	#else
	221	ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(setjmp))
	222	#endif
	223
	224	.comm _ZN14__interception12real__setjmpE,8,8
	225	.globl ASM_SYMBOL_INTERCEPTOR(_setjmp)
	226	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(_setjmp))
	227	ASM_SYMBOL_INTERCEPTOR(_setjmp):
	228	CFI_STARTPROC
	229	// save env parameter
	230	push %rdi
	231	CFI_ADJUST_CFA_OFFSET(8)
	232	CFI_REL_OFFSET(%rdi, 0)
	233	// obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)`
	234	#if defined(__FreeBSD__) \|\| defined(__NetBSD__)
	235	lea 8(%rsp), %rdi
	236	#elif defined(__linux__) \|\| defined(__APPLE__)
	237	lea 16(%rsp), %rdi
	238	#else
	239	# error "Unknown platform"
	240	#endif
	241	// call tsan interceptor
	242	call ASM_SYMBOL(__tsan_setjmp)
	243	// restore env parameter
	244	pop %rdi
	245	CFI_ADJUST_CFA_OFFSET(-8)
	246	CFI_RESTORE(%rdi)
	247	// tail jump to libc setjmp
	248	movl $0, %eax
	249	#if !defined(__APPLE__)
	250	movq _ZN14__interception12real__setjmpE@GOTPCREL(%rip), %rdx
	251	jmp *(%rdx)
	252	#else
	253	jmp ASM_SYMBOL(_setjmp)
	254	#endif
	255	CFI_ENDPROC
	256	ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(_setjmp))
	257
	258	#if defined(__NetBSD__)
	259	.comm _ZN14__interception18real___sigsetjmp14E,8,8
	260	.globl ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14)
	261	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14))
	262	ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14):
	263	#else
	264	.comm _ZN14__interception14real_sigsetjmpE,8,8
	265	.globl ASM_SYMBOL_INTERCEPTOR(sigsetjmp)
	266	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))
	267	ASM_SYMBOL_INTERCEPTOR(sigsetjmp):
	268	#endif
	269	CFI_STARTPROC
	270	// save env parameter
	271	push %rdi
	272	CFI_ADJUST_CFA_OFFSET(8)
	273	CFI_REL_OFFSET(%rdi, 0)
	274	// save savesigs parameter
	275	push %rsi
	276	CFI_ADJUST_CFA_OFFSET(8)
	277	CFI_REL_OFFSET(%rsi, 0)
	278	// align stack frame
	279	sub $8, %rsp
	280	CFI_ADJUST_CFA_OFFSET(8)
	281	// obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)`
	282	#if defined(__FreeBSD__) \|\| defined(__NetBSD__)
	283	lea 24(%rsp), %rdi
	284	#elif defined(__linux__) \|\| defined(__APPLE__)
	285	lea 32(%rsp), %rdi
	286	#else
	287	# error "Unknown platform"
	288	#endif
	289	// call tsan interceptor
	290	call ASM_SYMBOL(__tsan_setjmp)
	291	// unalign stack frame
	292	add $8, %rsp
	293	CFI_ADJUST_CFA_OFFSET(-8)
	294	// restore savesigs parameter
	295	pop %rsi
	296	CFI_ADJUST_CFA_OFFSET(-8)
	297	CFI_RESTORE(%rsi)
	298	// restore env parameter
	299	pop %rdi
	300	CFI_ADJUST_CFA_OFFSET(-8)
	301	CFI_RESTORE(%rdi)
	302	// tail jump to libc sigsetjmp
	303	movl $0, %eax
	304	#if defined(__NetBSD__)
	305	movq _ZN14__interception18real___sigsetjmp14E@GOTPCREL(%rip), %rdx
	306	jmp *(%rdx)
	307	#elif !defined(__APPLE__)
	308	movq _ZN14__interception14real_sigsetjmpE@GOTPCREL(%rip), %rdx
	309	jmp *(%rdx)
	310	#else
	311	jmp ASM_SYMBOL(sigsetjmp)
	312	#endif
	313	CFI_ENDPROC
	314	#if defined(__NetBSD__)
	315	ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14))
	316	#else
	317	ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))
	318	#endif
	319
	320	#if !defined(__APPLE__) && !defined(__NetBSD__)
	321	.comm _ZN14__interception16real___sigsetjmpE,8,8
	322	.globl ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)
	323	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))
	324	ASM_SYMBOL_INTERCEPTOR(__sigsetjmp):
	325	CFI_STARTPROC
	326	// save env parameter
	327	push %rdi
	328	CFI_ADJUST_CFA_OFFSET(8)
	329	CFI_REL_OFFSET(%rdi, 0)
	330	// save savesigs parameter
	331	push %rsi
	332	CFI_ADJUST_CFA_OFFSET(8)
	333	CFI_REL_OFFSET(%rsi, 0)
	334	// align stack frame
	335	sub $8, %rsp
	336	CFI_ADJUST_CFA_OFFSET(8)
	337	// obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)`
	338	#if defined(__FreeBSD__)
	339	lea 24(%rsp), %rdi
	340	#else
	341	lea 32(%rsp), %rdi
	342	#endif
	343	// call tsan interceptor
	344	call ASM_SYMBOL(__tsan_setjmp)
	345	// unalign stack frame
	346	add $8, %rsp
	347	CFI_ADJUST_CFA_OFFSET(-8)
	348	// restore savesigs parameter
	349	pop %rsi
	350	CFI_ADJUST_CFA_OFFSET(-8)
	351	CFI_RESTORE(%rsi)
	352	// restore env parameter
	353	pop %rdi
	354	CFI_ADJUST_CFA_OFFSET(-8)
	355	CFI_RESTORE(%rdi)
	356	// tail jump to libc sigsetjmp
	357	movl $0, %eax
	358	movq _ZN14__interception16real___sigsetjmpE@GOTPCREL(%rip), %rdx
	359	jmp *(%rdx)
	360	CFI_ENDPROC
	361	ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))
	362	#endif // !defined(__APPLE__) && !defined(__NetBSD__)
	363
	364	NO_EXEC_STACK_DIRECTIVE
	365
	366	#endif

lib/tsan/tsan_rtl_mips64.S created+214

...	...	@@ -0,0 +1,214 @@
	1	.section .text
	2	.set noreorder
	3
	4	.hidden __tsan_setjmp
	5	.comm _ZN14__interception11real_setjmpE,8,8
	6	.globl setjmp
	7	.type setjmp, @function
	8	setjmp:
	9
	10	// save env parameters
	11	daddiu $sp,$sp,-40
	12	sd $s0,32($sp)
	13	sd $ra,24($sp)
	14	sd $fp,16($sp)
	15	sd $gp,8($sp)
	16
	17	// calculate and save pointer to GOT
	18	lui $gp,%hi(%neg(%gp_rel(setjmp)))
	19	daddu $gp,$gp,$t9
	20	daddiu $gp,$gp,%lo(%neg(%gp_rel(setjmp)))
	21	move $s0,$gp
	22
	23	// save jmp_buf
	24	sd $a0,0($sp)
	25
	26	// obtain $sp
	27	dadd $a0,$zero,$sp
	28
	29	// call tsan interceptor
	30	jal __tsan_setjmp
	31	daddiu $a1,$a0,40
	32
	33	// restore jmp_buf
	34	ld $a0,0($sp)
	35
	36	// restore gp
	37	move $gp,$s0
	38
	39	// load pointer of libc setjmp to t9
	40	dla $t9,(_ZN14__interception11real_setjmpE)
	41
	42	// restore env parameters
	43	ld $gp,8($sp)
	44	ld $fp,16($sp)
	45	ld $ra,24($sp)
	46	ld $s0,32($sp)
	47	daddiu $sp,$sp,40
	48
	49	// tail jump to libc setjmp
	50	ld $t9,0($t9)
	51	jr $t9
	52	nop
	53
	54	.size setjmp, .-setjmp
	55
	56	.hidden __tsan_setjmp
	57	.globl _setjmp
	58	.comm _ZN14__interception12real__setjmpE,8,8
	59	.type _setjmp, @function
	60	_setjmp:
	61
	62	// Save env parameters
	63	daddiu $sp,$sp,-40
	64	sd $s0,32($sp)
	65	sd $ra,24($sp)
	66	sd $fp,16($sp)
	67	sd $gp,8($sp)
	68
	69	// calculate and save pointer to GOT
	70	lui $gp,%hi(%neg(%gp_rel(_setjmp)))
	71	daddu $gp,$gp,$t9
	72	daddiu $gp,$gp,%lo(%neg(%gp_rel(_setjmp)))
	73	move $s0,$gp
	74
	75	// save jmp_buf
	76	sd $a0,0($sp)
	77
	78	// obtain $sp
	79	dadd $a0,$zero,$sp
	80
	81	// call tsan interceptor
	82	jal __tsan_setjmp
	83	daddiu $a1,$a0,40
	84
	85	// restore jmp_buf
	86	ld $a0,0($sp)
	87
	88	// restore gp
	89	move $gp,$s0
	90
	91	// load pointer of libc _setjmp to t9
	92	dla $t9,(_ZN14__interception12real__setjmpE)
	93
	94	// restore env parameters
	95	ld $gp,8($sp)
	96	ld $fp,16($sp)
	97	ld $ra,24($sp)
	98	ld $s0,32($sp)
	99	daddiu $sp,$sp,40
	100
	101	// tail jump to libc _setjmp
	102	ld $t9,0($t9)
	103	jr $t9
	104	nop
	105
	106	.size _setjmp, .-_setjmp
	107
	108	.hidden __tsan_setjmp
	109	.globl sigsetjmp
	110	.comm _ZN14__interception14real_sigsetjmpE,8,8
	111	.type sigsetjmp, @function
	112	sigsetjmp:
	113
	114	// Save env parameters
	115	daddiu $sp,$sp,-48
	116	sd $s0,40($sp)
	117	sd $ra,32($sp)
	118	sd $fp,24($sp)
	119	sd $gp,16($sp)
	120
	121	// calculate and save pointer to GOT
	122	lui $gp,%hi(%neg(%gp_rel(sigsetjmp)))
	123	daddu $gp,$gp,$t9
	124	daddiu $gp,$gp,%lo(%neg(%gp_rel(sigsetjmp)))
	125	move $s0,$gp
	126
	127	// save jmp_buf and savesig
	128	sd $a0,0($sp)
	129	sd $a1,8($sp)
	130
	131	// obtain $sp
	132	dadd $a0,$zero,$sp
	133
	134	// call tsan interceptor
	135	jal __tsan_setjmp
	136	daddiu $a1,$a0,48
	137
	138	// restore jmp_buf and savesig
	139	ld $a0,0($sp)
	140	ld $a1,8($sp)
	141
	142	// restore gp
	143	move $gp,$s0
	144
	145	// load pointer of libc sigsetjmp to t9
	146	dla $t9,(_ZN14__interception14real_sigsetjmpE)
	147
	148	// restore env parameters
	149	ld $gp,16($sp)
	150	ld $fp,24($sp)
	151	ld $ra,32($sp)
	152	ld $s0,40($sp)
	153	daddiu $sp,$sp,48
	154
	155	// tail jump to libc sigsetjmp
	156	ld $t9,0($t9)
	157	jr $t9
	158	nop
	159
	160	.size sigsetjmp, .-sigsetjmp
	161
	162	.hidden __tsan_setjmp
	163	.comm _ZN14__interception16real___sigsetjmpE,8,8
	164	.globl __sigsetjmp
	165	.type __sigsetjmp, @function
	166	__sigsetjmp:
	167
	168	// Save env parameters
	169	daddiu $sp,$sp,-48
	170	sd $s0,40($sp)
	171	sd $ra,32($sp)
	172	sd $fp,24($sp)
	173	sd $gp,16($sp)
	174
	175	// calculate and save pointer to GOT
	176	lui $gp,%hi(%neg(%gp_rel(__sigsetjmp)))
	177	daddu $gp,$gp,$t9
	178	daddiu $gp,$gp,%lo(%neg(%gp_rel(__sigsetjmp)))
	179	move $s0,$gp
	180
	181	// save jmp_buf and savesig
	182	sd $a0,0($sp)
	183	sd $a1,8($sp)
	184
	185	// obtain $sp
	186	dadd $a0,$zero,$sp
	187
	188	// call tsan interceptor
	189	jal __tsan_setjmp
	190	daddiu $a1,$a0,48
	191
	192	// restore jmp_buf and savesig
	193	ld $a0,0($sp)
	194	ld $a1,8($sp)
	195
	196	// restore gp
	197	move $gp,$s0
	198
	199	// load pointer to libc __sigsetjmp in t9
	200	dla $t9,(_ZN14__interception16real___sigsetjmpE)
	201
	202	// restore env parameters
	203	ld $gp,16($sp)
	204	ld $fp,24($sp)
	205	ld $ra,32($sp)
	206	ld $s0,40($sp)
	207	daddiu $sp,$sp,48
	208
	209	// tail jump to libc __sigsetjmp
	210	ld $t9,0($t9)
	211	jr $t9
	212	nop
	213
	214	.size __sigsetjmp, .-__sigsetjmp

lib/tsan/tsan_rtl_ppc64.S created+288

...	...	@@ -0,0 +1,288 @@
	1	#include "tsan_ppc_regs.h"
	2
	3	.section .text
	4	.hidden __tsan_setjmp
	5	.globl _setjmp
	6	.type _setjmp, @function
	7	.align 4
	8	#if _CALL_ELF == 2
	9	_setjmp:
	10	#else
	11	.section ".opd","aw"
	12	.align 3
	13	_setjmp:
	14	.quad .L._setjmp,.TOC.@tocbase,0
	15	.previous
	16	#endif
	17	.L._setjmp:
	18	mflr r0
	19	stdu r1,-48(r1)
	20	std r2,24(r1)
	21	std r3,32(r1)
	22	std r0,40(r1)
	23	// r3 is the original stack pointer.
	24	addi r3,r1,48
	25	// r4 is the mangled stack pointer (see glibc)
	26	ld r4,-28696(r13)
	27	xor r4,r3,r4
	28	// Materialize a TOC in case we were called from libc.
	29	// For big-endian, we load the TOC from the OPD. For little-
	30	// endian, we use the .TOC. symbol to find it.
	31	nop
	32	bcl 20,31,0f
	33	0:
	34	mflr r2
	35	#if _CALL_ELF == 2
	36	addis r2,r2,.TOC.-0b@ha
	37	addi r2,r2,.TOC.-0b@l
	38	#else
	39	addis r2,r2,_setjmp-0b@ha
	40	addi r2,r2,_setjmp-0b@l
	41	ld r2,8(r2)
	42	#endif
	43	// Call the interceptor.
	44	bl __tsan_setjmp
	45	nop
	46	// Restore regs needed for setjmp.
	47	ld r3,32(r1)
	48	ld r0,40(r1)
	49	// Emulate the real setjmp function. We do this because we can't
	50	// perform a sibcall: The real setjmp function trashes the TOC
	51	// pointer, and with a sibcall we have no way to restore it.
	52	// This way we can make sure our caller's stack pointer and
	53	// link register are saved correctly in the jmpbuf.
	54	ld r6,-28696(r13)
	55	addi r5,r1,48 // original stack ptr of caller
	56	xor r5,r6,r5
	57	std r5,0(r3) // mangled stack ptr of caller
	58	ld r5,24(r1)
	59	std r5,8(r3) // caller's saved TOC pointer
	60	xor r0,r6,r0
	61	std r0,16(r3) // caller's mangled return address
	62	mfcr r0
	63	// Nonvolatiles.
	64	std r14,24(r3)
	65	stfd f14,176(r3)
	66	stw r0,172(r3) // CR
	67	std r15,32(r3)
	68	stfd f15,184(r3)
	69	std r16,40(r3)
	70	stfd f16,192(r3)
	71	std r17,48(r3)
	72	stfd f17,200(r3)
	73	std r18,56(r3)
	74	stfd f18,208(r3)
	75	std r19,64(r3)
	76	stfd f19,216(r3)
	77	std r20,72(r3)
	78	stfd f20,224(r3)
	79	std r21,80(r3)
	80	stfd f21,232(r3)
	81	std r22,88(r3)
	82	stfd f22,240(r3)
	83	std r23,96(r3)
	84	stfd f23,248(r3)
	85	std r24,104(r3)
	86	stfd f24,256(r3)
	87	std r25,112(r3)
	88	stfd f25,264(r3)
	89	std r26,120(r3)
	90	stfd f26,272(r3)
	91	std r27,128(r3)
	92	stfd f27,280(r3)
	93	std r28,136(r3)
	94	stfd f28,288(r3)
	95	std r29,144(r3)
	96	stfd f29,296(r3)
	97	std r30,152(r3)
	98	stfd f30,304(r3)
	99	std r31,160(r3)
	100	stfd f31,312(r3)
	101	addi r5,r3,320
	102	mfspr r0,256
	103	stw r0,168(r3) // VRSAVE
	104	addi r6,r5,16
	105	stvx v20,0,r5
	106	addi r5,r5,32
	107	stvx v21,0,r6
	108	addi r6,r6,32
	109	stvx v22,0,r5
	110	addi r5,r5,32
	111	stvx v23,0,r6
	112	addi r6,r6,32
	113	stvx v24,0,r5
	114	addi r5,r5,32
	115	stvx v25,0,r6
	116	addi r6,r6,32
	117	stvx v26,0,r5
	118	addi r5,r5,32
	119	stvx v27,0,r6
	120	addi r6,r6,32
	121	stvx v28,0,r5
	122	addi r5,r5,32
	123	stvx v29,0,r6
	124	addi r6,r6,32
	125	stvx v30,0,r5
	126	stvx v31,0,r6
	127	// Clear the "mask-saved" slot.
	128	li r4,0
	129	stw r4,512(r3)
	130	// Restore TOC, LR, and stack and return to caller.
	131	ld r2,24(r1)
	132	ld r0,40(r1)
	133	addi r1,r1,48
	134	li r3,0 // This is the setjmp return path
	135	mtlr r0
	136	blr
	137	.size _setjmp, .-.L._setjmp
	138
	139	.globl setjmp
	140	.type setjmp, @function
	141	.align 4
	142	setjmp:
	143	b _setjmp
	144	.size setjmp, .-setjmp
	145
	146	// sigsetjmp is like setjmp, except that the mask in r4 needs
	147	// to be saved at offset 512 of the jump buffer.
	148	.globl __sigsetjmp
	149	.type __sigsetjmp, @function
	150	.align 4
	151	#if _CALL_ELF == 2
	152	__sigsetjmp:
	153	#else
	154	.section ".opd","aw"
	155	.align 3
	156	__sigsetjmp:
	157	.quad .L.__sigsetjmp,.TOC.@tocbase,0
	158	.previous
	159	#endif
	160	.L.__sigsetjmp:
	161	mflr r0
	162	stdu r1,-64(r1)
	163	std r2,24(r1)
	164	std r3,32(r1)
	165	std r4,40(r1)
	166	std r0,48(r1)
	167	// r3 is the original stack pointer.
	168	addi r3,r1,64
	169	// r4 is the mangled stack pointer (see glibc)
	170	ld r4,-28696(r13)
	171	xor r4,r3,r4
	172	// Materialize a TOC in case we were called from libc.
	173	// For big-endian, we load the TOC from the OPD. For little-
	174	// endian, we use the .TOC. symbol to find it.
	175	nop
	176	bcl 20,31,1f
	177	1:
	178	mflr r2
	179	#if _CALL_ELF == 2
	180	addis r2,r2,.TOC.-1b@ha
	181	addi r2,r2,.TOC.-1b@l
	182	#else
	183	addis r2,r2,_setjmp-1b@ha
	184	addi r2,r2,_setjmp-1b@l
	185	ld r2,8(r2)
	186	#endif
	187	// Call the interceptor.
	188	bl __tsan_setjmp
	189	nop
	190	// Restore regs needed for __sigsetjmp.
	191	ld r3,32(r1)
	192	ld r4,40(r1)
	193	ld r0,48(r1)
	194	// Emulate the real sigsetjmp function. We do this because we can't
	195	// perform a sibcall: The real sigsetjmp function trashes the TOC
	196	// pointer, and with a sibcall we have no way to restore it.
	197	// This way we can make sure our caller's stack pointer and
	198	// link register are saved correctly in the jmpbuf.
	199	ld r6,-28696(r13)
	200	addi r5,r1,64 // original stack ptr of caller
	201	xor r5,r6,r5
	202	std r5,0(r3) // mangled stack ptr of caller
	203	ld r5,24(r1)
	204	std r5,8(r3) // caller's saved TOC pointer
	205	xor r0,r6,r0
	206	std r0,16(r3) // caller's mangled return address
	207	mfcr r0
	208	// Nonvolatiles.
	209	std r14,24(r3)
	210	stfd f14,176(r3)
	211	stw r0,172(r3) // CR
	212	std r15,32(r3)
	213	stfd f15,184(r3)
	214	std r16,40(r3)
	215	stfd f16,192(r3)
	216	std r17,48(r3)
	217	stfd f17,200(r3)
	218	std r18,56(r3)
	219	stfd f18,208(r3)
	220	std r19,64(r3)
	221	stfd f19,216(r3)
	222	std r20,72(r3)
	223	stfd f20,224(r3)
	224	std r21,80(r3)
	225	stfd f21,232(r3)
	226	std r22,88(r3)
	227	stfd f22,240(r3)
	228	std r23,96(r3)
	229	stfd f23,248(r3)
	230	std r24,104(r3)
	231	stfd f24,256(r3)
	232	std r25,112(r3)
	233	stfd f25,264(r3)
	234	std r26,120(r3)
	235	stfd f26,272(r3)
	236	std r27,128(r3)
	237	stfd f27,280(r3)
	238	std r28,136(r3)
	239	stfd f28,288(r3)
	240	std r29,144(r3)
	241	stfd f29,296(r3)
	242	std r30,152(r3)
	243	stfd f30,304(r3)
	244	std r31,160(r3)
	245	stfd f31,312(r3)
	246	addi r5,r3,320
	247	mfspr r0,256
	248	stw r0,168(r3) // VRSAVE
	249	addi r6,r5,16
	250	stvx v20,0,r5
	251	addi r5,r5,32
	252	stvx v21,0,r6
	253	addi r6,r6,32
	254	stvx v22,0,r5
	255	addi r5,r5,32
	256	stvx v23,0,r6
	257	addi r6,r6,32
	258	stvx v24,0,r5
	259	addi r5,r5,32
	260	stvx v25,0,r6
	261	addi r6,r6,32
	262	stvx v26,0,r5
	263	addi r5,r5,32
	264	stvx v27,0,r6
	265	addi r6,r6,32
	266	stvx v28,0,r5
	267	addi r5,r5,32
	268	stvx v29,0,r6
	269	addi r6,r6,32
	270	stvx v30,0,r5
	271	stvx v31,0,r6
	272	// Save into the "mask-saved" slot.
	273	stw r4,512(r3)
	274	// Restore TOC, LR, and stack and return to caller.
	275	ld r2,24(r1)
	276	ld r0,48(r1)
	277	addi r1,r1,64
	278	li r3,0 // This is the sigsetjmp return path
	279	mtlr r0
	280	blr
	281	.size __sigsetjmp, .-.L.__sigsetjmp
	282
	283	.globl sigsetjmp
	284	.type sigsetjmp, @function
	285	.align 4
	286	sigsetjmp:
	287	b __sigsetjmp
	288	.size sigsetjmp, .-sigsetjmp

src/Compilation.zig+19-14

...	...	@@ -429,7 +429,11 @@ pub const InitOptions = struct {
429	429	verbose_llvm_cpu_features: bool = false,
430	430	is_test: bool = false,
431	431	test_evented_io: bool = false,
432		is_compiler_rt_or_libc: bool = false,
	432	/// Normally when you create a `Compilation`, Zig will automatically build
	433	/// and link in required dependencies, such as compiler-rt and libc. When
	434	/// building such dependencies themselves, this flag must be set to avoid
	435	/// infinite recursion.
	436	skip_linker_dependencies: bool = false,
433	437	parent_compilation_link_libc: bool = false,
434	438	stack_size_override: ?u64 = null,
435	439	image_base_override: ?u64 = null,
...	...	@@ -499,7 +503,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !Compilation {
499	503	.Lib => is_dyn_lib,
500	504	.Exe => true,
501	505	};
502		const needs_c_symbols = !options.is_compiler_rt_or_libc and
	506	const needs_c_symbols = !options.skip_linker_dependencies and
503	507	(is_exe_or_dyn_lib or (options.target.isWasm() and options.output_mode != .Obj));
504	508
505	509	const comp: *Compilation = comp: {
...	...	@@ -677,6 +681,9 @@ pub fn create(gpa: Allocator, options: InitOptions) !Compilation {
677	681	break :pic explicit;
678	682	} else pie or must_pic;
679	683
	684	// TSAN is implemented in C++ so it requires linking libc++.
	685	const link_libcpp = options.link_libcpp or tsan;
	686
680	687	// Make a decision on whether to use Clang for translate-c and compiling C files.
681	688	const use_clang = if (options.use_clang) \|explicit\| explicit else blk: {
682	689	if (build_options.have_llvm) {
...	...	@@ -765,7 +772,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !Compilation {
765	772	cache.hash.add(options.function_sections);
766	773	cache.hash.add(strip);
767	774	cache.hash.add(link_libc);
768		cache.hash.add(options.link_libcpp);
	775	cache.hash.add(link_libcpp);
769	776	cache.hash.add(options.output_mode);
770	777	cache.hash.add(options.machine_code_model);
771	778	cache.hash.addOptionalEmitLoc(options.emit_bin);
...	...	@@ -793,7 +800,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !Compilation {
793	800	hash.add(single_threaded);
794	801	hash.add(dll_export_fns);
795	802	hash.add(options.is_test);
796		hash.add(options.is_compiler_rt_or_libc);
	803	hash.add(options.skip_linker_dependencies);
797	804	hash.add(options.parent_compilation_link_libc);
798	805
799	806	const digest = hash.final();
...	...	@@ -930,7 +937,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !Compilation {
930	937	.use_llvm = use_llvm,
931	938	.system_linker_hack = darwin_options.system_linker_hack,
932	939	.link_libc = link_libc,
933		.link_libcpp = options.link_libcpp,
	940	.link_libcpp = link_libcpp,
934	941	.objects = options.link_objects,
935	942	.frameworks = options.frameworks,
936	943	.framework_dirs = options.framework_dirs,
...	...	@@ -970,7 +977,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !Compilation {
970	977	.dll_export_fns = dll_export_fns,
971	978	.error_return_tracing = error_return_tracing,
972	979	.llvm_cpu_features = llvm_cpu_features,
973		.is_compiler_rt_or_libc = options.is_compiler_rt_or_libc,
	980	.skip_linker_dependencies = options.skip_linker_dependencies,
974	981	.parent_compilation_link_libc = options.parent_compilation_link_libc,
975	982	.each_lib_rpath = options.each_lib_rpath orelse options.is_native_os,
976	983	.disable_lld_caching = options.disable_lld_caching,
...	...	@@ -1044,7 +1051,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !Compilation {
1044	1051	comp.c_object_table.putAssumeCapacityNoClobber(c_object, {});
1045	1052	}
1046	1053
1047		if (comp.bin_file.options.emit != null and !comp.bin_file.options.is_compiler_rt_or_libc) {
	1054	if (comp.bin_file.options.emit != null and !comp.bin_file.options.skip_linker_dependencies) {
1048	1055	// If we need to build glibc for the target, add work items for it.
1049	1056	// We go through the work queue so that building can be done in parallel.
1050	1057	if (comp.wantBuildGLibCFromSource()) {
...	...	@@ -1097,9 +1104,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !Compilation {
1097	1104	if (comp.wantBuildLibUnwindFromSource()) {
1098	1105	try comp.work_queue.writeItem(.{ .libunwind = {} });
1099	1106	}
1100		if (build_options.have_llvm and comp.bin_file.options.output_mode != .Obj and
1101		comp.bin_file.options.link_libcpp)
1102		{
	1107	if (build_options.have_llvm and is_exe_or_dyn_lib and comp.bin_file.options.link_libcpp) {
1103	1108	try comp.work_queue.writeItem(.libcxx);
1104	1109	try comp.work_queue.writeItem(.libcxxabi);
1105	1110	}
...	...	@@ -2772,7 +2777,7 @@ pub fn generateBuiltinZigSource(comp: Compilation, allocator: Allocator) ![]u8
2772	2777	// in. For example, compiler_rt will not export the __chkstk symbol if it
2773	2778	// knows libc will provide it, and likewise c.zig will not export memcpy.
2774	2779	const link_libc = comp.bin_file.options.link_libc or
2775		(comp.bin_file.options.is_compiler_rt_or_libc and comp.bin_file.options.parent_compilation_link_libc);
	2780	(comp.bin_file.options.skip_linker_dependencies and comp.bin_file.options.parent_compilation_link_libc);
2776	2781
2777	2782	try buffer.writer().print(
2778	2783	\\pub const object_format = ObjectFormat.{};
...	...	@@ -2927,7 +2932,7 @@ fn buildOutputFromZig(
2927	2932	.verbose_cimport = comp.verbose_cimport,
2928	2933	.verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features,
2929	2934	.clang_passthrough_mode = comp.clang_passthrough_mode,
2930		.is_compiler_rt_or_libc = true,
	2935	.skip_linker_dependencies = true,
2931	2936	.parent_compilation_link_libc = comp.bin_file.options.link_libc,
2932	2937	});
2933	2938	defer sub_compilation.destroy();
...	...	@@ -3305,7 +3310,7 @@ pub fn build_crt_file(
3305	3310	.verbose_cimport = comp.verbose_cimport,
3306	3311	.verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features,
3307	3312	.clang_passthrough_mode = comp.clang_passthrough_mode,
3308		.is_compiler_rt_or_libc = true,
	3313	.skip_linker_dependencies = true,
3309	3314	.parent_compilation_link_libc = comp.bin_file.options.link_libc,
3310	3315	});
3311	3316	defer sub_compilation.destroy();
...	...	@@ -3326,7 +3331,7 @@ pub fn stage1AddLinkLib(comp: *Compilation, lib_name: []const u8) !void {
3326	3331	// Avoid deadlocking on building import libs such as kernel32.lib
3327	3332	// This can happen when the user uses `build-exe foo.obj -lkernel32` and then
3328	3333	// when we create a sub-Compilation for zig libc, it also tries to build kernel32.lib.
3329		if (comp.bin_file.options.is_compiler_rt_or_libc) return;
	3334	if (comp.bin_file.options.skip_linker_dependencies) return;
3330	3335
3331	3336	// This happens when an `extern "foo"` function is referenced by the stage1 backend.
3332	3337	// If we haven't seen this library yet and we're targeting Windows, we need to queue up

src/glibc.zig+1-1

...	...	@@ -962,7 +962,7 @@ fn buildSharedLib(
962	962	.version_script = map_file_path,
963	963	.soname = soname,
964	964	.c_source_files = &c_source_files,
965		.is_compiler_rt_or_libc = true,
	965	.skip_linker_dependencies = true,
966	966	});
967	967	defer sub_compilation.destroy();
968	968

src/libcxx.zig+2

...	...	@@ -188,6 +188,7 @@ pub fn buildLibCXX(comp: *Compilation) !void {
188	188	.verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features,
189	189	.clang_passthrough_mode = comp.clang_passthrough_mode,
190	190	.link_libc = true,
	191	.skip_linker_dependencies = true,
191	192	});
192	193	defer sub_compilation.destroy();
193	194
...	...	@@ -308,6 +309,7 @@ pub fn buildLibCXXABI(comp: *Compilation) !void {
308	309	.verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features,
309	310	.clang_passthrough_mode = comp.clang_passthrough_mode,
310	311	.link_libc = true,
	312	.skip_linker_dependencies = true,
311	313	});
312	314	defer sub_compilation.destroy();
313	315

src/libtsan.zig+183-15

...	...	@@ -34,7 +34,7 @@ pub fn buildTsan(comp: *Compilation) !void {
34	34	};
35	35
36	36	var c_source_files = std.ArrayList(Compilation.CSourceFile).init(arena);
37		try c_source_files.ensureCapacity(tsan_sources.len + sanitizer_common_sources.len);
	37	try c_source_files.ensureCapacity(c_source_files.items.len + tsan_sources.len);
38	38
39	39	const tsan_include_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{"tsan"});
40	40	for (tsan_sources) \|tsan_src\| {
...	...	@@ -48,6 +48,7 @@ pub fn buildTsan(comp: *Compilation) !void {
48	48	try cflags.append("-nostdinc++");
49	49	try cflags.append("-fvisibility-inlines-hidden");
50	50	try cflags.append("-std=c++14");
	51	try cflags.append("-fno-rtti");
51	52
52	53	c_source_files.appendAssumeCapacity(.{
53	54	.src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ "tsan", tsan_src }),
...	...	@@ -55,6 +56,51 @@ pub fn buildTsan(comp: *Compilation) !void {
55	56	});
56	57	}
57	58
	59	const platform_tsan_sources = if (target.isDarwin())
	60	&darwin_tsan_sources
	61	else
	62	&unix_tsan_sources;
	63	try c_source_files.ensureCapacity(c_source_files.items.len + platform_tsan_sources.len);
	64	for (platform_tsan_sources) \|tsan_src\| {
	65	var cflags = std.ArrayList([]const u8).init(arena);
	66
	67	try cflags.append("-I");
	68	try cflags.append(tsan_include_path);
	69
	70	try cflags.append("-O3");
	71	try cflags.append("-DNDEBUG");
	72	try cflags.append("-nostdinc++");
	73	try cflags.append("-fvisibility-inlines-hidden");
	74	try cflags.append("-std=c++14");
	75	try cflags.append("-fno-rtti");
	76
	77	c_source_files.appendAssumeCapacity(.{
	78	.src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ "tsan", tsan_src }),
	79	.extra_flags = cflags.items,
	80	});
	81	}
	82	{
	83	const asm_source = switch (target.cpu.arch) {
	84	.aarch64 => "tsan_rtl_aarch64.S",
	85	.x86_64 => "tsan_rtl_amd64.S",
	86	.mips64 => "tsan_rtl_mips64.S",
	87	.powerpc64 => "tsan_rtl_ppc64.S",
	88	else => return error.TSANUnsupportedCPUArchitecture,
	89	};
	90	var cflags = std.ArrayList([]const u8).init(arena);
	91
	92	try cflags.append("-I");
	93	try cflags.append(tsan_include_path);
	94
	95	try cflags.append("-DNDEBUG");
	96
	97	c_source_files.appendAssumeCapacity(.{
	98	.src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ "tsan", asm_source }),
	99	.extra_flags = cflags.items,
	100	});
	101	}
	102
	103	try c_source_files.ensureCapacity(c_source_files.items.len + sanitizer_common_sources.len);
58	104	const sanitizer_common_include_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{
59	105	"tsan", "sanitizer_common",
60	106	});
...	...	@@ -69,6 +115,7 @@ pub fn buildTsan(comp: *Compilation) !void {
69	115	try cflags.append("-nostdinc++");
70	116	try cflags.append("-fvisibility-inlines-hidden");
71	117	try cflags.append("-std=c++14");
	118	try cflags.append("-fno-rtti");
72	119
73	120	c_source_files.appendAssumeCapacity(.{
74	121	.src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{
...	...	@@ -78,6 +125,88 @@ pub fn buildTsan(comp: *Compilation) !void {
78	125	});
79	126	}
80	127
	128	const to_c_or_not_to_c_sources = if (comp.bin_file.options.link_libc)
	129	&sanitizer_libcdep_sources
	130	else
	131	&sanitizer_nolibc_sources;
	132	try c_source_files.ensureCapacity(c_source_files.items.len + to_c_or_not_to_c_sources.len);
	133	for (to_c_or_not_to_c_sources) \|c_src\| {
	134	var cflags = std.ArrayList([]const u8).init(arena);
	135
	136	try cflags.append("-I");
	137	try cflags.append(sanitizer_common_include_path);
	138
	139	try cflags.append("-O3");
	140	try cflags.append("-DNDEBUG");
	141	try cflags.append("-nostdinc++");
	142	try cflags.append("-fvisibility-inlines-hidden");
	143	try cflags.append("-std=c++14");
	144	try cflags.append("-fno-rtti");
	145
	146	c_source_files.appendAssumeCapacity(.{
	147	.src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{
	148	"tsan", "sanitizer_common", c_src,
	149	}),
	150	.extra_flags = cflags.items,
	151	});
	152	}
	153
	154	try c_source_files.ensureCapacity(c_source_files.items.len + sanitizer_symbolizer_sources.len);
	155	for (sanitizer_symbolizer_sources) \|c_src\| {
	156	var cflags = std.ArrayList([]const u8).init(arena);
	157
	158	try cflags.append("-I");
	159	try cflags.append(tsan_include_path);
	160
	161	try cflags.append("-O3");
	162	try cflags.append("-DNDEBUG");
	163	try cflags.append("-nostdinc++");
	164	try cflags.append("-fvisibility-inlines-hidden");
	165	try cflags.append("-std=c++14");
	166	try cflags.append("-fno-rtti");
	167
	168	c_source_files.appendAssumeCapacity(.{
	169	.src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{
	170	"tsan", "sanitizer_common", c_src,
	171	}),
	172	.extra_flags = cflags.items,
	173	});
	174	}
	175
	176	const interception_include_path = try comp.zig_lib_directory.join(
	177	arena,
	178	&[_][]const u8{"interception"},
	179	);
	180
	181	try c_source_files.ensureCapacity(c_source_files.items.len + interception_sources.len);
	182	for (interception_sources) \|c_src\| {
	183	var cflags = std.ArrayList([]const u8).init(arena);
	184
	185	try cflags.append("-I");
	186	try cflags.append(interception_include_path);
	187
	188	try cflags.append("-I");
	189	try cflags.append(tsan_include_path);
	190
	191	try cflags.append("-O3");
	192	try cflags.append("-DNDEBUG");
	193	try cflags.append("-nostdinc++");
	194	try cflags.append("-fvisibility-inlines-hidden");
	195	try cflags.append("-std=c++14");
	196	try cflags.append("-fno-rtti");
	197
	198	c_source_files.appendAssumeCapacity(.{
	199	.src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{
	200	"tsan", "interception", c_src,
	201	}),
	202	.extra_flags = cflags.items,
	203	});
	204	}
	205
	206	const common_flags = [_][]const u8{
	207	"-DTSAN_CONTAINS_UBSAN=0",
	208	};
	209
81	210	const sub_compilation = try Compilation.create(comp.gpa, .{
82	211	.local_cache_directory = comp.global_cache_directory,
83	212	.global_cache_directory = comp.global_cache_directory,
...	...	@@ -113,6 +242,8 @@ pub fn buildTsan(comp: *Compilation) !void {
113	242	.verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features,
114	243	.clang_passthrough_mode = comp.clang_passthrough_mode,
115	244	.link_libc = true,
	245	.skip_linker_dependencies = true,
	246	.clang_argv = &common_flags,
116	247	});
117	248	defer sub_compilation.destroy();
118	249
...	...	@@ -159,6 +290,18 @@ const tsan_sources = [_][]const u8{
159	290	"tsan_sync.cpp",
160	291	};
161	292
	293	const darwin_tsan_sources = [_][]const u8{
	294	"tsan_interceptors_mac.cpp",
	295	"tsan_interceptors_mach_vm.cpp",
	296	"tsan_platform_mac.cpp",
	297	"tsan_platform_posix.cpp",
	298	};
	299
	300	const unix_tsan_sources = [_][]const u8{
	301	"tsan_platform_linux.cpp",
	302	"tsan_platform_posix.cpp",
	303	};
	304
162	305	const sanitizer_common_sources = [_][]const u8{
163	306	"sanitizer_allocator.cpp",
164	307	"sanitizer_common.cpp",
...	...	@@ -203,17 +346,42 @@ const sanitizer_common_sources = [_][]const u8{
203	346	"sanitizer_win.cpp",
204	347	};
205	348
206		// TODO This is next up
207		//const sanitizer_nolibc_sources = [_][]const u8{
208		// "sanitizer_common_nolibc.cpp",
209		//};
210		//
211		//const sanitizer_libcdep_sources = [_][]const u8{
212		// "sanitizer_common_libcdep.cpp",
213		// "sanitizer_allocator_checks.cpp",
214		// "sanitizer_linux_libcdep.cpp",
215		// "sanitizer_mac_libcdep.cpp",
216		// "sanitizer_posix_libcdep.cpp",
217		// "sanitizer_stoptheworld_linux_libcdep.cpp",
218		// "sanitizer_stoptheworld_netbsd_libcdep.cpp",
219		//};
	349	const sanitizer_nolibc_sources = [_][]const u8{
	350	"sanitizer_common_nolibc.cpp",
	351	};
	352
	353	const sanitizer_libcdep_sources = [_][]const u8{
	354	"sanitizer_common_libcdep.cpp",
	355	"sanitizer_allocator_checks.cpp",
	356	"sanitizer_linux_libcdep.cpp",
	357	"sanitizer_mac_libcdep.cpp",
	358	"sanitizer_posix_libcdep.cpp",
	359	"sanitizer_stoptheworld_linux_libcdep.cpp",
	360	"sanitizer_stoptheworld_netbsd_libcdep.cpp",
	361	};
	362
	363	const sanitizer_symbolizer_sources = [_][]const u8{
	364	"sanitizer_allocator_report.cpp",
	365	"sanitizer_stackdepot.cpp",
	366	"sanitizer_stacktrace.cpp",
	367	"sanitizer_stacktrace_libcdep.cpp",
	368	"sanitizer_stacktrace_printer.cpp",
	369	"sanitizer_stacktrace_sparc.cpp",
	370	"sanitizer_symbolizer.cpp",
	371	"sanitizer_symbolizer_libbacktrace.cpp",
	372	"sanitizer_symbolizer_libcdep.cpp",
	373	"sanitizer_symbolizer_mac.cpp",
	374	"sanitizer_symbolizer_markup.cpp",
	375	"sanitizer_symbolizer_posix_libcdep.cpp",
	376	"sanitizer_symbolizer_report.cpp",
	377	"sanitizer_symbolizer_win.cpp",
	378	"sanitizer_unwind_linux_libcdep.cpp",
	379	"sanitizer_unwind_win.cpp",
	380	};
	381
	382	const interception_sources = [_][]const u8{
	383	"interception_linux.cpp",
	384	"interception_mac.cpp",
	385	"interception_win.cpp",
	386	"interception_type_test.cpp",
	387	};

src/libunwind.zig+1

...	...	@@ -122,6 +122,7 @@ pub fn buildStaticLib(comp: *Compilation) !void {
122	122	.verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features,
123	123	.clang_passthrough_mode = comp.clang_passthrough_mode,
124	124	.link_libc = true,
	125	.skip_linker_dependencies = true,
125	126	});
126	127	defer sub_compilation.destroy();
127	128

src/link.zig+1-1

...	...	@@ -81,7 +81,7 @@ pub const Options = struct {
81	81	verbose_link: bool,
82	82	dll_export_fns: bool,
83	83	error_return_tracing: bool,
84		is_compiler_rt_or_libc: bool,
	84	skip_linker_dependencies: bool,
85	85	parent_compilation_link_libc: bool,
86	86	each_lib_rpath: bool,
87	87	disable_lld_caching: bool,

src/link/Coff.zig+2-2

...	...	@@ -835,7 +835,7 @@ fn linkWithLLD(self: Coff, comp: Compilation) !void {
835	835	man.hash.addOptional(self.base.options.image_base_override);
836	836	man.hash.addListOfBytes(self.base.options.extra_lld_args);
837	837	man.hash.addListOfBytes(self.base.options.lib_dirs);
838		man.hash.add(self.base.options.is_compiler_rt_or_libc);
	838	man.hash.add(self.base.options.skip_linker_dependencies);
839	839	if (self.base.options.link_libc) {
840	840	man.hash.add(self.base.options.libc_installation != null);
841	841	if (self.base.options.libc_installation) \|libc_installation\| {
...	...	@@ -1125,7 +1125,7 @@ fn linkWithLLD(self: Coff, comp: Compilation) !void {
1125	1125	}
1126	1126
1127	1127	// compiler-rt, libc and libssp
1128		if (is_exe_or_dyn_lib and !self.base.options.is_compiler_rt_or_libc) {
	1128	if (is_exe_or_dyn_lib and !self.base.options.skip_linker_dependencies) {
1129	1129	if (!self.base.options.link_libc) {
1130	1130	try argv.append(comp.libc_static_lib.?.full_object_path);
1131	1131	}

src/link/Elf.zig+2-4

...	...	@@ -1310,7 +1310,7 @@ fn linkWithLLD(self: Elf, comp: Compilation) !void {
1310	1310	man.hash.addListOfBytes(self.base.options.lib_dirs);
1311	1311	man.hash.addListOfBytes(self.base.options.rpath_list);
1312	1312	man.hash.add(self.base.options.each_lib_rpath);
1313		man.hash.add(self.base.options.is_compiler_rt_or_libc);
	1313	man.hash.add(self.base.options.skip_linker_dependencies);
1314	1314	man.hash.add(self.base.options.z_nodelete);
1315	1315	man.hash.add(self.base.options.z_defs);
1316	1316	if (self.base.options.link_libc) {
...	...	@@ -1552,7 +1552,7 @@ fn linkWithLLD(self: Elf, comp: Compilation) !void {
1552	1552	}
1553	1553
1554	1554	// libc
1555		if (is_exe_or_dyn_lib and !self.base.options.is_compiler_rt_or_libc and !self.base.options.link_libc) {
	1555	if (is_exe_or_dyn_lib and !self.base.options.skip_linker_dependencies and !self.base.options.link_libc) {
1556	1556	try argv.append(comp.libc_static_lib.?.full_object_path);
1557	1557	}
1558	1558
...	...	@@ -1574,9 +1574,7 @@ fn linkWithLLD(self: Elf, comp: Compilation) !void {
1574	1574	const arg = if (ext == .shared_library) link_lib else try std.fmt.allocPrint(arena, "-l{}", .{link_lib});
1575	1575	argv.appendAssumeCapacity(arg);
1576	1576	}
1577		}
1578	1577
1579		if (!is_obj) {
1580	1578	// libc++ dep
1581	1579	if (self.base.options.link_libcpp) {
1582	1580	try argv.append(comp.libcxxabi_static_lib.?.full_object_path);

src/link/MachO.zig+2-2

...	...	@@ -438,7 +438,7 @@ fn linkWithLLD(self: MachO, comp: Compilation) !void {
438	438	man.hash.addListOfBytes(self.base.options.framework_dirs);
439	439	man.hash.addListOfBytes(self.base.options.frameworks);
440	440	man.hash.addListOfBytes(self.base.options.rpath_list);
441		man.hash.add(self.base.options.is_compiler_rt_or_libc);
	441	man.hash.add(self.base.options.skip_linker_dependencies);
442	442	man.hash.add(self.base.options.z_nodelete);
443	443	man.hash.add(self.base.options.z_defs);
444	444	if (is_dyn_lib) {
...	...	@@ -633,7 +633,7 @@ fn linkWithLLD(self: MachO, comp: Compilation) !void {
633	633	}
634	634
635	635	// compiler_rt on darwin is missing some stuff, so we still build it and rely on LinkOnce
636		if (is_exe_or_dyn_lib and !self.base.options.is_compiler_rt_or_libc) {
	636	if (is_exe_or_dyn_lib and !self.base.options.skip_linker_dependencies) {
637	637	try argv.append(comp.compiler_rt_static_lib.?.full_object_path);
638	638	}
639	639

src/link/Wasm.zig+1-1

...	...	@@ -387,7 +387,7 @@ fn linkWithLLD(self: Wasm, comp: Compilation) !void {
387	387	}
388	388
389	389	if (self.base.options.output_mode != .Obj and
390		!self.base.options.is_compiler_rt_or_libc and
	390	!self.base.options.skip_linker_dependencies and
391	391	!self.base.options.link_libc)
392	392	{
393	393	try argv.append(comp.libc_static_lib.?.full_object_path);

src/musl.zig+1-1

...	...	@@ -225,7 +225,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void {
225	225	.c_source_files = &[_]Compilation.CSourceFile{
226	226	.{ .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", "musl", "libc.s" }) },
227	227	},
228		.is_compiler_rt_or_libc = true,
	228	.skip_linker_dependencies = true,
229	229	.soname = "libc.so",
230	230	});
231	231	defer sub_compilation.destroy();