Commit 8219d92987c7d9641d6b24c4d5be29e3a80fd6b9 | zig

lib/tsan/interception/interception_linux.cpp created+83

...	@@ -0,0 +1,83 @@
1	//===-- interception_linux.cpp ----------------------------------- C++ --===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is a part of AddressSanitizer, an address sanity checker.
10	//
11	// Linux-specific interception methods.
12	//===----------------------------------------------------------------------===//
13
14	#include "interception.h"
15
16	#if SANITIZER_LINUX \|\| SANITIZER_FREEBSD \|\| SANITIZER_NETBSD \|\| \
17	SANITIZER_OPENBSD \|\| SANITIZER_SOLARIS
18
19	#include <dlfcn.h> // for dlsym() and dlvsym()
20
21	namespace __interception {
22
23	#if SANITIZER_NETBSD
24	static int StrCmp(const char s1, const char s2) {
25	while (true) {
26	if (s1 != s2)
27	return false;
28	if (*s1 == 0)
29	return true;
30	s1++;
31	s2++;
32	}
33	}
34	#endif
35
36	static void GetFuncAddr(const char name, uptr wrapper_addr) {
37	#if SANITIZER_NETBSD
38	// FIXME: Find a better way to handle renames
39	if (StrCmp(name, "sigaction"))
40	name = "__sigaction14";
41	#endif
42	void *addr = dlsym(RTLD_NEXT, name);
43	if (!addr) {
44	// If the lookup using RTLD_NEXT failed, the sanitizer runtime library is
45	// later in the library search order than the DSO that we are trying to
46	// intercept, which means that we cannot intercept this function. We still
47	// want the address of the real definition, though, so look it up using
48	// RTLD_DEFAULT.
49	addr = dlsym(RTLD_DEFAULT, name);
50
51	// In case `name' is not loaded, dlsym ends up finding the actual wrapper.
52	// We don't want to intercept the wrapper and have it point to itself.
53	if ((uptr)addr == wrapper_addr)
54	addr = nullptr;
55	}
56	return addr;
57	}
58
59	bool InterceptFunction(const char name, uptr ptr_to_real, uptr func,
60	uptr wrapper) {
61	void *addr = GetFuncAddr(name, wrapper);
62	*ptr_to_real = (uptr)addr;
63	return addr && (func == wrapper);
64	}
65
66	// Android and Solaris do not have dlvsym
67	#if !SANITIZER_ANDROID && !SANITIZER_SOLARIS && !SANITIZER_OPENBSD
68	static void GetFuncAddr(const char name, const char *ver) {
69	return dlvsym(RTLD_NEXT, name, ver);
70	}
71
72	bool InterceptFunction(const char name, const char ver, uptr *ptr_to_real,
73	uptr func, uptr wrapper) {
74	void *addr = GetFuncAddr(name, ver);
75	*ptr_to_real = (uptr)addr;
76	return addr && (func == wrapper);
77	}
78	#endif // !SANITIZER_ANDROID
79
80	} // namespace __interception
81
82	#endif // SANITIZER_LINUX \|\| SANITIZER_FREEBSD \|\| SANITIZER_NETBSD \|\|
83	// SANITIZER_OPENBSD \|\| SANITIZER_SOLARIS

lib/tsan/interception/interception_mac.cpp created+18

...	@@ -0,0 +1,18 @@
1	//===-- interception_mac.cpp ------------------------------------- C++ --===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is a part of AddressSanitizer, an address sanity checker.
10	//
11	// Mac-specific interception methods.
12	//===----------------------------------------------------------------------===//
13
14	#include "interception.h"
15
16	#if SANITIZER_MAC
17
18	#endif // SANITIZER_MAC

lib/tsan/interception/interception_type_test.cpp created+39

...	@@ -0,0 +1,39 @@
1	//===-- interception_type_test.cpp ------------------------------- C++ --===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is a part of AddressSanitizer, an address sanity checker.
10	//
11	// Compile-time tests of the internal type definitions.
12	//===----------------------------------------------------------------------===//
13
14	#include "interception.h"
15
16	#if SANITIZER_LINUX \|\| SANITIZER_MAC
17
18	#include <sys/types.h>
19	#include <stddef.h>
20	#include <stdint.h>
21
22	COMPILER_CHECK(sizeof(::SIZE_T) == sizeof(size_t));
23	COMPILER_CHECK(sizeof(::SSIZE_T) == sizeof(ssize_t));
24	COMPILER_CHECK(sizeof(::PTRDIFF_T) == sizeof(ptrdiff_t));
25	COMPILER_CHECK(sizeof(::INTMAX_T) == sizeof(intmax_t));
26
27	#if !SANITIZER_MAC
28	COMPILER_CHECK(sizeof(::OFF64_T) == sizeof(off64_t));
29	#endif
30
31	// The following are the cases when pread (and friends) is used instead of
32	// pread64. In those cases we need OFF_T to match off_t. We don't care about the
33	// rest (they depend on _FILE_OFFSET_BITS setting when building an application).
34	# if SANITIZER_ANDROID \|\| !defined _FILE_OFFSET_BITS \|\| \
35	_FILE_OFFSET_BITS != 64
36	COMPILER_CHECK(sizeof(::OFF_T) == sizeof(off_t));
37	# endif
38
39	#endif

lib/tsan/interception/interception_win.cpp created+1022

...	@@ -0,0 +1,1022 @@
1	//===-- interception_linux.cpp ----------------------------------- C++ --===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is a part of AddressSanitizer, an address sanity checker.
10	//
11	// Windows-specific interception methods.
12	//
13	// This file is implementing several hooking techniques to intercept calls
14	// to functions. The hooks are dynamically installed by modifying the assembly
15	// code.
16	//
17	// The hooking techniques are making assumptions on the way the code is
18	// generated and are safe under these assumptions.
19	//
20	// On 64-bit architecture, there is no direct 64-bit jump instruction. To allow
21	// arbitrary branching on the whole memory space, the notion of trampoline
22	// region is used. A trampoline region is a memory space withing 2G boundary
23	// where it is safe to add custom assembly code to build 64-bit jumps.
24	//
25	// Hooking techniques
26	// ==================
27	//
28	// 1) Detour
29	//
30	// The Detour hooking technique is assuming the presence of an header with
31	// padding and an overridable 2-bytes nop instruction (mov edi, edi). The
32	// nop instruction can safely be replaced by a 2-bytes jump without any need
33	// to save the instruction. A jump to the target is encoded in the function
34	// header and the nop instruction is replaced by a short jump to the header.
35	//
36	// head: 5 x nop head: jmp <hook>
37	// func: mov edi, edi --> func: jmp short <head>
38	// [...] real: [...]
39	//
40	// This technique is only implemented on 32-bit architecture.
41	// Most of the time, Windows API are hookable with the detour technique.
42	//
43	// 2) Redirect Jump
44	//
45	// The redirect jump is applicable when the first instruction is a direct
46	// jump. The instruction is replaced by jump to the hook.
47	//
48	// func: jmp <label> --> func: jmp <hook>
49	//
50	// On an 64-bit architecture, a trampoline is inserted.
51	//
52	// func: jmp <label> --> func: jmp <tramp>
53	// [...]
54	//
55	// [trampoline]
56	// tramp: jmp QWORD [addr]
57	// addr: .bytes <hook>
58	//
59	// Note: <real> is equilavent to <label>.
60	//
61	// 3) HotPatch
62	//
63	// The HotPatch hooking is assuming the presence of an header with padding
64	// and a first instruction with at least 2-bytes.
65	//
66	// The reason to enforce the 2-bytes limitation is to provide the minimal
67	// space to encode a short jump. HotPatch technique is only rewriting one
68	// instruction to avoid breaking a sequence of instructions containing a
69	// branching target.
70	//
71	// Assumptions are enforced by MSVC compiler by using the /HOTPATCH flag.
72	// see: https://msdn.microsoft.com/en-us/library/ms173507.aspx
73	// Default padding length is 5 bytes in 32-bits and 6 bytes in 64-bits.
74	//
75	// head: 5 x nop head: jmp <hook>
76	// func: <instr> --> func: jmp short <head>
77	// [...] body: [...]
78	//
79	// [trampoline]
80	// real: <instr>
81	// jmp <body>
82	//
83	// On an 64-bit architecture:
84	//
85	// head: 6 x nop head: jmp QWORD [addr1]
86	// func: <instr> --> func: jmp short <head>
87	// [...] body: [...]
88	//
89	// [trampoline]
90	// addr1: .bytes <hook>
91	// real: <instr>
92	// jmp QWORD [addr2]
93	// addr2: .bytes <body>
94	//
95	// 4) Trampoline
96	//
97	// The Trampoline hooking technique is the most aggressive one. It is
98	// assuming that there is a sequence of instructions that can be safely
99	// replaced by a jump (enough room and no incoming branches).
100	//
101	// Unfortunately, these assumptions can't be safely presumed and code may
102	// be broken after hooking.
103	//
104	// func: <instr> --> func: jmp <hook>
105	// <instr>
106	// [...] body: [...]
107	//
108	// [trampoline]
109	// real: <instr>
110	// <instr>
111	// jmp <body>
112	//
113	// On an 64-bit architecture:
114	//
115	// func: <instr> --> func: jmp QWORD [addr1]
116	// <instr>
117	// [...] body: [...]
118	//
119	// [trampoline]
120	// addr1: .bytes <hook>
121	// real: <instr>
122	// <instr>
123	// jmp QWORD [addr2]
124	// addr2: .bytes <body>
125	//===----------------------------------------------------------------------===//
126
127	#include "interception.h"
128
129	#if SANITIZER_WINDOWS
130	#include "sanitizer_common/sanitizer_platform.h"
131	#define WIN32_LEAN_AND_MEAN
132	#include <windows.h>
133
134	namespace __interception {
135
136	static const int kAddressLength = FIRST_32_SECOND_64(4, 8);
137	static const int kJumpInstructionLength = 5;
138	static const int kShortJumpInstructionLength = 2;
139	static const int kIndirectJumpInstructionLength = 6;
140	static const int kBranchLength =
141	FIRST_32_SECOND_64(kJumpInstructionLength, kIndirectJumpInstructionLength);
142	static const int kDirectBranchLength = kBranchLength + kAddressLength;
143
144	static void InterceptionFailed() {
145	// Do we have a good way to abort with an error message here?
146	__debugbreak();
147	}
148
149	static bool DistanceIsWithin2Gig(uptr from, uptr target) {
150	#if SANITIZER_WINDOWS64
151	if (from < target)
152	return target - from <= (uptr)0x7FFFFFFFU;
153	else
154	return from - target <= (uptr)0x80000000U;
155	#else
156	// In a 32-bit address space, the address calculation will wrap, so this check
157	// is unnecessary.
158	return true;
159	#endif
160	}
161
162	static uptr GetMmapGranularity() {
163	SYSTEM_INFO si;
164	GetSystemInfo(&si);
165	return si.dwAllocationGranularity;
166	}
167
168	static uptr RoundUpTo(uptr size, uptr boundary) {
169	return (size + boundary - 1) & ~(boundary - 1);
170	}
171
172	// FIXME: internal_str* and internal_mem* functions should be moved from the
173	// ASan sources into interception/.
174
175	static size_t _strlen(const char *str) {
176	const char* p = str;
177	while (*p != '\0') ++p;
178	return p - str;
179	}
180
181	static char* _strchr(char* str, char c) {
182	while (*str) {
183	if (*str == c)
184	return str;
185	++str;
186	}
187	return nullptr;
188	}
189
190	static void _memset(void *p, int value, size_t sz) {
191	for (size_t i = 0; i < sz; ++i)
192	((char*)p)[i] = (char)value;
193	}
194
195	static void _memcpy(void dst, void src, size_t sz) {
196	char dst_c = (char)dst,
197	src_c = (char)src;
198	for (size_t i = 0; i < sz; ++i)
199	dst_c[i] = src_c[i];
200	}
201
202	static bool ChangeMemoryProtection(
203	uptr address, uptr size, DWORD *old_protection) {
204	return ::VirtualProtect((void*)address, size,
205	PAGE_EXECUTE_READWRITE,
206	old_protection) != FALSE;
207	}
208
209	static bool RestoreMemoryProtection(
210	uptr address, uptr size, DWORD old_protection) {
211	DWORD unused;
212	return ::VirtualProtect((void*)address, size,
213	old_protection,
214	&unused) != FALSE;
215	}
216
217	static bool IsMemoryPadding(uptr address, uptr size) {
218	u8* function = (u8*)address;
219	for (size_t i = 0; i < size; ++i)
220	if (function[i] != 0x90 && function[i] != 0xCC)
221	return false;
222	return true;
223	}
224
225	static const u8 kHintNop8Bytes[] = {
226	0x0F, 0x1F, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00
227	};
228
229	template<class T>
230	static bool FunctionHasPrefix(uptr address, const T &pattern) {
231	u8* function = (u8*)address - sizeof(pattern);
232	for (size_t i = 0; i < sizeof(pattern); ++i)
233	if (function[i] != pattern[i])
234	return false;
235	return true;
236	}
237
238	static bool FunctionHasPadding(uptr address, uptr size) {
239	if (IsMemoryPadding(address - size, size))
240	return true;
241	if (size <= sizeof(kHintNop8Bytes) &&
242	FunctionHasPrefix(address, kHintNop8Bytes))
243	return true;
244	return false;
245	}
246
247	static void WritePadding(uptr from, uptr size) {
248	_memset((void*)from, 0xCC, (size_t)size);
249	}
250
251	static void WriteJumpInstruction(uptr from, uptr target) {
252	if (!DistanceIsWithin2Gig(from + kJumpInstructionLength, target))
253	InterceptionFailed();
254	ptrdiff_t offset = target - from - kJumpInstructionLength;
255	(u8)from = 0xE9;
256	(u32)(from + 1) = offset;
257	}
258
259	static void WriteShortJumpInstruction(uptr from, uptr target) {
260	sptr offset = target - from - kShortJumpInstructionLength;
261	if (offset < -128 \|\| offset > 127)
262	InterceptionFailed();
263	(u8)from = 0xEB;
264	(u8)(from + 1) = (u8)offset;
265	}
266
267	#if SANITIZER_WINDOWS64
268	static void WriteIndirectJumpInstruction(uptr from, uptr indirect_target) {
269	// jmp [rip + <offset>] = FF 25 <offset> where <offset> is a relative
270	// offset.
271	// The offset is the distance from then end of the jump instruction to the
272	// memory location containing the targeted address. The displacement is still
273	// 32-bit in x64, so indirect_target must be located within +/- 2GB range.
274	int offset = indirect_target - from - kIndirectJumpInstructionLength;
275	if (!DistanceIsWithin2Gig(from + kIndirectJumpInstructionLength,
276	indirect_target)) {
277	InterceptionFailed();
278	}
279	(u16)from = 0x25FF;
280	(u32)(from + 2) = offset;
281	}
282	#endif
283
284	static void WriteBranch(
285	uptr from, uptr indirect_target, uptr target) {
286	#if SANITIZER_WINDOWS64
287	WriteIndirectJumpInstruction(from, indirect_target);
288	(u64)indirect_target = target;
289	#else
290	(void)indirect_target;
291	WriteJumpInstruction(from, target);
292	#endif
293	}
294
295	static void WriteDirectBranch(uptr from, uptr target) {
296	#if SANITIZER_WINDOWS64
297	// Emit an indirect jump through immediately following bytes:
298	// jmp [rip + kBranchLength]
299	// .quad <target>
300	WriteBranch(from, from + kBranchLength, target);
301	#else
302	WriteJumpInstruction(from, target);
303	#endif
304	}
305
306	struct TrampolineMemoryRegion {
307	uptr content;
308	uptr allocated_size;
309	uptr max_size;
310	};
311
312	static const uptr kTrampolineScanLimitRange = 1 << 31; // 2 gig
313	static const int kMaxTrampolineRegion = 1024;
314	static TrampolineMemoryRegion TrampolineRegions[kMaxTrampolineRegion];
315
316	static void *AllocateTrampolineRegion(uptr image_address, size_t granularity) {
317	#if SANITIZER_WINDOWS64
318	uptr address = image_address;
319	uptr scanned = 0;
320	while (scanned < kTrampolineScanLimitRange) {
321	MEMORY_BASIC_INFORMATION info;
322	if (!::VirtualQuery((void*)address, &info, sizeof(info)))
323	return nullptr;
324
325	// Check whether a region can be allocated at \|address\|.
326	if (info.State == MEM_FREE && info.RegionSize >= granularity) {
327	void page = ::VirtualAlloc((void)RoundUpTo(address, granularity),
328	granularity,
329	MEM_RESERVE \| MEM_COMMIT,
330	PAGE_EXECUTE_READWRITE);
331	return page;
332	}
333
334	// Move to the next region.
335	address = (uptr)info.BaseAddress + info.RegionSize;
336	scanned += info.RegionSize;
337	}
338	return nullptr;
339	#else
340	return ::VirtualAlloc(nullptr,
341	granularity,
342	MEM_RESERVE \| MEM_COMMIT,
343	PAGE_EXECUTE_READWRITE);
344	#endif
345	}
346
347	// Used by unittests to release mapped memory space.
348	void TestOnlyReleaseTrampolineRegions() {
349	for (size_t bucket = 0; bucket < kMaxTrampolineRegion; ++bucket) {
350	TrampolineMemoryRegion *current = &TrampolineRegions[bucket];
351	if (current->content == 0)
352	return;
353	::VirtualFree((void*)current->content, 0, MEM_RELEASE);
354	current->content = 0;
355	}
356	}
357
358	static uptr AllocateMemoryForTrampoline(uptr image_address, size_t size) {
359	// Find a region within 2G with enough space to allocate \|size\| bytes.
360	TrampolineMemoryRegion *region = nullptr;
361	for (size_t bucket = 0; bucket < kMaxTrampolineRegion; ++bucket) {
362	TrampolineMemoryRegion* current = &TrampolineRegions[bucket];
363	if (current->content == 0) {
364	// No valid region found, allocate a new region.
365	size_t bucket_size = GetMmapGranularity();
366	void *content = AllocateTrampolineRegion(image_address, bucket_size);
367	if (content == nullptr)
368	return 0U;
369
370	current->content = (uptr)content;
371	current->allocated_size = 0;
372	current->max_size = bucket_size;
373	region = current;
374	break;
375	} else if (current->max_size - current->allocated_size > size) {
376	#if SANITIZER_WINDOWS64
377	// In 64-bits, the memory space must be allocated within 2G boundary.
378	uptr next_address = current->content + current->allocated_size;
379	if (next_address < image_address \|\|
380	next_address - image_address >= 0x7FFF0000)
381	continue;
382	#endif
383	// The space can be allocated in the current region.
384	region = current;
385	break;
386	}
387	}
388
389	// Failed to find a region.
390	if (region == nullptr)
391	return 0U;
392
393	// Allocate the space in the current region.
394	uptr allocated_space = region->content + region->allocated_size;
395	region->allocated_size += size;
396	WritePadding(allocated_space, size);
397
398	return allocated_space;
399	}
400
401	// Returns 0 on error.
402	static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
403	switch ((u64)address) {
404	case 0x90909090909006EB: // stub: jmp over 6 x nop.
405	return 8;
406	}
407
408	switch ((u8)address) {
409	case 0x90: // 90 : nop
410	return 1;
411
412	case 0x50: // push eax / rax
413	case 0x51: // push ecx / rcx
414	case 0x52: // push edx / rdx
415	case 0x53: // push ebx / rbx
416	case 0x54: // push esp / rsp
417	case 0x55: // push ebp / rbp
418	case 0x56: // push esi / rsi
419	case 0x57: // push edi / rdi
420	case 0x5D: // pop ebp / rbp
421	return 1;
422
423	case 0x6A: // 6A XX = push XX
424	return 2;
425
426	case 0xb8: // b8 XX XX XX XX : mov eax, XX XX XX XX
427	case 0xB9: // b9 XX XX XX XX : mov ecx, XX XX XX XX
428	return 5;
429
430	// Cannot overwrite control-instruction. Return 0 to indicate failure.
431	case 0xE9: // E9 XX XX XX XX : jmp <label>
432	case 0xE8: // E8 XX XX XX XX : call <func>
433	case 0xC3: // C3 : ret
434	case 0xEB: // EB XX : jmp XX (short jump)
435	case 0x70: // 7Y YY : jy XX (short conditional jump)
436	case 0x71:
437	case 0x72:
438	case 0x73:
439	case 0x74:
440	case 0x75:
441	case 0x76:
442	case 0x77:
443	case 0x78:
444	case 0x79:
445	case 0x7A:
446	case 0x7B:
447	case 0x7C:
448	case 0x7D:
449	case 0x7E:
450	case 0x7F:
451	return 0;
452	}
453
454	switch ((u16)(address)) {
455	case 0x018A: // 8A 01 : mov al, byte ptr [ecx]
456	case 0xFF8B: // 8B FF : mov edi, edi
457	case 0xEC8B: // 8B EC : mov ebp, esp
458	case 0xc889: // 89 C8 : mov eax, ecx
459	case 0xC18B: // 8B C1 : mov eax, ecx
460	case 0xC033: // 33 C0 : xor eax, eax
461	case 0xC933: // 33 C9 : xor ecx, ecx
462	case 0xD233: // 33 D2 : xor edx, edx
463	return 2;
464
465	// Cannot overwrite control-instruction. Return 0 to indicate failure.
466	case 0x25FF: // FF 25 XX XX XX XX : jmp [XXXXXXXX]
467	return 0;
468	}
469
470	switch (0x00FFFFFF & (u32)address) {
471	case 0x24A48D: // 8D A4 24 XX XX XX XX : lea esp, [esp + XX XX XX XX]
472	return 7;
473	}
474
475	#if SANITIZER_WINDOWS64
476	switch ((u8)address) {
477	case 0xA1: // A1 XX XX XX XX XX XX XX XX :
478	// movabs eax, dword ptr ds:[XXXXXXXX]
479	return 9;
480	}
481
482	switch ((u16)address) {
483	case 0x5040: // push rax
484	case 0x5140: // push rcx
485	case 0x5240: // push rdx
486	case 0x5340: // push rbx
487	case 0x5440: // push rsp
488	case 0x5540: // push rbp
489	case 0x5640: // push rsi
490	case 0x5740: // push rdi
491	case 0x5441: // push r12
492	case 0x5541: // push r13
493	case 0x5641: // push r14
494	case 0x5741: // push r15
495	case 0x9066: // Two-byte NOP
496	return 2;
497
498	case 0x058B: // 8B 05 XX XX XX XX : mov eax, dword ptr [XX XX XX XX]
499	if (rel_offset)
500	*rel_offset = 2;
501	return 6;
502	}
503
504	switch (0x00FFFFFF & (u32)address) {
505	case 0xe58948: // 48 8b c4 : mov rbp, rsp
506	case 0xc18b48: // 48 8b c1 : mov rax, rcx
507	case 0xc48b48: // 48 8b c4 : mov rax, rsp
508	case 0xd9f748: // 48 f7 d9 : neg rcx
509	case 0xd12b48: // 48 2b d1 : sub rdx, rcx
510	case 0x07c1f6: // f6 c1 07 : test cl, 0x7
511	case 0xc98548: // 48 85 C9 : test rcx, rcx
512	case 0xc0854d: // 4d 85 c0 : test r8, r8
513	case 0xc2b60f: // 0f b6 c2 : movzx eax, dl
514	case 0xc03345: // 45 33 c0 : xor r8d, r8d
515	case 0xc93345: // 45 33 c9 : xor r9d, r9d
516	case 0xdb3345: // 45 33 DB : xor r11d, r11d
517	case 0xd98b4c: // 4c 8b d9 : mov r11, rcx
518	case 0xd28b4c: // 4c 8b d2 : mov r10, rdx
519	case 0xc98b4c: // 4C 8B C9 : mov r9, rcx
520	case 0xc18b4c: // 4C 8B C1 : mov r8, rcx
521	case 0xd2b60f: // 0f b6 d2 : movzx edx, dl
522	case 0xca2b48: // 48 2b ca : sub rcx, rdx
523	case 0x10b70f: // 0f b7 10 : movzx edx, WORD PTR [rax]
524	case 0xc00b4d: // 3d 0b c0 : or r8, r8
525	case 0xd18b48: // 48 8b d1 : mov rdx, rcx
526	case 0xdc8b4c: // 4c 8b dc : mov r11, rsp
527	case 0xd18b4c: // 4c 8b d1 : mov r10, rcx
528	case 0xE0E483: // 83 E4 E0 : and esp, 0xFFFFFFE0
529	return 3;
530
531	case 0xec8348: // 48 83 ec XX : sub rsp, XX
532	case 0xf88349: // 49 83 f8 XX : cmp r8, XX
533	case 0x588948: // 48 89 58 XX : mov QWORD PTR[rax + XX], rbx
534	return 4;
535
536	case 0xec8148: // 48 81 EC XX XX XX XX : sub rsp, XXXXXXXX
537	return 7;
538
539	case 0x058b48: // 48 8b 05 XX XX XX XX :
540	// mov rax, QWORD PTR [rip + XXXXXXXX]
541	case 0x25ff48: // 48 ff 25 XX XX XX XX :
542	// rex.W jmp QWORD PTR [rip + XXXXXXXX]
543
544	// Instructions having offset relative to 'rip' need offset adjustment.
545	if (rel_offset)
546	*rel_offset = 3;
547	return 7;
548
549	case 0x2444c7: // C7 44 24 XX YY YY YY YY
550	// mov dword ptr [rsp + XX], YYYYYYYY
551	return 8;
552	}
553
554	switch ((u32)(address)) {
555	case 0x24448b48: // 48 8b 44 24 XX : mov rax, QWORD ptr [rsp + XX]
556	case 0x246c8948: // 48 89 6C 24 XX : mov QWORD ptr [rsp + XX], rbp
557	case 0x245c8948: // 48 89 5c 24 XX : mov QWORD PTR [rsp + XX], rbx
558	case 0x24748948: // 48 89 74 24 XX : mov QWORD PTR [rsp + XX], rsi
559	case 0x244C8948: // 48 89 4C 24 XX : mov QWORD PTR [rsp + XX], rcx
560	case 0x24548948: // 48 89 54 24 XX : mov QWORD PTR [rsp + XX], rdx
561	case 0x244c894c: // 4c 89 4c 24 XX : mov QWORD PTR [rsp + XX], r9
562	case 0x2444894c: // 4c 89 44 24 XX : mov QWORD PTR [rsp + XX], r8
563	return 5;
564	case 0x24648348: // 48 83 64 24 XX : and QWORD PTR [rsp + XX], YY
565	return 6;
566	}
567
568	#else
569
570	switch ((u8)address) {
571	case 0xA1: // A1 XX XX XX XX : mov eax, dword ptr ds:[XXXXXXXX]
572	return 5;
573	}
574	switch ((u16)address) {
575	case 0x458B: // 8B 45 XX : mov eax, dword ptr [ebp + XX]
576	case 0x5D8B: // 8B 5D XX : mov ebx, dword ptr [ebp + XX]
577	case 0x7D8B: // 8B 7D XX : mov edi, dword ptr [ebp + XX]
578	case 0xEC83: // 83 EC XX : sub esp, XX
579	case 0x75FF: // FF 75 XX : push dword ptr [ebp + XX]
580	return 3;
581	case 0xC1F7: // F7 C1 XX YY ZZ WW : test ecx, WWZZYYXX
582	case 0x25FF: // FF 25 XX YY ZZ WW : jmp dword ptr ds:[WWZZYYXX]
583	return 6;
584	case 0x3D83: // 83 3D XX YY ZZ WW TT : cmp TT, WWZZYYXX
585	return 7;
586	case 0x7D83: // 83 7D XX YY : cmp dword ptr [ebp + XX], YY
587	return 4;
588	}
589
590	switch (0x00FFFFFF & (u32)address) {
591	case 0x24448A: // 8A 44 24 XX : mov eal, dword ptr [esp + XX]
592	case 0x24448B: // 8B 44 24 XX : mov eax, dword ptr [esp + XX]
593	case 0x244C8B: // 8B 4C 24 XX : mov ecx, dword ptr [esp + XX]
594	case 0x24548B: // 8B 54 24 XX : mov edx, dword ptr [esp + XX]
595	case 0x24748B: // 8B 74 24 XX : mov esi, dword ptr [esp + XX]
596	case 0x247C8B: // 8B 7C 24 XX : mov edi, dword ptr [esp + XX]
597	return 4;
598	}
599
600	switch ((u32)address) {
601	case 0x2444B60F: // 0F B6 44 24 XX : movzx eax, byte ptr [esp + XX]
602	return 5;
603	}
604	#endif
605
606	// Unknown instruction!
607	// FIXME: Unknown instruction failures might happen when we add a new
608	// interceptor or a new compiler version. In either case, they should result
609	// in visible and readable error messages. However, merely calling abort()
610	// leads to an infinite recursion in CheckFailed.
611	InterceptionFailed();
612	return 0;
613	}
614
615	// Returns 0 on error.
616	static size_t RoundUpToInstrBoundary(size_t size, uptr address) {
617	size_t cursor = 0;
618	while (cursor < size) {
619	size_t instruction_size = GetInstructionSize(address + cursor);
620	if (!instruction_size)
621	return 0;
622	cursor += instruction_size;
623	}
624	return cursor;
625	}
626
627	static bool CopyInstructions(uptr to, uptr from, size_t size) {
628	size_t cursor = 0;
629	while (cursor != size) {
630	size_t rel_offset = 0;
631	size_t instruction_size = GetInstructionSize(from + cursor, &rel_offset);
632	_memcpy((void)(to + cursor), (void)(from + cursor),
633	(size_t)instruction_size);
634	if (rel_offset) {
635	uptr delta = to - from;
636	uptr relocated_offset = (u32)(to + cursor + rel_offset) - delta;
637	#if SANITIZER_WINDOWS64
638	if (relocated_offset + 0x80000000U >= 0xFFFFFFFFU)
639	return false;
640	#endif
641	(u32)(to + cursor + rel_offset) = relocated_offset;
642	}
643	cursor += instruction_size;
644	}
645	return true;
646	}
647
648
649	#if !SANITIZER_WINDOWS64
650	bool OverrideFunctionWithDetour(
651	uptr old_func, uptr new_func, uptr *orig_old_func) {
652	const int kDetourHeaderLen = 5;
653	const u16 kDetourInstruction = 0xFF8B;
654
655	uptr header = (uptr)old_func - kDetourHeaderLen;
656	uptr patch_length = kDetourHeaderLen + kShortJumpInstructionLength;
657
658	// Validate that the function is hookable.
659	if ((u16)old_func != kDetourInstruction \|\|
660	!IsMemoryPadding(header, kDetourHeaderLen))
661	return false;
662
663	// Change memory protection to writable.
664	DWORD protection = 0;
665	if (!ChangeMemoryProtection(header, patch_length, &protection))
666	return false;
667
668	// Write a relative jump to the redirected function.
669	WriteJumpInstruction(header, new_func);
670
671	// Write the short jump to the function prefix.
672	WriteShortJumpInstruction(old_func, header);
673
674	// Restore previous memory protection.
675	if (!RestoreMemoryProtection(header, patch_length, protection))
676	return false;
677
678	if (orig_old_func)
679	*orig_old_func = old_func + kShortJumpInstructionLength;
680
681	return true;
682	}
683	#endif
684
685	bool OverrideFunctionWithRedirectJump(
686	uptr old_func, uptr new_func, uptr *orig_old_func) {
687	// Check whether the first instruction is a relative jump.
688	if ((u8)old_func != 0xE9)
689	return false;
690
691	if (orig_old_func) {
692	uptr relative_offset = (u32)(old_func + 1);
693	uptr absolute_target = old_func + relative_offset + kJumpInstructionLength;
694	*orig_old_func = absolute_target;
695	}
696
697	#if SANITIZER_WINDOWS64
698	// If needed, get memory space for a trampoline jump.
699	uptr trampoline = AllocateMemoryForTrampoline(old_func, kDirectBranchLength);
700	if (!trampoline)
701	return false;
702	WriteDirectBranch(trampoline, new_func);
703	#endif
704
705	// Change memory protection to writable.
706	DWORD protection = 0;
707	if (!ChangeMemoryProtection(old_func, kJumpInstructionLength, &protection))
708	return false;
709
710	// Write a relative jump to the redirected function.
711	WriteJumpInstruction(old_func, FIRST_32_SECOND_64(new_func, trampoline));
712
713	// Restore previous memory protection.
714	if (!RestoreMemoryProtection(old_func, kJumpInstructionLength, protection))
715	return false;
716
717	return true;
718	}
719
720	bool OverrideFunctionWithHotPatch(
721	uptr old_func, uptr new_func, uptr *orig_old_func) {
722	const int kHotPatchHeaderLen = kBranchLength;
723
724	uptr header = (uptr)old_func - kHotPatchHeaderLen;
725	uptr patch_length = kHotPatchHeaderLen + kShortJumpInstructionLength;
726
727	// Validate that the function is hot patchable.
728	size_t instruction_size = GetInstructionSize(old_func);
729	if (instruction_size < kShortJumpInstructionLength \|\|
730	!FunctionHasPadding(old_func, kHotPatchHeaderLen))
731	return false;
732
733	if (orig_old_func) {
734	// Put the needed instructions into the trampoline bytes.
735	uptr trampoline_length = instruction_size + kDirectBranchLength;
736	uptr trampoline = AllocateMemoryForTrampoline(old_func, trampoline_length);
737	if (!trampoline)
738	return false;
739	if (!CopyInstructions(trampoline, old_func, instruction_size))
740	return false;
741	WriteDirectBranch(trampoline + instruction_size,
742	old_func + instruction_size);
743	*orig_old_func = trampoline;
744	}
745
746	// If needed, get memory space for indirect address.
747	uptr indirect_address = 0;
748	#if SANITIZER_WINDOWS64
749	indirect_address = AllocateMemoryForTrampoline(old_func, kAddressLength);
750	if (!indirect_address)
751	return false;
752	#endif
753
754	// Change memory protection to writable.
755	DWORD protection = 0;
756	if (!ChangeMemoryProtection(header, patch_length, &protection))
757	return false;
758
759	// Write jumps to the redirected function.
760	WriteBranch(header, indirect_address, new_func);
761	WriteShortJumpInstruction(old_func, header);
762
763	// Restore previous memory protection.
764	if (!RestoreMemoryProtection(header, patch_length, protection))
765	return false;
766
767	return true;
768	}
769
770	bool OverrideFunctionWithTrampoline(
771	uptr old_func, uptr new_func, uptr *orig_old_func) {
772
773	size_t instructions_length = kBranchLength;
774	size_t padding_length = 0;
775	uptr indirect_address = 0;
776
777	if (orig_old_func) {
778	// Find out the number of bytes of the instructions we need to copy
779	// to the trampoline.
780	instructions_length = RoundUpToInstrBoundary(kBranchLength, old_func);
781	if (!instructions_length)
782	return false;
783
784	// Put the needed instructions into the trampoline bytes.
785	uptr trampoline_length = instructions_length + kDirectBranchLength;
786	uptr trampoline = AllocateMemoryForTrampoline(old_func, trampoline_length);
787	if (!trampoline)
788	return false;
789	if (!CopyInstructions(trampoline, old_func, instructions_length))
790	return false;
791	WriteDirectBranch(trampoline + instructions_length,
792	old_func + instructions_length);
793	*orig_old_func = trampoline;
794	}
795
796	#if SANITIZER_WINDOWS64
797	// Check if the targeted address can be encoded in the function padding.
798	// Otherwise, allocate it in the trampoline region.
799	if (IsMemoryPadding(old_func - kAddressLength, kAddressLength)) {
800	indirect_address = old_func - kAddressLength;
801	padding_length = kAddressLength;
802	} else {
803	indirect_address = AllocateMemoryForTrampoline(old_func, kAddressLength);
804	if (!indirect_address)
805	return false;
806	}
807	#endif
808
809	// Change memory protection to writable.
810	uptr patch_address = old_func - padding_length;
811	uptr patch_length = instructions_length + padding_length;
812	DWORD protection = 0;
813	if (!ChangeMemoryProtection(patch_address, patch_length, &protection))
814	return false;
815
816	// Patch the original function.
817	WriteBranch(old_func, indirect_address, new_func);
818
819	// Restore previous memory protection.
820	if (!RestoreMemoryProtection(patch_address, patch_length, protection))
821	return false;
822
823	return true;
824	}
825
826	bool OverrideFunction(
827	uptr old_func, uptr new_func, uptr *orig_old_func) {
828	#if !SANITIZER_WINDOWS64
829	if (OverrideFunctionWithDetour(old_func, new_func, orig_old_func))
830	return true;
831	#endif
832	if (OverrideFunctionWithRedirectJump(old_func, new_func, orig_old_func))
833	return true;
834	if (OverrideFunctionWithHotPatch(old_func, new_func, orig_old_func))
835	return true;
836	if (OverrideFunctionWithTrampoline(old_func, new_func, orig_old_func))
837	return true;
838	return false;
839	}
840
841	static void **InterestingDLLsAvailable() {
842	static const char *InterestingDLLs[] = {
843	"kernel32.dll",
844	"msvcr100.dll", // VS2010
845	"msvcr110.dll", // VS2012
846	"msvcr120.dll", // VS2013
847	"vcruntime140.dll", // VS2015
848	"ucrtbase.dll", // Universal CRT
849	// NTDLL should go last as it exports some functions that we should
850	// override in the CRT [presumably only used internally].
851	"ntdll.dll", NULL};
852	static void *result[ARRAY_SIZE(InterestingDLLs)] = { 0 };
853	if (!result[0]) {
854	for (size_t i = 0, j = 0; InterestingDLLs[i]; ++i) {
855	if (HMODULE h = GetModuleHandleA(InterestingDLLs[i]))
856	result[j++] = (void *)h;
857	}
858	}
859	return &result[0];
860	}
861
862	namespace {
863	// Utility for reading loaded PE images.
864	template <typename T> class RVAPtr {
865	public:
866	RVAPtr(void *module, uptr rva)
867	: ptr_(reinterpret_cast<T >(reinterpret_cast<char >(module) + rva)) {}
868	operator T *() { return ptr_; }
869	T *operator->() { return ptr_; }
870	T *operator++() { return ++ptr_; }
871
872	private:
873	T *ptr_;
874	};
875	} // namespace
876
877	// Internal implementation of GetProcAddress. At least since Windows 8,
878	// GetProcAddress appears to initialize DLLs before returning function pointers
879	// into them. This is problematic for the sanitizers, because they typically
880	// want to intercept malloc before MSVCRT initializes. Our internal
881	// implementation walks the export list manually without doing initialization.
882	uptr InternalGetProcAddress(void module, const char func_name) {
883	// Check that the module header is full and present.
884	RVAPtr<IMAGE_DOS_HEADER> dos_stub(module, 0);
885	RVAPtr<IMAGE_NT_HEADERS> headers(module, dos_stub->e_lfanew);
886	if (!module \|\| dos_stub->e_magic != IMAGE_DOS_SIGNATURE \|\| // "MZ"
887	headers->Signature != IMAGE_NT_SIGNATURE \|\| // "PE\0\0"
888	headers->FileHeader.SizeOfOptionalHeader <
889	sizeof(IMAGE_OPTIONAL_HEADER)) {
890	return 0;
891	}
892
893	IMAGE_DATA_DIRECTORY *export_directory =
894	&headers->OptionalHeader.DataDirectory[IMAGE_DIRECTORY_ENTRY_EXPORT];
895	if (export_directory->Size == 0)
896	return 0;
897	RVAPtr<IMAGE_EXPORT_DIRECTORY> exports(module,
898	export_directory->VirtualAddress);
899	RVAPtr<DWORD> functions(module, exports->AddressOfFunctions);
900	RVAPtr<DWORD> names(module, exports->AddressOfNames);
901	RVAPtr<WORD> ordinals(module, exports->AddressOfNameOrdinals);
902
903	for (DWORD i = 0; i < exports->NumberOfNames; i++) {
904	RVAPtr<char> name(module, names[i]);
905	if (!strcmp(func_name, name)) {
906	DWORD index = ordinals[i];
907	RVAPtr<char> func(module, functions[index]);
908
909	// Handle forwarded functions.
910	DWORD offset = functions[index];
911	if (offset >= export_directory->VirtualAddress &&
912	offset < export_directory->VirtualAddress + export_directory->Size) {
913	// An entry for a forwarded function is a string with the following
914	// format: "<module> . <function_name>" that is stored into the
915	// exported directory.
916	char function_name[256];
917	size_t funtion_name_length = _strlen(func);
918	if (funtion_name_length >= sizeof(function_name) - 1)
919	InterceptionFailed();
920
921	_memcpy(function_name, func, funtion_name_length);
922	function_name[funtion_name_length] = '\0';
923	char* separator = _strchr(function_name, '.');
924	if (!separator)
925	InterceptionFailed();
926	*separator = '\0';
927
928	void* redirected_module = GetModuleHandleA(function_name);
929	if (!redirected_module)
930	InterceptionFailed();
931	return InternalGetProcAddress(redirected_module, separator + 1);
932	}
933
934	return (uptr)(char *)func;
935	}
936	}
937
938	return 0;
939	}
940
941	bool OverrideFunction(
942	const char func_name, uptr new_func, uptr orig_old_func) {
943	bool hooked = false;
944	void **DLLs = InterestingDLLsAvailable();
945	for (size_t i = 0; DLLs[i]; ++i) {
946	uptr func_addr = InternalGetProcAddress(DLLs[i], func_name);
947	if (func_addr &&
948	OverrideFunction(func_addr, new_func, orig_old_func)) {
949	hooked = true;
950	}
951	}
952	return hooked;
953	}
954
955	bool OverrideImportedFunction(const char *module_to_patch,
956	const char *imported_module,
957	const char *function_name, uptr new_function,
958	uptr *orig_old_func) {
959	HMODULE module = GetModuleHandleA(module_to_patch);
960	if (!module)
961	return false;
962
963	// Check that the module header is full and present.
964	RVAPtr<IMAGE_DOS_HEADER> dos_stub(module, 0);
965	RVAPtr<IMAGE_NT_HEADERS> headers(module, dos_stub->e_lfanew);
966	if (!module \|\| dos_stub->e_magic != IMAGE_DOS_SIGNATURE \|\| // "MZ"
967	headers->Signature != IMAGE_NT_SIGNATURE \|\| // "PE\0\0"
968	headers->FileHeader.SizeOfOptionalHeader <
969	sizeof(IMAGE_OPTIONAL_HEADER)) {
970	return false;
971	}
972
973	IMAGE_DATA_DIRECTORY *import_directory =
974	&headers->OptionalHeader.DataDirectory[IMAGE_DIRECTORY_ENTRY_IMPORT];
975
976	// Iterate the list of imported DLLs. FirstThunk will be null for the last
977	// entry.
978	RVAPtr<IMAGE_IMPORT_DESCRIPTOR> imports(module,
979	import_directory->VirtualAddress);
980	for (; imports->FirstThunk != 0; ++imports) {
981	RVAPtr<const char> modname(module, imports->Name);
982	if (_stricmp(&*modname, imported_module) == 0)
983	break;
984	}
985	if (imports->FirstThunk == 0)
986	return false;
987
988	// We have two parallel arrays: the import address table (IAT) and the table
989	// of names. They start out containing the same data, but the loader rewrites
990	// the IAT to hold imported addresses and leaves the name table in
991	// OriginalFirstThunk alone.
992	RVAPtr<IMAGE_THUNK_DATA> name_table(module, imports->OriginalFirstThunk);
993	RVAPtr<IMAGE_THUNK_DATA> iat(module, imports->FirstThunk);
994	for (; name_table->u1.Ordinal != 0; ++name_table, ++iat) {
995	if (!IMAGE_SNAP_BY_ORDINAL(name_table->u1.Ordinal)) {
996	RVAPtr<IMAGE_IMPORT_BY_NAME> import_by_name(
997	module, name_table->u1.ForwarderString);
998	const char *funcname = &import_by_name->Name[0];
999	if (strcmp(funcname, function_name) == 0)
1000	break;
1001	}
1002	}
1003	if (name_table->u1.Ordinal == 0)
1004	return false;
1005
1006	// Now we have the correct IAT entry. Do the swap. We have to make the page
1007	// read/write first.
1008	if (orig_old_func)
1009	*orig_old_func = iat->u1.AddressOfData;
1010	DWORD old_prot, unused_prot;
1011	if (!VirtualProtect(&iat->u1.AddressOfData, 4, PAGE_EXECUTE_READWRITE,
1012	&old_prot))
1013	return false;
1014	iat->u1.AddressOfData = new_function;
1015	if (!VirtualProtect(&iat->u1.AddressOfData, 4, old_prot, &unused_prot))
1016	return false; // Not clear if this failure bothers us.
1017	return true;
1018	}
1019
1020	} // namespace __interception
1021
1022	#endif // SANITIZER_MAC

lib/tsan/sanitizer_common/sanitizer_allocator_checks.cpp created+22

...	@@ -0,0 +1,22 @@
1	//===-- sanitizer_allocator_checks.cpp --------------------------- C++ --===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// Various checks shared between ThreadSanitizer, MemorySanitizer, etc. memory
10	// allocators.
11	//
12	//===----------------------------------------------------------------------===//
13
14	#include "sanitizer_errno.h"
15
16	namespace __sanitizer {
17
18	void SetErrnoToENOMEM() {
19	errno = errno_ENOMEM;
20	}
21
22	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_allocator_report.cpp created+137

...	@@ -0,0 +1,137 @@
1	//===-- sanitizer_allocator_report.cpp --------------------------- C++ --===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	///
9	/// \file
10	/// Shared allocator error reporting for ThreadSanitizer, MemorySanitizer, etc.
11	///
12	//===----------------------------------------------------------------------===//
13
14	#include "sanitizer_allocator.h"
15	#include "sanitizer_allocator_report.h"
16	#include "sanitizer_common.h"
17	#include "sanitizer_report_decorator.h"
18
19	namespace __sanitizer {
20
21	class ScopedAllocatorErrorReport {
22	public:
23	ScopedAllocatorErrorReport(const char *error_summary_,
24	const StackTrace *stack_)
25	: error_summary(error_summary_),
26	stack(stack_) {
27	Printf("%s", d.Error());
28	}
29	~ScopedAllocatorErrorReport() {
30	Printf("%s", d.Default());
31	stack->Print();
32	PrintHintAllocatorCannotReturnNull();
33	ReportErrorSummary(error_summary, stack);
34	}
35
36	private:
37	ScopedErrorReportLock lock;
38	const char *error_summary;
39	const StackTrace* const stack;
40	const SanitizerCommonDecorator d;
41	};
42
43	void NORETURN ReportCallocOverflow(uptr count, uptr size,
44	const StackTrace *stack) {
45	{
46	ScopedAllocatorErrorReport report("calloc-overflow", stack);
47	Report("ERROR: %s: calloc parameters overflow: count * size (%zd * %zd) "
48	"cannot be represented in type size_t\n", SanitizerToolName, count,
49	size);
50	}
51	Die();
52	}
53
54	void NORETURN ReportReallocArrayOverflow(uptr count, uptr size,
55	const StackTrace *stack) {
56	{
57	ScopedAllocatorErrorReport report("reallocarray-overflow", stack);
58	Report(
59	"ERROR: %s: reallocarray parameters overflow: count * size (%zd * %zd) "
60	"cannot be represented in type size_t\n",
61	SanitizerToolName, count, size);
62	}
63	Die();
64	}
65
66	void NORETURN ReportPvallocOverflow(uptr size, const StackTrace *stack) {
67	{
68	ScopedAllocatorErrorReport report("pvalloc-overflow", stack);
69	Report("ERROR: %s: pvalloc parameters overflow: size 0x%zx rounded up to "
70	"system page size 0x%zx cannot be represented in type size_t\n",
71	SanitizerToolName, size, GetPageSizeCached());
72	}
73	Die();
74	}
75
76	void NORETURN ReportInvalidAllocationAlignment(uptr alignment,
77	const StackTrace *stack) {
78	{
79	ScopedAllocatorErrorReport report("invalid-allocation-alignment", stack);
80	Report("ERROR: %s: invalid allocation alignment: %zd, alignment must be a "
81	"power of two\n", SanitizerToolName, alignment);
82	}
83	Die();
84	}
85
86	void NORETURN ReportInvalidAlignedAllocAlignment(uptr size, uptr alignment,
87	const StackTrace *stack) {
88	{
89	ScopedAllocatorErrorReport report("invalid-aligned-alloc-alignment", stack);
90	#if SANITIZER_POSIX
91	Report("ERROR: %s: invalid alignment requested in "
92	"aligned_alloc: %zd, alignment must be a power of two and the "
93	"requested size 0x%zx must be a multiple of alignment\n",
94	SanitizerToolName, alignment, size);
95	#else
96	Report("ERROR: %s: invalid alignment requested in aligned_alloc: %zd, "
97	"the requested size 0x%zx must be a multiple of alignment\n",
98	SanitizerToolName, alignment, size);
99	#endif
100	}
101	Die();
102	}
103
104	void NORETURN ReportInvalidPosixMemalignAlignment(uptr alignment,
105	const StackTrace *stack) {
106	{
107	ScopedAllocatorErrorReport report("invalid-posix-memalign-alignment",
108	stack);
109	Report(
110	"ERROR: %s: invalid alignment requested in "
111	"posix_memalign: %zd, alignment must be a power of two and a "
112	"multiple of sizeof(void*) == %zd\n",
113	SanitizerToolName, alignment, sizeof(void *));
114	}
115	Die();
116	}
117
118	void NORETURN ReportAllocationSizeTooBig(uptr user_size, uptr max_size,
119	const StackTrace *stack) {
120	{
121	ScopedAllocatorErrorReport report("allocation-size-too-big", stack);
122	Report("ERROR: %s: requested allocation size 0x%zx exceeds maximum "
123	"supported size of 0x%zx\n", SanitizerToolName, user_size, max_size);
124	}
125	Die();
126	}
127
128	void NORETURN ReportOutOfMemory(uptr requested_size, const StackTrace *stack) {
129	{
130	ScopedAllocatorErrorReport report("out-of-memory", stack);
131	Report("ERROR: %s: allocator is out of memory trying to allocate 0x%zx "
132	"bytes\n", SanitizerToolName, requested_size);
133	}
134	Die();
135	}
136
137	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_common_libcdep.cpp created+149

...	@@ -0,0 +1,149 @@
1	//===-- sanitizer_common_libcdep.cpp --------------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is shared between AddressSanitizer and ThreadSanitizer
10	// run-time libraries.
11	//===----------------------------------------------------------------------===//
12
13	#include "sanitizer_allocator_interface.h"
14	#include "sanitizer_common.h"
15	#include "sanitizer_flags.h"
16	#include "sanitizer_procmaps.h"
17
18
19	namespace __sanitizer {
20
21	static void (*SoftRssLimitExceededCallback)(bool exceeded);
22	void SetSoftRssLimitExceededCallback(void (*Callback)(bool exceeded)) {
23	CHECK_EQ(SoftRssLimitExceededCallback, nullptr);
24	SoftRssLimitExceededCallback = Callback;
25	}
26
27	#if (SANITIZER_LINUX \|\| SANITIZER_NETBSD) && !SANITIZER_GO
28	// Weak default implementation for when sanitizer_stackdepot is not linked in.
29	SANITIZER_WEAK_ATTRIBUTE StackDepotStats *StackDepotGetStats() {
30	return nullptr;
31	}
32
33	void BackgroundThread(void arg) {
34	const uptr hard_rss_limit_mb = common_flags()->hard_rss_limit_mb;
35	const uptr soft_rss_limit_mb = common_flags()->soft_rss_limit_mb;
36	const bool heap_profile = common_flags()->heap_profile;
37	uptr prev_reported_rss = 0;
38	uptr prev_reported_stack_depot_size = 0;
39	bool reached_soft_rss_limit = false;
40	uptr rss_during_last_reported_profile = 0;
41	while (true) {
42	SleepForMillis(100);
43	const uptr current_rss_mb = GetRSS() >> 20;
44	if (Verbosity()) {
45	// If RSS has grown 10% since last time, print some information.
46	if (prev_reported_rss * 11 / 10 < current_rss_mb) {
47	Printf("%s: RSS: %zdMb\n", SanitizerToolName, current_rss_mb);
48	prev_reported_rss = current_rss_mb;
49	}
50	// If stack depot has grown 10% since last time, print it too.
51	StackDepotStats *stack_depot_stats = StackDepotGetStats();
52	if (stack_depot_stats) {
53	if (prev_reported_stack_depot_size * 11 / 10 <
54	stack_depot_stats->allocated) {
55	Printf("%s: StackDepot: %zd ids; %zdM allocated\n",
56	SanitizerToolName,
57	stack_depot_stats->n_uniq_ids,
58	stack_depot_stats->allocated >> 20);
59	prev_reported_stack_depot_size = stack_depot_stats->allocated;
60	}
61	}
62	}
63	// Check RSS against the limit.
64	if (hard_rss_limit_mb && hard_rss_limit_mb < current_rss_mb) {
65	Report("%s: hard rss limit exhausted (%zdMb vs %zdMb)\n",
66	SanitizerToolName, hard_rss_limit_mb, current_rss_mb);
67	DumpProcessMap();
68	Die();
69	}
70	if (soft_rss_limit_mb) {
71	if (soft_rss_limit_mb < current_rss_mb && !reached_soft_rss_limit) {
72	reached_soft_rss_limit = true;
73	Report("%s: soft rss limit exhausted (%zdMb vs %zdMb)\n",
74	SanitizerToolName, soft_rss_limit_mb, current_rss_mb);
75	if (SoftRssLimitExceededCallback)
76	SoftRssLimitExceededCallback(true);
77	} else if (soft_rss_limit_mb >= current_rss_mb &&
78	reached_soft_rss_limit) {
79	reached_soft_rss_limit = false;
80	if (SoftRssLimitExceededCallback)
81	SoftRssLimitExceededCallback(false);
82	}
83	}
84	if (heap_profile &&
85	current_rss_mb > rss_during_last_reported_profile * 1.1) {
86	Printf("\n\nHEAP PROFILE at RSS %zdMb\n", current_rss_mb);
87	__sanitizer_print_memory_profile(90, 20);
88	rss_during_last_reported_profile = current_rss_mb;
89	}
90	}
91	}
92	#endif
93
94	void WriteToSyslog(const char *msg) {
95	InternalScopedString msg_copy(kErrorMessageBufferSize);
96	msg_copy.append("%s", msg);
97	char *p = msg_copy.data();
98	char *q;
99
100	// Print one line at a time.
101	// syslog, at least on Android, has an implicit message length limit.
102	while ((q = internal_strchr(p, '\n'))) {
103	*q = '\0';
104	WriteOneLineToSyslog(p);
105	p = q + 1;
106	}
107	// Print remaining characters, if there are any.
108	// Note that this will add an extra newline at the end.
109	// FIXME: buffer extra output. This would need a thread-local buffer, which
110	// on Android requires plugging into the tools (ex. ASan's) Thread class.
111	if (*p)
112	WriteOneLineToSyslog(p);
113	}
114
115	void MaybeStartBackgroudThread() {
116	#if (SANITIZER_LINUX \|\| SANITIZER_NETBSD) && \
117	!SANITIZER_GO // Need to implement/test on other platforms.
118	// Start the background thread if one of the rss limits is given.
119	if (!common_flags()->hard_rss_limit_mb &&
120	!common_flags()->soft_rss_limit_mb &&
121	!common_flags()->heap_profile) return;
122	if (!&real_pthread_create) return; // Can't spawn the thread anyway.
123	internal_start_thread(BackgroundThread, nullptr);
124	#endif
125	}
126
127	static void (*sandboxing_callback)();
128	void SetSandboxingCallback(void (*f)()) {
129	sandboxing_callback = f;
130	}
131
132	uptr ReservedAddressRange::InitAligned(uptr size, uptr align,
133	const char *name) {
134	CHECK(IsPowerOfTwo(align));
135	if (align <= GetPageSizeCached())
136	return Init(size, name);
137	uptr start = Init(size + align, name);
138	start += align - (start & (align - 1));
139	return start;
140	}
141
142	} // namespace __sanitizer
143
144	SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_sandbox_on_notify,
145	__sanitizer_sandbox_arguments *args) {
146	__sanitizer::PlatformPrepareForSandboxing(args);
147	if (__sanitizer::sandboxing_callback)
148	__sanitizer::sandboxing_callback();
149	}

lib/tsan/sanitizer_common/sanitizer_common_nolibc.cpp created+34

...	@@ -0,0 +1,34 @@
1	//===-- sanitizer_common_nolibc.cpp ---------------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file contains stubs for libc function to facilitate optional use of
10	// libc in no-libcdep sources.
11	//===----------------------------------------------------------------------===//
12
13	#include "sanitizer_platform.h"
14	#include "sanitizer_common.h"
15	#include "sanitizer_libc.h"
16
17	namespace __sanitizer {
18
19	// The Windows implementations of these functions use the win32 API directly,
20	// bypassing libc.
21	#if !SANITIZER_WINDOWS
22	#if SANITIZER_LINUX
23	void LogMessageOnPrintf(const char *str) {}
24	#endif
25	void WriteToSyslog(const char *buffer) {}
26	void Abort() { internal__exit(1); }
27	void SleepForSeconds(int seconds) { internal_sleep(seconds); }
28	#endif // !SANITIZER_WINDOWS
29
30	#if !SANITIZER_WINDOWS && !SANITIZER_MAC
31	void ListOfModules::init() {}
32	#endif
33
34	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_linux_libcdep.cpp created+846

...	@@ -0,0 +1,846 @@
1	//===-- sanitizer_linux_libcdep.cpp ---------------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is shared between AddressSanitizer and ThreadSanitizer
10	// run-time libraries and implements linux-specific functions from
11	// sanitizer_libc.h.
12	//===----------------------------------------------------------------------===//
13
14	#include "sanitizer_platform.h"
15
16	#if SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_NETBSD \|\| \
17	SANITIZER_OPENBSD \|\| SANITIZER_SOLARIS
18
19	#include "sanitizer_allocator_internal.h"
20	#include "sanitizer_atomic.h"
21	#include "sanitizer_common.h"
22	#include "sanitizer_file.h"
23	#include "sanitizer_flags.h"
24	#include "sanitizer_freebsd.h"
25	#include "sanitizer_getauxval.h"
26	#include "sanitizer_glibc_version.h"
27	#include "sanitizer_linux.h"
28	#include "sanitizer_placement_new.h"
29	#include "sanitizer_procmaps.h"
30
31	#include <dlfcn.h> // for dlsym()
32	#include <link.h>
33	#include <pthread.h>
34	#include <signal.h>
35	#include <sys/resource.h>
36	#include <syslog.h>
37
38	#if !defined(ElfW)
39	#define ElfW(type) Elf_##type
40	#endif
41
42	#if SANITIZER_FREEBSD
43	#include <pthread_np.h>
44	#include <osreldate.h>
45	#include <sys/sysctl.h>
46	#define pthread_getattr_np pthread_attr_get_np
47	#endif
48
49	#if SANITIZER_OPENBSD
50	#include <pthread_np.h>
51	#include <sys/sysctl.h>
52	#endif
53
54	#if SANITIZER_NETBSD
55	#include <sys/sysctl.h>
56	#include <sys/tls.h>
57	#include <lwp.h>
58	#endif
59
60	#if SANITIZER_SOLARIS
61	#include <stdlib.h>
62	#include <thread.h>
63	#endif
64
65	#if SANITIZER_ANDROID
66	#include <android/api-level.h>
67	#if !defined(CPU_COUNT) && !defined(__aarch64__)
68	#include <dirent.h>
69	#include <fcntl.h>
70	struct __sanitizer::linux_dirent {
71	long d_ino;
72	off_t d_off;
73	unsigned short d_reclen;
74	char d_name[];
75	};
76	#endif
77	#endif
78
79	#if !SANITIZER_ANDROID
80	#include <elf.h>
81	#include <unistd.h>
82	#endif
83
84	namespace __sanitizer {
85
86	SANITIZER_WEAK_ATTRIBUTE int
87	real_sigaction(int signum, const void act, void oldact);
88
89	int internal_sigaction(int signum, const void act, void oldact) {
90	#if !SANITIZER_GO
91	if (&real_sigaction)
92	return real_sigaction(signum, act, oldact);
93	#endif
94	return sigaction(signum, (const struct sigaction *)act,
95	(struct sigaction *)oldact);
96	}
97
98	void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top,
99	uptr *stack_bottom) {
100	CHECK(stack_top);
101	CHECK(stack_bottom);
102	if (at_initialization) {
103	// This is the main thread. Libpthread may not be initialized yet.
104	struct rlimit rl;
105	CHECK_EQ(getrlimit(RLIMIT_STACK, &rl), 0);
106
107	// Find the mapping that contains a stack variable.
108	MemoryMappingLayout proc_maps(/cache_enabled/true);
109	if (proc_maps.Error()) {
110	stack_top = stack_bottom = 0;
111	return;
112	}
113	MemoryMappedSegment segment;
114	uptr prev_end = 0;
115	while (proc_maps.Next(&segment)) {
116	if ((uptr)&rl < segment.end) break;
117	prev_end = segment.end;
118	}
119	CHECK((uptr)&rl >= segment.start && (uptr)&rl < segment.end);
120
121	// Get stacksize from rlimit, but clip it so that it does not overlap
122	// with other mappings.
123	uptr stacksize = rl.rlim_cur;
124	if (stacksize > segment.end - prev_end) stacksize = segment.end - prev_end;
125	// When running with unlimited stack size, we still want to set some limit.
126	// The unlimited stack size is caused by 'ulimit -s unlimited'.
127	// Also, for some reason, GNU make spawns subprocesses with unlimited stack.
128	if (stacksize > kMaxThreadStackSize)
129	stacksize = kMaxThreadStackSize;
130	*stack_top = segment.end;
131	*stack_bottom = segment.end - stacksize;
132	return;
133	}
134	uptr stacksize = 0;
135	void *stackaddr = nullptr;
136	#if SANITIZER_SOLARIS
137	stack_t ss;
138	CHECK_EQ(thr_stksegment(&ss), 0);
139	stacksize = ss.ss_size;
140	stackaddr = (char *)ss.ss_sp - stacksize;
141	#elif SANITIZER_OPENBSD
142	stack_t sattr;
143	CHECK_EQ(pthread_stackseg_np(pthread_self(), &sattr), 0);
144	stackaddr = sattr.ss_sp;
145	stacksize = sattr.ss_size;
146	#else // !SANITIZER_SOLARIS
147	pthread_attr_t attr;
148	pthread_attr_init(&attr);
149	CHECK_EQ(pthread_getattr_np(pthread_self(), &attr), 0);
150	my_pthread_attr_getstack(&attr, &stackaddr, &stacksize);
151	pthread_attr_destroy(&attr);
152	#endif // SANITIZER_SOLARIS
153
154	*stack_top = (uptr)stackaddr + stacksize;
155	*stack_bottom = (uptr)stackaddr;
156	}
157
158	#if !SANITIZER_GO
159	bool SetEnv(const char name, const char value) {
160	void *f = dlsym(RTLD_NEXT, "setenv");
161	if (!f)
162	return false;
163	typedef int(setenv_ft)(const char name, const char *value, int overwrite);
164	setenv_ft setenv_f;
165	CHECK_EQ(sizeof(setenv_f), sizeof(f));
166	internal_memcpy(&setenv_f, &f, sizeof(f));
167	return setenv_f(name, value, 1) == 0;
168	}
169	#endif
170
171	__attribute__((unused)) static bool GetLibcVersion(int major, int minor,
172	int *patch) {
173	#ifdef _CS_GNU_LIBC_VERSION
174	char buf[64];
175	uptr len = confstr(_CS_GNU_LIBC_VERSION, buf, sizeof(buf));
176	if (len >= sizeof(buf))
177	return false;
178	buf[len] = 0;
179	static const char kGLibC[] = "glibc ";
180	if (internal_strncmp(buf, kGLibC, sizeof(kGLibC) - 1) != 0)
181	return false;
182	const char *p = buf + sizeof(kGLibC) - 1;
183	*major = internal_simple_strtoll(p, &p, 10);
184	minor = (p == '.') ? internal_simple_strtoll(p + 1, &p, 10) : 0;
185	patch = (p == '.') ? internal_simple_strtoll(p + 1, &p, 10) : 0;
186	return true;
187	#else
188	return false;
189	#endif
190	}
191
192	#if !SANITIZER_FREEBSD && !SANITIZER_ANDROID && !SANITIZER_GO && \
193	!SANITIZER_NETBSD && !SANITIZER_OPENBSD && !SANITIZER_SOLARIS
194	static uptr g_tls_size;
195
196	#ifdef __i386__
197	# define CHECK_GET_TLS_STATIC_INFO_VERSION (!__GLIBC_PREREQ(2, 27))
198	#else
199	# define CHECK_GET_TLS_STATIC_INFO_VERSION 0
200	#endif
201
202	#if CHECK_GET_TLS_STATIC_INFO_VERSION
203	# define DL_INTERNAL_FUNCTION __attribute__((regparm(3), stdcall))
204	#else
205	# define DL_INTERNAL_FUNCTION
206	#endif
207
208	namespace {
209	struct GetTlsStaticInfoCall {
210	typedef void (get_tls_func)(size_t, size_t*);
211	};
212	struct GetTlsStaticInfoRegparmCall {
213	typedef void (get_tls_func)(size_t, size_t*) DL_INTERNAL_FUNCTION;
214	};
215
216	template <typename T>
217	void CallGetTls(void* ptr, size_t* size, size_t* align) {
218	typename T::get_tls_func get_tls;
219	CHECK_EQ(sizeof(get_tls), sizeof(ptr));
220	internal_memcpy(&get_tls, &ptr, sizeof(ptr));
221	CHECK_NE(get_tls, 0);
222	get_tls(size, align);
223	}
224
225	bool CmpLibcVersion(int major, int minor, int patch) {
226	int ma;
227	int mi;
228	int pa;
229	if (!GetLibcVersion(&ma, &mi, &pa))
230	return false;
231	if (ma > major)
232	return true;
233	if (ma < major)
234	return false;
235	if (mi > minor)
236	return true;
237	if (mi < minor)
238	return false;
239	return pa >= patch;
240	}
241
242	} // namespace
243
244	void InitTlsSize() {
245	// all current supported platforms have 16 bytes stack alignment
246	const size_t kStackAlign = 16;
247	void *get_tls_static_info_ptr = dlsym(RTLD_NEXT, "_dl_get_tls_static_info");
248	size_t tls_size = 0;
249	size_t tls_align = 0;
250	// On i?86, _dl_get_tls_static_info used to be internal_function, i.e.
251	// __attribute__((regparm(3), stdcall)) before glibc 2.27 and is normal
252	// function in 2.27 and later.
253	if (CHECK_GET_TLS_STATIC_INFO_VERSION && !CmpLibcVersion(2, 27, 0))
254	CallGetTls<GetTlsStaticInfoRegparmCall>(get_tls_static_info_ptr,
255	&tls_size, &tls_align);
256	else
257	CallGetTls<GetTlsStaticInfoCall>(get_tls_static_info_ptr,
258	&tls_size, &tls_align);
259	if (tls_align < kStackAlign)
260	tls_align = kStackAlign;
261	g_tls_size = RoundUpTo(tls_size, tls_align);
262	}
263	#else
264	void InitTlsSize() { }
265	#endif // !SANITIZER_FREEBSD && !SANITIZER_ANDROID && !SANITIZER_GO &&
266	// !SANITIZER_NETBSD && !SANITIZER_SOLARIS
267
268	#if (defined(__x86_64__) \|\| defined(__i386__) \|\| defined(__mips__) \|\| \
269	defined(__aarch64__) \|\| defined(__powerpc64__) \|\| defined(__s390__) \|\| \
270	defined(__arm__)) && \
271	SANITIZER_LINUX && !SANITIZER_ANDROID
272	// sizeof(struct pthread) from glibc.
273	static atomic_uintptr_t thread_descriptor_size;
274
275	uptr ThreadDescriptorSize() {
276	uptr val = atomic_load_relaxed(&thread_descriptor_size);
277	if (val)
278	return val;
279	#if defined(__x86_64__) \|\| defined(__i386__) \|\| defined(__arm__)
280	int major;
281	int minor;
282	int patch;
283	if (GetLibcVersion(&major, &minor, &patch) && major == 2) {
284	/* sizeof(struct pthread) values from various glibc versions. */
285	if (SANITIZER_X32)
286	val = 1728; // Assume only one particular version for x32.
287	// For ARM sizeof(struct pthread) changed in Glibc 2.23.
288	else if (SANITIZER_ARM)
289	val = minor <= 22 ? 1120 : 1216;
290	else if (minor <= 3)
291	val = FIRST_32_SECOND_64(1104, 1696);
292	else if (minor == 4)
293	val = FIRST_32_SECOND_64(1120, 1728);
294	else if (minor == 5)
295	val = FIRST_32_SECOND_64(1136, 1728);
296	else if (minor <= 9)
297	val = FIRST_32_SECOND_64(1136, 1712);
298	else if (minor == 10)
299	val = FIRST_32_SECOND_64(1168, 1776);
300	else if (minor == 11 \|\| (minor == 12 && patch == 1))
301	val = FIRST_32_SECOND_64(1168, 2288);
302	else if (minor <= 14)
303	val = FIRST_32_SECOND_64(1168, 2304);
304	else
305	val = FIRST_32_SECOND_64(1216, 2304);
306	}
307	#elif defined(__mips__)
308	// TODO(sagarthakur): add more values as per different glibc versions.
309	val = FIRST_32_SECOND_64(1152, 1776);
310	#elif defined(__aarch64__)
311	// The sizeof (struct pthread) is the same from GLIBC 2.17 to 2.22.
312	val = 1776;
313	#elif defined(__powerpc64__)
314	val = 1776; // from glibc.ppc64le 2.20-8.fc21
315	#elif defined(__s390__)
316	val = FIRST_32_SECOND_64(1152, 1776); // valid for glibc 2.22
317	#endif
318	if (val)
319	atomic_store_relaxed(&thread_descriptor_size, val);
320	return val;
321	}
322
323	// The offset at which pointer to self is located in the thread descriptor.
324	const uptr kThreadSelfOffset = FIRST_32_SECOND_64(8, 16);
325
326	uptr ThreadSelfOffset() {
327	return kThreadSelfOffset;
328	}
329
330	#if defined(__mips__) \|\| defined(__powerpc64__)
331	// TlsPreTcbSize includes size of struct pthread_descr and size of tcb
332	// head structure. It lies before the static tls blocks.
333	static uptr TlsPreTcbSize() {
334	# if defined(__mips__)
335	const uptr kTcbHead = 16; // sizeof (tcbhead_t)
336	# elif defined(__powerpc64__)
337	const uptr kTcbHead = 88; // sizeof (tcbhead_t)
338	# endif
339	const uptr kTlsAlign = 16;
340	const uptr kTlsPreTcbSize =
341	RoundUpTo(ThreadDescriptorSize() + kTcbHead, kTlsAlign);
342	return kTlsPreTcbSize;
343	}
344	#endif
345
346	uptr ThreadSelf() {
347	uptr descr_addr;
348	# if defined(__i386__)
349	asm("mov %%gs:%c1,%0" : "=r"(descr_addr) : "i"(kThreadSelfOffset));
350	# elif defined(__x86_64__)
351	asm("mov %%fs:%c1,%0" : "=r"(descr_addr) : "i"(kThreadSelfOffset));
352	# elif defined(__mips__)
353	// MIPS uses TLS variant I. The thread pointer (in hardware register $29)
354	// points to the end of the TCB + 0x7000. The pthread_descr structure is
355	// immediately in front of the TCB. TlsPreTcbSize() includes the size of the
356	// TCB and the size of pthread_descr.
357	const uptr kTlsTcbOffset = 0x7000;
358	uptr thread_pointer;
359	asm volatile(".set push;\
360	.set mips64r2;\
361	rdhwr %0,$29;\
362	.set pop" : "=r" (thread_pointer));
363	descr_addr = thread_pointer - kTlsTcbOffset - TlsPreTcbSize();
364	# elif defined(__aarch64__) \|\| defined(__arm__)
365	descr_addr = reinterpret_cast<uptr>(__builtin_thread_pointer()) -
366	ThreadDescriptorSize();
367	# elif defined(__s390__)
368	descr_addr = reinterpret_cast<uptr>(__builtin_thread_pointer());
369	# elif defined(__powerpc64__)
370	// PPC64LE uses TLS variant I. The thread pointer (in GPR 13)
371	// points to the end of the TCB + 0x7000. The pthread_descr structure is
372	// immediately in front of the TCB. TlsPreTcbSize() includes the size of the
373	// TCB and the size of pthread_descr.
374	const uptr kTlsTcbOffset = 0x7000;
375	uptr thread_pointer;
376	asm("addi %0,13,%1" : "=r"(thread_pointer) : "I"(-kTlsTcbOffset));
377	descr_addr = thread_pointer - TlsPreTcbSize();
378	# else
379	# error "unsupported CPU arch"
380	# endif
381	return descr_addr;
382	}
383	#endif // (x86_64 \|\| i386 \|\| MIPS) && SANITIZER_LINUX
384
385	#if SANITIZER_FREEBSD
386	static void **ThreadSelfSegbase() {
387	void **segbase = 0;
388	# if defined(__i386__)
389	// sysarch(I386_GET_GSBASE, segbase);
390	__asm __volatile("mov %%gs:0, %0" : "=r" (segbase));
391	# elif defined(__x86_64__)
392	// sysarch(AMD64_GET_FSBASE, segbase);
393	__asm __volatile("movq %%fs:0, %0" : "=r" (segbase));
394	# else
395	# error "unsupported CPU arch"
396	# endif
397	return segbase;
398	}
399
400	uptr ThreadSelf() {
401	return (uptr)ThreadSelfSegbase()[2];
402	}
403	#endif // SANITIZER_FREEBSD
404
405	#if SANITIZER_NETBSD
406	static struct tls_tcb * ThreadSelfTlsTcb() {
407	return (struct tls_tcb *)_lwp_getprivate();
408	}
409
410	uptr ThreadSelf() {
411	return (uptr)ThreadSelfTlsTcb()->tcb_pthread;
412	}
413
414	int GetSizeFromHdr(struct dl_phdr_info info, size_t size, void data) {
415	const Elf_Phdr *hdr = info->dlpi_phdr;
416	const Elf_Phdr *last_hdr = hdr + info->dlpi_phnum;
417
418	for (; hdr != last_hdr; ++hdr) {
419	if (hdr->p_type == PT_TLS && info->dlpi_tls_modid == 1) {
420	(uptr)data = hdr->p_memsz;
421	break;
422	}
423	}
424	return 0;
425	}
426	#endif // SANITIZER_NETBSD
427
428	#if !SANITIZER_GO
429	static void GetTls(uptr addr, uptr size) {
430	#if SANITIZER_LINUX && !SANITIZER_ANDROID
431	# if defined(__x86_64__) \|\| defined(__i386__) \|\| defined(__s390__)
432	*addr = ThreadSelf();
433	*size = GetTlsSize();
434	addr -= size;
435	*addr += ThreadDescriptorSize();
436	# elif defined(__mips__) \|\| defined(__aarch64__) \|\| defined(__powerpc64__) \
437	\|\| defined(__arm__)
438	*addr = ThreadSelf();
439	*size = GetTlsSize();
440	# else
441	*addr = 0;
442	*size = 0;
443	# endif
444	#elif SANITIZER_FREEBSD
445	void** segbase = ThreadSelfSegbase();
446	*addr = 0;
447	*size = 0;
448	if (segbase != 0) {
449	// tcbalign = 16
450	// tls_size = round(tls_static_space, tcbalign);
451	// dtv = segbase[1];
452	// dtv[2] = segbase - tls_static_space;
453	void dtv = (void) segbase[1];
454	*addr = (uptr) dtv[2];
455	size = (addr == 0) ? 0 : ((uptr) segbase[0] - (uptr) dtv[2]);
456	}
457	#elif SANITIZER_NETBSD
458	struct tls_tcb * const tcb = ThreadSelfTlsTcb();
459	*addr = 0;
460	*size = 0;
461	if (tcb != 0) {
462	// Find size (p_memsz) of dlpi_tls_modid 1 (TLS block of the main program).
463	// ld.elf_so hardcodes the index 1.
464	dl_iterate_phdr(GetSizeFromHdr, size);
465
466	if (*size != 0) {
467	// The block has been found and tcb_dtv[1] contains the base address
468	*addr = (uptr)tcb->tcb_dtv[1];
469	}
470	}
471	#elif SANITIZER_OPENBSD
472	*addr = 0;
473	*size = 0;
474	#elif SANITIZER_ANDROID
475	*addr = 0;
476	*size = 0;
477	#elif SANITIZER_SOLARIS
478	// FIXME
479	*addr = 0;
480	*size = 0;
481	#else
482	# error "Unknown OS"
483	#endif
484	}
485	#endif
486
487	#if !SANITIZER_GO
488	uptr GetTlsSize() {
489	#if SANITIZER_FREEBSD \|\| SANITIZER_ANDROID \|\| SANITIZER_NETBSD \|\| \
490	SANITIZER_OPENBSD \|\| SANITIZER_SOLARIS
491	uptr addr, size;
492	GetTls(&addr, &size);
493	return size;
494	#elif defined(__mips__) \|\| defined(__powerpc64__)
495	return RoundUpTo(g_tls_size + TlsPreTcbSize(), 16);
496	#else
497	return g_tls_size;
498	#endif
499	}
500	#endif
501
502	void GetThreadStackAndTls(bool main, uptr stk_addr, uptr stk_size,
503	uptr tls_addr, uptr tls_size) {
504	#if SANITIZER_GO
505	// Stub implementation for Go.
506	stk_addr = stk_size = tls_addr = tls_size = 0;
507	#else
508	GetTls(tls_addr, tls_size);
509
510	uptr stack_top, stack_bottom;
511	GetThreadStackTopAndBottom(main, &stack_top, &stack_bottom);
512	*stk_addr = stack_bottom;
513	*stk_size = stack_top - stack_bottom;
514
515	if (!main) {
516	// If stack and tls intersect, make them non-intersecting.
517	if (tls_addr > stk_addr && tls_addr < stk_addr + *stk_size) {
518	CHECK_GT(tls_addr + tls_size, *stk_addr);
519	CHECK_LE(tls_addr + tls_size, stk_addr + stk_size);
520	stk_size -= tls_size;
521	tls_addr = stk_addr + *stk_size;
522	}
523	}
524	#endif
525	}
526
527	#if !SANITIZER_FREEBSD && !SANITIZER_OPENBSD
528	typedef ElfW(Phdr) Elf_Phdr;
529	#elif SANITIZER_WORDSIZE == 32 && __FreeBSD_version <= 902001 // v9.2
530	#define Elf_Phdr XElf32_Phdr
531	#define dl_phdr_info xdl_phdr_info
532	#define dl_iterate_phdr(c, b) xdl_iterate_phdr((c), (b))
533	#endif // !SANITIZER_FREEBSD && !SANITIZER_OPENBSD
534
535	struct DlIteratePhdrData {
536	InternalMmapVectorNoCtor<LoadedModule> *modules;
537	bool first;
538	};
539
540	static int dl_iterate_phdr_cb(dl_phdr_info info, size_t size, void arg) {
541	DlIteratePhdrData data = (DlIteratePhdrData)arg;
542	InternalScopedString module_name(kMaxPathLength);
543	if (data->first) {
544	data->first = false;
545	// First module is the binary itself.
546	ReadBinaryNameCached(module_name.data(), module_name.size());
547	} else if (info->dlpi_name) {
548	module_name.append("%s", info->dlpi_name);
549	}
550	if (module_name[0] == '\0')
551	return 0;
552	LoadedModule cur_module;
553	cur_module.set(module_name.data(), info->dlpi_addr);
554	for (int i = 0; i < (int)info->dlpi_phnum; i++) {
555	const Elf_Phdr *phdr = &info->dlpi_phdr[i];
556	if (phdr->p_type == PT_LOAD) {
557	uptr cur_beg = info->dlpi_addr + phdr->p_vaddr;
558	uptr cur_end = cur_beg + phdr->p_memsz;
559	bool executable = phdr->p_flags & PF_X;
560	bool writable = phdr->p_flags & PF_W;
561	cur_module.addAddressRange(cur_beg, cur_end, executable,
562	writable);
563	}
564	}
565	data->modules->push_back(cur_module);
566	return 0;
567	}
568
569	#if SANITIZER_ANDROID && __ANDROID_API__ < 21
570	extern "C" __attribute__((weak)) int dl_iterate_phdr(
571	int ()(struct dl_phdr_info , size_t, void ), void );
572	#endif
573
574	static bool requiresProcmaps() {
575	#if SANITIZER_ANDROID && __ANDROID_API__ <= 22
576	// Fall back to /proc/maps if dl_iterate_phdr is unavailable or broken.
577	// The runtime check allows the same library to work with
578	// both K and L (and future) Android releases.
579	return AndroidGetApiLevel() <= ANDROID_LOLLIPOP_MR1;
580	#else
581	return false;
582	#endif
583	}
584
585	static void procmapsInit(InternalMmapVectorNoCtor<LoadedModule> *modules) {
586	MemoryMappingLayout memory_mapping(/cache_enabled/true);
587	memory_mapping.DumpListOfModules(modules);
588	}
589
590	void ListOfModules::init() {
591	clearOrInit();
592	if (requiresProcmaps()) {
593	procmapsInit(&modules_);
594	} else {
595	DlIteratePhdrData data = {&modules_, true};
596	dl_iterate_phdr(dl_iterate_phdr_cb, &data);
597	}
598	}
599
600	// When a custom loader is used, dl_iterate_phdr may not contain the full
601	// list of modules. Allow callers to fall back to using procmaps.
602	void ListOfModules::fallbackInit() {
603	if (!requiresProcmaps()) {
604	clearOrInit();
605	procmapsInit(&modules_);
606	} else {
607	clear();
608	}
609	}
610
611	// getrusage does not give us the current RSS, only the max RSS.
612	// Still, this is better than nothing if /proc/self/statm is not available
613	// for some reason, e.g. due to a sandbox.
614	static uptr GetRSSFromGetrusage() {
615	struct rusage usage;
616	if (getrusage(RUSAGE_SELF, &usage)) // Failed, probably due to a sandbox.
617	return 0;
618	return usage.ru_maxrss << 10; // ru_maxrss is in Kb.
619	}
620
621	uptr GetRSS() {
622	if (!common_flags()->can_use_proc_maps_statm)
623	return GetRSSFromGetrusage();
624	fd_t fd = OpenFile("/proc/self/statm", RdOnly);
625	if (fd == kInvalidFd)
626	return GetRSSFromGetrusage();
627	char buf[64];
628	uptr len = internal_read(fd, buf, sizeof(buf) - 1);
629	internal_close(fd);
630	if ((sptr)len <= 0)
631	return 0;
632	buf[len] = 0;
633	// The format of the file is:
634	// 1084 89 69 11 0 79 0
635	// We need the second number which is RSS in pages.
636	char *pos = buf;
637	// Skip the first number.
638	while (pos >= '0' && pos <= '9')
639	pos++;
640	// Skip whitespaces.
641	while (!(pos >= '0' && pos <= '9') && *pos != 0)
642	pos++;
643	// Read the number.
644	uptr rss = 0;
645	while (pos >= '0' && pos <= '9')
646	rss = rss * 10 + *pos++ - '0';
647	return rss * GetPageSizeCached();
648	}
649
650	// sysconf(_SC_NPROCESSORS_{CONF,ONLN}) cannot be used on most platforms as
651	// they allocate memory.
652	u32 GetNumberOfCPUs() {
653	#if SANITIZER_FREEBSD \|\| SANITIZER_NETBSD \|\| SANITIZER_OPENBSD
654	u32 ncpu;
655	int req[2];
656	uptr len = sizeof(ncpu);
657	req[0] = CTL_HW;
658	req[1] = HW_NCPU;
659	CHECK_EQ(internal_sysctl(req, 2, &ncpu, &len, NULL, 0), 0);
660	return ncpu;
661	#elif SANITIZER_ANDROID && !defined(CPU_COUNT) && !defined(__aarch64__)
662	// Fall back to /sys/devices/system/cpu on Android when cpu_set_t doesn't
663	// exist in sched.h. That is the case for toolchains generated with older
664	// NDKs.
665	// This code doesn't work on AArch64 because internal_getdents makes use of
666	// the 64bit getdents syscall, but cpu_set_t seems to always exist on AArch64.
667	uptr fd = internal_open("/sys/devices/system/cpu", O_RDONLY \| O_DIRECTORY);
668	if (internal_iserror(fd))
669	return 0;
670	InternalMmapVector<u8> buffer(4096);
671	uptr bytes_read = buffer.size();
672	uptr n_cpus = 0;
673	u8 *d_type;
674	struct linux_dirent entry = (struct linux_dirent )&buffer[bytes_read];
675	while (true) {
676	if ((u8 *)entry >= &buffer[bytes_read]) {
677	bytes_read = internal_getdents(fd, (struct linux_dirent *)buffer.data(),
678	buffer.size());
679	if (internal_iserror(bytes_read) \|\| !bytes_read)
680	break;
681	entry = (struct linux_dirent *)buffer.data();
682	}
683	d_type = (u8 *)entry + entry->d_reclen - 1;
684	if (d_type >= &buffer[bytes_read] \|\|
685	(u8 *)&entry->d_name[3] >= &buffer[bytes_read])
686	break;
687	if (entry->d_ino != 0 && *d_type == DT_DIR) {
688	if (entry->d_name[0] == 'c' && entry->d_name[1] == 'p' &&
689	entry->d_name[2] == 'u' &&
690	entry->d_name[3] >= '0' && entry->d_name[3] <= '9')
691	n_cpus++;
692	}
693	entry = (struct linux_dirent )(((u8 )entry) + entry->d_reclen);
694	}
695	internal_close(fd);
696	return n_cpus;
697	#elif SANITIZER_SOLARIS
698	return sysconf(_SC_NPROCESSORS_ONLN);
699	#else
700	cpu_set_t CPUs;
701	CHECK_EQ(sched_getaffinity(0, sizeof(cpu_set_t), &CPUs), 0);
702	return CPU_COUNT(&CPUs);
703	#endif
704	}
705
706	#if SANITIZER_LINUX
707
708	# if SANITIZER_ANDROID
709	static atomic_uint8_t android_log_initialized;
710
711	void AndroidLogInit() {
712	openlog(GetProcessName(), 0, LOG_USER);
713	atomic_store(&android_log_initialized, 1, memory_order_release);
714	}
715
716	static bool ShouldLogAfterPrintf() {
717	return atomic_load(&android_log_initialized, memory_order_acquire);
718	}
719
720	extern "C" SANITIZER_WEAK_ATTRIBUTE
721	int async_safe_write_log(int pri, const char* tag, const char* msg);
722	extern "C" SANITIZER_WEAK_ATTRIBUTE
723	int __android_log_write(int prio, const char* tag, const char* msg);
724
725	// ANDROID_LOG_INFO is 4, but can't be resolved at runtime.
726	#define SANITIZER_ANDROID_LOG_INFO 4
727
728	// async_safe_write_log is a new public version of __libc_write_log that is
729	// used behind syslog. It is preferable to syslog as it will not do any dynamic
730	// memory allocation or formatting.
731	// If the function is not available, syslog is preferred for L+ (it was broken
732	// pre-L) as __android_log_write triggers a racey behavior with the strncpy
733	// interceptor. Fallback to __android_log_write pre-L.
734	void WriteOneLineToSyslog(const char *s) {
735	if (&async_safe_write_log) {
736	async_safe_write_log(SANITIZER_ANDROID_LOG_INFO, GetProcessName(), s);
737	} else if (AndroidGetApiLevel() > ANDROID_KITKAT) {
738	syslog(LOG_INFO, "%s", s);
739	} else {
740	CHECK(&__android_log_write);
741	__android_log_write(SANITIZER_ANDROID_LOG_INFO, nullptr, s);
742	}
743	}
744
745	extern "C" SANITIZER_WEAK_ATTRIBUTE
746	void android_set_abort_message(const char *);
747
748	void SetAbortMessage(const char *str) {
749	if (&android_set_abort_message)
750	android_set_abort_message(str);
751	}
752	# else
753	void AndroidLogInit() {}
754
755	static bool ShouldLogAfterPrintf() { return true; }
756
757	void WriteOneLineToSyslog(const char *s) { syslog(LOG_INFO, "%s", s); }
758
759	void SetAbortMessage(const char *str) {}
760	# endif // SANITIZER_ANDROID
761
762	void LogMessageOnPrintf(const char *str) {
763	if (common_flags()->log_to_syslog && ShouldLogAfterPrintf())
764	WriteToSyslog(str);
765	}
766
767	#endif // SANITIZER_LINUX
768
769	#if SANITIZER_LINUX && !SANITIZER_GO
770	// glibc crashes when using clock_gettime from a preinit_array function as the
771	// vDSO function pointers haven't been initialized yet. __progname is
772	// initialized after the vDSO function pointers, so if it exists, is not null
773	// and is not empty, we can use clock_gettime.
774	extern "C" SANITIZER_WEAK_ATTRIBUTE char *__progname;
775	INLINE bool CanUseVDSO() {
776	// Bionic is safe, it checks for the vDSO function pointers to be initialized.
777	if (SANITIZER_ANDROID)
778	return true;
779	if (&__progname && __progname && *__progname)
780	return true;
781	return false;
782	}
783
784	// MonotonicNanoTime is a timing function that can leverage the vDSO by calling
785	// clock_gettime. real_clock_gettime only exists if clock_gettime is
786	// intercepted, so define it weakly and use it if available.
787	extern "C" SANITIZER_WEAK_ATTRIBUTE
788	int real_clock_gettime(u32 clk_id, void *tp);
789	u64 MonotonicNanoTime() {
790	timespec ts;
791	if (CanUseVDSO()) {
792	if (&real_clock_gettime)
793	real_clock_gettime(CLOCK_MONOTONIC, &ts);
794	else
795	clock_gettime(CLOCK_MONOTONIC, &ts);
796	} else {
797	internal_clock_gettime(CLOCK_MONOTONIC, &ts);
798	}
799	return (u64)ts.tv_sec * (1000ULL * 1000 * 1000) + ts.tv_nsec;
800	}
801	#else
802	// Non-Linux & Go always use the syscall.
803	u64 MonotonicNanoTime() {
804	timespec ts;
805	internal_clock_gettime(CLOCK_MONOTONIC, &ts);
806	return (u64)ts.tv_sec * (1000ULL * 1000 * 1000) + ts.tv_nsec;
807	}
808	#endif // SANITIZER_LINUX && !SANITIZER_GO
809
810	#if !SANITIZER_OPENBSD
811	void ReExec() {
812	const char *pathname = "/proc/self/exe";
813
814	#if SANITIZER_NETBSD
815	static const int name[] = {
816	CTL_KERN,
817	KERN_PROC_ARGS,
818	-1,
819	KERN_PROC_PATHNAME,
820	};
821	char path[400];
822	uptr len;
823
824	len = sizeof(path);
825	if (internal_sysctl(name, ARRAY_SIZE(name), path, &len, NULL, 0) != -1)
826	pathname = path;
827	#elif SANITIZER_SOLARIS
828	pathname = getexecname();
829	CHECK_NE(pathname, NULL);
830	#elif SANITIZER_USE_GETAUXVAL
831	// Calling execve with /proc/self/exe sets that as $EXEC_ORIGIN. Binaries that
832	// rely on that will fail to load shared libraries. Query AT_EXECFN instead.
833	pathname = reinterpret_cast<const char *>(getauxval(AT_EXECFN));
834	#endif
835
836	uptr rv = internal_execve(pathname, GetArgv(), GetEnviron());
837	int rverrno;
838	CHECK_EQ(internal_iserror(rv, &rverrno), true);
839	Printf("execve failed, errno %d\n", rverrno);
840	Die();
841	}
842	#endif // !SANITIZER_OPENBSD
843
844	} // namespace __sanitizer
845
846	#endif

lib/tsan/sanitizer_common/sanitizer_mac_libcdep.cpp created+29

...	@@ -0,0 +1,29 @@
1	//===-- sanitizer_mac_libcdep.cpp -----------------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is shared between various sanitizers' runtime libraries and
10	// implements OSX-specific functions.
11	//===----------------------------------------------------------------------===//
12
13	#include "sanitizer_platform.h"
14	#if SANITIZER_MAC
15	#include "sanitizer_mac.h"
16
17	#include <sys/mman.h>
18
19	namespace __sanitizer {
20
21	void RestrictMemoryToMaxAddress(uptr max_address) {
22	uptr size_to_mmap = GetMaxUserVirtualAddress() + 1 - max_address;
23	void *res = MmapFixedNoAccess(max_address, size_to_mmap, "high gap");
24	CHECK(res != MAP_FAILED);
25	}
26
27	} // namespace __sanitizer
28
29	#endif // SANITIZER_MAC

lib/tsan/sanitizer_common/sanitizer_posix_libcdep.cpp created+509

...	@@ -0,0 +1,509 @@
1	//===-- sanitizer_posix_libcdep.cpp ---------------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is shared between AddressSanitizer and ThreadSanitizer
10	// run-time libraries and implements libc-dependent POSIX-specific functions
11	// from sanitizer_libc.h.
12	//===----------------------------------------------------------------------===//
13
14	#include "sanitizer_platform.h"
15
16	#if SANITIZER_POSIX
17
18	#include "sanitizer_common.h"
19	#include "sanitizer_flags.h"
20	#include "sanitizer_platform_limits_netbsd.h"
21	#include "sanitizer_platform_limits_openbsd.h"
22	#include "sanitizer_platform_limits_posix.h"
23	#include "sanitizer_platform_limits_solaris.h"
24	#include "sanitizer_posix.h"
25	#include "sanitizer_procmaps.h"
26
27	#include <errno.h>
28	#include <fcntl.h>
29	#include <pthread.h>
30	#include <signal.h>
31	#include <stdlib.h>
32	#include <sys/mman.h>
33	#include <sys/resource.h>
34	#include <sys/stat.h>
35	#include <sys/time.h>
36	#include <sys/types.h>
37	#include <sys/wait.h>
38	#include <unistd.h>
39
40	#if SANITIZER_FREEBSD
41	// The MAP_NORESERVE define has been removed in FreeBSD 11.x, and even before
42	// that, it was never implemented. So just define it to zero.
43	#undef MAP_NORESERVE
44	#define MAP_NORESERVE 0
45	#endif
46
47	typedef void (sa_sigaction_t)(int, siginfo_t , void *);
48
49	namespace __sanitizer {
50
51	u32 GetUid() {
52	return getuid();
53	}
54
55	uptr GetThreadSelf() {
56	return (uptr)pthread_self();
57	}
58
59	void ReleaseMemoryPagesToOS(uptr beg, uptr end) {
60	uptr page_size = GetPageSizeCached();
61	uptr beg_aligned = RoundUpTo(beg, page_size);
62	uptr end_aligned = RoundDownTo(end, page_size);
63	if (beg_aligned < end_aligned)
64	// In the default Solaris compilation environment, madvise() is declared
65	// to take a caddr_t arg; casting it to void * results in an invalid
66	// conversion error, so use char * instead.
67	madvise((char *)beg_aligned, end_aligned - beg_aligned,
68	SANITIZER_MADVISE_DONTNEED);
69	}
70
71	void SetShadowRegionHugePageMode(uptr addr, uptr size) {
72	#ifdef MADV_NOHUGEPAGE // May not be defined on old systems.
73	if (common_flags()->no_huge_pages_for_shadow)
74	madvise((char *)addr, size, MADV_NOHUGEPAGE);
75	else
76	madvise((char *)addr, size, MADV_HUGEPAGE);
77	#endif // MADV_NOHUGEPAGE
78	}
79
80	bool DontDumpShadowMemory(uptr addr, uptr length) {
81	#if defined(MADV_DONTDUMP)
82	return madvise((char *)addr, length, MADV_DONTDUMP) == 0;
83	#elif defined(MADV_NOCORE)
84	return madvise((char *)addr, length, MADV_NOCORE) == 0;
85	#else
86	return true;
87	#endif // MADV_DONTDUMP
88	}
89
90	static rlim_t getlim(int res) {
91	rlimit rlim;
92	CHECK_EQ(0, getrlimit(res, &rlim));
93	return rlim.rlim_cur;
94	}
95
96	static void setlim(int res, rlim_t lim) {
97	struct rlimit rlim;
98	if (getrlimit(res, const_cast<struct rlimit *>(&rlim))) {
99	Report("ERROR: %s getrlimit() failed %d\n", SanitizerToolName, errno);
100	Die();
101	}
102	rlim.rlim_cur = lim;
103	if (setrlimit(res, const_cast<struct rlimit *>(&rlim))) {
104	Report("ERROR: %s setrlimit() failed %d\n", SanitizerToolName, errno);
105	Die();
106	}
107	}
108
109	void DisableCoreDumperIfNecessary() {
110	if (common_flags()->disable_coredump) {
111	setlim(RLIMIT_CORE, 0);
112	}
113	}
114
115	bool StackSizeIsUnlimited() {
116	rlim_t stack_size = getlim(RLIMIT_STACK);
117	return (stack_size == RLIM_INFINITY);
118	}
119
120	void SetStackSizeLimitInBytes(uptr limit) {
121	setlim(RLIMIT_STACK, (rlim_t)limit);
122	CHECK(!StackSizeIsUnlimited());
123	}
124
125	bool AddressSpaceIsUnlimited() {
126	rlim_t as_size = getlim(RLIMIT_AS);
127	return (as_size == RLIM_INFINITY);
128	}
129
130	void SetAddressSpaceUnlimited() {
131	setlim(RLIMIT_AS, RLIM_INFINITY);
132	CHECK(AddressSpaceIsUnlimited());
133	}
134
135	void SleepForSeconds(int seconds) {
136	sleep(seconds);
137	}
138
139	void SleepForMillis(int millis) {
140	usleep(millis * 1000);
141	}
142
143	void Abort() {
144	#if !SANITIZER_GO
145	// If we are handling SIGABRT, unhandle it first.
146	// TODO(vitalybuka): Check if handler belongs to sanitizer.
147	if (GetHandleSignalMode(SIGABRT) != kHandleSignalNo) {
148	struct sigaction sigact;
149	internal_memset(&sigact, 0, sizeof(sigact));
150	sigact.sa_sigaction = (sa_sigaction_t)SIG_DFL;
151	internal_sigaction(SIGABRT, &sigact, nullptr);
152	}
153	#endif
154
155	abort();
156	}
157
158	int Atexit(void (*function)(void)) {
159	#if !SANITIZER_GO
160	return atexit(function);
161	#else
162	return 0;
163	#endif
164	}
165
166	bool SupportsColoredOutput(fd_t fd) {
167	return isatty(fd) != 0;
168	}
169
170	#if !SANITIZER_GO
171	// TODO(glider): different tools may require different altstack size.
172	static const uptr kAltStackSize = SIGSTKSZ * 4; // SIGSTKSZ is not enough.
173
174	void SetAlternateSignalStack() {
175	stack_t altstack, oldstack;
176	CHECK_EQ(0, sigaltstack(nullptr, &oldstack));
177	// If the alternate stack is already in place, do nothing.
178	// Android always sets an alternate stack, but it's too small for us.
179	if (!SANITIZER_ANDROID && !(oldstack.ss_flags & SS_DISABLE)) return;
180	// TODO(glider): the mapped stack should have the MAP_STACK flag in the
181	// future. It is not required by man 2 sigaltstack now (they're using
182	// malloc()).
183	void* base = MmapOrDie(kAltStackSize, __func__);
184	altstack.ss_sp = (char*) base;
185	altstack.ss_flags = 0;
186	altstack.ss_size = kAltStackSize;
187	CHECK_EQ(0, sigaltstack(&altstack, nullptr));
188	}
189
190	void UnsetAlternateSignalStack() {
191	stack_t altstack, oldstack;
192	altstack.ss_sp = nullptr;
193	altstack.ss_flags = SS_DISABLE;
194	altstack.ss_size = kAltStackSize; // Some sane value required on Darwin.
195	CHECK_EQ(0, sigaltstack(&altstack, &oldstack));
196	UnmapOrDie(oldstack.ss_sp, oldstack.ss_size);
197	}
198
199	static void MaybeInstallSigaction(int signum,
200	SignalHandlerType handler) {
201	if (GetHandleSignalMode(signum) == kHandleSignalNo) return;
202
203	struct sigaction sigact;
204	internal_memset(&sigact, 0, sizeof(sigact));
205	sigact.sa_sigaction = (sa_sigaction_t)handler;
206	// Do not block the signal from being received in that signal's handler.
207	// Clients are responsible for handling this correctly.
208	sigact.sa_flags = SA_SIGINFO \| SA_NODEFER;
209	if (common_flags()->use_sigaltstack) sigact.sa_flags \|= SA_ONSTACK;
210	CHECK_EQ(0, internal_sigaction(signum, &sigact, nullptr));
211	VReport(1, "Installed the sigaction for signal %d\n", signum);
212	}
213
214	void InstallDeadlySignalHandlers(SignalHandlerType handler) {
215	// Set the alternate signal stack for the main thread.
216	// This will cause SetAlternateSignalStack to be called twice, but the stack
217	// will be actually set only once.
218	if (common_flags()->use_sigaltstack) SetAlternateSignalStack();
219	MaybeInstallSigaction(SIGSEGV, handler);
220	MaybeInstallSigaction(SIGBUS, handler);
221	MaybeInstallSigaction(SIGABRT, handler);
222	MaybeInstallSigaction(SIGFPE, handler);
223	MaybeInstallSigaction(SIGILL, handler);
224	MaybeInstallSigaction(SIGTRAP, handler);
225	}
226
227	bool SignalContext::IsStackOverflow() const {
228	// Access at a reasonable offset above SP, or slightly below it (to account
229	// for x86_64 or PowerPC redzone, ARM push of multiple registers, etc) is
230	// probably a stack overflow.
231	#ifdef __s390__
232	// On s390, the fault address in siginfo points to start of the page, not
233	// to the precise word that was accessed. Mask off the low bits of sp to
234	// take it into account.
235	bool IsStackAccess = addr >= (sp & ~0xFFF) && addr < sp + 0xFFFF;
236	#else
237	// Let's accept up to a page size away from top of stack. Things like stack
238	// probing can trigger accesses with such large offsets.
239	bool IsStackAccess = addr + GetPageSizeCached() > sp && addr < sp + 0xFFFF;
240	#endif
241
242	#if __powerpc__
243	// Large stack frames can be allocated with e.g.
244	// lis r0,-10000
245	// stdux r1,r1,r0 # store sp to [sp-10000] and update sp by -10000
246	// If the store faults then sp will not have been updated, so test above
247	// will not work, because the fault address will be more than just "slightly"
248	// below sp.
249	if (!IsStackAccess && IsAccessibleMemoryRange(pc, 4)) {
250	u32 inst = (unsigned )pc;
251	u32 ra = (inst >> 16) & 0x1F;
252	u32 opcd = inst >> 26;
253	u32 xo = (inst >> 1) & 0x3FF;
254	// Check for store-with-update to sp. The instructions we accept are:
255	// stbu rs,d(ra) stbux rs,ra,rb
256	// sthu rs,d(ra) sthux rs,ra,rb
257	// stwu rs,d(ra) stwux rs,ra,rb
258	// stdu rs,ds(ra) stdux rs,ra,rb
259	// where ra is r1 (the stack pointer).
260	if (ra == 1 &&
261	(opcd == 39 \|\| opcd == 45 \|\| opcd == 37 \|\| opcd == 62 \|\|
262	(opcd == 31 && (xo == 247 \|\| xo == 439 \|\| xo == 183 \|\| xo == 181))))
263	IsStackAccess = true;
264	}
265	#endif // __powerpc__
266
267	// We also check si_code to filter out SEGV caused by something else other
268	// then hitting the guard page or unmapped memory, like, for example,
269	// unaligned memory access.
270	auto si = static_cast<const siginfo_t *>(siginfo);
271	return IsStackAccess &&
272	(si->si_code == si_SEGV_MAPERR \|\| si->si_code == si_SEGV_ACCERR);
273	}
274
275	#endif // SANITIZER_GO
276
277	bool IsAccessibleMemoryRange(uptr beg, uptr size) {
278	uptr page_size = GetPageSizeCached();
279	// Checking too large memory ranges is slow.
280	CHECK_LT(size, page_size * 10);
281	int sock_pair[2];
282	if (pipe(sock_pair))
283	return false;
284	uptr bytes_written =
285	internal_write(sock_pair[1], reinterpret_cast<void *>(beg), size);
286	int write_errno;
287	bool result;
288	if (internal_iserror(bytes_written, &write_errno)) {
289	CHECK_EQ(EFAULT, write_errno);
290	result = false;
291	} else {
292	result = (bytes_written == size);
293	}
294	internal_close(sock_pair[0]);
295	internal_close(sock_pair[1]);
296	return result;
297	}
298
299	void PlatformPrepareForSandboxing(__sanitizer_sandbox_arguments *args) {
300	// Some kinds of sandboxes may forbid filesystem access, so we won't be able
301	// to read the file mappings from /proc/self/maps. Luckily, neither the
302	// process will be able to load additional libraries, so it's fine to use the
303	// cached mappings.
304	MemoryMappingLayout::CacheMemoryMappings();
305	}
306
307	static bool MmapFixed(uptr fixed_addr, uptr size, int additional_flags,
308	const char *name) {
309	size = RoundUpTo(size, GetPageSizeCached());
310	fixed_addr = RoundDownTo(fixed_addr, GetPageSizeCached());
311	uptr p =
312	MmapNamed((void *)fixed_addr, size, PROT_READ \| PROT_WRITE,
313	MAP_PRIVATE \| MAP_FIXED \| additional_flags \| MAP_ANON, name);
314	int reserrno;
315	if (internal_iserror(p, &reserrno)) {
316	Report("ERROR: %s failed to "
317	"allocate 0x%zx (%zd) bytes at address %zx (errno: %d)\n",
318	SanitizerToolName, size, size, fixed_addr, reserrno);
319	return false;
320	}
321	IncreaseTotalMmap(size);
322	return true;
323	}
324
325	bool MmapFixedNoReserve(uptr fixed_addr, uptr size, const char *name) {
326	return MmapFixed(fixed_addr, size, MAP_NORESERVE, name);
327	}
328
329	bool MmapFixedSuperNoReserve(uptr fixed_addr, uptr size, const char *name) {
330	#if SANITIZER_FREEBSD
331	if (common_flags()->no_huge_pages_for_shadow)
332	return MmapFixedNoReserve(fixed_addr, size, name);
333	// MAP_NORESERVE is implicit with FreeBSD
334	return MmapFixed(fixed_addr, size, MAP_ALIGNED_SUPER, name);
335	#else
336	bool r = MmapFixedNoReserve(fixed_addr, size, name);
337	if (r)
338	SetShadowRegionHugePageMode(fixed_addr, size);
339	return r;
340	#endif
341	}
342
343	uptr ReservedAddressRange::Init(uptr size, const char *name, uptr fixed_addr) {
344	base_ = fixed_addr ? MmapFixedNoAccess(fixed_addr, size, name)
345	: MmapNoAccess(size);
346	size_ = size;
347	name_ = name;
348	(void)os_handle_; // unsupported
349	return reinterpret_cast<uptr>(base_);
350	}
351
352	// Uses fixed_addr for now.
353	// Will use offset instead once we've implemented this function for real.
354	uptr ReservedAddressRange::Map(uptr fixed_addr, uptr size, const char *name) {
355	return reinterpret_cast<uptr>(
356	MmapFixedOrDieOnFatalError(fixed_addr, size, name));
357	}
358
359	uptr ReservedAddressRange::MapOrDie(uptr fixed_addr, uptr size,
360	const char *name) {
361	return reinterpret_cast<uptr>(MmapFixedOrDie(fixed_addr, size, name));
362	}
363
364	void ReservedAddressRange::Unmap(uptr addr, uptr size) {
365	CHECK_LE(size, size_);
366	if (addr == reinterpret_cast<uptr>(base_))
367	// If we unmap the whole range, just null out the base.
368	base_ = (size == size_) ? nullptr : reinterpret_cast<void*>(addr + size);
369	else
370	CHECK_EQ(addr + size, reinterpret_cast<uptr>(base_) + size_);
371	size_ -= size;
372	UnmapOrDie(reinterpret_cast<void*>(addr), size);
373	}
374
375	void MmapFixedNoAccess(uptr fixed_addr, uptr size, const char name) {
376	return (void )MmapNamed((void )fixed_addr, size, PROT_NONE,
377	MAP_PRIVATE \| MAP_FIXED \| MAP_NORESERVE \| MAP_ANON,
378	name);
379	}
380
381	void *MmapNoAccess(uptr size) {
382	unsigned flags = MAP_PRIVATE \| MAP_ANON \| MAP_NORESERVE;
383	return (void *)internal_mmap(nullptr, size, PROT_NONE, flags, -1, 0);
384	}
385
386	// This function is defined elsewhere if we intercepted pthread_attr_getstack.
387	extern "C" {
388	SANITIZER_WEAK_ATTRIBUTE int
389	real_pthread_attr_getstack(void attr, void addr, size_t size);
390	} // extern "C"
391
392	int my_pthread_attr_getstack(void attr, void addr, uptr size) {
393	#if !SANITIZER_GO && !SANITIZER_MAC
394	if (&real_pthread_attr_getstack)
395	return real_pthread_attr_getstack((pthread_attr_t *)attr, addr,
396	(size_t *)size);
397	#endif
398	return pthread_attr_getstack((pthread_attr_t )attr, addr, (size_t )size);
399	}
400
401	#if !SANITIZER_GO
402	void AdjustStackSize(void *attr_) {
403	pthread_attr_t attr = (pthread_attr_t )attr_;
404	uptr stackaddr = 0;
405	uptr stacksize = 0;
406	my_pthread_attr_getstack(attr, (void**)&stackaddr, &stacksize);
407	// GLibC will return (0 - stacksize) as the stack address in the case when
408	// stacksize is set, but stackaddr is not.
409	bool stack_set = (stackaddr != 0) && (stackaddr + stacksize != 0);
410	// We place a lot of tool data into TLS, account for that.
411	const uptr minstacksize = GetTlsSize() + 128*1024;
412	if (stacksize < minstacksize) {
413	if (!stack_set) {
414	if (stacksize != 0) {
415	VPrintf(1, "Sanitizer: increasing stacksize %zu->%zu\n", stacksize,
416	minstacksize);
417	pthread_attr_setstacksize(attr, minstacksize);
418	}
419	} else {
420	Printf("Sanitizer: pre-allocated stack size is insufficient: "
421	"%zu < %zu\n", stacksize, minstacksize);
422	Printf("Sanitizer: pthread_create is likely to fail.\n");
423	}
424	}
425	}
426	#endif // !SANITIZER_GO
427
428	pid_t StartSubprocess(const char program, const char const argv[],
429	const char *const envp[], fd_t stdin_fd, fd_t stdout_fd,
430	fd_t stderr_fd) {
431	auto file_closer = at_scope_exit([&] {
432	if (stdin_fd != kInvalidFd) {
433	internal_close(stdin_fd);
434	}
435	if (stdout_fd != kInvalidFd) {
436	internal_close(stdout_fd);
437	}
438	if (stderr_fd != kInvalidFd) {
439	internal_close(stderr_fd);
440	}
441	});
442
443	int pid = internal_fork();
444
445	if (pid < 0) {
446	int rverrno;
447	if (internal_iserror(pid, &rverrno)) {
448	Report("WARNING: failed to fork (errno %d)\n", rverrno);
449	}
450	return pid;
451	}
452
453	if (pid == 0) {
454	// Child subprocess
455	if (stdin_fd != kInvalidFd) {
456	internal_close(STDIN_FILENO);
457	internal_dup2(stdin_fd, STDIN_FILENO);
458	internal_close(stdin_fd);
459	}
460	if (stdout_fd != kInvalidFd) {
461	internal_close(STDOUT_FILENO);
462	internal_dup2(stdout_fd, STDOUT_FILENO);
463	internal_close(stdout_fd);
464	}
465	if (stderr_fd != kInvalidFd) {
466	internal_close(STDERR_FILENO);
467	internal_dup2(stderr_fd, STDERR_FILENO);
468	internal_close(stderr_fd);
469	}
470
471	for (int fd = sysconf(_SC_OPEN_MAX); fd > 2; fd--) internal_close(fd);
472
473	internal_execve(program, const_cast<char **>(&argv[0]),
474	const_cast<char const >(envp));
475	internal__exit(1);
476	}
477
478	return pid;
479	}
480
481	bool IsProcessRunning(pid_t pid) {
482	int process_status;
483	uptr waitpid_status = internal_waitpid(pid, &process_status, WNOHANG);
484	int local_errno;
485	if (internal_iserror(waitpid_status, &local_errno)) {
486	VReport(1, "Waiting on the process failed (errno %d).\n", local_errno);
487	return false;
488	}
489	return waitpid_status == 0;
490	}
491
492	int WaitForProcess(pid_t pid) {
493	int process_status;
494	uptr waitpid_status = internal_waitpid(pid, &process_status, 0);
495	int local_errno;
496	if (internal_iserror(waitpid_status, &local_errno)) {
497	VReport(1, "Waiting on the process failed (errno %d).\n", local_errno);
498	return -1;
499	}
500	return process_status;
501	}
502
503	bool IsStateDetached(int state) {
504	return state == PTHREAD_CREATE_DETACHED;
505	}
506
507	} // namespace __sanitizer
508
509	#endif // SANITIZER_POSIX

lib/tsan/sanitizer_common/sanitizer_stackdepot.cpp created+149

...	@@ -0,0 +1,149 @@
1	//===-- sanitizer_stackdepot.cpp ------------------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is shared between AddressSanitizer and ThreadSanitizer
10	// run-time libraries.
11	//===----------------------------------------------------------------------===//
12
13	#include "sanitizer_stackdepot.h"
14
15	#include "sanitizer_common.h"
16	#include "sanitizer_hash.h"
17	#include "sanitizer_stackdepotbase.h"
18
19	namespace __sanitizer {
20
21	struct StackDepotNode {
22	StackDepotNode *link;
23	u32 id;
24	atomic_uint32_t hash_and_use_count; // hash_bits : 12; use_count : 20;
25	u32 size;
26	u32 tag;
27	uptr stack[1]; // [size]
28
29	static const u32 kTabSizeLog = SANITIZER_ANDROID ? 16 : 20;
30	// Lower kTabSizeLog bits are equal for all items in one bucket.
31	// We use these bits to store the per-stack use counter.
32	static const u32 kUseCountBits = kTabSizeLog;
33	static const u32 kMaxUseCount = 1 << kUseCountBits;
34	static const u32 kUseCountMask = (1 << kUseCountBits) - 1;
35	static const u32 kHashMask = ~kUseCountMask;
36
37	typedef StackTrace args_type;
38	bool eq(u32 hash, const args_type &args) const {
39	u32 hash_bits =
40	atomic_load(&hash_and_use_count, memory_order_relaxed) & kHashMask;
41	if ((hash & kHashMask) != hash_bits \|\| args.size != size \|\| args.tag != tag)
42	return false;
43	uptr i = 0;
44	for (; i < size; i++) {
45	if (stack[i] != args.trace[i]) return false;
46	}
47	return true;
48	}
49	static uptr storage_size(const args_type &args) {
50	return sizeof(StackDepotNode) + (args.size - 1) * sizeof(uptr);
51	}
52	static u32 hash(const args_type &args) {
53	MurMur2HashBuilder H(args.size * sizeof(uptr));
54	for (uptr i = 0; i < args.size; i++) H.add(args.trace[i]);
55	return H.get();
56	}
57	static bool is_valid(const args_type &args) {
58	return args.size > 0 && args.trace;
59	}
60	void store(const args_type &args, u32 hash) {
61	atomic_store(&hash_and_use_count, hash & kHashMask, memory_order_relaxed);
62	size = args.size;
63	tag = args.tag;
64	internal_memcpy(stack, args.trace, size * sizeof(uptr));
65	}
66	args_type load() const {
67	return args_type(&stack[0], size, tag);
68	}
69	StackDepotHandle get_handle() { return StackDepotHandle(this); }
70
71	typedef StackDepotHandle handle_type;
72	};
73
74	COMPILER_CHECK(StackDepotNode::kMaxUseCount == (u32)kStackDepotMaxUseCount);
75
76	u32 StackDepotHandle::id() { return node_->id; }
77	int StackDepotHandle::use_count() {
78	return atomic_load(&node_->hash_and_use_count, memory_order_relaxed) &
79	StackDepotNode::kUseCountMask;
80	}
81	void StackDepotHandle::inc_use_count_unsafe() {
82	u32 prev =
83	atomic_fetch_add(&node_->hash_and_use_count, 1, memory_order_relaxed) &
84	StackDepotNode::kUseCountMask;
85	CHECK_LT(prev + 1, StackDepotNode::kMaxUseCount);
86	}
87
88	// FIXME(dvyukov): this single reserved bit is used in TSan.
89	typedef StackDepotBase<StackDepotNode, 1, StackDepotNode::kTabSizeLog>
90	StackDepot;
91	static StackDepot theDepot;
92
93	StackDepotStats *StackDepotGetStats() {
94	return theDepot.GetStats();
95	}
96
97	u32 StackDepotPut(StackTrace stack) {
98	StackDepotHandle h = theDepot.Put(stack);
99	return h.valid() ? h.id() : 0;
100	}
101
102	StackDepotHandle StackDepotPut_WithHandle(StackTrace stack) {
103	return theDepot.Put(stack);
104	}
105
106	StackTrace StackDepotGet(u32 id) {
107	return theDepot.Get(id);
108	}
109
110	void StackDepotLockAll() {
111	theDepot.LockAll();
112	}
113
114	void StackDepotUnlockAll() {
115	theDepot.UnlockAll();
116	}
117
118	bool StackDepotReverseMap::IdDescPair::IdComparator(
119	const StackDepotReverseMap::IdDescPair &a,
120	const StackDepotReverseMap::IdDescPair &b) {
121	return a.id < b.id;
122	}
123
124	StackDepotReverseMap::StackDepotReverseMap() {
125	map_.reserve(StackDepotGetStats()->n_uniq_ids + 100);
126	for (int idx = 0; idx < StackDepot::kTabSize; idx++) {
127	atomic_uintptr_t *p = &theDepot.tab[idx];
128	uptr v = atomic_load(p, memory_order_consume);
129	StackDepotNode s = (StackDepotNode)(v & ~1);
130	for (; s; s = s->link) {
131	IdDescPair pair = {s->id, s};
132	map_.push_back(pair);
133	}
134	}
135	Sort(map_.data(), map_.size(), &IdDescPair::IdComparator);
136	}
137
138	StackTrace StackDepotReverseMap::Get(u32 id) {
139	if (!map_.size())
140	return StackTrace();
141	IdDescPair pair = {id, nullptr};
142	uptr idx =
143	InternalLowerBound(map_, 0, map_.size(), pair, IdDescPair::IdComparator);
144	if (idx > map_.size() \|\| map_[idx].id != id)
145	return StackTrace();
146	return map_[idx].desc->load();
147	}
148
149	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_stacktrace.cpp created+133

...	@@ -0,0 +1,133 @@
1	//===-- sanitizer_stacktrace.cpp ------------------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is shared between AddressSanitizer and ThreadSanitizer
10	// run-time libraries.
11	//===----------------------------------------------------------------------===//
12
13	#include "sanitizer_common.h"
14	#include "sanitizer_flags.h"
15	#include "sanitizer_stacktrace.h"
16
17	namespace __sanitizer {
18
19	uptr StackTrace::GetNextInstructionPc(uptr pc) {
20	#if defined(__sparc__) \|\| defined(__mips__)
21	return pc + 8;
22	#elif defined(__powerpc__) \|\| defined(__arm__) \|\| defined(__aarch64__)
23	return pc + 4;
24	#else
25	return pc + 1;
26	#endif
27	}
28
29	uptr StackTrace::GetCurrentPc() {
30	return GET_CALLER_PC();
31	}
32
33	void BufferedStackTrace::Init(const uptr *pcs, uptr cnt, uptr extra_top_pc) {
34	size = cnt + !!extra_top_pc;
35	CHECK_LE(size, kStackTraceMax);
36	internal_memcpy(trace_buffer, pcs, cnt * sizeof(trace_buffer[0]));
37	if (extra_top_pc)
38	trace_buffer[cnt] = extra_top_pc;
39	top_frame_bp = 0;
40	}
41
42	// Sparc implemention is in its own file.
43	#if !defined(__sparc__)
44
45	// In GCC on ARM bp points to saved lr, not fp, so we should check the next
46	// cell in stack to be a saved frame pointer. GetCanonicFrame returns the
47	// pointer to saved frame pointer in any case.
48	static inline uhwptr *GetCanonicFrame(uptr bp,
49	uptr stack_top,
50	uptr stack_bottom) {
51	CHECK_GT(stack_top, stack_bottom);
52	#ifdef __arm__
53	if (!IsValidFrame(bp, stack_top, stack_bottom)) return 0;
54	uhwptr bp_prev = (uhwptr )bp;
55	if (IsValidFrame((uptr)bp_prev[0], stack_top, stack_bottom)) return bp_prev;
56	// The next frame pointer does not look right. This could be a GCC frame, step
57	// back by 1 word and try again.
58	if (IsValidFrame((uptr)bp_prev[-1], stack_top, stack_bottom))
59	return bp_prev - 1;
60	// Nope, this does not look right either. This means the frame after next does
61	// not have a valid frame pointer, but we can still extract the caller PC.
62	// Unfortunately, there is no way to decide between GCC and LLVM frame
63	// layouts. Assume LLVM.
64	return bp_prev;
65	#else
66	return (uhwptr*)bp;
67	#endif
68	}
69
70	void BufferedStackTrace::UnwindFast(uptr pc, uptr bp, uptr stack_top,
71	uptr stack_bottom, u32 max_depth) {
72	// TODO(yln): add arg sanity check for stack_top/stack_bottom
73	CHECK_GE(max_depth, 2);
74	const uptr kPageSize = GetPageSizeCached();
75	trace_buffer[0] = pc;
76	size = 1;
77	if (stack_top < 4096) return; // Sanity check for stack top.
78	uhwptr *frame = GetCanonicFrame(bp, stack_top, stack_bottom);
79	// Lowest possible address that makes sense as the next frame pointer.
80	// Goes up as we walk the stack.
81	uptr bottom = stack_bottom;
82	// Avoid infinite loop when frame == frame[0] by using frame > prev_frame.
83	while (IsValidFrame((uptr)frame, stack_top, bottom) &&
84	IsAligned((uptr)frame, sizeof(*frame)) &&
85	size < max_depth) {
86	#ifdef __powerpc__
87	// PowerPC ABIs specify that the return address is saved at offset
88	// 16 of the caller's stack frame. Thus we must dereference the
89	// back chain to find the caller frame before extracting it.
90	uhwptr caller_frame = (uhwptr)frame[0];
91	if (!IsValidFrame((uptr)caller_frame, stack_top, bottom) \|\|
92	!IsAligned((uptr)caller_frame, sizeof(uhwptr)))
93	break;
94	uhwptr pc1 = caller_frame[2];
95	#elif defined(__s390__)
96	uhwptr pc1 = frame[14];
97	#else
98	uhwptr pc1 = frame[1];
99	#endif
100	// Let's assume that any pointer in the 0th page (i.e. <0x1000 on i386 and
101	// x86_64) is invalid and stop unwinding here. If we're adding support for
102	// a platform where this isn't true, we need to reconsider this check.
103	if (pc1 < kPageSize)
104	break;
105	if (pc1 != pc) {
106	trace_buffer[size++] = (uptr) pc1;
107	}
108	bottom = (uptr)frame;
109	frame = GetCanonicFrame((uptr)frame[0], stack_top, bottom);
110	}
111	}
112
113	#endif // !defined(__sparc__)
114
115	void BufferedStackTrace::PopStackFrames(uptr count) {
116	CHECK_LT(count, size);
117	size -= count;
118	for (uptr i = 0; i < size; ++i) {
119	trace_buffer[i] = trace_buffer[i + count];
120	}
121	}
122
123	static uptr Distance(uptr a, uptr b) { return a < b ? b - a : a - b; }
124
125	uptr BufferedStackTrace::LocatePcInTrace(uptr pc) {
126	uptr best = 0;
127	for (uptr i = 1; i < size; ++i) {
128	if (Distance(trace[i], pc) < Distance(trace[best], pc)) best = i;
129	}
130	return best;
131	}
132
133	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_stacktrace_libcdep.cpp created+159

...	@@ -0,0 +1,159 @@
1	//===-- sanitizer_stacktrace_libcdep.cpp ----------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is shared between AddressSanitizer and ThreadSanitizer
10	// run-time libraries.
11	//===----------------------------------------------------------------------===//
12
13	#include "sanitizer_common.h"
14	#include "sanitizer_placement_new.h"
15	#include "sanitizer_stacktrace.h"
16	#include "sanitizer_stacktrace_printer.h"
17	#include "sanitizer_symbolizer.h"
18
19	namespace __sanitizer {
20
21	void StackTrace::Print() const {
22	if (trace == nullptr \|\| size == 0) {
23	Printf(" <empty stack>\n\n");
24	return;
25	}
26	InternalScopedString frame_desc(GetPageSizeCached() * 2);
27	InternalScopedString dedup_token(GetPageSizeCached());
28	int dedup_frames = common_flags()->dedup_token_length;
29	uptr frame_num = 0;
30	for (uptr i = 0; i < size && trace[i]; i++) {
31	// PCs in stack traces are actually the return addresses, that is,
32	// addresses of the next instructions after the call.
33	uptr pc = GetPreviousInstructionPc(trace[i]);
34	SymbolizedStack *frames = Symbolizer::GetOrInit()->SymbolizePC(pc);
35	CHECK(frames);
36	for (SymbolizedStack *cur = frames; cur; cur = cur->next) {
37	frame_desc.clear();
38	RenderFrame(&frame_desc, common_flags()->stack_trace_format, frame_num++,
39	cur->info, common_flags()->symbolize_vs_style,
40	common_flags()->strip_path_prefix);
41	Printf("%s\n", frame_desc.data());
42	if (dedup_frames-- > 0) {
43	if (dedup_token.length())
44	dedup_token.append("--");
45	if (cur->info.function != nullptr)
46	dedup_token.append(cur->info.function);
47	}
48	}
49	frames->ClearAll();
50	}
51	// Always print a trailing empty line after stack trace.
52	Printf("\n");
53	if (dedup_token.length())
54	Printf("DEDUP_TOKEN: %s\n", dedup_token.data());
55	}
56
57	void BufferedStackTrace::Unwind(u32 max_depth, uptr pc, uptr bp, void *context,
58	uptr stack_top, uptr stack_bottom,
59	bool request_fast_unwind) {
60	// Ensures all call sites get what they requested.
61	CHECK_EQ(request_fast_unwind, WillUseFastUnwind(request_fast_unwind));
62	top_frame_bp = (max_depth > 0) ? bp : 0;
63	// Avoid doing any work for small max_depth.
64	if (max_depth == 0) {
65	size = 0;
66	return;
67	}
68	if (max_depth == 1) {
69	size = 1;
70	trace_buffer[0] = pc;
71	return;
72	}
73	if (!WillUseFastUnwind(request_fast_unwind)) {
74	#if SANITIZER_CAN_SLOW_UNWIND
75	if (context)
76	UnwindSlow(pc, context, max_depth);
77	else
78	UnwindSlow(pc, max_depth);
79	#else
80	UNREACHABLE("slow unwind requested but not available");
81	#endif
82	} else {
83	UnwindFast(pc, bp, stack_top, stack_bottom, max_depth);
84	}
85	}
86
87	static int GetModuleAndOffsetForPc(uptr pc, char *module_name,
88	uptr module_name_len, uptr *pc_offset) {
89	const char *found_module_name = nullptr;
90	bool ok = Symbolizer::GetOrInit()->GetModuleNameAndOffsetForPC(
91	pc, &found_module_name, pc_offset);
92
93	if (!ok) return false;
94
95	if (module_name && module_name_len) {
96	internal_strncpy(module_name, found_module_name, module_name_len);
97	module_name[module_name_len - 1] = '\x00';
98	}
99	return true;
100	}
101
102	} // namespace __sanitizer
103	using namespace __sanitizer;
104
105	extern "C" {
106	SANITIZER_INTERFACE_ATTRIBUTE
107	void __sanitizer_symbolize_pc(uptr pc, const char fmt, char out_buf,
108	uptr out_buf_size) {
109	if (!out_buf_size) return;
110	pc = StackTrace::GetPreviousInstructionPc(pc);
111	SymbolizedStack *frame = Symbolizer::GetOrInit()->SymbolizePC(pc);
112	if (!frame) {
113	internal_strncpy(out_buf, "<can't symbolize>", out_buf_size);
114	out_buf[out_buf_size - 1] = 0;
115	return;
116	}
117	InternalScopedString frame_desc(GetPageSizeCached());
118	uptr frame_num = 0;
119	// Reserve one byte for the final 0.
120	char *out_end = out_buf + out_buf_size - 1;
121	for (SymbolizedStack *cur = frame; cur && out_buf < out_end;
122	cur = cur->next) {
123	frame_desc.clear();
124	RenderFrame(&frame_desc, fmt, frame_num++, cur->info,
125	common_flags()->symbolize_vs_style,
126	common_flags()->strip_path_prefix);
127	if (!frame_desc.length())
128	continue;
129	// Reserve one byte for the terminating 0.
130	uptr n = out_end - out_buf - 1;
131	internal_strncpy(out_buf, frame_desc.data(), n);
132	out_buf += __sanitizer::Min<uptr>(n, frame_desc.length());
133	*out_buf++ = 0;
134	}
135	CHECK(out_buf <= out_end);
136	*out_buf = 0;
137	}
138
139	SANITIZER_INTERFACE_ATTRIBUTE
140	void __sanitizer_symbolize_global(uptr data_addr, const char *fmt,
141	char *out_buf, uptr out_buf_size) {
142	if (!out_buf_size) return;
143	out_buf[0] = 0;
144	DataInfo DI;
145	if (!Symbolizer::GetOrInit()->SymbolizeData(data_addr, &DI)) return;
146	InternalScopedString data_desc(GetPageSizeCached());
147	RenderData(&data_desc, fmt, &DI, common_flags()->strip_path_prefix);
148	internal_strncpy(out_buf, data_desc.data(), out_buf_size);
149	out_buf[out_buf_size - 1] = 0;
150	}
151
152	SANITIZER_INTERFACE_ATTRIBUTE
153	int __sanitizer_get_module_and_offset_for_pc(uptr pc, char *module_name,
154	uptr module_name_len,
155	uptr *pc_offset) {
156	return __sanitizer::GetModuleAndOffsetForPc(pc, module_name, module_name_len,
157	pc_offset);
158	}
159	} // extern "C"

lib/tsan/sanitizer_common/sanitizer_stacktrace_printer.cpp created+263

...	@@ -0,0 +1,263 @@
1	//===-- sanitizer_common.cpp ----------------------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is shared between sanitizers' run-time libraries.
10	//
11	//===----------------------------------------------------------------------===//
12
13	#include "sanitizer_stacktrace_printer.h"
14	#include "sanitizer_file.h"
15	#include "sanitizer_fuchsia.h"
16
17	namespace __sanitizer {
18
19	// sanitizer_symbolizer_markup.cpp implements these differently.
20	#if !SANITIZER_SYMBOLIZER_MARKUP
21
22	static const char StripFunctionName(const char function, const char *prefix) {
23	if (!function) return nullptr;
24	if (!prefix) return function;
25	uptr prefix_len = internal_strlen(prefix);
26	if (0 == internal_strncmp(function, prefix, prefix_len))
27	return function + prefix_len;
28	return function;
29	}
30
31	static const char DemangleFunctionName(const char function) {
32	if (!function) return nullptr;
33
34	// NetBSD uses indirection for old threading functions for historical reasons
35	// The mangled names are internal implementation detail and should not be
36	// exposed even in backtraces.
37	#if SANITIZER_NETBSD
38	if (!internal_strcmp(function, "__libc_mutex_init"))
39	return "pthread_mutex_init";
40	if (!internal_strcmp(function, "__libc_mutex_lock"))
41	return "pthread_mutex_lock";
42	if (!internal_strcmp(function, "__libc_mutex_trylock"))
43	return "pthread_mutex_trylock";
44	if (!internal_strcmp(function, "__libc_mutex_unlock"))
45	return "pthread_mutex_unlock";
46	if (!internal_strcmp(function, "__libc_mutex_destroy"))
47	return "pthread_mutex_destroy";
48	if (!internal_strcmp(function, "__libc_mutexattr_init"))
49	return "pthread_mutexattr_init";
50	if (!internal_strcmp(function, "__libc_mutexattr_settype"))
51	return "pthread_mutexattr_settype";
52	if (!internal_strcmp(function, "__libc_mutexattr_destroy"))
53	return "pthread_mutexattr_destroy";
54	if (!internal_strcmp(function, "__libc_cond_init"))
55	return "pthread_cond_init";
56	if (!internal_strcmp(function, "__libc_cond_signal"))
57	return "pthread_cond_signal";
58	if (!internal_strcmp(function, "__libc_cond_broadcast"))
59	return "pthread_cond_broadcast";
60	if (!internal_strcmp(function, "__libc_cond_wait"))
61	return "pthread_cond_wait";
62	if (!internal_strcmp(function, "__libc_cond_timedwait"))
63	return "pthread_cond_timedwait";
64	if (!internal_strcmp(function, "__libc_cond_destroy"))
65	return "pthread_cond_destroy";
66	if (!internal_strcmp(function, "__libc_rwlock_init"))
67	return "pthread_rwlock_init";
68	if (!internal_strcmp(function, "__libc_rwlock_rdlock"))
69	return "pthread_rwlock_rdlock";
70	if (!internal_strcmp(function, "__libc_rwlock_wrlock"))
71	return "pthread_rwlock_wrlock";
72	if (!internal_strcmp(function, "__libc_rwlock_tryrdlock"))
73	return "pthread_rwlock_tryrdlock";
74	if (!internal_strcmp(function, "__libc_rwlock_trywrlock"))
75	return "pthread_rwlock_trywrlock";
76	if (!internal_strcmp(function, "__libc_rwlock_unlock"))
77	return "pthread_rwlock_unlock";
78	if (!internal_strcmp(function, "__libc_rwlock_destroy"))
79	return "pthread_rwlock_destroy";
80	if (!internal_strcmp(function, "__libc_thr_keycreate"))
81	return "pthread_key_create";
82	if (!internal_strcmp(function, "__libc_thr_setspecific"))
83	return "pthread_setspecific";
84	if (!internal_strcmp(function, "__libc_thr_getspecific"))
85	return "pthread_getspecific";
86	if (!internal_strcmp(function, "__libc_thr_keydelete"))
87	return "pthread_key_delete";
88	if (!internal_strcmp(function, "__libc_thr_once"))
89	return "pthread_once";
90	if (!internal_strcmp(function, "__libc_thr_self"))
91	return "pthread_self";
92	if (!internal_strcmp(function, "__libc_thr_exit"))
93	return "pthread_exit";
94	if (!internal_strcmp(function, "__libc_thr_setcancelstate"))
95	return "pthread_setcancelstate";
96	if (!internal_strcmp(function, "__libc_thr_equal"))
97	return "pthread_equal";
98	if (!internal_strcmp(function, "__libc_thr_curcpu"))
99	return "pthread_curcpu_np";
100	if (!internal_strcmp(function, "__libc_thr_sigsetmask"))
101	return "pthread_sigmask";
102	#endif
103
104	return function;
105	}
106
107	static const char kDefaultFormat[] = " #%n %p %F %L";
108
109	void RenderFrame(InternalScopedString buffer, const char format, int frame_no,
110	const AddressInfo &info, bool vs_style,
111	const char strip_path_prefix, const char strip_func_prefix) {
112	if (0 == internal_strcmp(format, "DEFAULT"))
113	format = kDefaultFormat;
114	for (const char p = format; p != '\0'; p++) {
115	if (*p != '%') {
116	buffer->append("%c", *p);
117	continue;
118	}
119	p++;
120	switch (*p) {
121	case '%':
122	buffer->append("%%");
123	break;
124	// Frame number and all fields of AddressInfo structure.
125	case 'n':
126	buffer->append("%zu", frame_no);
127	break;
128	case 'p':
129	buffer->append("0x%zx", info.address);
130	break;
131	case 'm':
132	buffer->append("%s", StripPathPrefix(info.module, strip_path_prefix));
133	break;
134	case 'o':
135	buffer->append("0x%zx", info.module_offset);
136	break;
137	case 'f':
138	buffer->append("%s",
139	DemangleFunctionName(
140	StripFunctionName(info.function, strip_func_prefix)));
141	break;
142	case 'q':
143	buffer->append("0x%zx", info.function_offset != AddressInfo::kUnknown
144	? info.function_offset
145	: 0x0);
146	break;
147	case 's':
148	buffer->append("%s", StripPathPrefix(info.file, strip_path_prefix));
149	break;
150	case 'l':
151	buffer->append("%d", info.line);
152	break;
153	case 'c':
154	buffer->append("%d", info.column);
155	break;
156	// Smarter special cases.
157	case 'F':
158	// Function name and offset, if file is unknown.
159	if (info.function) {
160	buffer->append("in %s",
161	DemangleFunctionName(
162	StripFunctionName(info.function, strip_func_prefix)));
163	if (!info.file && info.function_offset != AddressInfo::kUnknown)
164	buffer->append("+0x%zx", info.function_offset);
165	}
166	break;
167	case 'S':
168	// File/line information.
169	RenderSourceLocation(buffer, info.file, info.line, info.column, vs_style,
170	strip_path_prefix);
171	break;
172	case 'L':
173	// Source location, or module location.
174	if (info.file) {
175	RenderSourceLocation(buffer, info.file, info.line, info.column,
176	vs_style, strip_path_prefix);
177	} else if (info.module) {
178	RenderModuleLocation(buffer, info.module, info.module_offset,
179	info.module_arch, strip_path_prefix);
180	} else {
181	buffer->append("(<unknown module>)");
182	}
183	break;
184	case 'M':
185	// Module basename and offset, or PC.
186	if (info.address & kExternalPCBit)
187	{} // There PCs are not meaningful.
188	else if (info.module)
189	// Always strip the module name for %M.
190	RenderModuleLocation(buffer, StripModuleName(info.module),
191	info.module_offset, info.module_arch, "");
192	else
193	buffer->append("(%p)", (void *)info.address);
194	break;
195	default:
196	Report("Unsupported specifier in stack frame format: %c (0x%zx)!\n", *p,
197	*p);
198	Die();
199	}
200	}
201	}
202
203	void RenderData(InternalScopedString buffer, const char format,
204	const DataInfo DI, const char strip_path_prefix) {
205	for (const char p = format; p != '\0'; p++) {
206	if (*p != '%') {
207	buffer->append("%c", *p);
208	continue;
209	}
210	p++;
211	switch (*p) {
212	case '%':
213	buffer->append("%%");
214	break;
215	case 's':
216	buffer->append("%s", StripPathPrefix(DI->file, strip_path_prefix));
217	break;
218	case 'l':
219	buffer->append("%d", DI->line);
220	break;
221	case 'g':
222	buffer->append("%s", DI->name);
223	break;
224	default:
225	Report("Unsupported specifier in stack frame format: %c (0x%zx)!\n", *p,
226	*p);
227	Die();
228	}
229	}
230	}
231
232	#endif // !SANITIZER_SYMBOLIZER_MARKUP
233
234	void RenderSourceLocation(InternalScopedString buffer, const char file,
235	int line, int column, bool vs_style,
236	const char *strip_path_prefix) {
237	if (vs_style && line > 0) {
238	buffer->append("%s(%d", StripPathPrefix(file, strip_path_prefix), line);
239	if (column > 0)
240	buffer->append(",%d", column);
241	buffer->append(")");
242	return;
243	}
244
245	buffer->append("%s", StripPathPrefix(file, strip_path_prefix));
246	if (line > 0) {
247	buffer->append(":%d", line);
248	if (column > 0)
249	buffer->append(":%d", column);
250	}
251	}
252
253	void RenderModuleLocation(InternalScopedString buffer, const char module,
254	uptr offset, ModuleArch arch,
255	const char *strip_path_prefix) {
256	buffer->append("(%s", StripPathPrefix(module, strip_path_prefix));
257	if (arch != kModuleArchUnknown) {
258	buffer->append(":%s", ModuleArchToString(arch));
259	}
260	buffer->append("+0x%zx)", offset);
261	}
262
263	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_stacktrace_sparc.cpp created+85

...	@@ -0,0 +1,85 @@
1	//===-- sanitizer_stacktrace_sparc.cpp ------------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is shared between AddressSanitizer and ThreadSanitizer
10	// run-time libraries.
11	//
12	// Implemention of fast stack unwinding for Sparc.
13	//===----------------------------------------------------------------------===//
14
15	#if defined(__sparc__)
16
17	#if defined(__arch64__) \|\| defined(__sparcv9)
18	#define STACK_BIAS 2047
19	#else
20	#define STACK_BIAS 0
21	#endif
22
23	#include "sanitizer_common.h"
24	#include "sanitizer_stacktrace.h"
25
26	namespace __sanitizer {
27
28	void BufferedStackTrace::UnwindFast(uptr pc, uptr bp, uptr stack_top,
29	uptr stack_bottom, u32 max_depth) {
30	// TODO(yln): add arg sanity check for stack_top/stack_bottom
31	CHECK_GE(max_depth, 2);
32	const uptr kPageSize = GetPageSizeCached();
33	#if defined(__GNUC__)
34	// __builtin_return_address returns the address of the call instruction
35	// on the SPARC and not the return address, so we need to compensate.
36	trace_buffer[0] = GetNextInstructionPc(pc);
37	#else
38	trace_buffer[0] = pc;
39	#endif
40	size = 1;
41	if (stack_top < 4096) return; // Sanity check for stack top.
42	// Flush register windows to memory
43	#if defined(__sparc_v9__) \|\| defined(__sparcv9__) \|\| defined(__sparcv9)
44	asm volatile("flushw" ::: "memory");
45	#else
46	asm volatile("ta 3" ::: "memory");
47	#endif
48	// On the SPARC, the return address is not in the frame, it is in a
49	// register. There is no way to access it off of the current frame
50	// pointer, but it can be accessed off the previous frame pointer by
51	// reading the value from the register window save area.
52	uptr prev_bp = GET_CURRENT_FRAME();
53	uptr next_bp = prev_bp;
54	unsigned int i = 0;
55	while (next_bp != bp && IsAligned(next_bp, sizeof(uhwptr)) && i++ < 8) {
56	prev_bp = next_bp;
57	next_bp = (uptr)((uhwptr *)next_bp)[14] + STACK_BIAS;
58	}
59	if (next_bp == bp)
60	bp = prev_bp;
61	// Lowest possible address that makes sense as the next frame pointer.
62	// Goes up as we walk the stack.
63	uptr bottom = stack_bottom;
64	// Avoid infinite loop when frame == frame[0] by using frame > prev_frame.
65	while (IsValidFrame(bp, stack_top, bottom) && IsAligned(bp, sizeof(uhwptr)) &&
66	size < max_depth) {
67	uhwptr pc1 = ((uhwptr *)bp)[15];
68	// Let's assume that any pointer in the 0th page is invalid and
69	// stop unwinding here. If we're adding support for a platform
70	// where this isn't true, we need to reconsider this check.
71	if (pc1 < kPageSize)
72	break;
73	if (pc1 != pc) {
74	// %o7 contains the address of the call instruction and not the
75	// return address, so we need to compensate.
76	trace_buffer[size++] = GetNextInstructionPc((uptr)pc1);
77	}
78	bottom = bp;
79	bp = (uptr)((uhwptr *)bp)[14] + STACK_BIAS;
80	}
81	}
82
83	} // namespace __sanitizer
84
85	#endif // !defined(__sparc__)

lib/tsan/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cpp created+573

...	@@ -0,0 +1,573 @@
1	//===-- sanitizer_stoptheworld_linux_libcdep.cpp --------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// See sanitizer_stoptheworld.h for details.
10	// This implementation was inspired by Markus Gutschke's linuxthreads.cc.
11	//
12	//===----------------------------------------------------------------------===//
13
14	#include "sanitizer_platform.h"
15
16	#if SANITIZER_LINUX && (defined(__x86_64__) \|\| defined(__mips__) \|\| \
17	defined(__aarch64__) \|\| defined(__powerpc64__) \|\| \
18	defined(__s390__) \|\| defined(__i386__) \|\| \
19	defined(__arm__))
20
21	#include "sanitizer_stoptheworld.h"
22
23	#include "sanitizer_platform_limits_posix.h"
24	#include "sanitizer_atomic.h"
25
26	#include <errno.h>
27	#include <sched.h> // for CLONE_* definitions
28	#include <stddef.h>
29	#include <sys/prctl.h> // for PR_* definitions
30	#include <sys/ptrace.h> // for PTRACE_* definitions
31	#include <sys/types.h> // for pid_t
32	#include <sys/uio.h> // for iovec
33	#include <elf.h> // for NT_PRSTATUS
34	#if defined(__aarch64__) && !SANITIZER_ANDROID
35	// GLIBC 2.20+ sys/user does not include asm/ptrace.h
36	# include <asm/ptrace.h>
37	#endif
38	#include <sys/user.h> // for user_regs_struct
39	#if SANITIZER_ANDROID && SANITIZER_MIPS
40	# include <asm/reg.h> // for mips SP register in sys/user.h
41	#endif
42	#include <sys/wait.h> // for signal-related stuff
43
44	#ifdef sa_handler
45	# undef sa_handler
46	#endif
47
48	#ifdef sa_sigaction
49	# undef sa_sigaction
50	#endif
51
52	#include "sanitizer_common.h"
53	#include "sanitizer_flags.h"
54	#include "sanitizer_libc.h"
55	#include "sanitizer_linux.h"
56	#include "sanitizer_mutex.h"
57	#include "sanitizer_placement_new.h"
58
59	// Sufficiently old kernel headers don't provide this value, but we can still
60	// call prctl with it. If the runtime kernel is new enough, the prctl call will
61	// have the desired effect; if the kernel is too old, the call will error and we
62	// can ignore said error.
63	#ifndef PR_SET_PTRACER
64	#define PR_SET_PTRACER 0x59616d61
65	#endif
66
67	// This module works by spawning a Linux task which then attaches to every
68	// thread in the caller process with ptrace. This suspends the threads, and
69	// PTRACE_GETREGS can then be used to obtain their register state. The callback
70	// supplied to StopTheWorld() is run in the tracer task while the threads are
71	// suspended.
72	// The tracer task must be placed in a different thread group for ptrace to
73	// work, so it cannot be spawned as a pthread. Instead, we use the low-level
74	// clone() interface (we want to share the address space with the caller
75	// process, so we prefer clone() over fork()).
76	//
77	// We don't use any libc functions, relying instead on direct syscalls. There
78	// are two reasons for this:
79	// 1. calling a library function while threads are suspended could cause a
80	// deadlock, if one of the treads happens to be holding a libc lock;
81	// 2. it's generally not safe to call libc functions from the tracer task,
82	// because clone() does not set up a thread-local storage for it. Any
83	// thread-local variables used by libc will be shared between the tracer task
84	// and the thread which spawned it.
85
86	namespace __sanitizer {
87
88	class SuspendedThreadsListLinux : public SuspendedThreadsList {
89	public:
90	SuspendedThreadsListLinux() { thread_ids_.reserve(1024); }
91
92	tid_t GetThreadID(uptr index) const;
93	uptr ThreadCount() const;
94	bool ContainsTid(tid_t thread_id) const;
95	void Append(tid_t tid);
96
97	PtraceRegistersStatus GetRegistersAndSP(uptr index, uptr *buffer,
98	uptr *sp) const;
99	uptr RegisterCount() const;
100
101	private:
102	InternalMmapVector<tid_t> thread_ids_;
103	};
104
105	// Structure for passing arguments into the tracer thread.
106	struct TracerThreadArgument {
107	StopTheWorldCallback callback;
108	void *callback_argument;
109	// The tracer thread waits on this mutex while the parent finishes its
110	// preparations.
111	BlockingMutex mutex;
112	// Tracer thread signals its completion by setting done.
113	atomic_uintptr_t done;
114	uptr parent_pid;
115	};
116
117	// This class handles thread suspending/unsuspending in the tracer thread.
118	class ThreadSuspender {
119	public:
120	explicit ThreadSuspender(pid_t pid, TracerThreadArgument *arg)
121	: arg(arg)
122	, pid_(pid) {
123	CHECK_GE(pid, 0);
124	}
125	bool SuspendAllThreads();
126	void ResumeAllThreads();
127	void KillAllThreads();
128	SuspendedThreadsListLinux &suspended_threads_list() {
129	return suspended_threads_list_;
130	}
131	TracerThreadArgument *arg;
132	private:
133	SuspendedThreadsListLinux suspended_threads_list_;
134	pid_t pid_;
135	bool SuspendThread(tid_t thread_id);
136	};
137
138	bool ThreadSuspender::SuspendThread(tid_t tid) {
139	// Are we already attached to this thread?
140	// Currently this check takes linear time, however the number of threads is
141	// usually small.
142	if (suspended_threads_list_.ContainsTid(tid)) return false;
143	int pterrno;
144	if (internal_iserror(internal_ptrace(PTRACE_ATTACH, tid, nullptr, nullptr),
145	&pterrno)) {
146	// Either the thread is dead, or something prevented us from attaching.
147	// Log this event and move on.
148	VReport(1, "Could not attach to thread %zu (errno %d).\n", (uptr)tid,
149	pterrno);
150	return false;
151	} else {
152	VReport(2, "Attached to thread %zu.\n", (uptr)tid);
153	// The thread is not guaranteed to stop before ptrace returns, so we must
154	// wait on it. Note: if the thread receives a signal concurrently,
155	// we can get notification about the signal before notification about stop.
156	// In such case we need to forward the signal to the thread, otherwise
157	// the signal will be missed (as we do PTRACE_DETACH with arg=0) and
158	// any logic relying on signals will break. After forwarding we need to
159	// continue to wait for stopping, because the thread is not stopped yet.
160	// We do ignore delivery of SIGSTOP, because we want to make stop-the-world
161	// as invisible as possible.
162	for (;;) {
163	int status;
164	uptr waitpid_status;
165	HANDLE_EINTR(waitpid_status, internal_waitpid(tid, &status, __WALL));
166	int wperrno;
167	if (internal_iserror(waitpid_status, &wperrno)) {
168	// Got a ECHILD error. I don't think this situation is possible, but it
169	// doesn't hurt to report it.
170	VReport(1, "Waiting on thread %zu failed, detaching (errno %d).\n",
171	(uptr)tid, wperrno);
172	internal_ptrace(PTRACE_DETACH, tid, nullptr, nullptr);
173	return false;
174	}
175	if (WIFSTOPPED(status) && WSTOPSIG(status) != SIGSTOP) {
176	internal_ptrace(PTRACE_CONT, tid, nullptr,
177	(void*)(uptr)WSTOPSIG(status));
178	continue;
179	}
180	break;
181	}
182	suspended_threads_list_.Append(tid);
183	return true;
184	}
185	}
186
187	void ThreadSuspender::ResumeAllThreads() {
188	for (uptr i = 0; i < suspended_threads_list_.ThreadCount(); i++) {
189	pid_t tid = suspended_threads_list_.GetThreadID(i);
190	int pterrno;
191	if (!internal_iserror(internal_ptrace(PTRACE_DETACH, tid, nullptr, nullptr),
192	&pterrno)) {
193	VReport(2, "Detached from thread %d.\n", tid);
194	} else {
195	// Either the thread is dead, or we are already detached.
196	// The latter case is possible, for instance, if this function was called
197	// from a signal handler.
198	VReport(1, "Could not detach from thread %d (errno %d).\n", tid, pterrno);
199	}
200	}
201	}
202
203	void ThreadSuspender::KillAllThreads() {
204	for (uptr i = 0; i < suspended_threads_list_.ThreadCount(); i++)
205	internal_ptrace(PTRACE_KILL, suspended_threads_list_.GetThreadID(i),
206	nullptr, nullptr);
207	}
208
209	bool ThreadSuspender::SuspendAllThreads() {
210	ThreadLister thread_lister(pid_);
211	bool retry = true;
212	InternalMmapVector<tid_t> threads;
213	threads.reserve(128);
214	for (int i = 0; i < 30 && retry; ++i) {
215	retry = false;
216	switch (thread_lister.ListThreads(&threads)) {
217	case ThreadLister::Error:
218	ResumeAllThreads();
219	return false;
220	case ThreadLister::Incomplete:
221	retry = true;
222	break;
223	case ThreadLister::Ok:
224	break;
225	}
226	for (tid_t tid : threads) {
227	if (SuspendThread(tid))
228	retry = true;
229	}
230	}
231	return suspended_threads_list_.ThreadCount();
232	}
233
234	// Pointer to the ThreadSuspender instance for use in signal handler.
235	static ThreadSuspender *thread_suspender_instance = nullptr;
236
237	// Synchronous signals that should not be blocked.
238	static const int kSyncSignals[] = { SIGABRT, SIGILL, SIGFPE, SIGSEGV, SIGBUS,
239	SIGXCPU, SIGXFSZ };
240
241	static void TracerThreadDieCallback() {
242	// Generally a call to Die() in the tracer thread should be fatal to the
243	// parent process as well, because they share the address space.
244	// This really only works correctly if all the threads are suspended at this
245	// point. So we correctly handle calls to Die() from within the callback, but
246	// not those that happen before or after the callback. Hopefully there aren't
247	// a lot of opportunities for that to happen...
248	ThreadSuspender *inst = thread_suspender_instance;
249	if (inst && stoptheworld_tracer_pid == internal_getpid()) {
250	inst->KillAllThreads();
251	thread_suspender_instance = nullptr;
252	}
253	}
254
255	// Signal handler to wake up suspended threads when the tracer thread dies.
256	static void TracerThreadSignalHandler(int signum, __sanitizer_siginfo *siginfo,
257	void *uctx) {
258	SignalContext ctx(siginfo, uctx);
259	Printf("Tracer caught signal %d: addr=0x%zx pc=0x%zx sp=0x%zx\n", signum,
260	ctx.addr, ctx.pc, ctx.sp);
261	ThreadSuspender *inst = thread_suspender_instance;
262	if (inst) {
263	if (signum == SIGABRT)
264	inst->KillAllThreads();
265	else
266	inst->ResumeAllThreads();
267	RAW_CHECK(RemoveDieCallback(TracerThreadDieCallback));
268	thread_suspender_instance = nullptr;
269	atomic_store(&inst->arg->done, 1, memory_order_relaxed);
270	}
271	internal__exit((signum == SIGABRT) ? 1 : 2);
272	}
273
274	// Size of alternative stack for signal handlers in the tracer thread.
275	static const int kHandlerStackSize = 8192;
276
277	// This function will be run as a cloned task.
278	static int TracerThread(void* argument) {
279	TracerThreadArgument *tracer_thread_argument =
280	(TracerThreadArgument *)argument;
281
282	internal_prctl(PR_SET_PDEATHSIG, SIGKILL, 0, 0, 0);
283	// Check if parent is already dead.
284	if (internal_getppid() != tracer_thread_argument->parent_pid)
285	internal__exit(4);
286
287	// Wait for the parent thread to finish preparations.
288	tracer_thread_argument->mutex.Lock();
289	tracer_thread_argument->mutex.Unlock();
290
291	RAW_CHECK(AddDieCallback(TracerThreadDieCallback));
292
293	ThreadSuspender thread_suspender(internal_getppid(), tracer_thread_argument);
294	// Global pointer for the signal handler.
295	thread_suspender_instance = &thread_suspender;
296
297	// Alternate stack for signal handling.
298	InternalMmapVector<char> handler_stack_memory(kHandlerStackSize);
299	stack_t handler_stack;
300	internal_memset(&handler_stack, 0, sizeof(handler_stack));
301	handler_stack.ss_sp = handler_stack_memory.data();
302	handler_stack.ss_size = kHandlerStackSize;
303	internal_sigaltstack(&handler_stack, nullptr);
304
305	// Install our handler for synchronous signals. Other signals should be
306	// blocked by the mask we inherited from the parent thread.
307	for (uptr i = 0; i < ARRAY_SIZE(kSyncSignals); i++) {
308	__sanitizer_sigaction act;
309	internal_memset(&act, 0, sizeof(act));
310	act.sigaction = TracerThreadSignalHandler;
311	act.sa_flags = SA_ONSTACK \| SA_SIGINFO;
312	internal_sigaction_norestorer(kSyncSignals[i], &act, 0);
313	}
314
315	int exit_code = 0;
316	if (!thread_suspender.SuspendAllThreads()) {
317	VReport(1, "Failed suspending threads.\n");
318	exit_code = 3;
319	} else {
320	tracer_thread_argument->callback(thread_suspender.suspended_threads_list(),
321	tracer_thread_argument->callback_argument);
322	thread_suspender.ResumeAllThreads();
323	exit_code = 0;
324	}
325	RAW_CHECK(RemoveDieCallback(TracerThreadDieCallback));
326	thread_suspender_instance = nullptr;
327	atomic_store(&tracer_thread_argument->done, 1, memory_order_relaxed);
328	return exit_code;
329	}
330
331	class ScopedStackSpaceWithGuard {
332	public:
333	explicit ScopedStackSpaceWithGuard(uptr stack_size) {
334	stack_size_ = stack_size;
335	guard_size_ = GetPageSizeCached();
336	// FIXME: Omitting MAP_STACK here works in current kernels but might break
337	// in the future.
338	guard_start_ = (uptr)MmapOrDie(stack_size_ + guard_size_,
339	"ScopedStackWithGuard");
340	CHECK(MprotectNoAccess((uptr)guard_start_, guard_size_));
341	}
342	~ScopedStackSpaceWithGuard() {
343	UnmapOrDie((void *)guard_start_, stack_size_ + guard_size_);
344	}
345	void *Bottom() const {
346	return (void *)(guard_start_ + stack_size_ + guard_size_);
347	}
348
349	private:
350	uptr stack_size_;
351	uptr guard_size_;
352	uptr guard_start_;
353	};
354
355	// We have a limitation on the stack frame size, so some stuff had to be moved
356	// into globals.
357	static __sanitizer_sigset_t blocked_sigset;
358	static __sanitizer_sigset_t old_sigset;
359
360	class StopTheWorldScope {
361	public:
362	StopTheWorldScope() {
363	// Make this process dumpable. Processes that are not dumpable cannot be
364	// attached to.
365	process_was_dumpable_ = internal_prctl(PR_GET_DUMPABLE, 0, 0, 0, 0);
366	if (!process_was_dumpable_)
367	internal_prctl(PR_SET_DUMPABLE, 1, 0, 0, 0);
368	}
369
370	~StopTheWorldScope() {
371	// Restore the dumpable flag.
372	if (!process_was_dumpable_)
373	internal_prctl(PR_SET_DUMPABLE, 0, 0, 0, 0);
374	}
375
376	private:
377	int process_was_dumpable_;
378	};
379
380	// When sanitizer output is being redirected to file (i.e. by using log_path),
381	// the tracer should write to the parent's log instead of trying to open a new
382	// file. Alert the logging code to the fact that we have a tracer.
383	struct ScopedSetTracerPID {
384	explicit ScopedSetTracerPID(uptr tracer_pid) {
385	stoptheworld_tracer_pid = tracer_pid;
386	stoptheworld_tracer_ppid = internal_getpid();
387	}
388	~ScopedSetTracerPID() {
389	stoptheworld_tracer_pid = 0;
390	stoptheworld_tracer_ppid = 0;
391	}
392	};
393
394	void StopTheWorld(StopTheWorldCallback callback, void *argument) {
395	StopTheWorldScope in_stoptheworld;
396	// Prepare the arguments for TracerThread.
397	struct TracerThreadArgument tracer_thread_argument;
398	tracer_thread_argument.callback = callback;
399	tracer_thread_argument.callback_argument = argument;
400	tracer_thread_argument.parent_pid = internal_getpid();
401	atomic_store(&tracer_thread_argument.done, 0, memory_order_relaxed);
402	const uptr kTracerStackSize = 2 * 1024 * 1024;
403	ScopedStackSpaceWithGuard tracer_stack(kTracerStackSize);
404	// Block the execution of TracerThread until after we have set ptrace
405	// permissions.
406	tracer_thread_argument.mutex.Lock();
407	// Signal handling story.
408	// We don't want async signals to be delivered to the tracer thread,
409	// so we block all async signals before creating the thread. An async signal
410	// handler can temporary modify errno, which is shared with this thread.
411	// We ought to use pthread_sigmask here, because sigprocmask has undefined
412	// behavior in multithreaded programs. However, on linux sigprocmask is
413	// equivalent to pthread_sigmask with the exception that pthread_sigmask
414	// does not allow to block some signals used internally in pthread
415	// implementation. We are fine with blocking them here, we are really not
416	// going to pthread_cancel the thread.
417	// The tracer thread should not raise any synchronous signals. But in case it
418	// does, we setup a special handler for sync signals that properly kills the
419	// parent as well. Note: we don't pass CLONE_SIGHAND to clone, so handlers
420	// in the tracer thread won't interfere with user program. Double note: if a
421	// user does something along the lines of 'kill -11 pid', that can kill the
422	// process even if user setup own handler for SEGV.
423	// Thing to watch out for: this code should not change behavior of user code
424	// in any observable way. In particular it should not override user signal
425	// handlers.
426	internal_sigfillset(&blocked_sigset);
427	for (uptr i = 0; i < ARRAY_SIZE(kSyncSignals); i++)
428	internal_sigdelset(&blocked_sigset, kSyncSignals[i]);
429	int rv = internal_sigprocmask(SIG_BLOCK, &blocked_sigset, &old_sigset);
430	CHECK_EQ(rv, 0);
431	uptr tracer_pid = internal_clone(
432	TracerThread, tracer_stack.Bottom(),
433	CLONE_VM \| CLONE_FS \| CLONE_FILES \| CLONE_UNTRACED,
434	&tracer_thread_argument, nullptr /* parent_tidptr */,
435	nullptr /* newtls /, nullptr / child_tidptr */);
436	internal_sigprocmask(SIG_SETMASK, &old_sigset, 0);
437	int local_errno = 0;
438	if (internal_iserror(tracer_pid, &local_errno)) {
439	VReport(1, "Failed spawning a tracer thread (errno %d).\n", local_errno);
440	tracer_thread_argument.mutex.Unlock();
441	} else {
442	ScopedSetTracerPID scoped_set_tracer_pid(tracer_pid);
443	// On some systems we have to explicitly declare that we want to be traced
444	// by the tracer thread.
445	internal_prctl(PR_SET_PTRACER, tracer_pid, 0, 0, 0);
446	// Allow the tracer thread to start.
447	tracer_thread_argument.mutex.Unlock();
448	// NOTE: errno is shared between this thread and the tracer thread.
449	// internal_waitpid() may call syscall() which can access/spoil errno,
450	// so we can't call it now. Instead we for the tracer thread to finish using
451	// the spin loop below. Man page for sched_yield() says "In the Linux
452	// implementation, sched_yield() always succeeds", so let's hope it does not
453	// spoil errno. Note that this spin loop runs only for brief periods before
454	// the tracer thread has suspended us and when it starts unblocking threads.
455	while (atomic_load(&tracer_thread_argument.done, memory_order_relaxed) == 0)
456	sched_yield();
457	// Now the tracer thread is about to exit and does not touch errno,
458	// wait for it.
459	for (;;) {
460	uptr waitpid_status = internal_waitpid(tracer_pid, nullptr, __WALL);
461	if (!internal_iserror(waitpid_status, &local_errno))
462	break;
463	if (local_errno == EINTR)
464	continue;
465	VReport(1, "Waiting on the tracer thread failed (errno %d).\n",
466	local_errno);
467	break;
468	}
469	}
470	}
471
472	// Platform-specific methods from SuspendedThreadsList.
473	#if SANITIZER_ANDROID && defined(__arm__)
474	typedef pt_regs regs_struct;
475	#define REG_SP ARM_sp
476
477	#elif SANITIZER_LINUX && defined(__arm__)
478	typedef user_regs regs_struct;
479	#define REG_SP uregs[13]
480
481	#elif defined(__i386__) \|\| defined(__x86_64__)
482	typedef user_regs_struct regs_struct;
483	#if defined(__i386__)
484	#define REG_SP esp
485	#else
486	#define REG_SP rsp
487	#endif
488
489	#elif defined(__powerpc__) \|\| defined(__powerpc64__)
490	typedef pt_regs regs_struct;
491	#define REG_SP gpr[PT_R1]
492
493	#elif defined(__mips__)
494	typedef struct user regs_struct;
495	# if SANITIZER_ANDROID
496	# define REG_SP regs[EF_R29]
497	# else
498	# define REG_SP regs[EF_REG29]
499	# endif
500
501	#elif defined(__aarch64__)
502	typedef struct user_pt_regs regs_struct;
503	#define REG_SP sp
504	#define ARCH_IOVEC_FOR_GETREGSET
505
506	#elif defined(__s390__)
507	typedef _user_regs_struct regs_struct;
508	#define REG_SP gprs[15]
509	#define ARCH_IOVEC_FOR_GETREGSET
510
511	#else
512	#error "Unsupported architecture"
513	#endif // SANITIZER_ANDROID && defined(__arm__)
514
515	tid_t SuspendedThreadsListLinux::GetThreadID(uptr index) const {
516	CHECK_LT(index, thread_ids_.size());
517	return thread_ids_[index];
518	}
519
520	uptr SuspendedThreadsListLinux::ThreadCount() const {
521	return thread_ids_.size();
522	}
523
524	bool SuspendedThreadsListLinux::ContainsTid(tid_t thread_id) const {
525	for (uptr i = 0; i < thread_ids_.size(); i++) {
526	if (thread_ids_[i] == thread_id) return true;
527	}
528	return false;
529	}
530
531	void SuspendedThreadsListLinux::Append(tid_t tid) {
532	thread_ids_.push_back(tid);
533	}
534
535	PtraceRegistersStatus SuspendedThreadsListLinux::GetRegistersAndSP(
536	uptr index, uptr buffer, uptr sp) const {
537	pid_t tid = GetThreadID(index);
538	regs_struct regs;
539	int pterrno;
540	#ifdef ARCH_IOVEC_FOR_GETREGSET
541	struct iovec regset_io;
542	regset_io.iov_base = &regs;
543	regset_io.iov_len = sizeof(regs_struct);
544	bool isErr = internal_iserror(internal_ptrace(PTRACE_GETREGSET, tid,
545	(void)NT_PRSTATUS, (void)&regset_io),
546	&pterrno);
547	#else
548	bool isErr = internal_iserror(internal_ptrace(PTRACE_GETREGS, tid, nullptr,
549	&regs), &pterrno);
550	#endif
551	if (isErr) {
552	VReport(1, "Could not get registers from thread %d (errno %d).\n", tid,
553	pterrno);
554	// ESRCH means that the given thread is not suspended or already dead.
555	// Therefore it's unsafe to inspect its data (e.g. walk through stack) and
556	// we should notify caller about this.
557	return pterrno == ESRCH ? REGISTERS_UNAVAILABLE_FATAL
558	: REGISTERS_UNAVAILABLE;
559	}
560
561	*sp = regs.REG_SP;
562	internal_memcpy(buffer, &regs, sizeof(regs));
563	return REGISTERS_AVAILABLE;
564	}
565
566	uptr SuspendedThreadsListLinux::RegisterCount() const {
567	return sizeof(regs_struct) / sizeof(uptr);
568	}
569	} // namespace __sanitizer
570
571	#endif // SANITIZER_LINUX && (defined(__x86_64__) \|\| defined(__mips__)
572	// \|\| defined(__aarch64__) \|\| defined(__powerpc64__)
573	// \|\| defined(__s390__) \|\| defined(__i386__) \|\| defined(__arm__)

lib/tsan/sanitizer_common/sanitizer_stoptheworld_netbsd_libcdep.cpp created+364

...	@@ -0,0 +1,364 @@
1	//===-- sanitizer_stoptheworld_netbsd_libcdep.cpp -------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// See sanitizer_stoptheworld.h for details.
10	// This implementation was inspired by Markus Gutschke's linuxthreads.cc.
11	//
12	// This is a NetBSD variation of Linux stoptheworld implementation
13	// See sanitizer_stoptheworld_linux_libcdep.cpp for code comments.
14	//
15	//===----------------------------------------------------------------------===//
16
17	#include "sanitizer_platform.h"
18
19	#if SANITIZER_NETBSD
20
21	#include "sanitizer_stoptheworld.h"
22
23	#include "sanitizer_atomic.h"
24	#include "sanitizer_platform_limits_posix.h"
25
26	#include <sys/types.h>
27
28	#include <sys/ptrace.h>
29	#include <sys/uio.h>
30	#include <sys/wait.h>
31
32	#include <machine/reg.h>
33
34	#include <elf.h>
35	#include <errno.h>
36	#include <sched.h>
37	#include <signal.h>
38	#include <stddef.h>
39
40	#define internal_sigaction_norestorer internal_sigaction
41
42	#include "sanitizer_common.h"
43	#include "sanitizer_flags.h"
44	#include "sanitizer_libc.h"
45	#include "sanitizer_linux.h"
46	#include "sanitizer_mutex.h"
47	#include "sanitizer_placement_new.h"
48
49	namespace __sanitizer {
50
51	class SuspendedThreadsListNetBSD : public SuspendedThreadsList {
52	public:
53	SuspendedThreadsListNetBSD() { thread_ids_.reserve(1024); }
54
55	tid_t GetThreadID(uptr index) const;
56	uptr ThreadCount() const;
57	bool ContainsTid(tid_t thread_id) const;
58	void Append(tid_t tid);
59
60	PtraceRegistersStatus GetRegistersAndSP(uptr index, uptr *buffer,
61	uptr *sp) const;
62	uptr RegisterCount() const;
63
64	private:
65	InternalMmapVector<tid_t> thread_ids_;
66	};
67
68	struct TracerThreadArgument {
69	StopTheWorldCallback callback;
70	void *callback_argument;
71	BlockingMutex mutex;
72	atomic_uintptr_t done;
73	uptr parent_pid;
74	};
75
76	class ThreadSuspender {
77	public:
78	explicit ThreadSuspender(pid_t pid, TracerThreadArgument *arg)
79	: arg(arg), pid_(pid) {
80	CHECK_GE(pid, 0);
81	}
82	bool SuspendAllThreads();
83	void ResumeAllThreads();
84	void KillAllThreads();
85	SuspendedThreadsListNetBSD &suspended_threads_list() {
86	return suspended_threads_list_;
87	}
88	TracerThreadArgument *arg;
89
90	private:
91	SuspendedThreadsListNetBSD suspended_threads_list_;
92	pid_t pid_;
93	};
94
95	void ThreadSuspender::ResumeAllThreads() {
96	int pterrno;
97	if (!internal_iserror(internal_ptrace(PT_DETACH, pid_, (void *)(uptr)1, 0),
98	&pterrno)) {
99	VReport(2, "Detached from process %d.\n", pid_);
100	} else {
101	VReport(1, "Could not detach from process %d (errno %d).\n", pid_, pterrno);
102	}
103	}
104
105	void ThreadSuspender::KillAllThreads() {
106	internal_ptrace(PT_KILL, pid_, nullptr, 0);
107	}
108
109	bool ThreadSuspender::SuspendAllThreads() {
110	int pterrno;
111	if (internal_iserror(internal_ptrace(PT_ATTACH, pid_, nullptr, 0),
112	&pterrno)) {
113	Printf("Could not attach to process %d (errno %d).\n", pid_, pterrno);
114	return false;
115	}
116
117	int status;
118	uptr waitpid_status;
119	HANDLE_EINTR(waitpid_status, internal_waitpid(pid_, &status, 0));
120
121	VReport(2, "Attached to process %d.\n", pid_);
122
123	#ifdef PT_LWPNEXT
124	struct ptrace_lwpstatus pl;
125	int op = PT_LWPNEXT;
126	#else
127	struct ptrace_lwpinfo pl;
128	int op = PT_LWPINFO;
129	#endif
130
131	pl.pl_lwpid = 0;
132
133	int val;
134	while ((val = ptrace(op, pid_, (void *)&pl, sizeof(pl))) != -1 &&
135	pl.pl_lwpid != 0) {
136	suspended_threads_list_.Append(pl.pl_lwpid);
137	VReport(2, "Appended thread %d in process %d.\n", pl.pl_lwpid, pid_);
138	}
139	return true;
140	}
141
142	// Pointer to the ThreadSuspender instance for use in signal handler.
143	static ThreadSuspender *thread_suspender_instance = nullptr;
144
145	// Synchronous signals that should not be blocked.
146	static const int kSyncSignals[] = {SIGABRT, SIGILL, SIGFPE, SIGSEGV,
147	SIGBUS, SIGXCPU, SIGXFSZ};
148
149	static void TracerThreadDieCallback() {
150	ThreadSuspender *inst = thread_suspender_instance;
151	if (inst && stoptheworld_tracer_pid == internal_getpid()) {
152	inst->KillAllThreads();
153	thread_suspender_instance = nullptr;
154	}
155	}
156
157	// Signal handler to wake up suspended threads when the tracer thread dies.
158	static void TracerThreadSignalHandler(int signum, __sanitizer_siginfo *siginfo,
159	void *uctx) {
160	SignalContext ctx(siginfo, uctx);
161	Printf("Tracer caught signal %d: addr=0x%zx pc=0x%zx sp=0x%zx\n", signum,
162	ctx.addr, ctx.pc, ctx.sp);
163	ThreadSuspender *inst = thread_suspender_instance;
164	if (inst) {
165	if (signum == SIGABRT)
166	inst->KillAllThreads();
167	else
168	inst->ResumeAllThreads();
169	RAW_CHECK(RemoveDieCallback(TracerThreadDieCallback));
170	thread_suspender_instance = nullptr;
171	atomic_store(&inst->arg->done, 1, memory_order_relaxed);
172	}
173	internal__exit((signum == SIGABRT) ? 1 : 2);
174	}
175
176	// Size of alternative stack for signal handlers in the tracer thread.
177	static const int kHandlerStackSize = 8192;
178
179	// This function will be run as a cloned task.
180	static int TracerThread(void *argument) {
181	TracerThreadArgument *tracer_thread_argument =
182	(TracerThreadArgument *)argument;
183
184	// Check if parent is already dead.
185	if (internal_getppid() != tracer_thread_argument->parent_pid)
186	internal__exit(4);
187
188	// Wait for the parent thread to finish preparations.
189	tracer_thread_argument->mutex.Lock();
190	tracer_thread_argument->mutex.Unlock();
191
192	RAW_CHECK(AddDieCallback(TracerThreadDieCallback));
193
194	ThreadSuspender thread_suspender(internal_getppid(), tracer_thread_argument);
195	// Global pointer for the signal handler.
196	thread_suspender_instance = &thread_suspender;
197
198	// Alternate stack for signal handling.
199	InternalMmapVector<char> handler_stack_memory(kHandlerStackSize);
200	stack_t handler_stack;
201	internal_memset(&handler_stack, 0, sizeof(handler_stack));
202	handler_stack.ss_sp = handler_stack_memory.data();
203	handler_stack.ss_size = kHandlerStackSize;
204	internal_sigaltstack(&handler_stack, nullptr);
205
206	// Install our handler for synchronous signals. Other signals should be
207	// blocked by the mask we inherited from the parent thread.
208	for (uptr i = 0; i < ARRAY_SIZE(kSyncSignals); i++) {
209	__sanitizer_sigaction act;
210	internal_memset(&act, 0, sizeof(act));
211	act.sigaction = TracerThreadSignalHandler;
212	act.sa_flags = SA_ONSTACK \| SA_SIGINFO;
213	internal_sigaction_norestorer(kSyncSignals[i], &act, 0);
214	}
215
216	int exit_code = 0;
217	if (!thread_suspender.SuspendAllThreads()) {
218	VReport(1, "Failed suspending threads.\n");
219	exit_code = 3;
220	} else {
221	tracer_thread_argument->callback(thread_suspender.suspended_threads_list(),
222	tracer_thread_argument->callback_argument);
223	thread_suspender.ResumeAllThreads();
224	exit_code = 0;
225	}
226	RAW_CHECK(RemoveDieCallback(TracerThreadDieCallback));
227	thread_suspender_instance = nullptr;
228	atomic_store(&tracer_thread_argument->done, 1, memory_order_relaxed);
229	return exit_code;
230	}
231
232	class ScopedStackSpaceWithGuard {
233	public:
234	explicit ScopedStackSpaceWithGuard(uptr stack_size) {
235	stack_size_ = stack_size;
236	guard_size_ = GetPageSizeCached();
237	// FIXME: Omitting MAP_STACK here works in current kernels but might break
238	// in the future.
239	guard_start_ =
240	(uptr)MmapOrDie(stack_size_ + guard_size_, "ScopedStackWithGuard");
241	CHECK(MprotectNoAccess((uptr)guard_start_, guard_size_));
242	}
243	~ScopedStackSpaceWithGuard() {
244	UnmapOrDie((void *)guard_start_, stack_size_ + guard_size_);
245	}
246	void *Bottom() const {
247	return (void *)(guard_start_ + stack_size_ + guard_size_);
248	}
249
250	private:
251	uptr stack_size_;
252	uptr guard_size_;
253	uptr guard_start_;
254	};
255
256	static __sanitizer_sigset_t blocked_sigset;
257	static __sanitizer_sigset_t old_sigset;
258
259	struct ScopedSetTracerPID {
260	explicit ScopedSetTracerPID(uptr tracer_pid) {
261	stoptheworld_tracer_pid = tracer_pid;
262	stoptheworld_tracer_ppid = internal_getpid();
263	}
264	~ScopedSetTracerPID() {
265	stoptheworld_tracer_pid = 0;
266	stoptheworld_tracer_ppid = 0;
267	}
268	};
269
270	void StopTheWorld(StopTheWorldCallback callback, void *argument) {
271	// Prepare the arguments for TracerThread.
272	struct TracerThreadArgument tracer_thread_argument;
273	tracer_thread_argument.callback = callback;
274	tracer_thread_argument.callback_argument = argument;
275	tracer_thread_argument.parent_pid = internal_getpid();
276	atomic_store(&tracer_thread_argument.done, 0, memory_order_relaxed);
277	const uptr kTracerStackSize = 2 * 1024 * 1024;
278	ScopedStackSpaceWithGuard tracer_stack(kTracerStackSize);
279
280	tracer_thread_argument.mutex.Lock();
281
282	internal_sigfillset(&blocked_sigset);
283	for (uptr i = 0; i < ARRAY_SIZE(kSyncSignals); i++)
284	internal_sigdelset(&blocked_sigset, kSyncSignals[i]);
285	int rv = internal_sigprocmask(SIG_BLOCK, &blocked_sigset, &old_sigset);
286	CHECK_EQ(rv, 0);
287	uptr tracer_pid = internal_clone(TracerThread, tracer_stack.Bottom(),
288	CLONE_VM \| CLONE_FS \| CLONE_FILES,
289	&tracer_thread_argument);
290	internal_sigprocmask(SIG_SETMASK, &old_sigset, 0);
291	int local_errno = 0;
292	if (internal_iserror(tracer_pid, &local_errno)) {
293	VReport(1, "Failed spawning a tracer thread (errno %d).\n", local_errno);
294	tracer_thread_argument.mutex.Unlock();
295	} else {
296	ScopedSetTracerPID scoped_set_tracer_pid(tracer_pid);
297
298	tracer_thread_argument.mutex.Unlock();
299
300	while (atomic_load(&tracer_thread_argument.done, memory_order_relaxed) == 0)
301	sched_yield();
302
303	for (;;) {
304	uptr waitpid_status = internal_waitpid(tracer_pid, nullptr, __WALL);
305	if (!internal_iserror(waitpid_status, &local_errno))
306	break;
307	if (local_errno == EINTR)
308	continue;
309	VReport(1, "Waiting on the tracer thread failed (errno %d).\n",
310	local_errno);
311	break;
312	}
313	}
314	}
315
316	tid_t SuspendedThreadsListNetBSD::GetThreadID(uptr index) const {
317	CHECK_LT(index, thread_ids_.size());
318	return thread_ids_[index];
319	}
320
321	uptr SuspendedThreadsListNetBSD::ThreadCount() const {
322	return thread_ids_.size();
323	}
324
325	bool SuspendedThreadsListNetBSD::ContainsTid(tid_t thread_id) const {
326	for (uptr i = 0; i < thread_ids_.size(); i++) {
327	if (thread_ids_[i] == thread_id)
328	return true;
329	}
330	return false;
331	}
332
333	void SuspendedThreadsListNetBSD::Append(tid_t tid) {
334	thread_ids_.push_back(tid);
335	}
336
337	PtraceRegistersStatus SuspendedThreadsListNetBSD::GetRegistersAndSP(
338	uptr index, uptr buffer, uptr sp) const {
339	lwpid_t tid = GetThreadID(index);
340	pid_t ppid = internal_getppid();
341	struct reg regs;
342	int pterrno;
343	bool isErr =
344	internal_iserror(internal_ptrace(PT_GETREGS, ppid, &regs, tid), &pterrno);
345	if (isErr) {
346	VReport(1,
347	"Could not get registers from process %d thread %d (errno %d).\n",
348	ppid, tid, pterrno);
349	return pterrno == ESRCH ? REGISTERS_UNAVAILABLE_FATAL
350	: REGISTERS_UNAVAILABLE;
351	}
352
353	*sp = PTRACE_REG_SP(&regs);
354	internal_memcpy(buffer, &regs, sizeof(regs));
355
356	return REGISTERS_AVAILABLE;
357	}
358
359	uptr SuspendedThreadsListNetBSD::RegisterCount() const {
360	return sizeof(struct reg) / sizeof(uptr);
361	}
362	} // namespace __sanitizer
363
364	#endif

lib/tsan/sanitizer_common/sanitizer_symbolizer.cpp created+135

...	@@ -0,0 +1,135 @@
1	//===-- sanitizer_symbolizer.cpp ------------------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is shared between AddressSanitizer and ThreadSanitizer
10	// run-time libraries.
11	//===----------------------------------------------------------------------===//
12
13	#include "sanitizer_allocator_internal.h"
14	#include "sanitizer_platform.h"
15	#include "sanitizer_internal_defs.h"
16	#include "sanitizer_libc.h"
17	#include "sanitizer_placement_new.h"
18	#include "sanitizer_symbolizer_internal.h"
19
20	namespace __sanitizer {
21
22	AddressInfo::AddressInfo() {
23	internal_memset(this, 0, sizeof(AddressInfo));
24	function_offset = kUnknown;
25	}
26
27	void AddressInfo::Clear() {
28	InternalFree(module);
29	InternalFree(function);
30	InternalFree(file);
31	internal_memset(this, 0, sizeof(AddressInfo));
32	function_offset = kUnknown;
33	}
34
35	void AddressInfo::FillModuleInfo(const char *mod_name, uptr mod_offset,
36	ModuleArch mod_arch) {
37	module = internal_strdup(mod_name);
38	module_offset = mod_offset;
39	module_arch = mod_arch;
40	}
41
42	SymbolizedStack::SymbolizedStack() : next(nullptr), info() {}
43
44	SymbolizedStack *SymbolizedStack::New(uptr addr) {
45	void *mem = InternalAlloc(sizeof(SymbolizedStack));
46	SymbolizedStack *res = new(mem) SymbolizedStack();
47	res->info.address = addr;
48	return res;
49	}
50
51	void SymbolizedStack::ClearAll() {
52	info.Clear();
53	if (next)
54	next->ClearAll();
55	InternalFree(this);
56	}
57
58	DataInfo::DataInfo() {
59	internal_memset(this, 0, sizeof(DataInfo));
60	}
61
62	void DataInfo::Clear() {
63	InternalFree(module);
64	InternalFree(file);
65	InternalFree(name);
66	internal_memset(this, 0, sizeof(DataInfo));
67	}
68
69	void FrameInfo::Clear() {
70	InternalFree(module);
71	for (LocalInfo &local : locals) {
72	InternalFree(local.function_name);
73	InternalFree(local.name);
74	InternalFree(local.decl_file);
75	}
76	locals.clear();
77	}
78
79	Symbolizer *Symbolizer::symbolizer_;
80	StaticSpinMutex Symbolizer::init_mu_;
81	LowLevelAllocator Symbolizer::symbolizer_allocator_;
82
83	void Symbolizer::InvalidateModuleList() {
84	modules_fresh_ = false;
85	}
86
87	void Symbolizer::AddHooks(Symbolizer::StartSymbolizationHook start_hook,
88	Symbolizer::EndSymbolizationHook end_hook) {
89	CHECK(start_hook_ == 0 && end_hook_ == 0);
90	start_hook_ = start_hook;
91	end_hook_ = end_hook;
92	}
93
94	const char Symbolizer::ModuleNameOwner::GetOwnedCopy(const char str) {
95	mu_->CheckLocked();
96
97	// 'str' will be the same string multiple times in a row, optimize this case.
98	if (last_match_ && !internal_strcmp(last_match_, str))
99	return last_match_;
100
101	// FIXME: this is linear search.
102	// We should optimize this further if this turns out to be a bottleneck later.
103	for (uptr i = 0; i < storage_.size(); ++i) {
104	if (!internal_strcmp(storage_[i], str)) {
105	last_match_ = storage_[i];
106	return last_match_;
107	}
108	}
109	last_match_ = internal_strdup(str);
110	storage_.push_back(last_match_);
111	return last_match_;
112	}
113
114	Symbolizer::Symbolizer(IntrusiveList<SymbolizerTool> tools)
115	: module_names_(&mu_), modules_(), modules_fresh_(false), tools_(tools),
116	start_hook_(0), end_hook_(0) {}
117
118	Symbolizer::SymbolizerScope::SymbolizerScope(const Symbolizer *sym)
119	: sym_(sym) {
120	if (sym_->start_hook_)
121	sym_->start_hook_();
122	}
123
124	Symbolizer::SymbolizerScope::~SymbolizerScope() {
125	if (sym_->end_hook_)
126	sym_->end_hook_();
127	}
128
129	void Symbolizer::LateInitializeTools() {
130	for (auto &tool : tools_) {
131	tool.LateInitialize();
132	}
133	}
134
135	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_symbolizer_libbacktrace.cpp created+209

...	@@ -0,0 +1,209 @@
1	//===-- sanitizer_symbolizer_libbacktrace.cpp -----------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is shared between AddressSanitizer and ThreadSanitizer
10	// run-time libraries.
11	// Libbacktrace implementation of symbolizer parts.
12	//===----------------------------------------------------------------------===//
13
14	#include "sanitizer_platform.h"
15
16	#include "sanitizer_internal_defs.h"
17	#include "sanitizer_symbolizer.h"
18	#include "sanitizer_symbolizer_libbacktrace.h"
19
20	#if SANITIZER_LIBBACKTRACE
21	# include "backtrace-supported.h"
22	# if SANITIZER_POSIX && BACKTRACE_SUPPORTED && !BACKTRACE_USES_MALLOC
23	# include "backtrace.h"
24	# if SANITIZER_CP_DEMANGLE
25	# undef ARRAY_SIZE
26	# include "demangle.h"
27	# endif
28	# else
29	# define SANITIZER_LIBBACKTRACE 0
30	# endif
31	#endif
32
33	namespace __sanitizer {
34
35	static char DemangleAlloc(const char name, bool always_alloc);
36
37	#if SANITIZER_LIBBACKTRACE
38
39	namespace {
40
41	# if SANITIZER_CP_DEMANGLE
42	struct CplusV3DemangleData {
43	char *buf;
44	uptr size, allocated;
45	};
46
47	extern "C" {
48	static void CplusV3DemangleCallback(const char s, size_t l, void vdata) {
49	CplusV3DemangleData data = (CplusV3DemangleData )vdata;
50	uptr needed = data->size + l + 1;
51	if (needed > data->allocated) {
52	data->allocated *= 2;
53	if (needed > data->allocated)
54	data->allocated = needed;
55	char buf = (char )InternalAlloc(data->allocated);
56	if (data->buf) {
57	internal_memcpy(buf, data->buf, data->size);
58	InternalFree(data->buf);
59	}
60	data->buf = buf;
61	}
62	internal_memcpy(data->buf + data->size, s, l);
63	data->buf[data->size + l] = '\0';
64	data->size += l;
65	}
66	} // extern "C"
67
68	char CplusV3Demangle(const char name) {
69	CplusV3DemangleData data;
70	data.buf = 0;
71	data.size = 0;
72	data.allocated = 0;
73	if (cplus_demangle_v3_callback(name, DMGL_PARAMS \| DMGL_ANSI,
74	CplusV3DemangleCallback, &data)) {
75	if (data.size + 64 > data.allocated)
76	return data.buf;
77	char *buf = internal_strdup(data.buf);
78	InternalFree(data.buf);
79	return buf;
80	}
81	if (data.buf)
82	InternalFree(data.buf);
83	return 0;
84	}
85	# endif // SANITIZER_CP_DEMANGLE
86
87	struct SymbolizeCodeCallbackArg {
88	SymbolizedStack *first;
89	SymbolizedStack *last;
90	uptr frames_symbolized;
91
92	AddressInfo *get_new_frame(uintptr_t addr) {
93	CHECK(last);
94	if (frames_symbolized > 0) {
95	SymbolizedStack *cur = SymbolizedStack::New(addr);
96	AddressInfo *info = &cur->info;
97	info->FillModuleInfo(first->info.module, first->info.module_offset,
98	first->info.module_arch);
99	last->next = cur;
100	last = cur;
101	}
102	CHECK_EQ(addr, first->info.address);
103	CHECK_EQ(addr, last->info.address);
104	return &last->info;
105	}
106	};
107
108	extern "C" {
109	static int SymbolizeCodePCInfoCallback(void *vdata, uintptr_t addr,
110	const char *filename, int lineno,
111	const char *function) {
112	SymbolizeCodeCallbackArg cdata = (SymbolizeCodeCallbackArg )vdata;
113	if (function) {
114	AddressInfo *info = cdata->get_new_frame(addr);
115	info->function = DemangleAlloc(function, /always_alloc/ true);
116	if (filename)
117	info->file = internal_strdup(filename);
118	info->line = lineno;
119	cdata->frames_symbolized++;
120	}
121	return 0;
122	}
123
124	static void SymbolizeCodeCallback(void *vdata, uintptr_t addr,
125	const char *symname, uintptr_t, uintptr_t) {
126	SymbolizeCodeCallbackArg cdata = (SymbolizeCodeCallbackArg )vdata;
127	if (symname) {
128	AddressInfo *info = cdata->get_new_frame(addr);
129	info->function = DemangleAlloc(symname, /always_alloc/ true);
130	cdata->frames_symbolized++;
131	}
132	}
133
134	static void SymbolizeDataCallback(void vdata, uintptr_t, const char symname,
135	uintptr_t symval, uintptr_t symsize) {
136	DataInfo info = (DataInfo )vdata;
137	if (symname && symval) {
138	info->name = DemangleAlloc(symname, /always_alloc/ true);
139	info->start = symval;
140	info->size = symsize;
141	}
142	}
143
144	static void ErrorCallback(void , const char , int) {}
145	} // extern "C"
146
147	} // namespace
148
149	LibbacktraceSymbolizer LibbacktraceSymbolizer::get(LowLevelAllocator alloc) {
150	// State created in backtrace_create_state is leaked.
151	void state = (void )(backtrace_create_state("/proc/self/exe", 0,
152	ErrorCallback, NULL));
153	if (!state)
154	return 0;
155	return new(*alloc) LibbacktraceSymbolizer(state);
156	}
157
158	bool LibbacktraceSymbolizer::SymbolizePC(uptr addr, SymbolizedStack *stack) {
159	SymbolizeCodeCallbackArg data;
160	data.first = stack;
161	data.last = stack;
162	data.frames_symbolized = 0;
163	backtrace_pcinfo((backtrace_state *)state_, addr, SymbolizeCodePCInfoCallback,
164	ErrorCallback, &data);
165	if (data.frames_symbolized > 0)
166	return true;
167	backtrace_syminfo((backtrace_state *)state_, addr, SymbolizeCodeCallback,
168	ErrorCallback, &data);
169	return (data.frames_symbolized > 0);
170	}
171
172	bool LibbacktraceSymbolizer::SymbolizeData(uptr addr, DataInfo *info) {
173	backtrace_syminfo((backtrace_state *)state_, addr, SymbolizeDataCallback,
174	ErrorCallback, info);
175	return true;
176	}
177
178	#else // SANITIZER_LIBBACKTRACE
179
180	LibbacktraceSymbolizer LibbacktraceSymbolizer::get(LowLevelAllocator alloc) {
181	return 0;
182	}
183
184	bool LibbacktraceSymbolizer::SymbolizePC(uptr addr, SymbolizedStack *stack) {
185	(void)state_;
186	return false;
187	}
188
189	bool LibbacktraceSymbolizer::SymbolizeData(uptr addr, DataInfo *info) {
190	return false;
191	}
192
193	#endif // SANITIZER_LIBBACKTRACE
194
195	static char DemangleAlloc(const char name, bool always_alloc) {
196	#if SANITIZER_LIBBACKTRACE && SANITIZER_CP_DEMANGLE
197	if (char *demangled = CplusV3Demangle(name))
198	return demangled;
199	#endif
200	if (always_alloc)
201	return internal_strdup(name);
202	return 0;
203	}
204
205	const char LibbacktraceSymbolizer::Demangle(const char name) {
206	return DemangleAlloc(name, /always_alloc/ false);
207	}
208
209	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_symbolizer_libcdep.cpp created+554

...	@@ -0,0 +1,554 @@
1	//===-- sanitizer_symbolizer_libcdep.cpp ----------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is shared between AddressSanitizer and ThreadSanitizer
10	// run-time libraries.
11	//===----------------------------------------------------------------------===//
12
13	#include "sanitizer_allocator_internal.h"
14	#include "sanitizer_internal_defs.h"
15	#include "sanitizer_symbolizer_internal.h"
16
17	namespace __sanitizer {
18
19	Symbolizer *Symbolizer::GetOrInit() {
20	SpinMutexLock l(&init_mu_);
21	if (symbolizer_)
22	return symbolizer_;
23	symbolizer_ = PlatformInit();
24	CHECK(symbolizer_);
25	return symbolizer_;
26	}
27
28	// See sanitizer_symbolizer_markup.cpp.
29	#if !SANITIZER_SYMBOLIZER_MARKUP
30
31	const char ExtractToken(const char str, const char delims, char *result) {
32	uptr prefix_len = internal_strcspn(str, delims);
33	result = (char)InternalAlloc(prefix_len + 1);
34	internal_memcpy(*result, str, prefix_len);
35	(*result)[prefix_len] = '\0';
36	const char *prefix_end = str + prefix_len;
37	if (*prefix_end != '\0') prefix_end++;
38	return prefix_end;
39	}
40
41	const char ExtractInt(const char str, const char delims, int result) {
42	char *buff = nullptr;
43	const char *ret = ExtractToken(str, delims, &buff);
44	if (buff) {
45	*result = (int)internal_atoll(buff);
46	}
47	InternalFree(buff);
48	return ret;
49	}
50
51	const char ExtractUptr(const char str, const char delims, uptr result) {
52	char *buff = nullptr;
53	const char *ret = ExtractToken(str, delims, &buff);
54	if (buff) {
55	*result = (uptr)internal_atoll(buff);
56	}
57	InternalFree(buff);
58	return ret;
59	}
60
61	const char ExtractSptr(const char str, const char delims, sptr result) {
62	char *buff = nullptr;
63	const char *ret = ExtractToken(str, delims, &buff);
64	if (buff) {
65	*result = (sptr)internal_atoll(buff);
66	}
67	InternalFree(buff);
68	return ret;
69	}
70
71	const char ExtractTokenUpToDelimiter(const char str, const char *delimiter,
72	char **result) {
73	const char *found_delimiter = internal_strstr(str, delimiter);
74	uptr prefix_len =
75	found_delimiter ? found_delimiter - str : internal_strlen(str);
76	result = (char )InternalAlloc(prefix_len + 1);
77	internal_memcpy(*result, str, prefix_len);
78	(*result)[prefix_len] = '\0';
79	const char *prefix_end = str + prefix_len;
80	if (*prefix_end != '\0') prefix_end += internal_strlen(delimiter);
81	return prefix_end;
82	}
83
84	SymbolizedStack *Symbolizer::SymbolizePC(uptr addr) {
85	BlockingMutexLock l(&mu_);
86	const char *module_name = nullptr;
87	uptr module_offset;
88	ModuleArch arch;
89	SymbolizedStack *res = SymbolizedStack::New(addr);
90	if (!FindModuleNameAndOffsetForAddress(addr, &module_name, &module_offset,
91	&arch))
92	return res;
93	// Always fill data about module name and offset.
94	res->info.FillModuleInfo(module_name, module_offset, arch);
95	for (auto &tool : tools_) {
96	SymbolizerScope sym_scope(this);
97	if (tool.SymbolizePC(addr, res)) {
98	return res;
99	}
100	}
101	return res;
102	}
103
104	bool Symbolizer::SymbolizeData(uptr addr, DataInfo *info) {
105	BlockingMutexLock l(&mu_);
106	const char *module_name = nullptr;
107	uptr module_offset;
108	ModuleArch arch;
109	if (!FindModuleNameAndOffsetForAddress(addr, &module_name, &module_offset,
110	&arch))
111	return false;
112	info->Clear();
113	info->module = internal_strdup(module_name);
114	info->module_offset = module_offset;
115	info->module_arch = arch;
116	for (auto &tool : tools_) {
117	SymbolizerScope sym_scope(this);
118	if (tool.SymbolizeData(addr, info)) {
119	return true;
120	}
121	}
122	return true;
123	}
124
125	bool Symbolizer::SymbolizeFrame(uptr addr, FrameInfo *info) {
126	BlockingMutexLock l(&mu_);
127	const char *module_name = nullptr;
128	if (!FindModuleNameAndOffsetForAddress(
129	addr, &module_name, &info->module_offset, &info->module_arch))
130	return false;
131	info->module = internal_strdup(module_name);
132	for (auto &tool : tools_) {
133	SymbolizerScope sym_scope(this);
134	if (tool.SymbolizeFrame(addr, info)) {
135	return true;
136	}
137	}
138	return true;
139	}
140
141	bool Symbolizer::GetModuleNameAndOffsetForPC(uptr pc, const char **module_name,
142	uptr *module_address) {
143	BlockingMutexLock l(&mu_);
144	const char *internal_module_name = nullptr;
145	ModuleArch arch;
146	if (!FindModuleNameAndOffsetForAddress(pc, &internal_module_name,
147	module_address, &arch))
148	return false;
149
150	if (module_name)
151	*module_name = module_names_.GetOwnedCopy(internal_module_name);
152	return true;
153	}
154
155	void Symbolizer::Flush() {
156	BlockingMutexLock l(&mu_);
157	for (auto &tool : tools_) {
158	SymbolizerScope sym_scope(this);
159	tool.Flush();
160	}
161	}
162
163	const char Symbolizer::Demangle(const char name) {
164	BlockingMutexLock l(&mu_);
165	for (auto &tool : tools_) {
166	SymbolizerScope sym_scope(this);
167	if (const char *demangled = tool.Demangle(name))
168	return demangled;
169	}
170	return PlatformDemangle(name);
171	}
172
173	bool Symbolizer::FindModuleNameAndOffsetForAddress(uptr address,
174	const char **module_name,
175	uptr *module_offset,
176	ModuleArch *module_arch) {
177	const LoadedModule *module = FindModuleForAddress(address);
178	if (!module)
179	return false;
180	*module_name = module->full_name();
181	*module_offset = address - module->base_address();
182	*module_arch = module->arch();
183	return true;
184	}
185
186	void Symbolizer::RefreshModules() {
187	modules_.init();
188	fallback_modules_.fallbackInit();
189	RAW_CHECK(modules_.size() > 0);
190	modules_fresh_ = true;
191	}
192
193	static const LoadedModule *SearchForModule(const ListOfModules &modules,
194	uptr address) {
195	for (uptr i = 0; i < modules.size(); i++) {
196	if (modules[i].containsAddress(address)) {
197	return &modules[i];
198	}
199	}
200	return nullptr;
201	}
202
203	const LoadedModule *Symbolizer::FindModuleForAddress(uptr address) {
204	bool modules_were_reloaded = false;
205	if (!modules_fresh_) {
206	RefreshModules();
207	modules_were_reloaded = true;
208	}
209	const LoadedModule *module = SearchForModule(modules_, address);
210	if (module) return module;
211
212	// dlopen/dlclose interceptors invalidate the module list, but when
213	// interception is disabled, we need to retry if the lookup fails in
214	// case the module list changed.
215	#if !SANITIZER_INTERCEPT_DLOPEN_DLCLOSE
216	if (!modules_were_reloaded) {
217	RefreshModules();
218	module = SearchForModule(modules_, address);
219	if (module) return module;
220	}
221	#endif
222
223	if (fallback_modules_.size()) {
224	module = SearchForModule(fallback_modules_, address);
225	}
226	return module;
227	}
228
229	// For now we assume the following protocol:
230	// For each request of the form
231	// <module_name> <module_offset>
232	// passed to STDIN, external symbolizer prints to STDOUT response:
233	// <function_name>
234	// <file_name>:<line_number>:<column_number>
235	// <function_name>
236	// <file_name>:<line_number>:<column_number>
237	// ...
238	// <empty line>
239	class LLVMSymbolizerProcess : public SymbolizerProcess {
240	public:
241	explicit LLVMSymbolizerProcess(const char *path)
242	: SymbolizerProcess(path, /use_posix_spawn=/SANITIZER_MAC) {}
243
244	private:
245	bool ReachedEndOfOutput(const char *buffer, uptr length) const override {
246	// Empty line marks the end of llvm-symbolizer output.
247	return length >= 2 && buffer[length - 1] == '\n' &&
248	buffer[length - 2] == '\n';
249	}
250
251	// When adding a new architecture, don't forget to also update
252	// script/asan_symbolize.py and sanitizer_common.h.
253	void GetArgV(const char *path_to_binary,
254	const char *(&argv)[kArgVMax]) const override {
255	#if defined(__x86_64h__)
256	const char* const kSymbolizerArch = "--default-arch=x86_64h";
257	#elif defined(__x86_64__)
258	const char* const kSymbolizerArch = "--default-arch=x86_64";
259	#elif defined(__i386__)
260	const char* const kSymbolizerArch = "--default-arch=i386";
261	#elif defined(__aarch64__)
262	const char* const kSymbolizerArch = "--default-arch=arm64";
263	#elif defined(__arm__)
264	const char* const kSymbolizerArch = "--default-arch=arm";
265	#elif defined(__powerpc64__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
266	const char* const kSymbolizerArch = "--default-arch=powerpc64";
267	#elif defined(__powerpc64__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
268	const char* const kSymbolizerArch = "--default-arch=powerpc64le";
269	#elif defined(__s390x__)
270	const char* const kSymbolizerArch = "--default-arch=s390x";
271	#elif defined(__s390__)
272	const char* const kSymbolizerArch = "--default-arch=s390";
273	#else
274	const char* const kSymbolizerArch = "--default-arch=unknown";
275	#endif
276
277	const char *const inline_flag = common_flags()->symbolize_inline_frames
278	? "--inlining=true"
279	: "--inlining=false";
280	int i = 0;
281	argv[i++] = path_to_binary;
282	argv[i++] = inline_flag;
283	argv[i++] = kSymbolizerArch;
284	argv[i++] = nullptr;
285	}
286	};
287
288	LLVMSymbolizer::LLVMSymbolizer(const char path, LowLevelAllocator allocator)
289	: symbolizer_process_(new(*allocator) LLVMSymbolizerProcess(path)) {}
290
291	// Parse a <file>:<line>[:<column>] buffer. The file path may contain colons on
292	// Windows, so extract tokens from the right hand side first. The column info is
293	// also optional.
294	static const char ParseFileLineInfo(AddressInfo info, const char *str) {
295	char *file_line_info = nullptr;
296	str = ExtractToken(str, "\n", &file_line_info);
297	CHECK(file_line_info);
298
299	if (uptr size = internal_strlen(file_line_info)) {
300	char *back = file_line_info + size - 1;
301	for (int i = 0; i < 2; ++i) {
302	while (back > file_line_info && IsDigit(*back)) --back;
303	if (*back != ':' \|\| !IsDigit(back[1])) break;
304	info->column = info->line;
305	info->line = internal_atoll(back + 1);
306	// Truncate the string at the colon to keep only filename.
307	*back = '\0';
308	--back;
309	}
310	ExtractToken(file_line_info, "", &info->file);
311	}
312
313	InternalFree(file_line_info);
314	return str;
315	}
316
317	// Parses one or more two-line strings in the following format:
318	// <function_name>
319	// <file_name>:<line_number>[:<column_number>]
320	// Used by LLVMSymbolizer, Addr2LinePool and InternalSymbolizer, since all of
321	// them use the same output format.
322	void ParseSymbolizePCOutput(const char str, SymbolizedStack res) {
323	bool top_frame = true;
324	SymbolizedStack *last = res;
325	while (true) {
326	char *function_name = nullptr;
327	str = ExtractToken(str, "\n", &function_name);
328	CHECK(function_name);
329	if (function_name[0] == '\0') {
330	// There are no more frames.
331	InternalFree(function_name);
332	break;
333	}
334	SymbolizedStack *cur;
335	if (top_frame) {
336	cur = res;
337	top_frame = false;
338	} else {
339	cur = SymbolizedStack::New(res->info.address);
340	cur->info.FillModuleInfo(res->info.module, res->info.module_offset,
341	res->info.module_arch);
342	last->next = cur;
343	last = cur;
344	}
345
346	AddressInfo *info = &cur->info;
347	info->function = function_name;
348	str = ParseFileLineInfo(info, str);
349
350	// Functions and filenames can be "??", in which case we write 0
351	// to address info to mark that names are unknown.
352	if (0 == internal_strcmp(info->function, "??")) {
353	InternalFree(info->function);
354	info->function = 0;
355	}
356	if (0 == internal_strcmp(info->file, "??")) {
357	InternalFree(info->file);
358	info->file = 0;
359	}
360	}
361	}
362
363	// Parses a two-line string in the following format:
364	// <symbol_name>
365	// <start_address> <size>
366	// Used by LLVMSymbolizer and InternalSymbolizer.
367	void ParseSymbolizeDataOutput(const char str, DataInfo info) {
368	str = ExtractToken(str, "\n", &info->name);
369	str = ExtractUptr(str, " ", &info->start);
370	str = ExtractUptr(str, "\n", &info->size);
371	}
372
373	static void ParseSymbolizeFrameOutput(const char *str,
374	InternalMmapVector<LocalInfo> *locals) {
375	if (internal_strncmp(str, "??", 2) == 0)
376	return;
377
378	while (*str) {
379	LocalInfo local;
380	str = ExtractToken(str, "\n", &local.function_name);
381	str = ExtractToken(str, "\n", &local.name);
382
383	AddressInfo addr;
384	str = ParseFileLineInfo(&addr, str);
385	local.decl_file = addr.file;
386	local.decl_line = addr.line;
387
388	local.has_frame_offset = internal_strncmp(str, "??", 2) != 0;
389	str = ExtractSptr(str, " ", &local.frame_offset);
390
391	local.has_size = internal_strncmp(str, "??", 2) != 0;
392	str = ExtractUptr(str, " ", &local.size);
393
394	local.has_tag_offset = internal_strncmp(str, "??", 2) != 0;
395	str = ExtractUptr(str, "\n", &local.tag_offset);
396
397	locals->push_back(local);
398	}
399	}
400
401	bool LLVMSymbolizer::SymbolizePC(uptr addr, SymbolizedStack *stack) {
402	AddressInfo *info = &stack->info;
403	const char *buf = FormatAndSendCommand(
404	"CODE", info->module, info->module_offset, info->module_arch);
405	if (!buf)
406	return false;
407	ParseSymbolizePCOutput(buf, stack);
408	return true;
409	}
410
411	bool LLVMSymbolizer::SymbolizeData(uptr addr, DataInfo *info) {
412	const char *buf = FormatAndSendCommand(
413	"DATA", info->module, info->module_offset, info->module_arch);
414	if (!buf)
415	return false;
416	ParseSymbolizeDataOutput(buf, info);
417	info->start += (addr - info->module_offset); // Add the base address.
418	return true;
419	}
420
421	bool LLVMSymbolizer::SymbolizeFrame(uptr addr, FrameInfo *info) {
422	const char *buf = FormatAndSendCommand(
423	"FRAME", info->module, info->module_offset, info->module_arch);
424	if (!buf)
425	return false;
426	ParseSymbolizeFrameOutput(buf, &info->locals);
427	return true;
428	}
429
430	const char LLVMSymbolizer::FormatAndSendCommand(const char command_prefix,
431	const char *module_name,
432	uptr module_offset,
433	ModuleArch arch) {
434	CHECK(module_name);
435	int size_needed = 0;
436	if (arch == kModuleArchUnknown)
437	size_needed = internal_snprintf(buffer_, kBufferSize, "%s \"%s\" 0x%zx\n",
438	command_prefix, module_name, module_offset);
439	else
440	size_needed = internal_snprintf(buffer_, kBufferSize,
441	"%s \"%s:%s\" 0x%zx\n", command_prefix,
442	module_name, ModuleArchToString(arch),
443	module_offset);
444
445	if (size_needed >= static_cast<int>(kBufferSize)) {
446	Report("WARNING: Command buffer too small");
447	return nullptr;
448	}
449
450	return symbolizer_process_->SendCommand(buffer_);
451	}
452
453	SymbolizerProcess::SymbolizerProcess(const char *path, bool use_posix_spawn)
454	: path_(path),
455	input_fd_(kInvalidFd),
456	output_fd_(kInvalidFd),
457	times_restarted_(0),
458	failed_to_start_(false),
459	reported_invalid_path_(false),
460	use_posix_spawn_(use_posix_spawn) {
461	CHECK(path_);
462	CHECK_NE(path_[0], '\0');
463	}
464
465	static bool IsSameModule(const char* path) {
466	if (const char* ProcessName = GetProcessName()) {
467	if (const char* SymbolizerName = StripModuleName(path)) {
468	return !internal_strcmp(ProcessName, SymbolizerName);
469	}
470	}
471	return false;
472	}
473
474	const char SymbolizerProcess::SendCommand(const char command) {
475	if (failed_to_start_)
476	return nullptr;
477	if (IsSameModule(path_)) {
478	Report("WARNING: Symbolizer was blocked from starting itself!\n");
479	failed_to_start_ = true;
480	return nullptr;
481	}
482	for (; times_restarted_ < kMaxTimesRestarted; times_restarted_++) {
483	// Start or restart symbolizer if we failed to send command to it.
484	if (const char *res = SendCommandImpl(command))
485	return res;
486	Restart();
487	}
488	if (!failed_to_start_) {
489	Report("WARNING: Failed to use and restart external symbolizer!\n");
490	failed_to_start_ = true;
491	}
492	return nullptr;
493	}
494
495	const char SymbolizerProcess::SendCommandImpl(const char command) {
496	if (input_fd_ == kInvalidFd \|\| output_fd_ == kInvalidFd)
497	return nullptr;
498	if (!WriteToSymbolizer(command, internal_strlen(command)))
499	return nullptr;
500	if (!ReadFromSymbolizer(buffer_, kBufferSize))
501	return nullptr;
502	return buffer_;
503	}
504
505	bool SymbolizerProcess::Restart() {
506	if (input_fd_ != kInvalidFd)
507	CloseFile(input_fd_);
508	if (output_fd_ != kInvalidFd)
509	CloseFile(output_fd_);
510	return StartSymbolizerSubprocess();
511	}
512
513	bool SymbolizerProcess::ReadFromSymbolizer(char *buffer, uptr max_length) {
514	if (max_length == 0)
515	return true;
516	uptr read_len = 0;
517	while (true) {
518	uptr just_read = 0;
519	bool success = ReadFromFile(input_fd_, buffer + read_len,
520	max_length - read_len - 1, &just_read);
521	// We can't read 0 bytes, as we don't expect external symbolizer to close
522	// its stdout.
523	if (!success \|\| just_read == 0) {
524	Report("WARNING: Can't read from symbolizer at fd %d\n", input_fd_);
525	return false;
526	}
527	read_len += just_read;
528	if (ReachedEndOfOutput(buffer, read_len))
529	break;
530	if (read_len + 1 == max_length) {
531	Report("WARNING: Symbolizer buffer too small\n");
532	read_len = 0;
533	break;
534	}
535	}
536	buffer[read_len] = '\0';
537	return true;
538	}
539
540	bool SymbolizerProcess::WriteToSymbolizer(const char *buffer, uptr length) {
541	if (length == 0)
542	return true;
543	uptr write_len = 0;
544	bool success = WriteToFile(output_fd_, buffer, length, &write_len);
545	if (!success \|\| write_len != length) {
546	Report("WARNING: Can't write to symbolizer at fd %d\n", output_fd_);
547	return false;
548	}
549	return true;
550	}
551
552	#endif // !SANITIZER_SYMBOLIZER_MARKUP
553
554	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_symbolizer_mac.cpp created+249

...	@@ -0,0 +1,249 @@
1	//===-- sanitizer_symbolizer_mac.cpp --------------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is shared between various sanitizers' runtime libraries.
10	//
11	// Implementation of Mac-specific "atos" symbolizer.
12	//===----------------------------------------------------------------------===//
13
14	#include "sanitizer_platform.h"
15	#if SANITIZER_MAC
16
17	#include "sanitizer_allocator_internal.h"
18	#include "sanitizer_mac.h"
19	#include "sanitizer_symbolizer_mac.h"
20
21	#include <dlfcn.h>
22	#include <errno.h>
23	#include <mach/mach.h>
24	#include <stdlib.h>
25	#include <sys/wait.h>
26	#include <unistd.h>
27	#include <util.h>
28
29	namespace __sanitizer {
30
31	bool DlAddrSymbolizer::SymbolizePC(uptr addr, SymbolizedStack *stack) {
32	Dl_info info;
33	int result = dladdr((const void *)addr, &info);
34	if (!result) return false;
35
36	CHECK(addr >= reinterpret_cast<uptr>(info.dli_saddr));
37	stack->info.function_offset = addr - reinterpret_cast<uptr>(info.dli_saddr);
38	const char *demangled = DemangleSwiftAndCXX(info.dli_sname);
39	if (!demangled) return false;
40	stack->info.function = internal_strdup(demangled);
41	return true;
42	}
43
44	bool DlAddrSymbolizer::SymbolizeData(uptr addr, DataInfo *datainfo) {
45	Dl_info info;
46	int result = dladdr((const void *)addr, &info);
47	if (!result) return false;
48	const char *demangled = DemangleSwiftAndCXX(info.dli_sname);
49	datainfo->name = internal_strdup(demangled);
50	datainfo->start = (uptr)info.dli_saddr;
51	return true;
52	}
53
54	#define K_ATOS_ENV_VAR "__check_mach_ports_lookup"
55
56	// This cannot live in `AtosSymbolizerProcess` because instances of that object
57	// are allocated by the internal allocator which under ASan is poisoned with
58	// kAsanInternalHeapMagic.
59	static char kAtosMachPortEnvEntry[] = K_ATOS_ENV_VAR "=000000000000000";
60
61	class AtosSymbolizerProcess : public SymbolizerProcess {
62	public:
63	explicit AtosSymbolizerProcess(const char *path)
64	: SymbolizerProcess(path, /use_posix_spawn/ true) {
65	pid_str_[0] = '\0';
66	}
67
68	void LateInitialize() {
69	if (SANITIZER_IOSSIM) {
70	// `putenv()` may call malloc/realloc so it is only safe to do this
71	// during LateInitialize() or later (i.e. we can't do this in the
72	// constructor). We also can't do this in `StartSymbolizerSubprocess()`
73	// because in TSan we switch allocators when we're symbolizing.
74	// We use `putenv()` rather than `setenv()` so that we can later directly
75	// write into the storage without LibC getting involved to change what the
76	// variable is set to
77	int result = putenv(kAtosMachPortEnvEntry);
78	CHECK_EQ(result, 0);
79	}
80	}
81
82	private:
83	bool StartSymbolizerSubprocess() override {
84	// Configure sandbox before starting atos process.
85
86	// Put the string command line argument in the object so that it outlives
87	// the call to GetArgV.
88	internal_snprintf(pid_str_, sizeof(pid_str_), "%d", internal_getpid());
89
90	if (SANITIZER_IOSSIM) {
91	// `atos` in the simulator is restricted in its ability to retrieve the
92	// task port for the target process (us) so we need to do extra work
93	// to pass our task port to it.
94	mach_port_t ports[]{mach_task_self()};
95	kern_return_t ret =
96	mach_ports_register(mach_task_self(), ports, /count=/1);
97	CHECK_EQ(ret, KERN_SUCCESS);
98
99	// Set environment variable that signals to `atos` that it should look
100	// for our task port. We can't call `setenv()` here because it might call
101	// malloc/realloc. To avoid that we instead update the
102	// `mach_port_env_var_entry_` variable with our current PID.
103	uptr count = internal_snprintf(kAtosMachPortEnvEntry,
104	sizeof(kAtosMachPortEnvEntry),
105	K_ATOS_ENV_VAR "=%s", pid_str_);
106	CHECK_GE(count, sizeof(K_ATOS_ENV_VAR) + internal_strlen(pid_str_));
107	// Document our assumption but without calling `getenv()` in normal
108	// builds.
109	DCHECK(getenv(K_ATOS_ENV_VAR));
110	DCHECK_EQ(internal_strcmp(getenv(K_ATOS_ENV_VAR), pid_str_), 0);
111	}
112
113	return SymbolizerProcess::StartSymbolizerSubprocess();
114	}
115
116	bool ReachedEndOfOutput(const char *buffer, uptr length) const override {
117	return (length >= 1 && buffer[length - 1] == '\n');
118	}
119
120	void GetArgV(const char *path_to_binary,
121	const char *(&argv)[kArgVMax]) const override {
122	int i = 0;
123	argv[i++] = path_to_binary;
124	argv[i++] = "-p";
125	argv[i++] = &pid_str_[0];
126	if (GetMacosAlignedVersion() == MacosVersion(10, 9)) {
127	// On Mavericks atos prints a deprecation warning which we suppress by
128	// passing -d. The warning isn't present on other OSX versions, even the
129	// newer ones.
130	argv[i++] = "-d";
131	}
132	argv[i++] = nullptr;
133	}
134
135	char pid_str_[16];
136	// Space for `\0` in `K_ATOS_ENV_VAR` is reused for `=`.
137	static_assert(sizeof(kAtosMachPortEnvEntry) ==
138	(sizeof(K_ATOS_ENV_VAR) + sizeof(pid_str_)),
139	"sizes should match");
140	};
141
142	#undef K_ATOS_ENV_VAR
143
144	static bool ParseCommandOutput(const char str, uptr addr, char *out_name,
145	char out_module, char out_file, uptr *line,
146	uptr *start_address) {
147	// Trim ending newlines.
148	char *trim;
149	ExtractTokenUpToDelimiter(str, "\n", &trim);
150
151	// The line from `atos` is in one of these formats:
152	// myfunction (in library.dylib) (sourcefile.c:17)
153	// myfunction (in library.dylib) + 0x1fe
154	// myfunction (in library.dylib) + 15
155	// 0xdeadbeef (in library.dylib) + 0x1fe
156	// 0xdeadbeef (in library.dylib) + 15
157	// 0xdeadbeef (in library.dylib)
158	// 0xdeadbeef
159
160	const char *rest = trim;
161	char *symbol_name;
162	rest = ExtractTokenUpToDelimiter(rest, " (in ", &symbol_name);
163	if (rest[0] == '\0') {
164	InternalFree(symbol_name);
165	InternalFree(trim);
166	return false;
167	}
168
169	if (internal_strncmp(symbol_name, "0x", 2) != 0)
170	*out_name = symbol_name;
171	else
172	InternalFree(symbol_name);
173	rest = ExtractTokenUpToDelimiter(rest, ") ", out_module);
174
175	if (rest[0] == '(') {
176	if (out_file) {
177	rest++;
178	rest = ExtractTokenUpToDelimiter(rest, ":", out_file);
179	char *extracted_line_number;
180	rest = ExtractTokenUpToDelimiter(rest, ")", &extracted_line_number);
181	if (line) *line = (uptr)internal_atoll(extracted_line_number);
182	InternalFree(extracted_line_number);
183	}
184	} else if (rest[0] == '+') {
185	rest += 2;
186	uptr offset = internal_atoll(rest);
187	if (start_address) *start_address = addr - offset;
188	}
189
190	InternalFree(trim);
191	return true;
192	}
193
194	AtosSymbolizer::AtosSymbolizer(const char path, LowLevelAllocator allocator)
195	: process_(new (*allocator) AtosSymbolizerProcess(path)) {}
196
197	bool AtosSymbolizer::SymbolizePC(uptr addr, SymbolizedStack *stack) {
198	if (!process_) return false;
199	if (addr == 0) return false;
200	char command[32];
201	internal_snprintf(command, sizeof(command), "0x%zx\n", addr);
202	const char *buf = process_->SendCommand(command);
203	if (!buf) return false;
204	uptr line;
205	uptr start_address = AddressInfo::kUnknown;
206	if (!ParseCommandOutput(buf, addr, &stack->info.function, &stack->info.module,
207	&stack->info.file, &line, &start_address)) {
208	process_ = nullptr;
209	return false;
210	}
211	stack->info.line = (int)line;
212
213	if (start_address == AddressInfo::kUnknown) {
214	// Fallback to dladdr() to get function start address if atos doesn't report
215	// it.
216	Dl_info info;
217	int result = dladdr((const void *)addr, &info);
218	if (result)
219	start_address = reinterpret_cast<uptr>(info.dli_saddr);
220	}
221
222	// Only assig to `function_offset` if we were able to get the function's
223	// start address.
224	if (start_address != AddressInfo::kUnknown) {
225	CHECK(addr >= start_address);
226	stack->info.function_offset = addr - start_address;
227	}
228	return true;
229	}
230
231	bool AtosSymbolizer::SymbolizeData(uptr addr, DataInfo *info) {
232	if (!process_) return false;
233	char command[32];
234	internal_snprintf(command, sizeof(command), "0x%zx\n", addr);
235	const char *buf = process_->SendCommand(command);
236	if (!buf) return false;
237	if (!ParseCommandOutput(buf, addr, &info->name, &info->module, nullptr,
238	nullptr, &info->start)) {
239	process_ = nullptr;
240	return false;
241	}
242	return true;
243	}
244
245	void AtosSymbolizer::LateInitialize() { process_->LateInitialize(); }
246
247	} // namespace __sanitizer
248
249	#endif // SANITIZER_MAC

lib/tsan/sanitizer_common/sanitizer_symbolizer_markup.cpp created+146

...	@@ -0,0 +1,146 @@
1	//===-- sanitizer_symbolizer_markup.cpp -----------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is shared between various sanitizers' runtime libraries.
10	//
11	// Implementation of offline markup symbolizer.
12	//===----------------------------------------------------------------------===//
13
14	#include "sanitizer_platform.h"
15	#if SANITIZER_SYMBOLIZER_MARKUP
16
17	#if SANITIZER_FUCHSIA
18	#include "sanitizer_symbolizer_fuchsia.h"
19	#elif SANITIZER_RTEMS
20	#include "sanitizer_symbolizer_rtems.h"
21	#endif
22	#include "sanitizer_stacktrace.h"
23	#include "sanitizer_symbolizer.h"
24
25	#include <limits.h>
26	#include <unwind.h>
27
28	namespace __sanitizer {
29
30	// This generic support for offline symbolizing is based on the
31	// Fuchsia port. We don't do any actual symbolization per se.
32	// Instead, we emit text containing raw addresses and raw linkage
33	// symbol names, embedded in Fuchsia's symbolization markup format.
34	// Fuchsia's logging infrastructure emits enough information about
35	// process memory layout that a post-processing filter can do the
36	// symbolization and pretty-print the markup. See the spec at:
37	// https://fuchsia.googlesource.com/zircon/+/master/docs/symbolizer_markup.md
38
39	// This is used by UBSan for type names, and by ASan for global variable names.
40	// It's expected to return a static buffer that will be reused on each call.
41	const char Symbolizer::Demangle(const char name) {
42	static char buffer[kFormatDemangleMax];
43	internal_snprintf(buffer, sizeof(buffer), kFormatDemangle, name);
44	return buffer;
45	}
46
47	// This is used mostly for suppression matching. Making it work
48	// would enable "interceptor_via_lib" suppressions. It's also used
49	// once in UBSan to say "in module ..." in a message that also
50	// includes an address in the module, so post-processing can already
51	// pretty-print that so as to indicate the module.
52	bool Symbolizer::GetModuleNameAndOffsetForPC(uptr pc, const char **module_name,
53	uptr *module_address) {
54	return false;
55	}
56
57	// This is used in some places for suppression checking, which we
58	// don't really support for Fuchsia. It's also used in UBSan to
59	// identify a PC location to a function name, so we always fill in
60	// the function member with a string containing markup around the PC
61	// value.
62	// TODO(mcgrathr): Under SANITIZER_GO, it's currently used by TSan
63	// to render stack frames, but that should be changed to use
64	// RenderStackFrame.
65	SymbolizedStack *Symbolizer::SymbolizePC(uptr addr) {
66	SymbolizedStack *s = SymbolizedStack::New(addr);
67	char buffer[kFormatFunctionMax];
68	internal_snprintf(buffer, sizeof(buffer), kFormatFunction, addr);
69	s->info.function = internal_strdup(buffer);
70	return s;
71	}
72
73	// Always claim we succeeded, so that RenderDataInfo will be called.
74	bool Symbolizer::SymbolizeData(uptr addr, DataInfo *info) {
75	info->Clear();
76	info->start = addr;
77	return true;
78	}
79
80	// We ignore the format argument to __sanitizer_symbolize_global.
81	void RenderData(InternalScopedString buffer, const char format,
82	const DataInfo DI, const char strip_path_prefix) {
83	buffer->append(kFormatData, DI->start);
84	}
85
86	// We don't support the stack_trace_format flag at all.
87	void RenderFrame(InternalScopedString buffer, const char format, int frame_no,
88	const AddressInfo &info, bool vs_style,
89	const char strip_path_prefix, const char strip_func_prefix) {
90	buffer->append(kFormatFrame, frame_no, info.address);
91	}
92
93	Symbolizer *Symbolizer::PlatformInit() {
94	return new (symbolizer_allocator_) Symbolizer({});
95	}
96
97	void Symbolizer::LateInitialize() {
98	Symbolizer::GetOrInit()->LateInitializeTools();
99	}
100
101	void StartReportDeadlySignal() {}
102	void ReportDeadlySignal(const SignalContext &sig, u32 tid,
103	UnwindSignalStackCallbackType unwind,
104	const void *unwind_context) {}
105
106	#if SANITIZER_CAN_SLOW_UNWIND
107	struct UnwindTraceArg {
108	BufferedStackTrace *stack;
109	u32 max_depth;
110	};
111
112	_Unwind_Reason_Code Unwind_Trace(struct _Unwind_Context ctx, void param) {
113	UnwindTraceArg arg = static_cast<UnwindTraceArg >(param);
114	CHECK_LT(arg->stack->size, arg->max_depth);
115	uptr pc = _Unwind_GetIP(ctx);
116	if (pc < PAGE_SIZE) return _URC_NORMAL_STOP;
117	arg->stack->trace_buffer[arg->stack->size++] = pc;
118	return (arg->stack->size == arg->max_depth ? _URC_NORMAL_STOP
119	: _URC_NO_REASON);
120	}
121
122	void BufferedStackTrace::UnwindSlow(uptr pc, u32 max_depth) {
123	CHECK_GE(max_depth, 2);
124	size = 0;
125	UnwindTraceArg arg = {this, Min(max_depth + 1, kStackTraceMax)};
126	_Unwind_Backtrace(Unwind_Trace, &arg);
127	CHECK_GT(size, 0);
128	// We need to pop a few frames so that pc is on top.
129	uptr to_pop = LocatePcInTrace(pc);
130	// trace_buffer[0] belongs to the current function so we always pop it,
131	// unless there is only 1 frame in the stack trace (1 frame is always better
132	// than 0!).
133	PopStackFrames(Min(to_pop, static_cast<uptr>(1)));
134	trace_buffer[0] = pc;
135	}
136
137	void BufferedStackTrace::UnwindSlow(uptr pc, void *context, u32 max_depth) {
138	CHECK(context);
139	CHECK_GE(max_depth, 2);
140	UNREACHABLE("signal context doesn't exist");
141	}
142	#endif // SANITIZER_CAN_SLOW_UNWIND
143
144	} // namespace __sanitizer
145
146	#endif // SANITIZER_SYMBOLIZER_MARKUP

lib/tsan/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cpp created+492

...	@@ -0,0 +1,492 @@
1	//===-- sanitizer_symbolizer_posix_libcdep.cpp ----------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is shared between AddressSanitizer and ThreadSanitizer
10	// run-time libraries.
11	// POSIX-specific implementation of symbolizer parts.
12	//===----------------------------------------------------------------------===//
13
14	#include "sanitizer_platform.h"
15	#if SANITIZER_POSIX
16	#include "sanitizer_allocator_internal.h"
17	#include "sanitizer_common.h"
18	#include "sanitizer_file.h"
19	#include "sanitizer_flags.h"
20	#include "sanitizer_internal_defs.h"
21	#include "sanitizer_linux.h"
22	#include "sanitizer_placement_new.h"
23	#include "sanitizer_posix.h"
24	#include "sanitizer_procmaps.h"
25	#include "sanitizer_symbolizer_internal.h"
26	#include "sanitizer_symbolizer_libbacktrace.h"
27	#include "sanitizer_symbolizer_mac.h"
28
29	#include <dlfcn.h> // for dlsym()
30	#include <errno.h>
31	#include <stdint.h>
32	#include <stdlib.h>
33	#include <sys/wait.h>
34	#include <unistd.h>
35
36	// C++ demangling function, as required by Itanium C++ ABI. This is weak,
37	// because we do not require a C++ ABI library to be linked to a program
38	// using sanitizers; if it's not present, we'll just use the mangled name.
39	namespace __cxxabiv1 {
40	extern "C" SANITIZER_WEAK_ATTRIBUTE
41	char __cxa_demangle(const char mangled, char *buffer,
42	size_t length, int status);
43	}
44
45	namespace __sanitizer {
46
47	// Attempts to demangle the name via __cxa_demangle from __cxxabiv1.
48	const char DemangleCXXABI(const char name) {
49	// FIXME: __cxa_demangle aggressively insists on allocating memory.
50	// There's not much we can do about that, short of providing our
51	// own demangler (libc++abi's implementation could be adapted so that
52	// it does not allocate). For now, we just call it anyway, and we leak
53	// the returned value.
54	if (&__cxxabiv1::__cxa_demangle)
55	if (const char *demangled_name =
56	__cxxabiv1::__cxa_demangle(name, 0, 0, 0))
57	return demangled_name;
58
59	return name;
60	}
61
62	// As of now, there are no headers for the Swift runtime. Once they are
63	// present, we will weakly link since we do not require Swift runtime to be
64	// linked.
65	typedef char (swift_demangle_ft)(const char *mangledName,
66	size_t mangledNameLength, char *outputBuffer,
67	size_t *outputBufferSize, uint32_t flags);
68	static swift_demangle_ft swift_demangle_f;
69
70	// This must not happen lazily at symbolication time, because dlsym uses
71	// malloc and thread-local storage, which is not a good thing to do during
72	// symbolication.
73	static void InitializeSwiftDemangler() {
74	swift_demangle_f = (swift_demangle_ft)dlsym(RTLD_DEFAULT, "swift_demangle");
75	(void)dlerror(); // Cleanup error message in case of failure
76	}
77
78	// Attempts to demangle a Swift name. The demangler will return nullptr if a
79	// non-Swift name is passed in.
80	const char DemangleSwift(const char name) {
81	if (swift_demangle_f)
82	return swift_demangle_f(name, internal_strlen(name), 0, 0, 0);
83
84	return nullptr;
85	}
86
87	const char DemangleSwiftAndCXX(const char name) {
88	if (!name) return nullptr;
89	if (const char *swift_demangled_name = DemangleSwift(name))
90	return swift_demangled_name;
91	return DemangleCXXABI(name);
92	}
93
94	static bool CreateTwoHighNumberedPipes(int infd_, int outfd_) {
95	int *infd = NULL;
96	int *outfd = NULL;
97	// The client program may close its stdin and/or stdout and/or stderr
98	// thus allowing socketpair to reuse file descriptors 0, 1 or 2.
99	// In this case the communication between the forked processes may be
100	// broken if either the parent or the child tries to close or duplicate
101	// these descriptors. The loop below produces two pairs of file
102	// descriptors, each greater than 2 (stderr).
103	int sock_pair[5][2];
104	for (int i = 0; i < 5; i++) {
105	if (pipe(sock_pair[i]) == -1) {
106	for (int j = 0; j < i; j++) {
107	internal_close(sock_pair[j][0]);
108	internal_close(sock_pair[j][1]);
109	}
110	return false;
111	} else if (sock_pair[i][0] > 2 && sock_pair[i][1] > 2) {
112	if (infd == NULL) {
113	infd = sock_pair[i];
114	} else {
115	outfd = sock_pair[i];
116	for (int j = 0; j < i; j++) {
117	if (sock_pair[j] == infd) continue;
118	internal_close(sock_pair[j][0]);
119	internal_close(sock_pair[j][1]);
120	}
121	break;
122	}
123	}
124	}
125	CHECK(infd);
126	CHECK(outfd);
127	infd_[0] = infd[0];
128	infd_[1] = infd[1];
129	outfd_[0] = outfd[0];
130	outfd_[1] = outfd[1];
131	return true;
132	}
133
134	bool SymbolizerProcess::StartSymbolizerSubprocess() {
135	if (!FileExists(path_)) {
136	if (!reported_invalid_path_) {
137	Report("WARNING: invalid path to external symbolizer!\n");
138	reported_invalid_path_ = true;
139	}
140	return false;
141	}
142
143	const char *argv[kArgVMax];
144	GetArgV(path_, argv);
145	pid_t pid;
146
147	// Report how symbolizer is being launched for debugging purposes.
148	if (Verbosity() >= 3) {
149	// Only use `Report` for first line so subsequent prints don't get prefixed
150	// with current PID.
151	Report("Launching Symbolizer process: ");
152	for (unsigned index = 0; index < kArgVMax && argv[index]; ++index)
153	Printf("%s ", argv[index]);
154	Printf("\n");
155	}
156
157	if (use_posix_spawn_) {
158	#if SANITIZER_MAC
159	fd_t fd = internal_spawn(argv, const_cast<const char **>(GetEnvP()), &pid);
160	if (fd == kInvalidFd) {
161	Report("WARNING: failed to spawn external symbolizer (errno: %d)\n",
162	errno);
163	return false;
164	}
165
166	input_fd_ = fd;
167	output_fd_ = fd;
168	#else // SANITIZER_MAC
169	UNIMPLEMENTED();
170	#endif // SANITIZER_MAC
171	} else {
172	fd_t infd[2] = {}, outfd[2] = {};
173	if (!CreateTwoHighNumberedPipes(infd, outfd)) {
174	Report("WARNING: Can't create a socket pair to start "
175	"external symbolizer (errno: %d)\n", errno);
176	return false;
177	}
178
179	pid = StartSubprocess(path_, argv, GetEnvP(), /* stdin */ outfd[0],
180	/* stdout */ infd[1]);
181	if (pid < 0) {
182	internal_close(infd[0]);
183	internal_close(outfd[1]);
184	return false;
185	}
186
187	input_fd_ = infd[0];
188	output_fd_ = outfd[1];
189	}
190
191	CHECK_GT(pid, 0);
192
193	// Check that symbolizer subprocess started successfully.
194	SleepForMillis(kSymbolizerStartupTimeMillis);
195	if (!IsProcessRunning(pid)) {
196	// Either waitpid failed, or child has already exited.
197	Report("WARNING: external symbolizer didn't start up correctly!\n");
198	return false;
199	}
200
201	return true;
202	}
203
204	class Addr2LineProcess : public SymbolizerProcess {
205	public:
206	Addr2LineProcess(const char path, const char module_name)
207	: SymbolizerProcess(path), module_name_(internal_strdup(module_name)) {}
208
209	const char *module_name() const { return module_name_; }
210
211	private:
212	void GetArgV(const char *path_to_binary,
213	const char *(&argv)[kArgVMax]) const override {
214	int i = 0;
215	argv[i++] = path_to_binary;
216	argv[i++] = "-iCfe";
217	argv[i++] = module_name_;
218	argv[i++] = nullptr;
219	}
220
221	bool ReachedEndOfOutput(const char *buffer, uptr length) const override;
222
223	bool ReadFromSymbolizer(char *buffer, uptr max_length) override {
224	if (!SymbolizerProcess::ReadFromSymbolizer(buffer, max_length))
225	return false;
226	// The returned buffer is empty when output is valid, but exceeds
227	// max_length.
228	if (*buffer == '\0')
229	return true;
230	// We should cut out output_terminator_ at the end of given buffer,
231	// appended by addr2line to mark the end of its meaningful output.
232	// We cannot scan buffer from it's beginning, because it is legal for it
233	// to start with output_terminator_ in case given offset is invalid. So,
234	// scanning from second character.
235	char *garbage = internal_strstr(buffer + 1, output_terminator_);
236	// This should never be NULL since buffer must end up with
237	// output_terminator_.
238	CHECK(garbage);
239	// Trim the buffer.
240	garbage[0] = '\0';
241	return true;
242	}
243
244	const char *module_name_; // Owned, leaked.
245	static const char output_terminator_[];
246	};
247
248	const char Addr2LineProcess::output_terminator_[] = "??\n??:0\n";
249
250	bool Addr2LineProcess::ReachedEndOfOutput(const char *buffer,
251	uptr length) const {
252	const size_t kTerminatorLen = sizeof(output_terminator_) - 1;
253	// Skip, if we read just kTerminatorLen bytes, because Addr2Line output
254	// should consist at least of two pairs of lines:
255	// 1. First one, corresponding to given offset to be symbolized
256	// (may be equal to output_terminator_, if offset is not valid).
257	// 2. Second one for output_terminator_, itself to mark the end of output.
258	if (length <= kTerminatorLen) return false;
259	// Addr2Line output should end up with output_terminator_.
260	return !internal_memcmp(buffer + length - kTerminatorLen,
261	output_terminator_, kTerminatorLen);
262	}
263
264	class Addr2LinePool : public SymbolizerTool {
265	public:
266	explicit Addr2LinePool(const char *addr2line_path,
267	LowLevelAllocator *allocator)
268	: addr2line_path_(addr2line_path), allocator_(allocator) {
269	addr2line_pool_.reserve(16);
270	}
271
272	bool SymbolizePC(uptr addr, SymbolizedStack *stack) override {
273	if (const char *buf =
274	SendCommand(stack->info.module, stack->info.module_offset)) {
275	ParseSymbolizePCOutput(buf, stack);
276	return true;
277	}
278	return false;
279	}
280
281	bool SymbolizeData(uptr addr, DataInfo *info) override {
282	return false;
283	}
284
285	private:
286	const char SendCommand(const char module_name, uptr module_offset) {
287	Addr2LineProcess *addr2line = 0;
288	for (uptr i = 0; i < addr2line_pool_.size(); ++i) {
289	if (0 ==
290	internal_strcmp(module_name, addr2line_pool_[i]->module_name())) {
291	addr2line = addr2line_pool_[i];
292	break;
293	}
294	}
295	if (!addr2line) {
296	addr2line =
297	new(*allocator_) Addr2LineProcess(addr2line_path_, module_name);
298	addr2line_pool_.push_back(addr2line);
299	}
300	CHECK_EQ(0, internal_strcmp(module_name, addr2line->module_name()));
301	char buffer[kBufferSize];
302	internal_snprintf(buffer, kBufferSize, "0x%zx\n0x%zx\n",
303	module_offset, dummy_address_);
304	return addr2line->SendCommand(buffer);
305	}
306
307	static const uptr kBufferSize = 64;
308	const char *addr2line_path_;
309	LowLevelAllocator *allocator_;
310	InternalMmapVector<Addr2LineProcess*> addr2line_pool_;
311	static const uptr dummy_address_ =
312	FIRST_32_SECOND_64(UINT32_MAX, UINT64_MAX);
313	};
314
315	#if SANITIZER_SUPPORTS_WEAK_HOOKS
316	extern "C" {
317	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE bool
318	__sanitizer_symbolize_code(const char *ModuleName, u64 ModuleOffset,
319	char *Buffer, int MaxLength,
320	bool SymbolizeInlineFrames);
321	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
322	bool __sanitizer_symbolize_data(const char *ModuleName, u64 ModuleOffset,
323	char *Buffer, int MaxLength);
324	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
325	void __sanitizer_symbolize_flush();
326	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
327	int __sanitizer_symbolize_demangle(const char Name, char Buffer,
328	int MaxLength);
329	} // extern "C"
330
331	class InternalSymbolizer : public SymbolizerTool {
332	public:
333	static InternalSymbolizer get(LowLevelAllocator alloc) {
334	if (__sanitizer_symbolize_code != 0 &&
335	__sanitizer_symbolize_data != 0) {
336	return new(*alloc) InternalSymbolizer();
337	}
338	return 0;
339	}
340
341	bool SymbolizePC(uptr addr, SymbolizedStack *stack) override {
342	bool result = __sanitizer_symbolize_code(
343	stack->info.module, stack->info.module_offset, buffer_, kBufferSize,
344	common_flags()->symbolize_inline_frames);
345	if (result) ParseSymbolizePCOutput(buffer_, stack);
346	return result;
347	}
348
349	bool SymbolizeData(uptr addr, DataInfo *info) override {
350	bool result = __sanitizer_symbolize_data(info->module, info->module_offset,
351	buffer_, kBufferSize);
352	if (result) {
353	ParseSymbolizeDataOutput(buffer_, info);
354	info->start += (addr - info->module_offset); // Add the base address.
355	}
356	return result;
357	}
358
359	void Flush() override {
360	if (__sanitizer_symbolize_flush)
361	__sanitizer_symbolize_flush();
362	}
363
364	const char Demangle(const char name) override {
365	if (__sanitizer_symbolize_demangle) {
366	for (uptr res_length = 1024;
367	res_length <= InternalSizeClassMap::kMaxSize;) {
368	char res_buff = static_cast<char>(InternalAlloc(res_length));
369	uptr req_length =
370	__sanitizer_symbolize_demangle(name, res_buff, res_length);
371	if (req_length > res_length) {
372	res_length = req_length + 1;
373	InternalFree(res_buff);
374	continue;
375	}
376	return res_buff;
377	}
378	}
379	return name;
380	}
381
382	private:
383	InternalSymbolizer() { }
384
385	static const int kBufferSize = 16 * 1024;
386	char buffer_[kBufferSize];
387	};
388	#else // SANITIZER_SUPPORTS_WEAK_HOOKS
389
390	class InternalSymbolizer : public SymbolizerTool {
391	public:
392	static InternalSymbolizer get(LowLevelAllocator alloc) { return 0; }
393	};
394
395	#endif // SANITIZER_SUPPORTS_WEAK_HOOKS
396
397	const char Symbolizer::PlatformDemangle(const char name) {
398	return DemangleSwiftAndCXX(name);
399	}
400
401	static SymbolizerTool ChooseExternalSymbolizer(LowLevelAllocator allocator) {
402	const char *path = common_flags()->external_symbolizer_path;
403	const char *binary_name = path ? StripModuleName(path) : "";
404	if (path && path[0] == '\0') {
405	VReport(2, "External symbolizer is explicitly disabled.\n");
406	return nullptr;
407	} else if (!internal_strcmp(binary_name, "llvm-symbolizer")) {
408	VReport(2, "Using llvm-symbolizer at user-specified path: %s\n", path);
409	return new(*allocator) LLVMSymbolizer(path, allocator);
410	} else if (!internal_strcmp(binary_name, "atos")) {
411	#if SANITIZER_MAC
412	VReport(2, "Using atos at user-specified path: %s\n", path);
413	return new(*allocator) AtosSymbolizer(path, allocator);
414	#else // SANITIZER_MAC
415	Report("ERROR: Using `atos` is only supported on Darwin.\n");
416	Die();
417	#endif // SANITIZER_MAC
418	} else if (!internal_strcmp(binary_name, "addr2line")) {
419	VReport(2, "Using addr2line at user-specified path: %s\n", path);
420	return new(*allocator) Addr2LinePool(path, allocator);
421	} else if (path) {
422	Report("ERROR: External symbolizer path is set to '%s' which isn't "
423	"a known symbolizer. Please set the path to the llvm-symbolizer "
424	"binary or other known tool.\n", path);
425	Die();
426	}
427
428	// Otherwise symbolizer program is unknown, let's search $PATH
429	CHECK(path == nullptr);
430	#if SANITIZER_MAC
431	if (const char *found_path = FindPathToBinary("atos")) {
432	VReport(2, "Using atos found at: %s\n", found_path);
433	return new(*allocator) AtosSymbolizer(found_path, allocator);
434	}
435	#endif // SANITIZER_MAC
436	if (const char *found_path = FindPathToBinary("llvm-symbolizer")) {
437	VReport(2, "Using llvm-symbolizer found at: %s\n", found_path);
438	return new(*allocator) LLVMSymbolizer(found_path, allocator);
439	}
440	if (common_flags()->allow_addr2line) {
441	if (const char *found_path = FindPathToBinary("addr2line")) {
442	VReport(2, "Using addr2line found at: %s\n", found_path);
443	return new(*allocator) Addr2LinePool(found_path, allocator);
444	}
445	}
446	return nullptr;
447	}
448
449	static void ChooseSymbolizerTools(IntrusiveList<SymbolizerTool> *list,
450	LowLevelAllocator *allocator) {
451	if (!common_flags()->symbolize) {
452	VReport(2, "Symbolizer is disabled.\n");
453	return;
454	}
455	if (IsAllocatorOutOfMemory()) {
456	VReport(2, "Cannot use internal symbolizer: out of memory\n");
457	} else if (SymbolizerTool *tool = InternalSymbolizer::get(allocator)) {
458	VReport(2, "Using internal symbolizer.\n");
459	list->push_back(tool);
460	return;
461	}
462	if (SymbolizerTool *tool = LibbacktraceSymbolizer::get(allocator)) {
463	VReport(2, "Using libbacktrace symbolizer.\n");
464	list->push_back(tool);
465	return;
466	}
467
468	if (SymbolizerTool *tool = ChooseExternalSymbolizer(allocator)) {
469	list->push_back(tool);
470	}
471
472	#if SANITIZER_MAC
473	VReport(2, "Using dladdr symbolizer.\n");
474	list->push_back(new(*allocator) DlAddrSymbolizer());
475	#endif // SANITIZER_MAC
476	}
477
478	Symbolizer *Symbolizer::PlatformInit() {
479	IntrusiveList<SymbolizerTool> list;
480	list.clear();
481	ChooseSymbolizerTools(&list, &symbolizer_allocator_);
482	return new(symbolizer_allocator_) Symbolizer(list);
483	}
484
485	void Symbolizer::LateInitialize() {
486	Symbolizer::GetOrInit()->LateInitializeTools();
487	InitializeSwiftDemangler();
488	}
489
490	} // namespace __sanitizer
491
492	#endif // SANITIZER_POSIX

lib/tsan/sanitizer_common/sanitizer_symbolizer_report.cpp created+293

...	@@ -0,0 +1,293 @@
1	//===-- sanitizer_symbolizer_report.cpp -----------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	///
9	/// This file is shared between AddressSanitizer and other sanitizer run-time
10	/// libraries and implements symbolized reports related functions.
11	///
12	//===----------------------------------------------------------------------===//
13
14	#include "sanitizer_common.h"
15	#include "sanitizer_file.h"
16	#include "sanitizer_flags.h"
17	#include "sanitizer_procmaps.h"
18	#include "sanitizer_report_decorator.h"
19	#include "sanitizer_stacktrace.h"
20	#include "sanitizer_stacktrace_printer.h"
21	#include "sanitizer_symbolizer.h"
22
23	#if SANITIZER_POSIX
24	# include "sanitizer_posix.h"
25	# include <sys/mman.h>
26	#endif
27
28	namespace __sanitizer {
29
30	#if !SANITIZER_GO
31	void ReportErrorSummary(const char *error_type, const AddressInfo &info,
32	const char *alt_tool_name) {
33	if (!common_flags()->print_summary) return;
34	InternalScopedString buff(kMaxSummaryLength);
35	buff.append("%s ", error_type);
36	RenderFrame(&buff, "%L %F", 0, info, common_flags()->symbolize_vs_style,
37	common_flags()->strip_path_prefix);
38	ReportErrorSummary(buff.data(), alt_tool_name);
39	}
40	#endif
41
42	#if !SANITIZER_FUCHSIA
43
44	bool ReportFile::SupportsColors() {
45	SpinMutexLock l(mu);
46	ReopenIfNecessary();
47	return SupportsColoredOutput(fd);
48	}
49
50	static INLINE bool ReportSupportsColors() {
51	return report_file.SupportsColors();
52	}
53
54	#else // SANITIZER_FUCHSIA
55
56	// Fuchsia's logs always go through post-processing that handles colorization.
57	static INLINE bool ReportSupportsColors() { return true; }
58
59	#endif // !SANITIZER_FUCHSIA
60
61	bool ColorizeReports() {
62	// FIXME: Add proper Windows support to AnsiColorDecorator and re-enable color
63	// printing on Windows.
64	if (SANITIZER_WINDOWS)
65	return false;
66
67	const char *flag = common_flags()->color;
68	return internal_strcmp(flag, "always") == 0 \|\|
69	(internal_strcmp(flag, "auto") == 0 && ReportSupportsColors());
70	}
71
72	void ReportErrorSummary(const char error_type, const StackTrace stack,
73	const char *alt_tool_name) {
74	#if !SANITIZER_GO
75	if (!common_flags()->print_summary)
76	return;
77	if (stack->size == 0) {
78	ReportErrorSummary(error_type);
79	return;
80	}
81	// Currently, we include the first stack frame into the report summary.
82	// Maybe sometimes we need to choose another frame (e.g. skip memcpy/etc).
83	uptr pc = StackTrace::GetPreviousInstructionPc(stack->trace[0]);
84	SymbolizedStack *frame = Symbolizer::GetOrInit()->SymbolizePC(pc);
85	ReportErrorSummary(error_type, frame->info, alt_tool_name);
86	frame->ClearAll();
87	#endif
88	}
89
90	void ReportMmapWriteExec(int prot) {
91	#if SANITIZER_POSIX && (!SANITIZER_GO && !SANITIZER_ANDROID)
92	if ((prot & (PROT_WRITE \| PROT_EXEC)) != (PROT_WRITE \| PROT_EXEC))
93	return;
94
95	ScopedErrorReportLock l;
96	SanitizerCommonDecorator d;
97
98	InternalMmapVector<BufferedStackTrace> stack_buffer(1);
99	BufferedStackTrace *stack = stack_buffer.data();
100	stack->Reset();
101	uptr top = 0;
102	uptr bottom = 0;
103	GET_CALLER_PC_BP_SP;
104	(void)sp;
105	bool fast = common_flags()->fast_unwind_on_fatal;
106	if (StackTrace::WillUseFastUnwind(fast)) {
107	GetThreadStackTopAndBottom(false, &top, &bottom);
108	stack->Unwind(kStackTraceMax, pc, bp, nullptr, top, bottom, true);
109	} else {
110	stack->Unwind(kStackTraceMax, pc, 0, nullptr, 0, 0, false);
111	}
112
113	Printf("%s", d.Warning());
114	Report("WARNING: %s: writable-executable page usage\n", SanitizerToolName);
115	Printf("%s", d.Default());
116
117	stack->Print();
118	ReportErrorSummary("w-and-x-usage", stack);
119	#endif
120	}
121
122	#if !SANITIZER_FUCHSIA && !SANITIZER_RTEMS && !SANITIZER_GO
123	void StartReportDeadlySignal() {
124	// Write the first message using fd=2, just in case.
125	// It may actually fail to write in case stderr is closed.
126	CatastrophicErrorWrite(SanitizerToolName, internal_strlen(SanitizerToolName));
127	static const char kDeadlySignal[] = ":DEADLYSIGNAL\n";
128	CatastrophicErrorWrite(kDeadlySignal, sizeof(kDeadlySignal) - 1);
129	}
130
131	static void MaybeReportNonExecRegion(uptr pc) {
132	#if SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_NETBSD
133	MemoryMappingLayout proc_maps(/cache_enabled/ true);
134	MemoryMappedSegment segment;
135	while (proc_maps.Next(&segment)) {
136	if (pc >= segment.start && pc < segment.end && !segment.IsExecutable())
137	Report("Hint: PC is at a non-executable region. Maybe a wild jump?\n");
138	}
139	#endif
140	}
141
142	static void PrintMemoryByte(InternalScopedString str, const char before,
143	u8 byte) {
144	SanitizerCommonDecorator d;
145	str->append("%s%s%x%x%s ", before, d.MemoryByte(), byte >> 4, byte & 15,
146	d.Default());
147	}
148
149	static void MaybeDumpInstructionBytes(uptr pc) {
150	if (!common_flags()->dump_instruction_bytes \|\| (pc < GetPageSizeCached()))
151	return;
152	InternalScopedString str(1024);
153	str.append("First 16 instruction bytes at pc: ");
154	if (IsAccessibleMemoryRange(pc, 16)) {
155	for (int i = 0; i < 16; ++i) {
156	PrintMemoryByte(&str, "", ((u8 *)pc)[i]);
157	}
158	str.append("\n");
159	} else {
160	str.append("unaccessible\n");
161	}
162	Report("%s", str.data());
163	}
164
165	static void MaybeDumpRegisters(void *context) {
166	if (!common_flags()->dump_registers) return;
167	SignalContext::DumpAllRegisters(context);
168	}
169
170	static void ReportStackOverflowImpl(const SignalContext &sig, u32 tid,
171	UnwindSignalStackCallbackType unwind,
172	const void *unwind_context) {
173	SanitizerCommonDecorator d;
174	Printf("%s", d.Warning());
175	static const char kDescription[] = "stack-overflow";
176	Report("ERROR: %s: %s on address %p (pc %p bp %p sp %p T%d)\n",
177	SanitizerToolName, kDescription, (void )sig.addr, (void )sig.pc,
178	(void )sig.bp, (void )sig.sp, tid);
179	Printf("%s", d.Default());
180	InternalMmapVector<BufferedStackTrace> stack_buffer(1);
181	BufferedStackTrace *stack = stack_buffer.data();
182	stack->Reset();
183	unwind(sig, unwind_context, stack);
184	stack->Print();
185	ReportErrorSummary(kDescription, stack);
186	}
187
188	static void ReportDeadlySignalImpl(const SignalContext &sig, u32 tid,
189	UnwindSignalStackCallbackType unwind,
190	const void *unwind_context) {
191	SanitizerCommonDecorator d;
192	Printf("%s", d.Warning());
193	const char *description = sig.Describe();
194	if (sig.is_memory_access && !sig.is_true_faulting_addr)
195	Report("ERROR: %s: %s on unknown address (pc %p bp %p sp %p T%d)\n",
196	SanitizerToolName, description, (void )sig.pc, (void )sig.bp,
197	(void *)sig.sp, tid);
198	else
199	Report("ERROR: %s: %s on unknown address %p (pc %p bp %p sp %p T%d)\n",
200	SanitizerToolName, description, (void )sig.addr, (void )sig.pc,
201	(void )sig.bp, (void )sig.sp, tid);
202	Printf("%s", d.Default());
203	if (sig.pc < GetPageSizeCached())
204	Report("Hint: pc points to the zero page.\n");
205	if (sig.is_memory_access) {
206	const char *access_type =
207	sig.write_flag == SignalContext::WRITE
208	? "WRITE"
209	: (sig.write_flag == SignalContext::READ ? "READ" : "UNKNOWN");
210	Report("The signal is caused by a %s memory access.\n", access_type);
211	if (!sig.is_true_faulting_addr)
212	Report("Hint: this fault was caused by a dereference of a high value "
213	"address (see register values below). Dissassemble the provided "
214	"pc to learn which register was used.\n");
215	else if (sig.addr < GetPageSizeCached())
216	Report("Hint: address points to the zero page.\n");
217	}
218	MaybeReportNonExecRegion(sig.pc);
219	InternalMmapVector<BufferedStackTrace> stack_buffer(1);
220	BufferedStackTrace *stack = stack_buffer.data();
221	stack->Reset();
222	unwind(sig, unwind_context, stack);
223	stack->Print();
224	MaybeDumpInstructionBytes(sig.pc);
225	MaybeDumpRegisters(sig.context);
226	Printf("%s can not provide additional info.\n", SanitizerToolName);
227	ReportErrorSummary(description, stack);
228	}
229
230	void ReportDeadlySignal(const SignalContext &sig, u32 tid,
231	UnwindSignalStackCallbackType unwind,
232	const void *unwind_context) {
233	if (sig.IsStackOverflow())
234	ReportStackOverflowImpl(sig, tid, unwind, unwind_context);
235	else
236	ReportDeadlySignalImpl(sig, tid, unwind, unwind_context);
237	}
238
239	void HandleDeadlySignal(void siginfo, void context, u32 tid,
240	UnwindSignalStackCallbackType unwind,
241	const void *unwind_context) {
242	StartReportDeadlySignal();
243	ScopedErrorReportLock rl;
244	SignalContext sig(siginfo, context);
245	ReportDeadlySignal(sig, tid, unwind, unwind_context);
246	Report("ABORTING\n");
247	Die();
248	}
249
250	#endif // !SANITIZER_FUCHSIA && !SANITIZER_GO
251
252	static atomic_uintptr_t reporting_thread = {0};
253	static StaticSpinMutex CommonSanitizerReportMutex;
254
255	ScopedErrorReportLock::ScopedErrorReportLock() {
256	uptr current = GetThreadSelf();
257	for (;;) {
258	uptr expected = 0;
259	if (atomic_compare_exchange_strong(&reporting_thread, &expected, current,
260	memory_order_relaxed)) {
261	// We've claimed reporting_thread so proceed.
262	CommonSanitizerReportMutex.Lock();
263	return;
264	}
265
266	if (expected == current) {
267	// This is either asynch signal or nested error during error reporting.
268	// Fail simple to avoid deadlocks in Report().
269
270	// Can't use Report() here because of potential deadlocks in nested
271	// signal handlers.
272	CatastrophicErrorWrite(SanitizerToolName,
273	internal_strlen(SanitizerToolName));
274	static const char msg[] = ": nested bug in the same thread, aborting.\n";
275	CatastrophicErrorWrite(msg, sizeof(msg) - 1);
276
277	internal__exit(common_flags()->exitcode);
278	}
279
280	internal_sched_yield();
281	}
282	}
283
284	ScopedErrorReportLock::~ScopedErrorReportLock() {
285	CommonSanitizerReportMutex.Unlock();
286	atomic_store_relaxed(&reporting_thread, 0);
287	}
288
289	void ScopedErrorReportLock::CheckLocked() {
290	CommonSanitizerReportMutex.CheckLocked();
291	}
292
293	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_symbolizer_win.cpp created+318

...	@@ -0,0 +1,318 @@
1	//===-- sanitizer_symbolizer_win.cpp --------------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is shared between AddressSanitizer and ThreadSanitizer
10	// run-time libraries.
11	// Windows-specific implementation of symbolizer parts.
12	//===----------------------------------------------------------------------===//
13
14	#include "sanitizer_platform.h"
15	#if SANITIZER_WINDOWS
16
17	#include "sanitizer_dbghelp.h"
18	#include "sanitizer_symbolizer_internal.h"
19
20	namespace __sanitizer {
21
22	decltype(::StackWalk64) *StackWalk64;
23	decltype(::SymCleanup) *SymCleanup;
24	decltype(::SymFromAddr) *SymFromAddr;
25	decltype(::SymFunctionTableAccess64) *SymFunctionTableAccess64;
26	decltype(::SymGetLineFromAddr64) *SymGetLineFromAddr64;
27	decltype(::SymGetModuleBase64) *SymGetModuleBase64;
28	decltype(::SymGetSearchPathW) *SymGetSearchPathW;
29	decltype(::SymInitialize) *SymInitialize;
30	decltype(::SymSetOptions) *SymSetOptions;
31	decltype(::SymSetSearchPathW) *SymSetSearchPathW;
32	decltype(::UnDecorateSymbolName) *UnDecorateSymbolName;
33
34	namespace {
35
36	class WinSymbolizerTool : public SymbolizerTool {
37	public:
38	// The constructor is provided to avoid synthesized memsets.
39	WinSymbolizerTool() {}
40
41	bool SymbolizePC(uptr addr, SymbolizedStack *stack) override;
42	bool SymbolizeData(uptr addr, DataInfo *info) override {
43	return false;
44	}
45	const char Demangle(const char name) override;
46	};
47
48	bool is_dbghelp_initialized = false;
49
50	bool TrySymInitialize() {
51	SymSetOptions(SYMOPT_DEFERRED_LOADS \| SYMOPT_UNDNAME \| SYMOPT_LOAD_LINES);
52	return SymInitialize(GetCurrentProcess(), 0, TRUE);
53	// FIXME: We don't call SymCleanup() on exit yet - should we?
54	}
55
56	} // namespace
57
58	// Initializes DbgHelp library, if it's not yet initialized. Calls to this
59	// function should be synchronized with respect to other calls to DbgHelp API
60	// (e.g. from WinSymbolizerTool).
61	void InitializeDbgHelpIfNeeded() {
62	if (is_dbghelp_initialized)
63	return;
64
65	HMODULE dbghelp = LoadLibraryA("dbghelp.dll");
66	CHECK(dbghelp && "failed to load dbghelp.dll");
67
68	#define DBGHELP_IMPORT(name) \
69	do { \
70	name = \
71	reinterpret_cast<decltype(::name) *>(GetProcAddress(dbghelp, #name)); \
72	CHECK(name != nullptr); \
73	} while (0)
74	DBGHELP_IMPORT(StackWalk64);
75	DBGHELP_IMPORT(SymCleanup);
76	DBGHELP_IMPORT(SymFromAddr);
77	DBGHELP_IMPORT(SymFunctionTableAccess64);
78	DBGHELP_IMPORT(SymGetLineFromAddr64);
79	DBGHELP_IMPORT(SymGetModuleBase64);
80	DBGHELP_IMPORT(SymGetSearchPathW);
81	DBGHELP_IMPORT(SymInitialize);
82	DBGHELP_IMPORT(SymSetOptions);
83	DBGHELP_IMPORT(SymSetSearchPathW);
84	DBGHELP_IMPORT(UnDecorateSymbolName);
85	#undef DBGHELP_IMPORT
86
87	if (!TrySymInitialize()) {
88	// OK, maybe the client app has called SymInitialize already.
89	// That's a bit unfortunate for us as all the DbgHelp functions are
90	// single-threaded and we can't coordinate with the app.
91	// FIXME: Can we stop the other threads at this point?
92	// Anyways, we have to reconfigure stuff to make sure that SymInitialize
93	// has all the appropriate options set.
94	// Cross our fingers and reinitialize DbgHelp.
95	Report("* WARNING: Failed to initialize DbgHelp! *\n");
96	Report("* Most likely this means that the app is already *\n");
97	Report("* using DbgHelp, possibly with incompatible flags. *\n");
98	Report("* Due to technical reasons, symbolization might crash *\n");
99	Report("* or produce wrong results. *\n");
100	SymCleanup(GetCurrentProcess());
101	TrySymInitialize();
102	}
103	is_dbghelp_initialized = true;
104
105	// When an executable is run from a location different from the one where it
106	// was originally built, we may not see the nearby PDB files.
107	// To work around this, let's append the directory of the main module
108	// to the symbol search path. All the failures below are not fatal.
109	const size_t kSymPathSize = 2048;
110	static wchar_t path_buffer[kSymPathSize + 1 + MAX_PATH];
111	if (!SymGetSearchPathW(GetCurrentProcess(), path_buffer, kSymPathSize)) {
112	Report("* WARNING: Failed to SymGetSearchPathW *\n");
113	return;
114	}
115	size_t sz = wcslen(path_buffer);
116	if (sz) {
117	CHECK_EQ(0, wcscat_s(path_buffer, L";"));
118	sz++;
119	}
120	DWORD res = GetModuleFileNameW(NULL, path_buffer + sz, MAX_PATH);
121	if (res == 0 \|\| res == MAX_PATH) {
122	Report("* WARNING: Failed to getting the EXE directory *\n");
123	return;
124	}
125	// Write the zero character in place of the last backslash to get the
126	// directory of the main module at the end of path_buffer.
127	wchar_t *last_bslash = wcsrchr(path_buffer + sz, L'\\');
128	CHECK_NE(last_bslash, 0);
129	*last_bslash = L'\0';
130	if (!SymSetSearchPathW(GetCurrentProcess(), path_buffer)) {
131	Report("*** WARNING: Failed to SymSetSearchPathW\n");
132	return;
133	}
134	}
135
136	bool WinSymbolizerTool::SymbolizePC(uptr addr, SymbolizedStack *frame) {
137	InitializeDbgHelpIfNeeded();
138
139	// See http://msdn.microsoft.com/en-us/library/ms680578(VS.85).aspx
140	char buffer[sizeof(SYMBOL_INFO) + MAX_SYM_NAME * sizeof(CHAR)];
141	PSYMBOL_INFO symbol = (PSYMBOL_INFO)buffer;
142	symbol->SizeOfStruct = sizeof(SYMBOL_INFO);
143	symbol->MaxNameLen = MAX_SYM_NAME;
144	DWORD64 offset = 0;
145	BOOL got_objname = SymFromAddr(GetCurrentProcess(),
146	(DWORD64)addr, &offset, symbol);
147	if (!got_objname)
148	return false;
149
150	DWORD unused;
151	IMAGEHLP_LINE64 line_info;
152	line_info.SizeOfStruct = sizeof(IMAGEHLP_LINE64);
153	BOOL got_fileline = SymGetLineFromAddr64(GetCurrentProcess(), (DWORD64)addr,
154	&unused, &line_info);
155	frame->info.function = internal_strdup(symbol->Name);
156	frame->info.function_offset = (uptr)offset;
157	if (got_fileline) {
158	frame->info.file = internal_strdup(line_info.FileName);
159	frame->info.line = line_info.LineNumber;
160	}
161	// Only consider this a successful symbolization attempt if we got file info.
162	// Otherwise, try llvm-symbolizer.
163	return got_fileline;
164	}
165
166	const char WinSymbolizerTool::Demangle(const char name) {
167	CHECK(is_dbghelp_initialized);
168	static char demangle_buffer[1000];
169	if (name[0] == '\01' &&
170	UnDecorateSymbolName(name + 1, demangle_buffer, sizeof(demangle_buffer),
171	UNDNAME_NAME_ONLY))
172	return demangle_buffer;
173	else
174	return name;
175	}
176
177	const char Symbolizer::PlatformDemangle(const char name) {
178	return name;
179	}
180
181	namespace {
182	struct ScopedHandle {
183	ScopedHandle() : h_(nullptr) {}
184	explicit ScopedHandle(HANDLE h) : h_(h) {}
185	~ScopedHandle() {
186	if (h_)
187	::CloseHandle(h_);
188	}
189	HANDLE get() { return h_; }
190	HANDLE *receive() { return &h_; }
191	HANDLE release() {
192	HANDLE h = h_;
193	h_ = nullptr;
194	return h;
195	}
196	HANDLE h_;
197	};
198	} // namespace
199
200	bool SymbolizerProcess::StartSymbolizerSubprocess() {
201	// Create inherited pipes for stdin and stdout.
202	ScopedHandle stdin_read, stdin_write;
203	ScopedHandle stdout_read, stdout_write;
204	SECURITY_ATTRIBUTES attrs;
205	attrs.nLength = sizeof(SECURITY_ATTRIBUTES);
206	attrs.bInheritHandle = TRUE;
207	attrs.lpSecurityDescriptor = nullptr;
208	if (!::CreatePipe(stdin_read.receive(), stdin_write.receive(), &attrs, 0) \|\|
209	!::CreatePipe(stdout_read.receive(), stdout_write.receive(), &attrs, 0)) {
210	VReport(2, "WARNING: %s CreatePipe failed (error code: %d)\n",
211	SanitizerToolName, path_, GetLastError());
212	return false;
213	}
214
215	// Don't inherit the writing end of stdin or the reading end of stdout.
216	if (!SetHandleInformation(stdin_write.get(), HANDLE_FLAG_INHERIT, 0) \|\|
217	!SetHandleInformation(stdout_read.get(), HANDLE_FLAG_INHERIT, 0)) {
218	VReport(2, "WARNING: %s SetHandleInformation failed (error code: %d)\n",
219	SanitizerToolName, path_, GetLastError());
220	return false;
221	}
222
223	// Compute the command line. Wrap double quotes around everything.
224	const char *argv[kArgVMax];
225	GetArgV(path_, argv);
226	InternalScopedString command_line(kMaxPathLength * 3);
227	for (int i = 0; argv[i]; i++) {
228	const char *arg = argv[i];
229	int arglen = internal_strlen(arg);
230	// Check that tool command lines are simple and that complete escaping is
231	// unnecessary.
232	CHECK(!internal_strchr(arg, '"') && "quotes in args unsupported");
233	CHECK(!internal_strstr(arg, "\\\\") &&
234	"double backslashes in args unsupported");
235	CHECK(arglen > 0 && arg[arglen - 1] != '\\' &&
236	"args ending in backslash and empty args unsupported");
237	command_line.append("\"%s\" ", arg);
238	}
239	VReport(3, "Launching symbolizer command: %s\n", command_line.data());
240
241	// Launch llvm-symbolizer with stdin and stdout redirected.
242	STARTUPINFOA si;
243	memset(&si, 0, sizeof(si));
244	si.cb = sizeof(si);
245	si.dwFlags \|= STARTF_USESTDHANDLES;
246	si.hStdInput = stdin_read.get();
247	si.hStdOutput = stdout_write.get();
248	PROCESS_INFORMATION pi;
249	memset(&pi, 0, sizeof(pi));
250	if (!CreateProcessA(path_, // Executable
251	command_line.data(), // Command line
252	nullptr, // Process handle not inheritable
253	nullptr, // Thread handle not inheritable
254	TRUE, // Set handle inheritance to TRUE
255	0, // Creation flags
256	nullptr, // Use parent's environment block
257	nullptr, // Use parent's starting directory
258	&si, &pi)) {
259	VReport(2, "WARNING: %s failed to create process for %s (error code: %d)\n",
260	SanitizerToolName, path_, GetLastError());
261	return false;
262	}
263
264	// Process creation succeeded, so transfer handle ownership into the fields.
265	input_fd_ = stdout_read.release();
266	output_fd_ = stdin_write.release();
267
268	// The llvm-symbolizer process is responsible for quitting itself when the
269	// stdin pipe is closed, so we don't need these handles. Close them to prevent
270	// leaks. If we ever want to try to kill the symbolizer process from the
271	// parent, we'll want to hang on to these handles.
272	CloseHandle(pi.hProcess);
273	CloseHandle(pi.hThread);
274	return true;
275	}
276
277	static void ChooseSymbolizerTools(IntrusiveList<SymbolizerTool> *list,
278	LowLevelAllocator *allocator) {
279	if (!common_flags()->symbolize) {
280	VReport(2, "Symbolizer is disabled.\n");
281	return;
282	}
283
284	// Add llvm-symbolizer in case the binary has dwarf.
285	const char *user_path = common_flags()->external_symbolizer_path;
286	const char *path =
287	user_path ? user_path : FindPathToBinary("llvm-symbolizer.exe");
288	if (path) {
289	VReport(2, "Using llvm-symbolizer at %spath: %s\n",
290	user_path ? "user-specified " : "", path);
291	list->push_back(new(*allocator) LLVMSymbolizer(path, allocator));
292	} else {
293	if (user_path && user_path[0] == '\0') {
294	VReport(2, "External symbolizer is explicitly disabled.\n");
295	} else {
296	VReport(2, "External symbolizer is not present.\n");
297	}
298	}
299
300	// Add the dbghelp based symbolizer.
301	list->push_back(new(*allocator) WinSymbolizerTool());
302	}
303
304	Symbolizer *Symbolizer::PlatformInit() {
305	IntrusiveList<SymbolizerTool> list;
306	list.clear();
307	ChooseSymbolizerTools(&list, &symbolizer_allocator_);
308
309	return new(symbolizer_allocator_) Symbolizer(list);
310	}
311
312	void Symbolizer::LateInitialize() {
313	Symbolizer::GetOrInit()->LateInitializeTools();
314	}
315
316	} // namespace __sanitizer
317
318	#endif // _WIN32

lib/tsan/sanitizer_common/sanitizer_unwind_linux_libcdep.cpp created+180

...	@@ -0,0 +1,180 @@
1	//===-- sanitizer_unwind_linux_libcdep.cpp --------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file contains the unwind.h-based (aka "slow") stack unwinding routines
10	// available to the tools on Linux, Android, NetBSD, FreeBSD, and Solaris.
11	//===----------------------------------------------------------------------===//
12
13	#include "sanitizer_platform.h"
14	#if SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_NETBSD \|\| \
15	SANITIZER_SOLARIS
16	#include "sanitizer_common.h"
17	#include "sanitizer_stacktrace.h"
18
19	#if SANITIZER_ANDROID
20	#include <dlfcn.h> // for dlopen()
21	#endif
22
23	#if SANITIZER_FREEBSD
24	#define _GNU_SOURCE // to declare _Unwind_Backtrace() from <unwind.h>
25	#endif
26	#include <unwind.h>
27
28	namespace __sanitizer {
29
30	namespace {
31
32	//---------------------------- UnwindSlow --------------------------------------
33
34	typedef struct {
35	uptr absolute_pc;
36	uptr stack_top;
37	uptr stack_size;
38	} backtrace_frame_t;
39
40	extern "C" {
41	typedef void (acquire_my_map_info_list_func)();
42	typedef void (release_my_map_info_list_func)(void map);
43	typedef sptr (*unwind_backtrace_signal_arch_func)(
44	void siginfo, void sigcontext, void *map_info_list,
45	backtrace_frame_t *backtrace, uptr ignore_depth, uptr max_depth);
46	acquire_my_map_info_list_func acquire_my_map_info_list;
47	release_my_map_info_list_func release_my_map_info_list;
48	unwind_backtrace_signal_arch_func unwind_backtrace_signal_arch;
49	} // extern "C"
50
51	#if defined(__arm__) && !SANITIZER_NETBSD
52	// NetBSD uses dwarf EH
53	#define UNWIND_STOP _URC_END_OF_STACK
54	#define UNWIND_CONTINUE _URC_NO_REASON
55	#else
56	#define UNWIND_STOP _URC_NORMAL_STOP
57	#define UNWIND_CONTINUE _URC_NO_REASON
58	#endif
59
60	uptr Unwind_GetIP(struct _Unwind_Context *ctx) {
61	#if defined(__arm__) && !SANITIZER_MAC
62	uptr val;
63	_Unwind_VRS_Result res = _Unwind_VRS_Get(ctx, _UVRSC_CORE,
64	15 /* r15 = PC */, _UVRSD_UINT32, &val);
65	CHECK(res == _UVRSR_OK && "_Unwind_VRS_Get failed");
66	// Clear the Thumb bit.
67	return val & ~(uptr)1;
68	#else
69	return (uptr)_Unwind_GetIP(ctx);
70	#endif
71	}
72
73	struct UnwindTraceArg {
74	BufferedStackTrace *stack;
75	u32 max_depth;
76	};
77
78	_Unwind_Reason_Code Unwind_Trace(struct _Unwind_Context ctx, void param) {
79	UnwindTraceArg arg = (UnwindTraceArg)param;
80	CHECK_LT(arg->stack->size, arg->max_depth);
81	uptr pc = Unwind_GetIP(ctx);
82	const uptr kPageSize = GetPageSizeCached();
83	// Let's assume that any pointer in the 0th page (i.e. <0x1000 on i386 and
84	// x86_64) is invalid and stop unwinding here. If we're adding support for
85	// a platform where this isn't true, we need to reconsider this check.
86	if (pc < kPageSize) return UNWIND_STOP;
87	arg->stack->trace_buffer[arg->stack->size++] = pc;
88	if (arg->stack->size == arg->max_depth) return UNWIND_STOP;
89	return UNWIND_CONTINUE;
90	}
91
92	} // namespace
93
94	#if SANITIZER_ANDROID
95	void SanitizerInitializeUnwinder() {
96	if (AndroidGetApiLevel() >= ANDROID_LOLLIPOP_MR1) return;
97
98	// Pre-lollipop Android can not unwind through signal handler frames with
99	// libgcc unwinder, but it has a libcorkscrew.so library with the necessary
100	// workarounds.
101	void *p = dlopen("libcorkscrew.so", RTLD_LAZY);
102	if (!p) {
103	VReport(1,
104	"Failed to open libcorkscrew.so. You may see broken stack traces "
105	"in SEGV reports.");
106	return;
107	}
108	acquire_my_map_info_list =
109	(acquire_my_map_info_list_func)(uptr)dlsym(p, "acquire_my_map_info_list");
110	release_my_map_info_list =
111	(release_my_map_info_list_func)(uptr)dlsym(p, "release_my_map_info_list");
112	unwind_backtrace_signal_arch = (unwind_backtrace_signal_arch_func)(uptr)dlsym(
113	p, "unwind_backtrace_signal_arch");
114	if (!acquire_my_map_info_list \|\| !release_my_map_info_list \|\|
115	!unwind_backtrace_signal_arch) {
116	VReport(1,
117	"Failed to find one of the required symbols in libcorkscrew.so. "
118	"You may see broken stack traces in SEGV reports.");
119	acquire_my_map_info_list = 0;
120	unwind_backtrace_signal_arch = 0;
121	release_my_map_info_list = 0;
122	}
123	}
124	#endif
125
126	void BufferedStackTrace::UnwindSlow(uptr pc, u32 max_depth) {
127	CHECK_GE(max_depth, 2);
128	size = 0;
129	UnwindTraceArg arg = {this, Min(max_depth + 1, kStackTraceMax)};
130	_Unwind_Backtrace(Unwind_Trace, &arg);
131	// We need to pop a few frames so that pc is on top.
132	uptr to_pop = LocatePcInTrace(pc);
133	// trace_buffer[0] belongs to the current function so we always pop it,
134	// unless there is only 1 frame in the stack trace (1 frame is always better
135	// than 0!).
136	// 1-frame stacks don't normally happen, but this depends on the actual
137	// unwinder implementation (libgcc, libunwind, etc) which is outside of our
138	// control.
139	if (to_pop == 0 && size > 1)
140	to_pop = 1;
141	PopStackFrames(to_pop);
142	#if defined(__GNUC__) && defined(__sparc__)
143	// __builtin_return_address returns the address of the call instruction
144	// on the SPARC and not the return address, so we need to compensate.
145	trace_buffer[0] = GetNextInstructionPc(pc);
146	#else
147	trace_buffer[0] = pc;
148	#endif
149	}
150
151	void BufferedStackTrace::UnwindSlow(uptr pc, void *context, u32 max_depth) {
152	CHECK(context);
153	CHECK_GE(max_depth, 2);
154	if (!unwind_backtrace_signal_arch) {
155	UnwindSlow(pc, max_depth);
156	return;
157	}
158
159	void *map = acquire_my_map_info_list();
160	CHECK(map);
161	InternalMmapVector<backtrace_frame_t> frames(kStackTraceMax);
162	// siginfo argument appears to be unused.
163	sptr res = unwind_backtrace_signal_arch(/* siginfo */ 0, context, map,
164	frames.data(),
165	/* ignore_depth */ 0, max_depth);
166	release_my_map_info_list(map);
167	if (res < 0) return;
168	CHECK_LE((uptr)res, kStackTraceMax);
169
170	size = 0;
171	// +2 compensate for libcorkscrew unwinder returning addresses of call
172	// instructions instead of raw return addresses.
173	for (sptr i = 0; i < res; ++i)
174	trace_buffer[size++] = frames[i].absolute_pc + 2;
175	}
176
177	} // namespace __sanitizer
178
179	#endif // SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_NETBSD \|\|
180	// SANITIZER_SOLARIS

lib/tsan/sanitizer_common/sanitizer_unwind_win.cpp created+75

...	@@ -0,0 +1,75 @@
1	//===-- sanitizer_unwind_win.cpp ------------------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	/// Sanitizer unwind Windows specific functions.
10	//
11	//===----------------------------------------------------------------------===//
12
13	#include "sanitizer_platform.h"
14	#if SANITIZER_WINDOWS
15
16	#define WIN32_LEAN_AND_MEAN
17	#define NOGDI
18	#include <windows.h>
19
20	#include "sanitizer_dbghelp.h" // for StackWalk64
21	#include "sanitizer_stacktrace.h"
22	#include "sanitizer_symbolizer.h" // for InitializeDbgHelpIfNeeded
23
24	using namespace __sanitizer;
25
26	#if !SANITIZER_GO
27	void BufferedStackTrace::UnwindSlow(uptr pc, u32 max_depth) {
28	CHECK_GE(max_depth, 2);
29	// FIXME: CaptureStackBackTrace might be too slow for us.
30	// FIXME: Compare with StackWalk64.
31	// FIXME: Look at LLVMUnhandledExceptionFilter in Signals.inc
32	size = CaptureStackBackTrace(1, Min(max_depth, kStackTraceMax),
33	(void **)&trace_buffer[0], 0);
34	if (size == 0)
35	return;
36
37	// Skip the RTL frames by searching for the PC in the stacktrace.
38	uptr pc_location = LocatePcInTrace(pc);
39	PopStackFrames(pc_location);
40	}
41
42	void BufferedStackTrace::UnwindSlow(uptr pc, void *context, u32 max_depth) {
43	CHECK(context);
44	CHECK_GE(max_depth, 2);
45	CONTEXT ctx = (CONTEXT )context;
46	STACKFRAME64 stack_frame;
47	memset(&stack_frame, 0, sizeof(stack_frame));
48
49	InitializeDbgHelpIfNeeded();
50
51	size = 0;
52	#if defined(_WIN64)
53	int machine_type = IMAGE_FILE_MACHINE_AMD64;
54	stack_frame.AddrPC.Offset = ctx.Rip;
55	stack_frame.AddrFrame.Offset = ctx.Rbp;
56	stack_frame.AddrStack.Offset = ctx.Rsp;
57	#else
58	int machine_type = IMAGE_FILE_MACHINE_I386;
59	stack_frame.AddrPC.Offset = ctx.Eip;
60	stack_frame.AddrFrame.Offset = ctx.Ebp;
61	stack_frame.AddrStack.Offset = ctx.Esp;
62	#endif
63	stack_frame.AddrPC.Mode = AddrModeFlat;
64	stack_frame.AddrFrame.Mode = AddrModeFlat;
65	stack_frame.AddrStack.Mode = AddrModeFlat;
66	while (StackWalk64(machine_type, GetCurrentProcess(), GetCurrentThread(),
67	&stack_frame, &ctx, NULL, SymFunctionTableAccess64,
68	SymGetModuleBase64, NULL) &&
69	size < Min(max_depth, kStackTraceMax)) {
70	trace_buffer[size++] = (uptr)stack_frame.AddrPC.Offset;
71	}
72	}
73	#endif // #if !SANITIZER_GO
74
75	#endif // SANITIZER_WINDOWS

lib/tsan/tsan_interceptors_mac.cpp created+519

...	@@ -0,0 +1,519 @@
1	//===-- tsan_interceptors_mac.cpp -----------------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is a part of ThreadSanitizer (TSan), a race detector.
10	//
11	// Mac-specific interceptors.
12	//===----------------------------------------------------------------------===//
13
14	#include "sanitizer_common/sanitizer_platform.h"
15	#if SANITIZER_MAC
16
17	#include "interception/interception.h"
18	#include "tsan_interceptors.h"
19	#include "tsan_interface.h"
20	#include "tsan_interface_ann.h"
21	#include "sanitizer_common/sanitizer_addrhashmap.h"
22
23	#include <errno.h>
24	#include <libkern/OSAtomic.h>
25	#include <objc/objc-sync.h>
26	#include <os/lock.h>
27	#include <sys/ucontext.h>
28
29	#if defined(__has_include) && __has_include(<xpc/xpc.h>)
30	#include <xpc/xpc.h>
31	#endif // #if defined(__has_include) && __has_include(<xpc/xpc.h>)
32
33	typedef long long_t;
34
35	extern "C" {
36	int getcontext(ucontext_t *ucp) __attribute__((returns_twice));
37	int setcontext(const ucontext_t *ucp);
38	}
39
40	namespace __tsan {
41
42	// The non-barrier versions of OSAtomic* functions are semantically mo_relaxed,
43	// but the two variants (e.g. OSAtomicAdd32 and OSAtomicAdd32Barrier) are
44	// actually aliases of each other, and we cannot have different interceptors for
45	// them, because they're actually the same function. Thus, we have to stay
46	// conservative and treat the non-barrier versions as mo_acq_rel.
47	static const morder kMacOrderBarrier = mo_acq_rel;
48	static const morder kMacOrderNonBarrier = mo_acq_rel;
49
50	#define OSATOMIC_INTERCEPTOR(return_t, t, tsan_t, f, tsan_atomic_f, mo) \
51	TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \
52	SCOPED_TSAN_INTERCEPTOR(f, x, ptr); \
53	return tsan_atomic_f((volatile tsan_t *)ptr, x, mo); \
54	}
55
56	#define OSATOMIC_INTERCEPTOR_PLUS_X(return_t, t, tsan_t, f, tsan_atomic_f, mo) \
57	TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \
58	SCOPED_TSAN_INTERCEPTOR(f, x, ptr); \
59	return tsan_atomic_f((volatile tsan_t *)ptr, x, mo) + x; \
60	}
61
62	#define OSATOMIC_INTERCEPTOR_PLUS_1(return_t, t, tsan_t, f, tsan_atomic_f, mo) \
63	TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) { \
64	SCOPED_TSAN_INTERCEPTOR(f, ptr); \
65	return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) + 1; \
66	}
67
68	#define OSATOMIC_INTERCEPTOR_MINUS_1(return_t, t, tsan_t, f, tsan_atomic_f, \
69	mo) \
70	TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) { \
71	SCOPED_TSAN_INTERCEPTOR(f, ptr); \
72	return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) - 1; \
73	}
74
75	#define OSATOMIC_INTERCEPTORS_ARITHMETIC(f, tsan_atomic_f, m) \
76	m(int32_t, int32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f, \
77	kMacOrderNonBarrier) \
78	m(int32_t, int32_t, a32, f##32##Barrier, __tsan_atomic32_##tsan_atomic_f, \
79	kMacOrderBarrier) \
80	m(int64_t, int64_t, a64, f##64, __tsan_atomic64_##tsan_atomic_f, \
81	kMacOrderNonBarrier) \
82	m(int64_t, int64_t, a64, f##64##Barrier, __tsan_atomic64_##tsan_atomic_f, \
83	kMacOrderBarrier)
84
85	#define OSATOMIC_INTERCEPTORS_BITWISE(f, tsan_atomic_f, m, m_orig) \
86	m(int32_t, uint32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f, \
87	kMacOrderNonBarrier) \
88	m(int32_t, uint32_t, a32, f##32##Barrier, __tsan_atomic32_##tsan_atomic_f, \
89	kMacOrderBarrier) \
90	m_orig(int32_t, uint32_t, a32, f##32##Orig, __tsan_atomic32_##tsan_atomic_f, \
91	kMacOrderNonBarrier) \
92	m_orig(int32_t, uint32_t, a32, f##32##OrigBarrier, \
93	__tsan_atomic32_##tsan_atomic_f, kMacOrderBarrier)
94
95	OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicAdd, fetch_add,
96	OSATOMIC_INTERCEPTOR_PLUS_X)
97	OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicIncrement, fetch_add,
98	OSATOMIC_INTERCEPTOR_PLUS_1)
99	OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicDecrement, fetch_sub,
100	OSATOMIC_INTERCEPTOR_MINUS_1)
101	OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicOr, fetch_or, OSATOMIC_INTERCEPTOR_PLUS_X,
102	OSATOMIC_INTERCEPTOR)
103	OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicAnd, fetch_and,
104	OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR)
105	OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicXor, fetch_xor,
106	OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR)
107
108	#define OSATOMIC_INTERCEPTORS_CAS(f, tsan_atomic_f, tsan_t, t) \
109	TSAN_INTERCEPTOR(bool, f, t old_value, t new_value, t volatile *ptr) { \
110	SCOPED_TSAN_INTERCEPTOR(f, old_value, new_value, ptr); \
111	return tsan_atomic_f##_compare_exchange_strong( \
112	(volatile tsan_t )ptr, (tsan_t )&old_value, (tsan_t)new_value, \
113	kMacOrderNonBarrier, kMacOrderNonBarrier); \
114	} \
115	\
116	TSAN_INTERCEPTOR(bool, f##Barrier, t old_value, t new_value, \
117	t volatile *ptr) { \
118	SCOPED_TSAN_INTERCEPTOR(f##Barrier, old_value, new_value, ptr); \
119	return tsan_atomic_f##_compare_exchange_strong( \
120	(volatile tsan_t )ptr, (tsan_t )&old_value, (tsan_t)new_value, \
121	kMacOrderBarrier, kMacOrderNonBarrier); \
122	}
123
124	OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapInt, __tsan_atomic32, a32, int)
125	OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapLong, __tsan_atomic64, a64,
126	long_t)
127	OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapPtr, __tsan_atomic64, a64,
128	void *)
129	OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap32, __tsan_atomic32, a32,
130	int32_t)
131	OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap64, __tsan_atomic64, a64,
132	int64_t)
133
134	#define OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, mo) \
135	TSAN_INTERCEPTOR(bool, f, uint32_t n, volatile void *ptr) { \
136	SCOPED_TSAN_INTERCEPTOR(f, n, ptr); \
137	volatile char byte_ptr = ((volatile char )ptr) + (n >> 3); \
138	char bit = 0x80u >> (n & 7); \
139	char mask = clear ? ~bit : bit; \
140	char orig_byte = op((volatile a8 *)byte_ptr, mask, mo); \
141	return orig_byte & bit; \
142	}
143
144	#define OSATOMIC_INTERCEPTORS_BITOP(f, op, clear) \
145	OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, kMacOrderNonBarrier) \
146	OSATOMIC_INTERCEPTOR_BITOP(f##Barrier, op, clear, kMacOrderBarrier)
147
148	OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndSet, __tsan_atomic8_fetch_or, false)
149	OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndClear, __tsan_atomic8_fetch_and,
150	true)
151
152	TSAN_INTERCEPTOR(void, OSAtomicEnqueue, OSQueueHead list, void item,
153	size_t offset) {
154	SCOPED_TSAN_INTERCEPTOR(OSAtomicEnqueue, list, item, offset);
155	__tsan_release(item);
156	REAL(OSAtomicEnqueue)(list, item, offset);
157	}
158
159	TSAN_INTERCEPTOR(void , OSAtomicDequeue, OSQueueHead list, size_t offset) {
160	SCOPED_TSAN_INTERCEPTOR(OSAtomicDequeue, list, offset);
161	void *item = REAL(OSAtomicDequeue)(list, offset);
162	if (item) __tsan_acquire(item);
163	return item;
164	}
165
166	// OSAtomicFifoEnqueue and OSAtomicFifoDequeue are only on OS X.
167	#if !SANITIZER_IOS
168
169	TSAN_INTERCEPTOR(void, OSAtomicFifoEnqueue, OSFifoQueueHead list, void item,
170	size_t offset) {
171	SCOPED_TSAN_INTERCEPTOR(OSAtomicFifoEnqueue, list, item, offset);
172	__tsan_release(item);
173	REAL(OSAtomicFifoEnqueue)(list, item, offset);
174	}
175
176	TSAN_INTERCEPTOR(void , OSAtomicFifoDequeue, OSFifoQueueHead list,
177	size_t offset) {
178	SCOPED_TSAN_INTERCEPTOR(OSAtomicFifoDequeue, list, offset);
179	void *item = REAL(OSAtomicFifoDequeue)(list, offset);
180	if (item) __tsan_acquire(item);
181	return item;
182	}
183
184	#endif
185
186	TSAN_INTERCEPTOR(void, OSSpinLockLock, volatile OSSpinLock *lock) {
187	CHECK(!cur_thread()->is_dead);
188	if (!cur_thread()->is_inited) {
189	return REAL(OSSpinLockLock)(lock);
190	}
191	SCOPED_TSAN_INTERCEPTOR(OSSpinLockLock, lock);
192	REAL(OSSpinLockLock)(lock);
193	Acquire(thr, pc, (uptr)lock);
194	}
195
196	TSAN_INTERCEPTOR(bool, OSSpinLockTry, volatile OSSpinLock *lock) {
197	CHECK(!cur_thread()->is_dead);
198	if (!cur_thread()->is_inited) {
199	return REAL(OSSpinLockTry)(lock);
200	}
201	SCOPED_TSAN_INTERCEPTOR(OSSpinLockTry, lock);
202	bool result = REAL(OSSpinLockTry)(lock);
203	if (result)
204	Acquire(thr, pc, (uptr)lock);
205	return result;
206	}
207
208	TSAN_INTERCEPTOR(void, OSSpinLockUnlock, volatile OSSpinLock *lock) {
209	CHECK(!cur_thread()->is_dead);
210	if (!cur_thread()->is_inited) {
211	return REAL(OSSpinLockUnlock)(lock);
212	}
213	SCOPED_TSAN_INTERCEPTOR(OSSpinLockUnlock, lock);
214	Release(thr, pc, (uptr)lock);
215	REAL(OSSpinLockUnlock)(lock);
216	}
217
218	TSAN_INTERCEPTOR(void, os_lock_lock, void *lock) {
219	CHECK(!cur_thread()->is_dead);
220	if (!cur_thread()->is_inited) {
221	return REAL(os_lock_lock)(lock);
222	}
223	SCOPED_TSAN_INTERCEPTOR(os_lock_lock, lock);
224	REAL(os_lock_lock)(lock);
225	Acquire(thr, pc, (uptr)lock);
226	}
227
228	TSAN_INTERCEPTOR(bool, os_lock_trylock, void *lock) {
229	CHECK(!cur_thread()->is_dead);
230	if (!cur_thread()->is_inited) {
231	return REAL(os_lock_trylock)(lock);
232	}
233	SCOPED_TSAN_INTERCEPTOR(os_lock_trylock, lock);
234	bool result = REAL(os_lock_trylock)(lock);
235	if (result)
236	Acquire(thr, pc, (uptr)lock);
237	return result;
238	}
239
240	TSAN_INTERCEPTOR(void, os_lock_unlock, void *lock) {
241	CHECK(!cur_thread()->is_dead);
242	if (!cur_thread()->is_inited) {
243	return REAL(os_lock_unlock)(lock);
244	}
245	SCOPED_TSAN_INTERCEPTOR(os_lock_unlock, lock);
246	Release(thr, pc, (uptr)lock);
247	REAL(os_lock_unlock)(lock);
248	}
249
250	TSAN_INTERCEPTOR(void, os_unfair_lock_lock, os_unfair_lock_t lock) {
251	if (!cur_thread()->is_inited \|\| cur_thread()->is_dead) {
252	return REAL(os_unfair_lock_lock)(lock);
253	}
254	SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_lock, lock);
255	REAL(os_unfair_lock_lock)(lock);
256	Acquire(thr, pc, (uptr)lock);
257	}
258
259	TSAN_INTERCEPTOR(void, os_unfair_lock_lock_with_options, os_unfair_lock_t lock,
260	u32 options) {
261	if (!cur_thread()->is_inited \|\| cur_thread()->is_dead) {
262	return REAL(os_unfair_lock_lock_with_options)(lock, options);
263	}
264	SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_lock_with_options, lock, options);
265	REAL(os_unfair_lock_lock_with_options)(lock, options);
266	Acquire(thr, pc, (uptr)lock);
267	}
268
269	TSAN_INTERCEPTOR(bool, os_unfair_lock_trylock, os_unfair_lock_t lock) {
270	if (!cur_thread()->is_inited \|\| cur_thread()->is_dead) {
271	return REAL(os_unfair_lock_trylock)(lock);
272	}
273	SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_trylock, lock);
274	bool result = REAL(os_unfair_lock_trylock)(lock);
275	if (result)
276	Acquire(thr, pc, (uptr)lock);
277	return result;
278	}
279
280	TSAN_INTERCEPTOR(void, os_unfair_lock_unlock, os_unfair_lock_t lock) {
281	if (!cur_thread()->is_inited \|\| cur_thread()->is_dead) {
282	return REAL(os_unfair_lock_unlock)(lock);
283	}
284	SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_unlock, lock);
285	Release(thr, pc, (uptr)lock);
286	REAL(os_unfair_lock_unlock)(lock);
287	}
288
289	#if defined(__has_include) && __has_include(<xpc/xpc.h>)
290
291	TSAN_INTERCEPTOR(void, xpc_connection_set_event_handler,
292	xpc_connection_t connection, xpc_handler_t handler) {
293	SCOPED_TSAN_INTERCEPTOR(xpc_connection_set_event_handler, connection,
294	handler);
295	Release(thr, pc, (uptr)connection);
296	xpc_handler_t new_handler = ^(xpc_object_t object) {
297	{
298	SCOPED_INTERCEPTOR_RAW(xpc_connection_set_event_handler);
299	Acquire(thr, pc, (uptr)connection);
300	}
301	handler(object);
302	};
303	REAL(xpc_connection_set_event_handler)(connection, new_handler);
304	}
305
306	TSAN_INTERCEPTOR(void, xpc_connection_send_barrier, xpc_connection_t connection,
307	dispatch_block_t barrier) {
308	SCOPED_TSAN_INTERCEPTOR(xpc_connection_send_barrier, connection, barrier);
309	Release(thr, pc, (uptr)connection);
310	dispatch_block_t new_barrier = ^() {
311	{
312	SCOPED_INTERCEPTOR_RAW(xpc_connection_send_barrier);
313	Acquire(thr, pc, (uptr)connection);
314	}
315	barrier();
316	};
317	REAL(xpc_connection_send_barrier)(connection, new_barrier);
318	}
319
320	TSAN_INTERCEPTOR(void, xpc_connection_send_message_with_reply,
321	xpc_connection_t connection, xpc_object_t message,
322	dispatch_queue_t replyq, xpc_handler_t handler) {
323	SCOPED_TSAN_INTERCEPTOR(xpc_connection_send_message_with_reply, connection,
324	message, replyq, handler);
325	Release(thr, pc, (uptr)connection);
326	xpc_handler_t new_handler = ^(xpc_object_t object) {
327	{
328	SCOPED_INTERCEPTOR_RAW(xpc_connection_send_message_with_reply);
329	Acquire(thr, pc, (uptr)connection);
330	}
331	handler(object);
332	};
333	REAL(xpc_connection_send_message_with_reply)
334	(connection, message, replyq, new_handler);
335	}
336
337	TSAN_INTERCEPTOR(void, xpc_connection_cancel, xpc_connection_t connection) {
338	SCOPED_TSAN_INTERCEPTOR(xpc_connection_cancel, connection);
339	Release(thr, pc, (uptr)connection);
340	REAL(xpc_connection_cancel)(connection);
341	}
342
343	#endif // #if defined(__has_include) && __has_include(<xpc/xpc.h>)
344
345	// Determines whether the Obj-C object pointer is a tagged pointer. Tagged
346	// pointers encode the object data directly in their pointer bits and do not
347	// have an associated memory allocation. The Obj-C runtime uses tagged pointers
348	// to transparently optimize small objects.
349	static bool IsTaggedObjCPointer(id obj) {
350	const uptr kPossibleTaggedBits = 0x8000000000000001ull;
351	return ((uptr)obj & kPossibleTaggedBits) != 0;
352	}
353
354	// Returns an address which can be used to inform TSan about synchronization
355	// points (MutexLock/Unlock). The TSan infrastructure expects this to be a valid
356	// address in the process space. We do a small allocation here to obtain a
357	// stable address (the array backing the hash map can change). The memory is
358	// never free'd (leaked) and allocation and locking are slow, but this code only
359	// runs for @synchronized with tagged pointers, which is very rare.
360	static uptr GetOrCreateSyncAddress(uptr addr, ThreadState *thr, uptr pc) {
361	typedef AddrHashMap<uptr, 5> Map;
362	static Map Addresses;
363	Map::Handle h(&Addresses, addr);
364	if (h.created()) {
365	ThreadIgnoreBegin(thr, pc);
366	h = (uptr) user_alloc(thr, pc, /size=*/1);
367	ThreadIgnoreEnd(thr, pc);
368	}
369	return *h;
370	}
371
372	// Returns an address on which we can synchronize given an Obj-C object pointer.
373	// For normal object pointers, this is just the address of the object in memory.
374	// Tagged pointers are not backed by an actual memory allocation, so we need to
375	// synthesize a valid address.
376	static uptr SyncAddressForObjCObject(id obj, ThreadState *thr, uptr pc) {
377	if (IsTaggedObjCPointer(obj))
378	return GetOrCreateSyncAddress((uptr)obj, thr, pc);
379	return (uptr)obj;
380	}
381
382	TSAN_INTERCEPTOR(int, objc_sync_enter, id obj) {
383	SCOPED_TSAN_INTERCEPTOR(objc_sync_enter, obj);
384	if (!obj) return REAL(objc_sync_enter)(obj);
385	uptr addr = SyncAddressForObjCObject(obj, thr, pc);
386	MutexPreLock(thr, pc, addr, MutexFlagWriteReentrant);
387	int result = REAL(objc_sync_enter)(obj);
388	CHECK_EQ(result, OBJC_SYNC_SUCCESS);
389	MutexPostLock(thr, pc, addr, MutexFlagWriteReentrant);
390	return result;
391	}
392
393	TSAN_INTERCEPTOR(int, objc_sync_exit, id obj) {
394	SCOPED_TSAN_INTERCEPTOR(objc_sync_exit, obj);
395	if (!obj) return REAL(objc_sync_exit)(obj);
396	uptr addr = SyncAddressForObjCObject(obj, thr, pc);
397	MutexUnlock(thr, pc, addr);
398	int result = REAL(objc_sync_exit)(obj);
399	if (result != OBJC_SYNC_SUCCESS) MutexInvalidAccess(thr, pc, addr);
400	return result;
401	}
402
403	TSAN_INTERCEPTOR(int, swapcontext, ucontext_t oucp, const ucontext_t ucp) {
404	{
405	SCOPED_INTERCEPTOR_RAW(swapcontext, oucp, ucp);
406	}
407	// Bacause of swapcontext() semantics we have no option but to copy its
408	// impementation here
409	if (!oucp \|\| !ucp) {
410	errno = EINVAL;
411	return -1;
412	}
413	ThreadState *thr = cur_thread();
414	const int UCF_SWAPPED = 0x80000000;
415	oucp->uc_onstack &= ~UCF_SWAPPED;
416	thr->ignore_interceptors++;
417	int ret = getcontext(oucp);
418	if (!(oucp->uc_onstack & UCF_SWAPPED)) {
419	thr->ignore_interceptors--;
420	if (!ret) {
421	oucp->uc_onstack \|= UCF_SWAPPED;
422	ret = setcontext(ucp);
423	}
424	}
425	return ret;
426	}
427
428	// On macOS, libc++ is always linked dynamically, so intercepting works the
429	// usual way.
430	#define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR
431
432	namespace {
433	struct fake_shared_weak_count {
434	volatile a64 shared_owners;
435	volatile a64 shared_weak_owners;
436	virtual void _unused_0x0() = 0;
437	virtual void _unused_0x8() = 0;
438	virtual void on_zero_shared() = 0;
439	virtual void _unused_0x18() = 0;
440	virtual void on_zero_shared_weak() = 0;
441	};
442	} // namespace
443
444	// The following code adds libc++ interceptors for:
445	// void __shared_weak_count::__release_shared() _NOEXCEPT;
446	// bool __shared_count::__release_shared() _NOEXCEPT;
447	// Shared and weak pointers in C++ maintain reference counts via atomics in
448	// libc++.dylib, which are TSan-invisible, and this leads to false positives in
449	// destructor code. These interceptors re-implements the whole functions so that
450	// the mo_acq_rel semantics of the atomic decrement are visible.
451	//
452	// Unfortunately, the interceptors cannot simply Acquire/Release some sync
453	// object and call the original function, because it would have a race between
454	// the sync and the destruction of the object. Calling both under a lock will
455	// not work because the destructor can invoke this interceptor again (and even
456	// in a different thread, so recursive locks don't help).
457
458	STDCXX_INTERCEPTOR(void, _ZNSt3__119__shared_weak_count16__release_sharedEv,
459	fake_shared_weak_count *o) {
460	if (!flags()->shared_ptr_interceptor)
461	return REAL(_ZNSt3__119__shared_weak_count16__release_sharedEv)(o);
462
463	SCOPED_TSAN_INTERCEPTOR(_ZNSt3__119__shared_weak_count16__release_sharedEv,
464	o);
465	if (__tsan_atomic64_fetch_add(&o->shared_owners, -1, mo_release) == 0) {
466	Acquire(thr, pc, (uptr)&o->shared_owners);
467	o->on_zero_shared();
468	if (__tsan_atomic64_fetch_add(&o->shared_weak_owners, -1, mo_release) ==
469	0) {
470	Acquire(thr, pc, (uptr)&o->shared_weak_owners);
471	o->on_zero_shared_weak();
472	}
473	}
474	}
475
476	STDCXX_INTERCEPTOR(bool, _ZNSt3__114__shared_count16__release_sharedEv,
477	fake_shared_weak_count *o) {
478	if (!flags()->shared_ptr_interceptor)
479	return REAL(_ZNSt3__114__shared_count16__release_sharedEv)(o);
480
481	SCOPED_TSAN_INTERCEPTOR(_ZNSt3__114__shared_count16__release_sharedEv, o);
482	if (__tsan_atomic64_fetch_add(&o->shared_owners, -1, mo_release) == 0) {
483	Acquire(thr, pc, (uptr)&o->shared_owners);
484	o->on_zero_shared();
485	return true;
486	}
487	return false;
488	}
489
490	namespace {
491	struct call_once_callback_args {
492	void (orig_func)(void arg);
493	void *orig_arg;
494	void *flag;
495	};
496
497	void call_once_callback_wrapper(void *arg) {
498	call_once_callback_args new_args = (call_once_callback_args )arg;
499	new_args->orig_func(new_args->orig_arg);
500	__tsan_release(new_args->flag);
501	}
502	} // namespace
503
504	// This adds a libc++ interceptor for:
505	// void __call_once(volatile unsigned long&, void, void()(void*));
506	// C++11 call_once is implemented via an internal function __call_once which is
507	// inside libc++.dylib, and the atomic release store inside it is thus
508	// TSan-invisible. To avoid false positives, this interceptor wraps the callback
509	// function and performs an explicit Release after the user code has run.
510	STDCXX_INTERCEPTOR(void, _ZNSt3__111__call_onceERVmPvPFvS2_E, void *flag,
511	void arg, void (func)(void *arg)) {
512	call_once_callback_args new_args = {func, arg, flag};
513	REAL(_ZNSt3__111__call_onceERVmPvPFvS2_E)(flag, &new_args,
514	call_once_callback_wrapper);
515	}
516
517	} // namespace __tsan
518
519	#endif // SANITIZER_MAC

lib/tsan/tsan_interceptors_mach_vm.cpp created+52

...	@@ -0,0 +1,52 @@
1	//===-- tsan_interceptors_mach_vm.cpp -------------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is a part of ThreadSanitizer (TSan), a race detector.
10	//
11	// Interceptors for mach_vm_* user space memory routines on Darwin.
12	//===----------------------------------------------------------------------===//
13
14	#include "interception/interception.h"
15	#include "tsan_interceptors.h"
16	#include "tsan_platform.h"
17
18	#include <mach/mach.h>
19
20	namespace __tsan {
21
22	static bool intersects_with_shadow(mach_vm_address_t *address,
23	mach_vm_size_t size, int flags) {
24	// VM_FLAGS_FIXED is 0x0, so we have to test for VM_FLAGS_ANYWHERE.
25	if (flags & VM_FLAGS_ANYWHERE) return false;
26	uptr ptr = *address;
27	return !IsAppMem(ptr) \|\| !IsAppMem(ptr + size - 1);
28	}
29
30	TSAN_INTERCEPTOR(kern_return_t, mach_vm_allocate, vm_map_t target,
31	mach_vm_address_t *address, mach_vm_size_t size, int flags) {
32	SCOPED_TSAN_INTERCEPTOR(mach_vm_allocate, target, address, size, flags);
33	if (target != mach_task_self())
34	return REAL(mach_vm_allocate)(target, address, size, flags);
35	if (intersects_with_shadow(address, size, flags))
36	return KERN_NO_SPACE;
37	kern_return_t res = REAL(mach_vm_allocate)(target, address, size, flags);
38	if (res == KERN_SUCCESS)
39	MemoryRangeImitateWriteOrResetRange(thr, pc, *address, size);
40	return res;
41	}
42
43	TSAN_INTERCEPTOR(kern_return_t, mach_vm_deallocate, vm_map_t target,
44	mach_vm_address_t address, mach_vm_size_t size) {
45	SCOPED_TSAN_INTERCEPTOR(mach_vm_deallocate, target, address, size);
46	if (target != mach_task_self())
47	return REAL(mach_vm_deallocate)(target, address, size);
48	UnmapShadow(thr, address, size);
49	return REAL(mach_vm_deallocate)(target, address, size);
50	}
51
52	} // namespace __tsan

lib/tsan/tsan_platform_linux.cpp created+517

...	@@ -0,0 +1,517 @@
1	//===-- tsan_platform_linux.cpp -------------------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is a part of ThreadSanitizer (TSan), a race detector.
10	//
11	// Linux- and BSD-specific code.
12	//===----------------------------------------------------------------------===//
13
14	#include "sanitizer_common/sanitizer_platform.h"
15	#if SANITIZER_LINUX \|\| SANITIZER_FREEBSD \|\| SANITIZER_NETBSD \|\| \
16	SANITIZER_OPENBSD
17
18	#include "sanitizer_common/sanitizer_common.h"
19	#include "sanitizer_common/sanitizer_libc.h"
20	#include "sanitizer_common/sanitizer_linux.h"
21	#include "sanitizer_common/sanitizer_platform_limits_netbsd.h"
22	#include "sanitizer_common/sanitizer_platform_limits_openbsd.h"
23	#include "sanitizer_common/sanitizer_platform_limits_posix.h"
24	#include "sanitizer_common/sanitizer_posix.h"
25	#include "sanitizer_common/sanitizer_procmaps.h"
26	#include "sanitizer_common/sanitizer_stackdepot.h"
27	#include "sanitizer_common/sanitizer_stoptheworld.h"
28	#include "tsan_flags.h"
29	#include "tsan_platform.h"
30	#include "tsan_rtl.h"
31
32	#include <fcntl.h>
33	#include <pthread.h>
34	#include <signal.h>
35	#include <stdio.h>
36	#include <stdlib.h>
37	#include <string.h>
38	#include <stdarg.h>
39	#include <sys/mman.h>
40	#if SANITIZER_LINUX
41	#include <sys/personality.h>
42	#include <setjmp.h>
43	#endif
44	#include <sys/syscall.h>
45	#include <sys/socket.h>
46	#include <sys/time.h>
47	#include <sys/types.h>
48	#include <sys/resource.h>
49	#include <sys/stat.h>
50	#include <unistd.h>
51	#include <sched.h>
52	#include <dlfcn.h>
53	#if SANITIZER_LINUX
54	#define __need_res_state
55	#include <resolv.h>
56	#endif
57
58	#ifdef sa_handler
59	# undef sa_handler
60	#endif
61
62	#ifdef sa_sigaction
63	# undef sa_sigaction
64	#endif
65
66	#if SANITIZER_FREEBSD
67	extern "C" void *__libc_stack_end;
68	void *__libc_stack_end = 0;
69	#endif
70
71	#if SANITIZER_LINUX && defined(__aarch64__) && !SANITIZER_GO
72	# define INIT_LONGJMP_XOR_KEY 1
73	#else
74	# define INIT_LONGJMP_XOR_KEY 0
75	#endif
76
77	#if INIT_LONGJMP_XOR_KEY
78	#include "interception/interception.h"
79	// Must be declared outside of other namespaces.
80	DECLARE_REAL(int, _setjmp, void *env)
81	#endif
82
83	namespace __tsan {
84
85	#if INIT_LONGJMP_XOR_KEY
86	static void InitializeLongjmpXorKey();
87	static uptr longjmp_xor_key;
88	#endif
89
90	#ifdef TSAN_RUNTIME_VMA
91	// Runtime detected VMA size.
92	uptr vmaSize;
93	#endif
94
95	enum {
96	MemTotal = 0,
97	MemShadow = 1,
98	MemMeta = 2,
99	MemFile = 3,
100	MemMmap = 4,
101	MemTrace = 5,
102	MemHeap = 6,
103	MemOther = 7,
104	MemCount = 8,
105	};
106
107	void FillProfileCallback(uptr p, uptr rss, bool file,
108	uptr *mem, uptr stats_size) {
109	mem[MemTotal] += rss;
110	if (p >= ShadowBeg() && p < ShadowEnd())
111	mem[MemShadow] += rss;
112	else if (p >= MetaShadowBeg() && p < MetaShadowEnd())
113	mem[MemMeta] += rss;
114	#if !SANITIZER_GO
115	else if (p >= HeapMemBeg() && p < HeapMemEnd())
116	mem[MemHeap] += rss;
117	else if (p >= LoAppMemBeg() && p < LoAppMemEnd())
118	mem[file ? MemFile : MemMmap] += rss;
119	else if (p >= HiAppMemBeg() && p < HiAppMemEnd())
120	mem[file ? MemFile : MemMmap] += rss;
121	#else
122	else if (p >= AppMemBeg() && p < AppMemEnd())
123	mem[file ? MemFile : MemMmap] += rss;
124	#endif
125	else if (p >= TraceMemBeg() && p < TraceMemEnd())
126	mem[MemTrace] += rss;
127	else
128	mem[MemOther] += rss;
129	}
130
131	void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive) {
132	uptr mem[MemCount];
133	internal_memset(mem, 0, sizeof(mem[0]) * MemCount);
134	__sanitizer::GetMemoryProfile(FillProfileCallback, mem, 7);
135	StackDepotStats *stacks = StackDepotGetStats();
136	internal_snprintf(buf, buf_size,
137	"RSS %zd MB: shadow:%zd meta:%zd file:%zd mmap:%zd"
138	" trace:%zd heap:%zd other:%zd stacks=%zd[%zd] nthr=%zd/%zd\n",
139	mem[MemTotal] >> 20, mem[MemShadow] >> 20, mem[MemMeta] >> 20,
140	mem[MemFile] >> 20, mem[MemMmap] >> 20, mem[MemTrace] >> 20,
141	mem[MemHeap] >> 20, mem[MemOther] >> 20,
142	stacks->allocated >> 20, stacks->n_uniq_ids,
143	nlive, nthread);
144	}
145
146	#if SANITIZER_LINUX
147	void FlushShadowMemoryCallback(
148	const SuspendedThreadsList &suspended_threads_list,
149	void *argument) {
150	ReleaseMemoryPagesToOS(ShadowBeg(), ShadowEnd());
151	}
152	#endif
153
154	void FlushShadowMemory() {
155	#if SANITIZER_LINUX
156	StopTheWorld(FlushShadowMemoryCallback, 0);
157	#endif
158	}
159
160	#if !SANITIZER_GO
161	// Mark shadow for .rodata sections with the special kShadowRodata marker.
162	// Accesses to .rodata can't race, so this saves time, memory and trace space.
163	static void MapRodata() {
164	// First create temp file.
165	const char *tmpdir = GetEnv("TMPDIR");
166	if (tmpdir == 0)
167	tmpdir = GetEnv("TEST_TMPDIR");
168	#ifdef P_tmpdir
169	if (tmpdir == 0)
170	tmpdir = P_tmpdir;
171	#endif
172	if (tmpdir == 0)
173	return;
174	char name[256];
175	internal_snprintf(name, sizeof(name), "%s/tsan.rodata.%d",
176	tmpdir, (int)internal_getpid());
177	uptr openrv = internal_open(name, O_RDWR \| O_CREAT \| O_EXCL, 0600);
178	if (internal_iserror(openrv))
179	return;
180	internal_unlink(name); // Unlink it now, so that we can reuse the buffer.
181	fd_t fd = openrv;
182	// Fill the file with kShadowRodata.
183	const uptr kMarkerSize = 512 * 1024 / sizeof(u64);
184	InternalMmapVector<u64> marker(kMarkerSize);
185	// volatile to prevent insertion of memset
186	for (volatile u64 *p = marker.data(); p < marker.data() + kMarkerSize; p++)
187	*p = kShadowRodata;
188	internal_write(fd, marker.data(), marker.size() * sizeof(u64));
189	// Map the file into memory.
190	uptr page = internal_mmap(0, GetPageSizeCached(), PROT_READ \| PROT_WRITE,
191	MAP_PRIVATE \| MAP_ANONYMOUS, fd, 0);
192	if (internal_iserror(page)) {
193	internal_close(fd);
194	return;
195	}
196	// Map the file into shadow of .rodata sections.
197	MemoryMappingLayout proc_maps(/cache_enabled/true);
198	// Reusing the buffer 'name'.
199	MemoryMappedSegment segment(name, ARRAY_SIZE(name));
200	while (proc_maps.Next(&segment)) {
201	if (segment.filename[0] != 0 && segment.filename[0] != '[' &&
202	segment.IsReadable() && segment.IsExecutable() &&
203	!segment.IsWritable() && IsAppMem(segment.start)) {
204	// Assume it's .rodata
205	char shadow_start = (char )MemToShadow(segment.start);
206	char shadow_end = (char )MemToShadow(segment.end);
207	for (char *p = shadow_start; p < shadow_end;
208	p += marker.size() * sizeof(u64)) {
209	internal_mmap(p, Min<uptr>(marker.size() * sizeof(u64), shadow_end - p),
210	PROT_READ, MAP_PRIVATE \| MAP_FIXED, fd, 0);
211	}
212	}
213	}
214	internal_close(fd);
215	}
216
217	void InitializeShadowMemoryPlatform() {
218	MapRodata();
219	}
220
221	#endif // #if !SANITIZER_GO
222
223	void InitializePlatformEarly() {
224	#ifdef TSAN_RUNTIME_VMA
225	vmaSize =
226	(MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1);
227	#if defined(__aarch64__)
228	# if !SANITIZER_GO
229	if (vmaSize != 39 && vmaSize != 42 && vmaSize != 48) {
230	Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
231	Printf("FATAL: Found %zd - Supported 39, 42 and 48\n", vmaSize);
232	Die();
233	}
234	#else
235	if (vmaSize != 48) {
236	Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
237	Printf("FATAL: Found %zd - Supported 48\n", vmaSize);
238	Die();
239	}
240	#endif
241	#elif defined(__powerpc64__)
242	# if !SANITIZER_GO
243	if (vmaSize != 44 && vmaSize != 46 && vmaSize != 47) {
244	Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
245	Printf("FATAL: Found %zd - Supported 44, 46, and 47\n", vmaSize);
246	Die();
247	}
248	# else
249	if (vmaSize != 46 && vmaSize != 47) {
250	Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
251	Printf("FATAL: Found %zd - Supported 46, and 47\n", vmaSize);
252	Die();
253	}
254	# endif
255	#endif
256	#endif
257	}
258
259	void InitializePlatform() {
260	DisableCoreDumperIfNecessary();
261
262	// Go maps shadow memory lazily and works fine with limited address space.
263	// Unlimited stack is not a problem as well, because the executable
264	// is not compiled with -pie.
265	#if !SANITIZER_GO
266	{
267	bool reexec = false;
268	// TSan doesn't play well with unlimited stack size (as stack
269	// overlaps with shadow memory). If we detect unlimited stack size,
270	// we re-exec the program with limited stack size as a best effort.
271	if (StackSizeIsUnlimited()) {
272	const uptr kMaxStackSize = 32 * 1024 * 1024;
273	VReport(1, "Program is run with unlimited stack size, which wouldn't "
274	"work with ThreadSanitizer.\n"
275	"Re-execing with stack size limited to %zd bytes.\n",
276	kMaxStackSize);
277	SetStackSizeLimitInBytes(kMaxStackSize);
278	reexec = true;
279	}
280
281	if (!AddressSpaceIsUnlimited()) {
282	Report("WARNING: Program is run with limited virtual address space,"
283	" which wouldn't work with ThreadSanitizer.\n");
284	Report("Re-execing with unlimited virtual address space.\n");
285	SetAddressSpaceUnlimited();
286	reexec = true;
287	}
288	#if SANITIZER_LINUX && defined(__aarch64__)
289	// After patch "arm64: mm: support ARCH_MMAP_RND_BITS." is introduced in
290	// linux kernel, the random gap between stack and mapped area is increased
291	// from 128M to 36G on 39-bit aarch64. As it is almost impossible to cover
292	// this big range, we should disable randomized virtual space on aarch64.
293	int old_personality = personality(0xffffffff);
294	if (old_personality != -1 && (old_personality & ADDR_NO_RANDOMIZE) == 0) {
295	VReport(1, "WARNING: Program is run with randomized virtual address "
296	"space, which wouldn't work with ThreadSanitizer.\n"
297	"Re-execing with fixed virtual address space.\n");
298	CHECK_NE(personality(old_personality \| ADDR_NO_RANDOMIZE), -1);
299	reexec = true;
300	}
301	// Initialize the xor key used in {sig}{set,long}jump.
302	InitializeLongjmpXorKey();
303	#endif
304	if (reexec)
305	ReExec();
306	}
307
308	CheckAndProtect();
309	InitTlsSize();
310	#endif // !SANITIZER_GO
311	}
312
313	#if !SANITIZER_GO
314	// Extract file descriptors passed to glibc internal __res_iclose function.
315	// This is required to properly "close" the fds, because we do not see internal
316	// closes within glibc. The code is a pure hack.
317	int ExtractResolvFDs(void state, int fds, int nfd) {
318	#if SANITIZER_LINUX && !SANITIZER_ANDROID
319	int cnt = 0;
320	struct __res_state statp = (struct __res_state)state;
321	for (int i = 0; i < MAXNS && cnt < nfd; i++) {
322	if (statp->_u._ext.nsaddrs[i] && statp->_u._ext.nssocks[i] != -1)
323	fds[cnt++] = statp->_u._ext.nssocks[i];
324	}
325	return cnt;
326	#else
327	return 0;
328	#endif
329	}
330
331	// Extract file descriptors passed via UNIX domain sockets.
332	// This is requried to properly handle "open" of these fds.
333	// see 'man recvmsg' and 'man 3 cmsg'.
334	int ExtractRecvmsgFDs(void msgp, int fds, int nfd) {
335	int res = 0;
336	msghdr msg = (msghdr)msgp;
337	struct cmsghdr *cmsg = CMSG_FIRSTHDR(msg);
338	for (; cmsg; cmsg = CMSG_NXTHDR(msg, cmsg)) {
339	if (cmsg->cmsg_level != SOL_SOCKET \|\| cmsg->cmsg_type != SCM_RIGHTS)
340	continue;
341	int n = (cmsg->cmsg_len - CMSG_LEN(0)) / sizeof(fds[0]);
342	for (int i = 0; i < n; i++) {
343	fds[res++] = ((int*)CMSG_DATA(cmsg))[i];
344	if (res == nfd)
345	return res;
346	}
347	}
348	return res;
349	}
350
351	// Reverse operation of libc stack pointer mangling
352	static uptr UnmangleLongJmpSp(uptr mangled_sp) {
353	#if defined(__x86_64__)
354	# if SANITIZER_LINUX
355	// Reverse of:
356	// xor %fs:0x30, %rsi
357	// rol $0x11, %rsi
358	uptr sp;
359	asm("ror $0x11, %0 \n"
360	"xor %%fs:0x30, %0 \n"
361	: "=r" (sp)
362	: "0" (mangled_sp));
363	return sp;
364	# else
365	return mangled_sp;
366	# endif
367	#elif defined(__aarch64__)
368	# if SANITIZER_LINUX
369	return mangled_sp ^ longjmp_xor_key;
370	# else
371	return mangled_sp;
372	# endif
373	#elif defined(__powerpc64__)
374	// Reverse of:
375	// ld r4, -28696(r13)
376	// xor r4, r3, r4
377	uptr xor_key;
378	asm("ld %0, -28696(%%r13)" : "=r" (xor_key));
379	return mangled_sp ^ xor_key;
380	#elif defined(__mips__)
381	return mangled_sp;
382	#else
383	#error "Unknown platform"
384	#endif
385	}
386
387	#ifdef __powerpc__
388	# define LONG_JMP_SP_ENV_SLOT 0
389	#elif SANITIZER_FREEBSD
390	# define LONG_JMP_SP_ENV_SLOT 2
391	#elif SANITIZER_NETBSD
392	# define LONG_JMP_SP_ENV_SLOT 6
393	#elif SANITIZER_LINUX
394	# ifdef __aarch64__
395	# define LONG_JMP_SP_ENV_SLOT 13
396	# elif defined(__mips64)
397	# define LONG_JMP_SP_ENV_SLOT 1
398	# else
399	# define LONG_JMP_SP_ENV_SLOT 6
400	# endif
401	#endif
402
403	uptr ExtractLongJmpSp(uptr *env) {
404	uptr mangled_sp = env[LONG_JMP_SP_ENV_SLOT];
405	return UnmangleLongJmpSp(mangled_sp);
406	}
407
408	#if INIT_LONGJMP_XOR_KEY
409	// GLIBC mangles the function pointers in jmp_buf (used in {set,long}*jmp
410	// functions) by XORing them with a random key. For AArch64 it is a global
411	// variable rather than a TCB one (as for x86_64/powerpc). We obtain the key by
412	// issuing a setjmp and XORing the SP pointer values to derive the key.
413	static void InitializeLongjmpXorKey() {
414	// 1. Call REAL(setjmp), which stores the mangled SP in env.
415	jmp_buf env;
416	REAL(_setjmp)(env);
417
418	// 2. Retrieve vanilla/mangled SP.
419	uptr sp;
420	asm("mov %0, sp" : "=r" (sp));
421	uptr mangled_sp = ((uptr *)&env)[LONG_JMP_SP_ENV_SLOT];
422
423	// 3. xor SPs to obtain key.
424	longjmp_xor_key = mangled_sp ^ sp;
425	}
426	#endif
427
428	void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size) {
429	// Check that the thr object is in tls;
430	const uptr thr_beg = (uptr)thr;
431	const uptr thr_end = (uptr)thr + sizeof(*thr);
432	CHECK_GE(thr_beg, tls_addr);
433	CHECK_LE(thr_beg, tls_addr + tls_size);
434	CHECK_GE(thr_end, tls_addr);
435	CHECK_LE(thr_end, tls_addr + tls_size);
436	// Since the thr object is huge, skip it.
437	MemoryRangeImitateWrite(thr, /pc=/2, tls_addr, thr_beg - tls_addr);
438	MemoryRangeImitateWrite(thr, /pc=/2, thr_end,
439	tls_addr + tls_size - thr_end);
440	}
441
442	// Note: this function runs with async signals enabled,
443	// so it must not touch any tsan state.
444	int call_pthread_cancel_with_cleanup(int(fn)(void c, void *m,
445	void abstime), void c, void m, void abstime,
446	void(cleanup)(void arg), void *arg) {
447	// pthread_cleanup_push/pop are hardcore macros mess.
448	// We can't intercept nor call them w/o including pthread.h.
449	int res;
450	pthread_cleanup_push(cleanup, arg);
451	res = fn(c, m, abstime);
452	pthread_cleanup_pop(0);
453	return res;
454	}
455	#endif // !SANITIZER_GO
456
457	#if !SANITIZER_GO
458	void ReplaceSystemMalloc() { }
459	#endif
460
461	#if !SANITIZER_GO
462	#if SANITIZER_ANDROID
463	// On Android, one thread can call intercepted functions after
464	// DestroyThreadState(), so add a fake thread state for "dead" threads.
465	static ThreadState *dead_thread_state = nullptr;
466
467	ThreadState *cur_thread() {
468	ThreadState* thr = reinterpret_cast<ThreadState>(get_android_tls_ptr());
469	if (thr == nullptr) {
470	__sanitizer_sigset_t emptyset;
471	internal_sigfillset(&emptyset);
472	__sanitizer_sigset_t oldset;
473	CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, &emptyset, &oldset));
474	thr = reinterpret_cast<ThreadState>(get_android_tls_ptr());
475	if (thr == nullptr) {
476	thr = reinterpret_cast<ThreadState*>(MmapOrDie(sizeof(ThreadState),
477	"ThreadState"));
478	*get_android_tls_ptr() = reinterpret_cast<uptr>(thr);
479	if (dead_thread_state == nullptr) {
480	dead_thread_state = reinterpret_cast<ThreadState*>(
481	MmapOrDie(sizeof(ThreadState), "ThreadState"));
482	dead_thread_state->fast_state.SetIgnoreBit();
483	dead_thread_state->ignore_interceptors = 1;
484	dead_thread_state->is_dead = true;
485	const_cast<int>(&dead_thread_state->tid) = -1;
486	CHECK_EQ(0, internal_mprotect(dead_thread_state, sizeof(ThreadState),
487	PROT_READ));
488	}
489	}
490	CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, &oldset, nullptr));
491	}
492	return thr;
493	}
494
495	void set_cur_thread(ThreadState *thr) {
496	*get_android_tls_ptr() = reinterpret_cast<uptr>(thr);
497	}
498
499	void cur_thread_finalize() {
500	__sanitizer_sigset_t emptyset;
501	internal_sigfillset(&emptyset);
502	__sanitizer_sigset_t oldset;
503	CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, &emptyset, &oldset));
504	ThreadState* thr = reinterpret_cast<ThreadState>(get_android_tls_ptr());
505	if (thr != dead_thread_state) {
506	*get_android_tls_ptr() = reinterpret_cast<uptr>(dead_thread_state);
507	UnmapOrDie(thr, sizeof(ThreadState));
508	}
509	CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, &oldset, nullptr));
510	}
511	#endif // SANITIZER_ANDROID
512	#endif // if !SANITIZER_GO
513
514	} // namespace __tsan
515
516	#endif // SANITIZER_LINUX \|\| SANITIZER_FREEBSD \|\| SANITIZER_NETBSD \|\|
517	// SANITIZER_OPENBSD

lib/tsan/tsan_platform_mac.cpp created+324

...	@@ -0,0 +1,324 @@
1	//===-- tsan_platform_mac.cpp ---------------------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is a part of ThreadSanitizer (TSan), a race detector.
10	//
11	// Mac-specific code.
12	//===----------------------------------------------------------------------===//
13
14	#include "sanitizer_common/sanitizer_platform.h"
15	#if SANITIZER_MAC
16
17	#include "sanitizer_common/sanitizer_atomic.h"
18	#include "sanitizer_common/sanitizer_common.h"
19	#include "sanitizer_common/sanitizer_libc.h"
20	#include "sanitizer_common/sanitizer_posix.h"
21	#include "sanitizer_common/sanitizer_procmaps.h"
22	#include "sanitizer_common/sanitizer_ptrauth.h"
23	#include "sanitizer_common/sanitizer_stackdepot.h"
24	#include "tsan_platform.h"
25	#include "tsan_rtl.h"
26	#include "tsan_flags.h"
27
28	#include <mach/mach.h>
29	#include <pthread.h>
30	#include <signal.h>
31	#include <stdio.h>
32	#include <stdlib.h>
33	#include <string.h>
34	#include <stdarg.h>
35	#include <sys/mman.h>
36	#include <sys/syscall.h>
37	#include <sys/time.h>
38	#include <sys/types.h>
39	#include <sys/resource.h>
40	#include <sys/stat.h>
41	#include <unistd.h>
42	#include <errno.h>
43	#include <sched.h>
44
45	namespace __tsan {
46
47	#if !SANITIZER_GO
48	static void SignalSafeGetOrAllocate(uptr dst, uptr size) {
49	atomic_uintptr_t a = (atomic_uintptr_t )dst;
50	void val = (void )atomic_load_relaxed(a);
51	atomic_signal_fence(memory_order_acquire); // Turns the previous load into
52	// acquire wrt signals.
53	if (UNLIKELY(val == nullptr)) {
54	val = (void *)internal_mmap(nullptr, size, PROT_READ \| PROT_WRITE,
55	MAP_PRIVATE \| MAP_ANON, -1, 0);
56	CHECK(val);
57	void *cmp = nullptr;
58	if (!atomic_compare_exchange_strong(a, (uintptr_t *)&cmp, (uintptr_t)val,
59	memory_order_acq_rel)) {
60	internal_munmap(val, size);
61	val = cmp;
62	}
63	}
64	return val;
65	}
66
67	// On OS X, accessing TLVs via __thread or manually by using pthread_key_* is
68	// problematic, because there are several places where interceptors are called
69	// when TLVs are not accessible (early process startup, thread cleanup, ...).
70	// The following provides a "poor man's TLV" implementation, where we use the
71	// shadow memory of the pointer returned by pthread_self() to store a pointer to
72	// the ThreadState object. The main thread's ThreadState is stored separately
73	// in a static variable, because we need to access it even before the
74	// shadow memory is set up.
75	static uptr main_thread_identity = 0;
76	ALIGNED(64) static char main_thread_state[sizeof(ThreadState)];
77	static ThreadState main_thread_state_loc = (ThreadState )main_thread_state;
78
79	// We cannot use pthread_self() before libpthread has been initialized. Our
80	// current heuristic for guarding this is checking `main_thread_identity` which
81	// is only assigned in `__tsan::InitializePlatform`.
82	static ThreadState **cur_thread_location() {
83	if (main_thread_identity == 0)
84	return &main_thread_state_loc;
85	uptr thread_identity = (uptr)pthread_self();
86	if (thread_identity == main_thread_identity)
87	return &main_thread_state_loc;
88	return (ThreadState **)MemToShadow(thread_identity);
89	}
90
91	ThreadState *cur_thread() {
92	return (ThreadState *)SignalSafeGetOrAllocate(
93	(uptr *)cur_thread_location(), sizeof(ThreadState));
94	}
95
96	void set_cur_thread(ThreadState *thr) {
97	*cur_thread_location() = thr;
98	}
99
100	// TODO(kuba.brecka): This is not async-signal-safe. In particular, we call
101	// munmap first and then clear `fake_tls`; if we receive a signal in between,
102	// handler will try to access the unmapped ThreadState.
103	void cur_thread_finalize() {
104	ThreadState **thr_state_loc = cur_thread_location();
105	if (thr_state_loc == &main_thread_state_loc) {
106	// Calling dispatch_main() or xpc_main() actually invokes pthread_exit to
107	// exit the main thread. Let's keep the main thread's ThreadState.
108	return;
109	}
110	internal_munmap(*thr_state_loc, sizeof(ThreadState));
111	*thr_state_loc = nullptr;
112	}
113	#endif
114
115	void FlushShadowMemory() {
116	}
117
118	static void RegionMemUsage(uptr start, uptr end, uptr res, uptr dirty) {
119	vm_address_t address = start;
120	vm_address_t end_address = end;
121	uptr resident_pages = 0;
122	uptr dirty_pages = 0;
123	while (address < end_address) {
124	vm_size_t vm_region_size;
125	mach_msg_type_number_t count = VM_REGION_EXTENDED_INFO_COUNT;
126	vm_region_extended_info_data_t vm_region_info;
127	mach_port_t object_name;
128	kern_return_t ret = vm_region_64(
129	mach_task_self(), &address, &vm_region_size, VM_REGION_EXTENDED_INFO,
130	(vm_region_info_t)&vm_region_info, &count, &object_name);
131	if (ret != KERN_SUCCESS) break;
132
133	resident_pages += vm_region_info.pages_resident;
134	dirty_pages += vm_region_info.pages_dirtied;
135
136	address += vm_region_size;
137	}
138	res = resident_pages GetPageSizeCached();
139	dirty = dirty_pages GetPageSizeCached();
140	}
141
142	void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive) {
143	uptr shadow_res, shadow_dirty;
144	uptr meta_res, meta_dirty;
145	uptr trace_res, trace_dirty;
146	RegionMemUsage(ShadowBeg(), ShadowEnd(), &shadow_res, &shadow_dirty);
147	RegionMemUsage(MetaShadowBeg(), MetaShadowEnd(), &meta_res, &meta_dirty);
148	RegionMemUsage(TraceMemBeg(), TraceMemEnd(), &trace_res, &trace_dirty);
149
150	#if !SANITIZER_GO
151	uptr low_res, low_dirty;
152	uptr high_res, high_dirty;
153	uptr heap_res, heap_dirty;
154	RegionMemUsage(LoAppMemBeg(), LoAppMemEnd(), &low_res, &low_dirty);
155	RegionMemUsage(HiAppMemBeg(), HiAppMemEnd(), &high_res, &high_dirty);
156	RegionMemUsage(HeapMemBeg(), HeapMemEnd(), &heap_res, &heap_dirty);
157	#else // !SANITIZER_GO
158	uptr app_res, app_dirty;
159	RegionMemUsage(AppMemBeg(), AppMemEnd(), &app_res, &app_dirty);
160	#endif
161
162	StackDepotStats *stacks = StackDepotGetStats();
163	internal_snprintf(buf, buf_size,
164	"shadow (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
165	"meta (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
166	"traces (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
167	#if !SANITIZER_GO
168	"low app (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
169	"high app (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
170	"heap (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
171	#else // !SANITIZER_GO
172	"app (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
173	#endif
174	"stacks: %zd unique IDs, %zd kB allocated\n"
175	"threads: %zd total, %zd live\n"
176	"------------------------------\n",
177	ShadowBeg(), ShadowEnd(), shadow_res / 1024, shadow_dirty / 1024,
178	MetaShadowBeg(), MetaShadowEnd(), meta_res / 1024, meta_dirty / 1024,
179	TraceMemBeg(), TraceMemEnd(), trace_res / 1024, trace_dirty / 1024,
180	#if !SANITIZER_GO
181	LoAppMemBeg(), LoAppMemEnd(), low_res / 1024, low_dirty / 1024,
182	HiAppMemBeg(), HiAppMemEnd(), high_res / 1024, high_dirty / 1024,
183	HeapMemBeg(), HeapMemEnd(), heap_res / 1024, heap_dirty / 1024,
184	#else // !SANITIZER_GO
185	AppMemBeg(), AppMemEnd(), app_res / 1024, app_dirty / 1024,
186	#endif
187	stacks->n_uniq_ids, stacks->allocated / 1024,
188	nthread, nlive);
189	}
190
191	#if !SANITIZER_GO
192	void InitializeShadowMemoryPlatform() { }
193
194	// On OS X, GCD worker threads are created without a call to pthread_create. We
195	// need to properly register these threads with ThreadCreate and ThreadStart.
196	// These threads don't have a parent thread, as they are created "spuriously".
197	// We're using a libpthread API that notifies us about a newly created thread.
198	// The `thread == pthread_self()` check indicates this is actually a worker
199	// thread. If it's just a regular thread, this hook is called on the parent
200	// thread.
201	typedef void (*pthread_introspection_hook_t)(unsigned int event,
202	pthread_t thread, void *addr,
203	size_t size);
204	extern "C" pthread_introspection_hook_t pthread_introspection_hook_install(
205	pthread_introspection_hook_t hook);
206	static const uptr PTHREAD_INTROSPECTION_THREAD_CREATE = 1;
207	static const uptr PTHREAD_INTROSPECTION_THREAD_TERMINATE = 3;
208	static pthread_introspection_hook_t prev_pthread_introspection_hook;
209	static void my_pthread_introspection_hook(unsigned int event, pthread_t thread,
210	void *addr, size_t size) {
211	if (event == PTHREAD_INTROSPECTION_THREAD_CREATE) {
212	if (thread == pthread_self()) {
213	// The current thread is a newly created GCD worker thread.
214	ThreadState *thr = cur_thread();
215	Processor *proc = ProcCreate();
216	ProcWire(proc, thr);
217	ThreadState *parent_thread_state = nullptr; // No parent.
218	int tid = ThreadCreate(parent_thread_state, 0, (uptr)thread, true);
219	CHECK_NE(tid, 0);
220	ThreadStart(thr, tid, GetTid(), ThreadType::Worker);
221	}
222	} else if (event == PTHREAD_INTROSPECTION_THREAD_TERMINATE) {
223	if (thread == pthread_self()) {
224	ThreadState *thr = cur_thread();
225	if (thr->tctx) {
226	DestroyThreadState();
227	}
228	}
229	}
230
231	if (prev_pthread_introspection_hook != nullptr)
232	prev_pthread_introspection_hook(event, thread, addr, size);
233	}
234	#endif
235
236	void InitializePlatformEarly() {
237	#if defined(__aarch64__)
238	uptr max_vm = GetMaxUserVirtualAddress() + 1;
239	if (max_vm != Mapping::kHiAppMemEnd) {
240	Printf("ThreadSanitizer: unsupported vm address limit %p, expected %p.\n",
241	max_vm, Mapping::kHiAppMemEnd);
242	Die();
243	}
244	#endif
245	}
246
247	static uptr longjmp_xor_key = 0;
248
249	void InitializePlatform() {
250	DisableCoreDumperIfNecessary();
251	#if !SANITIZER_GO
252	CheckAndProtect();
253
254	CHECK_EQ(main_thread_identity, 0);
255	main_thread_identity = (uptr)pthread_self();
256
257	prev_pthread_introspection_hook =
258	pthread_introspection_hook_install(&my_pthread_introspection_hook);
259	#endif
260
261	if (GetMacosAlignedVersion() >= MacosVersion(10, 14)) {
262	// Libsystem currently uses a process-global key; this might change.
263	const unsigned kTLSLongjmpXorKeySlot = 0x7;
264	longjmp_xor_key = (uptr)pthread_getspecific(kTLSLongjmpXorKeySlot);
265	}
266	}
267
268	#ifdef __aarch64__
269	# define LONG_JMP_SP_ENV_SLOT \
270	((GetMacosAlignedVersion() >= MacosVersion(10, 14)) ? 12 : 13)
271	#else
272	# define LONG_JMP_SP_ENV_SLOT 2
273	#endif
274
275	uptr ExtractLongJmpSp(uptr *env) {
276	uptr mangled_sp = env[LONG_JMP_SP_ENV_SLOT];
277	uptr sp = mangled_sp ^ longjmp_xor_key;
278	sp = (uptr)ptrauth_auth_data((void *)sp, ptrauth_key_asdb,
279	ptrauth_string_discriminator("sp"));
280	return sp;
281	}
282
283	#if !SANITIZER_GO
284	void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size) {
285	// The pointer to the ThreadState object is stored in the shadow memory
286	// of the tls.
287	uptr tls_end = tls_addr + tls_size;
288	uptr thread_identity = (uptr)pthread_self();
289	if (thread_identity == main_thread_identity) {
290	MemoryRangeImitateWrite(thr, /pc=/2, tls_addr, tls_size);
291	} else {
292	uptr thr_state_start = thread_identity;
293	uptr thr_state_end = thr_state_start + sizeof(uptr);
294	CHECK_GE(thr_state_start, tls_addr);
295	CHECK_LE(thr_state_start, tls_addr + tls_size);
296	CHECK_GE(thr_state_end, tls_addr);
297	CHECK_LE(thr_state_end, tls_addr + tls_size);
298	MemoryRangeImitateWrite(thr, /pc=/2, tls_addr,
299	thr_state_start - tls_addr);
300	MemoryRangeImitateWrite(thr, /pc=/2, thr_state_end,
301	tls_end - thr_state_end);
302	}
303	}
304	#endif
305
306	#if !SANITIZER_GO
307	// Note: this function runs with async signals enabled,
308	// so it must not touch any tsan state.
309	int call_pthread_cancel_with_cleanup(int(fn)(void c, void *m,
310	void abstime), void c, void m, void abstime,
311	void(cleanup)(void arg), void *arg) {
312	// pthread_cleanup_push/pop are hardcore macros mess.
313	// We can't intercept nor call them w/o including pthread.h.
314	int res;
315	pthread_cleanup_push(cleanup, arg);
316	res = fn(c, m, abstime);
317	pthread_cleanup_pop(0);
318	return res;
319	}
320	#endif
321
322	} // namespace __tsan
323
324	#endif // SANITIZER_MAC

lib/tsan/tsan_platform_posix.cpp created+167

...	@@ -0,0 +1,167 @@
1	//===-- tsan_platform_posix.cpp -------------------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is a part of ThreadSanitizer (TSan), a race detector.
10	//
11	// POSIX-specific code.
12	//===----------------------------------------------------------------------===//
13
14	#include "sanitizer_common/sanitizer_platform.h"
15	#if SANITIZER_POSIX
16
17	#include "sanitizer_common/sanitizer_common.h"
18	#include "sanitizer_common/sanitizer_errno.h"
19	#include "sanitizer_common/sanitizer_libc.h"
20	#include "sanitizer_common/sanitizer_procmaps.h"
21	#include "tsan_platform.h"
22	#include "tsan_rtl.h"
23
24	namespace __tsan {
25
26	static const char kShadowMemoryMappingWarning[] =
27	"FATAL: %s can not madvise shadow region [%zx, %zx] with %s (errno: %d)\n";
28	static const char kShadowMemoryMappingHint[] =
29	"HINT: if %s is not supported in your environment, you may set "
30	"TSAN_OPTIONS=%s=0\n";
31
32	static void NoHugePagesInShadow(uptr addr, uptr size) {
33	SetShadowRegionHugePageMode(addr, size);
34	}
35
36	static void DontDumpShadow(uptr addr, uptr size) {
37	if (common_flags()->use_madv_dontdump)
38	if (!DontDumpShadowMemory(addr, size)) {
39	Printf(kShadowMemoryMappingWarning, SanitizerToolName, addr, addr + size,
40	"MADV_DONTDUMP", errno);
41	Printf(kShadowMemoryMappingHint, "MADV_DONTDUMP", "use_madv_dontdump");
42	Die();
43	}
44	}
45
46	#if !SANITIZER_GO
47	void InitializeShadowMemory() {
48	// Map memory shadow.
49	if (!MmapFixedNoReserve(ShadowBeg(), ShadowEnd() - ShadowBeg(), "shadow")) {
50	Printf("FATAL: ThreadSanitizer can not mmap the shadow memory\n");
51	Printf("FATAL: Make sure to compile with -fPIE and to link with -pie.\n");
52	Die();
53	}
54	// This memory range is used for thread stacks and large user mmaps.
55	// Frequently a thread uses only a small part of stack and similarly
56	// a program uses a small part of large mmap. On some programs
57	// we see 20% memory usage reduction without huge pages for this range.
58	// FIXME: don't use constants here.
59	#if defined(__x86_64__)
60	const uptr kMadviseRangeBeg = 0x7f0000000000ull;
61	const uptr kMadviseRangeSize = 0x010000000000ull;
62	#elif defined(__mips64)
63	const uptr kMadviseRangeBeg = 0xff00000000ull;
64	const uptr kMadviseRangeSize = 0x0100000000ull;
65	#elif defined(__aarch64__) && defined(__APPLE__)
66	uptr kMadviseRangeBeg = LoAppMemBeg();
67	uptr kMadviseRangeSize = LoAppMemEnd() - LoAppMemBeg();
68	#elif defined(__aarch64__)
69	uptr kMadviseRangeBeg = 0;
70	uptr kMadviseRangeSize = 0;
71	if (vmaSize == 39) {
72	kMadviseRangeBeg = 0x7d00000000ull;
73	kMadviseRangeSize = 0x0300000000ull;
74	} else if (vmaSize == 42) {
75	kMadviseRangeBeg = 0x3f000000000ull;
76	kMadviseRangeSize = 0x01000000000ull;
77	} else {
78	DCHECK(0);
79	}
80	#elif defined(__powerpc64__)
81	uptr kMadviseRangeBeg = 0;
82	uptr kMadviseRangeSize = 0;
83	if (vmaSize == 44) {
84	kMadviseRangeBeg = 0x0f60000000ull;
85	kMadviseRangeSize = 0x0010000000ull;
86	} else if (vmaSize == 46) {
87	kMadviseRangeBeg = 0x3f0000000000ull;
88	kMadviseRangeSize = 0x010000000000ull;
89	} else {
90	DCHECK(0);
91	}
92	#endif
93	NoHugePagesInShadow(MemToShadow(kMadviseRangeBeg),
94	kMadviseRangeSize * kShadowMultiplier);
95	DontDumpShadow(ShadowBeg(), ShadowEnd() - ShadowBeg());
96	DPrintf("memory shadow: %zx-%zx (%zuGB)\n",
97	ShadowBeg(), ShadowEnd(),
98	(ShadowEnd() - ShadowBeg()) >> 30);
99
100	// Map meta shadow.
101	const uptr meta = MetaShadowBeg();
102	const uptr meta_size = MetaShadowEnd() - meta;
103	if (!MmapFixedNoReserve(meta, meta_size, "meta shadow")) {
104	Printf("FATAL: ThreadSanitizer can not mmap the shadow memory\n");
105	Printf("FATAL: Make sure to compile with -fPIE and to link with -pie.\n");
106	Die();
107	}
108	NoHugePagesInShadow(meta, meta_size);
109	DontDumpShadow(meta, meta_size);
110	DPrintf("meta shadow: %zx-%zx (%zuGB)\n",
111	meta, meta + meta_size, meta_size >> 30);
112
113	InitializeShadowMemoryPlatform();
114	}
115
116	static void ProtectRange(uptr beg, uptr end) {
117	CHECK_LE(beg, end);
118	if (beg == end)
119	return;
120	if (beg != (uptr)MmapFixedNoAccess(beg, end - beg)) {
121	Printf("FATAL: ThreadSanitizer can not protect [%zx,%zx]\n", beg, end);
122	Printf("FATAL: Make sure you are not using unlimited stack\n");
123	Die();
124	}
125	}
126
127	void CheckAndProtect() {
128	// Ensure that the binary is indeed compiled with -pie.
129	MemoryMappingLayout proc_maps(true);
130	MemoryMappedSegment segment;
131	while (proc_maps.Next(&segment)) {
132	if (IsAppMem(segment.start)) continue;
133	if (segment.start >= HeapMemEnd() && segment.start < HeapEnd()) continue;
134	if (segment.protection == 0) // Zero page or mprotected.
135	continue;
136	if (segment.start >= VdsoBeg()) // vdso
137	break;
138	Printf("FATAL: ThreadSanitizer: unexpected memory mapping %p-%p\n",
139	segment.start, segment.end);
140	Die();
141	}
142
143	#if defined(__aarch64__) && defined(__APPLE__)
144	ProtectRange(HeapMemEnd(), ShadowBeg());
145	ProtectRange(ShadowEnd(), MetaShadowBeg());
146	ProtectRange(MetaShadowEnd(), TraceMemBeg());
147	#else
148	ProtectRange(LoAppMemEnd(), ShadowBeg());
149	ProtectRange(ShadowEnd(), MetaShadowBeg());
150	#ifdef TSAN_MID_APP_RANGE
151	ProtectRange(MetaShadowEnd(), MidAppMemBeg());
152	ProtectRange(MidAppMemEnd(), TraceMemBeg());
153	#else
154	ProtectRange(MetaShadowEnd(), TraceMemBeg());
155	#endif
156	// Memory for traces is mapped lazily in MapThreadTrace.
157	// Protect the whole range for now, so that user does not map something here.
158	ProtectRange(TraceMemBeg(), TraceMemEnd());
159	ProtectRange(TraceMemEnd(), HeapMemBeg());
160	ProtectRange(HeapEnd(), HiAppMemBeg());
161	#endif
162	}
163	#endif
164
165	} // namespace __tsan
166
167	#endif // SANITIZER_POSIX

lib/tsan/tsan_rtl_aarch64.S created+245

...	@@ -0,0 +1,245 @@
1	// The content of this file is AArch64-only:
2	#if defined(__aarch64__)
3
4	#include "sanitizer_common/sanitizer_asm.h"
5
6	#if defined(__APPLE__)
7	.align 2
8
9	.section __DATA,__nl_symbol_ptr,non_lazy_symbol_pointers
10	.long _setjmp$non_lazy_ptr
11	_setjmp$non_lazy_ptr:
12	.indirect_symbol _setjmp
13	.long 0
14
15	.section __DATA,__nl_symbol_ptr,non_lazy_symbol_pointers
16	.long __setjmp$non_lazy_ptr
17	__setjmp$non_lazy_ptr:
18	.indirect_symbol __setjmp
19	.long 0
20
21	.section __DATA,__nl_symbol_ptr,non_lazy_symbol_pointers
22	.long _sigsetjmp$non_lazy_ptr
23	_sigsetjmp$non_lazy_ptr:
24	.indirect_symbol _sigsetjmp
25	.long 0
26	#endif
27
28	#if !defined(__APPLE__)
29	.section .text
30	#else
31	.section __TEXT,__text
32	.align 3
33	#endif
34
35	ASM_HIDDEN(__tsan_setjmp)
36	.comm _ZN14__interception11real_setjmpE,8,8
37	.globl ASM_SYMBOL_INTERCEPTOR(setjmp)
38	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(setjmp))
39	ASM_SYMBOL_INTERCEPTOR(setjmp):
40	CFI_STARTPROC
41
42	// Save frame/link register
43	stp x29, x30, [sp, -32]!
44	CFI_DEF_CFA_OFFSET (32)
45	CFI_OFFSET (29, -32)
46	CFI_OFFSET (30, -24)
47
48	// Adjust the SP for previous frame
49	add x29, sp, 0
50	CFI_DEF_CFA_REGISTER (29)
51
52	// Save env parameter
53	str x0, [sp, 16]
54	CFI_OFFSET (0, -16)
55
56	// Obtain SP, first argument to `void __tsan_setjmp(uptr sp)`
57	add x0, x29, 32
58
59	// call tsan interceptor
60	bl ASM_SYMBOL(__tsan_setjmp)
61
62	// Restore env parameter
63	ldr x0, [sp, 16]
64	CFI_RESTORE (0)
65
66	// Restore frame/link register
67	ldp x29, x30, [sp], 32
68	CFI_RESTORE (29)
69	CFI_RESTORE (30)
70	CFI_DEF_CFA (31, 0)
71
72	// tail jump to libc setjmp
73	#if !defined(__APPLE__)
74	adrp x1, :got:_ZN14__interception11real_setjmpE
75	ldr x1, [x1, #:got_lo12:_ZN14__interception11real_setjmpE]
76	ldr x1, [x1]
77	#else
78	adrp x1, _setjmp$non_lazy_ptr@page
79	add x1, x1, _setjmp$non_lazy_ptr@pageoff
80	ldr x1, [x1]
81	#endif
82	br x1
83
84	CFI_ENDPROC
85	ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(setjmp))
86
87	.comm _ZN14__interception12real__setjmpE,8,8
88	.globl ASM_SYMBOL_INTERCEPTOR(_setjmp)
89	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(_setjmp))
90	ASM_SYMBOL_INTERCEPTOR(_setjmp):
91	CFI_STARTPROC
92
93	// Save frame/link register
94	stp x29, x30, [sp, -32]!
95	CFI_DEF_CFA_OFFSET (32)
96	CFI_OFFSET (29, -32)
97	CFI_OFFSET (30, -24)
98
99	// Adjust the SP for previous frame
100	add x29, sp, 0
101	CFI_DEF_CFA_REGISTER (29)
102
103	// Save env parameter
104	str x0, [sp, 16]
105	CFI_OFFSET (0, -16)
106
107	// Obtain SP, first argument to `void __tsan_setjmp(uptr sp)`
108	add x0, x29, 32
109
110	// call tsan interceptor
111	bl ASM_SYMBOL(__tsan_setjmp)
112
113	// Restore env parameter
114	ldr x0, [sp, 16]
115	CFI_RESTORE (0)
116
117	// Restore frame/link register
118	ldp x29, x30, [sp], 32
119	CFI_RESTORE (29)
120	CFI_RESTORE (30)
121	CFI_DEF_CFA (31, 0)
122
123	// tail jump to libc setjmp
124	#if !defined(__APPLE__)
125	adrp x1, :got:_ZN14__interception12real__setjmpE
126	ldr x1, [x1, #:got_lo12:_ZN14__interception12real__setjmpE]
127	ldr x1, [x1]
128	#else
129	adrp x1, __setjmp$non_lazy_ptr@page
130	add x1, x1, __setjmp$non_lazy_ptr@pageoff
131	ldr x1, [x1]
132	#endif
133	br x1
134
135	CFI_ENDPROC
136	ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(_setjmp))
137
138	.comm _ZN14__interception14real_sigsetjmpE,8,8
139	.globl ASM_SYMBOL_INTERCEPTOR(sigsetjmp)
140	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))
141	ASM_SYMBOL_INTERCEPTOR(sigsetjmp):
142	CFI_STARTPROC
143
144	// Save frame/link register
145	stp x29, x30, [sp, -32]!
146	CFI_DEF_CFA_OFFSET (32)
147	CFI_OFFSET (29, -32)
148	CFI_OFFSET (30, -24)
149
150	// Adjust the SP for previous frame
151	add x29, sp, 0
152	CFI_DEF_CFA_REGISTER (29)
153
154	// Save env and savesigs parameter
155	stp x0, x1, [sp, 16]
156	CFI_OFFSET (0, -16)
157	CFI_OFFSET (1, -8)
158
159	// Obtain SP, first argument to `void __tsan_setjmp(uptr sp)`
160	add x0, x29, 32
161
162	// call tsan interceptor
163	bl ASM_SYMBOL(__tsan_setjmp)
164
165	// Restore env and savesigs parameter
166	ldp x0, x1, [sp, 16]
167	CFI_RESTORE (0)
168	CFI_RESTORE (1)
169
170	// Restore frame/link register
171	ldp x29, x30, [sp], 32
172	CFI_RESTORE (29)
173	CFI_RESTORE (30)
174	CFI_DEF_CFA (31, 0)
175
176	// tail jump to libc sigsetjmp
177	#if !defined(__APPLE__)
178	adrp x2, :got:_ZN14__interception14real_sigsetjmpE
179	ldr x2, [x2, #:got_lo12:_ZN14__interception14real_sigsetjmpE]
180	ldr x2, [x2]
181	#else
182	adrp x2, _sigsetjmp$non_lazy_ptr@page
183	add x2, x2, _sigsetjmp$non_lazy_ptr@pageoff
184	ldr x2, [x2]
185	#endif
186	br x2
187	CFI_ENDPROC
188	ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))
189
190	#if !defined(__APPLE__)
191	.comm _ZN14__interception16real___sigsetjmpE,8,8
192	.globl ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)
193	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))
194	ASM_SYMBOL_INTERCEPTOR(__sigsetjmp):
195	CFI_STARTPROC
196
197	// Save frame/link register
198	stp x29, x30, [sp, -32]!
199	CFI_DEF_CFA_OFFSET (32)
200	CFI_OFFSET (29, -32)
201	CFI_OFFSET (30, -24)
202
203	// Adjust the SP for previous frame
204	add x29, sp, 0
205	CFI_DEF_CFA_REGISTER (29)
206
207	// Save env and savesigs parameter
208	stp x0, x1, [sp, 16]
209	CFI_OFFSET (0, -16)
210	CFI_OFFSET (1, -8)
211
212	// Obtain SP, first argument to `void __tsan_setjmp(uptr sp)`
213	add x0, x29, 32
214
215	// call tsan interceptor
216	bl ASM_SYMBOL(__tsan_setjmp)
217
218	// Restore env and savesigs parameter
219	ldp x0, x1, [sp, 16]
220	CFI_RESTORE (0)
221	CFI_RESTORE (1)
222
223	// Restore frame/link register
224	ldp x29, x30, [sp], 32
225	CFI_RESTORE (29)
226	CFI_RESTORE (30)
227	CFI_DEF_CFA (31, 0)
228
229	// tail jump to libc __sigsetjmp
230	#if !defined(__APPLE__)
231	adrp x2, :got:_ZN14__interception16real___sigsetjmpE
232	ldr x2, [x2, #:got_lo12:_ZN14__interception16real___sigsetjmpE]
233	ldr x2, [x2]
234	#else
235	adrp x2, ASM_SYMBOL(__sigsetjmp)@page
236	add x2, x2, ASM_SYMBOL(__sigsetjmp)@pageoff
237	#endif
238	br x2
239	CFI_ENDPROC
240	ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))
241	#endif
242
243	NO_EXEC_STACK_DIRECTIVE
244
245	#endif

lib/tsan/tsan_rtl_amd64.S created+366

...	@@ -0,0 +1,366 @@
1	// The content of this file is x86_64-only:
2	#if defined(__x86_64__)
3
4	#include "sanitizer_common/sanitizer_asm.h"
5
6	#if !defined(__APPLE__)
7	.section .text
8	#else
9	.section __TEXT,__text
10	#endif
11
12	ASM_HIDDEN(__tsan_trace_switch)
13	.globl ASM_SYMBOL(__tsan_trace_switch_thunk)
14	ASM_SYMBOL(__tsan_trace_switch_thunk):
15	CFI_STARTPROC
16	# Save scratch registers.
17	push %rax
18	CFI_ADJUST_CFA_OFFSET(8)
19	CFI_REL_OFFSET(%rax, 0)
20	push %rcx
21	CFI_ADJUST_CFA_OFFSET(8)
22	CFI_REL_OFFSET(%rcx, 0)
23	push %rdx
24	CFI_ADJUST_CFA_OFFSET(8)
25	CFI_REL_OFFSET(%rdx, 0)
26	push %rsi
27	CFI_ADJUST_CFA_OFFSET(8)
28	CFI_REL_OFFSET(%rsi, 0)
29	push %rdi
30	CFI_ADJUST_CFA_OFFSET(8)
31	CFI_REL_OFFSET(%rdi, 0)
32	push %r8
33	CFI_ADJUST_CFA_OFFSET(8)
34	CFI_REL_OFFSET(%r8, 0)
35	push %r9
36	CFI_ADJUST_CFA_OFFSET(8)
37	CFI_REL_OFFSET(%r9, 0)
38	push %r10
39	CFI_ADJUST_CFA_OFFSET(8)
40	CFI_REL_OFFSET(%r10, 0)
41	push %r11
42	CFI_ADJUST_CFA_OFFSET(8)
43	CFI_REL_OFFSET(%r11, 0)
44	# Align stack frame.
45	push %rbx # non-scratch
46	CFI_ADJUST_CFA_OFFSET(8)
47	CFI_REL_OFFSET(%rbx, 0)
48	mov %rsp, %rbx # save current rsp
49	CFI_DEF_CFA_REGISTER(%rbx)
50	shr $4, %rsp # clear 4 lsb, align to 16
51	shl $4, %rsp
52
53	call ASM_SYMBOL(__tsan_trace_switch)
54
55	# Unalign stack frame back.
56	mov %rbx, %rsp # restore the original rsp
57	CFI_DEF_CFA_REGISTER(%rsp)
58	pop %rbx
59	CFI_ADJUST_CFA_OFFSET(-8)
60	# Restore scratch registers.
61	pop %r11
62	CFI_ADJUST_CFA_OFFSET(-8)
63	pop %r10
64	CFI_ADJUST_CFA_OFFSET(-8)
65	pop %r9
66	CFI_ADJUST_CFA_OFFSET(-8)
67	pop %r8
68	CFI_ADJUST_CFA_OFFSET(-8)
69	pop %rdi
70	CFI_ADJUST_CFA_OFFSET(-8)
71	pop %rsi
72	CFI_ADJUST_CFA_OFFSET(-8)
73	pop %rdx
74	CFI_ADJUST_CFA_OFFSET(-8)
75	pop %rcx
76	CFI_ADJUST_CFA_OFFSET(-8)
77	pop %rax
78	CFI_ADJUST_CFA_OFFSET(-8)
79	CFI_RESTORE(%rax)
80	CFI_RESTORE(%rbx)
81	CFI_RESTORE(%rcx)
82	CFI_RESTORE(%rdx)
83	CFI_RESTORE(%rsi)
84	CFI_RESTORE(%rdi)
85	CFI_RESTORE(%r8)
86	CFI_RESTORE(%r9)
87	CFI_RESTORE(%r10)
88	CFI_RESTORE(%r11)
89	ret
90	CFI_ENDPROC
91
92	ASM_HIDDEN(__tsan_report_race)
93	.globl ASM_SYMBOL(__tsan_report_race_thunk)
94	ASM_SYMBOL(__tsan_report_race_thunk):
95	CFI_STARTPROC
96	# Save scratch registers.
97	push %rax
98	CFI_ADJUST_CFA_OFFSET(8)
99	CFI_REL_OFFSET(%rax, 0)
100	push %rcx
101	CFI_ADJUST_CFA_OFFSET(8)
102	CFI_REL_OFFSET(%rcx, 0)
103	push %rdx
104	CFI_ADJUST_CFA_OFFSET(8)
105	CFI_REL_OFFSET(%rdx, 0)
106	push %rsi
107	CFI_ADJUST_CFA_OFFSET(8)
108	CFI_REL_OFFSET(%rsi, 0)
109	push %rdi
110	CFI_ADJUST_CFA_OFFSET(8)
111	CFI_REL_OFFSET(%rdi, 0)
112	push %r8
113	CFI_ADJUST_CFA_OFFSET(8)
114	CFI_REL_OFFSET(%r8, 0)
115	push %r9
116	CFI_ADJUST_CFA_OFFSET(8)
117	CFI_REL_OFFSET(%r9, 0)
118	push %r10
119	CFI_ADJUST_CFA_OFFSET(8)
120	CFI_REL_OFFSET(%r10, 0)
121	push %r11
122	CFI_ADJUST_CFA_OFFSET(8)
123	CFI_REL_OFFSET(%r11, 0)
124	# Align stack frame.
125	push %rbx # non-scratch
126	CFI_ADJUST_CFA_OFFSET(8)
127	CFI_REL_OFFSET(%rbx, 0)
128	mov %rsp, %rbx # save current rsp
129	CFI_DEF_CFA_REGISTER(%rbx)
130	shr $4, %rsp # clear 4 lsb, align to 16
131	shl $4, %rsp
132
133	call ASM_SYMBOL(__tsan_report_race)
134
135	# Unalign stack frame back.
136	mov %rbx, %rsp # restore the original rsp
137	CFI_DEF_CFA_REGISTER(%rsp)
138	pop %rbx
139	CFI_ADJUST_CFA_OFFSET(-8)
140	# Restore scratch registers.
141	pop %r11
142	CFI_ADJUST_CFA_OFFSET(-8)
143	pop %r10
144	CFI_ADJUST_CFA_OFFSET(-8)
145	pop %r9
146	CFI_ADJUST_CFA_OFFSET(-8)
147	pop %r8
148	CFI_ADJUST_CFA_OFFSET(-8)
149	pop %rdi
150	CFI_ADJUST_CFA_OFFSET(-8)
151	pop %rsi
152	CFI_ADJUST_CFA_OFFSET(-8)
153	pop %rdx
154	CFI_ADJUST_CFA_OFFSET(-8)
155	pop %rcx
156	CFI_ADJUST_CFA_OFFSET(-8)
157	pop %rax
158	CFI_ADJUST_CFA_OFFSET(-8)
159	CFI_RESTORE(%rax)
160	CFI_RESTORE(%rbx)
161	CFI_RESTORE(%rcx)
162	CFI_RESTORE(%rdx)
163	CFI_RESTORE(%rsi)
164	CFI_RESTORE(%rdi)
165	CFI_RESTORE(%r8)
166	CFI_RESTORE(%r9)
167	CFI_RESTORE(%r10)
168	CFI_RESTORE(%r11)
169	ret
170	CFI_ENDPROC
171
172	ASM_HIDDEN(__tsan_setjmp)
173	#if defined(__NetBSD__)
174	.comm _ZN14__interception15real___setjmp14E,8,8
175	#elif !defined(__APPLE__)
176	.comm _ZN14__interception11real_setjmpE,8,8
177	#endif
178	#if defined(__NetBSD__)
179	.globl ASM_SYMBOL_INTERCEPTOR(__setjmp14)
180	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__setjmp14))
181	ASM_SYMBOL_INTERCEPTOR(__setjmp14):
182	#else
183	.globl ASM_SYMBOL_INTERCEPTOR(setjmp)
184	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(setjmp))
185	ASM_SYMBOL_INTERCEPTOR(setjmp):
186	#endif
187	CFI_STARTPROC
188	// save env parameter
189	push %rdi
190	CFI_ADJUST_CFA_OFFSET(8)
191	CFI_REL_OFFSET(%rdi, 0)
192	// obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)`
193	#if defined(__FreeBSD__) \|\| defined(__NetBSD__)
194	lea 8(%rsp), %rdi
195	#elif defined(__linux__) \|\| defined(__APPLE__)
196	lea 16(%rsp), %rdi
197	#else
198	# error "Unknown platform"
199	#endif
200	// call tsan interceptor
201	call ASM_SYMBOL(__tsan_setjmp)
202	// restore env parameter
203	pop %rdi
204	CFI_ADJUST_CFA_OFFSET(-8)
205	CFI_RESTORE(%rdi)
206	// tail jump to libc setjmp
207	movl $0, %eax
208	#if defined(__NetBSD__)
209	movq _ZN14__interception15real___setjmp14E@GOTPCREL(%rip), %rdx
210	jmp *(%rdx)
211	#elif !defined(__APPLE__)
212	movq _ZN14__interception11real_setjmpE@GOTPCREL(%rip), %rdx
213	jmp *(%rdx)
214	#else
215	jmp ASM_SYMBOL(setjmp)
216	#endif
217	CFI_ENDPROC
218	#if defined(__NetBSD__)
219	ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__setjmp14))
220	#else
221	ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(setjmp))
222	#endif
223
224	.comm _ZN14__interception12real__setjmpE,8,8
225	.globl ASM_SYMBOL_INTERCEPTOR(_setjmp)
226	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(_setjmp))
227	ASM_SYMBOL_INTERCEPTOR(_setjmp):
228	CFI_STARTPROC
229	// save env parameter
230	push %rdi
231	CFI_ADJUST_CFA_OFFSET(8)
232	CFI_REL_OFFSET(%rdi, 0)
233	// obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)`
234	#if defined(__FreeBSD__) \|\| defined(__NetBSD__)
235	lea 8(%rsp), %rdi
236	#elif defined(__linux__) \|\| defined(__APPLE__)
237	lea 16(%rsp), %rdi
238	#else
239	# error "Unknown platform"
240	#endif
241	// call tsan interceptor
242	call ASM_SYMBOL(__tsan_setjmp)
243	// restore env parameter
244	pop %rdi
245	CFI_ADJUST_CFA_OFFSET(-8)
246	CFI_RESTORE(%rdi)
247	// tail jump to libc setjmp
248	movl $0, %eax
249	#if !defined(__APPLE__)
250	movq _ZN14__interception12real__setjmpE@GOTPCREL(%rip), %rdx
251	jmp *(%rdx)
252	#else
253	jmp ASM_SYMBOL(_setjmp)
254	#endif
255	CFI_ENDPROC
256	ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(_setjmp))
257
258	#if defined(__NetBSD__)
259	.comm _ZN14__interception18real___sigsetjmp14E,8,8
260	.globl ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14)
261	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14))
262	ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14):
263	#else
264	.comm _ZN14__interception14real_sigsetjmpE,8,8
265	.globl ASM_SYMBOL_INTERCEPTOR(sigsetjmp)
266	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))
267	ASM_SYMBOL_INTERCEPTOR(sigsetjmp):
268	#endif
269	CFI_STARTPROC
270	// save env parameter
271	push %rdi
272	CFI_ADJUST_CFA_OFFSET(8)
273	CFI_REL_OFFSET(%rdi, 0)
274	// save savesigs parameter
275	push %rsi
276	CFI_ADJUST_CFA_OFFSET(8)
277	CFI_REL_OFFSET(%rsi, 0)
278	// align stack frame
279	sub $8, %rsp
280	CFI_ADJUST_CFA_OFFSET(8)
281	// obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)`
282	#if defined(__FreeBSD__) \|\| defined(__NetBSD__)
283	lea 24(%rsp), %rdi
284	#elif defined(__linux__) \|\| defined(__APPLE__)
285	lea 32(%rsp), %rdi
286	#else
287	# error "Unknown platform"
288	#endif
289	// call tsan interceptor
290	call ASM_SYMBOL(__tsan_setjmp)
291	// unalign stack frame
292	add $8, %rsp
293	CFI_ADJUST_CFA_OFFSET(-8)
294	// restore savesigs parameter
295	pop %rsi
296	CFI_ADJUST_CFA_OFFSET(-8)
297	CFI_RESTORE(%rsi)
298	// restore env parameter
299	pop %rdi
300	CFI_ADJUST_CFA_OFFSET(-8)
301	CFI_RESTORE(%rdi)
302	// tail jump to libc sigsetjmp
303	movl $0, %eax
304	#if defined(__NetBSD__)
305	movq _ZN14__interception18real___sigsetjmp14E@GOTPCREL(%rip), %rdx
306	jmp *(%rdx)
307	#elif !defined(__APPLE__)
308	movq _ZN14__interception14real_sigsetjmpE@GOTPCREL(%rip), %rdx
309	jmp *(%rdx)
310	#else
311	jmp ASM_SYMBOL(sigsetjmp)
312	#endif
313	CFI_ENDPROC
314	#if defined(__NetBSD__)
315	ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14))
316	#else
317	ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))
318	#endif
319
320	#if !defined(__APPLE__) && !defined(__NetBSD__)
321	.comm _ZN14__interception16real___sigsetjmpE,8,8
322	.globl ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)
323	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))
324	ASM_SYMBOL_INTERCEPTOR(__sigsetjmp):
325	CFI_STARTPROC
326	// save env parameter
327	push %rdi
328	CFI_ADJUST_CFA_OFFSET(8)
329	CFI_REL_OFFSET(%rdi, 0)
330	// save savesigs parameter
331	push %rsi
332	CFI_ADJUST_CFA_OFFSET(8)
333	CFI_REL_OFFSET(%rsi, 0)
334	// align stack frame
335	sub $8, %rsp
336	CFI_ADJUST_CFA_OFFSET(8)
337	// obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)`
338	#if defined(__FreeBSD__)
339	lea 24(%rsp), %rdi
340	#else
341	lea 32(%rsp), %rdi
342	#endif
343	// call tsan interceptor
344	call ASM_SYMBOL(__tsan_setjmp)
345	// unalign stack frame
346	add $8, %rsp
347	CFI_ADJUST_CFA_OFFSET(-8)
348	// restore savesigs parameter
349	pop %rsi
350	CFI_ADJUST_CFA_OFFSET(-8)
351	CFI_RESTORE(%rsi)
352	// restore env parameter
353	pop %rdi
354	CFI_ADJUST_CFA_OFFSET(-8)
355	CFI_RESTORE(%rdi)
356	// tail jump to libc sigsetjmp
357	movl $0, %eax
358	movq _ZN14__interception16real___sigsetjmpE@GOTPCREL(%rip), %rdx
359	jmp *(%rdx)
360	CFI_ENDPROC
361	ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))
362	#endif // !defined(__APPLE__) && !defined(__NetBSD__)
363
364	NO_EXEC_STACK_DIRECTIVE
365
366	#endif

lib/tsan/tsan_rtl_mips64.S created+214

...	@@ -0,0 +1,214 @@
1	.section .text
2	.set noreorder
3
4	.hidden __tsan_setjmp
5	.comm _ZN14__interception11real_setjmpE,8,8
6	.globl setjmp
7	.type setjmp, @function
8	setjmp:
9
10	// save env parameters
11	daddiu $sp,$sp,-40
12	sd $s0,32($sp)
13	sd $ra,24($sp)
14	sd $fp,16($sp)
15	sd $gp,8($sp)
16
17	// calculate and save pointer to GOT
18	lui $gp,%hi(%neg(%gp_rel(setjmp)))
19	daddu $gp,$gp,$t9
20	daddiu $gp,$gp,%lo(%neg(%gp_rel(setjmp)))
21	move $s0,$gp
22
23	// save jmp_buf
24	sd $a0,0($sp)
25
26	// obtain $sp
27	dadd $a0,$zero,$sp
28
29	// call tsan interceptor
30	jal __tsan_setjmp
31	daddiu $a1,$a0,40
32
33	// restore jmp_buf
34	ld $a0,0($sp)
35
36	// restore gp
37	move $gp,$s0
38
39	// load pointer of libc setjmp to t9
40	dla $t9,(_ZN14__interception11real_setjmpE)
41
42	// restore env parameters
43	ld $gp,8($sp)
44	ld $fp,16($sp)
45	ld $ra,24($sp)
46	ld $s0,32($sp)
47	daddiu $sp,$sp,40
48
49	// tail jump to libc setjmp
50	ld $t9,0($t9)
51	jr $t9
52	nop
53
54	.size setjmp, .-setjmp
55
56	.hidden __tsan_setjmp
57	.globl _setjmp
58	.comm _ZN14__interception12real__setjmpE,8,8
59	.type _setjmp, @function
60	_setjmp:
61
62	// Save env parameters
63	daddiu $sp,$sp,-40
64	sd $s0,32($sp)
65	sd $ra,24($sp)
66	sd $fp,16($sp)
67	sd $gp,8($sp)
68
69	// calculate and save pointer to GOT
70	lui $gp,%hi(%neg(%gp_rel(_setjmp)))
71	daddu $gp,$gp,$t9
72	daddiu $gp,$gp,%lo(%neg(%gp_rel(_setjmp)))
73	move $s0,$gp
74
75	// save jmp_buf
76	sd $a0,0($sp)
77
78	// obtain $sp
79	dadd $a0,$zero,$sp
80
81	// call tsan interceptor
82	jal __tsan_setjmp
83	daddiu $a1,$a0,40
84
85	// restore jmp_buf
86	ld $a0,0($sp)
87
88	// restore gp
89	move $gp,$s0
90
91	// load pointer of libc _setjmp to t9
92	dla $t9,(_ZN14__interception12real__setjmpE)
93
94	// restore env parameters
95	ld $gp,8($sp)
96	ld $fp,16($sp)
97	ld $ra,24($sp)
98	ld $s0,32($sp)
99	daddiu $sp,$sp,40
100
101	// tail jump to libc _setjmp
102	ld $t9,0($t9)
103	jr $t9
104	nop
105
106	.size _setjmp, .-_setjmp
107
108	.hidden __tsan_setjmp
109	.globl sigsetjmp
110	.comm _ZN14__interception14real_sigsetjmpE,8,8
111	.type sigsetjmp, @function
112	sigsetjmp:
113
114	// Save env parameters
115	daddiu $sp,$sp,-48
116	sd $s0,40($sp)
117	sd $ra,32($sp)
118	sd $fp,24($sp)
119	sd $gp,16($sp)
120
121	// calculate and save pointer to GOT
122	lui $gp,%hi(%neg(%gp_rel(sigsetjmp)))
123	daddu $gp,$gp,$t9
124	daddiu $gp,$gp,%lo(%neg(%gp_rel(sigsetjmp)))
125	move $s0,$gp
126
127	// save jmp_buf and savesig
128	sd $a0,0($sp)
129	sd $a1,8($sp)
130
131	// obtain $sp
132	dadd $a0,$zero,$sp
133
134	// call tsan interceptor
135	jal __tsan_setjmp
136	daddiu $a1,$a0,48
137
138	// restore jmp_buf and savesig
139	ld $a0,0($sp)
140	ld $a1,8($sp)
141
142	// restore gp
143	move $gp,$s0
144
145	// load pointer of libc sigsetjmp to t9
146	dla $t9,(_ZN14__interception14real_sigsetjmpE)
147
148	// restore env parameters
149	ld $gp,16($sp)
150	ld $fp,24($sp)
151	ld $ra,32($sp)
152	ld $s0,40($sp)
153	daddiu $sp,$sp,48
154
155	// tail jump to libc sigsetjmp
156	ld $t9,0($t9)
157	jr $t9
158	nop
159
160	.size sigsetjmp, .-sigsetjmp
161
162	.hidden __tsan_setjmp
163	.comm _ZN14__interception16real___sigsetjmpE,8,8
164	.globl __sigsetjmp
165	.type __sigsetjmp, @function
166	__sigsetjmp:
167
168	// Save env parameters
169	daddiu $sp,$sp,-48
170	sd $s0,40($sp)
171	sd $ra,32($sp)
172	sd $fp,24($sp)
173	sd $gp,16($sp)
174
175	// calculate and save pointer to GOT
176	lui $gp,%hi(%neg(%gp_rel(__sigsetjmp)))
177	daddu $gp,$gp,$t9
178	daddiu $gp,$gp,%lo(%neg(%gp_rel(__sigsetjmp)))
179	move $s0,$gp
180
181	// save jmp_buf and savesig
182	sd $a0,0($sp)
183	sd $a1,8($sp)
184
185	// obtain $sp
186	dadd $a0,$zero,$sp
187
188	// call tsan interceptor
189	jal __tsan_setjmp
190	daddiu $a1,$a0,48
191
192	// restore jmp_buf and savesig
193	ld $a0,0($sp)
194	ld $a1,8($sp)
195
196	// restore gp
197	move $gp,$s0
198
199	// load pointer to libc __sigsetjmp in t9
200	dla $t9,(_ZN14__interception16real___sigsetjmpE)
201
202	// restore env parameters
203	ld $gp,16($sp)
204	ld $fp,24($sp)
205	ld $ra,32($sp)
206	ld $s0,40($sp)
207	daddiu $sp,$sp,48
208
209	// tail jump to libc __sigsetjmp
210	ld $t9,0($t9)
211	jr $t9
212	nop
213
214	.size __sigsetjmp, .-__sigsetjmp

lib/tsan/tsan_rtl_ppc64.S created+288

...	@@ -0,0 +1,288 @@
1	#include "tsan_ppc_regs.h"
2
3	.section .text
4	.hidden __tsan_setjmp
5	.globl _setjmp
6	.type _setjmp, @function
7	.align 4
8	#if _CALL_ELF == 2
9	_setjmp:
10	#else
11	.section ".opd","aw"
12	.align 3
13	_setjmp:
14	.quad .L._setjmp,.TOC.@tocbase,0
15	.previous
16	#endif
17	.L._setjmp:
18	mflr r0
19	stdu r1,-48(r1)
20	std r2,24(r1)
21	std r3,32(r1)
22	std r0,40(r1)
23	// r3 is the original stack pointer.
24	addi r3,r1,48
25	// r4 is the mangled stack pointer (see glibc)
26	ld r4,-28696(r13)
27	xor r4,r3,r4
28	// Materialize a TOC in case we were called from libc.
29	// For big-endian, we load the TOC from the OPD. For little-
30	// endian, we use the .TOC. symbol to find it.
31	nop
32	bcl 20,31,0f
33	0:
34	mflr r2
35	#if _CALL_ELF == 2
36	addis r2,r2,.TOC.-0b@ha
37	addi r2,r2,.TOC.-0b@l
38	#else
39	addis r2,r2,_setjmp-0b@ha
40	addi r2,r2,_setjmp-0b@l
41	ld r2,8(r2)
42	#endif
43	// Call the interceptor.
44	bl __tsan_setjmp
45	nop
46	// Restore regs needed for setjmp.
47	ld r3,32(r1)
48	ld r0,40(r1)
49	// Emulate the real setjmp function. We do this because we can't
50	// perform a sibcall: The real setjmp function trashes the TOC
51	// pointer, and with a sibcall we have no way to restore it.
52	// This way we can make sure our caller's stack pointer and
53	// link register are saved correctly in the jmpbuf.
54	ld r6,-28696(r13)
55	addi r5,r1,48 // original stack ptr of caller
56	xor r5,r6,r5
57	std r5,0(r3) // mangled stack ptr of caller
58	ld r5,24(r1)
59	std r5,8(r3) // caller's saved TOC pointer
60	xor r0,r6,r0
61	std r0,16(r3) // caller's mangled return address
62	mfcr r0
63	// Nonvolatiles.
64	std r14,24(r3)
65	stfd f14,176(r3)
66	stw r0,172(r3) // CR
67	std r15,32(r3)
68	stfd f15,184(r3)
69	std r16,40(r3)
70	stfd f16,192(r3)
71	std r17,48(r3)
72	stfd f17,200(r3)
73	std r18,56(r3)
74	stfd f18,208(r3)
75	std r19,64(r3)
76	stfd f19,216(r3)
77	std r20,72(r3)
78	stfd f20,224(r3)
79	std r21,80(r3)
80	stfd f21,232(r3)
81	std r22,88(r3)
82	stfd f22,240(r3)
83	std r23,96(r3)
84	stfd f23,248(r3)
85	std r24,104(r3)
86	stfd f24,256(r3)
87	std r25,112(r3)
88	stfd f25,264(r3)
89	std r26,120(r3)
90	stfd f26,272(r3)
91	std r27,128(r3)
92	stfd f27,280(r3)
93	std r28,136(r3)
94	stfd f28,288(r3)
95	std r29,144(r3)
96	stfd f29,296(r3)
97	std r30,152(r3)
98	stfd f30,304(r3)
99	std r31,160(r3)
100	stfd f31,312(r3)
101	addi r5,r3,320
102	mfspr r0,256
103	stw r0,168(r3) // VRSAVE
104	addi r6,r5,16
105	stvx v20,0,r5
106	addi r5,r5,32
107	stvx v21,0,r6
108	addi r6,r6,32
109	stvx v22,0,r5
110	addi r5,r5,32
111	stvx v23,0,r6
112	addi r6,r6,32
113	stvx v24,0,r5
114	addi r5,r5,32
115	stvx v25,0,r6
116	addi r6,r6,32
117	stvx v26,0,r5
118	addi r5,r5,32
119	stvx v27,0,r6
120	addi r6,r6,32
121	stvx v28,0,r5
122	addi r5,r5,32
123	stvx v29,0,r6
124	addi r6,r6,32
125	stvx v30,0,r5
126	stvx v31,0,r6
127	// Clear the "mask-saved" slot.
128	li r4,0
129	stw r4,512(r3)
130	// Restore TOC, LR, and stack and return to caller.
131	ld r2,24(r1)
132	ld r0,40(r1)
133	addi r1,r1,48
134	li r3,0 // This is the setjmp return path
135	mtlr r0
136	blr
137	.size _setjmp, .-.L._setjmp
138
139	.globl setjmp
140	.type setjmp, @function
141	.align 4
142	setjmp:
143	b _setjmp
144	.size setjmp, .-setjmp
145
146	// sigsetjmp is like setjmp, except that the mask in r4 needs
147	// to be saved at offset 512 of the jump buffer.
148	.globl __sigsetjmp
149	.type __sigsetjmp, @function
150	.align 4
151	#if _CALL_ELF == 2
152	__sigsetjmp:
153	#else
154	.section ".opd","aw"
155	.align 3
156	__sigsetjmp:
157	.quad .L.__sigsetjmp,.TOC.@tocbase,0
158	.previous
159	#endif
160	.L.__sigsetjmp:
161	mflr r0
162	stdu r1,-64(r1)
163	std r2,24(r1)
164	std r3,32(r1)
165	std r4,40(r1)
166	std r0,48(r1)
167	// r3 is the original stack pointer.
168	addi r3,r1,64
169	// r4 is the mangled stack pointer (see glibc)
170	ld r4,-28696(r13)
171	xor r4,r3,r4
172	// Materialize a TOC in case we were called from libc.
173	// For big-endian, we load the TOC from the OPD. For little-
174	// endian, we use the .TOC. symbol to find it.
175	nop
176	bcl 20,31,1f
177	1:
178	mflr r2
179	#if _CALL_ELF == 2
180	addis r2,r2,.TOC.-1b@ha
181	addi r2,r2,.TOC.-1b@l
182	#else
183	addis r2,r2,_setjmp-1b@ha
184	addi r2,r2,_setjmp-1b@l
185	ld r2,8(r2)
186	#endif
187	// Call the interceptor.
188	bl __tsan_setjmp
189	nop
190	// Restore regs needed for __sigsetjmp.
191	ld r3,32(r1)
192	ld r4,40(r1)
193	ld r0,48(r1)
194	// Emulate the real sigsetjmp function. We do this because we can't
195	// perform a sibcall: The real sigsetjmp function trashes the TOC
196	// pointer, and with a sibcall we have no way to restore it.
197	// This way we can make sure our caller's stack pointer and
198	// link register are saved correctly in the jmpbuf.
199	ld r6,-28696(r13)
200	addi r5,r1,64 // original stack ptr of caller
201	xor r5,r6,r5
202	std r5,0(r3) // mangled stack ptr of caller
203	ld r5,24(r1)
204	std r5,8(r3) // caller's saved TOC pointer
205	xor r0,r6,r0
206	std r0,16(r3) // caller's mangled return address
207	mfcr r0
208	// Nonvolatiles.
209	std r14,24(r3)
210	stfd f14,176(r3)
211	stw r0,172(r3) // CR
212	std r15,32(r3)
213	stfd f15,184(r3)
214	std r16,40(r3)
215	stfd f16,192(r3)
216	std r17,48(r3)
217	stfd f17,200(r3)
218	std r18,56(r3)
219	stfd f18,208(r3)
220	std r19,64(r3)
221	stfd f19,216(r3)
222	std r20,72(r3)
223	stfd f20,224(r3)
224	std r21,80(r3)
225	stfd f21,232(r3)
226	std r22,88(r3)
227	stfd f22,240(r3)
228	std r23,96(r3)
229	stfd f23,248(r3)
230	std r24,104(r3)
231	stfd f24,256(r3)
232	std r25,112(r3)
233	stfd f25,264(r3)
234	std r26,120(r3)
235	stfd f26,272(r3)
236	std r27,128(r3)
237	stfd f27,280(r3)
238	std r28,136(r3)
239	stfd f28,288(r3)
240	std r29,144(r3)
241	stfd f29,296(r3)
242	std r30,152(r3)
243	stfd f30,304(r3)
244	std r31,160(r3)
245	stfd f31,312(r3)
246	addi r5,r3,320
247	mfspr r0,256
248	stw r0,168(r3) // VRSAVE
249	addi r6,r5,16
250	stvx v20,0,r5
251	addi r5,r5,32
252	stvx v21,0,r6
253	addi r6,r6,32
254	stvx v22,0,r5
255	addi r5,r5,32
256	stvx v23,0,r6
257	addi r6,r6,32
258	stvx v24,0,r5
259	addi r5,r5,32
260	stvx v25,0,r6
261	addi r6,r6,32
262	stvx v26,0,r5
263	addi r5,r5,32
264	stvx v27,0,r6
265	addi r6,r6,32
266	stvx v28,0,r5
267	addi r5,r5,32
268	stvx v29,0,r6
269	addi r6,r6,32
270	stvx v30,0,r5
271	stvx v31,0,r6
272	// Save into the "mask-saved" slot.
273	stw r4,512(r3)
274	// Restore TOC, LR, and stack and return to caller.
275	ld r2,24(r1)
276	ld r0,48(r1)
277	addi r1,r1,64
278	li r3,0 // This is the sigsetjmp return path
279	mtlr r0
280	blr
281	.size __sigsetjmp, .-.L.__sigsetjmp
282
283	.globl sigsetjmp
284	.type sigsetjmp, @function
285	.align 4
286	sigsetjmp:
287	b __sigsetjmp
288	.size sigsetjmp, .-sigsetjmp

src/Compilation.zig+19-14

src/glibc.zig+1-1

...	@@ -962,7 +962,7 @@ fn buildSharedLib(	...	@@ -962,7 +962,7 @@ fn buildSharedLib(
962	.version_script = map_file_path,	962	.version_script = map_file_path,
963	.soname = soname,	963	.soname = soname,
964	.c_source_files = &c_source_files,	964	.c_source_files = &c_source_files,
965	.is_compiler_rt_or_libc = true,	965	.skip_linker_dependencies = true,
966	});	966	});
967	defer sub_compilation.destroy();	967	defer sub_compilation.destroy();
968		968

src/libcxx.zig+2

...	@@ -188,6 +188,7 @@ pub fn buildLibCXX(comp: *Compilation) !void {	...	@@ -188,6 +188,7 @@ pub fn buildLibCXX(comp: *Compilation) !void {
188	.verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features,	188	.verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features,
189	.clang_passthrough_mode = comp.clang_passthrough_mode,	189	.clang_passthrough_mode = comp.clang_passthrough_mode,
190	.link_libc = true,	190	.link_libc = true,
		191	.skip_linker_dependencies = true,
191	});	192	});
192	defer sub_compilation.destroy();	193	defer sub_compilation.destroy();
193		194
...	@@ -308,6 +309,7 @@ pub fn buildLibCXXABI(comp: *Compilation) !void {	...	@@ -308,6 +309,7 @@ pub fn buildLibCXXABI(comp: *Compilation) !void {
308	.verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features,	309	.verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features,
309	.clang_passthrough_mode = comp.clang_passthrough_mode,	310	.clang_passthrough_mode = comp.clang_passthrough_mode,
310	.link_libc = true,	311	.link_libc = true,
		312	.skip_linker_dependencies = true,
311	});	313	});
312	defer sub_compilation.destroy();	314	defer sub_compilation.destroy();
313		315

src/libtsan.zig+183-15

...	@@ -34,7 +34,7 @@ pub fn buildTsan(comp: *Compilation) !void {	...	@@ -34,7 +34,7 @@ pub fn buildTsan(comp: *Compilation) !void {
34	};	34	};
35		35
36	var c_source_files = std.ArrayList(Compilation.CSourceFile).init(arena);	36	var c_source_files = std.ArrayList(Compilation.CSourceFile).init(arena);
37	try c_source_files.ensureCapacity(tsan_sources.len + sanitizer_common_sources.len);	37	try c_source_files.ensureCapacity(c_source_files.items.len + tsan_sources.len);
38		38
39	const tsan_include_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{"tsan"});	39	const tsan_include_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{"tsan"});
40	for (tsan_sources) \|tsan_src\| {	40	for (tsan_sources) \|tsan_src\| {
...	@@ -48,6 +48,7 @@ pub fn buildTsan(comp: *Compilation) !void {	...	@@ -48,6 +48,7 @@ pub fn buildTsan(comp: *Compilation) !void {
48	try cflags.append("-nostdinc++");	48	try cflags.append("-nostdinc++");
49	try cflags.append("-fvisibility-inlines-hidden");	49	try cflags.append("-fvisibility-inlines-hidden");
50	try cflags.append("-std=c++14");	50	try cflags.append("-std=c++14");
		51	try cflags.append("-fno-rtti");
51		52
52	c_source_files.appendAssumeCapacity(.{	53	c_source_files.appendAssumeCapacity(.{
53	.src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ "tsan", tsan_src }),	54	.src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ "tsan", tsan_src }),
...	@@ -55,6 +56,51 @@ pub fn buildTsan(comp: *Compilation) !void {	...	@@ -55,6 +56,51 @@ pub fn buildTsan(comp: *Compilation) !void {
55	});	56	});
56	}	57	}
57		58
		59	const platform_tsan_sources = if (target.isDarwin())
		60	&darwin_tsan_sources
		61	else
		62	&unix_tsan_sources;
		63	try c_source_files.ensureCapacity(c_source_files.items.len + platform_tsan_sources.len);
		64	for (platform_tsan_sources) \|tsan_src\| {
		65	var cflags = std.ArrayList([]const u8).init(arena);
		66
		67	try cflags.append("-I");
		68	try cflags.append(tsan_include_path);
		69
		70	try cflags.append("-O3");
		71	try cflags.append("-DNDEBUG");
		72	try cflags.append("-nostdinc++");
		73	try cflags.append("-fvisibility-inlines-hidden");
		74	try cflags.append("-std=c++14");
		75	try cflags.append("-fno-rtti");
		76
		77	c_source_files.appendAssumeCapacity(.{
		78	.src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ "tsan", tsan_src }),
		79	.extra_flags = cflags.items,
		80	});
		81	}
		82	{
		83	const asm_source = switch (target.cpu.arch) {
		84	.aarch64 => "tsan_rtl_aarch64.S",
		85	.x86_64 => "tsan_rtl_amd64.S",
		86	.mips64 => "tsan_rtl_mips64.S",
		87	.powerpc64 => "tsan_rtl_ppc64.S",
		88	else => return error.TSANUnsupportedCPUArchitecture,
		89	};
		90	var cflags = std.ArrayList([]const u8).init(arena);
		91
		92	try cflags.append("-I");
		93	try cflags.append(tsan_include_path);
		94
		95	try cflags.append("-DNDEBUG");
		96
		97	c_source_files.appendAssumeCapacity(.{
		98	.src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ "tsan", asm_source }),
		99	.extra_flags = cflags.items,
		100	});
		101	}
		102
		103	try c_source_files.ensureCapacity(c_source_files.items.len + sanitizer_common_sources.len);
58	const sanitizer_common_include_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{	104	const sanitizer_common_include_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{
59	"tsan", "sanitizer_common",	105	"tsan", "sanitizer_common",
60	});	106	});
...	@@ -69,6 +115,7 @@ pub fn buildTsan(comp: *Compilation) !void {	...	@@ -69,6 +115,7 @@ pub fn buildTsan(comp: *Compilation) !void {
69	try cflags.append("-nostdinc++");	115	try cflags.append("-nostdinc++");
70	try cflags.append("-fvisibility-inlines-hidden");	116	try cflags.append("-fvisibility-inlines-hidden");
71	try cflags.append("-std=c++14");	117	try cflags.append("-std=c++14");
		118	try cflags.append("-fno-rtti");
72		119
73	c_source_files.appendAssumeCapacity(.{	120	c_source_files.appendAssumeCapacity(.{
74	.src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{	121	.src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{
...	@@ -78,6 +125,88 @@ pub fn buildTsan(comp: *Compilation) !void {	...	@@ -78,6 +125,88 @@ pub fn buildTsan(comp: *Compilation) !void {
78	});	125	});
79	}	126	}
80		127
		128	const to_c_or_not_to_c_sources = if (comp.bin_file.options.link_libc)
		129	&sanitizer_libcdep_sources
		130	else
		131	&sanitizer_nolibc_sources;
		132	try c_source_files.ensureCapacity(c_source_files.items.len + to_c_or_not_to_c_sources.len);
		133	for (to_c_or_not_to_c_sources) \|c_src\| {
		134	var cflags = std.ArrayList([]const u8).init(arena);
		135
		136	try cflags.append("-I");
		137	try cflags.append(sanitizer_common_include_path);
		138
		139	try cflags.append("-O3");
		140	try cflags.append("-DNDEBUG");
		141	try cflags.append("-nostdinc++");
		142	try cflags.append("-fvisibility-inlines-hidden");
		143	try cflags.append("-std=c++14");
		144	try cflags.append("-fno-rtti");
		145
		146	c_source_files.appendAssumeCapacity(.{
		147	.src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{
		148	"tsan", "sanitizer_common", c_src,
		149	}),
		150	.extra_flags = cflags.items,
		151	});
		152	}
		153
		154	try c_source_files.ensureCapacity(c_source_files.items.len + sanitizer_symbolizer_sources.len);
		155	for (sanitizer_symbolizer_sources) \|c_src\| {
		156	var cflags = std.ArrayList([]const u8).init(arena);
		157
		158	try cflags.append("-I");
		159	try cflags.append(tsan_include_path);
		160
		161	try cflags.append("-O3");
		162	try cflags.append("-DNDEBUG");
		163	try cflags.append("-nostdinc++");
		164	try cflags.append("-fvisibility-inlines-hidden");
		165	try cflags.append("-std=c++14");
		166	try cflags.append("-fno-rtti");
		167
		168	c_source_files.appendAssumeCapacity(.{
		169	.src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{
		170	"tsan", "sanitizer_common", c_src,
		171	}),
		172	.extra_flags = cflags.items,
		173	});
		174	}
		175
		176	const interception_include_path = try comp.zig_lib_directory.join(
		177	arena,
		178	&[_][]const u8{"interception"},
		179	);
		180
		181	try c_source_files.ensureCapacity(c_source_files.items.len + interception_sources.len);
		182	for (interception_sources) \|c_src\| {
		183	var cflags = std.ArrayList([]const u8).init(arena);
		184
		185	try cflags.append("-I");
		186	try cflags.append(interception_include_path);
		187
		188	try cflags.append("-I");
		189	try cflags.append(tsan_include_path);
		190
		191	try cflags.append("-O3");
		192	try cflags.append("-DNDEBUG");
		193	try cflags.append("-nostdinc++");
		194	try cflags.append("-fvisibility-inlines-hidden");
		195	try cflags.append("-std=c++14");
		196	try cflags.append("-fno-rtti");
		197
		198	c_source_files.appendAssumeCapacity(.{
		199	.src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{
		200	"tsan", "interception", c_src,
		201	}),
		202	.extra_flags = cflags.items,
		203	});
		204	}
		205
		206	const common_flags = [_][]const u8{
		207	"-DTSAN_CONTAINS_UBSAN=0",
		208	};
		209
81	const sub_compilation = try Compilation.create(comp.gpa, .{	210	const sub_compilation = try Compilation.create(comp.gpa, .{
82	.local_cache_directory = comp.global_cache_directory,	211	.local_cache_directory = comp.global_cache_directory,
83	.global_cache_directory = comp.global_cache_directory,	212	.global_cache_directory = comp.global_cache_directory,
...	@@ -113,6 +242,8 @@ pub fn buildTsan(comp: *Compilation) !void {	...	@@ -113,6 +242,8 @@ pub fn buildTsan(comp: *Compilation) !void {
113	.verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features,	242	.verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features,
114	.clang_passthrough_mode = comp.clang_passthrough_mode,	243	.clang_passthrough_mode = comp.clang_passthrough_mode,
115	.link_libc = true,	244	.link_libc = true,
		245	.skip_linker_dependencies = true,
		246	.clang_argv = &common_flags,
116	});	247	});
117	defer sub_compilation.destroy();	248	defer sub_compilation.destroy();
118		249
...	@@ -159,6 +290,18 @@ const tsan_sources = [_][]const u8{	...	@@ -159,6 +290,18 @@ const tsan_sources = [_][]const u8{
159	"tsan_sync.cpp",	290	"tsan_sync.cpp",
160	};	291	};
161		292
		293	const darwin_tsan_sources = [_][]const u8{
		294	"tsan_interceptors_mac.cpp",
		295	"tsan_interceptors_mach_vm.cpp",
		296	"tsan_platform_mac.cpp",
		297	"tsan_platform_posix.cpp",
		298	};
		299
		300	const unix_tsan_sources = [_][]const u8{
		301	"tsan_platform_linux.cpp",
		302	"tsan_platform_posix.cpp",
		303	};
		304
162	const sanitizer_common_sources = [_][]const u8{	305	const sanitizer_common_sources = [_][]const u8{
163	"sanitizer_allocator.cpp",	306	"sanitizer_allocator.cpp",
164	"sanitizer_common.cpp",	307	"sanitizer_common.cpp",
...	@@ -203,17 +346,42 @@ const sanitizer_common_sources = [_][]const u8{	...	@@ -203,17 +346,42 @@ const sanitizer_common_sources = [_][]const u8{
203	"sanitizer_win.cpp",	346	"sanitizer_win.cpp",
204	};	347	};
205		348
206	// TODO This is next up	349	const sanitizer_nolibc_sources = [_][]const u8{
207	//const sanitizer_nolibc_sources = [_][]const u8{	350	"sanitizer_common_nolibc.cpp",
208	// "sanitizer_common_nolibc.cpp",	351	};
209	//};	352
210	//	353	const sanitizer_libcdep_sources = [_][]const u8{
211	//const sanitizer_libcdep_sources = [_][]const u8{	354	"sanitizer_common_libcdep.cpp",
212	// "sanitizer_common_libcdep.cpp",	355	"sanitizer_allocator_checks.cpp",
213	// "sanitizer_allocator_checks.cpp",	356	"sanitizer_linux_libcdep.cpp",
214	// "sanitizer_linux_libcdep.cpp",	357	"sanitizer_mac_libcdep.cpp",
215	// "sanitizer_mac_libcdep.cpp",	358	"sanitizer_posix_libcdep.cpp",
216	// "sanitizer_posix_libcdep.cpp",	359	"sanitizer_stoptheworld_linux_libcdep.cpp",
217	// "sanitizer_stoptheworld_linux_libcdep.cpp",	360	"sanitizer_stoptheworld_netbsd_libcdep.cpp",
218	// "sanitizer_stoptheworld_netbsd_libcdep.cpp",	361	};
219	//};	362
		363	const sanitizer_symbolizer_sources = [_][]const u8{
		364	"sanitizer_allocator_report.cpp",
		365	"sanitizer_stackdepot.cpp",
		366	"sanitizer_stacktrace.cpp",
		367	"sanitizer_stacktrace_libcdep.cpp",
		368	"sanitizer_stacktrace_printer.cpp",
		369	"sanitizer_stacktrace_sparc.cpp",
		370	"sanitizer_symbolizer.cpp",
		371	"sanitizer_symbolizer_libbacktrace.cpp",
		372	"sanitizer_symbolizer_libcdep.cpp",
		373	"sanitizer_symbolizer_mac.cpp",
		374	"sanitizer_symbolizer_markup.cpp",
		375	"sanitizer_symbolizer_posix_libcdep.cpp",
		376	"sanitizer_symbolizer_report.cpp",
		377	"sanitizer_symbolizer_win.cpp",
		378	"sanitizer_unwind_linux_libcdep.cpp",
		379	"sanitizer_unwind_win.cpp",
		380	};
		381
		382	const interception_sources = [_][]const u8{
		383	"interception_linux.cpp",
		384	"interception_mac.cpp",
		385	"interception_win.cpp",
		386	"interception_type_test.cpp",
		387	};

src/libunwind.zig+1

...	@@ -122,6 +122,7 @@ pub fn buildStaticLib(comp: *Compilation) !void {	...	@@ -122,6 +122,7 @@ pub fn buildStaticLib(comp: *Compilation) !void {
122	.verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features,	122	.verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features,
123	.clang_passthrough_mode = comp.clang_passthrough_mode,	123	.clang_passthrough_mode = comp.clang_passthrough_mode,
124	.link_libc = true,	124	.link_libc = true,
		125	.skip_linker_dependencies = true,
125	});	126	});
126	defer sub_compilation.destroy();	127	defer sub_compilation.destroy();
127		128

src/link.zig+1-1

...	@@ -81,7 +81,7 @@ pub const Options = struct {	...	@@ -81,7 +81,7 @@ pub const Options = struct {
81	verbose_link: bool,	81	verbose_link: bool,
82	dll_export_fns: bool,	82	dll_export_fns: bool,
83	error_return_tracing: bool,	83	error_return_tracing: bool,
84	is_compiler_rt_or_libc: bool,	84	skip_linker_dependencies: bool,
85	parent_compilation_link_libc: bool,	85	parent_compilation_link_libc: bool,
86	each_lib_rpath: bool,	86	each_lib_rpath: bool,
87	disable_lld_caching: bool,	87	disable_lld_caching: bool,

src/link/Coff.zig+2-2

...	@@ -835,7 +835,7 @@ fn linkWithLLD(self: Coff, comp: Compilation) !void {	...	@@ -835,7 +835,7 @@ fn linkWithLLD(self: Coff, comp: Compilation) !void {
835	man.hash.addOptional(self.base.options.image_base_override);	835	man.hash.addOptional(self.base.options.image_base_override);
836	man.hash.addListOfBytes(self.base.options.extra_lld_args);	836	man.hash.addListOfBytes(self.base.options.extra_lld_args);
837	man.hash.addListOfBytes(self.base.options.lib_dirs);	837	man.hash.addListOfBytes(self.base.options.lib_dirs);
838	man.hash.add(self.base.options.is_compiler_rt_or_libc);	838	man.hash.add(self.base.options.skip_linker_dependencies);
839	if (self.base.options.link_libc) {	839	if (self.base.options.link_libc) {
840	man.hash.add(self.base.options.libc_installation != null);	840	man.hash.add(self.base.options.libc_installation != null);
841	if (self.base.options.libc_installation) \|libc_installation\| {	841	if (self.base.options.libc_installation) \|libc_installation\| {
...	@@ -1125,7 +1125,7 @@ fn linkWithLLD(self: Coff, comp: Compilation) !void {	...	@@ -1125,7 +1125,7 @@ fn linkWithLLD(self: Coff, comp: Compilation) !void {
1125	}	1125	}
1126		1126
1127	// compiler-rt, libc and libssp	1127	// compiler-rt, libc and libssp
1128	if (is_exe_or_dyn_lib and !self.base.options.is_compiler_rt_or_libc) {	1128	if (is_exe_or_dyn_lib and !self.base.options.skip_linker_dependencies) {
1129	if (!self.base.options.link_libc) {	1129	if (!self.base.options.link_libc) {
1130	try argv.append(comp.libc_static_lib.?.full_object_path);	1130	try argv.append(comp.libc_static_lib.?.full_object_path);
1131	}	1131	}

src/link/Elf.zig+2-4

src/link/MachO.zig+2-2

...	@@ -438,7 +438,7 @@ fn linkWithLLD(self: MachO, comp: Compilation) !void {	...	@@ -438,7 +438,7 @@ fn linkWithLLD(self: MachO, comp: Compilation) !void {
438	man.hash.addListOfBytes(self.base.options.framework_dirs);	438	man.hash.addListOfBytes(self.base.options.framework_dirs);
439	man.hash.addListOfBytes(self.base.options.frameworks);	439	man.hash.addListOfBytes(self.base.options.frameworks);
440	man.hash.addListOfBytes(self.base.options.rpath_list);	440	man.hash.addListOfBytes(self.base.options.rpath_list);
441	man.hash.add(self.base.options.is_compiler_rt_or_libc);	441	man.hash.add(self.base.options.skip_linker_dependencies);
442	man.hash.add(self.base.options.z_nodelete);	442	man.hash.add(self.base.options.z_nodelete);
443	man.hash.add(self.base.options.z_defs);	443	man.hash.add(self.base.options.z_defs);
444	if (is_dyn_lib) {	444	if (is_dyn_lib) {
...	@@ -633,7 +633,7 @@ fn linkWithLLD(self: MachO, comp: Compilation) !void {	...	@@ -633,7 +633,7 @@ fn linkWithLLD(self: MachO, comp: Compilation) !void {
633	}	633	}
634		634
635	// compiler_rt on darwin is missing some stuff, so we still build it and rely on LinkOnce	635	// compiler_rt on darwin is missing some stuff, so we still build it and rely on LinkOnce
636	if (is_exe_or_dyn_lib and !self.base.options.is_compiler_rt_or_libc) {	636	if (is_exe_or_dyn_lib and !self.base.options.skip_linker_dependencies) {
637	try argv.append(comp.compiler_rt_static_lib.?.full_object_path);	637	try argv.append(comp.compiler_rt_static_lib.?.full_object_path);
638	}	638	}
639		639

src/link/Wasm.zig+1-1

...	@@ -387,7 +387,7 @@ fn linkWithLLD(self: Wasm, comp: Compilation) !void {	...	@@ -387,7 +387,7 @@ fn linkWithLLD(self: Wasm, comp: Compilation) !void {
387	}	387	}
388		388
389	if (self.base.options.output_mode != .Obj and	389	if (self.base.options.output_mode != .Obj and
390	!self.base.options.is_compiler_rt_or_libc and	390	!self.base.options.skip_linker_dependencies and
391	!self.base.options.link_libc)	391	!self.base.options.link_libc)
392	{	392	{
393	try argv.append(comp.libc_static_lib.?.full_object_path);	393	try argv.append(comp.libc_static_lib.?.full_object_path);

src/musl.zig+1-1

...	@@ -225,7 +225,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void {	...	@@ -225,7 +225,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void {
225	.c_source_files = &[_]Compilation.CSourceFile{	225	.c_source_files = &[_]Compilation.CSourceFile{
226	.{ .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", "musl", "libc.s" }) },	226	.{ .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", "musl", "libc.s" }) },
227	},	227	},
228	.is_compiler_rt_or_libc = true,	228	.skip_linker_dependencies = true,
229	.soname = "libc.so",	229	.soname = "libc.so",
230	});	230	});
231	defer sub_compilation.destroy();	231	defer sub_compilation.destroy();

author	Andrew Kelley <andrew@ziglang.org> 2020-12-22 19:25:24-07:00
committer	Andrew Kelley <andrew@ziglang.org> 2020-12-24 01:18:48-07:00
log	8219d92987c7d9641d6b24c4d5be29e3a80fd6b9
tree	026578eeb7d11f5813d081025763f064401536e1
parent	42b4a48bc96ce22562230cd1a266f93a013c76dd

...	@@ -429,7 +429,11 @@ pub const InitOptions = struct {	...	@@ -429,7 +429,11 @@ pub const InitOptions = struct {
429	verbose_llvm_cpu_features: bool = false,	429	verbose_llvm_cpu_features: bool = false,
430	is_test: bool = false,	430	is_test: bool = false,
431	test_evented_io: bool = false,	431	test_evented_io: bool = false,
432	is_compiler_rt_or_libc: bool = false,	432	/// Normally when you create a `Compilation`, Zig will automatically build
		433	/// and link in required dependencies, such as compiler-rt and libc. When
		434	/// building such dependencies themselves, this flag must be set to avoid
		435	/// infinite recursion.
		436	skip_linker_dependencies: bool = false,
433	parent_compilation_link_libc: bool = false,	437	parent_compilation_link_libc: bool = false,
434	stack_size_override: ?u64 = null,	438	stack_size_override: ?u64 = null,
435	image_base_override: ?u64 = null,	439	image_base_override: ?u64 = null,
...	@@ -499,7 +503,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !Compilation {	...	@@ -499,7 +503,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !Compilation {
499	.Lib => is_dyn_lib,	503	.Lib => is_dyn_lib,
500	.Exe => true,	504	.Exe => true,
501	};	505	};
502	const needs_c_symbols = !options.is_compiler_rt_or_libc and	506	const needs_c_symbols = !options.skip_linker_dependencies and
503	(is_exe_or_dyn_lib or (options.target.isWasm() and options.output_mode != .Obj));	507	(is_exe_or_dyn_lib or (options.target.isWasm() and options.output_mode != .Obj));
504		508
505	const comp: *Compilation = comp: {	509	const comp: *Compilation = comp: {
...	@@ -677,6 +681,9 @@ pub fn create(gpa: Allocator, options: InitOptions) !Compilation {	...	@@ -677,6 +681,9 @@ pub fn create(gpa: Allocator, options: InitOptions) !Compilation {
677	break :pic explicit;	681	break :pic explicit;
678	} else pie or must_pic;	682	} else pie or must_pic;
679		683
		684	// TSAN is implemented in C++ so it requires linking libc++.
		685	const link_libcpp = options.link_libcpp or tsan;
		686
680	// Make a decision on whether to use Clang for translate-c and compiling C files.	687	// Make a decision on whether to use Clang for translate-c and compiling C files.
681	const use_clang = if (options.use_clang) \|explicit\| explicit else blk: {	688	const use_clang = if (options.use_clang) \|explicit\| explicit else blk: {
682	if (build_options.have_llvm) {	689	if (build_options.have_llvm) {
...	@@ -765,7 +772,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !Compilation {	...	@@ -765,7 +772,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !Compilation {
765	cache.hash.add(options.function_sections);	772	cache.hash.add(options.function_sections);
766	cache.hash.add(strip);	773	cache.hash.add(strip);
767	cache.hash.add(link_libc);	774	cache.hash.add(link_libc);
768	cache.hash.add(options.link_libcpp);	775	cache.hash.add(link_libcpp);
769	cache.hash.add(options.output_mode);	776	cache.hash.add(options.output_mode);
770	cache.hash.add(options.machine_code_model);	777	cache.hash.add(options.machine_code_model);
771	cache.hash.addOptionalEmitLoc(options.emit_bin);	778	cache.hash.addOptionalEmitLoc(options.emit_bin);
...	@@ -793,7 +800,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !Compilation {	...	@@ -793,7 +800,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !Compilation {
793	hash.add(single_threaded);	800	hash.add(single_threaded);
794	hash.add(dll_export_fns);	801	hash.add(dll_export_fns);
795	hash.add(options.is_test);	802	hash.add(options.is_test);
796	hash.add(options.is_compiler_rt_or_libc);	803	hash.add(options.skip_linker_dependencies);
797	hash.add(options.parent_compilation_link_libc);	804	hash.add(options.parent_compilation_link_libc);
798		805
799	const digest = hash.final();	806	const digest = hash.final();
...	@@ -930,7 +937,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !Compilation {	...	@@ -930,7 +937,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !Compilation {
930	.use_llvm = use_llvm,	937	.use_llvm = use_llvm,
931	.system_linker_hack = darwin_options.system_linker_hack,	938	.system_linker_hack = darwin_options.system_linker_hack,
932	.link_libc = link_libc,	939	.link_libc = link_libc,
933	.link_libcpp = options.link_libcpp,	940	.link_libcpp = link_libcpp,
934	.objects = options.link_objects,	941	.objects = options.link_objects,
935	.frameworks = options.frameworks,	942	.frameworks = options.frameworks,
936	.framework_dirs = options.framework_dirs,	943	.framework_dirs = options.framework_dirs,
...	@@ -970,7 +977,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !Compilation {	...	@@ -970,7 +977,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !Compilation {
970	.dll_export_fns = dll_export_fns,	977	.dll_export_fns = dll_export_fns,
971	.error_return_tracing = error_return_tracing,	978	.error_return_tracing = error_return_tracing,
972	.llvm_cpu_features = llvm_cpu_features,	979	.llvm_cpu_features = llvm_cpu_features,
973	.is_compiler_rt_or_libc = options.is_compiler_rt_or_libc,	980	.skip_linker_dependencies = options.skip_linker_dependencies,
974	.parent_compilation_link_libc = options.parent_compilation_link_libc,	981	.parent_compilation_link_libc = options.parent_compilation_link_libc,
975	.each_lib_rpath = options.each_lib_rpath orelse options.is_native_os,	982	.each_lib_rpath = options.each_lib_rpath orelse options.is_native_os,
976	.disable_lld_caching = options.disable_lld_caching,	983	.disable_lld_caching = options.disable_lld_caching,
...	@@ -1044,7 +1051,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !Compilation {	...	@@ -1044,7 +1051,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !Compilation {
1044	comp.c_object_table.putAssumeCapacityNoClobber(c_object, {});	1051	comp.c_object_table.putAssumeCapacityNoClobber(c_object, {});
1045	}	1052	}
1046		1053
1047	if (comp.bin_file.options.emit != null and !comp.bin_file.options.is_compiler_rt_or_libc) {	1054	if (comp.bin_file.options.emit != null and !comp.bin_file.options.skip_linker_dependencies) {
1048	// If we need to build glibc for the target, add work items for it.	1055	// If we need to build glibc for the target, add work items for it.
1049	// We go through the work queue so that building can be done in parallel.	1056	// We go through the work queue so that building can be done in parallel.
1050	if (comp.wantBuildGLibCFromSource()) {	1057	if (comp.wantBuildGLibCFromSource()) {
...	@@ -1097,9 +1104,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !Compilation {	...	@@ -1097,9 +1104,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !Compilation {
1097	if (comp.wantBuildLibUnwindFromSource()) {	1104	if (comp.wantBuildLibUnwindFromSource()) {
1098	try comp.work_queue.writeItem(.{ .libunwind = {} });	1105	try comp.work_queue.writeItem(.{ .libunwind = {} });
1099	}	1106	}
1100	if (build_options.have_llvm and comp.bin_file.options.output_mode != .Obj and	1107	if (build_options.have_llvm and is_exe_or_dyn_lib and comp.bin_file.options.link_libcpp) {
1101	comp.bin_file.options.link_libcpp)
1102	{
1103	try comp.work_queue.writeItem(.libcxx);	1108	try comp.work_queue.writeItem(.libcxx);
1104	try comp.work_queue.writeItem(.libcxxabi);	1109	try comp.work_queue.writeItem(.libcxxabi);
1105	}	1110	}
...	@@ -2772,7 +2777,7 @@ pub fn generateBuiltinZigSource(comp: Compilation, allocator: Allocator) ![]u8	...	@@ -2772,7 +2777,7 @@ pub fn generateBuiltinZigSource(comp: Compilation, allocator: Allocator) ![]u8
2772	// in. For example, compiler_rt will not export the __chkstk symbol if it	2777	// in. For example, compiler_rt will not export the __chkstk symbol if it
2773	// knows libc will provide it, and likewise c.zig will not export memcpy.	2778	// knows libc will provide it, and likewise c.zig will not export memcpy.
2774	const link_libc = comp.bin_file.options.link_libc or	2779	const link_libc = comp.bin_file.options.link_libc or
2775	(comp.bin_file.options.is_compiler_rt_or_libc and comp.bin_file.options.parent_compilation_link_libc);	2780	(comp.bin_file.options.skip_linker_dependencies and comp.bin_file.options.parent_compilation_link_libc);
2776		2781
2777	try buffer.writer().print(	2782	try buffer.writer().print(
2778	\\pub const object_format = ObjectFormat.{};	2783	\\pub const object_format = ObjectFormat.{};
...	@@ -2927,7 +2932,7 @@ fn buildOutputFromZig(	...	@@ -2927,7 +2932,7 @@ fn buildOutputFromZig(
2927	.verbose_cimport = comp.verbose_cimport,	2932	.verbose_cimport = comp.verbose_cimport,
2928	.verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features,	2933	.verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features,
2929	.clang_passthrough_mode = comp.clang_passthrough_mode,	2934	.clang_passthrough_mode = comp.clang_passthrough_mode,
2930	.is_compiler_rt_or_libc = true,	2935	.skip_linker_dependencies = true,
2931	.parent_compilation_link_libc = comp.bin_file.options.link_libc,	2936	.parent_compilation_link_libc = comp.bin_file.options.link_libc,
2932	});	2937	});
2933	defer sub_compilation.destroy();	2938	defer sub_compilation.destroy();
...	@@ -3305,7 +3310,7 @@ pub fn build_crt_file(	...	@@ -3305,7 +3310,7 @@ pub fn build_crt_file(
3305	.verbose_cimport = comp.verbose_cimport,	3310	.verbose_cimport = comp.verbose_cimport,
3306	.verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features,	3311	.verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features,
3307	.clang_passthrough_mode = comp.clang_passthrough_mode,	3312	.clang_passthrough_mode = comp.clang_passthrough_mode,
3308	.is_compiler_rt_or_libc = true,	3313	.skip_linker_dependencies = true,
3309	.parent_compilation_link_libc = comp.bin_file.options.link_libc,	3314	.parent_compilation_link_libc = comp.bin_file.options.link_libc,
3310	});	3315	});
3311	defer sub_compilation.destroy();	3316	defer sub_compilation.destroy();
...	@@ -3326,7 +3331,7 @@ pub fn stage1AddLinkLib(comp: *Compilation, lib_name: []const u8) !void {	...	@@ -3326,7 +3331,7 @@ pub fn stage1AddLinkLib(comp: *Compilation, lib_name: []const u8) !void {
3326	// Avoid deadlocking on building import libs such as kernel32.lib	3331	// Avoid deadlocking on building import libs such as kernel32.lib
3327	// This can happen when the user uses `build-exe foo.obj -lkernel32` and then	3332	// This can happen when the user uses `build-exe foo.obj -lkernel32` and then
3328	// when we create a sub-Compilation for zig libc, it also tries to build kernel32.lib.	3333	// when we create a sub-Compilation for zig libc, it also tries to build kernel32.lib.
3329	if (comp.bin_file.options.is_compiler_rt_or_libc) return;	3334	if (comp.bin_file.options.skip_linker_dependencies) return;
3330		3335
3331	// This happens when an `extern "foo"` function is referenced by the stage1 backend.	3336	// This happens when an `extern "foo"` function is referenced by the stage1 backend.
3332	// If we haven't seen this library yet and we're targeting Windows, we need to queue up	3337	// If we haven't seen this library yet and we're targeting Windows, we need to queue up

...	@@ -1310,7 +1310,7 @@ fn linkWithLLD(self: Elf, comp: Compilation) !void {	...	@@ -1310,7 +1310,7 @@ fn linkWithLLD(self: Elf, comp: Compilation) !void {
1310	man.hash.addListOfBytes(self.base.options.lib_dirs);	1310	man.hash.addListOfBytes(self.base.options.lib_dirs);
1311	man.hash.addListOfBytes(self.base.options.rpath_list);	1311	man.hash.addListOfBytes(self.base.options.rpath_list);
1312	man.hash.add(self.base.options.each_lib_rpath);	1312	man.hash.add(self.base.options.each_lib_rpath);
1313	man.hash.add(self.base.options.is_compiler_rt_or_libc);	1313	man.hash.add(self.base.options.skip_linker_dependencies);
1314	man.hash.add(self.base.options.z_nodelete);	1314	man.hash.add(self.base.options.z_nodelete);
1315	man.hash.add(self.base.options.z_defs);	1315	man.hash.add(self.base.options.z_defs);
1316	if (self.base.options.link_libc) {	1316	if (self.base.options.link_libc) {
...	@@ -1552,7 +1552,7 @@ fn linkWithLLD(self: Elf, comp: Compilation) !void {	...	@@ -1552,7 +1552,7 @@ fn linkWithLLD(self: Elf, comp: Compilation) !void {
1552	}	1552	}
1553		1553
1554	// libc	1554	// libc
1555	if (is_exe_or_dyn_lib and !self.base.options.is_compiler_rt_or_libc and !self.base.options.link_libc) {	1555	if (is_exe_or_dyn_lib and !self.base.options.skip_linker_dependencies and !self.base.options.link_libc) {
1556	try argv.append(comp.libc_static_lib.?.full_object_path);	1556	try argv.append(comp.libc_static_lib.?.full_object_path);
1557	}	1557	}
1558		1558
...	@@ -1574,9 +1574,7 @@ fn linkWithLLD(self: Elf, comp: Compilation) !void {	...	@@ -1574,9 +1574,7 @@ fn linkWithLLD(self: Elf, comp: Compilation) !void {
1574	const arg = if (ext == .shared_library) link_lib else try std.fmt.allocPrint(arena, "-l{}", .{link_lib});	1574	const arg = if (ext == .shared_library) link_lib else try std.fmt.allocPrint(arena, "-l{}", .{link_lib});
1575	argv.appendAssumeCapacity(arg);	1575	argv.appendAssumeCapacity(arg);
1576	}	1576	}
1577	}
1578		1577
1579	if (!is_obj) {
1580	// libc++ dep	1578	// libc++ dep
1581	if (self.base.options.link_libcpp) {	1579	if (self.base.options.link_libcpp) {
1582	try argv.append(comp.libcxxabi_static_lib.?.full_object_path);	1580	try argv.append(comp.libcxxabi_static_lib.?.full_object_path);

stage2: fix Cache deadlock and build more of TSAN