authorgravatar for thatlemon@gmail.comLemonBoy <thatlemon@gmail.com> 2019-05-04 12:02:55+02:00
committergravatar for thatlemon@gmail.comLemonBoy <thatlemon@gmail.com> 2019-05-07 13:09:18+02:00
logd8ab301aa81758a7918de446271201c044e7835f
tree6651864a9104e691c0347e76581dfc2ce73aa2a3
parent7432fb04d6e3c52b40e81911d6c608e4d17317df

std: Implement TLS support for Linux

Tested on x86_64, i386, ARM, AARCH64

5 files changed, 257 insertions(+), 72 deletions(-)

CMakeLists.txt+1
...@@ -611,6 +611,7 @@ set(ZIG_STD_FILES...@@ -611,6 +611,7 @@ set(ZIG_STD_FILES
611 "os/linux.zig"611 "os/linux.zig"
612 "os/linux/arm64.zig"612 "os/linux/arm64.zig"
613 "os/linux/errno.zig"613 "os/linux/errno.zig"
614 "os/linux/tls.zig"
614 "os/linux/vdso.zig"615 "os/linux/vdso.zig"
615 "os/linux/x86_64.zig"616 "os/linux/x86_64.zig"
616 "os/netbsd.zig"617 "os/netbsd.zig"
std/os.zig+5-12
...@@ -3126,9 +3126,6 @@ pub const SpawnThreadError = error{...@@ -3126,9 +3126,6 @@ pub const SpawnThreadError = error{
3126 Unexpected,3126 Unexpected,
3127};3127};
31283128
3129pub var linux_tls_phdr: ?*std.elf.Phdr = null;
3130pub var linux_tls_img_src: [*]const u8 = undefined; // defined if linux_tls_phdr is
3131
3132/// caller must call wait on the returned thread3129/// caller must call wait on the returned thread
3133/// fn startFn(@typeOf(context)) T3130/// fn startFn(@typeOf(context)) T
3134/// where T is u8, noreturn, void, or !void3131/// where T is u8, noreturn, void, or !void
...@@ -3238,12 +3235,10 @@ pub fn spawnThread(context: var, comptime startFn: var) SpawnThreadError!*Thread...@@ -3238,12 +3235,10 @@ pub fn spawnThread(context: var, comptime startFn: var) SpawnThreadError!*Thread
3238 }3235 }
3239 // Finally, the Thread Local Storage, if any.3236 // Finally, the Thread Local Storage, if any.
3240 if (!Thread.use_pthreads) {3237 if (!Thread.use_pthreads) {
3241 if (linux_tls_phdr) |tls_phdr| {3238 if (linux.tls.tls_image) |tls_img| {
3242 l = mem.alignForward(l, tls_phdr.p_align);3239 l = mem.alignForward(l, @alignOf(usize));
3243 tls_start_offset = l;3240 tls_start_offset = l;
3244 l += tls_phdr.p_memsz;3241 l += tls_img.alloc_size;
3245 // the fs register address
3246 l += @sizeOf(usize);
3247 }3242 }
3248 }3243 }
3249 break :blk l;3244 break :blk l;
...@@ -3284,10 +3279,8 @@ pub fn spawnThread(context: var, comptime startFn: var) SpawnThreadError!*Thread...@@ -3284,10 +3279,8 @@ pub fn spawnThread(context: var, comptime startFn: var) SpawnThreadError!*Thread
3284 posix.CLONE_THREAD | posix.CLONE_SYSVSEM | posix.CLONE_PARENT_SETTID | posix.CLONE_CHILD_CLEARTID |3279 posix.CLONE_THREAD | posix.CLONE_SYSVSEM | posix.CLONE_PARENT_SETTID | posix.CLONE_CHILD_CLEARTID |
3285 posix.CLONE_DETACHED;3280 posix.CLONE_DETACHED;
3286 var newtls: usize = undefined;3281 var newtls: usize = undefined;
3287 if (linux_tls_phdr) |tls_phdr| {3282 if (linux.tls.tls_image) |tls_img| {
3288 @memcpy(@intToPtr([*]u8, mmap_addr + tls_start_offset), linux_tls_img_src, tls_phdr.p_filesz);3283 newtls = linux.tls.copyTLS(mmap_addr + tls_start_offset);
3289 newtls = mmap_addr + mmap_len - @sizeOf(usize);
3290 @intToPtr(*usize, newtls).* = newtls;
3291 flags |= posix.CLONE_SETTLS;3284 flags |= posix.CLONE_SETTLS;
3292 }3285 }
3293 const rc = posix.clone(MainFuncs.linuxThreadMain, mmap_addr + stack_end_offset, flags, arg, &thread_ptr.data.handle, newtls, &thread_ptr.data.handle);3286 const rc = posix.clone(MainFuncs.linuxThreadMain, mmap_addr + stack_end_offset, flags, arg, &thread_ptr.data.handle, newtls, &thread_ptr.data.handle);
std/os/linux.zig+1
...@@ -3,6 +3,7 @@ const assert = std.debug.assert;...@@ -3,6 +3,7 @@ const assert = std.debug.assert;
3const builtin = @import("builtin");3const builtin = @import("builtin");
4const maxInt = std.math.maxInt;4const maxInt = std.math.maxInt;
5const elf = std.elf;5const elf = std.elf;
6pub const tls = @import("linux/tls.zig");
6const vdso = @import("linux/vdso.zig");7const vdso = @import("linux/vdso.zig");
7const dl = @import("../dynamic_library.zig");8const dl = @import("../dynamic_library.zig");
8pub use switch (builtin.arch) {9pub use switch (builtin.arch) {
std/os/linux/tls.zig created+242
...@@ -0,0 +1,242 @@
1const std = @import("std");
2const mem = std.mem;
3const posix = std.posix;
4const elf = std.elf;
5const builtin = @import("builtin");
6const assert = std.debug.assert;
7
8// This file implements the two TLS variants [1] used by ELF-based systems.
9//
10// The variant I has the following layout in memory:
11// -------------------------------------------------------
12// | DTV | Zig | DTV | Alignment | TLS |
13// | storage | thread data | pointer | | block |
14// ------------------------^------------------------------
15// `-- The thread pointer register points here
16//
17// In this case we allocate additional space for our control structure that's
18// placed _before_ the DTV pointer together with the DTV.
19//
20// NOTE: Some systems such as power64 or mips use this variant with a twist: the
21// alignment is not present and the tp and DTV addresses are offset by a
22// constant.
23//
24// On the other hand the variant II has the following layout in memory:
25// ---------------------------------------
26// | TLS | TCB | Zig | DTV |
27// | block | | thread data | storage |
28// --------^------------------------------
29// `-- The thread pointer register points here
30//
31// The structure of the TCB is not defined by the ABI so we reserve enough space
32// for a single pointer as some architectures such as i386 and x86_64 need a
33// pointer to the TCB block itself at the address pointed by the tp.
34//
35// In this case the control structure and DTV are placed one after another right
36// after the TLS block data.
37//
38// At the moment the DTV is very simple since we only support static TLS, all we
39// need is a two word vector to hold the number of entries (1) and the address
40// of the first TLS block.
41//
42// [1] https://www.akkadia.org/drepper/tls.pdf
43
44const TLSVariant = enum {
45 VariantI,
46 VariantII,
47};
48
49const tls_variant = switch (builtin.arch) {
50 .arm, .armeb, .aarch64, .aarch64_be => TLSVariant.VariantI,
51 .x86_64, .i386 => TLSVariant.VariantII,
52 else => @compileError("undefined tls_variant for this architecture"),
53};
54
55// Controls how many bytes are reserved for the Thread Control Block
56const tls_tcb_size = switch (builtin.arch) {
57 // ARM EABI mandates enough space for two pointers: the first one points to
58 // the DTV while the second one is unspecified but reserved
59 .arm, .armeb, .aarch64, .aarch64_be => 2 * @sizeOf(usize),
60 .i386, .x86_64 => @sizeOf(usize),
61 else => 0,
62};
63
64// Controls if the TCB should be aligned according to the TLS segment p_align
65const tls_tcb_align_size = switch (builtin.arch) {
66 .arm, .armeb, .aarch64, .aarch64_be => true,
67 else => false,
68};
69
70// Check if the architecture-specific parameters look correct
71comptime {
72 if (tls_tcb_align_size and tls_variant != TLSVariant.VariantI) {
73 @compileError("tls_tcb_align_size is only meaningful for variant I TLS");
74 }
75}
76
77// Some architectures add some offset to the tp and dtv addresses in order to
78// make the generated code more efficient
79
80const tls_tp_offset = switch (builtin.arch) {
81 else => 0,
82};
83
84const tls_dtv_offset = switch (builtin.arch) {
85 else => 0,
86};
87
88// Per-thread storage for Zig's use
89const CustomData = packed struct {
90};
91
92// Dynamic Thread Vector
93const DTV = packed struct {
94 entries: usize,
95 tls_block: [1]usize,
96};
97
98// Holds all the information about the process TLS image
99const TLSImage = struct {
100 data_src: []u8,
101 alloc_size: usize,
102 tcb_offset: usize,
103 dtv_offset: usize,
104 data_offset: usize,
105};
106
107pub var tls_image: ?TLSImage = null;
108
109pub fn setThreadPointer(addr: usize) void {
110 switch (builtin.arch) {
111 .x86_64 => {
112 const ARCH_SET_FS = 0x1002;
113 const rc = std.os.linux.syscall2(std.os.linux.SYS_arch_prctl, ARCH_SET_FS, addr);
114 // arch_prctl is documented to never fail
115 assert(rc == 0);
116 },
117 .aarch64 => {
118 asm volatile (
119 \\ msr tpidr_el0, %[addr]
120 : : [addr] "r" (addr)
121 );
122 },
123 else => @compileError("Unsupported architecture"),
124 }
125}
126
127pub fn initTLS() void {
128 var tls_phdr: ?*elf.Phdr = null;
129 var img_base: usize = 0;
130
131 if (std.os.linux_elf_aux_maybe) |auxv| {
132 var at_phent: usize = undefined;
133 var at_phnum: usize = undefined;
134 var at_phdr: usize = undefined;
135
136 var i: usize = 0;
137 while (auxv[i].a_type != std.elf.AT_NULL) : (i += 1) {
138 switch (auxv[i].a_type) {
139 elf.AT_PHENT => at_phent = auxv[i].a_un.a_val,
140 elf.AT_PHNUM => at_phnum = auxv[i].a_un.a_val,
141 elf.AT_PHDR => at_phdr = auxv[i].a_un.a_val,
142 else => continue,
143 }
144 }
145
146 // Sanity check
147 assert(at_phent == @sizeOf(elf.Phdr));
148
149 // Search the TLS section
150 const phdrs = (@intToPtr([*]elf.Phdr, at_phdr))[0..at_phnum];
151
152 for (phdrs) |*phdr| {
153 switch (phdr.p_type) {
154 elf.PT_PHDR => img_base = at_phdr - phdr.p_vaddr,
155 elf.PT_TLS => tls_phdr = phdr,
156 else => continue,
157 }
158 }
159 } else {
160 @panic("no auxv vector available!");
161 }
162
163 if (tls_phdr) |phdr| {
164 // Offsets into the allocated TLS area
165 var tcb_offset: usize = undefined;
166 var dtv_offset: usize = undefined;
167 var data_offset: usize = undefined;
168 var thread_data_offset: usize = undefined;
169 // Compute the total size of the ABI-specific data plus our own control
170 // structures
171 const alloc_size = switch (tls_variant) {
172 .VariantI => blk: {
173 var l: usize = 0;
174 dtv_offset = l;
175 l += @sizeOf(DTV);
176 thread_data_offset = l;
177 l += @sizeOf(CustomData);
178 l = mem.alignForward(l, phdr.p_align);
179 tcb_offset = l;
180 if (tls_tcb_align_size) {
181 l += mem.alignForward(tls_tcb_size, phdr.p_align);
182 } else {
183 l += tls_tcb_size;
184 }
185 data_offset = l;
186 l += phdr.p_memsz;
187 break :blk l;
188 },
189 .VariantII => blk: {
190 var l: usize = 0;
191 data_offset = l;
192 l += phdr.p_memsz;
193 l = mem.alignForward(l, phdr.p_align);
194 tcb_offset = l;
195 l += tls_tcb_size;
196 thread_data_offset = l;
197 l += @sizeOf(CustomData);
198 dtv_offset = l;
199 l += @sizeOf(DTV);
200 break :blk l;
201 }
202 };
203
204 tls_image = TLSImage{
205 .data_src = @intToPtr([*]u8, phdr.p_vaddr + img_base)[0..phdr.p_filesz],
206 .alloc_size = alloc_size,
207 .tcb_offset = tcb_offset,
208 .dtv_offset = dtv_offset,
209 .data_offset = data_offset,
210 };
211 }
212}
213
214pub fn copyTLS(addr: usize) usize {
215 const tls_img = tls_image orelse @panic("copyTLS called with no TLS section!");
216
217 // Be paranoid, clear the area we're going to use
218 @memset(@intToPtr([*]u8, addr), 0, tls_img.alloc_size);
219 // Prepare the DTV
220 const dtv = @intToPtr(*DTV, addr + tls_img.dtv_offset);
221 dtv.entries = 1;
222 dtv.tls_block[0] = addr + tls_img.data_offset + tls_dtv_offset;
223 // Set-up the TCB
224 const tcb_ptr = @intToPtr(*usize, addr + tls_img.tcb_offset);
225 if (tls_variant == TLSVariant.VariantI) {
226 tcb_ptr.* = addr + tls_img.dtv_offset;
227 } else {
228 tcb_ptr.* = addr + tls_img.tcb_offset;
229 }
230 // Copy the data
231 @memcpy(@intToPtr([*]u8, addr + tls_img.data_offset), tls_img.data_src.ptr, tls_img.data_src.len);
232
233 // Return the corrected (if needed) value for the tp register
234 return addr + tls_img.tcb_offset + tls_tp_offset;
235}
236
237var main_thread_tls_buffer: [64]u8 align(32) = undefined;
238
239pub fn allocateTLS(size: usize) usize {
240 assert(size < main_thread_tls_buffer.len);
241 return @ptrToInt(&main_thread_tls_buffer);
242}
std/special/bootstrap.zig+8-60
...@@ -67,24 +67,19 @@ fn posixCallMainAndExit() noreturn {...@@ -67,24 +67,19 @@ fn posixCallMainAndExit() noreturn {
67 var envp_count: usize = 0;67 var envp_count: usize = 0;
68 while (envp_optional[envp_count]) |_| : (envp_count += 1) {}68 while (envp_optional[envp_count]) |_| : (envp_count += 1) {}
69 const envp = @ptrCast([*][*]u8, envp_optional)[0..envp_count];69 const envp = @ptrCast([*][*]u8, envp_optional)[0..envp_count];
70
70 if (builtin.os == builtin.Os.linux) {71 if (builtin.os == builtin.Os.linux) {
71 // Scan auxiliary vector.
72 const auxv = @ptrCast([*]std.elf.Auxv, envp.ptr + envp_count + 1);72 const auxv = @ptrCast([*]std.elf.Auxv, envp.ptr + envp_count + 1);
73 std.os.linux_elf_aux_maybe = auxv;73 std.os.linux_elf_aux_maybe = auxv;
74 var i: usize = 0;74
75 var at_phdr: usize = 0;75 std.os.linux.tls.initTLS();
76 var at_phnum: usize = 0;76 if (!builtin.single_threaded) {
77 var at_phent: usize = 0;77 if (std.os.linux.tls.tls_image) |tls_img| {
78 while (auxv[i].a_un.a_val != 0) : (i += 1) {78 const tls_addr = std.os.linux.tls.allocateTLS(tls_img.alloc_size);
79 switch (auxv[i].a_type) {79 const tp = std.os.linux.tls.copyTLS(tls_addr);
80 std.elf.AT_PAGESZ => assert(auxv[i].a_un.a_val == std.os.page_size),80 std.os.linux.tls.setThreadPointer(tp);
81 std.elf.AT_PHDR => at_phdr = auxv[i].a_un.a_val,
82 std.elf.AT_PHNUM => at_phnum = auxv[i].a_un.a_val,
83 std.elf.AT_PHENT => at_phent = auxv[i].a_un.a_val,
84 else => {},
85 }81 }
86 }82 }
87 if (!builtin.single_threaded) linuxInitializeThreadLocalStorage(at_phdr, at_phnum, at_phent);
88 }83 }
8984
90 std.os.posix.exit(callMainWithArgs(argc, argv, envp));85 std.os.posix.exit(callMainWithArgs(argc, argv, envp));
...@@ -140,50 +135,3 @@ inline fn callMain() u8 {...@@ -140,50 +135,3 @@ inline fn callMain() u8 {
140135
141const main_thread_tls_align = 32;136const main_thread_tls_align = 32;
142var main_thread_tls_bytes: [64]u8 align(main_thread_tls_align) = [1]u8{0} ** 64;137var main_thread_tls_bytes: [64]u8 align(main_thread_tls_align) = [1]u8{0} ** 64;
143
144fn linuxInitializeThreadLocalStorage(at_phdr: usize, at_phnum: usize, at_phent: usize) void {
145 var phdr_addr = at_phdr;
146 var n = at_phnum;
147 var base: usize = 0;
148 while (n != 0) : ({
149 n -= 1;
150 phdr_addr += at_phent;
151 }) {
152 const phdr = @intToPtr(*std.elf.Phdr, phdr_addr);
153 // TODO look for PT_DYNAMIC when we have https://github.com/ziglang/zig/issues/1917
154 switch (phdr.p_type) {
155 std.elf.PT_PHDR => base = at_phdr - phdr.p_vaddr,
156 std.elf.PT_TLS => std.os.linux_tls_phdr = phdr,
157 else => continue,
158 }
159 }
160 const tls_phdr = std.os.linux_tls_phdr orelse return;
161 std.os.linux_tls_img_src = @intToPtr([*]const u8, base + tls_phdr.p_vaddr);
162 const end_addr = @ptrToInt(&main_thread_tls_bytes) + tls_phdr.p_memsz;
163 const max_end_addr = @ptrToInt(&main_thread_tls_bytes) + main_thread_tls_bytes.len;
164 assert(max_end_addr >= end_addr + @sizeOf(usize)); // not enough preallocated Thread Local Storage
165 assert(main_thread_tls_align >= tls_phdr.p_align); // preallocated Thread Local Storage not aligned enough
166 @memcpy(&main_thread_tls_bytes, std.os.linux_tls_img_src, tls_phdr.p_filesz);
167 const end_ptr = @intToPtr(*usize, end_addr);
168 end_ptr.* = end_addr;
169 linuxSetThreadArea(end_addr);
170}
171
172fn linuxSetThreadArea(addr: usize) void {
173 switch (builtin.arch) {
174 builtin.Arch.x86_64 => {
175 const ARCH_SET_FS = 0x1002;
176 const rc = std.os.linux.syscall2(std.os.linux.SYS_arch_prctl, ARCH_SET_FS, addr);
177 // acrh_prctl is documented to never fail
178 assert(rc == 0);
179 },
180 builtin.Arch.aarch64 => {
181 asm volatile (
182 \\ msr tpidr_el0,x0
183 \\ mov w0,#0
184 \\ ret
185 );
186 },
187 else => @compileError("Unsupported architecture"),
188 }
189}