authorgravatar for thatlemon@gmail.comLemonBoy <thatlemon@gmail.com> 2020-03-25 12:08:50+01:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-03-28 11:20:38-04:00
logd788b0cd8b0b12ec4f3dddf4ecdb59556b73d4c2
treebc0dbd74dbdeb7f0fc8bdfaac3d2cf4a1b92006a
parent12e1c6e21c1d21238450903ad7f8fa4ce52ee758

std: Minor changes to TLS handling

* Always allocate an info block per-thread so that libc can store important stuff there. * Respect ABI-mandated alignment in more places. * Nicer code, use slices/pointers instead of raw addresses whenever possible.

4 files changed, 153 insertions(+), 141 deletions(-)

lib/std/os/linux/tls.zig+135-117
......@@ -1,8 +1,9 @@
11const std = @import("std");
2const builtin = std.builtin;
23const os = std.os;
34const mem = std.mem;
45const elf = std.elf;
5const builtin = @import("builtin");
6const math = std.math;
67const assert = std.debug.assert;
78
89// This file implements the two TLS variants [1] used by ELF-based systems.
......@@ -57,28 +58,16 @@ const tls_tcb_size = switch (builtin.arch) {
5758 // ARM EABI mandates enough space for two pointers: the first one points to
5859 // the DTV while the second one is unspecified but reserved
5960 .arm, .armeb, .aarch64, .aarch64_be => 2 * @sizeOf(usize),
61 // One pointer-sized word that points either to the DTV or the TCB itself
6062 else => @sizeOf(usize),
6163};
6264
63// Controls if the TCB should be aligned according to the TLS segment p_align
64const tls_tcb_align_size = switch (builtin.arch) {
65 .arm, .armeb, .aarch64, .aarch64_be => true,
66 else => false,
67};
68
6965// Controls if the TP points to the end of the TCB instead of its beginning
7066const tls_tp_points_past_tcb = switch (builtin.arch) {
7167 .riscv32, .riscv64, .mipsel, .powerpc64, .powerpc64le => true,
7268 else => false,
7369};
7470
75// Check if the architecture-specific parameters look correct
76comptime {
77 if (tls_tcb_align_size and tls_variant != TLSVariant.VariantI) {
78 @compileError("tls_tcb_align_size is only meaningful for variant I TLS");
79 }
80}
81
8271// Some architectures add some offset to the tp and dtv addresses in order to
8372// make the generated code more efficient
8473
......@@ -94,32 +83,36 @@ const tls_dtv_offset = switch (builtin.arch) {
9483};
9584
9685// Per-thread storage for Zig's use
97const CustomData = packed struct {};
86const CustomData = struct {
87 dummy: usize,
88};
9889
9990// Dynamic Thread Vector
100const DTV = packed struct {
91const DTV = extern struct {
10192 entries: usize,
102 tls_block: [1]usize,
93 tls_block: [1][*]u8,
10394};
10495
10596// Holds all the information about the process TLS image
10697const TLSImage = struct {
107 data_src: []u8,
98 init_data: []const u8,
10899 alloc_size: usize,
100 alloc_align: usize,
109101 tcb_offset: usize,
110102 dtv_offset: usize,
111103 data_offset: usize,
104 data_size: usize,
112105 // Only used on the i386 architecture
113106 gdt_entry_number: usize,
114107};
115108
116pub var tls_image: ?TLSImage = null;
109pub var tls_image: TLSImage = undefined;
117110
118111pub fn setThreadPointer(addr: usize) void {
119112 switch (builtin.arch) {
120113 .i386 => {
121114 var user_desc = std.os.linux.user_desc{
122 .entry_number = tls_image.?.gdt_entry_number,
115 .entry_number = tls_image.gdt_entry_number,
123116 .base_addr = addr,
124117 .limit = 0xfffff,
125118 .seg_32bit = 1,
......@@ -134,7 +127,7 @@ pub fn setThreadPointer(addr: usize) void {
134127
135128 const gdt_entry_number = user_desc.entry_number;
136129 // We have to keep track of our slot as it's also needed for clone()
137 tls_image.?.gdt_entry_number = gdt_entry_number;
130 tls_image.gdt_entry_number = gdt_entry_number;
138131 // Update the %gs selector
139132 asm volatile ("movl %[gs_val], %%gs"
140133 :
......@@ -171,7 +164,7 @@ pub fn setThreadPointer(addr: usize) void {
171164 }
172165}
173166
174pub fn initTLS() ?*elf.Phdr {
167fn initTLS() void {
175168 var tls_phdr: ?*elf.Phdr = null;
176169 var img_base: usize = 0;
177170
......@@ -195,124 +188,149 @@ pub fn initTLS() ?*elf.Phdr {
195188 // Sanity check
196189 assert(at_phent == @sizeOf(elf.Phdr));
197190
198 // Search the TLS section
191 // Find the TLS section
199192 const phdrs = (@intToPtr([*]elf.Phdr, at_phdr))[0..at_phnum];
200193
201 var gnu_stack: ?*elf.Phdr = null;
202
203194 for (phdrs) |*phdr| {
204195 switch (phdr.p_type) {
205196 elf.PT_PHDR => img_base = at_phdr - phdr.p_vaddr,
206197 elf.PT_TLS => tls_phdr = phdr,
207 elf.PT_GNU_STACK => gnu_stack = phdr,
208 else => continue,
198 else => {},
209199 }
210200 }
211201
212 if (tls_phdr) |phdr| {
213 // If the cpu is arm-based, check if it supports the TLS register
214 if (builtin.arch == .arm and at_hwcap & std.os.linux.HWCAP_TLS == 0) {
215 // If the CPU does not support TLS via a coprocessor register,
216 // a kernel helper function can be used instead on certain linux kernels.
217 // See linux/arch/arm/include/asm/tls.h and musl/src/thread/arm/__set_thread_area.c.
218 @panic("TODO: Implement ARM fallback TLS functionality");
219 }
202 // If the cpu is ARM-based, check if it supports the TLS register
203 if (comptime builtin.arch.isARM() and at_hwcap & std.os.linux.HWCAP_TLS == 0) {
204 // If the CPU does not support TLS via a coprocessor register,
205 // a kernel helper function can be used instead on certain linux kernels.
206 // See linux/arch/arm/include/asm/tls.h and musl/src/thread/arm/__set_thread_area.c.
207 @panic("TODO: Implement ARM fallback TLS functionality");
208 }
220209
221 // Offsets into the allocated TLS area
222 var tcb_offset: usize = undefined;
223 var dtv_offset: usize = undefined;
224 var data_offset: usize = undefined;
225 var thread_data_offset: usize = undefined;
226 // Compute the total size of the ABI-specific data plus our own control
227 // structures
228 const alloc_size = switch (tls_variant) {
229 .VariantI => blk: {
230 var l: usize = 0;
231 dtv_offset = l;
232 l += @sizeOf(DTV);
233 thread_data_offset = l;
234 l += @sizeOf(CustomData);
235 l = mem.alignForward(l, phdr.p_align);
236 tcb_offset = l;
237 if (tls_tcb_align_size) {
238 l += mem.alignForward(tls_tcb_size, phdr.p_align);
239 } else {
240 l += tls_tcb_size;
241 }
242 data_offset = l;
243 l += phdr.p_memsz;
244 break :blk l;
245 },
246 .VariantII => blk: {
247 var l: usize = 0;
248 data_offset = l;
249 l += phdr.p_memsz;
250 l = mem.alignForward(l, phdr.p_align);
251 tcb_offset = l;
252 l += tls_tcb_size;
253 thread_data_offset = l;
254 l += @sizeOf(CustomData);
255 dtv_offset = l;
256 l += @sizeOf(DTV);
257 break :blk l;
258 },
259 };
260
261 tls_image = TLSImage{
262 .data_src = @intToPtr([*]u8, phdr.p_vaddr + img_base)[0..phdr.p_filesz],
263 .alloc_size = alloc_size,
264 .tcb_offset = tcb_offset,
265 .dtv_offset = dtv_offset,
266 .data_offset = data_offset,
267 .gdt_entry_number = @bitCast(usize, @as(isize, -1)),
268 };
210 var tls_align_factor: usize = undefined;
211 var tls_data: []const u8 = undefined;
212 var tls_data_alloc_size: usize = undefined;
213 if (tls_phdr) |phdr| {
214 // The effective size in memory is represented by p_memsz, the length of
215 // the data stored in the PT_TLS segment is p_filesz and may be less
216 // than the former
217 tls_align_factor = phdr.p_align;
218 tls_data = @intToPtr([*]u8, img_base + phdr.p_vaddr)[0..phdr.p_filesz];
219 tls_data_alloc_size = phdr.p_memsz;
220 } else {
221 tls_align_factor = @alignOf(*usize);
222 tls_data = &[_]u8{};
223 tls_data_alloc_size = 0;
269224 }
270225
271 return gnu_stack;
226 // Offsets into the allocated TLS area
227 var tcb_offset: usize = undefined;
228 var dtv_offset: usize = undefined;
229 var data_offset: usize = undefined;
230 // Compute the total size of the ABI-specific data plus our own control
231 // structures. All the offset calculated here assume a well-aligned base
232 // address.
233 const alloc_size = switch (tls_variant) {
234 .VariantI => blk: {
235 var l: usize = 0;
236 dtv_offset = l;
237 l += @sizeOf(DTV);
238 // Add some padding here so that the thread pointer (tcb_offset) is
239 // aligned to p_align and the CustomData structure can be found by
240 // simply subtracting its @sizeOf from the tp value
241 const delta = (l + @sizeOf(CustomData)) & (tls_align_factor - 1);
242 if (delta > 0)
243 l += tls_align_factor - delta;
244 l += @sizeOf(CustomData);
245 tcb_offset = l;
246 l += mem.alignForward(tls_tcb_size, tls_align_factor);
247 data_offset = l;
248 l += tls_data_alloc_size;
249 break :blk l;
250 },
251 .VariantII => blk: {
252 var l: usize = 0;
253 data_offset = l;
254 l += mem.alignForward(tls_data_alloc_size, tls_align_factor);
255 // The thread pointer is aligned to p_align
256 tcb_offset = l;
257 l += tls_tcb_size;
258 // The CustomData structure is right after the TCB with no padding
259 // in between so it can be easily found
260 l += @sizeOf(CustomData);
261 l = mem.alignForward(l, @alignOf(DTV));
262 dtv_offset = l;
263 l += @sizeOf(DTV);
264 break :blk l;
265 },
266 };
267
268 tls_image = TLSImage{
269 .init_data = tls_data,
270 .alloc_size = alloc_size,
271 .alloc_align = tls_align_factor,
272 .tcb_offset = tcb_offset,
273 .dtv_offset = dtv_offset,
274 .data_offset = data_offset,
275 .data_size = tls_data_alloc_size,
276 .gdt_entry_number = @bitCast(usize, @as(isize, -1)),
277 };
272278}
273279
274pub fn copyTLS(addr: usize) usize {
275 const tls_img = tls_image.?;
280inline fn alignPtrCast(comptime T: type, ptr: [*]u8) *T {
281 return @ptrCast(*T, @alignCast(@alignOf(*T), ptr));
282}
276283
277 // Be paranoid, clear the area we're going to use
278 @memset(@intToPtr([*]u8, addr), 0, tls_img.alloc_size);
284/// Initializes all the fields of the static TLS area and returns the computed
285/// architecture-specific value of the thread-pointer register
286pub fn prepareTLS(area: []u8) usize {
287 // Clear the area we're going to use, just to be safe
288 mem.set(u8, area, 0);
279289 // Prepare the DTV
280 const dtv = @intToPtr(*DTV, addr + tls_img.dtv_offset);
290 const dtv = alignPtrCast(DTV, area.ptr + tls_image.dtv_offset);
281291 dtv.entries = 1;
282 dtv.tls_block[0] = addr + tls_img.data_offset + tls_dtv_offset;
283 // Set-up the TCB
284 // Force the alignment to 1 byte as the TCB may start from a non-aligned
285 // address under the variant II model
286 const tcb_ptr = @intToPtr(*align(1) usize, addr + tls_img.tcb_offset);
287 if (tls_variant == TLSVariant.VariantI) {
288 tcb_ptr.* = addr + tls_img.dtv_offset;
289 } else {
290 tcb_ptr.* = addr + tls_img.tcb_offset;
291 }
292 dtv.tls_block[0] = area.ptr + tls_dtv_offset + tls_image.data_offset;
293 // Prepare the TCB
294 const tcb_ptr = alignPtrCast([*]u8, area.ptr + tls_image.tcb_offset);
295 tcb_ptr.* = switch (tls_variant) {
296 .VariantI => area.ptr + tls_image.dtv_offset,
297 .VariantII => area.ptr + tls_image.tcb_offset,
298 };
292299 // Copy the data
293 @memcpy(@intToPtr([*]u8, addr + tls_img.data_offset), tls_img.data_src.ptr, tls_img.data_src.len);
300 mem.copy(u8, area[tls_image.data_offset..], tls_image.init_data);
294301
295302 // Return the corrected (if needed) value for the tp register
296 return addr + tls_tp_offset +
297 if (tls_tp_points_past_tcb) tls_img.data_offset else tls_img.tcb_offset;
303 return @ptrToInt(area.ptr) + tls_tp_offset +
304 if (tls_tp_points_past_tcb) tls_image.data_offset else tls_image.tcb_offset;
298305}
299306
300var main_thread_tls_buffer: [256]u8 align(32) = undefined;
307var main_thread_tls_buffer: [256]u8 = undefined;
301308
302pub fn allocateTLS(size: usize) usize {
303 // Small TLS allocation, use our local buffer
304 if (size < main_thread_tls_buffer.len) {
305 return @ptrToInt(&main_thread_tls_buffer);
306 }
309pub fn initStaticTLS() void {
310 initTLS();
311
312 const alloc_tls_area: []u8 = blk: {
313 const full_alloc_size = tls_image.alloc_size + tls_image.alloc_align - 1;
314
315 // Fast path for the common case where the TLS data is really small,
316 // avoid an allocation and use our local buffer
317 if (full_alloc_size < main_thread_tls_buffer.len)
318 break :blk main_thread_tls_buffer[0..];
319
320 break :blk os.mmap(
321 null,
322 full_alloc_size,
323 os.PROT_READ | os.PROT_WRITE,
324 os.MAP_PRIVATE | os.MAP_ANONYMOUS,
325 -1,
326 0,
327 ) catch os.abort();
328 };
307329
308 const slice = os.mmap(
309 null,
310 size,
311 os.PROT_READ | os.PROT_WRITE,
312 os.MAP_PRIVATE | os.MAP_ANONYMOUS,
313 -1,
314 0,
315 ) catch @panic("out of memory");
330 // Make sure the slice is correctly aligned
331 const start = @ptrToInt(alloc_tls_area.ptr) & (tls_image.alloc_align - 1);
332 const tls_area = alloc_tls_area[start .. start + tls_image.alloc_size];
316333
317 return @ptrToInt(slice.ptr);
334 const tp_value = prepareTLS(tls_area);
335 setThreadPointer(tp_value);
318336}
lib/std/start.zig+1-7
......@@ -152,13 +152,7 @@ fn posixCallMainAndExit() noreturn {
152152 const auxv = @ptrCast([*]std.elf.Auxv, @alignCast(@alignOf(usize), envp.ptr + envp_count + 1));
153153 std.os.linux.elf_aux_maybe = auxv;
154154 // Initialize the TLS area
155 const gnu_stack_phdr = std.os.linux.tls.initTLS() orelse @panic("ELF missing stack size");
156
157 if (std.os.linux.tls.tls_image) |tls_img| {
158 const tls_addr = std.os.linux.tls.allocateTLS(tls_img.alloc_size);
159 const tp = std.os.linux.tls.copyTLS(tls_addr);
160 std.os.linux.tls.setThreadPointer(tp);
161 }
155 std.os.linux.tls.initStaticTLS();
162156
163157 // TODO This is disabled because what should we do when linking libc and this code
164158 // does not execute? And also it's causing a test failure in stack traces in release modes.
lib/std/thread.zig+16-16
......@@ -286,11 +286,9 @@ pub const Thread = struct {
286286 }
287287 // Finally, the Thread Local Storage, if any.
288288 if (!Thread.use_pthreads) {
289 if (os.linux.tls.tls_image) |tls_img| {
290 l = mem.alignForward(l, @alignOf(usize));
291 tls_start_offset = l;
292 l += tls_img.alloc_size;
293 }
289 l = mem.alignForward(l, os.linux.tls.tls_image.alloc_align);
290 tls_start_offset = l;
291 l += os.linux.tls.tls_image.alloc_size;
294292 }
295293 // Round the size to the page size.
296294 break :blk mem.alignForward(l, mem.page_size);
......@@ -396,18 +394,21 @@ pub const Thread = struct {
396394 else => return os.unexpectedErrno(@intCast(usize, err)),
397395 }
398396 } else if (std.Target.current.os.tag == .linux) {
399 var flags: u32 = os.CLONE_VM | os.CLONE_FS | os.CLONE_FILES | os.CLONE_SIGHAND |
400 os.CLONE_THREAD | os.CLONE_SYSVSEM | os.CLONE_PARENT_SETTID | os.CLONE_CHILD_CLEARTID |
401 os.CLONE_DETACHED;
402 var newtls: usize = undefined;
397 const flags: u32 = os.CLONE_VM | os.CLONE_FS | os.CLONE_FILES |
398 os.CLONE_SIGHAND | os.CLONE_THREAD | os.CLONE_SYSVSEM |
399 os.CLONE_PARENT_SETTID | os.CLONE_CHILD_CLEARTID |
400 os.CLONE_DETACHED | os.CLONE_SETTLS;
403401 // This structure is only needed when targeting i386
404402 var user_desc: if (std.Target.current.cpu.arch == .i386) os.linux.user_desc else void = undefined;
405403
406 if (os.linux.tls.tls_image) |tls_img| {
404 const tls_area = mmap_slice[tls_start_offset..];
405 const tp_value = os.linux.tls.prepareTLS(tls_area);
406
407 const newtls = blk: {
407408 if (std.Target.current.cpu.arch == .i386) {
408409 user_desc = os.linux.user_desc{
409 .entry_number = tls_img.gdt_entry_number,
410 .base_addr = os.linux.tls.copyTLS(mmap_addr + tls_start_offset),
410 .entry_number = os.linux.tls.tls_image.gdt_entry_number,
411 .base_addr = tp_value,
411412 .limit = 0xfffff,
412413 .seg_32bit = 1,
413414 .contents = 0, // Data
......@@ -416,12 +417,11 @@ pub const Thread = struct {
416417 .seg_not_present = 0,
417418 .useable = 1,
418419 };
419 newtls = @ptrToInt(&user_desc);
420 break :blk @ptrToInt(&user_desc);
420421 } else {
421 newtls = os.linux.tls.copyTLS(mmap_addr + tls_start_offset);
422 break :blk tp_value;
422423 }
423 flags |= os.CLONE_SETTLS;
424 }
424 };
425425
426426 const rc = os.linux.clone(
427427 MainFuncs.linuxThreadMain,
test/stack_traces.zig+1-1
......@@ -282,7 +282,7 @@ pub fn addCases(cases: *tests.StackTracesContext) void {
282282 \\source.zig:10:8: [address] in main (test)
283283 \\ foo();
284284 \\ ^
285 \\start.zig:256:29: [address] in std.start.posixCallMainAndExit (test)
285 \\start.zig:250:29: [address] in std.start.posixCallMainAndExit (test)
286286 \\ return root.main();
287287 \\ ^
288288 \\start.zig:123:5: [address] in std.start._start (test)