authorgravatar for thatlemon@gmail.comLemonBoy <thatlemon@gmail.com> 2020-03-25 12:08:50+01:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-03-28 11:20:38-04:00
logd788b0cd8b0b12ec4f3dddf4ecdb59556b73d4c2
treebc0dbd74dbdeb7f0fc8bdfaac3d2cf4a1b92006a
parent12e1c6e21c1d21238450903ad7f8fa4ce52ee758

std: Minor changes to TLS handling

* Always allocate an info block per-thread so that libc can store important stuff there. * Respect ABI-mandated alignment in more places. * Nicer code, use slices/pointers instead of raw addresses whenever possible.

4 files changed, 153 insertions(+), 141 deletions(-)

lib/std/os/linux/tls.zig+135-117
...@@ -1,8 +1,9 @@...@@ -1,8 +1,9 @@
1const std = @import("std");1const std = @import("std");
2const builtin = std.builtin;
2const os = std.os;3const os = std.os;
3const mem = std.mem;4const mem = std.mem;
4const elf = std.elf;5const elf = std.elf;
5const builtin = @import("builtin");6const math = std.math;
6const assert = std.debug.assert;7const assert = std.debug.assert;
78
8// This file implements the two TLS variants [1] used by ELF-based systems.9// This file implements the two TLS variants [1] used by ELF-based systems.
...@@ -57,28 +58,16 @@ const tls_tcb_size = switch (builtin.arch) {...@@ -57,28 +58,16 @@ const tls_tcb_size = switch (builtin.arch) {
57 // ARM EABI mandates enough space for two pointers: the first one points to58 // ARM EABI mandates enough space for two pointers: the first one points to
58 // the DTV while the second one is unspecified but reserved59 // the DTV while the second one is unspecified but reserved
59 .arm, .armeb, .aarch64, .aarch64_be => 2 * @sizeOf(usize),60 .arm, .armeb, .aarch64, .aarch64_be => 2 * @sizeOf(usize),
61 // One pointer-sized word that points either to the DTV or the TCB itself
60 else => @sizeOf(usize),62 else => @sizeOf(usize),
61};63};
6264
63// Controls if the TCB should be aligned according to the TLS segment p_align
64const tls_tcb_align_size = switch (builtin.arch) {
65 .arm, .armeb, .aarch64, .aarch64_be => true,
66 else => false,
67};
68
69// Controls if the TP points to the end of the TCB instead of its beginning65// Controls if the TP points to the end of the TCB instead of its beginning
70const tls_tp_points_past_tcb = switch (builtin.arch) {66const tls_tp_points_past_tcb = switch (builtin.arch) {
71 .riscv32, .riscv64, .mipsel, .powerpc64, .powerpc64le => true,67 .riscv32, .riscv64, .mipsel, .powerpc64, .powerpc64le => true,
72 else => false,68 else => false,
73};69};
7470
75// Check if the architecture-specific parameters look correct
76comptime {
77 if (tls_tcb_align_size and tls_variant != TLSVariant.VariantI) {
78 @compileError("tls_tcb_align_size is only meaningful for variant I TLS");
79 }
80}
81
82// Some architectures add some offset to the tp and dtv addresses in order to71// Some architectures add some offset to the tp and dtv addresses in order to
83// make the generated code more efficient72// make the generated code more efficient
8473
...@@ -94,32 +83,36 @@ const tls_dtv_offset = switch (builtin.arch) {...@@ -94,32 +83,36 @@ const tls_dtv_offset = switch (builtin.arch) {
94};83};
9584
96// Per-thread storage for Zig's use85// Per-thread storage for Zig's use
97const CustomData = packed struct {};86const CustomData = struct {
87 dummy: usize,
88};
9889
99// Dynamic Thread Vector90// Dynamic Thread Vector
100const DTV = packed struct {91const DTV = extern struct {
101 entries: usize,92 entries: usize,
102 tls_block: [1]usize,93 tls_block: [1][*]u8,
103};94};
10495
105// Holds all the information about the process TLS image96// Holds all the information about the process TLS image
106const TLSImage = struct {97const TLSImage = struct {
107 data_src: []u8,98 init_data: []const u8,
108 alloc_size: usize,99 alloc_size: usize,
100 alloc_align: usize,
109 tcb_offset: usize,101 tcb_offset: usize,
110 dtv_offset: usize,102 dtv_offset: usize,
111 data_offset: usize,103 data_offset: usize,
104 data_size: usize,
112 // Only used on the i386 architecture105 // Only used on the i386 architecture
113 gdt_entry_number: usize,106 gdt_entry_number: usize,
114};107};
115108
116pub var tls_image: ?TLSImage = null;109pub var tls_image: TLSImage = undefined;
117110
118pub fn setThreadPointer(addr: usize) void {111pub fn setThreadPointer(addr: usize) void {
119 switch (builtin.arch) {112 switch (builtin.arch) {
120 .i386 => {113 .i386 => {
121 var user_desc = std.os.linux.user_desc{114 var user_desc = std.os.linux.user_desc{
122 .entry_number = tls_image.?.gdt_entry_number,115 .entry_number = tls_image.gdt_entry_number,
123 .base_addr = addr,116 .base_addr = addr,
124 .limit = 0xfffff,117 .limit = 0xfffff,
125 .seg_32bit = 1,118 .seg_32bit = 1,
...@@ -134,7 +127,7 @@ pub fn setThreadPointer(addr: usize) void {...@@ -134,7 +127,7 @@ pub fn setThreadPointer(addr: usize) void {
134127
135 const gdt_entry_number = user_desc.entry_number;128 const gdt_entry_number = user_desc.entry_number;
136 // We have to keep track of our slot as it's also needed for clone()129 // We have to keep track of our slot as it's also needed for clone()
137 tls_image.?.gdt_entry_number = gdt_entry_number;130 tls_image.gdt_entry_number = gdt_entry_number;
138 // Update the %gs selector131 // Update the %gs selector
139 asm volatile ("movl %[gs_val], %%gs"132 asm volatile ("movl %[gs_val], %%gs"
140 :133 :
...@@ -171,7 +164,7 @@ pub fn setThreadPointer(addr: usize) void {...@@ -171,7 +164,7 @@ pub fn setThreadPointer(addr: usize) void {
171 }164 }
172}165}
173166
174pub fn initTLS() ?*elf.Phdr {167fn initTLS() void {
175 var tls_phdr: ?*elf.Phdr = null;168 var tls_phdr: ?*elf.Phdr = null;
176 var img_base: usize = 0;169 var img_base: usize = 0;
177170
...@@ -195,124 +188,149 @@ pub fn initTLS() ?*elf.Phdr {...@@ -195,124 +188,149 @@ pub fn initTLS() ?*elf.Phdr {
195 // Sanity check188 // Sanity check
196 assert(at_phent == @sizeOf(elf.Phdr));189 assert(at_phent == @sizeOf(elf.Phdr));
197190
198 // Search the TLS section191 // Find the TLS section
199 const phdrs = (@intToPtr([*]elf.Phdr, at_phdr))[0..at_phnum];192 const phdrs = (@intToPtr([*]elf.Phdr, at_phdr))[0..at_phnum];
200193
201 var gnu_stack: ?*elf.Phdr = null;
202
203 for (phdrs) |*phdr| {194 for (phdrs) |*phdr| {
204 switch (phdr.p_type) {195 switch (phdr.p_type) {
205 elf.PT_PHDR => img_base = at_phdr - phdr.p_vaddr,196 elf.PT_PHDR => img_base = at_phdr - phdr.p_vaddr,
206 elf.PT_TLS => tls_phdr = phdr,197 elf.PT_TLS => tls_phdr = phdr,
207 elf.PT_GNU_STACK => gnu_stack = phdr,198 else => {},
208 else => continue,
209 }199 }
210 }200 }
211201
212 if (tls_phdr) |phdr| {202 // If the cpu is ARM-based, check if it supports the TLS register
213 // If the cpu is arm-based, check if it supports the TLS register203 if (comptime builtin.arch.isARM() and at_hwcap & std.os.linux.HWCAP_TLS == 0) {
214 if (builtin.arch == .arm and at_hwcap & std.os.linux.HWCAP_TLS == 0) {204 // If the CPU does not support TLS via a coprocessor register,
215 // If the CPU does not support TLS via a coprocessor register,205 // a kernel helper function can be used instead on certain linux kernels.
216 // a kernel helper function can be used instead on certain linux kernels.206 // See linux/arch/arm/include/asm/tls.h and musl/src/thread/arm/__set_thread_area.c.
217 // See linux/arch/arm/include/asm/tls.h and musl/src/thread/arm/__set_thread_area.c.207 @panic("TODO: Implement ARM fallback TLS functionality");
218 @panic("TODO: Implement ARM fallback TLS functionality");208 }
219 }
220209
221 // Offsets into the allocated TLS area210 var tls_align_factor: usize = undefined;
222 var tcb_offset: usize = undefined;211 var tls_data: []const u8 = undefined;
223 var dtv_offset: usize = undefined;212 var tls_data_alloc_size: usize = undefined;
224 var data_offset: usize = undefined;213 if (tls_phdr) |phdr| {
225 var thread_data_offset: usize = undefined;214 // The effective size in memory is represented by p_memsz, the length of
226 // Compute the total size of the ABI-specific data plus our own control215 // the data stored in the PT_TLS segment is p_filesz and may be less
227 // structures216 // than the former
228 const alloc_size = switch (tls_variant) {217 tls_align_factor = phdr.p_align;
229 .VariantI => blk: {218 tls_data = @intToPtr([*]u8, img_base + phdr.p_vaddr)[0..phdr.p_filesz];
230 var l: usize = 0;219 tls_data_alloc_size = phdr.p_memsz;
231 dtv_offset = l;220 } else {
232 l += @sizeOf(DTV);221 tls_align_factor = @alignOf(*usize);
233 thread_data_offset = l;222 tls_data = &[_]u8{};
234 l += @sizeOf(CustomData);223 tls_data_alloc_size = 0;
235 l = mem.alignForward(l, phdr.p_align);
236 tcb_offset = l;
237 if (tls_tcb_align_size) {
238 l += mem.alignForward(tls_tcb_size, phdr.p_align);
239 } else {
240 l += tls_tcb_size;
241 }
242 data_offset = l;
243 l += phdr.p_memsz;
244 break :blk l;
245 },
246 .VariantII => blk: {
247 var l: usize = 0;
248 data_offset = l;
249 l += phdr.p_memsz;
250 l = mem.alignForward(l, phdr.p_align);
251 tcb_offset = l;
252 l += tls_tcb_size;
253 thread_data_offset = l;
254 l += @sizeOf(CustomData);
255 dtv_offset = l;
256 l += @sizeOf(DTV);
257 break :blk l;
258 },
259 };
260
261 tls_image = TLSImage{
262 .data_src = @intToPtr([*]u8, phdr.p_vaddr + img_base)[0..phdr.p_filesz],
263 .alloc_size = alloc_size,
264 .tcb_offset = tcb_offset,
265 .dtv_offset = dtv_offset,
266 .data_offset = data_offset,
267 .gdt_entry_number = @bitCast(usize, @as(isize, -1)),
268 };
269 }224 }
270225
271 return gnu_stack;226 // Offsets into the allocated TLS area
227 var tcb_offset: usize = undefined;
228 var dtv_offset: usize = undefined;
229 var data_offset: usize = undefined;
230 // Compute the total size of the ABI-specific data plus our own control
231 // structures. All the offset calculated here assume a well-aligned base
232 // address.
233 const alloc_size = switch (tls_variant) {
234 .VariantI => blk: {
235 var l: usize = 0;
236 dtv_offset = l;
237 l += @sizeOf(DTV);
238 // Add some padding here so that the thread pointer (tcb_offset) is
239 // aligned to p_align and the CustomData structure can be found by
240 // simply subtracting its @sizeOf from the tp value
241 const delta = (l + @sizeOf(CustomData)) & (tls_align_factor - 1);
242 if (delta > 0)
243 l += tls_align_factor - delta;
244 l += @sizeOf(CustomData);
245 tcb_offset = l;
246 l += mem.alignForward(tls_tcb_size, tls_align_factor);
247 data_offset = l;
248 l += tls_data_alloc_size;
249 break :blk l;
250 },
251 .VariantII => blk: {
252 var l: usize = 0;
253 data_offset = l;
254 l += mem.alignForward(tls_data_alloc_size, tls_align_factor);
255 // The thread pointer is aligned to p_align
256 tcb_offset = l;
257 l += tls_tcb_size;
258 // The CustomData structure is right after the TCB with no padding
259 // in between so it can be easily found
260 l += @sizeOf(CustomData);
261 l = mem.alignForward(l, @alignOf(DTV));
262 dtv_offset = l;
263 l += @sizeOf(DTV);
264 break :blk l;
265 },
266 };
267
268 tls_image = TLSImage{
269 .init_data = tls_data,
270 .alloc_size = alloc_size,
271 .alloc_align = tls_align_factor,
272 .tcb_offset = tcb_offset,
273 .dtv_offset = dtv_offset,
274 .data_offset = data_offset,
275 .data_size = tls_data_alloc_size,
276 .gdt_entry_number = @bitCast(usize, @as(isize, -1)),
277 };
272}278}
273279
274pub fn copyTLS(addr: usize) usize {280inline fn alignPtrCast(comptime T: type, ptr: [*]u8) *T {
275 const tls_img = tls_image.?;281 return @ptrCast(*T, @alignCast(@alignOf(*T), ptr));
282}
276283
277 // Be paranoid, clear the area we're going to use284/// Initializes all the fields of the static TLS area and returns the computed
278 @memset(@intToPtr([*]u8, addr), 0, tls_img.alloc_size);285/// architecture-specific value of the thread-pointer register
286pub fn prepareTLS(area: []u8) usize {
287 // Clear the area we're going to use, just to be safe
288 mem.set(u8, area, 0);
279 // Prepare the DTV289 // Prepare the DTV
280 const dtv = @intToPtr(*DTV, addr + tls_img.dtv_offset);290 const dtv = alignPtrCast(DTV, area.ptr + tls_image.dtv_offset);
281 dtv.entries = 1;291 dtv.entries = 1;
282 dtv.tls_block[0] = addr + tls_img.data_offset + tls_dtv_offset;292 dtv.tls_block[0] = area.ptr + tls_dtv_offset + tls_image.data_offset;
283 // Set-up the TCB293 // Prepare the TCB
284 // Force the alignment to 1 byte as the TCB may start from a non-aligned294 const tcb_ptr = alignPtrCast([*]u8, area.ptr + tls_image.tcb_offset);
285 // address under the variant II model295 tcb_ptr.* = switch (tls_variant) {
286 const tcb_ptr = @intToPtr(*align(1) usize, addr + tls_img.tcb_offset);296 .VariantI => area.ptr + tls_image.dtv_offset,
287 if (tls_variant == TLSVariant.VariantI) {297 .VariantII => area.ptr + tls_image.tcb_offset,
288 tcb_ptr.* = addr + tls_img.dtv_offset;298 };
289 } else {
290 tcb_ptr.* = addr + tls_img.tcb_offset;
291 }
292 // Copy the data299 // Copy the data
293 @memcpy(@intToPtr([*]u8, addr + tls_img.data_offset), tls_img.data_src.ptr, tls_img.data_src.len);300 mem.copy(u8, area[tls_image.data_offset..], tls_image.init_data);
294301
295 // Return the corrected (if needed) value for the tp register302 // Return the corrected (if needed) value for the tp register
296 return addr + tls_tp_offset +303 return @ptrToInt(area.ptr) + tls_tp_offset +
297 if (tls_tp_points_past_tcb) tls_img.data_offset else tls_img.tcb_offset;304 if (tls_tp_points_past_tcb) tls_image.data_offset else tls_image.tcb_offset;
298}305}
299306
300var main_thread_tls_buffer: [256]u8 align(32) = undefined;307var main_thread_tls_buffer: [256]u8 = undefined;
301308
302pub fn allocateTLS(size: usize) usize {309pub fn initStaticTLS() void {
303 // Small TLS allocation, use our local buffer310 initTLS();
304 if (size < main_thread_tls_buffer.len) {311
305 return @ptrToInt(&main_thread_tls_buffer);312 const alloc_tls_area: []u8 = blk: {
306 }313 const full_alloc_size = tls_image.alloc_size + tls_image.alloc_align - 1;
314
315 // Fast path for the common case where the TLS data is really small,
316 // avoid an allocation and use our local buffer
317 if (full_alloc_size < main_thread_tls_buffer.len)
318 break :blk main_thread_tls_buffer[0..];
319
320 break :blk os.mmap(
321 null,
322 full_alloc_size,
323 os.PROT_READ | os.PROT_WRITE,
324 os.MAP_PRIVATE | os.MAP_ANONYMOUS,
325 -1,
326 0,
327 ) catch os.abort();
328 };
307329
308 const slice = os.mmap(330 // Make sure the slice is correctly aligned
309 null,331 const start = @ptrToInt(alloc_tls_area.ptr) & (tls_image.alloc_align - 1);
310 size,332 const tls_area = alloc_tls_area[start .. start + tls_image.alloc_size];
311 os.PROT_READ | os.PROT_WRITE,
312 os.MAP_PRIVATE | os.MAP_ANONYMOUS,
313 -1,
314 0,
315 ) catch @panic("out of memory");
316333
317 return @ptrToInt(slice.ptr);334 const tp_value = prepareTLS(tls_area);
335 setThreadPointer(tp_value);
318}336}
lib/std/start.zig+1-7
...@@ -152,13 +152,7 @@ fn posixCallMainAndExit() noreturn {...@@ -152,13 +152,7 @@ fn posixCallMainAndExit() noreturn {
152 const auxv = @ptrCast([*]std.elf.Auxv, @alignCast(@alignOf(usize), envp.ptr + envp_count + 1));152 const auxv = @ptrCast([*]std.elf.Auxv, @alignCast(@alignOf(usize), envp.ptr + envp_count + 1));
153 std.os.linux.elf_aux_maybe = auxv;153 std.os.linux.elf_aux_maybe = auxv;
154 // Initialize the TLS area154 // Initialize the TLS area
155 const gnu_stack_phdr = std.os.linux.tls.initTLS() orelse @panic("ELF missing stack size");155 std.os.linux.tls.initStaticTLS();
156
157 if (std.os.linux.tls.tls_image) |tls_img| {
158 const tls_addr = std.os.linux.tls.allocateTLS(tls_img.alloc_size);
159 const tp = std.os.linux.tls.copyTLS(tls_addr);
160 std.os.linux.tls.setThreadPointer(tp);
161 }
162156
163 // TODO This is disabled because what should we do when linking libc and this code157 // TODO This is disabled because what should we do when linking libc and this code
164 // does not execute? And also it's causing a test failure in stack traces in release modes.158 // does not execute? And also it's causing a test failure in stack traces in release modes.
lib/std/thread.zig+16-16
...@@ -286,11 +286,9 @@ pub const Thread = struct {...@@ -286,11 +286,9 @@ pub const Thread = struct {
286 }286 }
287 // Finally, the Thread Local Storage, if any.287 // Finally, the Thread Local Storage, if any.
288 if (!Thread.use_pthreads) {288 if (!Thread.use_pthreads) {
289 if (os.linux.tls.tls_image) |tls_img| {289 l = mem.alignForward(l, os.linux.tls.tls_image.alloc_align);
290 l = mem.alignForward(l, @alignOf(usize));290 tls_start_offset = l;
291 tls_start_offset = l;291 l += os.linux.tls.tls_image.alloc_size;
292 l += tls_img.alloc_size;
293 }
294 }292 }
295 // Round the size to the page size.293 // Round the size to the page size.
296 break :blk mem.alignForward(l, mem.page_size);294 break :blk mem.alignForward(l, mem.page_size);
...@@ -396,18 +394,21 @@ pub const Thread = struct {...@@ -396,18 +394,21 @@ pub const Thread = struct {
396 else => return os.unexpectedErrno(@intCast(usize, err)),394 else => return os.unexpectedErrno(@intCast(usize, err)),
397 }395 }
398 } else if (std.Target.current.os.tag == .linux) {396 } else if (std.Target.current.os.tag == .linux) {
399 var flags: u32 = os.CLONE_VM | os.CLONE_FS | os.CLONE_FILES | os.CLONE_SIGHAND |397 const flags: u32 = os.CLONE_VM | os.CLONE_FS | os.CLONE_FILES |
400 os.CLONE_THREAD | os.CLONE_SYSVSEM | os.CLONE_PARENT_SETTID | os.CLONE_CHILD_CLEARTID |398 os.CLONE_SIGHAND | os.CLONE_THREAD | os.CLONE_SYSVSEM |
401 os.CLONE_DETACHED;399 os.CLONE_PARENT_SETTID | os.CLONE_CHILD_CLEARTID |
402 var newtls: usize = undefined;400 os.CLONE_DETACHED | os.CLONE_SETTLS;
403 // This structure is only needed when targeting i386401 // This structure is only needed when targeting i386
404 var user_desc: if (std.Target.current.cpu.arch == .i386) os.linux.user_desc else void = undefined;402 var user_desc: if (std.Target.current.cpu.arch == .i386) os.linux.user_desc else void = undefined;
405403
406 if (os.linux.tls.tls_image) |tls_img| {404 const tls_area = mmap_slice[tls_start_offset..];
405 const tp_value = os.linux.tls.prepareTLS(tls_area);
406
407 const newtls = blk: {
407 if (std.Target.current.cpu.arch == .i386) {408 if (std.Target.current.cpu.arch == .i386) {
408 user_desc = os.linux.user_desc{409 user_desc = os.linux.user_desc{
409 .entry_number = tls_img.gdt_entry_number,410 .entry_number = os.linux.tls.tls_image.gdt_entry_number,
410 .base_addr = os.linux.tls.copyTLS(mmap_addr + tls_start_offset),411 .base_addr = tp_value,
411 .limit = 0xfffff,412 .limit = 0xfffff,
412 .seg_32bit = 1,413 .seg_32bit = 1,
413 .contents = 0, // Data414 .contents = 0, // Data
...@@ -416,12 +417,11 @@ pub const Thread = struct {...@@ -416,12 +417,11 @@ pub const Thread = struct {
416 .seg_not_present = 0,417 .seg_not_present = 0,
417 .useable = 1,418 .useable = 1,
418 };419 };
419 newtls = @ptrToInt(&user_desc);420 break :blk @ptrToInt(&user_desc);
420 } else {421 } else {
421 newtls = os.linux.tls.copyTLS(mmap_addr + tls_start_offset);422 break :blk tp_value;
422 }423 }
423 flags |= os.CLONE_SETTLS;424 };
424 }
425425
426 const rc = os.linux.clone(426 const rc = os.linux.clone(
427 MainFuncs.linuxThreadMain,427 MainFuncs.linuxThreadMain,
test/stack_traces.zig+1-1
...@@ -282,7 +282,7 @@ pub fn addCases(cases: *tests.StackTracesContext) void {...@@ -282,7 +282,7 @@ pub fn addCases(cases: *tests.StackTracesContext) void {
282 \\source.zig:10:8: [address] in main (test)282 \\source.zig:10:8: [address] in main (test)
283 \\ foo();283 \\ foo();
284 \\ ^284 \\ ^
285 \\start.zig:256:29: [address] in std.start.posixCallMainAndExit (test)285 \\start.zig:250:29: [address] in std.start.posixCallMainAndExit (test)
286 \\ return root.main();286 \\ return root.main();
287 \\ ^287 \\ ^
288 \\start.zig:123:5: [address] in std.start._start (test)288 \\start.zig:123:5: [address] in std.start._start (test)