authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-03-25 18:53:04-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2020-03-25 18:53:04-04:00
logee6fda2297bf75432b8d7115ec4c60c213535bbe
treeb24ecd83a49cf2142820eba4094e488044589ca5
parentf313ab18aecea1ade0b6a90d671352a641ad351a
parentabcd9ac9d01f6915ceecdb690ed5531495c8cd9a
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #4807 from LemonBoy/tls-touchups

std: Minor changes to TLS handling

3 files changed, 142 insertions(+), 136 deletions(-)

lib/std/os/linux/tls.zig+124-113
......@@ -1,8 +1,9 @@
11const std = @import("std");
2const builtin = std.builtin;
23const os = std.os;
34const mem = std.mem;
45const elf = std.elf;
5const builtin = @import("builtin");
6const math = std.math;
67const assert = std.debug.assert;
78
89// This file implements the two TLS variants [1] used by ELF-based systems.
......@@ -60,10 +61,11 @@ const tls_tcb_size = switch (builtin.arch) {
6061 else => @sizeOf(usize),
6162};
6263
63// Controls if the TCB should be aligned according to the TLS segment p_align
64// Controls the minimum alignment of the TCB end address. The effective value
65// used by the code is min(this_value, tls_segment.p_align)
6466const tls_tcb_align_size = switch (builtin.arch) {
65 .arm, .armeb, .aarch64, .aarch64_be => true,
66 else => false,
67 .arm, .armeb, .aarch64, .aarch64_be => 16,
68 else => 1,
6769};
6870
6971// Controls if the TP points to the end of the TCB instead of its beginning
......@@ -72,13 +74,6 @@ const tls_tp_points_past_tcb = switch (builtin.arch) {
7274 else => false,
7375};
7476
75// Check if the architecture-specific parameters look correct
76comptime {
77 if (tls_tcb_align_size and tls_variant != TLSVariant.VariantI) {
78 @compileError("tls_tcb_align_size is only meaningful for variant I TLS");
79 }
80}
81
8277// Some architectures add some offset to the tp and dtv addresses in order to
8378// make the generated code more efficient
8479
......@@ -94,17 +89,19 @@ const tls_dtv_offset = switch (builtin.arch) {
9489};
9590
9691// Per-thread storage for Zig's use
97const CustomData = packed struct {};
92const CustomData = struct {
93 padding: [16]usize,
94};
9895
9996// Dynamic Thread Vector
100const DTV = packed struct {
97const DTV = extern struct {
10198 entries: usize,
102 tls_block: [1]usize,
99 tls_block: [1][*]u8,
103100};
104101
105102// Holds all the information about the process TLS image
106103const TLSImage = struct {
107 data_src: []u8,
104 data_src: []const u8,
108105 alloc_size: usize,
109106 tcb_offset: usize,
110107 dtv_offset: usize,
......@@ -113,13 +110,13 @@ const TLSImage = struct {
113110 gdt_entry_number: usize,
114111};
115112
116pub var tls_image: ?TLSImage = null;
113pub var tls_image: TLSImage = undefined;
117114
118115pub fn setThreadPointer(addr: usize) void {
119116 switch (builtin.arch) {
120117 .i386 => {
121118 var user_desc = std.os.linux.user_desc{
122 .entry_number = tls_image.?.gdt_entry_number,
119 .entry_number = tls_image.gdt_entry_number,
123120 .base_addr = addr,
124121 .limit = 0xfffff,
125122 .seg_32bit = 1,
......@@ -134,7 +131,7 @@ pub fn setThreadPointer(addr: usize) void {
134131
135132 const gdt_entry_number = user_desc.entry_number;
136133 // We have to keep track of our slot as it's also needed for clone()
137 tls_image.?.gdt_entry_number = gdt_entry_number;
134 tls_image.gdt_entry_number = gdt_entry_number;
138135 // Update the %gs selector
139136 asm volatile ("movl %[gs_val], %%gs"
140137 :
......@@ -171,7 +168,7 @@ pub fn setThreadPointer(addr: usize) void {
171168 }
172169}
173170
174pub fn initTLS() ?*elf.Phdr {
171fn initTLS() void {
175172 var tls_phdr: ?*elf.Phdr = null;
176173 var img_base: usize = 0;
177174
......@@ -195,124 +192,138 @@ pub fn initTLS() ?*elf.Phdr {
195192 // Sanity check
196193 assert(at_phent == @sizeOf(elf.Phdr));
197194
198 // Search the TLS section
195 // Find the TLS section
199196 const phdrs = (@intToPtr([*]elf.Phdr, at_phdr))[0..at_phnum];
200197
201 var gnu_stack: ?*elf.Phdr = null;
202
203198 for (phdrs) |*phdr| {
204199 switch (phdr.p_type) {
205200 elf.PT_PHDR => img_base = at_phdr - phdr.p_vaddr,
206201 elf.PT_TLS => tls_phdr = phdr,
207 elf.PT_GNU_STACK => gnu_stack = phdr,
208 else => continue,
202 else => {},
209203 }
210204 }
211205
212 if (tls_phdr) |phdr| {
213 // If the cpu is arm-based, check if it supports the TLS register
214 if (builtin.arch == .arm and at_hwcap & std.os.linux.HWCAP_TLS == 0) {
215 // If the CPU does not support TLS via a coprocessor register,
216 // a kernel helper function can be used instead on certain linux kernels.
217 // See linux/arch/arm/include/asm/tls.h and musl/src/thread/arm/__set_thread_area.c.
218 @panic("TODO: Implement ARM fallback TLS functionality");
219 }
206 // If the cpu is ARM-based, check if it supports the TLS register
207 if (comptime builtin.arch.isARM() and at_hwcap & std.os.linux.HWCAP_TLS == 0) {
208 // If the CPU does not support TLS via a coprocessor register,
209 // a kernel helper function can be used instead on certain linux kernels.
210 // See linux/arch/arm/include/asm/tls.h and musl/src/thread/arm/__set_thread_area.c.
211 @panic("TODO: Implement ARM fallback TLS functionality");
212 }
220213
221 // Offsets into the allocated TLS area
222 var tcb_offset: usize = undefined;
223 var dtv_offset: usize = undefined;
224 var data_offset: usize = undefined;
225 var thread_data_offset: usize = undefined;
226 // Compute the total size of the ABI-specific data plus our own control
227 // structures
228 const alloc_size = switch (tls_variant) {
229 .VariantI => blk: {
230 var l: usize = 0;
231 dtv_offset = l;
232 l += @sizeOf(DTV);
233 thread_data_offset = l;
234 l += @sizeOf(CustomData);
235 l = mem.alignForward(l, phdr.p_align);
236 tcb_offset = l;
237 if (tls_tcb_align_size) {
238 l += mem.alignForward(tls_tcb_size, phdr.p_align);
239 } else {
240 l += tls_tcb_size;
241 }
242 data_offset = l;
243 l += phdr.p_memsz;
244 break :blk l;
245 },
246 .VariantII => blk: {
247 var l: usize = 0;
248 data_offset = l;
249 l += phdr.p_memsz;
250 l = mem.alignForward(l, phdr.p_align);
251 tcb_offset = l;
252 l += tls_tcb_size;
253 thread_data_offset = l;
254 l += @sizeOf(CustomData);
255 dtv_offset = l;
256 l += @sizeOf(DTV);
257 break :blk l;
258 },
259 };
260
261 tls_image = TLSImage{
262 .data_src = @intToPtr([*]u8, phdr.p_vaddr + img_base)[0..phdr.p_filesz],
263 .alloc_size = alloc_size,
264 .tcb_offset = tcb_offset,
265 .dtv_offset = dtv_offset,
266 .data_offset = data_offset,
267 .gdt_entry_number = @bitCast(usize, @as(isize, -1)),
268 };
214 var tls_align_factor: usize = undefined;
215 var tls_data: []const u8 = undefined;
216 if (tls_phdr) |phdr| {
217 tls_align_factor = phdr.p_align;
218 tls_data = @intToPtr([*]u8, img_base + phdr.p_vaddr)[0..phdr.p_memsz];
219 } else {
220 tls_align_factor = @alignOf(*usize);
221 tls_data = &[_]u8{};
269222 }
270223
271 return gnu_stack;
224 // Offsets into the allocated TLS area
225 var tcb_offset: usize = undefined;
226 var dtv_offset: usize = undefined;
227 var data_offset: usize = undefined;
228 var thread_data_offset: usize = undefined;
229 // Compute the total size of the ABI-specific data plus our own control
230 // structures
231 const alloc_size = switch (tls_variant) {
232 .VariantI => blk: {
233 var l: usize = 0;
234 // Unneeded because l is zero
235 // l = mem.alignForward(l, @alignOf(DTV));
236 dtv_offset = l;
237 l += @sizeOf(DTV);
238 l = mem.alignForward(l, @alignOf(CustomData));
239 thread_data_offset = l;
240 l += @sizeOf(CustomData);
241 // Make sure the TP is aligned
242 l = mem.alignForward(l, tls_align_factor);
243 tcb_offset = l;
244 // Ensure there are at least tls_tcb_align_size bytes of padding
245 const min_align = math.max(tls_tcb_align_size, tls_align_factor);
246 l += mem.alignForward(tls_tcb_size, min_align);
247 data_offset = l;
248 l += mem.alignForward(tls_data.len, tls_align_factor);
249 break :blk l;
250 },
251 .VariantII => blk: {
252 var l: usize = 0;
253 data_offset = l;
254 l = mem.alignForward(tls_data.len, tls_align_factor);
255 // The TP is aligned to p_align
256 tcb_offset = l;
257 l += tls_tcb_size;
258 l = mem.alignForward(l, @alignOf(CustomData));
259 thread_data_offset = l;
260 l += @sizeOf(CustomData);
261 l = mem.alignForward(l, @alignOf(DTV));
262 dtv_offset = l;
263 l += @sizeOf(DTV);
264 break :blk l;
265 },
266 };
267
268 tls_image = TLSImage{
269 .data_src = tls_data,
270 .alloc_size = alloc_size,
271 .tcb_offset = tcb_offset,
272 .dtv_offset = dtv_offset,
273 .data_offset = data_offset,
274 .gdt_entry_number = @bitCast(usize, @as(isize, -1)),
275 };
272276}
273277
274pub fn copyTLS(addr: usize) usize {
275 const tls_img = tls_image.?;
278inline fn alignPtrCast(comptime T: type, ptr: [*]u8) *T {
279 return @ptrCast(*T, @alignCast(@alignOf(*T), ptr));
280}
276281
277 // Be paranoid, clear the area we're going to use
278 @memset(@intToPtr([*]u8, addr), 0, tls_img.alloc_size);
282/// Initializes all the fields of the static TLS area and returns the computed
283/// architecture-specific value of the thread-pointer register
284pub fn prepareTLS(area: []u8) usize {
285 // Clear the area we're going to use, just to be safe
286 mem.set(u8, area, 0);
279287 // Prepare the DTV
280 const dtv = @intToPtr(*DTV, addr + tls_img.dtv_offset);
288 const dtv = alignPtrCast(DTV, area.ptr + tls_image.dtv_offset);
281289 dtv.entries = 1;
282 dtv.tls_block[0] = addr + tls_img.data_offset + tls_dtv_offset;
283 // Set-up the TCB
284 // Force the alignment to 1 byte as the TCB may start from a non-aligned
285 // address under the variant II model
286 const tcb_ptr = @intToPtr(*align(1) usize, addr + tls_img.tcb_offset);
287 if (tls_variant == TLSVariant.VariantI) {
288 tcb_ptr.* = addr + tls_img.dtv_offset;
289 } else {
290 tcb_ptr.* = addr + tls_img.tcb_offset;
291 }
290 dtv.tls_block[0] = area.ptr + tls_dtv_offset + tls_image.data_offset;
291 // Prepare the TCB
292 const tcb_ptr = alignPtrCast([*]u8, area.ptr + tls_image.tcb_offset);
293 tcb_ptr.* = switch (tls_variant) {
294 .VariantI => area.ptr + tls_image.dtv_offset,
295 .VariantII => area.ptr + tls_image.tcb_offset,
296 };
292297 // Copy the data
293 @memcpy(@intToPtr([*]u8, addr + tls_img.data_offset), tls_img.data_src.ptr, tls_img.data_src.len);
298 mem.copy(u8, area[tls_image.data_offset..], tls_image.data_src);
294299
295300 // Return the corrected (if needed) value for the tp register
296 return addr + tls_tp_offset +
297 if (tls_tp_points_past_tcb) tls_img.data_offset else tls_img.tcb_offset;
301 return @ptrToInt(area.ptr) + tls_tp_offset +
302 if (tls_tp_points_past_tcb) tls_image.data_offset else tls_image.tcb_offset;
298303}
299304
300305var main_thread_tls_buffer: [256]u8 align(32) = undefined;
301306
302pub fn allocateTLS(size: usize) usize {
303 // Small TLS allocation, use our local buffer
304 if (size < main_thread_tls_buffer.len) {
305 return @ptrToInt(&main_thread_tls_buffer);
306 }
307pub fn initStaticTLS() void {
308 initTLS();
307309
308 const slice = os.mmap(
309 null,
310 size,
311 os.PROT_READ | os.PROT_WRITE,
312 os.MAP_PRIVATE | os.MAP_ANONYMOUS,
313 -1,
314 0,
315 ) catch @panic("out of memory");
310 var tls_area = blk: {
311 // Fast path for the common case where the TLS data is really small,
312 // avoid an allocation and use our local buffer
313 if (tls_image.alloc_size < main_thread_tls_buffer.len) {
314 break :blk main_thread_tls_buffer[0..tls_image.alloc_size];
315 }
316316
317 return @ptrToInt(slice.ptr);
317 break :blk os.mmap(
318 null,
319 tls_image.alloc_size,
320 os.PROT_READ | os.PROT_WRITE,
321 os.MAP_PRIVATE | os.MAP_ANONYMOUS,
322 -1,
323 0,
324 ) catch @panic("out of memory");
325 };
326
327 const tp_value = prepareTLS(tls_area);
328 setThreadPointer(tp_value);
318329}
lib/std/start.zig+1-7
......@@ -152,13 +152,7 @@ fn posixCallMainAndExit() noreturn {
152152 const auxv = @ptrCast([*]std.elf.Auxv, @alignCast(@alignOf(usize), envp.ptr + envp_count + 1));
153153 std.os.linux.elf_aux_maybe = auxv;
154154 // Initialize the TLS area
155 const gnu_stack_phdr = std.os.linux.tls.initTLS() orelse @panic("ELF missing stack size");
156
157 if (std.os.linux.tls.tls_image) |tls_img| {
158 const tls_addr = std.os.linux.tls.allocateTLS(tls_img.alloc_size);
159 const tp = std.os.linux.tls.copyTLS(tls_addr);
160 std.os.linux.tls.setThreadPointer(tp);
161 }
155 std.os.linux.tls.initStaticTLS();
162156
163157 // TODO This is disabled because what should we do when linking libc and this code
164158 // does not execute? And also it's causing a test failure in stack traces in release modes.
lib/std/thread.zig+17-16
......@@ -286,11 +286,10 @@ pub const Thread = struct {
286286 }
287287 // Finally, the Thread Local Storage, if any.
288288 if (!Thread.use_pthreads) {
289 if (os.linux.tls.tls_image) |tls_img| {
290 l = mem.alignForward(l, @alignOf(usize));
291 tls_start_offset = l;
292 l += tls_img.alloc_size;
293 }
289 // XXX: Is this alignment enough?
290 l = mem.alignForward(l, @alignOf(usize));
291 tls_start_offset = l;
292 l += os.linux.tls.tls_image.alloc_size;
294293 }
295294 // Round the size to the page size.
296295 break :blk mem.alignForward(l, mem.page_size);
......@@ -396,18 +395,21 @@ pub const Thread = struct {
396395 else => return os.unexpectedErrno(@intCast(usize, err)),
397396 }
398397 } else if (std.Target.current.os.tag == .linux) {
399 var flags: u32 = os.CLONE_VM | os.CLONE_FS | os.CLONE_FILES | os.CLONE_SIGHAND |
400 os.CLONE_THREAD | os.CLONE_SYSVSEM | os.CLONE_PARENT_SETTID | os.CLONE_CHILD_CLEARTID |
401 os.CLONE_DETACHED;
402 var newtls: usize = undefined;
398 const flags: u32 = os.CLONE_VM | os.CLONE_FS | os.CLONE_FILES |
399 os.CLONE_SIGHAND | os.CLONE_THREAD | os.CLONE_SYSVSEM |
400 os.CLONE_PARENT_SETTID | os.CLONE_CHILD_CLEARTID |
401 os.CLONE_DETACHED | os.CLONE_SETTLS;
403402 // This structure is only needed when targeting i386
404403 var user_desc: if (std.Target.current.cpu.arch == .i386) os.linux.user_desc else void = undefined;
405404
406 if (os.linux.tls.tls_image) |tls_img| {
405 const tls_area = mmap_slice[tls_start_offset..];
406 const tp_value = os.linux.tls.prepareTLS(tls_area);
407
408 const newtls = blk: {
407409 if (std.Target.current.cpu.arch == .i386) {
408410 user_desc = os.linux.user_desc{
409 .entry_number = tls_img.gdt_entry_number,
410 .base_addr = os.linux.tls.copyTLS(mmap_addr + tls_start_offset),
411 .entry_number = os.linux.tls.tls_image.gdt_entry_number,
412 .base_addr = tp_value,
411413 .limit = 0xfffff,
412414 .seg_32bit = 1,
413415 .contents = 0, // Data
......@@ -416,12 +418,11 @@ pub const Thread = struct {
416418 .seg_not_present = 0,
417419 .useable = 1,
418420 };
419 newtls = @ptrToInt(&user_desc);
421 break :blk @ptrToInt(&user_desc);
420422 } else {
421 newtls = os.linux.tls.copyTLS(mmap_addr + tls_start_offset);
423 break :blk tp_value;
422424 }
423 flags |= os.CLONE_SETTLS;
424 }
425 };
425426
426427 const rc = os.linux.clone(
427428 MainFuncs.linuxThreadMain,