authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-03-25 18:53:04-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2020-03-25 18:53:04-04:00
logee6fda2297bf75432b8d7115ec4c60c213535bbe
treeb24ecd83a49cf2142820eba4094e488044589ca5
parentf313ab18aecea1ade0b6a90d671352a641ad351a
parentabcd9ac9d01f6915ceecdb690ed5531495c8cd9a
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #4807 from LemonBoy/tls-touchups

std: Minor changes to TLS handling

3 files changed, 142 insertions(+), 136 deletions(-)

lib/std/os/linux/tls.zig+124-113
...@@ -1,8 +1,9 @@...@@ -1,8 +1,9 @@
1const std = @import("std");1const std = @import("std");
2const builtin = std.builtin;
2const os = std.os;3const os = std.os;
3const mem = std.mem;4const mem = std.mem;
4const elf = std.elf;5const elf = std.elf;
5const builtin = @import("builtin");6const math = std.math;
6const assert = std.debug.assert;7const assert = std.debug.assert;
78
8// This file implements the two TLS variants [1] used by ELF-based systems.9// This file implements the two TLS variants [1] used by ELF-based systems.
...@@ -60,10 +61,11 @@ const tls_tcb_size = switch (builtin.arch) {...@@ -60,10 +61,11 @@ const tls_tcb_size = switch (builtin.arch) {
60 else => @sizeOf(usize),61 else => @sizeOf(usize),
61};62};
6263
63// Controls if the TCB should be aligned according to the TLS segment p_align64// Controls the minimum alignment of the TCB end address. The effective value
65// used by the code is min(this_value, tls_segment.p_align)
64const tls_tcb_align_size = switch (builtin.arch) {66const tls_tcb_align_size = switch (builtin.arch) {
65 .arm, .armeb, .aarch64, .aarch64_be => true,67 .arm, .armeb, .aarch64, .aarch64_be => 16,
66 else => false,68 else => 1,
67};69};
6870
69// Controls if the TP points to the end of the TCB instead of its beginning71// Controls if the TP points to the end of the TCB instead of its beginning
...@@ -72,13 +74,6 @@ const tls_tp_points_past_tcb = switch (builtin.arch) {...@@ -72,13 +74,6 @@ const tls_tp_points_past_tcb = switch (builtin.arch) {
72 else => false,74 else => false,
73};75};
7476
75// Check if the architecture-specific parameters look correct
76comptime {
77 if (tls_tcb_align_size and tls_variant != TLSVariant.VariantI) {
78 @compileError("tls_tcb_align_size is only meaningful for variant I TLS");
79 }
80}
81
82// Some architectures add some offset to the tp and dtv addresses in order to77// Some architectures add some offset to the tp and dtv addresses in order to
83// make the generated code more efficient78// make the generated code more efficient
8479
...@@ -94,17 +89,19 @@ const tls_dtv_offset = switch (builtin.arch) {...@@ -94,17 +89,19 @@ const tls_dtv_offset = switch (builtin.arch) {
94};89};
9590
96// Per-thread storage for Zig's use91// Per-thread storage for Zig's use
97const CustomData = packed struct {};92const CustomData = struct {
93 padding: [16]usize,
94};
9895
99// Dynamic Thread Vector96// Dynamic Thread Vector
100const DTV = packed struct {97const DTV = extern struct {
101 entries: usize,98 entries: usize,
102 tls_block: [1]usize,99 tls_block: [1][*]u8,
103};100};
104101
105// Holds all the information about the process TLS image102// Holds all the information about the process TLS image
106const TLSImage = struct {103const TLSImage = struct {
107 data_src: []u8,104 data_src: []const u8,
108 alloc_size: usize,105 alloc_size: usize,
109 tcb_offset: usize,106 tcb_offset: usize,
110 dtv_offset: usize,107 dtv_offset: usize,
...@@ -113,13 +110,13 @@ const TLSImage = struct {...@@ -113,13 +110,13 @@ const TLSImage = struct {
113 gdt_entry_number: usize,110 gdt_entry_number: usize,
114};111};
115112
116pub var tls_image: ?TLSImage = null;113pub var tls_image: TLSImage = undefined;
117114
118pub fn setThreadPointer(addr: usize) void {115pub fn setThreadPointer(addr: usize) void {
119 switch (builtin.arch) {116 switch (builtin.arch) {
120 .i386 => {117 .i386 => {
121 var user_desc = std.os.linux.user_desc{118 var user_desc = std.os.linux.user_desc{
122 .entry_number = tls_image.?.gdt_entry_number,119 .entry_number = tls_image.gdt_entry_number,
123 .base_addr = addr,120 .base_addr = addr,
124 .limit = 0xfffff,121 .limit = 0xfffff,
125 .seg_32bit = 1,122 .seg_32bit = 1,
...@@ -134,7 +131,7 @@ pub fn setThreadPointer(addr: usize) void {...@@ -134,7 +131,7 @@ pub fn setThreadPointer(addr: usize) void {
134131
135 const gdt_entry_number = user_desc.entry_number;132 const gdt_entry_number = user_desc.entry_number;
136 // We have to keep track of our slot as it's also needed for clone()133 // We have to keep track of our slot as it's also needed for clone()
137 tls_image.?.gdt_entry_number = gdt_entry_number;134 tls_image.gdt_entry_number = gdt_entry_number;
138 // Update the %gs selector135 // Update the %gs selector
139 asm volatile ("movl %[gs_val], %%gs"136 asm volatile ("movl %[gs_val], %%gs"
140 :137 :
...@@ -171,7 +168,7 @@ pub fn setThreadPointer(addr: usize) void {...@@ -171,7 +168,7 @@ pub fn setThreadPointer(addr: usize) void {
171 }168 }
172}169}
173170
174pub fn initTLS() ?*elf.Phdr {171fn initTLS() void {
175 var tls_phdr: ?*elf.Phdr = null;172 var tls_phdr: ?*elf.Phdr = null;
176 var img_base: usize = 0;173 var img_base: usize = 0;
177174
...@@ -195,124 +192,138 @@ pub fn initTLS() ?*elf.Phdr {...@@ -195,124 +192,138 @@ pub fn initTLS() ?*elf.Phdr {
195 // Sanity check192 // Sanity check
196 assert(at_phent == @sizeOf(elf.Phdr));193 assert(at_phent == @sizeOf(elf.Phdr));
197194
198 // Search the TLS section195 // Find the TLS section
199 const phdrs = (@intToPtr([*]elf.Phdr, at_phdr))[0..at_phnum];196 const phdrs = (@intToPtr([*]elf.Phdr, at_phdr))[0..at_phnum];
200197
201 var gnu_stack: ?*elf.Phdr = null;
202
203 for (phdrs) |*phdr| {198 for (phdrs) |*phdr| {
204 switch (phdr.p_type) {199 switch (phdr.p_type) {
205 elf.PT_PHDR => img_base = at_phdr - phdr.p_vaddr,200 elf.PT_PHDR => img_base = at_phdr - phdr.p_vaddr,
206 elf.PT_TLS => tls_phdr = phdr,201 elf.PT_TLS => tls_phdr = phdr,
207 elf.PT_GNU_STACK => gnu_stack = phdr,202 else => {},
208 else => continue,
209 }203 }
210 }204 }
211205
212 if (tls_phdr) |phdr| {206 // If the cpu is ARM-based, check if it supports the TLS register
213 // If the cpu is arm-based, check if it supports the TLS register207 if (comptime builtin.arch.isARM() and at_hwcap & std.os.linux.HWCAP_TLS == 0) {
214 if (builtin.arch == .arm and at_hwcap & std.os.linux.HWCAP_TLS == 0) {208 // If the CPU does not support TLS via a coprocessor register,
215 // If the CPU does not support TLS via a coprocessor register,209 // a kernel helper function can be used instead on certain linux kernels.
216 // a kernel helper function can be used instead on certain linux kernels.210 // See linux/arch/arm/include/asm/tls.h and musl/src/thread/arm/__set_thread_area.c.
217 // See linux/arch/arm/include/asm/tls.h and musl/src/thread/arm/__set_thread_area.c.211 @panic("TODO: Implement ARM fallback TLS functionality");
218 @panic("TODO: Implement ARM fallback TLS functionality");212 }
219 }
220213
221 // Offsets into the allocated TLS area214 var tls_align_factor: usize = undefined;
222 var tcb_offset: usize = undefined;215 var tls_data: []const u8 = undefined;
223 var dtv_offset: usize = undefined;216 if (tls_phdr) |phdr| {
224 var data_offset: usize = undefined;217 tls_align_factor = phdr.p_align;
225 var thread_data_offset: usize = undefined;218 tls_data = @intToPtr([*]u8, img_base + phdr.p_vaddr)[0..phdr.p_memsz];
226 // Compute the total size of the ABI-specific data plus our own control219 } else {
227 // structures220 tls_align_factor = @alignOf(*usize);
228 const alloc_size = switch (tls_variant) {221 tls_data = &[_]u8{};
229 .VariantI => blk: {
230 var l: usize = 0;
231 dtv_offset = l;
232 l += @sizeOf(DTV);
233 thread_data_offset = l;
234 l += @sizeOf(CustomData);
235 l = mem.alignForward(l, phdr.p_align);
236 tcb_offset = l;
237 if (tls_tcb_align_size) {
238 l += mem.alignForward(tls_tcb_size, phdr.p_align);
239 } else {
240 l += tls_tcb_size;
241 }
242 data_offset = l;
243 l += phdr.p_memsz;
244 break :blk l;
245 },
246 .VariantII => blk: {
247 var l: usize = 0;
248 data_offset = l;
249 l += phdr.p_memsz;
250 l = mem.alignForward(l, phdr.p_align);
251 tcb_offset = l;
252 l += tls_tcb_size;
253 thread_data_offset = l;
254 l += @sizeOf(CustomData);
255 dtv_offset = l;
256 l += @sizeOf(DTV);
257 break :blk l;
258 },
259 };
260
261 tls_image = TLSImage{
262 .data_src = @intToPtr([*]u8, phdr.p_vaddr + img_base)[0..phdr.p_filesz],
263 .alloc_size = alloc_size,
264 .tcb_offset = tcb_offset,
265 .dtv_offset = dtv_offset,
266 .data_offset = data_offset,
267 .gdt_entry_number = @bitCast(usize, @as(isize, -1)),
268 };
269 }222 }
270223
271 return gnu_stack;224 // Offsets into the allocated TLS area
225 var tcb_offset: usize = undefined;
226 var dtv_offset: usize = undefined;
227 var data_offset: usize = undefined;
228 var thread_data_offset: usize = undefined;
229 // Compute the total size of the ABI-specific data plus our own control
230 // structures
231 const alloc_size = switch (tls_variant) {
232 .VariantI => blk: {
233 var l: usize = 0;
234 // Unneeded because l is zero
235 // l = mem.alignForward(l, @alignOf(DTV));
236 dtv_offset = l;
237 l += @sizeOf(DTV);
238 l = mem.alignForward(l, @alignOf(CustomData));
239 thread_data_offset = l;
240 l += @sizeOf(CustomData);
241 // Make sure the TP is aligned
242 l = mem.alignForward(l, tls_align_factor);
243 tcb_offset = l;
244 // Ensure there are at least tls_tcb_align_size bytes of padding
245 const min_align = math.max(tls_tcb_align_size, tls_align_factor);
246 l += mem.alignForward(tls_tcb_size, min_align);
247 data_offset = l;
248 l += mem.alignForward(tls_data.len, tls_align_factor);
249 break :blk l;
250 },
251 .VariantII => blk: {
252 var l: usize = 0;
253 data_offset = l;
254 l = mem.alignForward(tls_data.len, tls_align_factor);
255 // The TP is aligned to p_align
256 tcb_offset = l;
257 l += tls_tcb_size;
258 l = mem.alignForward(l, @alignOf(CustomData));
259 thread_data_offset = l;
260 l += @sizeOf(CustomData);
261 l = mem.alignForward(l, @alignOf(DTV));
262 dtv_offset = l;
263 l += @sizeOf(DTV);
264 break :blk l;
265 },
266 };
267
268 tls_image = TLSImage{
269 .data_src = tls_data,
270 .alloc_size = alloc_size,
271 .tcb_offset = tcb_offset,
272 .dtv_offset = dtv_offset,
273 .data_offset = data_offset,
274 .gdt_entry_number = @bitCast(usize, @as(isize, -1)),
275 };
272}276}
273277
274pub fn copyTLS(addr: usize) usize {278inline fn alignPtrCast(comptime T: type, ptr: [*]u8) *T {
275 const tls_img = tls_image.?;279 return @ptrCast(*T, @alignCast(@alignOf(*T), ptr));
280}
276281
277 // Be paranoid, clear the area we're going to use282/// Initializes all the fields of the static TLS area and returns the computed
278 @memset(@intToPtr([*]u8, addr), 0, tls_img.alloc_size);283/// architecture-specific value of the thread-pointer register
284pub fn prepareTLS(area: []u8) usize {
285 // Clear the area we're going to use, just to be safe
286 mem.set(u8, area, 0);
279 // Prepare the DTV287 // Prepare the DTV
280 const dtv = @intToPtr(*DTV, addr + tls_img.dtv_offset);288 const dtv = alignPtrCast(DTV, area.ptr + tls_image.dtv_offset);
281 dtv.entries = 1;289 dtv.entries = 1;
282 dtv.tls_block[0] = addr + tls_img.data_offset + tls_dtv_offset;290 dtv.tls_block[0] = area.ptr + tls_dtv_offset + tls_image.data_offset;
283 // Set-up the TCB291 // Prepare the TCB
284 // Force the alignment to 1 byte as the TCB may start from a non-aligned292 const tcb_ptr = alignPtrCast([*]u8, area.ptr + tls_image.tcb_offset);
285 // address under the variant II model293 tcb_ptr.* = switch (tls_variant) {
286 const tcb_ptr = @intToPtr(*align(1) usize, addr + tls_img.tcb_offset);294 .VariantI => area.ptr + tls_image.dtv_offset,
287 if (tls_variant == TLSVariant.VariantI) {295 .VariantII => area.ptr + tls_image.tcb_offset,
288 tcb_ptr.* = addr + tls_img.dtv_offset;296 };
289 } else {
290 tcb_ptr.* = addr + tls_img.tcb_offset;
291 }
292 // Copy the data297 // Copy the data
293 @memcpy(@intToPtr([*]u8, addr + tls_img.data_offset), tls_img.data_src.ptr, tls_img.data_src.len);298 mem.copy(u8, area[tls_image.data_offset..], tls_image.data_src);
294299
295 // Return the corrected (if needed) value for the tp register300 // Return the corrected (if needed) value for the tp register
296 return addr + tls_tp_offset +301 return @ptrToInt(area.ptr) + tls_tp_offset +
297 if (tls_tp_points_past_tcb) tls_img.data_offset else tls_img.tcb_offset;302 if (tls_tp_points_past_tcb) tls_image.data_offset else tls_image.tcb_offset;
298}303}
299304
300var main_thread_tls_buffer: [256]u8 align(32) = undefined;305var main_thread_tls_buffer: [256]u8 align(32) = undefined;
301306
302pub fn allocateTLS(size: usize) usize {307pub fn initStaticTLS() void {
303 // Small TLS allocation, use our local buffer308 initTLS();
304 if (size < main_thread_tls_buffer.len) {
305 return @ptrToInt(&main_thread_tls_buffer);
306 }
307309
308 const slice = os.mmap(310 var tls_area = blk: {
309 null,311 // Fast path for the common case where the TLS data is really small,
310 size,312 // avoid an allocation and use our local buffer
311 os.PROT_READ | os.PROT_WRITE,313 if (tls_image.alloc_size < main_thread_tls_buffer.len) {
312 os.MAP_PRIVATE | os.MAP_ANONYMOUS,314 break :blk main_thread_tls_buffer[0..tls_image.alloc_size];
313 -1,315 }
314 0,
315 ) catch @panic("out of memory");
316316
317 return @ptrToInt(slice.ptr);317 break :blk os.mmap(
318 null,
319 tls_image.alloc_size,
320 os.PROT_READ | os.PROT_WRITE,
321 os.MAP_PRIVATE | os.MAP_ANONYMOUS,
322 -1,
323 0,
324 ) catch @panic("out of memory");
325 };
326
327 const tp_value = prepareTLS(tls_area);
328 setThreadPointer(tp_value);
318}329}
lib/std/start.zig+1-7
...@@ -152,13 +152,7 @@ fn posixCallMainAndExit() noreturn {...@@ -152,13 +152,7 @@ fn posixCallMainAndExit() noreturn {
152 const auxv = @ptrCast([*]std.elf.Auxv, @alignCast(@alignOf(usize), envp.ptr + envp_count + 1));152 const auxv = @ptrCast([*]std.elf.Auxv, @alignCast(@alignOf(usize), envp.ptr + envp_count + 1));
153 std.os.linux.elf_aux_maybe = auxv;153 std.os.linux.elf_aux_maybe = auxv;
154 // Initialize the TLS area154 // Initialize the TLS area
155 const gnu_stack_phdr = std.os.linux.tls.initTLS() orelse @panic("ELF missing stack size");155 std.os.linux.tls.initStaticTLS();
156
157 if (std.os.linux.tls.tls_image) |tls_img| {
158 const tls_addr = std.os.linux.tls.allocateTLS(tls_img.alloc_size);
159 const tp = std.os.linux.tls.copyTLS(tls_addr);
160 std.os.linux.tls.setThreadPointer(tp);
161 }
162156
163 // TODO This is disabled because what should we do when linking libc and this code157 // TODO This is disabled because what should we do when linking libc and this code
164 // does not execute? And also it's causing a test failure in stack traces in release modes.158 // does not execute? And also it's causing a test failure in stack traces in release modes.
lib/std/thread.zig+17-16
...@@ -286,11 +286,10 @@ pub const Thread = struct {...@@ -286,11 +286,10 @@ pub const Thread = struct {
286 }286 }
287 // Finally, the Thread Local Storage, if any.287 // Finally, the Thread Local Storage, if any.
288 if (!Thread.use_pthreads) {288 if (!Thread.use_pthreads) {
289 if (os.linux.tls.tls_image) |tls_img| {289 // XXX: Is this alignment enough?
290 l = mem.alignForward(l, @alignOf(usize));290 l = mem.alignForward(l, @alignOf(usize));
291 tls_start_offset = l;291 tls_start_offset = l;
292 l += tls_img.alloc_size;292 l += os.linux.tls.tls_image.alloc_size;
293 }
294 }293 }
295 // Round the size to the page size.294 // Round the size to the page size.
296 break :blk mem.alignForward(l, mem.page_size);295 break :blk mem.alignForward(l, mem.page_size);
...@@ -396,18 +395,21 @@ pub const Thread = struct {...@@ -396,18 +395,21 @@ pub const Thread = struct {
396 else => return os.unexpectedErrno(@intCast(usize, err)),395 else => return os.unexpectedErrno(@intCast(usize, err)),
397 }396 }
398 } else if (std.Target.current.os.tag == .linux) {397 } else if (std.Target.current.os.tag == .linux) {
399 var flags: u32 = os.CLONE_VM | os.CLONE_FS | os.CLONE_FILES | os.CLONE_SIGHAND |398 const flags: u32 = os.CLONE_VM | os.CLONE_FS | os.CLONE_FILES |
400 os.CLONE_THREAD | os.CLONE_SYSVSEM | os.CLONE_PARENT_SETTID | os.CLONE_CHILD_CLEARTID |399 os.CLONE_SIGHAND | os.CLONE_THREAD | os.CLONE_SYSVSEM |
401 os.CLONE_DETACHED;400 os.CLONE_PARENT_SETTID | os.CLONE_CHILD_CLEARTID |
402 var newtls: usize = undefined;401 os.CLONE_DETACHED | os.CLONE_SETTLS;
403 // This structure is only needed when targeting i386402 // This structure is only needed when targeting i386
404 var user_desc: if (std.Target.current.cpu.arch == .i386) os.linux.user_desc else void = undefined;403 var user_desc: if (std.Target.current.cpu.arch == .i386) os.linux.user_desc else void = undefined;
405404
406 if (os.linux.tls.tls_image) |tls_img| {405 const tls_area = mmap_slice[tls_start_offset..];
406 const tp_value = os.linux.tls.prepareTLS(tls_area);
407
408 const newtls = blk: {
407 if (std.Target.current.cpu.arch == .i386) {409 if (std.Target.current.cpu.arch == .i386) {
408 user_desc = os.linux.user_desc{410 user_desc = os.linux.user_desc{
409 .entry_number = tls_img.gdt_entry_number,411 .entry_number = os.linux.tls.tls_image.gdt_entry_number,
410 .base_addr = os.linux.tls.copyTLS(mmap_addr + tls_start_offset),412 .base_addr = tp_value,
411 .limit = 0xfffff,413 .limit = 0xfffff,
412 .seg_32bit = 1,414 .seg_32bit = 1,
413 .contents = 0, // Data415 .contents = 0, // Data
...@@ -416,12 +418,11 @@ pub const Thread = struct {...@@ -416,12 +418,11 @@ pub const Thread = struct {
416 .seg_not_present = 0,418 .seg_not_present = 0,
417 .useable = 1,419 .useable = 1,
418 };420 };
419 newtls = @ptrToInt(&user_desc);421 break :blk @ptrToInt(&user_desc);
420 } else {422 } else {
421 newtls = os.linux.tls.copyTLS(mmap_addr + tls_start_offset);423 break :blk tp_value;
422 }424 }
423 flags |= os.CLONE_SETTLS;425 };
424 }
425426
426 const rc = os.linux.clone(427 const rc = os.linux.clone(
427 MainFuncs.linuxThreadMain,428 MainFuncs.linuxThreadMain,