authorgravatar for alex@alexrp.comAlex Rønne Petersen <alex@alexrp.com> 2026-08-18 22:43:52+02:00
committergravatar for alex@alexrp.comAlex Rønne Petersen <alex@alexrp.com> 2026-10-05 03:26:41+02:00
log736d37ad161b255cf29fb3050026ad4852639b11
tree0a1534cf4c6ca94c2341bdd08add79dcab5faa63
parent292edd2440baa02864f1d2fe2dc08140d0c3c7a6
signature Signed by SSH key SHA256:7B/LJ7bpR1eX8aCXSr4mtd5M45VMPKcx9zY8e95b5QM

move std.os.linux.tls to compiler_rt as __zig_elf_static_tls_*

As with __zig_elf_auxv, we need a canonical location for __tls_get_addr (or __tls_get_offset on s390x), and compiler-rt is the obvious place. But that also means that moving all of std.os.linux.tls there just makes sense. I decided not to namespace this under linux because it's expected that most of this code will apply to the BSDs too. The ABI that std now expects is: extern fn __zig_elf_static_tls_init(phdrs: [*]elf.ElfN.Phdr, phnum: usize) void; extern fn __zig_elf_static_tls_fill(area: [*]u8) usize; extern var __zig_elf_static_tls: extern struct { size: usize, alignment: usize, gdt_entry_number: usize, }; Users can override all three weak symbols with strong symbols if they want to provide their own ELF TLS implementation, which may be useful to do when doing manual dynamic loading from a static executable. Note that __tls_get_addr (or __tls_get_offset on s390x) must also be provided, per the ELF TLS specification.

6 files changed, 801 insertions(+), 779 deletions(-)

lib/compiler_rt/elf.zig+1
......@@ -1,3 +1,4 @@
11comptime {
22 _ = @import("elf/auxv.zig");
3 _ = @import("elf/tls.zig");
34}
lib/compiler_rt/elf/tls.zig created+778
......@@ -0,0 +1,778 @@
1//! This file implements the two TLS variants [1] used by ELF-based systems. Note that, in reality,
2//! Variant I has two sub-variants.
3//!
4//! It is important to understand that the term TCB (Thread Control Block) is overloaded here.
5//! Official ABI documentation uses it simply to mean the ABI TCB, i.e. a small area of ABI-defined
6//! data, usually one or two words (see the `AbiTcb` type below). People will also often use TCB to
7//! refer to the libc TCB, which can be any size and contain anything. (One could even omit it!) We
8//! refer to the latter as the Zig TCB; see the `ZigTcb` type below.
9//!
10//! [1] https://www.akkadia.org/drepper/tls.pdf
11
12const builtin = @import("builtin");
13const std = @import("std");
14
15const compiler_rt = @import("../../compiler_rt.zig");
16const symbol = compiler_rt.symbol;
17
18const elf = std.elf;
19const assert = std.debug.assert;
20const native_arch = builtin.cpu.arch;
21const linux = std.os.linux;
22const page_size_min = std.heap.page_size_min;
23
24comptime {
25 // FreeBSD, NetBSD, and OpenBSD are expected to also use this in the future.
26 if (!builtin.link_libc and builtin.target.os.tag == .linux) {
27 symbol(&__zig_elf_static_tls, "__zig_elf_static_tls");
28 symbol(&__zig_elf_static_tls_init, "__zig_elf_static_tls_init");
29 symbol(&__zig_elf_static_tls_fill, "__zig_elf_static_tls_fill");
30
31 // To make static executables without libc work, it is our job to provide the TLS accessor
32 // function for the GD and LD models. This function is unlikely to actually be used, since
33 // the linker should be able to relax every TLS access to the LE model and therefore
34 // eliminate all calls to this function, but that isn't guaranteed.
35 if (native_arch == .s390x)
36 symbol(&__tls_get_offset, "__tls_get_offset")
37 else
38 symbol(&__tls_get_addr, "__tls_get_addr");
39 }
40}
41
42/// Represents an ELF TLS variant.
43///
44/// In all variants, the TP and the TLS blocks must be aligned to the `p_align` value in the
45/// `PT.TLS` ELF program header. Everything else has natural alignment.
46///
47/// The location of the DTV does not actually matter. For simplicity, we put it in the TLS area, but
48/// there is no actual ABI requirement that it reside there.
49const Variant = enum {
50 /// The original Variant I:
51 ///
52 /// ----------------------------------------
53 /// | DTV | Zig TCB | ABI TCB | TLS Blocks |
54 /// ----------------^-----------------------
55 /// `-- The TP register points here.
56 ///
57 /// The layout in this variant necessitates separate alignment of both the TP and the TLS
58 /// blocks.
59 ///
60 /// The first word in the ABI TCB points to the DTV. For some architectures, there may be a
61 /// second word with an unspecified meaning.
62 I_original,
63 /// The modified Variant I:
64 ///
65 /// --------------------------------------------
66 /// | DTV | Zig TCB | ABI TCB | TLS Blocks |
67 /// ------------------------------^-------------
68 /// `-- The TP register points here (*inside* the TLS blocks).
69 ///
70 /// The offset from the start of the TLS blocks to the TP register is `current_tp_offset`. It
71 /// may be zero, in which case the TP register points to the start of the TLS blocks.
72 ///
73 /// The first (and only) word in the ABI TCB points to the DTV.
74 I_modified,
75 /// Variant II:
76 ///
77 /// ----------------------------------------
78 /// | TLS Blocks | ABI TCB | Zig TCB | DTV |
79 /// -------------^--------------------------
80 /// `-- The TP register points here.
81 ///
82 /// The first (and only) word in the ABI TCB points to the ABI TCB itself.
83 II,
84};
85
86const current_variant: Variant = switch (native_arch) {
87 .aarch64,
88 .aarch64_be,
89 .alpha,
90 .arc,
91 .arceb,
92 .arm,
93 .armeb,
94 .csky,
95 .hppa,
96 .microblaze,
97 .microblazeel,
98 .sh,
99 .sheb,
100 .thumb,
101 .thumbeb,
102 .xtensa,
103 .xtensaeb,
104 => .I_original,
105 .loongarch32,
106 .loongarch64,
107 .m68k,
108 .mips,
109 .mipsel,
110 .mips64,
111 .mips64el,
112 .or1k,
113 .powerpc,
114 .powerpcle,
115 .powerpc64,
116 .powerpc64le,
117 .riscv32,
118 .riscv64,
119 => .I_modified,
120 .hexagon,
121 .s390x,
122 .sparc,
123 .sparc64,
124 .x86,
125 .x86_64,
126 => .II,
127 else => @compileError("undefined TLS variant for this architecture"),
128};
129
130/// The offset value for the modified Variant I.
131const current_tp_offset = switch (native_arch) {
132 .m68k,
133 .mips,
134 .mipsel,
135 .mips64,
136 .mips64el,
137 .powerpc,
138 .powerpcle,
139 .powerpc64,
140 .powerpc64le,
141 => 0x7000,
142 else => 0,
143};
144
145/// Usually only used by the modified Variant I.
146const current_dtv_offset = switch (native_arch) {
147 .m68k,
148 .mips,
149 .mipsel,
150 .mips64,
151 .mips64el,
152 .powerpc,
153 .powerpcle,
154 .powerpc64,
155 .powerpc64le,
156 => 0x8000,
157 .riscv32,
158 .riscv64,
159 => 0x800,
160 else => 0,
161};
162
163/// Per-thread storage for the ELF TLS ABI.
164const AbiTcb = switch (current_variant) {
165 .I_original, .I_modified => switch (native_arch) {
166 .aarch64,
167 .aarch64_be,
168 .alpha,
169 .arm,
170 .armeb,
171 .hppa,
172 .microblaze,
173 .microblazeel,
174 .sh,
175 .sheb,
176 .thumb,
177 .thumbeb,
178 .xtensa,
179 .xtensaeb,
180 => extern struct {
181 /// This is offset by `current_dtv_offset`.
182 dtv: usize,
183 _reserved: ?*anyopaque,
184 },
185 else => extern struct {
186 /// This is offset by `current_dtv_offset`.
187 dtv: usize,
188 },
189 },
190 .II => extern struct {
191 /// This is self-referential.
192 self: *AbiTcb,
193 },
194};
195
196/// Per-thread storage for Zig's use. Currently unused.
197const ZigTcb = struct {
198 dummy: usize,
199};
200
201/// Dynamic Thread Vector as specified in the ELF TLS ABI. Ordinarily, there is a block pointer per
202/// dynamically-loaded module, but since we only support static TLS, we only need one block pointer.
203const Dtv = extern struct {
204 len: usize = 1,
205 tls_block: [*]u8,
206};
207
208/// Describes a process's TLS area. The area encompasses the DTV, both TCBs, and the TLS block, with
209/// the exact layout of these being dependent primarily on `current_variant`. See also `area_info`.
210///
211/// The layout of this variable is ABI between compiler-rt and std.
212var __zig_elf_static_tls: extern struct {
213 size: usize,
214 alignment: usize,
215 /// Only used on the 32-bit x86 architecture (not x86_64, nor x32).
216 gdt_entry_number: usize,
217} = undefined;
218
219var area_info: struct {
220 dtv: struct {
221 /// Offset into the TLS area.
222 offset: usize,
223 },
224
225 abi_tcb: struct {
226 /// Offset into the TLS area.
227 offset: usize,
228 },
229
230 block: struct {
231 /// The initial data to be copied into the TLS block. Note that this may be smaller than
232 /// `size`, in which case any remaining data in the TLS block is simply left uninitialized.
233 init: []const u8,
234 /// Offset into the TLS area.
235 offset: usize,
236 /// This is the effective size of the TLS block, which may be greater than `init.len`.
237 size: usize,
238 },
239} = undefined;
240
241fn setThreadPointer(addr: usize) void {
242 @setRuntimeSafety(false);
243 @disableInstrumentation();
244
245 switch (native_arch) {
246 .x86 => {
247 var user_desc: linux.user_desc = .{
248 .entry_number = __zig_elf_static_tls.gdt_entry_number,
249 .base_addr = addr,
250 .limit = 0xfffff,
251 .flags = .{
252 .seg_32bit = 1,
253 .contents = 0, // Data
254 .read_exec_only = 0,
255 .limit_in_pages = 1,
256 .seg_not_present = 0,
257 .useable = 1,
258 },
259 };
260 const rc = @call(.always_inline, linux.syscall1, .{ .set_thread_area, @intFromPtr(&user_desc) });
261 assert(rc == 0);
262
263 const gdt_entry_number = user_desc.entry_number;
264 // We have to keep track of our slot as it's also needed for clone()
265 __zig_elf_static_tls.gdt_entry_number = gdt_entry_number;
266 // Update the %gs selector
267 asm volatile ("movl %[gs_val], %%gs"
268 :
269 : [gs_val] "r" (gdt_entry_number << 3 | 3),
270 );
271 },
272 .x86_64 => {
273 const rc = @call(.always_inline, linux.syscall2, .{ .arch_prctl, linux.ARCH.SET_FS, addr });
274 assert(rc == 0);
275 },
276 .aarch64, .aarch64_be => {
277 asm volatile (
278 \\ msr tpidr_el0, %[addr]
279 :
280 : [addr] "r" (addr),
281 );
282 },
283 .alpha => {
284 asm volatile (
285 \\ wruniq
286 :
287 : [addr] "{$16}" (addr),
288 );
289 },
290 .arc, .arceb => {
291 // We apparently need to both set r25 (TP) *and* inform the kernel...
292 asm volatile (
293 \\ mov r25, %[addr]
294 :
295 : [addr] "r" (addr),
296 );
297 const rc = @call(.always_inline, linux.syscall1, .{ .arc_settls, addr });
298 assert(rc == 0);
299 },
300 .arm, .armeb, .thumb, .thumbeb => {
301 const rc = @call(.always_inline, linux.syscall1, .{ .set_tls, addr });
302 assert(rc == 0);
303 },
304 .m68k => {
305 const rc = @call(.always_inline, linux.syscall1, .{ .set_thread_area, addr });
306 assert(rc == 0);
307 },
308 .hexagon => {
309 asm volatile (
310 \\ ugp = %[addr]
311 :
312 : [addr] "r" (addr),
313 );
314 },
315 .hppa => {
316 asm volatile (
317 \\ ble 0xe0(%%sr2, %%r0)
318 \\ nop
319 :
320 : [addr] "{r26}" (addr),
321 : .{ .r31 = true });
322 },
323 .loongarch32, .loongarch64 => {
324 asm volatile (
325 \\ move $tp, %[addr]
326 :
327 : [addr] "r" (addr),
328 );
329 },
330 .riscv32, .riscv64 => {
331 asm volatile (
332 \\ mv tp, %[addr]
333 :
334 : [addr] "r" (addr),
335 );
336 },
337 .csky, .mips, .mipsel, .mips64, .mips64el => {
338 const rc = @call(.always_inline, linux.syscall1, .{ .set_thread_area, addr });
339 assert(rc == 0);
340 },
341 .microblaze, .microblazeel => {
342 asm volatile (
343 \\ ori r21, %[addr], 0
344 :
345 : [addr] "r" (addr),
346 );
347 },
348 .or1k => {
349 asm volatile (
350 \\ l.ori r10, %[addr], 0
351 :
352 : [addr] "r" (addr),
353 );
354 },
355 .powerpc, .powerpcle => {
356 asm volatile (
357 \\ mr 2, %[addr]
358 :
359 : [addr] "r" (addr),
360 );
361 },
362 .powerpc64, .powerpc64le => {
363 asm volatile (
364 \\ mr 13, %[addr]
365 :
366 : [addr] "r" (addr),
367 );
368 },
369 .s390x => {
370 asm volatile (
371 \\ lgr %%r0, %[addr]
372 \\ sar %%a1, %%r0
373 \\ srlg %%r0, %%r0, 32
374 \\ sar %%a0, %%r0
375 :
376 : [addr] "r" (addr),
377 : .{ .r0 = true });
378 },
379 .sh, .sheb => {
380 asm volatile (
381 \\ ldc %[addr], gbr
382 :
383 : [addr] "r" (addr),
384 );
385 },
386 .sparc, .sparc64 => {
387 asm volatile (
388 \\ mov %[addr], %%g7
389 :
390 : [addr] "r" (addr),
391 );
392 },
393 .xtensa, .xtensaeb => {
394 asm volatile (
395 \\ wur %[addr], threadptr
396 :
397 : [addr] "a" (addr),
398 );
399 },
400 else => @compileError("Unsupported architecture"),
401 }
402}
403
404fn getThreadPointer() usize {
405 @setRuntimeSafety(false);
406 @disableInstrumentation();
407
408 return switch (native_arch) {
409 .aarch64, .aarch64_be => asm (
410 \\ mrs %[ret], tpidr_el0
411 : [ret] "=r" (-> usize),
412 ),
413 .alpha => asm (
414 \\ rduniq
415 : [ret] "={$0}" (-> usize),
416 ),
417 .arc, .arceb => asm (
418 \\ mov %[ret], r25
419 : [ret] "=r" (-> usize),
420 ),
421 .arm, .armeb, .thumb, .thumbeb => asm (
422 \\ mrc p15, 0, %[ret], c13, c0, 3
423 : [ret] "=r" (-> usize),
424 ),
425 .csky => asm (
426 \\ mov %[ret], r31
427 : [ret] "=r" (-> usize),
428 ),
429 .hexagon => asm (
430 \\ %[ret] = ugp
431 : [ret] "=r" (-> usize),
432 ),
433 .hppa => asm (
434 \\ mfctl %%cr27, %[ret]
435 : [ret] "=r" (-> usize),
436 ),
437 .loongarch32, .loongarch64 => asm (
438 \\ move %[ret], $tp
439 : [ret] "=r" (-> usize),
440 ),
441 .m68k => linux.syscall0(.get_thread_area),
442 .mips, .mipsel, .mips64, .mips64el => asm (
443 \\ rdhwr %[ret], $29
444 : [ret] "=r" (-> usize),
445 ),
446 .microblaze, .microblazeel => asm (
447 \\ ori %[ret], r21, 0
448 : [ret] "=r" (-> usize),
449 ),
450 .or1k => asm (
451 \\ l.ori %[ret], r10, 0
452 : [ret] "=r" (-> usize),
453 ),
454 .riscv32, .riscv64 => asm (
455 \\ mv %[ret], tp
456 : [ret] "=r" (-> usize),
457 ),
458 .powerpc, .powerpcle => asm (
459 \\ mr %[ret], 2
460 : [ret] "=r" (-> usize),
461 ),
462 .powerpc64, .powerpc64le => asm (
463 \\ mr %[ret], 13
464 : [ret] "=r" (-> usize),
465 ),
466 .s390x => asm (
467 \\ ear %[ret], %%a0
468 \\ sllg %[ret], %[ret], 32
469 \\ ear %[ret], %%a1
470 : [ret] "=r" (-> usize),
471 ),
472 .sh, .sheb => asm (
473 \\ stc gbr, %[ret]
474 : [ret] "=r" (-> usize),
475 ),
476 .sparc, .sparc64 => asm (
477 \\ mov %%g7, %[ret]
478 : [ret] "=r" (-> usize),
479 ),
480 .x86 => asm (
481 \\ movl %%gs:0, %[ret]
482 : [ret] "=r" (-> usize),
483 ),
484 .x86_64 => switch (@sizeOf(usize)) {
485 8 => asm (
486 \\ movq %%fs:0, %[ret]
487 : [ret] "=r" (-> usize),
488 ),
489 // On x32, usize is 32 bits.
490 4 => asm (
491 \\ movl %%fs:0, %[ret]
492 : [ret] "=r" (-> usize),
493 ),
494 else => comptime unreachable,
495 },
496 .xtensa, .xtensaeb => asm (
497 \\ rur %[ret], threadptr
498 : [ret] "=r" (-> usize),
499 ),
500 else => @compileError("Unsupported architecture"),
501 };
502}
503
504fn computeAreaDesc(phdrs: []elf.ElfN.Phdr) void {
505 @setRuntimeSafety(false);
506 @disableInstrumentation();
507
508 var tls_phdr: ?*elf.ElfN.Phdr = null;
509 var img_base: usize = 0;
510
511 for (phdrs) |*phdr| {
512 switch (phdr.type) {
513 .PHDR => img_base = @intFromPtr(phdrs.ptr) - phdr.vaddr,
514 .TLS => tls_phdr = phdr,
515 else => {},
516 }
517 }
518
519 var align_factor: usize = undefined;
520 var block_init: []const u8 = undefined;
521 var block_size: usize = undefined;
522
523 if (tls_phdr) |phdr| {
524 align_factor = phdr.@"align";
525
526 // The effective size in memory is represented by `memsz`; the length of the data stored
527 // in the `PT.TLS` segment is `filesz` and may be less than the former.
528 block_init = @as([*]u8, @ptrFromInt(img_base + phdr.vaddr))[0..phdr.filesz];
529 block_size = phdr.memsz;
530 } else {
531 align_factor = @alignOf(usize);
532
533 block_init = &[_]u8{};
534 block_size = 0;
535 }
536
537 // Offsets into the allocated TLS area.
538 var dtv_offset: usize = undefined;
539 var abi_tcb_offset: usize = undefined;
540 var block_offset: usize = undefined;
541
542 // Compute the total size of the ABI-specific data plus our own `ZigTcb` structure. All the
543 // offsets calculated here assume a well-aligned base address.
544 const area_size = switch (current_variant) {
545 .I_original => blk: {
546 var l: usize = 0;
547 dtv_offset = l;
548 l += @sizeOf(Dtv);
549 // Add some padding here so that the TP (`abi_tcb_offset`) is aligned to `align_factor`
550 // and the `ZigTcb` structure can be found by simply subtracting `@sizeOf(ZigTcb)` from
551 // the TP.
552 const delta = (l + @sizeOf(ZigTcb)) & (align_factor - 1);
553 if (delta > 0)
554 l += align_factor - delta;
555 l += @sizeOf(ZigTcb);
556 abi_tcb_offset = l;
557 l += alignForward(@sizeOf(AbiTcb), align_factor);
558 block_offset = l;
559 l += block_size;
560 break :blk l;
561 },
562 .I_modified => blk: {
563 var l: usize = 0;
564 dtv_offset = l;
565 l += @sizeOf(Dtv);
566 // In this variant, the TLS blocks must begin immediately after the end of the ABI TCB,
567 // with the TP pointing to the beginning of the TLS blocks. Add padding so that the TP
568 // (`abi_tcb_offset`) is aligned to `align_factor` and the `ZigTcb` structure can be
569 // found by subtracting `@sizeOf(AbiTcb) + @sizeOf(ZigTcb)` from the TP.
570 const delta = (l + @sizeOf(ZigTcb) + @sizeOf(AbiTcb)) & (align_factor - 1);
571 if (delta > 0)
572 l += align_factor - delta;
573 l += @sizeOf(ZigTcb);
574 abi_tcb_offset = l;
575 l += @sizeOf(AbiTcb);
576 block_offset = l;
577 l += block_size;
578 break :blk l;
579 },
580 .II => blk: {
581 var l: usize = 0;
582 block_offset = l;
583 l += alignForward(block_size, align_factor);
584 // The TP is aligned to `align_factor`.
585 abi_tcb_offset = l;
586 l += @sizeOf(AbiTcb);
587 // The `ZigTcb` structure is right after the `AbiTcb` with no padding in between so it
588 // can be easily found.
589 l += @sizeOf(ZigTcb);
590 // It doesn't really matter where we put the DTV, so give it natural alignment.
591 l = alignForward(l, @alignOf(Dtv));
592 dtv_offset = l;
593 l += @sizeOf(Dtv);
594 break :blk l;
595 },
596 };
597
598 __zig_elf_static_tls = .{
599 .size = area_size,
600 .alignment = align_factor,
601 .gdt_entry_number = @as(usize, @bitCast(@as(isize, -1))),
602 };
603
604 area_info = .{
605 .dtv = .{
606 .offset = dtv_offset,
607 },
608
609 .abi_tcb = .{
610 .offset = abi_tcb_offset,
611 },
612
613 .block = .{
614 .init = block_init,
615 .offset = block_offset,
616 .size = block_size,
617 },
618 };
619}
620
621/// Inline because TLS is not set up yet.
622inline fn alignForward(addr: usize, alignment: usize) usize {
623 return alignBackward(addr + (alignment - 1), alignment);
624}
625
626/// Inline because TLS is not set up yet.
627inline fn alignBackward(addr: usize, alignment: usize) usize {
628 return addr & ~(alignment - 1);
629}
630
631/// Inline because TLS is not set up yet.
632inline fn alignPtrCast(comptime T: type, ptr: [*]u8) *T {
633 return @ptrCast(@alignCast(ptr));
634}
635
636/// Initializes all the fields of the static TLS area and returns the computed architecture-specific
637/// value of the TP register.
638fn __zig_elf_static_tls_fill(area: [*]u8) callconv(.c) usize {
639 @setRuntimeSafety(false);
640 @disableInstrumentation();
641
642 const area_slice = area[0..__zig_elf_static_tls.size];
643
644 // Clear the area we're going to use, just to be safe.
645 @memset(area_slice, 0);
646
647 // Prepare the ABI TCB.
648 const abi_tcb = alignPtrCast(AbiTcb, area + area_info.abi_tcb.offset);
649 switch (current_variant) {
650 .I_original, .I_modified => abi_tcb.dtv = @intFromPtr(area + area_info.dtv.offset),
651 .II => abi_tcb.self = abi_tcb,
652 }
653
654 // Prepare the DTV.
655 const dtv = alignPtrCast(Dtv, area + area_info.dtv.offset);
656 dtv.len = 1;
657 dtv.tls_block = area + current_dtv_offset + area_info.block.offset;
658
659 // Copy the initial data.
660 @memcpy(area_slice[area_info.block.offset..][0..area_info.block.init.len], area_info.block.init);
661
662 // Return the corrected value (if needed) for the TP register. Overflow here is not a problem;
663 // the pointer arithmetic involving the TP is done with wrapping semantics.
664 return @intFromPtr(area) +% switch (current_variant) {
665 .I_original, .II => area_info.abi_tcb.offset,
666 .I_modified => area_info.block.offset +% current_tp_offset,
667 };
668}
669
670/// The main motivation for the size chosen here is to be larger than total
671/// amount of thread-local variables for most programs. Putting this allocation
672/// in the ELF like this is equivalent to moving the `mmap` call below into the
673/// kernel, avoiding syscall overhead.
674var main_thread_area_buffer: [0x1000]u8 align(page_size_min) = undefined;
675
676/// Computes the layout of the static TLS area, allocates the area, initializes all of its fields,
677/// and assigns the architecture-specific value to the TP register.
678fn __zig_elf_static_tls_init(phdrs: [*]elf.ElfN.Phdr, phnum: usize) callconv(.c) void {
679 @setRuntimeSafety(false);
680 @disableInstrumentation();
681
682 computeAreaDesc(phdrs[0..phnum]);
683
684 const area: [*]u8 = blk: {
685 // Fast path for the common case where the TLS data is really small, avoid an allocation and
686 // use our local buffer.
687 if (__zig_elf_static_tls.alignment <= page_size_min and __zig_elf_static_tls.size <= main_thread_area_buffer.len) {
688 break :blk &main_thread_area_buffer;
689 }
690
691 const begin_addr = mmap(__zig_elf_static_tls.size + __zig_elf_static_tls.alignment - 1);
692 if (@call(.always_inline, linux.errno, .{begin_addr}) != .SUCCESS) @trap();
693
694 const area_ptr: [*]align(page_size_min) u8 = @ptrFromInt(begin_addr);
695
696 // Make sure the slice is correctly aligned.
697 const begin_aligned_addr = alignForward(begin_addr, __zig_elf_static_tls.alignment);
698 const start = begin_aligned_addr - begin_addr;
699 break :blk area_ptr + start;
700 };
701
702 const tp_value = __zig_elf_static_tls_fill(area);
703 setThreadPointer(tp_value);
704}
705
706inline fn mmap(length: usize) usize {
707 const prot: linux.PROT = .{ .READ = true, .WRITE = true };
708 const flags: linux.MAP = .{ .TYPE = .PRIVATE, .ANONYMOUS = true };
709
710 if (@hasField(linux.SYS, "mmap2")) {
711 return @call(.always_inline, linux.syscall6, .{
712 .mmap2,
713 0,
714 length,
715 @as(u32, @bitCast(prot)),
716 @as(u32, @bitCast(flags)),
717 @as(usize, @bitCast(@as(isize, -1))),
718 0,
719 });
720 } else {
721 // The s390x mmap() syscall existed before Linux supported syscalls with 5+ parameters, so
722 // it takes a single pointer to an array of arguments instead.
723 return if (native_arch == .s390x) @call(.always_inline, linux.syscall1, .{
724 .mmap,
725 @intFromPtr(&[_]usize{
726 0,
727 length,
728 @as(u32, @bitCast(prot)),
729 @as(u32, @bitCast(flags)),
730 @as(usize, @bitCast(@as(isize, -1))),
731 0,
732 }),
733 }) else @call(.always_inline, linux.syscall6, .{
734 .mmap,
735 0,
736 length,
737 @as(u32, @bitCast(prot)),
738 @as(u32, @bitCast(flags)),
739 @as(usize, @bitCast(@as(isize, -1))),
740 0,
741 });
742 }
743}
744
745const TlsIndex = switch (native_arch) {
746 .x86_64 => extern struct { module: u64, offset: u64 }, // Even for x32...
747 else => extern struct { module: usize, offset: usize }, // ...but not MIPS N32!
748};
749
750fn __tls_get_addr(ti: *const TlsIndex) callconv(.c) *anyopaque {
751 comptime assert(native_arch != .s390x);
752
753 assert(ti.module == 1); // The executable's module ID is always 1
754 const tp = getThreadPointer();
755 const block: [*]u8 = switch (current_variant) {
756 .I_original => @ptrFromInt(tp -% area_info.abi_tcb.offset +% area_info.block.offset),
757 .I_modified => @ptrFromInt(tp -% current_tp_offset),
758 // The `.I_original` approach would also work for `.II`, but there is an
759 // alternative strategy which is one less operation:
760 .II => @ptrFromInt(tp -% area_info.block.size),
761 };
762 return block[@intCast(ti.offset)..];
763}
764
765fn __tls_get_offset() callconv(.naked) noreturn {
766 comptime assert(native_arch == .s390x);
767
768 // We receive the module's GOT pointer in r12 and the GOT offset in r2.
769 asm volatile (
770 \\ la %%r1, 0(%%r12, %%r2)
771 \\ lg %%r2, 8(%%r1)
772 \\ lgrl %%r0, %[block_size]
773 \\ sgr %%r2, %%r0
774 \\ br %%r14
775 :
776 : [block_size] "s" (&area_info.block.size),
777 );
778}
lib/std/Thread.zig+14-4
......@@ -1434,6 +1434,16 @@ const LinuxThreadImpl = struct {
14341434 }
14351435 };
14361436
1437 /// Provided by compiler-rt and populated by `std.start`.
1438 extern var __zig_elf_static_tls: extern struct {
1439 size: usize,
1440 alignment: usize,
1441 gdt_entry_number: usize,
1442 };
1443
1444 /// Provided by compiler-rt.
1445 extern fn __zig_elf_static_tls_fill(area: [*]u8) usize;
1446
14371447 fn spawn(config: SpawnConfig, comptime f: anytype, args: anytype) !Impl {
14381448 const page_size = std.heap.pageSize();
14391449 const Args = @TypeOf(args);
......@@ -1465,9 +1475,9 @@ const LinuxThreadImpl = struct {
14651475 bytes = std.mem.alignForward(usize, bytes, page_size);
14661476 stack_offset = bytes;
14671477
1468 bytes = std.mem.alignForward(usize, bytes, linux.tls.area_desc.alignment);
1478 bytes = std.mem.alignForward(usize, bytes, __zig_elf_static_tls.alignment);
14691479 tls_offset = bytes;
1470 bytes += linux.tls.area_desc.size;
1480 bytes += __zig_elf_static_tls.size;
14711481
14721482 bytes = std.mem.alignForward(usize, bytes, @alignOf(Instance));
14731483 instance_offset = bytes;
......@@ -1509,12 +1519,12 @@ const LinuxThreadImpl = struct {
15091519 }
15101520
15111521 // Prepare the TLS segment and prepare a user_desc struct when needed on x86
1512 var tls_ptr = linux.tls.prepareArea(mapped[tls_offset..][0..linux.tls.area_desc.size]);
1522 var tls_ptr = __zig_elf_static_tls_fill(mapped.ptr + tls_offset);
15131523 var user_desc: if (target.cpu.arch == .x86) linux.user_desc else void = undefined;
15141524 if (target.cpu.arch == .x86) {
15151525 defer tls_ptr = @intFromPtr(&user_desc);
15161526 user_desc = .{
1517 .entry_number = linux.tls.area_desc.gdt_entry_number,
1527 .entry_number = __zig_elf_static_tls.gdt_entry_number,
15181528 .base_addr = tls_ptr,
15191529 .limit = 0xfffff,
15201530 .flags = .{
lib/std/os/linux.zig-1
......@@ -123,7 +123,6 @@ pub const gid_t = u32;
123123pub const clock_t = isize;
124124pub const time_t = arch_bits.time_t;
125125
126pub const tls = @import("linux/tls.zig");
127126pub const BPF = @import("linux/bpf.zig");
128127pub const IOCTL = @import("linux/ioctl.zig");
129128pub const SECCOMP = @import("linux/seccomp.zig");
lib/std/os/linux/tls.zig deleted-769
......@@ -1,769 +0,0 @@
1//! This file implements the two TLS variants [1] used by ELF-based systems. Note that, in reality,
2//! Variant I has two sub-variants.
3//!
4//! It is important to understand that the term TCB (Thread Control Block) is overloaded here.
5//! Official ABI documentation uses it simply to mean the ABI TCB, i.e. a small area of ABI-defined
6//! data, usually one or two words (see the `AbiTcb` type below). People will also often use TCB to
7//! refer to the libc TCB, which can be any size and contain anything. (One could even omit it!) We
8//! refer to the latter as the Zig TCB; see the `ZigTcb` type below.
9//!
10//! [1] https://www.akkadia.org/drepper/tls.pdf
11
12const std = @import("std");
13const mem = std.mem;
14const elf = std.elf;
15const math = std.math;
16const assert = std.debug.assert;
17const builtin = @import("builtin");
18const native_arch = builtin.cpu.arch;
19const linux = std.os.linux;
20const page_size_min = std.heap.page_size_min;
21
22/// Represents an ELF TLS variant.
23///
24/// In all variants, the TP and the TLS blocks must be aligned to the `p_align` value in the
25/// `PT.TLS` ELF program header. Everything else has natural alignment.
26///
27/// The location of the DTV does not actually matter. For simplicity, we put it in the TLS area, but
28/// there is no actual ABI requirement that it reside there.
29const Variant = enum {
30 /// The original Variant I:
31 ///
32 /// ----------------------------------------
33 /// | DTV | Zig TCB | ABI TCB | TLS Blocks |
34 /// ----------------^-----------------------
35 /// `-- The TP register points here.
36 ///
37 /// The layout in this variant necessitates separate alignment of both the TP and the TLS
38 /// blocks.
39 ///
40 /// The first word in the ABI TCB points to the DTV. For some architectures, there may be a
41 /// second word with an unspecified meaning.
42 I_original,
43 /// The modified Variant I:
44 ///
45 /// --------------------------------------------
46 /// | DTV | Zig TCB | ABI TCB | TLS Blocks |
47 /// ------------------------------^-------------
48 /// `-- The TP register points here (*inside* the TLS blocks).
49 ///
50 /// The offset from the start of the TLS blocks to the TP register is `current_tp_offset`. It
51 /// may be zero, in which case the TP register points to the start of the TLS blocks.
52 ///
53 /// The first (and only) word in the ABI TCB points to the DTV.
54 I_modified,
55 /// Variant II:
56 ///
57 /// ----------------------------------------
58 /// | TLS Blocks | ABI TCB | Zig TCB | DTV |
59 /// -------------^--------------------------
60 /// `-- The TP register points here.
61 ///
62 /// The first (and only) word in the ABI TCB points to the ABI TCB itself.
63 II,
64};
65
66const current_variant: Variant = switch (native_arch) {
67 .aarch64,
68 .aarch64_be,
69 .alpha,
70 .arc,
71 .arceb,
72 .arm,
73 .armeb,
74 .csky,
75 .hppa,
76 .microblaze,
77 .microblazeel,
78 .sh,
79 .sheb,
80 .thumb,
81 .thumbeb,
82 .xtensa,
83 .xtensaeb,
84 => .I_original,
85 .loongarch32,
86 .loongarch64,
87 .m68k,
88 .mips,
89 .mipsel,
90 .mips64,
91 .mips64el,
92 .or1k,
93 .powerpc,
94 .powerpcle,
95 .powerpc64,
96 .powerpc64le,
97 .riscv32,
98 .riscv64,
99 => .I_modified,
100 .hexagon,
101 .s390x,
102 .sparc,
103 .sparc64,
104 .x86,
105 .x86_64,
106 => .II,
107 else => @compileError("undefined TLS variant for this architecture"),
108};
109
110/// The offset value for the modified Variant I.
111const current_tp_offset = switch (native_arch) {
112 .m68k,
113 .mips,
114 .mipsel,
115 .mips64,
116 .mips64el,
117 .powerpc,
118 .powerpcle,
119 .powerpc64,
120 .powerpc64le,
121 => 0x7000,
122 else => 0,
123};
124
125/// Usually only used by the modified Variant I.
126const current_dtv_offset = switch (native_arch) {
127 .m68k,
128 .mips,
129 .mipsel,
130 .mips64,
131 .mips64el,
132 .powerpc,
133 .powerpcle,
134 .powerpc64,
135 .powerpc64le,
136 => 0x8000,
137 .riscv32,
138 .riscv64,
139 => 0x800,
140 else => 0,
141};
142
143/// Per-thread storage for the ELF TLS ABI.
144const AbiTcb = switch (current_variant) {
145 .I_original, .I_modified => switch (native_arch) {
146 .aarch64,
147 .aarch64_be,
148 .alpha,
149 .arm,
150 .armeb,
151 .hppa,
152 .microblaze,
153 .microblazeel,
154 .sh,
155 .sheb,
156 .thumb,
157 .thumbeb,
158 .xtensa,
159 .xtensaeb,
160 => extern struct {
161 /// This is offset by `current_dtv_offset`.
162 dtv: usize,
163 _reserved: ?*anyopaque,
164 },
165 else => extern struct {
166 /// This is offset by `current_dtv_offset`.
167 dtv: usize,
168 },
169 },
170 .II => extern struct {
171 /// This is self-referential.
172 self: *AbiTcb,
173 },
174};
175
176/// Per-thread storage for Zig's use. Currently unused.
177const ZigTcb = struct {
178 dummy: usize,
179};
180
181/// Dynamic Thread Vector as specified in the ELF TLS ABI. Ordinarily, there is a block pointer per
182/// dynamically-loaded module, but since we only support static TLS, we only need one block pointer.
183const Dtv = extern struct {
184 len: usize = 1,
185 tls_block: [*]u8,
186};
187
188/// Describes a process's TLS area. The area encompasses the DTV, both TCBs, and the TLS block, with
189/// the exact layout of these being dependent primarily on `current_variant`.
190const AreaDesc = struct {
191 size: usize,
192 alignment: usize,
193
194 dtv: struct {
195 /// Offset into the TLS area.
196 offset: usize,
197 },
198
199 abi_tcb: struct {
200 /// Offset into the TLS area.
201 offset: usize,
202 },
203
204 block: struct {
205 /// The initial data to be copied into the TLS block. Note that this may be smaller than
206 /// `size`, in which case any remaining data in the TLS block is simply left uninitialized.
207 init: []const u8,
208 /// Offset into the TLS area.
209 offset: usize,
210 /// This is the effective size of the TLS block, which may be greater than `init.len`.
211 size: usize,
212 },
213
214 /// Only used on the 32-bit x86 architecture (not x86_64, nor x32).
215 gdt_entry_number: usize,
216};
217
218pub var area_desc: AreaDesc = undefined;
219
220pub fn setThreadPointer(addr: usize) void {
221 @setRuntimeSafety(false);
222 @disableInstrumentation();
223
224 switch (native_arch) {
225 .x86 => {
226 var user_desc: linux.user_desc = .{
227 .entry_number = area_desc.gdt_entry_number,
228 .base_addr = addr,
229 .limit = 0xfffff,
230 .flags = .{
231 .seg_32bit = 1,
232 .contents = 0, // Data
233 .read_exec_only = 0,
234 .limit_in_pages = 1,
235 .seg_not_present = 0,
236 .useable = 1,
237 },
238 };
239 const rc = @call(.always_inline, linux.syscall1, .{ .set_thread_area, @intFromPtr(&user_desc) });
240 assert(rc == 0);
241
242 const gdt_entry_number = user_desc.entry_number;
243 // We have to keep track of our slot as it's also needed for clone()
244 area_desc.gdt_entry_number = gdt_entry_number;
245 // Update the %gs selector
246 asm volatile ("movl %[gs_val], %%gs"
247 :
248 : [gs_val] "r" (gdt_entry_number << 3 | 3),
249 );
250 },
251 .x86_64 => {
252 const rc = @call(.always_inline, linux.syscall2, .{ .arch_prctl, linux.ARCH.SET_FS, addr });
253 assert(rc == 0);
254 },
255 .aarch64, .aarch64_be => {
256 asm volatile (
257 \\ msr tpidr_el0, %[addr]
258 :
259 : [addr] "r" (addr),
260 );
261 },
262 .alpha => {
263 asm volatile (
264 \\ wruniq
265 :
266 : [addr] "{$16}" (addr),
267 );
268 },
269 .arc, .arceb => {
270 // We apparently need to both set r25 (TP) *and* inform the kernel...
271 asm volatile (
272 \\ mov r25, %[addr]
273 :
274 : [addr] "r" (addr),
275 );
276 const rc = @call(.always_inline, linux.syscall1, .{ .arc_settls, addr });
277 assert(rc == 0);
278 },
279 .arm, .armeb, .thumb, .thumbeb => {
280 const rc = @call(.always_inline, linux.syscall1, .{ .set_tls, addr });
281 assert(rc == 0);
282 },
283 .m68k => {
284 const rc = @call(.always_inline, linux.syscall1, .{ .set_thread_area, addr });
285 assert(rc == 0);
286 },
287 .hexagon => {
288 asm volatile (
289 \\ ugp = %[addr]
290 :
291 : [addr] "r" (addr),
292 );
293 },
294 .hppa => {
295 asm volatile (
296 \\ ble 0xe0(%%sr2, %%r0)
297 \\ nop
298 :
299 : [addr] "{r26}" (addr),
300 : .{ .r31 = true });
301 },
302 .loongarch32, .loongarch64 => {
303 asm volatile (
304 \\ move $tp, %[addr]
305 :
306 : [addr] "r" (addr),
307 );
308 },
309 .riscv32, .riscv64 => {
310 asm volatile (
311 \\ mv tp, %[addr]
312 :
313 : [addr] "r" (addr),
314 );
315 },
316 .csky, .mips, .mipsel, .mips64, .mips64el => {
317 const rc = @call(.always_inline, linux.syscall1, .{ .set_thread_area, addr });
318 assert(rc == 0);
319 },
320 .microblaze, .microblazeel => {
321 asm volatile (
322 \\ ori r21, %[addr], 0
323 :
324 : [addr] "r" (addr),
325 );
326 },
327 .or1k => {
328 asm volatile (
329 \\ l.ori r10, %[addr], 0
330 :
331 : [addr] "r" (addr),
332 );
333 },
334 .powerpc, .powerpcle => {
335 asm volatile (
336 \\ mr 2, %[addr]
337 :
338 : [addr] "r" (addr),
339 );
340 },
341 .powerpc64, .powerpc64le => {
342 asm volatile (
343 \\ mr 13, %[addr]
344 :
345 : [addr] "r" (addr),
346 );
347 },
348 .s390x => {
349 asm volatile (
350 \\ lgr %%r0, %[addr]
351 \\ sar %%a1, %%r0
352 \\ srlg %%r0, %%r0, 32
353 \\ sar %%a0, %%r0
354 :
355 : [addr] "r" (addr),
356 : .{ .r0 = true });
357 },
358 .sh, .sheb => {
359 asm volatile (
360 \\ ldc %[addr], gbr
361 :
362 : [addr] "r" (addr),
363 );
364 },
365 .sparc, .sparc64 => {
366 asm volatile (
367 \\ mov %[addr], %%g7
368 :
369 : [addr] "r" (addr),
370 );
371 },
372 .xtensa, .xtensaeb => {
373 asm volatile (
374 \\ wur %[addr], threadptr
375 :
376 : [addr] "a" (addr),
377 );
378 },
379 else => @compileError("Unsupported architecture"),
380 }
381}
382
383pub fn getThreadPointer() usize {
384 @setRuntimeSafety(false);
385 @disableInstrumentation();
386
387 return switch (native_arch) {
388 .aarch64, .aarch64_be => asm (
389 \\ mrs %[ret], tpidr_el0
390 : [ret] "=r" (-> usize),
391 ),
392 .alpha => asm (
393 \\ rduniq
394 : [ret] "={$0}" (-> usize),
395 ),
396 .arc, .arceb => asm (
397 \\ mov %[ret], r25
398 : [ret] "=r" (-> usize),
399 ),
400 .arm, .armeb, .thumb, .thumbeb => asm (
401 \\ mrc p15, 0, %[ret], c13, c0, 3
402 : [ret] "=r" (-> usize),
403 ),
404 .csky => asm (
405 \\ mov %[ret], r31
406 : [ret] "=r" (-> usize),
407 ),
408 .hexagon => asm (
409 \\ %[ret] = ugp
410 : [ret] "=r" (-> usize),
411 ),
412 .hppa => asm (
413 \\ mfctl %%cr27, %[ret]
414 : [ret] "=r" (-> usize),
415 ),
416 .loongarch32, .loongarch64 => asm (
417 \\ move %[ret], $tp
418 : [ret] "=r" (-> usize),
419 ),
420 .m68k => linux.syscall0(.get_thread_area),
421 .mips, .mipsel, .mips64, .mips64el => asm (
422 \\ rdhwr %[ret], $29
423 : [ret] "=r" (-> usize),
424 ),
425 .microblaze, .microblazeel => asm (
426 \\ ori %[ret], r21, 0
427 : [ret] "=r" (-> usize),
428 ),
429 .or1k => asm (
430 \\ l.ori %[ret], r10, 0
431 : [ret] "=r" (-> usize),
432 ),
433 .riscv32, .riscv64 => asm (
434 \\ mv %[ret], tp
435 : [ret] "=r" (-> usize),
436 ),
437 .powerpc, .powerpcle => asm (
438 \\ mr %[ret], 2
439 : [ret] "=r" (-> usize),
440 ),
441 .powerpc64, .powerpc64le => asm (
442 \\ mr %[ret], 13
443 : [ret] "=r" (-> usize),
444 ),
445 .s390x => asm (
446 \\ ear %[ret], %%a0
447 \\ sllg %[ret], %[ret], 32
448 \\ ear %[ret], %%a1
449 : [ret] "=r" (-> usize),
450 ),
451 .sh, .sheb => asm (
452 \\ stc gbr, %[ret]
453 : [ret] "=r" (-> usize),
454 ),
455 .sparc, .sparc64 => asm (
456 \\ mov %%g7, %[ret]
457 : [ret] "=r" (-> usize),
458 ),
459 .x86 => asm (
460 \\ movl %%gs:0, %[ret]
461 : [ret] "=r" (-> usize),
462 ),
463 .x86_64 => switch (@sizeOf(usize)) {
464 8 => asm (
465 \\ movq %%fs:0, %[ret]
466 : [ret] "=r" (-> usize),
467 ),
468 // On x32, usize is 32 bits.
469 4 => asm (
470 \\ movl %%fs:0, %[ret]
471 : [ret] "=r" (-> usize),
472 ),
473 else => comptime unreachable,
474 },
475 .xtensa, .xtensaeb => asm (
476 \\ rur %[ret], threadptr
477 : [ret] "=r" (-> usize),
478 ),
479 else => @compileError("Unsupported architecture"),
480 };
481}
482
483fn computeAreaDesc(phdrs: []elf.ElfN.Phdr) void {
484 @setRuntimeSafety(false);
485 @disableInstrumentation();
486
487 var tls_phdr: ?*elf.ElfN.Phdr = null;
488 var img_base: usize = 0;
489
490 for (phdrs) |*phdr| {
491 switch (phdr.type) {
492 .PHDR => img_base = @intFromPtr(phdrs.ptr) - phdr.vaddr,
493 .TLS => tls_phdr = phdr,
494 else => {},
495 }
496 }
497
498 var align_factor: usize = undefined;
499 var block_init: []const u8 = undefined;
500 var block_size: usize = undefined;
501
502 if (tls_phdr) |phdr| {
503 align_factor = phdr.@"align";
504
505 // The effective size in memory is represented by `memsz`; the length of the data stored
506 // in the `PT.TLS` segment is `filesz` and may be less than the former.
507 block_init = @as([*]u8, @ptrFromInt(img_base + phdr.vaddr))[0..phdr.filesz];
508 block_size = phdr.memsz;
509 } else {
510 align_factor = @alignOf(usize);
511
512 block_init = &[_]u8{};
513 block_size = 0;
514 }
515
516 // Offsets into the allocated TLS area.
517 var dtv_offset: usize = undefined;
518 var abi_tcb_offset: usize = undefined;
519 var block_offset: usize = undefined;
520
521 // Compute the total size of the ABI-specific data plus our own `ZigTcb` structure. All the
522 // offsets calculated here assume a well-aligned base address.
523 const area_size = switch (current_variant) {
524 .I_original => blk: {
525 var l: usize = 0;
526 dtv_offset = l;
527 l += @sizeOf(Dtv);
528 // Add some padding here so that the TP (`abi_tcb_offset`) is aligned to `align_factor`
529 // and the `ZigTcb` structure can be found by simply subtracting `@sizeOf(ZigTcb)` from
530 // the TP.
531 const delta = (l + @sizeOf(ZigTcb)) & (align_factor - 1);
532 if (delta > 0)
533 l += align_factor - delta;
534 l += @sizeOf(ZigTcb);
535 abi_tcb_offset = l;
536 l += alignForward(@sizeOf(AbiTcb), align_factor);
537 block_offset = l;
538 l += block_size;
539 break :blk l;
540 },
541 .I_modified => blk: {
542 var l: usize = 0;
543 dtv_offset = l;
544 l += @sizeOf(Dtv);
545 // In this variant, the TLS blocks must begin immediately after the end of the ABI TCB,
546 // with the TP pointing to the beginning of the TLS blocks. Add padding so that the TP
547 // (`abi_tcb_offset`) is aligned to `align_factor` and the `ZigTcb` structure can be
548 // found by subtracting `@sizeOf(AbiTcb) + @sizeOf(ZigTcb)` from the TP.
549 const delta = (l + @sizeOf(ZigTcb) + @sizeOf(AbiTcb)) & (align_factor - 1);
550 if (delta > 0)
551 l += align_factor - delta;
552 l += @sizeOf(ZigTcb);
553 abi_tcb_offset = l;
554 l += @sizeOf(AbiTcb);
555 block_offset = l;
556 l += block_size;
557 break :blk l;
558 },
559 .II => blk: {
560 var l: usize = 0;
561 block_offset = l;
562 l += alignForward(block_size, align_factor);
563 // The TP is aligned to `align_factor`.
564 abi_tcb_offset = l;
565 l += @sizeOf(AbiTcb);
566 // The `ZigTcb` structure is right after the `AbiTcb` with no padding in between so it
567 // can be easily found.
568 l += @sizeOf(ZigTcb);
569 // It doesn't really matter where we put the DTV, so give it natural alignment.
570 l = alignForward(l, @alignOf(Dtv));
571 dtv_offset = l;
572 l += @sizeOf(Dtv);
573 break :blk l;
574 },
575 };
576
577 area_desc = .{
578 .size = area_size,
579 .alignment = align_factor,
580
581 .dtv = .{
582 .offset = dtv_offset,
583 },
584
585 .abi_tcb = .{
586 .offset = abi_tcb_offset,
587 },
588
589 .block = .{
590 .init = block_init,
591 .offset = block_offset,
592 .size = block_size,
593 },
594
595 .gdt_entry_number = @as(usize, @bitCast(@as(isize, -1))),
596 };
597}
598
599/// Inline because TLS is not set up yet.
600inline fn alignForward(addr: usize, alignment: usize) usize {
601 return alignBackward(addr + (alignment - 1), alignment);
602}
603
604/// Inline because TLS is not set up yet.
605inline fn alignBackward(addr: usize, alignment: usize) usize {
606 return addr & ~(alignment - 1);
607}
608
609/// Inline because TLS is not set up yet.
610inline fn alignPtrCast(comptime T: type, ptr: [*]u8) *T {
611 return @ptrCast(@alignCast(ptr));
612}
613
614/// Initializes all the fields of the static TLS area and returns the computed architecture-specific
615/// value of the TP register.
616pub fn prepareArea(area: []u8) usize {
617 @setRuntimeSafety(false);
618 @disableInstrumentation();
619
620 // Clear the area we're going to use, just to be safe.
621 @memset(area, 0);
622
623 // Prepare the ABI TCB.
624 const abi_tcb = alignPtrCast(AbiTcb, area.ptr + area_desc.abi_tcb.offset);
625 switch (current_variant) {
626 .I_original, .I_modified => abi_tcb.dtv = @intFromPtr(area.ptr + area_desc.dtv.offset),
627 .II => abi_tcb.self = abi_tcb,
628 }
629
630 // Prepare the DTV.
631 const dtv = alignPtrCast(Dtv, area.ptr + area_desc.dtv.offset);
632 dtv.len = 1;
633 dtv.tls_block = area.ptr + current_dtv_offset + area_desc.block.offset;
634
635 // Copy the initial data.
636 @memcpy(area[area_desc.block.offset..][0..area_desc.block.init.len], area_desc.block.init);
637
638 // Return the corrected value (if needed) for the TP register. Overflow here is not a problem;
639 // the pointer arithmetic involving the TP is done with wrapping semantics.
640 return @intFromPtr(area.ptr) +% switch (current_variant) {
641 .I_original, .II => area_desc.abi_tcb.offset,
642 .I_modified => area_desc.block.offset +% current_tp_offset,
643 };
644}
645
646/// The main motivation for the size chosen here is to be larger than total
647/// amount of thread-local variables for most programs. Putting this allocation
648/// in the ELF like this is equivalent to moving the `mmap` call below into the
649/// kernel, avoiding syscall overhead.
650var main_thread_area_buffer: [0x1000]u8 align(page_size_min) = undefined;
651
652/// Computes the layout of the static TLS area, allocates the area, initializes all of its fields,
653/// and assigns the architecture-specific value to the TP register.
654pub fn initStatic(phdrs: []elf.ElfN.Phdr) void {
655 @setRuntimeSafety(false);
656 @disableInstrumentation();
657
658 computeAreaDesc(phdrs);
659
660 const area = blk: {
661 // Fast path for the common case where the TLS data is really small, avoid an allocation and
662 // use our local buffer.
663 if (area_desc.alignment <= page_size_min and area_desc.size <= main_thread_area_buffer.len) {
664 break :blk main_thread_area_buffer[0..area_desc.size];
665 }
666
667 const begin_addr = mmap_tls(area_desc.size + area_desc.alignment - 1);
668 if (@call(.always_inline, linux.errno, .{begin_addr}) != .SUCCESS) @trap();
669
670 const area_ptr: [*]align(page_size_min) u8 = @ptrFromInt(begin_addr);
671
672 // Make sure the slice is correctly aligned.
673 const begin_aligned_addr = alignForward(begin_addr, area_desc.alignment);
674 const start = begin_aligned_addr - begin_addr;
675 break :blk area_ptr[start..][0..area_desc.size];
676 };
677
678 const tp_value = prepareArea(area);
679 setThreadPointer(tp_value);
680}
681
682inline fn mmap_tls(length: usize) usize {
683 const prot: linux.PROT = .{ .READ = true, .WRITE = true };
684 const flags: linux.MAP = .{ .TYPE = .PRIVATE, .ANONYMOUS = true };
685
686 if (@hasField(linux.SYS, "mmap2")) {
687 return @call(.always_inline, linux.syscall6, .{
688 .mmap2,
689 0,
690 length,
691 @as(u32, @bitCast(prot)),
692 @as(u32, @bitCast(flags)),
693 @as(usize, @bitCast(@as(isize, -1))),
694 0,
695 });
696 } else {
697 // The s390x mmap() syscall existed before Linux supported syscalls with 5+ parameters, so
698 // it takes a single pointer to an array of arguments instead.
699 return if (native_arch == .s390x) @call(.always_inline, linux.syscall1, .{
700 .mmap,
701 @intFromPtr(&[_]usize{
702 0,
703 length,
704 @as(u32, @bitCast(prot)),
705 @as(u32, @bitCast(flags)),
706 @as(usize, @bitCast(@as(isize, -1))),
707 0,
708 }),
709 }) else @call(.always_inline, linux.syscall6, .{
710 .mmap,
711 0,
712 length,
713 @as(u32, @bitCast(prot)),
714 @as(u32, @bitCast(flags)),
715 @as(usize, @bitCast(@as(isize, -1))),
716 0,
717 });
718 }
719}
720
721comptime {
722 assert(!builtin.link_libc); // otherwise libc should control TLS
723
724 if (builtin.output_mode == .Exe and builtin.link_mode == .static) {
725 // This is a static executable without libc, so it is our job to provide the TLS accessor
726 // function for the GD and LD models. This function is unlikely to actually be used, since
727 // the linker should be able to relax every TLS access to the LE model and therefore
728 // eliminate all calls to this function, but that isn't guaranteed.
729 const Fns = struct {
730 const TlsIndex = switch (native_arch) {
731 .x86_64 => extern struct { module: u64, offset: u64 }, // Even for x32...
732 else => extern struct { module: usize, offset: usize }, // ...but not MIPS N32!
733 };
734 fn __tls_get_addr(ti: *const TlsIndex) callconv(.c) *anyopaque {
735 comptime assert(native_arch != .s390x);
736
737 assert(ti.module == 1); // The executable's module ID is always 1
738 const tp = getThreadPointer();
739 const block: [*]u8 = switch (current_variant) {
740 .I_original => @ptrFromInt(tp -% area_desc.abi_tcb.offset +% area_desc.block.offset),
741 .I_modified => @ptrFromInt(tp -% current_tp_offset),
742 // The `.I_original` approach would also work for `.II`, but there is an
743 // alternative strategy which is one less operation:
744 .II => @ptrFromInt(tp -% area_desc.block.size),
745 };
746 return block[@intCast(ti.offset)..];
747 }
748 fn __tls_get_offset() callconv(.naked) noreturn {
749 comptime assert(native_arch == .s390x);
750
751 // We receive the module's GOT pointer in r12 and the GOT offset in r2.
752 asm volatile (
753 \\ la %%r1, 0(%%r12, %%r2)
754 \\ lg %%r2, 8(%%r1)
755 \\ lgrl %%r0, %[block_size]
756 \\ sgr %%r2, %%r0
757 \\ br %%r14
758 :
759 : [block_size] "s" (&area_desc.block.size),
760 );
761 }
762 };
763
764 if (native_arch == .s390x)
765 @export(&Fns.__tls_get_offset, .{ .name = "__tls_get_offset" })
766 else
767 @export(&Fns.__tls_get_addr, .{ .name = "__tls_get_addr" });
768 }
769}
lib/std/start.zig+8-5
......@@ -599,6 +599,9 @@ fn wWinMain(hInstance: *anyopaque, hPrevInstance: ?*anyopaque, pCmdLine: [*:0]u1
599599/// Provided by compiler-rt.
600600extern var __zig_elf_auxv: [*]std.elf.Auxv;
601601
602/// Provided by compiler-rt.
603extern fn __zig_elf_static_tls_init(phdrs: [*]elf.ElfN.Phdr, phnum: usize) void;
604
602605fn posixCallMainAndExit(argc_argv_ptr: [*]usize) callconv(.c) noreturn {
603606 // We're not ready to panic until thread local storage is initialized.
604607 @setRuntimeSafety(false);
......@@ -616,7 +619,7 @@ fn posixCallMainAndExit(argc_argv_ptr: [*]usize) callconv(.c) noreturn {
616619 const auxv: [*]elf.Auxv = @ptrCast(@alignCast(envp.ptr + envp_count + 1));
617620
618621 var at_hwcap: usize = 0;
619 const phdrs = init: {
622 const phdrs: [*]elf.ElfN.Phdr, const phnum: usize = init: {
620623 var i: usize = 0;
621624 var at_phdr: usize = 0;
622625 var at_phnum: usize = 0;
......@@ -628,14 +631,14 @@ fn posixCallMainAndExit(argc_argv_ptr: [*]usize) callconv(.c) noreturn {
628631 else => continue,
629632 }
630633 }
631 break :init @as([*]elf.ElfN.Phdr, @ptrFromInt(at_phdr))[0..at_phnum];
634 break :init .{ @ptrFromInt(at_phdr), at_phnum };
632635 };
633636
634637 // Apply the initial relocations as early as possible in the startup process. We cannot
635638 // make calls yet on some architectures (e.g. MIPS) *because* they haven't been applied yet,
636639 // so this must be fully inlined.
637640 if (builtin.link_mode == .static and builtin.position_independent_executable) {
638 @call(.always_inline, std.pie.relocate, .{phdrs});
641 @call(.always_inline, std.pie.relocate, .{phdrs[0..phnum]});
639642 }
640643
641644 if (native_os == .linux) {
......@@ -657,14 +660,14 @@ fn posixCallMainAndExit(argc_argv_ptr: [*]usize) callconv(.c) noreturn {
657660 }
658661
659662 // Initialize the TLS area.
660 std.os.linux.tls.initStatic(phdrs);
663 __zig_elf_static_tls_init(phdrs, phnum);
661664 }
662665
663666 // The way Linux executables represent stack size is via the PT.GNU_STACK
664667 // program header. However the kernel does not recognize it; it always gives 8 MiB.
665668 // Here we look for the stack size in our program headers and use setrlimit
666669 // to ask for more stack space.
667 expandStackSize(phdrs);
670 expandStackSize(phdrs[0..phnum]);
668671 }
669672
670673 const opt_init_array_start = @extern([*]const *const fn () callconv(.c) void, .{