authorgravatar for alex@alexrp.comAlex Rønne Petersen <alex@alexrp.com> 2026-10-07 00:26:56+02:00
committergravatar for alex@alexrp.comAlex Rønne Petersen <alex@alexrp.com> 2026-10-07 00:26:56+02:00
log6c78e9250de36f2e21e710940e6f8d0e8516556f
treecc074df4b3d0310ec4090dd345f41253f66f4928
parentbff490a23889750de7096d9057c0a432efc929a8
parentf0d0412942e52a38bcd4914a50f686f5a59ed37b

Merge pull request 'move no-libc ELF auxiliary vector and thread-local storage initialization to compiler-rt' (#36564) from alexrp/zig:compiler-rt-tls-auxv into master

Reviewed-on: https://codeberg.org/ziglang/zig/pulls/36564

14 files changed, 871 insertions(+), 819 deletions(-)

CMakeLists.txt+1-1
......@@ -690,7 +690,7 @@ add_custom_command(
690690
691691set(BUILD_COMPILER_RT_ARGS
692692 "${PROJECT_SOURCE_DIR}/lib"
693 build-obj -ofmt=c -OReleaseSmall
693 build-obj -ofmt=c -lc -OReleaseSmall
694694 --name compiler_rt
695695 -femit-bin="${ZIG_COMPILER_RT_C_SOURCE}"
696696 -target "${ZIG_HOST_TARGET_TRIPLE}"
bootstrap.c+1-1
......@@ -212,7 +212,7 @@ int main(int argc, char **argv) {
212212 {
213213 const char *child_argv[] = {
214214 "./zig1", "lib", "build-obj",
215 "-ofmt=c", "-OReleaseSmall",
215 "-ofmt=c", "-lc", "-OReleaseSmall",
216216 "--name", "compiler_rt", "-femit-bin=compiler_rt.c",
217217 "-target", host_triple,
218218 "-Mroot=lib/compiler_rt.zig",
lib/compiler_rt.zig+2
......@@ -164,6 +164,8 @@ comptime {
164164 symbol(&strlen, "strlen");
165165 }
166166
167 _ = @import("compiler_rt/elf.zig");
168
167169 // Temporarily used for uefi until https://github.com/ziglang/zig/issues/21630 is addressed.
168170 if (!builtin.link_libc and (builtin.os.tag == .windows or builtin.os.tag == .uefi) and (builtin.abi == .none or builtin.abi == .msvc)) {
169171 symbol(&_fltused, "_fltused");
lib/compiler_rt/elf.zig created+23
......@@ -0,0 +1,23 @@
1const builtin = @import("builtin");
2const std = @import("std");
3
4const compiler_rt = @import("../compiler_rt.zig");
5const symbol = compiler_rt.symbol;
6
7/// More operating systems are expected to be added here:
8///
9/// * FreeBSD: https://codeberg.org/ziglang/zig/issues/30981
10/// * NetBSD: https://codeberg.org/ziglang/zig/issues/30980
11/// * OpenBSD: https://codeberg.org/ziglang/zig/issues/30982
12const use_zig_start = switch (builtin.target.os.tag) {
13 .linux,
14 => !builtin.link_libc,
15 else => false,
16};
17
18comptime {
19 if (use_zig_start) {
20 _ = @import("elf/auxv.zig");
21 _ = @import("elf/tls.zig");
22 }
23}
lib/compiler_rt/elf/auxv.zig created+18
......@@ -0,0 +1,18 @@
1const builtin = @import("builtin");
2const std = @import("std");
3
4const compiler_rt = @import("../../compiler_rt.zig");
5const symbol = compiler_rt.symbol;
6
7comptime {
8 symbol(@ptrCast(&__zig_elf_auxv), "__zig_elf_auxv");
9}
10
11/// Populated by `std.start` when libc is not linked on targets that provide an ELF auxiliary vector
12/// to the process on startup. Provided by compiler-rt so that there is no confusion about where
13/// this state lives regardless of how the final artifact (executable or shared library) is
14/// compiled.
15///
16/// It is expected that this will gradually be replaced with better mechanisms as operating systems
17/// improve; see https://codeberg.org/ziglang/zig/issues/31290.
18var __zig_elf_auxv: [*]std.elf.Auxv = undefined;
lib/compiler_rt/elf/tls.zig created+774
......@@ -0,0 +1,774 @@
1//! This file implements the two TLS variants [1] used by ELF-based systems. Note that, in reality,
2//! Variant I has two sub-variants.
3//!
4//! It is important to understand that the term TCB (Thread Control Block) is overloaded here.
5//! Official ABI documentation uses it simply to mean the ABI TCB, i.e. a small area of ABI-defined
6//! data, usually one or two words (see the `AbiTcb` type below). People will also often use TCB to
7//! refer to the libc TCB, which can be any size and contain anything. (One could even omit it!) We
8//! refer to the latter as the Zig TCB; see the `ZigTcb` type below.
9//!
10//! [1] https://www.akkadia.org/drepper/tls.pdf
11
12const builtin = @import("builtin");
13const std = @import("std");
14const elf = std.elf;
15const assert = std.debug.assert;
16const native_arch = builtin.cpu.arch;
17const linux = std.os.linux;
18const page_size_min = std.heap.page_size_min;
19
20const compiler_rt = @import("../../compiler_rt.zig");
21const symbol = compiler_rt.symbol;
22
23comptime {
24 symbol(&__zig_elf_static_tls, "__zig_elf_static_tls");
25 symbol(&__zig_elf_static_tls_init, "__zig_elf_static_tls_init");
26 symbol(&__zig_elf_static_tls_fill, "__zig_elf_static_tls_fill");
27
28 // To make static executables without libc work, it is our job to provide the TLS accessor
29 // function for the GD and LD models. This function is unlikely to actually be used, since the
30 // linker should be able to relax every TLS access to the LE model and therefore eliminate all
31 // calls to this function, but that isn't guaranteed.
32 if (native_arch == .s390x)
33 symbol(&__tls_get_offset, "__tls_get_offset")
34 else
35 symbol(&__tls_get_addr, "__tls_get_addr");
36}
37
38/// Represents an ELF TLS variant.
39///
40/// In all variants, the TP and the TLS blocks must be aligned to the `p_align` value in the
41/// `PT.TLS` ELF program header. Everything else has natural alignment.
42///
43/// The location of the DTV does not actually matter. For simplicity, we put it in the TLS area, but
44/// there is no actual ABI requirement that it reside there.
45const Variant = enum {
46 /// The original Variant I:
47 ///
48 /// ----------------------------------------
49 /// | DTV | Zig TCB | ABI TCB | TLS Blocks |
50 /// ----------------^-----------------------
51 /// `-- The TP register points here.
52 ///
53 /// The layout in this variant necessitates separate alignment of both the TP and the TLS
54 /// blocks.
55 ///
56 /// The first word in the ABI TCB points to the DTV. For some architectures, there may be a
57 /// second word with an unspecified meaning.
58 I_original,
59 /// The modified Variant I:
60 ///
61 /// --------------------------------------------
62 /// | DTV | Zig TCB | ABI TCB | TLS Blocks |
63 /// ------------------------------^-------------
64 /// `-- The TP register points here (*inside* the TLS blocks).
65 ///
66 /// The offset from the start of the TLS blocks to the TP register is `current_tp_offset`. It
67 /// may be zero, in which case the TP register points to the start of the TLS blocks.
68 ///
69 /// The first (and only) word in the ABI TCB points to the DTV.
70 I_modified,
71 /// Variant II:
72 ///
73 /// ----------------------------------------
74 /// | TLS Blocks | ABI TCB | Zig TCB | DTV |
75 /// -------------^--------------------------
76 /// `-- The TP register points here.
77 ///
78 /// The first (and only) word in the ABI TCB points to the ABI TCB itself.
79 II,
80};
81
82const current_variant: Variant = switch (native_arch) {
83 .aarch64,
84 .aarch64_be,
85 .alpha,
86 .arc,
87 .arceb,
88 .arm,
89 .armeb,
90 .csky,
91 .hppa,
92 .microblaze,
93 .microblazeel,
94 .sh,
95 .sheb,
96 .thumb,
97 .thumbeb,
98 .xtensa,
99 .xtensaeb,
100 => .I_original,
101 .loongarch32,
102 .loongarch64,
103 .m68k,
104 .mips,
105 .mipsel,
106 .mips64,
107 .mips64el,
108 .or1k,
109 .powerpc,
110 .powerpcle,
111 .powerpc64,
112 .powerpc64le,
113 .riscv32,
114 .riscv64,
115 => .I_modified,
116 .hexagon,
117 .s390x,
118 .sparc,
119 .sparc64,
120 .x86,
121 .x86_64,
122 => .II,
123 else => @compileError("undefined TLS variant for this architecture"),
124};
125
126/// The offset value for the modified Variant I.
127const current_tp_offset = switch (native_arch) {
128 .m68k,
129 .mips,
130 .mipsel,
131 .mips64,
132 .mips64el,
133 .powerpc,
134 .powerpcle,
135 .powerpc64,
136 .powerpc64le,
137 => 0x7000,
138 else => 0,
139};
140
141/// Usually only used by the modified Variant I.
142const current_dtv_offset = switch (native_arch) {
143 .m68k,
144 .mips,
145 .mipsel,
146 .mips64,
147 .mips64el,
148 .powerpc,
149 .powerpcle,
150 .powerpc64,
151 .powerpc64le,
152 => 0x8000,
153 .riscv32,
154 .riscv64,
155 => 0x800,
156 else => 0,
157};
158
159/// Per-thread storage for the ELF TLS ABI.
160const AbiTcb = switch (current_variant) {
161 .I_original, .I_modified => switch (native_arch) {
162 .aarch64,
163 .aarch64_be,
164 .alpha,
165 .arm,
166 .armeb,
167 .hppa,
168 .microblaze,
169 .microblazeel,
170 .sh,
171 .sheb,
172 .thumb,
173 .thumbeb,
174 .xtensa,
175 .xtensaeb,
176 => extern struct {
177 /// This is offset by `current_dtv_offset`.
178 dtv: usize,
179 _reserved: ?*anyopaque,
180 },
181 else => extern struct {
182 /// This is offset by `current_dtv_offset`.
183 dtv: usize,
184 },
185 },
186 .II => extern struct {
187 /// This is self-referential.
188 self: *AbiTcb,
189 },
190};
191
192/// Per-thread storage for Zig's use. Currently unused.
193const ZigTcb = struct {
194 dummy: usize,
195};
196
197/// Dynamic Thread Vector as specified in the ELF TLS ABI. Ordinarily, there is a block pointer per
198/// dynamically-loaded module, but since we only support static TLS, we only need one block pointer.
199const Dtv = extern struct {
200 len: usize = 1,
201 tls_block: [*]u8,
202};
203
204/// Describes a process's TLS area. The area encompasses the DTV, both TCBs, and the TLS block, with
205/// the exact layout of these being dependent primarily on `current_variant`. See also `area_info`.
206///
207/// The layout of this variable is ABI between compiler-rt and std.
208var __zig_elf_static_tls: extern struct {
209 size: usize,
210 alignment: usize,
211 /// Only used on the 32-bit x86 architecture (not x86_64, nor x32).
212 gdt_entry_number: usize,
213} = undefined;
214
215var area_info: struct {
216 dtv: struct {
217 /// Offset into the TLS area.
218 offset: usize,
219 },
220
221 abi_tcb: struct {
222 /// Offset into the TLS area.
223 offset: usize,
224 },
225
226 block: struct {
227 /// The initial data to be copied into the TLS block. Note that this may be smaller than
228 /// `size`, in which case any remaining data in the TLS block is simply left uninitialized.
229 init: []const u8,
230 /// Offset into the TLS area.
231 offset: usize,
232 /// This is the effective size of the TLS block, which may be greater than `init.len`.
233 size: usize,
234 },
235} = undefined;
236
237fn setThreadPointer(addr: usize) void {
238 @setRuntimeSafety(false);
239 @disableInstrumentation();
240
241 switch (native_arch) {
242 .x86 => {
243 var user_desc: linux.user_desc = .{
244 .entry_number = __zig_elf_static_tls.gdt_entry_number,
245 .base_addr = addr,
246 .limit = 0xfffff,
247 .flags = .{
248 .seg_32bit = 1,
249 .contents = 0, // Data
250 .read_exec_only = 0,
251 .limit_in_pages = 1,
252 .seg_not_present = 0,
253 .useable = 1,
254 },
255 };
256 const rc = @call(.always_inline, linux.syscall1, .{ .set_thread_area, @intFromPtr(&user_desc) });
257 assert(rc == 0);
258
259 const gdt_entry_number = user_desc.entry_number;
260 // We have to keep track of our slot as it's also needed for clone()
261 __zig_elf_static_tls.gdt_entry_number = gdt_entry_number;
262 // Update the %gs selector
263 asm volatile ("movl %[gs_val], %%gs"
264 :
265 : [gs_val] "r" (gdt_entry_number << 3 | 3),
266 );
267 },
268 .x86_64 => {
269 const rc = @call(.always_inline, linux.syscall2, .{ .arch_prctl, linux.ARCH.SET_FS, addr });
270 assert(rc == 0);
271 },
272 .aarch64, .aarch64_be => {
273 asm volatile (
274 \\ msr tpidr_el0, %[addr]
275 :
276 : [addr] "r" (addr),
277 );
278 },
279 .alpha => {
280 asm volatile (
281 \\ wruniq
282 :
283 : [addr] "{$16}" (addr),
284 );
285 },
286 .arc, .arceb => {
287 // We apparently need to both set r25 (TP) *and* inform the kernel...
288 asm volatile (
289 \\ mov r25, %[addr]
290 :
291 : [addr] "r" (addr),
292 );
293 const rc = @call(.always_inline, linux.syscall1, .{ .arc_settls, addr });
294 assert(rc == 0);
295 },
296 .arm, .armeb, .thumb, .thumbeb => {
297 const rc = @call(.always_inline, linux.syscall1, .{ .set_tls, addr });
298 assert(rc == 0);
299 },
300 .m68k => {
301 const rc = @call(.always_inline, linux.syscall1, .{ .set_thread_area, addr });
302 assert(rc == 0);
303 },
304 .hexagon => {
305 asm volatile (
306 \\ ugp = %[addr]
307 :
308 : [addr] "r" (addr),
309 );
310 },
311 .hppa => {
312 asm volatile (
313 \\ ble 0xe0(%%sr2, %%r0)
314 \\ nop
315 :
316 : [addr] "{r26}" (addr),
317 : .{ .r31 = true });
318 },
319 .loongarch32, .loongarch64 => {
320 asm volatile (
321 \\ move $tp, %[addr]
322 :
323 : [addr] "r" (addr),
324 );
325 },
326 .riscv32, .riscv64 => {
327 asm volatile (
328 \\ mv tp, %[addr]
329 :
330 : [addr] "r" (addr),
331 );
332 },
333 .csky, .mips, .mipsel, .mips64, .mips64el => {
334 const rc = @call(.always_inline, linux.syscall1, .{ .set_thread_area, addr });
335 assert(rc == 0);
336 },
337 .microblaze, .microblazeel => {
338 asm volatile (
339 \\ ori r21, %[addr], 0
340 :
341 : [addr] "r" (addr),
342 );
343 },
344 .or1k => {
345 asm volatile (
346 \\ l.ori r10, %[addr], 0
347 :
348 : [addr] "r" (addr),
349 );
350 },
351 .powerpc, .powerpcle => {
352 asm volatile (
353 \\ mr 2, %[addr]
354 :
355 : [addr] "r" (addr),
356 );
357 },
358 .powerpc64, .powerpc64le => {
359 asm volatile (
360 \\ mr 13, %[addr]
361 :
362 : [addr] "r" (addr),
363 );
364 },
365 .s390x => {
366 asm volatile (
367 \\ lgr %%r0, %[addr]
368 \\ sar %%a1, %%r0
369 \\ srlg %%r0, %%r0, 32
370 \\ sar %%a0, %%r0
371 :
372 : [addr] "r" (addr),
373 : .{ .r0 = true });
374 },
375 .sh, .sheb => {
376 asm volatile (
377 \\ ldc %[addr], gbr
378 :
379 : [addr] "r" (addr),
380 );
381 },
382 .sparc, .sparc64 => {
383 asm volatile (
384 \\ mov %[addr], %%g7
385 :
386 : [addr] "r" (addr),
387 );
388 },
389 .xtensa, .xtensaeb => {
390 asm volatile (
391 \\ wur %[addr], threadptr
392 :
393 : [addr] "a" (addr),
394 );
395 },
396 else => @compileError("Unsupported architecture"),
397 }
398}
399
400fn getThreadPointer() usize {
401 @setRuntimeSafety(false);
402 @disableInstrumentation();
403
404 return switch (native_arch) {
405 .aarch64, .aarch64_be => asm (
406 \\ mrs %[ret], tpidr_el0
407 : [ret] "=r" (-> usize),
408 ),
409 .alpha => asm (
410 \\ rduniq
411 : [ret] "={$0}" (-> usize),
412 ),
413 .arc, .arceb => asm (
414 \\ mov %[ret], r25
415 : [ret] "=r" (-> usize),
416 ),
417 .arm, .armeb, .thumb, .thumbeb => asm (
418 \\ mrc p15, 0, %[ret], c13, c0, 3
419 : [ret] "=r" (-> usize),
420 ),
421 .csky => asm (
422 \\ mov %[ret], r31
423 : [ret] "=r" (-> usize),
424 ),
425 .hexagon => asm (
426 \\ %[ret] = ugp
427 : [ret] "=r" (-> usize),
428 ),
429 .hppa => asm (
430 \\ mfctl %%cr27, %[ret]
431 : [ret] "=r" (-> usize),
432 ),
433 .loongarch32, .loongarch64 => asm (
434 \\ move %[ret], $tp
435 : [ret] "=r" (-> usize),
436 ),
437 .m68k => linux.syscall0(.get_thread_area),
438 .mips, .mipsel, .mips64, .mips64el => asm (
439 \\ rdhwr %[ret], $29
440 : [ret] "=r" (-> usize),
441 ),
442 .microblaze, .microblazeel => asm (
443 \\ ori %[ret], r21, 0
444 : [ret] "=r" (-> usize),
445 ),
446 .or1k => asm (
447 \\ l.ori %[ret], r10, 0
448 : [ret] "=r" (-> usize),
449 ),
450 .riscv32, .riscv64 => asm (
451 \\ mv %[ret], tp
452 : [ret] "=r" (-> usize),
453 ),
454 .powerpc, .powerpcle => asm (
455 \\ mr %[ret], 2
456 : [ret] "=r" (-> usize),
457 ),
458 .powerpc64, .powerpc64le => asm (
459 \\ mr %[ret], 13
460 : [ret] "=r" (-> usize),
461 ),
462 .s390x => asm (
463 \\ ear %[ret], %%a0
464 \\ sllg %[ret], %[ret], 32
465 \\ ear %[ret], %%a1
466 : [ret] "=r" (-> usize),
467 ),
468 .sh, .sheb => asm (
469 \\ stc gbr, %[ret]
470 : [ret] "=r" (-> usize),
471 ),
472 .sparc, .sparc64 => asm (
473 \\ mov %%g7, %[ret]
474 : [ret] "=r" (-> usize),
475 ),
476 .x86 => asm (
477 \\ movl %%gs:0, %[ret]
478 : [ret] "=r" (-> usize),
479 ),
480 .x86_64 => switch (@sizeOf(usize)) {
481 8 => asm (
482 \\ movq %%fs:0, %[ret]
483 : [ret] "=r" (-> usize),
484 ),
485 // On x32, usize is 32 bits.
486 4 => asm (
487 \\ movl %%fs:0, %[ret]
488 : [ret] "=r" (-> usize),
489 ),
490 else => comptime unreachable,
491 },
492 .xtensa, .xtensaeb => asm (
493 \\ rur %[ret], threadptr
494 : [ret] "=r" (-> usize),
495 ),
496 else => @compileError("Unsupported architecture"),
497 };
498}
499
500fn computeAreaDesc(phdrs: []elf.ElfN.Phdr) void {
501 @setRuntimeSafety(false);
502 @disableInstrumentation();
503
504 var tls_phdr: ?*elf.ElfN.Phdr = null;
505 var img_base: usize = 0;
506
507 for (phdrs) |*phdr| {
508 switch (phdr.type) {
509 .PHDR => img_base = @intFromPtr(phdrs.ptr) - phdr.vaddr,
510 .TLS => tls_phdr = phdr,
511 else => {},
512 }
513 }
514
515 var align_factor: usize = undefined;
516 var block_init: []const u8 = undefined;
517 var block_size: usize = undefined;
518
519 if (tls_phdr) |phdr| {
520 align_factor = phdr.@"align";
521
522 // The effective size in memory is represented by `memsz`; the length of the data stored
523 // in the `PT.TLS` segment is `filesz` and may be less than the former.
524 block_init = @as([*]u8, @ptrFromInt(img_base + phdr.vaddr))[0..phdr.filesz];
525 block_size = phdr.memsz;
526 } else {
527 align_factor = @alignOf(usize);
528
529 block_init = &[_]u8{};
530 block_size = 0;
531 }
532
533 // Offsets into the allocated TLS area.
534 var dtv_offset: usize = undefined;
535 var abi_tcb_offset: usize = undefined;
536 var block_offset: usize = undefined;
537
538 // Compute the total size of the ABI-specific data plus our own `ZigTcb` structure. All the
539 // offsets calculated here assume a well-aligned base address.
540 const area_size = switch (current_variant) {
541 .I_original => blk: {
542 var l: usize = 0;
543 dtv_offset = l;
544 l += @sizeOf(Dtv);
545 // Add some padding here so that the TP (`abi_tcb_offset`) is aligned to `align_factor`
546 // and the `ZigTcb` structure can be found by simply subtracting `@sizeOf(ZigTcb)` from
547 // the TP.
548 const delta = (l + @sizeOf(ZigTcb)) & (align_factor - 1);
549 if (delta > 0)
550 l += align_factor - delta;
551 l += @sizeOf(ZigTcb);
552 abi_tcb_offset = l;
553 l += alignForward(@sizeOf(AbiTcb), align_factor);
554 block_offset = l;
555 l += block_size;
556 break :blk l;
557 },
558 .I_modified => blk: {
559 var l: usize = 0;
560 dtv_offset = l;
561 l += @sizeOf(Dtv);
562 // In this variant, the TLS blocks must begin immediately after the end of the ABI TCB,
563 // with the TP pointing to the beginning of the TLS blocks. Add padding so that the TP
564 // (`abi_tcb_offset`) is aligned to `align_factor` and the `ZigTcb` structure can be
565 // found by subtracting `@sizeOf(AbiTcb) + @sizeOf(ZigTcb)` from the TP.
566 const delta = (l + @sizeOf(ZigTcb) + @sizeOf(AbiTcb)) & (align_factor - 1);
567 if (delta > 0)
568 l += align_factor - delta;
569 l += @sizeOf(ZigTcb);
570 abi_tcb_offset = l;
571 l += @sizeOf(AbiTcb);
572 block_offset = l;
573 l += block_size;
574 break :blk l;
575 },
576 .II => blk: {
577 var l: usize = 0;
578 block_offset = l;
579 l += alignForward(block_size, align_factor);
580 // The TP is aligned to `align_factor`.
581 abi_tcb_offset = l;
582 l += @sizeOf(AbiTcb);
583 // The `ZigTcb` structure is right after the `AbiTcb` with no padding in between so it
584 // can be easily found.
585 l += @sizeOf(ZigTcb);
586 // It doesn't really matter where we put the DTV, so give it natural alignment.
587 l = alignForward(l, @alignOf(Dtv));
588 dtv_offset = l;
589 l += @sizeOf(Dtv);
590 break :blk l;
591 },
592 };
593
594 __zig_elf_static_tls = .{
595 .size = area_size,
596 .alignment = align_factor,
597 .gdt_entry_number = @as(usize, @bitCast(@as(isize, -1))),
598 };
599
600 area_info = .{
601 .dtv = .{
602 .offset = dtv_offset,
603 },
604
605 .abi_tcb = .{
606 .offset = abi_tcb_offset,
607 },
608
609 .block = .{
610 .init = block_init,
611 .offset = block_offset,
612 .size = block_size,
613 },
614 };
615}
616
617/// Inline because TLS is not set up yet.
618inline fn alignForward(addr: usize, alignment: usize) usize {
619 return alignBackward(addr + (alignment - 1), alignment);
620}
621
622/// Inline because TLS is not set up yet.
623inline fn alignBackward(addr: usize, alignment: usize) usize {
624 return addr & ~(alignment - 1);
625}
626
627/// Inline because TLS is not set up yet.
628inline fn alignPtrCast(comptime T: type, ptr: [*]u8) *T {
629 return @ptrCast(@alignCast(ptr));
630}
631
632/// Initializes all the fields of the static TLS area and returns the computed architecture-specific
633/// value of the TP register.
634fn __zig_elf_static_tls_fill(area: [*]u8) callconv(.c) usize {
635 @setRuntimeSafety(false);
636 @disableInstrumentation();
637
638 const area_slice = area[0..__zig_elf_static_tls.size];
639
640 // Clear the area we're going to use, just to be safe.
641 @memset(area_slice, 0);
642
643 // Prepare the ABI TCB.
644 const abi_tcb = alignPtrCast(AbiTcb, area + area_info.abi_tcb.offset);
645 switch (current_variant) {
646 .I_original, .I_modified => abi_tcb.dtv = @intFromPtr(area + area_info.dtv.offset),
647 .II => abi_tcb.self = abi_tcb,
648 }
649
650 // Prepare the DTV.
651 const dtv = alignPtrCast(Dtv, area + area_info.dtv.offset);
652 dtv.len = 1;
653 dtv.tls_block = area + current_dtv_offset + area_info.block.offset;
654
655 // Copy the initial data.
656 @memcpy(area_slice[area_info.block.offset..][0..area_info.block.init.len], area_info.block.init);
657
658 // Return the corrected value (if needed) for the TP register. Overflow here is not a problem;
659 // the pointer arithmetic involving the TP is done with wrapping semantics.
660 return @intFromPtr(area) +% switch (current_variant) {
661 .I_original, .II => area_info.abi_tcb.offset,
662 .I_modified => area_info.block.offset +% current_tp_offset,
663 };
664}
665
666/// The main motivation for the size chosen here is to be larger than total
667/// amount of thread-local variables for most programs. Putting this allocation
668/// in the ELF like this is equivalent to moving the `mmap` call below into the
669/// kernel, avoiding syscall overhead.
670var main_thread_area_buffer: [0x1000]u8 align(page_size_min) = undefined;
671
672/// Computes the layout of the static TLS area, allocates the area, initializes all of its fields,
673/// and assigns the architecture-specific value to the TP register.
674fn __zig_elf_static_tls_init(phdrs: [*]elf.ElfN.Phdr, phnum: usize) callconv(.c) void {
675 @setRuntimeSafety(false);
676 @disableInstrumentation();
677
678 computeAreaDesc(phdrs[0..phnum]);
679
680 const area: [*]u8 = blk: {
681 // Fast path for the common case where the TLS data is really small, avoid an allocation and
682 // use our local buffer.
683 if (__zig_elf_static_tls.alignment <= page_size_min and __zig_elf_static_tls.size <= main_thread_area_buffer.len) {
684 break :blk &main_thread_area_buffer;
685 }
686
687 const begin_addr = mmap(__zig_elf_static_tls.size + __zig_elf_static_tls.alignment - 1);
688 if (@call(.always_inline, linux.errno, .{begin_addr}) != .SUCCESS) @trap();
689
690 const area_ptr: [*]align(page_size_min) u8 = @ptrFromInt(begin_addr);
691
692 // Make sure the slice is correctly aligned.
693 const begin_aligned_addr = alignForward(begin_addr, __zig_elf_static_tls.alignment);
694 const start = begin_aligned_addr - begin_addr;
695 break :blk area_ptr + start;
696 };
697
698 const tp_value = __zig_elf_static_tls_fill(area);
699 setThreadPointer(tp_value);
700}
701
702inline fn mmap(length: usize) usize {
703 const prot: linux.PROT = .{ .READ = true, .WRITE = true };
704 const flags: linux.MAP = .{ .TYPE = .PRIVATE, .ANONYMOUS = true };
705
706 if (@hasField(linux.SYS, "mmap2")) {
707 return @call(.always_inline, linux.syscall6, .{
708 .mmap2,
709 0,
710 length,
711 @as(u32, @bitCast(prot)),
712 @as(u32, @bitCast(flags)),
713 @as(usize, @bitCast(@as(isize, -1))),
714 0,
715 });
716 } else {
717 // The s390x mmap() syscall existed before Linux supported syscalls with 5+ parameters, so
718 // it takes a single pointer to an array of arguments instead.
719 return if (native_arch == .s390x) @call(.always_inline, linux.syscall1, .{
720 .mmap,
721 @intFromPtr(&[_]usize{
722 0,
723 length,
724 @as(u32, @bitCast(prot)),
725 @as(u32, @bitCast(flags)),
726 @as(usize, @bitCast(@as(isize, -1))),
727 0,
728 }),
729 }) else @call(.always_inline, linux.syscall6, .{
730 .mmap,
731 0,
732 length,
733 @as(u32, @bitCast(prot)),
734 @as(u32, @bitCast(flags)),
735 @as(usize, @bitCast(@as(isize, -1))),
736 0,
737 });
738 }
739}
740
741const TlsIndex = switch (native_arch) {
742 .x86_64 => extern struct { module: u64, offset: u64 }, // Even for x32...
743 else => extern struct { module: usize, offset: usize }, // ...but not MIPS N32!
744};
745
746fn __tls_get_addr(ti: *const TlsIndex) callconv(.c) *anyopaque {
747 comptime assert(native_arch != .s390x);
748
749 assert(ti.module == 1); // The executable's module ID is always 1
750 const tp = getThreadPointer();
751 const block: [*]u8 = switch (current_variant) {
752 .I_original => @ptrFromInt(tp -% area_info.abi_tcb.offset +% area_info.block.offset),
753 .I_modified => @ptrFromInt(tp -% current_tp_offset),
754 // The `.I_original` approach would also work for `.II`, but there is an
755 // alternative strategy which is one less operation:
756 .II => @ptrFromInt(tp -% area_info.block.size),
757 };
758 return block[@intCast(ti.offset)..];
759}
760
761fn __tls_get_offset() callconv(.naked) noreturn {
762 comptime assert(native_arch == .s390x);
763
764 // We receive the module's GOT pointer in r12 and the GOT offset in r2.
765 asm volatile (
766 \\ la %%r1, 0(%%r12, %%r2)
767 \\ lg %%r2, 8(%%r1)
768 \\ lgrl %%r0, %[block_size]
769 \\ sgr %%r2, %%r0
770 \\ br %%r14
771 :
772 : [block_size] "s" (&area_info.block.size),
773 );
774}
lib/std/Thread.zig+14-4
......@@ -1434,6 +1434,16 @@ const LinuxThreadImpl = struct {
14341434 }
14351435 };
14361436
1437 /// Provided by compiler-rt and populated by `std.start`.
1438 extern var __zig_elf_static_tls: extern struct {
1439 size: usize,
1440 alignment: usize,
1441 gdt_entry_number: usize,
1442 };
1443
1444 /// Provided by compiler-rt.
1445 extern fn __zig_elf_static_tls_fill(area: [*]u8) usize;
1446
14371447 fn spawn(config: SpawnConfig, comptime f: anytype, args: anytype) !Impl {
14381448 const page_size = std.heap.pageSize();
14391449 const Args = @TypeOf(args);
......@@ -1465,9 +1475,9 @@ const LinuxThreadImpl = struct {
14651475 bytes = std.mem.alignForward(usize, bytes, page_size);
14661476 stack_offset = bytes;
14671477
1468 bytes = std.mem.alignForward(usize, bytes, linux.tls.area_desc.alignment);
1478 bytes = std.mem.alignForward(usize, bytes, __zig_elf_static_tls.alignment);
14691479 tls_offset = bytes;
1470 bytes += linux.tls.area_desc.size;
1480 bytes += __zig_elf_static_tls.size;
14711481
14721482 bytes = std.mem.alignForward(usize, bytes, @alignOf(Instance));
14731483 instance_offset = bytes;
......@@ -1509,12 +1519,12 @@ const LinuxThreadImpl = struct {
15091519 }
15101520
15111521 // Prepare the TLS segment and prepare a user_desc struct when needed on x86
1512 var tls_ptr = linux.tls.prepareArea(mapped[tls_offset..][0..linux.tls.area_desc.size]);
1522 var tls_ptr = __zig_elf_static_tls_fill(mapped.ptr + tls_offset);
15131523 var user_desc: if (target.cpu.arch == .x86) linux.user_desc else void = undefined;
15141524 if (target.cpu.arch == .x86) {
15151525 defer tls_ptr = @intFromPtr(&user_desc);
15161526 user_desc = .{
1517 .entry_number = linux.tls.area_desc.gdt_entry_number,
1527 .entry_number = __zig_elf_static_tls.gdt_entry_number,
15181528 .base_addr = tls_ptr,
15191529 .limit = 0xfffff,
15201530 .flags = .{
lib/std/elf.zig+1-1
......@@ -1602,7 +1602,7 @@ pub const Elf32_auxv_t = extern struct {
16021602 },
16031603};
16041604pub const Elf64_auxv_t = extern struct {
1605 a_type: u64,
1605 a_type: if (native_os == .netbsd or native_os == .openbsd) u32 else u64,
16061606 a_un: extern union {
16071607 a_val: u64,
16081608 },
lib/std/os/linux.zig+14-36
......@@ -123,7 +123,6 @@ pub const gid_t = u32;
123123pub const clock_t = isize;
124124pub const time_t = arch_bits.time_t;
125125
126pub const tls = @import("linux/tls.zig");
127126pub const BPF = @import("linux/bpf.zig");
128127pub const IOCTL = @import("linux/ioctl.zig");
129128pub const SECCOMP = @import("linux/seccomp.zig");
......@@ -960,43 +959,22 @@ pub const RENAME = packed struct(u32) {
960959 _: u29 = 0,
961960};
962961
963/// Set by startup code, used by `getauxval`.
964pub var elf_aux_maybe: ?[*]std.elf.Auxv = null;
965
966/// Whether an external or internal getauxval implementation is used.
967const extern_getauxval = switch (builtin.zig_backend) {
968 // Calling extern functions is not yet supported with these backends
969 .stage2_arm,
970 .stage2_loongarch,
971 .stage2_powerpc,
972 .stage2_riscv64,
973 .stage2_sparc64,
974 => false,
975 else => !builtin.link_libc,
976};
977
978pub const getauxval = if (extern_getauxval) struct {
979 comptime {
980 const root = @import("root");
981 // Export this only when building an executable, otherwise it is overriding
982 // the libc implementation
983 if (builtin.output_mode == .Exe or @hasDecl(root, "main")) {
984 @export(&getauxvalImpl, .{ .name = "getauxval", .linkage = .weak });
962/// Available only when libc is *not* linked; use `std.c.getauxval` if libc is linked.
963pub const getauxval = if (builtin.link_libc) {} else struct {
964 /// Provided by compiler-rt and set by `std.start`.
965 extern var __zig_elf_auxv: [*]std.elf.Auxv;
966
967 pub fn getauxval(index: usize) usize {
968 @disableInstrumentation();
969
970 var i: usize = 0;
971 while (__zig_elf_auxv[i].a_type != std.elf.AT.NULL) : (i += 1) {
972 if (__zig_elf_auxv[i].a_type == index) return __zig_elf_auxv[i].a_un.a_val;
985973 }
974
975 return 0;
986976 }
987 extern fn getauxval(index: usize) usize;
988}.getauxval else getauxvalImpl;
989
990fn getauxvalImpl(index: usize) callconv(.c) usize {
991 @disableInstrumentation();
992 const auxv = elf_aux_maybe orelse return 0;
993 var i: usize = 0;
994 while (auxv[i].a_type != std.elf.AT.NULL) : (i += 1) {
995 if (auxv[i].a_type == index)
996 return auxv[i].a_un.a_val;
997 }
998 return 0;
999}
977}.getauxval;
1000978
1001979// Some architectures (and some syscalls) require 64bit parameters to be passed
1002980// in a even-aligned register pair.
lib/std/os/linux/tls.zig deleted-769
......@@ -1,769 +0,0 @@
1//! This file implements the two TLS variants [1] used by ELF-based systems. Note that, in reality,
2//! Variant I has two sub-variants.
3//!
4//! It is important to understand that the term TCB (Thread Control Block) is overloaded here.
5//! Official ABI documentation uses it simply to mean the ABI TCB, i.e. a small area of ABI-defined
6//! data, usually one or two words (see the `AbiTcb` type below). People will also often use TCB to
7//! refer to the libc TCB, which can be any size and contain anything. (One could even omit it!) We
8//! refer to the latter as the Zig TCB; see the `ZigTcb` type below.
9//!
10//! [1] https://www.akkadia.org/drepper/tls.pdf
11
12const std = @import("std");
13const mem = std.mem;
14const elf = std.elf;
15const math = std.math;
16const assert = std.debug.assert;
17const builtin = @import("builtin");
18const native_arch = builtin.cpu.arch;
19const linux = std.os.linux;
20const page_size_min = std.heap.page_size_min;
21
22/// Represents an ELF TLS variant.
23///
24/// In all variants, the TP and the TLS blocks must be aligned to the `p_align` value in the
25/// `PT.TLS` ELF program header. Everything else has natural alignment.
26///
27/// The location of the DTV does not actually matter. For simplicity, we put it in the TLS area, but
28/// there is no actual ABI requirement that it reside there.
29const Variant = enum {
30 /// The original Variant I:
31 ///
32 /// ----------------------------------------
33 /// | DTV | Zig TCB | ABI TCB | TLS Blocks |
34 /// ----------------^-----------------------
35 /// `-- The TP register points here.
36 ///
37 /// The layout in this variant necessitates separate alignment of both the TP and the TLS
38 /// blocks.
39 ///
40 /// The first word in the ABI TCB points to the DTV. For some architectures, there may be a
41 /// second word with an unspecified meaning.
42 I_original,
43 /// The modified Variant I:
44 ///
45 /// --------------------------------------------
46 /// | DTV | Zig TCB | ABI TCB | TLS Blocks |
47 /// ------------------------------^-------------
48 /// `-- The TP register points here (*inside* the TLS blocks).
49 ///
50 /// The offset from the start of the TLS blocks to the TP register is `current_tp_offset`. It
51 /// may be zero, in which case the TP register points to the start of the TLS blocks.
52 ///
53 /// The first (and only) word in the ABI TCB points to the DTV.
54 I_modified,
55 /// Variant II:
56 ///
57 /// ----------------------------------------
58 /// | TLS Blocks | ABI TCB | Zig TCB | DTV |
59 /// -------------^--------------------------
60 /// `-- The TP register points here.
61 ///
62 /// The first (and only) word in the ABI TCB points to the ABI TCB itself.
63 II,
64};
65
66const current_variant: Variant = switch (native_arch) {
67 .aarch64,
68 .aarch64_be,
69 .alpha,
70 .arc,
71 .arceb,
72 .arm,
73 .armeb,
74 .csky,
75 .hppa,
76 .microblaze,
77 .microblazeel,
78 .sh,
79 .sheb,
80 .thumb,
81 .thumbeb,
82 .xtensa,
83 .xtensaeb,
84 => .I_original,
85 .loongarch32,
86 .loongarch64,
87 .m68k,
88 .mips,
89 .mipsel,
90 .mips64,
91 .mips64el,
92 .or1k,
93 .powerpc,
94 .powerpcle,
95 .powerpc64,
96 .powerpc64le,
97 .riscv32,
98 .riscv64,
99 => .I_modified,
100 .hexagon,
101 .s390x,
102 .sparc,
103 .sparc64,
104 .x86,
105 .x86_64,
106 => .II,
107 else => @compileError("undefined TLS variant for this architecture"),
108};
109
110/// The offset value for the modified Variant I.
111const current_tp_offset = switch (native_arch) {
112 .m68k,
113 .mips,
114 .mipsel,
115 .mips64,
116 .mips64el,
117 .powerpc,
118 .powerpcle,
119 .powerpc64,
120 .powerpc64le,
121 => 0x7000,
122 else => 0,
123};
124
125/// Usually only used by the modified Variant I.
126const current_dtv_offset = switch (native_arch) {
127 .m68k,
128 .mips,
129 .mipsel,
130 .mips64,
131 .mips64el,
132 .powerpc,
133 .powerpcle,
134 .powerpc64,
135 .powerpc64le,
136 => 0x8000,
137 .riscv32,
138 .riscv64,
139 => 0x800,
140 else => 0,
141};
142
143/// Per-thread storage for the ELF TLS ABI.
144const AbiTcb = switch (current_variant) {
145 .I_original, .I_modified => switch (native_arch) {
146 .aarch64,
147 .aarch64_be,
148 .alpha,
149 .arm,
150 .armeb,
151 .hppa,
152 .microblaze,
153 .microblazeel,
154 .sh,
155 .sheb,
156 .thumb,
157 .thumbeb,
158 .xtensa,
159 .xtensaeb,
160 => extern struct {
161 /// This is offset by `current_dtv_offset`.
162 dtv: usize,
163 _reserved: ?*anyopaque,
164 },
165 else => extern struct {
166 /// This is offset by `current_dtv_offset`.
167 dtv: usize,
168 },
169 },
170 .II => extern struct {
171 /// This is self-referential.
172 self: *AbiTcb,
173 },
174};
175
176/// Per-thread storage for Zig's use. Currently unused.
177const ZigTcb = struct {
178 dummy: usize,
179};
180
181/// Dynamic Thread Vector as specified in the ELF TLS ABI. Ordinarily, there is a block pointer per
182/// dynamically-loaded module, but since we only support static TLS, we only need one block pointer.
183const Dtv = extern struct {
184 len: usize = 1,
185 tls_block: [*]u8,
186};
187
188/// Describes a process's TLS area. The area encompasses the DTV, both TCBs, and the TLS block, with
189/// the exact layout of these being dependent primarily on `current_variant`.
190const AreaDesc = struct {
191 size: usize,
192 alignment: usize,
193
194 dtv: struct {
195 /// Offset into the TLS area.
196 offset: usize,
197 },
198
199 abi_tcb: struct {
200 /// Offset into the TLS area.
201 offset: usize,
202 },
203
204 block: struct {
205 /// The initial data to be copied into the TLS block. Note that this may be smaller than
206 /// `size`, in which case any remaining data in the TLS block is simply left uninitialized.
207 init: []const u8,
208 /// Offset into the TLS area.
209 offset: usize,
210 /// This is the effective size of the TLS block, which may be greater than `init.len`.
211 size: usize,
212 },
213
214 /// Only used on the 32-bit x86 architecture (not x86_64, nor x32).
215 gdt_entry_number: usize,
216};
217
218pub var area_desc: AreaDesc = undefined;
219
220pub fn setThreadPointer(addr: usize) void {
221 @setRuntimeSafety(false);
222 @disableInstrumentation();
223
224 switch (native_arch) {
225 .x86 => {
226 var user_desc: linux.user_desc = .{
227 .entry_number = area_desc.gdt_entry_number,
228 .base_addr = addr,
229 .limit = 0xfffff,
230 .flags = .{
231 .seg_32bit = 1,
232 .contents = 0, // Data
233 .read_exec_only = 0,
234 .limit_in_pages = 1,
235 .seg_not_present = 0,
236 .useable = 1,
237 },
238 };
239 const rc = @call(.always_inline, linux.syscall1, .{ .set_thread_area, @intFromPtr(&user_desc) });
240 assert(rc == 0);
241
242 const gdt_entry_number = user_desc.entry_number;
243 // We have to keep track of our slot as it's also needed for clone()
244 area_desc.gdt_entry_number = gdt_entry_number;
245 // Update the %gs selector
246 asm volatile ("movl %[gs_val], %%gs"
247 :
248 : [gs_val] "r" (gdt_entry_number << 3 | 3),
249 );
250 },
251 .x86_64 => {
252 const rc = @call(.always_inline, linux.syscall2, .{ .arch_prctl, linux.ARCH.SET_FS, addr });
253 assert(rc == 0);
254 },
255 .aarch64, .aarch64_be => {
256 asm volatile (
257 \\ msr tpidr_el0, %[addr]
258 :
259 : [addr] "r" (addr),
260 );
261 },
262 .alpha => {
263 asm volatile (
264 \\ wruniq
265 :
266 : [addr] "{$16}" (addr),
267 );
268 },
269 .arc, .arceb => {
270 // We apparently need to both set r25 (TP) *and* inform the kernel...
271 asm volatile (
272 \\ mov r25, %[addr]
273 :
274 : [addr] "r" (addr),
275 );
276 const rc = @call(.always_inline, linux.syscall1, .{ .arc_settls, addr });
277 assert(rc == 0);
278 },
279 .arm, .armeb, .thumb, .thumbeb => {
280 const rc = @call(.always_inline, linux.syscall1, .{ .set_tls, addr });
281 assert(rc == 0);
282 },
283 .m68k => {
284 const rc = @call(.always_inline, linux.syscall1, .{ .set_thread_area, addr });
285 assert(rc == 0);
286 },
287 .hexagon => {
288 asm volatile (
289 \\ ugp = %[addr]
290 :
291 : [addr] "r" (addr),
292 );
293 },
294 .hppa => {
295 asm volatile (
296 \\ ble 0xe0(%%sr2, %%r0)
297 \\ nop
298 :
299 : [addr] "{r26}" (addr),
300 : .{ .r31 = true });
301 },
302 .loongarch32, .loongarch64 => {
303 asm volatile (
304 \\ move $tp, %[addr]
305 :
306 : [addr] "r" (addr),
307 );
308 },
309 .riscv32, .riscv64 => {
310 asm volatile (
311 \\ mv tp, %[addr]
312 :
313 : [addr] "r" (addr),
314 );
315 },
316 .csky, .mips, .mipsel, .mips64, .mips64el => {
317 const rc = @call(.always_inline, linux.syscall1, .{ .set_thread_area, addr });
318 assert(rc == 0);
319 },
320 .microblaze, .microblazeel => {
321 asm volatile (
322 \\ ori r21, %[addr], 0
323 :
324 : [addr] "r" (addr),
325 );
326 },
327 .or1k => {
328 asm volatile (
329 \\ l.ori r10, %[addr], 0
330 :
331 : [addr] "r" (addr),
332 );
333 },
334 .powerpc, .powerpcle => {
335 asm volatile (
336 \\ mr 2, %[addr]
337 :
338 : [addr] "r" (addr),
339 );
340 },
341 .powerpc64, .powerpc64le => {
342 asm volatile (
343 \\ mr 13, %[addr]
344 :
345 : [addr] "r" (addr),
346 );
347 },
348 .s390x => {
349 asm volatile (
350 \\ lgr %%r0, %[addr]
351 \\ sar %%a1, %%r0
352 \\ srlg %%r0, %%r0, 32
353 \\ sar %%a0, %%r0
354 :
355 : [addr] "r" (addr),
356 : .{ .r0 = true });
357 },
358 .sh, .sheb => {
359 asm volatile (
360 \\ ldc %[addr], gbr
361 :
362 : [addr] "r" (addr),
363 );
364 },
365 .sparc, .sparc64 => {
366 asm volatile (
367 \\ mov %[addr], %%g7
368 :
369 : [addr] "r" (addr),
370 );
371 },
372 .xtensa, .xtensaeb => {
373 asm volatile (
374 \\ wur %[addr], threadptr
375 :
376 : [addr] "a" (addr),
377 );
378 },
379 else => @compileError("Unsupported architecture"),
380 }
381}
382
383pub fn getThreadPointer() usize {
384 @setRuntimeSafety(false);
385 @disableInstrumentation();
386
387 return switch (native_arch) {
388 .aarch64, .aarch64_be => asm (
389 \\ mrs %[ret], tpidr_el0
390 : [ret] "=r" (-> usize),
391 ),
392 .alpha => asm (
393 \\ rduniq
394 : [ret] "={$0}" (-> usize),
395 ),
396 .arc, .arceb => asm (
397 \\ mov %[ret], r25
398 : [ret] "=r" (-> usize),
399 ),
400 .arm, .armeb, .thumb, .thumbeb => asm (
401 \\ mrc p15, 0, %[ret], c13, c0, 3
402 : [ret] "=r" (-> usize),
403 ),
404 .csky => asm (
405 \\ mov %[ret], r31
406 : [ret] "=r" (-> usize),
407 ),
408 .hexagon => asm (
409 \\ %[ret] = ugp
410 : [ret] "=r" (-> usize),
411 ),
412 .hppa => asm (
413 \\ mfctl %%cr27, %[ret]
414 : [ret] "=r" (-> usize),
415 ),
416 .loongarch32, .loongarch64 => asm (
417 \\ move %[ret], $tp
418 : [ret] "=r" (-> usize),
419 ),
420 .m68k => linux.syscall0(.get_thread_area),
421 .mips, .mipsel, .mips64, .mips64el => asm (
422 \\ rdhwr %[ret], $29
423 : [ret] "=r" (-> usize),
424 ),
425 .microblaze, .microblazeel => asm (
426 \\ ori %[ret], r21, 0
427 : [ret] "=r" (-> usize),
428 ),
429 .or1k => asm (
430 \\ l.ori %[ret], r10, 0
431 : [ret] "=r" (-> usize),
432 ),
433 .riscv32, .riscv64 => asm (
434 \\ mv %[ret], tp
435 : [ret] "=r" (-> usize),
436 ),
437 .powerpc, .powerpcle => asm (
438 \\ mr %[ret], 2
439 : [ret] "=r" (-> usize),
440 ),
441 .powerpc64, .powerpc64le => asm (
442 \\ mr %[ret], 13
443 : [ret] "=r" (-> usize),
444 ),
445 .s390x => asm (
446 \\ ear %[ret], %%a0
447 \\ sllg %[ret], %[ret], 32
448 \\ ear %[ret], %%a1
449 : [ret] "=r" (-> usize),
450 ),
451 .sh, .sheb => asm (
452 \\ stc gbr, %[ret]
453 : [ret] "=r" (-> usize),
454 ),
455 .sparc, .sparc64 => asm (
456 \\ mov %%g7, %[ret]
457 : [ret] "=r" (-> usize),
458 ),
459 .x86 => asm (
460 \\ movl %%gs:0, %[ret]
461 : [ret] "=r" (-> usize),
462 ),
463 .x86_64 => switch (@sizeOf(usize)) {
464 8 => asm (
465 \\ movq %%fs:0, %[ret]
466 : [ret] "=r" (-> usize),
467 ),
468 // On x32, usize is 32 bits.
469 4 => asm (
470 \\ movl %%fs:0, %[ret]
471 : [ret] "=r" (-> usize),
472 ),
473 else => comptime unreachable,
474 },
475 .xtensa, .xtensaeb => asm (
476 \\ rur %[ret], threadptr
477 : [ret] "=r" (-> usize),
478 ),
479 else => @compileError("Unsupported architecture"),
480 };
481}
482
483fn computeAreaDesc(phdrs: []elf.ElfN.Phdr) void {
484 @setRuntimeSafety(false);
485 @disableInstrumentation();
486
487 var tls_phdr: ?*elf.ElfN.Phdr = null;
488 var img_base: usize = 0;
489
490 for (phdrs) |*phdr| {
491 switch (phdr.type) {
492 .PHDR => img_base = @intFromPtr(phdrs.ptr) - phdr.vaddr,
493 .TLS => tls_phdr = phdr,
494 else => {},
495 }
496 }
497
498 var align_factor: usize = undefined;
499 var block_init: []const u8 = undefined;
500 var block_size: usize = undefined;
501
502 if (tls_phdr) |phdr| {
503 align_factor = phdr.@"align";
504
505 // The effective size in memory is represented by `memsz`; the length of the data stored
506 // in the `PT.TLS` segment is `filesz` and may be less than the former.
507 block_init = @as([*]u8, @ptrFromInt(img_base + phdr.vaddr))[0..phdr.filesz];
508 block_size = phdr.memsz;
509 } else {
510 align_factor = @alignOf(usize);
511
512 block_init = &[_]u8{};
513 block_size = 0;
514 }
515
516 // Offsets into the allocated TLS area.
517 var dtv_offset: usize = undefined;
518 var abi_tcb_offset: usize = undefined;
519 var block_offset: usize = undefined;
520
521 // Compute the total size of the ABI-specific data plus our own `ZigTcb` structure. All the
522 // offsets calculated here assume a well-aligned base address.
523 const area_size = switch (current_variant) {
524 .I_original => blk: {
525 var l: usize = 0;
526 dtv_offset = l;
527 l += @sizeOf(Dtv);
528 // Add some padding here so that the TP (`abi_tcb_offset`) is aligned to `align_factor`
529 // and the `ZigTcb` structure can be found by simply subtracting `@sizeOf(ZigTcb)` from
530 // the TP.
531 const delta = (l + @sizeOf(ZigTcb)) & (align_factor - 1);
532 if (delta > 0)
533 l += align_factor - delta;
534 l += @sizeOf(ZigTcb);
535 abi_tcb_offset = l;
536 l += alignForward(@sizeOf(AbiTcb), align_factor);
537 block_offset = l;
538 l += block_size;
539 break :blk l;
540 },
541 .I_modified => blk: {
542 var l: usize = 0;
543 dtv_offset = l;
544 l += @sizeOf(Dtv);
545 // In this variant, the TLS blocks must begin immediately after the end of the ABI TCB,
546 // with the TP pointing to the beginning of the TLS blocks. Add padding so that the TP
547 // (`abi_tcb_offset`) is aligned to `align_factor` and the `ZigTcb` structure can be
548 // found by subtracting `@sizeOf(AbiTcb) + @sizeOf(ZigTcb)` from the TP.
549 const delta = (l + @sizeOf(ZigTcb) + @sizeOf(AbiTcb)) & (align_factor - 1);
550 if (delta > 0)
551 l += align_factor - delta;
552 l += @sizeOf(ZigTcb);
553 abi_tcb_offset = l;
554 l += @sizeOf(AbiTcb);
555 block_offset = l;
556 l += block_size;
557 break :blk l;
558 },
559 .II => blk: {
560 var l: usize = 0;
561 block_offset = l;
562 l += alignForward(block_size, align_factor);
563 // The TP is aligned to `align_factor`.
564 abi_tcb_offset = l;
565 l += @sizeOf(AbiTcb);
566 // The `ZigTcb` structure is right after the `AbiTcb` with no padding in between so it
567 // can be easily found.
568 l += @sizeOf(ZigTcb);
569 // It doesn't really matter where we put the DTV, so give it natural alignment.
570 l = alignForward(l, @alignOf(Dtv));
571 dtv_offset = l;
572 l += @sizeOf(Dtv);
573 break :blk l;
574 },
575 };
576
577 area_desc = .{
578 .size = area_size,
579 .alignment = align_factor,
580
581 .dtv = .{
582 .offset = dtv_offset,
583 },
584
585 .abi_tcb = .{
586 .offset = abi_tcb_offset,
587 },
588
589 .block = .{
590 .init = block_init,
591 .offset = block_offset,
592 .size = block_size,
593 },
594
595 .gdt_entry_number = @as(usize, @bitCast(@as(isize, -1))),
596 };
597}
598
599/// Inline because TLS is not set up yet.
600inline fn alignForward(addr: usize, alignment: usize) usize {
601 return alignBackward(addr + (alignment - 1), alignment);
602}
603
604/// Inline because TLS is not set up yet.
605inline fn alignBackward(addr: usize, alignment: usize) usize {
606 return addr & ~(alignment - 1);
607}
608
609/// Inline because TLS is not set up yet.
610inline fn alignPtrCast(comptime T: type, ptr: [*]u8) *T {
611 return @ptrCast(@alignCast(ptr));
612}
613
614/// Initializes all the fields of the static TLS area and returns the computed architecture-specific
615/// value of the TP register.
616pub fn prepareArea(area: []u8) usize {
617 @setRuntimeSafety(false);
618 @disableInstrumentation();
619
620 // Clear the area we're going to use, just to be safe.
621 @memset(area, 0);
622
623 // Prepare the ABI TCB.
624 const abi_tcb = alignPtrCast(AbiTcb, area.ptr + area_desc.abi_tcb.offset);
625 switch (current_variant) {
626 .I_original, .I_modified => abi_tcb.dtv = @intFromPtr(area.ptr + area_desc.dtv.offset),
627 .II => abi_tcb.self = abi_tcb,
628 }
629
630 // Prepare the DTV.
631 const dtv = alignPtrCast(Dtv, area.ptr + area_desc.dtv.offset);
632 dtv.len = 1;
633 dtv.tls_block = area.ptr + current_dtv_offset + area_desc.block.offset;
634
635 // Copy the initial data.
636 @memcpy(area[area_desc.block.offset..][0..area_desc.block.init.len], area_desc.block.init);
637
638 // Return the corrected value (if needed) for the TP register. Overflow here is not a problem;
639 // the pointer arithmetic involving the TP is done with wrapping semantics.
640 return @intFromPtr(area.ptr) +% switch (current_variant) {
641 .I_original, .II => area_desc.abi_tcb.offset,
642 .I_modified => area_desc.block.offset +% current_tp_offset,
643 };
644}
645
646/// The main motivation for the size chosen here is to be larger than total
647/// amount of thread-local variables for most programs. Putting this allocation
648/// in the ELF like this is equivalent to moving the `mmap` call below into the
649/// kernel, avoiding syscall overhead.
650var main_thread_area_buffer: [0x1000]u8 align(page_size_min) = undefined;
651
652/// Computes the layout of the static TLS area, allocates the area, initializes all of its fields,
653/// and assigns the architecture-specific value to the TP register.
654pub fn initStatic(phdrs: []elf.ElfN.Phdr) void {
655 @setRuntimeSafety(false);
656 @disableInstrumentation();
657
658 computeAreaDesc(phdrs);
659
660 const area = blk: {
661 // Fast path for the common case where the TLS data is really small, avoid an allocation and
662 // use our local buffer.
663 if (area_desc.alignment <= page_size_min and area_desc.size <= main_thread_area_buffer.len) {
664 break :blk main_thread_area_buffer[0..area_desc.size];
665 }
666
667 const begin_addr = mmap_tls(area_desc.size + area_desc.alignment - 1);
668 if (@call(.always_inline, linux.errno, .{begin_addr}) != .SUCCESS) @trap();
669
670 const area_ptr: [*]align(page_size_min) u8 = @ptrFromInt(begin_addr);
671
672 // Make sure the slice is correctly aligned.
673 const begin_aligned_addr = alignForward(begin_addr, area_desc.alignment);
674 const start = begin_aligned_addr - begin_addr;
675 break :blk area_ptr[start..][0..area_desc.size];
676 };
677
678 const tp_value = prepareArea(area);
679 setThreadPointer(tp_value);
680}
681
682inline fn mmap_tls(length: usize) usize {
683 const prot: linux.PROT = .{ .READ = true, .WRITE = true };
684 const flags: linux.MAP = .{ .TYPE = .PRIVATE, .ANONYMOUS = true };
685
686 if (@hasField(linux.SYS, "mmap2")) {
687 return @call(.always_inline, linux.syscall6, .{
688 .mmap2,
689 0,
690 length,
691 @as(u32, @bitCast(prot)),
692 @as(u32, @bitCast(flags)),
693 @as(usize, @bitCast(@as(isize, -1))),
694 0,
695 });
696 } else {
697 // The s390x mmap() syscall existed before Linux supported syscalls with 5+ parameters, so
698 // it takes a single pointer to an array of arguments instead.
699 return if (native_arch == .s390x) @call(.always_inline, linux.syscall1, .{
700 .mmap,
701 @intFromPtr(&[_]usize{
702 0,
703 length,
704 @as(u32, @bitCast(prot)),
705 @as(u32, @bitCast(flags)),
706 @as(usize, @bitCast(@as(isize, -1))),
707 0,
708 }),
709 }) else @call(.always_inline, linux.syscall6, .{
710 .mmap,
711 0,
712 length,
713 @as(u32, @bitCast(prot)),
714 @as(u32, @bitCast(flags)),
715 @as(usize, @bitCast(@as(isize, -1))),
716 0,
717 });
718 }
719}
720
721comptime {
722 assert(!builtin.link_libc); // otherwise libc should control TLS
723
724 if (builtin.output_mode == .Exe and builtin.link_mode == .static) {
725 // This is a static executable without libc, so it is our job to provide the TLS accessor
726 // function for the GD and LD models. This function is unlikely to actually be used, since
727 // the linker should be able to relax every TLS access to the LE model and therefore
728 // eliminate all calls to this function, but that isn't guaranteed.
729 const Fns = struct {
730 const TlsIndex = switch (native_arch) {
731 .x86_64 => extern struct { module: u64, offset: u64 }, // Even for x32...
732 else => extern struct { module: usize, offset: usize }, // ...but not MIPS N32!
733 };
734 fn __tls_get_addr(ti: *const TlsIndex) callconv(.c) *anyopaque {
735 comptime assert(native_arch != .s390x);
736
737 assert(ti.module == 1); // The executable's module ID is always 1
738 const tp = getThreadPointer();
739 const block: [*]u8 = switch (current_variant) {
740 .I_original => @ptrFromInt(tp -% area_desc.abi_tcb.offset +% area_desc.block.offset),
741 .I_modified => @ptrFromInt(tp -% current_tp_offset),
742 // The `.I_original` approach would also work for `.II`, but there is an
743 // alternative strategy which is one less operation:
744 .II => @ptrFromInt(tp -% area_desc.block.size),
745 };
746 return block[@intCast(ti.offset)..];
747 }
748 fn __tls_get_offset() callconv(.naked) noreturn {
749 comptime assert(native_arch == .s390x);
750
751 // We receive the module's GOT pointer in r12 and the GOT offset in r2.
752 asm volatile (
753 \\ la %%r1, 0(%%r12, %%r2)
754 \\ lg %%r2, 8(%%r1)
755 \\ lgrl %%r0, %[block_size]
756 \\ sgr %%r2, %%r0
757 \\ br %%r14
758 :
759 : [block_size] "s" (&area_desc.block.size),
760 );
761 }
762 };
763
764 if (native_arch == .s390x)
765 @export(&Fns.__tls_get_offset, .{ .name = "__tls_get_offset" })
766 else
767 @export(&Fns.__tls_get_addr, .{ .name = "__tls_get_addr" });
768 }
769}
lib/std/os/linux/vdso.zig+3-1
......@@ -1,3 +1,4 @@
1const builtin = @import("builtin");
12const std = @import("../../std.zig");
23const elf = std.elf;
34const linux = std.os.linux;
......@@ -5,7 +6,8 @@ const mem = std.mem;
56const maxInt = std.math.maxInt;
67
78pub fn lookup(vername: []const u8, name: []const u8) usize {
8 const vdso_addr = linux.getauxval(std.elf.AT.SYSINFO_EHDR);
9 const getauxval = if (builtin.link_libc) std.c.getauxval else linux.getauxval;
10 const vdso_addr = getauxval(std.elf.AT.SYSINFO_EHDR);
911 if (vdso_addr == 0) return 0;
1012
1113 const eh = @as(*elf.Ehdr, @ptrFromInt(vdso_addr));
lib/std/start.zig+12-6
......@@ -596,6 +596,12 @@ fn wWinMain(hInstance: *anyopaque, hPrevInstance: ?*anyopaque, pCmdLine: [*:0]u1
596596 return root.wWinMain(@ptrCast(hInstance), @ptrCast(hPrevInstance), pCmdLine, @intCast(nCmdShow));
597597}
598598
599/// Provided by compiler-rt.
600extern var __zig_elf_auxv: [*]std.elf.Auxv;
601
602/// Provided by compiler-rt.
603extern fn __zig_elf_static_tls_init(phdrs: [*]elf.ElfN.Phdr, phnum: usize) void;
604
599605fn posixCallMainAndExit(argc_argv_ptr: [*]usize) callconv(.c) noreturn {
600606 // We're not ready to panic until thread local storage is initialized.
601607 @setRuntimeSafety(false);
......@@ -613,7 +619,7 @@ fn posixCallMainAndExit(argc_argv_ptr: [*]usize) callconv(.c) noreturn {
613619 const auxv: [*]elf.Auxv = @ptrCast(@alignCast(envp.ptr + envp_count + 1));
614620
615621 var at_hwcap: usize = 0;
616 const phdrs = init: {
622 const phdrs: [*]elf.ElfN.Phdr, const phnum: usize = init: {
617623 var i: usize = 0;
618624 var at_phdr: usize = 0;
619625 var at_phnum: usize = 0;
......@@ -625,20 +631,20 @@ fn posixCallMainAndExit(argc_argv_ptr: [*]usize) callconv(.c) noreturn {
625631 else => continue,
626632 }
627633 }
628 break :init @as([*]elf.ElfN.Phdr, @ptrFromInt(at_phdr))[0..at_phnum];
634 break :init .{ @ptrFromInt(at_phdr), at_phnum };
629635 };
630636
631637 // Apply the initial relocations as early as possible in the startup process. We cannot
632638 // make calls yet on some architectures (e.g. MIPS) *because* they haven't been applied yet,
633639 // so this must be fully inlined.
634640 if (builtin.link_mode == .static and builtin.position_independent_executable) {
635 @call(.always_inline, std.pie.relocate, .{phdrs});
641 @call(.always_inline, std.pie.relocate, .{phdrs[0..phnum]});
636642 }
637643
638644 if (native_os == .linux) {
639645 // This must be done after PIE relocations have been applied or we may crash
640646 // while trying to access the global variable (happens on MIPS at least).
641 std.os.linux.elf_aux_maybe = auxv;
647 __zig_elf_auxv = auxv;
642648
643649 if (!builtin.single_threaded) {
644650 // ARMv6 targets (and earlier) have no support for TLS in hardware.
......@@ -654,14 +660,14 @@ fn posixCallMainAndExit(argc_argv_ptr: [*]usize) callconv(.c) noreturn {
654660 }
655661
656662 // Initialize the TLS area.
657 std.os.linux.tls.initStatic(phdrs);
663 __zig_elf_static_tls_init(phdrs, phnum);
658664 }
659665
660666 // The way Linux executables represent stack size is via the PT.GNU_STACK
661667 // program header. However the kernel does not recognize it; it always gives 8 MiB.
662668 // Here we look for the stack size in our program headers and use setrlimit
663669 // to ask for more stack space.
664 expandStackSize(phdrs);
670 expandStackSize(phdrs[0..phnum]);
665671 }
666672
667673 const opt_init_array_start = @extern([*]const *const fn () callconv(.c) void, .{
src/codegen/c.zig+7
......@@ -2552,6 +2552,13 @@ pub fn genDeclFwd(dg: *DeclGen, w: *Writer) Error!void {
25522552 .{ .@"const" = nav.resolved.?.@"const" },
25532553 nav.resolved.?.@"align",
25542554 );
2555 const extern_name = nav.name.toSlice(ip);
2556 const is_mangled = isMangledIdent(extern_name, true);
2557 if (is_mangled) {
2558 try w.print(" zig_mangled({f}, {f})", .{
2559 fmtIdentSolo(extern_name), fmtStringLiteral(extern_name, null),
2560 });
2561 }
25552562 try w.writeAll(";\n");
25562563 return;
25572564 },
test/standalone/mix_o_files/build.zig+1
......@@ -13,6 +13,7 @@ pub fn build(b: *std.Build) void {
1313 .root_source_file = b.path("base64.zig"),
1414 .optimize = optimize,
1515 .target = target,
16 .link_libc = true,
1617 }),
1718 });
1819