1//! This file implements the two TLS variants [1] used by ELF-based systems. Note that, in reality,
2//! Variant I has two sub-variants.
3//!
4//! It is important to understand that the term TCB (Thread Control Block) is overloaded here.
5//! Official ABI documentation uses it simply to mean the ABI TCB, i.e. a small area of ABI-defined
6//! data, usually one or two words (see the `AbiTcb` type below). People will also often use TCB to
7//! refer to the libc TCB, which can be any size and contain anything. (One could even omit it!) We
8//! refer to the latter as the Zig TCB; see the `ZigTcb` type below.
9//!
10//! [1] https://www.akkadia.org/drepper/tls.pdf
11
12const builtin = @import("builtin");
13const std = @import("std");
14const elf = std.elf;
15const assert = std.debug.assert;
16const native_arch = builtin.cpu.arch;
17const linux = std.os.linux;
18const page_size_min = std.heap.page_size_min;
19
20const compiler_rt = @import("../../compiler_rt.zig");
21const symbol = compiler_rt.symbol;
22
23comptime {
24 symbol(&__zig_elf_static_tls, "__zig_elf_static_tls");
25 symbol(&__zig_elf_static_tls_init, "__zig_elf_static_tls_init");
26 symbol(&__zig_elf_static_tls_fill, "__zig_elf_static_tls_fill");
27
28 // To make static executables without libc work, it is our job to provide the TLS accessor
29 // function for the GD and LD models. This function is unlikely to actually be used, since the
30 // linker should be able to relax every TLS access to the LE model and therefore eliminate all
31 // calls to this function, but that isn't guaranteed.
32 if (native_arch == .s390x) {
33 symbol(&__tls_get_offset, "__tls_get_offset");
34 } else {
35 symbol(&__tls_get_addr, "__tls_get_addr");
36 if (native_arch == .m68k) {
37 symbol(&__m68k_read_tp, "__m68k_read_tp");
38 }
39 if (native_arch == .x86) {
40 symbol(&___tls_get_addr, "___tls_get_addr");
41 }
42 }
43}
44
45/// Represents an ELF TLS variant.
46///
47/// In all variants, the TP and the TLS blocks must be aligned to the `p_align` value in the
48/// `PT.TLS` ELF program header. Everything else has natural alignment.
49///
50/// The location of the DTV does not actually matter. For simplicity, we put it in the TLS area, but
51/// there is no actual ABI requirement that it reside there.
52const Variant = enum {
53 /// The original Variant I:
54 ///
55 /// ----------------------------------------
56 /// | DTV | Zig TCB | ABI TCB | TLS Blocks |
57 /// ----------------^-----------------------
58 /// `-- The TP register points here.
59 ///
60 /// The layout in this variant necessitates separate alignment of both the TP and the TLS
61 /// blocks.
62 ///
63 /// The first word in the ABI TCB points to the DTV. For some architectures, there may be a
64 /// second word with an unspecified meaning.
65 I_original,
66 /// The modified Variant I:
67 ///
68 /// --------------------------------------------
69 /// | DTV | Zig TCB | ABI TCB | TLS Blocks |
70 /// ------------------------------^-------------
71 /// `-- The TP register points here (*inside* the TLS blocks).
72 ///
73 /// The offset from the start of the TLS blocks to the TP register is `current_tp_offset`. It
74 /// may be zero, in which case the TP register points to the start of the TLS blocks.
75 ///
76 /// The first (and only) word in the ABI TCB points to the DTV.
77 I_modified,
78 /// Variant II:
79 ///
80 /// ----------------------------------------
81 /// | TLS Blocks | ABI TCB | Zig TCB | DTV |
82 /// -------------^--------------------------
83 /// `-- The TP register points here.
84 ///
85 /// The first (and only) word in the ABI TCB points to the ABI TCB itself.
86 II,
87};
88
89const current_variant: Variant = switch (native_arch) {
90 .aarch64,
91 .aarch64_be,
92 .alpha,
93 .arc,
94 .arceb,
95 .arm,
96 .armeb,
97 .csky,
98 .hppa,
99 .microblaze,
100 .microblazeel,
101 .sh,
102 .sheb,
103 .thumb,
104 .thumbeb,
105 .xtensa,
106 .xtensaeb,
107 => .I_original,
108 .loongarch32,
109 .loongarch64,
110 .m68k,
111 .mips,
112 .mipsel,
113 .mips64,
114 .mips64el,
115 .or1k,
116 .powerpc,
117 .powerpcle,
118 .powerpc64,
119 .powerpc64le,
120 .riscv32,
121 .riscv64,
122 => .I_modified,
123 .hexagon,
124 .s390x,
125 .sparc,
126 .sparc64,
127 .x86,
128 .x86_64,
129 => .II,
130 else => @compileError("undefined TLS variant for this architecture"),
131};
132
133/// The offset value for the modified Variant I.
134const current_tp_offset = switch (native_arch) {
135 .m68k,
136 .mips,
137 .mipsel,
138 .mips64,
139 .mips64el,
140 .powerpc,
141 .powerpcle,
142 .powerpc64,
143 .powerpc64le,
144 => 0x7000,
145 else => 0,
146};
147
148/// Usually only used by the modified Variant I.
149const current_dtv_offset = switch (native_arch) {
150 .m68k,
151 .mips,
152 .mipsel,
153 .mips64,
154 .mips64el,
155 .powerpc,
156 .powerpcle,
157 .powerpc64,
158 .powerpc64le,
159 => 0x8000,
160 .riscv32,
161 .riscv64,
162 => 0x800,
163 else => 0,
164};
165
166/// Per-thread storage for the ELF TLS ABI.
167const AbiTcb = switch (current_variant) {
168 .I_original => extern struct {
169 dtv: usize,
170 _reserved: ?*anyopaque,
171 },
172 .I_modified => extern struct {
173 dtv: usize,
174 },
175 .II => extern struct {
176 /// This is self-referential.
177 self: *AbiTcb,
178 },
179};
180
181/// Per-thread storage for Zig's use. Currently unused.
182const ZigTcb = struct {
183 dummy: usize,
184};
185
186/// Dynamic Thread Vector as specified in the ELF TLS ABI. Ordinarily, there is a block pointer per
187/// dynamically-loaded module, but since we only support static TLS, we only need one block pointer.
188const Dtv = extern struct {
189 len: usize = 1,
190 tls_block: [*]u8,
191};
192
193/// Describes a process's TLS area. The area encompasses the DTV, both TCBs, and the TLS block, with
194/// the exact layout of these being dependent primarily on `current_variant`. See also `area_info`.
195///
196/// The layout of this variable is ABI between compiler-rt and std.
197var __zig_elf_static_tls: extern struct {
198 size: usize,
199 alignment: usize,
200 /// Only used on the 32-bit x86 architecture (not x86_64, nor x32).
201 gdt_entry_number: usize,
202} = undefined;
203
204var area_info: struct {
205 dtv: struct {
206 /// Offset into the TLS area.
207 offset: usize,
208 },
209
210 abi_tcb: struct {
211 /// Offset into the TLS area.
212 offset: usize,
213 },
214
215 block: struct {
216 /// The initial data to be copied into the TLS block. Note that this may be smaller than
217 /// `size`, in which case any remaining data in the TLS block is simply left uninitialized.
218 init: []const u8,
219 /// Offset into the TLS area.
220 offset: usize,
221 /// This is the effective size of the TLS block, which may be greater than `init.len`.
222 size: usize,
223 },
224} = undefined;
225
226fn setThreadPointer(addr: usize) void {
227 @setRuntimeSafety(false);
228 @disableInstrumentation();
229
230 switch (native_arch) {
231 .x86 => {
232 var user_desc: linux.user_desc = .{
233 .entry_number = __zig_elf_static_tls.gdt_entry_number,
234 .base_addr = addr,
235 .limit = 0xfffff,
236 .flags = .{
237 .seg_32bit = 1,
238 .contents = 0, // Data
239 .read_exec_only = 0,
240 .limit_in_pages = 1,
241 .seg_not_present = 0,
242 .useable = 1,
243 },
244 };
245 const rc = @call(.always_inline, linux.syscall1, .{ .set_thread_area, @intFromPtr(&user_desc) });
246 assert(rc == 0);
247
248 const gdt_entry_number = user_desc.entry_number;
249 // We have to keep track of our slot as it's also needed for clone()
250 __zig_elf_static_tls.gdt_entry_number = gdt_entry_number;
251 // Update the %gs selector
252 asm volatile ("movl %[gs_val], %%gs"
253 :
254 : [gs_val] "r" ((gdt_entry_number << 3) | 3),
255 );
256 },
257 .x86_64 => {
258 const rc = @call(.always_inline, linux.syscall2, .{ .arch_prctl, linux.ARCH.SET_FS, addr });
259 assert(rc == 0);
260 },
261 .aarch64, .aarch64_be => {
262 asm volatile (
263 \\ msr tpidr_el0, %[addr]
264 :
265 : [addr] "r" (addr),
266 );
267 },
268 .alpha => {
269 asm volatile (
270 \\ wruniq
271 :
272 : [addr] "{$16}" (addr),
273 );
274 },
275 .arc, .arceb => {
276 // We apparently need to both set r25 (TP) *and* inform the kernel...
277 asm volatile (
278 \\ mov r25, %[addr]
279 :
280 : [addr] "r" (addr),
281 );
282 const rc = @call(.always_inline, linux.syscall1, .{ .arc_settls, addr });
283 assert(rc == 0);
284 },
285 .arm, .armeb, .thumb, .thumbeb => {
286 const rc = @call(.always_inline, linux.syscall1, .{ .set_tls, addr });
287 assert(rc == 0);
288 },
289 .m68k => {
290 const rc = @call(.always_inline, linux.syscall1, .{ .set_thread_area, addr });
291 assert(rc == 0);
292 },
293 .hexagon => {
294 asm volatile (
295 \\ ugp = %[addr]
296 :
297 : [addr] "r" (addr),
298 );
299 },
300 .hppa => {
301 asm volatile (
302 \\ ble 0xe0(%%sr2, %%r0)
303 \\ nop
304 :
305 : [addr] "{r26}" (addr),
306 : .{ .r31 = true });
307 },
308 .loongarch32, .loongarch64 => {
309 asm volatile (
310 \\ move $tp, %[addr]
311 :
312 : [addr] "r" (addr),
313 );
314 },
315 .riscv32, .riscv64 => {
316 asm volatile (
317 \\ mv tp, %[addr]
318 :
319 : [addr] "r" (addr),
320 );
321 },
322 .csky, .mips, .mipsel, .mips64, .mips64el => {
323 const rc = @call(.always_inline, linux.syscall1, .{ .set_thread_area, addr });
324 assert(rc == 0);
325 },
326 .microblaze, .microblazeel => {
327 asm volatile (
328 \\ ori r21, %[addr], 0
329 :
330 : [addr] "r" (addr),
331 );
332 },
333 .or1k => {
334 asm volatile (
335 \\ l.ori r10, %[addr], 0
336 :
337 : [addr] "r" (addr),
338 );
339 },
340 .powerpc, .powerpcle => {
341 asm volatile (
342 \\ mr 2, %[addr]
343 :
344 : [addr] "r" (addr),
345 );
346 },
347 .powerpc64, .powerpc64le => {
348 asm volatile (
349 \\ mr 13, %[addr]
350 :
351 : [addr] "r" (addr),
352 );
353 },
354 .s390x => {
355 asm volatile (
356 \\ lgr %%r0, %[addr]
357 \\ sar %%a1, %%r0
358 \\ srlg %%r0, %%r0, 32
359 \\ sar %%a0, %%r0
360 :
361 : [addr] "r" (addr),
362 : .{ .r0 = true });
363 },
364 .sh, .sheb => {
365 asm volatile (
366 \\ ldc %[addr], gbr
367 :
368 : [addr] "r" (addr),
369 );
370 },
371 .sparc, .sparc64 => {
372 asm volatile (
373 \\ mov %[addr], %%g7
374 :
375 : [addr] "r" (addr),
376 );
377 },
378 .xtensa, .xtensaeb => {
379 asm volatile (
380 \\ wur %[addr], threadptr
381 :
382 : [addr] "a" (addr),
383 );
384 },
385 else => @compileError("Unsupported architecture"),
386 }
387}
388
389fn getThreadPointer() usize {
390 @setRuntimeSafety(false);
391 @disableInstrumentation();
392
393 return switch (native_arch) {
394 .aarch64, .aarch64_be => asm (
395 \\ mrs %[ret], tpidr_el0
396 : [ret] "=r" (-> usize),
397 ),
398 .alpha => asm (
399 \\ rduniq
400 : [ret] "={$0}" (-> usize),
401 ),
402 .arc, .arceb => asm (
403 \\ mov %[ret], r25
404 : [ret] "=r" (-> usize),
405 ),
406 .arm, .armeb, .thumb, .thumbeb => asm (
407 \\ mrc p15, 0, %[ret], c13, c0, 3
408 : [ret] "=r" (-> usize),
409 ),
410 .csky => asm (
411 \\ mov %[ret], r31
412 : [ret] "=r" (-> usize),
413 ),
414 .hexagon => asm (
415 \\ %[ret] = ugp
416 : [ret] "=r" (-> usize),
417 ),
418 .hppa => asm (
419 \\ mfctl %%cr27, %[ret]
420 : [ret] "=r" (-> usize),
421 ),
422 .loongarch32, .loongarch64 => asm (
423 \\ move %[ret], $tp
424 : [ret] "=r" (-> usize),
425 ),
426 .m68k => linux.syscall0(.get_thread_area),
427 .mips, .mipsel, .mips64, .mips64el => asm (
428 \\ rdhwr %[ret], $29
429 : [ret] "=r" (-> usize),
430 ),
431 .microblaze, .microblazeel => asm (
432 \\ ori %[ret], r21, 0
433 : [ret] "=r" (-> usize),
434 ),
435 .or1k => asm (
436 \\ l.ori %[ret], r10, 0
437 : [ret] "=r" (-> usize),
438 ),
439 .riscv32, .riscv64 => asm (
440 \\ mv %[ret], tp
441 : [ret] "=r" (-> usize),
442 ),
443 .powerpc, .powerpcle => asm (
444 \\ mr %[ret], 2
445 : [ret] "=r" (-> usize),
446 ),
447 .powerpc64, .powerpc64le => asm (
448 \\ mr %[ret], 13
449 : [ret] "=r" (-> usize),
450 ),
451 .s390x => asm (
452 \\ ear %[ret], %%a0
453 \\ sllg %[ret], %[ret], 32
454 \\ ear %[ret], %%a1
455 : [ret] "=r" (-> usize),
456 ),
457 .sh, .sheb => asm (
458 \\ stc gbr, %[ret]
459 : [ret] "=r" (-> usize),
460 ),
461 .sparc, .sparc64 => asm (
462 \\ mov %%g7, %[ret]
463 : [ret] "=r" (-> usize),
464 ),
465 .x86 => asm (
466 \\ movl %%gs:0, %[ret]
467 : [ret] "=r" (-> usize),
468 ),
469 .x86_64 => switch (@sizeOf(usize)) {
470 8 => asm (
471 \\ movq %%fs:0, %[ret]
472 : [ret] "=r" (-> usize),
473 ),
474 // On x32, usize is 32 bits.
475 4 => asm (
476 \\ movl %%fs:0, %[ret]
477 : [ret] "=r" (-> usize),
478 ),
479 else => comptime unreachable,
480 },
481 .xtensa, .xtensaeb => asm (
482 \\ rur %[ret], threadptr
483 : [ret] "=r" (-> usize),
484 ),
485 else => @compileError("Unsupported architecture"),
486 };
487}
488
489fn computeAreaDesc(phdrs: []elf.ElfN.Phdr) void {
490 @setRuntimeSafety(false);
491 @disableInstrumentation();
492
493 var tls_phdr: ?*elf.ElfN.Phdr = null;
494 var img_base: usize = 0;
495
496 for (phdrs) |*phdr| {
497 switch (phdr.type) {
498 .PHDR => img_base = @intFromPtr(phdrs.ptr) - phdr.vaddr,
499 .TLS => tls_phdr = phdr,
500 else => {},
501 }
502 }
503
504 var align_factor: usize = undefined;
505 var block_init: []const u8 = undefined;
506 var block_size: usize = undefined;
507
508 if (tls_phdr) |phdr| {
509 align_factor = @max(phdr.@"align", 1);
510
511 // The effective size in memory is represented by `memsz`; the length of the data stored
512 // in the `PT.TLS` segment is `filesz` and may be less than the former.
513 block_init = @as([*]u8, @ptrFromInt(img_base + phdr.vaddr))[0..phdr.filesz];
514 block_size = phdr.memsz;
515 } else {
516 align_factor = @alignOf(usize);
517
518 block_init = &[_]u8{};
519 block_size = 0;
520 }
521
522 // Offsets into the allocated TLS area.
523 var dtv_offset: usize = undefined;
524 var abi_tcb_offset: usize = undefined;
525 var block_offset: usize = undefined;
526
527 // Compute the total size of the ABI-specific data plus our own `ZigTcb` structure. All the
528 // offsets calculated here assume a well-aligned base address.
529 const area_size = switch (current_variant) {
530 .I_original => blk: {
531 var l: usize = 0;
532 dtv_offset = l;
533 l += @sizeOf(Dtv);
534 // Add some padding here so that the TP (`abi_tcb_offset`) is aligned to `align_factor`
535 // and the `ZigTcb` structure can be found by simply subtracting `@sizeOf(ZigTcb)` from
536 // the TP.
537 const delta = (l + @sizeOf(ZigTcb)) & (align_factor - 1);
538 if (delta > 0)
539 l += align_factor - delta;
540 l += @sizeOf(ZigTcb);
541 abi_tcb_offset = l;
542 l += alignForward(@sizeOf(AbiTcb), align_factor);
543 block_offset = l;
544 l += block_size;
545 break :blk l;
546 },
547 .I_modified => blk: {
548 var l: usize = 0;
549 dtv_offset = l;
550 l += @sizeOf(Dtv);
551 // In this variant, the TLS blocks must begin immediately after the end of the ABI TCB,
552 // with the TP pointing to the beginning of the TLS blocks. Add padding so that the TP
553 // (`abi_tcb_offset`) is aligned to `align_factor` and the `ZigTcb` structure can be
554 // found by subtracting `@sizeOf(AbiTcb) + @sizeOf(ZigTcb)` from the TP.
555 const delta = (l + @sizeOf(ZigTcb) + @sizeOf(AbiTcb)) & (align_factor - 1);
556 if (delta > 0)
557 l += align_factor - delta;
558 l += @sizeOf(ZigTcb);
559 abi_tcb_offset = l;
560 l += @sizeOf(AbiTcb);
561 block_offset = l;
562 l += block_size;
563 break :blk l;
564 },
565 .II => blk: {
566 var l: usize = 0;
567 block_offset = l;
568 l += alignForward(block_size, align_factor);
569 // The TP is aligned to `align_factor`.
570 abi_tcb_offset = l;
571 l += @sizeOf(AbiTcb);
572 // The `ZigTcb` structure is right after the `AbiTcb` with no padding in between so it
573 // can be easily found.
574 l += @sizeOf(ZigTcb);
575 // It doesn't really matter where we put the DTV, so give it natural alignment.
576 l = alignForward(l, @alignOf(Dtv));
577 dtv_offset = l;
578 l += @sizeOf(Dtv);
579 break :blk l;
580 },
581 };
582
583 __zig_elf_static_tls = .{
584 .size = area_size,
585 .alignment = align_factor,
586 .gdt_entry_number = @as(usize, @bitCast(@as(isize, -1))),
587 };
588
589 area_info = .{
590 .dtv = .{
591 .offset = dtv_offset,
592 },
593
594 .abi_tcb = .{
595 .offset = abi_tcb_offset,
596 },
597
598 .block = .{
599 .init = block_init,
600 .offset = block_offset,
601 .size = block_size,
602 },
603 };
604}
605
606/// Inline because TLS is not set up yet.
607inline fn alignForward(addr: usize, alignment: usize) usize {
608 return alignBackward(addr + (alignment - 1), alignment);
609}
610
611/// Inline because TLS is not set up yet.
612inline fn alignBackward(addr: usize, alignment: usize) usize {
613 return addr & ~(alignment - 1);
614}
615
616/// Inline because TLS is not set up yet.
617inline fn alignPtrCast(comptime T: type, ptr: [*]u8) *T {
618 return @ptrCast(@alignCast(ptr));
619}
620
621/// Initializes all the fields of the static TLS area and returns the computed architecture-specific
622/// value of the TP register.
623fn __zig_elf_static_tls_fill(area: [*]u8) callconv(.c) usize {
624 @setRuntimeSafety(false);
625 @disableInstrumentation();
626
627 const area_slice = area[0..__zig_elf_static_tls.size];
628
629 // Clear the area we're going to use, just to be safe.
630 @memset(area_slice, 0);
631
632 // Prepare the ABI TCB.
633 const abi_tcb = alignPtrCast(AbiTcb, area + area_info.abi_tcb.offset);
634 switch (current_variant) {
635 .I_original, .I_modified => abi_tcb.dtv = @intFromPtr(area + area_info.dtv.offset),
636 .II => abi_tcb.self = abi_tcb,
637 }
638
639 // Prepare the DTV.
640 const dtv = alignPtrCast(Dtv, area + area_info.dtv.offset);
641 dtv.len = 1;
642 dtv.tls_block = area + current_dtv_offset + area_info.block.offset;
643
644 // Copy the initial data.
645 @memcpy(area_slice[area_info.block.offset..][0..area_info.block.init.len], area_info.block.init);
646
647 // Return the corrected value (if needed) for the TP register. Overflow here is not a problem;
648 // the pointer arithmetic involving the TP is done with wrapping semantics.
649 return @intFromPtr(area) +% switch (current_variant) {
650 .I_original, .II => area_info.abi_tcb.offset,
651 .I_modified => area_info.block.offset +% current_tp_offset,
652 };
653}
654
655/// The main motivation for the size chosen here is to be larger than total
656/// amount of thread-local variables for most programs. Putting this allocation
657/// in the ELF like this is equivalent to moving the `mmap` call below into the
658/// kernel, avoiding syscall overhead.
659var main_thread_area_buffer: [0x1000]u8 align(page_size_min) = undefined;
660
661/// Computes the layout of the static TLS area, allocates the area, initializes all of its fields,
662/// and assigns the architecture-specific value to the TP register.
663fn __zig_elf_static_tls_init(phdrs: [*]elf.ElfN.Phdr, phnum: usize) callconv(.c) void {
664 @setRuntimeSafety(false);
665 @disableInstrumentation();
666
667 computeAreaDesc(phdrs[0..phnum]);
668
669 const area: [*]u8 = blk: {
670 // Fast path for the common case where the TLS data is really small, avoid an allocation and
671 // use our local buffer.
672 if (__zig_elf_static_tls.alignment <= page_size_min and __zig_elf_static_tls.size <= main_thread_area_buffer.len) {
673 break :blk &main_thread_area_buffer;
674 }
675
676 const begin_addr = mmap(__zig_elf_static_tls.size + __zig_elf_static_tls.alignment - 1);
677 if (@call(.always_inline, linux.errno, .{begin_addr}) != .SUCCESS) @trap();
678
679 const area_ptr: [*]align(page_size_min) u8 = @ptrFromInt(begin_addr);
680
681 // Make sure the slice is correctly aligned.
682 const begin_aligned_addr = alignForward(begin_addr, __zig_elf_static_tls.alignment);
683 const start = begin_aligned_addr - begin_addr;
684 break :blk area_ptr + start;
685 };
686
687 const tp_value = __zig_elf_static_tls_fill(area);
688 setThreadPointer(tp_value);
689}
690
691inline fn mmap(length: usize) usize {
692 const prot: linux.PROT = .{ .READ = true, .WRITE = true };
693 const flags: linux.MAP = .{ .TYPE = .PRIVATE, .ANONYMOUS = true };
694
695 if (@hasField(linux.SYS, "mmap2")) {
696 return @call(.always_inline, linux.syscall6, .{
697 .mmap2,
698 0,
699 length,
700 @as(u32, @bitCast(prot)),
701 @as(u32, @bitCast(flags)),
702 @as(usize, @bitCast(@as(isize, -1))),
703 0,
704 });
705 } else {
706 // The s390x mmap() syscall existed before Linux supported syscalls with 5+ parameters, so
707 // it takes a single pointer to an array of arguments instead.
708 return if (native_arch == .s390x) @call(.always_inline, linux.syscall1, .{
709 .mmap,
710 @intFromPtr(&[_]usize{
711 0,
712 length,
713 @as(u32, @bitCast(prot)),
714 @as(u32, @bitCast(flags)),
715 @as(usize, @bitCast(@as(isize, -1))),
716 0,
717 }),
718 }) else @call(.always_inline, linux.syscall6, .{
719 .mmap,
720 0,
721 length,
722 @as(u32, @bitCast(prot)),
723 @as(u32, @bitCast(flags)),
724 @as(usize, @bitCast(@as(isize, -1))),
725 0,
726 });
727 }
728}
729
730fn __m68k_read_tp() callconv(.c) ?*anyopaque {
731 comptime assert(native_arch == .m68k);
732
733 return @ptrFromInt(getThreadPointer());
734}
735
736const TlsIndex = switch (native_arch) {
737 .x86_64 => extern struct { module: u64, offset: u64 }, // Even for x32...
738 else => extern struct { module: usize, offset: usize }, // ...but not MIPS N32!
739};
740
741fn __tls_get_addr(ti: *const TlsIndex) callconv(.c) *anyopaque {
742 comptime assert(native_arch != .s390x);
743
744 assert(ti.module == 1); // The executable's module ID is always 1
745 const tp = getThreadPointer();
746 const block = switch (current_variant) {
747 .I_original => tp -% area_info.abi_tcb.offset +% area_info.block.offset,
748 .I_modified => tp -% current_tp_offset,
749 // The `.I_original` approach would also work for `.II`, but there is an
750 // alternative strategy which is one less operation:
751 .II => tp -% area_info.abi_tcb.offset,
752 };
753 return @ptrFromInt(block +% current_dtv_offset +% @as(usize, @intCast(ti.offset)));
754}
755
756fn ___tls_get_addr() callconv(.naked) *anyopaque {
757 comptime assert(native_arch == .x86);
758
759 // We receive the `TlsIndex` pointer in eax.
760 comptime assert(@sizeOf(AbiTcb) + @sizeOf(ZigTcb) + @offsetOf(Dtv, "tls_block") == 12);
761 asm volatile (
762 \\ mov 4(%%eax), %%eax
763 \\ add %%gs:12, %%eax
764 \\ ret
765 );
766}
767
768fn __tls_get_offset() callconv(.naked) noreturn {
769 comptime assert(native_arch == .s390x);
770
771 // We receive the module's GOT pointer in r12 and the GOT offset in r2.
772 asm volatile (
773 \\ la %%r1, 0(%%r12, %%r2)
774 \\ lg %%r2, 8(%%r1)
775 \\ lgrl %%r0, %[abi_tcb_offset]
776 \\ sgr %%r2, %%r0
777 \\ br %%r14
778 :
779 : [abi_tcb_offset] "s" (&area_info.abi_tcb.offset),
780 );
781}