From 736d37ad161b255cf29fb3050026ad4852639b11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20R=C3=B8nne=20Petersen?= Date: Tue, 18 Aug 2026 22:43:52 +0200 Subject: [PATCH] move std.os.linux.tls to compiler_rt as __zig_elf_static_tls_* As with __zig_elf_auxv, we need a canonical location for __tls_get_addr (or __tls_get_offset on s390x), and compiler-rt is the obvious place. But that also means that moving all of std.os.linux.tls there just makes sense. I decided not to namespace this under linux because it's expected that most of this code will apply to the BSDs too. The ABI that std now expects is: extern fn __zig_elf_static_tls_init(phdrs: [*]elf.ElfN.Phdr, phnum: usize) void; extern fn __zig_elf_static_tls_fill(area: [*]u8) usize; extern var __zig_elf_static_tls: extern struct { size: usize, alignment: usize, gdt_entry_number: usize, }; Users can override all three weak symbols with strong symbols if they want to provide their own ELF TLS implementation, which may be useful to do when doing manual dynamic loading from a static executable. Note that __tls_get_addr (or __tls_get_offset on s390x) must also be provided, per the ELF TLS specification. --- lib/compiler_rt/elf.zig | 1 + lib/{std/os/linux => compiler_rt/elf}/tls.zig | 173 +++++++++--------- lib/std/Thread.zig | 18 +- lib/std/os/linux.zig | 1 - lib/std/start.zig | 13 +- 5 files changed, 114 insertions(+), 92 deletions(-) rename lib/{std/os/linux => compiler_rt/elf}/tls.zig (85%) diff --git a/lib/compiler_rt/elf.zig b/lib/compiler_rt/elf.zig index e68d5ad7c459cccfc3fefe34739507349a5ed350..11f934d30edde31a6c5f4ec01092636fa602620c 100644 --- a/lib/compiler_rt/elf.zig +++ b/lib/compiler_rt/elf.zig @@ -1,3 +1,4 @@ comptime { _ = @import("elf/auxv.zig"); + _ = @import("elf/tls.zig"); } diff --git a/lib/std/os/linux/tls.zig b/lib/compiler_rt/elf/tls.zig similarity index 85% rename from lib/std/os/linux/tls.zig rename to lib/compiler_rt/elf/tls.zig index aed76fc0cb2ee83318e5c0e2d72b0016cf44eb91..b4231387593edb88fdaff50d5d526b5eec5e3eda 100644 --- a/lib/std/os/linux/tls.zig +++ b/lib/compiler_rt/elf/tls.zig @@ -9,16 +9,36 @@ //! //! [1] https://www.akkadia.org/drepper/tls.pdf +const builtin = @import("builtin"); const std = @import("std"); -const mem = std.mem; + +const compiler_rt = @import("../../compiler_rt.zig"); +const symbol = compiler_rt.symbol; + const elf = std.elf; -const math = std.math; const assert = std.debug.assert; -const builtin = @import("builtin"); const native_arch = builtin.cpu.arch; const linux = std.os.linux; const page_size_min = std.heap.page_size_min; +comptime { + // FreeBSD, NetBSD, and OpenBSD are expected to also use this in the future. + if (!builtin.link_libc and builtin.target.os.tag == .linux) { + symbol(&__zig_elf_static_tls, "__zig_elf_static_tls"); + symbol(&__zig_elf_static_tls_init, "__zig_elf_static_tls_init"); + symbol(&__zig_elf_static_tls_fill, "__zig_elf_static_tls_fill"); + + // To make static executables without libc work, it is our job to provide the TLS accessor + // function for the GD and LD models. This function is unlikely to actually be used, since + // the linker should be able to relax every TLS access to the LE model and therefore + // eliminate all calls to this function, but that isn't guaranteed. + if (native_arch == .s390x) + symbol(&__tls_get_offset, "__tls_get_offset") + else + symbol(&__tls_get_addr, "__tls_get_addr"); + } +} + /// Represents an ELF TLS variant. /// /// In all variants, the TP and the TLS blocks must be aligned to the `p_align` value in the @@ -186,11 +206,17 @@ const Dtv = extern struct { }; /// Describes a process's TLS area. The area encompasses the DTV, both TCBs, and the TLS block, with -/// the exact layout of these being dependent primarily on `current_variant`. -const AreaDesc = struct { +/// the exact layout of these being dependent primarily on `current_variant`. See also `area_info`. +/// +/// The layout of this variable is ABI between compiler-rt and std. +var __zig_elf_static_tls: extern struct { size: usize, alignment: usize, + /// Only used on the 32-bit x86 architecture (not x86_64, nor x32). + gdt_entry_number: usize, +} = undefined; +var area_info: struct { dtv: struct { /// Offset into the TLS area. offset: usize, @@ -210,21 +236,16 @@ const AreaDesc = struct { /// This is the effective size of the TLS block, which may be greater than `init.len`. size: usize, }, +} = undefined; - /// Only used on the 32-bit x86 architecture (not x86_64, nor x32). - gdt_entry_number: usize, -}; - -pub var area_desc: AreaDesc = undefined; - -pub fn setThreadPointer(addr: usize) void { +fn setThreadPointer(addr: usize) void { @setRuntimeSafety(false); @disableInstrumentation(); switch (native_arch) { .x86 => { var user_desc: linux.user_desc = .{ - .entry_number = area_desc.gdt_entry_number, + .entry_number = __zig_elf_static_tls.gdt_entry_number, .base_addr = addr, .limit = 0xfffff, .flags = .{ @@ -241,7 +262,7 @@ pub fn setThreadPointer(addr: usize) void { const gdt_entry_number = user_desc.entry_number; // We have to keep track of our slot as it's also needed for clone() - area_desc.gdt_entry_number = gdt_entry_number; + __zig_elf_static_tls.gdt_entry_number = gdt_entry_number; // Update the %gs selector asm volatile ("movl %[gs_val], %%gs" : @@ -380,7 +401,7 @@ pub fn setThreadPointer(addr: usize) void { } } -pub fn getThreadPointer() usize { +fn getThreadPointer() usize { @setRuntimeSafety(false); @disableInstrumentation(); @@ -574,10 +595,13 @@ fn computeAreaDesc(phdrs: []elf.ElfN.Phdr) void { }, }; - area_desc = .{ + __zig_elf_static_tls = .{ .size = area_size, .alignment = align_factor, + .gdt_entry_number = @as(usize, @bitCast(@as(isize, -1))), + }; + area_info = .{ .dtv = .{ .offset = dtv_offset, }, @@ -591,8 +615,6 @@ fn computeAreaDesc(phdrs: []elf.ElfN.Phdr) void { .offset = block_offset, .size = block_size, }, - - .gdt_entry_number = @as(usize, @bitCast(@as(isize, -1))), }; } @@ -613,33 +635,35 @@ inline fn alignPtrCast(comptime T: type, ptr: [*]u8) *T { /// Initializes all the fields of the static TLS area and returns the computed architecture-specific /// value of the TP register. -pub fn prepareArea(area: []u8) usize { +fn __zig_elf_static_tls_fill(area: [*]u8) callconv(.c) usize { @setRuntimeSafety(false); @disableInstrumentation(); + const area_slice = area[0..__zig_elf_static_tls.size]; + // Clear the area we're going to use, just to be safe. - @memset(area, 0); + @memset(area_slice, 0); // Prepare the ABI TCB. - const abi_tcb = alignPtrCast(AbiTcb, area.ptr + area_desc.abi_tcb.offset); + const abi_tcb = alignPtrCast(AbiTcb, area + area_info.abi_tcb.offset); switch (current_variant) { - .I_original, .I_modified => abi_tcb.dtv = @intFromPtr(area.ptr + area_desc.dtv.offset), + .I_original, .I_modified => abi_tcb.dtv = @intFromPtr(area + area_info.dtv.offset), .II => abi_tcb.self = abi_tcb, } // Prepare the DTV. - const dtv = alignPtrCast(Dtv, area.ptr + area_desc.dtv.offset); + const dtv = alignPtrCast(Dtv, area + area_info.dtv.offset); dtv.len = 1; - dtv.tls_block = area.ptr + current_dtv_offset + area_desc.block.offset; + dtv.tls_block = area + current_dtv_offset + area_info.block.offset; // Copy the initial data. - @memcpy(area[area_desc.block.offset..][0..area_desc.block.init.len], area_desc.block.init); + @memcpy(area_slice[area_info.block.offset..][0..area_info.block.init.len], area_info.block.init); // Return the corrected value (if needed) for the TP register. Overflow here is not a problem; // the pointer arithmetic involving the TP is done with wrapping semantics. - return @intFromPtr(area.ptr) +% switch (current_variant) { - .I_original, .II => area_desc.abi_tcb.offset, - .I_modified => area_desc.block.offset +% current_tp_offset, + return @intFromPtr(area) +% switch (current_variant) { + .I_original, .II => area_info.abi_tcb.offset, + .I_modified => area_info.block.offset +% current_tp_offset, }; } @@ -651,35 +675,35 @@ var main_thread_area_buffer: [0x1000]u8 align(page_size_min) = undefined; /// Computes the layout of the static TLS area, allocates the area, initializes all of its fields, /// and assigns the architecture-specific value to the TP register. -pub fn initStatic(phdrs: []elf.ElfN.Phdr) void { +fn __zig_elf_static_tls_init(phdrs: [*]elf.ElfN.Phdr, phnum: usize) callconv(.c) void { @setRuntimeSafety(false); @disableInstrumentation(); - computeAreaDesc(phdrs); + computeAreaDesc(phdrs[0..phnum]); - const area = blk: { + const area: [*]u8 = blk: { // Fast path for the common case where the TLS data is really small, avoid an allocation and // use our local buffer. - if (area_desc.alignment <= page_size_min and area_desc.size <= main_thread_area_buffer.len) { - break :blk main_thread_area_buffer[0..area_desc.size]; + if (__zig_elf_static_tls.alignment <= page_size_min and __zig_elf_static_tls.size <= main_thread_area_buffer.len) { + break :blk &main_thread_area_buffer; } - const begin_addr = mmap_tls(area_desc.size + area_desc.alignment - 1); + const begin_addr = mmap(__zig_elf_static_tls.size + __zig_elf_static_tls.alignment - 1); if (@call(.always_inline, linux.errno, .{begin_addr}) != .SUCCESS) @trap(); const area_ptr: [*]align(page_size_min) u8 = @ptrFromInt(begin_addr); // Make sure the slice is correctly aligned. - const begin_aligned_addr = alignForward(begin_addr, area_desc.alignment); + const begin_aligned_addr = alignForward(begin_addr, __zig_elf_static_tls.alignment); const start = begin_aligned_addr - begin_addr; - break :blk area_ptr[start..][0..area_desc.size]; + break :blk area_ptr + start; }; - const tp_value = prepareArea(area); + const tp_value = __zig_elf_static_tls_fill(area); setThreadPointer(tp_value); } -inline fn mmap_tls(length: usize) usize { +inline fn mmap(length: usize) usize { const prot: linux.PROT = .{ .READ = true, .WRITE = true }; const flags: linux.MAP = .{ .TYPE = .PRIVATE, .ANONYMOUS = true }; @@ -718,52 +742,37 @@ inline fn mmap_tls(length: usize) usize { } } -comptime { - assert(!builtin.link_libc); // otherwise libc should control TLS +const TlsIndex = switch (native_arch) { + .x86_64 => extern struct { module: u64, offset: u64 }, // Even for x32... + else => extern struct { module: usize, offset: usize }, // ...but not MIPS N32! +}; - if (builtin.output_mode == .Exe and builtin.link_mode == .static) { - // This is a static executable without libc, so it is our job to provide the TLS accessor - // function for the GD and LD models. This function is unlikely to actually be used, since - // the linker should be able to relax every TLS access to the LE model and therefore - // eliminate all calls to this function, but that isn't guaranteed. - const Fns = struct { - const TlsIndex = switch (native_arch) { - .x86_64 => extern struct { module: u64, offset: u64 }, // Even for x32... - else => extern struct { module: usize, offset: usize }, // ...but not MIPS N32! - }; - fn __tls_get_addr(ti: *const TlsIndex) callconv(.c) *anyopaque { - comptime assert(native_arch != .s390x); +fn __tls_get_addr(ti: *const TlsIndex) callconv(.c) *anyopaque { + comptime assert(native_arch != .s390x); - assert(ti.module == 1); // The executable's module ID is always 1 - const tp = getThreadPointer(); - const block: [*]u8 = switch (current_variant) { - .I_original => @ptrFromInt(tp -% area_desc.abi_tcb.offset +% area_desc.block.offset), - .I_modified => @ptrFromInt(tp -% current_tp_offset), - // The `.I_original` approach would also work for `.II`, but there is an - // alternative strategy which is one less operation: - .II => @ptrFromInt(tp -% area_desc.block.size), - }; - return block[@intCast(ti.offset)..]; - } - fn __tls_get_offset() callconv(.naked) noreturn { - comptime assert(native_arch == .s390x); + assert(ti.module == 1); // The executable's module ID is always 1 + const tp = getThreadPointer(); + const block: [*]u8 = switch (current_variant) { + .I_original => @ptrFromInt(tp -% area_info.abi_tcb.offset +% area_info.block.offset), + .I_modified => @ptrFromInt(tp -% current_tp_offset), + // The `.I_original` approach would also work for `.II`, but there is an + // alternative strategy which is one less operation: + .II => @ptrFromInt(tp -% area_info.block.size), + }; + return block[@intCast(ti.offset)..]; +} - // We receive the module's GOT pointer in r12 and the GOT offset in r2. - asm volatile ( - \\ la %%r1, 0(%%r12, %%r2) - \\ lg %%r2, 8(%%r1) - \\ lgrl %%r0, %[block_size] - \\ sgr %%r2, %%r0 - \\ br %%r14 - : - : [block_size] "s" (&area_desc.block.size), - ); - } - }; +fn __tls_get_offset() callconv(.naked) noreturn { + comptime assert(native_arch == .s390x); - if (native_arch == .s390x) - @export(&Fns.__tls_get_offset, .{ .name = "__tls_get_offset" }) - else - @export(&Fns.__tls_get_addr, .{ .name = "__tls_get_addr" }); - } + // We receive the module's GOT pointer in r12 and the GOT offset in r2. + asm volatile ( + \\ la %%r1, 0(%%r12, %%r2) + \\ lg %%r2, 8(%%r1) + \\ lgrl %%r0, %[block_size] + \\ sgr %%r2, %%r0 + \\ br %%r14 + : + : [block_size] "s" (&area_info.block.size), + ); } diff --git a/lib/std/Thread.zig b/lib/std/Thread.zig index 66a3662eedc92bae67d06cb9fac9625e8bde87ff..f093e3899267f2f5afb93432fb32f3c8fbd4ce20 100644 --- a/lib/std/Thread.zig +++ b/lib/std/Thread.zig @@ -1434,6 +1434,16 @@ const LinuxThreadImpl = struct { } }; + /// Provided by compiler-rt and populated by `std.start`. + extern var __zig_elf_static_tls: extern struct { + size: usize, + alignment: usize, + gdt_entry_number: usize, + }; + + /// Provided by compiler-rt. + extern fn __zig_elf_static_tls_fill(area: [*]u8) usize; + fn spawn(config: SpawnConfig, comptime f: anytype, args: anytype) !Impl { const page_size = std.heap.pageSize(); const Args = @TypeOf(args); @@ -1465,9 +1475,9 @@ const LinuxThreadImpl = struct { bytes = std.mem.alignForward(usize, bytes, page_size); stack_offset = bytes; - bytes = std.mem.alignForward(usize, bytes, linux.tls.area_desc.alignment); + bytes = std.mem.alignForward(usize, bytes, __zig_elf_static_tls.alignment); tls_offset = bytes; - bytes += linux.tls.area_desc.size; + bytes += __zig_elf_static_tls.size; bytes = std.mem.alignForward(usize, bytes, @alignOf(Instance)); instance_offset = bytes; @@ -1509,12 +1519,12 @@ const LinuxThreadImpl = struct { } // Prepare the TLS segment and prepare a user_desc struct when needed on x86 - var tls_ptr = linux.tls.prepareArea(mapped[tls_offset..][0..linux.tls.area_desc.size]); + var tls_ptr = __zig_elf_static_tls_fill(mapped.ptr + tls_offset); var user_desc: if (target.cpu.arch == .x86) linux.user_desc else void = undefined; if (target.cpu.arch == .x86) { defer tls_ptr = @intFromPtr(&user_desc); user_desc = .{ - .entry_number = linux.tls.area_desc.gdt_entry_number, + .entry_number = __zig_elf_static_tls.gdt_entry_number, .base_addr = tls_ptr, .limit = 0xfffff, .flags = .{ diff --git a/lib/std/os/linux.zig b/lib/std/os/linux.zig index e75a0d822467481b245997d4818989a9c7654b50..7c18ef62011d3cd07a6612d78b652446ce602344 100644 --- a/lib/std/os/linux.zig +++ b/lib/std/os/linux.zig @@ -123,7 +123,6 @@ pub const gid_t = u32; pub const clock_t = isize; pub const time_t = arch_bits.time_t; -pub const tls = @import("linux/tls.zig"); pub const BPF = @import("linux/bpf.zig"); pub const IOCTL = @import("linux/ioctl.zig"); pub const SECCOMP = @import("linux/seccomp.zig"); diff --git a/lib/std/start.zig b/lib/std/start.zig index f8c85adeded05726074c5ed0516b336892d34154..d77c093d23792f1422d4cfdc2e8d97a2af181b7d 100644 --- a/lib/std/start.zig +++ b/lib/std/start.zig @@ -599,6 +599,9 @@ fn wWinMain(hInstance: *anyopaque, hPrevInstance: ?*anyopaque, pCmdLine: [*:0]u1 /// Provided by compiler-rt. extern var __zig_elf_auxv: [*]std.elf.Auxv; +/// Provided by compiler-rt. +extern fn __zig_elf_static_tls_init(phdrs: [*]elf.ElfN.Phdr, phnum: usize) void; + fn posixCallMainAndExit(argc_argv_ptr: [*]usize) callconv(.c) noreturn { // We're not ready to panic until thread local storage is initialized. @setRuntimeSafety(false); @@ -616,7 +619,7 @@ fn posixCallMainAndExit(argc_argv_ptr: [*]usize) callconv(.c) noreturn { const auxv: [*]elf.Auxv = @ptrCast(@alignCast(envp.ptr + envp_count + 1)); var at_hwcap: usize = 0; - const phdrs = init: { + const phdrs: [*]elf.ElfN.Phdr, const phnum: usize = init: { var i: usize = 0; var at_phdr: usize = 0; var at_phnum: usize = 0; @@ -628,14 +631,14 @@ fn posixCallMainAndExit(argc_argv_ptr: [*]usize) callconv(.c) noreturn { else => continue, } } - break :init @as([*]elf.ElfN.Phdr, @ptrFromInt(at_phdr))[0..at_phnum]; + break :init .{ @ptrFromInt(at_phdr), at_phnum }; }; // Apply the initial relocations as early as possible in the startup process. We cannot // make calls yet on some architectures (e.g. MIPS) *because* they haven't been applied yet, // so this must be fully inlined. if (builtin.link_mode == .static and builtin.position_independent_executable) { - @call(.always_inline, std.pie.relocate, .{phdrs}); + @call(.always_inline, std.pie.relocate, .{phdrs[0..phnum]}); } if (native_os == .linux) { @@ -657,14 +660,14 @@ fn posixCallMainAndExit(argc_argv_ptr: [*]usize) callconv(.c) noreturn { } // Initialize the TLS area. - std.os.linux.tls.initStatic(phdrs); + __zig_elf_static_tls_init(phdrs, phnum); } // The way Linux executables represent stack size is via the PT.GNU_STACK // program header. However the kernel does not recognize it; it always gives 8 MiB. // Here we look for the stack size in our program headers and use setrlimit // to ask for more stack space. - expandStackSize(phdrs); + expandStackSize(phdrs[0..phnum]); } const opt_init_array_start = @extern([*]const *const fn () callconv(.c) void, .{ -- 2.54.0