diff --git a/CMakeLists.txt b/CMakeLists.txt index 9e74b6f17b313a6c4bef52610f2d49b33ecda978..837a5bcd1b8de6225797bdb92031b306fdd7bf92 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -690,7 +690,7 @@ add_custom_command( set(BUILD_COMPILER_RT_ARGS "${PROJECT_SOURCE_DIR}/lib" - build-obj -ofmt=c -OReleaseSmall + build-obj -ofmt=c -lc -OReleaseSmall --name compiler_rt -femit-bin="${ZIG_COMPILER_RT_C_SOURCE}" -target "${ZIG_HOST_TARGET_TRIPLE}" diff --git a/bootstrap.c b/bootstrap.c index f0cdc8e0a2e2cb418ea0029cbb0a4aaae6c258be..4e41d2ebdc8a0839182901b33e6214ad3f6ae3f4 100644 --- a/bootstrap.c +++ b/bootstrap.c @@ -212,7 +212,7 @@ int main(int argc, char **argv) { { const char *child_argv[] = { "./zig1", "lib", "build-obj", - "-ofmt=c", "-OReleaseSmall", + "-ofmt=c", "-lc", "-OReleaseSmall", "--name", "compiler_rt", "-femit-bin=compiler_rt.c", "-target", host_triple, "-Mroot=lib/compiler_rt.zig", diff --git a/lib/compiler_rt.zig b/lib/compiler_rt.zig index 5660847ce84fbbedc64d4e980202e3865f8df592..e4d4824a38654c26da6a763f75e8df8cfd408681 100644 --- a/lib/compiler_rt.zig +++ b/lib/compiler_rt.zig @@ -164,6 +164,8 @@ comptime { symbol(&strlen, "strlen"); } + _ = @import("compiler_rt/elf.zig"); + // Temporarily used for uefi until https://github.com/ziglang/zig/issues/21630 is addressed. if (!builtin.link_libc and (builtin.os.tag == .windows or builtin.os.tag == .uefi) and (builtin.abi == .none or builtin.abi == .msvc)) { symbol(&_fltused, "_fltused"); diff --git a/lib/compiler_rt/elf.zig b/lib/compiler_rt/elf.zig new file mode 100644 index 0000000000000000000000000000000000000000..713fe4845c9f77d1dcf7903a5b23b3a1484e7ed0 --- /dev/null +++ b/lib/compiler_rt/elf.zig @@ -0,0 +1,23 @@ +const builtin = @import("builtin"); +const std = @import("std"); + +const compiler_rt = @import("../compiler_rt.zig"); +const symbol = compiler_rt.symbol; + +/// More operating systems are expected to be added here: +/// +/// * FreeBSD: https://codeberg.org/ziglang/zig/issues/30981 +/// * NetBSD: https://codeberg.org/ziglang/zig/issues/30980 +/// * OpenBSD: https://codeberg.org/ziglang/zig/issues/30982 +const use_zig_start = switch (builtin.target.os.tag) { + .linux, + => !builtin.link_libc, + else => false, +}; + +comptime { + if (use_zig_start) { + _ = @import("elf/auxv.zig"); + _ = @import("elf/tls.zig"); + } +} diff --git a/lib/compiler_rt/elf/auxv.zig b/lib/compiler_rt/elf/auxv.zig new file mode 100644 index 0000000000000000000000000000000000000000..e8452202a8b085144a08a925153271b94b39ad7f --- /dev/null +++ b/lib/compiler_rt/elf/auxv.zig @@ -0,0 +1,18 @@ +const builtin = @import("builtin"); +const std = @import("std"); + +const compiler_rt = @import("../../compiler_rt.zig"); +const symbol = compiler_rt.symbol; + +comptime { + symbol(@ptrCast(&__zig_elf_auxv), "__zig_elf_auxv"); +} + +/// Populated by `std.start` when libc is not linked on targets that provide an ELF auxiliary vector +/// to the process on startup. Provided by compiler-rt so that there is no confusion about where +/// this state lives regardless of how the final artifact (executable or shared library) is +/// compiled. +/// +/// It is expected that this will gradually be replaced with better mechanisms as operating systems +/// improve; see https://codeberg.org/ziglang/zig/issues/31290. +var __zig_elf_auxv: [*]std.elf.Auxv = undefined; diff --git a/lib/compiler_rt/elf/tls.zig b/lib/compiler_rt/elf/tls.zig new file mode 100644 index 0000000000000000000000000000000000000000..75dda8d300ec7e079979528af638fdc3e8e90347 --- /dev/null +++ b/lib/compiler_rt/elf/tls.zig @@ -0,0 +1,774 @@ +//! This file implements the two TLS variants [1] used by ELF-based systems. Note that, in reality, +//! Variant I has two sub-variants. +//! +//! It is important to understand that the term TCB (Thread Control Block) is overloaded here. +//! Official ABI documentation uses it simply to mean the ABI TCB, i.e. a small area of ABI-defined +//! data, usually one or two words (see the `AbiTcb` type below). People will also often use TCB to +//! refer to the libc TCB, which can be any size and contain anything. (One could even omit it!) We +//! refer to the latter as the Zig TCB; see the `ZigTcb` type below. +//! +//! [1] https://www.akkadia.org/drepper/tls.pdf + +const builtin = @import("builtin"); +const std = @import("std"); +const elf = std.elf; +const assert = std.debug.assert; +const native_arch = builtin.cpu.arch; +const linux = std.os.linux; +const page_size_min = std.heap.page_size_min; + +const compiler_rt = @import("../../compiler_rt.zig"); +const symbol = compiler_rt.symbol; + +comptime { + symbol(&__zig_elf_static_tls, "__zig_elf_static_tls"); + symbol(&__zig_elf_static_tls_init, "__zig_elf_static_tls_init"); + symbol(&__zig_elf_static_tls_fill, "__zig_elf_static_tls_fill"); + + // To make static executables without libc work, it is our job to provide the TLS accessor + // function for the GD and LD models. This function is unlikely to actually be used, since the + // linker should be able to relax every TLS access to the LE model and therefore eliminate all + // calls to this function, but that isn't guaranteed. + if (native_arch == .s390x) + symbol(&__tls_get_offset, "__tls_get_offset") + else + symbol(&__tls_get_addr, "__tls_get_addr"); +} + +/// Represents an ELF TLS variant. +/// +/// In all variants, the TP and the TLS blocks must be aligned to the `p_align` value in the +/// `PT.TLS` ELF program header. Everything else has natural alignment. +/// +/// The location of the DTV does not actually matter. For simplicity, we put it in the TLS area, but +/// there is no actual ABI requirement that it reside there. +const Variant = enum { + /// The original Variant I: + /// + /// ---------------------------------------- + /// | DTV | Zig TCB | ABI TCB | TLS Blocks | + /// ----------------^----------------------- + /// `-- The TP register points here. + /// + /// The layout in this variant necessitates separate alignment of both the TP and the TLS + /// blocks. + /// + /// The first word in the ABI TCB points to the DTV. For some architectures, there may be a + /// second word with an unspecified meaning. + I_original, + /// The modified Variant I: + /// + /// -------------------------------------------- + /// | DTV | Zig TCB | ABI TCB | TLS Blocks | + /// ------------------------------^------------- + /// `-- The TP register points here (*inside* the TLS blocks). + /// + /// The offset from the start of the TLS blocks to the TP register is `current_tp_offset`. It + /// may be zero, in which case the TP register points to the start of the TLS blocks. + /// + /// The first (and only) word in the ABI TCB points to the DTV. + I_modified, + /// Variant II: + /// + /// ---------------------------------------- + /// | TLS Blocks | ABI TCB | Zig TCB | DTV | + /// -------------^-------------------------- + /// `-- The TP register points here. + /// + /// The first (and only) word in the ABI TCB points to the ABI TCB itself. + II, +}; + +const current_variant: Variant = switch (native_arch) { + .aarch64, + .aarch64_be, + .alpha, + .arc, + .arceb, + .arm, + .armeb, + .csky, + .hppa, + .microblaze, + .microblazeel, + .sh, + .sheb, + .thumb, + .thumbeb, + .xtensa, + .xtensaeb, + => .I_original, + .loongarch32, + .loongarch64, + .m68k, + .mips, + .mipsel, + .mips64, + .mips64el, + .or1k, + .powerpc, + .powerpcle, + .powerpc64, + .powerpc64le, + .riscv32, + .riscv64, + => .I_modified, + .hexagon, + .s390x, + .sparc, + .sparc64, + .x86, + .x86_64, + => .II, + else => @compileError("undefined TLS variant for this architecture"), +}; + +/// The offset value for the modified Variant I. +const current_tp_offset = switch (native_arch) { + .m68k, + .mips, + .mipsel, + .mips64, + .mips64el, + .powerpc, + .powerpcle, + .powerpc64, + .powerpc64le, + => 0x7000, + else => 0, +}; + +/// Usually only used by the modified Variant I. +const current_dtv_offset = switch (native_arch) { + .m68k, + .mips, + .mipsel, + .mips64, + .mips64el, + .powerpc, + .powerpcle, + .powerpc64, + .powerpc64le, + => 0x8000, + .riscv32, + .riscv64, + => 0x800, + else => 0, +}; + +/// Per-thread storage for the ELF TLS ABI. +const AbiTcb = switch (current_variant) { + .I_original, .I_modified => switch (native_arch) { + .aarch64, + .aarch64_be, + .alpha, + .arm, + .armeb, + .hppa, + .microblaze, + .microblazeel, + .sh, + .sheb, + .thumb, + .thumbeb, + .xtensa, + .xtensaeb, + => extern struct { + /// This is offset by `current_dtv_offset`. + dtv: usize, + _reserved: ?*anyopaque, + }, + else => extern struct { + /// This is offset by `current_dtv_offset`. + dtv: usize, + }, + }, + .II => extern struct { + /// This is self-referential. + self: *AbiTcb, + }, +}; + +/// Per-thread storage for Zig's use. Currently unused. +const ZigTcb = struct { + dummy: usize, +}; + +/// Dynamic Thread Vector as specified in the ELF TLS ABI. Ordinarily, there is a block pointer per +/// dynamically-loaded module, but since we only support static TLS, we only need one block pointer. +const Dtv = extern struct { + len: usize = 1, + tls_block: [*]u8, +}; + +/// Describes a process's TLS area. The area encompasses the DTV, both TCBs, and the TLS block, with +/// the exact layout of these being dependent primarily on `current_variant`. See also `area_info`. +/// +/// The layout of this variable is ABI between compiler-rt and std. +var __zig_elf_static_tls: extern struct { + size: usize, + alignment: usize, + /// Only used on the 32-bit x86 architecture (not x86_64, nor x32). + gdt_entry_number: usize, +} = undefined; + +var area_info: struct { + dtv: struct { + /// Offset into the TLS area. + offset: usize, + }, + + abi_tcb: struct { + /// Offset into the TLS area. + offset: usize, + }, + + block: struct { + /// The initial data to be copied into the TLS block. Note that this may be smaller than + /// `size`, in which case any remaining data in the TLS block is simply left uninitialized. + init: []const u8, + /// Offset into the TLS area. + offset: usize, + /// This is the effective size of the TLS block, which may be greater than `init.len`. + size: usize, + }, +} = undefined; + +fn setThreadPointer(addr: usize) void { + @setRuntimeSafety(false); + @disableInstrumentation(); + + switch (native_arch) { + .x86 => { + var user_desc: linux.user_desc = .{ + .entry_number = __zig_elf_static_tls.gdt_entry_number, + .base_addr = addr, + .limit = 0xfffff, + .flags = .{ + .seg_32bit = 1, + .contents = 0, // Data + .read_exec_only = 0, + .limit_in_pages = 1, + .seg_not_present = 0, + .useable = 1, + }, + }; + const rc = @call(.always_inline, linux.syscall1, .{ .set_thread_area, @intFromPtr(&user_desc) }); + assert(rc == 0); + + const gdt_entry_number = user_desc.entry_number; + // We have to keep track of our slot as it's also needed for clone() + __zig_elf_static_tls.gdt_entry_number = gdt_entry_number; + // Update the %gs selector + asm volatile ("movl %[gs_val], %%gs" + : + : [gs_val] "r" (gdt_entry_number << 3 | 3), + ); + }, + .x86_64 => { + const rc = @call(.always_inline, linux.syscall2, .{ .arch_prctl, linux.ARCH.SET_FS, addr }); + assert(rc == 0); + }, + .aarch64, .aarch64_be => { + asm volatile ( + \\ msr tpidr_el0, %[addr] + : + : [addr] "r" (addr), + ); + }, + .alpha => { + asm volatile ( + \\ wruniq + : + : [addr] "{$16}" (addr), + ); + }, + .arc, .arceb => { + // We apparently need to both set r25 (TP) *and* inform the kernel... + asm volatile ( + \\ mov r25, %[addr] + : + : [addr] "r" (addr), + ); + const rc = @call(.always_inline, linux.syscall1, .{ .arc_settls, addr }); + assert(rc == 0); + }, + .arm, .armeb, .thumb, .thumbeb => { + const rc = @call(.always_inline, linux.syscall1, .{ .set_tls, addr }); + assert(rc == 0); + }, + .m68k => { + const rc = @call(.always_inline, linux.syscall1, .{ .set_thread_area, addr }); + assert(rc == 0); + }, + .hexagon => { + asm volatile ( + \\ ugp = %[addr] + : + : [addr] "r" (addr), + ); + }, + .hppa => { + asm volatile ( + \\ ble 0xe0(%%sr2, %%r0) + \\ nop + : + : [addr] "{r26}" (addr), + : .{ .r31 = true }); + }, + .loongarch32, .loongarch64 => { + asm volatile ( + \\ move $tp, %[addr] + : + : [addr] "r" (addr), + ); + }, + .riscv32, .riscv64 => { + asm volatile ( + \\ mv tp, %[addr] + : + : [addr] "r" (addr), + ); + }, + .csky, .mips, .mipsel, .mips64, .mips64el => { + const rc = @call(.always_inline, linux.syscall1, .{ .set_thread_area, addr }); + assert(rc == 0); + }, + .microblaze, .microblazeel => { + asm volatile ( + \\ ori r21, %[addr], 0 + : + : [addr] "r" (addr), + ); + }, + .or1k => { + asm volatile ( + \\ l.ori r10, %[addr], 0 + : + : [addr] "r" (addr), + ); + }, + .powerpc, .powerpcle => { + asm volatile ( + \\ mr 2, %[addr] + : + : [addr] "r" (addr), + ); + }, + .powerpc64, .powerpc64le => { + asm volatile ( + \\ mr 13, %[addr] + : + : [addr] "r" (addr), + ); + }, + .s390x => { + asm volatile ( + \\ lgr %%r0, %[addr] + \\ sar %%a1, %%r0 + \\ srlg %%r0, %%r0, 32 + \\ sar %%a0, %%r0 + : + : [addr] "r" (addr), + : .{ .r0 = true }); + }, + .sh, .sheb => { + asm volatile ( + \\ ldc %[addr], gbr + : + : [addr] "r" (addr), + ); + }, + .sparc, .sparc64 => { + asm volatile ( + \\ mov %[addr], %%g7 + : + : [addr] "r" (addr), + ); + }, + .xtensa, .xtensaeb => { + asm volatile ( + \\ wur %[addr], threadptr + : + : [addr] "a" (addr), + ); + }, + else => @compileError("Unsupported architecture"), + } +} + +fn getThreadPointer() usize { + @setRuntimeSafety(false); + @disableInstrumentation(); + + return switch (native_arch) { + .aarch64, .aarch64_be => asm ( + \\ mrs %[ret], tpidr_el0 + : [ret] "=r" (-> usize), + ), + .alpha => asm ( + \\ rduniq + : [ret] "={$0}" (-> usize), + ), + .arc, .arceb => asm ( + \\ mov %[ret], r25 + : [ret] "=r" (-> usize), + ), + .arm, .armeb, .thumb, .thumbeb => asm ( + \\ mrc p15, 0, %[ret], c13, c0, 3 + : [ret] "=r" (-> usize), + ), + .csky => asm ( + \\ mov %[ret], r31 + : [ret] "=r" (-> usize), + ), + .hexagon => asm ( + \\ %[ret] = ugp + : [ret] "=r" (-> usize), + ), + .hppa => asm ( + \\ mfctl %%cr27, %[ret] + : [ret] "=r" (-> usize), + ), + .loongarch32, .loongarch64 => asm ( + \\ move %[ret], $tp + : [ret] "=r" (-> usize), + ), + .m68k => linux.syscall0(.get_thread_area), + .mips, .mipsel, .mips64, .mips64el => asm ( + \\ rdhwr %[ret], $29 + : [ret] "=r" (-> usize), + ), + .microblaze, .microblazeel => asm ( + \\ ori %[ret], r21, 0 + : [ret] "=r" (-> usize), + ), + .or1k => asm ( + \\ l.ori %[ret], r10, 0 + : [ret] "=r" (-> usize), + ), + .riscv32, .riscv64 => asm ( + \\ mv %[ret], tp + : [ret] "=r" (-> usize), + ), + .powerpc, .powerpcle => asm ( + \\ mr %[ret], 2 + : [ret] "=r" (-> usize), + ), + .powerpc64, .powerpc64le => asm ( + \\ mr %[ret], 13 + : [ret] "=r" (-> usize), + ), + .s390x => asm ( + \\ ear %[ret], %%a0 + \\ sllg %[ret], %[ret], 32 + \\ ear %[ret], %%a1 + : [ret] "=r" (-> usize), + ), + .sh, .sheb => asm ( + \\ stc gbr, %[ret] + : [ret] "=r" (-> usize), + ), + .sparc, .sparc64 => asm ( + \\ mov %%g7, %[ret] + : [ret] "=r" (-> usize), + ), + .x86 => asm ( + \\ movl %%gs:0, %[ret] + : [ret] "=r" (-> usize), + ), + .x86_64 => switch (@sizeOf(usize)) { + 8 => asm ( + \\ movq %%fs:0, %[ret] + : [ret] "=r" (-> usize), + ), + // On x32, usize is 32 bits. + 4 => asm ( + \\ movl %%fs:0, %[ret] + : [ret] "=r" (-> usize), + ), + else => comptime unreachable, + }, + .xtensa, .xtensaeb => asm ( + \\ rur %[ret], threadptr + : [ret] "=r" (-> usize), + ), + else => @compileError("Unsupported architecture"), + }; +} + +fn computeAreaDesc(phdrs: []elf.ElfN.Phdr) void { + @setRuntimeSafety(false); + @disableInstrumentation(); + + var tls_phdr: ?*elf.ElfN.Phdr = null; + var img_base: usize = 0; + + for (phdrs) |*phdr| { + switch (phdr.type) { + .PHDR => img_base = @intFromPtr(phdrs.ptr) - phdr.vaddr, + .TLS => tls_phdr = phdr, + else => {}, + } + } + + var align_factor: usize = undefined; + var block_init: []const u8 = undefined; + var block_size: usize = undefined; + + if (tls_phdr) |phdr| { + align_factor = phdr.@"align"; + + // The effective size in memory is represented by `memsz`; the length of the data stored + // in the `PT.TLS` segment is `filesz` and may be less than the former. + block_init = @as([*]u8, @ptrFromInt(img_base + phdr.vaddr))[0..phdr.filesz]; + block_size = phdr.memsz; + } else { + align_factor = @alignOf(usize); + + block_init = &[_]u8{}; + block_size = 0; + } + + // Offsets into the allocated TLS area. + var dtv_offset: usize = undefined; + var abi_tcb_offset: usize = undefined; + var block_offset: usize = undefined; + + // Compute the total size of the ABI-specific data plus our own `ZigTcb` structure. All the + // offsets calculated here assume a well-aligned base address. + const area_size = switch (current_variant) { + .I_original => blk: { + var l: usize = 0; + dtv_offset = l; + l += @sizeOf(Dtv); + // Add some padding here so that the TP (`abi_tcb_offset`) is aligned to `align_factor` + // and the `ZigTcb` structure can be found by simply subtracting `@sizeOf(ZigTcb)` from + // the TP. + const delta = (l + @sizeOf(ZigTcb)) & (align_factor - 1); + if (delta > 0) + l += align_factor - delta; + l += @sizeOf(ZigTcb); + abi_tcb_offset = l; + l += alignForward(@sizeOf(AbiTcb), align_factor); + block_offset = l; + l += block_size; + break :blk l; + }, + .I_modified => blk: { + var l: usize = 0; + dtv_offset = l; + l += @sizeOf(Dtv); + // In this variant, the TLS blocks must begin immediately after the end of the ABI TCB, + // with the TP pointing to the beginning of the TLS blocks. Add padding so that the TP + // (`abi_tcb_offset`) is aligned to `align_factor` and the `ZigTcb` structure can be + // found by subtracting `@sizeOf(AbiTcb) + @sizeOf(ZigTcb)` from the TP. + const delta = (l + @sizeOf(ZigTcb) + @sizeOf(AbiTcb)) & (align_factor - 1); + if (delta > 0) + l += align_factor - delta; + l += @sizeOf(ZigTcb); + abi_tcb_offset = l; + l += @sizeOf(AbiTcb); + block_offset = l; + l += block_size; + break :blk l; + }, + .II => blk: { + var l: usize = 0; + block_offset = l; + l += alignForward(block_size, align_factor); + // The TP is aligned to `align_factor`. + abi_tcb_offset = l; + l += @sizeOf(AbiTcb); + // The `ZigTcb` structure is right after the `AbiTcb` with no padding in between so it + // can be easily found. + l += @sizeOf(ZigTcb); + // It doesn't really matter where we put the DTV, so give it natural alignment. + l = alignForward(l, @alignOf(Dtv)); + dtv_offset = l; + l += @sizeOf(Dtv); + break :blk l; + }, + }; + + __zig_elf_static_tls = .{ + .size = area_size, + .alignment = align_factor, + .gdt_entry_number = @as(usize, @bitCast(@as(isize, -1))), + }; + + area_info = .{ + .dtv = .{ + .offset = dtv_offset, + }, + + .abi_tcb = .{ + .offset = abi_tcb_offset, + }, + + .block = .{ + .init = block_init, + .offset = block_offset, + .size = block_size, + }, + }; +} + +/// Inline because TLS is not set up yet. +inline fn alignForward(addr: usize, alignment: usize) usize { + return alignBackward(addr + (alignment - 1), alignment); +} + +/// Inline because TLS is not set up yet. +inline fn alignBackward(addr: usize, alignment: usize) usize { + return addr & ~(alignment - 1); +} + +/// Inline because TLS is not set up yet. +inline fn alignPtrCast(comptime T: type, ptr: [*]u8) *T { + return @ptrCast(@alignCast(ptr)); +} + +/// Initializes all the fields of the static TLS area and returns the computed architecture-specific +/// value of the TP register. +fn __zig_elf_static_tls_fill(area: [*]u8) callconv(.c) usize { + @setRuntimeSafety(false); + @disableInstrumentation(); + + const area_slice = area[0..__zig_elf_static_tls.size]; + + // Clear the area we're going to use, just to be safe. + @memset(area_slice, 0); + + // Prepare the ABI TCB. + const abi_tcb = alignPtrCast(AbiTcb, area + area_info.abi_tcb.offset); + switch (current_variant) { + .I_original, .I_modified => abi_tcb.dtv = @intFromPtr(area + area_info.dtv.offset), + .II => abi_tcb.self = abi_tcb, + } + + // Prepare the DTV. + const dtv = alignPtrCast(Dtv, area + area_info.dtv.offset); + dtv.len = 1; + dtv.tls_block = area + current_dtv_offset + area_info.block.offset; + + // Copy the initial data. + @memcpy(area_slice[area_info.block.offset..][0..area_info.block.init.len], area_info.block.init); + + // Return the corrected value (if needed) for the TP register. Overflow here is not a problem; + // the pointer arithmetic involving the TP is done with wrapping semantics. + return @intFromPtr(area) +% switch (current_variant) { + .I_original, .II => area_info.abi_tcb.offset, + .I_modified => area_info.block.offset +% current_tp_offset, + }; +} + +/// The main motivation for the size chosen here is to be larger than total +/// amount of thread-local variables for most programs. Putting this allocation +/// in the ELF like this is equivalent to moving the `mmap` call below into the +/// kernel, avoiding syscall overhead. +var main_thread_area_buffer: [0x1000]u8 align(page_size_min) = undefined; + +/// Computes the layout of the static TLS area, allocates the area, initializes all of its fields, +/// and assigns the architecture-specific value to the TP register. +fn __zig_elf_static_tls_init(phdrs: [*]elf.ElfN.Phdr, phnum: usize) callconv(.c) void { + @setRuntimeSafety(false); + @disableInstrumentation(); + + computeAreaDesc(phdrs[0..phnum]); + + const area: [*]u8 = blk: { + // Fast path for the common case where the TLS data is really small, avoid an allocation and + // use our local buffer. + if (__zig_elf_static_tls.alignment <= page_size_min and __zig_elf_static_tls.size <= main_thread_area_buffer.len) { + break :blk &main_thread_area_buffer; + } + + const begin_addr = mmap(__zig_elf_static_tls.size + __zig_elf_static_tls.alignment - 1); + if (@call(.always_inline, linux.errno, .{begin_addr}) != .SUCCESS) @trap(); + + const area_ptr: [*]align(page_size_min) u8 = @ptrFromInt(begin_addr); + + // Make sure the slice is correctly aligned. + const begin_aligned_addr = alignForward(begin_addr, __zig_elf_static_tls.alignment); + const start = begin_aligned_addr - begin_addr; + break :blk area_ptr + start; + }; + + const tp_value = __zig_elf_static_tls_fill(area); + setThreadPointer(tp_value); +} + +inline fn mmap(length: usize) usize { + const prot: linux.PROT = .{ .READ = true, .WRITE = true }; + const flags: linux.MAP = .{ .TYPE = .PRIVATE, .ANONYMOUS = true }; + + if (@hasField(linux.SYS, "mmap2")) { + return @call(.always_inline, linux.syscall6, .{ + .mmap2, + 0, + length, + @as(u32, @bitCast(prot)), + @as(u32, @bitCast(flags)), + @as(usize, @bitCast(@as(isize, -1))), + 0, + }); + } else { + // The s390x mmap() syscall existed before Linux supported syscalls with 5+ parameters, so + // it takes a single pointer to an array of arguments instead. + return if (native_arch == .s390x) @call(.always_inline, linux.syscall1, .{ + .mmap, + @intFromPtr(&[_]usize{ + 0, + length, + @as(u32, @bitCast(prot)), + @as(u32, @bitCast(flags)), + @as(usize, @bitCast(@as(isize, -1))), + 0, + }), + }) else @call(.always_inline, linux.syscall6, .{ + .mmap, + 0, + length, + @as(u32, @bitCast(prot)), + @as(u32, @bitCast(flags)), + @as(usize, @bitCast(@as(isize, -1))), + 0, + }); + } +} + +const TlsIndex = switch (native_arch) { + .x86_64 => extern struct { module: u64, offset: u64 }, // Even for x32... + else => extern struct { module: usize, offset: usize }, // ...but not MIPS N32! +}; + +fn __tls_get_addr(ti: *const TlsIndex) callconv(.c) *anyopaque { + comptime assert(native_arch != .s390x); + + assert(ti.module == 1); // The executable's module ID is always 1 + const tp = getThreadPointer(); + const block: [*]u8 = switch (current_variant) { + .I_original => @ptrFromInt(tp -% area_info.abi_tcb.offset +% area_info.block.offset), + .I_modified => @ptrFromInt(tp -% current_tp_offset), + // The `.I_original` approach would also work for `.II`, but there is an + // alternative strategy which is one less operation: + .II => @ptrFromInt(tp -% area_info.block.size), + }; + return block[@intCast(ti.offset)..]; +} + +fn __tls_get_offset() callconv(.naked) noreturn { + comptime assert(native_arch == .s390x); + + // We receive the module's GOT pointer in r12 and the GOT offset in r2. + asm volatile ( + \\ la %%r1, 0(%%r12, %%r2) + \\ lg %%r2, 8(%%r1) + \\ lgrl %%r0, %[block_size] + \\ sgr %%r2, %%r0 + \\ br %%r14 + : + : [block_size] "s" (&area_info.block.size), + ); +} diff --git a/lib/std/Thread.zig b/lib/std/Thread.zig index 66a3662eedc92bae67d06cb9fac9625e8bde87ff..f093e3899267f2f5afb93432fb32f3c8fbd4ce20 100644 --- a/lib/std/Thread.zig +++ b/lib/std/Thread.zig @@ -1434,6 +1434,16 @@ const LinuxThreadImpl = struct { } }; + /// Provided by compiler-rt and populated by `std.start`. + extern var __zig_elf_static_tls: extern struct { + size: usize, + alignment: usize, + gdt_entry_number: usize, + }; + + /// Provided by compiler-rt. + extern fn __zig_elf_static_tls_fill(area: [*]u8) usize; + fn spawn(config: SpawnConfig, comptime f: anytype, args: anytype) !Impl { const page_size = std.heap.pageSize(); const Args = @TypeOf(args); @@ -1465,9 +1475,9 @@ const LinuxThreadImpl = struct { bytes = std.mem.alignForward(usize, bytes, page_size); stack_offset = bytes; - bytes = std.mem.alignForward(usize, bytes, linux.tls.area_desc.alignment); + bytes = std.mem.alignForward(usize, bytes, __zig_elf_static_tls.alignment); tls_offset = bytes; - bytes += linux.tls.area_desc.size; + bytes += __zig_elf_static_tls.size; bytes = std.mem.alignForward(usize, bytes, @alignOf(Instance)); instance_offset = bytes; @@ -1509,12 +1519,12 @@ const LinuxThreadImpl = struct { } // Prepare the TLS segment and prepare a user_desc struct when needed on x86 - var tls_ptr = linux.tls.prepareArea(mapped[tls_offset..][0..linux.tls.area_desc.size]); + var tls_ptr = __zig_elf_static_tls_fill(mapped.ptr + tls_offset); var user_desc: if (target.cpu.arch == .x86) linux.user_desc else void = undefined; if (target.cpu.arch == .x86) { defer tls_ptr = @intFromPtr(&user_desc); user_desc = .{ - .entry_number = linux.tls.area_desc.gdt_entry_number, + .entry_number = __zig_elf_static_tls.gdt_entry_number, .base_addr = tls_ptr, .limit = 0xfffff, .flags = .{ diff --git a/lib/std/elf.zig b/lib/std/elf.zig index 966dbb6e49eae3c04ec6a4a9eb0213fb6e462098..28dc645307c31a5dacaaa0fca5664d0551659aa8 100644 --- a/lib/std/elf.zig +++ b/lib/std/elf.zig @@ -1602,7 +1602,7 @@ pub const Elf32_auxv_t = extern struct { }, }; pub const Elf64_auxv_t = extern struct { - a_type: u64, + a_type: if (native_os == .netbsd or native_os == .openbsd) u32 else u64, a_un: extern union { a_val: u64, }, diff --git a/lib/std/os/linux.zig b/lib/std/os/linux.zig index 38e4abd197ba6c17245e7fe7038d7db0528ca395..7c18ef62011d3cd07a6612d78b652446ce602344 100644 --- a/lib/std/os/linux.zig +++ b/lib/std/os/linux.zig @@ -123,7 +123,6 @@ pub const gid_t = u32; pub const clock_t = isize; pub const time_t = arch_bits.time_t; -pub const tls = @import("linux/tls.zig"); pub const BPF = @import("linux/bpf.zig"); pub const IOCTL = @import("linux/ioctl.zig"); pub const SECCOMP = @import("linux/seccomp.zig"); @@ -960,43 +959,22 @@ pub const RENAME = packed struct(u32) { _: u29 = 0, }; -/// Set by startup code, used by `getauxval`. -pub var elf_aux_maybe: ?[*]std.elf.Auxv = null; +/// Available only when libc is *not* linked; use `std.c.getauxval` if libc is linked. +pub const getauxval = if (builtin.link_libc) {} else struct { + /// Provided by compiler-rt and set by `std.start`. + extern var __zig_elf_auxv: [*]std.elf.Auxv; -/// Whether an external or internal getauxval implementation is used. -const extern_getauxval = switch (builtin.zig_backend) { - // Calling extern functions is not yet supported with these backends - .stage2_arm, - .stage2_loongarch, - .stage2_powerpc, - .stage2_riscv64, - .stage2_sparc64, - => false, - else => !builtin.link_libc, -}; + pub fn getauxval(index: usize) usize { + @disableInstrumentation(); -pub const getauxval = if (extern_getauxval) struct { - comptime { - const root = @import("root"); - // Export this only when building an executable, otherwise it is overriding - // the libc implementation - if (builtin.output_mode == .Exe or @hasDecl(root, "main")) { - @export(&getauxvalImpl, .{ .name = "getauxval", .linkage = .weak }); + var i: usize = 0; + while (__zig_elf_auxv[i].a_type != std.elf.AT.NULL) : (i += 1) { + if (__zig_elf_auxv[i].a_type == index) return __zig_elf_auxv[i].a_un.a_val; } - } - extern fn getauxval(index: usize) usize; -}.getauxval else getauxvalImpl; -fn getauxvalImpl(index: usize) callconv(.c) usize { - @disableInstrumentation(); - const auxv = elf_aux_maybe orelse return 0; - var i: usize = 0; - while (auxv[i].a_type != std.elf.AT.NULL) : (i += 1) { - if (auxv[i].a_type == index) - return auxv[i].a_un.a_val; + return 0; } - return 0; -} +}.getauxval; // Some architectures (and some syscalls) require 64bit parameters to be passed // in a even-aligned register pair. diff --git a/lib/std/os/linux/tls.zig b/lib/std/os/linux/tls.zig deleted file mode 100644 index aed76fc0cb2ee83318e5c0e2d72b0016cf44eb91..0000000000000000000000000000000000000000 --- a/lib/std/os/linux/tls.zig +++ /dev/null @@ -1,769 +0,0 @@ -//! This file implements the two TLS variants [1] used by ELF-based systems. Note that, in reality, -//! Variant I has two sub-variants. -//! -//! It is important to understand that the term TCB (Thread Control Block) is overloaded here. -//! Official ABI documentation uses it simply to mean the ABI TCB, i.e. a small area of ABI-defined -//! data, usually one or two words (see the `AbiTcb` type below). People will also often use TCB to -//! refer to the libc TCB, which can be any size and contain anything. (One could even omit it!) We -//! refer to the latter as the Zig TCB; see the `ZigTcb` type below. -//! -//! [1] https://www.akkadia.org/drepper/tls.pdf - -const std = @import("std"); -const mem = std.mem; -const elf = std.elf; -const math = std.math; -const assert = std.debug.assert; -const builtin = @import("builtin"); -const native_arch = builtin.cpu.arch; -const linux = std.os.linux; -const page_size_min = std.heap.page_size_min; - -/// Represents an ELF TLS variant. -/// -/// In all variants, the TP and the TLS blocks must be aligned to the `p_align` value in the -/// `PT.TLS` ELF program header. Everything else has natural alignment. -/// -/// The location of the DTV does not actually matter. For simplicity, we put it in the TLS area, but -/// there is no actual ABI requirement that it reside there. -const Variant = enum { - /// The original Variant I: - /// - /// ---------------------------------------- - /// | DTV | Zig TCB | ABI TCB | TLS Blocks | - /// ----------------^----------------------- - /// `-- The TP register points here. - /// - /// The layout in this variant necessitates separate alignment of both the TP and the TLS - /// blocks. - /// - /// The first word in the ABI TCB points to the DTV. For some architectures, there may be a - /// second word with an unspecified meaning. - I_original, - /// The modified Variant I: - /// - /// -------------------------------------------- - /// | DTV | Zig TCB | ABI TCB | TLS Blocks | - /// ------------------------------^------------- - /// `-- The TP register points here (*inside* the TLS blocks). - /// - /// The offset from the start of the TLS blocks to the TP register is `current_tp_offset`. It - /// may be zero, in which case the TP register points to the start of the TLS blocks. - /// - /// The first (and only) word in the ABI TCB points to the DTV. - I_modified, - /// Variant II: - /// - /// ---------------------------------------- - /// | TLS Blocks | ABI TCB | Zig TCB | DTV | - /// -------------^-------------------------- - /// `-- The TP register points here. - /// - /// The first (and only) word in the ABI TCB points to the ABI TCB itself. - II, -}; - -const current_variant: Variant = switch (native_arch) { - .aarch64, - .aarch64_be, - .alpha, - .arc, - .arceb, - .arm, - .armeb, - .csky, - .hppa, - .microblaze, - .microblazeel, - .sh, - .sheb, - .thumb, - .thumbeb, - .xtensa, - .xtensaeb, - => .I_original, - .loongarch32, - .loongarch64, - .m68k, - .mips, - .mipsel, - .mips64, - .mips64el, - .or1k, - .powerpc, - .powerpcle, - .powerpc64, - .powerpc64le, - .riscv32, - .riscv64, - => .I_modified, - .hexagon, - .s390x, - .sparc, - .sparc64, - .x86, - .x86_64, - => .II, - else => @compileError("undefined TLS variant for this architecture"), -}; - -/// The offset value for the modified Variant I. -const current_tp_offset = switch (native_arch) { - .m68k, - .mips, - .mipsel, - .mips64, - .mips64el, - .powerpc, - .powerpcle, - .powerpc64, - .powerpc64le, - => 0x7000, - else => 0, -}; - -/// Usually only used by the modified Variant I. -const current_dtv_offset = switch (native_arch) { - .m68k, - .mips, - .mipsel, - .mips64, - .mips64el, - .powerpc, - .powerpcle, - .powerpc64, - .powerpc64le, - => 0x8000, - .riscv32, - .riscv64, - => 0x800, - else => 0, -}; - -/// Per-thread storage for the ELF TLS ABI. -const AbiTcb = switch (current_variant) { - .I_original, .I_modified => switch (native_arch) { - .aarch64, - .aarch64_be, - .alpha, - .arm, - .armeb, - .hppa, - .microblaze, - .microblazeel, - .sh, - .sheb, - .thumb, - .thumbeb, - .xtensa, - .xtensaeb, - => extern struct { - /// This is offset by `current_dtv_offset`. - dtv: usize, - _reserved: ?*anyopaque, - }, - else => extern struct { - /// This is offset by `current_dtv_offset`. - dtv: usize, - }, - }, - .II => extern struct { - /// This is self-referential. - self: *AbiTcb, - }, -}; - -/// Per-thread storage for Zig's use. Currently unused. -const ZigTcb = struct { - dummy: usize, -}; - -/// Dynamic Thread Vector as specified in the ELF TLS ABI. Ordinarily, there is a block pointer per -/// dynamically-loaded module, but since we only support static TLS, we only need one block pointer. -const Dtv = extern struct { - len: usize = 1, - tls_block: [*]u8, -}; - -/// Describes a process's TLS area. The area encompasses the DTV, both TCBs, and the TLS block, with -/// the exact layout of these being dependent primarily on `current_variant`. -const AreaDesc = struct { - size: usize, - alignment: usize, - - dtv: struct { - /// Offset into the TLS area. - offset: usize, - }, - - abi_tcb: struct { - /// Offset into the TLS area. - offset: usize, - }, - - block: struct { - /// The initial data to be copied into the TLS block. Note that this may be smaller than - /// `size`, in which case any remaining data in the TLS block is simply left uninitialized. - init: []const u8, - /// Offset into the TLS area. - offset: usize, - /// This is the effective size of the TLS block, which may be greater than `init.len`. - size: usize, - }, - - /// Only used on the 32-bit x86 architecture (not x86_64, nor x32). - gdt_entry_number: usize, -}; - -pub var area_desc: AreaDesc = undefined; - -pub fn setThreadPointer(addr: usize) void { - @setRuntimeSafety(false); - @disableInstrumentation(); - - switch (native_arch) { - .x86 => { - var user_desc: linux.user_desc = .{ - .entry_number = area_desc.gdt_entry_number, - .base_addr = addr, - .limit = 0xfffff, - .flags = .{ - .seg_32bit = 1, - .contents = 0, // Data - .read_exec_only = 0, - .limit_in_pages = 1, - .seg_not_present = 0, - .useable = 1, - }, - }; - const rc = @call(.always_inline, linux.syscall1, .{ .set_thread_area, @intFromPtr(&user_desc) }); - assert(rc == 0); - - const gdt_entry_number = user_desc.entry_number; - // We have to keep track of our slot as it's also needed for clone() - area_desc.gdt_entry_number = gdt_entry_number; - // Update the %gs selector - asm volatile ("movl %[gs_val], %%gs" - : - : [gs_val] "r" (gdt_entry_number << 3 | 3), - ); - }, - .x86_64 => { - const rc = @call(.always_inline, linux.syscall2, .{ .arch_prctl, linux.ARCH.SET_FS, addr }); - assert(rc == 0); - }, - .aarch64, .aarch64_be => { - asm volatile ( - \\ msr tpidr_el0, %[addr] - : - : [addr] "r" (addr), - ); - }, - .alpha => { - asm volatile ( - \\ wruniq - : - : [addr] "{$16}" (addr), - ); - }, - .arc, .arceb => { - // We apparently need to both set r25 (TP) *and* inform the kernel... - asm volatile ( - \\ mov r25, %[addr] - : - : [addr] "r" (addr), - ); - const rc = @call(.always_inline, linux.syscall1, .{ .arc_settls, addr }); - assert(rc == 0); - }, - .arm, .armeb, .thumb, .thumbeb => { - const rc = @call(.always_inline, linux.syscall1, .{ .set_tls, addr }); - assert(rc == 0); - }, - .m68k => { - const rc = @call(.always_inline, linux.syscall1, .{ .set_thread_area, addr }); - assert(rc == 0); - }, - .hexagon => { - asm volatile ( - \\ ugp = %[addr] - : - : [addr] "r" (addr), - ); - }, - .hppa => { - asm volatile ( - \\ ble 0xe0(%%sr2, %%r0) - \\ nop - : - : [addr] "{r26}" (addr), - : .{ .r31 = true }); - }, - .loongarch32, .loongarch64 => { - asm volatile ( - \\ move $tp, %[addr] - : - : [addr] "r" (addr), - ); - }, - .riscv32, .riscv64 => { - asm volatile ( - \\ mv tp, %[addr] - : - : [addr] "r" (addr), - ); - }, - .csky, .mips, .mipsel, .mips64, .mips64el => { - const rc = @call(.always_inline, linux.syscall1, .{ .set_thread_area, addr }); - assert(rc == 0); - }, - .microblaze, .microblazeel => { - asm volatile ( - \\ ori r21, %[addr], 0 - : - : [addr] "r" (addr), - ); - }, - .or1k => { - asm volatile ( - \\ l.ori r10, %[addr], 0 - : - : [addr] "r" (addr), - ); - }, - .powerpc, .powerpcle => { - asm volatile ( - \\ mr 2, %[addr] - : - : [addr] "r" (addr), - ); - }, - .powerpc64, .powerpc64le => { - asm volatile ( - \\ mr 13, %[addr] - : - : [addr] "r" (addr), - ); - }, - .s390x => { - asm volatile ( - \\ lgr %%r0, %[addr] - \\ sar %%a1, %%r0 - \\ srlg %%r0, %%r0, 32 - \\ sar %%a0, %%r0 - : - : [addr] "r" (addr), - : .{ .r0 = true }); - }, - .sh, .sheb => { - asm volatile ( - \\ ldc %[addr], gbr - : - : [addr] "r" (addr), - ); - }, - .sparc, .sparc64 => { - asm volatile ( - \\ mov %[addr], %%g7 - : - : [addr] "r" (addr), - ); - }, - .xtensa, .xtensaeb => { - asm volatile ( - \\ wur %[addr], threadptr - : - : [addr] "a" (addr), - ); - }, - else => @compileError("Unsupported architecture"), - } -} - -pub fn getThreadPointer() usize { - @setRuntimeSafety(false); - @disableInstrumentation(); - - return switch (native_arch) { - .aarch64, .aarch64_be => asm ( - \\ mrs %[ret], tpidr_el0 - : [ret] "=r" (-> usize), - ), - .alpha => asm ( - \\ rduniq - : [ret] "={$0}" (-> usize), - ), - .arc, .arceb => asm ( - \\ mov %[ret], r25 - : [ret] "=r" (-> usize), - ), - .arm, .armeb, .thumb, .thumbeb => asm ( - \\ mrc p15, 0, %[ret], c13, c0, 3 - : [ret] "=r" (-> usize), - ), - .csky => asm ( - \\ mov %[ret], r31 - : [ret] "=r" (-> usize), - ), - .hexagon => asm ( - \\ %[ret] = ugp - : [ret] "=r" (-> usize), - ), - .hppa => asm ( - \\ mfctl %%cr27, %[ret] - : [ret] "=r" (-> usize), - ), - .loongarch32, .loongarch64 => asm ( - \\ move %[ret], $tp - : [ret] "=r" (-> usize), - ), - .m68k => linux.syscall0(.get_thread_area), - .mips, .mipsel, .mips64, .mips64el => asm ( - \\ rdhwr %[ret], $29 - : [ret] "=r" (-> usize), - ), - .microblaze, .microblazeel => asm ( - \\ ori %[ret], r21, 0 - : [ret] "=r" (-> usize), - ), - .or1k => asm ( - \\ l.ori %[ret], r10, 0 - : [ret] "=r" (-> usize), - ), - .riscv32, .riscv64 => asm ( - \\ mv %[ret], tp - : [ret] "=r" (-> usize), - ), - .powerpc, .powerpcle => asm ( - \\ mr %[ret], 2 - : [ret] "=r" (-> usize), - ), - .powerpc64, .powerpc64le => asm ( - \\ mr %[ret], 13 - : [ret] "=r" (-> usize), - ), - .s390x => asm ( - \\ ear %[ret], %%a0 - \\ sllg %[ret], %[ret], 32 - \\ ear %[ret], %%a1 - : [ret] "=r" (-> usize), - ), - .sh, .sheb => asm ( - \\ stc gbr, %[ret] - : [ret] "=r" (-> usize), - ), - .sparc, .sparc64 => asm ( - \\ mov %%g7, %[ret] - : [ret] "=r" (-> usize), - ), - .x86 => asm ( - \\ movl %%gs:0, %[ret] - : [ret] "=r" (-> usize), - ), - .x86_64 => switch (@sizeOf(usize)) { - 8 => asm ( - \\ movq %%fs:0, %[ret] - : [ret] "=r" (-> usize), - ), - // On x32, usize is 32 bits. - 4 => asm ( - \\ movl %%fs:0, %[ret] - : [ret] "=r" (-> usize), - ), - else => comptime unreachable, - }, - .xtensa, .xtensaeb => asm ( - \\ rur %[ret], threadptr - : [ret] "=r" (-> usize), - ), - else => @compileError("Unsupported architecture"), - }; -} - -fn computeAreaDesc(phdrs: []elf.ElfN.Phdr) void { - @setRuntimeSafety(false); - @disableInstrumentation(); - - var tls_phdr: ?*elf.ElfN.Phdr = null; - var img_base: usize = 0; - - for (phdrs) |*phdr| { - switch (phdr.type) { - .PHDR => img_base = @intFromPtr(phdrs.ptr) - phdr.vaddr, - .TLS => tls_phdr = phdr, - else => {}, - } - } - - var align_factor: usize = undefined; - var block_init: []const u8 = undefined; - var block_size: usize = undefined; - - if (tls_phdr) |phdr| { - align_factor = phdr.@"align"; - - // The effective size in memory is represented by `memsz`; the length of the data stored - // in the `PT.TLS` segment is `filesz` and may be less than the former. - block_init = @as([*]u8, @ptrFromInt(img_base + phdr.vaddr))[0..phdr.filesz]; - block_size = phdr.memsz; - } else { - align_factor = @alignOf(usize); - - block_init = &[_]u8{}; - block_size = 0; - } - - // Offsets into the allocated TLS area. - var dtv_offset: usize = undefined; - var abi_tcb_offset: usize = undefined; - var block_offset: usize = undefined; - - // Compute the total size of the ABI-specific data plus our own `ZigTcb` structure. All the - // offsets calculated here assume a well-aligned base address. - const area_size = switch (current_variant) { - .I_original => blk: { - var l: usize = 0; - dtv_offset = l; - l += @sizeOf(Dtv); - // Add some padding here so that the TP (`abi_tcb_offset`) is aligned to `align_factor` - // and the `ZigTcb` structure can be found by simply subtracting `@sizeOf(ZigTcb)` from - // the TP. - const delta = (l + @sizeOf(ZigTcb)) & (align_factor - 1); - if (delta > 0) - l += align_factor - delta; - l += @sizeOf(ZigTcb); - abi_tcb_offset = l; - l += alignForward(@sizeOf(AbiTcb), align_factor); - block_offset = l; - l += block_size; - break :blk l; - }, - .I_modified => blk: { - var l: usize = 0; - dtv_offset = l; - l += @sizeOf(Dtv); - // In this variant, the TLS blocks must begin immediately after the end of the ABI TCB, - // with the TP pointing to the beginning of the TLS blocks. Add padding so that the TP - // (`abi_tcb_offset`) is aligned to `align_factor` and the `ZigTcb` structure can be - // found by subtracting `@sizeOf(AbiTcb) + @sizeOf(ZigTcb)` from the TP. - const delta = (l + @sizeOf(ZigTcb) + @sizeOf(AbiTcb)) & (align_factor - 1); - if (delta > 0) - l += align_factor - delta; - l += @sizeOf(ZigTcb); - abi_tcb_offset = l; - l += @sizeOf(AbiTcb); - block_offset = l; - l += block_size; - break :blk l; - }, - .II => blk: { - var l: usize = 0; - block_offset = l; - l += alignForward(block_size, align_factor); - // The TP is aligned to `align_factor`. - abi_tcb_offset = l; - l += @sizeOf(AbiTcb); - // The `ZigTcb` structure is right after the `AbiTcb` with no padding in between so it - // can be easily found. - l += @sizeOf(ZigTcb); - // It doesn't really matter where we put the DTV, so give it natural alignment. - l = alignForward(l, @alignOf(Dtv)); - dtv_offset = l; - l += @sizeOf(Dtv); - break :blk l; - }, - }; - - area_desc = .{ - .size = area_size, - .alignment = align_factor, - - .dtv = .{ - .offset = dtv_offset, - }, - - .abi_tcb = .{ - .offset = abi_tcb_offset, - }, - - .block = .{ - .init = block_init, - .offset = block_offset, - .size = block_size, - }, - - .gdt_entry_number = @as(usize, @bitCast(@as(isize, -1))), - }; -} - -/// Inline because TLS is not set up yet. -inline fn alignForward(addr: usize, alignment: usize) usize { - return alignBackward(addr + (alignment - 1), alignment); -} - -/// Inline because TLS is not set up yet. -inline fn alignBackward(addr: usize, alignment: usize) usize { - return addr & ~(alignment - 1); -} - -/// Inline because TLS is not set up yet. -inline fn alignPtrCast(comptime T: type, ptr: [*]u8) *T { - return @ptrCast(@alignCast(ptr)); -} - -/// Initializes all the fields of the static TLS area and returns the computed architecture-specific -/// value of the TP register. -pub fn prepareArea(area: []u8) usize { - @setRuntimeSafety(false); - @disableInstrumentation(); - - // Clear the area we're going to use, just to be safe. - @memset(area, 0); - - // Prepare the ABI TCB. - const abi_tcb = alignPtrCast(AbiTcb, area.ptr + area_desc.abi_tcb.offset); - switch (current_variant) { - .I_original, .I_modified => abi_tcb.dtv = @intFromPtr(area.ptr + area_desc.dtv.offset), - .II => abi_tcb.self = abi_tcb, - } - - // Prepare the DTV. - const dtv = alignPtrCast(Dtv, area.ptr + area_desc.dtv.offset); - dtv.len = 1; - dtv.tls_block = area.ptr + current_dtv_offset + area_desc.block.offset; - - // Copy the initial data. - @memcpy(area[area_desc.block.offset..][0..area_desc.block.init.len], area_desc.block.init); - - // Return the corrected value (if needed) for the TP register. Overflow here is not a problem; - // the pointer arithmetic involving the TP is done with wrapping semantics. - return @intFromPtr(area.ptr) +% switch (current_variant) { - .I_original, .II => area_desc.abi_tcb.offset, - .I_modified => area_desc.block.offset +% current_tp_offset, - }; -} - -/// The main motivation for the size chosen here is to be larger than total -/// amount of thread-local variables for most programs. Putting this allocation -/// in the ELF like this is equivalent to moving the `mmap` call below into the -/// kernel, avoiding syscall overhead. -var main_thread_area_buffer: [0x1000]u8 align(page_size_min) = undefined; - -/// Computes the layout of the static TLS area, allocates the area, initializes all of its fields, -/// and assigns the architecture-specific value to the TP register. -pub fn initStatic(phdrs: []elf.ElfN.Phdr) void { - @setRuntimeSafety(false); - @disableInstrumentation(); - - computeAreaDesc(phdrs); - - const area = blk: { - // Fast path for the common case where the TLS data is really small, avoid an allocation and - // use our local buffer. - if (area_desc.alignment <= page_size_min and area_desc.size <= main_thread_area_buffer.len) { - break :blk main_thread_area_buffer[0..area_desc.size]; - } - - const begin_addr = mmap_tls(area_desc.size + area_desc.alignment - 1); - if (@call(.always_inline, linux.errno, .{begin_addr}) != .SUCCESS) @trap(); - - const area_ptr: [*]align(page_size_min) u8 = @ptrFromInt(begin_addr); - - // Make sure the slice is correctly aligned. - const begin_aligned_addr = alignForward(begin_addr, area_desc.alignment); - const start = begin_aligned_addr - begin_addr; - break :blk area_ptr[start..][0..area_desc.size]; - }; - - const tp_value = prepareArea(area); - setThreadPointer(tp_value); -} - -inline fn mmap_tls(length: usize) usize { - const prot: linux.PROT = .{ .READ = true, .WRITE = true }; - const flags: linux.MAP = .{ .TYPE = .PRIVATE, .ANONYMOUS = true }; - - if (@hasField(linux.SYS, "mmap2")) { - return @call(.always_inline, linux.syscall6, .{ - .mmap2, - 0, - length, - @as(u32, @bitCast(prot)), - @as(u32, @bitCast(flags)), - @as(usize, @bitCast(@as(isize, -1))), - 0, - }); - } else { - // The s390x mmap() syscall existed before Linux supported syscalls with 5+ parameters, so - // it takes a single pointer to an array of arguments instead. - return if (native_arch == .s390x) @call(.always_inline, linux.syscall1, .{ - .mmap, - @intFromPtr(&[_]usize{ - 0, - length, - @as(u32, @bitCast(prot)), - @as(u32, @bitCast(flags)), - @as(usize, @bitCast(@as(isize, -1))), - 0, - }), - }) else @call(.always_inline, linux.syscall6, .{ - .mmap, - 0, - length, - @as(u32, @bitCast(prot)), - @as(u32, @bitCast(flags)), - @as(usize, @bitCast(@as(isize, -1))), - 0, - }); - } -} - -comptime { - assert(!builtin.link_libc); // otherwise libc should control TLS - - if (builtin.output_mode == .Exe and builtin.link_mode == .static) { - // This is a static executable without libc, so it is our job to provide the TLS accessor - // function for the GD and LD models. This function is unlikely to actually be used, since - // the linker should be able to relax every TLS access to the LE model and therefore - // eliminate all calls to this function, but that isn't guaranteed. - const Fns = struct { - const TlsIndex = switch (native_arch) { - .x86_64 => extern struct { module: u64, offset: u64 }, // Even for x32... - else => extern struct { module: usize, offset: usize }, // ...but not MIPS N32! - }; - fn __tls_get_addr(ti: *const TlsIndex) callconv(.c) *anyopaque { - comptime assert(native_arch != .s390x); - - assert(ti.module == 1); // The executable's module ID is always 1 - const tp = getThreadPointer(); - const block: [*]u8 = switch (current_variant) { - .I_original => @ptrFromInt(tp -% area_desc.abi_tcb.offset +% area_desc.block.offset), - .I_modified => @ptrFromInt(tp -% current_tp_offset), - // The `.I_original` approach would also work for `.II`, but there is an - // alternative strategy which is one less operation: - .II => @ptrFromInt(tp -% area_desc.block.size), - }; - return block[@intCast(ti.offset)..]; - } - fn __tls_get_offset() callconv(.naked) noreturn { - comptime assert(native_arch == .s390x); - - // We receive the module's GOT pointer in r12 and the GOT offset in r2. - asm volatile ( - \\ la %%r1, 0(%%r12, %%r2) - \\ lg %%r2, 8(%%r1) - \\ lgrl %%r0, %[block_size] - \\ sgr %%r2, %%r0 - \\ br %%r14 - : - : [block_size] "s" (&area_desc.block.size), - ); - } - }; - - if (native_arch == .s390x) - @export(&Fns.__tls_get_offset, .{ .name = "__tls_get_offset" }) - else - @export(&Fns.__tls_get_addr, .{ .name = "__tls_get_addr" }); - } -} diff --git a/lib/std/os/linux/vdso.zig b/lib/std/os/linux/vdso.zig index bdf46337b59ecc863a543d3722ddf47bd967374a..5add23f5c42bd6f93cfdc39825b42cc969d8e321 100644 --- a/lib/std/os/linux/vdso.zig +++ b/lib/std/os/linux/vdso.zig @@ -1,3 +1,4 @@ +const builtin = @import("builtin"); const std = @import("../../std.zig"); const elf = std.elf; const linux = std.os.linux; @@ -5,7 +6,8 @@ const mem = std.mem; const maxInt = std.math.maxInt; pub fn lookup(vername: []const u8, name: []const u8) usize { - const vdso_addr = linux.getauxval(std.elf.AT.SYSINFO_EHDR); + const getauxval = if (builtin.link_libc) std.c.getauxval else linux.getauxval; + const vdso_addr = getauxval(std.elf.AT.SYSINFO_EHDR); if (vdso_addr == 0) return 0; const eh = @as(*elf.Ehdr, @ptrFromInt(vdso_addr)); diff --git a/lib/std/start.zig b/lib/std/start.zig index 96190d835614718793d94832f9a0b72e415d964d..d77c093d23792f1422d4cfdc2e8d97a2af181b7d 100644 --- a/lib/std/start.zig +++ b/lib/std/start.zig @@ -596,6 +596,12 @@ fn wWinMain(hInstance: *anyopaque, hPrevInstance: ?*anyopaque, pCmdLine: [*:0]u1 return root.wWinMain(@ptrCast(hInstance), @ptrCast(hPrevInstance), pCmdLine, @intCast(nCmdShow)); } +/// Provided by compiler-rt. +extern var __zig_elf_auxv: [*]std.elf.Auxv; + +/// Provided by compiler-rt. +extern fn __zig_elf_static_tls_init(phdrs: [*]elf.ElfN.Phdr, phnum: usize) void; + fn posixCallMainAndExit(argc_argv_ptr: [*]usize) callconv(.c) noreturn { // We're not ready to panic until thread local storage is initialized. @setRuntimeSafety(false); @@ -613,7 +619,7 @@ fn posixCallMainAndExit(argc_argv_ptr: [*]usize) callconv(.c) noreturn { const auxv: [*]elf.Auxv = @ptrCast(@alignCast(envp.ptr + envp_count + 1)); var at_hwcap: usize = 0; - const phdrs = init: { + const phdrs: [*]elf.ElfN.Phdr, const phnum: usize = init: { var i: usize = 0; var at_phdr: usize = 0; var at_phnum: usize = 0; @@ -625,20 +631,20 @@ fn posixCallMainAndExit(argc_argv_ptr: [*]usize) callconv(.c) noreturn { else => continue, } } - break :init @as([*]elf.ElfN.Phdr, @ptrFromInt(at_phdr))[0..at_phnum]; + break :init .{ @ptrFromInt(at_phdr), at_phnum }; }; // Apply the initial relocations as early as possible in the startup process. We cannot // make calls yet on some architectures (e.g. MIPS) *because* they haven't been applied yet, // so this must be fully inlined. if (builtin.link_mode == .static and builtin.position_independent_executable) { - @call(.always_inline, std.pie.relocate, .{phdrs}); + @call(.always_inline, std.pie.relocate, .{phdrs[0..phnum]}); } if (native_os == .linux) { // This must be done after PIE relocations have been applied or we may crash // while trying to access the global variable (happens on MIPS at least). - std.os.linux.elf_aux_maybe = auxv; + __zig_elf_auxv = auxv; if (!builtin.single_threaded) { // ARMv6 targets (and earlier) have no support for TLS in hardware. @@ -654,14 +660,14 @@ fn posixCallMainAndExit(argc_argv_ptr: [*]usize) callconv(.c) noreturn { } // Initialize the TLS area. - std.os.linux.tls.initStatic(phdrs); + __zig_elf_static_tls_init(phdrs, phnum); } // The way Linux executables represent stack size is via the PT.GNU_STACK // program header. However the kernel does not recognize it; it always gives 8 MiB. // Here we look for the stack size in our program headers and use setrlimit // to ask for more stack space. - expandStackSize(phdrs); + expandStackSize(phdrs[0..phnum]); } const opt_init_array_start = @extern([*]const *const fn () callconv(.c) void, .{ diff --git a/src/codegen/c.zig b/src/codegen/c.zig index 4e58ede308fcaefe9b58ee073b3f1b21eaba8e76..c871f2d765bea5eac10790eebb9c4cff53276832 100644 --- a/src/codegen/c.zig +++ b/src/codegen/c.zig @@ -2552,6 +2552,13 @@ pub fn genDeclFwd(dg: *DeclGen, w: *Writer) Error!void { .{ .@"const" = nav.resolved.?.@"const" }, nav.resolved.?.@"align", ); + const extern_name = nav.name.toSlice(ip); + const is_mangled = isMangledIdent(extern_name, true); + if (is_mangled) { + try w.print(" zig_mangled({f}, {f})", .{ + fmtIdentSolo(extern_name), fmtStringLiteral(extern_name, null), + }); + } try w.writeAll(";\n"); return; }, diff --git a/test/standalone/mix_o_files/build.zig b/test/standalone/mix_o_files/build.zig index dcc5a09a0203d39440e92fdf0f3d7585c760e02f..3143834ffb3880ace94e618015902105e8104d94 100644 --- a/test/standalone/mix_o_files/build.zig +++ b/test/standalone/mix_o_files/build.zig @@ -13,6 +13,7 @@ pub fn build(b: *std.Build) void { .root_source_file = b.path("base64.zig"), .optimize = optimize, .target = target, + .link_libc = true, }), });