| ... | ... | @@ -1,3 +1,14 @@ |
| 1 | //! This file implements the two TLS variants [1] used by ELF-based systems. Note that, in reality, |
| 2 | //! Variant I has two sub-variants. |
| 3 | //! |
| 4 | //! It is important to understand that the term TCB (Thread Control Block) is overloaded here. |
| 5 | //! Official ABI documentation uses it simply to mean the ABI TCB, i.e. a small area of ABI-defined |
| 6 | //! data, usually one or two words (see the `AbiTcb` type below). People will also often use TCB to |
| 7 | //! refer to the libc TCB, which can be any size and contain anything. (One could even omit it!) We |
| 8 | //! refer to the latter as the Zig TCB; see the `ZigTcb` type below. |
| 9 | //! |
| 10 | //! [1] https://www.akkadia.org/drepper/tls.pdf |
| 11 | |
| 1 | 12 | const std = @import("std"); |
| 2 | 13 | const mem = std.mem; |
| 3 | 14 | const elf = std.elf; |
| ... | ... | @@ -7,56 +18,63 @@ const native_arch = @import("builtin").cpu.arch; |
| 7 | 18 | const linux = std.os.linux; |
| 8 | 19 | const posix = std.posix; |
| 9 | 20 | |
| 10 | | // This file implements the two TLS variants [1] used by ELF-based systems. |
| 11 | | // |
| 12 | | // The variant I has the following layout in memory: |
| 13 | | // ------------------------------------------------------- |
| 14 | | // | DTV | Zig | DTV | Alignment | TLS | |
| 15 | | // | storage | thread data | pointer | | block | |
| 16 | | // ------------------------^------------------------------ |
| 17 | | // `-- The thread pointer register points here |
| 18 | | // |
| 19 | | // In this case we allocate additional space for our control structure that's |
| 20 | | // placed _before_ the DTV pointer together with the DTV. |
| 21 | | // |
| 22 | | // NOTE: Some systems such as power64 or mips use this variant with a twist: the |
| 23 | | // alignment is not present and the tp and DTV addresses are offset by a |
| 24 | | // constant. |
| 25 | | // |
| 26 | | // On the other hand the variant II has the following layout in memory: |
| 27 | | // --------------------------------------- |
| 28 | | // | TLS | TCB | Zig | DTV | |
| 29 | | // | block | | thread data | storage | |
| 30 | | // --------^------------------------------ |
| 31 | | // `-- The thread pointer register points here |
| 32 | | // |
| 33 | | // The structure of the TCB is not defined by the ABI so we reserve enough space |
| 34 | | // for a single pointer as some architectures such as x86 and x86_64 need a |
| 35 | | // pointer to the TCB block itself at the address pointed by the tp. |
| 36 | | // |
| 37 | | // In this case the control structure and DTV are placed one after another right |
| 38 | | // after the TLS block data. |
| 39 | | // |
| 40 | | // At the moment the DTV is very simple since we only support static TLS, all we |
| 41 | | // need is a two word vector to hold the number of entries (1) and the address |
| 42 | | // of the first TLS block. |
| 43 | | // |
| 44 | | // [1] https://www.akkadia.org/drepper/tls.pdf |
| 45 | | |
| 46 | | const TLSVariant = enum { |
| 47 | | VariantI, |
| 48 | | VariantII, |
| 21 | /// Represents an ELF TLS variant. |
| 22 | /// |
| 23 | /// In all variants, the TP and the TLS blocks must be aligned to the `p_align` value in the |
| 24 | /// `PT_TLS` ELF program header. Everything else has natural alignment. |
| 25 | /// |
| 26 | /// The location of the DTV does not actually matter. For simplicity, we put it in the TLS area, but |
| 27 | /// there is no actual ABI requirement that it reside there. |
| 28 | const Variant = enum { |
| 29 | /// The original Variant I: |
| 30 | /// |
| 31 | /// ---------------------------------------- |
| 32 | /// | DTV | Zig TCB | ABI TCB | TLS Blocks | |
| 33 | /// ----------------^----------------------- |
| 34 | /// `-- The TP register points here. |
| 35 | /// |
| 36 | /// The layout in this variant necessitates separate alignment of both the TP and the TLS |
| 37 | /// blocks. |
| 38 | /// |
| 39 | /// The first word in the ABI TCB points to the DTV. For some architectures, there may be a |
| 40 | /// second word with an unspecified meaning. |
| 41 | I_original, |
| 42 | /// The modified Variant I: |
| 43 | /// |
| 44 | /// --------------------------------------------------- |
| 45 | /// | DTV | Zig TCB | ABI TCB | [Offset] | TLS Blocks | |
| 46 | /// -------------------------------------^------------- |
| 47 | /// `-- The TP register points here. |
| 48 | /// |
| 49 | /// The offset (which can be zero) is applied to the TP only; there is never physical gap |
| 50 | /// between the ABI TCB and the TLS blocks. This implies that we only need to align the TP. |
| 51 | /// |
| 52 | /// The first (and only) word in the ABI TCB points to the DTV. |
| 53 | I_modified, |
| 54 | /// Variant II: |
| 55 | /// |
| 56 | /// ---------------------------------------- |
| 57 | /// | TLS Blocks | ABI TCB | Zig TCB | DTV | |
| 58 | /// -------------^-------------------------- |
| 59 | /// `-- The TP register points here. |
| 60 | /// |
| 61 | /// The first (and only) word in the ABI TCB points to the ABI TCB itself. |
| 62 | II, |
| 49 | 63 | }; |
| 50 | 64 | |
| 51 | | const tls_variant = switch (native_arch) { |
| 65 | const current_variant: Variant = switch (native_arch) { |
| 66 | .arc, |
| 52 | 67 | .arm, |
| 53 | 68 | .armeb, |
| 54 | | .thumb, |
| 55 | | .thumbeb, |
| 56 | 69 | .aarch64, |
| 57 | 70 | .aarch64_be, |
| 58 | | .riscv32, |
| 59 | | .riscv64, |
| 71 | .csky, |
| 72 | .thumb, |
| 73 | .thumbeb, |
| 74 | => .I_original, |
| 75 | .loongarch32, |
| 76 | .loongarch64, |
| 77 | .m68k, |
| 60 | 78 | .mips, |
| 61 | 79 | .mipsel, |
| 62 | 80 | .mips64, |
| ... | ... | @@ -65,73 +83,130 @@ const tls_variant = switch (native_arch) { |
| 65 | 83 | .powerpcle, |
| 66 | 84 | .powerpc64, |
| 67 | 85 | .powerpc64le, |
| 68 | | => TLSVariant.VariantI, |
| 69 | | .x86_64, .x86, .sparc64 => TLSVariant.VariantII, |
| 70 | | else => @compileError("undefined tls_variant for this architecture"), |
| 71 | | }; |
| 72 | | |
| 73 | | // Controls how many bytes are reserved for the Thread Control Block |
| 74 | | const tls_tcb_size = switch (native_arch) { |
| 75 | | // ARM EABI mandates enough space for two pointers: the first one points to |
| 76 | | // the DTV while the second one is unspecified but reserved |
| 77 | | .arm, .armeb, .thumb, .thumbeb, .aarch64, .aarch64_be => 2 * @sizeOf(usize), |
| 78 | | // One pointer-sized word that points either to the DTV or the TCB itself |
| 79 | | else => @sizeOf(usize), |
| 86 | .riscv32, |
| 87 | .riscv64, |
| 88 | => .I_modified, |
| 89 | .hexagon, |
| 90 | .s390x, |
| 91 | .sparc64, |
| 92 | .x86, |
| 93 | .x86_64, |
| 94 | => .II, |
| 95 | else => @compileError("undefined TLS variant for this architecture"), |
| 80 | 96 | }; |
| 81 | 97 | |
| 82 | | // Controls if the TP points to the end of the TCB instead of its beginning |
| 83 | | const tls_tp_points_past_tcb = switch (native_arch) { |
| 84 | | .riscv32, .riscv64, .mips, .mipsel, .mips64, .mips64el, .powerpc, .powerpcle, .powerpc64, .powerpc64le => true, |
| 85 | | else => false, |
| 98 | /// The Offset value for the modified Variant I. |
| 99 | const current_tp_offset = switch (native_arch) { |
| 100 | .m68k, |
| 101 | .mips, |
| 102 | .mipsel, |
| 103 | .mips64, |
| 104 | .mips64el, |
| 105 | .powerpc, |
| 106 | .powerpcle, |
| 107 | .powerpc64, |
| 108 | .powerpc64le, |
| 109 | => 0x7000, |
| 110 | else => 0, |
| 86 | 111 | }; |
| 87 | 112 | |
| 88 | | // Some architectures add some offset to the tp and dtv addresses in order to |
| 89 | | // make the generated code more efficient |
| 90 | | |
| 91 | | const tls_tp_offset = switch (native_arch) { |
| 92 | | .mips, .mipsel, .mips64, .mips64el, .powerpc, .powerpcle, .powerpc64, .powerpc64le => 0x7000, |
| 113 | /// Usually only used by the modified Variant I. |
| 114 | const current_dtv_offset = switch (native_arch) { |
| 115 | .m68k, |
| 116 | .mips, |
| 117 | .mipsel, |
| 118 | .mips64, |
| 119 | .mips64el, |
| 120 | .powerpc, |
| 121 | .powerpcle, |
| 122 | .powerpc64, |
| 123 | .powerpc64le, |
| 124 | => 0x8000, |
| 125 | .riscv32, |
| 126 | .riscv64, |
| 127 | => 0x800, |
| 93 | 128 | else => 0, |
| 94 | 129 | }; |
| 95 | 130 | |
| 96 | | const tls_dtv_offset = switch (native_arch) { |
| 97 | | .mips, .mipsel, .mips64, .mips64el, .powerpc, .powerpcle, .powerpc64, .powerpc64le => 0x8000, |
| 98 | | .riscv32, .riscv64 => 0x800, |
| 99 | | else => 0, |
| 131 | /// Per-thread storage for the ELF TLS ABI. |
| 132 | const AbiTcb = switch (current_variant) { |
| 133 | .I_original, .I_modified => switch (native_arch) { |
| 134 | // ARM EABI mandates enough space for two pointers: the first one points to the DTV as |
| 135 | // usual, while the second one is unspecified. |
| 136 | .aarch64, |
| 137 | .aarch64_be, |
| 138 | .arm, |
| 139 | .armeb, |
| 140 | .thumb, |
| 141 | .thumbeb, |
| 142 | => extern struct { |
| 143 | /// This is offset by `current_dtv_offset`. |
| 144 | dtv: usize, |
| 145 | reserved: ?*anyopaque, |
| 146 | }, |
| 147 | else => extern struct { |
| 148 | /// This is offset by `current_dtv_offset`. |
| 149 | dtv: usize, |
| 150 | }, |
| 151 | }, |
| 152 | .II => extern struct { |
| 153 | /// This is self-referential. |
| 154 | self: *AbiTcb, |
| 155 | }, |
| 100 | 156 | }; |
| 101 | 157 | |
| 102 | | // Per-thread storage for Zig's use |
| 103 | | const CustomData = struct { |
| 158 | /// Per-thread storage for Zig's use. Currently unused. |
| 159 | const ZigTcb = struct { |
| 104 | 160 | dummy: usize, |
| 105 | 161 | }; |
| 106 | 162 | |
| 107 | | // Dynamic Thread Vector |
| 108 | | const DTV = extern struct { |
| 109 | | entries: usize, |
| 110 | | tls_block: [1][*]u8, |
| 163 | /// Dynamic Thread Vector as specified in the ELF TLS ABI. Ordinarily, there is a block pointer per |
| 164 | /// dynamically-loaded module, but since we only support static TLS, we only need one block pointer. |
| 165 | const Dtv = extern struct { |
| 166 | len: usize = 1, |
| 167 | tls_block: [*]u8, |
| 111 | 168 | }; |
| 112 | 169 | |
| 113 | | // Holds all the information about the process TLS image |
| 114 | | const TLSImage = struct { |
| 115 | | init_data: []const u8, |
| 116 | | alloc_size: usize, |
| 117 | | alloc_align: usize, |
| 118 | | tcb_offset: usize, |
| 119 | | dtv_offset: usize, |
| 120 | | data_offset: usize, |
| 121 | | data_size: usize, |
| 122 | | // Only used on the x86 architecture |
| 170 | /// Describes a process's TLS area. The area encompasses the DTV, both TCBs, and the TLS block, with |
| 171 | /// the exact layout of these being dependent primarily on `current_variant`. |
| 172 | const AreaDesc = struct { |
| 173 | size: usize, |
| 174 | alignment: usize, |
| 175 | |
| 176 | dtv: struct { |
| 177 | /// Offset into the TLS area. |
| 178 | offset: usize, |
| 179 | }, |
| 180 | |
| 181 | abi_tcb: struct { |
| 182 | /// Offset into the TLS area. |
| 183 | offset: usize, |
| 184 | }, |
| 185 | |
| 186 | block: struct { |
| 187 | /// The initial data to be copied into the TLS block. Note that this may be smaller than |
| 188 | /// `size`, in which case any remaining data in the TLS block is simply left uninitialized. |
| 189 | init: []const u8, |
| 190 | /// Offset into the TLS area. |
| 191 | offset: usize, |
| 192 | /// This is the effective size of the TLS block, which may be greater than `init.len`. |
| 193 | size: usize, |
| 194 | }, |
| 195 | |
| 196 | /// Only used on the 32-bit x86 architecture (not x86_64, nor x32). |
| 123 | 197 | gdt_entry_number: usize, |
| 124 | 198 | }; |
| 125 | 199 | |
| 126 | | pub var tls_image: TLSImage = undefined; |
| 200 | pub var area_desc: AreaDesc = undefined; |
| 127 | 201 | |
| 128 | 202 | pub fn setThreadPointer(addr: usize) void { |
| 129 | 203 | @setRuntimeSafety(false); |
| 130 | 204 | @disableInstrumentation(); |
| 205 | |
| 131 | 206 | switch (native_arch) { |
| 132 | 207 | .x86 => { |
| 133 | 208 | var user_desc: linux.user_desc = .{ |
| 134 | | .entry_number = tls_image.gdt_entry_number, |
| 209 | .entry_number = area_desc.gdt_entry_number, |
| 135 | 210 | .base_addr = addr, |
| 136 | 211 | .limit = 0xfffff, |
| 137 | 212 | .flags = .{ |
| ... | ... | @@ -148,7 +223,7 @@ pub fn setThreadPointer(addr: usize) void { |
| 148 | 223 | |
| 149 | 224 | const gdt_entry_number = user_desc.entry_number; |
| 150 | 225 | // We have to keep track of our slot as it's also needed for clone() |
| 151 | | tls_image.gdt_entry_number = gdt_entry_number; |
| 226 | area_desc.gdt_entry_number = gdt_entry_number; |
| 152 | 227 | // Update the %gs selector |
| 153 | 228 | asm volatile ("movl %[gs_val], %%gs" |
| 154 | 229 | : |
| ... | ... | @@ -166,10 +241,38 @@ pub fn setThreadPointer(addr: usize) void { |
| 166 | 241 | : [addr] "r" (addr), |
| 167 | 242 | ); |
| 168 | 243 | }, |
| 244 | .arc => { |
| 245 | // We apparently need to both set r25 (TP) *and* inform the kernel... |
| 246 | asm volatile ( |
| 247 | \\ mov r25, %[addr] |
| 248 | : |
| 249 | : [addr] "r" (addr), |
| 250 | ); |
| 251 | const rc = @call(.always_inline, linux.syscall1, .{ .arc_settls, addr }); |
| 252 | assert(rc == 0); |
| 253 | }, |
| 169 | 254 | .arm, .armeb, .thumb, .thumbeb => { |
| 170 | 255 | const rc = @call(.always_inline, linux.syscall1, .{ .set_tls, addr }); |
| 171 | 256 | assert(rc == 0); |
| 172 | 257 | }, |
| 258 | .m68k => { |
| 259 | const rc = linux.syscall1(.set_thread_area, addr); |
| 260 | assert(rc == 0); |
| 261 | }, |
| 262 | .hexagon => { |
| 263 | asm volatile ( |
| 264 | \\ ugp = %[addr] |
| 265 | : |
| 266 | : [addr] "r" (addr), |
| 267 | ); |
| 268 | }, |
| 269 | .loongarch32, .loongarch64 => { |
| 270 | asm volatile ( |
| 271 | \\ mv tp, %[addr] |
| 272 | : |
| 273 | : [addr] "r" (addr), |
| 274 | ); |
| 275 | }, |
| 173 | 276 | .riscv32, .riscv64 => { |
| 174 | 277 | asm volatile ( |
| 175 | 278 | \\ mv tp, %[addr] |
| ... | ... | @@ -177,7 +280,7 @@ pub fn setThreadPointer(addr: usize) void { |
| 177 | 280 | : [addr] "r" (addr), |
| 178 | 281 | ); |
| 179 | 282 | }, |
| 180 | | .mips, .mipsel, .mips64, .mips64el => { |
| 283 | .csky, .mips, .mipsel, .mips64, .mips64el => { |
| 181 | 284 | const rc = @call(.always_inline, linux.syscall1, .{ .set_thread_area, addr }); |
| 182 | 285 | assert(rc == 0); |
| 183 | 286 | }, |
| ... | ... | @@ -195,6 +298,17 @@ pub fn setThreadPointer(addr: usize) void { |
| 195 | 298 | : [addr] "r" (addr), |
| 196 | 299 | ); |
| 197 | 300 | }, |
| 301 | .s390x => { |
| 302 | asm volatile ( |
| 303 | \\ lgr %%r0, %[addr] |
| 304 | \\ sar %%a1, %%r0 |
| 305 | \\ srlg %%r0, %%r0, 32 |
| 306 | \\ sar %%a0, %%r0 |
| 307 | : |
| 308 | : [addr] "r" (addr), |
| 309 | : "r0" |
| 310 | ); |
| 311 | }, |
| 198 | 312 | .sparc64 => { |
| 199 | 313 | asm volatile ( |
| 200 | 314 | \\ mov %[addr], %%g7 |
| ... | ... | @@ -206,7 +320,7 @@ pub fn setThreadPointer(addr: usize) void { |
| 206 | 320 | } |
| 207 | 321 | } |
| 208 | 322 | |
| 209 | | fn initTLS(phdrs: []elf.Phdr) void { |
| 323 | fn computeAreaDesc(phdrs: []elf.Phdr) void { |
| 210 | 324 | @setRuntimeSafety(false); |
| 211 | 325 | @disableInstrumentation(); |
| 212 | 326 | |
| ... | ... | @@ -221,72 +335,103 @@ fn initTLS(phdrs: []elf.Phdr) void { |
| 221 | 335 | } |
| 222 | 336 | } |
| 223 | 337 | |
| 224 | | var tls_align_factor: usize = undefined; |
| 225 | | var tls_data: []const u8 = undefined; |
| 226 | | var tls_data_alloc_size: usize = undefined; |
| 338 | var align_factor: usize = undefined; |
| 339 | var block_init: []const u8 = undefined; |
| 340 | var block_size: usize = undefined; |
| 341 | |
| 227 | 342 | if (tls_phdr) |phdr| { |
| 228 | | // The effective size in memory is represented by p_memsz, the length of |
| 229 | | // the data stored in the PT_TLS segment is p_filesz and may be less |
| 230 | | // than the former |
| 231 | | tls_align_factor = phdr.p_align; |
| 232 | | tls_data = @as([*]u8, @ptrFromInt(img_base + phdr.p_vaddr))[0..phdr.p_filesz]; |
| 233 | | tls_data_alloc_size = phdr.p_memsz; |
| 343 | align_factor = phdr.p_align; |
| 344 | |
| 345 | // The effective size in memory is represented by `p_memsz`; the length of the data stored |
| 346 | // in the `PT_TLS` segment is `p_filesz` and may be less than the former. |
| 347 | block_init = @as([*]u8, @ptrFromInt(img_base + phdr.p_vaddr))[0..phdr.p_filesz]; |
| 348 | block_size = phdr.p_memsz; |
| 234 | 349 | } else { |
| 235 | | tls_align_factor = @alignOf(usize); |
| 236 | | tls_data = &[_]u8{}; |
| 237 | | tls_data_alloc_size = 0; |
| 350 | align_factor = @alignOf(usize); |
| 351 | |
| 352 | block_init = &[_]u8{}; |
| 353 | block_size = 0; |
| 238 | 354 | } |
| 239 | 355 | |
| 240 | | // Offsets into the allocated TLS area |
| 241 | | var tcb_offset: usize = undefined; |
| 356 | // Offsets into the allocated TLS area. |
| 242 | 357 | var dtv_offset: usize = undefined; |
| 243 | | var data_offset: usize = undefined; |
| 244 | | // Compute the total size of the ABI-specific data plus our own control |
| 245 | | // structures. All the offset calculated here assume a well-aligned base |
| 246 | | // address. |
| 247 | | const alloc_size = switch (tls_variant) { |
| 248 | | .VariantI => blk: { |
| 358 | var abi_tcb_offset: usize = undefined; |
| 359 | var block_offset: usize = undefined; |
| 360 | |
| 361 | // Compute the total size of the ABI-specific data plus our own `ZigTcb` structure. All the |
| 362 | // offsets calculated here assume a well-aligned base address. |
| 363 | const area_size = switch (current_variant) { |
| 364 | .I_original => blk: { |
| 365 | var l: usize = 0; |
| 366 | dtv_offset = l; |
| 367 | l += @sizeOf(Dtv); |
| 368 | // Add some padding here so that the TP (`abi_tcb_offset`) is aligned to `align_factor` |
| 369 | // and the `ZigTcb` structure can be found by simply subtracting `@sizeOf(ZigTcb)` from |
| 370 | // the TP. |
| 371 | const delta = (l + @sizeOf(ZigTcb)) & (align_factor - 1); |
| 372 | if (delta > 0) |
| 373 | l += align_factor - delta; |
| 374 | l += @sizeOf(ZigTcb); |
| 375 | abi_tcb_offset = l; |
| 376 | l += alignForward(@sizeOf(AbiTcb), align_factor); |
| 377 | block_offset = l; |
| 378 | l += block_size; |
| 379 | break :blk l; |
| 380 | }, |
| 381 | .I_modified => blk: { |
| 249 | 382 | var l: usize = 0; |
| 250 | 383 | dtv_offset = l; |
| 251 | | l += @sizeOf(DTV); |
| 252 | | // Add some padding here so that the thread pointer (tcb_offset) is |
| 253 | | // aligned to p_align and the CustomData structure can be found by |
| 254 | | // simply subtracting its @sizeOf from the tp value |
| 255 | | const delta = (l + @sizeOf(CustomData)) & (tls_align_factor - 1); |
| 384 | l += @sizeOf(Dtv); |
| 385 | // In this variant, the TLS blocks must begin immediately after the end of the ABI TCB, |
| 386 | // with the TP pointing to the beginning of the TLS blocks. Add padding so that the TP |
| 387 | // (`abi_tcb_offset`) is aligned to `align_factor` and the `ZigTcb` structure can be |
| 388 | // found by subtracting `@sizeOf(AbiTcb) + @sizeOf(ZigTcb)` from the TP. |
| 389 | const delta = (l + @sizeOf(ZigTcb) + @sizeOf(AbiTcb)) & (align_factor - 1); |
| 256 | 390 | if (delta > 0) |
| 257 | | l += tls_align_factor - delta; |
| 258 | | l += @sizeOf(CustomData); |
| 259 | | tcb_offset = l; |
| 260 | | l += alignForward(tls_tcb_size, tls_align_factor); |
| 261 | | data_offset = l; |
| 262 | | l += tls_data_alloc_size; |
| 391 | l += align_factor - delta; |
| 392 | l += @sizeOf(ZigTcb); |
| 393 | abi_tcb_offset = l; |
| 394 | l += @sizeOf(AbiTcb); |
| 395 | block_offset = l; |
| 396 | l += block_size; |
| 263 | 397 | break :blk l; |
| 264 | 398 | }, |
| 265 | | .VariantII => blk: { |
| 399 | .II => blk: { |
| 266 | 400 | var l: usize = 0; |
| 267 | | data_offset = l; |
| 268 | | l += alignForward(tls_data_alloc_size, tls_align_factor); |
| 269 | | // The thread pointer is aligned to p_align |
| 270 | | tcb_offset = l; |
| 271 | | l += tls_tcb_size; |
| 272 | | // The CustomData structure is right after the TCB with no padding |
| 273 | | // in between so it can be easily found |
| 274 | | l += @sizeOf(CustomData); |
| 275 | | l = alignForward(l, @alignOf(DTV)); |
| 401 | block_offset = l; |
| 402 | l += alignForward(block_size, align_factor); |
| 403 | // The TP is aligned to `align_factor`. |
| 404 | abi_tcb_offset = l; |
| 405 | l += @sizeOf(AbiTcb); |
| 406 | // The `ZigTcb` structure is right after the `AbiTcb` with no padding in between so it |
| 407 | // can be easily found. |
| 408 | l += @sizeOf(ZigTcb); |
| 409 | // It doesn't really matter where we put the DTV, so give it natural alignment. |
| 410 | l = alignForward(l, @alignOf(Dtv)); |
| 276 | 411 | dtv_offset = l; |
| 277 | | l += @sizeOf(DTV); |
| 412 | l += @sizeOf(Dtv); |
| 278 | 413 | break :blk l; |
| 279 | 414 | }, |
| 280 | 415 | }; |
| 281 | 416 | |
| 282 | | tls_image = TLSImage{ |
| 283 | | .init_data = tls_data, |
| 284 | | .alloc_size = alloc_size, |
| 285 | | .alloc_align = tls_align_factor, |
| 286 | | .tcb_offset = tcb_offset, |
| 287 | | .dtv_offset = dtv_offset, |
| 288 | | .data_offset = data_offset, |
| 289 | | .data_size = tls_data_alloc_size, |
| 417 | area_desc = .{ |
| 418 | .size = area_size, |
| 419 | .alignment = align_factor, |
| 420 | |
| 421 | .dtv = .{ |
| 422 | .offset = dtv_offset, |
| 423 | }, |
| 424 | |
| 425 | .abi_tcb = .{ |
| 426 | .offset = abi_tcb_offset, |
| 427 | }, |
| 428 | |
| 429 | .block = .{ |
| 430 | .init = block_init, |
| 431 | .offset = block_offset, |
| 432 | .size = block_size, |
| 433 | }, |
| 434 | |
| 290 | 435 | .gdt_entry_number = @as(usize, @bitCast(@as(isize, -1))), |
| 291 | 436 | }; |
| 292 | 437 | } |
| ... | ... | @@ -306,78 +451,80 @@ inline fn alignPtrCast(comptime T: type, ptr: [*]u8) *T { |
| 306 | 451 | return @ptrCast(@alignCast(ptr)); |
| 307 | 452 | } |
| 308 | 453 | |
| 309 | | /// Initializes all the fields of the static TLS area and returns the computed |
| 310 | | /// architecture-specific value of the thread-pointer register |
| 311 | | /// |
| 312 | | /// This function is inline because thread local storage is not set up yet. |
| 313 | | pub fn prepareTLS(area: []u8) usize { |
| 454 | /// Initializes all the fields of the static TLS area and returns the computed architecture-specific |
| 455 | /// value of the TP register. |
| 456 | pub fn prepareArea(area: []u8) usize { |
| 314 | 457 | @setRuntimeSafety(false); |
| 315 | 458 | @disableInstrumentation(); |
| 316 | | // Clear the area we're going to use, just to be safe |
| 459 | |
| 460 | // Clear the area we're going to use, just to be safe. |
| 317 | 461 | @memset(area, 0); |
| 318 | | // Prepare the DTV |
| 319 | | const dtv = alignPtrCast(DTV, area.ptr + tls_image.dtv_offset); |
| 320 | | dtv.entries = 1; |
| 321 | | dtv.tls_block[0] = area.ptr + tls_dtv_offset + tls_image.data_offset; |
| 322 | | // Prepare the TCB |
| 323 | | const tcb_ptr = alignPtrCast([*]u8, area.ptr + tls_image.tcb_offset); |
| 324 | | tcb_ptr.* = switch (tls_variant) { |
| 325 | | .VariantI => area.ptr + tls_image.dtv_offset, |
| 326 | | .VariantII => area.ptr + tls_image.tcb_offset, |
| 462 | |
| 463 | // Prepare the ABI TCB. |
| 464 | const abi_tcb = alignPtrCast(AbiTcb, area.ptr + area_desc.abi_tcb.offset); |
| 465 | switch (current_variant) { |
| 466 | .I_original, .I_modified => abi_tcb.dtv = @intFromPtr(area.ptr + area_desc.dtv.offset), |
| 467 | .II => abi_tcb.self = abi_tcb, |
| 468 | } |
| 469 | |
| 470 | // Prepare the DTV. |
| 471 | const dtv = alignPtrCast(Dtv, area.ptr + area_desc.dtv.offset); |
| 472 | dtv.len = 1; |
| 473 | dtv.tls_block = area.ptr + current_dtv_offset + area_desc.block.offset; |
| 474 | |
| 475 | // Copy the initial data. |
| 476 | @memcpy(area[area_desc.block.offset..][0..area_desc.block.init.len], area_desc.block.init); |
| 477 | |
| 478 | // Return the corrected value (if needed) for the TP register. Overflow here is not a problem; |
| 479 | // the pointer arithmetic involving the TP is done with wrapping semantics. |
| 480 | return @intFromPtr(area.ptr) +% switch (current_variant) { |
| 481 | .I_original, .II => area_desc.abi_tcb.offset, |
| 482 | .I_modified => area_desc.block.offset +% current_tp_offset, |
| 327 | 483 | }; |
| 328 | | // Copy the data |
| 329 | | @memcpy(area[tls_image.data_offset..][0..tls_image.init_data.len], tls_image.init_data); |
| 330 | | |
| 331 | | // Return the corrected value (if needed) for the tp register. |
| 332 | | // Overflow here is not a problem, the pointer arithmetic involving the tp |
| 333 | | // is done with wrapping semantics. |
| 334 | | return @intFromPtr(area.ptr) +% tls_tp_offset +% |
| 335 | | if (tls_tp_points_past_tcb) tls_image.data_offset else tls_image.tcb_offset; |
| 336 | 484 | } |
| 337 | 485 | |
| 338 | | // The main motivation for the size chosen here is this is how much ends up being |
| 339 | | // requested for the thread local variables of the std.crypto.random implementation. |
| 340 | | // I'm not sure why it ends up being so much; the struct itself is only 64 bytes. |
| 341 | | // I think it has to do with being page aligned and LLVM or LLD is not smart enough |
| 342 | | // to lay out the TLS data in a space conserving way. Anyway I think it's fine |
| 343 | | // because it's less than 3 pages of memory, and putting it in the ELF like this |
| 344 | | // is equivalent to moving the mmap call below into the kernel, avoiding syscall |
| 345 | | // overhead. |
| 346 | | var main_thread_tls_buffer: [0x2100]u8 align(mem.page_size) = undefined; |
| 347 | | |
| 348 | | pub fn initStaticTLS(phdrs: []elf.Phdr) void { |
| 486 | // The main motivation for the size chosen here is that this is how much ends up being requested for |
| 487 | // the thread-local variables of the `std.crypto.random` implementation. I'm not sure why it ends up |
| 488 | // being so much; the struct itself is only 64 bytes. I think it has to do with being page-aligned |
| 489 | // and LLVM or LLD is not smart enough to lay out the TLS data in a space-conserving way. Anyway, I |
| 490 | // think it's fine because it's less than 3 pages of memory, and putting it in the ELF like this is |
| 491 | // equivalent to moving the `mmap` call below into the kernel, avoiding syscall overhead. |
| 492 | var main_thread_area_buffer: [0x2100]u8 align(mem.page_size) = undefined; |
| 493 | |
| 494 | /// Computes the layout of the static TLS area, allocates the area, initializes all of its fields, |
| 495 | /// and assigns the architecture-specific value to the TP register. |
| 496 | pub fn initStatic(phdrs: []elf.Phdr) void { |
| 349 | 497 | @setRuntimeSafety(false); |
| 350 | 498 | @disableInstrumentation(); |
| 351 | 499 | |
| 352 | | initTLS(phdrs); |
| 500 | computeAreaDesc(phdrs); |
| 353 | 501 | |
| 354 | | const tls_area = blk: { |
| 355 | | // Fast path for the common case where the TLS data is really small, |
| 356 | | // avoid an allocation and use our local buffer. |
| 357 | | if (tls_image.alloc_align <= mem.page_size and |
| 358 | | tls_image.alloc_size <= main_thread_tls_buffer.len) |
| 359 | | { |
| 360 | | break :blk main_thread_tls_buffer[0..tls_image.alloc_size]; |
| 502 | const area = blk: { |
| 503 | // Fast path for the common case where the TLS data is really small, avoid an allocation and |
| 504 | // use our local buffer. |
| 505 | if (area_desc.alignment <= mem.page_size and area_desc.size <= main_thread_area_buffer.len) { |
| 506 | break :blk main_thread_area_buffer[0..area_desc.size]; |
| 361 | 507 | } |
| 362 | 508 | |
| 363 | 509 | const begin_addr = mmap( |
| 364 | 510 | null, |
| 365 | | tls_image.alloc_size + tls_image.alloc_align - 1, |
| 511 | area_desc.size + area_desc.alignment - 1, |
| 366 | 512 | posix.PROT.READ | posix.PROT.WRITE, |
| 367 | 513 | .{ .TYPE = .PRIVATE, .ANONYMOUS = true }, |
| 368 | 514 | -1, |
| 369 | 515 | 0, |
| 370 | 516 | ); |
| 371 | 517 | if (@as(isize, @bitCast(begin_addr)) < 0) @trap(); |
| 372 | | const alloc_tls_area: [*]align(mem.page_size) u8 = @ptrFromInt(begin_addr); |
| 518 | |
| 519 | const area_ptr: [*]align(mem.page_size) u8 = @ptrFromInt(begin_addr); |
| 373 | 520 | |
| 374 | 521 | // Make sure the slice is correctly aligned. |
| 375 | | const begin_aligned_addr = alignForward(begin_addr, tls_image.alloc_align); |
| 522 | const begin_aligned_addr = alignForward(begin_addr, area_desc.alignment); |
| 376 | 523 | const start = begin_aligned_addr - begin_addr; |
| 377 | | break :blk alloc_tls_area[start..][0..tls_image.alloc_size]; |
| 524 | break :blk area_ptr[start..][0..area_desc.size]; |
| 378 | 525 | }; |
| 379 | 526 | |
| 380 | | const tp_value = prepareTLS(tls_area); |
| 527 | const tp_value = prepareArea(area); |
| 381 | 528 | setThreadPointer(tp_value); |
| 382 | 529 | } |
| 383 | 530 | |
Andrew Kelley <andrew@ziglang.org> 2024-08-01 01:15:17-07:00
GitHub <noreply@github.com> 2024-08-01 01:15:17-07:00
Signed by PGP key B5690EEEBB952194
lib