| ... | ... | @@ -1,3 +1,14 @@ |
| 1 | //! This file implements the two TLS variants [1] used by ELF-based systems. Note that, in reality, |
| 2 | //! Variant I has two sub-variants. |
| 3 | //! |
| 4 | //! It is important to understand that the term TCB (Thread Control Block) is overloaded here. |
| 5 | //! Official ABI documentation uses it simply to mean the ABI TCB, i.e. a small area of ABI-defined |
| 6 | //! data, usually one or two words (see the `AbiTcb` type below). People will also often use TCB to |
| 7 | //! refer to the libc TCB, which can be any size and contain anything. (One could even omit it!) We |
| 8 | //! refer to the latter as the Zig TCB; see the `ZigTcb` type below. |
| 9 | //! |
| 10 | //! [1] https://www.akkadia.org/drepper/tls.pdf |
| 11 | |
| 1 | 12 | const std = @import("std"); |
| 2 | 13 | const mem = std.mem; |
| 3 | 14 | const elf = std.elf; |
| ... | ... | @@ -7,56 +18,58 @@ const native_arch = @import("builtin").cpu.arch; |
| 7 | 18 | const linux = std.os.linux; |
| 8 | 19 | const posix = std.posix; |
| 9 | 20 | |
| 10 | | // This file implements the two TLS variants [1] used by ELF-based systems. |
| 11 | | // |
| 12 | | // The variant I has the following layout in memory: |
| 13 | | // ------------------------------------------------------- |
| 14 | | // | DTV | Zig | DTV | Alignment | TLS | |
| 15 | | // | storage | thread data | pointer | | block | |
| 16 | | // ------------------------^------------------------------ |
| 17 | | // `-- The thread pointer register points here |
| 18 | | // |
| 19 | | // In this case we allocate additional space for our control structure that's |
| 20 | | // placed _before_ the DTV pointer together with the DTV. |
| 21 | | // |
| 22 | | // NOTE: Some systems such as power64 or mips use this variant with a twist: the |
| 23 | | // alignment is not present and the tp and DTV addresses are offset by a |
| 24 | | // constant. |
| 25 | | // |
| 26 | | // On the other hand the variant II has the following layout in memory: |
| 27 | | // --------------------------------------- |
| 28 | | // | TLS | TCB | Zig | DTV | |
| 29 | | // | block | | thread data | storage | |
| 30 | | // --------^------------------------------ |
| 31 | | // `-- The thread pointer register points here |
| 32 | | // |
| 33 | | // The structure of the TCB is not defined by the ABI so we reserve enough space |
| 34 | | // for a single pointer as some architectures such as x86 and x86_64 need a |
| 35 | | // pointer to the TCB block itself at the address pointed by the tp. |
| 36 | | // |
| 37 | | // In this case the control structure and DTV are placed one after another right |
| 38 | | // after the TLS block data. |
| 39 | | // |
| 40 | | // At the moment the DTV is very simple since we only support static TLS, all we |
| 41 | | // need is a two word vector to hold the number of entries (1) and the address |
| 42 | | // of the first TLS block. |
| 43 | | // |
| 44 | | // [1] https://www.akkadia.org/drepper/tls.pdf |
| 45 | | |
| 46 | | const TLSVariant = enum { |
| 47 | | VariantI, |
| 48 | | VariantII, |
| 21 | /// Represents an ELF TLS variant. |
| 22 | /// |
| 23 | /// In all variants, the TP and the TLS blocks must be aligned to the `p_align` value in the |
| 24 | /// `PT_TLS` ELF program header. Everything else has natural alignment. |
| 25 | /// |
| 26 | /// The location of the DTV does not actually matter. For simplicity, we put it in the TLS area, but |
| 27 | /// there is no actual ABI requirement that it reside there. |
| 28 | const Variant = enum { |
| 29 | /// The original Variant I: |
| 30 | /// |
| 31 | /// ---------------------------------------- |
| 32 | /// | DTV | Zig TCB | ABI TCB | TLS Blocks | |
| 33 | /// ----------------^----------------------- |
| 34 | /// `-- The TP register points here. |
| 35 | /// |
| 36 | /// The layout in this variant necessitates separate alignment of both the TP and the TLS |
| 37 | /// blocks. |
| 38 | /// |
| 39 | /// The first word in the ABI TCB points to the DTV. For some architectures, there may be a |
| 40 | /// second word with an unspecified meaning. |
| 41 | I_original, |
| 42 | /// The modified Variant I: |
| 43 | /// |
| 44 | /// --------------------------------------------------- |
| 45 | /// | DTV | Zig TCB | ABI TCB | [Offset] | TLS Blocks | |
| 46 | /// -------------------------------------^------------- |
| 47 | /// `-- The TP register points here. |
| 48 | /// |
| 49 | /// The offset (which can be zero) is applied to the TP only; there is never physical gap |
| 50 | /// between the ABI TCB and the TLS blocks. This implies that we only need to align the TP. |
| 51 | /// |
| 52 | /// The first (and only) word in the ABI TCB points to the DTV. |
| 53 | I_modified, |
| 54 | /// Variant II: |
| 55 | /// |
| 56 | /// ---------------------------------------- |
| 57 | /// | TLS Blocks | ABI TCB | Zig TCB | DTV | |
| 58 | /// -------------^-------------------------- |
| 59 | /// `-- The TP register points here. |
| 60 | /// |
| 61 | /// The first (and only) word in the ABI TCB points to the ABI TCB itself. |
| 62 | II, |
| 49 | 63 | }; |
| 50 | 64 | |
| 51 | | const tls_variant = switch (native_arch) { |
| 65 | const current_variant: Variant = switch (native_arch) { |
| 52 | 66 | .arm, |
| 53 | 67 | .armeb, |
| 54 | | .thumb, |
| 55 | | .thumbeb, |
| 56 | 68 | .aarch64, |
| 57 | 69 | .aarch64_be, |
| 58 | | .riscv32, |
| 59 | | .riscv64, |
| 70 | .thumb, |
| 71 | .thumbeb, |
| 72 | => .I_original, |
| 60 | 73 | .mips, |
| 61 | 74 | .mipsel, |
| 62 | 75 | .mips64, |
| ... | ... | @@ -65,73 +78,126 @@ const tls_variant = switch (native_arch) { |
| 65 | 78 | .powerpcle, |
| 66 | 79 | .powerpc64, |
| 67 | 80 | .powerpc64le, |
| 68 | | => TLSVariant.VariantI, |
| 69 | | .x86_64, .x86, .sparc64 => TLSVariant.VariantII, |
| 70 | | else => @compileError("undefined tls_variant for this architecture"), |
| 71 | | }; |
| 72 | | |
| 73 | | // Controls how many bytes are reserved for the Thread Control Block |
| 74 | | const tls_tcb_size = switch (native_arch) { |
| 75 | | // ARM EABI mandates enough space for two pointers: the first one points to |
| 76 | | // the DTV while the second one is unspecified but reserved |
| 77 | | .arm, .armeb, .thumb, .thumbeb, .aarch64, .aarch64_be => 2 * @sizeOf(usize), |
| 78 | | // One pointer-sized word that points either to the DTV or the TCB itself |
| 79 | | else => @sizeOf(usize), |
| 81 | .riscv32, |
| 82 | .riscv64, |
| 83 | => .I_modified, |
| 84 | .sparc64, |
| 85 | .x86, |
| 86 | .x86_64, |
| 87 | => .II, |
| 88 | else => @compileError("undefined TLS variant for this architecture"), |
| 80 | 89 | }; |
| 81 | 90 | |
| 82 | | // Controls if the TP points to the end of the TCB instead of its beginning |
| 83 | | const tls_tp_points_past_tcb = switch (native_arch) { |
| 84 | | .riscv32, .riscv64, .mips, .mipsel, .mips64, .mips64el, .powerpc, .powerpcle, .powerpc64, .powerpc64le => true, |
| 85 | | else => false, |
| 91 | /// The Offset value for the modified Variant I. |
| 92 | const current_tp_offset = switch (native_arch) { |
| 93 | .mips, |
| 94 | .mipsel, |
| 95 | .mips64, |
| 96 | .mips64el, |
| 97 | .powerpc, |
| 98 | .powerpcle, |
| 99 | .powerpc64, |
| 100 | .powerpc64le, |
| 101 | => 0x7000, |
| 102 | else => 0, |
| 86 | 103 | }; |
| 87 | 104 | |
| 88 | | // Some architectures add some offset to the tp and dtv addresses in order to |
| 89 | | // make the generated code more efficient |
| 90 | | |
| 91 | | const tls_tp_offset = switch (native_arch) { |
| 92 | | .mips, .mipsel, .mips64, .mips64el, .powerpc, .powerpcle, .powerpc64, .powerpc64le => 0x7000, |
| 105 | /// Usually only used by the modified Variant I. |
| 106 | const current_dtv_offset = switch (native_arch) { |
| 107 | .mips, |
| 108 | .mipsel, |
| 109 | .mips64, |
| 110 | .mips64el, |
| 111 | .powerpc, |
| 112 | .powerpcle, |
| 113 | .powerpc64, |
| 114 | .powerpc64le, |
| 115 | => 0x8000, |
| 116 | .riscv32, |
| 117 | .riscv64, |
| 118 | => 0x800, |
| 93 | 119 | else => 0, |
| 94 | 120 | }; |
| 95 | 121 | |
| 96 | | const tls_dtv_offset = switch (native_arch) { |
| 97 | | .mips, .mipsel, .mips64, .mips64el, .powerpc, .powerpcle, .powerpc64, .powerpc64le => 0x8000, |
| 98 | | .riscv32, .riscv64 => 0x800, |
| 99 | | else => 0, |
| 122 | /// Per-thread storage for the ELF TLS ABI. |
| 123 | const AbiTcb = switch (current_variant) { |
| 124 | .I_original, .I_modified => switch (native_arch) { |
| 125 | // ARM EABI mandates enough space for two pointers: the first one points to the DTV as |
| 126 | // usual, while the second one is unspecified. |
| 127 | .aarch64, |
| 128 | .aarch64_be, |
| 129 | .arm, |
| 130 | .armeb, |
| 131 | .thumb, |
| 132 | .thumbeb, |
| 133 | => extern struct { |
| 134 | /// This is offset by `current_dtv_offset`. |
| 135 | dtv: usize, |
| 136 | reserved: ?*anyopaque, |
| 137 | }, |
| 138 | else => extern struct { |
| 139 | /// This is offset by `current_dtv_offset`. |
| 140 | dtv: usize, |
| 141 | }, |
| 142 | }, |
| 143 | .II => extern struct { |
| 144 | /// This is self-referential. |
| 145 | self: *AbiTcb, |
| 146 | }, |
| 100 | 147 | }; |
| 101 | 148 | |
| 102 | | // Per-thread storage for Zig's use |
| 103 | | const CustomData = struct { |
| 149 | /// Per-thread storage for Zig's use. Currently unused. |
| 150 | const ZigTcb = struct { |
| 104 | 151 | dummy: usize, |
| 105 | 152 | }; |
| 106 | 153 | |
| 107 | | // Dynamic Thread Vector |
| 108 | | const DTV = extern struct { |
| 109 | | entries: usize, |
| 110 | | tls_block: [1][*]u8, |
| 154 | /// Dynamic Thread Vector as specified in the ELF TLS ABI. Ordinarily, there is a block pointer per |
| 155 | /// dynamically-loaded module, but since we only support static TLS, we only need one block pointer. |
| 156 | const Dtv = extern struct { |
| 157 | len: usize = 1, |
| 158 | tls_block: [*]u8, |
| 111 | 159 | }; |
| 112 | 160 | |
| 113 | | // Holds all the information about the process TLS image |
| 114 | | const TLSImage = struct { |
| 115 | | init_data: []const u8, |
| 116 | | alloc_size: usize, |
| 117 | | alloc_align: usize, |
| 118 | | tcb_offset: usize, |
| 119 | | dtv_offset: usize, |
| 120 | | data_offset: usize, |
| 121 | | data_size: usize, |
| 122 | | // Only used on the x86 architecture |
| 161 | /// Describes a process's TLS area. The area encompasses the DTV, both TCBs, and the TLS block, with |
| 162 | /// the exact layout of these being dependent primarily on `current_variant`. |
| 163 | const AreaDesc = struct { |
| 164 | size: usize, |
| 165 | alignment: usize, |
| 166 | |
| 167 | dtv: struct { |
| 168 | /// Offset into the TLS area. |
| 169 | offset: usize, |
| 170 | }, |
| 171 | |
| 172 | abi_tcb: struct { |
| 173 | /// Offset into the TLS area. |
| 174 | offset: usize, |
| 175 | }, |
| 176 | |
| 177 | block: struct { |
| 178 | /// The initial data to be copied into the TLS block. Note that this may be smaller than |
| 179 | /// `size`, in which case any remaining data in the TLS block is simply left uninitialized. |
| 180 | init: []const u8, |
| 181 | /// Offset into the TLS area. |
| 182 | offset: usize, |
| 183 | /// This is the effective size of the TLS block, which may be greater than `init.len`. |
| 184 | size: usize, |
| 185 | }, |
| 186 | |
| 187 | /// Only used on the 32-bit x86 architecture (not x86_64, nor x32). |
| 123 | 188 | gdt_entry_number: usize, |
| 124 | 189 | }; |
| 125 | 190 | |
| 126 | | pub var tls_image: TLSImage = undefined; |
| 191 | pub var area_desc: AreaDesc = undefined; |
| 127 | 192 | |
| 128 | 193 | pub fn setThreadPointer(addr: usize) void { |
| 129 | 194 | @setRuntimeSafety(false); |
| 130 | 195 | @disableInstrumentation(); |
| 196 | |
| 131 | 197 | switch (native_arch) { |
| 132 | 198 | .x86 => { |
| 133 | 199 | var user_desc: linux.user_desc = .{ |
| 134 | | .entry_number = tls_image.gdt_entry_number, |
| 200 | .entry_number = area_desc.gdt_entry_number, |
| 135 | 201 | .base_addr = addr, |
| 136 | 202 | .limit = 0xfffff, |
| 137 | 203 | .flags = .{ |
| ... | ... | @@ -148,7 +214,7 @@ pub fn setThreadPointer(addr: usize) void { |
| 148 | 214 | |
| 149 | 215 | const gdt_entry_number = user_desc.entry_number; |
| 150 | 216 | // We have to keep track of our slot as it's also needed for clone() |
| 151 | | tls_image.gdt_entry_number = gdt_entry_number; |
| 217 | area_desc.gdt_entry_number = gdt_entry_number; |
| 152 | 218 | // Update the %gs selector |
| 153 | 219 | asm volatile ("movl %[gs_val], %%gs" |
| 154 | 220 | : |
| ... | ... | @@ -206,7 +272,7 @@ pub fn setThreadPointer(addr: usize) void { |
| 206 | 272 | } |
| 207 | 273 | } |
| 208 | 274 | |
| 209 | | fn initTLS(phdrs: []elf.Phdr) void { |
| 275 | fn computeAreaDesc(phdrs: []elf.Phdr) void { |
| 210 | 276 | @setRuntimeSafety(false); |
| 211 | 277 | @disableInstrumentation(); |
| 212 | 278 | |
| ... | ... | @@ -221,72 +287,85 @@ fn initTLS(phdrs: []elf.Phdr) void { |
| 221 | 287 | } |
| 222 | 288 | } |
| 223 | 289 | |
| 224 | | var tls_align_factor: usize = undefined; |
| 225 | | var tls_data: []const u8 = undefined; |
| 226 | | var tls_data_alloc_size: usize = undefined; |
| 290 | var align_factor: usize = undefined; |
| 291 | var block_init: []const u8 = undefined; |
| 292 | var block_size: usize = undefined; |
| 293 | |
| 227 | 294 | if (tls_phdr) |phdr| { |
| 228 | | // The effective size in memory is represented by p_memsz, the length of |
| 229 | | // the data stored in the PT_TLS segment is p_filesz and may be less |
| 230 | | // than the former |
| 231 | | tls_align_factor = phdr.p_align; |
| 232 | | tls_data = @as([*]u8, @ptrFromInt(img_base + phdr.p_vaddr))[0..phdr.p_filesz]; |
| 233 | | tls_data_alloc_size = phdr.p_memsz; |
| 295 | align_factor = phdr.p_align; |
| 296 | |
| 297 | // The effective size in memory is represented by `p_memsz`; the length of the data stored |
| 298 | // in the `PT_TLS` segment is `p_filesz` and may be less than the former. |
| 299 | block_init = @as([*]u8, @ptrFromInt(img_base + phdr.p_vaddr))[0..phdr.p_filesz]; |
| 300 | block_size = phdr.p_memsz; |
| 234 | 301 | } else { |
| 235 | | tls_align_factor = @alignOf(usize); |
| 236 | | tls_data = &[_]u8{}; |
| 237 | | tls_data_alloc_size = 0; |
| 302 | align_factor = @alignOf(usize); |
| 303 | |
| 304 | block_init = &[_]u8{}; |
| 305 | block_size = 0; |
| 238 | 306 | } |
| 239 | 307 | |
| 240 | | // Offsets into the allocated TLS area |
| 241 | | var tcb_offset: usize = undefined; |
| 308 | // Offsets into the allocated TLS area. |
| 242 | 309 | var dtv_offset: usize = undefined; |
| 243 | | var data_offset: usize = undefined; |
| 244 | | // Compute the total size of the ABI-specific data plus our own control |
| 245 | | // structures. All the offset calculated here assume a well-aligned base |
| 246 | | // address. |
| 247 | | const alloc_size = switch (tls_variant) { |
| 248 | | .VariantI => blk: { |
| 310 | var abi_tcb_offset: usize = undefined; |
| 311 | var block_offset: usize = undefined; |
| 312 | |
| 313 | // Compute the total size of the ABI-specific data plus our own `ZigTcb` structure. All the |
| 314 | // offsets calculated here assume a well-aligned base address. |
| 315 | const area_size = switch (current_variant) { |
| 316 | .I_original, .I_modified => blk: { |
| 249 | 317 | var l: usize = 0; |
| 250 | 318 | dtv_offset = l; |
| 251 | | l += @sizeOf(DTV); |
| 252 | | // Add some padding here so that the thread pointer (tcb_offset) is |
| 253 | | // aligned to p_align and the CustomData structure can be found by |
| 254 | | // simply subtracting its @sizeOf from the tp value |
| 255 | | const delta = (l + @sizeOf(CustomData)) & (tls_align_factor - 1); |
| 319 | l += @sizeOf(Dtv); |
| 320 | // Add some padding here so that the TP (`abi_tcb_offset`) is aligned to `align_factor` |
| 321 | // and the `ZigTcb` structure can be found by simply subtracting `@sizeOf(ZigTcb)` from |
| 322 | // the TP. |
| 323 | const delta = (l + @sizeOf(ZigTcb)) & (align_factor - 1); |
| 256 | 324 | if (delta > 0) |
| 257 | | l += tls_align_factor - delta; |
| 258 | | l += @sizeOf(CustomData); |
| 259 | | tcb_offset = l; |
| 260 | | l += alignForward(tls_tcb_size, tls_align_factor); |
| 261 | | data_offset = l; |
| 262 | | l += tls_data_alloc_size; |
| 325 | l += align_factor - delta; |
| 326 | l += @sizeOf(ZigTcb); |
| 327 | abi_tcb_offset = l; |
| 328 | l += alignForward(@sizeOf(AbiTcb), align_factor); |
| 329 | block_offset = l; |
| 330 | l += block_size; |
| 263 | 331 | break :blk l; |
| 264 | 332 | }, |
| 265 | | .VariantII => blk: { |
| 333 | .II => blk: { |
| 266 | 334 | var l: usize = 0; |
| 267 | | data_offset = l; |
| 268 | | l += alignForward(tls_data_alloc_size, tls_align_factor); |
| 269 | | // The thread pointer is aligned to p_align |
| 270 | | tcb_offset = l; |
| 271 | | l += tls_tcb_size; |
| 272 | | // The CustomData structure is right after the TCB with no padding |
| 273 | | // in between so it can be easily found |
| 274 | | l += @sizeOf(CustomData); |
| 275 | | l = alignForward(l, @alignOf(DTV)); |
| 335 | block_offset = l; |
| 336 | l += alignForward(block_size, align_factor); |
| 337 | // The TP is aligned to `align_factor`. |
| 338 | abi_tcb_offset = l; |
| 339 | l += @sizeOf(AbiTcb); |
| 340 | // The `ZigTcb` structure is right after the `AbiTcb` with no padding in between so it |
| 341 | // can be easily found. |
| 342 | l += @sizeOf(ZigTcb); |
| 343 | // It doesn't really matter where we put the DTV, so give it natural alignment. |
| 344 | l = alignForward(l, @alignOf(Dtv)); |
| 276 | 345 | dtv_offset = l; |
| 277 | | l += @sizeOf(DTV); |
| 346 | l += @sizeOf(Dtv); |
| 278 | 347 | break :blk l; |
| 279 | 348 | }, |
| 280 | 349 | }; |
| 281 | 350 | |
| 282 | | tls_image = TLSImage{ |
| 283 | | .init_data = tls_data, |
| 284 | | .alloc_size = alloc_size, |
| 285 | | .alloc_align = tls_align_factor, |
| 286 | | .tcb_offset = tcb_offset, |
| 287 | | .dtv_offset = dtv_offset, |
| 288 | | .data_offset = data_offset, |
| 289 | | .data_size = tls_data_alloc_size, |
| 351 | area_desc = .{ |
| 352 | .size = area_size, |
| 353 | .alignment = align_factor, |
| 354 | |
| 355 | .dtv = .{ |
| 356 | .offset = dtv_offset, |
| 357 | }, |
| 358 | |
| 359 | .abi_tcb = .{ |
| 360 | .offset = abi_tcb_offset, |
| 361 | }, |
| 362 | |
| 363 | .block = .{ |
| 364 | .init = block_init, |
| 365 | .offset = block_offset, |
| 366 | .size = block_size, |
| 367 | }, |
| 368 | |
| 290 | 369 | .gdt_entry_number = @as(usize, @bitCast(@as(isize, -1))), |
| 291 | 370 | }; |
| 292 | 371 | } |
| ... | ... | @@ -306,78 +385,80 @@ inline fn alignPtrCast(comptime T: type, ptr: [*]u8) *T { |
| 306 | 385 | return @ptrCast(@alignCast(ptr)); |
| 307 | 386 | } |
| 308 | 387 | |
| 309 | | /// Initializes all the fields of the static TLS area and returns the computed |
| 310 | | /// architecture-specific value of the thread-pointer register |
| 311 | | /// |
| 312 | | /// This function is inline because thread local storage is not set up yet. |
| 313 | | pub fn prepareTLS(area: []u8) usize { |
| 388 | /// Initializes all the fields of the static TLS area and returns the computed architecture-specific |
| 389 | /// value of the TP register. |
| 390 | pub fn prepareArea(area: []u8) usize { |
| 314 | 391 | @setRuntimeSafety(false); |
| 315 | 392 | @disableInstrumentation(); |
| 316 | | // Clear the area we're going to use, just to be safe |
| 393 | |
| 394 | // Clear the area we're going to use, just to be safe. |
| 317 | 395 | @memset(area, 0); |
| 318 | | // Prepare the DTV |
| 319 | | const dtv = alignPtrCast(DTV, area.ptr + tls_image.dtv_offset); |
| 320 | | dtv.entries = 1; |
| 321 | | dtv.tls_block[0] = area.ptr + tls_dtv_offset + tls_image.data_offset; |
| 322 | | // Prepare the TCB |
| 323 | | const tcb_ptr = alignPtrCast([*]u8, area.ptr + tls_image.tcb_offset); |
| 324 | | tcb_ptr.* = switch (tls_variant) { |
| 325 | | .VariantI => area.ptr + tls_image.dtv_offset, |
| 326 | | .VariantII => area.ptr + tls_image.tcb_offset, |
| 396 | |
| 397 | // Prepare the ABI TCB. |
| 398 | const abi_tcb = alignPtrCast(AbiTcb, area.ptr + area_desc.abi_tcb.offset); |
| 399 | switch (current_variant) { |
| 400 | .I_original, .I_modified => abi_tcb.dtv = @intFromPtr(area.ptr + area_desc.dtv.offset), |
| 401 | .II => abi_tcb.self = abi_tcb, |
| 402 | } |
| 403 | |
| 404 | // Prepare the DTV. |
| 405 | const dtv = alignPtrCast(Dtv, area.ptr + area_desc.dtv.offset); |
| 406 | dtv.len = 1; |
| 407 | dtv.tls_block = area.ptr + current_dtv_offset + area_desc.block.offset; |
| 408 | |
| 409 | // Copy the initial data. |
| 410 | @memcpy(area[area_desc.block.offset..][0..area_desc.block.init.len], area_desc.block.init); |
| 411 | |
| 412 | // Return the corrected value (if needed) for the TP register. Overflow here is not a problem; |
| 413 | // the pointer arithmetic involving the TP is done with wrapping semantics. |
| 414 | return @intFromPtr(area.ptr) +% switch (current_variant) { |
| 415 | .I_original, .II => area_desc.abi_tcb.offset, |
| 416 | .I_modified => area_desc.block.offset +% current_tp_offset, |
| 327 | 417 | }; |
| 328 | | // Copy the data |
| 329 | | @memcpy(area[tls_image.data_offset..][0..tls_image.init_data.len], tls_image.init_data); |
| 330 | | |
| 331 | | // Return the corrected value (if needed) for the tp register. |
| 332 | | // Overflow here is not a problem, the pointer arithmetic involving the tp |
| 333 | | // is done with wrapping semantics. |
| 334 | | return @intFromPtr(area.ptr) +% tls_tp_offset +% |
| 335 | | if (tls_tp_points_past_tcb) tls_image.data_offset else tls_image.tcb_offset; |
| 336 | 418 | } |
| 337 | 419 | |
| 338 | | // The main motivation for the size chosen here is this is how much ends up being |
| 339 | | // requested for the thread local variables of the std.crypto.random implementation. |
| 340 | | // I'm not sure why it ends up being so much; the struct itself is only 64 bytes. |
| 341 | | // I think it has to do with being page aligned and LLVM or LLD is not smart enough |
| 342 | | // to lay out the TLS data in a space conserving way. Anyway I think it's fine |
| 343 | | // because it's less than 3 pages of memory, and putting it in the ELF like this |
| 344 | | // is equivalent to moving the mmap call below into the kernel, avoiding syscall |
| 345 | | // overhead. |
| 346 | | var main_thread_tls_buffer: [0x2100]u8 align(mem.page_size) = undefined; |
| 347 | | |
| 348 | | pub fn initStaticTLS(phdrs: []elf.Phdr) void { |
| 420 | // The main motivation for the size chosen here is that this is how much ends up being requested for |
| 421 | // the thread-local variables of the `std.crypto.random` implementation. I'm not sure why it ends up |
| 422 | // being so much; the struct itself is only 64 bytes. I think it has to do with being page-aligned |
| 423 | // and LLVM or LLD is not smart enough to lay out the TLS data in a space-conserving way. Anyway, I |
| 424 | // think it's fine because it's less than 3 pages of memory, and putting it in the ELF like this is |
| 425 | // equivalent to moving the `mmap` call below into the kernel, avoiding syscall overhead. |
| 426 | var main_thread_area_buffer: [0x2100]u8 align(mem.page_size) = undefined; |
| 427 | |
| 428 | /// Computes the layout of the static TLS area, allocates the area, initializes all of its fields, |
| 429 | /// and assigns the architecture-specific value to the TP register. |
| 430 | pub fn initStatic(phdrs: []elf.Phdr) void { |
| 349 | 431 | @setRuntimeSafety(false); |
| 350 | 432 | @disableInstrumentation(); |
| 351 | 433 | |
| 352 | | initTLS(phdrs); |
| 434 | computeAreaDesc(phdrs); |
| 353 | 435 | |
| 354 | | const tls_area = blk: { |
| 355 | | // Fast path for the common case where the TLS data is really small, |
| 356 | | // avoid an allocation and use our local buffer. |
| 357 | | if (tls_image.alloc_align <= mem.page_size and |
| 358 | | tls_image.alloc_size <= main_thread_tls_buffer.len) |
| 359 | | { |
| 360 | | break :blk main_thread_tls_buffer[0..tls_image.alloc_size]; |
| 436 | const area = blk: { |
| 437 | // Fast path for the common case where the TLS data is really small, avoid an allocation and |
| 438 | // use our local buffer. |
| 439 | if (area_desc.alignment <= mem.page_size and area_desc.size <= main_thread_area_buffer.len) { |
| 440 | break :blk main_thread_area_buffer[0..area_desc.size]; |
| 361 | 441 | } |
| 362 | 442 | |
| 363 | 443 | const begin_addr = mmap( |
| 364 | 444 | null, |
| 365 | | tls_image.alloc_size + tls_image.alloc_align - 1, |
| 445 | area_desc.size + area_desc.alignment - 1, |
| 366 | 446 | posix.PROT.READ | posix.PROT.WRITE, |
| 367 | 447 | .{ .TYPE = .PRIVATE, .ANONYMOUS = true }, |
| 368 | 448 | -1, |
| 369 | 449 | 0, |
| 370 | 450 | ); |
| 371 | 451 | if (@as(isize, @bitCast(begin_addr)) < 0) @trap(); |
| 372 | | const alloc_tls_area: [*]align(mem.page_size) u8 = @ptrFromInt(begin_addr); |
| 452 | |
| 453 | const area_ptr: [*]align(mem.page_size) u8 = @ptrFromInt(begin_addr); |
| 373 | 454 | |
| 374 | 455 | // Make sure the slice is correctly aligned. |
| 375 | | const begin_aligned_addr = alignForward(begin_addr, tls_image.alloc_align); |
| 456 | const begin_aligned_addr = alignForward(begin_addr, area_desc.alignment); |
| 376 | 457 | const start = begin_aligned_addr - begin_addr; |
| 377 | | break :blk alloc_tls_area[start..][0..tls_image.alloc_size]; |
| 458 | break :blk area_ptr[start..][0..area_desc.size]; |
| 378 | 459 | }; |
| 379 | 460 | |
| 380 | | const tp_value = prepareTLS(tls_area); |
| 461 | const tp_value = prepareArea(area); |
| 381 | 462 | setThreadPointer(tp_value); |
| 382 | 463 | } |
| 383 | 464 | |