1//! __emutls_get_address specific builtin
2//!
3//! derived work from LLVM Compiler Infrastructure - release 8.0 (MIT)
4//! https://github.com/llvm-mirror/compiler-rt/blob/release_80/lib/builtins/emutls.c
5
6const std = @import("std");
7const builtin = @import("builtin");
8const compiler_rt = @import("../compiler_rt.zig");
9const symbol = compiler_rt.symbol;
10
11const abort = std.process.abort;
12const assert = std.debug.assert;
13const expect = std.testing.expect;
14
15/// defined in C as:
16/// typedef unsigned int gcc_word __attribute__((mode(word)));
17const gcc_word = usize;
18
19comptime {
20 if (builtin.link_libc and (builtin.abi.isAndroid() or builtin.abi.isOpenHarmony() or builtin.os.tag == .openbsd)) {
21 symbol(&__emutls_get_address, "__emutls_get_address");
22 }
23}
24
25/// public entrypoint for generated code using EmulatedTLS
26pub fn __emutls_get_address(control: *emutls_control) callconv(.c) *anyopaque {
27 return control.getPointer();
28}
29
30/// Simple allocator interface, to avoid pulling in the while
31/// std allocator implementation.
32const simple_allocator = struct {
33 /// Allocate a memory chunk for requested type. Return a pointer on the data.
34 pub fn alloc(comptime T: type) *T {
35 return @ptrCast(@alignCast(advancedAlloc(@alignOf(T), @sizeOf(T))));
36 }
37
38 /// Allocate a slice of T, with len elements.
39 pub fn allocSlice(comptime T: type, len: usize) []T {
40 return @as([*]T, @ptrCast(@alignCast(
41 advancedAlloc(@alignOf(T), @sizeOf(T) * len),
42 )))[0 .. len - 1];
43 }
44
45 /// Allocate a memory chunk.
46 pub fn advancedAlloc(alignment: u29, size: usize) [*]u8 {
47 const minimal_alignment = @max(@alignOf(usize), alignment);
48
49 var aligned_ptr: ?*anyopaque = undefined;
50 if (std.c.posix_memalign(&aligned_ptr, minimal_alignment, size) != 0) {
51 abort();
52 }
53
54 return @ptrCast(aligned_ptr);
55 }
56
57 /// Resize a slice.
58 pub fn reallocSlice(comptime T: type, slice: []T, len: usize) []T {
59 const c_ptr: *anyopaque = @ptrCast(slice.ptr);
60 const new_array: [*]T = @ptrCast(@alignCast(std.c.realloc(c_ptr, @sizeOf(T) * len) orelse abort()));
61 return new_array[0..len];
62 }
63
64 /// Free a memory chunk allocated with simple_allocator.
65 pub fn free(ptr: anytype) void {
66 std.c.free(@ptrCast(ptr));
67 }
68};
69
70/// Simple array of ?ObjectPointer with automatic resizing and
71/// automatic storage allocation.
72const ObjectArray = struct {
73 const ObjectPointer = *anyopaque;
74
75 // content of the array
76 slots: []?ObjectPointer,
77
78 /// create a new ObjectArray with n slots. must call deinit() to deallocate.
79 pub fn init(n: usize) *ObjectArray {
80 const array = simple_allocator.alloc(ObjectArray);
81
82 array.* = ObjectArray{
83 .slots = simple_allocator.allocSlice(?ObjectPointer, n),
84 };
85
86 for (array.slots) |*object| {
87 object.* = null;
88 }
89
90 return array;
91 }
92
93 /// deallocate the ObjectArray.
94 pub fn deinit(self: *ObjectArray) void {
95 // deallocated used objects in the array
96 for (self.slots) |*object| {
97 simple_allocator.free(object.*);
98 }
99 simple_allocator.free(self.slots);
100 simple_allocator.free(self);
101 }
102
103 /// resize the ObjectArray if needed.
104 pub fn ensureLength(self: *ObjectArray, new_len: usize) *ObjectArray {
105 const old_len = self.slots.len;
106
107 if (old_len > new_len) {
108 return self;
109 }
110
111 // reallocate
112 self.slots = simple_allocator.reallocSlice(?ObjectPointer, self.slots, new_len);
113
114 // init newly added slots
115 for (self.slots[old_len..]) |*object| {
116 object.* = null;
117 }
118
119 return self;
120 }
121
122 /// Retrieve the pointer at request index, using control to initialize it if needed.
123 pub fn getPointer(self: *ObjectArray, index: usize, control: *emutls_control) ObjectPointer {
124 if (self.slots[index] == null) {
125 // initialize the slot
126 const size = control.size;
127 const alignment: u29 = @truncate(control.alignment);
128
129 var data = simple_allocator.advancedAlloc(alignment, size);
130 errdefer simple_allocator.free(data);
131
132 if (control.default_value) |value| {
133 // default value: copy the content to newly allocated object.
134 @memcpy(data[0..size], @as([*]const u8, @ptrCast(value)));
135 } else {
136 // no default: return zeroed memory.
137 @memset(data[0..size], 0);
138 }
139
140 self.slots[index] = data;
141 }
142
143 return self.slots[index].?;
144 }
145};
146
147// Global structure for Thread Storage.
148// It provides thread-safety for on-demand storage of Thread Objects.
149const current_thread_storage = struct {
150 var key: std.c.pthread_key_t = undefined;
151 var init_mutex: std.c.pthread_mutex_t = std.c.PTHREAD_MUTEX_INITIALIZER;
152 var init_done: bool = false;
153
154 /// Return a per thread ObjectArray with at least the expected index.
155 pub fn getArray(index: usize) *ObjectArray {
156 if (current_thread_storage.getspecific()) |array| {
157 // we already have a specific. just ensure the array is
158 // big enough for the wanted index.
159 return array.ensureLength(index);
160 }
161
162 // no specific. we need to create a new array.
163
164 // make it to contains at least 16 objects (to avoid too much
165 // reallocation at startup).
166 const size = @max(16, index);
167
168 // create a new array and store it.
169 const array: *ObjectArray = ObjectArray.init(size);
170 current_thread_storage.setspecific(array);
171 return array;
172 }
173
174 /// Return casted thread specific value.
175 fn getspecific() ?*ObjectArray {
176 return @ptrCast(@alignCast(std.c.pthread_getspecific(current_thread_storage.key)));
177 }
178
179 /// Set casted thread specific value.
180 fn setspecific(new: ?*ObjectArray) void {
181 if (std.c.pthread_setspecific(current_thread_storage.key, @ptrCast(new)) != 0) {
182 abort();
183 }
184 }
185
186 /// Initialize pthread_key_t.
187 fn init() void {
188 if (@atomicLoad(bool, &init_done, .monotonic)) return;
189 _ = std.c.pthread_mutex_lock(&init_mutex);
190 if (std.c.pthread_key_create(&current_thread_storage.key, current_thread_storage.deinit) != .SUCCESS) {
191 abort();
192 }
193 @atomicStore(bool, &init_done, true, .release);
194 _ = std.c.pthread_mutex_unlock(&init_mutex);
195 }
196
197 /// Invoked by pthread specific destructor. the passed argument is the ObjectArray pointer.
198 fn deinit(arrayPtr: *anyopaque) callconv(.c) void {
199 var array: *ObjectArray = @ptrCast(@alignCast(arrayPtr));
200 array.deinit();
201 }
202};
203
204const emutls_control = extern struct {
205 // A emutls_control value is a global value across all
206 // threads. The threads shares the index of TLS variable. The data
207 // array (containing address of allocated variables) is thread
208 // specific and stored using pthread_setspecific().
209
210 // size of the object in bytes
211 size: gcc_word,
212
213 // alignment of the object in bytes
214 alignment: gcc_word,
215
216 object: extern union {
217 // data[index-1] is the object address / 0 = uninit
218 index: usize,
219
220 // object address, when in single thread env (not used)
221 address: *anyopaque,
222 },
223
224 // null or non-zero initial value for the object
225 default_value: ?*const anyopaque,
226
227 // global Mutex used to serialize control.index initialization.
228 var mutex: std.c.pthread_mutex_t = std.c.PTHREAD_MUTEX_INITIALIZER;
229
230 // global counter for keeping track of requested indexes.
231 // access should be done with mutex held.
232 var next_index: usize = 1;
233
234 /// Simple wrapper for global lock.
235 fn lock() void {
236 if (std.c.pthread_mutex_lock(&emutls_control.mutex) != .SUCCESS) {
237 abort();
238 }
239 }
240
241 /// Simple wrapper for global unlock.
242 fn unlock() void {
243 if (std.c.pthread_mutex_unlock(&emutls_control.mutex) != .SUCCESS) {
244 abort();
245 }
246 }
247
248 /// Helper to retrieve nad initialize global unique index per emutls variable.
249 pub fn getIndex(self: *emutls_control) usize {
250 // Two threads could race against the same emutls_control.
251
252 // Use atomic for reading coherent value lockless.
253 const index_lockless = @atomicLoad(usize, &self.object.index, .acquire);
254
255 if (index_lockless != 0) {
256 // index is already initialized, return it.
257 return index_lockless;
258 }
259
260 // index is uninitialized: take global lock to avoid possible race.
261 emutls_control.lock();
262 defer emutls_control.unlock();
263
264 const index_locked = self.object.index;
265 if (index_locked != 0) {
266 // we lost a race, but index is already initialized: nothing particular to do.
267 return index_locked;
268 }
269
270 // Store a new index atomically (for having coherent index_lockless reading).
271 @atomicStore(usize, &self.object.index, emutls_control.next_index, .release);
272
273 // Increment the next available index
274 emutls_control.next_index += 1;
275
276 return self.object.index;
277 }
278
279 /// Simple helper for testing purpose.
280 pub fn init(comptime T: type, default_value: ?*const T) emutls_control {
281 return emutls_control{
282 .size = @sizeOf(T),
283 .alignment = @alignOf(T),
284 .object = .{ .index = 0 },
285 .default_value = @ptrCast(default_value),
286 };
287 }
288
289 /// Get the pointer on allocated storage for emutls variable.
290 pub fn getPointer(self: *emutls_control) *anyopaque {
291 // ensure current_thread_storage initialization is done
292 current_thread_storage.init();
293
294 const index = self.getIndex();
295 var array = current_thread_storage.getArray(index);
296
297 return array.getPointer(index - 1, self);
298 }
299
300 /// Testing helper for retrieving typed pointer.
301 pub fn get_typed_pointer(self: *emutls_control, comptime T: type) *T {
302 assert(self.size == @sizeOf(T));
303 assert(self.alignment == @alignOf(T));
304 return @ptrCast(@alignCast(self.getPointer()));
305 }
306};
307
308test "simple_allocator" {
309 if (!builtin.link_libc or builtin.os.tag != .openbsd) return error.SkipZigTest;
310
311 const data1: *[64]u8 = simple_allocator.alloc([64]u8);
312 defer simple_allocator.free(data1);
313 for (data1) |*c| {
314 c.* = 0xff;
315 }
316
317 const data2: [*]u8 = simple_allocator.advancedAlloc(@alignOf(u8), 64);
318 defer simple_allocator.free(data2);
319 for (data2[0..63]) |*c| {
320 c.* = 0xff;
321 }
322}
323
324test "__emutls_get_address zeroed" {
325 if (!builtin.link_libc or builtin.os.tag != .openbsd) return error.SkipZigTest;
326
327 var ctl = emutls_control.init(usize, null);
328 try expect(ctl.object.index == 0);
329
330 // retrieve a variable from ctl
331 const x: *usize = @ptrCast(@alignCast(__emutls_get_address(&ctl)));
332 try expect(ctl.object.index != 0); // index has been allocated for this ctl
333 try expect(x.* == 0); // storage has been zeroed
334
335 // modify the storage
336 x.* = 1234;
337
338 // retrieve a variable from ctl (same ctl)
339 const y: *usize = @ptrCast(@alignCast(__emutls_get_address(&ctl)));
340
341 try expect(y.* == 1234); // same content that x.*
342 try expect(x == y); // same pointer
343}
344
345test "__emutls_get_address with default_value" {
346 if (!builtin.link_libc or builtin.os.tag != .openbsd) return error.SkipZigTest;
347
348 const value: usize = 5678; // default value
349 var ctl = emutls_control.init(usize, &value);
350 try expect(ctl.object.index == 0);
351
352 const x: *usize = @ptrCast(@alignCast(__emutls_get_address(&ctl)));
353 try expect(ctl.object.index != 0);
354 try expect(x.* == 5678); // storage initialized with default value
355
356 // modify the storage
357 x.* = 9012;
358
359 try expect(value == 5678); // the default value didn't change
360
361 const y: *usize = @ptrCast(@alignCast(__emutls_get_address(&ctl)));
362 try expect(y.* == 9012); // the modified storage persists
363}
364
365test "test default_value with different sizes" {
366 if (!builtin.link_libc or builtin.os.tag != .openbsd) return error.SkipZigTest;
367
368 const testType = struct {
369 fn _testType(comptime T: type, value: T) !void {
370 var ctl = emutls_control.init(T, &value);
371 const x = ctl.get_typed_pointer(T);
372 try expect(x.* == value);
373 }
374 }._testType;
375
376 try testType(usize, 1234);
377 try testType(u32, 1234);
378 try testType(i16, -12);
379 try testType(f64, -12.0);
380 try testType(
381 @TypeOf("012345678901234567890123456789"),
382 "012345678901234567890123456789",
383 );
384}