authorgravatar for thatlemon@gmail.comLemonBoy <thatlemon@gmail.com> 2020-04-03 19:13:21+02:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-04-03 16:04:44-04:00
loged69821f5b4e47b25b95d2a1099e95ece477b66a
tree73eef00c49f1a1e9fc5a44a0643f4cceeb934d05
parentcf52f3f99a371fd4cb897afb2ed515ea00927808
signaturelock-open Commit is signed but in an unrecognized format.

compiler-rt: Add the __atomic family of builtins

The implementation was checked against a few files using std::atomic and compiled using zig c++. Closes #4887

2 files changed, 255 insertions(+), 0 deletions(-)

lib/std/special/compiler_rt.zig+2
...@@ -317,6 +317,8 @@ comptime {...@@ -317,6 +317,8 @@ comptime {
317 @export(@import("compiler_rt/mulodi4.zig").__mulodi4, .{ .name = "__mulodi4", .linkage = linkage });317 @export(@import("compiler_rt/mulodi4.zig").__mulodi4, .{ .name = "__mulodi4", .linkage = linkage });
318}318}
319319
320pub usingnamespace @import("compiler_rt/atomics.zig");
321
320// Avoid dragging in the runtime safety mechanisms into this .o file,322// Avoid dragging in the runtime safety mechanisms into this .o file,
321// unless we're trying to test this file.323// unless we're trying to test this file.
322pub fn panic(msg: []const u8, error_return_trace: ?*builtin.StackTrace) noreturn {324pub fn panic(msg: []const u8, error_return_trace: ?*builtin.StackTrace) noreturn {
lib/std/special/compiler_rt/atomics.zig created+253
...@@ -0,0 +1,253 @@
1const std = @import("std");
2const builtin = std.builtin;
3
4const linkage: builtin.GlobalLinkage = if (builtin.is_test) .Internal else .Weak;
5
6const cache_line_size = 64;
7
8const SpinlockTable = struct {
9 // Allocate ~4096 bytes of memory for the spinlock table
10 const max_spinlocks = 64;
11
12 const Spinlock = struct {
13 // Prevent false sharing by providing enough padding between two
14 // consecutive spinlock elements
15 v: enum(usize) { Unlocked = 0, Locked } align(cache_line_size) = .Unlocked,
16
17 fn acquire(self: *@This()) void {
18 while (true) {
19 switch (@atomicRmw(@TypeOf(self.v), &self.v, .Xchg, .Locked, .Acquire)) {
20 .Unlocked => break,
21 .Locked => {},
22 }
23 }
24 }
25 fn release(self: *@This()) void {
26 @atomicStore(@TypeOf(self.v), &self.v, .Unlocked, .Release);
27 }
28 };
29
30 list: [max_spinlocks]Spinlock = [_]Spinlock{.{}} ** max_spinlocks,
31
32 // The spinlock table behaves as a really simple hash table, mapping
33 // addresses to spinlocks. The mapping is not unique but that's only a
34 // performance problem as the lock will be contended by more than a pair of
35 // threads.
36 fn get(self: *@This(), address: usize) *Spinlock {
37 var sl = &self.list[(address >> 3) % max_spinlocks];
38 sl.acquire();
39 return sl;
40 }
41};
42
43var spinlocks: SpinlockTable = SpinlockTable{};
44
45// The following builtins do not respect the specified memory model and instead
46// uses seq_cst, the strongest one, for simplicity sake.
47
48// Generic version of GCC atomic builtin functions.
49// Those work on any object no matter the pointer alignment nor its size.
50
51fn __atomic_load(size: u32, src: [*]u8, dest: [*]u8, model: i32) callconv(.C) void {
52 var sl = spinlocks.get(@ptrToInt(src));
53 defer sl.release();
54 @memcpy(dest, src, size);
55}
56
57fn __atomic_store(size: u32, dest: [*]u8, src: [*]u8, model: i32) callconv(.C) void {
58 var sl = spinlocks.get(@ptrToInt(dest));
59 defer sl.release();
60 @memcpy(dest, src, size);
61}
62
63fn __atomic_exchange(size: u32, ptr: [*]u8, val: [*]u8, old: [*]u8, model: i32) callconv(.C) void {
64 var sl = spinlocks.get(@ptrToInt(ptr));
65 defer sl.release();
66 @memcpy(old, ptr, size);
67 @memcpy(ptr, val, size);
68}
69
70fn __atomic_compare_exchange(
71 size: u32,
72 ptr: [*]u8,
73 expected: [*]u8,
74 desired: [*]u8,
75 success: i32,
76 failure: i32,
77) callconv(.C) i32 {
78 var sl = spinlocks.get(@ptrToInt(ptr));
79 defer sl.release();
80 for (ptr[0..size]) |b, i| {
81 if (expected[i] != b) break;
82 } else {
83 // The two objects, ptr and expected, are equal
84 @memcpy(ptr, desired, size);
85 return 1;
86 }
87 @memcpy(expected, ptr, size);
88 return 0;
89}
90
91// Specialized versions of the GCC atomic builtin functions.
92// LLVM emits those iff the object size is known and the pointers are correctly
93// aligned.
94
95// The size (in bytes) of the biggest object that the architecture can access
96// atomically. Objects bigger than this threshold require the use of a lock.
97const largest_atomic_size = switch (builtin.arch) {
98 .x86_64 => 16,
99 else => @sizeOf(usize),
100};
101
102fn makeAtomicLoadFn(comptime T: type) type {
103 return struct {
104 fn atomic_load_N(src: *T, model: i32) callconv(.C) T {
105 if (@sizeOf(T) > largest_atomic_size) {
106 var sl = spinlocks.get(@ptrToInt(src));
107 defer sl.release();
108 return src.*;
109 } else {
110 return @atomicLoad(T, src, .SeqCst);
111 }
112 }
113 };
114}
115
116comptime {
117 @export(makeAtomicLoadFn(u8).atomic_load_N, .{ .name = "__atomic_load_1", .linkage = linkage });
118 @export(makeAtomicLoadFn(u16).atomic_load_N, .{ .name = "__atomic_load_2", .linkage = linkage });
119 @export(makeAtomicLoadFn(u32).atomic_load_N, .{ .name = "__atomic_load_4", .linkage = linkage });
120 @export(makeAtomicLoadFn(u64).atomic_load_N, .{ .name = "__atomic_load_8", .linkage = linkage });
121}
122
123fn makeAtomicStoreFn(comptime T: type) type {
124 return struct {
125 fn atomic_store_N(dst: *T, value: T, model: i32) callconv(.C) void {
126 if (@sizeOf(T) > largest_atomic_size) {
127 var sl = spinlocks.get(@ptrToInt(dst));
128 defer sl.release();
129 dst.* = value;
130 } else {
131 @atomicStore(T, dst, value, .SeqCst);
132 }
133 }
134 };
135}
136
137comptime {
138 @export(makeAtomicStoreFn(u8).atomic_store_N, .{ .name = "__atomic_store_1", .linkage = linkage });
139 @export(makeAtomicStoreFn(u16).atomic_store_N, .{ .name = "__atomic_store_2", .linkage = linkage });
140 @export(makeAtomicStoreFn(u32).atomic_store_N, .{ .name = "__atomic_store_4", .linkage = linkage });
141 @export(makeAtomicStoreFn(u64).atomic_store_N, .{ .name = "__atomic_store_8", .linkage = linkage });
142}
143
144fn makeAtomicExchangeFn(comptime T: type) type {
145 return struct {
146 fn atomic_exchange_N(ptr: *T, val: T, model: i32) callconv(.C) T {
147 if (@sizeOf(T) > largest_atomic_size) {
148 var sl = spinlocks.get(@ptrToInt(ptr));
149 defer sl.release();
150 var value = ptr.*;
151 ptr.* = val;
152 return value;
153 } else {
154 return @atomicRmw(T, ptr, .Xchg, val, .SeqCst);
155 }
156 }
157 };
158}
159
160comptime {
161 @export(makeAtomicExchangeFn(u8).atomic_exchange_N, .{ .name = "__atomic_exchange_1", .linkage = linkage });
162 @export(makeAtomicExchangeFn(u16).atomic_exchange_N, .{ .name = "__atomic_exchange_2", .linkage = linkage });
163 @export(makeAtomicExchangeFn(u32).atomic_exchange_N, .{ .name = "__atomic_exchange_4", .linkage = linkage });
164 @export(makeAtomicExchangeFn(u64).atomic_exchange_N, .{ .name = "__atomic_exchange_8", .linkage = linkage });
165}
166
167fn makeAtomicCompareExchangeFn(comptime T: type) type {
168 return struct {
169 fn atomic_compare_exchange_N(ptr: *T, expected: *T, desired: T, success: i32, failure: i32) callconv(.C) i32 {
170 if (@sizeOf(T) > largest_atomic_size) {
171 var sl = spinlocks.get(@ptrToInt(ptr));
172 defer sl.release();
173 if (ptr.* == expected.*) {
174 ptr.* = desired;
175 return 1;
176 }
177 expected.* = ptr.*;
178 return 0;
179 } else {
180 if (@cmpxchgStrong(T, ptr, expected.*, desired, .SeqCst, .SeqCst)) |old_value| {
181 expected.* = old_value;
182 return 0;
183 }
184 return 1;
185 }
186 }
187 };
188}
189
190comptime {
191 @export(makeAtomicCompareExchangeFn(u8).atomic_compare_exchange_N, .{ .name = "__atomic_compare_exchange_1", .linkage = linkage });
192 @export(makeAtomicCompareExchangeFn(u16).atomic_compare_exchange_N, .{ .name = "__atomic_compare_exchange_2", .linkage = linkage });
193 @export(makeAtomicCompareExchangeFn(u32).atomic_compare_exchange_N, .{ .name = "__atomic_compare_exchange_4", .linkage = linkage });
194 @export(makeAtomicCompareExchangeFn(u64).atomic_compare_exchange_N, .{ .name = "__atomic_compare_exchange_8", .linkage = linkage });
195}
196
197fn makeFetchFn(comptime T: type, comptime op: builtin.AtomicRmwOp) type {
198 return struct {
199 pub fn fetch_op_N(ptr: *T, val: T, model: i32) callconv(.C) T {
200 if (@sizeOf(T) > largest_atomic_size) {
201 var sl = spinlocks.get(@ptrToInt(ptr));
202 defer sl.release();
203
204 var value = ptr.*;
205 ptr.* = switch (op) {
206 .Add => ptr.* +% val,
207 .Sub => ptr.* -% val,
208 .And => ptr.* & val,
209 .Nand => ~(ptr.* & val),
210 .Or => ptr.* | val,
211 .Xor => ptr.* ^ val,
212 else => @compileError("unsupported atomic op"),
213 };
214
215 return value;
216 }
217
218 return @atomicRmw(T, ptr, op, val, .SeqCst);
219 }
220 };
221}
222
223comptime {
224 @export(makeFetchFn(u8, .Add).fetch_op_N, .{ .name = "__atomic_fetch_add_1", .linkage = linkage });
225 @export(makeFetchFn(u16, .Add).fetch_op_N, .{ .name = "__atomic_fetch_add_2", .linkage = linkage });
226 @export(makeFetchFn(u32, .Add).fetch_op_N, .{ .name = "__atomic_fetch_add_4", .linkage = linkage });
227 @export(makeFetchFn(u64, .Add).fetch_op_N, .{ .name = "__atomic_fetch_add_8", .linkage = linkage });
228
229 @export(makeFetchFn(u8, .Sub).fetch_op_N, .{ .name = "__atomic_fetch_sub_1", .linkage = linkage });
230 @export(makeFetchFn(u16, .Sub).fetch_op_N, .{ .name = "__atomic_fetch_sub_2", .linkage = linkage });
231 @export(makeFetchFn(u32, .Sub).fetch_op_N, .{ .name = "__atomic_fetch_sub_4", .linkage = linkage });
232 @export(makeFetchFn(u64, .Sub).fetch_op_N, .{ .name = "__atomic_fetch_sub_8", .linkage = linkage });
233
234 @export(makeFetchFn(u8, .And).fetch_op_N, .{ .name = "__atomic_fetch_and_1", .linkage = linkage });
235 @export(makeFetchFn(u16, .And).fetch_op_N, .{ .name = "__atomic_fetch_and_2", .linkage = linkage });
236 @export(makeFetchFn(u32, .And).fetch_op_N, .{ .name = "__atomic_fetch_and_4", .linkage = linkage });
237 @export(makeFetchFn(u64, .And).fetch_op_N, .{ .name = "__atomic_fetch_and_8", .linkage = linkage });
238
239 @export(makeFetchFn(u8, .Or).fetch_op_N, .{ .name = "__atomic_fetch_or_1", .linkage = linkage });
240 @export(makeFetchFn(u16, .Or).fetch_op_N, .{ .name = "__atomic_fetch_or_2", .linkage = linkage });
241 @export(makeFetchFn(u32, .Or).fetch_op_N, .{ .name = "__atomic_fetch_or_4", .linkage = linkage });
242 @export(makeFetchFn(u64, .Or).fetch_op_N, .{ .name = "__atomic_fetch_or_8", .linkage = linkage });
243
244 @export(makeFetchFn(u8, .Xor).fetch_op_N, .{ .name = "__atomic_fetch_xor_1", .linkage = linkage });
245 @export(makeFetchFn(u16, .Xor).fetch_op_N, .{ .name = "__atomic_fetch_xor_2", .linkage = linkage });
246 @export(makeFetchFn(u32, .Xor).fetch_op_N, .{ .name = "__atomic_fetch_xor_4", .linkage = linkage });
247 @export(makeFetchFn(u64, .Xor).fetch_op_N, .{ .name = "__atomic_fetch_xor_8", .linkage = linkage });
248
249 @export(makeFetchFn(u8, .Nand).fetch_op_N, .{ .name = "__atomic_fetch_nand_1", .linkage = linkage });
250 @export(makeFetchFn(u16, .Nand).fetch_op_N, .{ .name = "__atomic_fetch_nand_2", .linkage = linkage });
251 @export(makeFetchFn(u32, .Nand).fetch_op_N, .{ .name = "__atomic_fetch_nand_4", .linkage = linkage });
252 @export(makeFetchFn(u64, .Nand).fetch_op_N, .{ .name = "__atomic_fetch_nand_8", .linkage = linkage });
253}