authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-04-03 16:08:52-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-04-03 16:08:52-04:00
log7beea47178f311ce7bb1e5f9bad4d3c8c14caec5
tree90e1ded47c6d0576ba3a7789ed4e46c64759e82e
parentcf52f3f99a371fd4cb897afb2ed515ea00927808
parente03cbb117ee661824a1025ffee39f3ae80e660a9
signaturelock-open Commit is signed but in an unrecognized format.

Merge branch 'LemonBoy-compiler-rt-atomics'

closes #4924

2 files changed, 262 insertions(+), 0 deletions(-)

lib/std/special/compiler_rt.zig+2
......@@ -317,6 +317,8 @@ comptime {
317317 @export(@import("compiler_rt/mulodi4.zig").__mulodi4, .{ .name = "__mulodi4", .linkage = linkage });
318318}
319319
320pub usingnamespace @import("compiler_rt/atomics.zig");
321
320322// Avoid dragging in the runtime safety mechanisms into this .o file,
321323// unless we're trying to test this file.
322324pub fn panic(msg: []const u8, error_return_trace: ?*builtin.StackTrace) noreturn {
lib/std/special/compiler_rt/atomics.zig created+260
......@@ -0,0 +1,260 @@
1const std = @import("std");
2const builtin = std.builtin;
3
4const linkage: builtin.GlobalLinkage = if (builtin.is_test) .Internal else .Weak;
5
6const cache_line_size = 64;
7
8const SpinlockTable = struct {
9 // Allocate ~4096 bytes of memory for the spinlock table
10 const max_spinlocks = 64;
11
12 const Spinlock = struct {
13 // Prevent false sharing by providing enough padding between two
14 // consecutive spinlock elements
15 v: enum(usize) { Unlocked = 0, Locked } align(cache_line_size) = .Unlocked,
16
17 fn acquire(self: *@This()) void {
18 while (true) {
19 switch (@atomicRmw(@TypeOf(self.v), &self.v, .Xchg, .Locked, .Acquire)) {
20 .Unlocked => break,
21 .Locked => {},
22 }
23 }
24 }
25 fn release(self: *@This()) void {
26 @atomicStore(@TypeOf(self.v), &self.v, .Unlocked, .Release);
27 }
28 };
29
30 list: [max_spinlocks]Spinlock = [_]Spinlock{.{}} ** max_spinlocks,
31
32 // The spinlock table behaves as a really simple hash table, mapping
33 // addresses to spinlocks. The mapping is not unique but that's only a
34 // performance problem as the lock will be contended by more than a pair of
35 // threads.
36 fn get(self: *@This(), address: usize) *Spinlock {
37 var sl = &self.list[(address >> 3) % max_spinlocks];
38 sl.acquire();
39 return sl;
40 }
41};
42
43var spinlocks: SpinlockTable = SpinlockTable{};
44
45// The following builtins do not respect the specified memory model and instead
46// uses seq_cst, the strongest one, for simplicity sake.
47
48// Generic version of GCC atomic builtin functions.
49// Those work on any object no matter the pointer alignment nor its size.
50
51fn __atomic_load(size: u32, src: [*]u8, dest: [*]u8, model: i32) callconv(.C) void {
52 var sl = spinlocks.get(@ptrToInt(src));
53 defer sl.release();
54 @memcpy(dest, src, size);
55}
56
57fn __atomic_store(size: u32, dest: [*]u8, src: [*]u8, model: i32) callconv(.C) void {
58 var sl = spinlocks.get(@ptrToInt(dest));
59 defer sl.release();
60 @memcpy(dest, src, size);
61}
62
63fn __atomic_exchange(size: u32, ptr: [*]u8, val: [*]u8, old: [*]u8, model: i32) callconv(.C) void {
64 var sl = spinlocks.get(@ptrToInt(ptr));
65 defer sl.release();
66 @memcpy(old, ptr, size);
67 @memcpy(ptr, val, size);
68}
69
70fn __atomic_compare_exchange(
71 size: u32,
72 ptr: [*]u8,
73 expected: [*]u8,
74 desired: [*]u8,
75 success: i32,
76 failure: i32,
77) callconv(.C) i32 {
78 var sl = spinlocks.get(@ptrToInt(ptr));
79 defer sl.release();
80 for (ptr[0..size]) |b, i| {
81 if (expected[i] != b) break;
82 } else {
83 // The two objects, ptr and expected, are equal
84 @memcpy(ptr, desired, size);
85 return 1;
86 }
87 @memcpy(expected, ptr, size);
88 return 0;
89}
90
91comptime {
92 @export(__atomic_load, .{ .name = "__atomic_load", .linkage = linkage });
93 @export(__atomic_store, .{ .name = "__atomic_store", .linkage = linkage });
94 @export(__atomic_exchange, .{ .name = "__atomic_exchange", .linkage = linkage });
95 @export(__atomic_compare_exchange, .{ .name = "__atomic_compare_exchange", .linkage = linkage });
96}
97
98// Specialized versions of the GCC atomic builtin functions.
99// LLVM emits those iff the object size is known and the pointers are correctly
100// aligned.
101
102// The size (in bytes) of the biggest object that the architecture can access
103// atomically. Objects bigger than this threshold require the use of a lock.
104const largest_atomic_size = switch (builtin.arch) {
105 .x86_64 => 16,
106 else => @sizeOf(usize),
107};
108
109fn makeAtomicLoadFn(comptime T: type) type {
110 return struct {
111 fn atomic_load_N(src: *T, model: i32) callconv(.C) T {
112 if (@sizeOf(T) > largest_atomic_size) {
113 var sl = spinlocks.get(@ptrToInt(src));
114 defer sl.release();
115 return src.*;
116 } else {
117 return @atomicLoad(T, src, .SeqCst);
118 }
119 }
120 };
121}
122
123comptime {
124 @export(makeAtomicLoadFn(u8).atomic_load_N, .{ .name = "__atomic_load_1", .linkage = linkage });
125 @export(makeAtomicLoadFn(u16).atomic_load_N, .{ .name = "__atomic_load_2", .linkage = linkage });
126 @export(makeAtomicLoadFn(u32).atomic_load_N, .{ .name = "__atomic_load_4", .linkage = linkage });
127 @export(makeAtomicLoadFn(u64).atomic_load_N, .{ .name = "__atomic_load_8", .linkage = linkage });
128}
129
130fn makeAtomicStoreFn(comptime T: type) type {
131 return struct {
132 fn atomic_store_N(dst: *T, value: T, model: i32) callconv(.C) void {
133 if (@sizeOf(T) > largest_atomic_size) {
134 var sl = spinlocks.get(@ptrToInt(dst));
135 defer sl.release();
136 dst.* = value;
137 } else {
138 @atomicStore(T, dst, value, .SeqCst);
139 }
140 }
141 };
142}
143
144comptime {
145 @export(makeAtomicStoreFn(u8).atomic_store_N, .{ .name = "__atomic_store_1", .linkage = linkage });
146 @export(makeAtomicStoreFn(u16).atomic_store_N, .{ .name = "__atomic_store_2", .linkage = linkage });
147 @export(makeAtomicStoreFn(u32).atomic_store_N, .{ .name = "__atomic_store_4", .linkage = linkage });
148 @export(makeAtomicStoreFn(u64).atomic_store_N, .{ .name = "__atomic_store_8", .linkage = linkage });
149}
150
151fn makeAtomicExchangeFn(comptime T: type) type {
152 return struct {
153 fn atomic_exchange_N(ptr: *T, val: T, model: i32) callconv(.C) T {
154 if (@sizeOf(T) > largest_atomic_size) {
155 var sl = spinlocks.get(@ptrToInt(ptr));
156 defer sl.release();
157 var value = ptr.*;
158 ptr.* = val;
159 return value;
160 } else {
161 return @atomicRmw(T, ptr, .Xchg, val, .SeqCst);
162 }
163 }
164 };
165}
166
167comptime {
168 @export(makeAtomicExchangeFn(u8).atomic_exchange_N, .{ .name = "__atomic_exchange_1", .linkage = linkage });
169 @export(makeAtomicExchangeFn(u16).atomic_exchange_N, .{ .name = "__atomic_exchange_2", .linkage = linkage });
170 @export(makeAtomicExchangeFn(u32).atomic_exchange_N, .{ .name = "__atomic_exchange_4", .linkage = linkage });
171 @export(makeAtomicExchangeFn(u64).atomic_exchange_N, .{ .name = "__atomic_exchange_8", .linkage = linkage });
172}
173
174fn makeAtomicCompareExchangeFn(comptime T: type) type {
175 return struct {
176 fn atomic_compare_exchange_N(ptr: *T, expected: *T, desired: T, success: i32, failure: i32) callconv(.C) i32 {
177 if (@sizeOf(T) > largest_atomic_size) {
178 var sl = spinlocks.get(@ptrToInt(ptr));
179 defer sl.release();
180 if (ptr.* == expected.*) {
181 ptr.* = desired;
182 return 1;
183 }
184 expected.* = ptr.*;
185 return 0;
186 } else {
187 if (@cmpxchgStrong(T, ptr, expected.*, desired, .SeqCst, .SeqCst)) |old_value| {
188 expected.* = old_value;
189 return 0;
190 }
191 return 1;
192 }
193 }
194 };
195}
196
197comptime {
198 @export(makeAtomicCompareExchangeFn(u8).atomic_compare_exchange_N, .{ .name = "__atomic_compare_exchange_1", .linkage = linkage });
199 @export(makeAtomicCompareExchangeFn(u16).atomic_compare_exchange_N, .{ .name = "__atomic_compare_exchange_2", .linkage = linkage });
200 @export(makeAtomicCompareExchangeFn(u32).atomic_compare_exchange_N, .{ .name = "__atomic_compare_exchange_4", .linkage = linkage });
201 @export(makeAtomicCompareExchangeFn(u64).atomic_compare_exchange_N, .{ .name = "__atomic_compare_exchange_8", .linkage = linkage });
202}
203
204fn makeFetchFn(comptime T: type, comptime op: builtin.AtomicRmwOp) type {
205 return struct {
206 pub fn fetch_op_N(ptr: *T, val: T, model: i32) callconv(.C) T {
207 if (@sizeOf(T) > largest_atomic_size) {
208 var sl = spinlocks.get(@ptrToInt(ptr));
209 defer sl.release();
210
211 var value = ptr.*;
212 ptr.* = switch (op) {
213 .Add => ptr.* +% val,
214 .Sub => ptr.* -% val,
215 .And => ptr.* & val,
216 .Nand => ~(ptr.* & val),
217 .Or => ptr.* | val,
218 .Xor => ptr.* ^ val,
219 else => @compileError("unsupported atomic op"),
220 };
221
222 return value;
223 }
224
225 return @atomicRmw(T, ptr, op, val, .SeqCst);
226 }
227 };
228}
229
230comptime {
231 @export(makeFetchFn(u8, .Add).fetch_op_N, .{ .name = "__atomic_fetch_add_1", .linkage = linkage });
232 @export(makeFetchFn(u16, .Add).fetch_op_N, .{ .name = "__atomic_fetch_add_2", .linkage = linkage });
233 @export(makeFetchFn(u32, .Add).fetch_op_N, .{ .name = "__atomic_fetch_add_4", .linkage = linkage });
234 @export(makeFetchFn(u64, .Add).fetch_op_N, .{ .name = "__atomic_fetch_add_8", .linkage = linkage });
235
236 @export(makeFetchFn(u8, .Sub).fetch_op_N, .{ .name = "__atomic_fetch_sub_1", .linkage = linkage });
237 @export(makeFetchFn(u16, .Sub).fetch_op_N, .{ .name = "__atomic_fetch_sub_2", .linkage = linkage });
238 @export(makeFetchFn(u32, .Sub).fetch_op_N, .{ .name = "__atomic_fetch_sub_4", .linkage = linkage });
239 @export(makeFetchFn(u64, .Sub).fetch_op_N, .{ .name = "__atomic_fetch_sub_8", .linkage = linkage });
240
241 @export(makeFetchFn(u8, .And).fetch_op_N, .{ .name = "__atomic_fetch_and_1", .linkage = linkage });
242 @export(makeFetchFn(u16, .And).fetch_op_N, .{ .name = "__atomic_fetch_and_2", .linkage = linkage });
243 @export(makeFetchFn(u32, .And).fetch_op_N, .{ .name = "__atomic_fetch_and_4", .linkage = linkage });
244 @export(makeFetchFn(u64, .And).fetch_op_N, .{ .name = "__atomic_fetch_and_8", .linkage = linkage });
245
246 @export(makeFetchFn(u8, .Or).fetch_op_N, .{ .name = "__atomic_fetch_or_1", .linkage = linkage });
247 @export(makeFetchFn(u16, .Or).fetch_op_N, .{ .name = "__atomic_fetch_or_2", .linkage = linkage });
248 @export(makeFetchFn(u32, .Or).fetch_op_N, .{ .name = "__atomic_fetch_or_4", .linkage = linkage });
249 @export(makeFetchFn(u64, .Or).fetch_op_N, .{ .name = "__atomic_fetch_or_8", .linkage = linkage });
250
251 @export(makeFetchFn(u8, .Xor).fetch_op_N, .{ .name = "__atomic_fetch_xor_1", .linkage = linkage });
252 @export(makeFetchFn(u16, .Xor).fetch_op_N, .{ .name = "__atomic_fetch_xor_2", .linkage = linkage });
253 @export(makeFetchFn(u32, .Xor).fetch_op_N, .{ .name = "__atomic_fetch_xor_4", .linkage = linkage });
254 @export(makeFetchFn(u64, .Xor).fetch_op_N, .{ .name = "__atomic_fetch_xor_8", .linkage = linkage });
255
256 @export(makeFetchFn(u8, .Nand).fetch_op_N, .{ .name = "__atomic_fetch_nand_1", .linkage = linkage });
257 @export(makeFetchFn(u16, .Nand).fetch_op_N, .{ .name = "__atomic_fetch_nand_2", .linkage = linkage });
258 @export(makeFetchFn(u32, .Nand).fetch_op_N, .{ .name = "__atomic_fetch_nand_4", .linkage = linkage });
259 @export(makeFetchFn(u64, .Nand).fetch_op_N, .{ .name = "__atomic_fetch_nand_8", .linkage = linkage });
260}