1/* $OpenBSD: atomic.h,v 1.19 2022/08/29 02:01:18 jsg Exp $ */
2
3/* Public Domain */
4
5#ifndef _ARM_ATOMIC_H_
6#define _ARM_ATOMIC_H_
7
8/*
9 * Compare and set:
10 * ret = *ptr
11 * if (ret == expect)
12 * *ptr = new
13 * return (ret)
14 */
15#define _def_atomic_cas(_f, _t) \
16static inline _t \
17_f(volatile _t *p, _t e, _t n) \
18{ \
19 _t ret, modified; \
20 \
21 __asm volatile ( \
22 "1: ldrex %0, [%4] \n\t" \
23 " cmp %0, %3 \n\t" \
24 " bne 2f \n\t" \
25 " strex %1, %2, [%4] \n\t" \
26 " cmp %1, #0 \n\t" \
27 " bne 1b \n\t" \
28 " b 3f \n\t" \
29 "2: clrex \n\t" \
30 "3: \n\t" \
31 : "=&r" (ret), "=&r" (modified) \
32 : "r" (n), "r" (e), "r" (p) \
33 : "memory", "cc" \
34 ); \
35 return (ret); \
36}
37_def_atomic_cas(_atomic_cas_uint, unsigned int)
38_def_atomic_cas(_atomic_cas_ulong, unsigned long)
39#undef _def_atomic_cas
40
41#define atomic_cas_uint(_p, _e, _n) _atomic_cas_uint((_p), (_e), (_n))
42#define atomic_cas_ulong(_p, _e, _n) _atomic_cas_ulong((_p), (_e), (_n))
43
44static inline void *
45_atomic_cas_ptr(volatile void *p, void *e, void *n)
46{
47 void *ret;
48 unsigned long modified;
49
50 __asm volatile (
51 "1: ldrex %0, [%4] \n\t"
52 " cmp %0, %3 \n\t"
53 " bne 2f \n\t"
54 " strex %1, %2, [%4] \n\t"
55 " cmp %1, #0 \n\t"
56 " bne 1b \n\t"
57 " b 3f \n\t"
58 "2: clrex \n\t"
59 "3: \n\t"
60 : "=&r" (ret), "=&r" (modified)
61 : "r" (n), "r" (e), "r" (p)
62 : "memory", "cc"
63 );
64 return (ret);
65}
66#define atomic_cas_ptr(_p, _e, _n) _atomic_cas_ptr((_p), (_e), (_n))
67
68/*
69 * Swap:
70 * ret = *p
71 * *p = val
72 * return (ret)
73 */
74#define _def_atomic_swap(_f, _t) \
75static inline _t \
76_f(volatile _t *p, _t v) \
77{ \
78 _t ret, modified; \
79 \
80 __asm volatile ( \
81 "1: ldrex %0, [%3] \n\t" \
82 " strex %1, %2, [%3] \n\t" \
83 " cmp %1, #0 \n\t" \
84 " bne 1b \n\t" \
85 : "=&r" (ret), "=&r" (modified) \
86 : "r" (v), "r" (p) \
87 : "memory", "cc" \
88 ); \
89 return (ret); \
90}
91_def_atomic_swap(_atomic_swap_uint, unsigned int)
92_def_atomic_swap(_atomic_swap_ulong, unsigned long)
93#undef _def_atomic_swap
94
95#define atomic_swap_uint(_p, _v) _atomic_swap_uint((_p), (_v))
96#define atomic_swap_ulong(_p, _v) _atomic_swap_ulong((_p), (_v))
97
98static inline void *
99_atomic_swap_ptr(volatile void *p, void *v)
100{
101 void *ret;
102 unsigned long modified;
103
104 __asm volatile (
105 "1: ldrex %0, [%3] \n\t"
106 " strex %1, %2, [%3] \n\t"
107 " cmp %1, #0 \n\t"
108 " bne 1b \n\t"
109 : "=&r" (ret), "=&r" (modified)
110 : "r" (v), "r" (p)
111 : "memory", "cc"
112 );
113 return (ret);
114}
115#define atomic_swap_ptr(_p, _v) _atomic_swap_ptr((_p), (_v))
116
117/*
118 * Increment returning the new value
119 * *p += 1
120 * return (*p)
121 */
122#define _def_atomic_inc_nv(_f, _t) \
123static inline _t \
124_f(volatile _t *p) \
125{ \
126 _t ret, modified; \
127 \
128 __asm volatile ( \
129 "1: ldrex %0, [%2] \n\t" \
130 " add %0, %0, #1 \n\t" \
131 " strex %1, %0, [%2] \n\t" \
132 " cmp %1, #0 \n\t" \
133 " bne 1b \n\t" \
134 : "=&r" (ret), "=&r" (modified) \
135 : "r" (p) \
136 : "memory", "cc" \
137 ); \
138 return (ret); \
139}
140_def_atomic_inc_nv(_atomic_inc_int_nv, unsigned int)
141_def_atomic_inc_nv(_atomic_inc_long_nv, unsigned long)
142#undef _def_atomic_inc_nv
143
144#define atomic_inc_int_nv(_p) _atomic_inc_int_nv((_p))
145#define atomic_inc_long_nv(_p) _atomic_inc_long_nv((_p))
146
147/*
148 * Decrement returning the new value
149 * *p -= 1
150 * return (*p)
151 */
152#define _def_atomic_dec_nv(_f, _t) \
153static inline _t \
154_f(volatile _t *p) \
155{ \
156 _t ret, modified; \
157 \
158 __asm volatile ( \
159 "1: ldrex %0, [%2] \n\t" \
160 " sub %0, %0, #1 \n\t" \
161 " strex %1, %0, [%2] \n\t" \
162 " cmp %1, #0 \n\t" \
163 " bne 1b \n\t" \
164 : "=&r" (ret), "=&r" (modified) \
165 : "r" (p) \
166 : "memory", "cc" \
167 ); \
168 return (ret); \
169}
170_def_atomic_dec_nv(_atomic_dec_int_nv, unsigned int)
171_def_atomic_dec_nv(_atomic_dec_long_nv, unsigned long)
172#undef _def_atomic_dec_nv
173
174#define atomic_dec_int_nv(_p) _atomic_dec_int_nv((_p))
175#define atomic_dec_long_nv(_p) _atomic_dec_long_nv((_p))
176
177/*
178 * Addition returning the new value
179 * *p += v
180 * return (*p)
181 */
182#define _def_atomic_add_nv(_f, _t) \
183static inline _t \
184_f(volatile _t *p, _t v) \
185{ \
186 _t ret, modified; \
187 \
188 __asm volatile ( \
189 "1: ldrex %0, [%2] \n\t" \
190 " add %0, %0, %3 \n\t" \
191 " strex %1, %0, [%2] \n\t" \
192 " cmp %1, #0 \n\t" \
193 " bne 1b \n\t" \
194 : "=&r" (ret), "=&r" (modified) \
195 : "r" (p), "r" (v) \
196 : "memory", "cc" \
197 ); \
198 return (ret); \
199}
200_def_atomic_add_nv(_atomic_add_int_nv, unsigned int)
201_def_atomic_add_nv(_atomic_add_long_nv, unsigned long)
202#undef _def_atomic_add_nv
203
204#define atomic_add_int_nv(_p, _v) _atomic_add_int_nv((_p), (_v))
205#define atomic_add_long_nv(_p, _v) _atomic_add_long_nv((_p), (_v))
206
207/*
208 * Subtraction returning the new value
209 * *p -= v
210 * return (*p)
211 */
212#define _def_atomic_sub_nv(_f, _t) \
213static inline _t \
214_f(volatile _t *p, _t v) \
215{ \
216 _t ret, modified; \
217 \
218 __asm volatile ( \
219 "1: ldrex %0, [%2] \n\t" \
220 " sub %0, %0, %3 \n\t" \
221 " strex %1, %0, [%2] \n\t" \
222 " cmp %1, #0 \n\t" \
223 " bne 1b \n\t" \
224 : "=&r" (ret), "=&r" (modified) \
225 : "r" (p), "r" (v) \
226 : "memory", "cc" \
227 ); \
228 return (ret); \
229}
230_def_atomic_sub_nv(_atomic_sub_int_nv, unsigned int)
231_def_atomic_sub_nv(_atomic_sub_long_nv, unsigned long)
232#undef _def_atomic_sub_nv
233
234#define atomic_sub_int_nv(_p, _v) _atomic_sub_int_nv((_p), (_v))
235#define atomic_sub_long_nv(_p, _v) _atomic_sub_long_nv((_p), (_v))
236
237#define __membar(_f) do { __asm volatile(_f ::: "memory"); } while (0)
238
239#define membar_enter() __membar("dmb sy")
240#define membar_exit() __membar("dmb sy")
241#define membar_producer() __membar("dmb st")
242#define membar_consumer() __membar("dmb sy")
243#define membar_sync() __membar("dmb sy")
244
245#if defined(_KERNEL)
246
247/* virtio needs MP membars even on SP kernels */
248#define virtio_membar_producer() __membar("dmb st")
249#define virtio_membar_consumer() __membar("dmb sy")
250#define virtio_membar_sync() __membar("dmb sy")
251
252/*
253 * Set bits
254 * *p = *p | v
255 */
256static inline void
257atomic_setbits_int(volatile unsigned int *p, unsigned int v)
258{
259 unsigned int modified, tmp;
260
261 __asm volatile (
262 "1: ldrex %0, [%3] \n\t"
263 " orr %0, %0, %2 \n\t"
264 " strex %1, %0, [%3] \n\t"
265 " cmp %1, #0 \n\t"
266 " bne 1b \n\t"
267 : "=&r" (tmp), "=&r" (modified)
268 : "r" (v), "r" (p)
269 : "memory", "cc"
270 );
271}
272
273/*
274 * Clear bits
275 * *p = *p & (~v)
276 */
277static inline void
278atomic_clearbits_int(volatile unsigned int *p, unsigned int v)
279{
280 unsigned int modified, tmp;
281
282 __asm volatile (
283 "1: ldrex %0, [%3] \n\t"
284 " bic %0, %0, %2 \n\t"
285 " strex %1, %0, [%3] \n\t"
286 " cmp %1, #0 \n\t"
287 " bne 1b \n\t"
288 : "=&r" (tmp), "=&r" (modified)
289 : "r" (v), "r" (p)
290 : "memory", "cc"
291 );
292}
293
294#endif /* defined(_KERNEL) */
295#endif /* _ARM_ATOMIC_H_ */