1/* $OpenBSD: atomic.h,v 1.20 2022/08/29 02:01:18 jsg Exp $ */
2/* $NetBSD: atomic.h,v 1.1.2.2 2000/02/21 18:54:07 sommerfeld Exp $ */
3
4/*-
5 * Copyright (c) 2000 The NetBSD Foundation, Inc.
6 * All rights reserved.
7 *
8 * This code is derived from software contributed to The NetBSD Foundation
9 * by RedBack Networks Inc.
10 *
11 * Author: Bill Sommerfeld
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 * 1. Redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer.
18 * 2. Redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
23 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
24 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
25 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
26 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
27 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
28 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
29 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
30 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
31 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
32 * POSSIBILITY OF SUCH DAMAGE.
33 */
34
35#ifndef _MACHINE_ATOMIC_H_
36#define _MACHINE_ATOMIC_H_
37
38/*
39 * Perform atomic operations on memory. Should be atomic with respect
40 * to interrupts and multiple processors.
41 *
42 * void atomic_setbits_int(volatile u_int *a, u_int mask) { *a |= mask; }
43 * void atomic_clearbits_int(volatile u_int *a, u_int mas) { *a &= ~mask; }
44 */
45#if !defined(_LOCORE)
46
47#if defined(MULTIPROCESSOR) || !defined(_KERNEL)
48#define _LOCK "lock"
49#else
50#define _LOCK
51#endif
52
53static inline unsigned int
54_atomic_cas_uint(volatile unsigned int *p, unsigned int e, unsigned int n)
55{
56 __asm volatile(_LOCK " cmpxchgl %2, %1"
57 : "=a" (n), "=m" (*p)
58 : "r" (n), "a" (e), "m" (*p));
59
60 return (n);
61}
62#define atomic_cas_uint(_p, _e, _n) _atomic_cas_uint((_p), (_e), (_n))
63
64static inline unsigned long
65_atomic_cas_ulong(volatile unsigned long *p, unsigned long e, unsigned long n)
66{
67 __asm volatile(_LOCK " cmpxchgl %2, %1"
68 : "=a" (n), "=m" (*p)
69 : "r" (n), "a" (e), "m" (*p));
70
71 return (n);
72}
73#define atomic_cas_ulong(_p, _e, _n) _atomic_cas_ulong((_p), (_e), (_n))
74
75static inline void *
76_atomic_cas_ptr(volatile void *p, void *e, void *n)
77{
78 __asm volatile(_LOCK " cmpxchgl %2, %1"
79 : "=a" (n), "=m" (*(unsigned long *)p)
80 : "r" (n), "a" (e), "m" (*(unsigned long *)p));
81
82 return (n);
83}
84#define atomic_cas_ptr(_p, _e, _n) _atomic_cas_ptr((_p), (_e), (_n))
85
86static inline unsigned int
87_atomic_swap_uint(volatile unsigned int *p, unsigned int n)
88{
89 __asm volatile("xchgl %0, %1"
90 : "=a" (n), "=m" (*p)
91 : "0" (n), "m" (*p));
92
93 return (n);
94}
95#define atomic_swap_uint(_p, _n) _atomic_swap_uint((_p), (_n))
96#define atomic_swap_32(_p, _n) _atomic_swap_uint((_p), (_n))
97
98static inline unsigned long
99_atomic_swap_ulong(volatile unsigned long *p, unsigned long n)
100{
101 __asm volatile("xchgl %0, %1"
102 : "=a" (n), "=m" (*p)
103 : "0" (n), "m" (*p));
104
105 return (n);
106}
107#define atomic_swap_ulong(_p, _n) _atomic_swap_ulong((_p), (_n))
108
109static inline void *
110_atomic_swap_ptr(volatile void *p, void *n)
111{
112 __asm volatile("xchgl %0, %1"
113 : "=a" (n), "=m" (*(unsigned long *)p)
114 : "0" (n), "m" (*(unsigned long *)p));
115
116 return (n);
117}
118#define atomic_swap_ptr(_p, _n) _atomic_swap_ptr((_p), (_n))
119
120static inline void
121_atomic_inc_int(volatile unsigned int *p)
122{
123 __asm volatile(_LOCK " incl %0"
124 : "+m" (*p));
125}
126#define atomic_inc_int(_p) _atomic_inc_int(_p)
127
128static inline void
129_atomic_inc_long(volatile unsigned long *p)
130{
131 __asm volatile(_LOCK " incl %0"
132 : "+m" (*p));
133}
134#define atomic_inc_long(_p) _atomic_inc_long(_p)
135
136static inline void
137_atomic_dec_int(volatile unsigned int *p)
138{
139 __asm volatile(_LOCK " decl %0"
140 : "+m" (*p));
141}
142#define atomic_dec_int(_p) _atomic_dec_int(_p)
143
144static inline void
145_atomic_dec_long(volatile unsigned long *p)
146{
147 __asm volatile(_LOCK " decl %0"
148 : "+m" (*p));
149}
150#define atomic_dec_long(_p) _atomic_dec_long(_p)
151
152static inline void
153_atomic_add_int(volatile unsigned int *p, unsigned int v)
154{
155 __asm volatile(_LOCK " addl %1,%0"
156 : "+m" (*p)
157 : "a" (v));
158}
159#define atomic_add_int(_p, _v) _atomic_add_int(_p, _v)
160
161static inline void
162_atomic_add_long(volatile unsigned long *p, unsigned long v)
163{
164 __asm volatile(_LOCK " addl %1,%0"
165 : "+m" (*p)
166 : "a" (v));
167}
168#define atomic_add_long(_p, _v) _atomic_add_long(_p, _v)
169
170static inline void
171_atomic_sub_int(volatile unsigned int *p, unsigned int v)
172{
173 __asm volatile(_LOCK " subl %1,%0"
174 : "+m" (*p)
175 : "a" (v));
176}
177#define atomic_sub_int(_p, _v) _atomic_sub_int(_p, _v)
178
179static inline void
180_atomic_sub_long(volatile unsigned long *p, unsigned long v)
181{
182 __asm volatile(_LOCK " subl %1,%0"
183 : "+m" (*p)
184 : "a" (v));
185}
186#define atomic_sub_long(_p, _v) _atomic_sub_long(_p, _v)
187
188
189static inline unsigned long
190_atomic_add_int_nv(volatile unsigned int *p, unsigned int v)
191{
192 unsigned int rv = v;
193
194 __asm volatile(_LOCK " xaddl %0,%1"
195 : "+a" (rv), "+m" (*p));
196
197 return (rv + v);
198}
199#define atomic_add_int_nv(_p, _v) _atomic_add_int_nv(_p, _v)
200
201static inline unsigned long
202_atomic_add_long_nv(volatile unsigned long *p, unsigned long v)
203{
204 unsigned long rv = v;
205
206 __asm volatile(_LOCK " xaddl %0,%1"
207 : "+a" (rv), "+m" (*p));
208
209 return (rv + v);
210}
211#define atomic_add_long_nv(_p, _v) _atomic_add_long_nv(_p, _v)
212
213static inline unsigned long
214_atomic_sub_int_nv(volatile unsigned int *p, unsigned int v)
215{
216 unsigned int rv = 0 - v;
217
218 __asm volatile(_LOCK " xaddl %0,%1"
219 : "+a" (rv), "+m" (*p));
220
221 return (rv - v);
222}
223#define atomic_sub_int_nv(_p, _v) _atomic_sub_int_nv(_p, _v)
224
225static inline unsigned long
226_atomic_sub_long_nv(volatile unsigned long *p, unsigned long v)
227{
228 unsigned long rv = 0 - v;
229
230 __asm volatile(_LOCK " xaddl %0,%1"
231 : "+a" (rv), "+m" (*p));
232
233 return (rv - v);
234}
235#define atomic_sub_long_nv(_p, _v) _atomic_sub_long_nv(_p, _v)
236
237/*
238 * The IA-32 architecture is rather strongly ordered. When accessing
239 * normal write-back cacheable memory, only reads may be reordered with
240 * older writes to different locations. There are a few instructions
241 * (clfush, non-temporal move instructions) that obey weaker ordering
242 * rules, but those instructions will only be used in (inline)
243 * assembly code where we can add the necessary fence instructions
244 * ourselves.
245 */
246
247#define __membar(_f) do { __asm volatile(_f ::: "memory"); } while (0)
248
249#if defined(MULTIPROCESSOR) || !defined(_KERNEL)
250#define membar_enter() __membar("lock; addl $0,0(%%esp)")
251#define membar_exit() __membar("")
252#define membar_producer() __membar("")
253#define membar_consumer() __membar("")
254#define membar_sync() __membar("lock; addl $0,0(%%esp)")
255#else
256#define membar_enter() __membar("")
257#define membar_exit() __membar("")
258#define membar_producer() __membar("")
259#define membar_consumer() __membar("")
260#define membar_sync() __membar("")
261#endif
262
263#define membar_enter_after_atomic() __membar("")
264#define membar_exit_before_atomic() __membar("")
265
266#ifdef _KERNEL
267
268/* virtio needs MP membars even on SP kernels */
269#define virtio_membar_producer() __membar("")
270#define virtio_membar_consumer() __membar("")
271#define virtio_membar_sync() __membar("lock; addl $0,0(%%esp)")
272
273static __inline u_int64_t
274i386_atomic_testset_uq(volatile u_int64_t *ptr, u_int64_t val)
275{
276 __asm__ volatile ("\n1:\t" _LOCK " cmpxchg8b (%1); jnz 1b" : "+A" (val) :
277 "r" (ptr), "b" ((u_int32_t)val), "c" ((u_int32_t)(val >> 32)));
278 return val;
279}
280
281static __inline u_int32_t
282i386_atomic_testset_ul(volatile u_int32_t *ptr, unsigned long val)
283{
284 __asm__ volatile ("xchgl %0,(%2)" :"=r" (val):"0" (val),"r" (ptr));
285 return val;
286}
287
288static __inline int
289i386_atomic_testset_i(volatile int *ptr, unsigned long val)
290{
291 __asm__ volatile ("xchgl %0,(%2)" :"=r" (val):"0" (val),"r" (ptr));
292 return val;
293}
294
295static __inline void
296i386_atomic_setbits_l(volatile u_int32_t *ptr, unsigned long bits)
297{
298 __asm volatile(_LOCK " orl %1,%0" : "=m" (*ptr) : "ir" (bits));
299}
300
301static __inline void
302i386_atomic_clearbits_l(volatile u_int32_t *ptr, unsigned long bits)
303{
304 bits = ~bits;
305 __asm volatile(_LOCK " andl %1,%0" : "=m" (*ptr) : "ir" (bits));
306}
307
308#define atomic_setbits_int i386_atomic_setbits_l
309#define atomic_clearbits_int i386_atomic_clearbits_l
310
311#endif /* _KERNEL */
312
313#undef _LOCK
314
315#endif /* !defined(_LOCORE) */
316#endif /* _MACHINE_ATOMIC_H_ */