1/*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 1998 Doug Rabson
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28#ifndef _MACHINE_ATOMIC_H_
29#define _MACHINE_ATOMIC_H_
30
31#include <sys/atomic_common.h>
32
33#ifdef _KERNEL
34#include <machine/md_var.h>
35#include <machine/specialreg.h>
36#endif
37
38#ifndef __OFFSETOF_MONITORBUF
39/*
40 * __OFFSETOF_MONITORBUF == __pcpu_offset(pc_monitorbuf).
41 *
42 * The open-coded number is used instead of the symbolic expression to
43 * avoid a dependency on sys/pcpu.h in machine/atomic.h consumers.
44 * An assertion in i386/vm_machdep.c ensures that the value is correct.
45 */
46#define __OFFSETOF_MONITORBUF 0x80
47
48static __inline void
49__mbk(void)
50{
51
52 __asm __volatile("lock; addl $0,%%fs:%c0"
53 : : "i" (__OFFSETOF_MONITORBUF) : "memory", "cc");
54}
55
56static __inline void
57__mbu(void)
58{
59
60 __asm __volatile("lock; addl $0,(%%esp)" : : : "memory", "cc");
61}
62#endif
63
64/*
65 * Various simple operations on memory, each of which is atomic in the
66 * presence of interrupts and multiple processors.
67 *
68 * atomic_set_char(P, V) (*(u_char *)(P) |= (V))
69 * atomic_clear_char(P, V) (*(u_char *)(P) &= ~(V))
70 * atomic_add_char(P, V) (*(u_char *)(P) += (V))
71 * atomic_subtract_char(P, V) (*(u_char *)(P) -= (V))
72 *
73 * atomic_set_short(P, V) (*(u_short *)(P) |= (V))
74 * atomic_clear_short(P, V) (*(u_short *)(P) &= ~(V))
75 * atomic_add_short(P, V) (*(u_short *)(P) += (V))
76 * atomic_subtract_short(P, V) (*(u_short *)(P) -= (V))
77 *
78 * atomic_set_int(P, V) (*(u_int *)(P) |= (V))
79 * atomic_clear_int(P, V) (*(u_int *)(P) &= ~(V))
80 * atomic_add_int(P, V) (*(u_int *)(P) += (V))
81 * atomic_subtract_int(P, V) (*(u_int *)(P) -= (V))
82 * atomic_swap_int(P, V) (return (*(u_int *)(P)); *(u_int *)(P) = (V);)
83 * atomic_readandclear_int(P) (return (*(u_int *)(P)); *(u_int *)(P) = 0;)
84 *
85 * atomic_set_long(P, V) (*(u_long *)(P) |= (V))
86 * atomic_clear_long(P, V) (*(u_long *)(P) &= ~(V))
87 * atomic_add_long(P, V) (*(u_long *)(P) += (V))
88 * atomic_subtract_long(P, V) (*(u_long *)(P) -= (V))
89 * atomic_swap_long(P, V) (return (*(u_long *)(P)); *(u_long *)(P) = (V);)
90 * atomic_readandclear_long(P) (return (*(u_long *)(P)); *(u_long *)(P) = 0;)
91 */
92
93/*
94 * Always use lock prefixes. The result is slightly less optimal for
95 * UP systems, but it matters less now, and sometimes UP is emulated
96 * over SMP.
97 *
98 * The assembly is volatilized to avoid code chunk removal by the compiler.
99 * GCC aggressively reorders operations and memory clobbering is necessary
100 * in order to avoid that for memory barriers.
101 */
102#define ATOMIC_ASM(NAME, TYPE, OP, CONS, V) \
103static __inline void \
104atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\
105{ \
106 __asm __volatile("lock; " OP \
107 : "+m" (*p) \
108 : CONS (V) \
109 : "cc"); \
110} \
111 \
112static __inline void \
113atomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\
114{ \
115 __asm __volatile("lock; " OP \
116 : "+m" (*p) \
117 : CONS (V) \
118 : "memory", "cc"); \
119} \
120struct __hack
121
122/*
123 * Atomic compare and set, used by the mutex functions.
124 *
125 * cmpset:
126 * if (*dst == expect)
127 * *dst = src
128 *
129 * fcmpset:
130 * if (*dst == *expect)
131 * *dst = src
132 * else
133 * *expect = *dst
134 *
135 * Returns 0 on failure, non-zero on success.
136 */
137#define ATOMIC_CMPSET(TYPE, CONS) \
138static __inline int \
139atomic_cmpset_##TYPE(volatile u_##TYPE *dst, u_##TYPE expect, u_##TYPE src) \
140{ \
141 u_char res; \
142 \
143 __asm __volatile( \
144 " lock; cmpxchg %3,%1 ; " \
145 " sete %0 ; " \
146 "# atomic_cmpset_" #TYPE " " \
147 : "=q" (res), /* 0 */ \
148 "+m" (*dst), /* 1 */ \
149 "+a" (expect) /* 2 */ \
150 : CONS (src) /* 3 */ \
151 : "memory", "cc"); \
152 return (res); \
153} \
154 \
155static __inline int \
156atomic_fcmpset_##TYPE(volatile u_##TYPE *dst, u_##TYPE *expect, u_##TYPE src) \
157{ \
158 u_char res; \
159 \
160 __asm __volatile( \
161 " lock; cmpxchg %3,%1 ; " \
162 " sete %0 ; " \
163 "# atomic_fcmpset_" #TYPE " " \
164 : "=q" (res), /* 0 */ \
165 "+m" (*dst), /* 1 */ \
166 "+a" (*expect) /* 2 */ \
167 : CONS (src) /* 3 */ \
168 : "memory", "cc"); \
169 return (res); \
170}
171
172ATOMIC_CMPSET(char, "q");
173ATOMIC_CMPSET(short, "r");
174ATOMIC_CMPSET(int, "r");
175
176/*
177 * Atomically add the value of v to the integer pointed to by p and return
178 * the previous value of *p.
179 */
180static __inline u_int
181atomic_fetchadd_int(volatile u_int *p, u_int v)
182{
183
184 __asm __volatile(
185 " lock; xaddl %0,%1 ; "
186 "# atomic_fetchadd_int"
187 : "+r" (v), /* 0 */
188 "+m" (*p) /* 1 */
189 : : "cc");
190 return (v);
191}
192
193static __inline int
194atomic_testandset_int(volatile u_int *p, u_int v)
195{
196 u_char res;
197
198 __asm __volatile(
199 " lock; btsl %2,%1 ; "
200 " setc %0 ; "
201 "# atomic_testandset_int"
202 : "=q" (res), /* 0 */
203 "+m" (*p) /* 1 */
204 : "Ir" (v & 0x1f) /* 2 */
205 : "cc");
206 return (res);
207}
208
209static __inline int
210atomic_testandclear_int(volatile u_int *p, u_int v)
211{
212 u_char res;
213
214 __asm __volatile(
215 " lock; btrl %2,%1 ; "
216 " setc %0 ; "
217 "# atomic_testandclear_int"
218 : "=q" (res), /* 0 */
219 "+m" (*p) /* 1 */
220 : "Ir" (v & 0x1f) /* 2 */
221 : "cc");
222 return (res);
223}
224
225/*
226 * We assume that a = b will do atomic loads and stores. Due to the
227 * IA32 memory model, a simple store guarantees release semantics.
228 *
229 * However, a load may pass a store if they are performed on distinct
230 * addresses, so we need Store/Load barrier for sequentially
231 * consistent fences in SMP kernels. We use "lock addl $0,mem" for a
232 * Store/Load barrier, as recommended by the AMD Software Optimization
233 * Guide, and not mfence. In the kernel, we use a private per-cpu
234 * cache line for "mem", to avoid introducing false data
235 * dependencies. In user space, we use the word at the top of the
236 * stack.
237 *
238 * For UP kernels, however, the memory of the single processor is
239 * always consistent, so we only need to stop the compiler from
240 * reordering accesses in a way that violates the semantics of acquire
241 * and release.
242 */
243
244#if defined(_KERNEL)
245#define __storeload_barrier() __mbk()
246#else /* !_KERNEL */
247#define __storeload_barrier() __mbu()
248#endif /* _KERNEL*/
249
250#define ATOMIC_LOAD(TYPE) \
251static __inline u_##TYPE \
252atomic_load_acq_##TYPE(const volatile u_##TYPE *p) \
253{ \
254 u_##TYPE res; \
255 \
256 res = *p; \
257 __compiler_membar(); \
258 return (res); \
259} \
260struct __hack
261
262#define ATOMIC_STORE(TYPE) \
263static __inline void \
264atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \
265{ \
266 \
267 __compiler_membar(); \
268 *p = v; \
269} \
270struct __hack
271
272static __inline void
273atomic_thread_fence_acq(void)
274{
275
276 __compiler_membar();
277}
278
279static __inline void
280atomic_thread_fence_rel(void)
281{
282
283 __compiler_membar();
284}
285
286static __inline void
287atomic_thread_fence_acq_rel(void)
288{
289
290 __compiler_membar();
291}
292
293static __inline void
294atomic_thread_fence_seq_cst(void)
295{
296
297 __storeload_barrier();
298}
299
300#ifdef _KERNEL
301
302/* I486 does not support SMP or CMPXCHG8B. */
303static __inline int
304atomic_cmpset_64_i386(volatile uint64_t *dst, uint64_t expect, uint64_t src)
305{
306 volatile uint32_t *p;
307 u_char res;
308
309 p = (volatile uint32_t *)dst;
310 __asm __volatile(
311 " pushfl ; "
312 " cli ; "
313 " xorl %1,%%eax ; "
314 " xorl %2,%%edx ; "
315 " orl %%edx,%%eax ; "
316 " jne 1f ; "
317 " movl %4,%1 ; "
318 " movl %5,%2 ; "
319 "1: "
320 " sete %3 ; "
321 " popfl"
322 : "+A" (expect), /* 0 */
323 "+m" (*p), /* 1 */
324 "+m" (*(p + 1)), /* 2 */
325 "=q" (res) /* 3 */
326 : "r" ((uint32_t)src), /* 4 */
327 "r" ((uint32_t)(src >> 32)) /* 5 */
328 : "memory", "cc");
329 return (res);
330}
331
332static __inline int
333atomic_fcmpset_64_i386(volatile uint64_t *dst, uint64_t *expect, uint64_t src)
334{
335
336 if (atomic_cmpset_64_i386(dst, *expect, src)) {
337 return (1);
338 } else {
339 *expect = *dst;
340 return (0);
341 }
342}
343
344static __inline uint64_t
345atomic_load_acq_64_i386(const volatile uint64_t *p)
346{
347 const volatile uint32_t *q;
348 uint64_t res;
349
350 q = (const volatile uint32_t *)p;
351 __asm __volatile(
352 " pushfl ; "
353 " cli ; "
354 " movl %1,%%eax ; "
355 " movl %2,%%edx ; "
356 " popfl"
357 : "=&A" (res) /* 0 */
358 : "m" (*q), /* 1 */
359 "m" (*(q + 1)) /* 2 */
360 : "memory");
361 return (res);
362}
363
364static __inline void
365atomic_store_rel_64_i386(volatile uint64_t *p, uint64_t v)
366{
367 volatile uint32_t *q;
368
369 q = (volatile uint32_t *)p;
370 __asm __volatile(
371 " pushfl ; "
372 " cli ; "
373 " movl %%eax,%0 ; "
374 " movl %%edx,%1 ; "
375 " popfl"
376 : "=m" (*q), /* 0 */
377 "=m" (*(q + 1)) /* 1 */
378 : "A" (v) /* 2 */
379 : "memory");
380}
381
382static __inline uint64_t
383atomic_swap_64_i386(volatile uint64_t *p, uint64_t v)
384{
385 volatile uint32_t *q;
386 uint64_t res;
387
388 q = (volatile uint32_t *)p;
389 __asm __volatile(
390 " pushfl ; "
391 " cli ; "
392 " movl %1,%%eax ; "
393 " movl %2,%%edx ; "
394 " movl %4,%2 ; "
395 " movl %3,%1 ; "
396 " popfl"
397 : "=&A" (res), /* 0 */
398 "+m" (*q), /* 1 */
399 "+m" (*(q + 1)) /* 2 */
400 : "r" ((uint32_t)v), /* 3 */
401 "r" ((uint32_t)(v >> 32))); /* 4 */
402 return (res);
403}
404
405static __inline int
406atomic_cmpset_64_i586(volatile uint64_t *dst, uint64_t expect, uint64_t src)
407{
408 u_char res;
409
410 __asm __volatile(
411 " lock; cmpxchg8b %1 ; "
412 " sete %0"
413 : "=q" (res), /* 0 */
414 "+m" (*dst), /* 1 */
415 "+A" (expect) /* 2 */
416 : "b" ((uint32_t)src), /* 3 */
417 "c" ((uint32_t)(src >> 32)) /* 4 */
418 : "memory", "cc");
419 return (res);
420}
421
422static __inline int
423atomic_fcmpset_64_i586(volatile uint64_t *dst, uint64_t *expect, uint64_t src)
424{
425 u_char res;
426
427 __asm __volatile(
428 " lock; cmpxchg8b %1 ; "
429 " sete %0"
430 : "=q" (res), /* 0 */
431 "+m" (*dst), /* 1 */
432 "+A" (*expect) /* 2 */
433 : "b" ((uint32_t)src), /* 3 */
434 "c" ((uint32_t)(src >> 32)) /* 4 */
435 : "memory", "cc");
436 return (res);
437}
438
439/*
440 * Architecturally always writes back some value to '*p' so will trigger
441 * a #GP(0) on read-only mappings.
442 */
443static __inline uint64_t
444atomic_load_acq_64_i586(const volatile uint64_t *p)
445{
446 uint64_t res;
447
448 __asm __volatile(
449 " movl %%ebx,%%eax ; "
450 " movl %%ecx,%%edx ; "
451 " lock; cmpxchg8b %1"
452 : "=&A" (res) /* 0 */
453 : "m" (*p) /* 1 */
454 : "memory", "cc");
455 return (res);
456}
457
458static __inline void
459atomic_store_rel_64_i586(volatile uint64_t *p, uint64_t v)
460{
461
462 __asm __volatile(
463 " movl %%eax,%%ebx ; "
464 " movl %%edx,%%ecx ; "
465 "1: "
466 " lock; cmpxchg8b %0 ; "
467 " jne 1b"
468 : "+m" (*p), /* 0 */
469 "+A" (v) /* 1 */
470 : : "ebx", "ecx", "memory", "cc");
471}
472
473static __inline uint64_t
474atomic_swap_64_i586(volatile uint64_t *p, uint64_t v)
475{
476
477 __asm __volatile(
478 " movl %%eax,%%ebx ; "
479 " movl %%edx,%%ecx ; "
480 "1: "
481 " lock; cmpxchg8b %0 ; "
482 " jne 1b"
483 : "+m" (*p), /* 0 */
484 "+A" (v) /* 1 */
485 : : "ebx", "ecx", "memory", "cc");
486 return (v);
487}
488
489static __inline int
490atomic_cmpset_64(volatile uint64_t *dst, uint64_t expect, uint64_t src)
491{
492
493 if ((cpu_feature & CPUID_CX8) == 0)
494 return (atomic_cmpset_64_i386(dst, expect, src));
495 else
496 return (atomic_cmpset_64_i586(dst, expect, src));
497}
498
499static __inline int
500atomic_fcmpset_64(volatile uint64_t *dst, uint64_t *expect, uint64_t src)
501{
502
503 if ((cpu_feature & CPUID_CX8) == 0)
504 return (atomic_fcmpset_64_i386(dst, expect, src));
505 else
506 return (atomic_fcmpset_64_i586(dst, expect, src));
507}
508
509static __inline uint64_t
510atomic_load_acq_64(const volatile uint64_t *p)
511{
512
513 if ((cpu_feature & CPUID_CX8) == 0)
514 return (atomic_load_acq_64_i386(p));
515 else
516 return (atomic_load_acq_64_i586(p));
517}
518
519static __inline void
520atomic_store_rel_64(volatile uint64_t *p, uint64_t v)
521{
522
523 if ((cpu_feature & CPUID_CX8) == 0)
524 atomic_store_rel_64_i386(p, v);
525 else
526 atomic_store_rel_64_i586(p, v);
527}
528
529static __inline uint64_t
530atomic_swap_64(volatile uint64_t *p, uint64_t v)
531{
532
533 if ((cpu_feature & CPUID_CX8) == 0)
534 return (atomic_swap_64_i386(p, v));
535 else
536 return (atomic_swap_64_i586(p, v));
537}
538
539static __inline uint64_t
540atomic_fetchadd_64(volatile uint64_t *p, uint64_t v)
541{
542
543 for (;;) {
544 uint64_t t = *p;
545 if (atomic_cmpset_64(p, t, t + v))
546 return (t);
547 }
548}
549
550static __inline void
551atomic_add_64(volatile uint64_t *p, uint64_t v)
552{
553 uint64_t t;
554
555 for (;;) {
556 t = *p;
557 if (atomic_cmpset_64(p, t, t + v))
558 break;
559 }
560}
561
562static __inline void
563atomic_subtract_64(volatile uint64_t *p, uint64_t v)
564{
565 uint64_t t;
566
567 for (;;) {
568 t = *p;
569 if (atomic_cmpset_64(p, t, t - v))
570 break;
571 }
572}
573
574#endif /* _KERNEL */
575
576ATOMIC_ASM(set, char, "orb %b1,%0", "iq", v);
577ATOMIC_ASM(clear, char, "andb %b1,%0", "iq", ~v);
578ATOMIC_ASM(add, char, "addb %b1,%0", "iq", v);
579ATOMIC_ASM(subtract, char, "subb %b1,%0", "iq", v);
580
581ATOMIC_ASM(set, short, "orw %w1,%0", "ir", v);
582ATOMIC_ASM(clear, short, "andw %w1,%0", "ir", ~v);
583ATOMIC_ASM(add, short, "addw %w1,%0", "ir", v);
584ATOMIC_ASM(subtract, short, "subw %w1,%0", "ir", v);
585
586ATOMIC_ASM(set, int, "orl %1,%0", "ir", v);
587ATOMIC_ASM(clear, int, "andl %1,%0", "ir", ~v);
588ATOMIC_ASM(add, int, "addl %1,%0", "ir", v);
589ATOMIC_ASM(subtract, int, "subl %1,%0", "ir", v);
590
591ATOMIC_ASM(set, long, "orl %1,%0", "ir", v);
592ATOMIC_ASM(clear, long, "andl %1,%0", "ir", ~v);
593ATOMIC_ASM(add, long, "addl %1,%0", "ir", v);
594ATOMIC_ASM(subtract, long, "subl %1,%0", "ir", v);
595
596#define ATOMIC_LOADSTORE(TYPE) \
597 ATOMIC_LOAD(TYPE); \
598 ATOMIC_STORE(TYPE)
599
600ATOMIC_LOADSTORE(char);
601ATOMIC_LOADSTORE(short);
602ATOMIC_LOADSTORE(int);
603ATOMIC_LOADSTORE(long);
604
605#undef ATOMIC_ASM
606#undef ATOMIC_LOAD
607#undef ATOMIC_STORE
608#undef ATOMIC_LOADSTORE
609
610static __inline int
611atomic_cmpset_long(volatile u_long *dst, u_long expect, u_long src)
612{
613
614 return (atomic_cmpset_int((volatile u_int *)dst, (u_int)expect,
615 (u_int)src));
616}
617
618static __inline int
619atomic_fcmpset_long(volatile u_long *dst, u_long *expect, u_long src)
620{
621
622 return (atomic_fcmpset_int((volatile u_int *)dst, (u_int *)expect,
623 (u_int)src));
624}
625
626static __inline u_long
627atomic_fetchadd_long(volatile u_long *p, u_long v)
628{
629
630 return (atomic_fetchadd_int((volatile u_int *)p, (u_int)v));
631}
632
633static __inline int
634atomic_testandset_long(volatile u_long *p, u_int v)
635{
636
637 return (atomic_testandset_int((volatile u_int *)p, v));
638}
639
640static __inline int
641atomic_testandclear_long(volatile u_long *p, u_int v)
642{
643
644 return (atomic_testandclear_int((volatile u_int *)p, v));
645}
646
647/* Read the current value and store a new value in the destination. */
648static __inline u_int
649atomic_swap_int(volatile u_int *p, u_int v)
650{
651
652 __asm __volatile(
653 " xchgl %1,%0 ; "
654 "# atomic_swap_int"
655 : "+r" (v), /* 0 */
656 "+m" (*p)); /* 1 */
657 return (v);
658}
659
660static __inline u_long
661atomic_swap_long(volatile u_long *p, u_long v)
662{
663
664 return (atomic_swap_int((volatile u_int *)p, (u_int)v));
665}
666
667#define atomic_set_acq_char atomic_set_barr_char
668#define atomic_set_rel_char atomic_set_barr_char
669#define atomic_clear_acq_char atomic_clear_barr_char
670#define atomic_clear_rel_char atomic_clear_barr_char
671#define atomic_add_acq_char atomic_add_barr_char
672#define atomic_add_rel_char atomic_add_barr_char
673#define atomic_subtract_acq_char atomic_subtract_barr_char
674#define atomic_subtract_rel_char atomic_subtract_barr_char
675#define atomic_cmpset_acq_char atomic_cmpset_char
676#define atomic_cmpset_rel_char atomic_cmpset_char
677#define atomic_fcmpset_acq_char atomic_fcmpset_char
678#define atomic_fcmpset_rel_char atomic_fcmpset_char
679
680#define atomic_set_acq_short atomic_set_barr_short
681#define atomic_set_rel_short atomic_set_barr_short
682#define atomic_clear_acq_short atomic_clear_barr_short
683#define atomic_clear_rel_short atomic_clear_barr_short
684#define atomic_add_acq_short atomic_add_barr_short
685#define atomic_add_rel_short atomic_add_barr_short
686#define atomic_subtract_acq_short atomic_subtract_barr_short
687#define atomic_subtract_rel_short atomic_subtract_barr_short
688#define atomic_cmpset_acq_short atomic_cmpset_short
689#define atomic_cmpset_rel_short atomic_cmpset_short
690#define atomic_fcmpset_acq_short atomic_fcmpset_short
691#define atomic_fcmpset_rel_short atomic_fcmpset_short
692
693#define atomic_set_acq_int atomic_set_barr_int
694#define atomic_set_rel_int atomic_set_barr_int
695#define atomic_clear_acq_int atomic_clear_barr_int
696#define atomic_clear_rel_int atomic_clear_barr_int
697#define atomic_add_acq_int atomic_add_barr_int
698#define atomic_add_rel_int atomic_add_barr_int
699#define atomic_subtract_acq_int atomic_subtract_barr_int
700#define atomic_subtract_rel_int atomic_subtract_barr_int
701#define atomic_cmpset_acq_int atomic_cmpset_int
702#define atomic_cmpset_rel_int atomic_cmpset_int
703#define atomic_fcmpset_acq_int atomic_fcmpset_int
704#define atomic_fcmpset_rel_int atomic_fcmpset_int
705
706#define atomic_set_acq_long atomic_set_barr_long
707#define atomic_set_rel_long atomic_set_barr_long
708#define atomic_clear_acq_long atomic_clear_barr_long
709#define atomic_clear_rel_long atomic_clear_barr_long
710#define atomic_add_acq_long atomic_add_barr_long
711#define atomic_add_rel_long atomic_add_barr_long
712#define atomic_subtract_acq_long atomic_subtract_barr_long
713#define atomic_subtract_rel_long atomic_subtract_barr_long
714#define atomic_cmpset_acq_long atomic_cmpset_long
715#define atomic_cmpset_rel_long atomic_cmpset_long
716#define atomic_fcmpset_acq_long atomic_fcmpset_long
717#define atomic_fcmpset_rel_long atomic_fcmpset_long
718
719#define atomic_readandclear_int(p) atomic_swap_int(p, 0)
720#define atomic_readandclear_long(p) atomic_swap_long(p, 0)
721#define atomic_testandset_acq_long atomic_testandset_long
722
723/* Operations on 8-bit bytes. */
724#define atomic_set_8 atomic_set_char
725#define atomic_set_acq_8 atomic_set_acq_char
726#define atomic_set_rel_8 atomic_set_rel_char
727#define atomic_clear_8 atomic_clear_char
728#define atomic_clear_acq_8 atomic_clear_acq_char
729#define atomic_clear_rel_8 atomic_clear_rel_char
730#define atomic_add_8 atomic_add_char
731#define atomic_add_acq_8 atomic_add_acq_char
732#define atomic_add_rel_8 atomic_add_rel_char
733#define atomic_subtract_8 atomic_subtract_char
734#define atomic_subtract_acq_8 atomic_subtract_acq_char
735#define atomic_subtract_rel_8 atomic_subtract_rel_char
736#define atomic_load_acq_8 atomic_load_acq_char
737#define atomic_store_rel_8 atomic_store_rel_char
738#define atomic_cmpset_8 atomic_cmpset_char
739#define atomic_cmpset_acq_8 atomic_cmpset_acq_char
740#define atomic_cmpset_rel_8 atomic_cmpset_rel_char
741#define atomic_fcmpset_8 atomic_fcmpset_char
742#define atomic_fcmpset_acq_8 atomic_fcmpset_acq_char
743#define atomic_fcmpset_rel_8 atomic_fcmpset_rel_char
744
745/* Operations on 16-bit words. */
746#define atomic_set_16 atomic_set_short
747#define atomic_set_acq_16 atomic_set_acq_short
748#define atomic_set_rel_16 atomic_set_rel_short
749#define atomic_clear_16 atomic_clear_short
750#define atomic_clear_acq_16 atomic_clear_acq_short
751#define atomic_clear_rel_16 atomic_clear_rel_short
752#define atomic_add_16 atomic_add_short
753#define atomic_add_acq_16 atomic_add_acq_short
754#define atomic_add_rel_16 atomic_add_rel_short
755#define atomic_subtract_16 atomic_subtract_short
756#define atomic_subtract_acq_16 atomic_subtract_acq_short
757#define atomic_subtract_rel_16 atomic_subtract_rel_short
758#define atomic_load_acq_16 atomic_load_acq_short
759#define atomic_store_rel_16 atomic_store_rel_short
760#define atomic_cmpset_16 atomic_cmpset_short
761#define atomic_cmpset_acq_16 atomic_cmpset_acq_short
762#define atomic_cmpset_rel_16 atomic_cmpset_rel_short
763#define atomic_fcmpset_16 atomic_fcmpset_short
764#define atomic_fcmpset_acq_16 atomic_fcmpset_acq_short
765#define atomic_fcmpset_rel_16 atomic_fcmpset_rel_short
766
767/* Operations on 32-bit double words. */
768#define atomic_set_32 atomic_set_int
769#define atomic_set_acq_32 atomic_set_acq_int
770#define atomic_set_rel_32 atomic_set_rel_int
771#define atomic_clear_32 atomic_clear_int
772#define atomic_clear_acq_32 atomic_clear_acq_int
773#define atomic_clear_rel_32 atomic_clear_rel_int
774#define atomic_add_32 atomic_add_int
775#define atomic_add_acq_32 atomic_add_acq_int
776#define atomic_add_rel_32 atomic_add_rel_int
777#define atomic_subtract_32 atomic_subtract_int
778#define atomic_subtract_acq_32 atomic_subtract_acq_int
779#define atomic_subtract_rel_32 atomic_subtract_rel_int
780#define atomic_load_acq_32 atomic_load_acq_int
781#define atomic_store_rel_32 atomic_store_rel_int
782#define atomic_cmpset_32 atomic_cmpset_int
783#define atomic_cmpset_acq_32 atomic_cmpset_acq_int
784#define atomic_cmpset_rel_32 atomic_cmpset_rel_int
785#define atomic_fcmpset_32 atomic_fcmpset_int
786#define atomic_fcmpset_acq_32 atomic_fcmpset_acq_int
787#define atomic_fcmpset_rel_32 atomic_fcmpset_rel_int
788#define atomic_swap_32 atomic_swap_int
789#define atomic_readandclear_32 atomic_readandclear_int
790#define atomic_fetchadd_32 atomic_fetchadd_int
791#define atomic_testandset_32 atomic_testandset_int
792#define atomic_testandclear_32 atomic_testandclear_int
793
794#ifdef _KERNEL
795/* Operations on 64-bit quad words. */
796#define atomic_cmpset_acq_64 atomic_cmpset_64
797#define atomic_cmpset_rel_64 atomic_cmpset_64
798#define atomic_fcmpset_acq_64 atomic_fcmpset_64
799#define atomic_fcmpset_rel_64 atomic_fcmpset_64
800#define atomic_fetchadd_acq_64 atomic_fetchadd_64
801#define atomic_fetchadd_rel_64 atomic_fetchadd_64
802#define atomic_add_acq_64 atomic_add_64
803#define atomic_add_rel_64 atomic_add_64
804#define atomic_subtract_acq_64 atomic_subtract_64
805#define atomic_subtract_rel_64 atomic_subtract_64
806#define atomic_load_64 atomic_load_acq_64
807#define atomic_store_64 atomic_store_rel_64
808#endif
809
810/* Operations on pointers. */
811#define atomic_set_ptr(p, v) \
812 atomic_set_int((volatile u_int *)(p), (u_int)(v))
813#define atomic_set_acq_ptr(p, v) \
814 atomic_set_acq_int((volatile u_int *)(p), (u_int)(v))
815#define atomic_set_rel_ptr(p, v) \
816 atomic_set_rel_int((volatile u_int *)(p), (u_int)(v))
817#define atomic_clear_ptr(p, v) \
818 atomic_clear_int((volatile u_int *)(p), (u_int)(v))
819#define atomic_clear_acq_ptr(p, v) \
820 atomic_clear_acq_int((volatile u_int *)(p), (u_int)(v))
821#define atomic_clear_rel_ptr(p, v) \
822 atomic_clear_rel_int((volatile u_int *)(p), (u_int)(v))
823#define atomic_add_ptr(p, v) \
824 atomic_add_int((volatile u_int *)(p), (u_int)(v))
825#define atomic_add_acq_ptr(p, v) \
826 atomic_add_acq_int((volatile u_int *)(p), (u_int)(v))
827#define atomic_add_rel_ptr(p, v) \
828 atomic_add_rel_int((volatile u_int *)(p), (u_int)(v))
829#define atomic_subtract_ptr(p, v) \
830 atomic_subtract_int((volatile u_int *)(p), (u_int)(v))
831#define atomic_subtract_acq_ptr(p, v) \
832 atomic_subtract_acq_int((volatile u_int *)(p), (u_int)(v))
833#define atomic_subtract_rel_ptr(p, v) \
834 atomic_subtract_rel_int((volatile u_int *)(p), (u_int)(v))
835#define atomic_load_acq_ptr(p) \
836 atomic_load_acq_int((const volatile u_int *)(p))
837#define atomic_store_rel_ptr(p, v) \
838 atomic_store_rel_int((volatile u_int *)(p), (v))
839#define atomic_cmpset_ptr(dst, old, new) \
840 atomic_cmpset_int((volatile u_int *)(dst), (u_int)(old), (u_int)(new))
841#define atomic_cmpset_acq_ptr(dst, old, new) \
842 atomic_cmpset_acq_int((volatile u_int *)(dst), (u_int)(old), \
843 (u_int)(new))
844#define atomic_cmpset_rel_ptr(dst, old, new) \
845 atomic_cmpset_rel_int((volatile u_int *)(dst), (u_int)(old), \
846 (u_int)(new))
847#define atomic_fcmpset_ptr(dst, old, new) \
848 atomic_fcmpset_int((volatile u_int *)(dst), (u_int *)(old), (u_int)(new))
849#define atomic_fcmpset_acq_ptr(dst, old, new) \
850 atomic_fcmpset_acq_int((volatile u_int *)(dst), (u_int *)(old), \
851 (u_int)(new))
852#define atomic_fcmpset_rel_ptr(dst, old, new) \
853 atomic_fcmpset_rel_int((volatile u_int *)(dst), (u_int *)(old), \
854 (u_int)(new))
855#define atomic_swap_ptr(p, v) \
856 atomic_swap_int((volatile u_int *)(p), (u_int)(v))
857#define atomic_readandclear_ptr(p) \
858 atomic_readandclear_int((volatile u_int *)(p))
859#define atomic_testandclear_ptr(p, val) \
860 atomic_testandclear_int((volatile u_int *)(p), (val))
861#define atomic_testandset_ptr(p, val) \
862 atomic_testandset_int((volatile u_int *)(p), (val))
863
864#if defined(_KERNEL)
865#define mb() __mbk()
866#define wmb() __mbk()
867#define rmb() __mbk()
868#else
869#define mb() __mbu()
870#define wmb() __mbu()
871#define rmb() __mbu()
872#endif
873
874#endif /* !_MACHINE_ATOMIC_H_ */