1/*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2008 Marcel Moolenaar
5 * Copyright (c) 2001 Benno Rice
6 * Copyright (c) 2001 David E. O'Brien
7 * Copyright (c) 1998 Doug Rabson
8 * All rights reserved.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * SUCH DAMAGE.
30 */
31
32#ifndef _MACHINE_ATOMIC_H_
33#define _MACHINE_ATOMIC_H_
34
35#include <sys/atomic_common.h>
36
37#ifndef __powerpc64__
38#include <sys/_atomic64e.h>
39#endif
40
41/*
42 * The __ATOMIC_REL/ACQ() macros provide memory barriers only in conjunction
43 * with the atomic lXarx/stXcx. sequences below. They are not exposed outside
44 * of this file. See also Appendix B.2 of Book II of the architecture manual.
45 *
46 * Note that not all Book-E processors accept the light-weight sync variant.
47 * In particular, early models of E500 cores are known to wedge. Bank on all
48 * 64-bit capable CPUs to accept lwsync properly and pressimize 32-bit CPUs
49 * to use the heavier-weight sync.
50 */
51
52#ifdef __powerpc64__
53#define mb() __asm __volatile("sync" : : : "memory")
54#define rmb() __asm __volatile("lwsync" : : : "memory")
55#define wmb() __asm __volatile("lwsync" : : : "memory")
56#define __ATOMIC_REL() __asm __volatile("lwsync" : : : "memory")
57#define __ATOMIC_ACQ() __asm __volatile("isync" : : : "memory")
58#else
59#define mb() __asm __volatile("sync" : : : "memory")
60#define rmb() __asm __volatile("sync" : : : "memory")
61#define wmb() __asm __volatile("sync" : : : "memory")
62#define __ATOMIC_REL() __asm __volatile("sync" : : : "memory")
63#define __ATOMIC_ACQ() __asm __volatile("isync" : : : "memory")
64#endif
65
66static __inline void
67powerpc_lwsync(void)
68{
69
70#ifdef __powerpc64__
71 __asm __volatile("lwsync" : : : "memory");
72#else
73 __asm __volatile("sync" : : : "memory");
74#endif
75}
76
77/*
78 * atomic_add(p, v)
79 * { *p += v; }
80 */
81
82#define __atomic_add_int(p, v, t) \
83 __asm __volatile( \
84 "1: lwarx %0, 0, %2\n" \
85 " add %0, %3, %0\n" \
86 " stwcx. %0, 0, %2\n" \
87 " bne- 1b\n" \
88 : "=&r" (t), "=m" (*p) \
89 : "r" (p), "r" (v), "m" (*p) \
90 : "cr0", "memory") \
91 /* __atomic_add_int */
92
93#ifdef __powerpc64__
94#define __atomic_add_long(p, v, t) \
95 __asm __volatile( \
96 "1: ldarx %0, 0, %2\n" \
97 " add %0, %3, %0\n" \
98 " stdcx. %0, 0, %2\n" \
99 " bne- 1b\n" \
100 : "=&r" (t), "=m" (*p) \
101 : "r" (p), "r" (v), "m" (*p) \
102 : "cr0", "memory") \
103 /* __atomic_add_long */
104#else
105#define __atomic_add_long(p, v, t) \
106 __asm __volatile( \
107 "1: lwarx %0, 0, %2\n" \
108 " add %0, %3, %0\n" \
109 " stwcx. %0, 0, %2\n" \
110 " bne- 1b\n" \
111 : "=&r" (t), "=m" (*p) \
112 : "r" (p), "r" (v), "m" (*p) \
113 : "cr0", "memory") \
114 /* __atomic_add_long */
115#endif
116
117#define _ATOMIC_ADD(type) \
118 static __inline void \
119 atomic_add_##type(volatile u_##type *p, u_##type v) { \
120 u_##type t; \
121 __atomic_add_##type(p, v, t); \
122 } \
123 \
124 static __inline void \
125 atomic_add_acq_##type(volatile u_##type *p, u_##type v) { \
126 u_##type t; \
127 __atomic_add_##type(p, v, t); \
128 __ATOMIC_ACQ(); \
129 } \
130 \
131 static __inline void \
132 atomic_add_rel_##type(volatile u_##type *p, u_##type v) { \
133 u_##type t; \
134 __ATOMIC_REL(); \
135 __atomic_add_##type(p, v, t); \
136 } \
137 /* _ATOMIC_ADD */
138
139_ATOMIC_ADD(int)
140_ATOMIC_ADD(long)
141
142#define atomic_add_32 atomic_add_int
143#define atomic_add_acq_32 atomic_add_acq_int
144#define atomic_add_rel_32 atomic_add_rel_int
145
146#ifdef __powerpc64__
147#define atomic_add_64 atomic_add_long
148#define atomic_add_acq_64 atomic_add_acq_long
149#define atomic_add_rel_64 atomic_add_rel_long
150
151#define atomic_add_ptr atomic_add_long
152#define atomic_add_acq_ptr atomic_add_acq_long
153#define atomic_add_rel_ptr atomic_add_rel_long
154#else
155#define atomic_add_ptr atomic_add_int
156#define atomic_add_acq_ptr atomic_add_acq_int
157#define atomic_add_rel_ptr atomic_add_rel_int
158#endif
159#undef _ATOMIC_ADD
160#undef __atomic_add_long
161#undef __atomic_add_int
162
163/*
164 * atomic_clear(p, v)
165 * { *p &= ~v; }
166 */
167
168#define __atomic_clear_int(p, v, t) \
169 __asm __volatile( \
170 "1: lwarx %0, 0, %2\n" \
171 " andc %0, %0, %3\n" \
172 " stwcx. %0, 0, %2\n" \
173 " bne- 1b\n" \
174 : "=&r" (t), "=m" (*p) \
175 : "r" (p), "r" (v), "m" (*p) \
176 : "cr0", "memory") \
177 /* __atomic_clear_int */
178
179#ifdef __powerpc64__
180#define __atomic_clear_long(p, v, t) \
181 __asm __volatile( \
182 "1: ldarx %0, 0, %2\n" \
183 " andc %0, %0, %3\n" \
184 " stdcx. %0, 0, %2\n" \
185 " bne- 1b\n" \
186 : "=&r" (t), "=m" (*p) \
187 : "r" (p), "r" (v), "m" (*p) \
188 : "cr0", "memory") \
189 /* __atomic_clear_long */
190#else
191#define __atomic_clear_long(p, v, t) \
192 __asm __volatile( \
193 "1: lwarx %0, 0, %2\n" \
194 " andc %0, %0, %3\n" \
195 " stwcx. %0, 0, %2\n" \
196 " bne- 1b\n" \
197 : "=&r" (t), "=m" (*p) \
198 : "r" (p), "r" (v), "m" (*p) \
199 : "cr0", "memory") \
200 /* __atomic_clear_long */
201#endif
202
203#define _ATOMIC_CLEAR(type) \
204 static __inline void \
205 atomic_clear_##type(volatile u_##type *p, u_##type v) { \
206 u_##type t; \
207 __atomic_clear_##type(p, v, t); \
208 } \
209 \
210 static __inline void \
211 atomic_clear_acq_##type(volatile u_##type *p, u_##type v) { \
212 u_##type t; \
213 __atomic_clear_##type(p, v, t); \
214 __ATOMIC_ACQ(); \
215 } \
216 \
217 static __inline void \
218 atomic_clear_rel_##type(volatile u_##type *p, u_##type v) { \
219 u_##type t; \
220 __ATOMIC_REL(); \
221 __atomic_clear_##type(p, v, t); \
222 } \
223 /* _ATOMIC_CLEAR */
224
225_ATOMIC_CLEAR(int)
226_ATOMIC_CLEAR(long)
227
228#define atomic_clear_32 atomic_clear_int
229#define atomic_clear_acq_32 atomic_clear_acq_int
230#define atomic_clear_rel_32 atomic_clear_rel_int
231
232#ifdef __powerpc64__
233#define atomic_clear_64 atomic_clear_long
234#define atomic_clear_acq_64 atomic_clear_acq_long
235#define atomic_clear_rel_64 atomic_clear_rel_long
236
237#define atomic_clear_ptr atomic_clear_long
238#define atomic_clear_acq_ptr atomic_clear_acq_long
239#define atomic_clear_rel_ptr atomic_clear_rel_long
240#else
241#define atomic_clear_ptr atomic_clear_int
242#define atomic_clear_acq_ptr atomic_clear_acq_int
243#define atomic_clear_rel_ptr atomic_clear_rel_int
244#endif
245#undef _ATOMIC_CLEAR
246#undef __atomic_clear_long
247#undef __atomic_clear_int
248
249/*
250 * atomic_cmpset(p, o, n)
251 */
252/* TODO -- see below */
253
254/*
255 * atomic_load_acq(p)
256 */
257/* TODO -- see below */
258
259/*
260 * atomic_readandclear(p)
261 */
262/* TODO -- see below */
263
264/*
265 * atomic_set(p, v)
266 * { *p |= v; }
267 */
268
269#define __atomic_set_int(p, v, t) \
270 __asm __volatile( \
271 "1: lwarx %0, 0, %2\n" \
272 " or %0, %3, %0\n" \
273 " stwcx. %0, 0, %2\n" \
274 " bne- 1b\n" \
275 : "=&r" (t), "=m" (*p) \
276 : "r" (p), "r" (v), "m" (*p) \
277 : "cr0", "memory") \
278 /* __atomic_set_int */
279
280#ifdef __powerpc64__
281#define __atomic_set_long(p, v, t) \
282 __asm __volatile( \
283 "1: ldarx %0, 0, %2\n" \
284 " or %0, %3, %0\n" \
285 " stdcx. %0, 0, %2\n" \
286 " bne- 1b\n" \
287 : "=&r" (t), "=m" (*p) \
288 : "r" (p), "r" (v), "m" (*p) \
289 : "cr0", "memory") \
290 /* __atomic_set_long */
291#else
292#define __atomic_set_long(p, v, t) \
293 __asm __volatile( \
294 "1: lwarx %0, 0, %2\n" \
295 " or %0, %3, %0\n" \
296 " stwcx. %0, 0, %2\n" \
297 " bne- 1b\n" \
298 : "=&r" (t), "=m" (*p) \
299 : "r" (p), "r" (v), "m" (*p) \
300 : "cr0", "memory") \
301 /* __atomic_set_long */
302#endif
303
304#define _ATOMIC_SET(type) \
305 static __inline void \
306 atomic_set_##type(volatile u_##type *p, u_##type v) { \
307 u_##type t; \
308 __atomic_set_##type(p, v, t); \
309 } \
310 \
311 static __inline void \
312 atomic_set_acq_##type(volatile u_##type *p, u_##type v) { \
313 u_##type t; \
314 __atomic_set_##type(p, v, t); \
315 __ATOMIC_ACQ(); \
316 } \
317 \
318 static __inline void \
319 atomic_set_rel_##type(volatile u_##type *p, u_##type v) { \
320 u_##type t; \
321 __ATOMIC_REL(); \
322 __atomic_set_##type(p, v, t); \
323 } \
324 /* _ATOMIC_SET */
325
326_ATOMIC_SET(int)
327_ATOMIC_SET(long)
328
329#define atomic_set_32 atomic_set_int
330#define atomic_set_acq_32 atomic_set_acq_int
331#define atomic_set_rel_32 atomic_set_rel_int
332
333#ifdef __powerpc64__
334#define atomic_set_64 atomic_set_long
335#define atomic_set_acq_64 atomic_set_acq_long
336#define atomic_set_rel_64 atomic_set_rel_long
337
338#define atomic_set_ptr atomic_set_long
339#define atomic_set_acq_ptr atomic_set_acq_long
340#define atomic_set_rel_ptr atomic_set_rel_long
341#else
342#define atomic_set_ptr atomic_set_int
343#define atomic_set_acq_ptr atomic_set_acq_int
344#define atomic_set_rel_ptr atomic_set_rel_int
345#endif
346#undef _ATOMIC_SET
347#undef __atomic_set_long
348#undef __atomic_set_int
349
350/*
351 * atomic_subtract(p, v)
352 * { *p -= v; }
353 */
354
355#define __atomic_subtract_int(p, v, t) \
356 __asm __volatile( \
357 "1: lwarx %0, 0, %2\n" \
358 " subf %0, %3, %0\n" \
359 " stwcx. %0, 0, %2\n" \
360 " bne- 1b\n" \
361 : "=&r" (t), "=m" (*p) \
362 : "r" (p), "r" (v), "m" (*p) \
363 : "cr0", "memory") \
364 /* __atomic_subtract_int */
365
366#ifdef __powerpc64__
367#define __atomic_subtract_long(p, v, t) \
368 __asm __volatile( \
369 "1: ldarx %0, 0, %2\n" \
370 " subf %0, %3, %0\n" \
371 " stdcx. %0, 0, %2\n" \
372 " bne- 1b\n" \
373 : "=&r" (t), "=m" (*p) \
374 : "r" (p), "r" (v), "m" (*p) \
375 : "cr0", "memory") \
376 /* __atomic_subtract_long */
377#else
378#define __atomic_subtract_long(p, v, t) \
379 __asm __volatile( \
380 "1: lwarx %0, 0, %2\n" \
381 " subf %0, %3, %0\n" \
382 " stwcx. %0, 0, %2\n" \
383 " bne- 1b\n" \
384 : "=&r" (t), "=m" (*p) \
385 : "r" (p), "r" (v), "m" (*p) \
386 : "cr0", "memory") \
387 /* __atomic_subtract_long */
388#endif
389
390#define _ATOMIC_SUBTRACT(type) \
391 static __inline void \
392 atomic_subtract_##type(volatile u_##type *p, u_##type v) { \
393 u_##type t; \
394 __atomic_subtract_##type(p, v, t); \
395 } \
396 \
397 static __inline void \
398 atomic_subtract_acq_##type(volatile u_##type *p, u_##type v) { \
399 u_##type t; \
400 __atomic_subtract_##type(p, v, t); \
401 __ATOMIC_ACQ(); \
402 } \
403 \
404 static __inline void \
405 atomic_subtract_rel_##type(volatile u_##type *p, u_##type v) { \
406 u_##type t; \
407 __ATOMIC_REL(); \
408 __atomic_subtract_##type(p, v, t); \
409 } \
410 /* _ATOMIC_SUBTRACT */
411
412_ATOMIC_SUBTRACT(int)
413_ATOMIC_SUBTRACT(long)
414
415#define atomic_subtract_32 atomic_subtract_int
416#define atomic_subtract_acq_32 atomic_subtract_acq_int
417#define atomic_subtract_rel_32 atomic_subtract_rel_int
418
419#ifdef __powerpc64__
420#define atomic_subtract_64 atomic_subtract_long
421#define atomic_subtract_acq_64 atomic_subract_acq_long
422#define atomic_subtract_rel_64 atomic_subtract_rel_long
423
424#define atomic_subtract_ptr atomic_subtract_long
425#define atomic_subtract_acq_ptr atomic_subtract_acq_long
426#define atomic_subtract_rel_ptr atomic_subtract_rel_long
427#else
428#define atomic_subtract_ptr atomic_subtract_int
429#define atomic_subtract_acq_ptr atomic_subtract_acq_int
430#define atomic_subtract_rel_ptr atomic_subtract_rel_int
431#endif
432#undef _ATOMIC_SUBTRACT
433#undef __atomic_subtract_long
434#undef __atomic_subtract_int
435
436/*
437 * atomic_store_rel(p, v)
438 */
439/* TODO -- see below */
440
441/*
442 * Old/original implementations that still need revisiting.
443 */
444
445static __inline u_int
446atomic_readandclear_int(volatile u_int *addr)
447{
448 u_int result,temp;
449
450 __asm __volatile (
451 "\tsync\n" /* drain writes */
452 "1:\tlwarx %0, 0, %3\n\t" /* load old value */
453 "li %1, 0\n\t" /* load new value */
454 "stwcx. %1, 0, %3\n\t" /* attempt to store */
455 "bne- 1b\n\t" /* spin if failed */
456 : "=&r"(result), "=&r"(temp), "=m" (*addr)
457 : "r" (addr), "m" (*addr)
458 : "cr0", "memory");
459
460 return (result);
461}
462
463#ifdef __powerpc64__
464static __inline u_long
465atomic_readandclear_long(volatile u_long *addr)
466{
467 u_long result,temp;
468
469 __asm __volatile (
470 "\tsync\n" /* drain writes */
471 "1:\tldarx %0, 0, %3\n\t" /* load old value */
472 "li %1, 0\n\t" /* load new value */
473 "stdcx. %1, 0, %3\n\t" /* attempt to store */
474 "bne- 1b\n\t" /* spin if failed */
475 : "=&r"(result), "=&r"(temp), "=m" (*addr)
476 : "r" (addr), "m" (*addr)
477 : "cr0", "memory");
478
479 return (result);
480}
481#endif
482
483#define atomic_readandclear_32 atomic_readandclear_int
484
485#ifdef __powerpc64__
486#define atomic_readandclear_64 atomic_readandclear_long
487
488#define atomic_readandclear_ptr atomic_readandclear_long
489#else
490static __inline u_long
491atomic_readandclear_long(volatile u_long *addr)
492{
493
494 return ((u_long)atomic_readandclear_int((volatile u_int *)addr));
495}
496
497#define atomic_readandclear_ptr atomic_readandclear_int
498#endif
499
500/*
501 * We assume that a = b will do atomic loads and stores.
502 */
503#define ATOMIC_STORE_LOAD(TYPE) \
504static __inline u_##TYPE \
505atomic_load_acq_##TYPE(const volatile u_##TYPE *p) \
506{ \
507 u_##TYPE v; \
508 \
509 v = *p; \
510 powerpc_lwsync(); \
511 return (v); \
512} \
513 \
514static __inline void \
515atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \
516{ \
517 \
518 powerpc_lwsync(); \
519 *p = v; \
520}
521
522ATOMIC_STORE_LOAD(int)
523
524#define atomic_load_acq_32 atomic_load_acq_int
525#define atomic_store_rel_32 atomic_store_rel_int
526
527#ifdef __powerpc64__
528ATOMIC_STORE_LOAD(long)
529
530#define atomic_load_acq_64 atomic_load_acq_long
531#define atomic_store_rel_64 atomic_store_rel_long
532
533#define atomic_load_acq_ptr atomic_load_acq_long
534#define atomic_store_rel_ptr atomic_store_rel_long
535#else
536static __inline u_long
537atomic_load_acq_long(const volatile u_long *addr)
538{
539
540 return ((u_long)atomic_load_acq_int((const volatile u_int *)addr));
541}
542
543static __inline void
544atomic_store_rel_long(volatile u_long *addr, u_long val)
545{
546
547 atomic_store_rel_int((volatile u_int *)addr, (u_int)val);
548}
549
550#define atomic_load_acq_ptr atomic_load_acq_int
551#define atomic_store_rel_ptr atomic_store_rel_int
552#endif
553#undef ATOMIC_STORE_LOAD
554
555/*
556 * Atomically compare the value stored at *p with cmpval and if the
557 * two values are equal, update the value of *p with newval. Returns
558 * zero if the compare failed, nonzero otherwise.
559 */
560#ifdef ISA_206_ATOMICS
561static __inline int
562atomic_cmpset_char(volatile u_char *p, u_char cmpval, u_char newval)
563{
564 int ret;
565
566 __asm __volatile (
567 "1:\tlbarx %0, 0, %2\n\t" /* load old value */
568 "cmplw %3, %0\n\t" /* compare */
569 "bne- 2f\n\t" /* exit if not equal */
570 "stbcx. %4, 0, %2\n\t" /* attempt to store */
571 "bne- 1b\n\t" /* spin if failed */
572 "li %0, 1\n\t" /* success - retval = 1 */
573 "b 3f\n\t" /* we've succeeded */
574 "2:\n\t"
575 "stbcx. %0, 0, %2\n\t" /* clear reservation (74xx) */
576 "li %0, 0\n\t" /* failure - retval = 0 */
577 "3:\n\t"
578 : "=&r" (ret), "=m" (*p)
579 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p)
580 : "cr0", "memory");
581
582 return (ret);
583}
584
585static __inline int
586atomic_cmpset_short(volatile u_short *p, u_short cmpval, u_short newval)
587{
588 int ret;
589
590 __asm __volatile (
591 "1:\tlharx %0, 0, %2\n\t" /* load old value */
592 "cmplw %3, %0\n\t" /* compare */
593 "bne- 2f\n\t" /* exit if not equal */
594 "sthcx. %4, 0, %2\n\t" /* attempt to store */
595 "bne- 1b\n\t" /* spin if failed */
596 "li %0, 1\n\t" /* success - retval = 1 */
597 "b 3f\n\t" /* we've succeeded */
598 "2:\n\t"
599 "sthcx. %0, 0, %2\n\t" /* clear reservation (74xx) */
600 "li %0, 0\n\t" /* failure - retval = 0 */
601 "3:\n\t"
602 : "=&r" (ret), "=m" (*p)
603 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p)
604 : "cr0", "memory");
605
606 return (ret);
607}
608#else
609static __inline int
610atomic_cmpset_masked(uint32_t *p, uint32_t cmpval, uint32_t newval,
611 uint32_t mask)
612{
613 int ret;
614 uint32_t tmp;
615
616 __asm __volatile (
617 "1:\tlwarx %2, 0, %3\n\t" /* load old value */
618 "and %0, %2, %7\n\t"
619 "cmplw %4, %0\n\t" /* compare */
620 "bne- 2f\n\t" /* exit if not equal */
621 "andc %2, %2, %7\n\t"
622 "or %2, %2, %5\n\t"
623 "stwcx. %2, 0, %3\n\t" /* attempt to store */
624 "bne- 1b\n\t" /* spin if failed */
625 "li %0, 1\n\t" /* success - retval = 1 */
626 "b 3f\n\t" /* we've succeeded */
627 "2:\n\t"
628 "stwcx. %2, 0, %3\n\t" /* clear reservation (74xx) */
629 "li %0, 0\n\t" /* failure - retval = 0 */
630 "3:\n\t"
631 : "=&r" (ret), "=m" (*p), "+&r" (tmp)
632 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p),
633 "r" (mask)
634 : "cr0", "memory");
635
636 return (ret);
637}
638
639#define _atomic_cmpset_masked_word(a,o,v,m) atomic_cmpset_masked(a, o, v, m)
640#endif
641
642static __inline int
643atomic_cmpset_int(volatile u_int* p, u_int cmpval, u_int newval)
644{
645 int ret;
646
647 __asm __volatile (
648 "1:\tlwarx %0, 0, %2\n\t" /* load old value */
649 "cmplw %3, %0\n\t" /* compare */
650 "bne- 2f\n\t" /* exit if not equal */
651 "stwcx. %4, 0, %2\n\t" /* attempt to store */
652 "bne- 1b\n\t" /* spin if failed */
653 "li %0, 1\n\t" /* success - retval = 1 */
654 "b 3f\n\t" /* we've succeeded */
655 "2:\n\t"
656 "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */
657 "li %0, 0\n\t" /* failure - retval = 0 */
658 "3:\n\t"
659 : "=&r" (ret), "=m" (*p)
660 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p)
661 : "cr0", "memory");
662
663 return (ret);
664}
665static __inline int
666atomic_cmpset_long(volatile u_long* p, u_long cmpval, u_long newval)
667{
668 int ret;
669
670 __asm __volatile (
671 #ifdef __powerpc64__
672 "1:\tldarx %0, 0, %2\n\t" /* load old value */
673 "cmpld %3, %0\n\t" /* compare */
674 "bne- 2f\n\t" /* exit if not equal */
675 "stdcx. %4, 0, %2\n\t" /* attempt to store */
676 #else
677 "1:\tlwarx %0, 0, %2\n\t" /* load old value */
678 "cmplw %3, %0\n\t" /* compare */
679 "bne- 2f\n\t" /* exit if not equal */
680 "stwcx. %4, 0, %2\n\t" /* attempt to store */
681 #endif
682 "bne- 1b\n\t" /* spin if failed */
683 "li %0, 1\n\t" /* success - retval = 1 */
684 "b 3f\n\t" /* we've succeeded */
685 "2:\n\t"
686 #ifdef __powerpc64__
687 "stdcx. %0, 0, %2\n\t" /* clear reservation (74xx) */
688 #else
689 "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */
690 #endif
691 "li %0, 0\n\t" /* failure - retval = 0 */
692 "3:\n\t"
693 : "=&r" (ret), "=m" (*p)
694 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p)
695 : "cr0", "memory");
696
697 return (ret);
698}
699
700#define ATOMIC_CMPSET_ACQ_REL(type) \
701 static __inline int \
702 atomic_cmpset_acq_##type(volatile u_##type *p, \
703 u_##type cmpval, u_##type newval)\
704 {\
705 u_##type retval; \
706 retval = atomic_cmpset_##type(p, cmpval, newval);\
707 __ATOMIC_ACQ();\
708 return (retval);\
709 }\
710 static __inline int \
711 atomic_cmpset_rel_##type(volatile u_##type *p, \
712 u_##type cmpval, u_##type newval)\
713 {\
714 __ATOMIC_REL();\
715 return (atomic_cmpset_##type(p, cmpval, newval));\
716 }\
717 struct hack
718
719ATOMIC_CMPSET_ACQ_REL(int);
720ATOMIC_CMPSET_ACQ_REL(long);
721
722#ifdef ISA_206_ATOMICS
723#define atomic_cmpset_8 atomic_cmpset_char
724#endif
725#define atomic_cmpset_acq_8 atomic_cmpset_acq_char
726#define atomic_cmpset_rel_8 atomic_cmpset_rel_char
727
728#ifdef ISA_206_ATOMICS
729#define atomic_cmpset_16 atomic_cmpset_short
730#endif
731#define atomic_cmpset_acq_16 atomic_cmpset_acq_short
732#define atomic_cmpset_rel_16 atomic_cmpset_rel_short
733
734#define atomic_cmpset_32 atomic_cmpset_int
735#define atomic_cmpset_acq_32 atomic_cmpset_acq_int
736#define atomic_cmpset_rel_32 atomic_cmpset_rel_int
737
738#ifdef __powerpc64__
739#define atomic_cmpset_64 atomic_cmpset_long
740#define atomic_cmpset_acq_64 atomic_cmpset_acq_long
741#define atomic_cmpset_rel_64 atomic_cmpset_rel_long
742
743#define atomic_cmpset_ptr atomic_cmpset_long
744#define atomic_cmpset_acq_ptr atomic_cmpset_acq_long
745#define atomic_cmpset_rel_ptr atomic_cmpset_rel_long
746#else
747#define atomic_cmpset_ptr atomic_cmpset_int
748#define atomic_cmpset_acq_ptr atomic_cmpset_acq_int
749#define atomic_cmpset_rel_ptr atomic_cmpset_rel_int
750#endif
751
752/*
753 * Atomically compare the value stored at *p with *cmpval and if the
754 * two values are equal, update the value of *p with newval. Returns
755 * zero if the compare failed and sets *cmpval to the read value from *p,
756 * nonzero otherwise.
757 */
758#ifdef ISA_206_ATOMICS
759static __inline int
760atomic_fcmpset_char(volatile u_char *p, u_char *cmpval, u_char newval)
761{
762 int ret;
763
764 __asm __volatile (
765 "lbarx %0, 0, %3\n\t" /* load old value */
766 "cmplw %4, %0\n\t" /* compare */
767 "bne- 1f\n\t" /* exit if not equal */
768 "stbcx. %5, 0, %3\n\t" /* attempt to store */
769 "bne- 1f\n\t" /* exit if failed */
770 "li %0, 1\n\t" /* success - retval = 1 */
771 "b 2f\n\t" /* we've succeeded */
772 "1:\n\t"
773 "stbcx. %0, 0, %3\n\t" /* clear reservation (74xx) */
774 "stbx %0, 0, %7\n\t"
775 "li %0, 0\n\t" /* failure - retval = 0 */
776 "2:\n\t"
777 : "=&r" (ret), "=m" (*p), "=m" (*cmpval)
778 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval)
779 : "cr0", "memory");
780
781 return (ret);
782}
783
784static __inline int
785atomic_fcmpset_short(volatile u_short *p, u_short *cmpval, u_short newval)
786{
787 int ret;
788
789 __asm __volatile (
790 "lharx %0, 0, %3\n\t" /* load old value */
791 "cmplw %4, %0\n\t" /* compare */
792 "bne- 1f\n\t" /* exit if not equal */
793 "sthcx. %5, 0, %3\n\t" /* attempt to store */
794 "bne- 1f\n\t" /* exit if failed */
795 "li %0, 1\n\t" /* success - retval = 1 */
796 "b 2f\n\t" /* we've succeeded */
797 "1:\n\t"
798 "sthcx. %0, 0, %3\n\t" /* clear reservation (74xx) */
799 "sthx %0, 0, %7\n\t"
800 "li %0, 0\n\t" /* failure - retval = 0 */
801 "2:\n\t"
802 : "=&r" (ret), "=m" (*p), "=m" (*cmpval)
803 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval)
804 : "cr0", "memory");
805
806 return (ret);
807}
808#endif /* ISA_206_ATOMICS */
809
810static __inline int
811atomic_fcmpset_int(volatile u_int *p, u_int *cmpval, u_int newval)
812{
813 int ret;
814
815 __asm __volatile (
816 "lwarx %0, 0, %3\n\t" /* load old value */
817 "cmplw %4, %0\n\t" /* compare */
818 "bne- 1f\n\t" /* exit if not equal */
819 "stwcx. %5, 0, %3\n\t" /* attempt to store */
820 "bne- 1f\n\t" /* exit if failed */
821 "li %0, 1\n\t" /* success - retval = 1 */
822 "b 2f\n\t" /* we've succeeded */
823 "1:\n\t"
824 "stwcx. %0, 0, %3\n\t" /* clear reservation (74xx) */
825 "stwx %0, 0, %7\n\t"
826 "li %0, 0\n\t" /* failure - retval = 0 */
827 "2:\n\t"
828 : "=&r" (ret), "=m" (*p), "=m" (*cmpval)
829 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval)
830 : "cr0", "memory");
831
832 return (ret);
833}
834static __inline int
835atomic_fcmpset_long(volatile u_long *p, u_long *cmpval, u_long newval)
836{
837 int ret;
838
839 __asm __volatile (
840 #ifdef __powerpc64__
841 "ldarx %0, 0, %3\n\t" /* load old value */
842 "cmpld %4, %0\n\t" /* compare */
843 "bne- 1f\n\t" /* exit if not equal */
844 "stdcx. %5, 0, %3\n\t" /* attempt to store */
845 #else
846 "lwarx %0, 0, %3\n\t" /* load old value */
847 "cmplw %4, %0\n\t" /* compare */
848 "bne- 1f\n\t" /* exit if not equal */
849 "stwcx. %5, 0, %3\n\t" /* attempt to store */
850 #endif
851 "bne- 1f\n\t" /* exit if failed */
852 "li %0, 1\n\t" /* success - retval = 1 */
853 "b 2f\n\t" /* we've succeeded */
854 "1:\n\t"
855 #ifdef __powerpc64__
856 "stdcx. %0, 0, %3\n\t" /* clear reservation (74xx) */
857 "stdx %0, 0, %7\n\t"
858 #else
859 "stwcx. %0, 0, %3\n\t" /* clear reservation (74xx) */
860 "stwx %0, 0, %7\n\t"
861 #endif
862 "li %0, 0\n\t" /* failure - retval = 0 */
863 "2:\n\t"
864 : "=&r" (ret), "=m" (*p), "=m" (*cmpval)
865 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval)
866 : "cr0", "memory");
867
868 return (ret);
869}
870
871#define ATOMIC_FCMPSET_ACQ_REL(type) \
872 static __inline int \
873 atomic_fcmpset_acq_##type(volatile u_##type *p, \
874 u_##type *cmpval, u_##type newval)\
875 {\
876 u_##type retval; \
877 retval = atomic_fcmpset_##type(p, cmpval, newval);\
878 __ATOMIC_ACQ();\
879 return (retval);\
880 }\
881 static __inline int \
882 atomic_fcmpset_rel_##type(volatile u_##type *p, \
883 u_##type *cmpval, u_##type newval)\
884 {\
885 __ATOMIC_REL();\
886 return (atomic_fcmpset_##type(p, cmpval, newval));\
887 }\
888 struct hack
889
890ATOMIC_FCMPSET_ACQ_REL(int);
891ATOMIC_FCMPSET_ACQ_REL(long);
892
893#ifdef ISA_206_ATOMICS
894#define atomic_fcmpset_8 atomic_fcmpset_char
895#endif
896#define atomic_fcmpset_acq_8 atomic_fcmpset_acq_char
897#define atomic_fcmpset_rel_8 atomic_fcmpset_rel_char
898
899#ifdef ISA_206_ATOMICS
900#define atomic_fcmpset_16 atomic_fcmpset_short
901#endif
902#define atomic_fcmpset_acq_16 atomic_fcmpset_acq_short
903#define atomic_fcmpset_rel_16 atomic_fcmpset_rel_short
904
905#define atomic_fcmpset_32 atomic_fcmpset_int
906#define atomic_fcmpset_acq_32 atomic_fcmpset_acq_int
907#define atomic_fcmpset_rel_32 atomic_fcmpset_rel_int
908
909#ifdef __powerpc64__
910#define atomic_fcmpset_64 atomic_fcmpset_long
911#define atomic_fcmpset_acq_64 atomic_fcmpset_acq_long
912#define atomic_fcmpset_rel_64 atomic_fcmpset_rel_long
913
914#define atomic_fcmpset_ptr atomic_fcmpset_long
915#define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_long
916#define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_long
917#else
918#define atomic_fcmpset_ptr atomic_fcmpset_int
919#define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_int
920#define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_int
921#endif
922
923static __inline u_int
924atomic_fetchadd_int(volatile u_int *p, u_int v)
925{
926 u_int value;
927
928 do {
929 value = *p;
930 } while (!atomic_cmpset_int(p, value, value + v));
931 return (value);
932}
933
934static __inline u_long
935atomic_fetchadd_long(volatile u_long *p, u_long v)
936{
937 u_long value;
938
939 do {
940 value = *p;
941 } while (!atomic_cmpset_long(p, value, value + v));
942 return (value);
943}
944
945static __inline u_int
946atomic_swap_32(volatile u_int *p, u_int v)
947{
948 u_int prev;
949
950 __asm __volatile(
951 "1: lwarx %0,0,%2\n"
952 " stwcx. %3,0,%2\n"
953 " bne- 1b\n"
954 : "=&r" (prev), "+m" (*(volatile u_int *)p)
955 : "r" (p), "r" (v)
956 : "cr0", "memory");
957
958 return (prev);
959}
960
961#ifdef __powerpc64__
962static __inline u_long
963atomic_swap_64(volatile u_long *p, u_long v)
964{
965 u_long prev;
966
967 __asm __volatile(
968 "1: ldarx %0,0,%2\n"
969 " stdcx. %3,0,%2\n"
970 " bne- 1b\n"
971 : "=&r" (prev), "+m" (*(volatile u_long *)p)
972 : "r" (p), "r" (v)
973 : "cr0", "memory");
974
975 return (prev);
976}
977#endif
978
979#define atomic_fetchadd_32 atomic_fetchadd_int
980#define atomic_swap_int atomic_swap_32
981
982#ifdef __powerpc64__
983#define atomic_fetchadd_64 atomic_fetchadd_long
984#define atomic_swap_long atomic_swap_64
985#define atomic_swap_ptr atomic_swap_64
986#else
987#define atomic_swap_long(p,v) atomic_swap_32((volatile u_int *)(p), v)
988#define atomic_swap_ptr(p,v) atomic_swap_32((volatile u_int *)(p), v)
989#endif
990
991static __inline int
992atomic_testandset_int(volatile u_int *p, u_int v)
993{
994 u_int m = (1u << (v & 0x1f));
995 u_int res;
996 u_int tmp;
997
998 __asm __volatile(
999 "1: lwarx %0,0,%3\n"
1000 " and %1,%0,%4\n"
1001 " or %0,%0,%4\n"
1002 " stwcx. %0,0,%3\n"
1003 " bne- 1b\n"
1004 : "=&r"(tmp), "=&r"(res), "+m"(*p)
1005 : "r"(p), "r"(m)
1006 : "cr0", "memory");
1007
1008 return (res != 0);
1009}
1010
1011static __inline int
1012atomic_testandclear_int(volatile u_int *p, u_int v)
1013{
1014 u_int m = (1u << (v & 0x1f));
1015 u_int res;
1016 u_int tmp;
1017
1018 __asm __volatile(
1019 "1: lwarx %0,0,%3\n"
1020 " and %1,%0,%4\n"
1021 " andc %0,%0,%4\n"
1022 " stwcx. %0,0,%3\n"
1023 " bne- 1b\n"
1024 : "=&r"(tmp), "=&r"(res), "+m"(*p)
1025 : "r"(p), "r"(m)
1026 : "cr0", "memory");
1027
1028 return (res != 0);
1029}
1030
1031#ifdef __powerpc64__
1032static __inline int
1033atomic_testandset_long(volatile u_long *p, u_int v)
1034{
1035 u_long m = (1ul << (v & 0x3f));
1036 u_long res;
1037 u_long tmp;
1038
1039 __asm __volatile(
1040 "1: ldarx %0,0,%3\n"
1041 " and %1,%0,%4\n"
1042 " or %0,%0,%4\n"
1043 " stdcx. %0,0,%3\n"
1044 " bne- 1b\n"
1045 : "=&r"(tmp), "=&r"(res), "+m"(*(volatile u_long *)p)
1046 : "r"(p), "r"(m)
1047 : "cr0", "memory");
1048
1049 return (res != 0);
1050}
1051
1052static __inline int
1053atomic_testandclear_long(volatile u_long *p, u_int v)
1054{
1055 u_long m = (1ul << (v & 0x3f));
1056 u_long res;
1057 u_long tmp;
1058
1059 __asm __volatile(
1060 "1: ldarx %0,0,%3\n"
1061 " and %1,%0,%4\n"
1062 " andc %0,%0,%4\n"
1063 " stdcx. %0,0,%3\n"
1064 " bne- 1b\n"
1065 : "=&r"(tmp), "=&r"(res), "+m"(*p)
1066 : "r"(p), "r"(m)
1067 : "cr0", "memory");
1068
1069 return (res != 0);
1070}
1071#else
1072static __inline int
1073atomic_testandset_long(volatile u_long *p, u_int v)
1074{
1075 return (atomic_testandset_int((volatile u_int *)p, v));
1076}
1077
1078static __inline int
1079atomic_testandclear_long(volatile u_long *p, u_int v)
1080{
1081 return (atomic_testandclear_int((volatile u_int *)p, v));
1082}
1083#endif
1084
1085#define atomic_testandclear_32 atomic_testandclear_int
1086#define atomic_testandset_32 atomic_testandset_int
1087
1088static __inline int
1089atomic_testandset_acq_long(volatile u_long *p, u_int v)
1090{
1091 u_int a = atomic_testandset_long(p, v);
1092 __ATOMIC_ACQ();
1093 return (a);
1094}
1095
1096#ifdef __powerpc64__
1097#define atomic_testandclear_ptr atomic_testandclear_long
1098#define atomic_testandset_ptr atomic_testandset_long
1099#else
1100#define atomic_testandclear_ptr(p,v) \
1101 atomic_testandclear_32((volatile u_int *)(p), v)
1102#define atomic_testandset_ptr(p,v) \
1103 atomic_testandset_32((volatile u_int *)(p), v)
1104#endif
1105
1106static __inline void
1107atomic_thread_fence_acq(void)
1108{
1109
1110 powerpc_lwsync();
1111}
1112
1113static __inline void
1114atomic_thread_fence_rel(void)
1115{
1116
1117 powerpc_lwsync();
1118}
1119
1120static __inline void
1121atomic_thread_fence_acq_rel(void)
1122{
1123
1124 powerpc_lwsync();
1125}
1126
1127static __inline void
1128atomic_thread_fence_seq_cst(void)
1129{
1130
1131 __asm __volatile("sync" : : : "memory");
1132}
1133
1134#ifndef ISA_206_ATOMICS
1135#include <sys/_atomic_subword.h>
1136#define atomic_cmpset_char atomic_cmpset_8
1137#define atomic_cmpset_short atomic_cmpset_16
1138#define atomic_fcmpset_char atomic_fcmpset_8
1139#define atomic_fcmpset_short atomic_fcmpset_16
1140#define atomic_set_short atomic_set_16
1141#define atomic_clear_short atomic_clear_16
1142#else
1143
1144static __inline void
1145atomic_set_short(volatile u_short *p, u_short bit)
1146{
1147 u_short v;
1148
1149 v = atomic_load_short(p);
1150 for (;;) {
1151 if (atomic_fcmpset_16(p, &v, v | bit))
1152 break;
1153 }
1154}
1155
1156static __inline void
1157atomic_clear_short(volatile u_short *p, u_short bit)
1158{
1159 u_short v;
1160
1161 v = atomic_load_short(p);
1162 for (;;) {
1163 if (atomic_fcmpset_16(p, &v, v & ~bit))
1164 break;
1165 }
1166}
1167
1168#define atomic_set_16 atomic_set_short
1169#define atomic_clear_16 atomic_clear_short
1170
1171#endif /* ISA_206_ATOMICS */
1172
1173/* These need sys/_atomic_subword.h on non-ISA-2.06-atomic platforms. */
1174ATOMIC_CMPSET_ACQ_REL(char);
1175ATOMIC_CMPSET_ACQ_REL(short);
1176
1177ATOMIC_FCMPSET_ACQ_REL(char);
1178ATOMIC_FCMPSET_ACQ_REL(short);
1179
1180#undef __ATOMIC_REL
1181#undef __ATOMIC_ACQ
1182
1183#endif /* ! _MACHINE_ATOMIC_H_ */