1/* $NetBSD: atomic.h,v 1.27 2025/04/22 01:34:38 riastradh Exp $ */
2
3/*-
4 * Copyright (c) 2007, 2008 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Jason R. Thorpe.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32#ifndef _SYS_ATOMIC_H_
33#define _SYS_ATOMIC_H_
34
35#include <sys/types.h>
36#if !defined(_KERNEL) && !defined(_STANDALONE)
37#include <stdint.h>
38#endif
39
40#if defined(_KERNEL) && defined(_KERNEL_OPT)
41#include "opt_kasan.h"
42#include "opt_kcsan.h"
43#include "opt_kmsan.h"
44#endif
45
46#if defined(KASAN)
47#define ATOMIC_PROTO_ADD(name, tret, targ1, targ2) \
48 void kasan_atomic_add_##name(volatile targ1 *, targ2); \
49 tret kasan_atomic_add_##name##_nv(volatile targ1 *, targ2)
50#define ATOMIC_PROTO_AND(name, tret, targ1, targ2) \
51 void kasan_atomic_and_##name(volatile targ1 *, targ2); \
52 tret kasan_atomic_and_##name##_nv(volatile targ1 *, targ2)
53#define ATOMIC_PROTO_OR(name, tret, targ1, targ2) \
54 void kasan_atomic_or_##name(volatile targ1 *, targ2); \
55 tret kasan_atomic_or_##name##_nv(volatile targ1 *, targ2)
56#define ATOMIC_PROTO_CAS(name, tret, targ1, targ2) \
57 tret kasan_atomic_cas_##name(volatile targ1 *, targ2, targ2); \
58 tret kasan_atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2)
59#define ATOMIC_PROTO_SWAP(name, tret, targ1, targ2) \
60 tret kasan_atomic_swap_##name(volatile targ1 *, targ2)
61#define ATOMIC_PROTO_DEC(name, tret, targ1) \
62 void kasan_atomic_dec_##name(volatile targ1 *); \
63 tret kasan_atomic_dec_##name##_nv(volatile targ1 *)
64#define ATOMIC_PROTO_INC(name, tret, targ1) \
65 void kasan_atomic_inc_##name(volatile targ1 *); \
66 tret kasan_atomic_inc_##name##_nv(volatile targ1 *)
67#elif defined(KCSAN)
68#define ATOMIC_PROTO_ADD(name, tret, targ1, targ2) \
69 void kcsan_atomic_add_##name(volatile targ1 *, targ2); \
70 tret kcsan_atomic_add_##name##_nv(volatile targ1 *, targ2)
71#define ATOMIC_PROTO_AND(name, tret, targ1, targ2) \
72 void kcsan_atomic_and_##name(volatile targ1 *, targ2); \
73 tret kcsan_atomic_and_##name##_nv(volatile targ1 *, targ2)
74#define ATOMIC_PROTO_OR(name, tret, targ1, targ2) \
75 void kcsan_atomic_or_##name(volatile targ1 *, targ2); \
76 tret kcsan_atomic_or_##name##_nv(volatile targ1 *, targ2)
77#define ATOMIC_PROTO_CAS(name, tret, targ1, targ2) \
78 tret kcsan_atomic_cas_##name(volatile targ1 *, targ2, targ2); \
79 tret kcsan_atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2)
80#define ATOMIC_PROTO_SWAP(name, tret, targ1, targ2) \
81 tret kcsan_atomic_swap_##name(volatile targ1 *, targ2)
82#define ATOMIC_PROTO_DEC(name, tret, targ1) \
83 void kcsan_atomic_dec_##name(volatile targ1 *); \
84 tret kcsan_atomic_dec_##name##_nv(volatile targ1 *)
85#define ATOMIC_PROTO_INC(name, tret, targ1) \
86 void kcsan_atomic_inc_##name(volatile targ1 *); \
87 tret kcsan_atomic_inc_##name##_nv(volatile targ1 *)
88#elif defined(KMSAN)
89#define ATOMIC_PROTO_ADD(name, tret, targ1, targ2) \
90 void kmsan_atomic_add_##name(volatile targ1 *, targ2); \
91 tret kmsan_atomic_add_##name##_nv(volatile targ1 *, targ2)
92#define ATOMIC_PROTO_AND(name, tret, targ1, targ2) \
93 void kmsan_atomic_and_##name(volatile targ1 *, targ2); \
94 tret kmsan_atomic_and_##name##_nv(volatile targ1 *, targ2)
95#define ATOMIC_PROTO_OR(name, tret, targ1, targ2) \
96 void kmsan_atomic_or_##name(volatile targ1 *, targ2); \
97 tret kmsan_atomic_or_##name##_nv(volatile targ1 *, targ2)
98#define ATOMIC_PROTO_CAS(name, tret, targ1, targ2) \
99 tret kmsan_atomic_cas_##name(volatile targ1 *, targ2, targ2); \
100 tret kmsan_atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2)
101#define ATOMIC_PROTO_SWAP(name, tret, targ1, targ2) \
102 tret kmsan_atomic_swap_##name(volatile targ1 *, targ2)
103#define ATOMIC_PROTO_DEC(name, tret, targ1) \
104 void kmsan_atomic_dec_##name(volatile targ1 *); \
105 tret kmsan_atomic_dec_##name##_nv(volatile targ1 *)
106#define ATOMIC_PROTO_INC(name, tret, targ1) \
107 void kmsan_atomic_inc_##name(volatile targ1 *); \
108 tret kmsan_atomic_inc_##name##_nv(volatile targ1 *)
109#else
110#define ATOMIC_PROTO_ADD(name, tret, targ1, targ2) \
111 void atomic_add_##name(volatile targ1 *, targ2); \
112 tret atomic_add_##name##_nv(volatile targ1 *, targ2)
113#define ATOMIC_PROTO_AND(name, tret, targ1, targ2) \
114 void atomic_and_##name(volatile targ1 *, targ2); \
115 tret atomic_and_##name##_nv(volatile targ1 *, targ2)
116#define ATOMIC_PROTO_OR(name, tret, targ1, targ2) \
117 void atomic_or_##name(volatile targ1 *, targ2); \
118 tret atomic_or_##name##_nv(volatile targ1 *, targ2)
119#define ATOMIC_PROTO_CAS(name, tret, targ1, targ2) \
120 tret atomic_cas_##name(volatile targ1 *, targ2, targ2); \
121 tret atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2)
122#define ATOMIC_PROTO_SWAP(name, tret, targ1, targ2) \
123 tret atomic_swap_##name(volatile targ1 *, targ2)
124#define ATOMIC_PROTO_DEC(name, tret, targ1) \
125 void atomic_dec_##name(volatile targ1 *); \
126 tret atomic_dec_##name##_nv(volatile targ1 *)
127#define ATOMIC_PROTO_INC(name, tret, targ1) \
128 void atomic_inc_##name(volatile targ1 *); \
129 tret atomic_inc_##name##_nv(volatile targ1 *)
130#endif
131
132__BEGIN_DECLS
133
134ATOMIC_PROTO_ADD(32, uint32_t, uint32_t, int32_t);
135ATOMIC_PROTO_ADD(64, uint64_t, uint64_t, int64_t);
136ATOMIC_PROTO_ADD(int, unsigned int, unsigned int, int);
137ATOMIC_PROTO_ADD(long, unsigned long, unsigned long, long);
138ATOMIC_PROTO_ADD(ptr, void *, void, ssize_t);
139
140ATOMIC_PROTO_AND(32, uint32_t, uint32_t, uint32_t);
141ATOMIC_PROTO_AND(64, uint64_t, uint64_t, uint64_t);
142ATOMIC_PROTO_AND(uint, unsigned int, unsigned int, unsigned int);
143ATOMIC_PROTO_AND(ulong, unsigned long, unsigned long, unsigned long);
144
145ATOMIC_PROTO_OR(32, uint32_t, uint32_t, uint32_t);
146ATOMIC_PROTO_OR(64, uint64_t, uint64_t, uint64_t);
147ATOMIC_PROTO_OR(uint, unsigned int, unsigned int, unsigned int);
148ATOMIC_PROTO_OR(ulong, unsigned long, unsigned long, unsigned long);
149
150ATOMIC_PROTO_CAS(32, uint32_t, uint32_t, uint32_t);
151ATOMIC_PROTO_CAS(64, uint64_t, uint64_t, uint64_t);
152ATOMIC_PROTO_CAS(uint, unsigned int, unsigned int, unsigned int);
153ATOMIC_PROTO_CAS(ulong, unsigned long, unsigned long, unsigned long);
154ATOMIC_PROTO_CAS(ptr, void *, void, void *);
155
156ATOMIC_PROTO_SWAP(32, uint32_t, uint32_t, uint32_t);
157ATOMIC_PROTO_SWAP(64, uint64_t, uint64_t, uint64_t);
158ATOMIC_PROTO_SWAP(uint, unsigned int, unsigned int, unsigned int);
159ATOMIC_PROTO_SWAP(ulong, unsigned long, unsigned long, unsigned long);
160ATOMIC_PROTO_SWAP(ptr, void *, void, void *);
161
162ATOMIC_PROTO_DEC(32, uint32_t, uint32_t);
163ATOMIC_PROTO_DEC(64, uint64_t, uint64_t);
164ATOMIC_PROTO_DEC(uint, unsigned int, unsigned int);
165ATOMIC_PROTO_DEC(ulong, unsigned long, unsigned long);
166ATOMIC_PROTO_DEC(ptr, void *, void);
167
168ATOMIC_PROTO_INC(32, uint32_t, uint32_t);
169ATOMIC_PROTO_INC(64, uint64_t, uint64_t);
170ATOMIC_PROTO_INC(uint, unsigned int, unsigned int);
171ATOMIC_PROTO_INC(ulong, unsigned long, unsigned long);
172ATOMIC_PROTO_INC(ptr, void *, void);
173
174/*
175 * These operations will be provided for userland, but may not be
176 * implemented efficiently.
177 */
178uint16_t atomic_cas_16(volatile uint16_t *, uint16_t, uint16_t);
179uint8_t atomic_cas_8(volatile uint8_t *, uint8_t, uint8_t);
180
181/*
182 * Memory barrier operations
183 */
184void membar_acquire(void);
185void membar_release(void);
186void membar_producer(void);
187void membar_consumer(void);
188void membar_sync(void);
189
190/*
191 * Deprecated memory barriers
192 */
193void membar_enter(void);
194void membar_exit(void);
195
196#ifdef __HAVE_MEMBAR_DATADEP_CONSUMER
197void membar_datadep_consumer(void);
198#else
199#define membar_datadep_consumer() ((void)0)
200#endif
201
202__END_DECLS
203
204#if defined(KASAN)
205#define atomic_add_32 kasan_atomic_add_32
206#define atomic_add_int kasan_atomic_add_int
207#define atomic_add_long kasan_atomic_add_long
208#define atomic_add_ptr kasan_atomic_add_ptr
209#define atomic_add_64 kasan_atomic_add_64
210#define atomic_add_32_nv kasan_atomic_add_32_nv
211#define atomic_add_int_nv kasan_atomic_add_int_nv
212#define atomic_add_long_nv kasan_atomic_add_long_nv
213#define atomic_add_ptr_nv kasan_atomic_add_ptr_nv
214#define atomic_add_64_nv kasan_atomic_add_64_nv
215#define atomic_and_32 kasan_atomic_and_32
216#define atomic_and_uint kasan_atomic_and_uint
217#define atomic_and_ulong kasan_atomic_and_ulong
218#define atomic_and_64 kasan_atomic_and_64
219#define atomic_and_32_nv kasan_atomic_and_32_nv
220#define atomic_and_uint_nv kasan_atomic_and_uint_nv
221#define atomic_and_ulong_nv kasan_atomic_and_ulong_nv
222#define atomic_and_64_nv kasan_atomic_and_64_nv
223#define atomic_or_32 kasan_atomic_or_32
224#define atomic_or_uint kasan_atomic_or_uint
225#define atomic_or_ulong kasan_atomic_or_ulong
226#define atomic_or_64 kasan_atomic_or_64
227#define atomic_or_32_nv kasan_atomic_or_32_nv
228#define atomic_or_uint_nv kasan_atomic_or_uint_nv
229#define atomic_or_ulong_nv kasan_atomic_or_ulong_nv
230#define atomic_or_64_nv kasan_atomic_or_64_nv
231#define atomic_cas_32 kasan_atomic_cas_32
232#define atomic_cas_uint kasan_atomic_cas_uint
233#define atomic_cas_ulong kasan_atomic_cas_ulong
234#define atomic_cas_ptr kasan_atomic_cas_ptr
235#define atomic_cas_64 kasan_atomic_cas_64
236#define atomic_cas_32_ni kasan_atomic_cas_32_ni
237#define atomic_cas_uint_ni kasan_atomic_cas_uint_ni
238#define atomic_cas_ulong_ni kasan_atomic_cas_ulong_ni
239#define atomic_cas_ptr_ni kasan_atomic_cas_ptr_ni
240#define atomic_cas_64_ni kasan_atomic_cas_64_ni
241#define atomic_swap_32 kasan_atomic_swap_32
242#define atomic_swap_uint kasan_atomic_swap_uint
243#define atomic_swap_ulong kasan_atomic_swap_ulong
244#define atomic_swap_ptr kasan_atomic_swap_ptr
245#define atomic_swap_64 kasan_atomic_swap_64
246#define atomic_dec_32 kasan_atomic_dec_32
247#define atomic_dec_uint kasan_atomic_dec_uint
248#define atomic_dec_ulong kasan_atomic_dec_ulong
249#define atomic_dec_ptr kasan_atomic_dec_ptr
250#define atomic_dec_64 kasan_atomic_dec_64
251#define atomic_dec_32_nv kasan_atomic_dec_32_nv
252#define atomic_dec_uint_nv kasan_atomic_dec_uint_nv
253#define atomic_dec_ulong_nv kasan_atomic_dec_ulong_nv
254#define atomic_dec_ptr_nv kasan_atomic_dec_ptr_nv
255#define atomic_dec_64_nv kasan_atomic_dec_64_nv
256#define atomic_inc_32 kasan_atomic_inc_32
257#define atomic_inc_uint kasan_atomic_inc_uint
258#define atomic_inc_ulong kasan_atomic_inc_ulong
259#define atomic_inc_ptr kasan_atomic_inc_ptr
260#define atomic_inc_64 kasan_atomic_inc_64
261#define atomic_inc_32_nv kasan_atomic_inc_32_nv
262#define atomic_inc_uint_nv kasan_atomic_inc_uint_nv
263#define atomic_inc_ulong_nv kasan_atomic_inc_ulong_nv
264#define atomic_inc_ptr_nv kasan_atomic_inc_ptr_nv
265#define atomic_inc_64_nv kasan_atomic_inc_64_nv
266#elif defined(KCSAN)
267#define atomic_add_32 kcsan_atomic_add_32
268#define atomic_add_int kcsan_atomic_add_int
269#define atomic_add_long kcsan_atomic_add_long
270#define atomic_add_ptr kcsan_atomic_add_ptr
271#define atomic_add_64 kcsan_atomic_add_64
272#define atomic_add_32_nv kcsan_atomic_add_32_nv
273#define atomic_add_int_nv kcsan_atomic_add_int_nv
274#define atomic_add_long_nv kcsan_atomic_add_long_nv
275#define atomic_add_ptr_nv kcsan_atomic_add_ptr_nv
276#define atomic_add_64_nv kcsan_atomic_add_64_nv
277#define atomic_and_32 kcsan_atomic_and_32
278#define atomic_and_uint kcsan_atomic_and_uint
279#define atomic_and_ulong kcsan_atomic_and_ulong
280#define atomic_and_64 kcsan_atomic_and_64
281#define atomic_and_32_nv kcsan_atomic_and_32_nv
282#define atomic_and_uint_nv kcsan_atomic_and_uint_nv
283#define atomic_and_ulong_nv kcsan_atomic_and_ulong_nv
284#define atomic_and_64_nv kcsan_atomic_and_64_nv
285#define atomic_or_32 kcsan_atomic_or_32
286#define atomic_or_uint kcsan_atomic_or_uint
287#define atomic_or_ulong kcsan_atomic_or_ulong
288#define atomic_or_64 kcsan_atomic_or_64
289#define atomic_or_32_nv kcsan_atomic_or_32_nv
290#define atomic_or_uint_nv kcsan_atomic_or_uint_nv
291#define atomic_or_ulong_nv kcsan_atomic_or_ulong_nv
292#define atomic_or_64_nv kcsan_atomic_or_64_nv
293#define atomic_cas_32 kcsan_atomic_cas_32
294#define atomic_cas_uint kcsan_atomic_cas_uint
295#define atomic_cas_ulong kcsan_atomic_cas_ulong
296#define atomic_cas_ptr kcsan_atomic_cas_ptr
297#define atomic_cas_64 kcsan_atomic_cas_64
298#define atomic_cas_32_ni kcsan_atomic_cas_32_ni
299#define atomic_cas_uint_ni kcsan_atomic_cas_uint_ni
300#define atomic_cas_ulong_ni kcsan_atomic_cas_ulong_ni
301#define atomic_cas_ptr_ni kcsan_atomic_cas_ptr_ni
302#define atomic_cas_64_ni kcsan_atomic_cas_64_ni
303#define atomic_swap_32 kcsan_atomic_swap_32
304#define atomic_swap_uint kcsan_atomic_swap_uint
305#define atomic_swap_ulong kcsan_atomic_swap_ulong
306#define atomic_swap_ptr kcsan_atomic_swap_ptr
307#define atomic_swap_64 kcsan_atomic_swap_64
308#define atomic_dec_32 kcsan_atomic_dec_32
309#define atomic_dec_uint kcsan_atomic_dec_uint
310#define atomic_dec_ulong kcsan_atomic_dec_ulong
311#define atomic_dec_ptr kcsan_atomic_dec_ptr
312#define atomic_dec_64 kcsan_atomic_dec_64
313#define atomic_dec_32_nv kcsan_atomic_dec_32_nv
314#define atomic_dec_uint_nv kcsan_atomic_dec_uint_nv
315#define atomic_dec_ulong_nv kcsan_atomic_dec_ulong_nv
316#define atomic_dec_ptr_nv kcsan_atomic_dec_ptr_nv
317#define atomic_dec_64_nv kcsan_atomic_dec_64_nv
318#define atomic_inc_32 kcsan_atomic_inc_32
319#define atomic_inc_uint kcsan_atomic_inc_uint
320#define atomic_inc_ulong kcsan_atomic_inc_ulong
321#define atomic_inc_ptr kcsan_atomic_inc_ptr
322#define atomic_inc_64 kcsan_atomic_inc_64
323#define atomic_inc_32_nv kcsan_atomic_inc_32_nv
324#define atomic_inc_uint_nv kcsan_atomic_inc_uint_nv
325#define atomic_inc_ulong_nv kcsan_atomic_inc_ulong_nv
326#define atomic_inc_ptr_nv kcsan_atomic_inc_ptr_nv
327#define atomic_inc_64_nv kcsan_atomic_inc_64_nv
328#elif defined(KMSAN)
329#define atomic_add_32 kmsan_atomic_add_32
330#define atomic_add_int kmsan_atomic_add_int
331#define atomic_add_long kmsan_atomic_add_long
332#define atomic_add_ptr kmsan_atomic_add_ptr
333#define atomic_add_64 kmsan_atomic_add_64
334#define atomic_add_32_nv kmsan_atomic_add_32_nv
335#define atomic_add_int_nv kmsan_atomic_add_int_nv
336#define atomic_add_long_nv kmsan_atomic_add_long_nv
337#define atomic_add_ptr_nv kmsan_atomic_add_ptr_nv
338#define atomic_add_64_nv kmsan_atomic_add_64_nv
339#define atomic_and_32 kmsan_atomic_and_32
340#define atomic_and_uint kmsan_atomic_and_uint
341#define atomic_and_ulong kmsan_atomic_and_ulong
342#define atomic_and_64 kmsan_atomic_and_64
343#define atomic_and_32_nv kmsan_atomic_and_32_nv
344#define atomic_and_uint_nv kmsan_atomic_and_uint_nv
345#define atomic_and_ulong_nv kmsan_atomic_and_ulong_nv
346#define atomic_and_64_nv kmsan_atomic_and_64_nv
347#define atomic_or_32 kmsan_atomic_or_32
348#define atomic_or_uint kmsan_atomic_or_uint
349#define atomic_or_ulong kmsan_atomic_or_ulong
350#define atomic_or_64 kmsan_atomic_or_64
351#define atomic_or_32_nv kmsan_atomic_or_32_nv
352#define atomic_or_uint_nv kmsan_atomic_or_uint_nv
353#define atomic_or_ulong_nv kmsan_atomic_or_ulong_nv
354#define atomic_or_64_nv kmsan_atomic_or_64_nv
355#define atomic_cas_32 kmsan_atomic_cas_32
356#define atomic_cas_uint kmsan_atomic_cas_uint
357#define atomic_cas_ulong kmsan_atomic_cas_ulong
358#define atomic_cas_ptr kmsan_atomic_cas_ptr
359#define atomic_cas_64 kmsan_atomic_cas_64
360#define atomic_cas_32_ni kmsan_atomic_cas_32_ni
361#define atomic_cas_uint_ni kmsan_atomic_cas_uint_ni
362#define atomic_cas_ulong_ni kmsan_atomic_cas_ulong_ni
363#define atomic_cas_ptr_ni kmsan_atomic_cas_ptr_ni
364#define atomic_cas_64_ni kmsan_atomic_cas_64_ni
365#define atomic_swap_32 kmsan_atomic_swap_32
366#define atomic_swap_uint kmsan_atomic_swap_uint
367#define atomic_swap_ulong kmsan_atomic_swap_ulong
368#define atomic_swap_ptr kmsan_atomic_swap_ptr
369#define atomic_swap_64 kmsan_atomic_swap_64
370#define atomic_dec_32 kmsan_atomic_dec_32
371#define atomic_dec_uint kmsan_atomic_dec_uint
372#define atomic_dec_ulong kmsan_atomic_dec_ulong
373#define atomic_dec_ptr kmsan_atomic_dec_ptr
374#define atomic_dec_64 kmsan_atomic_dec_64
375#define atomic_dec_32_nv kmsan_atomic_dec_32_nv
376#define atomic_dec_uint_nv kmsan_atomic_dec_uint_nv
377#define atomic_dec_ulong_nv kmsan_atomic_dec_ulong_nv
378#define atomic_dec_ptr_nv kmsan_atomic_dec_ptr_nv
379#define atomic_dec_64_nv kmsan_atomic_dec_64_nv
380#define atomic_inc_32 kmsan_atomic_inc_32
381#define atomic_inc_uint kmsan_atomic_inc_uint
382#define atomic_inc_ulong kmsan_atomic_inc_ulong
383#define atomic_inc_ptr kmsan_atomic_inc_ptr
384#define atomic_inc_64 kmsan_atomic_inc_64
385#define atomic_inc_32_nv kmsan_atomic_inc_32_nv
386#define atomic_inc_uint_nv kmsan_atomic_inc_uint_nv
387#define atomic_inc_ulong_nv kmsan_atomic_inc_ulong_nv
388#define atomic_inc_ptr_nv kmsan_atomic_inc_ptr_nv
389#define atomic_inc_64_nv kmsan_atomic_inc_64_nv
390#endif
391
392#ifdef _KERNEL
393
394#if 1 // XXX: __STDC_VERSION__ < 201112L
395
396/* Pre-C11 definitions */
397
398#include <sys/cdefs.h>
399
400#include <lib/libkern/libkern.h>
401
402#ifdef _LP64
403#define __HAVE_ATOMIC64_LOADSTORE 1
404#define __ATOMIC_SIZE_MAX 8
405#else
406#define __ATOMIC_SIZE_MAX 4
407#endif
408
409/*
410 * We assume that access to an aligned pointer to a volatile object of
411 * at most __ATOMIC_SIZE_MAX bytes is guaranteed to be atomic. This is
412 * an assumption that may be wrong, but we hope it won't be wrong
413 * before we just adopt the C11 atomic API.
414 */
415#define __ATOMIC_PTR_CHECK(p) do \
416{ \
417 CTASSERT(sizeof(*(p)) <= __ATOMIC_SIZE_MAX); \
418 KASSERT(((uintptr_t)(p) & (sizeof(*(p)) - 1)) == 0); \
419} while (0)
420
421#ifdef KCSAN
422void kcsan_atomic_load(const volatile void *, void *, int);
423void kcsan_atomic_store(volatile void *, const void *, int);
424#define __BEGIN_ATOMIC_LOAD(p, v) \
425 union { __typeof__(*(p)) __al_val; char __al_buf[1]; } v; \
426 kcsan_atomic_load(p, v.__al_buf, sizeof(v.__al_val))
427#define __END_ATOMIC_LOAD(v) \
428 (v).__al_val
429#define __DO_ATOMIC_STORE(p, v) \
430 kcsan_atomic_store(p, __UNVOLATILE(&v), sizeof(v))
431#else
432#define __BEGIN_ATOMIC_LOAD(p, v) \
433 __typeof__(*(p)) v = *(p)
434#define __END_ATOMIC_LOAD(v) \
435 v
436#ifdef __HAVE_HASHLOCKED_ATOMICS
437#define __DO_ATOMIC_STORE(p, v) \
438 __do_atomic_store(p, __UNVOLATILE(&v), sizeof(v))
439#else /* !__HAVE_HASHLOCKED_ATOMICS */
440#define __DO_ATOMIC_STORE(p, v) \
441 *p = v
442#endif
443#endif
444
445#define atomic_load_relaxed(p) \
446({ \
447 const volatile __typeof__(*(p)) *__al_ptr = (p); \
448 __ATOMIC_PTR_CHECK(__al_ptr); \
449 __BEGIN_ATOMIC_LOAD(__al_ptr, __al_val); \
450 __END_ATOMIC_LOAD(__al_val); \
451})
452
453#define atomic_load_consume(p) \
454({ \
455 const volatile __typeof__(*(p)) *__al_ptr = (p); \
456 __ATOMIC_PTR_CHECK(__al_ptr); \
457 __BEGIN_ATOMIC_LOAD(__al_ptr, __al_val); \
458 membar_datadep_consumer(); \
459 __END_ATOMIC_LOAD(__al_val); \
460})
461
462#define atomic_load_acquire(p) \
463({ \
464 const volatile __typeof__(*(p)) *__al_ptr = (p); \
465 __ATOMIC_PTR_CHECK(__al_ptr); \
466 __BEGIN_ATOMIC_LOAD(__al_ptr, __al_val); \
467 membar_acquire(); \
468 __END_ATOMIC_LOAD(__al_val); \
469})
470
471#define atomic_store_relaxed(p,v) \
472({ \
473 volatile __typeof__(*(p)) *__as_ptr = (p); \
474 __typeof__(*(p)) __as_val = (v); \
475 __ATOMIC_PTR_CHECK(__as_ptr); \
476 __DO_ATOMIC_STORE(__as_ptr, __as_val); \
477})
478
479#define atomic_store_release(p,v) \
480({ \
481 volatile __typeof__(*(p)) *__as_ptr = (p); \
482 __typeof__(*(p)) __as_val = (v); \
483 __ATOMIC_PTR_CHECK(__as_ptr); \
484 membar_release(); \
485 __DO_ATOMIC_STORE(__as_ptr, __as_val); \
486})
487
488#ifdef __HAVE_HASHLOCKED_ATOMICS
489static __inline __always_inline void
490__do_atomic_store(volatile void *p, const void *q, size_t size)
491{
492 volatile uint32_t *p32 = (volatile uint32_t *)((uintptr_t)p & ~3);
493
494 switch (size) {
495 case 1: {
496 uint8_t v;
497 unsigned s = 8 * ((uintptr_t)p & 3);
498 uint32_t o, n, m = ~(0xffU << s);
499
500 memcpy(&v, q, 1);
501 do {
502 o = atomic_load_relaxed(p32);
503 n = (o & m) | ((uint32_t)v << s);
504 } while (atomic_cas_32(p32, o, n) != o);
505 break;
506 }
507 case 2: {
508 uint16_t v;
509 unsigned s = 8 * (((uintptr_t)p & 2) >> 1);
510 uint32_t o, n, m = ~(0xffffU << s);
511
512 memcpy(&v, q, 2);
513 do {
514 o = atomic_load_relaxed(p32);
515 n = (o & m) | ((uint32_t)v << s);
516 } while (atomic_cas_32(p32, o, n) != o);
517 break;
518 }
519 case 4: {
520 uint32_t v;
521
522 memcpy(&v, q, 4);
523 (void)atomic_swap_32(p, v);
524 break;
525 }
526#ifdef __HAVE_ATOMIC64_LOADSTORE
527 case 8: {
528 uint64_t v;
529 memcpy(&v, q, 8);
530 (void)atomic_swap_64(p, v);
531 break;
532 }
533#endif
534 }
535}
536#endif /* __HAVE_HASHLOCKED_ATOMICS */
537
538#else /* __STDC_VERSION__ >= 201112L */
539
540/* C11 definitions, not yet available */
541
542#include <stdatomic.h>
543
544#define atomic_load_relaxed(p) \
545 atomic_load_explicit((p), memory_order_relaxed)
546#if 0 /* memory_order_consume is not there yet */
547#define atomic_load_consume(p) \
548 atomic_load_explicit((p), memory_order_consume)
549#else
550#define atomic_load_consume(p) \
551({ \
552 const __typeof__(*(p)) __al_val = atomic_load_relaxed(p); \
553 membar_datadep_consumer(); \
554 __al_val; \
555})
556#endif
557#define atomic_load_acquire(p) \
558 atomic_load_explicit((p), memory_order_acquire)
559#define atomic_store_relaxed(p, v) \
560 atomic_store_explicit((p), (v), memory_order_relaxed)
561#define atomic_store_release(p, v) \
562 atomic_store_explicit((p), (v), memory_order_release)
563
564#endif /* __STDC_VERSION__ */
565
566#endif /* _KERNEL */
567
568#endif /* ! _SYS_ATOMIC_H_ */