1/* $OpenBSD: percpu.h,v 1.9 2023/09/16 09:33:27 mpi Exp $ */
2
3/*
4 * Copyright (c) 2016 David Gwynne <dlg@openbsd.org>
5 *
6 * Permission to use, copy, modify, and distribute this software for any
7 * purpose with or without fee is hereby granted, provided that the above
8 * copyright notice and this permission notice appear in all copies.
9 *
10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 */
18
19#ifndef _SYS_PERCPU_H_
20#define _SYS_PERCPU_H_
21
22#ifndef CACHELINESIZE
23#define CACHELINESIZE 64
24#endif
25
26#ifndef __upunused /* this should go in param.h */
27#ifdef MULTIPROCESSOR
28#define __upunused
29#else
30#define __upunused __attribute__((__unused__))
31#endif
32#endif
33
34struct cpumem {
35 void *mem;
36};
37
38struct cpumem_iter {
39 unsigned int cpu;
40} __upunused;
41
42struct counters_ref {
43 uint64_t g;
44 uint64_t *c;
45};
46
47#ifdef _KERNEL
48
49#include <sys/atomic.h>
50
51struct pool;
52
53struct cpumem *cpumem_get(struct pool *);
54void cpumem_put(struct pool *, struct cpumem *);
55
56struct cpumem *cpumem_malloc(size_t, int);
57struct cpumem *cpumem_malloc_ncpus(struct cpumem *, size_t, int);
58void cpumem_free(struct cpumem *, int, size_t);
59
60void *cpumem_first(struct cpumem_iter *, struct cpumem *);
61void *cpumem_next(struct cpumem_iter *, struct cpumem *);
62
63static inline void *
64cpumem_enter(struct cpumem *cm)
65{
66#ifdef MULTIPROCESSOR
67 return (cm[cpu_number()].mem);
68#else
69 return (cm);
70#endif
71}
72
73static inline void
74cpumem_leave(struct cpumem *cm, void *mem)
75{
76 /* KDASSERT? */
77}
78
79#ifdef MULTIPROCESSOR
80
81#define CPUMEM_BOOT_MEMORY(_name, _sz) \
82static struct { \
83 unsigned char mem[_sz]; \
84 struct cpumem cpumem; \
85} __aligned(CACHELINESIZE) _name##_boot_cpumem = { \
86 .cpumem = { _name##_boot_cpumem.mem } \
87}
88
89#define CPUMEM_BOOT_INITIALIZER(_name) \
90 { &_name##_boot_cpumem.cpumem }
91
92#else /* MULTIPROCESSOR */
93
94#define CPUMEM_BOOT_MEMORY(_name, _sz) \
95static struct { \
96 unsigned char mem[_sz]; \
97} __aligned(sizeof(uint64_t)) _name##_boot_cpumem
98
99#define CPUMEM_BOOT_INITIALIZER(_name) \
100 { (struct cpumem *)&_name##_boot_cpumem.mem }
101
102#endif /* MULTIPROCESSOR */
103
104#define CPUMEM_FOREACH(_var, _iter, _cpumem) \
105 for ((_var) = cpumem_first((_iter), (_cpumem)); \
106 (_var) != NULL; \
107 (_var) = cpumem_next((_iter), (_cpumem)))
108
109/*
110 * per cpu counters
111 */
112
113struct cpumem *counters_alloc(unsigned int);
114struct cpumem *counters_alloc_ncpus(struct cpumem *, unsigned int);
115void counters_free(struct cpumem *, unsigned int);
116void counters_read(struct cpumem *, uint64_t *, unsigned int,
117 uint64_t *);
118void counters_zero(struct cpumem *, unsigned int);
119
120static inline uint64_t *
121counters_enter(struct counters_ref *ref, struct cpumem *cm)
122{
123 ref->c = cpumem_enter(cm);
124#ifdef MULTIPROCESSOR
125 ref->g = ++(*ref->c); /* make the generation number odd */
126 membar_producer();
127 return (ref->c + 1);
128#else
129 return (ref->c);
130#endif
131}
132
133static inline void
134counters_leave(struct counters_ref *ref, struct cpumem *cm)
135{
136#ifdef MULTIPROCESSOR
137 membar_producer();
138 (*ref->c) = ++ref->g; /* make the generation number even again */
139#endif
140 cpumem_leave(cm, ref->c);
141}
142
143static inline void
144counters_inc(struct cpumem *cm, unsigned int c)
145{
146 struct counters_ref ref;
147 uint64_t *counters;
148
149 counters = counters_enter(&ref, cm);
150 counters[c]++;
151 counters_leave(&ref, cm);
152}
153
154static inline void
155counters_dec(struct cpumem *cm, unsigned int c)
156{
157 struct counters_ref ref;
158 uint64_t *counters;
159
160 counters = counters_enter(&ref, cm);
161 counters[c]--;
162 counters_leave(&ref, cm);
163}
164
165static inline void
166counters_add(struct cpumem *cm, unsigned int c, uint64_t v)
167{
168 struct counters_ref ref;
169 uint64_t *counters;
170
171 counters = counters_enter(&ref, cm);
172 counters[c] += v;
173 counters_leave(&ref, cm);
174}
175
176static inline void
177counters_pkt(struct cpumem *cm, unsigned int c, unsigned int b, uint64_t v)
178{
179 struct counters_ref ref;
180 uint64_t *counters;
181
182 counters = counters_enter(&ref, cm);
183 counters[c]++;
184 counters[b] += v;
185 counters_leave(&ref, cm);
186}
187
188#ifdef MULTIPROCESSOR
189#define COUNTERS_BOOT_MEMORY(_name, _n) \
190 CPUMEM_BOOT_MEMORY(_name, ((_n) + 1) * sizeof(uint64_t))
191#else
192#define COUNTERS_BOOT_MEMORY(_name, _n) \
193 CPUMEM_BOOT_MEMORY(_name, (_n) * sizeof(uint64_t))
194#endif
195
196#define COUNTERS_BOOT_INITIALIZER(_name) CPUMEM_BOOT_INITIALIZER(_name)
197
198#endif /* _KERNEL */
199#endif /* _SYS_PERCPU_H_ */