1/* $NetBSD: cpu.h,v 1.53 2024/12/30 19:17:21 jmcneill Exp $ */
2
3/*-
4 * Copyright (c) 2014, 2020 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Matt Thomas of 3am Software Foundry.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32#ifndef _AARCH64_CPU_H_
33#define _AARCH64_CPU_H_
34
35#include <arm/cpu.h>
36
37#ifdef __aarch64__
38
39#ifdef _KERNEL_OPT
40#include "opt_gprof.h"
41#include "opt_multiprocessor.h"
42#include "opt_pmap.h"
43#endif
44
45#include <sys/param.h>
46
47#if defined(_KERNEL) || defined(_KMEMUSER)
48#include <sys/evcnt.h>
49
50#include <aarch64/armreg.h>
51#include <aarch64/frame.h>
52
53struct clockframe {
54 struct trapframe cf_tf;
55};
56
57/* (spsr & 15) == SPSR_M_EL0T(64bit,0) or USER(32bit,0) */
58#define CLKF_USERMODE(cf) ((((cf)->cf_tf.tf_spsr) & 0x0f) == 0)
59#define CLKF_PC(cf) ((cf)->cf_tf.tf_pc)
60#define CLKF_INTR(cf) ((void)(cf), curcpu()->ci_intr_depth > 1)
61
62/*
63 * LWP_PC: Find out the program counter for the given lwp.
64 */
65#define LWP_PC(l) ((l)->l_md.md_utf->tf_pc)
66
67#include <sys/cpu_data.h>
68#include <sys/device_if.h>
69#include <sys/intr.h>
70
71struct aarch64_cpufuncs {
72 void (*cf_set_ttbr0)(uint64_t);
73 void (*cf_icache_sync_range)(vaddr_t, vsize_t);
74};
75
76#define MAX_CACHE_LEVEL 8 /* ARMv8 has maximum 8 level cache */
77
78struct aarch64_cache_unit {
79 u_int cache_type;
80#define CACHE_TYPE_VPIPT 0 /* VMID-aware PIPT */
81#define CACHE_TYPE_VIVT 1 /* ASID-tagged VIVT */
82#define CACHE_TYPE_VIPT 2
83#define CACHE_TYPE_PIPT 3
84 u_int cache_line_size;
85 u_int cache_ways;
86 u_int cache_sets;
87 u_int cache_way_size;
88 u_int cache_size;
89};
90
91struct aarch64_cache_info {
92 u_int cacheable;
93#define CACHE_CACHEABLE_NONE 0
94#define CACHE_CACHEABLE_ICACHE 1 /* instruction cache only */
95#define CACHE_CACHEABLE_DCACHE 2 /* data cache only */
96#define CACHE_CACHEABLE_IDCACHE 3 /* instruction and data caches */
97#define CACHE_CACHEABLE_UNIFIED 4 /* unified cache */
98 struct aarch64_cache_unit icache;
99 struct aarch64_cache_unit dcache;
100};
101
102struct aarch64_low_power_idle {
103 uint32_t min_res; /* minimum residency */
104 uint32_t wakeup_latency; /* worst case */
105 uint32_t save_restore_flags;
106#define LPI_SAVE_RESTORE_CORE __BIT(0)
107#define LPI_SAVE_RESTORE_TRACE __BIT(1)
108#define LPI_SAVE_RESTORE_GICR __BIT(2)
109#define LPI_SAVE_RESTORE_GICD __BIT(3)
110 uint32_t reg_addr;
111#define LPI_REG_ADDR_WFI 0xffffffff
112
113 char *name;
114 struct evcnt events;
115};
116
117struct cpu_info {
118 struct cpu_data ci_data;
119 device_t ci_dev;
120 cpuid_t ci_cpuid;
121
122 /*
123 * the following are in their own cache line, as they are stored to
124 * regularly by remote CPUs; when they were mixed with other fields
125 * we observed frequent cache misses.
126 */
127 int ci_want_resched __aligned(COHERENCY_UNIT);
128 /* XXX pending IPIs? */
129
130 /*
131 * this is stored frequently, and is fetched by remote CPUs.
132 */
133 struct lwp *ci_curlwp __aligned(COHERENCY_UNIT);
134 struct lwp *ci_onproc;
135
136 /*
137 * largely CPU-private.
138 */
139 struct lwp *ci_softlwps[SOFTINT_COUNT] __aligned(COHERENCY_UNIT);
140
141 uint64_t ci_lastintr;
142
143 int ci_mtx_oldspl;
144 int ci_mtx_count;
145
146 int ci_cpl; /* current processor level (spl) */
147 volatile int ci_hwpl; /* current hardware priority */
148 volatile u_int ci_softints;
149 volatile u_int ci_intr_depth;
150 volatile uint32_t ci_blocked_pics;
151 volatile uint32_t ci_pending_pics;
152 volatile uint32_t ci_pending_ipls;
153
154 int ci_kfpu_spl;
155
156#if defined(PMAP_MI)
157 struct pmap_tlb_info *ci_tlb_info;
158 struct pmap *ci_pmap_lastuser;
159 struct pmap *ci_pmap_cur;
160#endif
161
162 /* ASID of current pmap */
163 tlb_asid_t ci_pmap_asid_cur;
164
165 /* event counters */
166 struct evcnt ci_vfp_use;
167 struct evcnt ci_vfp_reuse;
168 struct evcnt ci_vfp_save;
169 struct evcnt ci_vfp_release;
170 struct evcnt ci_uct_trap;
171 struct evcnt ci_intr_preempt;
172 struct evcnt ci_rndrrs_fail;
173
174 /* FDT or similar supplied "cpu capacity" */
175 uint32_t ci_capacity_dmips_mhz;
176
177 /* interrupt controller */
178 u_int ci_gic_redist; /* GICv3 redistributor index */
179 uint64_t ci_gic_sgir; /* GICv3 SGIR target */
180
181 /* ACPI */
182 uint32_t ci_acpiid; /* ACPI Processor Unique ID */
183
184 /* ACPI low power idle */
185 uint32_t ci_nlpi;
186 struct aarch64_low_power_idle *ci_lpi;
187 uint64_t ci_last_idle;
188
189 /* cached system registers */
190 uint64_t ci_sctlr_el1;
191 uint64_t ci_sctlr_el2;
192
193 /* sysctl(9) exposed system registers */
194 struct aarch64_sysctl_cpu_id ci_id;
195#define ci_midr ci_id.ac_midr
196
197 /* cache information and function pointers */
198 struct aarch64_cache_info ci_cacheinfo[MAX_CACHE_LEVEL];
199 struct aarch64_cpufuncs ci_cpufuncs;
200
201#if defined(GPROF) && defined(MULTIPROCESSOR)
202 struct gmonparam *ci_gmon; /* MI per-cpu GPROF */
203#endif
204} __aligned(COHERENCY_UNIT);
205
206#ifdef _KERNEL
207static inline __always_inline struct lwp * __attribute__ ((const))
208aarch64_curlwp(void)
209{
210 struct lwp *l;
211 __asm("mrs %0, tpidr_el1" : "=r"(l));
212 return l;
213}
214
215/* forward declaration; defined in sys/lwp.h. */
216static __inline struct cpu_info *lwp_getcpu(struct lwp *);
217
218#define curcpu() (lwp_getcpu(aarch64_curlwp()))
219#define setsoftast(ci) (cpu_signotify((ci)->ci_onproc))
220#undef curlwp
221#define curlwp (aarch64_curlwp())
222#define curpcb ((struct pcb *)lwp_getpcb(curlwp))
223
224void cpu_signotify(struct lwp *l);
225void cpu_need_proftick(struct lwp *l);
226
227void cpu_hatch(struct cpu_info *);
228
229extern struct cpu_info *cpu_info[];
230extern struct cpu_info cpu_info_store[];
231
232#define CPU_INFO_ITERATOR int
233#if defined(MULTIPROCESSOR) || defined(_MODULE)
234#define cpu_number() (curcpu()->ci_index)
235#define CPU_IS_PRIMARY(ci) ((ci)->ci_index == 0)
236#define CPU_INFO_FOREACH(cii, ci) \
237 cii = 0, ci = cpu_info[0]; \
238 cii < (ncpu ? ncpu : 1) && (ci = cpu_info[cii]) != NULL; \
239 cii++
240#else /* MULTIPROCESSOR */
241#define cpu_number() 0
242#define CPU_IS_PRIMARY(ci) true
243#define CPU_INFO_FOREACH(cii, ci) \
244 cii = 0, __USE(cii), ci = curcpu(); ci != NULL; ci = NULL
245#endif /* MULTIPROCESSOR */
246
247#define LWP0_CPU_INFO (&cpu_info_store[0])
248
249#define __HAVE_CPU_DOSOFTINTS_CI
250
251static inline void
252cpu_dosoftints_ci(struct cpu_info *ci)
253{
254#if defined(__HAVE_FAST_SOFTINTS) && !defined(__HAVE_PIC_FAST_SOFTINTS)
255 void dosoftints(void);
256
257 if (ci->ci_intr_depth == 0 && (ci->ci_softints >> ci->ci_cpl) > 0) {
258 dosoftints();
259 }
260#endif
261}
262
263static inline void
264cpu_dosoftints(void)
265{
266#if defined(__HAVE_FAST_SOFTINTS) && !defined(__HAVE_PIC_FAST_SOFTINTS)
267 KDASSERT(kpreempt_disabled());
268 cpu_dosoftints_ci(curcpu());
269#endif
270}
271
272struct cpufeature_attach_args {
273 struct cpu_info *ci;
274};
275
276#endif /* _KERNEL */
277
278#endif /* _KERNEL || _KMEMUSER */
279
280#endif
281
282#endif /* _AARCH64_CPU_H_ */