1/*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2003-2007, Joseph Koshy
5 * Copyright (c) 2007 The FreeBSD Foundation
6 * All rights reserved.
7 *
8 * Portions of this software were developed by A. Joseph Koshy under
9 * sponsorship from the FreeBSD Foundation and Google, Inc.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 * SUCH DAMAGE.
31 */
32
33/*
34 * PMC interface used by the base kernel.
35 */
36
37#ifndef _SYS_PMCKERN_H_
38#define _SYS_PMCKERN_H_
39
40#include <sys/param.h>
41#include <sys/systm.h>
42#include <sys/kernel.h>
43#include <sys/lock.h>
44#include <sys/proc.h>
45#include <sys/sx.h>
46#include <sys/pmc.h>
47
48#include <machine/cpufunc.h>
49
50#define PMC_FN_PROCESS_EXEC 1
51#define PMC_FN_CSW_IN 2
52#define PMC_FN_CSW_OUT 3
53#define PMC_FN_DO_SAMPLES 4
54#define PMC_FN_UNUSED1 5
55#define PMC_FN_UNUSED2 6
56#define PMC_FN_MMAP 7
57#define PMC_FN_MUNMAP 8
58#define PMC_FN_USER_CALLCHAIN 9
59#define PMC_FN_USER_CALLCHAIN_SOFT 10
60#define PMC_FN_SOFT_SAMPLING 11
61#define PMC_FN_THR_CREATE 12
62#define PMC_FN_THR_EXIT 13
63#define PMC_FN_THR_USERRET 14
64#define PMC_FN_THR_CREATE_LOG 15
65#define PMC_FN_THR_EXIT_LOG 16
66#define PMC_FN_PROC_CREATE_LOG 17
67
68typedef enum ring_type {
69 PMC_HR = 0, /* Hardware ring buffer */
70 PMC_SR = 1, /* Software ring buffer */
71 PMC_UR = 2, /* userret ring buffer */
72 PMC_NUM_SR = PMC_UR+1
73} ring_type_t;
74
75struct pmckern_procexec {
76 int pm_credentialschanged;
77 uintptr_t pm_baseaddr;
78 uintptr_t pm_dynaddr;
79};
80
81struct pmckern_map_in {
82 void *pm_file; /* filename or vnode pointer */
83 uintfptr_t pm_address; /* address object is loaded at */
84};
85
86struct pmckern_map_out {
87 uintfptr_t pm_address; /* start address of region */
88 size_t pm_size; /* size of unmapped region */
89};
90
91struct pmckern_soft {
92 enum pmc_event pm_ev;
93 int pm_cpu;
94 struct trapframe *pm_tf;
95};
96
97/*
98 * Soft PMC.
99 */
100
101#define PMC_SOFT_DEFINE_EX(prov, mod, func, name, alloc, release) \
102 struct pmc_soft pmc_##prov##_##mod##_##func##_##name = \
103 { 0, alloc, release, { #prov "_" #mod "_" #func "." #name, 0 } }; \
104 SYSINIT(pmc_##prov##_##mod##_##func##_##name##_init, SI_SUB_KDTRACE, \
105 SI_ORDER_SECOND + 1, pmc_soft_ev_register, \
106 &pmc_##prov##_##mod##_##func##_##name ); \
107 SYSUNINIT(pmc_##prov##_##mod##_##func##_##name##_uninit, \
108 SI_SUB_KDTRACE, SI_ORDER_SECOND + 1, pmc_soft_ev_deregister, \
109 &pmc_##prov##_##mod##_##func##_##name )
110
111#define PMC_SOFT_DEFINE(prov, mod, func, name) \
112 PMC_SOFT_DEFINE_EX(prov, mod, func, name, NULL, NULL)
113
114#define PMC_SOFT_DECLARE(prov, mod, func, name) \
115 extern struct pmc_soft pmc_##prov##_##mod##_##func##_##name
116
117/*
118 * PMC_SOFT_CALL can be used anywhere in the kernel.
119 * Require md defined PMC_FAKE_TRAPFRAME.
120 */
121#ifdef PMC_FAKE_TRAPFRAME
122#define PMC_SOFT_CALL(pr, mo, fu, na) \
123do { \
124 if (__predict_false(pmc_##pr##_##mo##_##fu##_##na.ps_running)) { \
125 struct pmckern_soft ks; \
126 register_t intr; \
127 intr = intr_disable(); \
128 PMC_FAKE_TRAPFRAME(&pmc_tf[curcpu]); \
129 ks.pm_ev = pmc_##pr##_##mo##_##fu##_##na.ps_ev.pm_ev_code; \
130 ks.pm_cpu = PCPU_GET(cpuid); \
131 ks.pm_tf = &pmc_tf[curcpu]; \
132 PMC_CALL_HOOK_UNLOCKED(curthread, \
133 PMC_FN_SOFT_SAMPLING, (void *) &ks); \
134 intr_restore(intr); \
135 } \
136} while (0)
137#else
138#define PMC_SOFT_CALL(pr, mo, fu, na) \
139do { \
140} while (0)
141#endif
142
143/*
144 * PMC_SOFT_CALL_TF need to be used carefully.
145 * Userland capture will be done during AST processing.
146 */
147#define PMC_SOFT_CALL_TF(pr, mo, fu, na, tf) \
148do { \
149 if (__predict_false(pmc_##pr##_##mo##_##fu##_##na.ps_running)) { \
150 struct pmckern_soft ks; \
151 register_t intr; \
152 intr = intr_disable(); \
153 ks.pm_ev = pmc_##pr##_##mo##_##fu##_##na.ps_ev.pm_ev_code; \
154 ks.pm_cpu = PCPU_GET(cpuid); \
155 ks.pm_tf = tf; \
156 PMC_CALL_HOOK_UNLOCKED(curthread, \
157 PMC_FN_SOFT_SAMPLING, (void *) &ks); \
158 intr_restore(intr); \
159 } \
160} while (0)
161
162struct pmc_soft {
163 int ps_running;
164 void (*ps_alloc)(void);
165 void (*ps_release)(void);
166 struct pmc_dyn_event_descr ps_ev;
167};
168
169struct pmclog_buffer;
170
171struct pmc_domain_buffer_header {
172 struct mtx pdbh_mtx;
173 TAILQ_HEAD(, pmclog_buffer) pdbh_head;
174 struct pmclog_buffer *pdbh_plbs;
175 int pdbh_ncpus;
176} __aligned(CACHE_LINE_SIZE);
177
178/* hook */
179extern int (*pmc_hook)(struct thread *_td, int _function, void *_arg);
180extern int (*pmc_intr)(struct trapframe *_frame);
181
182/* SX lock protecting the hook */
183extern struct sx pmc_sx;
184
185/* Per-cpu flags indicating availability of sampling data */
186DPCPU_DECLARE(uint8_t, pmc_sampled);
187
188/* Count of system-wide sampling PMCs in existence */
189extern volatile int pmc_ss_count;
190
191/* kernel version number */
192extern const int pmc_kernel_version;
193
194/* PMC soft per cpu trapframe */
195extern struct trapframe pmc_tf[MAXCPU];
196
197/* per domain buffer header list */
198extern struct pmc_domain_buffer_header *pmc_dom_hdrs[MAXMEMDOM];
199
200/* Quick check if preparatory work is necessary */
201#define PMC_HOOK_INSTALLED(cmd) __predict_false(pmc_hook != NULL)
202
203/* Hook invocation; for use within the kernel */
204#define PMC_CALL_HOOK(t, cmd, arg) \
205do { \
206 struct epoch_tracker et; \
207 epoch_enter_preempt(global_epoch_preempt, &et); \
208 if (pmc_hook != NULL) \
209 (pmc_hook)((t), (cmd), (arg)); \
210 epoch_exit_preempt(global_epoch_preempt, &et); \
211} while (0)
212
213/* Hook invocation that needs an exclusive lock */
214#define PMC_CALL_HOOK_X(t, cmd, arg) \
215do { \
216 sx_xlock(&pmc_sx); \
217 if (pmc_hook != NULL) \
218 (pmc_hook)((t), (cmd), (arg)); \
219 sx_xunlock(&pmc_sx); \
220} while (0)
221
222/*
223 * Some hook invocations (e.g., from context switch and clock handling
224 * code) need to be lock-free.
225 */
226#define PMC_CALL_HOOK_UNLOCKED(t, cmd, arg) \
227do { \
228 if (pmc_hook != NULL) \
229 (pmc_hook)((t), (cmd), (arg)); \
230} while (0)
231
232#define PMC_SWITCH_CONTEXT(t,cmd) PMC_CALL_HOOK_UNLOCKED(t,cmd,NULL)
233
234/* Check if a process is using HWPMCs.*/
235#define PMC_PROC_IS_USING_PMCS(p) \
236 (__predict_false(p->p_flag & P_HWPMC))
237
238#define PMC_THREAD_HAS_SAMPLES(td) \
239 (__predict_false((td)->td_pmcpend))
240
241/* Check if a thread have pending user capture. */
242#define PMC_IS_PENDING_CALLCHAIN(p) \
243 (__predict_false((p)->td_pflags & TDP_CALLCHAIN))
244
245#define PMC_SYSTEM_SAMPLING_ACTIVE() (pmc_ss_count > 0)
246
247/* Check if a CPU has recorded samples. */
248#define PMC_CPU_HAS_SAMPLES(C) (__predict_false(DPCPU_ID_GET((C), pmc_sampled)))
249
250/*
251 * Helper functions.
252 */
253int pmc_cpu_is_disabled(int _cpu); /* deprecated */
254int pmc_cpu_is_active(int _cpu);
255int pmc_cpu_is_present(int _cpu);
256int pmc_cpu_is_primary(int _cpu);
257unsigned int pmc_cpu_max(void);
258
259#ifdef INVARIANTS
260int pmc_cpu_max_active(void);
261#endif
262
263/*
264 * Soft events functions.
265 */
266void pmc_soft_ev_register(struct pmc_soft *ps);
267void pmc_soft_ev_deregister(struct pmc_soft *ps);
268struct pmc_soft *pmc_soft_ev_acquire(enum pmc_event ev);
269void pmc_soft_ev_release(struct pmc_soft *ps);
270
271#endif /* _SYS_PMCKERN_H_ */