1 /* $NetBSD: tprof_x86_intel.c,v 1.8 2023/04/11 10:07:12 msaitoh Exp $ */ 2 3 /* 4 * Copyright (c) 2018 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Maxime Villard. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Copyright (c)2008,2009 YAMAMOTO Takashi, 34 * All rights reserved. 35 * 36 * Redistribution and use in source and binary forms, with or without 37 * modification, are permitted provided that the following conditions 38 * are met: 39 * 1. Redistributions of source code must retain the above copyright 40 * notice, this list of conditions and the following disclaimer. 41 * 2. Redistributions in binary form must reproduce the above copyright 42 * notice, this list of conditions and the following disclaimer in the 43 * documentation and/or other materials provided with the distribution. 44 * 45 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 46 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 47 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 48 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 49 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 50 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 51 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 52 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 53 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 54 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 55 * SUCH DAMAGE. 56 */ 57 58 #include <sys/cdefs.h> 59 __KERNEL_RCSID(0, "$NetBSD: tprof_x86_intel.c,v 1.8 2023/04/11 10:07:12 msaitoh Exp $"); 60 61 #include <sys/param.h> 62 #include <sys/systm.h> 63 #include <sys/kernel.h> 64 #include <sys/module.h> 65 66 #include <sys/cpu.h> 67 #include <sys/percpu.h> 68 #include <sys/xcall.h> 69 70 #include <dev/tprof/tprof.h> 71 72 #include <uvm/uvm.h> /* VM_MIN_KERNEL_ADDRESS */ 73 74 #include <x86/nmi.h> 75 76 #include <machine/cpufunc.h> 77 #include <machine/cputypes.h> /* CPUVENDOR_* */ 78 #include <machine/cpuvar.h> /* cpu_vendor */ 79 #include <machine/i82489reg.h> 80 #include <machine/i82489var.h> 81 82 static u_int counter_bitwidth; 83 84 #define PERFEVTSEL(i) (MSR_EVNTSEL0 + (i)) 85 #define PERFCTR(i) (MSR_PERFCTR0 + (i)) 86 87 #define PERFEVTSEL_EVENT_SELECT __BITS(0, 7) 88 #define PERFEVTSEL_UNIT_MASK __BITS(8, 15) 89 #define PERFEVTSEL_USR __BIT(16) 90 #define PERFEVTSEL_OS __BIT(17) 91 #define PERFEVTSEL_E __BIT(18) 92 #define PERFEVTSEL_PC __BIT(19) 93 #define PERFEVTSEL_INT __BIT(20) 94 #define PERFEVTSEL_EN __BIT(22) 95 #define PERFEVTSEL_INV __BIT(23) 96 #define PERFEVTSEL_COUNTER_MASK __BITS(24, 31) 97 98 static uint32_t intel_lapic_saved[MAXCPUS]; 99 static nmi_handler_t *intel_nmi_handle; 100 101 static uint32_t 102 tprof_intel_ncounters(void) 103 { 104 uint32_t descs[4]; 105 106 if (cpuid_level < 0x0a) 107 return 0; 108 109 x86_cpuid(0x0a, descs); 110 111 return __SHIFTOUT(descs[0], CPUID_PERF_NGPPC); 112 } 113 114 static u_int 115 tprof_intel_counter_bitwidth(u_int counter) 116 { 117 118 return counter_bitwidth; 119 } 120 121 static inline void 122 tprof_intel_counter_write(u_int counter, uint64_t val) 123 { 124 125 wrmsr(PERFCTR(counter), val); 126 } 127 128 static inline uint64_t 129 tprof_intel_counter_read(u_int counter) 130 { 131 132 return rdmsr(PERFCTR(counter)); 133 } 134 135 static void 136 tprof_intel_configure_event(u_int counter, const tprof_param_t *param) 137 { 138 uint64_t evtval; 139 140 evtval = 141 __SHIFTIN(param->p_event, PERFEVTSEL_EVENT_SELECT) | 142 __SHIFTIN(param->p_unit, PERFEVTSEL_UNIT_MASK) | 143 ((param->p_flags & TPROF_PARAM_USER) ? PERFEVTSEL_USR : 0) | 144 ((param->p_flags & TPROF_PARAM_KERN) ? PERFEVTSEL_OS : 0) | 145 PERFEVTSEL_INT; 146 wrmsr(PERFEVTSEL(counter), evtval); 147 148 /* Reset the counter */ 149 tprof_intel_counter_write(counter, param->p_value); 150 } 151 152 static void 153 tprof_intel_start(tprof_countermask_t runmask) 154 { 155 int bit; 156 157 while ((bit = ffs(runmask)) != 0) { 158 bit--; 159 CLR(runmask, __BIT(bit)); 160 wrmsr(PERFEVTSEL(bit), rdmsr(PERFEVTSEL(bit)) | PERFEVTSEL_EN); 161 } 162 } 163 164 static void 165 tprof_intel_stop(tprof_countermask_t stopmask) 166 { 167 int bit; 168 169 while ((bit = ffs(stopmask)) != 0) { 170 bit--; 171 CLR(stopmask, __BIT(bit)); 172 wrmsr(PERFEVTSEL(bit), rdmsr(PERFEVTSEL(bit)) & 173 ~PERFEVTSEL_EN); 174 } 175 } 176 177 static int 178 tprof_intel_nmi(const struct trapframe *tf, void *arg) 179 { 180 tprof_backend_softc_t *sc = arg; 181 tprof_frame_info_t tfi; 182 uint32_t pcint; 183 int bit; 184 185 uint64_t *counters_offset = 186 percpu_getptr_remote(sc->sc_ctr_offset_percpu, curcpu()); 187 tprof_countermask_t mask = sc->sc_ctr_ovf_mask; 188 while ((bit = ffs(mask)) != 0) { 189 bit--; 190 CLR(mask, __BIT(bit)); 191 192 /* If the highest bit is non zero, then it's not for us. */ 193 uint64_t ctr = tprof_intel_counter_read(bit); 194 if ((ctr & __BIT(counter_bitwidth - 1)) != 0) 195 continue; /* not overflowed */ 196 197 if (ISSET(sc->sc_ctr_prof_mask, __BIT(bit))) { 198 /* Account for the counter, and reset */ 199 tprof_intel_counter_write(bit, 200 sc->sc_count[bit].ctr_counter_reset_val); 201 counters_offset[bit] += 202 sc->sc_count[bit].ctr_counter_val + ctr; 203 204 /* Record a sample */ 205 #if defined(__x86_64__) 206 tfi.tfi_pc = tf->tf_rip; 207 #else 208 tfi.tfi_pc = tf->tf_eip; 209 #endif 210 tfi.tfi_counter = bit; 211 tfi.tfi_inkernel = tfi.tfi_pc >= VM_MIN_KERNEL_ADDRESS; 212 tprof_sample(NULL, &tfi); 213 } else { 214 /* Not profiled, but require to consider overflow */ 215 counters_offset[bit] += __BIT(counter_bitwidth); 216 } 217 } 218 219 /* Unmask PMI */ 220 pcint = lapic_readreg(LAPIC_LVT_PCINT); 221 KASSERT((pcint & LAPIC_LVT_MASKED) != 0); 222 lapic_writereg(LAPIC_LVT_PCINT, pcint & ~LAPIC_LVT_MASKED); 223 224 return 1; 225 } 226 227 static uint64_t 228 tprof_intel_counter_estimate_freq(u_int counter) 229 { 230 231 return curcpu()->ci_data.cpu_cc_freq; 232 } 233 234 static uint32_t 235 tprof_intel_ident(void) 236 { 237 uint32_t descs[4]; 238 239 if (cpu_vendor != CPUVENDOR_INTEL) 240 return TPROF_IDENT_NONE; 241 242 if (cpuid_level < 0x0a) 243 return TPROF_IDENT_NONE; 244 245 x86_cpuid(0x0a, descs); 246 if ((descs[0] & CPUID_PERF_VERSION) == 0) 247 return TPROF_IDENT_NONE; 248 249 if ((descs[0] & CPUID_PERF_NGPPC) == 0) 250 return TPROF_IDENT_NONE; 251 252 counter_bitwidth = __SHIFTOUT(descs[0], CPUID_PERF_NBWGPPC); 253 254 return TPROF_IDENT_INTEL_GENERIC; 255 } 256 257 static void 258 tprof_intel_establish_cpu(void *arg1, void *arg2) 259 { 260 struct cpu_info * const ci = curcpu(); 261 262 intel_lapic_saved[cpu_index(ci)] = lapic_readreg(LAPIC_LVT_PCINT); 263 lapic_writereg(LAPIC_LVT_PCINT, LAPIC_DLMODE_NMI); 264 } 265 266 static void 267 tprof_intel_disestablish_cpu(void *arg1, void *arg2) 268 { 269 struct cpu_info * const ci = curcpu(); 270 271 lapic_writereg(LAPIC_LVT_PCINT, intel_lapic_saved[cpu_index(ci)]); 272 } 273 274 static int 275 tprof_intel_establish(tprof_backend_softc_t *sc) 276 { 277 uint64_t xc; 278 279 if (tprof_intel_ident() == TPROF_IDENT_NONE) 280 return ENOTSUP; 281 282 KASSERT(intel_nmi_handle == NULL); 283 intel_nmi_handle = nmi_establish(tprof_intel_nmi, sc); 284 285 xc = xc_broadcast(0, tprof_intel_establish_cpu, sc, NULL); 286 xc_wait(xc); 287 288 return 0; 289 } 290 291 static void 292 tprof_intel_disestablish(tprof_backend_softc_t *sc) 293 { 294 uint64_t xc; 295 296 xc = xc_broadcast(0, tprof_intel_disestablish_cpu, sc, NULL); 297 xc_wait(xc); 298 299 KASSERT(intel_nmi_handle != NULL); 300 nmi_disestablish(intel_nmi_handle); 301 intel_nmi_handle = NULL; 302 } 303 304 const tprof_backend_ops_t tprof_intel_ops = { 305 .tbo_ident = tprof_intel_ident, 306 .tbo_ncounters = tprof_intel_ncounters, 307 .tbo_counter_bitwidth = tprof_intel_counter_bitwidth, 308 .tbo_counter_read = tprof_intel_counter_read, 309 .tbo_counter_estimate_freq = tprof_intel_counter_estimate_freq, 310 .tbo_valid_event = NULL, 311 .tbo_configure_event = tprof_intel_configure_event, 312 .tbo_start = tprof_intel_start, 313 .tbo_stop = tprof_intel_stop, 314 .tbo_establish = tprof_intel_establish, 315 .tbo_disestablish = tprof_intel_disestablish, 316 }; 317
Indexes created Mon Sep 22 10:09:38 GMT 2025