1 /* $NetBSD: tprof_x86_amd.c,v 1.8 2023/04/11 10:07:12 msaitoh Exp $ */ 2 3 /* 4 * Copyright (c) 2018 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Maxime Villard. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Copyright (c)2008,2009 YAMAMOTO Takashi, 34 * All rights reserved. 35 * 36 * Redistribution and use in source and binary forms, with or without 37 * modification, are permitted provided that the following conditions 38 * are met: 39 * 1. Redistributions of source code must retain the above copyright 40 * notice, this list of conditions and the following disclaimer. 41 * 2. Redistributions in binary form must reproduce the above copyright 42 * notice, this list of conditions and the following disclaimer in the 43 * documentation and/or other materials provided with the distribution. 44 * 45 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 46 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 47 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 48 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 49 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 50 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 51 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 52 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 53 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 54 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 55 * SUCH DAMAGE. 56 */ 57 58 #include <sys/cdefs.h> 59 __KERNEL_RCSID(0, "$NetBSD: tprof_x86_amd.c,v 1.8 2023/04/11 10:07:12 msaitoh Exp $"); 60 61 #include <sys/param.h> 62 #include <sys/systm.h> 63 #include <sys/kernel.h> 64 #include <sys/module.h> 65 66 #include <sys/cpu.h> 67 #include <sys/percpu.h> 68 #include <sys/xcall.h> 69 70 #include <dev/tprof/tprof.h> 71 72 #include <uvm/uvm.h> /* VM_MIN_KERNEL_ADDRESS */ 73 74 #include <x86/nmi.h> 75 76 #include <machine/cpufunc.h> 77 #include <machine/cputypes.h> /* CPUVENDOR_* */ 78 #include <machine/cpuvar.h> /* cpu_vendor */ 79 #include <machine/i82489reg.h> 80 #include <machine/i82489var.h> 81 82 #define NCTRS 4 83 #define COUNTER_BITWIDTH 48 84 85 #define PERFEVTSEL(i) (0xc0010000 + (i)) 86 #define PERFCTR(i) (0xc0010004 + (i)) 87 88 #define PESR_EVENT_MASK_LO __BITS(0, 7) 89 #define PESR_UNIT_MASK __BITS(8, 15) 90 #define PESR_USR __BIT(16) 91 #define PESR_OS __BIT(17) 92 #define PESR_E __BIT(18) 93 #define PESR_PC __BIT(19) 94 #define PESR_INT __BIT(20) 95 /* bit 21 reserved */ 96 #define PESR_EN __BIT(22) 97 #define PESR_INV __BIT(23) 98 #define PESR_COUNTER_MASK __BITS(24, 31) 99 #define PESR_EVENT_MASK_HI __BITS(32, 35) 100 /* bit 36-39 reserved */ 101 #define PESR_GO __BIT(40) 102 #define PESR_HO __BIT(41) 103 /* bit 42-63 reserved */ 104 105 /* 106 * Documents: 107 * http://support.amd.com/TechDocs/32559.pdf 108 * http://developer.amd.com/wordpress/media/2012/10/Basic_Performance_Measurements.pdf 109 */ 110 111 static uint32_t amd_lapic_saved[MAXCPUS]; 112 static nmi_handler_t *amd_nmi_handle; 113 114 static uint32_t 115 tprof_amd_ncounters(void) 116 { 117 return NCTRS; 118 } 119 120 static u_int 121 tprof_amd_counter_bitwidth(u_int counter) 122 { 123 return COUNTER_BITWIDTH; 124 } 125 126 static inline void 127 tprof_amd_counter_write(u_int counter, uint64_t val) 128 { 129 wrmsr(PERFCTR(counter), val); 130 } 131 132 static inline uint64_t 133 tprof_amd_counter_read(u_int counter) 134 { 135 return rdmsr(PERFCTR(counter)); 136 } 137 138 static void 139 tprof_amd_configure_event(u_int counter, const tprof_param_t *param) 140 { 141 uint64_t pesr; 142 uint64_t event_lo; 143 uint64_t event_hi; 144 145 event_hi = param->p_event >> 8; 146 event_lo = param->p_event & 0xff; 147 pesr = 148 ((param->p_flags & TPROF_PARAM_USER) ? PESR_USR : 0) | 149 ((param->p_flags & TPROF_PARAM_KERN) ? PESR_OS : 0) | 150 PESR_INT | 151 __SHIFTIN(event_lo, PESR_EVENT_MASK_LO) | 152 __SHIFTIN(event_hi, PESR_EVENT_MASK_HI) | 153 __SHIFTIN(0, PESR_COUNTER_MASK) | 154 __SHIFTIN(param->p_unit, PESR_UNIT_MASK); 155 wrmsr(PERFEVTSEL(counter), pesr); 156 157 /* Reset the counter */ 158 tprof_amd_counter_write(counter, param->p_value); 159 } 160 161 static void 162 tprof_amd_start(tprof_countermask_t runmask) 163 { 164 int bit; 165 166 while ((bit = ffs(runmask)) != 0) { 167 bit--; 168 CLR(runmask, __BIT(bit)); 169 wrmsr(PERFEVTSEL(bit), rdmsr(PERFEVTSEL(bit)) | PESR_EN); 170 } 171 } 172 173 static void 174 tprof_amd_stop(tprof_countermask_t stopmask) 175 { 176 int bit; 177 178 while ((bit = ffs(stopmask)) != 0) { 179 bit--; 180 CLR(stopmask, __BIT(bit)); 181 wrmsr(PERFEVTSEL(bit), rdmsr(PERFEVTSEL(bit)) & ~PESR_EN); 182 } 183 } 184 185 static int 186 tprof_amd_nmi(const struct trapframe *tf, void *arg) 187 { 188 tprof_backend_softc_t *sc = arg; 189 tprof_frame_info_t tfi; 190 int bit; 191 192 uint64_t *counters_offset = 193 percpu_getptr_remote(sc->sc_ctr_offset_percpu, curcpu()); 194 tprof_countermask_t mask = sc->sc_ctr_ovf_mask; 195 while ((bit = ffs(mask)) != 0) { 196 bit--; 197 CLR(mask, __BIT(bit)); 198 199 /* If the highest bit is non zero, then it's not for us. */ 200 uint64_t ctr = tprof_amd_counter_read(bit); 201 if ((ctr & __BIT(COUNTER_BITWIDTH - 1)) != 0) 202 continue; /* not overflowed */ 203 204 if (ISSET(sc->sc_ctr_prof_mask, __BIT(bit))) { 205 /* Account for the counter, and reset */ 206 tprof_amd_counter_write(bit, 207 sc->sc_count[bit].ctr_counter_reset_val); 208 counters_offset[bit] += 209 sc->sc_count[bit].ctr_counter_val + ctr; 210 211 /* Record a sample */ 212 #if defined(__x86_64__) 213 tfi.tfi_pc = tf->tf_rip; 214 #else 215 tfi.tfi_pc = tf->tf_eip; 216 #endif 217 tfi.tfi_counter = bit; 218 tfi.tfi_inkernel = tfi.tfi_pc >= VM_MIN_KERNEL_ADDRESS; 219 tprof_sample(NULL, &tfi); 220 } else { 221 /* Not profiled, but require to consider overflow */ 222 counters_offset[bit] += __BIT(COUNTER_BITWIDTH); 223 } 224 } 225 226 return 1; 227 } 228 229 static uint64_t 230 tprof_amd_counter_estimate_freq(u_int counter) 231 { 232 return curcpu()->ci_data.cpu_cc_freq; 233 } 234 235 static uint32_t 236 tprof_amd_ident(void) 237 { 238 struct cpu_info *ci = curcpu(); 239 240 if (cpu_vendor != CPUVENDOR_AMD) 241 return TPROF_IDENT_NONE; 242 243 switch (CPUID_TO_FAMILY(ci->ci_signature)) { 244 case 0x10: 245 case 0x15: 246 case 0x17: 247 case 0x19: 248 return TPROF_IDENT_AMD_GENERIC; 249 } 250 251 return TPROF_IDENT_NONE; 252 } 253 254 static void 255 tprof_amd_establish_cpu(void *arg1, void *arg2) 256 { 257 struct cpu_info * const ci = curcpu(); 258 259 amd_lapic_saved[cpu_index(ci)] = lapic_readreg(LAPIC_LVT_PCINT); 260 lapic_writereg(LAPIC_LVT_PCINT, LAPIC_DLMODE_NMI); 261 } 262 263 static void 264 tprof_amd_disestablish_cpu(void *arg1, void *arg2) 265 { 266 struct cpu_info * const ci = curcpu(); 267 268 lapic_writereg(LAPIC_LVT_PCINT, amd_lapic_saved[cpu_index(ci)]); 269 } 270 271 static int 272 tprof_amd_establish(tprof_backend_softc_t *sc) 273 { 274 uint64_t xc; 275 276 if (tprof_amd_ident() == TPROF_IDENT_NONE) 277 return ENOTSUP; 278 279 KASSERT(amd_nmi_handle == NULL); 280 amd_nmi_handle = nmi_establish(tprof_amd_nmi, sc); 281 282 xc = xc_broadcast(0, tprof_amd_establish_cpu, sc, NULL); 283 xc_wait(xc); 284 285 return 0; 286 } 287 288 static void 289 tprof_amd_disestablish(tprof_backend_softc_t *sc) 290 { 291 uint64_t xc; 292 293 xc = xc_broadcast(0, tprof_amd_disestablish_cpu, sc, NULL); 294 xc_wait(xc); 295 296 KASSERT(amd_nmi_handle != NULL); 297 nmi_disestablish(amd_nmi_handle); 298 amd_nmi_handle = NULL; 299 } 300 301 const tprof_backend_ops_t tprof_amd_ops = { 302 .tbo_ident = tprof_amd_ident, 303 .tbo_ncounters = tprof_amd_ncounters, 304 .tbo_counter_bitwidth = tprof_amd_counter_bitwidth, 305 .tbo_counter_read = tprof_amd_counter_read, 306 .tbo_counter_estimate_freq = tprof_amd_counter_estimate_freq, 307 .tbo_valid_event = NULL, 308 .tbo_configure_event = tprof_amd_configure_event, 309 .tbo_start = tprof_amd_start, 310 .tbo_stop = tprof_amd_stop, 311 .tbo_establish = tprof_amd_establish, 312 .tbo_disestablish = tprof_amd_disestablish, 313 }; 314
Indexes created Sat Sep 20 22:09:52 GMT 2025