1 /* $NetBSD: tsc.c,v 1.67 2026/09/30 06:29:55 andvar Exp $ */ 2 3 /*- 4 * Copyright (c) 2008, 2020 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 17 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 18 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 * POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __KERNEL_RCSID(0, "$NetBSD: tsc.c,v 1.67 2026/09/30 06:29:55 andvar Exp $"); 31 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <sys/time.h> 35 #include <sys/timetc.h> 36 #include <sys/lwp.h> 37 #include <sys/atomic.h> 38 #include <sys/kernel.h> 39 #include <sys/cpu.h> 40 #include <sys/xcall.h> 41 #include <sys/lock.h> 42 #ifdef BOOT_DURATION 43 #include <sys/boot_duration.h> 44 #endif 45 46 #include <machine/cpu_counter.h> 47 #include <machine/cpuvar.h> 48 #include <machine/cpufunc.h> 49 #include <machine/specialreg.h> 50 #include <machine/cputypes.h> 51 52 #include "tsc.h" 53 54 #define TSC_SYNC_ROUNDS 1000 55 #define ABS(a) ((a) >= 0 ? (a) : -(a)) 56 57 static u_int tsc_get_timecount(struct timecounter *); 58 59 static void tsc_delay(unsigned int); 60 61 static uint64_t tsc_dummy_cacheline __cacheline_aligned; 62 uint64_t tsc_freq __read_mostly; /* exported for sysctl */ 63 #ifdef BOOT_DURATION 64 extern uint32_t starttsc_lo; 65 extern uint32_t starttsc_hi; 66 #endif 67 static int64_t tsc_drift_max = 1000; /* max cycles */ 68 static int64_t tsc_drift_observed; 69 uint64_t (*rdtsc)(void) = rdtsc_cpuid; 70 uint64_t (*cpu_counter)(void) = cpu_counter_cpuid; 71 uint32_t (*cpu_counter32)(void) = cpu_counter32_cpuid; 72 73 int tsc_user_enabled = 1; 74 75 static volatile int64_t tsc_sync_val; 76 static volatile struct cpu_info *tsc_sync_cpu; 77 78 static struct timecounter tsc_timecounter = { 79 .tc_get_timecount = tsc_get_timecount, 80 .tc_counter_mask = ~0U, 81 .tc_name = "TSC", 82 .tc_quality = 3000, 83 }; 84 85 bool 86 tsc_is_invariant(void) 87 { 88 struct cpu_info *ci; 89 uint32_t descs[4]; 90 uint32_t family; 91 bool invariant; 92 93 if (!cpu_hascounter()) 94 return false; 95 96 ci = curcpu(); 97 invariant = false; 98 99 if (cpu_vendor == CPUVENDOR_INTEL) { 100 /* 101 * From Intel(tm) 64 and IA-32 Architectures Software 102 * Developer's Manual Volume 3A: System Programming Guide, 103 * Part 1, 17.13 TIME_STAMP COUNTER, these are the processors 104 * where the TSC is known invariant: 105 * 106 * Pentium 4, Intel Xeon (family 0f, models 03 and higher) 107 * Core Solo and Core Duo processors (family 06, model 0e) 108 * Xeon 5100 series and Core 2 Duo (family 06, model 0f) 109 * Core 2 and Xeon (family 06, model 17) 110 * Atom (family 06, model 1c) 111 * 112 * We'll also assume that it's safe on the Pentium, and 113 * that it's safe on P-II and P-III Xeons due to the 114 * typical configuration of those systems. 115 * 116 */ 117 switch (CPUID_TO_BASEFAMILY(ci->ci_signature)) { 118 case 0x05: 119 invariant = true; 120 break; 121 case 0x06: 122 invariant = CPUID_TO_MODEL(ci->ci_signature) == 0x0e || 123 CPUID_TO_MODEL(ci->ci_signature) == 0x0f || 124 CPUID_TO_MODEL(ci->ci_signature) == 0x17 || 125 CPUID_TO_MODEL(ci->ci_signature) == 0x1c; 126 break; 127 case 0x0f: 128 invariant = CPUID_TO_MODEL(ci->ci_signature) >= 0x03; 129 break; 130 } 131 } else if (cpu_vendor == CPUVENDOR_AMD) { 132 /* 133 * TSC and Power Management Events on AMD Processors 134 * Nov 2, 2005 Rich Brunner, AMD Fellow 135 * http://lkml.org/lkml/2005/11/4/173 136 * 137 * See Appendix E.4.7 CPUID Fn8000_0007_EDX Advanced Power 138 * Management Features, AMD64 Architecture Programmer's 139 * Manual Volume 3: General-Purpose and System Instructions. 140 * The check is done below. 141 */ 142 143 /* 144 * AMD Errata 778: Processor Core Time Stamp Counters May 145 * Experience Drift 146 * 147 * This affects all family 15h and family 16h processors. 148 */ 149 switch (CPUID_TO_FAMILY(ci->ci_signature)) { 150 case 0x15: 151 case 0x16: 152 return false; 153 } 154 } 155 156 /* 157 * The best way to check whether the TSC counter is invariant or not 158 * is to check CPUID 80000007. 159 */ 160 family = CPUID_TO_BASEFAMILY(ci->ci_signature); 161 if (((cpu_vendor == CPUVENDOR_INTEL) || (cpu_vendor == CPUVENDOR_AMD) 162 || (cpu_vendor == CPUVENDOR_IDT)) 163 && ((family == 0x06) || (family == 0x07) || (family == 0x0f))) { 164 x86_cpuid(0x80000000, descs); 165 if (descs[0] >= 0x80000007) { 166 x86_cpuid(0x80000007, descs); 167 invariant = (descs[3] & CPUID_APM_ITSC) != 0; 168 } 169 } 170 171 return invariant; 172 } 173 174 /* Setup function pointers for rdtsc() and timecounter(9). */ 175 void 176 tsc_setfunc(struct cpu_info *ci) 177 { 178 bool use_lfence, use_mfence; 179 uint32_t family; 180 181 use_lfence = use_mfence = false; 182 183 /* 184 * XXX On AMD, we might be able to use lfence for some cases: 185 * a) if MSR_DE_CFG exist and the bit 1 is set. 186 * b) family == 0x0f or 0x11. Those have no MSR_DE_CFG and 187 * lfence is always serializing. 188 * 189 * We don't use it because the test result showed mfence was better 190 * than lfence with MSR_DE_CFG. 191 */ 192 if (cpu_vendor == CPUVENDOR_AMD) 193 use_mfence = true; 194 else if (cpu_vendor == CPUVENDOR_INTEL) 195 use_lfence = true; 196 else if (cpu_vendor == CPUVENDOR_IDT) { 197 family = CPUID_TO_BASEFAMILY(ci->ci_signature); 198 use_lfence = family > 0x06 || (family == 0x06 && 199 CPUID_TO_MODEL(ci->ci_signature) == 0x0f); 200 } 201 202 /* LFENCE and MFENCE are applicable if SSE2 is set. */ 203 if ((ci->ci_feat_val[0] & CPUID_SSE2) == 0) 204 use_lfence = use_mfence = false; 205 206 #define TSC_SETFUNC(fence) \ 207 do { \ 208 rdtsc = rdtsc_##fence; \ 209 cpu_counter = cpu_counter_##fence; \ 210 cpu_counter32 = cpu_counter32_##fence; \ 211 } while (/* CONSTCOND */ 0) 212 213 if (use_lfence) 214 TSC_SETFUNC(lfence); 215 else if (use_mfence) 216 TSC_SETFUNC(mfence); 217 else 218 TSC_SETFUNC(cpuid); 219 220 aprint_verbose_dev(ci->ci_dev, "Use %s to serialize rdtsc\n", 221 use_lfence ? "lfence" : (use_mfence ? "mfence" : "cpuid")); 222 } 223 224 /* 225 * Initialize timecounter(9) and DELAY() function of TSC. 226 * 227 * This function is called after all secondary processors were brought up 228 * and drift has been measured, and after any other potential delay funcs 229 * have been installed (e.g. lapic_delay()). 230 */ 231 void 232 tsc_tc_init(void) 233 { 234 struct cpu_info *ci; 235 bool invariant; 236 237 if (!cpu_hascounter()) 238 return; 239 240 ci = curcpu(); 241 tsc_freq = ci->ci_data.cpu_cc_freq; 242 invariant = tsc_is_invariant(); 243 if (!invariant) { 244 aprint_debug("TSC not known invariant on this CPU\n"); 245 tsc_timecounter.tc_quality = -100; 246 } else if (tsc_drift_observed > tsc_drift_max) { 247 aprint_error("ERROR: %lld cycle TSC drift observed\n", 248 (long long)tsc_drift_observed); 249 tsc_timecounter.tc_quality = -100; 250 invariant = false; 251 } else if (vm_guest == VM_GUEST_NO) { 252 delay_func = tsc_delay; 253 } else if (vm_guest == VM_GUEST_VIRTUALBOX || 254 vm_guest == VM_GUEST_NVMM) { 255 /* 256 * nvmm doesn't advance guest tsc constantly unless 257 * the vcpu is pinned to a physical cpu. 258 */ 259 tsc_timecounter.tc_quality = -100; 260 } 261 262 if (tsc_freq != 0) { 263 tsc_timecounter.tc_frequency = tsc_freq; 264 tc_init(&tsc_timecounter); 265 } 266 } 267 268 /* 269 * Record drift (in clock cycles). Called during AP startup. 270 */ 271 void 272 tsc_sync_drift(int64_t drift) 273 { 274 275 if (drift < 0) 276 drift = -drift; 277 if (drift > tsc_drift_observed) 278 tsc_drift_observed = drift; 279 } 280 281 /* 282 * Called during startup of APs, by the boot processor. Interrupts 283 * are disabled on entry. 284 */ 285 static void __noinline 286 tsc_read_bp(struct cpu_info *ci, uint64_t *bptscp, uint64_t *aptscp) 287 { 288 uint64_t bptsc; 289 290 if (atomic_swap_ptr(&tsc_sync_cpu, ci) != NULL) { 291 panic("tsc_sync_bp: 1"); 292 } 293 294 /* Prepare a cache miss for the other side. */ 295 (void)atomic_swap_uint((void *)&tsc_dummy_cacheline, 0); 296 297 /* Flag our readiness. */ 298 atomic_or_uint(&ci->ci_flags, CPUF_SYNCTSC); 299 300 /* Wait for other side then read our TSC. */ 301 while ((ci->ci_flags & CPUF_SYNCTSC) != 0) { 302 __insn_barrier(); 303 } 304 bptsc = rdtsc(); 305 306 /* Wait for the results to come in. */ 307 while (tsc_sync_cpu == ci) { 308 x86_pause(); 309 } 310 if (tsc_sync_cpu != NULL) { 311 panic("tsc_sync_bp: 2"); 312 } 313 314 *bptscp = bptsc; 315 *aptscp = tsc_sync_val; 316 } 317 318 void 319 tsc_sync_bp(struct cpu_info *ci) 320 { 321 int64_t bptsc, aptsc, val, diff; 322 323 if (!cpu_hascounter()) 324 return; 325 326 val = INT64_MAX; 327 for (int i = 0; i < TSC_SYNC_ROUNDS; i++) { 328 tsc_read_bp(ci, &bptsc, &aptsc); 329 diff = bptsc - aptsc; 330 if (ABS(diff) < ABS(val)) { 331 val = diff; 332 } 333 } 334 335 ci->ci_data.cpu_cc_skew = val; 336 } 337 338 /* 339 * Called during startup of AP, by the AP itself. Interrupts are 340 * disabled on entry. 341 */ 342 static void __noinline 343 tsc_post_ap(struct cpu_info *ci) 344 { 345 uint64_t tsc; 346 347 /* Wait for go-ahead from primary. */ 348 while ((ci->ci_flags & CPUF_SYNCTSC) == 0) { 349 __insn_barrier(); 350 } 351 352 /* Instruct primary to read its counter. */ 353 atomic_and_uint(&ci->ci_flags, ~CPUF_SYNCTSC); 354 355 /* Suffer a cache miss, then read TSC. */ 356 __insn_barrier(); 357 tsc = tsc_dummy_cacheline; 358 __insn_barrier(); 359 tsc += rdtsc(); 360 361 /* Post result. Ensure the whole value goes out atomically. */ 362 (void)atomic_swap_64(&tsc_sync_val, tsc); 363 364 if (atomic_swap_ptr(&tsc_sync_cpu, NULL) != ci) { 365 panic("tsc_sync_ap"); 366 } 367 } 368 369 void 370 tsc_sync_ap(struct cpu_info *ci) 371 { 372 373 if (!cpu_hascounter()) 374 return; 375 376 for (int i = 0; i < TSC_SYNC_ROUNDS; i++) { 377 tsc_post_ap(ci); 378 } 379 } 380 381 static void 382 tsc_apply_cpu(void *arg1, void *arg2) 383 { 384 bool enable = arg1 != NULL; 385 if (enable) { 386 lcr4(rcr4() & ~CR4_TSD); 387 } else { 388 lcr4(rcr4() | CR4_TSD); 389 } 390 } 391 392 void 393 tsc_user_enable(void) 394 { 395 uint64_t xc; 396 397 xc = xc_broadcast(0, tsc_apply_cpu, (void *)true, NULL); 398 xc_wait(xc); 399 } 400 401 void 402 tsc_user_disable(void) 403 { 404 uint64_t xc; 405 406 xc = xc_broadcast(0, tsc_apply_cpu, (void *)false, NULL); 407 xc_wait(xc); 408 } 409 410 uint64_t 411 cpu_frequency(struct cpu_info *ci) 412 { 413 414 return ci->ci_data.cpu_cc_freq; 415 } 416 417 int 418 cpu_hascounter(void) 419 { 420 421 return cpu_feature[0] & CPUID_TSC; 422 } 423 424 static void 425 tsc_delay(unsigned int us) 426 { 427 uint64_t start, delta; 428 429 start = cpu_counter(); 430 delta = (uint64_t)us * tsc_freq / 1000000; 431 432 while ((cpu_counter() - start) < delta) { 433 x86_pause(); 434 } 435 } 436 437 static u_int 438 tsc_get_timecount(struct timecounter *tc) 439 { 440 #if defined(_LP64) && defined(DIAGNOSTIC) /* requires atomic 64-bit store */ 441 static __cpu_simple_lock_t lock = __SIMPLELOCK_UNLOCKED; 442 static int lastwarn; 443 uint64_t cur, prev; 444 lwp_t *l = curlwp; 445 int ticks; 446 447 /* 448 * Previous value must be read before the counter and stored to 449 * after, because this routine can be called from interrupt context 450 * and may run over the top of an existing invocation. Ordering is 451 * guaranteed by "volatile" on md_tsc. 452 */ 453 prev = l->l_md.md_tsc; 454 cur = cpu_counter(); 455 if (__predict_false(cur < prev) && (cur >> 63) == (prev >> 63) && 456 __cpu_simple_lock_try(&lock)) { 457 ticks = getticks(); 458 if (ticks - lastwarn >= hz) { 459 printf( 460 "WARNING: %s TSC went backwards by %u - " 461 "change sysctl(7) kern.timecounter?\n", 462 cpu_name(curcpu()), (unsigned)(prev - cur)); 463 lastwarn = ticks; 464 } 465 __cpu_simple_unlock(&lock); 466 } 467 l->l_md.md_tsc = cur; 468 return (uint32_t)cur; 469 #else 470 return cpu_counter32(); 471 #endif 472 } 473 474 /* 475 * tsc has been reset; zero the cached tsc of every lwp in the system 476 * so we don't spuriously report that the tsc has gone backward. 477 * Caller must ensure all LWPs are quiescent (except the current one, 478 * obviously) and interrupts are blocked while we update this. 479 */ 480 void 481 tsc_tc_reset(void) 482 { 483 struct lwp *l; 484 485 LIST_FOREACH(l, &alllwp, l_list) 486 l->l_md.md_tsc = 0; 487 } 488 489 #ifdef BOOT_DURATION 490 /* Returns the kernel boot time in milliseconds. */ 491 uint64_t 492 boot_duration_timer(void) 493 { 494 KASSERT(curcpu_stable()); 495 KASSERT(CPU_IS_PRIMARY(curcpu())); 496 return (rdtsc() - ((uint64_t)starttsc_hi << 32 | starttsc_lo)) / 497 (curcpu()->ci_data.cpu_cc_freq / 1000); 498 } 499 #endif 500
Indexes created Sun Oct 25 00:26:45 UTC 2026