1 /* $NetBSD: kern_tc.c,v 1.80 2026/01/04 01:54:31 riastradh Exp $ */ 2 3 /*- 4 * Copyright (c) 2008, 2009 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /*- 33 * ---------------------------------------------------------------------------- 34 * "THE BEER-WARE LICENSE" (Revision 42): 35 * <phk (at) FreeBSD.ORG> wrote this file. As long as you retain this notice you 36 * can do whatever you want with this stuff. If we meet some day, and you think 37 * this stuff is worth it, you can buy me a beer in return. Poul-Henning Kamp 38 * --------------------------------------------------------------------------- 39 */ 40 41 /* 42 * https://papers.freebsd.org/2002/phk-timecounters.files/timecounter.pdf 43 */ 44 45 #include <sys/cdefs.h> 46 /* __FBSDID("$FreeBSD: src/sys/kern/kern_tc.c,v 1.166 2005/09/19 22:16:31 andre Exp $"); */ 47 __KERNEL_RCSID(0, "$NetBSD: kern_tc.c,v 1.80 2026/01/04 01:54:31 riastradh Exp $"); 48 49 #ifdef _KERNEL_OPT 50 #include "opt_ntp.h" 51 #endif 52 53 #include <sys/param.h> 54 55 #include <sys/atomic.h> 56 #include <sys/evcnt.h> 57 #include <sys/kauth.h> 58 #include <sys/kernel.h> 59 #include <sys/lock.h> 60 #include <sys/mutex.h> 61 #include <sys/reboot.h> /* XXX just to get AB_VERBOSE */ 62 #include <sys/sdt.h> 63 #include <sys/sysctl.h> 64 #include <sys/syslog.h> 65 #include <sys/systm.h> 66 #include <sys/timepps.h> 67 #include <sys/timetc.h> 68 #include <sys/timex.h> 69 #include <sys/xcall.h> 70 71 /* 72 * A large step happens on boot. This constant detects such steps. 73 * It is relatively small so that ntp_update_second gets called enough 74 * in the typical 'missed a couple of seconds' case, but doesn't loop 75 * forever when the time step is large. 76 */ 77 #define LARGE_STEP 200 78 79 /* 80 * Implement a dummy timecounter which we can use until we get a real one 81 * in the air. This allows the console and other early stuff to use 82 * time services. 83 */ 84 85 static u_int 86 dummy_get_timecount(struct timecounter *tc) 87 { 88 static u_int now; 89 90 return ++now; 91 } 92 93 static struct timecounter dummy_timecounter = { 94 .tc_get_timecount = dummy_get_timecount, 95 .tc_counter_mask = ~0u, 96 .tc_frequency = 1000000, 97 .tc_name = "dummy", 98 .tc_quality = -1000000, 99 .tc_priv = NULL, 100 }; 101 102 struct timehands { 103 /* These fields must be initialized by the driver. */ 104 struct timecounter *th_counter; /* active timecounter */ 105 int64_t th_adjustment; /* frequency adjustment */ 106 /* (NTP/adjtime) */ 107 uint64_t th_scale; /* scale factor (counter */ 108 /* tick->time) */ 109 uint64_t th_offset_count; /* offset at last time */ 110 /* update (tc_windup()) */ 111 struct bintime th_offset; /* bin (up)time at windup */ 112 struct timeval th_microtime; /* cached microtime */ 113 struct timespec th_nanotime; /* cached nanotime */ 114 /* Fields not to be copied in tc_windup start with th_generation. */ 115 volatile u_int th_generation; /* current generation */ 116 struct timehands *th_next; /* next timehand */ 117 }; 118 119 static struct timehands th0; 120 static struct timehands th9 = { .th_next = &th0, }; 121 static struct timehands th8 = { .th_next = &th9, }; 122 static struct timehands th7 = { .th_next = &th8, }; 123 static struct timehands th6 = { .th_next = &th7, }; 124 static struct timehands th5 = { .th_next = &th6, }; 125 static struct timehands th4 = { .th_next = &th5, }; 126 static struct timehands th3 = { .th_next = &th4, }; 127 static struct timehands th2 = { .th_next = &th3, }; 128 static struct timehands th1 = { .th_next = &th2, }; 129 static struct timehands th0 = { 130 .th_counter = &dummy_timecounter, 131 .th_scale = (uint64_t)-1 / 1000000, 132 .th_offset = { .sec = 1, .frac = 0 }, 133 .th_generation = 1, 134 .th_next = &th1, 135 }; 136 137 static struct timehands *volatile timehands = &th0; 138 struct timecounter *timecounter = &dummy_timecounter; 139 static struct timecounter *timecounters = &dummy_timecounter; 140 141 /* used by savecore(8) */ 142 time_t time_second_legacy asm("time_second"); 143 144 #ifdef __HAVE_ATOMIC64_LOADSTORE 145 volatile time_t time__second __cacheline_aligned = 1; 146 volatile time_t time__uptime __cacheline_aligned = 1; 147 #else 148 static volatile struct { 149 uint32_t lo, hi; 150 } time__uptime32 __cacheline_aligned = { 151 .lo = 1, 152 }, time__second32 __cacheline_aligned = { 153 .lo = 1, 154 }; 155 #endif 156 157 static struct { 158 struct bintime bin; 159 volatile unsigned gen; /* even when stable, odd when changing */ 160 } timebase __cacheline_aligned; 161 162 static int timestepwarnings; 163 164 kmutex_t timecounter_lock; 165 static u_int timecounter_mods; 166 static volatile int timecounter_removals = 1; 167 static u_int timecounter_bad; 168 169 #ifdef __HAVE_ATOMIC64_LOADSTORE 170 171 static inline void 172 setrealuptime(time_t second, time_t uptime) 173 { 174 175 time_second_legacy = second; 176 177 atomic_store_relaxed(&time__second, second); 178 atomic_store_relaxed(&time__uptime, uptime); 179 } 180 181 #else 182 183 static inline void 184 setrealuptime(time_t second, time_t uptime) 185 { 186 uint32_t seclo = second & 0xffffffff, sechi = second >> 32; 187 uint32_t uplo = uptime & 0xffffffff, uphi = uptime >> 32; 188 189 KDASSERT(mutex_owned(&timecounter_lock)); 190 191 time_second_legacy = second; 192 193 /* 194 * Fast path -- no wraparound, just updating the low bits, so 195 * no need for seqlocked access. 196 */ 197 if (__predict_true(sechi == time__second32.hi) && 198 __predict_true(uphi == time__uptime32.hi)) { 199 atomic_store_relaxed(&time__second32.lo, seclo); 200 atomic_store_relaxed(&time__uptime32.lo, uplo); 201 return; 202 } 203 204 atomic_store_relaxed(&time__second32.hi, 0xffffffff); 205 atomic_store_relaxed(&time__uptime32.hi, 0xffffffff); 206 membar_producer(); 207 atomic_store_relaxed(&time__second32.lo, seclo); 208 atomic_store_relaxed(&time__uptime32.lo, uplo); 209 membar_producer(); 210 atomic_store_relaxed(&time__second32.hi, sechi); 211 atomic_store_relaxed(&time__uptime32.hi, uphi); 212 } 213 214 time_t 215 getrealtime(void) 216 { 217 uint32_t lo, hi; 218 219 do { 220 for (;;) { 221 hi = atomic_load_relaxed(&time__second32.hi); 222 if (__predict_true(hi != 0xffffffff)) 223 break; 224 SPINLOCK_BACKOFF_HOOK; 225 } 226 membar_consumer(); 227 lo = atomic_load_relaxed(&time__second32.lo); 228 membar_consumer(); 229 } while (hi != atomic_load_relaxed(&time__second32.hi)); 230 231 return ((time_t)hi << 32) | lo; 232 } 233 234 time_t 235 getuptime(void) 236 { 237 uint32_t lo, hi; 238 239 do { 240 for (;;) { 241 hi = atomic_load_relaxed(&time__uptime32.hi); 242 if (__predict_true(hi != 0xffffffff)) 243 break; 244 SPINLOCK_BACKOFF_HOOK; 245 } 246 membar_consumer(); 247 lo = atomic_load_relaxed(&time__uptime32.lo); 248 membar_consumer(); 249 } while (hi != atomic_load_relaxed(&time__uptime32.hi)); 250 251 return ((time_t)hi << 32) | lo; 252 } 253 254 time_t 255 getboottime(void) 256 { 257 258 return getrealtime() - getuptime(); 259 } 260 261 uint32_t 262 getuptime32(void) 263 { 264 265 return atomic_load_relaxed(&time__uptime32.lo); 266 } 267 268 #endif /* !defined(__HAVE_ATOMIC64_LOADSTORE) */ 269 270 /* 271 * sysctl helper routine for kern.timercounter.hardware 272 */ 273 static int 274 sysctl_kern_timecounter_hardware(SYSCTLFN_ARGS) 275 { 276 struct sysctlnode node; 277 int error; 278 char newname[MAX_TCNAMELEN]; 279 struct timecounter *newtc, *tc; 280 281 tc = timecounter; 282 283 strlcpy(newname, tc->tc_name, sizeof(newname)); 284 285 node = *rnode; 286 node.sysctl_data = newname; 287 node.sysctl_size = sizeof(newname); 288 289 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 290 291 if (error || 292 newp == NULL || 293 strncmp(newname, tc->tc_name, sizeof(newname)) == 0) 294 return error; 295 296 if (l != NULL && (error = kauth_authorize_system(l->l_cred, 297 KAUTH_SYSTEM_TIME, KAUTH_REQ_SYSTEM_TIME_TIMECOUNTERS, newname, 298 NULL, NULL)) != 0) 299 return error; 300 301 if (!cold) 302 mutex_spin_enter(&timecounter_lock); 303 error = SET_ERROR(EINVAL); 304 for (newtc = timecounters; newtc != NULL; newtc = newtc->tc_next) { 305 if (strcmp(newname, newtc->tc_name) != 0) 306 continue; 307 /* Warm up new timecounter. */ 308 (void)newtc->tc_get_timecount(newtc); 309 (void)newtc->tc_get_timecount(newtc); 310 timecounter = newtc; 311 error = 0; 312 break; 313 } 314 if (!cold) 315 mutex_spin_exit(&timecounter_lock); 316 return error; 317 } 318 319 static int 320 sysctl_kern_timecounter_choice(SYSCTLFN_ARGS) 321 { 322 char buf[MAX_TCNAMELEN+48]; 323 char *where; 324 const char *spc; 325 struct timecounter *tc; 326 size_t needed, left, slen; 327 int error, mods; 328 329 if (newp != NULL) 330 return SET_ERROR(EPERM); 331 if (namelen != 0) 332 return SET_ERROR(EINVAL); 333 334 mutex_spin_enter(&timecounter_lock); 335 retry: 336 spc = ""; 337 error = 0; 338 needed = 0; 339 left = *oldlenp; 340 where = oldp; 341 for (tc = timecounters; error == 0 && tc != NULL; tc = tc->tc_next) { 342 if (where == NULL) { 343 needed += sizeof(buf); /* be conservative */ 344 } else { 345 slen = snprintf(buf, sizeof(buf), "%s%s(q=%d, f=%" PRId64 346 " Hz)", spc, tc->tc_name, tc->tc_quality, 347 tc->tc_frequency); 348 if (left < slen + 1) 349 break; 350 mods = timecounter_mods; 351 mutex_spin_exit(&timecounter_lock); 352 error = copyout(buf, where, slen + 1); 353 mutex_spin_enter(&timecounter_lock); 354 if (mods != timecounter_mods) { 355 goto retry; 356 } 357 spc = " "; 358 where += slen; 359 needed += slen; 360 left -= slen; 361 } 362 } 363 mutex_spin_exit(&timecounter_lock); 364 365 *oldlenp = needed; 366 return error; 367 } 368 369 SYSCTL_SETUP(sysctl_timecounter_setup, "sysctl timecounter setup") 370 { 371 const struct sysctlnode *node; 372 373 sysctl_createv(clog, 0, NULL, &node, 374 CTLFLAG_PERMANENT, 375 CTLTYPE_NODE, "timecounter", 376 SYSCTL_DESCR("time counter information"), 377 NULL, 0, NULL, 0, 378 CTL_KERN, CTL_CREATE, CTL_EOL); 379 380 if (node != NULL) { 381 sysctl_createv(clog, 0, NULL, NULL, 382 CTLFLAG_PERMANENT, 383 CTLTYPE_STRING, "choice", 384 SYSCTL_DESCR("available counters"), 385 sysctl_kern_timecounter_choice, 0, NULL, 0, 386 CTL_KERN, node->sysctl_num, CTL_CREATE, CTL_EOL); 387 388 sysctl_createv(clog, 0, NULL, NULL, 389 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 390 CTLTYPE_STRING, "hardware", 391 SYSCTL_DESCR("currently active time counter"), 392 sysctl_kern_timecounter_hardware, 0, NULL, MAX_TCNAMELEN, 393 CTL_KERN, node->sysctl_num, CTL_CREATE, CTL_EOL); 394 395 sysctl_createv(clog, 0, NULL, NULL, 396 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 397 CTLTYPE_INT, "timestepwarnings", 398 SYSCTL_DESCR("log time steps"), 399 NULL, 0, ×tepwarnings, 0, 400 CTL_KERN, node->sysctl_num, CTL_CREATE, CTL_EOL); 401 } 402 } 403 404 #ifdef TC_COUNTERS 405 #define TC_STATS(name) \ 406 static struct evcnt n##name = \ 407 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "timecounter", #name); \ 408 EVCNT_ATTACH_STATIC(n##name) 409 TC_STATS(binuptime); TC_STATS(nanouptime); TC_STATS(microuptime); 410 TC_STATS(bintime); TC_STATS(nanotime); TC_STATS(microtime); 411 TC_STATS(getbinuptime); TC_STATS(getnanouptime); TC_STATS(getmicrouptime); 412 TC_STATS(getbintime); TC_STATS(getnanotime); TC_STATS(getmicrotime); 413 TC_STATS(setclock); 414 #define TC_COUNT(var) var.ev_count++ 415 #undef TC_STATS 416 #else 417 #define TC_COUNT(var) /* nothing */ 418 #endif /* TC_COUNTERS */ 419 420 static void tc_windup(void); 421 422 /* 423 * Return the difference between the timehands' counter value now and what 424 * was when we copied it to the timehands' offset_count. 425 */ 426 static inline u_int 427 tc_delta(struct timehands *th) 428 { 429 struct timecounter *tc; 430 431 tc = th->th_counter; 432 return (tc->tc_get_timecount(tc) - 433 th->th_offset_count) & tc->tc_counter_mask; 434 } 435 436 /* 437 * Functions for reading the time. We have to loop until we are sure that 438 * the timehands that we operated on was not updated under our feet. See 439 * the comment in <sys/timevar.h> for a description of these 12 functions. 440 */ 441 442 void 443 binuptime(struct bintime *bt) 444 { 445 struct timehands *th; 446 lwp_t *l; 447 u_int lgen, gen; 448 449 TC_COUNT(nbinuptime); 450 451 /* 452 * Provide exclusion against tc_detach(). 453 * 454 * We record the number of timecounter removals before accessing 455 * timecounter state. Note that the LWP can be using multiple 456 * "generations" at once, due to interrupts (interrupted while in 457 * this function). Hardware interrupts will borrow the interrupted 458 * LWP's l_tcgen value for this purpose, and can themselves be 459 * interrupted by higher priority interrupts. In this case we need 460 * to ensure that the oldest generation in use is recorded. 461 * 462 * splsched() is too expensive to use, so we take care to structure 463 * this code in such a way that it is not required. Likewise, we 464 * do not disable preemption. 465 * 466 * Memory barriers are also too expensive to use for such a 467 * performance critical function. The good news is that we do not 468 * need memory barriers for this type of exclusion, as the thread 469 * updating timecounter_removals will issue a broadcast cross call 470 * before inspecting our l_tcgen value (this elides memory ordering 471 * issues). 472 * 473 * XXX If the author of the above comment knows how to make it 474 * safe to avoid memory barriers around the access to 475 * th->th_generation, I'm all ears. 476 */ 477 l = curlwp; 478 lgen = l->l_tcgen; 479 if (__predict_true(lgen == 0)) { 480 l->l_tcgen = timecounter_removals; 481 } 482 __insn_barrier(); 483 484 do { 485 th = atomic_load_consume(&timehands); 486 gen = th->th_generation; 487 membar_consumer(); 488 *bt = th->th_offset; 489 bintime_addx(bt, th->th_scale * tc_delta(th)); 490 membar_consumer(); 491 } while (gen == 0 || gen != th->th_generation); 492 493 __insn_barrier(); 494 l->l_tcgen = lgen; 495 } 496 497 void 498 nanouptime(struct timespec *tsp) 499 { 500 struct bintime bt; 501 502 TC_COUNT(nnanouptime); 503 binuptime(&bt); 504 bintime2timespec(&bt, tsp); 505 } 506 507 void 508 microuptime(struct timeval *tvp) 509 { 510 struct bintime bt; 511 512 TC_COUNT(nmicrouptime); 513 binuptime(&bt); 514 bintime2timeval(&bt, tvp); 515 } 516 517 void 518 bintime(struct bintime *bt) 519 { 520 struct bintime boottime; 521 522 TC_COUNT(nbintime); 523 binuptime(bt); 524 getbinboottime(&boottime); 525 bintime_add(bt, &boottime); 526 } 527 528 void 529 nanotime(struct timespec *tsp) 530 { 531 struct bintime bt; 532 533 TC_COUNT(nnanotime); 534 bintime(&bt); 535 bintime2timespec(&bt, tsp); 536 } 537 538 void 539 microtime(struct timeval *tvp) 540 { 541 struct bintime bt; 542 543 TC_COUNT(nmicrotime); 544 bintime(&bt); 545 bintime2timeval(&bt, tvp); 546 } 547 548 void 549 getbinuptime(struct bintime *bt) 550 { 551 struct timehands *th; 552 u_int gen; 553 554 TC_COUNT(ngetbinuptime); 555 do { 556 th = atomic_load_consume(&timehands); 557 gen = th->th_generation; 558 membar_consumer(); 559 *bt = th->th_offset; 560 membar_consumer(); 561 } while (gen == 0 || gen != th->th_generation); 562 } 563 564 void 565 getnanouptime(struct timespec *tsp) 566 { 567 struct timehands *th; 568 u_int gen; 569 570 TC_COUNT(ngetnanouptime); 571 do { 572 th = atomic_load_consume(&timehands); 573 gen = th->th_generation; 574 membar_consumer(); 575 bintime2timespec(&th->th_offset, tsp); 576 membar_consumer(); 577 } while (gen == 0 || gen != th->th_generation); 578 } 579 580 void 581 getmicrouptime(struct timeval *tvp) 582 { 583 struct timehands *th; 584 u_int gen; 585 586 TC_COUNT(ngetmicrouptime); 587 do { 588 th = atomic_load_consume(&timehands); 589 gen = th->th_generation; 590 membar_consumer(); 591 bintime2timeval(&th->th_offset, tvp); 592 membar_consumer(); 593 } while (gen == 0 || gen != th->th_generation); 594 } 595 596 void 597 getbintime(struct bintime *bt) 598 { 599 struct timehands *th; 600 struct bintime boottime; 601 u_int gen; 602 603 TC_COUNT(ngetbintime); 604 do { 605 th = atomic_load_consume(&timehands); 606 gen = th->th_generation; 607 membar_consumer(); 608 *bt = th->th_offset; 609 membar_consumer(); 610 } while (gen == 0 || gen != th->th_generation); 611 getbinboottime(&boottime); 612 bintime_add(bt, &boottime); 613 } 614 615 static inline void 616 dogetnanotime(struct timespec *tsp) 617 { 618 struct timehands *th; 619 u_int gen; 620 621 TC_COUNT(ngetnanotime); 622 do { 623 th = atomic_load_consume(&timehands); 624 gen = th->th_generation; 625 membar_consumer(); 626 *tsp = th->th_nanotime; 627 membar_consumer(); 628 } while (gen == 0 || gen != th->th_generation); 629 } 630 631 void 632 getnanotime(struct timespec *tsp) 633 { 634 635 dogetnanotime(tsp); 636 } 637 638 void dtrace_getnanotime(struct timespec *tsp); 639 640 void 641 dtrace_getnanotime(struct timespec *tsp) 642 { 643 644 dogetnanotime(tsp); 645 } 646 647 void 648 getmicrotime(struct timeval *tvp) 649 { 650 struct timehands *th; 651 u_int gen; 652 653 TC_COUNT(ngetmicrotime); 654 do { 655 th = atomic_load_consume(&timehands); 656 gen = th->th_generation; 657 membar_consumer(); 658 *tvp = th->th_microtime; 659 membar_consumer(); 660 } while (gen == 0 || gen != th->th_generation); 661 } 662 663 void 664 getnanoboottime(struct timespec *tsp) 665 { 666 struct bintime bt; 667 668 getbinboottime(&bt); 669 bintime2timespec(&bt, tsp); 670 } 671 672 void 673 getmicroboottime(struct timeval *tvp) 674 { 675 struct bintime bt; 676 677 getbinboottime(&bt); 678 bintime2timeval(&bt, tvp); 679 } 680 681 void 682 getbinboottime(struct bintime *basep) 683 { 684 struct bintime base; 685 unsigned gen; 686 687 do { 688 /* Spin until the timebase isn't changing. */ 689 while ((gen = atomic_load_relaxed(&timebase.gen)) & 1) 690 SPINLOCK_BACKOFF_HOOK; 691 692 /* Read out a snapshot of the timebase. */ 693 membar_consumer(); 694 base = timebase.bin; 695 membar_consumer(); 696 697 /* Restart if it changed while we were reading. */ 698 } while (gen != atomic_load_relaxed(&timebase.gen)); 699 700 *basep = base; 701 } 702 703 /* 704 * Initialize a new timecounter and possibly use it. 705 */ 706 void 707 tc_init(struct timecounter *tc) 708 { 709 u_int u; 710 711 KASSERTMSG(tc->tc_next == NULL, "timecounter %s already initialised", 712 tc->tc_name); 713 714 u = tc->tc_frequency / tc->tc_counter_mask; 715 /* XXX: We need some margin here, 10% is a guess */ 716 u *= 11; 717 u /= 10; 718 if (u > hz && tc->tc_quality >= 0) { 719 tc->tc_quality = -2000; 720 aprint_verbose( 721 "timecounter: Timecounter \"%s\" frequency %ju Hz", 722 tc->tc_name, (uintmax_t)tc->tc_frequency); 723 aprint_verbose(" -- Insufficient hz, needs at least %u\n", u); 724 } else if (tc->tc_quality >= 0 || bootverbose) { 725 aprint_verbose( 726 "timecounter: Timecounter \"%s\" frequency %ju Hz " 727 "quality %d\n", tc->tc_name, (uintmax_t)tc->tc_frequency, 728 tc->tc_quality); 729 } 730 731 mutex_spin_enter(&timecounter_lock); 732 tc->tc_next = timecounters; 733 timecounters = tc; 734 timecounter_mods++; 735 /* 736 * Never automatically use a timecounter with negative quality. 737 * Even though we run on the dummy counter, switching here may be 738 * worse since this timecounter may not be monotonous. 739 */ 740 if (tc->tc_quality >= 0 && (tc->tc_quality > timecounter->tc_quality || 741 (tc->tc_quality == timecounter->tc_quality && 742 tc->tc_frequency > timecounter->tc_frequency))) { 743 (void)tc->tc_get_timecount(tc); 744 (void)tc->tc_get_timecount(tc); 745 timecounter = tc; 746 tc_windup(); 747 } 748 mutex_spin_exit(&timecounter_lock); 749 } 750 751 /* 752 * Pick a new timecounter due to the existing counter going bad. 753 */ 754 static void 755 tc_pick(void) 756 { 757 struct timecounter *best, *tc; 758 759 KASSERT(mutex_owned(&timecounter_lock)); 760 761 for (best = tc = timecounters; tc != NULL; tc = tc->tc_next) { 762 if (tc->tc_quality > best->tc_quality) 763 best = tc; 764 else if (tc->tc_quality < best->tc_quality) 765 continue; 766 else if (tc->tc_frequency > best->tc_frequency) 767 best = tc; 768 } 769 (void)best->tc_get_timecount(best); 770 (void)best->tc_get_timecount(best); 771 timecounter = best; 772 } 773 774 /* 775 * A timecounter has gone bad, arrange to pick a new one at the next 776 * clock tick. 777 */ 778 void 779 tc_gonebad(struct timecounter *tc) 780 { 781 782 tc->tc_quality = -100; 783 membar_producer(); 784 atomic_inc_uint(&timecounter_bad); 785 } 786 787 /* 788 * Stop using a timecounter and remove it from the timecounters list. 789 */ 790 int 791 tc_detach(struct timecounter *target) 792 { 793 struct timecounter *tc; 794 struct timecounter **tcp = NULL; 795 int removals; 796 lwp_t *l; 797 798 /* First, find the timecounter. */ 799 mutex_spin_enter(&timecounter_lock); 800 for (tcp = &timecounters, tc = timecounters; 801 tc != NULL; 802 tcp = &tc->tc_next, tc = tc->tc_next) { 803 if (tc == target) 804 break; 805 } 806 if (tc == NULL) { 807 mutex_spin_exit(&timecounter_lock); 808 return SET_ERROR(ESRCH); 809 } 810 811 /* And now, remove it. */ 812 *tcp = tc->tc_next; 813 if (timecounter == target) { 814 tc_pick(); 815 tc_windup(); 816 } 817 timecounter_mods++; 818 removals = timecounter_removals++; 819 mutex_spin_exit(&timecounter_lock); 820 821 /* 822 * We now have to determine if any threads in the system are still 823 * making use of this timecounter. 824 * 825 * We issue a broadcast cross call to elide memory ordering issues, 826 * then scan all LWPs in the system looking at each's timecounter 827 * generation number. We need to see a value of zero (not actively 828 * using a timecounter) or a value greater than our removal value. 829 * 830 * We may race with threads that read `timecounter_removals' and 831 * and then get preempted before updating `l_tcgen'. This is not 832 * a problem, since it means that these threads have not yet started 833 * accessing timecounter state. All we do need is one clean 834 * snapshot of the system where every thread appears not to be using 835 * old timecounter state. 836 */ 837 for (;;) { 838 xc_barrier(0); 839 840 mutex_enter(&proc_lock); 841 LIST_FOREACH(l, &alllwp, l_list) { 842 if (l->l_tcgen == 0 || l->l_tcgen > removals) { 843 /* 844 * Not using timecounter or old timecounter 845 * state at time of our xcall or later. 846 */ 847 continue; 848 } 849 break; 850 } 851 mutex_exit(&proc_lock); 852 853 /* 854 * If the timecounter is still in use, wait at least 10ms 855 * before retrying. 856 */ 857 if (l == NULL) { 858 break; 859 } 860 (void)kpause("tcdetach", false, mstohz(10), NULL); 861 } 862 863 tc->tc_next = NULL; 864 return 0; 865 } 866 867 /* Report the frequency of the current timecounter. */ 868 uint64_t 869 tc_getfrequency(void) 870 { 871 872 return atomic_load_consume(&timehands)->th_counter->tc_frequency; 873 } 874 875 /* 876 * Step our concept of UTC. This is done by modifying our estimate of 877 * when we booted. 878 */ 879 void 880 tc_setclock(const struct timespec *ts) 881 { 882 struct timespec ts2; 883 struct bintime bt, bt2; 884 885 mutex_spin_enter(&timecounter_lock); 886 TC_COUNT(nsetclock); 887 binuptime(&bt2); 888 timespec2bintime(ts, &bt); 889 bintime_sub(&bt, &bt2); 890 bintime_add(&bt2, &timebase.bin); 891 timebase.gen |= 1; /* change in progress */ 892 membar_producer(); 893 timebase.bin = bt; 894 membar_producer(); 895 timebase.gen++; /* commit change */ 896 tc_windup(); 897 mutex_spin_exit(&timecounter_lock); 898 899 if (timestepwarnings) { 900 bintime2timespec(&bt2, &ts2); 901 log(LOG_INFO, 902 "Time stepped from %lld.%09ld to %lld.%09ld\n", 903 (long long)ts2.tv_sec, ts2.tv_nsec, 904 (long long)ts->tv_sec, ts->tv_nsec); 905 } 906 } 907 908 /* 909 * Initialize the next struct timehands in the ring and make 910 * it the active timehands. Along the way we might switch to a different 911 * timecounter and/or do seconds processing in NTP. Slightly magic. 912 */ 913 static void 914 tc_windup(void) 915 { 916 struct bintime bt; 917 struct timehands *th, *tho; 918 uint64_t scale; 919 u_int delta, ncount, ogen; 920 int i, s_update; 921 time_t t; 922 923 KASSERT(mutex_owned(&timecounter_lock)); 924 925 s_update = 0; 926 927 /* 928 * Make the next timehands a copy of the current one, but do not 929 * overwrite the generation or next pointer. While we update 930 * the contents, the generation must be zero. Ensure global 931 * visibility of the generation before proceeding. 932 */ 933 tho = timehands; 934 th = tho->th_next; 935 ogen = th->th_generation; 936 th->th_generation = 0; 937 membar_producer(); 938 bcopy(tho, th, offsetof(struct timehands, th_generation)); 939 940 /* 941 * Capture a timecounter delta on the current timecounter and if 942 * changing timecounters, a counter value from the new timecounter. 943 * Update the offset fields accordingly. 944 */ 945 delta = tc_delta(th); 946 if (th->th_counter != timecounter) 947 ncount = timecounter->tc_get_timecount(timecounter); 948 else 949 ncount = 0; 950 th->th_offset_count += delta; 951 bintime_addx(&th->th_offset, th->th_scale * delta); 952 953 /* 954 * Hardware latching timecounters may not generate interrupts on 955 * PPS events, so instead we poll them. There is a finite risk that 956 * the hardware might capture a count which is later than the one we 957 * got above, and therefore possibly in the next NTP second which might 958 * have a different rate than the current NTP second. It doesn't 959 * matter in practice. 960 */ 961 if (tho->th_counter->tc_poll_pps) 962 tho->th_counter->tc_poll_pps(tho->th_counter); 963 964 /* 965 * Deal with NTP second processing. The for loop normally 966 * iterates at most once, but in extreme situations it might 967 * keep NTP sane if timeouts are not run for several seconds. 968 * At boot, the time step can be large when the TOD hardware 969 * has been read, so on really large steps, we call 970 * ntp_update_second only twice. We need to call it twice in 971 * case we missed a leap second. 972 * If NTP is not compiled in ntp_update_second still calculates 973 * the adjustment resulting from adjtime() calls. 974 */ 975 bt = th->th_offset; 976 bintime_add(&bt, &timebase.bin); 977 i = bt.sec - tho->th_microtime.tv_sec; 978 if (i > LARGE_STEP) 979 i = 2; 980 for (; i > 0; i--) { 981 t = bt.sec; 982 ntp_update_second(&th->th_adjustment, &bt.sec); 983 s_update = 1; 984 if (bt.sec != t) { 985 timebase.gen |= 1; /* change in progress */ 986 membar_producer(); 987 timebase.bin.sec += bt.sec - t; 988 membar_producer(); 989 timebase.gen++; /* commit change */ 990 } 991 } 992 993 /* Update the UTC timestamps used by the get*() functions. */ 994 /* XXX shouldn't do this here. Should force non-`get' versions. */ 995 bintime2timeval(&bt, &th->th_microtime); 996 bintime2timespec(&bt, &th->th_nanotime); 997 /* Now is a good time to change timecounters. */ 998 if (th->th_counter != timecounter) { 999 th->th_counter = timecounter; 1000 th->th_offset_count = ncount; 1001 s_update = 1; 1002 } 1003 1004 /*- 1005 * Recalculate the scaling factor. We want the number of 1/2^64 1006 * fractions of a second per period of the hardware counter, taking 1007 * into account the th_adjustment factor which the NTP PLL/adjtime(2) 1008 * processing provides us with. 1009 * 1010 * The th_adjustment is nanoseconds per second with 32 bit binary 1011 * fraction and we want 64 bit binary fraction of second: 1012 * 1013 * x = a * 2^32 / 10^9 = a * 4.294967296 1014 * 1015 * The range of th_adjustment is +/- 5000PPM so inside a 64bit int 1016 * we can only multiply by about 850 without overflowing, but that 1017 * leaves suitably precise fractions for multiply before divide. 1018 * 1019 * Divide before multiply with a fraction of 2199/512 results in a 1020 * systematic undercompensation of 10PPM of th_adjustment. On a 1021 * 5000PPM adjustment this is a 0.05PPM error. This is acceptable. 1022 * 1023 * We happily sacrifice the lowest of the 64 bits of our result 1024 * to the goddess of code clarity. 1025 * 1026 */ 1027 if (s_update) { 1028 scale = (uint64_t)1 << 63; 1029 scale += (th->th_adjustment / 1024) * 2199; 1030 scale /= th->th_counter->tc_frequency; 1031 th->th_scale = scale * 2; 1032 } 1033 /* 1034 * Now that the struct timehands is again consistent, set the new 1035 * generation number, making sure to not make it zero. Ensure 1036 * changes are globally visible before changing. 1037 */ 1038 if (++ogen == 0) 1039 ogen = 1; 1040 membar_producer(); 1041 th->th_generation = ogen; 1042 1043 /* 1044 * Go live with the new struct timehands. Ensure changes are 1045 * globally visible before changing. 1046 */ 1047 setrealuptime(th->th_microtime.tv_sec, th->th_offset.sec); 1048 atomic_store_release(&timehands, th); 1049 1050 /* 1051 * Force users of the old timehand to move on. This is 1052 * necessary for MP systems; we need to ensure that the 1053 * consumers will move away from the old timehand before 1054 * we begin updating it again when we eventually wrap 1055 * around. 1056 */ 1057 if (++tho->th_generation == 0) 1058 tho->th_generation = 1; 1059 } 1060 1061 /* 1062 * RFC 2783 PPS-API implementation. 1063 */ 1064 1065 int 1066 pps_ioctl(u_long cmd, void *data, struct pps_state *pps) 1067 { 1068 pps_params_t *app; 1069 pps_info_t *pipi; 1070 #ifdef PPS_SYNC 1071 int *epi; 1072 #endif 1073 1074 KASSERT(mutex_owned(&timecounter_lock)); 1075 1076 KASSERT(pps != NULL); 1077 1078 switch (cmd) { 1079 case PPS_IOC_CREATE: 1080 return 0; 1081 case PPS_IOC_DESTROY: 1082 return 0; 1083 case PPS_IOC_SETPARAMS: 1084 app = (pps_params_t *)data; 1085 if (app->mode & ~pps->ppscap) 1086 return SET_ERROR(EINVAL); 1087 pps->ppsparam = *app; 1088 return 0; 1089 case PPS_IOC_GETPARAMS: 1090 app = (pps_params_t *)data; 1091 *app = pps->ppsparam; 1092 app->api_version = PPS_API_VERS_1; 1093 return 0; 1094 case PPS_IOC_GETCAP: 1095 *(int*)data = pps->ppscap; 1096 return 0; 1097 case PPS_IOC_FETCH: 1098 pipi = (pps_info_t *)data; 1099 pps->ppsinfo.current_mode = pps->ppsparam.mode; 1100 *pipi = pps->ppsinfo; 1101 return 0; 1102 case PPS_IOC_KCBIND: 1103 #ifdef PPS_SYNC 1104 epi = (int *)data; 1105 /* XXX Only root should be able to do this */ 1106 if (*epi & ~pps->ppscap) 1107 return SET_ERROR(EINVAL); 1108 pps->kcmode = *epi; 1109 return 0; 1110 #else 1111 return SET_ERROR(EOPNOTSUPP); 1112 #endif 1113 default: 1114 return SET_ERROR(EPASSTHROUGH); 1115 } 1116 } 1117 1118 void 1119 pps_init(struct pps_state *pps) 1120 { 1121 1122 KASSERT(mutex_owned(&timecounter_lock)); 1123 1124 pps->ppscap |= PPS_TSFMT_TSPEC; 1125 if (pps->ppscap & PPS_CAPTUREASSERT) 1126 pps->ppscap |= PPS_OFFSETASSERT; 1127 if (pps->ppscap & PPS_CAPTURECLEAR) 1128 pps->ppscap |= PPS_OFFSETCLEAR; 1129 } 1130 1131 /* 1132 * capture a timestamp in the pps structure 1133 */ 1134 void 1135 pps_capture(struct pps_state *pps) 1136 { 1137 struct timehands *th; 1138 1139 KASSERT(mutex_owned(&timecounter_lock)); 1140 KASSERT(pps != NULL); 1141 1142 th = timehands; 1143 pps->capgen = th->th_generation; 1144 pps->capth = th; 1145 pps->capcount = (uint64_t)tc_delta(th) + th->th_offset_count; 1146 if (pps->capgen != th->th_generation) 1147 pps->capgen = 0; 1148 } 1149 1150 #ifdef PPS_DEBUG 1151 int ppsdebug = 0; 1152 #endif 1153 1154 /* 1155 * process a pps_capture()ed event 1156 */ 1157 void 1158 pps_event(struct pps_state *pps, int event) 1159 { 1160 pps_ref_event(pps, event, NULL, PPS_REFEVNT_PPS|PPS_REFEVNT_CAPTURE); 1161 } 1162 1163 /* 1164 * extended pps api / kernel pll/fll entry point 1165 * 1166 * feed reference time stamps to PPS engine 1167 * 1168 * will simulate a PPS event and feed 1169 * the NTP PLL/FLL if requested. 1170 * 1171 * the ref time stamps should be roughly once 1172 * a second but do not need to be exactly in phase 1173 * with the UTC second but should be close to it. 1174 * this relaxation of requirements allows callout 1175 * driven timestamping mechanisms to feed to pps 1176 * capture/kernel pll logic. 1177 * 1178 * calling pattern is: 1179 * pps_capture() (for PPS_REFEVNT_{CAPTURE|CAPCUR}) 1180 * read timestamp from reference source 1181 * pps_ref_event() 1182 * 1183 * supported refmodes: 1184 * PPS_REFEVNT_CAPTURE 1185 * use system timestamp of pps_capture() 1186 * PPS_REFEVNT_CURRENT 1187 * use system timestamp of this call 1188 * PPS_REFEVNT_CAPCUR 1189 * use average of read capture and current system time stamp 1190 * PPS_REFEVNT_PPS 1191 * assume timestamp on second mark - ref_ts is ignored 1192 * 1193 */ 1194 1195 void 1196 pps_ref_event(struct pps_state *pps, 1197 int event, 1198 struct bintime *ref_ts, 1199 int refmode 1200 ) 1201 { 1202 struct bintime bt; /* current time */ 1203 struct bintime btd; /* time difference */ 1204 struct bintime bt_ref; /* reference time */ 1205 struct timespec ts, *tsp, *osp; 1206 struct timehands *th; 1207 uint64_t tcount, acount, dcount, *pcount; 1208 int foff, gen; 1209 #ifdef PPS_SYNC 1210 int fhard; 1211 #endif 1212 pps_seq_t *pseq; 1213 1214 KASSERT(mutex_owned(&timecounter_lock)); 1215 1216 KASSERT(pps != NULL); 1217 1218 /* pick up current time stamp if needed */ 1219 if (refmode & (PPS_REFEVNT_CURRENT|PPS_REFEVNT_CAPCUR)) { 1220 /* pick up current time stamp */ 1221 th = timehands; 1222 gen = th->th_generation; 1223 tcount = (uint64_t)tc_delta(th) + th->th_offset_count; 1224 if (gen != th->th_generation) 1225 gen = 0; 1226 1227 /* If the timecounter was wound up underneath us, bail out. */ 1228 if (pps->capgen == 0 || 1229 pps->capgen != pps->capth->th_generation || 1230 gen == 0 || 1231 gen != pps->capgen) { 1232 #ifdef PPS_DEBUG 1233 if (ppsdebug & 0x1) { 1234 log(LOG_DEBUG, 1235 "pps_ref_event(pps=%p, event=%d, ...): DROP (wind-up)\n", 1236 pps, event); 1237 } 1238 #endif 1239 return; 1240 } 1241 } else { 1242 tcount = 0; /* keep GCC happy */ 1243 } 1244 1245 #ifdef PPS_DEBUG 1246 if (ppsdebug & 0x1) { 1247 struct timespec tmsp; 1248 1249 if (ref_ts == NULL) { 1250 tmsp.tv_sec = 0; 1251 tmsp.tv_nsec = 0; 1252 } else { 1253 bintime2timespec(ref_ts, &tmsp); 1254 } 1255 1256 log(LOG_DEBUG, 1257 "pps_ref_event(pps=%p, event=%d, ref_ts=%"PRIi64 1258 ".%09"PRIi32", refmode=0x%1x)\n", 1259 pps, event, tmsp.tv_sec, (int32_t)tmsp.tv_nsec, refmode); 1260 } 1261 #endif 1262 1263 /* setup correct event references */ 1264 if (event == PPS_CAPTUREASSERT) { 1265 tsp = &pps->ppsinfo.assert_timestamp; 1266 osp = &pps->ppsparam.assert_offset; 1267 foff = pps->ppsparam.mode & PPS_OFFSETASSERT; 1268 #ifdef PPS_SYNC 1269 fhard = pps->kcmode & PPS_CAPTUREASSERT; 1270 #endif 1271 pcount = &pps->ppscount[0]; 1272 pseq = &pps->ppsinfo.assert_sequence; 1273 } else { 1274 tsp = &pps->ppsinfo.clear_timestamp; 1275 osp = &pps->ppsparam.clear_offset; 1276 foff = pps->ppsparam.mode & PPS_OFFSETCLEAR; 1277 #ifdef PPS_SYNC 1278 fhard = pps->kcmode & PPS_CAPTURECLEAR; 1279 #endif 1280 pcount = &pps->ppscount[1]; 1281 pseq = &pps->ppsinfo.clear_sequence; 1282 } 1283 1284 /* determine system time stamp according to refmode */ 1285 dcount = 0; /* keep GCC happy */ 1286 switch (refmode & PPS_REFEVNT_RMASK) { 1287 case PPS_REFEVNT_CAPTURE: 1288 acount = pps->capcount; /* use capture timestamp */ 1289 break; 1290 1291 case PPS_REFEVNT_CURRENT: 1292 acount = tcount; /* use current timestamp */ 1293 break; 1294 1295 case PPS_REFEVNT_CAPCUR: 1296 /* 1297 * calculate counter value between pps_capture() and 1298 * pps_ref_event() 1299 */ 1300 dcount = tcount - pps->capcount; 1301 acount = (dcount / 2) + pps->capcount; 1302 break; 1303 1304 default: /* ignore call error silently */ 1305 return; 1306 } 1307 1308 /* 1309 * If the timecounter changed, we cannot compare the count values, so 1310 * we have to drop the rest of the PPS-stuff until the next event. 1311 */ 1312 if (pps->ppstc != pps->capth->th_counter) { 1313 pps->ppstc = pps->capth->th_counter; 1314 pps->capcount = acount; 1315 *pcount = acount; 1316 pps->ppscount[2] = acount; 1317 #ifdef PPS_DEBUG 1318 if (ppsdebug & 0x1) { 1319 log(LOG_DEBUG, 1320 "pps_ref_event(pps=%p, event=%d, ...): DROP (time-counter change)\n", 1321 pps, event); 1322 } 1323 #endif 1324 return; 1325 } 1326 1327 pps->capcount = acount; 1328 1329 /* Convert the count to a bintime. */ 1330 bt = pps->capth->th_offset; 1331 bintime_addx(&bt, pps->capth->th_scale * (acount - pps->capth->th_offset_count)); 1332 bintime_add(&bt, &timebase.bin); 1333 1334 if ((refmode & PPS_REFEVNT_PPS) == 0) { 1335 /* determine difference to reference time stamp */ 1336 bt_ref = *ref_ts; 1337 1338 btd = bt; 1339 bintime_sub(&btd, &bt_ref); 1340 1341 /* 1342 * simulate a PPS timestamp by dropping the fraction 1343 * and applying the offset 1344 */ 1345 if (bt.frac >= (uint64_t)1<<63) /* skip to nearest second */ 1346 bt.sec++; 1347 bt.frac = 0; 1348 bintime_add(&bt, &btd); 1349 } else { 1350 /* 1351 * create ref_ts from current time - 1352 * we are supposed to be called on 1353 * the second mark 1354 */ 1355 bt_ref = bt; 1356 if (bt_ref.frac >= (uint64_t)1<<63) /* skip to nearest second */ 1357 bt_ref.sec++; 1358 bt_ref.frac = 0; 1359 } 1360 1361 /* convert bintime to timestamp */ 1362 bintime2timespec(&bt, &ts); 1363 1364 /* If the timecounter was wound up underneath us, bail out. */ 1365 if (pps->capgen != pps->capth->th_generation) 1366 return; 1367 1368 /* store time stamp */ 1369 *pcount = pps->capcount; 1370 (*pseq)++; 1371 *tsp = ts; 1372 1373 /* add offset correction */ 1374 if (foff) { 1375 timespecadd(tsp, osp, tsp); 1376 if (tsp->tv_nsec < 0) { 1377 tsp->tv_nsec += 1000000000; 1378 tsp->tv_sec -= 1; 1379 } 1380 } 1381 1382 #ifdef PPS_DEBUG 1383 if (ppsdebug & 0x2) { 1384 struct timespec ts2; 1385 struct timespec ts3; 1386 1387 bintime2timespec(&bt_ref, &ts2); 1388 1389 bt.sec = 0; 1390 bt.frac = 0; 1391 1392 if (refmode & PPS_REFEVNT_CAPCUR) { 1393 bintime_addx(&bt, pps->capth->th_scale * dcount); 1394 } 1395 bintime2timespec(&bt, &ts3); 1396 1397 log(LOG_DEBUG, "ref_ts=%"PRIi64".%09"PRIi32 1398 ", ts=%"PRIi64".%09"PRIi32", read latency=%"PRIi64" ns\n", 1399 ts2.tv_sec, (int32_t)ts2.tv_nsec, 1400 tsp->tv_sec, (int32_t)tsp->tv_nsec, 1401 timespec2ns(&ts3)); 1402 } 1403 #endif 1404 1405 #ifdef PPS_SYNC 1406 if (fhard) { 1407 uint64_t scale; 1408 uint64_t div; 1409 1410 /* 1411 * Feed the NTP PLL/FLL. 1412 * The FLL wants to know how many (hardware) nanoseconds 1413 * elapsed since the previous event (mod 1 second) thus 1414 * we are actually looking at the frequency difference scaled 1415 * in nsec. 1416 * As the counter time stamps are not truly at 1Hz 1417 * we need to scale the count by the elapsed 1418 * reference time. 1419 * valid sampling interval: [0.5..2[ sec 1420 */ 1421 1422 /* calculate elapsed raw count */ 1423 tcount = pps->capcount - pps->ppscount[2]; 1424 pps->ppscount[2] = pps->capcount; 1425 tcount &= pps->capth->th_counter->tc_counter_mask; 1426 1427 /* calculate elapsed ref time */ 1428 btd = bt_ref; 1429 bintime_sub(&btd, &pps->ref_time); 1430 pps->ref_time = bt_ref; 1431 1432 /* check that we stay below 2 sec */ 1433 if (btd.sec < 0 || btd.sec > 1) 1434 return; 1435 1436 /* we want at least 0.5 sec between samples */ 1437 if (btd.sec == 0 && btd.frac < (uint64_t)1<<63) 1438 return; 1439 1440 /* 1441 * calculate cycles per period by multiplying 1442 * the frequency with the elapsed period 1443 * we pick a fraction of 30 bits 1444 * ~1ns resolution for elapsed time 1445 */ 1446 div = (uint64_t)btd.sec << 30; 1447 div |= (btd.frac >> 34) & (((uint64_t)1 << 30) - 1); 1448 div *= pps->capth->th_counter->tc_frequency; 1449 div >>= 30; 1450 1451 if (div == 0) /* safeguard */ 1452 return; 1453 1454 scale = (uint64_t)1 << 63; 1455 scale /= div; 1456 scale *= 2; 1457 1458 bt.sec = 0; 1459 bt.frac = 0; 1460 bintime_addx(&bt, scale * tcount); 1461 bintime2timespec(&bt, &ts); 1462 1463 #ifdef PPS_DEBUG 1464 if (ppsdebug & 0x4) { 1465 struct timespec ts2; 1466 int64_t df; 1467 1468 bintime2timespec(&bt_ref, &ts2); 1469 df = timespec2ns(&ts); 1470 if (df > 500000000) 1471 df -= 1000000000; 1472 log(LOG_DEBUG, "hardpps: ref_ts=%"PRIi64 1473 ".%09"PRIi32", ts=%"PRIi64".%09"PRIi32 1474 ", freqdiff=%"PRIi64" ns/s\n", 1475 ts2.tv_sec, (int32_t)ts2.tv_nsec, 1476 tsp->tv_sec, (int32_t)tsp->tv_nsec, 1477 df); 1478 } 1479 #endif 1480 1481 hardpps(tsp, timespec2ns(&ts)); 1482 } 1483 #endif 1484 } 1485 1486 /* 1487 * Timecounters need to be updated every so often to prevent the hardware 1488 * counter from overflowing. Updating also recalculates the cached values 1489 * used by the get*() family of functions, so their precision depends on 1490 * the update frequency. 1491 */ 1492 1493 static int tc_tick; 1494 1495 void 1496 tc_ticktock(void) 1497 { 1498 static int count; 1499 1500 if (++count < tc_tick) 1501 return; 1502 count = 0; 1503 mutex_spin_enter(&timecounter_lock); 1504 if (__predict_false(timecounter_bad != 0)) { 1505 /* An existing timecounter has gone bad, pick a new one. */ 1506 (void)atomic_swap_uint(&timecounter_bad, 0); 1507 if (timecounter->tc_quality < 0) { 1508 tc_pick(); 1509 } 1510 } 1511 tc_windup(); 1512 mutex_spin_exit(&timecounter_lock); 1513 } 1514 1515 void 1516 inittimecounter(void) 1517 { 1518 u_int p; 1519 1520 mutex_init(&timecounter_lock, MUTEX_DEFAULT, IPL_HIGH); 1521 1522 /* 1523 * Set the initial timeout to 1524 * max(1, <approx. number of hardclock ticks in a millisecond>). 1525 * People should probably not use the sysctl to set the timeout 1526 * to smaller than its initial value, since that value is the 1527 * smallest reasonable one. If they want better timestamps they 1528 * should use the non-"get"* functions. 1529 */ 1530 if (hz > 1000) 1531 tc_tick = (hz + 500) / 1000; 1532 else 1533 tc_tick = 1; 1534 p = (tc_tick * 1000000) / hz; 1535 aprint_verbose("timecounter: Timecounters tick every %d.%03u msec\n", 1536 p / 1000, p % 1000); 1537 1538 /* warm up new timecounter (again) and get rolling. */ 1539 (void)timecounter->tc_get_timecount(timecounter); 1540 (void)timecounter->tc_get_timecount(timecounter); 1541 } 1542
Indexes created Fri Feb 20 16:20:20 UTC 2026