Home | History | Annotate | Line # | Download | only in kern
kern_tc.c revision 1.69
      1  1.69  riastrad /* $NetBSD: kern_tc.c,v 1.69 2023/07/17 13:48:14 riastradh Exp $ */
      2  1.33        ad 
      3  1.33        ad /*-
      4  1.39        ad  * Copyright (c) 2008, 2009 The NetBSD Foundation, Inc.
      5  1.33        ad  * All rights reserved.
      6  1.33        ad  *
      7  1.39        ad  * This code is derived from software contributed to The NetBSD Foundation
      8  1.39        ad  * by Andrew Doran.
      9  1.39        ad  *
     10  1.33        ad  * Redistribution and use in source and binary forms, with or without
     11  1.33        ad  * modification, are permitted provided that the following conditions
     12  1.33        ad  * are met:
     13  1.33        ad  * 1. Redistributions of source code must retain the above copyright
     14  1.33        ad  *    notice, this list of conditions and the following disclaimer.
     15  1.33        ad  * 2. Redistributions in binary form must reproduce the above copyright
     16  1.33        ad  *    notice, this list of conditions and the following disclaimer in the
     17  1.33        ad  *    documentation and/or other materials provided with the distribution.
     18  1.33        ad  *
     19  1.33        ad  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     20  1.33        ad  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     21  1.33        ad  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     22  1.33        ad  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     23  1.33        ad  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     24  1.33        ad  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     25  1.33        ad  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     26  1.33        ad  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     27  1.33        ad  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     28  1.33        ad  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     29  1.33        ad  * POSSIBILITY OF SUCH DAMAGE.
     30  1.33        ad  */
     31   1.2    kardel 
     32   1.1    simonb /*-
     33   1.1    simonb  * ----------------------------------------------------------------------------
     34   1.1    simonb  * "THE BEER-WARE LICENSE" (Revision 42):
     35   1.1    simonb  * <phk (at) FreeBSD.ORG> wrote this file.  As long as you retain this notice you
     36   1.1    simonb  * can do whatever you want with this stuff. If we meet some day, and you think
     37   1.1    simonb  * this stuff is worth it, you can buy me a beer in return.   Poul-Henning Kamp
     38   1.2    kardel  * ---------------------------------------------------------------------------
     39   1.1    simonb  */
     40   1.1    simonb 
     41   1.1    simonb #include <sys/cdefs.h>
     42   1.2    kardel /* __FBSDID("$FreeBSD: src/sys/kern/kern_tc.c,v 1.166 2005/09/19 22:16:31 andre Exp $"); */
     43  1.69  riastrad __KERNEL_RCSID(0, "$NetBSD: kern_tc.c,v 1.69 2023/07/17 13:48:14 riastradh Exp $");
     44  1.58       rin 
     45  1.58       rin #ifdef _KERNEL_OPT
     46  1.58       rin #include "opt_ntp.h"
     47  1.58       rin #endif
     48   1.1    simonb 
     49   1.1    simonb #include <sys/param.h>
     50  1.63  riastrad 
     51  1.61    simonb #include <sys/atomic.h>
     52  1.61    simonb #include <sys/evcnt.h>
     53  1.67  riastrad #include <sys/ipi.h>
     54  1.61    simonb #include <sys/kauth.h>
     55   1.1    simonb #include <sys/kernel.h>
     56  1.63  riastrad #include <sys/lock.h>
     57  1.61    simonb #include <sys/mutex.h>
     58   1.2    kardel #include <sys/reboot.h>	/* XXX just to get AB_VERBOSE */
     59   1.1    simonb #include <sys/sysctl.h>
     60   1.1    simonb #include <sys/syslog.h>
     61   1.1    simonb #include <sys/systm.h>
     62   1.1    simonb #include <sys/timepps.h>
     63   1.1    simonb #include <sys/timetc.h>
     64   1.1    simonb #include <sys/timex.h>
     65  1.39        ad #include <sys/xcall.h>
     66   1.2    kardel 
     67   1.2    kardel /*
     68   1.1    simonb  * A large step happens on boot.  This constant detects such steps.
     69   1.1    simonb  * It is relatively small so that ntp_update_second gets called enough
     70   1.1    simonb  * in the typical 'missed a couple of seconds' case, but doesn't loop
     71   1.1    simonb  * forever when the time step is large.
     72   1.1    simonb  */
     73   1.1    simonb #define LARGE_STEP	200
     74   1.1    simonb 
     75   1.1    simonb /*
     76   1.1    simonb  * Implement a dummy timecounter which we can use until we get a real one
     77   1.1    simonb  * in the air.  This allows the console and other early stuff to use
     78   1.1    simonb  * time services.
     79   1.1    simonb  */
     80   1.1    simonb 
     81   1.1    simonb static u_int
     82  1.16      yamt dummy_get_timecount(struct timecounter *tc)
     83   1.1    simonb {
     84   1.1    simonb 	static u_int now;
     85   1.1    simonb 
     86  1.59       rin 	return ++now;
     87   1.1    simonb }
     88   1.1    simonb 
     89   1.1    simonb static struct timecounter dummy_timecounter = {
     90  1.48  riastrad 	.tc_get_timecount	= dummy_get_timecount,
     91  1.48  riastrad 	.tc_counter_mask	= ~0u,
     92  1.48  riastrad 	.tc_frequency		= 1000000,
     93  1.48  riastrad 	.tc_name		= "dummy",
     94  1.48  riastrad 	.tc_quality		= -1000000,
     95  1.48  riastrad 	.tc_priv		= NULL,
     96   1.1    simonb };
     97   1.1    simonb 
     98   1.1    simonb struct timehands {
     99   1.1    simonb 	/* These fields must be initialized by the driver. */
    100  1.40    kardel 	struct timecounter	*th_counter;     /* active timecounter */
    101  1.40    kardel 	int64_t			th_adjustment;   /* frequency adjustment */
    102  1.40    kardel 						 /* (NTP/adjtime) */
    103  1.57       rin 	uint64_t		th_scale;        /* scale factor (counter */
    104  1.40    kardel 						 /* tick->time) */
    105  1.57       rin 	uint64_t 		th_offset_count; /* offset at last time */
    106  1.40    kardel 						 /* update (tc_windup()) */
    107  1.40    kardel 	struct bintime		th_offset;       /* bin (up)time at windup */
    108  1.40    kardel 	struct timeval		th_microtime;    /* cached microtime */
    109  1.40    kardel 	struct timespec		th_nanotime;     /* cached nanotime */
    110   1.1    simonb 	/* Fields not to be copied in tc_windup start with th_generation. */
    111  1.40    kardel 	volatile u_int		th_generation;   /* current genration */
    112  1.40    kardel 	struct timehands	*th_next;        /* next timehand */
    113   1.1    simonb };
    114   1.1    simonb 
    115   1.1    simonb static struct timehands th0;
    116  1.10  christos static struct timehands th9 = { .th_next = &th0, };
    117  1.10  christos static struct timehands th8 = { .th_next = &th9, };
    118  1.10  christos static struct timehands th7 = { .th_next = &th8, };
    119  1.10  christos static struct timehands th6 = { .th_next = &th7, };
    120  1.10  christos static struct timehands th5 = { .th_next = &th6, };
    121  1.10  christos static struct timehands th4 = { .th_next = &th5, };
    122  1.10  christos static struct timehands th3 = { .th_next = &th4, };
    123  1.10  christos static struct timehands th2 = { .th_next = &th3, };
    124  1.10  christos static struct timehands th1 = { .th_next = &th2, };
    125   1.1    simonb static struct timehands th0 = {
    126  1.10  christos 	.th_counter = &dummy_timecounter,
    127  1.10  christos 	.th_scale = (uint64_t)-1 / 1000000,
    128  1.10  christos 	.th_offset = { .sec = 1, .frac = 0 },
    129  1.10  christos 	.th_generation = 1,
    130  1.10  christos 	.th_next = &th1,
    131   1.1    simonb };
    132   1.1    simonb 
    133   1.1    simonb static struct timehands *volatile timehands = &th0;
    134   1.1    simonb struct timecounter *timecounter = &dummy_timecounter;
    135   1.1    simonb static struct timecounter *timecounters = &dummy_timecounter;
    136   1.1    simonb 
    137  1.68  riastrad #ifdef __HAVE_ATOMIC64_LOADSTORE
    138  1.69  riastrad volatile time_t time__second __cacheline_aligned = 1;
    139  1.69  riastrad volatile time_t time__uptime __cacheline_aligned = 1;
    140  1.68  riastrad #else
    141  1.63  riastrad static volatile struct {
    142  1.63  riastrad 	uint32_t lo, hi;
    143  1.63  riastrad } time__uptime32 __cacheline_aligned = {
    144  1.63  riastrad 	.lo = 1,
    145  1.63  riastrad }, time__second32 __cacheline_aligned = {
    146  1.63  riastrad 	.lo = 1,
    147  1.63  riastrad };
    148  1.63  riastrad #endif
    149   1.1    simonb 
    150   1.4    kardel static struct bintime timebasebin;
    151   1.1    simonb 
    152   1.1    simonb static int timestepwarnings;
    153   1.2    kardel 
    154  1.33        ad kmutex_t timecounter_lock;
    155  1.35        ad static u_int timecounter_mods;
    156  1.39        ad static volatile int timecounter_removals = 1;
    157  1.35        ad static u_int timecounter_bad;
    158  1.25        ad 
    159  1.63  riastrad #ifdef __HAVE_ATOMIC64_LOADSTORE
    160  1.63  riastrad 
    161  1.63  riastrad static inline void
    162  1.63  riastrad setrealuptime(time_t second, time_t uptime)
    163  1.63  riastrad {
    164  1.63  riastrad 
    165  1.63  riastrad 	atomic_store_relaxed(&time__second, second);
    166  1.63  riastrad 	atomic_store_relaxed(&time__uptime, uptime);
    167  1.63  riastrad }
    168  1.63  riastrad 
    169  1.63  riastrad #else
    170  1.63  riastrad 
    171  1.67  riastrad static void
    172  1.67  riastrad nullipi(void *cookie)
    173  1.67  riastrad {
    174  1.67  riastrad }
    175  1.67  riastrad 
    176  1.67  riastrad /*
    177  1.67  riastrad  * Issue membar_release on this CPU, and force membar_acquire on all
    178  1.67  riastrad  * CPUs.
    179  1.67  riastrad  */
    180  1.67  riastrad static void
    181  1.67  riastrad ipi_barrier(void)
    182  1.67  riastrad {
    183  1.67  riastrad 	ipi_msg_t msg = { .func = nullipi };
    184  1.67  riastrad 
    185  1.67  riastrad 	ipi_broadcast(&msg, /*skip_self*/true);
    186  1.67  riastrad 	ipi_wait(&msg);
    187  1.67  riastrad }
    188  1.67  riastrad 
    189  1.63  riastrad static inline void
    190  1.63  riastrad setrealuptime(time_t second, time_t uptime)
    191  1.63  riastrad {
    192  1.63  riastrad 	uint32_t seclo = second & 0xffffffff, sechi = second >> 32;
    193  1.63  riastrad 	uint32_t uplo = uptime & 0xffffffff, uphi = uptime >> 32;
    194  1.63  riastrad 
    195  1.63  riastrad 	KDASSERT(mutex_owned(&timecounter_lock));
    196  1.63  riastrad 
    197  1.63  riastrad 	/*
    198  1.63  riastrad 	 * Fast path -- no wraparound, just updating the low bits, so
    199  1.63  riastrad 	 * no need for seqlocked access.
    200  1.63  riastrad 	 */
    201  1.63  riastrad 	if (__predict_true(sechi == time__second32.hi) &&
    202  1.63  riastrad 	    __predict_true(uphi == time__uptime32.hi)) {
    203  1.63  riastrad 		atomic_store_relaxed(&time__second32.lo, seclo);
    204  1.63  riastrad 		atomic_store_relaxed(&time__uptime32.lo, uplo);
    205  1.63  riastrad 		return;
    206  1.63  riastrad 	}
    207  1.63  riastrad 
    208  1.63  riastrad 	atomic_store_relaxed(&time__second32.hi, 0xffffffff);
    209  1.63  riastrad 	atomic_store_relaxed(&time__uptime32.hi, 0xffffffff);
    210  1.67  riastrad 	ipi_barrier();
    211  1.63  riastrad 	atomic_store_relaxed(&time__second32.lo, seclo);
    212  1.63  riastrad 	atomic_store_relaxed(&time__uptime32.lo, uplo);
    213  1.67  riastrad 	ipi_barrier();
    214  1.63  riastrad 	atomic_store_relaxed(&time__second32.hi, sechi);
    215  1.64  riastrad 	atomic_store_relaxed(&time__uptime32.hi, uphi);
    216  1.63  riastrad }
    217  1.63  riastrad 
    218  1.63  riastrad time_t
    219  1.63  riastrad getrealtime(void)
    220  1.63  riastrad {
    221  1.63  riastrad 	uint32_t lo, hi;
    222  1.63  riastrad 
    223  1.63  riastrad 	do {
    224  1.63  riastrad 		for (;;) {
    225  1.63  riastrad 			hi = atomic_load_relaxed(&time__second32.hi);
    226  1.63  riastrad 			if (__predict_true(hi != 0xffffffff))
    227  1.63  riastrad 				break;
    228  1.63  riastrad 			SPINLOCK_BACKOFF_HOOK;
    229  1.63  riastrad 		}
    230  1.67  riastrad 		__insn_barrier();
    231  1.63  riastrad 		lo = atomic_load_relaxed(&time__second32.lo);
    232  1.67  riastrad 		__insn_barrier();
    233  1.63  riastrad 	} while (hi != atomic_load_relaxed(&time__second32.hi));
    234  1.63  riastrad 
    235  1.63  riastrad 	return ((time_t)hi << 32) | lo;
    236  1.63  riastrad }
    237  1.63  riastrad 
    238  1.63  riastrad time_t
    239  1.63  riastrad getuptime(void)
    240  1.63  riastrad {
    241  1.63  riastrad 	uint32_t lo, hi;
    242  1.63  riastrad 
    243  1.63  riastrad 	do {
    244  1.63  riastrad 		for (;;) {
    245  1.63  riastrad 			hi = atomic_load_relaxed(&time__uptime32.hi);
    246  1.63  riastrad 			if (__predict_true(hi != 0xffffffff))
    247  1.63  riastrad 				break;
    248  1.63  riastrad 			SPINLOCK_BACKOFF_HOOK;
    249  1.63  riastrad 		}
    250  1.67  riastrad 		__insn_barrier();
    251  1.63  riastrad 		lo = atomic_load_relaxed(&time__uptime32.lo);
    252  1.67  riastrad 		__insn_barrier();
    253  1.63  riastrad 	} while (hi != atomic_load_relaxed(&time__uptime32.hi));
    254  1.63  riastrad 
    255  1.63  riastrad 	return ((time_t)hi << 32) | lo;
    256  1.63  riastrad }
    257  1.63  riastrad 
    258  1.63  riastrad time_t
    259  1.63  riastrad getboottime(void)
    260  1.63  riastrad {
    261  1.63  riastrad 
    262  1.63  riastrad 	return getrealtime() - getuptime();
    263  1.63  riastrad }
    264  1.63  riastrad 
    265  1.63  riastrad uint32_t
    266  1.63  riastrad getuptime32(void)
    267  1.63  riastrad {
    268  1.63  riastrad 
    269  1.63  riastrad 	return atomic_load_relaxed(&time__uptime32.lo);
    270  1.63  riastrad }
    271  1.63  riastrad 
    272  1.63  riastrad #endif	/* !defined(__HAVE_ATOMIC64_LOADSTORE) */
    273  1.63  riastrad 
    274   1.2    kardel /*
    275  1.28      yamt  * sysctl helper routine for kern.timercounter.hardware
    276   1.2    kardel  */
    277   1.2    kardel static int
    278   1.2    kardel sysctl_kern_timecounter_hardware(SYSCTLFN_ARGS)
    279   1.2    kardel {
    280   1.2    kardel 	struct sysctlnode node;
    281   1.2    kardel 	int error;
    282   1.2    kardel 	char newname[MAX_TCNAMELEN];
    283   1.2    kardel 	struct timecounter *newtc, *tc;
    284   1.2    kardel 
    285   1.2    kardel 	tc = timecounter;
    286   1.2    kardel 
    287   1.2    kardel 	strlcpy(newname, tc->tc_name, sizeof(newname));
    288   1.2    kardel 
    289   1.2    kardel 	node = *rnode;
    290   1.2    kardel 	node.sysctl_data = newname;
    291   1.2    kardel 	node.sysctl_size = sizeof(newname);
    292   1.2    kardel 
    293   1.2    kardel 	error = sysctl_lookup(SYSCTLFN_CALL(&node));
    294   1.2    kardel 
    295   1.2    kardel 	if (error ||
    296   1.2    kardel 	    newp == NULL ||
    297   1.2    kardel 	    strncmp(newname, tc->tc_name, sizeof(newname)) == 0)
    298   1.2    kardel 		return error;
    299   1.1    simonb 
    300  1.26      elad 	if (l != NULL && (error = kauth_authorize_system(l->l_cred,
    301  1.26      elad 	    KAUTH_SYSTEM_TIME, KAUTH_REQ_SYSTEM_TIME_TIMECOUNTERS, newname,
    302  1.26      elad 	    NULL, NULL)) != 0)
    303  1.59       rin 		return error;
    304   1.2    kardel 
    305  1.22        ad 	if (!cold)
    306  1.35        ad 		mutex_spin_enter(&timecounter_lock);
    307  1.23        ad 	error = EINVAL;
    308   1.2    kardel 	for (newtc = timecounters; newtc != NULL; newtc = newtc->tc_next) {
    309   1.2    kardel 		if (strcmp(newname, newtc->tc_name) != 0)
    310   1.2    kardel 			continue;
    311   1.2    kardel 		/* Warm up new timecounter. */
    312   1.2    kardel 		(void)newtc->tc_get_timecount(newtc);
    313   1.2    kardel 		(void)newtc->tc_get_timecount(newtc);
    314   1.2    kardel 		timecounter = newtc;
    315  1.22        ad 		error = 0;
    316  1.23        ad 		break;
    317  1.23        ad 	}
    318  1.22        ad 	if (!cold)
    319  1.35        ad 		mutex_spin_exit(&timecounter_lock);
    320  1.22        ad 	return error;
    321   1.2    kardel }
    322   1.2    kardel 
    323   1.2    kardel static int
    324   1.2    kardel sysctl_kern_timecounter_choice(SYSCTLFN_ARGS)
    325   1.2    kardel {
    326   1.9    kardel 	char buf[MAX_TCNAMELEN+48];
    327  1.35        ad 	char *where;
    328   1.2    kardel 	const char *spc;
    329   1.2    kardel 	struct timecounter *tc;
    330   1.2    kardel 	size_t needed, left, slen;
    331  1.35        ad 	int error, mods;
    332   1.2    kardel 
    333   1.2    kardel 	if (newp != NULL)
    334  1.59       rin 		return EPERM;
    335   1.2    kardel 	if (namelen != 0)
    336  1.59       rin 		return EINVAL;
    337   1.2    kardel 
    338  1.35        ad 	mutex_spin_enter(&timecounter_lock);
    339  1.35        ad  retry:
    340   1.2    kardel 	spc = "";
    341   1.2    kardel 	error = 0;
    342   1.2    kardel 	needed = 0;
    343   1.2    kardel 	left = *oldlenp;
    344  1.35        ad 	where = oldp;
    345   1.2    kardel 	for (tc = timecounters; error == 0 && tc != NULL; tc = tc->tc_next) {
    346   1.2    kardel 		if (where == NULL) {
    347   1.2    kardel 			needed += sizeof(buf);  /* be conservative */
    348   1.2    kardel 		} else {
    349   1.2    kardel 			slen = snprintf(buf, sizeof(buf), "%s%s(q=%d, f=%" PRId64
    350   1.2    kardel 					" Hz)", spc, tc->tc_name, tc->tc_quality,
    351   1.2    kardel 					tc->tc_frequency);
    352   1.2    kardel 			if (left < slen + 1)
    353   1.2    kardel 				break;
    354  1.35        ad 		 	mods = timecounter_mods;
    355  1.35        ad 			mutex_spin_exit(&timecounter_lock);
    356   1.2    kardel 			error = copyout(buf, where, slen + 1);
    357  1.35        ad 			mutex_spin_enter(&timecounter_lock);
    358  1.35        ad 			if (mods != timecounter_mods) {
    359  1.35        ad 				goto retry;
    360  1.35        ad 			}
    361   1.2    kardel 			spc = " ";
    362   1.2    kardel 			where += slen;
    363   1.2    kardel 			needed += slen;
    364   1.2    kardel 			left -= slen;
    365   1.2    kardel 		}
    366   1.2    kardel 	}
    367  1.35        ad 	mutex_spin_exit(&timecounter_lock);
    368   1.2    kardel 
    369   1.2    kardel 	*oldlenp = needed;
    370  1.59       rin 	return error;
    371   1.2    kardel }
    372   1.2    kardel 
    373   1.2    kardel SYSCTL_SETUP(sysctl_timecounter_setup, "sysctl timecounter setup")
    374   1.2    kardel {
    375   1.2    kardel 	const struct sysctlnode *node;
    376   1.2    kardel 
    377   1.2    kardel 	sysctl_createv(clog, 0, NULL, &node,
    378   1.2    kardel 		       CTLFLAG_PERMANENT,
    379   1.2    kardel 		       CTLTYPE_NODE, "timecounter",
    380   1.2    kardel 		       SYSCTL_DESCR("time counter information"),
    381   1.2    kardel 		       NULL, 0, NULL, 0,
    382   1.2    kardel 		       CTL_KERN, CTL_CREATE, CTL_EOL);
    383   1.2    kardel 
    384   1.2    kardel 	if (node != NULL) {
    385   1.2    kardel 		sysctl_createv(clog, 0, NULL, NULL,
    386   1.2    kardel 			       CTLFLAG_PERMANENT,
    387   1.2    kardel 			       CTLTYPE_STRING, "choice",
    388   1.2    kardel 			       SYSCTL_DESCR("available counters"),
    389   1.2    kardel 			       sysctl_kern_timecounter_choice, 0, NULL, 0,
    390   1.2    kardel 			       CTL_KERN, node->sysctl_num, CTL_CREATE, CTL_EOL);
    391   1.2    kardel 
    392   1.2    kardel 		sysctl_createv(clog, 0, NULL, NULL,
    393   1.2    kardel 			       CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
    394   1.2    kardel 			       CTLTYPE_STRING, "hardware",
    395   1.2    kardel 			       SYSCTL_DESCR("currently active time counter"),
    396   1.2    kardel 			       sysctl_kern_timecounter_hardware, 0, NULL, MAX_TCNAMELEN,
    397   1.2    kardel 			       CTL_KERN, node->sysctl_num, CTL_CREATE, CTL_EOL);
    398   1.2    kardel 
    399   1.2    kardel 		sysctl_createv(clog, 0, NULL, NULL,
    400   1.2    kardel 			       CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
    401   1.2    kardel 			       CTLTYPE_INT, "timestepwarnings",
    402   1.2    kardel 			       SYSCTL_DESCR("log time steps"),
    403   1.2    kardel 			       NULL, 0, &timestepwarnings, 0,
    404   1.2    kardel 			       CTL_KERN, node->sysctl_num, CTL_CREATE, CTL_EOL);
    405   1.2    kardel 	}
    406   1.2    kardel }
    407   1.2    kardel 
    408  1.32        ad #ifdef TC_COUNTERS
    409   1.2    kardel #define	TC_STATS(name)							\
    410   1.2    kardel static struct evcnt n##name =						\
    411   1.2    kardel     EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "timecounter", #name);	\
    412   1.2    kardel EVCNT_ATTACH_STATIC(n##name)
    413   1.2    kardel TC_STATS(binuptime);    TC_STATS(nanouptime);    TC_STATS(microuptime);
    414   1.2    kardel TC_STATS(bintime);      TC_STATS(nanotime);      TC_STATS(microtime);
    415   1.2    kardel TC_STATS(getbinuptime); TC_STATS(getnanouptime); TC_STATS(getmicrouptime);
    416   1.2    kardel TC_STATS(getbintime);   TC_STATS(getnanotime);   TC_STATS(getmicrotime);
    417   1.2    kardel TC_STATS(setclock);
    418  1.32        ad #define	TC_COUNT(var)	var.ev_count++
    419   1.1    simonb #undef TC_STATS
    420  1.32        ad #else
    421  1.32        ad #define	TC_COUNT(var)	/* nothing */
    422  1.32        ad #endif	/* TC_COUNTERS */
    423   1.1    simonb 
    424   1.1    simonb static void tc_windup(void);
    425   1.1    simonb 
    426   1.1    simonb /*
    427   1.1    simonb  * Return the difference between the timehands' counter value now and what
    428   1.1    simonb  * was when we copied it to the timehands' offset_count.
    429   1.1    simonb  */
    430  1.41  uebayasi static inline u_int
    431   1.1    simonb tc_delta(struct timehands *th)
    432   1.1    simonb {
    433   1.1    simonb 	struct timecounter *tc;
    434   1.1    simonb 
    435   1.1    simonb 	tc = th->th_counter;
    436  1.59       rin 	return (tc->tc_get_timecount(tc) -
    437  1.59       rin 		 th->th_offset_count) & tc->tc_counter_mask;
    438   1.1    simonb }
    439   1.1    simonb 
    440   1.1    simonb /*
    441   1.1    simonb  * Functions for reading the time.  We have to loop until we are sure that
    442   1.1    simonb  * the timehands that we operated on was not updated under our feet.  See
    443  1.21    simonb  * the comment in <sys/timevar.h> for a description of these 12 functions.
    444   1.1    simonb  */
    445   1.1    simonb 
    446   1.1    simonb void
    447   1.1    simonb binuptime(struct bintime *bt)
    448   1.1    simonb {
    449   1.1    simonb 	struct timehands *th;
    450  1.39        ad 	lwp_t *l;
    451  1.39        ad 	u_int lgen, gen;
    452   1.1    simonb 
    453  1.32        ad 	TC_COUNT(nbinuptime);
    454  1.39        ad 
    455  1.39        ad 	/*
    456  1.39        ad 	 * Provide exclusion against tc_detach().
    457  1.39        ad 	 *
    458  1.39        ad 	 * We record the number of timecounter removals before accessing
    459  1.39        ad 	 * timecounter state.  Note that the LWP can be using multiple
    460  1.39        ad 	 * "generations" at once, due to interrupts (interrupted while in
    461  1.39        ad 	 * this function).  Hardware interrupts will borrow the interrupted
    462  1.39        ad 	 * LWP's l_tcgen value for this purpose, and can themselves be
    463  1.39        ad 	 * interrupted by higher priority interrupts.  In this case we need
    464  1.39        ad 	 * to ensure that the oldest generation in use is recorded.
    465  1.39        ad 	 *
    466  1.39        ad 	 * splsched() is too expensive to use, so we take care to structure
    467  1.39        ad 	 * this code in such a way that it is not required.  Likewise, we
    468  1.39        ad 	 * do not disable preemption.
    469  1.39        ad 	 *
    470  1.39        ad 	 * Memory barriers are also too expensive to use for such a
    471  1.39        ad 	 * performance critical function.  The good news is that we do not
    472  1.39        ad 	 * need memory barriers for this type of exclusion, as the thread
    473  1.39        ad 	 * updating timecounter_removals will issue a broadcast cross call
    474  1.39        ad 	 * before inspecting our l_tcgen value (this elides memory ordering
    475  1.39        ad 	 * issues).
    476  1.39        ad 	 */
    477  1.39        ad 	l = curlwp;
    478  1.39        ad 	lgen = l->l_tcgen;
    479  1.39        ad 	if (__predict_true(lgen == 0)) {
    480  1.39        ad 		l->l_tcgen = timecounter_removals;
    481  1.39        ad 	}
    482  1.39        ad 	__insn_barrier();
    483  1.39        ad 
    484   1.1    simonb 	do {
    485   1.1    simonb 		th = timehands;
    486   1.1    simonb 		gen = th->th_generation;
    487   1.1    simonb 		*bt = th->th_offset;
    488   1.1    simonb 		bintime_addx(bt, th->th_scale * tc_delta(th));
    489   1.1    simonb 	} while (gen == 0 || gen != th->th_generation);
    490  1.39        ad 
    491  1.39        ad 	__insn_barrier();
    492  1.39        ad 	l->l_tcgen = lgen;
    493   1.1    simonb }
    494   1.1    simonb 
    495   1.1    simonb void
    496   1.1    simonb nanouptime(struct timespec *tsp)
    497   1.1    simonb {
    498   1.1    simonb 	struct bintime bt;
    499   1.1    simonb 
    500  1.32        ad 	TC_COUNT(nnanouptime);
    501   1.1    simonb 	binuptime(&bt);
    502   1.1    simonb 	bintime2timespec(&bt, tsp);
    503   1.1    simonb }
    504   1.1    simonb 
    505   1.1    simonb void
    506   1.1    simonb microuptime(struct timeval *tvp)
    507   1.1    simonb {
    508   1.1    simonb 	struct bintime bt;
    509   1.1    simonb 
    510  1.32        ad 	TC_COUNT(nmicrouptime);
    511   1.1    simonb 	binuptime(&bt);
    512   1.1    simonb 	bintime2timeval(&bt, tvp);
    513   1.1    simonb }
    514   1.1    simonb 
    515   1.1    simonb void
    516   1.1    simonb bintime(struct bintime *bt)
    517   1.1    simonb {
    518   1.1    simonb 
    519  1.32        ad 	TC_COUNT(nbintime);
    520   1.1    simonb 	binuptime(bt);
    521   1.4    kardel 	bintime_add(bt, &timebasebin);
    522   1.1    simonb }
    523   1.1    simonb 
    524   1.1    simonb void
    525   1.1    simonb nanotime(struct timespec *tsp)
    526   1.1    simonb {
    527   1.1    simonb 	struct bintime bt;
    528   1.1    simonb 
    529  1.32        ad 	TC_COUNT(nnanotime);
    530   1.1    simonb 	bintime(&bt);
    531   1.1    simonb 	bintime2timespec(&bt, tsp);
    532   1.1    simonb }
    533   1.1    simonb 
    534   1.1    simonb void
    535   1.1    simonb microtime(struct timeval *tvp)
    536   1.1    simonb {
    537   1.1    simonb 	struct bintime bt;
    538   1.1    simonb 
    539  1.32        ad 	TC_COUNT(nmicrotime);
    540   1.1    simonb 	bintime(&bt);
    541   1.1    simonb 	bintime2timeval(&bt, tvp);
    542   1.1    simonb }
    543   1.1    simonb 
    544   1.1    simonb void
    545   1.1    simonb getbinuptime(struct bintime *bt)
    546   1.1    simonb {
    547   1.1    simonb 	struct timehands *th;
    548   1.1    simonb 	u_int gen;
    549   1.1    simonb 
    550  1.32        ad 	TC_COUNT(ngetbinuptime);
    551   1.1    simonb 	do {
    552   1.1    simonb 		th = timehands;
    553   1.1    simonb 		gen = th->th_generation;
    554   1.1    simonb 		*bt = th->th_offset;
    555   1.1    simonb 	} while (gen == 0 || gen != th->th_generation);
    556   1.1    simonb }
    557   1.1    simonb 
    558   1.1    simonb void
    559   1.1    simonb getnanouptime(struct timespec *tsp)
    560   1.1    simonb {
    561   1.1    simonb 	struct timehands *th;
    562   1.1    simonb 	u_int gen;
    563   1.1    simonb 
    564  1.32        ad 	TC_COUNT(ngetnanouptime);
    565   1.1    simonb 	do {
    566   1.1    simonb 		th = timehands;
    567   1.1    simonb 		gen = th->th_generation;
    568   1.1    simonb 		bintime2timespec(&th->th_offset, tsp);
    569   1.1    simonb 	} while (gen == 0 || gen != th->th_generation);
    570   1.1    simonb }
    571   1.1    simonb 
    572   1.1    simonb void
    573   1.1    simonb getmicrouptime(struct timeval *tvp)
    574   1.1    simonb {
    575   1.1    simonb 	struct timehands *th;
    576   1.1    simonb 	u_int gen;
    577   1.1    simonb 
    578  1.32        ad 	TC_COUNT(ngetmicrouptime);
    579   1.1    simonb 	do {
    580   1.1    simonb 		th = timehands;
    581   1.1    simonb 		gen = th->th_generation;
    582   1.1    simonb 		bintime2timeval(&th->th_offset, tvp);
    583   1.1    simonb 	} while (gen == 0 || gen != th->th_generation);
    584   1.1    simonb }
    585   1.1    simonb 
    586   1.1    simonb void
    587   1.1    simonb getbintime(struct bintime *bt)
    588   1.1    simonb {
    589   1.1    simonb 	struct timehands *th;
    590   1.1    simonb 	u_int gen;
    591   1.1    simonb 
    592  1.32        ad 	TC_COUNT(ngetbintime);
    593   1.1    simonb 	do {
    594   1.1    simonb 		th = timehands;
    595   1.1    simonb 		gen = th->th_generation;
    596   1.1    simonb 		*bt = th->th_offset;
    597   1.1    simonb 	} while (gen == 0 || gen != th->th_generation);
    598   1.4    kardel 	bintime_add(bt, &timebasebin);
    599   1.1    simonb }
    600   1.1    simonb 
    601  1.47       chs static inline void
    602  1.47       chs dogetnanotime(struct timespec *tsp)
    603   1.1    simonb {
    604   1.1    simonb 	struct timehands *th;
    605   1.1    simonb 	u_int gen;
    606   1.1    simonb 
    607  1.32        ad 	TC_COUNT(ngetnanotime);
    608   1.1    simonb 	do {
    609   1.1    simonb 		th = timehands;
    610   1.1    simonb 		gen = th->th_generation;
    611   1.1    simonb 		*tsp = th->th_nanotime;
    612   1.1    simonb 	} while (gen == 0 || gen != th->th_generation);
    613   1.1    simonb }
    614   1.1    simonb 
    615   1.1    simonb void
    616  1.47       chs getnanotime(struct timespec *tsp)
    617  1.47       chs {
    618  1.47       chs 
    619  1.47       chs 	dogetnanotime(tsp);
    620  1.47       chs }
    621  1.47       chs 
    622  1.47       chs void dtrace_getnanotime(struct timespec *tsp);
    623  1.47       chs 
    624  1.47       chs void
    625  1.47       chs dtrace_getnanotime(struct timespec *tsp)
    626  1.47       chs {
    627  1.47       chs 
    628  1.47       chs 	dogetnanotime(tsp);
    629  1.47       chs }
    630  1.47       chs 
    631  1.47       chs void
    632   1.1    simonb getmicrotime(struct timeval *tvp)
    633   1.1    simonb {
    634   1.1    simonb 	struct timehands *th;
    635   1.1    simonb 	u_int gen;
    636   1.1    simonb 
    637  1.32        ad 	TC_COUNT(ngetmicrotime);
    638   1.1    simonb 	do {
    639   1.1    simonb 		th = timehands;
    640   1.1    simonb 		gen = th->th_generation;
    641   1.1    simonb 		*tvp = th->th_microtime;
    642   1.1    simonb 	} while (gen == 0 || gen != th->th_generation);
    643   1.1    simonb }
    644   1.1    simonb 
    645  1.54   thorpej void
    646  1.54   thorpej getnanoboottime(struct timespec *tsp)
    647  1.54   thorpej {
    648  1.54   thorpej 	struct bintime bt;
    649  1.54   thorpej 
    650  1.54   thorpej 	getbinboottime(&bt);
    651  1.54   thorpej 	bintime2timespec(&bt, tsp);
    652  1.54   thorpej }
    653  1.54   thorpej 
    654  1.54   thorpej void
    655  1.54   thorpej getmicroboottime(struct timeval *tvp)
    656  1.54   thorpej {
    657  1.54   thorpej 	struct bintime bt;
    658  1.54   thorpej 
    659  1.54   thorpej 	getbinboottime(&bt);
    660  1.54   thorpej 	bintime2timeval(&bt, tvp);
    661  1.54   thorpej }
    662  1.54   thorpej 
    663  1.54   thorpej void
    664  1.54   thorpej getbinboottime(struct bintime *bt)
    665  1.54   thorpej {
    666  1.54   thorpej 
    667  1.54   thorpej 	/*
    668  1.54   thorpej 	 * XXX Need lockless read synchronization around timebasebin
    669  1.54   thorpej 	 * (and not just here).
    670  1.54   thorpej 	 */
    671  1.54   thorpej 	*bt = timebasebin;
    672  1.54   thorpej }
    673  1.54   thorpej 
    674   1.1    simonb /*
    675   1.1    simonb  * Initialize a new timecounter and possibly use it.
    676   1.1    simonb  */
    677   1.1    simonb void
    678   1.1    simonb tc_init(struct timecounter *tc)
    679   1.1    simonb {
    680   1.1    simonb 	u_int u;
    681   1.1    simonb 
    682  1.60    simonb 	KASSERTMSG(tc->tc_next == NULL, "timecounter %s already initialised",
    683  1.60    simonb 	    tc->tc_name);
    684  1.60    simonb 
    685   1.1    simonb 	u = tc->tc_frequency / tc->tc_counter_mask;
    686   1.1    simonb 	/* XXX: We need some margin here, 10% is a guess */
    687   1.1    simonb 	u *= 11;
    688   1.1    simonb 	u /= 10;
    689   1.1    simonb 	if (u > hz && tc->tc_quality >= 0) {
    690   1.1    simonb 		tc->tc_quality = -2000;
    691  1.18        ad 		aprint_verbose(
    692  1.18        ad 		    "timecounter: Timecounter \"%s\" frequency %ju Hz",
    693   1.7     bjh21 			    tc->tc_name, (uintmax_t)tc->tc_frequency);
    694  1.18        ad 		aprint_verbose(" -- Insufficient hz, needs at least %u\n", u);
    695   1.1    simonb 	} else if (tc->tc_quality >= 0 || bootverbose) {
    696  1.18        ad 		aprint_verbose(
    697  1.18        ad 		    "timecounter: Timecounter \"%s\" frequency %ju Hz "
    698  1.18        ad 		    "quality %d\n", tc->tc_name, (uintmax_t)tc->tc_frequency,
    699   1.7     bjh21 		    tc->tc_quality);
    700   1.1    simonb 	}
    701   1.1    simonb 
    702  1.33        ad 	mutex_spin_enter(&timecounter_lock);
    703   1.1    simonb 	tc->tc_next = timecounters;
    704   1.1    simonb 	timecounters = tc;
    705  1.35        ad 	timecounter_mods++;
    706   1.1    simonb 	/*
    707   1.1    simonb 	 * Never automatically use a timecounter with negative quality.
    708   1.1    simonb 	 * Even though we run on the dummy counter, switching here may be
    709   1.1    simonb 	 * worse since this timecounter may not be monotonous.
    710   1.1    simonb 	 */
    711  1.22        ad 	if (tc->tc_quality >= 0 && (tc->tc_quality > timecounter->tc_quality ||
    712  1.24        ad 	    (tc->tc_quality == timecounter->tc_quality &&
    713  1.24        ad 	    tc->tc_frequency > timecounter->tc_frequency))) {
    714  1.22        ad 		(void)tc->tc_get_timecount(tc);
    715  1.22        ad 		(void)tc->tc_get_timecount(tc);
    716  1.22        ad 		timecounter = tc;
    717  1.22        ad 		tc_windup();
    718  1.22        ad 	}
    719  1.33        ad 	mutex_spin_exit(&timecounter_lock);
    720  1.35        ad }
    721  1.35        ad 
    722  1.35        ad /*
    723  1.35        ad  * Pick a new timecounter due to the existing counter going bad.
    724  1.35        ad  */
    725  1.35        ad static void
    726  1.35        ad tc_pick(void)
    727  1.35        ad {
    728  1.35        ad 	struct timecounter *best, *tc;
    729  1.35        ad 
    730  1.51  riastrad 	KASSERT(mutex_owned(&timecounter_lock));
    731  1.35        ad 
    732  1.35        ad 	for (best = tc = timecounters; tc != NULL; tc = tc->tc_next) {
    733  1.35        ad 		if (tc->tc_quality > best->tc_quality)
    734  1.35        ad 			best = tc;
    735  1.35        ad 		else if (tc->tc_quality < best->tc_quality)
    736  1.35        ad 			continue;
    737  1.35        ad 		else if (tc->tc_frequency > best->tc_frequency)
    738  1.35        ad 			best = tc;
    739  1.35        ad 	}
    740  1.35        ad 	(void)best->tc_get_timecount(best);
    741  1.35        ad 	(void)best->tc_get_timecount(best);
    742  1.35        ad 	timecounter = best;
    743  1.35        ad }
    744  1.35        ad 
    745  1.35        ad /*
    746  1.35        ad  * A timecounter has gone bad, arrange to pick a new one at the next
    747  1.35        ad  * clock tick.
    748  1.35        ad  */
    749  1.35        ad void
    750  1.35        ad tc_gonebad(struct timecounter *tc)
    751  1.35        ad {
    752  1.35        ad 
    753  1.35        ad 	tc->tc_quality = -100;
    754  1.35        ad 	membar_producer();
    755  1.35        ad 	atomic_inc_uint(&timecounter_bad);
    756   1.1    simonb }
    757   1.1    simonb 
    758  1.29    dyoung /*
    759  1.29    dyoung  * Stop using a timecounter and remove it from the timecounters list.
    760  1.29    dyoung  */
    761  1.29    dyoung int
    762  1.29    dyoung tc_detach(struct timecounter *target)
    763  1.29    dyoung {
    764  1.35        ad 	struct timecounter *tc;
    765  1.29    dyoung 	struct timecounter **tcp = NULL;
    766  1.39        ad 	int removals;
    767  1.39        ad 	lwp_t *l;
    768  1.29    dyoung 
    769  1.39        ad 	/* First, find the timecounter. */
    770  1.35        ad 	mutex_spin_enter(&timecounter_lock);
    771  1.29    dyoung 	for (tcp = &timecounters, tc = timecounters;
    772  1.29    dyoung 	     tc != NULL;
    773  1.29    dyoung 	     tcp = &tc->tc_next, tc = tc->tc_next) {
    774  1.29    dyoung 		if (tc == target)
    775  1.29    dyoung 			break;
    776  1.29    dyoung 	}
    777  1.29    dyoung 	if (tc == NULL) {
    778  1.39        ad 		mutex_spin_exit(&timecounter_lock);
    779  1.39        ad 		return ESRCH;
    780  1.39        ad 	}
    781  1.39        ad 
    782  1.39        ad 	/* And now, remove it. */
    783  1.39        ad 	*tcp = tc->tc_next;
    784  1.39        ad 	if (timecounter == target) {
    785  1.39        ad 		tc_pick();
    786  1.39        ad 		tc_windup();
    787  1.39        ad 	}
    788  1.39        ad 	timecounter_mods++;
    789  1.39        ad 	removals = timecounter_removals++;
    790  1.39        ad 	mutex_spin_exit(&timecounter_lock);
    791  1.39        ad 
    792  1.39        ad 	/*
    793  1.39        ad 	 * We now have to determine if any threads in the system are still
    794  1.39        ad 	 * making use of this timecounter.
    795  1.39        ad 	 *
    796  1.39        ad 	 * We issue a broadcast cross call to elide memory ordering issues,
    797  1.39        ad 	 * then scan all LWPs in the system looking at each's timecounter
    798  1.39        ad 	 * generation number.  We need to see a value of zero (not actively
    799  1.39        ad 	 * using a timecounter) or a value greater than our removal value.
    800  1.39        ad 	 *
    801  1.39        ad 	 * We may race with threads that read `timecounter_removals' and
    802  1.39        ad 	 * and then get preempted before updating `l_tcgen'.  This is not
    803  1.39        ad 	 * a problem, since it means that these threads have not yet started
    804  1.39        ad 	 * accessing timecounter state.  All we do need is one clean
    805  1.39        ad 	 * snapshot of the system where every thread appears not to be using
    806  1.39        ad 	 * old timecounter state.
    807  1.39        ad 	 */
    808  1.39        ad 	for (;;) {
    809  1.52       uwe 		xc_barrier(0);
    810  1.39        ad 
    811  1.55        ad 		mutex_enter(&proc_lock);
    812  1.39        ad 		LIST_FOREACH(l, &alllwp, l_list) {
    813  1.39        ad 			if (l->l_tcgen == 0 || l->l_tcgen > removals) {
    814  1.39        ad 				/*
    815  1.39        ad 				 * Not using timecounter or old timecounter
    816  1.39        ad 				 * state at time of our xcall or later.
    817  1.39        ad 				 */
    818  1.39        ad 				continue;
    819  1.39        ad 			}
    820  1.39        ad 			break;
    821  1.39        ad 		}
    822  1.55        ad 		mutex_exit(&proc_lock);
    823  1.39        ad 
    824  1.39        ad 		/*
    825  1.39        ad 		 * If the timecounter is still in use, wait at least 10ms
    826  1.39        ad 		 * before retrying.
    827  1.39        ad 		 */
    828  1.39        ad 		if (l == NULL) {
    829  1.62  riastrad 			break;
    830  1.35        ad 		}
    831  1.39        ad 		(void)kpause("tcdetach", false, mstohz(10), NULL);
    832  1.29    dyoung 	}
    833  1.62  riastrad 
    834  1.62  riastrad 	tc->tc_next = NULL;
    835  1.62  riastrad 	return 0;
    836  1.29    dyoung }
    837  1.29    dyoung 
    838   1.1    simonb /* Report the frequency of the current timecounter. */
    839  1.57       rin uint64_t
    840   1.1    simonb tc_getfrequency(void)
    841   1.1    simonb {
    842   1.1    simonb 
    843  1.59       rin 	return timehands->th_counter->tc_frequency;
    844   1.1    simonb }
    845   1.1    simonb 
    846   1.1    simonb /*
    847   1.1    simonb  * Step our concept of UTC.  This is done by modifying our estimate of
    848   1.1    simonb  * when we booted.
    849   1.1    simonb  */
    850   1.1    simonb void
    851  1.38  christos tc_setclock(const struct timespec *ts)
    852   1.1    simonb {
    853   1.1    simonb 	struct timespec ts2;
    854   1.1    simonb 	struct bintime bt, bt2;
    855   1.1    simonb 
    856  1.33        ad 	mutex_spin_enter(&timecounter_lock);
    857  1.32        ad 	TC_COUNT(nsetclock);
    858   1.1    simonb 	binuptime(&bt2);
    859   1.1    simonb 	timespec2bintime(ts, &bt);
    860   1.1    simonb 	bintime_sub(&bt, &bt2);
    861   1.4    kardel 	bintime_add(&bt2, &timebasebin);
    862   1.4    kardel 	timebasebin = bt;
    863  1.30        ad 	tc_windup();
    864  1.33        ad 	mutex_spin_exit(&timecounter_lock);
    865   1.1    simonb 
    866   1.1    simonb 	if (timestepwarnings) {
    867   1.1    simonb 		bintime2timespec(&bt2, &ts2);
    868  1.45    kardel 		log(LOG_INFO,
    869  1.45    kardel 		    "Time stepped from %lld.%09ld to %lld.%09ld\n",
    870  1.38  christos 		    (long long)ts2.tv_sec, ts2.tv_nsec,
    871  1.38  christos 		    (long long)ts->tv_sec, ts->tv_nsec);
    872   1.1    simonb 	}
    873   1.1    simonb }
    874   1.1    simonb 
    875   1.1    simonb /*
    876   1.1    simonb  * Initialize the next struct timehands in the ring and make
    877   1.1    simonb  * it the active timehands.  Along the way we might switch to a different
    878   1.1    simonb  * timecounter and/or do seconds processing in NTP.  Slightly magic.
    879   1.1    simonb  */
    880   1.1    simonb static void
    881   1.1    simonb tc_windup(void)
    882   1.1    simonb {
    883   1.1    simonb 	struct bintime bt;
    884   1.1    simonb 	struct timehands *th, *tho;
    885  1.57       rin 	uint64_t scale;
    886   1.1    simonb 	u_int delta, ncount, ogen;
    887  1.13    kardel 	int i, s_update;
    888   1.1    simonb 	time_t t;
    889   1.1    simonb 
    890  1.51  riastrad 	KASSERT(mutex_owned(&timecounter_lock));
    891  1.30        ad 
    892  1.13    kardel 	s_update = 0;
    893  1.20        ad 
    894   1.1    simonb 	/*
    895   1.1    simonb 	 * Make the next timehands a copy of the current one, but do not
    896   1.1    simonb 	 * overwrite the generation or next pointer.  While we update
    897  1.20        ad 	 * the contents, the generation must be zero.  Ensure global
    898  1.20        ad 	 * visibility of the generation before proceeding.
    899   1.1    simonb 	 */
    900   1.1    simonb 	tho = timehands;
    901   1.1    simonb 	th = tho->th_next;
    902   1.1    simonb 	ogen = th->th_generation;
    903   1.1    simonb 	th->th_generation = 0;
    904  1.27        ad 	membar_producer();
    905   1.1    simonb 	bcopy(tho, th, offsetof(struct timehands, th_generation));
    906   1.1    simonb 
    907   1.1    simonb 	/*
    908   1.1    simonb 	 * Capture a timecounter delta on the current timecounter and if
    909   1.1    simonb 	 * changing timecounters, a counter value from the new timecounter.
    910   1.1    simonb 	 * Update the offset fields accordingly.
    911   1.1    simonb 	 */
    912   1.1    simonb 	delta = tc_delta(th);
    913   1.1    simonb 	if (th->th_counter != timecounter)
    914   1.1    simonb 		ncount = timecounter->tc_get_timecount(timecounter);
    915   1.1    simonb 	else
    916   1.1    simonb 		ncount = 0;
    917   1.1    simonb 	th->th_offset_count += delta;
    918   1.1    simonb 	bintime_addx(&th->th_offset, th->th_scale * delta);
    919   1.1    simonb 
    920   1.1    simonb 	/*
    921   1.1    simonb 	 * Hardware latching timecounters may not generate interrupts on
    922   1.1    simonb 	 * PPS events, so instead we poll them.  There is a finite risk that
    923   1.1    simonb 	 * the hardware might capture a count which is later than the one we
    924   1.1    simonb 	 * got above, and therefore possibly in the next NTP second which might
    925   1.1    simonb 	 * have a different rate than the current NTP second.  It doesn't
    926   1.1    simonb 	 * matter in practice.
    927   1.1    simonb 	 */
    928   1.1    simonb 	if (tho->th_counter->tc_poll_pps)
    929   1.1    simonb 		tho->th_counter->tc_poll_pps(tho->th_counter);
    930   1.1    simonb 
    931   1.1    simonb 	/*
    932   1.1    simonb 	 * Deal with NTP second processing.  The for loop normally
    933   1.1    simonb 	 * iterates at most once, but in extreme situations it might
    934   1.1    simonb 	 * keep NTP sane if timeouts are not run for several seconds.
    935   1.1    simonb 	 * At boot, the time step can be large when the TOD hardware
    936   1.1    simonb 	 * has been read, so on really large steps, we call
    937   1.1    simonb 	 * ntp_update_second only twice.  We need to call it twice in
    938   1.1    simonb 	 * case we missed a leap second.
    939   1.2    kardel 	 * If NTP is not compiled in ntp_update_second still calculates
    940   1.2    kardel 	 * the adjustment resulting from adjtime() calls.
    941   1.1    simonb 	 */
    942   1.1    simonb 	bt = th->th_offset;
    943   1.4    kardel 	bintime_add(&bt, &timebasebin);
    944   1.1    simonb 	i = bt.sec - tho->th_microtime.tv_sec;
    945   1.1    simonb 	if (i > LARGE_STEP)
    946   1.1    simonb 		i = 2;
    947   1.1    simonb 	for (; i > 0; i--) {
    948   1.1    simonb 		t = bt.sec;
    949   1.1    simonb 		ntp_update_second(&th->th_adjustment, &bt.sec);
    950  1.13    kardel 		s_update = 1;
    951   1.1    simonb 		if (bt.sec != t)
    952   1.4    kardel 			timebasebin.sec += bt.sec - t;
    953   1.1    simonb 	}
    954   1.2    kardel 
    955   1.1    simonb 	/* Update the UTC timestamps used by the get*() functions. */
    956   1.1    simonb 	/* XXX shouldn't do this here.  Should force non-`get' versions. */
    957   1.1    simonb 	bintime2timeval(&bt, &th->th_microtime);
    958   1.1    simonb 	bintime2timespec(&bt, &th->th_nanotime);
    959   1.1    simonb 	/* Now is a good time to change timecounters. */
    960   1.1    simonb 	if (th->th_counter != timecounter) {
    961   1.1    simonb 		th->th_counter = timecounter;
    962   1.1    simonb 		th->th_offset_count = ncount;
    963  1.13    kardel 		s_update = 1;
    964   1.1    simonb 	}
    965   1.1    simonb 
    966   1.1    simonb 	/*-
    967   1.1    simonb 	 * Recalculate the scaling factor.  We want the number of 1/2^64
    968   1.1    simonb 	 * fractions of a second per period of the hardware counter, taking
    969   1.1    simonb 	 * into account the th_adjustment factor which the NTP PLL/adjtime(2)
    970   1.1    simonb 	 * processing provides us with.
    971   1.1    simonb 	 *
    972   1.1    simonb 	 * The th_adjustment is nanoseconds per second with 32 bit binary
    973   1.1    simonb 	 * fraction and we want 64 bit binary fraction of second:
    974   1.1    simonb 	 *
    975   1.1    simonb 	 *	 x = a * 2^32 / 10^9 = a * 4.294967296
    976   1.1    simonb 	 *
    977   1.1    simonb 	 * The range of th_adjustment is +/- 5000PPM so inside a 64bit int
    978   1.1    simonb 	 * we can only multiply by about 850 without overflowing, but that
    979   1.1    simonb 	 * leaves suitably precise fractions for multiply before divide.
    980   1.1    simonb 	 *
    981   1.1    simonb 	 * Divide before multiply with a fraction of 2199/512 results in a
    982   1.1    simonb 	 * systematic undercompensation of 10PPM of th_adjustment.  On a
    983   1.1    simonb 	 * 5000PPM adjustment this is a 0.05PPM error.  This is acceptable.
    984   1.1    simonb  	 *
    985   1.1    simonb 	 * We happily sacrifice the lowest of the 64 bits of our result
    986   1.1    simonb 	 * to the goddess of code clarity.
    987   1.1    simonb 	 *
    988   1.1    simonb 	 */
    989  1.13    kardel 	if (s_update) {
    990  1.57       rin 		scale = (uint64_t)1 << 63;
    991  1.13    kardel 		scale += (th->th_adjustment / 1024) * 2199;
    992  1.13    kardel 		scale /= th->th_counter->tc_frequency;
    993  1.13    kardel 		th->th_scale = scale * 2;
    994  1.13    kardel 	}
    995   1.1    simonb 	/*
    996   1.1    simonb 	 * Now that the struct timehands is again consistent, set the new
    997  1.20        ad 	 * generation number, making sure to not make it zero.  Ensure
    998  1.20        ad 	 * changes are globally visible before changing.
    999   1.1    simonb 	 */
   1000   1.1    simonb 	if (++ogen == 0)
   1001   1.1    simonb 		ogen = 1;
   1002  1.27        ad 	membar_producer();
   1003   1.1    simonb 	th->th_generation = ogen;
   1004   1.1    simonb 
   1005  1.20        ad 	/*
   1006  1.20        ad 	 * Go live with the new struct timehands.  Ensure changes are
   1007  1.20        ad 	 * globally visible before changing.
   1008  1.20        ad 	 */
   1009  1.63  riastrad 	setrealuptime(th->th_microtime.tv_sec, th->th_offset.sec);
   1010  1.27        ad 	membar_producer();
   1011   1.1    simonb 	timehands = th;
   1012  1.24        ad 
   1013  1.24        ad 	/*
   1014  1.24        ad 	 * Force users of the old timehand to move on.  This is
   1015  1.24        ad 	 * necessary for MP systems; we need to ensure that the
   1016  1.24        ad 	 * consumers will move away from the old timehand before
   1017  1.24        ad 	 * we begin updating it again when we eventually wrap
   1018  1.24        ad 	 * around.
   1019  1.24        ad 	 */
   1020  1.24        ad 	if (++tho->th_generation == 0)
   1021  1.24        ad 		tho->th_generation = 1;
   1022   1.1    simonb }
   1023   1.1    simonb 
   1024   1.1    simonb /*
   1025   1.1    simonb  * RFC 2783 PPS-API implementation.
   1026   1.1    simonb  */
   1027   1.1    simonb 
   1028   1.1    simonb int
   1029  1.19  christos pps_ioctl(u_long cmd, void *data, struct pps_state *pps)
   1030   1.1    simonb {
   1031   1.1    simonb 	pps_params_t *app;
   1032   1.2    kardel 	pps_info_t *pipi;
   1033   1.1    simonb #ifdef PPS_SYNC
   1034   1.2    kardel 	int *epi;
   1035   1.1    simonb #endif
   1036   1.1    simonb 
   1037  1.33        ad 	KASSERT(mutex_owned(&timecounter_lock));
   1038  1.33        ad 
   1039  1.45    kardel 	KASSERT(pps != NULL);
   1040  1.45    kardel 
   1041   1.1    simonb 	switch (cmd) {
   1042   1.1    simonb 	case PPS_IOC_CREATE:
   1043  1.59       rin 		return 0;
   1044   1.1    simonb 	case PPS_IOC_DESTROY:
   1045  1.59       rin 		return 0;
   1046   1.1    simonb 	case PPS_IOC_SETPARAMS:
   1047   1.1    simonb 		app = (pps_params_t *)data;
   1048   1.1    simonb 		if (app->mode & ~pps->ppscap)
   1049  1.59       rin 			return EINVAL;
   1050   1.1    simonb 		pps->ppsparam = *app;
   1051  1.59       rin 		return 0;
   1052   1.1    simonb 	case PPS_IOC_GETPARAMS:
   1053   1.1    simonb 		app = (pps_params_t *)data;
   1054   1.1    simonb 		*app = pps->ppsparam;
   1055   1.1    simonb 		app->api_version = PPS_API_VERS_1;
   1056  1.59       rin 		return 0;
   1057   1.1    simonb 	case PPS_IOC_GETCAP:
   1058   1.1    simonb 		*(int*)data = pps->ppscap;
   1059  1.59       rin 		return 0;
   1060   1.1    simonb 	case PPS_IOC_FETCH:
   1061   1.2    kardel 		pipi = (pps_info_t *)data;
   1062   1.1    simonb 		pps->ppsinfo.current_mode = pps->ppsparam.mode;
   1063   1.2    kardel 		*pipi = pps->ppsinfo;
   1064  1.59       rin 		return 0;
   1065   1.1    simonb 	case PPS_IOC_KCBIND:
   1066   1.1    simonb #ifdef PPS_SYNC
   1067   1.2    kardel 		epi = (int *)data;
   1068   1.1    simonb 		/* XXX Only root should be able to do this */
   1069   1.2    kardel 		if (*epi & ~pps->ppscap)
   1070  1.59       rin 			return EINVAL;
   1071   1.2    kardel 		pps->kcmode = *epi;
   1072  1.59       rin 		return 0;
   1073   1.1    simonb #else
   1074  1.59       rin 		return EOPNOTSUPP;
   1075   1.1    simonb #endif
   1076   1.1    simonb 	default:
   1077  1.59       rin 		return EPASSTHROUGH;
   1078   1.1    simonb 	}
   1079   1.1    simonb }
   1080   1.1    simonb 
   1081   1.1    simonb void
   1082   1.1    simonb pps_init(struct pps_state *pps)
   1083   1.1    simonb {
   1084  1.33        ad 
   1085  1.33        ad 	KASSERT(mutex_owned(&timecounter_lock));
   1086  1.33        ad 
   1087   1.1    simonb 	pps->ppscap |= PPS_TSFMT_TSPEC;
   1088   1.1    simonb 	if (pps->ppscap & PPS_CAPTUREASSERT)
   1089   1.1    simonb 		pps->ppscap |= PPS_OFFSETASSERT;
   1090   1.1    simonb 	if (pps->ppscap & PPS_CAPTURECLEAR)
   1091   1.1    simonb 		pps->ppscap |= PPS_OFFSETCLEAR;
   1092   1.1    simonb }
   1093   1.1    simonb 
   1094  1.45    kardel /*
   1095  1.45    kardel  * capture a timetamp in the pps structure
   1096  1.45    kardel  */
   1097   1.1    simonb void
   1098   1.1    simonb pps_capture(struct pps_state *pps)
   1099   1.1    simonb {
   1100   1.1    simonb 	struct timehands *th;
   1101   1.1    simonb 
   1102  1.33        ad 	KASSERT(mutex_owned(&timecounter_lock));
   1103  1.33        ad 	KASSERT(pps != NULL);
   1104  1.33        ad 
   1105   1.1    simonb 	th = timehands;
   1106   1.1    simonb 	pps->capgen = th->th_generation;
   1107   1.1    simonb 	pps->capth = th;
   1108  1.57       rin 	pps->capcount = (uint64_t)tc_delta(th) + th->th_offset_count;
   1109   1.1    simonb 	if (pps->capgen != th->th_generation)
   1110   1.1    simonb 		pps->capgen = 0;
   1111   1.1    simonb }
   1112   1.1    simonb 
   1113  1.45    kardel #ifdef PPS_DEBUG
   1114  1.45    kardel int ppsdebug = 0;
   1115  1.45    kardel #endif
   1116  1.45    kardel 
   1117  1.45    kardel /*
   1118  1.45    kardel  * process a pps_capture()ed event
   1119  1.45    kardel  */
   1120   1.1    simonb void
   1121   1.1    simonb pps_event(struct pps_state *pps, int event)
   1122   1.1    simonb {
   1123  1.45    kardel 	pps_ref_event(pps, event, NULL, PPS_REFEVNT_PPS|PPS_REFEVNT_CAPTURE);
   1124  1.45    kardel }
   1125  1.45    kardel 
   1126  1.45    kardel /*
   1127  1.45    kardel  * extended pps api /  kernel pll/fll entry point
   1128  1.45    kardel  *
   1129  1.45    kardel  * feed reference time stamps to PPS engine
   1130  1.45    kardel  *
   1131  1.45    kardel  * will simulate a PPS event and feed
   1132  1.45    kardel  * the NTP PLL/FLL if requested.
   1133  1.45    kardel  *
   1134  1.45    kardel  * the ref time stamps should be roughly once
   1135  1.45    kardel  * a second but do not need to be exactly in phase
   1136  1.45    kardel  * with the UTC second but should be close to it.
   1137  1.45    kardel  * this relaxation of requirements allows callout
   1138  1.45    kardel  * driven timestamping mechanisms to feed to pps
   1139  1.45    kardel  * capture/kernel pll logic.
   1140  1.45    kardel  *
   1141  1.45    kardel  * calling pattern is:
   1142  1.45    kardel  *  pps_capture() (for PPS_REFEVNT_{CAPTURE|CAPCUR})
   1143  1.45    kardel  *  read timestamp from reference source
   1144  1.45    kardel  *  pps_ref_event()
   1145  1.45    kardel  *
   1146  1.45    kardel  * supported refmodes:
   1147  1.45    kardel  *  PPS_REFEVNT_CAPTURE
   1148  1.45    kardel  *    use system timestamp of pps_capture()
   1149  1.45    kardel  *  PPS_REFEVNT_CURRENT
   1150  1.45    kardel  *    use system timestamp of this call
   1151  1.45    kardel  *  PPS_REFEVNT_CAPCUR
   1152  1.45    kardel  *    use average of read capture and current system time stamp
   1153  1.45    kardel  *  PPS_REFEVNT_PPS
   1154  1.45    kardel  *    assume timestamp on second mark - ref_ts is ignored
   1155  1.45    kardel  *
   1156  1.45    kardel  */
   1157  1.45    kardel 
   1158  1.45    kardel void
   1159  1.45    kardel pps_ref_event(struct pps_state *pps,
   1160  1.45    kardel 	      int event,
   1161  1.45    kardel 	      struct bintime *ref_ts,
   1162  1.45    kardel 	      int refmode
   1163  1.45    kardel 	)
   1164  1.45    kardel {
   1165  1.45    kardel 	struct bintime bt;	/* current time */
   1166  1.45    kardel 	struct bintime btd;	/* time difference */
   1167  1.45    kardel 	struct bintime bt_ref;	/* reference time */
   1168   1.1    simonb 	struct timespec ts, *tsp, *osp;
   1169  1.45    kardel 	struct timehands *th;
   1170  1.57       rin 	uint64_t tcount, acount, dcount, *pcount;
   1171  1.46    martin 	int foff, gen;
   1172  1.46    martin #ifdef PPS_SYNC
   1173  1.46    martin 	int fhard;
   1174  1.46    martin #endif
   1175   1.1    simonb 	pps_seq_t *pseq;
   1176   1.1    simonb 
   1177  1.33        ad 	KASSERT(mutex_owned(&timecounter_lock));
   1178  1.33        ad 
   1179  1.45    kardel 	KASSERT(pps != NULL);
   1180  1.45    kardel 
   1181  1.45    kardel         /* pick up current time stamp if needed */
   1182  1.45    kardel 	if (refmode & (PPS_REFEVNT_CURRENT|PPS_REFEVNT_CAPCUR)) {
   1183  1.45    kardel 		/* pick up current time stamp */
   1184  1.45    kardel 		th = timehands;
   1185  1.45    kardel 		gen = th->th_generation;
   1186  1.57       rin 		tcount = (uint64_t)tc_delta(th) + th->th_offset_count;
   1187  1.45    kardel 		if (gen != th->th_generation)
   1188  1.45    kardel 			gen = 0;
   1189  1.45    kardel 
   1190  1.45    kardel 		/* If the timecounter was wound up underneath us, bail out. */
   1191  1.45    kardel 		if (pps->capgen == 0 ||
   1192  1.45    kardel 		    pps->capgen != pps->capth->th_generation ||
   1193  1.45    kardel 		    gen == 0 ||
   1194  1.45    kardel 		    gen != pps->capgen) {
   1195  1.45    kardel #ifdef PPS_DEBUG
   1196  1.45    kardel 			if (ppsdebug & 0x1) {
   1197  1.45    kardel 				log(LOG_DEBUG,
   1198  1.45    kardel 				    "pps_ref_event(pps=%p, event=%d, ...): DROP (wind-up)\n",
   1199  1.45    kardel 				    pps, event);
   1200  1.45    kardel 			}
   1201  1.45    kardel #endif
   1202  1.45    kardel 			return;
   1203  1.45    kardel 		}
   1204  1.45    kardel 	} else {
   1205  1.45    kardel 		tcount = 0;	/* keep GCC happy */
   1206  1.45    kardel 	}
   1207  1.45    kardel 
   1208  1.45    kardel #ifdef PPS_DEBUG
   1209  1.45    kardel 	if (ppsdebug & 0x1) {
   1210  1.45    kardel 		struct timespec tmsp;
   1211  1.45    kardel 
   1212  1.45    kardel 		if (ref_ts == NULL) {
   1213  1.45    kardel 			tmsp.tv_sec = 0;
   1214  1.45    kardel 			tmsp.tv_nsec = 0;
   1215  1.45    kardel 		} else {
   1216  1.45    kardel 			bintime2timespec(ref_ts, &tmsp);
   1217  1.45    kardel 		}
   1218  1.45    kardel 
   1219  1.45    kardel 		log(LOG_DEBUG,
   1220  1.45    kardel 		    "pps_ref_event(pps=%p, event=%d, ref_ts=%"PRIi64
   1221  1.45    kardel 		    ".%09"PRIi32", refmode=0x%1x)\n",
   1222  1.45    kardel 		    pps, event, tmsp.tv_sec, (int32_t)tmsp.tv_nsec, refmode);
   1223  1.45    kardel 	}
   1224  1.45    kardel #endif
   1225   1.1    simonb 
   1226  1.45    kardel 	/* setup correct event references */
   1227   1.1    simonb 	if (event == PPS_CAPTUREASSERT) {
   1228   1.1    simonb 		tsp = &pps->ppsinfo.assert_timestamp;
   1229   1.1    simonb 		osp = &pps->ppsparam.assert_offset;
   1230   1.1    simonb 		foff = pps->ppsparam.mode & PPS_OFFSETASSERT;
   1231  1.46    martin #ifdef PPS_SYNC
   1232   1.1    simonb 		fhard = pps->kcmode & PPS_CAPTUREASSERT;
   1233  1.46    martin #endif
   1234   1.1    simonb 		pcount = &pps->ppscount[0];
   1235   1.1    simonb 		pseq = &pps->ppsinfo.assert_sequence;
   1236   1.1    simonb 	} else {
   1237   1.1    simonb 		tsp = &pps->ppsinfo.clear_timestamp;
   1238   1.1    simonb 		osp = &pps->ppsparam.clear_offset;
   1239   1.1    simonb 		foff = pps->ppsparam.mode & PPS_OFFSETCLEAR;
   1240  1.46    martin #ifdef PPS_SYNC
   1241   1.1    simonb 		fhard = pps->kcmode & PPS_CAPTURECLEAR;
   1242  1.46    martin #endif
   1243   1.1    simonb 		pcount = &pps->ppscount[1];
   1244   1.1    simonb 		pseq = &pps->ppsinfo.clear_sequence;
   1245   1.1    simonb 	}
   1246   1.1    simonb 
   1247  1.45    kardel 	/* determine system time stamp according to refmode */
   1248  1.45    kardel 	dcount = 0;		/* keep GCC happy */
   1249  1.45    kardel 	switch (refmode & PPS_REFEVNT_RMASK) {
   1250  1.45    kardel 	case PPS_REFEVNT_CAPTURE:
   1251  1.45    kardel 		acount = pps->capcount;	/* use capture timestamp */
   1252  1.45    kardel 		break;
   1253  1.45    kardel 
   1254  1.45    kardel 	case PPS_REFEVNT_CURRENT:
   1255  1.45    kardel 		acount = tcount; /* use current timestamp */
   1256  1.45    kardel 		break;
   1257  1.45    kardel 
   1258  1.45    kardel 	case PPS_REFEVNT_CAPCUR:
   1259  1.45    kardel 		/*
   1260  1.45    kardel 		 * calculate counter value between pps_capture() and
   1261  1.45    kardel 		 * pps_ref_event()
   1262  1.45    kardel 		 */
   1263  1.45    kardel 		dcount = tcount - pps->capcount;
   1264  1.45    kardel 		acount = (dcount / 2) + pps->capcount;
   1265  1.45    kardel 		break;
   1266  1.45    kardel 
   1267  1.45    kardel 	default:		/* ignore call error silently */
   1268  1.45    kardel 		return;
   1269  1.45    kardel 	}
   1270  1.45    kardel 
   1271   1.1    simonb 	/*
   1272   1.1    simonb 	 * If the timecounter changed, we cannot compare the count values, so
   1273   1.1    simonb 	 * we have to drop the rest of the PPS-stuff until the next event.
   1274   1.1    simonb 	 */
   1275   1.1    simonb 	if (pps->ppstc != pps->capth->th_counter) {
   1276   1.1    simonb 		pps->ppstc = pps->capth->th_counter;
   1277  1.45    kardel 		pps->capcount = acount;
   1278  1.45    kardel 		*pcount = acount;
   1279  1.45    kardel 		pps->ppscount[2] = acount;
   1280  1.45    kardel #ifdef PPS_DEBUG
   1281  1.45    kardel 		if (ppsdebug & 0x1) {
   1282  1.45    kardel 			log(LOG_DEBUG,
   1283  1.45    kardel 			    "pps_ref_event(pps=%p, event=%d, ...): DROP (time-counter change)\n",
   1284  1.45    kardel 			    pps, event);
   1285  1.45    kardel 		}
   1286  1.45    kardel #endif
   1287   1.1    simonb 		return;
   1288   1.1    simonb 	}
   1289   1.1    simonb 
   1290  1.45    kardel 	pps->capcount = acount;
   1291  1.45    kardel 
   1292  1.45    kardel 	/* Convert the count to a bintime. */
   1293   1.1    simonb 	bt = pps->capth->th_offset;
   1294  1.45    kardel 	bintime_addx(&bt, pps->capth->th_scale * (acount - pps->capth->th_offset_count));
   1295   1.4    kardel 	bintime_add(&bt, &timebasebin);
   1296  1.45    kardel 
   1297  1.45    kardel 	if ((refmode & PPS_REFEVNT_PPS) == 0) {
   1298  1.45    kardel 		/* determine difference to reference time stamp */
   1299  1.45    kardel 		bt_ref = *ref_ts;
   1300  1.45    kardel 
   1301  1.45    kardel 		btd = bt;
   1302  1.45    kardel 		bintime_sub(&btd, &bt_ref);
   1303  1.45    kardel 
   1304  1.45    kardel 		/*
   1305  1.45    kardel 		 * simulate a PPS timestamp by dropping the fraction
   1306  1.45    kardel 		 * and applying the offset
   1307  1.45    kardel 		 */
   1308  1.45    kardel 		if (bt.frac >= (uint64_t)1<<63)	/* skip to nearest second */
   1309  1.45    kardel 			bt.sec++;
   1310  1.45    kardel 		bt.frac = 0;
   1311  1.45    kardel 		bintime_add(&bt, &btd);
   1312  1.45    kardel 	} else {
   1313  1.45    kardel 		/*
   1314  1.45    kardel 		 * create ref_ts from current time -
   1315  1.45    kardel 		 * we are supposed to be called on
   1316  1.45    kardel 		 * the second mark
   1317  1.45    kardel 		 */
   1318  1.45    kardel 		bt_ref = bt;
   1319  1.45    kardel 		if (bt_ref.frac >= (uint64_t)1<<63)	/* skip to nearest second */
   1320  1.45    kardel 			bt_ref.sec++;
   1321  1.45    kardel 		bt_ref.frac = 0;
   1322  1.45    kardel 	}
   1323  1.45    kardel 
   1324  1.45    kardel 	/* convert bintime to timestamp */
   1325   1.1    simonb 	bintime2timespec(&bt, &ts);
   1326   1.1    simonb 
   1327   1.1    simonb 	/* If the timecounter was wound up underneath us, bail out. */
   1328   1.1    simonb 	if (pps->capgen != pps->capth->th_generation)
   1329   1.1    simonb 		return;
   1330   1.1    simonb 
   1331  1.45    kardel 	/* store time stamp */
   1332   1.1    simonb 	*pcount = pps->capcount;
   1333   1.1    simonb 	(*pseq)++;
   1334   1.1    simonb 	*tsp = ts;
   1335   1.1    simonb 
   1336  1.45    kardel 	/* add offset correction */
   1337   1.1    simonb 	if (foff) {
   1338   1.2    kardel 		timespecadd(tsp, osp, tsp);
   1339   1.1    simonb 		if (tsp->tv_nsec < 0) {
   1340   1.1    simonb 			tsp->tv_nsec += 1000000000;
   1341   1.1    simonb 			tsp->tv_sec -= 1;
   1342   1.1    simonb 		}
   1343   1.1    simonb 	}
   1344  1.45    kardel 
   1345  1.45    kardel #ifdef PPS_DEBUG
   1346  1.45    kardel 	if (ppsdebug & 0x2) {
   1347  1.45    kardel 		struct timespec ts2;
   1348  1.45    kardel 		struct timespec ts3;
   1349  1.45    kardel 
   1350  1.45    kardel 		bintime2timespec(&bt_ref, &ts2);
   1351  1.45    kardel 
   1352  1.45    kardel 		bt.sec = 0;
   1353  1.45    kardel 		bt.frac = 0;
   1354  1.45    kardel 
   1355  1.45    kardel 		if (refmode & PPS_REFEVNT_CAPCUR) {
   1356  1.45    kardel 			    bintime_addx(&bt, pps->capth->th_scale * dcount);
   1357  1.45    kardel 		}
   1358  1.45    kardel 		bintime2timespec(&bt, &ts3);
   1359  1.45    kardel 
   1360  1.45    kardel 		log(LOG_DEBUG, "ref_ts=%"PRIi64".%09"PRIi32
   1361  1.45    kardel 		    ", ts=%"PRIi64".%09"PRIi32", read latency=%"PRIi64" ns\n",
   1362  1.45    kardel 		    ts2.tv_sec, (int32_t)ts2.tv_nsec,
   1363  1.45    kardel 		    tsp->tv_sec, (int32_t)tsp->tv_nsec,
   1364  1.45    kardel 		    timespec2ns(&ts3));
   1365  1.45    kardel 	}
   1366  1.45    kardel #endif
   1367  1.45    kardel 
   1368   1.1    simonb #ifdef PPS_SYNC
   1369   1.1    simonb 	if (fhard) {
   1370  1.45    kardel 		uint64_t scale;
   1371  1.45    kardel 		uint64_t div;
   1372   1.1    simonb 
   1373   1.1    simonb 		/*
   1374   1.1    simonb 		 * Feed the NTP PLL/FLL.
   1375   1.1    simonb 		 * The FLL wants to know how many (hardware) nanoseconds
   1376  1.45    kardel 		 * elapsed since the previous event (mod 1 second) thus
   1377  1.45    kardel 		 * we are actually looking at the frequency difference scaled
   1378  1.45    kardel 		 * in nsec.
   1379  1.45    kardel 		 * As the counter time stamps are not truly at 1Hz
   1380  1.45    kardel 		 * we need to scale the count by the elapsed
   1381  1.45    kardel 		 * reference time.
   1382  1.45    kardel 		 * valid sampling interval: [0.5..2[ sec
   1383   1.1    simonb 		 */
   1384  1.45    kardel 
   1385  1.45    kardel 		/* calculate elapsed raw count */
   1386   1.1    simonb 		tcount = pps->capcount - pps->ppscount[2];
   1387   1.1    simonb 		pps->ppscount[2] = pps->capcount;
   1388   1.1    simonb 		tcount &= pps->capth->th_counter->tc_counter_mask;
   1389  1.45    kardel 
   1390  1.45    kardel 		/* calculate elapsed ref time */
   1391  1.45    kardel 		btd = bt_ref;
   1392  1.45    kardel 		bintime_sub(&btd, &pps->ref_time);
   1393  1.45    kardel 		pps->ref_time = bt_ref;
   1394  1.45    kardel 
   1395  1.45    kardel 		/* check that we stay below 2 sec */
   1396  1.45    kardel 		if (btd.sec < 0 || btd.sec > 1)
   1397  1.45    kardel 			return;
   1398  1.45    kardel 
   1399  1.45    kardel 		/* we want at least 0.5 sec between samples */
   1400  1.45    kardel 		if (btd.sec == 0 && btd.frac < (uint64_t)1<<63)
   1401  1.45    kardel 			return;
   1402  1.45    kardel 
   1403  1.45    kardel 		/*
   1404  1.45    kardel 		 * calculate cycles per period by multiplying
   1405  1.45    kardel 		 * the frequency with the elapsed period
   1406  1.45    kardel 		 * we pick a fraction of 30 bits
   1407  1.45    kardel 		 * ~1ns resolution for elapsed time
   1408  1.45    kardel 		 */
   1409  1.45    kardel 		div   = (uint64_t)btd.sec << 30;
   1410  1.45    kardel 		div  |= (btd.frac >> 34) & (((uint64_t)1 << 30) - 1);
   1411  1.45    kardel 		div  *= pps->capth->th_counter->tc_frequency;
   1412  1.45    kardel 		div >>= 30;
   1413  1.45    kardel 
   1414  1.45    kardel 		if (div == 0)	/* safeguard */
   1415  1.45    kardel 			return;
   1416  1.45    kardel 
   1417  1.45    kardel 		scale = (uint64_t)1 << 63;
   1418  1.45    kardel 		scale /= div;
   1419   1.1    simonb 		scale *= 2;
   1420  1.45    kardel 
   1421   1.1    simonb 		bt.sec = 0;
   1422   1.1    simonb 		bt.frac = 0;
   1423   1.1    simonb 		bintime_addx(&bt, scale * tcount);
   1424   1.1    simonb 		bintime2timespec(&bt, &ts);
   1425  1.45    kardel 
   1426  1.45    kardel #ifdef PPS_DEBUG
   1427  1.45    kardel 		if (ppsdebug & 0x4) {
   1428  1.45    kardel 			struct timespec ts2;
   1429  1.45    kardel 			int64_t df;
   1430  1.45    kardel 
   1431  1.45    kardel 			bintime2timespec(&bt_ref, &ts2);
   1432  1.45    kardel 			df = timespec2ns(&ts);
   1433  1.45    kardel 			if (df > 500000000)
   1434  1.45    kardel 				df -= 1000000000;
   1435  1.45    kardel 			log(LOG_DEBUG, "hardpps: ref_ts=%"PRIi64
   1436  1.45    kardel 			    ".%09"PRIi32", ts=%"PRIi64".%09"PRIi32
   1437  1.45    kardel 			    ", freqdiff=%"PRIi64" ns/s\n",
   1438  1.45    kardel 			    ts2.tv_sec, (int32_t)ts2.tv_nsec,
   1439  1.45    kardel 			    tsp->tv_sec, (int32_t)tsp->tv_nsec,
   1440  1.45    kardel 			    df);
   1441  1.45    kardel 		}
   1442  1.45    kardel #endif
   1443  1.45    kardel 
   1444  1.45    kardel 		hardpps(tsp, timespec2ns(&ts));
   1445   1.1    simonb 	}
   1446   1.1    simonb #endif
   1447   1.1    simonb }
   1448   1.1    simonb 
   1449   1.1    simonb /*
   1450   1.1    simonb  * Timecounters need to be updated every so often to prevent the hardware
   1451   1.1    simonb  * counter from overflowing.  Updating also recalculates the cached values
   1452   1.1    simonb  * used by the get*() family of functions, so their precision depends on
   1453   1.1    simonb  * the update frequency.
   1454   1.1    simonb  */
   1455   1.1    simonb 
   1456   1.1    simonb static int tc_tick;
   1457   1.1    simonb 
   1458   1.1    simonb void
   1459   1.1    simonb tc_ticktock(void)
   1460   1.1    simonb {
   1461   1.1    simonb 	static int count;
   1462   1.1    simonb 
   1463   1.1    simonb 	if (++count < tc_tick)
   1464   1.1    simonb 		return;
   1465   1.1    simonb 	count = 0;
   1466  1.51  riastrad 	mutex_spin_enter(&timecounter_lock);
   1467  1.56       rin 	if (__predict_false(timecounter_bad != 0)) {
   1468  1.35        ad 		/* An existing timecounter has gone bad, pick a new one. */
   1469  1.35        ad 		(void)atomic_swap_uint(&timecounter_bad, 0);
   1470  1.35        ad 		if (timecounter->tc_quality < 0) {
   1471  1.35        ad 			tc_pick();
   1472  1.35        ad 		}
   1473  1.35        ad 	}
   1474   1.1    simonb 	tc_windup();
   1475  1.51  riastrad 	mutex_spin_exit(&timecounter_lock);
   1476   1.1    simonb }
   1477   1.1    simonb 
   1478   1.2    kardel void
   1479   1.2    kardel inittimecounter(void)
   1480   1.1    simonb {
   1481   1.1    simonb 	u_int p;
   1482   1.1    simonb 
   1483  1.37    kardel 	mutex_init(&timecounter_lock, MUTEX_DEFAULT, IPL_HIGH);
   1484  1.30        ad 
   1485   1.1    simonb 	/*
   1486   1.1    simonb 	 * Set the initial timeout to
   1487   1.1    simonb 	 * max(1, <approx. number of hardclock ticks in a millisecond>).
   1488   1.1    simonb 	 * People should probably not use the sysctl to set the timeout
   1489  1.53   msaitoh 	 * to smaller than its initial value, since that value is the
   1490   1.1    simonb 	 * smallest reasonable one.  If they want better timestamps they
   1491   1.1    simonb 	 * should use the non-"get"* functions.
   1492   1.1    simonb 	 */
   1493   1.1    simonb 	if (hz > 1000)
   1494   1.1    simonb 		tc_tick = (hz + 500) / 1000;
   1495   1.1    simonb 	else
   1496   1.1    simonb 		tc_tick = 1;
   1497   1.1    simonb 	p = (tc_tick * 1000000) / hz;
   1498  1.18        ad 	aprint_verbose("timecounter: Timecounters tick every %d.%03u msec\n",
   1499  1.18        ad 	    p / 1000, p % 1000);
   1500   1.1    simonb 
   1501   1.1    simonb 	/* warm up new timecounter (again) and get rolling. */
   1502   1.1    simonb 	(void)timecounter->tc_get_timecount(timecounter);
   1503   1.1    simonb 	(void)timecounter->tc_get_timecount(timecounter);
   1504   1.1    simonb }
   1505