Home | History | Annotate | Line # | Download | only in kern
kern_timeout.c revision 1.47.6.3
      1  1.47.6.3     skrll /*	$NetBSD: kern_timeout.c,v 1.47.6.3 2017/08/28 17:53:07 skrll Exp $	*/
      2       1.1   thorpej 
      3       1.1   thorpej /*-
      4      1.44        ad  * Copyright (c) 2003, 2006, 2007, 2008, 2009 The NetBSD Foundation, Inc.
      5       1.1   thorpej  * All rights reserved.
      6       1.1   thorpej  *
      7       1.1   thorpej  * This code is derived from software contributed to The NetBSD Foundation
      8      1.22        ad  * by Jason R. Thorpe, and by Andrew Doran.
      9       1.1   thorpej  *
     10       1.1   thorpej  * Redistribution and use in source and binary forms, with or without
     11       1.1   thorpej  * modification, are permitted provided that the following conditions
     12       1.1   thorpej  * are met:
     13       1.1   thorpej  * 1. Redistributions of source code must retain the above copyright
     14       1.1   thorpej  *    notice, this list of conditions and the following disclaimer.
     15       1.1   thorpej  * 2. Redistributions in binary form must reproduce the above copyright
     16       1.1   thorpej  *    notice, this list of conditions and the following disclaimer in the
     17       1.1   thorpej  *    documentation and/or other materials provided with the distribution.
     18       1.1   thorpej  *
     19       1.1   thorpej  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     20       1.1   thorpej  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     21       1.1   thorpej  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     22       1.1   thorpej  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     23       1.1   thorpej  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     24       1.1   thorpej  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     25       1.1   thorpej  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     26       1.1   thorpej  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     27       1.1   thorpej  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     28       1.1   thorpej  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     29       1.1   thorpej  * POSSIBILITY OF SUCH DAMAGE.
     30       1.1   thorpej  */
     31       1.1   thorpej 
     32       1.1   thorpej /*
     33       1.1   thorpej  * Copyright (c) 2001 Thomas Nordin <nordin (at) openbsd.org>
     34       1.1   thorpej  * Copyright (c) 2000-2001 Artur Grabowski <art (at) openbsd.org>
     35      1.14     perry  * All rights reserved.
     36      1.14     perry  *
     37      1.14     perry  * Redistribution and use in source and binary forms, with or without
     38      1.14     perry  * modification, are permitted provided that the following conditions
     39      1.14     perry  * are met:
     40       1.1   thorpej  *
     41      1.14     perry  * 1. Redistributions of source code must retain the above copyright
     42      1.14     perry  *    notice, this list of conditions and the following disclaimer.
     43      1.14     perry  * 2. Redistributions in binary form must reproduce the above copyright
     44      1.14     perry  *    notice, this list of conditions and the following disclaimer in the
     45      1.14     perry  *    documentation and/or other materials provided with the distribution.
     46       1.1   thorpej  * 3. The name of the author may not be used to endorse or promote products
     47      1.14     perry  *    derived from this software without specific prior written permission.
     48       1.1   thorpej  *
     49       1.1   thorpej  * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
     50       1.1   thorpej  * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
     51       1.1   thorpej  * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
     52       1.1   thorpej  * THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
     53       1.1   thorpej  * EXEMPLARY, OR CONSEQUENTIAL  DAMAGES (INCLUDING, BUT NOT LIMITED TO,
     54       1.1   thorpej  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
     55       1.1   thorpej  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
     56       1.1   thorpej  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
     57       1.1   thorpej  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
     58      1.14     perry  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     59       1.1   thorpej  */
     60       1.7     lukem 
     61       1.7     lukem #include <sys/cdefs.h>
     62  1.47.6.3     skrll __KERNEL_RCSID(0, "$NetBSD: kern_timeout.c,v 1.47.6.3 2017/08/28 17:53:07 skrll Exp $");
     63       1.1   thorpej 
     64       1.1   thorpej /*
     65      1.22        ad  * Timeouts are kept in a hierarchical timing wheel.  The c_time is the
     66      1.36        ad  * value of c_cpu->cc_ticks when the timeout should be called.  There are
     67      1.36        ad  * four levels with 256 buckets each. See 'Scheme 7' in "Hashed and
     68      1.36        ad  * Hierarchical Timing Wheels: Efficient Data Structures for Implementing
     69      1.36        ad  * a Timer Facility" by George Varghese and Tony Lauck.
     70      1.22        ad  *
     71      1.22        ad  * Some of the "math" in here is a bit tricky.  We have to beware of
     72      1.22        ad  * wrapping ints.
     73      1.22        ad  *
     74      1.22        ad  * We use the fact that any element added to the queue must be added with
     75      1.22        ad  * a positive time.  That means that any element `to' on the queue cannot
     76      1.22        ad  * be scheduled to timeout further in time than INT_MAX, but c->c_time can
     77      1.22        ad  * be positive or negative so comparing it with anything is dangerous.
     78      1.22        ad  * The only way we can use the c->c_time value in any predictable way is
     79      1.22        ad  * when we calculate how far in the future `to' will timeout - "c->c_time
     80      1.36        ad  * - c->c_cpu->cc_ticks".  The result will always be positive for future
     81      1.22        ad  * timeouts and 0 or negative for due timeouts.
     82       1.1   thorpej  */
     83       1.1   thorpej 
     84      1.24        ad #define	_CALLOUT_PRIVATE
     85      1.24        ad 
     86       1.1   thorpej #include <sys/param.h>
     87       1.1   thorpej #include <sys/systm.h>
     88       1.1   thorpej #include <sys/kernel.h>
     89       1.1   thorpej #include <sys/callout.h>
     90      1.45     rmind #include <sys/lwp.h>
     91      1.20        ad #include <sys/mutex.h>
     92      1.22        ad #include <sys/proc.h>
     93      1.22        ad #include <sys/sleepq.h>
     94      1.22        ad #include <sys/syncobj.h>
     95      1.22        ad #include <sys/evcnt.h>
     96      1.27        ad #include <sys/intr.h>
     97      1.33        ad #include <sys/cpu.h>
     98      1.36        ad #include <sys/kmem.h>
     99       1.1   thorpej 
    100       1.1   thorpej #ifdef DDB
    101       1.1   thorpej #include <machine/db_machdep.h>
    102       1.1   thorpej #include <ddb/db_interface.h>
    103       1.1   thorpej #include <ddb/db_access.h>
    104  1.47.6.1     skrll #include <ddb/db_cpu.h>
    105       1.1   thorpej #include <ddb/db_sym.h>
    106       1.1   thorpej #include <ddb/db_output.h>
    107       1.1   thorpej #endif
    108       1.1   thorpej 
    109      1.22        ad #define BUCKETS		1024
    110      1.22        ad #define WHEELSIZE	256
    111      1.22        ad #define WHEELMASK	255
    112      1.22        ad #define WHEELBITS	8
    113      1.22        ad 
    114       1.1   thorpej #define MASKWHEEL(wheel, time) (((time) >> ((wheel)*WHEELBITS)) & WHEELMASK)
    115       1.1   thorpej 
    116      1.36        ad #define BUCKET(cc, rel, abs)						\
    117       1.1   thorpej     (((rel) <= (1 << (2*WHEELBITS)))					\
    118       1.1   thorpej     	? ((rel) <= (1 << WHEELBITS))					\
    119      1.36        ad             ? &(cc)->cc_wheel[MASKWHEEL(0, (abs))]			\
    120      1.36        ad             : &(cc)->cc_wheel[MASKWHEEL(1, (abs)) + WHEELSIZE]		\
    121       1.1   thorpej         : ((rel) <= (1 << (3*WHEELBITS)))				\
    122      1.36        ad             ? &(cc)->cc_wheel[MASKWHEEL(2, (abs)) + 2*WHEELSIZE]	\
    123      1.36        ad             : &(cc)->cc_wheel[MASKWHEEL(3, (abs)) + 3*WHEELSIZE])
    124       1.1   thorpej 
    125      1.36        ad #define MOVEBUCKET(cc, wheel, time)					\
    126      1.36        ad     CIRCQ_APPEND(&(cc)->cc_todo,					\
    127      1.36        ad         &(cc)->cc_wheel[MASKWHEEL((wheel), (time)) + (wheel)*WHEELSIZE])
    128       1.1   thorpej 
    129       1.1   thorpej /*
    130       1.1   thorpej  * Circular queue definitions.
    131       1.1   thorpej  */
    132       1.1   thorpej 
    133      1.11       scw #define CIRCQ_INIT(list)						\
    134       1.1   thorpej do {									\
    135      1.11       scw         (list)->cq_next_l = (list);					\
    136      1.11       scw         (list)->cq_prev_l = (list);					\
    137       1.1   thorpej } while (/*CONSTCOND*/0)
    138       1.1   thorpej 
    139       1.1   thorpej #define CIRCQ_INSERT(elem, list)					\
    140       1.1   thorpej do {									\
    141      1.11       scw         (elem)->cq_prev_e = (list)->cq_prev_e;				\
    142      1.11       scw         (elem)->cq_next_l = (list);					\
    143      1.11       scw         (list)->cq_prev_l->cq_next_l = (elem);				\
    144      1.11       scw         (list)->cq_prev_l = (elem);					\
    145       1.1   thorpej } while (/*CONSTCOND*/0)
    146       1.1   thorpej 
    147       1.1   thorpej #define CIRCQ_APPEND(fst, snd)						\
    148       1.1   thorpej do {									\
    149       1.1   thorpej         if (!CIRCQ_EMPTY(snd)) {					\
    150      1.11       scw                 (fst)->cq_prev_l->cq_next_l = (snd)->cq_next_l;		\
    151      1.11       scw                 (snd)->cq_next_l->cq_prev_l = (fst)->cq_prev_l;		\
    152      1.11       scw                 (snd)->cq_prev_l->cq_next_l = (fst);			\
    153      1.11       scw                 (fst)->cq_prev_l = (snd)->cq_prev_l;			\
    154       1.1   thorpej                 CIRCQ_INIT(snd);					\
    155       1.1   thorpej         }								\
    156       1.1   thorpej } while (/*CONSTCOND*/0)
    157       1.1   thorpej 
    158       1.1   thorpej #define CIRCQ_REMOVE(elem)						\
    159       1.1   thorpej do {									\
    160      1.11       scw         (elem)->cq_next_l->cq_prev_e = (elem)->cq_prev_e;		\
    161      1.11       scw         (elem)->cq_prev_l->cq_next_e = (elem)->cq_next_e;		\
    162       1.1   thorpej } while (/*CONSTCOND*/0)
    163       1.1   thorpej 
    164      1.11       scw #define CIRCQ_FIRST(list)	((list)->cq_next_e)
    165      1.11       scw #define CIRCQ_NEXT(elem)	((elem)->cq_next_e)
    166      1.11       scw #define CIRCQ_LAST(elem,list)	((elem)->cq_next_l == (list))
    167      1.11       scw #define CIRCQ_EMPTY(list)	((list)->cq_next_l == (list))
    168       1.1   thorpej 
    169      1.36        ad struct callout_cpu {
    170      1.44        ad 	kmutex_t	*cc_lock;
    171      1.36        ad 	sleepq_t	cc_sleepq;
    172      1.36        ad 	u_int		cc_nwait;
    173      1.36        ad 	u_int		cc_ticks;
    174      1.36        ad 	lwp_t		*cc_lwp;
    175      1.36        ad 	callout_impl_t	*cc_active;
    176      1.36        ad 	callout_impl_t	*cc_cancel;
    177      1.36        ad 	struct evcnt	cc_ev_late;
    178      1.36        ad 	struct evcnt	cc_ev_block;
    179      1.36        ad 	struct callout_circq cc_todo;		/* Worklist */
    180      1.36        ad 	struct callout_circq cc_wheel[BUCKETS];	/* Queues of timeouts */
    181      1.36        ad 	char		cc_name1[12];
    182      1.36        ad 	char		cc_name2[12];
    183      1.36        ad };
    184      1.36        ad 
    185  1.47.6.1     skrll #ifndef CRASH
    186  1.47.6.1     skrll 
    187  1.47.6.1     skrll static void	callout_softclock(void *);
    188      1.36        ad static struct callout_cpu callout_cpu0;
    189      1.36        ad static void *callout_sih;
    190      1.36        ad 
    191      1.36        ad static inline kmutex_t *
    192      1.36        ad callout_lock(callout_impl_t *c)
    193      1.36        ad {
    194      1.44        ad 	struct callout_cpu *cc;
    195      1.36        ad 	kmutex_t *lock;
    196      1.36        ad 
    197      1.36        ad 	for (;;) {
    198      1.44        ad 		cc = c->c_cpu;
    199      1.44        ad 		lock = cc->cc_lock;
    200      1.36        ad 		mutex_spin_enter(lock);
    201      1.44        ad 		if (__predict_true(cc == c->c_cpu))
    202      1.36        ad 			return lock;
    203      1.36        ad 		mutex_spin_exit(lock);
    204      1.36        ad 	}
    205      1.36        ad }
    206       1.5   thorpej 
    207       1.1   thorpej /*
    208       1.1   thorpej  * callout_startup:
    209       1.1   thorpej  *
    210       1.1   thorpej  *	Initialize the callout facility, called at system startup time.
    211      1.36        ad  *	Do just enough to allow callouts to be safely registered.
    212       1.1   thorpej  */
    213       1.1   thorpej void
    214       1.1   thorpej callout_startup(void)
    215       1.1   thorpej {
    216      1.36        ad 	struct callout_cpu *cc;
    217       1.1   thorpej 	int b;
    218       1.1   thorpej 
    219      1.36        ad 	KASSERT(curcpu()->ci_data.cpu_callout == NULL);
    220      1.22        ad 
    221      1.36        ad 	cc = &callout_cpu0;
    222      1.44        ad 	cc->cc_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SCHED);
    223      1.36        ad 	CIRCQ_INIT(&cc->cc_todo);
    224       1.1   thorpej 	for (b = 0; b < BUCKETS; b++)
    225      1.36        ad 		CIRCQ_INIT(&cc->cc_wheel[b]);
    226      1.36        ad 	curcpu()->ci_data.cpu_callout = cc;
    227      1.22        ad }
    228      1.22        ad 
    229      1.22        ad /*
    230      1.36        ad  * callout_init_cpu:
    231      1.22        ad  *
    232      1.36        ad  *	Per-CPU initialization.
    233      1.22        ad  */
    234      1.47    martin CTASSERT(sizeof(callout_impl_t) <= sizeof(callout_t));
    235      1.47    martin 
    236      1.22        ad void
    237      1.36        ad callout_init_cpu(struct cpu_info *ci)
    238      1.22        ad {
    239      1.36        ad 	struct callout_cpu *cc;
    240      1.36        ad 	int b;
    241      1.22        ad 
    242      1.36        ad 	if ((cc = ci->ci_data.cpu_callout) == NULL) {
    243      1.36        ad 		cc = kmem_zalloc(sizeof(*cc), KM_SLEEP);
    244      1.44        ad 		cc->cc_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SCHED);
    245      1.36        ad 		CIRCQ_INIT(&cc->cc_todo);
    246      1.36        ad 		for (b = 0; b < BUCKETS; b++)
    247      1.36        ad 			CIRCQ_INIT(&cc->cc_wheel[b]);
    248      1.36        ad 	} else {
    249      1.36        ad 		/* Boot CPU, one time only. */
    250      1.36        ad 		callout_sih = softint_establish(SOFTINT_CLOCK | SOFTINT_MPSAFE,
    251      1.36        ad 		    callout_softclock, NULL);
    252      1.36        ad 		if (callout_sih == NULL)
    253      1.36        ad 			panic("callout_init_cpu (2)");
    254      1.36        ad 	}
    255      1.36        ad 
    256      1.40        ad 	sleepq_init(&cc->cc_sleepq);
    257      1.36        ad 
    258      1.36        ad 	snprintf(cc->cc_name1, sizeof(cc->cc_name1), "late/%u",
    259      1.36        ad 	    cpu_index(ci));
    260      1.36        ad 	evcnt_attach_dynamic(&cc->cc_ev_late, EVCNT_TYPE_MISC,
    261      1.36        ad 	    NULL, "callout", cc->cc_name1);
    262      1.36        ad 
    263      1.36        ad 	snprintf(cc->cc_name2, sizeof(cc->cc_name2), "wait/%u",
    264      1.36        ad 	    cpu_index(ci));
    265      1.36        ad 	evcnt_attach_dynamic(&cc->cc_ev_block, EVCNT_TYPE_MISC,
    266      1.36        ad 	    NULL, "callout", cc->cc_name2);
    267      1.36        ad 
    268      1.36        ad 	ci->ci_data.cpu_callout = cc;
    269       1.1   thorpej }
    270       1.1   thorpej 
    271       1.1   thorpej /*
    272       1.1   thorpej  * callout_init:
    273       1.1   thorpej  *
    274      1.36        ad  *	Initialize a callout structure.  This must be quick, so we fill
    275      1.36        ad  *	only the minimum number of fields.
    276       1.1   thorpej  */
    277       1.1   thorpej void
    278      1.22        ad callout_init(callout_t *cs, u_int flags)
    279       1.1   thorpej {
    280      1.22        ad 	callout_impl_t *c = (callout_impl_t *)cs;
    281      1.36        ad 	struct callout_cpu *cc;
    282      1.22        ad 
    283      1.22        ad 	KASSERT((flags & ~CALLOUT_FLAGMASK) == 0);
    284       1.1   thorpej 
    285      1.36        ad 	cc = curcpu()->ci_data.cpu_callout;
    286      1.36        ad 	c->c_func = NULL;
    287      1.22        ad 	c->c_magic = CALLOUT_MAGIC;
    288      1.36        ad 	if (__predict_true((flags & CALLOUT_MPSAFE) != 0 && cc != NULL)) {
    289      1.36        ad 		c->c_flags = flags;
    290      1.36        ad 		c->c_cpu = cc;
    291      1.36        ad 		return;
    292      1.36        ad 	}
    293      1.36        ad 	c->c_flags = flags | CALLOUT_BOUND;
    294      1.36        ad 	c->c_cpu = &callout_cpu0;
    295      1.22        ad }
    296      1.22        ad 
    297      1.22        ad /*
    298      1.22        ad  * callout_destroy:
    299      1.22        ad  *
    300      1.22        ad  *	Destroy a callout structure.  The callout must be stopped.
    301      1.22        ad  */
    302      1.22        ad void
    303      1.22        ad callout_destroy(callout_t *cs)
    304      1.22        ad {
    305      1.22        ad 	callout_impl_t *c = (callout_impl_t *)cs;
    306      1.22        ad 
    307      1.22        ad 	/*
    308      1.22        ad 	 * It's not necessary to lock in order to see the correct value
    309      1.22        ad 	 * of c->c_flags.  If the callout could potentially have been
    310      1.22        ad 	 * running, the current thread should have stopped it.
    311      1.22        ad 	 */
    312  1.47.6.1     skrll 	KASSERTMSG((c->c_flags & CALLOUT_PENDING) == 0,
    313  1.47.6.1     skrll 	    "callout %p: c_func (%p) c_flags (%#x) destroyed from %p",
    314  1.47.6.1     skrll 	    c, c->c_func, c->c_flags, __builtin_return_address(0));
    315      1.36        ad 	KASSERT(c->c_cpu->cc_lwp == curlwp || c->c_cpu->cc_active != c);
    316      1.46      matt 	KASSERTMSG(c->c_magic == CALLOUT_MAGIC,
    317      1.46      matt 	    "callout %p: c_magic (%#x) != CALLOUT_MAGIC (%#x)",
    318      1.46      matt 	    c, c->c_magic, CALLOUT_MAGIC);
    319      1.22        ad 	c->c_magic = 0;
    320       1.1   thorpej }
    321       1.1   thorpej 
    322       1.1   thorpej /*
    323      1.29     joerg  * callout_schedule_locked:
    324       1.1   thorpej  *
    325      1.29     joerg  *	Schedule a callout to run.  The function and argument must
    326      1.29     joerg  *	already be set in the callout structure.  Must be called with
    327      1.29     joerg  *	callout_lock.
    328       1.1   thorpej  */
    329      1.29     joerg static void
    330      1.36        ad callout_schedule_locked(callout_impl_t *c, kmutex_t *lock, int to_ticks)
    331       1.1   thorpej {
    332      1.36        ad 	struct callout_cpu *cc, *occ;
    333      1.20        ad 	int old_time;
    334       1.1   thorpej 
    335       1.1   thorpej 	KASSERT(to_ticks >= 0);
    336      1.29     joerg 	KASSERT(c->c_func != NULL);
    337       1.1   thorpej 
    338       1.1   thorpej 	/* Initialize the time here, it won't change. */
    339      1.36        ad 	occ = c->c_cpu;
    340      1.43        ad 	c->c_flags &= ~(CALLOUT_FIRED | CALLOUT_INVOKING);
    341       1.1   thorpej 
    342       1.1   thorpej 	/*
    343       1.1   thorpej 	 * If this timeout is already scheduled and now is moved
    344      1.36        ad 	 * earlier, reschedule it now.  Otherwise leave it in place
    345       1.1   thorpej 	 * and let it be rescheduled later.
    346       1.1   thorpej 	 */
    347      1.22        ad 	if ((c->c_flags & CALLOUT_PENDING) != 0) {
    348      1.36        ad 		/* Leave on existing CPU. */
    349      1.36        ad 		old_time = c->c_time;
    350      1.36        ad 		c->c_time = to_ticks + occ->cc_ticks;
    351       1.4      yamt 		if (c->c_time - old_time < 0) {
    352       1.1   thorpej 			CIRCQ_REMOVE(&c->c_list);
    353      1.36        ad 			CIRCQ_INSERT(&c->c_list, &occ->cc_todo);
    354       1.1   thorpej 		}
    355      1.36        ad 		mutex_spin_exit(lock);
    356      1.36        ad 		return;
    357      1.36        ad 	}
    358      1.36        ad 
    359      1.36        ad 	cc = curcpu()->ci_data.cpu_callout;
    360      1.36        ad 	if ((c->c_flags & CALLOUT_BOUND) != 0 || cc == occ ||
    361      1.44        ad 	    !mutex_tryenter(cc->cc_lock)) {
    362      1.36        ad 		/* Leave on existing CPU. */
    363      1.36        ad 		c->c_time = to_ticks + occ->cc_ticks;
    364      1.36        ad 		c->c_flags |= CALLOUT_PENDING;
    365      1.36        ad 		CIRCQ_INSERT(&c->c_list, &occ->cc_todo);
    366       1.1   thorpej 	} else {
    367      1.36        ad 		/* Move to this CPU. */
    368      1.36        ad 		c->c_cpu = cc;
    369      1.36        ad 		c->c_time = to_ticks + cc->cc_ticks;
    370       1.1   thorpej 		c->c_flags |= CALLOUT_PENDING;
    371      1.36        ad 		CIRCQ_INSERT(&c->c_list, &cc->cc_todo);
    372      1.44        ad 		mutex_spin_exit(cc->cc_lock);
    373       1.1   thorpej 	}
    374      1.36        ad 	mutex_spin_exit(lock);
    375      1.29     joerg }
    376      1.29     joerg 
    377      1.29     joerg /*
    378      1.29     joerg  * callout_reset:
    379      1.29     joerg  *
    380      1.29     joerg  *	Reset a callout structure with a new function and argument, and
    381      1.29     joerg  *	schedule it to run.
    382      1.29     joerg  */
    383      1.29     joerg void
    384      1.29     joerg callout_reset(callout_t *cs, int to_ticks, void (*func)(void *), void *arg)
    385      1.29     joerg {
    386      1.29     joerg 	callout_impl_t *c = (callout_impl_t *)cs;
    387      1.36        ad 	kmutex_t *lock;
    388      1.29     joerg 
    389      1.29     joerg 	KASSERT(c->c_magic == CALLOUT_MAGIC);
    390      1.42     rmind 	KASSERT(func != NULL);
    391      1.29     joerg 
    392      1.36        ad 	lock = callout_lock(c);
    393      1.29     joerg 	c->c_func = func;
    394      1.29     joerg 	c->c_arg = arg;
    395      1.36        ad 	callout_schedule_locked(c, lock, to_ticks);
    396       1.1   thorpej }
    397       1.1   thorpej 
    398       1.1   thorpej /*
    399       1.1   thorpej  * callout_schedule:
    400       1.1   thorpej  *
    401       1.1   thorpej  *	Schedule a callout to run.  The function and argument must
    402       1.1   thorpej  *	already be set in the callout structure.
    403       1.1   thorpej  */
    404       1.1   thorpej void
    405      1.22        ad callout_schedule(callout_t *cs, int to_ticks)
    406       1.1   thorpej {
    407      1.22        ad 	callout_impl_t *c = (callout_impl_t *)cs;
    408      1.36        ad 	kmutex_t *lock;
    409       1.1   thorpej 
    410      1.22        ad 	KASSERT(c->c_magic == CALLOUT_MAGIC);
    411       1.1   thorpej 
    412      1.36        ad 	lock = callout_lock(c);
    413      1.36        ad 	callout_schedule_locked(c, lock, to_ticks);
    414       1.1   thorpej }
    415       1.1   thorpej 
    416       1.1   thorpej /*
    417       1.1   thorpej  * callout_stop:
    418       1.1   thorpej  *
    419      1.36        ad  *	Try to cancel a pending callout.  It may be too late: the callout
    420      1.36        ad  *	could be running on another CPU.  If called from interrupt context,
    421      1.36        ad  *	the callout could already be in progress at a lower priority.
    422       1.1   thorpej  */
    423      1.22        ad bool
    424      1.22        ad callout_stop(callout_t *cs)
    425       1.1   thorpej {
    426      1.22        ad 	callout_impl_t *c = (callout_impl_t *)cs;
    427      1.36        ad 	struct callout_cpu *cc;
    428      1.36        ad 	kmutex_t *lock;
    429      1.22        ad 	bool expired;
    430      1.22        ad 
    431      1.22        ad 	KASSERT(c->c_magic == CALLOUT_MAGIC);
    432       1.1   thorpej 
    433      1.36        ad 	lock = callout_lock(c);
    434      1.20        ad 
    435      1.22        ad 	if ((c->c_flags & CALLOUT_PENDING) != 0)
    436       1.1   thorpej 		CIRCQ_REMOVE(&c->c_list);
    437      1.32        ad 	expired = ((c->c_flags & CALLOUT_FIRED) != 0);
    438      1.32        ad 	c->c_flags &= ~(CALLOUT_PENDING|CALLOUT_FIRED);
    439      1.32        ad 
    440      1.36        ad 	cc = c->c_cpu;
    441      1.36        ad 	if (cc->cc_active == c) {
    442      1.32        ad 		/*
    443      1.32        ad 		 * This is for non-MPSAFE callouts only.  To synchronize
    444      1.32        ad 		 * effectively we must be called with kernel_lock held.
    445      1.32        ad 		 * It's also taken in callout_softclock.
    446      1.32        ad 		 */
    447      1.36        ad 		cc->cc_cancel = c;
    448      1.32        ad 	}
    449      1.32        ad 
    450      1.36        ad 	mutex_spin_exit(lock);
    451      1.32        ad 
    452      1.32        ad 	return expired;
    453      1.32        ad }
    454      1.32        ad 
    455      1.32        ad /*
    456      1.32        ad  * callout_halt:
    457      1.32        ad  *
    458      1.32        ad  *	Cancel a pending callout.  If in-flight, block until it completes.
    459      1.36        ad  *	May not be called from a hard interrupt handler.  If the callout
    460      1.36        ad  * 	can take locks, the caller of callout_halt() must not hold any of
    461      1.37        ad  *	those locks, otherwise the two could deadlock.  If 'interlock' is
    462      1.37        ad  *	non-NULL and we must wait for the callout to complete, it will be
    463      1.37        ad  *	released and re-acquired before returning.
    464      1.32        ad  */
    465      1.32        ad bool
    466      1.38        ad callout_halt(callout_t *cs, void *interlock)
    467      1.32        ad {
    468      1.32        ad 	callout_impl_t *c = (callout_impl_t *)cs;
    469      1.36        ad 	struct callout_cpu *cc;
    470      1.32        ad 	struct lwp *l;
    471      1.37        ad 	kmutex_t *lock, *relock;
    472      1.32        ad 	bool expired;
    473      1.32        ad 
    474      1.32        ad 	KASSERT(c->c_magic == CALLOUT_MAGIC);
    475      1.32        ad 	KASSERT(!cpu_intr_p());
    476      1.32        ad 
    477      1.36        ad 	lock = callout_lock(c);
    478      1.37        ad 	relock = NULL;
    479       1.1   thorpej 
    480      1.22        ad 	expired = ((c->c_flags & CALLOUT_FIRED) != 0);
    481      1.32        ad 	if ((c->c_flags & CALLOUT_PENDING) != 0)
    482      1.32        ad 		CIRCQ_REMOVE(&c->c_list);
    483       1.9        he 	c->c_flags &= ~(CALLOUT_PENDING|CALLOUT_FIRED);
    484       1.1   thorpej 
    485      1.32        ad 	l = curlwp;
    486      1.36        ad 	for (;;) {
    487      1.36        ad 		cc = c->c_cpu;
    488      1.36        ad 		if (__predict_true(cc->cc_active != c || cc->cc_lwp == l))
    489      1.36        ad 			break;
    490      1.37        ad 		if (interlock != NULL) {
    491      1.37        ad 			/*
    492      1.37        ad 			 * Avoid potential scheduler lock order problems by
    493      1.37        ad 			 * dropping the interlock without the callout lock
    494      1.37        ad 			 * held.
    495      1.37        ad 			 */
    496      1.37        ad 			mutex_spin_exit(lock);
    497      1.37        ad 			mutex_exit(interlock);
    498      1.37        ad 			relock = interlock;
    499      1.37        ad 			interlock = NULL;
    500      1.37        ad 		} else {
    501      1.37        ad 			/* XXX Better to do priority inheritance. */
    502      1.37        ad 			KASSERT(l->l_wchan == NULL);
    503      1.37        ad 			cc->cc_nwait++;
    504      1.37        ad 			cc->cc_ev_block.ev_count++;
    505      1.37        ad 			l->l_kpriority = true;
    506      1.44        ad 			sleepq_enter(&cc->cc_sleepq, l, cc->cc_lock);
    507      1.37        ad 			sleepq_enqueue(&cc->cc_sleepq, cc, "callout",
    508      1.37        ad 			    &sleep_syncobj);
    509      1.37        ad 			sleepq_block(0, false);
    510      1.37        ad 		}
    511      1.36        ad 		lock = callout_lock(c);
    512      1.32        ad 	}
    513      1.32        ad 
    514      1.36        ad 	mutex_spin_exit(lock);
    515      1.37        ad 	if (__predict_false(relock != NULL))
    516      1.37        ad 		mutex_enter(relock);
    517      1.22        ad 
    518      1.22        ad 	return expired;
    519      1.22        ad }
    520      1.22        ad 
    521      1.36        ad #ifdef notyet
    522      1.36        ad /*
    523      1.36        ad  * callout_bind:
    524      1.36        ad  *
    525      1.36        ad  *	Bind a callout so that it will only execute on one CPU.
    526      1.36        ad  *	The callout must be stopped, and must be MPSAFE.
    527      1.36        ad  *
    528      1.36        ad  *	XXX Disabled for now until it is decided how to handle
    529      1.36        ad  *	offlined CPUs.  We may want weak+strong binding.
    530      1.36        ad  */
    531      1.36        ad void
    532      1.36        ad callout_bind(callout_t *cs, struct cpu_info *ci)
    533      1.36        ad {
    534      1.36        ad 	callout_impl_t *c = (callout_impl_t *)cs;
    535      1.36        ad 	struct callout_cpu *cc;
    536      1.36        ad 	kmutex_t *lock;
    537      1.36        ad 
    538      1.36        ad 	KASSERT((c->c_flags & CALLOUT_PENDING) == 0);
    539      1.36        ad 	KASSERT(c->c_cpu->cc_active != c);
    540      1.36        ad 	KASSERT(c->c_magic == CALLOUT_MAGIC);
    541      1.36        ad 	KASSERT((c->c_flags & CALLOUT_MPSAFE) != 0);
    542      1.36        ad 
    543      1.36        ad 	lock = callout_lock(c);
    544      1.36        ad 	cc = ci->ci_data.cpu_callout;
    545      1.36        ad 	c->c_flags |= CALLOUT_BOUND;
    546      1.36        ad 	if (c->c_cpu != cc) {
    547      1.36        ad 		/*
    548      1.36        ad 		 * Assigning c_cpu effectively unlocks the callout
    549      1.36        ad 		 * structure, as we don't hold the new CPU's lock.
    550      1.36        ad 		 * Issue memory barrier to prevent accesses being
    551      1.36        ad 		 * reordered.
    552      1.36        ad 		 */
    553      1.36        ad 		membar_exit();
    554      1.36        ad 		c->c_cpu = cc;
    555      1.36        ad 	}
    556      1.36        ad 	mutex_spin_exit(lock);
    557      1.36        ad }
    558      1.36        ad #endif
    559      1.36        ad 
    560      1.22        ad void
    561      1.22        ad callout_setfunc(callout_t *cs, void (*func)(void *), void *arg)
    562      1.22        ad {
    563      1.22        ad 	callout_impl_t *c = (callout_impl_t *)cs;
    564      1.36        ad 	kmutex_t *lock;
    565      1.22        ad 
    566      1.22        ad 	KASSERT(c->c_magic == CALLOUT_MAGIC);
    567      1.42     rmind 	KASSERT(func != NULL);
    568      1.22        ad 
    569      1.36        ad 	lock = callout_lock(c);
    570      1.22        ad 	c->c_func = func;
    571      1.22        ad 	c->c_arg = arg;
    572      1.36        ad 	mutex_spin_exit(lock);
    573      1.22        ad }
    574      1.22        ad 
    575      1.22        ad bool
    576      1.22        ad callout_expired(callout_t *cs)
    577      1.22        ad {
    578      1.22        ad 	callout_impl_t *c = (callout_impl_t *)cs;
    579      1.36        ad 	kmutex_t *lock;
    580      1.22        ad 	bool rv;
    581      1.22        ad 
    582      1.22        ad 	KASSERT(c->c_magic == CALLOUT_MAGIC);
    583      1.22        ad 
    584      1.36        ad 	lock = callout_lock(c);
    585      1.22        ad 	rv = ((c->c_flags & CALLOUT_FIRED) != 0);
    586      1.36        ad 	mutex_spin_exit(lock);
    587      1.22        ad 
    588      1.22        ad 	return rv;
    589      1.22        ad }
    590      1.22        ad 
    591      1.22        ad bool
    592      1.22        ad callout_active(callout_t *cs)
    593      1.22        ad {
    594      1.22        ad 	callout_impl_t *c = (callout_impl_t *)cs;
    595      1.36        ad 	kmutex_t *lock;
    596      1.22        ad 	bool rv;
    597      1.22        ad 
    598      1.22        ad 	KASSERT(c->c_magic == CALLOUT_MAGIC);
    599      1.22        ad 
    600      1.36        ad 	lock = callout_lock(c);
    601      1.22        ad 	rv = ((c->c_flags & (CALLOUT_PENDING|CALLOUT_FIRED)) != 0);
    602      1.36        ad 	mutex_spin_exit(lock);
    603      1.22        ad 
    604      1.22        ad 	return rv;
    605      1.22        ad }
    606      1.22        ad 
    607      1.22        ad bool
    608      1.22        ad callout_pending(callout_t *cs)
    609      1.22        ad {
    610      1.22        ad 	callout_impl_t *c = (callout_impl_t *)cs;
    611      1.36        ad 	kmutex_t *lock;
    612      1.22        ad 	bool rv;
    613      1.22        ad 
    614      1.22        ad 	KASSERT(c->c_magic == CALLOUT_MAGIC);
    615      1.22        ad 
    616      1.36        ad 	lock = callout_lock(c);
    617      1.22        ad 	rv = ((c->c_flags & CALLOUT_PENDING) != 0);
    618      1.36        ad 	mutex_spin_exit(lock);
    619      1.22        ad 
    620      1.22        ad 	return rv;
    621      1.22        ad }
    622      1.22        ad 
    623      1.22        ad bool
    624      1.22        ad callout_invoking(callout_t *cs)
    625      1.22        ad {
    626      1.22        ad 	callout_impl_t *c = (callout_impl_t *)cs;
    627      1.36        ad 	kmutex_t *lock;
    628      1.22        ad 	bool rv;
    629      1.22        ad 
    630      1.22        ad 	KASSERT(c->c_magic == CALLOUT_MAGIC);
    631      1.22        ad 
    632      1.36        ad 	lock = callout_lock(c);
    633      1.22        ad 	rv = ((c->c_flags & CALLOUT_INVOKING) != 0);
    634      1.36        ad 	mutex_spin_exit(lock);
    635      1.22        ad 
    636      1.22        ad 	return rv;
    637      1.22        ad }
    638      1.22        ad 
    639      1.22        ad void
    640      1.22        ad callout_ack(callout_t *cs)
    641      1.22        ad {
    642      1.22        ad 	callout_impl_t *c = (callout_impl_t *)cs;
    643      1.36        ad 	kmutex_t *lock;
    644      1.22        ad 
    645      1.22        ad 	KASSERT(c->c_magic == CALLOUT_MAGIC);
    646      1.22        ad 
    647      1.36        ad 	lock = callout_lock(c);
    648      1.22        ad 	c->c_flags &= ~CALLOUT_INVOKING;
    649      1.36        ad 	mutex_spin_exit(lock);
    650       1.1   thorpej }
    651       1.1   thorpej 
    652       1.1   thorpej /*
    653      1.36        ad  * callout_hardclock:
    654      1.36        ad  *
    655      1.36        ad  *	Called from hardclock() once every tick.  We schedule a soft
    656      1.36        ad  *	interrupt if there is work to be done.
    657       1.1   thorpej  */
    658      1.22        ad void
    659       1.1   thorpej callout_hardclock(void)
    660       1.1   thorpej {
    661      1.36        ad 	struct callout_cpu *cc;
    662      1.36        ad 	int needsoftclock, ticks;
    663       1.1   thorpej 
    664      1.36        ad 	cc = curcpu()->ci_data.cpu_callout;
    665      1.44        ad 	mutex_spin_enter(cc->cc_lock);
    666       1.1   thorpej 
    667      1.36        ad 	ticks = ++cc->cc_ticks;
    668      1.36        ad 
    669      1.36        ad 	MOVEBUCKET(cc, 0, ticks);
    670      1.36        ad 	if (MASKWHEEL(0, ticks) == 0) {
    671      1.36        ad 		MOVEBUCKET(cc, 1, ticks);
    672      1.36        ad 		if (MASKWHEEL(1, ticks) == 0) {
    673      1.36        ad 			MOVEBUCKET(cc, 2, ticks);
    674      1.36        ad 			if (MASKWHEEL(2, ticks) == 0)
    675      1.36        ad 				MOVEBUCKET(cc, 3, ticks);
    676       1.1   thorpej 		}
    677       1.1   thorpej 	}
    678       1.1   thorpej 
    679      1.36        ad 	needsoftclock = !CIRCQ_EMPTY(&cc->cc_todo);
    680      1.44        ad 	mutex_spin_exit(cc->cc_lock);
    681       1.1   thorpej 
    682      1.22        ad 	if (needsoftclock)
    683      1.36        ad 		softint_schedule(callout_sih);
    684       1.1   thorpej }
    685       1.1   thorpej 
    686      1.36        ad /*
    687      1.36        ad  * callout_softclock:
    688      1.36        ad  *
    689      1.36        ad  *	Soft interrupt handler, scheduled above if there is work to
    690      1.36        ad  * 	be done.  Callouts are made in soft interrupt context.
    691      1.36        ad  */
    692      1.22        ad static void
    693      1.22        ad callout_softclock(void *v)
    694       1.1   thorpej {
    695      1.22        ad 	callout_impl_t *c;
    696      1.36        ad 	struct callout_cpu *cc;
    697       1.1   thorpej 	void (*func)(void *);
    698       1.1   thorpej 	void *arg;
    699      1.36        ad 	int mpsafe, count, ticks, delta;
    700      1.22        ad 	lwp_t *l;
    701       1.1   thorpej 
    702      1.22        ad 	l = curlwp;
    703      1.36        ad 	KASSERT(l->l_cpu == curcpu());
    704      1.36        ad 	cc = l->l_cpu->ci_data.cpu_callout;
    705       1.1   thorpej 
    706      1.44        ad 	mutex_spin_enter(cc->cc_lock);
    707      1.36        ad 	cc->cc_lwp = l;
    708      1.36        ad 	while (!CIRCQ_EMPTY(&cc->cc_todo)) {
    709      1.36        ad 		c = CIRCQ_FIRST(&cc->cc_todo);
    710      1.22        ad 		KASSERT(c->c_magic == CALLOUT_MAGIC);
    711      1.22        ad 		KASSERT(c->c_func != NULL);
    712      1.36        ad 		KASSERT(c->c_cpu == cc);
    713      1.26        ad 		KASSERT((c->c_flags & CALLOUT_PENDING) != 0);
    714      1.26        ad 		KASSERT((c->c_flags & CALLOUT_FIRED) == 0);
    715       1.1   thorpej 		CIRCQ_REMOVE(&c->c_list);
    716       1.1   thorpej 
    717       1.1   thorpej 		/* If due run it, otherwise insert it into the right bucket. */
    718      1.36        ad 		ticks = cc->cc_ticks;
    719      1.36        ad 		delta = c->c_time - ticks;
    720      1.36        ad 		if (delta > 0) {
    721      1.36        ad 			CIRCQ_INSERT(&c->c_list, BUCKET(cc, delta, c->c_time));
    722      1.36        ad 			continue;
    723      1.36        ad 		}
    724      1.36        ad 		if (delta < 0)
    725      1.36        ad 			cc->cc_ev_late.ev_count++;
    726      1.36        ad 
    727      1.43        ad 		c->c_flags = (c->c_flags & ~CALLOUT_PENDING) |
    728      1.43        ad 		    (CALLOUT_FIRED | CALLOUT_INVOKING);
    729      1.36        ad 		mpsafe = (c->c_flags & CALLOUT_MPSAFE);
    730      1.36        ad 		func = c->c_func;
    731      1.36        ad 		arg = c->c_arg;
    732      1.36        ad 		cc->cc_active = c;
    733      1.36        ad 
    734      1.44        ad 		mutex_spin_exit(cc->cc_lock);
    735      1.42     rmind 		KASSERT(func != NULL);
    736      1.44        ad 		if (__predict_false(!mpsafe)) {
    737      1.36        ad 			KERNEL_LOCK(1, NULL);
    738      1.36        ad 			(*func)(arg);
    739      1.36        ad 			KERNEL_UNLOCK_ONE(NULL);
    740      1.36        ad 		} else
    741      1.36        ad 			(*func)(arg);
    742      1.44        ad 		mutex_spin_enter(cc->cc_lock);
    743      1.36        ad 
    744      1.36        ad 		/*
    745      1.36        ad 		 * We can't touch 'c' here because it might be
    746      1.36        ad 		 * freed already.  If LWPs waiting for callout
    747      1.36        ad 		 * to complete, awaken them.
    748      1.36        ad 		 */
    749      1.36        ad 		cc->cc_active = NULL;
    750      1.36        ad 		if ((count = cc->cc_nwait) != 0) {
    751      1.36        ad 			cc->cc_nwait = 0;
    752      1.36        ad 			/* sleepq_wake() drops the lock. */
    753      1.44        ad 			sleepq_wake(&cc->cc_sleepq, cc, count, cc->cc_lock);
    754      1.44        ad 			mutex_spin_enter(cc->cc_lock);
    755       1.1   thorpej 		}
    756       1.1   thorpej 	}
    757      1.36        ad 	cc->cc_lwp = NULL;
    758      1.44        ad 	mutex_spin_exit(cc->cc_lock);
    759       1.1   thorpej }
    760  1.47.6.1     skrll #endif
    761       1.1   thorpej 
    762       1.1   thorpej #ifdef DDB
    763       1.1   thorpej static void
    764  1.47.6.2     skrll db_show_callout_bucket(struct callout_cpu *cc, struct callout_circq *kbucket,
    765  1.47.6.2     skrll     struct callout_circq *bucket)
    766       1.1   thorpej {
    767  1.47.6.1     skrll 	callout_impl_t *c, ci;
    768       1.1   thorpej 	db_expr_t offset;
    769      1.15  christos 	const char *name;
    770      1.15  christos 	static char question[] = "?";
    771      1.36        ad 	int b;
    772       1.1   thorpej 
    773  1.47.6.2     skrll 	if (CIRCQ_LAST(bucket, kbucket))
    774      1.11       scw 		return;
    775      1.11       scw 
    776  1.47.6.2     skrll 	for (c = CIRCQ_FIRST(bucket); /*nothing*/; c = CIRCQ_NEXT(&c->c_list)) {
    777  1.47.6.1     skrll 		db_read_bytes((db_addr_t)c, sizeof(ci), (char *)&ci);
    778  1.47.6.1     skrll 		c = &ci;
    779      1.10       scw 		db_find_sym_and_offset((db_addr_t)(intptr_t)c->c_func, &name,
    780      1.10       scw 		    &offset);
    781      1.15  christos 		name = name ? name : question;
    782      1.36        ad 		b = (bucket - cc->cc_wheel);
    783      1.36        ad 		if (b < 0)
    784      1.36        ad 			b = -WHEELSIZE;
    785      1.36        ad 		db_printf("%9d %2d/%-4d %16lx  %s\n",
    786      1.36        ad 		    c->c_time - cc->cc_ticks, b / WHEELSIZE, b,
    787      1.36        ad 		    (u_long)c->c_arg, name);
    788  1.47.6.2     skrll 		if (CIRCQ_LAST(&c->c_list, kbucket))
    789      1.11       scw 			break;
    790       1.1   thorpej 	}
    791       1.1   thorpej }
    792       1.1   thorpej 
    793       1.1   thorpej void
    794      1.21      matt db_show_callout(db_expr_t addr, bool haddr, db_expr_t count, const char *modif)
    795       1.1   thorpej {
    796  1.47.6.2     skrll 	struct callout_cpu *cc, ccb;
    797  1.47.6.2     skrll 	struct cpu_info *ci, cib;
    798       1.1   thorpej 	int b;
    799       1.1   thorpej 
    800  1.47.6.1     skrll #ifndef CRASH
    801       1.1   thorpej 	db_printf("hardclock_ticks now: %d\n", hardclock_ticks);
    802  1.47.6.1     skrll #endif
    803       1.1   thorpej 	db_printf("    ticks  wheel               arg  func\n");
    804       1.1   thorpej 
    805       1.1   thorpej 	/*
    806       1.1   thorpej 	 * Don't lock the callwheel; all the other CPUs are paused
    807       1.1   thorpej 	 * anyhow, and we might be called in a circumstance where
    808       1.1   thorpej 	 * some other CPU was paused while holding the lock.
    809       1.1   thorpej 	 */
    810  1.47.6.1     skrll 	for (ci = db_cpu_first(); ci != NULL; ci = db_cpu_next(ci)) {
    811  1.47.6.2     skrll 		db_read_bytes((db_addr_t)ci, sizeof(cib), (char *)&cib);
    812  1.47.6.2     skrll 		cc = cib.ci_data.cpu_callout;
    813  1.47.6.2     skrll 		db_read_bytes((db_addr_t)cc, sizeof(ccb), (char *)&ccb);
    814  1.47.6.2     skrll 		db_show_callout_bucket(&ccb, &cc->cc_todo, &ccb.cc_todo);
    815      1.36        ad 	}
    816      1.36        ad 	for (b = 0; b < BUCKETS; b++) {
    817  1.47.6.1     skrll 		for (ci = db_cpu_first(); ci != NULL; ci = db_cpu_next(ci)) {
    818  1.47.6.2     skrll 			db_read_bytes((db_addr_t)ci, sizeof(cib), (char *)&cib);
    819  1.47.6.2     skrll 			cc = cib.ci_data.cpu_callout;
    820  1.47.6.2     skrll 			db_read_bytes((db_addr_t)cc, sizeof(ccb), (char *)&ccb);
    821  1.47.6.2     skrll 			db_show_callout_bucket(&ccb, &cc->cc_wheel[b],
    822  1.47.6.2     skrll 			    &ccb.cc_wheel[b]);
    823      1.36        ad 		}
    824      1.36        ad 	}
    825       1.1   thorpej }
    826       1.1   thorpej #endif /* DDB */
    827