Home | History | Annotate | Line # | Download | only in kern
kern_synch.c revision 1.280.2.3
      1  1.280.2.2     rmind /*	$NetBSD: kern_synch.c,v 1.280.2.3 2011/04/21 01:42:08 rmind Exp $	*/
      2       1.63   thorpej 
      3       1.63   thorpej /*-
      4      1.260        ad  * Copyright (c) 1999, 2000, 2004, 2006, 2007, 2008, 2009
      5      1.260        ad  *    The NetBSD Foundation, Inc.
      6       1.63   thorpej  * All rights reserved.
      7       1.63   thorpej  *
      8       1.63   thorpej  * This code is derived from software contributed to The NetBSD Foundation
      9       1.63   thorpej  * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
     10      1.188      yamt  * NASA Ames Research Center, by Charles M. Hannum, Andrew Doran and
     11      1.188      yamt  * Daniel Sieger.
     12       1.63   thorpej  *
     13       1.63   thorpej  * Redistribution and use in source and binary forms, with or without
     14       1.63   thorpej  * modification, are permitted provided that the following conditions
     15       1.63   thorpej  * are met:
     16       1.63   thorpej  * 1. Redistributions of source code must retain the above copyright
     17       1.63   thorpej  *    notice, this list of conditions and the following disclaimer.
     18       1.63   thorpej  * 2. Redistributions in binary form must reproduce the above copyright
     19       1.63   thorpej  *    notice, this list of conditions and the following disclaimer in the
     20       1.63   thorpej  *    documentation and/or other materials provided with the distribution.
     21       1.63   thorpej  *
     22       1.63   thorpej  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     23       1.63   thorpej  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     24       1.63   thorpej  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     25       1.63   thorpej  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     26       1.63   thorpej  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     27       1.63   thorpej  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     28       1.63   thorpej  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     29       1.63   thorpej  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     30       1.63   thorpej  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     31       1.63   thorpej  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     32       1.63   thorpej  * POSSIBILITY OF SUCH DAMAGE.
     33       1.63   thorpej  */
     34       1.26       cgd 
     35       1.26       cgd /*-
     36       1.26       cgd  * Copyright (c) 1982, 1986, 1990, 1991, 1993
     37       1.26       cgd  *	The Regents of the University of California.  All rights reserved.
     38       1.26       cgd  * (c) UNIX System Laboratories, Inc.
     39       1.26       cgd  * All or some portions of this file are derived from material licensed
     40       1.26       cgd  * to the University of California by American Telephone and Telegraph
     41       1.26       cgd  * Co. or Unix System Laboratories, Inc. and are reproduced herein with
     42       1.26       cgd  * the permission of UNIX System Laboratories, Inc.
     43       1.26       cgd  *
     44       1.26       cgd  * Redistribution and use in source and binary forms, with or without
     45       1.26       cgd  * modification, are permitted provided that the following conditions
     46       1.26       cgd  * are met:
     47       1.26       cgd  * 1. Redistributions of source code must retain the above copyright
     48       1.26       cgd  *    notice, this list of conditions and the following disclaimer.
     49       1.26       cgd  * 2. Redistributions in binary form must reproduce the above copyright
     50       1.26       cgd  *    notice, this list of conditions and the following disclaimer in the
     51       1.26       cgd  *    documentation and/or other materials provided with the distribution.
     52      1.136       agc  * 3. Neither the name of the University nor the names of its contributors
     53       1.26       cgd  *    may be used to endorse or promote products derived from this software
     54       1.26       cgd  *    without specific prior written permission.
     55       1.26       cgd  *
     56       1.26       cgd  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
     57       1.26       cgd  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     58       1.26       cgd  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     59       1.26       cgd  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
     60       1.26       cgd  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     61       1.26       cgd  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
     62       1.26       cgd  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     63       1.26       cgd  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     64       1.26       cgd  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     65       1.26       cgd  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     66       1.26       cgd  * SUCH DAMAGE.
     67       1.26       cgd  *
     68       1.50      fvdl  *	@(#)kern_synch.c	8.9 (Berkeley) 5/19/95
     69       1.26       cgd  */
     70      1.106     lukem 
     71      1.106     lukem #include <sys/cdefs.h>
     72  1.280.2.2     rmind __KERNEL_RCSID(0, "$NetBSD: kern_synch.c,v 1.280.2.3 2011/04/21 01:42:08 rmind Exp $");
     73       1.48       mrg 
     74      1.109      yamt #include "opt_kstack.h"
     75      1.110    briggs #include "opt_perfctrs.h"
     76      1.252  wrstuden #include "opt_sa.h"
     77      1.277    darran #include "opt_dtrace.h"
     78       1.26       cgd 
     79      1.174        ad #define	__MUTEX_PRIVATE
     80      1.174        ad 
     81       1.26       cgd #include <sys/param.h>
     82       1.26       cgd #include <sys/systm.h>
     83       1.26       cgd #include <sys/proc.h>
     84       1.26       cgd #include <sys/kernel.h>
     85      1.111    briggs #if defined(PERFCTRS)
     86      1.110    briggs #include <sys/pmc.h>
     87      1.111    briggs #endif
     88      1.188      yamt #include <sys/cpu.h>
     89       1.26       cgd #include <sys/resourcevar.h>
     90       1.55      ross #include <sys/sched.h>
     91      1.252  wrstuden #include <sys/sa.h>
     92      1.252  wrstuden #include <sys/savar.h>
     93      1.179       dsl #include <sys/syscall_stats.h>
     94      1.174        ad #include <sys/sleepq.h>
     95      1.174        ad #include <sys/lockdebug.h>
     96      1.190        ad #include <sys/evcnt.h>
     97      1.199        ad #include <sys/intr.h>
     98      1.207        ad #include <sys/lwpctl.h>
     99      1.209        ad #include <sys/atomic.h>
    100      1.215        ad #include <sys/simplelock.h>
    101       1.47       mrg 
    102       1.47       mrg #include <uvm/uvm_extern.h>
    103       1.47       mrg 
    104      1.231        ad #include <dev/lockstat.h>
    105      1.231        ad 
    106      1.276    darran #include <sys/dtrace_bsd.h>
    107      1.279    darran int                             dtrace_vtime_active=0;
    108      1.276    darran dtrace_vtime_switch_func_t      dtrace_vtime_switch_func;
    109      1.276    darran 
    110      1.271     rmind static void	sched_unsleep(struct lwp *, bool);
    111      1.188      yamt static void	sched_changepri(struct lwp *, pri_t);
    112      1.188      yamt static void	sched_lendpri(struct lwp *, pri_t);
    113      1.250     rmind static void	resched_cpu(struct lwp *);
    114      1.122   thorpej 
    115      1.174        ad syncobj_t sleep_syncobj = {
    116      1.174        ad 	SOBJ_SLEEPQ_SORTED,
    117      1.174        ad 	sleepq_unsleep,
    118      1.184      yamt 	sleepq_changepri,
    119      1.184      yamt 	sleepq_lendpri,
    120      1.184      yamt 	syncobj_noowner,
    121      1.174        ad };
    122      1.174        ad 
    123      1.174        ad syncobj_t sched_syncobj = {
    124      1.174        ad 	SOBJ_SLEEPQ_SORTED,
    125      1.174        ad 	sched_unsleep,
    126      1.184      yamt 	sched_changepri,
    127      1.184      yamt 	sched_lendpri,
    128      1.184      yamt 	syncobj_noowner,
    129      1.174        ad };
    130      1.122   thorpej 
    131      1.223        ad unsigned	sched_pstats_ticks;
    132      1.223        ad kcondvar_t	lbolt;			/* once a second sleep address */
    133      1.223        ad 
    134      1.237     rmind /* Preemption event counters */
    135      1.231        ad static struct evcnt kpreempt_ev_crit;
    136      1.231        ad static struct evcnt kpreempt_ev_klock;
    137      1.231        ad static struct evcnt kpreempt_ev_immed;
    138      1.231        ad 
    139      1.231        ad /*
    140      1.174        ad  * During autoconfiguration or after a panic, a sleep will simply lower the
    141      1.174        ad  * priority briefly to allow interrupts, then return.  The priority to be
    142      1.174        ad  * used (safepri) is machine-dependent, thus this value is initialized and
    143      1.174        ad  * maintained in the machine-dependent layers.  This priority will typically
    144      1.174        ad  * be 0, or the lowest priority that is safe for use on the interrupt stack;
    145      1.174        ad  * it can be made higher to block network software interrupts after panics.
    146       1.26       cgd  */
    147      1.174        ad int	safepri;
    148       1.26       cgd 
    149      1.237     rmind void
    150      1.270      elad synch_init(void)
    151      1.237     rmind {
    152      1.237     rmind 
    153      1.237     rmind 	cv_init(&lbolt, "lbolt");
    154      1.237     rmind 
    155      1.239        ad 	evcnt_attach_dynamic(&kpreempt_ev_crit, EVCNT_TYPE_MISC, NULL,
    156      1.237     rmind 	   "kpreempt", "defer: critical section");
    157      1.239        ad 	evcnt_attach_dynamic(&kpreempt_ev_klock, EVCNT_TYPE_MISC, NULL,
    158      1.237     rmind 	   "kpreempt", "defer: kernel_lock");
    159      1.239        ad 	evcnt_attach_dynamic(&kpreempt_ev_immed, EVCNT_TYPE_MISC, NULL,
    160      1.237     rmind 	   "kpreempt", "immediate");
    161      1.237     rmind }
    162      1.237     rmind 
    163       1.26       cgd /*
    164      1.174        ad  * OBSOLETE INTERFACE
    165      1.174        ad  *
    166      1.255     skrll  * General sleep call.  Suspends the current LWP until a wakeup is
    167      1.255     skrll  * performed on the specified identifier.  The LWP will then be made
    168      1.174        ad  * runnable with the specified priority.  Sleeps at most timo/hz seconds (0
    169      1.174        ad  * means no timeout).  If pri includes PCATCH flag, signals are checked
    170       1.26       cgd  * before and after sleeping, else signals are not checked.  Returns 0 if
    171       1.26       cgd  * awakened, EWOULDBLOCK if the timeout expires.  If PCATCH is set and a
    172       1.26       cgd  * signal needs to be delivered, ERESTART is returned if the current system
    173       1.26       cgd  * call should be restarted if possible, and EINTR is returned if the system
    174       1.26       cgd  * call should be interrupted by the signal (return EINTR).
    175       1.77   thorpej  *
    176      1.174        ad  * The interlock is held until we are on a sleep queue. The interlock will
    177      1.174        ad  * be locked before returning back to the caller unless the PNORELOCK flag
    178      1.174        ad  * is specified, in which case the interlock will always be unlocked upon
    179      1.174        ad  * return.
    180       1.26       cgd  */
    181       1.26       cgd int
    182      1.185      yamt ltsleep(wchan_t ident, pri_t priority, const char *wmesg, int timo,
    183      1.174        ad 	volatile struct simplelock *interlock)
    184       1.26       cgd {
    185      1.122   thorpej 	struct lwp *l = curlwp;
    186      1.174        ad 	sleepq_t *sq;
    187      1.244        ad 	kmutex_t *mp;
    188      1.188      yamt 	int error;
    189       1.26       cgd 
    190      1.204        ad 	KASSERT((l->l_pflag & LP_INTR) == 0);
    191      1.272     pooka 	KASSERT(ident != &lbolt);
    192      1.204        ad 
    193      1.174        ad 	if (sleepq_dontsleep(l)) {
    194      1.174        ad 		(void)sleepq_abort(NULL, 0);
    195      1.174        ad 		if ((priority & PNORELOCK) != 0)
    196       1.77   thorpej 			simple_unlock(interlock);
    197      1.174        ad 		return 0;
    198       1.26       cgd 	}
    199       1.78  sommerfe 
    200      1.204        ad 	l->l_kpriority = true;
    201      1.244        ad 	sq = sleeptab_lookup(&sleeptab, ident, &mp);
    202      1.244        ad 	sleepq_enter(sq, l, mp);
    203      1.204        ad 	sleepq_enqueue(sq, ident, wmesg, &sleep_syncobj);
    204       1.42       cgd 
    205      1.174        ad 	if (interlock != NULL) {
    206      1.204        ad 		KASSERT(simple_lock_held(interlock));
    207      1.174        ad 		simple_unlock(interlock);
    208      1.150       chs 	}
    209      1.150       chs 
    210      1.188      yamt 	error = sleepq_block(timo, priority & PCATCH);
    211      1.126        pk 
    212      1.174        ad 	if (interlock != NULL && (priority & PNORELOCK) == 0)
    213      1.126        pk 		simple_lock(interlock);
    214      1.174        ad 
    215      1.174        ad 	return error;
    216       1.26       cgd }
    217       1.26       cgd 
    218      1.187        ad int
    219      1.187        ad mtsleep(wchan_t ident, pri_t priority, const char *wmesg, int timo,
    220      1.187        ad 	kmutex_t *mtx)
    221      1.187        ad {
    222      1.187        ad 	struct lwp *l = curlwp;
    223      1.187        ad 	sleepq_t *sq;
    224      1.244        ad 	kmutex_t *mp;
    225      1.188      yamt 	int error;
    226      1.187        ad 
    227      1.204        ad 	KASSERT((l->l_pflag & LP_INTR) == 0);
    228      1.272     pooka 	KASSERT(ident != &lbolt);
    229      1.204        ad 
    230      1.187        ad 	if (sleepq_dontsleep(l)) {
    231      1.187        ad 		(void)sleepq_abort(mtx, (priority & PNORELOCK) != 0);
    232      1.187        ad 		return 0;
    233      1.187        ad 	}
    234      1.187        ad 
    235      1.204        ad 	l->l_kpriority = true;
    236      1.244        ad 	sq = sleeptab_lookup(&sleeptab, ident, &mp);
    237      1.244        ad 	sleepq_enter(sq, l, mp);
    238      1.204        ad 	sleepq_enqueue(sq, ident, wmesg, &sleep_syncobj);
    239      1.187        ad 	mutex_exit(mtx);
    240      1.188      yamt 	error = sleepq_block(timo, priority & PCATCH);
    241      1.187        ad 
    242      1.187        ad 	if ((priority & PNORELOCK) == 0)
    243      1.187        ad 		mutex_enter(mtx);
    244      1.187        ad 
    245      1.187        ad 	return error;
    246      1.187        ad }
    247      1.187        ad 
    248       1.26       cgd /*
    249      1.174        ad  * General sleep call for situations where a wake-up is not expected.
    250       1.26       cgd  */
    251      1.174        ad int
    252      1.182   thorpej kpause(const char *wmesg, bool intr, int timo, kmutex_t *mtx)
    253       1.26       cgd {
    254      1.174        ad 	struct lwp *l = curlwp;
    255      1.244        ad 	kmutex_t *mp;
    256      1.174        ad 	sleepq_t *sq;
    257      1.174        ad 	int error;
    258       1.26       cgd 
    259  1.280.2.2     rmind 	KASSERT(!(timo == 0 && intr == false));
    260  1.280.2.2     rmind 
    261      1.174        ad 	if (sleepq_dontsleep(l))
    262      1.174        ad 		return sleepq_abort(NULL, 0);
    263       1.26       cgd 
    264      1.174        ad 	if (mtx != NULL)
    265      1.174        ad 		mutex_exit(mtx);
    266      1.204        ad 	l->l_kpriority = true;
    267      1.244        ad 	sq = sleeptab_lookup(&sleeptab, l, &mp);
    268      1.244        ad 	sleepq_enter(sq, l, mp);
    269      1.204        ad 	sleepq_enqueue(sq, l, wmesg, &sleep_syncobj);
    270      1.188      yamt 	error = sleepq_block(timo, intr);
    271      1.174        ad 	if (mtx != NULL)
    272      1.174        ad 		mutex_enter(mtx);
    273       1.83   thorpej 
    274      1.174        ad 	return error;
    275      1.139        cl }
    276      1.139        cl 
    277      1.252  wrstuden #ifdef KERN_SA
    278      1.252  wrstuden /*
    279      1.252  wrstuden  * sa_awaken:
    280      1.252  wrstuden  *
    281      1.252  wrstuden  *	We believe this lwp is an SA lwp. If it's yielding,
    282      1.252  wrstuden  * let it know it needs to wake up.
    283      1.252  wrstuden  *
    284      1.252  wrstuden  *	We are called and exit with the lwp locked. We are
    285      1.252  wrstuden  * called in the middle of wakeup operations, so we need
    286      1.252  wrstuden  * to not touch the locks at all.
    287      1.252  wrstuden  */
    288      1.252  wrstuden void
    289      1.252  wrstuden sa_awaken(struct lwp *l)
    290      1.252  wrstuden {
    291      1.252  wrstuden 	/* LOCK_ASSERT(lwp_locked(l, NULL)); */
    292      1.252  wrstuden 
    293      1.252  wrstuden 	if (l == l->l_savp->savp_lwp && l->l_flag & LW_SA_YIELD)
    294      1.252  wrstuden 		l->l_flag &= ~LW_SA_IDLE;
    295      1.252  wrstuden }
    296      1.252  wrstuden #endif /* KERN_SA */
    297      1.252  wrstuden 
    298       1.26       cgd /*
    299      1.174        ad  * OBSOLETE INTERFACE
    300      1.174        ad  *
    301      1.255     skrll  * Make all LWPs sleeping on the specified identifier runnable.
    302       1.26       cgd  */
    303       1.26       cgd void
    304      1.174        ad wakeup(wchan_t ident)
    305       1.26       cgd {
    306      1.174        ad 	sleepq_t *sq;
    307      1.244        ad 	kmutex_t *mp;
    308       1.83   thorpej 
    309      1.261     rmind 	if (__predict_false(cold))
    310      1.174        ad 		return;
    311       1.83   thorpej 
    312      1.244        ad 	sq = sleeptab_lookup(&sleeptab, ident, &mp);
    313      1.244        ad 	sleepq_wake(sq, ident, (u_int)-1, mp);
    314       1.63   thorpej }
    315       1.63   thorpej 
    316       1.63   thorpej /*
    317      1.174        ad  * OBSOLETE INTERFACE
    318      1.174        ad  *
    319      1.255     skrll  * Make the highest priority LWP first in line on the specified
    320       1.63   thorpej  * identifier runnable.
    321       1.63   thorpej  */
    322      1.174        ad void
    323      1.174        ad wakeup_one(wchan_t ident)
    324       1.63   thorpej {
    325      1.174        ad 	sleepq_t *sq;
    326      1.244        ad 	kmutex_t *mp;
    327       1.63   thorpej 
    328      1.261     rmind 	if (__predict_false(cold))
    329      1.174        ad 		return;
    330      1.188      yamt 
    331      1.244        ad 	sq = sleeptab_lookup(&sleeptab, ident, &mp);
    332      1.244        ad 	sleepq_wake(sq, ident, 1, mp);
    333      1.174        ad }
    334       1.63   thorpej 
    335      1.117  gmcgarry 
    336      1.117  gmcgarry /*
    337      1.255     skrll  * General yield call.  Puts the current LWP back on its run queue and
    338      1.117  gmcgarry  * performs a voluntary context switch.  Should only be called when the
    339      1.255     skrll  * current LWP explicitly requests it (eg sched_yield(2)).
    340      1.117  gmcgarry  */
    341      1.117  gmcgarry void
    342      1.117  gmcgarry yield(void)
    343      1.117  gmcgarry {
    344      1.122   thorpej 	struct lwp *l = curlwp;
    345      1.117  gmcgarry 
    346      1.174        ad 	KERNEL_UNLOCK_ALL(l, &l->l_biglocks);
    347      1.174        ad 	lwp_lock(l);
    348      1.217        ad 	KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_lwplock));
    349      1.188      yamt 	KASSERT(l->l_stat == LSONPROC);
    350      1.204        ad 	l->l_kpriority = false;
    351      1.188      yamt 	(void)mi_switch(l);
    352      1.174        ad 	KERNEL_LOCK(l->l_biglocks, l);
    353       1.69   thorpej }
    354       1.69   thorpej 
    355       1.69   thorpej /*
    356      1.255     skrll  * General preemption call.  Puts the current LWP back on its run queue
    357      1.156    rpaulo  * and performs an involuntary context switch.
    358       1.69   thorpej  */
    359       1.69   thorpej void
    360      1.174        ad preempt(void)
    361       1.69   thorpej {
    362      1.122   thorpej 	struct lwp *l = curlwp;
    363       1.69   thorpej 
    364      1.174        ad 	KERNEL_UNLOCK_ALL(l, &l->l_biglocks);
    365      1.174        ad 	lwp_lock(l);
    366      1.217        ad 	KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_lwplock));
    367      1.188      yamt 	KASSERT(l->l_stat == LSONPROC);
    368      1.204        ad 	l->l_kpriority = false;
    369      1.174        ad 	l->l_nivcsw++;
    370      1.188      yamt 	(void)mi_switch(l);
    371      1.174        ad 	KERNEL_LOCK(l->l_biglocks, l);
    372       1.69   thorpej }
    373       1.69   thorpej 
    374      1.234        ad /*
    375      1.234        ad  * Handle a request made by another agent to preempt the current LWP
    376      1.234        ad  * in-kernel.  Usually called when l_dopreempt may be non-zero.
    377      1.234        ad  *
    378      1.234        ad  * Character addresses for lockstat only.
    379      1.234        ad  */
    380      1.231        ad static char	in_critical_section;
    381      1.231        ad static char	kernel_lock_held;
    382      1.231        ad static char	is_softint;
    383      1.262      yamt static char	cpu_kpreempt_enter_fail;
    384      1.231        ad 
    385      1.231        ad bool
    386      1.231        ad kpreempt(uintptr_t where)
    387      1.231        ad {
    388      1.231        ad 	uintptr_t failed;
    389      1.231        ad 	lwp_t *l;
    390      1.264        ad 	int s, dop, lsflag;
    391      1.231        ad 
    392      1.231        ad 	l = curlwp;
    393      1.231        ad 	failed = 0;
    394      1.231        ad 	while ((dop = l->l_dopreempt) != 0) {
    395      1.231        ad 		if (l->l_stat != LSONPROC) {
    396      1.231        ad 			/*
    397      1.231        ad 			 * About to block (or die), let it happen.
    398      1.231        ad 			 * Doesn't really count as "preemption has
    399      1.231        ad 			 * been blocked", since we're going to
    400      1.231        ad 			 * context switch.
    401      1.231        ad 			 */
    402      1.231        ad 			l->l_dopreempt = 0;
    403      1.231        ad 			return true;
    404      1.231        ad 		}
    405      1.231        ad 		if (__predict_false((l->l_flag & LW_IDLE) != 0)) {
    406      1.231        ad 			/* Can't preempt idle loop, don't count as failure. */
    407      1.261     rmind 			l->l_dopreempt = 0;
    408      1.261     rmind 			return true;
    409      1.231        ad 		}
    410      1.231        ad 		if (__predict_false(l->l_nopreempt != 0)) {
    411      1.231        ad 			/* LWP holds preemption disabled, explicitly. */
    412      1.231        ad 			if ((dop & DOPREEMPT_COUNTED) == 0) {
    413      1.234        ad 				kpreempt_ev_crit.ev_count++;
    414      1.231        ad 			}
    415      1.231        ad 			failed = (uintptr_t)&in_critical_section;
    416      1.231        ad 			break;
    417      1.231        ad 		}
    418      1.231        ad 		if (__predict_false((l->l_pflag & LP_INTR) != 0)) {
    419      1.261     rmind 			/* Can't preempt soft interrupts yet. */
    420      1.261     rmind 			l->l_dopreempt = 0;
    421      1.261     rmind 			failed = (uintptr_t)&is_softint;
    422      1.261     rmind 			break;
    423      1.231        ad 		}
    424      1.231        ad 		s = splsched();
    425      1.231        ad 		if (__predict_false(l->l_blcnt != 0 ||
    426      1.231        ad 		    curcpu()->ci_biglock_wanted != NULL)) {
    427      1.231        ad 			/* Hold or want kernel_lock, code is not MT safe. */
    428      1.231        ad 			splx(s);
    429      1.231        ad 			if ((dop & DOPREEMPT_COUNTED) == 0) {
    430      1.234        ad 				kpreempt_ev_klock.ev_count++;
    431      1.231        ad 			}
    432      1.231        ad 			failed = (uintptr_t)&kernel_lock_held;
    433      1.231        ad 			break;
    434      1.231        ad 		}
    435      1.231        ad 		if (__predict_false(!cpu_kpreempt_enter(where, s))) {
    436      1.231        ad 			/*
    437      1.231        ad 			 * It may be that the IPL is too high.
    438      1.231        ad 			 * kpreempt_enter() can schedule an
    439      1.231        ad 			 * interrupt to retry later.
    440      1.231        ad 			 */
    441      1.231        ad 			splx(s);
    442      1.262      yamt 			failed = (uintptr_t)&cpu_kpreempt_enter_fail;
    443      1.231        ad 			break;
    444      1.231        ad 		}
    445      1.231        ad 		/* Do it! */
    446      1.231        ad 		if (__predict_true((dop & DOPREEMPT_COUNTED) == 0)) {
    447      1.234        ad 			kpreempt_ev_immed.ev_count++;
    448      1.231        ad 		}
    449      1.231        ad 		lwp_lock(l);
    450      1.231        ad 		mi_switch(l);
    451      1.231        ad 		l->l_nopreempt++;
    452      1.231        ad 		splx(s);
    453      1.231        ad 
    454      1.231        ad 		/* Take care of any MD cleanup. */
    455      1.231        ad 		cpu_kpreempt_exit(where);
    456      1.231        ad 		l->l_nopreempt--;
    457      1.231        ad 	}
    458      1.231        ad 
    459      1.264        ad 	if (__predict_true(!failed)) {
    460      1.264        ad 		return false;
    461      1.264        ad 	}
    462      1.264        ad 
    463      1.231        ad 	/* Record preemption failure for reporting via lockstat. */
    464      1.264        ad 	atomic_or_uint(&l->l_dopreempt, DOPREEMPT_COUNTED);
    465      1.264        ad 	lsflag = 0;
    466      1.264        ad 	LOCKSTAT_ENTER(lsflag);
    467      1.264        ad 	if (__predict_false(lsflag)) {
    468      1.264        ad 		if (where == 0) {
    469      1.264        ad 			where = (uintptr_t)__builtin_return_address(0);
    470      1.264        ad 		}
    471      1.264        ad 		/* Preemption is on, might recurse, so make it atomic. */
    472      1.264        ad 		if (atomic_cas_ptr_ni((void *)&l->l_pfailaddr, NULL,
    473      1.264        ad 		    (void *)where) == NULL) {
    474      1.264        ad 			LOCKSTAT_START_TIMER(lsflag, l->l_pfailtime);
    475      1.264        ad 			l->l_pfaillock = failed;
    476      1.231        ad 		}
    477      1.231        ad 	}
    478      1.264        ad 	LOCKSTAT_EXIT(lsflag);
    479      1.264        ad 	return true;
    480      1.231        ad }
    481      1.231        ad 
    482       1.69   thorpej /*
    483      1.231        ad  * Return true if preemption is explicitly disabled.
    484      1.230        ad  */
    485      1.231        ad bool
    486      1.231        ad kpreempt_disabled(void)
    487      1.231        ad {
    488      1.261     rmind 	const lwp_t *l = curlwp;
    489      1.231        ad 
    490      1.231        ad 	return l->l_nopreempt != 0 || l->l_stat == LSZOMB ||
    491      1.231        ad 	    (l->l_flag & LW_IDLE) != 0 || cpu_kpreempt_disabled();
    492      1.231        ad }
    493      1.230        ad 
    494      1.230        ad /*
    495      1.231        ad  * Disable kernel preemption.
    496      1.230        ad  */
    497      1.230        ad void
    498      1.231        ad kpreempt_disable(void)
    499      1.230        ad {
    500      1.230        ad 
    501      1.231        ad 	KPREEMPT_DISABLE(curlwp);
    502      1.230        ad }
    503      1.230        ad 
    504      1.230        ad /*
    505      1.231        ad  * Reenable kernel preemption.
    506      1.230        ad  */
    507      1.231        ad void
    508      1.231        ad kpreempt_enable(void)
    509      1.230        ad {
    510      1.230        ad 
    511      1.231        ad 	KPREEMPT_ENABLE(curlwp);
    512      1.230        ad }
    513      1.230        ad 
    514      1.230        ad /*
    515      1.188      yamt  * Compute the amount of time during which the current lwp was running.
    516      1.130   nathanw  *
    517      1.188      yamt  * - update l_rtime unless it's an idle lwp.
    518      1.188      yamt  */
    519      1.188      yamt 
    520      1.199        ad void
    521      1.212      yamt updatertime(lwp_t *l, const struct bintime *now)
    522      1.188      yamt {
    523      1.188      yamt 
    524      1.261     rmind 	if (__predict_false(l->l_flag & LW_IDLE))
    525      1.188      yamt 		return;
    526      1.188      yamt 
    527      1.212      yamt 	/* rtime += now - stime */
    528      1.212      yamt 	bintime_add(&l->l_rtime, now);
    529      1.212      yamt 	bintime_sub(&l->l_rtime, &l->l_stime);
    530      1.188      yamt }
    531      1.188      yamt 
    532      1.188      yamt /*
    533      1.245        ad  * Select next LWP from the current CPU to run..
    534      1.245        ad  */
    535      1.245        ad static inline lwp_t *
    536      1.245        ad nextlwp(struct cpu_info *ci, struct schedstate_percpu *spc)
    537      1.245        ad {
    538      1.245        ad 	lwp_t *newl;
    539      1.245        ad 
    540      1.245        ad 	/*
    541      1.245        ad 	 * Let sched_nextlwp() select the LWP to run the CPU next.
    542      1.245        ad 	 * If no LWP is runnable, select the idle LWP.
    543      1.245        ad 	 *
    544      1.245        ad 	 * Note that spc_lwplock might not necessary be held, and
    545      1.245        ad 	 * new thread would be unlocked after setting the LWP-lock.
    546      1.245        ad 	 */
    547      1.245        ad 	newl = sched_nextlwp();
    548      1.245        ad 	if (newl != NULL) {
    549      1.245        ad 		sched_dequeue(newl);
    550      1.245        ad 		KASSERT(lwp_locked(newl, spc->spc_mutex));
    551      1.274     rmind 		KASSERT(newl->l_cpu == ci);
    552      1.245        ad 		newl->l_stat = LSONPROC;
    553      1.248        ad 		newl->l_pflag |= LP_RUNNING;
    554      1.245        ad 		lwp_setlock(newl, spc->spc_lwplock);
    555      1.245        ad 	} else {
    556      1.245        ad 		newl = ci->ci_data.cpu_idlelwp;
    557      1.245        ad 		newl->l_stat = LSONPROC;
    558      1.248        ad 		newl->l_pflag |= LP_RUNNING;
    559      1.245        ad 	}
    560      1.261     rmind 
    561      1.245        ad 	/*
    562      1.245        ad 	 * Only clear want_resched if there are no pending (slow)
    563      1.245        ad 	 * software interrupts.
    564      1.245        ad 	 */
    565      1.245        ad 	ci->ci_want_resched = ci->ci_data.cpu_softints;
    566      1.245        ad 	spc->spc_flags &= ~SPCF_SWITCHCLEAR;
    567      1.245        ad 	spc->spc_curpriority = lwp_eprio(newl);
    568      1.245        ad 
    569      1.245        ad 	return newl;
    570      1.245        ad }
    571      1.245        ad 
    572      1.245        ad /*
    573      1.188      yamt  * The machine independent parts of context switch.
    574      1.188      yamt  *
    575      1.188      yamt  * Returns 1 if another LWP was actually run.
    576       1.26       cgd  */
    577      1.122   thorpej int
    578      1.199        ad mi_switch(lwp_t *l)
    579       1.26       cgd {
    580      1.246     rmind 	struct cpu_info *ci;
    581       1.76   thorpej 	struct schedstate_percpu *spc;
    582      1.188      yamt 	struct lwp *newl;
    583      1.174        ad 	int retval, oldspl;
    584      1.212      yamt 	struct bintime bt;
    585      1.199        ad 	bool returning;
    586       1.26       cgd 
    587      1.188      yamt 	KASSERT(lwp_locked(l, NULL));
    588      1.231        ad 	KASSERT(kpreempt_disabled());
    589      1.188      yamt 	LOCKDEBUG_BARRIER(l->l_mutex, 1);
    590      1.174        ad 
    591      1.174        ad 	kstack_check_magic(l);
    592       1.83   thorpej 
    593      1.212      yamt 	binuptime(&bt);
    594      1.199        ad 
    595      1.267      yamt 	KASSERT((l->l_pflag & LP_RUNNING) != 0);
    596      1.231        ad 	KASSERT(l->l_cpu == curcpu());
    597      1.196        ad 	ci = l->l_cpu;
    598      1.196        ad 	spc = &ci->ci_schedstate;
    599      1.199        ad 	returning = false;
    600      1.190        ad 	newl = NULL;
    601      1.190        ad 
    602      1.199        ad 	/*
    603      1.199        ad 	 * If we have been asked to switch to a specific LWP, then there
    604      1.199        ad 	 * is no need to inspect the run queues.  If a soft interrupt is
    605      1.199        ad 	 * blocking, then return to the interrupted thread without adjusting
    606      1.199        ad 	 * VM context or its start time: neither have been changed in order
    607      1.199        ad 	 * to take the interrupt.
    608      1.199        ad 	 */
    609      1.190        ad 	if (l->l_switchto != NULL) {
    610      1.204        ad 		if ((l->l_pflag & LP_INTR) != 0) {
    611      1.199        ad 			returning = true;
    612      1.199        ad 			softint_block(l);
    613      1.248        ad 			if ((l->l_pflag & LP_TIMEINTR) != 0)
    614      1.212      yamt 				updatertime(l, &bt);
    615      1.199        ad 		}
    616      1.190        ad 		newl = l->l_switchto;
    617      1.190        ad 		l->l_switchto = NULL;
    618      1.190        ad 	}
    619      1.204        ad #ifndef __HAVE_FAST_SOFTINTS
    620      1.204        ad 	else if (ci->ci_data.cpu_softints != 0) {
    621      1.204        ad 		/* There are pending soft interrupts, so pick one. */
    622      1.204        ad 		newl = softint_picklwp();
    623      1.204        ad 		newl->l_stat = LSONPROC;
    624      1.248        ad 		newl->l_pflag |= LP_RUNNING;
    625      1.204        ad 	}
    626      1.204        ad #endif	/* !__HAVE_FAST_SOFTINTS */
    627      1.190        ad 
    628      1.180       dsl 	/* Count time spent in current system call */
    629      1.199        ad 	if (!returning) {
    630      1.199        ad 		SYSCALL_TIME_SLEEP(l);
    631      1.180       dsl 
    632      1.199        ad 		/*
    633      1.199        ad 		 * XXXSMP If we are using h/w performance counters,
    634      1.199        ad 		 * save context.
    635      1.199        ad 		 */
    636      1.174        ad #if PERFCTRS
    637      1.199        ad 		if (PMC_ENABLED(l->l_proc)) {
    638      1.199        ad 			pmc_save_context(l->l_proc);
    639      1.199        ad 		}
    640      1.199        ad #endif
    641      1.212      yamt 		updatertime(l, &bt);
    642      1.174        ad 	}
    643      1.113  gmcgarry 
    644      1.246     rmind 	/* Lock the runqueue */
    645      1.246     rmind 	KASSERT(l->l_stat != LSRUN);
    646      1.246     rmind 	mutex_spin_enter(spc->spc_mutex);
    647      1.246     rmind 
    648      1.113  gmcgarry 	/*
    649      1.174        ad 	 * If on the CPU and we have gotten this far, then we must yield.
    650      1.113  gmcgarry 	 */
    651      1.246     rmind 	if (l->l_stat == LSONPROC && l != newl) {
    652      1.217        ad 		KASSERT(lwp_locked(l, spc->spc_lwplock));
    653      1.188      yamt 		if ((l->l_flag & LW_IDLE) == 0) {
    654      1.188      yamt 			l->l_stat = LSRUN;
    655      1.246     rmind 			lwp_setlock(l, spc->spc_mutex);
    656      1.246     rmind 			sched_enqueue(l, true);
    657  1.280.2.2     rmind 			/*
    658  1.280.2.2     rmind 			 * Handle migration.  Note that "migrating LWP" may
    659  1.280.2.2     rmind 			 * be reset here, if interrupt/preemption happens
    660  1.280.2.2     rmind 			 * early in idle LWP.
    661  1.280.2.2     rmind 			 */
    662  1.280.2.2     rmind 			if (l->l_target_cpu != NULL) {
    663  1.280.2.2     rmind 				KASSERT((l->l_pflag & LP_INTR) == 0);
    664      1.246     rmind 				spc->spc_migrating = l;
    665      1.216     rmind 			}
    666      1.246     rmind 		} else
    667      1.188      yamt 			l->l_stat = LSIDL;
    668      1.174        ad 	}
    669      1.174        ad 
    670      1.245        ad 	/* Pick new LWP to run. */
    671      1.190        ad 	if (newl == NULL) {
    672      1.245        ad 		newl = nextlwp(ci, spc);
    673      1.199        ad 	}
    674      1.199        ad 
    675      1.204        ad 	/* Items that must be updated with the CPU locked. */
    676      1.199        ad 	if (!returning) {
    677      1.204        ad 		/* Update the new LWP's start time. */
    678      1.212      yamt 		newl->l_stime = bt;
    679      1.204        ad 
    680      1.199        ad 		/*
    681      1.204        ad 		 * ci_curlwp changes when a fast soft interrupt occurs.
    682      1.204        ad 		 * We use cpu_onproc to keep track of which kernel or
    683      1.204        ad 		 * user thread is running 'underneath' the software
    684      1.204        ad 		 * interrupt.  This is important for time accounting,
    685      1.204        ad 		 * itimers and forcing user threads to preempt (aston).
    686      1.199        ad 		 */
    687      1.204        ad 		ci->ci_data.cpu_onproc = newl;
    688      1.188      yamt 	}
    689      1.188      yamt 
    690      1.241        ad 	/*
    691      1.241        ad 	 * Preemption related tasks.  Must be done with the current
    692      1.241        ad 	 * CPU locked.
    693      1.241        ad 	 */
    694      1.241        ad 	cpu_did_resched(l);
    695      1.231        ad 	l->l_dopreempt = 0;
    696      1.231        ad 	if (__predict_false(l->l_pfailaddr != 0)) {
    697      1.231        ad 		LOCKSTAT_FLAG(lsflag);
    698      1.231        ad 		LOCKSTAT_ENTER(lsflag);
    699      1.231        ad 		LOCKSTAT_STOP_TIMER(lsflag, l->l_pfailtime);
    700      1.231        ad 		LOCKSTAT_EVENT_RA(lsflag, l->l_pfaillock, LB_NOPREEMPT|LB_SPIN,
    701      1.231        ad 		    1, l->l_pfailtime, l->l_pfailaddr);
    702      1.231        ad 		LOCKSTAT_EXIT(lsflag);
    703      1.231        ad 		l->l_pfailtime = 0;
    704      1.231        ad 		l->l_pfaillock = 0;
    705      1.231        ad 		l->l_pfailaddr = 0;
    706      1.231        ad 	}
    707      1.231        ad 
    708      1.188      yamt 	if (l != newl) {
    709      1.188      yamt 		struct lwp *prevlwp;
    710      1.174        ad 
    711      1.209        ad 		/* Release all locks, but leave the current LWP locked */
    712      1.246     rmind 		if (l->l_mutex == spc->spc_mutex) {
    713      1.209        ad 			/*
    714      1.209        ad 			 * Drop spc_lwplock, if the current LWP has been moved
    715      1.209        ad 			 * to the run queue (it is now locked by spc_mutex).
    716      1.209        ad 			 */
    717      1.217        ad 			mutex_spin_exit(spc->spc_lwplock);
    718      1.188      yamt 		} else {
    719      1.209        ad 			/*
    720      1.209        ad 			 * Otherwise, drop the spc_mutex, we are done with the
    721      1.209        ad 			 * run queues.
    722      1.209        ad 			 */
    723      1.188      yamt 			mutex_spin_exit(spc->spc_mutex);
    724      1.188      yamt 		}
    725      1.188      yamt 
    726      1.209        ad 		/*
    727      1.253     skrll 		 * Mark that context switch is going to be performed
    728      1.209        ad 		 * for this LWP, to protect it from being switched
    729      1.209        ad 		 * to on another CPU.
    730      1.209        ad 		 */
    731      1.209        ad 		KASSERT(l->l_ctxswtch == 0);
    732      1.209        ad 		l->l_ctxswtch = 1;
    733      1.209        ad 		l->l_ncsw++;
    734      1.267      yamt 		KASSERT((l->l_pflag & LP_RUNNING) != 0);
    735      1.248        ad 		l->l_pflag &= ~LP_RUNNING;
    736      1.209        ad 
    737      1.209        ad 		/*
    738      1.209        ad 		 * Increase the count of spin-mutexes before the release
    739      1.209        ad 		 * of the last lock - we must remain at IPL_SCHED during
    740      1.209        ad 		 * the context switch.
    741      1.209        ad 		 */
    742  1.280.2.3     rmind 		KASSERTMSG(ci->ci_mtx_count == -1,
    743  1.280.2.3     rmind 		    ("%s: cpu%u: ci_mtx_count (%d) != -1",
    744  1.280.2.3     rmind 		     __func__, cpu_index(ci), ci->ci_mtx_count));
    745      1.209        ad 		oldspl = MUTEX_SPIN_OLDSPL(ci);
    746      1.209        ad 		ci->ci_mtx_count--;
    747      1.209        ad 		lwp_unlock(l);
    748      1.209        ad 
    749      1.218        ad 		/* Count the context switch on this CPU. */
    750      1.218        ad 		ci->ci_data.cpu_nswtch++;
    751      1.188      yamt 
    752      1.209        ad 		/* Update status for lwpctl, if present. */
    753      1.209        ad 		if (l->l_lwpctl != NULL)
    754      1.209        ad 			l->l_lwpctl->lc_curcpu = LWPCTL_CPU_NONE;
    755      1.209        ad 
    756      1.199        ad 		/*
    757      1.199        ad 		 * Save old VM context, unless a soft interrupt
    758      1.199        ad 		 * handler is blocking.
    759      1.199        ad 		 */
    760      1.199        ad 		if (!returning)
    761      1.199        ad 			pmap_deactivate(l);
    762      1.188      yamt 
    763      1.209        ad 		/*
    764      1.275     skrll 		 * We may need to spin-wait if 'newl' is still
    765      1.209        ad 		 * context switching on another CPU.
    766      1.209        ad 		 */
    767      1.261     rmind 		if (__predict_false(newl->l_ctxswtch != 0)) {
    768      1.209        ad 			u_int count;
    769      1.209        ad 			count = SPINLOCK_BACKOFF_MIN;
    770      1.209        ad 			while (newl->l_ctxswtch)
    771      1.209        ad 				SPINLOCK_BACKOFF(count);
    772      1.209        ad 		}
    773      1.207        ad 
    774      1.276    darran 		/*
    775      1.276    darran 		 * If DTrace has set the active vtime enum to anything
    776      1.276    darran 		 * other than INACTIVE (0), then it should have set the
    777      1.276    darran 		 * function to call.
    778      1.276    darran 		 */
    779      1.278    darran 		if (__predict_false(dtrace_vtime_active)) {
    780      1.276    darran 			(*dtrace_vtime_switch_func)(newl);
    781      1.276    darran 		}
    782      1.276    darran 
    783      1.188      yamt 		/* Switch to the new LWP.. */
    784      1.204        ad 		prevlwp = cpu_switchto(l, newl, returning);
    785      1.207        ad 		ci = curcpu();
    786      1.207        ad 
    787      1.188      yamt 		/*
    788      1.209        ad 		 * Switched away - we have new curlwp.
    789      1.209        ad 		 * Restore VM context and IPL.
    790      1.188      yamt 		 */
    791      1.209        ad 		pmap_activate(l);
    792      1.265     rmind 		uvm_emap_switch(l);
    793      1.265     rmind 
    794      1.188      yamt 		if (prevlwp != NULL) {
    795      1.209        ad 			/* Normalize the count of the spin-mutexes */
    796      1.209        ad 			ci->ci_mtx_count++;
    797      1.209        ad 			/* Unmark the state of context switch */
    798      1.209        ad 			membar_exit();
    799      1.209        ad 			prevlwp->l_ctxswtch = 0;
    800      1.188      yamt 		}
    801      1.209        ad 
    802      1.209        ad 		/* Update status for lwpctl, if present. */
    803      1.219        ad 		if (l->l_lwpctl != NULL) {
    804      1.209        ad 			l->l_lwpctl->lc_curcpu = (int)cpu_index(ci);
    805      1.219        ad 			l->l_lwpctl->lc_pctr++;
    806      1.219        ad 		}
    807      1.174        ad 
    808      1.231        ad 		KASSERT(l->l_cpu == ci);
    809      1.231        ad 		splx(oldspl);
    810      1.188      yamt 		retval = 1;
    811      1.188      yamt 	} else {
    812      1.188      yamt 		/* Nothing to do - just unlock and return. */
    813      1.246     rmind 		mutex_spin_exit(spc->spc_mutex);
    814      1.188      yamt 		lwp_unlock(l);
    815      1.122   thorpej 		retval = 0;
    816      1.122   thorpej 	}
    817      1.110    briggs 
    818      1.188      yamt 	KASSERT(l == curlwp);
    819      1.188      yamt 	KASSERT(l->l_stat == LSONPROC);
    820      1.188      yamt 
    821      1.110    briggs 	/*
    822      1.174        ad 	 * XXXSMP If we are using h/w performance counters, restore context.
    823      1.231        ad 	 * XXXSMP preemption problem.
    824       1.26       cgd 	 */
    825      1.114  gmcgarry #if PERFCTRS
    826      1.175  christos 	if (PMC_ENABLED(l->l_proc)) {
    827      1.175  christos 		pmc_restore_context(l->l_proc);
    828      1.166  christos 	}
    829      1.114  gmcgarry #endif
    830      1.180       dsl 	SYSCALL_TIME_WAKEUP(l);
    831      1.188      yamt 	LOCKDEBUG_BARRIER(NULL, 1);
    832      1.169      yamt 
    833      1.122   thorpej 	return retval;
    834       1.26       cgd }
    835       1.26       cgd 
    836       1.26       cgd /*
    837      1.245        ad  * The machine independent parts of context switch to oblivion.
    838      1.245        ad  * Does not return.  Call with the LWP unlocked.
    839      1.245        ad  */
    840      1.245        ad void
    841      1.245        ad lwp_exit_switchaway(lwp_t *l)
    842      1.245        ad {
    843      1.245        ad 	struct cpu_info *ci;
    844      1.245        ad 	struct lwp *newl;
    845      1.245        ad 	struct bintime bt;
    846      1.245        ad 
    847      1.245        ad 	ci = l->l_cpu;
    848      1.245        ad 
    849      1.245        ad 	KASSERT(kpreempt_disabled());
    850      1.245        ad 	KASSERT(l->l_stat == LSZOMB || l->l_stat == LSIDL);
    851      1.245        ad 	KASSERT(ci == curcpu());
    852      1.245        ad 	LOCKDEBUG_BARRIER(NULL, 0);
    853      1.245        ad 
    854      1.245        ad 	kstack_check_magic(l);
    855      1.245        ad 
    856      1.245        ad 	/* Count time spent in current system call */
    857      1.245        ad 	SYSCALL_TIME_SLEEP(l);
    858      1.245        ad 	binuptime(&bt);
    859      1.245        ad 	updatertime(l, &bt);
    860      1.245        ad 
    861      1.245        ad 	/* Must stay at IPL_SCHED even after releasing run queue lock. */
    862      1.245        ad 	(void)splsched();
    863      1.245        ad 
    864      1.245        ad 	/*
    865      1.245        ad 	 * Let sched_nextlwp() select the LWP to run the CPU next.
    866      1.245        ad 	 * If no LWP is runnable, select the idle LWP.
    867      1.245        ad 	 *
    868      1.245        ad 	 * Note that spc_lwplock might not necessary be held, and
    869      1.245        ad 	 * new thread would be unlocked after setting the LWP-lock.
    870      1.245        ad 	 */
    871      1.245        ad 	spc_lock(ci);
    872      1.245        ad #ifndef __HAVE_FAST_SOFTINTS
    873      1.245        ad 	if (ci->ci_data.cpu_softints != 0) {
    874      1.245        ad 		/* There are pending soft interrupts, so pick one. */
    875      1.245        ad 		newl = softint_picklwp();
    876      1.245        ad 		newl->l_stat = LSONPROC;
    877      1.248        ad 		newl->l_pflag |= LP_RUNNING;
    878      1.245        ad 	} else
    879      1.245        ad #endif	/* !__HAVE_FAST_SOFTINTS */
    880      1.245        ad 	{
    881      1.245        ad 		newl = nextlwp(ci, &ci->ci_schedstate);
    882      1.245        ad 	}
    883      1.245        ad 
    884      1.245        ad 	/* Update the new LWP's start time. */
    885      1.245        ad 	newl->l_stime = bt;
    886      1.248        ad 	l->l_pflag &= ~LP_RUNNING;
    887      1.245        ad 
    888      1.245        ad 	/*
    889      1.245        ad 	 * ci_curlwp changes when a fast soft interrupt occurs.
    890      1.245        ad 	 * We use cpu_onproc to keep track of which kernel or
    891      1.245        ad 	 * user thread is running 'underneath' the software
    892      1.245        ad 	 * interrupt.  This is important for time accounting,
    893      1.245        ad 	 * itimers and forcing user threads to preempt (aston).
    894      1.245        ad 	 */
    895      1.245        ad 	ci->ci_data.cpu_onproc = newl;
    896      1.245        ad 
    897      1.245        ad 	/*
    898      1.245        ad 	 * Preemption related tasks.  Must be done with the current
    899      1.245        ad 	 * CPU locked.
    900      1.245        ad 	 */
    901      1.245        ad 	cpu_did_resched(l);
    902      1.245        ad 
    903      1.245        ad 	/* Unlock the run queue. */
    904      1.245        ad 	spc_unlock(ci);
    905      1.245        ad 
    906      1.245        ad 	/* Count the context switch on this CPU. */
    907      1.245        ad 	ci->ci_data.cpu_nswtch++;
    908      1.245        ad 
    909      1.245        ad 	/* Update status for lwpctl, if present. */
    910      1.245        ad 	if (l->l_lwpctl != NULL)
    911      1.247        ad 		l->l_lwpctl->lc_curcpu = LWPCTL_CPU_EXITED;
    912      1.245        ad 
    913      1.245        ad 	/*
    914      1.275     skrll 	 * We may need to spin-wait if 'newl' is still
    915      1.245        ad 	 * context switching on another CPU.
    916      1.245        ad 	 */
    917      1.261     rmind 	if (__predict_false(newl->l_ctxswtch != 0)) {
    918      1.245        ad 		u_int count;
    919      1.245        ad 		count = SPINLOCK_BACKOFF_MIN;
    920      1.245        ad 		while (newl->l_ctxswtch)
    921      1.245        ad 			SPINLOCK_BACKOFF(count);
    922      1.245        ad 	}
    923      1.245        ad 
    924      1.279    darran 	/*
    925      1.279    darran 	 * If DTrace has set the active vtime enum to anything
    926      1.279    darran 	 * other than INACTIVE (0), then it should have set the
    927      1.279    darran 	 * function to call.
    928      1.279    darran 	 */
    929      1.279    darran 	if (__predict_false(dtrace_vtime_active)) {
    930      1.279    darran 		(*dtrace_vtime_switch_func)(newl);
    931      1.279    darran 	}
    932      1.276    darran 
    933      1.245        ad 	/* Switch to the new LWP.. */
    934      1.245        ad 	(void)cpu_switchto(NULL, newl, false);
    935      1.245        ad 
    936      1.251       uwe 	for (;;) continue;	/* XXX: convince gcc about "noreturn" */
    937      1.245        ad 	/* NOTREACHED */
    938      1.245        ad }
    939      1.245        ad 
    940      1.245        ad /*
    941      1.271     rmind  * setrunnable: change LWP state to be runnable, placing it on the run queue.
    942      1.174        ad  *
    943      1.174        ad  * Call with the process and LWP locked.  Will return with the LWP unlocked.
    944       1.26       cgd  */
    945       1.26       cgd void
    946      1.122   thorpej setrunnable(struct lwp *l)
    947       1.26       cgd {
    948      1.122   thorpej 	struct proc *p = l->l_proc;
    949      1.205        ad 	struct cpu_info *ci;
    950       1.26       cgd 
    951      1.188      yamt 	KASSERT((l->l_flag & LW_IDLE) == 0);
    952      1.229        ad 	KASSERT(mutex_owned(p->p_lock));
    953      1.183        ad 	KASSERT(lwp_locked(l, NULL));
    954      1.205        ad 	KASSERT(l->l_mutex != l->l_cpu->ci_schedstate.spc_mutex);
    955       1.83   thorpej 
    956      1.122   thorpej 	switch (l->l_stat) {
    957      1.122   thorpej 	case LSSTOP:
    958       1.33   mycroft 		/*
    959       1.33   mycroft 		 * If we're being traced (possibly because someone attached us
    960       1.33   mycroft 		 * while we were stopped), check for a signal from the debugger.
    961       1.33   mycroft 		 */
    962      1.256        ad 		if ((p->p_slflag & PSL_TRACED) != 0 && p->p_xstat != 0)
    963      1.174        ad 			signotify(l);
    964      1.174        ad 		p->p_nrlwps++;
    965       1.26       cgd 		break;
    966      1.174        ad 	case LSSUSPENDED:
    967      1.178     pavel 		l->l_flag &= ~LW_WSUSPEND;
    968      1.174        ad 		p->p_nrlwps++;
    969      1.192     rmind 		cv_broadcast(&p->p_lwpcv);
    970      1.122   thorpej 		break;
    971      1.174        ad 	case LSSLEEP:
    972      1.174        ad 		KASSERT(l->l_wchan != NULL);
    973       1.26       cgd 		break;
    974      1.174        ad 	default:
    975      1.174        ad 		panic("setrunnable: lwp %p state was %d", l, l->l_stat);
    976       1.26       cgd 	}
    977      1.139        cl 
    978      1.252  wrstuden #ifdef KERN_SA
    979      1.252  wrstuden 	if (l->l_proc->p_sa)
    980      1.252  wrstuden 		sa_awaken(l);
    981      1.252  wrstuden #endif /* KERN_SA */
    982      1.252  wrstuden 
    983      1.174        ad 	/*
    984  1.280.2.2     rmind 	 * If the LWP was sleeping, start it again.
    985      1.174        ad 	 */
    986      1.174        ad 	if (l->l_wchan != NULL) {
    987      1.174        ad 		l->l_stat = LSSLEEP;
    988      1.183        ad 		/* lwp_unsleep() will release the lock. */
    989      1.221        ad 		lwp_unsleep(l, true);
    990      1.174        ad 		return;
    991      1.174        ad 	}
    992      1.139        cl 
    993      1.174        ad 	/*
    994      1.174        ad 	 * If the LWP is still on the CPU, mark it as LSONPROC.  It may be
    995      1.174        ad 	 * about to call mi_switch(), in which case it will yield.
    996      1.174        ad 	 */
    997      1.248        ad 	if ((l->l_pflag & LP_RUNNING) != 0) {
    998      1.174        ad 		l->l_stat = LSONPROC;
    999      1.174        ad 		l->l_slptime = 0;
   1000      1.174        ad 		lwp_unlock(l);
   1001      1.174        ad 		return;
   1002      1.174        ad 	}
   1003      1.122   thorpej 
   1004      1.174        ad 	/*
   1005      1.205        ad 	 * Look for a CPU to run.
   1006      1.205        ad 	 * Set the LWP runnable.
   1007      1.174        ad 	 */
   1008      1.205        ad 	ci = sched_takecpu(l);
   1009      1.205        ad 	l->l_cpu = ci;
   1010      1.236        ad 	spc_lock(ci);
   1011      1.236        ad 	lwp_unlock_to(l, ci->ci_schedstate.spc_mutex);
   1012      1.188      yamt 	sched_setrunnable(l);
   1013      1.174        ad 	l->l_stat = LSRUN;
   1014      1.122   thorpej 	l->l_slptime = 0;
   1015      1.174        ad 
   1016      1.271     rmind 	sched_enqueue(l, false);
   1017      1.271     rmind 	resched_cpu(l);
   1018      1.271     rmind 	lwp_unlock(l);
   1019       1.26       cgd }
   1020       1.26       cgd 
   1021       1.26       cgd /*
   1022      1.174        ad  * suspendsched:
   1023      1.174        ad  *
   1024      1.266      yamt  *	Convert all non-LW_SYSTEM LSSLEEP or LSRUN LWPs to LSSUSPENDED.
   1025      1.174        ad  */
   1026       1.94    bouyer void
   1027      1.174        ad suspendsched(void)
   1028       1.94    bouyer {
   1029      1.174        ad 	CPU_INFO_ITERATOR cii;
   1030      1.174        ad 	struct cpu_info *ci;
   1031      1.122   thorpej 	struct lwp *l;
   1032      1.174        ad 	struct proc *p;
   1033       1.94    bouyer 
   1034       1.94    bouyer 	/*
   1035      1.174        ad 	 * We do this by process in order not to violate the locking rules.
   1036       1.94    bouyer 	 */
   1037      1.228        ad 	mutex_enter(proc_lock);
   1038      1.174        ad 	PROCLIST_FOREACH(p, &allproc) {
   1039      1.229        ad 		mutex_enter(p->p_lock);
   1040      1.178     pavel 		if ((p->p_flag & PK_SYSTEM) != 0) {
   1041      1.229        ad 			mutex_exit(p->p_lock);
   1042       1.94    bouyer 			continue;
   1043      1.174        ad 		}
   1044      1.174        ad 
   1045      1.174        ad 		p->p_stat = SSTOP;
   1046      1.174        ad 
   1047      1.174        ad 		LIST_FOREACH(l, &p->p_lwps, l_sibling) {
   1048      1.174        ad 			if (l == curlwp)
   1049      1.174        ad 				continue;
   1050      1.174        ad 
   1051      1.174        ad 			lwp_lock(l);
   1052      1.122   thorpej 
   1053       1.97     enami 			/*
   1054      1.174        ad 			 * Set L_WREBOOT so that the LWP will suspend itself
   1055      1.174        ad 			 * when it tries to return to user mode.  We want to
   1056      1.174        ad 			 * try and get to get as many LWPs as possible to
   1057      1.174        ad 			 * the user / kernel boundary, so that they will
   1058      1.174        ad 			 * release any locks that they hold.
   1059       1.97     enami 			 */
   1060      1.178     pavel 			l->l_flag |= (LW_WREBOOT | LW_WSUSPEND);
   1061      1.174        ad 
   1062      1.174        ad 			if (l->l_stat == LSSLEEP &&
   1063      1.178     pavel 			    (l->l_flag & LW_SINTR) != 0) {
   1064      1.174        ad 				/* setrunnable() will release the lock. */
   1065      1.174        ad 				setrunnable(l);
   1066      1.174        ad 				continue;
   1067      1.174        ad 			}
   1068      1.174        ad 
   1069      1.174        ad 			lwp_unlock(l);
   1070       1.94    bouyer 		}
   1071      1.174        ad 
   1072      1.229        ad 		mutex_exit(p->p_lock);
   1073       1.94    bouyer 	}
   1074      1.228        ad 	mutex_exit(proc_lock);
   1075      1.174        ad 
   1076      1.174        ad 	/*
   1077      1.174        ad 	 * Kick all CPUs to make them preempt any LWPs running in user mode.
   1078      1.174        ad 	 * They'll trap into the kernel and suspend themselves in userret().
   1079      1.174        ad 	 */
   1080      1.204        ad 	for (CPU_INFO_FOREACH(cii, ci)) {
   1081      1.204        ad 		spc_lock(ci);
   1082      1.204        ad 		cpu_need_resched(ci, RESCHED_IMMED);
   1083      1.204        ad 		spc_unlock(ci);
   1084      1.204        ad 	}
   1085      1.174        ad }
   1086      1.174        ad 
   1087      1.174        ad /*
   1088      1.174        ad  * sched_unsleep:
   1089      1.174        ad  *
   1090      1.174        ad  *	The is called when the LWP has not been awoken normally but instead
   1091      1.174        ad  *	interrupted: for example, if the sleep timed out.  Because of this,
   1092      1.174        ad  *	it's not a valid action for running or idle LWPs.
   1093      1.174        ad  */
   1094      1.271     rmind static void
   1095      1.221        ad sched_unsleep(struct lwp *l, bool cleanup)
   1096      1.174        ad {
   1097      1.174        ad 
   1098      1.174        ad 	lwp_unlock(l);
   1099      1.174        ad 	panic("sched_unsleep");
   1100      1.174        ad }
   1101      1.174        ad 
   1102      1.250     rmind static void
   1103      1.188      yamt resched_cpu(struct lwp *l)
   1104      1.188      yamt {
   1105      1.274     rmind 	struct cpu_info *ci = l->l_cpu;
   1106      1.188      yamt 
   1107      1.250     rmind 	KASSERT(lwp_locked(l, NULL));
   1108      1.204        ad 	if (lwp_eprio(l) > ci->ci_schedstate.spc_curpriority)
   1109      1.188      yamt 		cpu_need_resched(ci, 0);
   1110      1.188      yamt }
   1111      1.188      yamt 
   1112      1.188      yamt static void
   1113      1.185      yamt sched_changepri(struct lwp *l, pri_t pri)
   1114      1.174        ad {
   1115      1.174        ad 
   1116      1.188      yamt 	KASSERT(lwp_locked(l, NULL));
   1117      1.174        ad 
   1118      1.271     rmind 	if (l->l_stat == LSRUN) {
   1119      1.204        ad 		KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_mutex));
   1120      1.204        ad 		sched_dequeue(l);
   1121      1.204        ad 		l->l_priority = pri;
   1122      1.204        ad 		sched_enqueue(l, false);
   1123      1.204        ad 	} else {
   1124      1.174        ad 		l->l_priority = pri;
   1125      1.157      yamt 	}
   1126      1.188      yamt 	resched_cpu(l);
   1127      1.184      yamt }
   1128      1.184      yamt 
   1129      1.188      yamt static void
   1130      1.185      yamt sched_lendpri(struct lwp *l, pri_t pri)
   1131      1.184      yamt {
   1132      1.184      yamt 
   1133      1.188      yamt 	KASSERT(lwp_locked(l, NULL));
   1134      1.184      yamt 
   1135      1.271     rmind 	if (l->l_stat == LSRUN) {
   1136      1.204        ad 		KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_mutex));
   1137      1.204        ad 		sched_dequeue(l);
   1138      1.204        ad 		l->l_inheritedprio = pri;
   1139      1.204        ad 		sched_enqueue(l, false);
   1140      1.204        ad 	} else {
   1141      1.184      yamt 		l->l_inheritedprio = pri;
   1142      1.184      yamt 	}
   1143      1.188      yamt 	resched_cpu(l);
   1144      1.184      yamt }
   1145      1.184      yamt 
   1146      1.184      yamt struct lwp *
   1147      1.184      yamt syncobj_noowner(wchan_t wchan)
   1148      1.184      yamt {
   1149      1.184      yamt 
   1150      1.184      yamt 	return NULL;
   1151      1.151      yamt }
   1152      1.151      yamt 
   1153      1.250     rmind /* Decay 95% of proc::p_pctcpu in 60 seconds, ccpu = exp(-1/20) */
   1154  1.280.2.1     rmind const fixpt_t ccpu = 0.95122942450071400909 * FSCALE;
   1155  1.280.2.1     rmind 
   1156  1.280.2.1     rmind /*
   1157  1.280.2.1     rmind  * Constants for averages over 1, 5 and 15 minutes when sampling at
   1158  1.280.2.1     rmind  * 5 second intervals.
   1159  1.280.2.1     rmind  */
   1160  1.280.2.1     rmind static const fixpt_t cexp[ ] = {
   1161  1.280.2.1     rmind 	0.9200444146293232 * FSCALE,	/* exp(-1/12) */
   1162  1.280.2.1     rmind 	0.9834714538216174 * FSCALE,	/* exp(-1/60) */
   1163  1.280.2.1     rmind 	0.9944598480048967 * FSCALE,	/* exp(-1/180) */
   1164  1.280.2.1     rmind };
   1165      1.134      matt 
   1166      1.134      matt /*
   1167      1.188      yamt  * sched_pstats:
   1168      1.188      yamt  *
   1169  1.280.2.1     rmind  * => Update process statistics and check CPU resource allocation.
   1170  1.280.2.1     rmind  * => Call scheduler-specific hook to eventually adjust LWP priorities.
   1171  1.280.2.1     rmind  * => Compute load average of a quantity on 1, 5 and 15 minute intervals.
   1172      1.130   nathanw  */
   1173      1.113  gmcgarry void
   1174  1.280.2.1     rmind sched_pstats(void)
   1175      1.113  gmcgarry {
   1176  1.280.2.1     rmind 	extern struct loadavg averunnable;
   1177  1.280.2.1     rmind 	struct loadavg *avg = &averunnable;
   1178      1.249     rmind 	const int clkhz = (stathz != 0 ? stathz : hz);
   1179  1.280.2.1     rmind 	static bool backwards = false;
   1180  1.280.2.1     rmind 	static u_int lavg_count = 0;
   1181      1.188      yamt 	struct proc *p;
   1182  1.280.2.1     rmind 	int nrun;
   1183      1.113  gmcgarry 
   1184      1.188      yamt 	sched_pstats_ticks++;
   1185  1.280.2.1     rmind 	if (++lavg_count >= 5) {
   1186  1.280.2.1     rmind 		lavg_count = 0;
   1187  1.280.2.1     rmind 		nrun = 0;
   1188  1.280.2.1     rmind 	}
   1189      1.228        ad 	mutex_enter(proc_lock);
   1190      1.188      yamt 	PROCLIST_FOREACH(p, &allproc) {
   1191  1.280.2.1     rmind 		struct lwp *l;
   1192  1.280.2.1     rmind 		struct rlimit *rlim;
   1193  1.280.2.1     rmind 		long runtm;
   1194  1.280.2.1     rmind 		int sig;
   1195  1.280.2.1     rmind 
   1196      1.271     rmind 		/* Increment sleep time (if sleeping), ignore overflow. */
   1197      1.229        ad 		mutex_enter(p->p_lock);
   1198      1.212      yamt 		runtm = p->p_rtime.sec;
   1199      1.188      yamt 		LIST_FOREACH(l, &p->p_lwps, l_sibling) {
   1200  1.280.2.1     rmind 			fixpt_t lpctcpu;
   1201  1.280.2.1     rmind 			u_int lcpticks;
   1202  1.280.2.1     rmind 
   1203      1.249     rmind 			if (__predict_false((l->l_flag & LW_IDLE) != 0))
   1204      1.188      yamt 				continue;
   1205      1.188      yamt 			lwp_lock(l);
   1206      1.212      yamt 			runtm += l->l_rtime.sec;
   1207      1.188      yamt 			l->l_swtime++;
   1208      1.242     rmind 			sched_lwp_stats(l);
   1209  1.280.2.1     rmind 
   1210  1.280.2.1     rmind 			/* For load average calculation. */
   1211  1.280.2.1     rmind 			if (__predict_false(lavg_count == 0) &&
   1212  1.280.2.1     rmind 			    (l->l_flag & (LW_SINTR | LW_SYSTEM)) == 0) {
   1213  1.280.2.1     rmind 				switch (l->l_stat) {
   1214  1.280.2.1     rmind 				case LSSLEEP:
   1215  1.280.2.1     rmind 					if (l->l_slptime > 1) {
   1216  1.280.2.1     rmind 						break;
   1217  1.280.2.1     rmind 					}
   1218  1.280.2.1     rmind 				case LSRUN:
   1219  1.280.2.1     rmind 				case LSONPROC:
   1220  1.280.2.1     rmind 				case LSIDL:
   1221  1.280.2.1     rmind 					nrun++;
   1222  1.280.2.1     rmind 				}
   1223  1.280.2.1     rmind 			}
   1224      1.188      yamt 			lwp_unlock(l);
   1225      1.113  gmcgarry 
   1226      1.188      yamt 			l->l_pctcpu = (l->l_pctcpu * ccpu) >> FSHIFT;
   1227      1.249     rmind 			if (l->l_slptime != 0)
   1228      1.249     rmind 				continue;
   1229      1.249     rmind 
   1230      1.249     rmind 			lpctcpu = l->l_pctcpu;
   1231      1.249     rmind 			lcpticks = atomic_swap_uint(&l->l_cpticks, 0);
   1232      1.249     rmind 			lpctcpu += ((FSCALE - ccpu) *
   1233      1.249     rmind 			    (lcpticks * FSCALE / clkhz)) >> FSHIFT;
   1234      1.249     rmind 			l->l_pctcpu = lpctcpu;
   1235      1.188      yamt 		}
   1236      1.249     rmind 		/* Calculating p_pctcpu only for ps(1) */
   1237      1.188      yamt 		p->p_pctcpu = (p->p_pctcpu * ccpu) >> FSHIFT;
   1238      1.174        ad 
   1239      1.188      yamt 		/*
   1240      1.188      yamt 		 * Check if the process exceeds its CPU resource allocation.
   1241      1.188      yamt 		 * If over max, kill it.
   1242      1.188      yamt 		 */
   1243      1.188      yamt 		rlim = &p->p_rlimit[RLIMIT_CPU];
   1244      1.188      yamt 		sig = 0;
   1245      1.249     rmind 		if (__predict_false(runtm >= rlim->rlim_cur)) {
   1246      1.188      yamt 			if (runtm >= rlim->rlim_max)
   1247      1.188      yamt 				sig = SIGKILL;
   1248      1.188      yamt 			else {
   1249      1.188      yamt 				sig = SIGXCPU;
   1250      1.188      yamt 				if (rlim->rlim_cur < rlim->rlim_max)
   1251      1.188      yamt 					rlim->rlim_cur += 5;
   1252      1.188      yamt 			}
   1253      1.188      yamt 		}
   1254      1.229        ad 		mutex_exit(p->p_lock);
   1255      1.259     rmind 		if (__predict_false(runtm < 0)) {
   1256      1.260        ad 			if (!backwards) {
   1257      1.260        ad 				backwards = true;
   1258      1.260        ad 				printf("WARNING: negative runtime; "
   1259      1.260        ad 				    "monotonic clock has gone backwards\n");
   1260      1.260        ad 			}
   1261      1.259     rmind 		} else if (__predict_false(sig)) {
   1262      1.259     rmind 			KASSERT((p->p_flag & PK_SYSTEM) == 0);
   1263      1.188      yamt 			psignal(p, sig);
   1264      1.259     rmind 		}
   1265      1.174        ad 	}
   1266      1.228        ad 	mutex_exit(proc_lock);
   1267  1.280.2.1     rmind 
   1268  1.280.2.1     rmind 	/* Load average calculation. */
   1269  1.280.2.1     rmind 	if (__predict_false(lavg_count == 0)) {
   1270  1.280.2.1     rmind 		int i;
   1271  1.280.2.1     rmind 		CTASSERT(__arraycount(cexp) == __arraycount(avg->ldavg));
   1272  1.280.2.1     rmind 		for (i = 0; i < __arraycount(cexp); i++) {
   1273  1.280.2.1     rmind 			avg->ldavg[i] = (cexp[i] * avg->ldavg[i] +
   1274  1.280.2.1     rmind 			    nrun * FSCALE * (FSCALE - cexp[i])) >> FSHIFT;
   1275  1.280.2.1     rmind 		}
   1276  1.280.2.1     rmind 	}
   1277  1.280.2.1     rmind 
   1278  1.280.2.1     rmind 	/* Lightning bolt. */
   1279      1.273     pooka 	cv_broadcast(&lbolt);
   1280      1.113  gmcgarry }
   1281