Home | History | Annotate | Line # | Download | only in kern
kern_synch.c revision 1.286.2.1
      1  1.286.2.1    jruoho /*	$NetBSD: kern_synch.c,v 1.286.2.1 2011/06/06 09:09:32 jruoho Exp $	*/
      2       1.63   thorpej 
      3       1.63   thorpej /*-
      4      1.260        ad  * Copyright (c) 1999, 2000, 2004, 2006, 2007, 2008, 2009
      5      1.260        ad  *    The NetBSD Foundation, Inc.
      6       1.63   thorpej  * All rights reserved.
      7       1.63   thorpej  *
      8       1.63   thorpej  * This code is derived from software contributed to The NetBSD Foundation
      9       1.63   thorpej  * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
     10      1.188      yamt  * NASA Ames Research Center, by Charles M. Hannum, Andrew Doran and
     11      1.188      yamt  * Daniel Sieger.
     12       1.63   thorpej  *
     13       1.63   thorpej  * Redistribution and use in source and binary forms, with or without
     14       1.63   thorpej  * modification, are permitted provided that the following conditions
     15       1.63   thorpej  * are met:
     16       1.63   thorpej  * 1. Redistributions of source code must retain the above copyright
     17       1.63   thorpej  *    notice, this list of conditions and the following disclaimer.
     18       1.63   thorpej  * 2. Redistributions in binary form must reproduce the above copyright
     19       1.63   thorpej  *    notice, this list of conditions and the following disclaimer in the
     20       1.63   thorpej  *    documentation and/or other materials provided with the distribution.
     21       1.63   thorpej  *
     22       1.63   thorpej  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     23       1.63   thorpej  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     24       1.63   thorpej  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     25       1.63   thorpej  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     26       1.63   thorpej  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     27       1.63   thorpej  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     28       1.63   thorpej  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     29       1.63   thorpej  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     30       1.63   thorpej  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     31       1.63   thorpej  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     32       1.63   thorpej  * POSSIBILITY OF SUCH DAMAGE.
     33       1.63   thorpej  */
     34       1.26       cgd 
     35       1.26       cgd /*-
     36       1.26       cgd  * Copyright (c) 1982, 1986, 1990, 1991, 1993
     37       1.26       cgd  *	The Regents of the University of California.  All rights reserved.
     38       1.26       cgd  * (c) UNIX System Laboratories, Inc.
     39       1.26       cgd  * All or some portions of this file are derived from material licensed
     40       1.26       cgd  * to the University of California by American Telephone and Telegraph
     41       1.26       cgd  * Co. or Unix System Laboratories, Inc. and are reproduced herein with
     42       1.26       cgd  * the permission of UNIX System Laboratories, Inc.
     43       1.26       cgd  *
     44       1.26       cgd  * Redistribution and use in source and binary forms, with or without
     45       1.26       cgd  * modification, are permitted provided that the following conditions
     46       1.26       cgd  * are met:
     47       1.26       cgd  * 1. Redistributions of source code must retain the above copyright
     48       1.26       cgd  *    notice, this list of conditions and the following disclaimer.
     49       1.26       cgd  * 2. Redistributions in binary form must reproduce the above copyright
     50       1.26       cgd  *    notice, this list of conditions and the following disclaimer in the
     51       1.26       cgd  *    documentation and/or other materials provided with the distribution.
     52      1.136       agc  * 3. Neither the name of the University nor the names of its contributors
     53       1.26       cgd  *    may be used to endorse or promote products derived from this software
     54       1.26       cgd  *    without specific prior written permission.
     55       1.26       cgd  *
     56       1.26       cgd  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
     57       1.26       cgd  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     58       1.26       cgd  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     59       1.26       cgd  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
     60       1.26       cgd  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     61       1.26       cgd  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
     62       1.26       cgd  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     63       1.26       cgd  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     64       1.26       cgd  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     65       1.26       cgd  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     66       1.26       cgd  * SUCH DAMAGE.
     67       1.26       cgd  *
     68       1.50      fvdl  *	@(#)kern_synch.c	8.9 (Berkeley) 5/19/95
     69       1.26       cgd  */
     70      1.106     lukem 
     71      1.106     lukem #include <sys/cdefs.h>
     72  1.286.2.1    jruoho __KERNEL_RCSID(0, "$NetBSD: kern_synch.c,v 1.286.2.1 2011/06/06 09:09:32 jruoho Exp $");
     73       1.48       mrg 
     74      1.109      yamt #include "opt_kstack.h"
     75      1.110    briggs #include "opt_perfctrs.h"
     76      1.252  wrstuden #include "opt_sa.h"
     77      1.277    darran #include "opt_dtrace.h"
     78       1.26       cgd 
     79      1.174        ad #define	__MUTEX_PRIVATE
     80      1.174        ad 
     81       1.26       cgd #include <sys/param.h>
     82       1.26       cgd #include <sys/systm.h>
     83       1.26       cgd #include <sys/proc.h>
     84       1.26       cgd #include <sys/kernel.h>
     85      1.111    briggs #if defined(PERFCTRS)
     86      1.110    briggs #include <sys/pmc.h>
     87      1.111    briggs #endif
     88      1.188      yamt #include <sys/cpu.h>
     89       1.26       cgd #include <sys/resourcevar.h>
     90       1.55      ross #include <sys/sched.h>
     91      1.252  wrstuden #include <sys/sa.h>
     92      1.252  wrstuden #include <sys/savar.h>
     93      1.179       dsl #include <sys/syscall_stats.h>
     94      1.174        ad #include <sys/sleepq.h>
     95      1.174        ad #include <sys/lockdebug.h>
     96      1.190        ad #include <sys/evcnt.h>
     97      1.199        ad #include <sys/intr.h>
     98      1.207        ad #include <sys/lwpctl.h>
     99      1.209        ad #include <sys/atomic.h>
    100      1.215        ad #include <sys/simplelock.h>
    101       1.47       mrg 
    102       1.47       mrg #include <uvm/uvm_extern.h>
    103       1.47       mrg 
    104      1.231        ad #include <dev/lockstat.h>
    105      1.231        ad 
    106      1.276    darran #include <sys/dtrace_bsd.h>
    107      1.279    darran int                             dtrace_vtime_active=0;
    108      1.276    darran dtrace_vtime_switch_func_t      dtrace_vtime_switch_func;
    109      1.276    darran 
    110      1.271     rmind static void	sched_unsleep(struct lwp *, bool);
    111      1.188      yamt static void	sched_changepri(struct lwp *, pri_t);
    112      1.188      yamt static void	sched_lendpri(struct lwp *, pri_t);
    113      1.250     rmind static void	resched_cpu(struct lwp *);
    114      1.122   thorpej 
    115      1.174        ad syncobj_t sleep_syncobj = {
    116      1.174        ad 	SOBJ_SLEEPQ_SORTED,
    117      1.174        ad 	sleepq_unsleep,
    118      1.184      yamt 	sleepq_changepri,
    119      1.184      yamt 	sleepq_lendpri,
    120      1.184      yamt 	syncobj_noowner,
    121      1.174        ad };
    122      1.174        ad 
    123      1.174        ad syncobj_t sched_syncobj = {
    124      1.174        ad 	SOBJ_SLEEPQ_SORTED,
    125      1.174        ad 	sched_unsleep,
    126      1.184      yamt 	sched_changepri,
    127      1.184      yamt 	sched_lendpri,
    128      1.184      yamt 	syncobj_noowner,
    129      1.174        ad };
    130      1.122   thorpej 
    131  1.286.2.1    jruoho /* "Lightning bolt": once a second sleep address. */
    132  1.286.2.1    jruoho kcondvar_t		lbolt			__cacheline_aligned;
    133      1.223        ad 
    134  1.286.2.1    jruoho u_int			sched_pstats_ticks	__cacheline_aligned;
    135  1.286.2.1    jruoho 
    136  1.286.2.1    jruoho /* Preemption event counters. */
    137  1.286.2.1    jruoho static struct evcnt	kpreempt_ev_crit	__cacheline_aligned;
    138  1.286.2.1    jruoho static struct evcnt	kpreempt_ev_klock	__cacheline_aligned;
    139  1.286.2.1    jruoho static struct evcnt	kpreempt_ev_immed	__cacheline_aligned;
    140      1.231        ad 
    141      1.231        ad /*
    142      1.174        ad  * During autoconfiguration or after a panic, a sleep will simply lower the
    143      1.174        ad  * priority briefly to allow interrupts, then return.  The priority to be
    144      1.174        ad  * used (safepri) is machine-dependent, thus this value is initialized and
    145      1.174        ad  * maintained in the machine-dependent layers.  This priority will typically
    146      1.174        ad  * be 0, or the lowest priority that is safe for use on the interrupt stack;
    147      1.174        ad  * it can be made higher to block network software interrupts after panics.
    148       1.26       cgd  */
    149      1.174        ad int	safepri;
    150       1.26       cgd 
    151      1.237     rmind void
    152      1.270      elad synch_init(void)
    153      1.237     rmind {
    154      1.237     rmind 
    155      1.237     rmind 	cv_init(&lbolt, "lbolt");
    156      1.237     rmind 
    157      1.239        ad 	evcnt_attach_dynamic(&kpreempt_ev_crit, EVCNT_TYPE_MISC, NULL,
    158      1.237     rmind 	   "kpreempt", "defer: critical section");
    159      1.239        ad 	evcnt_attach_dynamic(&kpreempt_ev_klock, EVCNT_TYPE_MISC, NULL,
    160      1.237     rmind 	   "kpreempt", "defer: kernel_lock");
    161      1.239        ad 	evcnt_attach_dynamic(&kpreempt_ev_immed, EVCNT_TYPE_MISC, NULL,
    162      1.237     rmind 	   "kpreempt", "immediate");
    163      1.237     rmind }
    164      1.237     rmind 
    165       1.26       cgd /*
    166      1.174        ad  * OBSOLETE INTERFACE
    167      1.174        ad  *
    168      1.255     skrll  * General sleep call.  Suspends the current LWP until a wakeup is
    169      1.255     skrll  * performed on the specified identifier.  The LWP will then be made
    170      1.174        ad  * runnable with the specified priority.  Sleeps at most timo/hz seconds (0
    171      1.174        ad  * means no timeout).  If pri includes PCATCH flag, signals are checked
    172       1.26       cgd  * before and after sleeping, else signals are not checked.  Returns 0 if
    173       1.26       cgd  * awakened, EWOULDBLOCK if the timeout expires.  If PCATCH is set and a
    174       1.26       cgd  * signal needs to be delivered, ERESTART is returned if the current system
    175       1.26       cgd  * call should be restarted if possible, and EINTR is returned if the system
    176       1.26       cgd  * call should be interrupted by the signal (return EINTR).
    177       1.77   thorpej  *
    178      1.174        ad  * The interlock is held until we are on a sleep queue. The interlock will
    179      1.174        ad  * be locked before returning back to the caller unless the PNORELOCK flag
    180      1.174        ad  * is specified, in which case the interlock will always be unlocked upon
    181      1.174        ad  * return.
    182       1.26       cgd  */
    183       1.26       cgd int
    184      1.185      yamt ltsleep(wchan_t ident, pri_t priority, const char *wmesg, int timo,
    185      1.174        ad 	volatile struct simplelock *interlock)
    186       1.26       cgd {
    187      1.122   thorpej 	struct lwp *l = curlwp;
    188      1.174        ad 	sleepq_t *sq;
    189      1.244        ad 	kmutex_t *mp;
    190      1.188      yamt 	int error;
    191       1.26       cgd 
    192      1.204        ad 	KASSERT((l->l_pflag & LP_INTR) == 0);
    193      1.272     pooka 	KASSERT(ident != &lbolt);
    194      1.204        ad 
    195      1.174        ad 	if (sleepq_dontsleep(l)) {
    196      1.174        ad 		(void)sleepq_abort(NULL, 0);
    197      1.174        ad 		if ((priority & PNORELOCK) != 0)
    198       1.77   thorpej 			simple_unlock(interlock);
    199      1.174        ad 		return 0;
    200       1.26       cgd 	}
    201       1.78  sommerfe 
    202      1.204        ad 	l->l_kpriority = true;
    203      1.244        ad 	sq = sleeptab_lookup(&sleeptab, ident, &mp);
    204      1.244        ad 	sleepq_enter(sq, l, mp);
    205      1.204        ad 	sleepq_enqueue(sq, ident, wmesg, &sleep_syncobj);
    206       1.42       cgd 
    207      1.174        ad 	if (interlock != NULL) {
    208      1.204        ad 		KASSERT(simple_lock_held(interlock));
    209      1.174        ad 		simple_unlock(interlock);
    210      1.150       chs 	}
    211      1.150       chs 
    212      1.188      yamt 	error = sleepq_block(timo, priority & PCATCH);
    213      1.126        pk 
    214      1.174        ad 	if (interlock != NULL && (priority & PNORELOCK) == 0)
    215      1.126        pk 		simple_lock(interlock);
    216      1.174        ad 
    217      1.174        ad 	return error;
    218       1.26       cgd }
    219       1.26       cgd 
    220      1.187        ad int
    221      1.187        ad mtsleep(wchan_t ident, pri_t priority, const char *wmesg, int timo,
    222      1.187        ad 	kmutex_t *mtx)
    223      1.187        ad {
    224      1.187        ad 	struct lwp *l = curlwp;
    225      1.187        ad 	sleepq_t *sq;
    226      1.244        ad 	kmutex_t *mp;
    227      1.188      yamt 	int error;
    228      1.187        ad 
    229      1.204        ad 	KASSERT((l->l_pflag & LP_INTR) == 0);
    230      1.272     pooka 	KASSERT(ident != &lbolt);
    231      1.204        ad 
    232      1.187        ad 	if (sleepq_dontsleep(l)) {
    233      1.187        ad 		(void)sleepq_abort(mtx, (priority & PNORELOCK) != 0);
    234      1.187        ad 		return 0;
    235      1.187        ad 	}
    236      1.187        ad 
    237      1.204        ad 	l->l_kpriority = true;
    238      1.244        ad 	sq = sleeptab_lookup(&sleeptab, ident, &mp);
    239      1.244        ad 	sleepq_enter(sq, l, mp);
    240      1.204        ad 	sleepq_enqueue(sq, ident, wmesg, &sleep_syncobj);
    241      1.187        ad 	mutex_exit(mtx);
    242      1.188      yamt 	error = sleepq_block(timo, priority & PCATCH);
    243      1.187        ad 
    244      1.187        ad 	if ((priority & PNORELOCK) == 0)
    245      1.187        ad 		mutex_enter(mtx);
    246      1.187        ad 
    247      1.187        ad 	return error;
    248      1.187        ad }
    249      1.187        ad 
    250       1.26       cgd /*
    251      1.174        ad  * General sleep call for situations where a wake-up is not expected.
    252       1.26       cgd  */
    253      1.174        ad int
    254      1.182   thorpej kpause(const char *wmesg, bool intr, int timo, kmutex_t *mtx)
    255       1.26       cgd {
    256      1.174        ad 	struct lwp *l = curlwp;
    257      1.244        ad 	kmutex_t *mp;
    258      1.174        ad 	sleepq_t *sq;
    259      1.174        ad 	int error;
    260       1.26       cgd 
    261      1.284     pooka 	KASSERT(!(timo == 0 && intr == false));
    262      1.284     pooka 
    263      1.174        ad 	if (sleepq_dontsleep(l))
    264      1.174        ad 		return sleepq_abort(NULL, 0);
    265       1.26       cgd 
    266      1.174        ad 	if (mtx != NULL)
    267      1.174        ad 		mutex_exit(mtx);
    268      1.204        ad 	l->l_kpriority = true;
    269      1.244        ad 	sq = sleeptab_lookup(&sleeptab, l, &mp);
    270      1.244        ad 	sleepq_enter(sq, l, mp);
    271      1.204        ad 	sleepq_enqueue(sq, l, wmesg, &sleep_syncobj);
    272      1.188      yamt 	error = sleepq_block(timo, intr);
    273      1.174        ad 	if (mtx != NULL)
    274      1.174        ad 		mutex_enter(mtx);
    275       1.83   thorpej 
    276      1.174        ad 	return error;
    277      1.139        cl }
    278      1.139        cl 
    279      1.252  wrstuden #ifdef KERN_SA
    280      1.252  wrstuden /*
    281      1.252  wrstuden  * sa_awaken:
    282      1.252  wrstuden  *
    283      1.252  wrstuden  *	We believe this lwp is an SA lwp. If it's yielding,
    284      1.252  wrstuden  * let it know it needs to wake up.
    285      1.252  wrstuden  *
    286      1.252  wrstuden  *	We are called and exit with the lwp locked. We are
    287      1.252  wrstuden  * called in the middle of wakeup operations, so we need
    288      1.252  wrstuden  * to not touch the locks at all.
    289      1.252  wrstuden  */
    290      1.252  wrstuden void
    291      1.252  wrstuden sa_awaken(struct lwp *l)
    292      1.252  wrstuden {
    293      1.252  wrstuden 	/* LOCK_ASSERT(lwp_locked(l, NULL)); */
    294      1.252  wrstuden 
    295      1.252  wrstuden 	if (l == l->l_savp->savp_lwp && l->l_flag & LW_SA_YIELD)
    296      1.252  wrstuden 		l->l_flag &= ~LW_SA_IDLE;
    297      1.252  wrstuden }
    298      1.252  wrstuden #endif /* KERN_SA */
    299      1.252  wrstuden 
    300       1.26       cgd /*
    301      1.174        ad  * OBSOLETE INTERFACE
    302      1.174        ad  *
    303      1.255     skrll  * Make all LWPs sleeping on the specified identifier runnable.
    304       1.26       cgd  */
    305       1.26       cgd void
    306      1.174        ad wakeup(wchan_t ident)
    307       1.26       cgd {
    308      1.174        ad 	sleepq_t *sq;
    309      1.244        ad 	kmutex_t *mp;
    310       1.83   thorpej 
    311      1.261     rmind 	if (__predict_false(cold))
    312      1.174        ad 		return;
    313       1.83   thorpej 
    314      1.244        ad 	sq = sleeptab_lookup(&sleeptab, ident, &mp);
    315      1.244        ad 	sleepq_wake(sq, ident, (u_int)-1, mp);
    316       1.63   thorpej }
    317       1.63   thorpej 
    318       1.63   thorpej /*
    319      1.174        ad  * OBSOLETE INTERFACE
    320      1.174        ad  *
    321      1.255     skrll  * Make the highest priority LWP first in line on the specified
    322       1.63   thorpej  * identifier runnable.
    323       1.63   thorpej  */
    324      1.174        ad void
    325      1.174        ad wakeup_one(wchan_t ident)
    326       1.63   thorpej {
    327      1.174        ad 	sleepq_t *sq;
    328      1.244        ad 	kmutex_t *mp;
    329       1.63   thorpej 
    330      1.261     rmind 	if (__predict_false(cold))
    331      1.174        ad 		return;
    332      1.188      yamt 
    333      1.244        ad 	sq = sleeptab_lookup(&sleeptab, ident, &mp);
    334      1.244        ad 	sleepq_wake(sq, ident, 1, mp);
    335      1.174        ad }
    336       1.63   thorpej 
    337      1.117  gmcgarry 
    338      1.117  gmcgarry /*
    339      1.255     skrll  * General yield call.  Puts the current LWP back on its run queue and
    340      1.117  gmcgarry  * performs a voluntary context switch.  Should only be called when the
    341      1.255     skrll  * current LWP explicitly requests it (eg sched_yield(2)).
    342      1.117  gmcgarry  */
    343      1.117  gmcgarry void
    344      1.117  gmcgarry yield(void)
    345      1.117  gmcgarry {
    346      1.122   thorpej 	struct lwp *l = curlwp;
    347      1.117  gmcgarry 
    348      1.174        ad 	KERNEL_UNLOCK_ALL(l, &l->l_biglocks);
    349      1.174        ad 	lwp_lock(l);
    350      1.217        ad 	KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_lwplock));
    351      1.188      yamt 	KASSERT(l->l_stat == LSONPROC);
    352      1.204        ad 	l->l_kpriority = false;
    353      1.188      yamt 	(void)mi_switch(l);
    354      1.174        ad 	KERNEL_LOCK(l->l_biglocks, l);
    355       1.69   thorpej }
    356       1.69   thorpej 
    357       1.69   thorpej /*
    358      1.255     skrll  * General preemption call.  Puts the current LWP back on its run queue
    359      1.156    rpaulo  * and performs an involuntary context switch.
    360       1.69   thorpej  */
    361       1.69   thorpej void
    362      1.174        ad preempt(void)
    363       1.69   thorpej {
    364      1.122   thorpej 	struct lwp *l = curlwp;
    365       1.69   thorpej 
    366      1.174        ad 	KERNEL_UNLOCK_ALL(l, &l->l_biglocks);
    367      1.174        ad 	lwp_lock(l);
    368      1.217        ad 	KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_lwplock));
    369      1.188      yamt 	KASSERT(l->l_stat == LSONPROC);
    370      1.204        ad 	l->l_kpriority = false;
    371      1.174        ad 	l->l_nivcsw++;
    372      1.188      yamt 	(void)mi_switch(l);
    373      1.174        ad 	KERNEL_LOCK(l->l_biglocks, l);
    374       1.69   thorpej }
    375       1.69   thorpej 
    376      1.234        ad /*
    377      1.234        ad  * Handle a request made by another agent to preempt the current LWP
    378      1.234        ad  * in-kernel.  Usually called when l_dopreempt may be non-zero.
    379      1.234        ad  *
    380      1.234        ad  * Character addresses for lockstat only.
    381      1.234        ad  */
    382      1.231        ad static char	in_critical_section;
    383      1.231        ad static char	kernel_lock_held;
    384      1.231        ad static char	is_softint;
    385      1.262      yamt static char	cpu_kpreempt_enter_fail;
    386      1.231        ad 
    387      1.231        ad bool
    388      1.231        ad kpreempt(uintptr_t where)
    389      1.231        ad {
    390      1.231        ad 	uintptr_t failed;
    391      1.231        ad 	lwp_t *l;
    392      1.264        ad 	int s, dop, lsflag;
    393      1.231        ad 
    394      1.231        ad 	l = curlwp;
    395      1.231        ad 	failed = 0;
    396      1.231        ad 	while ((dop = l->l_dopreempt) != 0) {
    397      1.231        ad 		if (l->l_stat != LSONPROC) {
    398      1.231        ad 			/*
    399      1.231        ad 			 * About to block (or die), let it happen.
    400      1.231        ad 			 * Doesn't really count as "preemption has
    401      1.231        ad 			 * been blocked", since we're going to
    402      1.231        ad 			 * context switch.
    403      1.231        ad 			 */
    404      1.231        ad 			l->l_dopreempt = 0;
    405      1.231        ad 			return true;
    406      1.231        ad 		}
    407      1.231        ad 		if (__predict_false((l->l_flag & LW_IDLE) != 0)) {
    408      1.231        ad 			/* Can't preempt idle loop, don't count as failure. */
    409      1.261     rmind 			l->l_dopreempt = 0;
    410      1.261     rmind 			return true;
    411      1.231        ad 		}
    412      1.231        ad 		if (__predict_false(l->l_nopreempt != 0)) {
    413      1.231        ad 			/* LWP holds preemption disabled, explicitly. */
    414      1.231        ad 			if ((dop & DOPREEMPT_COUNTED) == 0) {
    415      1.234        ad 				kpreempt_ev_crit.ev_count++;
    416      1.231        ad 			}
    417      1.231        ad 			failed = (uintptr_t)&in_critical_section;
    418      1.231        ad 			break;
    419      1.231        ad 		}
    420      1.231        ad 		if (__predict_false((l->l_pflag & LP_INTR) != 0)) {
    421      1.261     rmind 			/* Can't preempt soft interrupts yet. */
    422      1.261     rmind 			l->l_dopreempt = 0;
    423      1.261     rmind 			failed = (uintptr_t)&is_softint;
    424      1.261     rmind 			break;
    425      1.231        ad 		}
    426      1.231        ad 		s = splsched();
    427      1.231        ad 		if (__predict_false(l->l_blcnt != 0 ||
    428      1.231        ad 		    curcpu()->ci_biglock_wanted != NULL)) {
    429      1.231        ad 			/* Hold or want kernel_lock, code is not MT safe. */
    430      1.231        ad 			splx(s);
    431      1.231        ad 			if ((dop & DOPREEMPT_COUNTED) == 0) {
    432      1.234        ad 				kpreempt_ev_klock.ev_count++;
    433      1.231        ad 			}
    434      1.231        ad 			failed = (uintptr_t)&kernel_lock_held;
    435      1.231        ad 			break;
    436      1.231        ad 		}
    437      1.231        ad 		if (__predict_false(!cpu_kpreempt_enter(where, s))) {
    438      1.231        ad 			/*
    439      1.231        ad 			 * It may be that the IPL is too high.
    440      1.231        ad 			 * kpreempt_enter() can schedule an
    441      1.231        ad 			 * interrupt to retry later.
    442      1.231        ad 			 */
    443      1.231        ad 			splx(s);
    444      1.262      yamt 			failed = (uintptr_t)&cpu_kpreempt_enter_fail;
    445      1.231        ad 			break;
    446      1.231        ad 		}
    447      1.231        ad 		/* Do it! */
    448      1.231        ad 		if (__predict_true((dop & DOPREEMPT_COUNTED) == 0)) {
    449      1.234        ad 			kpreempt_ev_immed.ev_count++;
    450      1.231        ad 		}
    451      1.231        ad 		lwp_lock(l);
    452      1.231        ad 		mi_switch(l);
    453      1.231        ad 		l->l_nopreempt++;
    454      1.231        ad 		splx(s);
    455      1.231        ad 
    456      1.231        ad 		/* Take care of any MD cleanup. */
    457      1.231        ad 		cpu_kpreempt_exit(where);
    458      1.231        ad 		l->l_nopreempt--;
    459      1.231        ad 	}
    460      1.231        ad 
    461      1.264        ad 	if (__predict_true(!failed)) {
    462      1.264        ad 		return false;
    463      1.264        ad 	}
    464      1.264        ad 
    465      1.231        ad 	/* Record preemption failure for reporting via lockstat. */
    466      1.264        ad 	atomic_or_uint(&l->l_dopreempt, DOPREEMPT_COUNTED);
    467      1.264        ad 	lsflag = 0;
    468      1.264        ad 	LOCKSTAT_ENTER(lsflag);
    469      1.264        ad 	if (__predict_false(lsflag)) {
    470      1.264        ad 		if (where == 0) {
    471      1.264        ad 			where = (uintptr_t)__builtin_return_address(0);
    472      1.264        ad 		}
    473      1.264        ad 		/* Preemption is on, might recurse, so make it atomic. */
    474      1.264        ad 		if (atomic_cas_ptr_ni((void *)&l->l_pfailaddr, NULL,
    475      1.264        ad 		    (void *)where) == NULL) {
    476      1.264        ad 			LOCKSTAT_START_TIMER(lsflag, l->l_pfailtime);
    477      1.264        ad 			l->l_pfaillock = failed;
    478      1.231        ad 		}
    479      1.231        ad 	}
    480      1.264        ad 	LOCKSTAT_EXIT(lsflag);
    481      1.264        ad 	return true;
    482      1.231        ad }
    483      1.231        ad 
    484       1.69   thorpej /*
    485      1.231        ad  * Return true if preemption is explicitly disabled.
    486      1.230        ad  */
    487      1.231        ad bool
    488      1.231        ad kpreempt_disabled(void)
    489      1.231        ad {
    490      1.261     rmind 	const lwp_t *l = curlwp;
    491      1.231        ad 
    492      1.231        ad 	return l->l_nopreempt != 0 || l->l_stat == LSZOMB ||
    493      1.231        ad 	    (l->l_flag & LW_IDLE) != 0 || cpu_kpreempt_disabled();
    494      1.231        ad }
    495      1.230        ad 
    496      1.230        ad /*
    497      1.231        ad  * Disable kernel preemption.
    498      1.230        ad  */
    499      1.230        ad void
    500      1.231        ad kpreempt_disable(void)
    501      1.230        ad {
    502      1.230        ad 
    503      1.231        ad 	KPREEMPT_DISABLE(curlwp);
    504      1.230        ad }
    505      1.230        ad 
    506      1.230        ad /*
    507      1.231        ad  * Reenable kernel preemption.
    508      1.230        ad  */
    509      1.231        ad void
    510      1.231        ad kpreempt_enable(void)
    511      1.230        ad {
    512      1.230        ad 
    513      1.231        ad 	KPREEMPT_ENABLE(curlwp);
    514      1.230        ad }
    515      1.230        ad 
    516      1.230        ad /*
    517      1.188      yamt  * Compute the amount of time during which the current lwp was running.
    518      1.130   nathanw  *
    519      1.188      yamt  * - update l_rtime unless it's an idle lwp.
    520      1.188      yamt  */
    521      1.188      yamt 
    522      1.199        ad void
    523      1.212      yamt updatertime(lwp_t *l, const struct bintime *now)
    524      1.188      yamt {
    525      1.188      yamt 
    526      1.261     rmind 	if (__predict_false(l->l_flag & LW_IDLE))
    527      1.188      yamt 		return;
    528      1.188      yamt 
    529      1.212      yamt 	/* rtime += now - stime */
    530      1.212      yamt 	bintime_add(&l->l_rtime, now);
    531      1.212      yamt 	bintime_sub(&l->l_rtime, &l->l_stime);
    532      1.188      yamt }
    533      1.188      yamt 
    534      1.188      yamt /*
    535      1.245        ad  * Select next LWP from the current CPU to run..
    536      1.245        ad  */
    537      1.245        ad static inline lwp_t *
    538      1.245        ad nextlwp(struct cpu_info *ci, struct schedstate_percpu *spc)
    539      1.245        ad {
    540      1.245        ad 	lwp_t *newl;
    541      1.245        ad 
    542      1.245        ad 	/*
    543      1.245        ad 	 * Let sched_nextlwp() select the LWP to run the CPU next.
    544      1.245        ad 	 * If no LWP is runnable, select the idle LWP.
    545      1.245        ad 	 *
    546      1.245        ad 	 * Note that spc_lwplock might not necessary be held, and
    547      1.245        ad 	 * new thread would be unlocked after setting the LWP-lock.
    548      1.245        ad 	 */
    549      1.245        ad 	newl = sched_nextlwp();
    550      1.245        ad 	if (newl != NULL) {
    551      1.245        ad 		sched_dequeue(newl);
    552      1.245        ad 		KASSERT(lwp_locked(newl, spc->spc_mutex));
    553      1.274     rmind 		KASSERT(newl->l_cpu == ci);
    554      1.245        ad 		newl->l_stat = LSONPROC;
    555      1.248        ad 		newl->l_pflag |= LP_RUNNING;
    556      1.245        ad 		lwp_setlock(newl, spc->spc_lwplock);
    557      1.245        ad 	} else {
    558      1.245        ad 		newl = ci->ci_data.cpu_idlelwp;
    559      1.245        ad 		newl->l_stat = LSONPROC;
    560      1.248        ad 		newl->l_pflag |= LP_RUNNING;
    561      1.245        ad 	}
    562      1.261     rmind 
    563      1.245        ad 	/*
    564      1.245        ad 	 * Only clear want_resched if there are no pending (slow)
    565      1.245        ad 	 * software interrupts.
    566      1.245        ad 	 */
    567      1.245        ad 	ci->ci_want_resched = ci->ci_data.cpu_softints;
    568      1.245        ad 	spc->spc_flags &= ~SPCF_SWITCHCLEAR;
    569      1.245        ad 	spc->spc_curpriority = lwp_eprio(newl);
    570      1.245        ad 
    571      1.245        ad 	return newl;
    572      1.245        ad }
    573      1.245        ad 
    574      1.245        ad /*
    575      1.188      yamt  * The machine independent parts of context switch.
    576      1.188      yamt  *
    577      1.188      yamt  * Returns 1 if another LWP was actually run.
    578       1.26       cgd  */
    579      1.122   thorpej int
    580      1.199        ad mi_switch(lwp_t *l)
    581       1.26       cgd {
    582      1.246     rmind 	struct cpu_info *ci;
    583       1.76   thorpej 	struct schedstate_percpu *spc;
    584      1.188      yamt 	struct lwp *newl;
    585      1.174        ad 	int retval, oldspl;
    586      1.212      yamt 	struct bintime bt;
    587      1.199        ad 	bool returning;
    588       1.26       cgd 
    589      1.188      yamt 	KASSERT(lwp_locked(l, NULL));
    590      1.231        ad 	KASSERT(kpreempt_disabled());
    591      1.188      yamt 	LOCKDEBUG_BARRIER(l->l_mutex, 1);
    592      1.174        ad 
    593      1.174        ad 	kstack_check_magic(l);
    594       1.83   thorpej 
    595      1.212      yamt 	binuptime(&bt);
    596      1.199        ad 
    597      1.267      yamt 	KASSERT((l->l_pflag & LP_RUNNING) != 0);
    598      1.231        ad 	KASSERT(l->l_cpu == curcpu());
    599      1.196        ad 	ci = l->l_cpu;
    600      1.196        ad 	spc = &ci->ci_schedstate;
    601      1.199        ad 	returning = false;
    602      1.190        ad 	newl = NULL;
    603      1.190        ad 
    604      1.199        ad 	/*
    605      1.199        ad 	 * If we have been asked to switch to a specific LWP, then there
    606      1.199        ad 	 * is no need to inspect the run queues.  If a soft interrupt is
    607      1.199        ad 	 * blocking, then return to the interrupted thread without adjusting
    608      1.199        ad 	 * VM context or its start time: neither have been changed in order
    609      1.199        ad 	 * to take the interrupt.
    610      1.199        ad 	 */
    611      1.190        ad 	if (l->l_switchto != NULL) {
    612      1.204        ad 		if ((l->l_pflag & LP_INTR) != 0) {
    613      1.199        ad 			returning = true;
    614      1.199        ad 			softint_block(l);
    615      1.248        ad 			if ((l->l_pflag & LP_TIMEINTR) != 0)
    616      1.212      yamt 				updatertime(l, &bt);
    617      1.199        ad 		}
    618      1.190        ad 		newl = l->l_switchto;
    619      1.190        ad 		l->l_switchto = NULL;
    620      1.190        ad 	}
    621      1.204        ad #ifndef __HAVE_FAST_SOFTINTS
    622      1.204        ad 	else if (ci->ci_data.cpu_softints != 0) {
    623      1.204        ad 		/* There are pending soft interrupts, so pick one. */
    624      1.204        ad 		newl = softint_picklwp();
    625      1.204        ad 		newl->l_stat = LSONPROC;
    626      1.248        ad 		newl->l_pflag |= LP_RUNNING;
    627      1.204        ad 	}
    628      1.204        ad #endif	/* !__HAVE_FAST_SOFTINTS */
    629      1.190        ad 
    630      1.180       dsl 	/* Count time spent in current system call */
    631      1.199        ad 	if (!returning) {
    632      1.199        ad 		SYSCALL_TIME_SLEEP(l);
    633      1.180       dsl 
    634      1.199        ad 		/*
    635      1.199        ad 		 * XXXSMP If we are using h/w performance counters,
    636      1.199        ad 		 * save context.
    637      1.199        ad 		 */
    638      1.174        ad #if PERFCTRS
    639      1.199        ad 		if (PMC_ENABLED(l->l_proc)) {
    640      1.199        ad 			pmc_save_context(l->l_proc);
    641      1.199        ad 		}
    642      1.199        ad #endif
    643      1.212      yamt 		updatertime(l, &bt);
    644      1.174        ad 	}
    645      1.113  gmcgarry 
    646      1.246     rmind 	/* Lock the runqueue */
    647      1.246     rmind 	KASSERT(l->l_stat != LSRUN);
    648      1.246     rmind 	mutex_spin_enter(spc->spc_mutex);
    649      1.246     rmind 
    650      1.113  gmcgarry 	/*
    651      1.174        ad 	 * If on the CPU and we have gotten this far, then we must yield.
    652      1.113  gmcgarry 	 */
    653      1.246     rmind 	if (l->l_stat == LSONPROC && l != newl) {
    654      1.217        ad 		KASSERT(lwp_locked(l, spc->spc_lwplock));
    655      1.188      yamt 		if ((l->l_flag & LW_IDLE) == 0) {
    656      1.188      yamt 			l->l_stat = LSRUN;
    657      1.246     rmind 			lwp_setlock(l, spc->spc_mutex);
    658      1.246     rmind 			sched_enqueue(l, true);
    659      1.285     rmind 			/*
    660      1.285     rmind 			 * Handle migration.  Note that "migrating LWP" may
    661      1.285     rmind 			 * be reset here, if interrupt/preemption happens
    662      1.285     rmind 			 * early in idle LWP.
    663      1.285     rmind 			 */
    664      1.285     rmind 			if (l->l_target_cpu != NULL) {
    665      1.285     rmind 				KASSERT((l->l_pflag & LP_INTR) == 0);
    666      1.246     rmind 				spc->spc_migrating = l;
    667      1.216     rmind 			}
    668      1.246     rmind 		} else
    669      1.188      yamt 			l->l_stat = LSIDL;
    670      1.174        ad 	}
    671      1.174        ad 
    672      1.245        ad 	/* Pick new LWP to run. */
    673      1.190        ad 	if (newl == NULL) {
    674      1.245        ad 		newl = nextlwp(ci, spc);
    675      1.199        ad 	}
    676      1.199        ad 
    677      1.204        ad 	/* Items that must be updated with the CPU locked. */
    678      1.199        ad 	if (!returning) {
    679      1.204        ad 		/* Update the new LWP's start time. */
    680      1.212      yamt 		newl->l_stime = bt;
    681      1.204        ad 
    682      1.199        ad 		/*
    683      1.204        ad 		 * ci_curlwp changes when a fast soft interrupt occurs.
    684      1.204        ad 		 * We use cpu_onproc to keep track of which kernel or
    685      1.204        ad 		 * user thread is running 'underneath' the software
    686      1.204        ad 		 * interrupt.  This is important for time accounting,
    687      1.204        ad 		 * itimers and forcing user threads to preempt (aston).
    688      1.199        ad 		 */
    689      1.204        ad 		ci->ci_data.cpu_onproc = newl;
    690      1.188      yamt 	}
    691      1.188      yamt 
    692      1.241        ad 	/*
    693      1.241        ad 	 * Preemption related tasks.  Must be done with the current
    694      1.241        ad 	 * CPU locked.
    695      1.241        ad 	 */
    696      1.241        ad 	cpu_did_resched(l);
    697      1.231        ad 	l->l_dopreempt = 0;
    698      1.231        ad 	if (__predict_false(l->l_pfailaddr != 0)) {
    699      1.231        ad 		LOCKSTAT_FLAG(lsflag);
    700      1.231        ad 		LOCKSTAT_ENTER(lsflag);
    701      1.231        ad 		LOCKSTAT_STOP_TIMER(lsflag, l->l_pfailtime);
    702      1.231        ad 		LOCKSTAT_EVENT_RA(lsflag, l->l_pfaillock, LB_NOPREEMPT|LB_SPIN,
    703      1.231        ad 		    1, l->l_pfailtime, l->l_pfailaddr);
    704      1.231        ad 		LOCKSTAT_EXIT(lsflag);
    705      1.231        ad 		l->l_pfailtime = 0;
    706      1.231        ad 		l->l_pfaillock = 0;
    707      1.231        ad 		l->l_pfailaddr = 0;
    708      1.231        ad 	}
    709      1.231        ad 
    710      1.188      yamt 	if (l != newl) {
    711      1.188      yamt 		struct lwp *prevlwp;
    712      1.174        ad 
    713      1.209        ad 		/* Release all locks, but leave the current LWP locked */
    714      1.246     rmind 		if (l->l_mutex == spc->spc_mutex) {
    715      1.209        ad 			/*
    716      1.209        ad 			 * Drop spc_lwplock, if the current LWP has been moved
    717      1.209        ad 			 * to the run queue (it is now locked by spc_mutex).
    718      1.209        ad 			 */
    719      1.217        ad 			mutex_spin_exit(spc->spc_lwplock);
    720      1.188      yamt 		} else {
    721      1.209        ad 			/*
    722      1.209        ad 			 * Otherwise, drop the spc_mutex, we are done with the
    723      1.209        ad 			 * run queues.
    724      1.209        ad 			 */
    725      1.188      yamt 			mutex_spin_exit(spc->spc_mutex);
    726      1.188      yamt 		}
    727      1.188      yamt 
    728      1.209        ad 		/*
    729      1.253     skrll 		 * Mark that context switch is going to be performed
    730      1.209        ad 		 * for this LWP, to protect it from being switched
    731      1.209        ad 		 * to on another CPU.
    732      1.209        ad 		 */
    733      1.209        ad 		KASSERT(l->l_ctxswtch == 0);
    734      1.209        ad 		l->l_ctxswtch = 1;
    735      1.209        ad 		l->l_ncsw++;
    736      1.267      yamt 		KASSERT((l->l_pflag & LP_RUNNING) != 0);
    737      1.248        ad 		l->l_pflag &= ~LP_RUNNING;
    738      1.209        ad 
    739      1.209        ad 		/*
    740      1.209        ad 		 * Increase the count of spin-mutexes before the release
    741      1.209        ad 		 * of the last lock - we must remain at IPL_SCHED during
    742      1.209        ad 		 * the context switch.
    743      1.209        ad 		 */
    744  1.286.2.1    jruoho 		KASSERTMSG(ci->ci_mtx_count == -1,
    745  1.286.2.1    jruoho 		    ("%s: cpu%u: ci_mtx_count (%d) != -1",
    746  1.286.2.1    jruoho 		     __func__, cpu_index(ci), ci->ci_mtx_count));
    747      1.209        ad 		oldspl = MUTEX_SPIN_OLDSPL(ci);
    748      1.209        ad 		ci->ci_mtx_count--;
    749      1.209        ad 		lwp_unlock(l);
    750      1.209        ad 
    751      1.218        ad 		/* Count the context switch on this CPU. */
    752      1.218        ad 		ci->ci_data.cpu_nswtch++;
    753      1.188      yamt 
    754      1.209        ad 		/* Update status for lwpctl, if present. */
    755      1.209        ad 		if (l->l_lwpctl != NULL)
    756      1.209        ad 			l->l_lwpctl->lc_curcpu = LWPCTL_CPU_NONE;
    757      1.209        ad 
    758      1.199        ad 		/*
    759      1.199        ad 		 * Save old VM context, unless a soft interrupt
    760      1.199        ad 		 * handler is blocking.
    761      1.199        ad 		 */
    762      1.199        ad 		if (!returning)
    763      1.199        ad 			pmap_deactivate(l);
    764      1.188      yamt 
    765      1.209        ad 		/*
    766      1.275     skrll 		 * We may need to spin-wait if 'newl' is still
    767      1.209        ad 		 * context switching on another CPU.
    768      1.209        ad 		 */
    769      1.261     rmind 		if (__predict_false(newl->l_ctxswtch != 0)) {
    770      1.209        ad 			u_int count;
    771      1.209        ad 			count = SPINLOCK_BACKOFF_MIN;
    772      1.209        ad 			while (newl->l_ctxswtch)
    773      1.209        ad 				SPINLOCK_BACKOFF(count);
    774      1.209        ad 		}
    775      1.207        ad 
    776      1.276    darran 		/*
    777      1.276    darran 		 * If DTrace has set the active vtime enum to anything
    778      1.276    darran 		 * other than INACTIVE (0), then it should have set the
    779      1.276    darran 		 * function to call.
    780      1.276    darran 		 */
    781      1.278    darran 		if (__predict_false(dtrace_vtime_active)) {
    782      1.276    darran 			(*dtrace_vtime_switch_func)(newl);
    783      1.276    darran 		}
    784      1.276    darran 
    785      1.188      yamt 		/* Switch to the new LWP.. */
    786      1.204        ad 		prevlwp = cpu_switchto(l, newl, returning);
    787      1.207        ad 		ci = curcpu();
    788      1.207        ad 
    789      1.188      yamt 		/*
    790      1.209        ad 		 * Switched away - we have new curlwp.
    791      1.209        ad 		 * Restore VM context and IPL.
    792      1.188      yamt 		 */
    793      1.209        ad 		pmap_activate(l);
    794      1.265     rmind 		uvm_emap_switch(l);
    795  1.286.2.1    jruoho 		pcu_switchpoint(l);
    796      1.265     rmind 
    797      1.188      yamt 		if (prevlwp != NULL) {
    798      1.209        ad 			/* Normalize the count of the spin-mutexes */
    799      1.209        ad 			ci->ci_mtx_count++;
    800      1.209        ad 			/* Unmark the state of context switch */
    801      1.209        ad 			membar_exit();
    802      1.209        ad 			prevlwp->l_ctxswtch = 0;
    803      1.188      yamt 		}
    804      1.209        ad 
    805      1.209        ad 		/* Update status for lwpctl, if present. */
    806      1.219        ad 		if (l->l_lwpctl != NULL) {
    807      1.209        ad 			l->l_lwpctl->lc_curcpu = (int)cpu_index(ci);
    808      1.219        ad 			l->l_lwpctl->lc_pctr++;
    809      1.219        ad 		}
    810      1.174        ad 
    811      1.231        ad 		KASSERT(l->l_cpu == ci);
    812      1.231        ad 		splx(oldspl);
    813      1.188      yamt 		retval = 1;
    814      1.188      yamt 	} else {
    815      1.188      yamt 		/* Nothing to do - just unlock and return. */
    816      1.246     rmind 		mutex_spin_exit(spc->spc_mutex);
    817      1.188      yamt 		lwp_unlock(l);
    818      1.122   thorpej 		retval = 0;
    819      1.122   thorpej 	}
    820      1.110    briggs 
    821      1.188      yamt 	KASSERT(l == curlwp);
    822      1.188      yamt 	KASSERT(l->l_stat == LSONPROC);
    823      1.188      yamt 
    824      1.110    briggs 	/*
    825      1.174        ad 	 * XXXSMP If we are using h/w performance counters, restore context.
    826      1.231        ad 	 * XXXSMP preemption problem.
    827       1.26       cgd 	 */
    828      1.114  gmcgarry #if PERFCTRS
    829      1.175  christos 	if (PMC_ENABLED(l->l_proc)) {
    830      1.175  christos 		pmc_restore_context(l->l_proc);
    831      1.166  christos 	}
    832      1.114  gmcgarry #endif
    833      1.180       dsl 	SYSCALL_TIME_WAKEUP(l);
    834      1.188      yamt 	LOCKDEBUG_BARRIER(NULL, 1);
    835      1.169      yamt 
    836      1.122   thorpej 	return retval;
    837       1.26       cgd }
    838       1.26       cgd 
    839       1.26       cgd /*
    840      1.245        ad  * The machine independent parts of context switch to oblivion.
    841      1.245        ad  * Does not return.  Call with the LWP unlocked.
    842      1.245        ad  */
    843      1.245        ad void
    844      1.245        ad lwp_exit_switchaway(lwp_t *l)
    845      1.245        ad {
    846      1.245        ad 	struct cpu_info *ci;
    847      1.245        ad 	struct lwp *newl;
    848      1.245        ad 	struct bintime bt;
    849      1.245        ad 
    850      1.245        ad 	ci = l->l_cpu;
    851      1.245        ad 
    852      1.245        ad 	KASSERT(kpreempt_disabled());
    853      1.245        ad 	KASSERT(l->l_stat == LSZOMB || l->l_stat == LSIDL);
    854      1.245        ad 	KASSERT(ci == curcpu());
    855      1.245        ad 	LOCKDEBUG_BARRIER(NULL, 0);
    856      1.245        ad 
    857      1.245        ad 	kstack_check_magic(l);
    858      1.245        ad 
    859      1.245        ad 	/* Count time spent in current system call */
    860      1.245        ad 	SYSCALL_TIME_SLEEP(l);
    861      1.245        ad 	binuptime(&bt);
    862      1.245        ad 	updatertime(l, &bt);
    863      1.245        ad 
    864      1.245        ad 	/* Must stay at IPL_SCHED even after releasing run queue lock. */
    865      1.245        ad 	(void)splsched();
    866      1.245        ad 
    867      1.245        ad 	/*
    868      1.245        ad 	 * Let sched_nextlwp() select the LWP to run the CPU next.
    869      1.245        ad 	 * If no LWP is runnable, select the idle LWP.
    870      1.245        ad 	 *
    871      1.245        ad 	 * Note that spc_lwplock might not necessary be held, and
    872      1.245        ad 	 * new thread would be unlocked after setting the LWP-lock.
    873      1.245        ad 	 */
    874      1.245        ad 	spc_lock(ci);
    875      1.245        ad #ifndef __HAVE_FAST_SOFTINTS
    876      1.245        ad 	if (ci->ci_data.cpu_softints != 0) {
    877      1.245        ad 		/* There are pending soft interrupts, so pick one. */
    878      1.245        ad 		newl = softint_picklwp();
    879      1.245        ad 		newl->l_stat = LSONPROC;
    880      1.248        ad 		newl->l_pflag |= LP_RUNNING;
    881      1.245        ad 	} else
    882      1.245        ad #endif	/* !__HAVE_FAST_SOFTINTS */
    883      1.245        ad 	{
    884      1.245        ad 		newl = nextlwp(ci, &ci->ci_schedstate);
    885      1.245        ad 	}
    886      1.245        ad 
    887      1.245        ad 	/* Update the new LWP's start time. */
    888      1.245        ad 	newl->l_stime = bt;
    889      1.248        ad 	l->l_pflag &= ~LP_RUNNING;
    890      1.245        ad 
    891      1.245        ad 	/*
    892      1.245        ad 	 * ci_curlwp changes when a fast soft interrupt occurs.
    893      1.245        ad 	 * We use cpu_onproc to keep track of which kernel or
    894      1.245        ad 	 * user thread is running 'underneath' the software
    895      1.245        ad 	 * interrupt.  This is important for time accounting,
    896      1.245        ad 	 * itimers and forcing user threads to preempt (aston).
    897      1.245        ad 	 */
    898      1.245        ad 	ci->ci_data.cpu_onproc = newl;
    899      1.245        ad 
    900      1.245        ad 	/*
    901      1.245        ad 	 * Preemption related tasks.  Must be done with the current
    902      1.245        ad 	 * CPU locked.
    903      1.245        ad 	 */
    904      1.245        ad 	cpu_did_resched(l);
    905      1.245        ad 
    906      1.245        ad 	/* Unlock the run queue. */
    907      1.245        ad 	spc_unlock(ci);
    908      1.245        ad 
    909      1.245        ad 	/* Count the context switch on this CPU. */
    910      1.245        ad 	ci->ci_data.cpu_nswtch++;
    911      1.245        ad 
    912      1.245        ad 	/* Update status for lwpctl, if present. */
    913      1.245        ad 	if (l->l_lwpctl != NULL)
    914      1.247        ad 		l->l_lwpctl->lc_curcpu = LWPCTL_CPU_EXITED;
    915      1.245        ad 
    916      1.245        ad 	/*
    917      1.275     skrll 	 * We may need to spin-wait if 'newl' is still
    918      1.245        ad 	 * context switching on another CPU.
    919      1.245        ad 	 */
    920      1.261     rmind 	if (__predict_false(newl->l_ctxswtch != 0)) {
    921      1.245        ad 		u_int count;
    922      1.245        ad 		count = SPINLOCK_BACKOFF_MIN;
    923      1.245        ad 		while (newl->l_ctxswtch)
    924      1.245        ad 			SPINLOCK_BACKOFF(count);
    925      1.245        ad 	}
    926      1.245        ad 
    927      1.279    darran 	/*
    928      1.279    darran 	 * If DTrace has set the active vtime enum to anything
    929      1.279    darran 	 * other than INACTIVE (0), then it should have set the
    930      1.279    darran 	 * function to call.
    931      1.279    darran 	 */
    932      1.279    darran 	if (__predict_false(dtrace_vtime_active)) {
    933      1.279    darran 		(*dtrace_vtime_switch_func)(newl);
    934      1.279    darran 	}
    935      1.276    darran 
    936      1.245        ad 	/* Switch to the new LWP.. */
    937      1.245        ad 	(void)cpu_switchto(NULL, newl, false);
    938      1.245        ad 
    939      1.251       uwe 	for (;;) continue;	/* XXX: convince gcc about "noreturn" */
    940      1.245        ad 	/* NOTREACHED */
    941      1.245        ad }
    942      1.245        ad 
    943      1.245        ad /*
    944      1.271     rmind  * setrunnable: change LWP state to be runnable, placing it on the run queue.
    945      1.174        ad  *
    946      1.174        ad  * Call with the process and LWP locked.  Will return with the LWP unlocked.
    947       1.26       cgd  */
    948       1.26       cgd void
    949      1.122   thorpej setrunnable(struct lwp *l)
    950       1.26       cgd {
    951      1.122   thorpej 	struct proc *p = l->l_proc;
    952      1.205        ad 	struct cpu_info *ci;
    953       1.26       cgd 
    954      1.188      yamt 	KASSERT((l->l_flag & LW_IDLE) == 0);
    955      1.229        ad 	KASSERT(mutex_owned(p->p_lock));
    956      1.183        ad 	KASSERT(lwp_locked(l, NULL));
    957      1.205        ad 	KASSERT(l->l_mutex != l->l_cpu->ci_schedstate.spc_mutex);
    958       1.83   thorpej 
    959      1.122   thorpej 	switch (l->l_stat) {
    960      1.122   thorpej 	case LSSTOP:
    961       1.33   mycroft 		/*
    962       1.33   mycroft 		 * If we're being traced (possibly because someone attached us
    963       1.33   mycroft 		 * while we were stopped), check for a signal from the debugger.
    964       1.33   mycroft 		 */
    965      1.256        ad 		if ((p->p_slflag & PSL_TRACED) != 0 && p->p_xstat != 0)
    966      1.174        ad 			signotify(l);
    967      1.174        ad 		p->p_nrlwps++;
    968       1.26       cgd 		break;
    969      1.174        ad 	case LSSUSPENDED:
    970      1.178     pavel 		l->l_flag &= ~LW_WSUSPEND;
    971      1.174        ad 		p->p_nrlwps++;
    972      1.192     rmind 		cv_broadcast(&p->p_lwpcv);
    973      1.122   thorpej 		break;
    974      1.174        ad 	case LSSLEEP:
    975      1.174        ad 		KASSERT(l->l_wchan != NULL);
    976       1.26       cgd 		break;
    977      1.174        ad 	default:
    978      1.174        ad 		panic("setrunnable: lwp %p state was %d", l, l->l_stat);
    979       1.26       cgd 	}
    980      1.139        cl 
    981      1.252  wrstuden #ifdef KERN_SA
    982      1.252  wrstuden 	if (l->l_proc->p_sa)
    983      1.252  wrstuden 		sa_awaken(l);
    984      1.252  wrstuden #endif /* KERN_SA */
    985      1.252  wrstuden 
    986      1.174        ad 	/*
    987      1.286     pooka 	 * If the LWP was sleeping, start it again.
    988      1.174        ad 	 */
    989      1.174        ad 	if (l->l_wchan != NULL) {
    990      1.174        ad 		l->l_stat = LSSLEEP;
    991      1.183        ad 		/* lwp_unsleep() will release the lock. */
    992      1.221        ad 		lwp_unsleep(l, true);
    993      1.174        ad 		return;
    994      1.174        ad 	}
    995      1.139        cl 
    996      1.174        ad 	/*
    997      1.174        ad 	 * If the LWP is still on the CPU, mark it as LSONPROC.  It may be
    998      1.174        ad 	 * about to call mi_switch(), in which case it will yield.
    999      1.174        ad 	 */
   1000      1.248        ad 	if ((l->l_pflag & LP_RUNNING) != 0) {
   1001      1.174        ad 		l->l_stat = LSONPROC;
   1002      1.174        ad 		l->l_slptime = 0;
   1003      1.174        ad 		lwp_unlock(l);
   1004      1.174        ad 		return;
   1005      1.174        ad 	}
   1006      1.122   thorpej 
   1007      1.174        ad 	/*
   1008      1.205        ad 	 * Look for a CPU to run.
   1009      1.205        ad 	 * Set the LWP runnable.
   1010      1.174        ad 	 */
   1011      1.205        ad 	ci = sched_takecpu(l);
   1012      1.205        ad 	l->l_cpu = ci;
   1013      1.236        ad 	spc_lock(ci);
   1014      1.236        ad 	lwp_unlock_to(l, ci->ci_schedstate.spc_mutex);
   1015      1.188      yamt 	sched_setrunnable(l);
   1016      1.174        ad 	l->l_stat = LSRUN;
   1017      1.122   thorpej 	l->l_slptime = 0;
   1018      1.174        ad 
   1019      1.271     rmind 	sched_enqueue(l, false);
   1020      1.271     rmind 	resched_cpu(l);
   1021      1.271     rmind 	lwp_unlock(l);
   1022       1.26       cgd }
   1023       1.26       cgd 
   1024       1.26       cgd /*
   1025      1.174        ad  * suspendsched:
   1026      1.174        ad  *
   1027      1.266      yamt  *	Convert all non-LW_SYSTEM LSSLEEP or LSRUN LWPs to LSSUSPENDED.
   1028      1.174        ad  */
   1029       1.94    bouyer void
   1030      1.174        ad suspendsched(void)
   1031       1.94    bouyer {
   1032      1.174        ad 	CPU_INFO_ITERATOR cii;
   1033      1.174        ad 	struct cpu_info *ci;
   1034      1.122   thorpej 	struct lwp *l;
   1035      1.174        ad 	struct proc *p;
   1036       1.94    bouyer 
   1037       1.94    bouyer 	/*
   1038      1.174        ad 	 * We do this by process in order not to violate the locking rules.
   1039       1.94    bouyer 	 */
   1040      1.228        ad 	mutex_enter(proc_lock);
   1041      1.174        ad 	PROCLIST_FOREACH(p, &allproc) {
   1042      1.229        ad 		mutex_enter(p->p_lock);
   1043      1.178     pavel 		if ((p->p_flag & PK_SYSTEM) != 0) {
   1044      1.229        ad 			mutex_exit(p->p_lock);
   1045       1.94    bouyer 			continue;
   1046      1.174        ad 		}
   1047      1.174        ad 
   1048      1.174        ad 		p->p_stat = SSTOP;
   1049      1.174        ad 
   1050      1.174        ad 		LIST_FOREACH(l, &p->p_lwps, l_sibling) {
   1051      1.174        ad 			if (l == curlwp)
   1052      1.174        ad 				continue;
   1053      1.174        ad 
   1054      1.174        ad 			lwp_lock(l);
   1055      1.122   thorpej 
   1056       1.97     enami 			/*
   1057      1.174        ad 			 * Set L_WREBOOT so that the LWP will suspend itself
   1058      1.174        ad 			 * when it tries to return to user mode.  We want to
   1059      1.174        ad 			 * try and get to get as many LWPs as possible to
   1060      1.174        ad 			 * the user / kernel boundary, so that they will
   1061      1.174        ad 			 * release any locks that they hold.
   1062       1.97     enami 			 */
   1063      1.178     pavel 			l->l_flag |= (LW_WREBOOT | LW_WSUSPEND);
   1064      1.174        ad 
   1065      1.174        ad 			if (l->l_stat == LSSLEEP &&
   1066      1.178     pavel 			    (l->l_flag & LW_SINTR) != 0) {
   1067      1.174        ad 				/* setrunnable() will release the lock. */
   1068      1.174        ad 				setrunnable(l);
   1069      1.174        ad 				continue;
   1070      1.174        ad 			}
   1071      1.174        ad 
   1072      1.174        ad 			lwp_unlock(l);
   1073       1.94    bouyer 		}
   1074      1.174        ad 
   1075      1.229        ad 		mutex_exit(p->p_lock);
   1076       1.94    bouyer 	}
   1077      1.228        ad 	mutex_exit(proc_lock);
   1078      1.174        ad 
   1079      1.174        ad 	/*
   1080      1.174        ad 	 * Kick all CPUs to make them preempt any LWPs running in user mode.
   1081      1.174        ad 	 * They'll trap into the kernel and suspend themselves in userret().
   1082      1.174        ad 	 */
   1083      1.204        ad 	for (CPU_INFO_FOREACH(cii, ci)) {
   1084      1.204        ad 		spc_lock(ci);
   1085      1.204        ad 		cpu_need_resched(ci, RESCHED_IMMED);
   1086      1.204        ad 		spc_unlock(ci);
   1087      1.204        ad 	}
   1088      1.174        ad }
   1089      1.174        ad 
   1090      1.174        ad /*
   1091      1.174        ad  * sched_unsleep:
   1092      1.174        ad  *
   1093      1.174        ad  *	The is called when the LWP has not been awoken normally but instead
   1094      1.174        ad  *	interrupted: for example, if the sleep timed out.  Because of this,
   1095      1.174        ad  *	it's not a valid action for running or idle LWPs.
   1096      1.174        ad  */
   1097      1.271     rmind static void
   1098      1.221        ad sched_unsleep(struct lwp *l, bool cleanup)
   1099      1.174        ad {
   1100      1.174        ad 
   1101      1.174        ad 	lwp_unlock(l);
   1102      1.174        ad 	panic("sched_unsleep");
   1103      1.174        ad }
   1104      1.174        ad 
   1105      1.250     rmind static void
   1106      1.188      yamt resched_cpu(struct lwp *l)
   1107      1.188      yamt {
   1108      1.274     rmind 	struct cpu_info *ci = l->l_cpu;
   1109      1.188      yamt 
   1110      1.250     rmind 	KASSERT(lwp_locked(l, NULL));
   1111      1.204        ad 	if (lwp_eprio(l) > ci->ci_schedstate.spc_curpriority)
   1112      1.188      yamt 		cpu_need_resched(ci, 0);
   1113      1.188      yamt }
   1114      1.188      yamt 
   1115      1.188      yamt static void
   1116      1.185      yamt sched_changepri(struct lwp *l, pri_t pri)
   1117      1.174        ad {
   1118      1.174        ad 
   1119      1.188      yamt 	KASSERT(lwp_locked(l, NULL));
   1120      1.174        ad 
   1121      1.271     rmind 	if (l->l_stat == LSRUN) {
   1122      1.204        ad 		KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_mutex));
   1123      1.204        ad 		sched_dequeue(l);
   1124      1.204        ad 		l->l_priority = pri;
   1125      1.204        ad 		sched_enqueue(l, false);
   1126      1.204        ad 	} else {
   1127      1.174        ad 		l->l_priority = pri;
   1128      1.157      yamt 	}
   1129      1.188      yamt 	resched_cpu(l);
   1130      1.184      yamt }
   1131      1.184      yamt 
   1132      1.188      yamt static void
   1133      1.185      yamt sched_lendpri(struct lwp *l, pri_t pri)
   1134      1.184      yamt {
   1135      1.184      yamt 
   1136      1.188      yamt 	KASSERT(lwp_locked(l, NULL));
   1137      1.184      yamt 
   1138      1.271     rmind 	if (l->l_stat == LSRUN) {
   1139      1.204        ad 		KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_mutex));
   1140      1.204        ad 		sched_dequeue(l);
   1141      1.204        ad 		l->l_inheritedprio = pri;
   1142      1.204        ad 		sched_enqueue(l, false);
   1143      1.204        ad 	} else {
   1144      1.184      yamt 		l->l_inheritedprio = pri;
   1145      1.184      yamt 	}
   1146      1.188      yamt 	resched_cpu(l);
   1147      1.184      yamt }
   1148      1.184      yamt 
   1149      1.184      yamt struct lwp *
   1150      1.184      yamt syncobj_noowner(wchan_t wchan)
   1151      1.184      yamt {
   1152      1.184      yamt 
   1153      1.184      yamt 	return NULL;
   1154      1.151      yamt }
   1155      1.151      yamt 
   1156      1.250     rmind /* Decay 95% of proc::p_pctcpu in 60 seconds, ccpu = exp(-1/20) */
   1157      1.281     rmind const fixpt_t ccpu = 0.95122942450071400909 * FSCALE;
   1158      1.281     rmind 
   1159      1.281     rmind /*
   1160      1.281     rmind  * Constants for averages over 1, 5 and 15 minutes when sampling at
   1161      1.281     rmind  * 5 second intervals.
   1162      1.281     rmind  */
   1163      1.281     rmind static const fixpt_t cexp[ ] = {
   1164      1.281     rmind 	0.9200444146293232 * FSCALE,	/* exp(-1/12) */
   1165      1.281     rmind 	0.9834714538216174 * FSCALE,	/* exp(-1/60) */
   1166      1.281     rmind 	0.9944598480048967 * FSCALE,	/* exp(-1/180) */
   1167      1.281     rmind };
   1168      1.134      matt 
   1169      1.134      matt /*
   1170      1.188      yamt  * sched_pstats:
   1171      1.188      yamt  *
   1172      1.281     rmind  * => Update process statistics and check CPU resource allocation.
   1173      1.281     rmind  * => Call scheduler-specific hook to eventually adjust LWP priorities.
   1174      1.281     rmind  * => Compute load average of a quantity on 1, 5 and 15 minute intervals.
   1175      1.130   nathanw  */
   1176      1.113  gmcgarry void
   1177      1.281     rmind sched_pstats(void)
   1178      1.113  gmcgarry {
   1179      1.281     rmind 	extern struct loadavg averunnable;
   1180      1.281     rmind 	struct loadavg *avg = &averunnable;
   1181      1.249     rmind 	const int clkhz = (stathz != 0 ? stathz : hz);
   1182      1.281     rmind 	static bool backwards = false;
   1183      1.281     rmind 	static u_int lavg_count = 0;
   1184      1.188      yamt 	struct proc *p;
   1185      1.281     rmind 	int nrun;
   1186      1.113  gmcgarry 
   1187      1.188      yamt 	sched_pstats_ticks++;
   1188      1.281     rmind 	if (++lavg_count >= 5) {
   1189      1.281     rmind 		lavg_count = 0;
   1190      1.281     rmind 		nrun = 0;
   1191      1.281     rmind 	}
   1192      1.228        ad 	mutex_enter(proc_lock);
   1193      1.188      yamt 	PROCLIST_FOREACH(p, &allproc) {
   1194      1.281     rmind 		struct lwp *l;
   1195      1.281     rmind 		struct rlimit *rlim;
   1196      1.281     rmind 		long runtm;
   1197      1.281     rmind 		int sig;
   1198      1.281     rmind 
   1199      1.271     rmind 		/* Increment sleep time (if sleeping), ignore overflow. */
   1200      1.229        ad 		mutex_enter(p->p_lock);
   1201      1.212      yamt 		runtm = p->p_rtime.sec;
   1202      1.188      yamt 		LIST_FOREACH(l, &p->p_lwps, l_sibling) {
   1203      1.281     rmind 			fixpt_t lpctcpu;
   1204      1.281     rmind 			u_int lcpticks;
   1205      1.281     rmind 
   1206      1.249     rmind 			if (__predict_false((l->l_flag & LW_IDLE) != 0))
   1207      1.188      yamt 				continue;
   1208      1.188      yamt 			lwp_lock(l);
   1209      1.212      yamt 			runtm += l->l_rtime.sec;
   1210      1.188      yamt 			l->l_swtime++;
   1211      1.242     rmind 			sched_lwp_stats(l);
   1212      1.281     rmind 
   1213      1.281     rmind 			/* For load average calculation. */
   1214      1.282     rmind 			if (__predict_false(lavg_count == 0) &&
   1215      1.282     rmind 			    (l->l_flag & (LW_SINTR | LW_SYSTEM)) == 0) {
   1216      1.281     rmind 				switch (l->l_stat) {
   1217      1.281     rmind 				case LSSLEEP:
   1218      1.281     rmind 					if (l->l_slptime > 1) {
   1219      1.281     rmind 						break;
   1220      1.281     rmind 					}
   1221      1.281     rmind 				case LSRUN:
   1222      1.281     rmind 				case LSONPROC:
   1223      1.281     rmind 				case LSIDL:
   1224      1.281     rmind 					nrun++;
   1225      1.281     rmind 				}
   1226      1.281     rmind 			}
   1227      1.282     rmind 			lwp_unlock(l);
   1228      1.282     rmind 
   1229      1.282     rmind 			l->l_pctcpu = (l->l_pctcpu * ccpu) >> FSHIFT;
   1230      1.282     rmind 			if (l->l_slptime != 0)
   1231      1.282     rmind 				continue;
   1232      1.282     rmind 
   1233      1.282     rmind 			lpctcpu = l->l_pctcpu;
   1234      1.282     rmind 			lcpticks = atomic_swap_uint(&l->l_cpticks, 0);
   1235      1.282     rmind 			lpctcpu += ((FSCALE - ccpu) *
   1236      1.282     rmind 			    (lcpticks * FSCALE / clkhz)) >> FSHIFT;
   1237      1.282     rmind 			l->l_pctcpu = lpctcpu;
   1238      1.188      yamt 		}
   1239      1.249     rmind 		/* Calculating p_pctcpu only for ps(1) */
   1240      1.188      yamt 		p->p_pctcpu = (p->p_pctcpu * ccpu) >> FSHIFT;
   1241      1.174        ad 
   1242      1.188      yamt 		/*
   1243      1.188      yamt 		 * Check if the process exceeds its CPU resource allocation.
   1244      1.188      yamt 		 * If over max, kill it.
   1245      1.188      yamt 		 */
   1246      1.188      yamt 		rlim = &p->p_rlimit[RLIMIT_CPU];
   1247      1.188      yamt 		sig = 0;
   1248      1.249     rmind 		if (__predict_false(runtm >= rlim->rlim_cur)) {
   1249      1.188      yamt 			if (runtm >= rlim->rlim_max)
   1250      1.188      yamt 				sig = SIGKILL;
   1251      1.188      yamt 			else {
   1252      1.188      yamt 				sig = SIGXCPU;
   1253      1.188      yamt 				if (rlim->rlim_cur < rlim->rlim_max)
   1254      1.188      yamt 					rlim->rlim_cur += 5;
   1255      1.188      yamt 			}
   1256      1.188      yamt 		}
   1257      1.229        ad 		mutex_exit(p->p_lock);
   1258      1.259     rmind 		if (__predict_false(runtm < 0)) {
   1259      1.260        ad 			if (!backwards) {
   1260      1.260        ad 				backwards = true;
   1261      1.260        ad 				printf("WARNING: negative runtime; "
   1262      1.260        ad 				    "monotonic clock has gone backwards\n");
   1263      1.260        ad 			}
   1264      1.259     rmind 		} else if (__predict_false(sig)) {
   1265      1.259     rmind 			KASSERT((p->p_flag & PK_SYSTEM) == 0);
   1266      1.188      yamt 			psignal(p, sig);
   1267      1.259     rmind 		}
   1268      1.174        ad 	}
   1269      1.228        ad 	mutex_exit(proc_lock);
   1270      1.281     rmind 
   1271      1.281     rmind 	/* Load average calculation. */
   1272      1.281     rmind 	if (__predict_false(lavg_count == 0)) {
   1273      1.281     rmind 		int i;
   1274      1.283    martin 		CTASSERT(__arraycount(cexp) == __arraycount(avg->ldavg));
   1275      1.281     rmind 		for (i = 0; i < __arraycount(cexp); i++) {
   1276      1.281     rmind 			avg->ldavg[i] = (cexp[i] * avg->ldavg[i] +
   1277      1.281     rmind 			    nrun * FSCALE * (FSCALE - cexp[i])) >> FSHIFT;
   1278      1.281     rmind 		}
   1279      1.281     rmind 	}
   1280      1.281     rmind 
   1281      1.281     rmind 	/* Lightning bolt. */
   1282      1.273     pooka 	cv_broadcast(&lbolt);
   1283      1.113  gmcgarry }
   1284