Home | History | Annotate | Line # | Download | only in kern
kern_synch.c revision 1.262
      1  1.262      yamt /*	$NetBSD: kern_synch.c,v 1.262 2009/04/15 11:44:20 yamt Exp $	*/
      2   1.63   thorpej 
      3   1.63   thorpej /*-
      4  1.260        ad  * Copyright (c) 1999, 2000, 2004, 2006, 2007, 2008, 2009
      5  1.260        ad  *    The NetBSD Foundation, Inc.
      6   1.63   thorpej  * All rights reserved.
      7   1.63   thorpej  *
      8   1.63   thorpej  * This code is derived from software contributed to The NetBSD Foundation
      9   1.63   thorpej  * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
     10  1.188      yamt  * NASA Ames Research Center, by Charles M. Hannum, Andrew Doran and
     11  1.188      yamt  * Daniel Sieger.
     12   1.63   thorpej  *
     13   1.63   thorpej  * Redistribution and use in source and binary forms, with or without
     14   1.63   thorpej  * modification, are permitted provided that the following conditions
     15   1.63   thorpej  * are met:
     16   1.63   thorpej  * 1. Redistributions of source code must retain the above copyright
     17   1.63   thorpej  *    notice, this list of conditions and the following disclaimer.
     18   1.63   thorpej  * 2. Redistributions in binary form must reproduce the above copyright
     19   1.63   thorpej  *    notice, this list of conditions and the following disclaimer in the
     20   1.63   thorpej  *    documentation and/or other materials provided with the distribution.
     21   1.63   thorpej  *
     22   1.63   thorpej  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     23   1.63   thorpej  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     24   1.63   thorpej  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     25   1.63   thorpej  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     26   1.63   thorpej  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     27   1.63   thorpej  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     28   1.63   thorpej  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     29   1.63   thorpej  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     30   1.63   thorpej  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     31   1.63   thorpej  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     32   1.63   thorpej  * POSSIBILITY OF SUCH DAMAGE.
     33   1.63   thorpej  */
     34   1.26       cgd 
     35   1.26       cgd /*-
     36   1.26       cgd  * Copyright (c) 1982, 1986, 1990, 1991, 1993
     37   1.26       cgd  *	The Regents of the University of California.  All rights reserved.
     38   1.26       cgd  * (c) UNIX System Laboratories, Inc.
     39   1.26       cgd  * All or some portions of this file are derived from material licensed
     40   1.26       cgd  * to the University of California by American Telephone and Telegraph
     41   1.26       cgd  * Co. or Unix System Laboratories, Inc. and are reproduced herein with
     42   1.26       cgd  * the permission of UNIX System Laboratories, Inc.
     43   1.26       cgd  *
     44   1.26       cgd  * Redistribution and use in source and binary forms, with or without
     45   1.26       cgd  * modification, are permitted provided that the following conditions
     46   1.26       cgd  * are met:
     47   1.26       cgd  * 1. Redistributions of source code must retain the above copyright
     48   1.26       cgd  *    notice, this list of conditions and the following disclaimer.
     49   1.26       cgd  * 2. Redistributions in binary form must reproduce the above copyright
     50   1.26       cgd  *    notice, this list of conditions and the following disclaimer in the
     51   1.26       cgd  *    documentation and/or other materials provided with the distribution.
     52  1.136       agc  * 3. Neither the name of the University nor the names of its contributors
     53   1.26       cgd  *    may be used to endorse or promote products derived from this software
     54   1.26       cgd  *    without specific prior written permission.
     55   1.26       cgd  *
     56   1.26       cgd  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
     57   1.26       cgd  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     58   1.26       cgd  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     59   1.26       cgd  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
     60   1.26       cgd  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     61   1.26       cgd  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
     62   1.26       cgd  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     63   1.26       cgd  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     64   1.26       cgd  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     65   1.26       cgd  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     66   1.26       cgd  * SUCH DAMAGE.
     67   1.26       cgd  *
     68   1.50      fvdl  *	@(#)kern_synch.c	8.9 (Berkeley) 5/19/95
     69   1.26       cgd  */
     70  1.106     lukem 
     71  1.106     lukem #include <sys/cdefs.h>
     72  1.262      yamt __KERNEL_RCSID(0, "$NetBSD: kern_synch.c,v 1.262 2009/04/15 11:44:20 yamt Exp $");
     73   1.48       mrg 
     74  1.109      yamt #include "opt_kstack.h"
     75  1.110    briggs #include "opt_perfctrs.h"
     76  1.252  wrstuden #include "opt_sa.h"
     77   1.26       cgd 
     78  1.174        ad #define	__MUTEX_PRIVATE
     79  1.174        ad 
     80   1.26       cgd #include <sys/param.h>
     81   1.26       cgd #include <sys/systm.h>
     82   1.26       cgd #include <sys/proc.h>
     83   1.26       cgd #include <sys/kernel.h>
     84  1.111    briggs #if defined(PERFCTRS)
     85  1.110    briggs #include <sys/pmc.h>
     86  1.111    briggs #endif
     87  1.188      yamt #include <sys/cpu.h>
     88   1.26       cgd #include <sys/resourcevar.h>
     89   1.55      ross #include <sys/sched.h>
     90  1.252  wrstuden #include <sys/sa.h>
     91  1.252  wrstuden #include <sys/savar.h>
     92  1.179       dsl #include <sys/syscall_stats.h>
     93  1.174        ad #include <sys/sleepq.h>
     94  1.174        ad #include <sys/lockdebug.h>
     95  1.190        ad #include <sys/evcnt.h>
     96  1.199        ad #include <sys/intr.h>
     97  1.207        ad #include <sys/lwpctl.h>
     98  1.209        ad #include <sys/atomic.h>
     99  1.215        ad #include <sys/simplelock.h>
    100   1.47       mrg 
    101   1.47       mrg #include <uvm/uvm_extern.h>
    102   1.47       mrg 
    103  1.231        ad #include <dev/lockstat.h>
    104  1.231        ad 
    105  1.221        ad static u_int	sched_unsleep(struct lwp *, bool);
    106  1.188      yamt static void	sched_changepri(struct lwp *, pri_t);
    107  1.188      yamt static void	sched_lendpri(struct lwp *, pri_t);
    108  1.250     rmind static void	resched_cpu(struct lwp *);
    109  1.122   thorpej 
    110  1.174        ad syncobj_t sleep_syncobj = {
    111  1.174        ad 	SOBJ_SLEEPQ_SORTED,
    112  1.174        ad 	sleepq_unsleep,
    113  1.184      yamt 	sleepq_changepri,
    114  1.184      yamt 	sleepq_lendpri,
    115  1.184      yamt 	syncobj_noowner,
    116  1.174        ad };
    117  1.174        ad 
    118  1.174        ad syncobj_t sched_syncobj = {
    119  1.174        ad 	SOBJ_SLEEPQ_SORTED,
    120  1.174        ad 	sched_unsleep,
    121  1.184      yamt 	sched_changepri,
    122  1.184      yamt 	sched_lendpri,
    123  1.184      yamt 	syncobj_noowner,
    124  1.174        ad };
    125  1.122   thorpej 
    126  1.223        ad callout_t 	sched_pstats_ch;
    127  1.223        ad unsigned	sched_pstats_ticks;
    128  1.223        ad kcondvar_t	lbolt;			/* once a second sleep address */
    129  1.223        ad 
    130  1.237     rmind /* Preemption event counters */
    131  1.231        ad static struct evcnt kpreempt_ev_crit;
    132  1.231        ad static struct evcnt kpreempt_ev_klock;
    133  1.231        ad static struct evcnt kpreempt_ev_immed;
    134  1.231        ad 
    135  1.231        ad /*
    136  1.174        ad  * During autoconfiguration or after a panic, a sleep will simply lower the
    137  1.174        ad  * priority briefly to allow interrupts, then return.  The priority to be
    138  1.174        ad  * used (safepri) is machine-dependent, thus this value is initialized and
    139  1.174        ad  * maintained in the machine-dependent layers.  This priority will typically
    140  1.174        ad  * be 0, or the lowest priority that is safe for use on the interrupt stack;
    141  1.174        ad  * it can be made higher to block network software interrupts after panics.
    142   1.26       cgd  */
    143  1.174        ad int	safepri;
    144   1.26       cgd 
    145  1.237     rmind void
    146  1.237     rmind sched_init(void)
    147  1.237     rmind {
    148  1.237     rmind 
    149  1.237     rmind 	cv_init(&lbolt, "lbolt");
    150  1.237     rmind 	callout_init(&sched_pstats_ch, CALLOUT_MPSAFE);
    151  1.237     rmind 	callout_setfunc(&sched_pstats_ch, sched_pstats, NULL);
    152  1.237     rmind 
    153  1.239        ad 	evcnt_attach_dynamic(&kpreempt_ev_crit, EVCNT_TYPE_MISC, NULL,
    154  1.237     rmind 	   "kpreempt", "defer: critical section");
    155  1.239        ad 	evcnt_attach_dynamic(&kpreempt_ev_klock, EVCNT_TYPE_MISC, NULL,
    156  1.237     rmind 	   "kpreempt", "defer: kernel_lock");
    157  1.239        ad 	evcnt_attach_dynamic(&kpreempt_ev_immed, EVCNT_TYPE_MISC, NULL,
    158  1.237     rmind 	   "kpreempt", "immediate");
    159  1.237     rmind 
    160  1.237     rmind 	sched_pstats(NULL);
    161  1.237     rmind }
    162  1.237     rmind 
    163   1.26       cgd /*
    164  1.174        ad  * OBSOLETE INTERFACE
    165  1.174        ad  *
    166  1.255     skrll  * General sleep call.  Suspends the current LWP until a wakeup is
    167  1.255     skrll  * performed on the specified identifier.  The LWP will then be made
    168  1.174        ad  * runnable with the specified priority.  Sleeps at most timo/hz seconds (0
    169  1.174        ad  * means no timeout).  If pri includes PCATCH flag, signals are checked
    170   1.26       cgd  * before and after sleeping, else signals are not checked.  Returns 0 if
    171   1.26       cgd  * awakened, EWOULDBLOCK if the timeout expires.  If PCATCH is set and a
    172   1.26       cgd  * signal needs to be delivered, ERESTART is returned if the current system
    173   1.26       cgd  * call should be restarted if possible, and EINTR is returned if the system
    174   1.26       cgd  * call should be interrupted by the signal (return EINTR).
    175   1.77   thorpej  *
    176  1.174        ad  * The interlock is held until we are on a sleep queue. The interlock will
    177  1.174        ad  * be locked before returning back to the caller unless the PNORELOCK flag
    178  1.174        ad  * is specified, in which case the interlock will always be unlocked upon
    179  1.174        ad  * return.
    180   1.26       cgd  */
    181   1.26       cgd int
    182  1.185      yamt ltsleep(wchan_t ident, pri_t priority, const char *wmesg, int timo,
    183  1.174        ad 	volatile struct simplelock *interlock)
    184   1.26       cgd {
    185  1.122   thorpej 	struct lwp *l = curlwp;
    186  1.174        ad 	sleepq_t *sq;
    187  1.244        ad 	kmutex_t *mp;
    188  1.188      yamt 	int error;
    189   1.26       cgd 
    190  1.204        ad 	KASSERT((l->l_pflag & LP_INTR) == 0);
    191  1.204        ad 
    192  1.174        ad 	if (sleepq_dontsleep(l)) {
    193  1.174        ad 		(void)sleepq_abort(NULL, 0);
    194  1.174        ad 		if ((priority & PNORELOCK) != 0)
    195   1.77   thorpej 			simple_unlock(interlock);
    196  1.174        ad 		return 0;
    197   1.26       cgd 	}
    198   1.78  sommerfe 
    199  1.204        ad 	l->l_kpriority = true;
    200  1.244        ad 	sq = sleeptab_lookup(&sleeptab, ident, &mp);
    201  1.244        ad 	sleepq_enter(sq, l, mp);
    202  1.204        ad 	sleepq_enqueue(sq, ident, wmesg, &sleep_syncobj);
    203   1.42       cgd 
    204  1.174        ad 	if (interlock != NULL) {
    205  1.204        ad 		KASSERT(simple_lock_held(interlock));
    206  1.174        ad 		simple_unlock(interlock);
    207  1.150       chs 	}
    208  1.150       chs 
    209  1.188      yamt 	error = sleepq_block(timo, priority & PCATCH);
    210  1.126        pk 
    211  1.174        ad 	if (interlock != NULL && (priority & PNORELOCK) == 0)
    212  1.126        pk 		simple_lock(interlock);
    213  1.174        ad 
    214  1.174        ad 	return error;
    215   1.26       cgd }
    216   1.26       cgd 
    217  1.187        ad int
    218  1.187        ad mtsleep(wchan_t ident, pri_t priority, const char *wmesg, int timo,
    219  1.187        ad 	kmutex_t *mtx)
    220  1.187        ad {
    221  1.187        ad 	struct lwp *l = curlwp;
    222  1.187        ad 	sleepq_t *sq;
    223  1.244        ad 	kmutex_t *mp;
    224  1.188      yamt 	int error;
    225  1.187        ad 
    226  1.204        ad 	KASSERT((l->l_pflag & LP_INTR) == 0);
    227  1.204        ad 
    228  1.187        ad 	if (sleepq_dontsleep(l)) {
    229  1.187        ad 		(void)sleepq_abort(mtx, (priority & PNORELOCK) != 0);
    230  1.187        ad 		return 0;
    231  1.187        ad 	}
    232  1.187        ad 
    233  1.204        ad 	l->l_kpriority = true;
    234  1.244        ad 	sq = sleeptab_lookup(&sleeptab, ident, &mp);
    235  1.244        ad 	sleepq_enter(sq, l, mp);
    236  1.204        ad 	sleepq_enqueue(sq, ident, wmesg, &sleep_syncobj);
    237  1.187        ad 	mutex_exit(mtx);
    238  1.188      yamt 	error = sleepq_block(timo, priority & PCATCH);
    239  1.187        ad 
    240  1.187        ad 	if ((priority & PNORELOCK) == 0)
    241  1.187        ad 		mutex_enter(mtx);
    242  1.187        ad 
    243  1.187        ad 	return error;
    244  1.187        ad }
    245  1.187        ad 
    246   1.26       cgd /*
    247  1.174        ad  * General sleep call for situations where a wake-up is not expected.
    248   1.26       cgd  */
    249  1.174        ad int
    250  1.182   thorpej kpause(const char *wmesg, bool intr, int timo, kmutex_t *mtx)
    251   1.26       cgd {
    252  1.174        ad 	struct lwp *l = curlwp;
    253  1.244        ad 	kmutex_t *mp;
    254  1.174        ad 	sleepq_t *sq;
    255  1.174        ad 	int error;
    256   1.26       cgd 
    257  1.174        ad 	if (sleepq_dontsleep(l))
    258  1.174        ad 		return sleepq_abort(NULL, 0);
    259   1.26       cgd 
    260  1.174        ad 	if (mtx != NULL)
    261  1.174        ad 		mutex_exit(mtx);
    262  1.204        ad 	l->l_kpriority = true;
    263  1.244        ad 	sq = sleeptab_lookup(&sleeptab, l, &mp);
    264  1.244        ad 	sleepq_enter(sq, l, mp);
    265  1.204        ad 	sleepq_enqueue(sq, l, wmesg, &sleep_syncobj);
    266  1.188      yamt 	error = sleepq_block(timo, intr);
    267  1.174        ad 	if (mtx != NULL)
    268  1.174        ad 		mutex_enter(mtx);
    269   1.83   thorpej 
    270  1.174        ad 	return error;
    271  1.139        cl }
    272  1.139        cl 
    273  1.252  wrstuden #ifdef KERN_SA
    274  1.252  wrstuden /*
    275  1.252  wrstuden  * sa_awaken:
    276  1.252  wrstuden  *
    277  1.252  wrstuden  *	We believe this lwp is an SA lwp. If it's yielding,
    278  1.252  wrstuden  * let it know it needs to wake up.
    279  1.252  wrstuden  *
    280  1.252  wrstuden  *	We are called and exit with the lwp locked. We are
    281  1.252  wrstuden  * called in the middle of wakeup operations, so we need
    282  1.252  wrstuden  * to not touch the locks at all.
    283  1.252  wrstuden  */
    284  1.252  wrstuden void
    285  1.252  wrstuden sa_awaken(struct lwp *l)
    286  1.252  wrstuden {
    287  1.252  wrstuden 	/* LOCK_ASSERT(lwp_locked(l, NULL)); */
    288  1.252  wrstuden 
    289  1.252  wrstuden 	if (l == l->l_savp->savp_lwp && l->l_flag & LW_SA_YIELD)
    290  1.252  wrstuden 		l->l_flag &= ~LW_SA_IDLE;
    291  1.252  wrstuden }
    292  1.252  wrstuden #endif /* KERN_SA */
    293  1.252  wrstuden 
    294   1.26       cgd /*
    295  1.174        ad  * OBSOLETE INTERFACE
    296  1.174        ad  *
    297  1.255     skrll  * Make all LWPs sleeping on the specified identifier runnable.
    298   1.26       cgd  */
    299   1.26       cgd void
    300  1.174        ad wakeup(wchan_t ident)
    301   1.26       cgd {
    302  1.174        ad 	sleepq_t *sq;
    303  1.244        ad 	kmutex_t *mp;
    304   1.83   thorpej 
    305  1.261     rmind 	if (__predict_false(cold))
    306  1.174        ad 		return;
    307   1.83   thorpej 
    308  1.244        ad 	sq = sleeptab_lookup(&sleeptab, ident, &mp);
    309  1.244        ad 	sleepq_wake(sq, ident, (u_int)-1, mp);
    310   1.63   thorpej }
    311   1.63   thorpej 
    312   1.63   thorpej /*
    313  1.174        ad  * OBSOLETE INTERFACE
    314  1.174        ad  *
    315  1.255     skrll  * Make the highest priority LWP first in line on the specified
    316   1.63   thorpej  * identifier runnable.
    317   1.63   thorpej  */
    318  1.174        ad void
    319  1.174        ad wakeup_one(wchan_t ident)
    320   1.63   thorpej {
    321  1.174        ad 	sleepq_t *sq;
    322  1.244        ad 	kmutex_t *mp;
    323   1.63   thorpej 
    324  1.261     rmind 	if (__predict_false(cold))
    325  1.174        ad 		return;
    326  1.188      yamt 
    327  1.244        ad 	sq = sleeptab_lookup(&sleeptab, ident, &mp);
    328  1.244        ad 	sleepq_wake(sq, ident, 1, mp);
    329  1.174        ad }
    330   1.63   thorpej 
    331  1.117  gmcgarry 
    332  1.117  gmcgarry /*
    333  1.255     skrll  * General yield call.  Puts the current LWP back on its run queue and
    334  1.117  gmcgarry  * performs a voluntary context switch.  Should only be called when the
    335  1.255     skrll  * current LWP explicitly requests it (eg sched_yield(2)).
    336  1.117  gmcgarry  */
    337  1.117  gmcgarry void
    338  1.117  gmcgarry yield(void)
    339  1.117  gmcgarry {
    340  1.122   thorpej 	struct lwp *l = curlwp;
    341  1.117  gmcgarry 
    342  1.174        ad 	KERNEL_UNLOCK_ALL(l, &l->l_biglocks);
    343  1.174        ad 	lwp_lock(l);
    344  1.217        ad 	KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_lwplock));
    345  1.188      yamt 	KASSERT(l->l_stat == LSONPROC);
    346  1.204        ad 	l->l_kpriority = false;
    347  1.188      yamt 	(void)mi_switch(l);
    348  1.174        ad 	KERNEL_LOCK(l->l_biglocks, l);
    349   1.69   thorpej }
    350   1.69   thorpej 
    351   1.69   thorpej /*
    352  1.255     skrll  * General preemption call.  Puts the current LWP back on its run queue
    353  1.156    rpaulo  * and performs an involuntary context switch.
    354   1.69   thorpej  */
    355   1.69   thorpej void
    356  1.174        ad preempt(void)
    357   1.69   thorpej {
    358  1.122   thorpej 	struct lwp *l = curlwp;
    359   1.69   thorpej 
    360  1.174        ad 	KERNEL_UNLOCK_ALL(l, &l->l_biglocks);
    361  1.174        ad 	lwp_lock(l);
    362  1.217        ad 	KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_lwplock));
    363  1.188      yamt 	KASSERT(l->l_stat == LSONPROC);
    364  1.204        ad 	l->l_kpriority = false;
    365  1.174        ad 	l->l_nivcsw++;
    366  1.188      yamt 	(void)mi_switch(l);
    367  1.174        ad 	KERNEL_LOCK(l->l_biglocks, l);
    368   1.69   thorpej }
    369   1.69   thorpej 
    370  1.234        ad /*
    371  1.234        ad  * Handle a request made by another agent to preempt the current LWP
    372  1.234        ad  * in-kernel.  Usually called when l_dopreempt may be non-zero.
    373  1.234        ad  *
    374  1.234        ad  * Character addresses for lockstat only.
    375  1.234        ad  */
    376  1.231        ad static char	in_critical_section;
    377  1.231        ad static char	kernel_lock_held;
    378  1.231        ad static char	is_softint;
    379  1.262      yamt static char	cpu_kpreempt_enter_fail;
    380  1.231        ad 
    381  1.231        ad bool
    382  1.231        ad kpreempt(uintptr_t where)
    383  1.231        ad {
    384  1.231        ad 	uintptr_t failed;
    385  1.231        ad 	lwp_t *l;
    386  1.231        ad 	int s, dop;
    387  1.231        ad 
    388  1.231        ad 	l = curlwp;
    389  1.231        ad 	failed = 0;
    390  1.231        ad 	while ((dop = l->l_dopreempt) != 0) {
    391  1.231        ad 		if (l->l_stat != LSONPROC) {
    392  1.231        ad 			/*
    393  1.231        ad 			 * About to block (or die), let it happen.
    394  1.231        ad 			 * Doesn't really count as "preemption has
    395  1.231        ad 			 * been blocked", since we're going to
    396  1.231        ad 			 * context switch.
    397  1.231        ad 			 */
    398  1.231        ad 			l->l_dopreempt = 0;
    399  1.231        ad 			return true;
    400  1.231        ad 		}
    401  1.231        ad 		if (__predict_false((l->l_flag & LW_IDLE) != 0)) {
    402  1.231        ad 			/* Can't preempt idle loop, don't count as failure. */
    403  1.261     rmind 			l->l_dopreempt = 0;
    404  1.261     rmind 			return true;
    405  1.231        ad 		}
    406  1.231        ad 		if (__predict_false(l->l_nopreempt != 0)) {
    407  1.231        ad 			/* LWP holds preemption disabled, explicitly. */
    408  1.231        ad 			if ((dop & DOPREEMPT_COUNTED) == 0) {
    409  1.234        ad 				kpreempt_ev_crit.ev_count++;
    410  1.231        ad 			}
    411  1.231        ad 			failed = (uintptr_t)&in_critical_section;
    412  1.231        ad 			break;
    413  1.231        ad 		}
    414  1.231        ad 		if (__predict_false((l->l_pflag & LP_INTR) != 0)) {
    415  1.261     rmind 			/* Can't preempt soft interrupts yet. */
    416  1.261     rmind 			l->l_dopreempt = 0;
    417  1.261     rmind 			failed = (uintptr_t)&is_softint;
    418  1.261     rmind 			break;
    419  1.231        ad 		}
    420  1.231        ad 		s = splsched();
    421  1.231        ad 		if (__predict_false(l->l_blcnt != 0 ||
    422  1.231        ad 		    curcpu()->ci_biglock_wanted != NULL)) {
    423  1.231        ad 			/* Hold or want kernel_lock, code is not MT safe. */
    424  1.231        ad 			splx(s);
    425  1.231        ad 			if ((dop & DOPREEMPT_COUNTED) == 0) {
    426  1.234        ad 				kpreempt_ev_klock.ev_count++;
    427  1.231        ad 			}
    428  1.231        ad 			failed = (uintptr_t)&kernel_lock_held;
    429  1.231        ad 			break;
    430  1.231        ad 		}
    431  1.231        ad 		if (__predict_false(!cpu_kpreempt_enter(where, s))) {
    432  1.231        ad 			/*
    433  1.231        ad 			 * It may be that the IPL is too high.
    434  1.231        ad 			 * kpreempt_enter() can schedule an
    435  1.231        ad 			 * interrupt to retry later.
    436  1.231        ad 			 */
    437  1.231        ad 			splx(s);
    438  1.262      yamt 			failed = (uintptr_t)&cpu_kpreempt_enter_fail;
    439  1.231        ad 			break;
    440  1.231        ad 		}
    441  1.231        ad 		/* Do it! */
    442  1.231        ad 		if (__predict_true((dop & DOPREEMPT_COUNTED) == 0)) {
    443  1.234        ad 			kpreempt_ev_immed.ev_count++;
    444  1.231        ad 		}
    445  1.231        ad 		lwp_lock(l);
    446  1.231        ad 		mi_switch(l);
    447  1.231        ad 		l->l_nopreempt++;
    448  1.231        ad 		splx(s);
    449  1.231        ad 
    450  1.231        ad 		/* Take care of any MD cleanup. */
    451  1.231        ad 		cpu_kpreempt_exit(where);
    452  1.231        ad 		l->l_nopreempt--;
    453  1.231        ad 	}
    454  1.231        ad 
    455  1.231        ad 	/* Record preemption failure for reporting via lockstat. */
    456  1.231        ad 	if (__predict_false(failed)) {
    457  1.240        ad 		int lsflag = 0;
    458  1.231        ad 		atomic_or_uint(&l->l_dopreempt, DOPREEMPT_COUNTED);
    459  1.231        ad 		LOCKSTAT_ENTER(lsflag);
    460  1.231        ad 		/* Might recurse, make it atomic. */
    461  1.231        ad 		if (__predict_false(lsflag)) {
    462  1.231        ad 			if (where == 0) {
    463  1.231        ad 				where = (uintptr_t)__builtin_return_address(0);
    464  1.231        ad 			}
    465  1.231        ad 			if (atomic_cas_ptr_ni((void *)&l->l_pfailaddr,
    466  1.231        ad 			    NULL, (void *)where) == NULL) {
    467  1.231        ad 				LOCKSTAT_START_TIMER(lsflag, l->l_pfailtime);
    468  1.231        ad 				l->l_pfaillock = failed;
    469  1.231        ad 			}
    470  1.231        ad 		}
    471  1.231        ad 		LOCKSTAT_EXIT(lsflag);
    472  1.231        ad 	}
    473  1.231        ad 
    474  1.231        ad 	return failed;
    475  1.231        ad }
    476  1.231        ad 
    477   1.69   thorpej /*
    478  1.231        ad  * Return true if preemption is explicitly disabled.
    479  1.230        ad  */
    480  1.231        ad bool
    481  1.231        ad kpreempt_disabled(void)
    482  1.231        ad {
    483  1.261     rmind 	const lwp_t *l = curlwp;
    484  1.231        ad 
    485  1.231        ad 	return l->l_nopreempt != 0 || l->l_stat == LSZOMB ||
    486  1.231        ad 	    (l->l_flag & LW_IDLE) != 0 || cpu_kpreempt_disabled();
    487  1.231        ad }
    488  1.230        ad 
    489  1.230        ad /*
    490  1.231        ad  * Disable kernel preemption.
    491  1.230        ad  */
    492  1.230        ad void
    493  1.231        ad kpreempt_disable(void)
    494  1.230        ad {
    495  1.230        ad 
    496  1.231        ad 	KPREEMPT_DISABLE(curlwp);
    497  1.230        ad }
    498  1.230        ad 
    499  1.230        ad /*
    500  1.231        ad  * Reenable kernel preemption.
    501  1.230        ad  */
    502  1.231        ad void
    503  1.231        ad kpreempt_enable(void)
    504  1.230        ad {
    505  1.230        ad 
    506  1.231        ad 	KPREEMPT_ENABLE(curlwp);
    507  1.230        ad }
    508  1.230        ad 
    509  1.230        ad /*
    510  1.188      yamt  * Compute the amount of time during which the current lwp was running.
    511  1.130   nathanw  *
    512  1.188      yamt  * - update l_rtime unless it's an idle lwp.
    513  1.188      yamt  */
    514  1.188      yamt 
    515  1.199        ad void
    516  1.212      yamt updatertime(lwp_t *l, const struct bintime *now)
    517  1.188      yamt {
    518  1.188      yamt 
    519  1.261     rmind 	if (__predict_false(l->l_flag & LW_IDLE))
    520  1.188      yamt 		return;
    521  1.188      yamt 
    522  1.212      yamt 	/* rtime += now - stime */
    523  1.212      yamt 	bintime_add(&l->l_rtime, now);
    524  1.212      yamt 	bintime_sub(&l->l_rtime, &l->l_stime);
    525  1.188      yamt }
    526  1.188      yamt 
    527  1.188      yamt /*
    528  1.245        ad  * Select next LWP from the current CPU to run..
    529  1.245        ad  */
    530  1.245        ad static inline lwp_t *
    531  1.245        ad nextlwp(struct cpu_info *ci, struct schedstate_percpu *spc)
    532  1.245        ad {
    533  1.245        ad 	lwp_t *newl;
    534  1.245        ad 
    535  1.245        ad 	/*
    536  1.245        ad 	 * Let sched_nextlwp() select the LWP to run the CPU next.
    537  1.245        ad 	 * If no LWP is runnable, select the idle LWP.
    538  1.245        ad 	 *
    539  1.245        ad 	 * Note that spc_lwplock might not necessary be held, and
    540  1.245        ad 	 * new thread would be unlocked after setting the LWP-lock.
    541  1.245        ad 	 */
    542  1.245        ad 	newl = sched_nextlwp();
    543  1.245        ad 	if (newl != NULL) {
    544  1.245        ad 		sched_dequeue(newl);
    545  1.245        ad 		KASSERT(lwp_locked(newl, spc->spc_mutex));
    546  1.245        ad 		newl->l_stat = LSONPROC;
    547  1.245        ad 		newl->l_cpu = ci;
    548  1.248        ad 		newl->l_pflag |= LP_RUNNING;
    549  1.245        ad 		lwp_setlock(newl, spc->spc_lwplock);
    550  1.245        ad 	} else {
    551  1.245        ad 		newl = ci->ci_data.cpu_idlelwp;
    552  1.245        ad 		newl->l_stat = LSONPROC;
    553  1.248        ad 		newl->l_pflag |= LP_RUNNING;
    554  1.245        ad 	}
    555  1.261     rmind 
    556  1.245        ad 	/*
    557  1.245        ad 	 * Only clear want_resched if there are no pending (slow)
    558  1.245        ad 	 * software interrupts.
    559  1.245        ad 	 */
    560  1.245        ad 	ci->ci_want_resched = ci->ci_data.cpu_softints;
    561  1.245        ad 	spc->spc_flags &= ~SPCF_SWITCHCLEAR;
    562  1.245        ad 	spc->spc_curpriority = lwp_eprio(newl);
    563  1.245        ad 
    564  1.245        ad 	return newl;
    565  1.245        ad }
    566  1.245        ad 
    567  1.245        ad /*
    568  1.188      yamt  * The machine independent parts of context switch.
    569  1.188      yamt  *
    570  1.188      yamt  * Returns 1 if another LWP was actually run.
    571   1.26       cgd  */
    572  1.122   thorpej int
    573  1.199        ad mi_switch(lwp_t *l)
    574   1.26       cgd {
    575  1.246     rmind 	struct cpu_info *ci;
    576   1.76   thorpej 	struct schedstate_percpu *spc;
    577  1.188      yamt 	struct lwp *newl;
    578  1.174        ad 	int retval, oldspl;
    579  1.212      yamt 	struct bintime bt;
    580  1.199        ad 	bool returning;
    581   1.26       cgd 
    582  1.188      yamt 	KASSERT(lwp_locked(l, NULL));
    583  1.231        ad 	KASSERT(kpreempt_disabled());
    584  1.188      yamt 	LOCKDEBUG_BARRIER(l->l_mutex, 1);
    585  1.174        ad 
    586  1.174        ad #ifdef KSTACK_CHECK_MAGIC
    587  1.174        ad 	kstack_check_magic(l);
    588  1.174        ad #endif
    589   1.83   thorpej 
    590  1.212      yamt 	binuptime(&bt);
    591  1.199        ad 
    592  1.231        ad 	KASSERT(l->l_cpu == curcpu());
    593  1.196        ad 	ci = l->l_cpu;
    594  1.196        ad 	spc = &ci->ci_schedstate;
    595  1.199        ad 	returning = false;
    596  1.190        ad 	newl = NULL;
    597  1.190        ad 
    598  1.199        ad 	/*
    599  1.199        ad 	 * If we have been asked to switch to a specific LWP, then there
    600  1.199        ad 	 * is no need to inspect the run queues.  If a soft interrupt is
    601  1.199        ad 	 * blocking, then return to the interrupted thread without adjusting
    602  1.199        ad 	 * VM context or its start time: neither have been changed in order
    603  1.199        ad 	 * to take the interrupt.
    604  1.199        ad 	 */
    605  1.190        ad 	if (l->l_switchto != NULL) {
    606  1.204        ad 		if ((l->l_pflag & LP_INTR) != 0) {
    607  1.199        ad 			returning = true;
    608  1.199        ad 			softint_block(l);
    609  1.248        ad 			if ((l->l_pflag & LP_TIMEINTR) != 0)
    610  1.212      yamt 				updatertime(l, &bt);
    611  1.199        ad 		}
    612  1.190        ad 		newl = l->l_switchto;
    613  1.190        ad 		l->l_switchto = NULL;
    614  1.190        ad 	}
    615  1.204        ad #ifndef __HAVE_FAST_SOFTINTS
    616  1.204        ad 	else if (ci->ci_data.cpu_softints != 0) {
    617  1.204        ad 		/* There are pending soft interrupts, so pick one. */
    618  1.204        ad 		newl = softint_picklwp();
    619  1.204        ad 		newl->l_stat = LSONPROC;
    620  1.248        ad 		newl->l_pflag |= LP_RUNNING;
    621  1.204        ad 	}
    622  1.204        ad #endif	/* !__HAVE_FAST_SOFTINTS */
    623  1.190        ad 
    624  1.180       dsl 	/* Count time spent in current system call */
    625  1.199        ad 	if (!returning) {
    626  1.199        ad 		SYSCALL_TIME_SLEEP(l);
    627  1.180       dsl 
    628  1.199        ad 		/*
    629  1.199        ad 		 * XXXSMP If we are using h/w performance counters,
    630  1.199        ad 		 * save context.
    631  1.199        ad 		 */
    632  1.174        ad #if PERFCTRS
    633  1.199        ad 		if (PMC_ENABLED(l->l_proc)) {
    634  1.199        ad 			pmc_save_context(l->l_proc);
    635  1.199        ad 		}
    636  1.199        ad #endif
    637  1.212      yamt 		updatertime(l, &bt);
    638  1.174        ad 	}
    639  1.113  gmcgarry 
    640  1.246     rmind 	/* Lock the runqueue */
    641  1.246     rmind 	KASSERT(l->l_stat != LSRUN);
    642  1.246     rmind 	mutex_spin_enter(spc->spc_mutex);
    643  1.246     rmind 
    644  1.113  gmcgarry 	/*
    645  1.174        ad 	 * If on the CPU and we have gotten this far, then we must yield.
    646  1.113  gmcgarry 	 */
    647  1.246     rmind 	if (l->l_stat == LSONPROC && l != newl) {
    648  1.217        ad 		KASSERT(lwp_locked(l, spc->spc_lwplock));
    649  1.188      yamt 		if ((l->l_flag & LW_IDLE) == 0) {
    650  1.188      yamt 			l->l_stat = LSRUN;
    651  1.246     rmind 			lwp_setlock(l, spc->spc_mutex);
    652  1.246     rmind 			sched_enqueue(l, true);
    653  1.246     rmind 			/* Handle migration case */
    654  1.246     rmind 			KASSERT(spc->spc_migrating == NULL);
    655  1.246     rmind 			if (l->l_target_cpu !=  NULL) {
    656  1.246     rmind 				spc->spc_migrating = l;
    657  1.216     rmind 			}
    658  1.246     rmind 		} else
    659  1.188      yamt 			l->l_stat = LSIDL;
    660  1.174        ad 	}
    661  1.174        ad 
    662  1.245        ad 	/* Pick new LWP to run. */
    663  1.190        ad 	if (newl == NULL) {
    664  1.245        ad 		newl = nextlwp(ci, spc);
    665  1.199        ad 	}
    666  1.199        ad 
    667  1.204        ad 	/* Items that must be updated with the CPU locked. */
    668  1.199        ad 	if (!returning) {
    669  1.204        ad 		/* Update the new LWP's start time. */
    670  1.212      yamt 		newl->l_stime = bt;
    671  1.204        ad 
    672  1.199        ad 		/*
    673  1.204        ad 		 * ci_curlwp changes when a fast soft interrupt occurs.
    674  1.204        ad 		 * We use cpu_onproc to keep track of which kernel or
    675  1.204        ad 		 * user thread is running 'underneath' the software
    676  1.204        ad 		 * interrupt.  This is important for time accounting,
    677  1.204        ad 		 * itimers and forcing user threads to preempt (aston).
    678  1.199        ad 		 */
    679  1.204        ad 		ci->ci_data.cpu_onproc = newl;
    680  1.188      yamt 	}
    681  1.188      yamt 
    682  1.241        ad 	/*
    683  1.241        ad 	 * Preemption related tasks.  Must be done with the current
    684  1.241        ad 	 * CPU locked.
    685  1.241        ad 	 */
    686  1.241        ad 	cpu_did_resched(l);
    687  1.231        ad 	l->l_dopreempt = 0;
    688  1.231        ad 	if (__predict_false(l->l_pfailaddr != 0)) {
    689  1.231        ad 		LOCKSTAT_FLAG(lsflag);
    690  1.231        ad 		LOCKSTAT_ENTER(lsflag);
    691  1.231        ad 		LOCKSTAT_STOP_TIMER(lsflag, l->l_pfailtime);
    692  1.231        ad 		LOCKSTAT_EVENT_RA(lsflag, l->l_pfaillock, LB_NOPREEMPT|LB_SPIN,
    693  1.231        ad 		    1, l->l_pfailtime, l->l_pfailaddr);
    694  1.231        ad 		LOCKSTAT_EXIT(lsflag);
    695  1.231        ad 		l->l_pfailtime = 0;
    696  1.231        ad 		l->l_pfaillock = 0;
    697  1.231        ad 		l->l_pfailaddr = 0;
    698  1.231        ad 	}
    699  1.231        ad 
    700  1.188      yamt 	if (l != newl) {
    701  1.188      yamt 		struct lwp *prevlwp;
    702  1.174        ad 
    703  1.209        ad 		/* Release all locks, but leave the current LWP locked */
    704  1.246     rmind 		if (l->l_mutex == spc->spc_mutex) {
    705  1.209        ad 			/*
    706  1.209        ad 			 * Drop spc_lwplock, if the current LWP has been moved
    707  1.209        ad 			 * to the run queue (it is now locked by spc_mutex).
    708  1.209        ad 			 */
    709  1.217        ad 			mutex_spin_exit(spc->spc_lwplock);
    710  1.188      yamt 		} else {
    711  1.209        ad 			/*
    712  1.209        ad 			 * Otherwise, drop the spc_mutex, we are done with the
    713  1.209        ad 			 * run queues.
    714  1.209        ad 			 */
    715  1.188      yamt 			mutex_spin_exit(spc->spc_mutex);
    716  1.188      yamt 		}
    717  1.188      yamt 
    718  1.209        ad 		/*
    719  1.253     skrll 		 * Mark that context switch is going to be performed
    720  1.209        ad 		 * for this LWP, to protect it from being switched
    721  1.209        ad 		 * to on another CPU.
    722  1.209        ad 		 */
    723  1.209        ad 		KASSERT(l->l_ctxswtch == 0);
    724  1.209        ad 		l->l_ctxswtch = 1;
    725  1.209        ad 		l->l_ncsw++;
    726  1.248        ad 		l->l_pflag &= ~LP_RUNNING;
    727  1.209        ad 
    728  1.209        ad 		/*
    729  1.209        ad 		 * Increase the count of spin-mutexes before the release
    730  1.209        ad 		 * of the last lock - we must remain at IPL_SCHED during
    731  1.209        ad 		 * the context switch.
    732  1.209        ad 		 */
    733  1.209        ad 		oldspl = MUTEX_SPIN_OLDSPL(ci);
    734  1.209        ad 		ci->ci_mtx_count--;
    735  1.209        ad 		lwp_unlock(l);
    736  1.209        ad 
    737  1.218        ad 		/* Count the context switch on this CPU. */
    738  1.218        ad 		ci->ci_data.cpu_nswtch++;
    739  1.188      yamt 
    740  1.209        ad 		/* Update status for lwpctl, if present. */
    741  1.209        ad 		if (l->l_lwpctl != NULL)
    742  1.209        ad 			l->l_lwpctl->lc_curcpu = LWPCTL_CPU_NONE;
    743  1.209        ad 
    744  1.199        ad 		/*
    745  1.199        ad 		 * Save old VM context, unless a soft interrupt
    746  1.199        ad 		 * handler is blocking.
    747  1.199        ad 		 */
    748  1.199        ad 		if (!returning)
    749  1.199        ad 			pmap_deactivate(l);
    750  1.188      yamt 
    751  1.209        ad 		/*
    752  1.209        ad 		 * We may need to spin-wait for if 'newl' is still
    753  1.209        ad 		 * context switching on another CPU.
    754  1.209        ad 		 */
    755  1.261     rmind 		if (__predict_false(newl->l_ctxswtch != 0)) {
    756  1.209        ad 			u_int count;
    757  1.209        ad 			count = SPINLOCK_BACKOFF_MIN;
    758  1.209        ad 			while (newl->l_ctxswtch)
    759  1.209        ad 				SPINLOCK_BACKOFF(count);
    760  1.209        ad 		}
    761  1.207        ad 
    762  1.188      yamt 		/* Switch to the new LWP.. */
    763  1.204        ad 		prevlwp = cpu_switchto(l, newl, returning);
    764  1.207        ad 		ci = curcpu();
    765  1.207        ad 
    766  1.188      yamt 		/*
    767  1.209        ad 		 * Switched away - we have new curlwp.
    768  1.209        ad 		 * Restore VM context and IPL.
    769  1.188      yamt 		 */
    770  1.209        ad 		pmap_activate(l);
    771  1.188      yamt 		if (prevlwp != NULL) {
    772  1.209        ad 			/* Normalize the count of the spin-mutexes */
    773  1.209        ad 			ci->ci_mtx_count++;
    774  1.209        ad 			/* Unmark the state of context switch */
    775  1.209        ad 			membar_exit();
    776  1.209        ad 			prevlwp->l_ctxswtch = 0;
    777  1.188      yamt 		}
    778  1.209        ad 
    779  1.209        ad 		/* Update status for lwpctl, if present. */
    780  1.219        ad 		if (l->l_lwpctl != NULL) {
    781  1.209        ad 			l->l_lwpctl->lc_curcpu = (int)cpu_index(ci);
    782  1.219        ad 			l->l_lwpctl->lc_pctr++;
    783  1.219        ad 		}
    784  1.174        ad 
    785  1.231        ad 		KASSERT(l->l_cpu == ci);
    786  1.231        ad 		splx(oldspl);
    787  1.188      yamt 		retval = 1;
    788  1.188      yamt 	} else {
    789  1.188      yamt 		/* Nothing to do - just unlock and return. */
    790  1.246     rmind 		mutex_spin_exit(spc->spc_mutex);
    791  1.188      yamt 		lwp_unlock(l);
    792  1.122   thorpej 		retval = 0;
    793  1.122   thorpej 	}
    794  1.110    briggs 
    795  1.188      yamt 	KASSERT(l == curlwp);
    796  1.188      yamt 	KASSERT(l->l_stat == LSONPROC);
    797  1.188      yamt 
    798  1.110    briggs 	/*
    799  1.174        ad 	 * XXXSMP If we are using h/w performance counters, restore context.
    800  1.231        ad 	 * XXXSMP preemption problem.
    801   1.26       cgd 	 */
    802  1.114  gmcgarry #if PERFCTRS
    803  1.175  christos 	if (PMC_ENABLED(l->l_proc)) {
    804  1.175  christos 		pmc_restore_context(l->l_proc);
    805  1.166  christos 	}
    806  1.114  gmcgarry #endif
    807  1.180       dsl 	SYSCALL_TIME_WAKEUP(l);
    808  1.188      yamt 	LOCKDEBUG_BARRIER(NULL, 1);
    809  1.169      yamt 
    810  1.122   thorpej 	return retval;
    811   1.26       cgd }
    812   1.26       cgd 
    813   1.26       cgd /*
    814  1.245        ad  * The machine independent parts of context switch to oblivion.
    815  1.245        ad  * Does not return.  Call with the LWP unlocked.
    816  1.245        ad  */
    817  1.245        ad void
    818  1.245        ad lwp_exit_switchaway(lwp_t *l)
    819  1.245        ad {
    820  1.245        ad 	struct cpu_info *ci;
    821  1.245        ad 	struct lwp *newl;
    822  1.245        ad 	struct bintime bt;
    823  1.245        ad 
    824  1.245        ad 	ci = l->l_cpu;
    825  1.245        ad 
    826  1.245        ad 	KASSERT(kpreempt_disabled());
    827  1.245        ad 	KASSERT(l->l_stat == LSZOMB || l->l_stat == LSIDL);
    828  1.245        ad 	KASSERT(ci == curcpu());
    829  1.245        ad 	LOCKDEBUG_BARRIER(NULL, 0);
    830  1.245        ad 
    831  1.245        ad #ifdef KSTACK_CHECK_MAGIC
    832  1.245        ad 	kstack_check_magic(l);
    833  1.245        ad #endif
    834  1.245        ad 
    835  1.245        ad 	/* Count time spent in current system call */
    836  1.245        ad 	SYSCALL_TIME_SLEEP(l);
    837  1.245        ad 	binuptime(&bt);
    838  1.245        ad 	updatertime(l, &bt);
    839  1.245        ad 
    840  1.245        ad 	/* Must stay at IPL_SCHED even after releasing run queue lock. */
    841  1.245        ad 	(void)splsched();
    842  1.245        ad 
    843  1.245        ad 	/*
    844  1.245        ad 	 * Let sched_nextlwp() select the LWP to run the CPU next.
    845  1.245        ad 	 * If no LWP is runnable, select the idle LWP.
    846  1.245        ad 	 *
    847  1.245        ad 	 * Note that spc_lwplock might not necessary be held, and
    848  1.245        ad 	 * new thread would be unlocked after setting the LWP-lock.
    849  1.245        ad 	 */
    850  1.245        ad 	spc_lock(ci);
    851  1.245        ad #ifndef __HAVE_FAST_SOFTINTS
    852  1.245        ad 	if (ci->ci_data.cpu_softints != 0) {
    853  1.245        ad 		/* There are pending soft interrupts, so pick one. */
    854  1.245        ad 		newl = softint_picklwp();
    855  1.245        ad 		newl->l_stat = LSONPROC;
    856  1.248        ad 		newl->l_pflag |= LP_RUNNING;
    857  1.245        ad 	} else
    858  1.245        ad #endif	/* !__HAVE_FAST_SOFTINTS */
    859  1.245        ad 	{
    860  1.245        ad 		newl = nextlwp(ci, &ci->ci_schedstate);
    861  1.245        ad 	}
    862  1.245        ad 
    863  1.245        ad 	/* Update the new LWP's start time. */
    864  1.245        ad 	newl->l_stime = bt;
    865  1.248        ad 	l->l_pflag &= ~LP_RUNNING;
    866  1.245        ad 
    867  1.245        ad 	/*
    868  1.245        ad 	 * ci_curlwp changes when a fast soft interrupt occurs.
    869  1.245        ad 	 * We use cpu_onproc to keep track of which kernel or
    870  1.245        ad 	 * user thread is running 'underneath' the software
    871  1.245        ad 	 * interrupt.  This is important for time accounting,
    872  1.245        ad 	 * itimers and forcing user threads to preempt (aston).
    873  1.245        ad 	 */
    874  1.245        ad 	ci->ci_data.cpu_onproc = newl;
    875  1.245        ad 
    876  1.245        ad 	/*
    877  1.245        ad 	 * Preemption related tasks.  Must be done with the current
    878  1.245        ad 	 * CPU locked.
    879  1.245        ad 	 */
    880  1.245        ad 	cpu_did_resched(l);
    881  1.245        ad 
    882  1.245        ad 	/* Unlock the run queue. */
    883  1.245        ad 	spc_unlock(ci);
    884  1.245        ad 
    885  1.245        ad 	/* Count the context switch on this CPU. */
    886  1.245        ad 	ci->ci_data.cpu_nswtch++;
    887  1.245        ad 
    888  1.245        ad 	/* Update status for lwpctl, if present. */
    889  1.245        ad 	if (l->l_lwpctl != NULL)
    890  1.247        ad 		l->l_lwpctl->lc_curcpu = LWPCTL_CPU_EXITED;
    891  1.245        ad 
    892  1.245        ad 	/*
    893  1.245        ad 	 * We may need to spin-wait for if 'newl' is still
    894  1.245        ad 	 * context switching on another CPU.
    895  1.245        ad 	 */
    896  1.261     rmind 	if (__predict_false(newl->l_ctxswtch != 0)) {
    897  1.245        ad 		u_int count;
    898  1.245        ad 		count = SPINLOCK_BACKOFF_MIN;
    899  1.245        ad 		while (newl->l_ctxswtch)
    900  1.245        ad 			SPINLOCK_BACKOFF(count);
    901  1.245        ad 	}
    902  1.245        ad 
    903  1.245        ad 	/* Switch to the new LWP.. */
    904  1.245        ad 	(void)cpu_switchto(NULL, newl, false);
    905  1.245        ad 
    906  1.251       uwe 	for (;;) continue;	/* XXX: convince gcc about "noreturn" */
    907  1.245        ad 	/* NOTREACHED */
    908  1.245        ad }
    909  1.245        ad 
    910  1.245        ad /*
    911  1.255     skrll  * Change LWP state to be runnable, placing it on the run queue if it is
    912  1.174        ad  * in memory, and awakening the swapper if it isn't in memory.
    913  1.174        ad  *
    914  1.174        ad  * Call with the process and LWP locked.  Will return with the LWP unlocked.
    915   1.26       cgd  */
    916   1.26       cgd void
    917  1.122   thorpej setrunnable(struct lwp *l)
    918   1.26       cgd {
    919  1.122   thorpej 	struct proc *p = l->l_proc;
    920  1.205        ad 	struct cpu_info *ci;
    921   1.26       cgd 
    922  1.188      yamt 	KASSERT((l->l_flag & LW_IDLE) == 0);
    923  1.229        ad 	KASSERT(mutex_owned(p->p_lock));
    924  1.183        ad 	KASSERT(lwp_locked(l, NULL));
    925  1.205        ad 	KASSERT(l->l_mutex != l->l_cpu->ci_schedstate.spc_mutex);
    926   1.83   thorpej 
    927  1.122   thorpej 	switch (l->l_stat) {
    928  1.122   thorpej 	case LSSTOP:
    929   1.33   mycroft 		/*
    930   1.33   mycroft 		 * If we're being traced (possibly because someone attached us
    931   1.33   mycroft 		 * while we were stopped), check for a signal from the debugger.
    932   1.33   mycroft 		 */
    933  1.256        ad 		if ((p->p_slflag & PSL_TRACED) != 0 && p->p_xstat != 0)
    934  1.174        ad 			signotify(l);
    935  1.174        ad 		p->p_nrlwps++;
    936   1.26       cgd 		break;
    937  1.174        ad 	case LSSUSPENDED:
    938  1.178     pavel 		l->l_flag &= ~LW_WSUSPEND;
    939  1.174        ad 		p->p_nrlwps++;
    940  1.192     rmind 		cv_broadcast(&p->p_lwpcv);
    941  1.122   thorpej 		break;
    942  1.174        ad 	case LSSLEEP:
    943  1.174        ad 		KASSERT(l->l_wchan != NULL);
    944   1.26       cgd 		break;
    945  1.174        ad 	default:
    946  1.174        ad 		panic("setrunnable: lwp %p state was %d", l, l->l_stat);
    947   1.26       cgd 	}
    948  1.139        cl 
    949  1.252  wrstuden #ifdef KERN_SA
    950  1.252  wrstuden 	if (l->l_proc->p_sa)
    951  1.252  wrstuden 		sa_awaken(l);
    952  1.252  wrstuden #endif /* KERN_SA */
    953  1.252  wrstuden 
    954  1.174        ad 	/*
    955  1.174        ad 	 * If the LWP was sleeping interruptably, then it's OK to start it
    956  1.174        ad 	 * again.  If not, mark it as still sleeping.
    957  1.174        ad 	 */
    958  1.174        ad 	if (l->l_wchan != NULL) {
    959  1.174        ad 		l->l_stat = LSSLEEP;
    960  1.183        ad 		/* lwp_unsleep() will release the lock. */
    961  1.221        ad 		lwp_unsleep(l, true);
    962  1.174        ad 		return;
    963  1.174        ad 	}
    964  1.139        cl 
    965  1.174        ad 	/*
    966  1.174        ad 	 * If the LWP is still on the CPU, mark it as LSONPROC.  It may be
    967  1.174        ad 	 * about to call mi_switch(), in which case it will yield.
    968  1.174        ad 	 */
    969  1.248        ad 	if ((l->l_pflag & LP_RUNNING) != 0) {
    970  1.174        ad 		l->l_stat = LSONPROC;
    971  1.174        ad 		l->l_slptime = 0;
    972  1.174        ad 		lwp_unlock(l);
    973  1.174        ad 		return;
    974  1.174        ad 	}
    975  1.122   thorpej 
    976  1.174        ad 	/*
    977  1.205        ad 	 * Look for a CPU to run.
    978  1.205        ad 	 * Set the LWP runnable.
    979  1.174        ad 	 */
    980  1.205        ad 	ci = sched_takecpu(l);
    981  1.205        ad 	l->l_cpu = ci;
    982  1.236        ad 	spc_lock(ci);
    983  1.236        ad 	lwp_unlock_to(l, ci->ci_schedstate.spc_mutex);
    984  1.188      yamt 	sched_setrunnable(l);
    985  1.174        ad 	l->l_stat = LSRUN;
    986  1.122   thorpej 	l->l_slptime = 0;
    987  1.174        ad 
    988  1.205        ad 	/*
    989  1.205        ad 	 * If thread is swapped out - wake the swapper to bring it back in.
    990  1.205        ad 	 * Otherwise, enter it into a run queue.
    991  1.205        ad 	 */
    992  1.178     pavel 	if (l->l_flag & LW_INMEM) {
    993  1.188      yamt 		sched_enqueue(l, false);
    994  1.188      yamt 		resched_cpu(l);
    995  1.174        ad 		lwp_unlock(l);
    996  1.174        ad 	} else {
    997  1.174        ad 		lwp_unlock(l);
    998  1.177        ad 		uvm_kick_scheduler();
    999  1.174        ad 	}
   1000   1.26       cgd }
   1001   1.26       cgd 
   1002   1.26       cgd /*
   1003  1.174        ad  * suspendsched:
   1004  1.174        ad  *
   1005  1.174        ad  *	Convert all non-L_SYSTEM LSSLEEP or LSRUN LWPs to LSSUSPENDED.
   1006  1.174        ad  */
   1007   1.94    bouyer void
   1008  1.174        ad suspendsched(void)
   1009   1.94    bouyer {
   1010  1.174        ad 	CPU_INFO_ITERATOR cii;
   1011  1.174        ad 	struct cpu_info *ci;
   1012  1.122   thorpej 	struct lwp *l;
   1013  1.174        ad 	struct proc *p;
   1014   1.94    bouyer 
   1015   1.94    bouyer 	/*
   1016  1.174        ad 	 * We do this by process in order not to violate the locking rules.
   1017   1.94    bouyer 	 */
   1018  1.228        ad 	mutex_enter(proc_lock);
   1019  1.174        ad 	PROCLIST_FOREACH(p, &allproc) {
   1020  1.238        ad 		if ((p->p_flag & PK_MARKER) != 0)
   1021  1.238        ad 			continue;
   1022  1.238        ad 
   1023  1.229        ad 		mutex_enter(p->p_lock);
   1024  1.178     pavel 		if ((p->p_flag & PK_SYSTEM) != 0) {
   1025  1.229        ad 			mutex_exit(p->p_lock);
   1026   1.94    bouyer 			continue;
   1027  1.174        ad 		}
   1028  1.174        ad 
   1029  1.174        ad 		p->p_stat = SSTOP;
   1030  1.174        ad 
   1031  1.174        ad 		LIST_FOREACH(l, &p->p_lwps, l_sibling) {
   1032  1.174        ad 			if (l == curlwp)
   1033  1.174        ad 				continue;
   1034  1.174        ad 
   1035  1.174        ad 			lwp_lock(l);
   1036  1.122   thorpej 
   1037   1.97     enami 			/*
   1038  1.174        ad 			 * Set L_WREBOOT so that the LWP will suspend itself
   1039  1.174        ad 			 * when it tries to return to user mode.  We want to
   1040  1.174        ad 			 * try and get to get as many LWPs as possible to
   1041  1.174        ad 			 * the user / kernel boundary, so that they will
   1042  1.174        ad 			 * release any locks that they hold.
   1043   1.97     enami 			 */
   1044  1.178     pavel 			l->l_flag |= (LW_WREBOOT | LW_WSUSPEND);
   1045  1.174        ad 
   1046  1.174        ad 			if (l->l_stat == LSSLEEP &&
   1047  1.178     pavel 			    (l->l_flag & LW_SINTR) != 0) {
   1048  1.174        ad 				/* setrunnable() will release the lock. */
   1049  1.174        ad 				setrunnable(l);
   1050  1.174        ad 				continue;
   1051  1.174        ad 			}
   1052  1.174        ad 
   1053  1.174        ad 			lwp_unlock(l);
   1054   1.94    bouyer 		}
   1055  1.174        ad 
   1056  1.229        ad 		mutex_exit(p->p_lock);
   1057   1.94    bouyer 	}
   1058  1.228        ad 	mutex_exit(proc_lock);
   1059  1.174        ad 
   1060  1.174        ad 	/*
   1061  1.174        ad 	 * Kick all CPUs to make them preempt any LWPs running in user mode.
   1062  1.174        ad 	 * They'll trap into the kernel and suspend themselves in userret().
   1063  1.174        ad 	 */
   1064  1.204        ad 	for (CPU_INFO_FOREACH(cii, ci)) {
   1065  1.204        ad 		spc_lock(ci);
   1066  1.204        ad 		cpu_need_resched(ci, RESCHED_IMMED);
   1067  1.204        ad 		spc_unlock(ci);
   1068  1.204        ad 	}
   1069  1.174        ad }
   1070  1.174        ad 
   1071  1.174        ad /*
   1072  1.174        ad  * sched_unsleep:
   1073  1.174        ad  *
   1074  1.174        ad  *	The is called when the LWP has not been awoken normally but instead
   1075  1.174        ad  *	interrupted: for example, if the sleep timed out.  Because of this,
   1076  1.174        ad  *	it's not a valid action for running or idle LWPs.
   1077  1.174        ad  */
   1078  1.221        ad static u_int
   1079  1.221        ad sched_unsleep(struct lwp *l, bool cleanup)
   1080  1.174        ad {
   1081  1.174        ad 
   1082  1.174        ad 	lwp_unlock(l);
   1083  1.174        ad 	panic("sched_unsleep");
   1084  1.174        ad }
   1085  1.174        ad 
   1086  1.250     rmind static void
   1087  1.188      yamt resched_cpu(struct lwp *l)
   1088  1.188      yamt {
   1089  1.250     rmind 	struct cpu_info *ci = ci = l->l_cpu;
   1090  1.188      yamt 
   1091  1.250     rmind 	KASSERT(lwp_locked(l, NULL));
   1092  1.204        ad 	if (lwp_eprio(l) > ci->ci_schedstate.spc_curpriority)
   1093  1.188      yamt 		cpu_need_resched(ci, 0);
   1094  1.188      yamt }
   1095  1.188      yamt 
   1096  1.188      yamt static void
   1097  1.185      yamt sched_changepri(struct lwp *l, pri_t pri)
   1098  1.174        ad {
   1099  1.174        ad 
   1100  1.188      yamt 	KASSERT(lwp_locked(l, NULL));
   1101  1.174        ad 
   1102  1.204        ad 	if (l->l_stat == LSRUN && (l->l_flag & LW_INMEM) != 0) {
   1103  1.204        ad 		KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_mutex));
   1104  1.204        ad 		sched_dequeue(l);
   1105  1.204        ad 		l->l_priority = pri;
   1106  1.204        ad 		sched_enqueue(l, false);
   1107  1.204        ad 	} else {
   1108  1.174        ad 		l->l_priority = pri;
   1109  1.157      yamt 	}
   1110  1.188      yamt 	resched_cpu(l);
   1111  1.184      yamt }
   1112  1.184      yamt 
   1113  1.188      yamt static void
   1114  1.185      yamt sched_lendpri(struct lwp *l, pri_t pri)
   1115  1.184      yamt {
   1116  1.184      yamt 
   1117  1.188      yamt 	KASSERT(lwp_locked(l, NULL));
   1118  1.184      yamt 
   1119  1.204        ad 	if (l->l_stat == LSRUN && (l->l_flag & LW_INMEM) != 0) {
   1120  1.204        ad 		KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_mutex));
   1121  1.204        ad 		sched_dequeue(l);
   1122  1.204        ad 		l->l_inheritedprio = pri;
   1123  1.204        ad 		sched_enqueue(l, false);
   1124  1.204        ad 	} else {
   1125  1.184      yamt 		l->l_inheritedprio = pri;
   1126  1.184      yamt 	}
   1127  1.188      yamt 	resched_cpu(l);
   1128  1.184      yamt }
   1129  1.184      yamt 
   1130  1.184      yamt struct lwp *
   1131  1.184      yamt syncobj_noowner(wchan_t wchan)
   1132  1.184      yamt {
   1133  1.184      yamt 
   1134  1.184      yamt 	return NULL;
   1135  1.151      yamt }
   1136  1.151      yamt 
   1137  1.250     rmind /* Decay 95% of proc::p_pctcpu in 60 seconds, ccpu = exp(-1/20) */
   1138  1.250     rmind const fixpt_t	ccpu = 0.95122942450071400909 * FSCALE;
   1139  1.134      matt 
   1140  1.134      matt /*
   1141  1.188      yamt  * sched_pstats:
   1142  1.188      yamt  *
   1143  1.188      yamt  * Update process statistics and check CPU resource allocation.
   1144  1.188      yamt  * Call scheduler-specific hook to eventually adjust process/LWP
   1145  1.188      yamt  * priorities.
   1146  1.130   nathanw  */
   1147  1.188      yamt /* ARGSUSED */
   1148  1.113  gmcgarry void
   1149  1.188      yamt sched_pstats(void *arg)
   1150  1.113  gmcgarry {
   1151  1.249     rmind 	const int clkhz = (stathz != 0 ? stathz : hz);
   1152  1.260        ad 	static bool backwards;
   1153  1.188      yamt 	struct rlimit *rlim;
   1154  1.188      yamt 	struct lwp *l;
   1155  1.188      yamt 	struct proc *p;
   1156  1.188      yamt 	long runtm;
   1157  1.249     rmind 	fixpt_t lpctcpu;
   1158  1.249     rmind 	u_int lcpticks;
   1159  1.249     rmind 	int sig;
   1160  1.113  gmcgarry 
   1161  1.188      yamt 	sched_pstats_ticks++;
   1162  1.174        ad 
   1163  1.228        ad 	mutex_enter(proc_lock);
   1164  1.188      yamt 	PROCLIST_FOREACH(p, &allproc) {
   1165  1.249     rmind 		if (__predict_false((p->p_flag & PK_MARKER) != 0))
   1166  1.238        ad 			continue;
   1167  1.238        ad 
   1168  1.188      yamt 		/*
   1169  1.250     rmind 		 * Increment time in/out of memory and sleep
   1170  1.250     rmind 		 * time (if sleeping), ignore overflow.
   1171  1.188      yamt 		 */
   1172  1.229        ad 		mutex_enter(p->p_lock);
   1173  1.212      yamt 		runtm = p->p_rtime.sec;
   1174  1.188      yamt 		LIST_FOREACH(l, &p->p_lwps, l_sibling) {
   1175  1.249     rmind 			if (__predict_false((l->l_flag & LW_IDLE) != 0))
   1176  1.188      yamt 				continue;
   1177  1.188      yamt 			lwp_lock(l);
   1178  1.212      yamt 			runtm += l->l_rtime.sec;
   1179  1.188      yamt 			l->l_swtime++;
   1180  1.242     rmind 			sched_lwp_stats(l);
   1181  1.188      yamt 			lwp_unlock(l);
   1182  1.113  gmcgarry 
   1183  1.188      yamt 			l->l_pctcpu = (l->l_pctcpu * ccpu) >> FSHIFT;
   1184  1.249     rmind 			if (l->l_slptime != 0)
   1185  1.249     rmind 				continue;
   1186  1.249     rmind 
   1187  1.249     rmind 			lpctcpu = l->l_pctcpu;
   1188  1.249     rmind 			lcpticks = atomic_swap_uint(&l->l_cpticks, 0);
   1189  1.249     rmind 			lpctcpu += ((FSCALE - ccpu) *
   1190  1.249     rmind 			    (lcpticks * FSCALE / clkhz)) >> FSHIFT;
   1191  1.249     rmind 			l->l_pctcpu = lpctcpu;
   1192  1.188      yamt 		}
   1193  1.249     rmind 		/* Calculating p_pctcpu only for ps(1) */
   1194  1.188      yamt 		p->p_pctcpu = (p->p_pctcpu * ccpu) >> FSHIFT;
   1195  1.174        ad 
   1196  1.188      yamt 		/*
   1197  1.188      yamt 		 * Check if the process exceeds its CPU resource allocation.
   1198  1.188      yamt 		 * If over max, kill it.
   1199  1.188      yamt 		 */
   1200  1.188      yamt 		rlim = &p->p_rlimit[RLIMIT_CPU];
   1201  1.188      yamt 		sig = 0;
   1202  1.249     rmind 		if (__predict_false(runtm >= rlim->rlim_cur)) {
   1203  1.188      yamt 			if (runtm >= rlim->rlim_max)
   1204  1.188      yamt 				sig = SIGKILL;
   1205  1.188      yamt 			else {
   1206  1.188      yamt 				sig = SIGXCPU;
   1207  1.188      yamt 				if (rlim->rlim_cur < rlim->rlim_max)
   1208  1.188      yamt 					rlim->rlim_cur += 5;
   1209  1.188      yamt 			}
   1210  1.188      yamt 		}
   1211  1.229        ad 		mutex_exit(p->p_lock);
   1212  1.259     rmind 		if (__predict_false(runtm < 0)) {
   1213  1.260        ad 			if (!backwards) {
   1214  1.260        ad 				backwards = true;
   1215  1.260        ad 				printf("WARNING: negative runtime; "
   1216  1.260        ad 				    "monotonic clock has gone backwards\n");
   1217  1.260        ad 			}
   1218  1.259     rmind 		} else if (__predict_false(sig)) {
   1219  1.259     rmind 			KASSERT((p->p_flag & PK_SYSTEM) == 0);
   1220  1.188      yamt 			psignal(p, sig);
   1221  1.259     rmind 		}
   1222  1.174        ad 	}
   1223  1.228        ad 	mutex_exit(proc_lock);
   1224  1.188      yamt 	uvm_meter();
   1225  1.191        ad 	cv_wakeup(&lbolt);
   1226  1.188      yamt 	callout_schedule(&sched_pstats_ch, hz);
   1227  1.113  gmcgarry }
   1228