Home | History | Annotate | Line # | Download | only in kern
kern_synch.c revision 1.298
      1  1.298     rmind /*	$NetBSD: kern_synch.c,v 1.298 2012/02/19 21:06:54 rmind Exp $	*/
      2   1.63   thorpej 
      3   1.63   thorpej /*-
      4  1.260        ad  * Copyright (c) 1999, 2000, 2004, 2006, 2007, 2008, 2009
      5  1.260        ad  *    The NetBSD Foundation, Inc.
      6   1.63   thorpej  * All rights reserved.
      7   1.63   thorpej  *
      8   1.63   thorpej  * This code is derived from software contributed to The NetBSD Foundation
      9   1.63   thorpej  * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
     10  1.188      yamt  * NASA Ames Research Center, by Charles M. Hannum, Andrew Doran and
     11  1.188      yamt  * Daniel Sieger.
     12   1.63   thorpej  *
     13   1.63   thorpej  * Redistribution and use in source and binary forms, with or without
     14   1.63   thorpej  * modification, are permitted provided that the following conditions
     15   1.63   thorpej  * are met:
     16   1.63   thorpej  * 1. Redistributions of source code must retain the above copyright
     17   1.63   thorpej  *    notice, this list of conditions and the following disclaimer.
     18   1.63   thorpej  * 2. Redistributions in binary form must reproduce the above copyright
     19   1.63   thorpej  *    notice, this list of conditions and the following disclaimer in the
     20   1.63   thorpej  *    documentation and/or other materials provided with the distribution.
     21   1.63   thorpej  *
     22   1.63   thorpej  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     23   1.63   thorpej  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     24   1.63   thorpej  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     25   1.63   thorpej  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     26   1.63   thorpej  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     27   1.63   thorpej  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     28   1.63   thorpej  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     29   1.63   thorpej  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     30   1.63   thorpej  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     31   1.63   thorpej  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     32   1.63   thorpej  * POSSIBILITY OF SUCH DAMAGE.
     33   1.63   thorpej  */
     34   1.26       cgd 
     35   1.26       cgd /*-
     36   1.26       cgd  * Copyright (c) 1982, 1986, 1990, 1991, 1993
     37   1.26       cgd  *	The Regents of the University of California.  All rights reserved.
     38   1.26       cgd  * (c) UNIX System Laboratories, Inc.
     39   1.26       cgd  * All or some portions of this file are derived from material licensed
     40   1.26       cgd  * to the University of California by American Telephone and Telegraph
     41   1.26       cgd  * Co. or Unix System Laboratories, Inc. and are reproduced herein with
     42   1.26       cgd  * the permission of UNIX System Laboratories, Inc.
     43   1.26       cgd  *
     44   1.26       cgd  * Redistribution and use in source and binary forms, with or without
     45   1.26       cgd  * modification, are permitted provided that the following conditions
     46   1.26       cgd  * are met:
     47   1.26       cgd  * 1. Redistributions of source code must retain the above copyright
     48   1.26       cgd  *    notice, this list of conditions and the following disclaimer.
     49   1.26       cgd  * 2. Redistributions in binary form must reproduce the above copyright
     50   1.26       cgd  *    notice, this list of conditions and the following disclaimer in the
     51   1.26       cgd  *    documentation and/or other materials provided with the distribution.
     52  1.136       agc  * 3. Neither the name of the University nor the names of its contributors
     53   1.26       cgd  *    may be used to endorse or promote products derived from this software
     54   1.26       cgd  *    without specific prior written permission.
     55   1.26       cgd  *
     56   1.26       cgd  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
     57   1.26       cgd  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     58   1.26       cgd  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     59   1.26       cgd  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
     60   1.26       cgd  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     61   1.26       cgd  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
     62   1.26       cgd  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     63   1.26       cgd  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     64   1.26       cgd  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     65   1.26       cgd  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     66   1.26       cgd  * SUCH DAMAGE.
     67   1.26       cgd  *
     68   1.50      fvdl  *	@(#)kern_synch.c	8.9 (Berkeley) 5/19/95
     69   1.26       cgd  */
     70  1.106     lukem 
     71  1.106     lukem #include <sys/cdefs.h>
     72  1.298     rmind __KERNEL_RCSID(0, "$NetBSD: kern_synch.c,v 1.298 2012/02/19 21:06:54 rmind Exp $");
     73   1.48       mrg 
     74  1.109      yamt #include "opt_kstack.h"
     75  1.110    briggs #include "opt_perfctrs.h"
     76  1.277    darran #include "opt_dtrace.h"
     77   1.26       cgd 
     78  1.174        ad #define	__MUTEX_PRIVATE
     79  1.174        ad 
     80   1.26       cgd #include <sys/param.h>
     81   1.26       cgd #include <sys/systm.h>
     82   1.26       cgd #include <sys/proc.h>
     83   1.26       cgd #include <sys/kernel.h>
     84  1.111    briggs #if defined(PERFCTRS)
     85  1.110    briggs #include <sys/pmc.h>
     86  1.111    briggs #endif
     87  1.188      yamt #include <sys/cpu.h>
     88  1.290  christos #include <sys/pserialize.h>
     89   1.26       cgd #include <sys/resourcevar.h>
     90   1.55      ross #include <sys/sched.h>
     91  1.179       dsl #include <sys/syscall_stats.h>
     92  1.174        ad #include <sys/sleepq.h>
     93  1.174        ad #include <sys/lockdebug.h>
     94  1.190        ad #include <sys/evcnt.h>
     95  1.199        ad #include <sys/intr.h>
     96  1.207        ad #include <sys/lwpctl.h>
     97  1.209        ad #include <sys/atomic.h>
     98  1.215        ad #include <sys/simplelock.h>
     99  1.295     njoly #include <sys/syslog.h>
    100   1.47       mrg 
    101   1.47       mrg #include <uvm/uvm_extern.h>
    102   1.47       mrg 
    103  1.231        ad #include <dev/lockstat.h>
    104  1.231        ad 
    105  1.276    darran #include <sys/dtrace_bsd.h>
    106  1.279    darran int                             dtrace_vtime_active=0;
    107  1.276    darran dtrace_vtime_switch_func_t      dtrace_vtime_switch_func;
    108  1.276    darran 
    109  1.271     rmind static void	sched_unsleep(struct lwp *, bool);
    110  1.188      yamt static void	sched_changepri(struct lwp *, pri_t);
    111  1.188      yamt static void	sched_lendpri(struct lwp *, pri_t);
    112  1.250     rmind static void	resched_cpu(struct lwp *);
    113  1.122   thorpej 
    114  1.174        ad syncobj_t sleep_syncobj = {
    115  1.174        ad 	SOBJ_SLEEPQ_SORTED,
    116  1.174        ad 	sleepq_unsleep,
    117  1.184      yamt 	sleepq_changepri,
    118  1.184      yamt 	sleepq_lendpri,
    119  1.184      yamt 	syncobj_noowner,
    120  1.174        ad };
    121  1.174        ad 
    122  1.174        ad syncobj_t sched_syncobj = {
    123  1.174        ad 	SOBJ_SLEEPQ_SORTED,
    124  1.174        ad 	sched_unsleep,
    125  1.184      yamt 	sched_changepri,
    126  1.184      yamt 	sched_lendpri,
    127  1.184      yamt 	syncobj_noowner,
    128  1.174        ad };
    129  1.122   thorpej 
    130  1.289     rmind /* "Lightning bolt": once a second sleep address. */
    131  1.289     rmind kcondvar_t		lbolt			__cacheline_aligned;
    132  1.223        ad 
    133  1.289     rmind u_int			sched_pstats_ticks	__cacheline_aligned;
    134  1.289     rmind 
    135  1.289     rmind /* Preemption event counters. */
    136  1.289     rmind static struct evcnt	kpreempt_ev_crit	__cacheline_aligned;
    137  1.289     rmind static struct evcnt	kpreempt_ev_klock	__cacheline_aligned;
    138  1.289     rmind static struct evcnt	kpreempt_ev_immed	__cacheline_aligned;
    139  1.231        ad 
    140  1.231        ad /*
    141  1.174        ad  * During autoconfiguration or after a panic, a sleep will simply lower the
    142  1.174        ad  * priority briefly to allow interrupts, then return.  The priority to be
    143  1.174        ad  * used (safepri) is machine-dependent, thus this value is initialized and
    144  1.174        ad  * maintained in the machine-dependent layers.  This priority will typically
    145  1.174        ad  * be 0, or the lowest priority that is safe for use on the interrupt stack;
    146  1.174        ad  * it can be made higher to block network software interrupts after panics.
    147   1.26       cgd  */
    148  1.174        ad int	safepri;
    149   1.26       cgd 
    150  1.237     rmind void
    151  1.270      elad synch_init(void)
    152  1.237     rmind {
    153  1.237     rmind 
    154  1.237     rmind 	cv_init(&lbolt, "lbolt");
    155  1.237     rmind 
    156  1.239        ad 	evcnt_attach_dynamic(&kpreempt_ev_crit, EVCNT_TYPE_MISC, NULL,
    157  1.237     rmind 	   "kpreempt", "defer: critical section");
    158  1.239        ad 	evcnt_attach_dynamic(&kpreempt_ev_klock, EVCNT_TYPE_MISC, NULL,
    159  1.237     rmind 	   "kpreempt", "defer: kernel_lock");
    160  1.239        ad 	evcnt_attach_dynamic(&kpreempt_ev_immed, EVCNT_TYPE_MISC, NULL,
    161  1.237     rmind 	   "kpreempt", "immediate");
    162  1.237     rmind }
    163  1.237     rmind 
    164   1.26       cgd /*
    165  1.174        ad  * OBSOLETE INTERFACE
    166  1.174        ad  *
    167  1.255     skrll  * General sleep call.  Suspends the current LWP until a wakeup is
    168  1.255     skrll  * performed on the specified identifier.  The LWP will then be made
    169  1.174        ad  * runnable with the specified priority.  Sleeps at most timo/hz seconds (0
    170  1.174        ad  * means no timeout).  If pri includes PCATCH flag, signals are checked
    171   1.26       cgd  * before and after sleeping, else signals are not checked.  Returns 0 if
    172   1.26       cgd  * awakened, EWOULDBLOCK if the timeout expires.  If PCATCH is set and a
    173   1.26       cgd  * signal needs to be delivered, ERESTART is returned if the current system
    174   1.26       cgd  * call should be restarted if possible, and EINTR is returned if the system
    175   1.26       cgd  * call should be interrupted by the signal (return EINTR).
    176   1.26       cgd  */
    177   1.26       cgd int
    178  1.297     rmind tsleep(wchan_t ident, pri_t priority, const char *wmesg, int timo)
    179   1.26       cgd {
    180  1.122   thorpej 	struct lwp *l = curlwp;
    181  1.174        ad 	sleepq_t *sq;
    182  1.244        ad 	kmutex_t *mp;
    183   1.26       cgd 
    184  1.204        ad 	KASSERT((l->l_pflag & LP_INTR) == 0);
    185  1.272     pooka 	KASSERT(ident != &lbolt);
    186  1.204        ad 
    187  1.174        ad 	if (sleepq_dontsleep(l)) {
    188  1.174        ad 		(void)sleepq_abort(NULL, 0);
    189  1.174        ad 		return 0;
    190   1.26       cgd 	}
    191   1.78  sommerfe 
    192  1.204        ad 	l->l_kpriority = true;
    193  1.244        ad 	sq = sleeptab_lookup(&sleeptab, ident, &mp);
    194  1.244        ad 	sleepq_enter(sq, l, mp);
    195  1.204        ad 	sleepq_enqueue(sq, ident, wmesg, &sleep_syncobj);
    196  1.297     rmind 	return sleepq_block(timo, priority & PCATCH);
    197   1.26       cgd }
    198   1.26       cgd 
    199  1.187        ad int
    200  1.187        ad mtsleep(wchan_t ident, pri_t priority, const char *wmesg, int timo,
    201  1.187        ad 	kmutex_t *mtx)
    202  1.187        ad {
    203  1.187        ad 	struct lwp *l = curlwp;
    204  1.187        ad 	sleepq_t *sq;
    205  1.244        ad 	kmutex_t *mp;
    206  1.188      yamt 	int error;
    207  1.187        ad 
    208  1.204        ad 	KASSERT((l->l_pflag & LP_INTR) == 0);
    209  1.272     pooka 	KASSERT(ident != &lbolt);
    210  1.204        ad 
    211  1.187        ad 	if (sleepq_dontsleep(l)) {
    212  1.187        ad 		(void)sleepq_abort(mtx, (priority & PNORELOCK) != 0);
    213  1.187        ad 		return 0;
    214  1.187        ad 	}
    215  1.187        ad 
    216  1.204        ad 	l->l_kpriority = true;
    217  1.244        ad 	sq = sleeptab_lookup(&sleeptab, ident, &mp);
    218  1.244        ad 	sleepq_enter(sq, l, mp);
    219  1.204        ad 	sleepq_enqueue(sq, ident, wmesg, &sleep_syncobj);
    220  1.187        ad 	mutex_exit(mtx);
    221  1.188      yamt 	error = sleepq_block(timo, priority & PCATCH);
    222  1.187        ad 
    223  1.187        ad 	if ((priority & PNORELOCK) == 0)
    224  1.187        ad 		mutex_enter(mtx);
    225  1.297     rmind 
    226  1.187        ad 	return error;
    227  1.187        ad }
    228  1.187        ad 
    229   1.26       cgd /*
    230  1.174        ad  * General sleep call for situations where a wake-up is not expected.
    231   1.26       cgd  */
    232  1.174        ad int
    233  1.182   thorpej kpause(const char *wmesg, bool intr, int timo, kmutex_t *mtx)
    234   1.26       cgd {
    235  1.174        ad 	struct lwp *l = curlwp;
    236  1.244        ad 	kmutex_t *mp;
    237  1.174        ad 	sleepq_t *sq;
    238  1.174        ad 	int error;
    239   1.26       cgd 
    240  1.284     pooka 	KASSERT(!(timo == 0 && intr == false));
    241  1.284     pooka 
    242  1.174        ad 	if (sleepq_dontsleep(l))
    243  1.174        ad 		return sleepq_abort(NULL, 0);
    244   1.26       cgd 
    245  1.174        ad 	if (mtx != NULL)
    246  1.174        ad 		mutex_exit(mtx);
    247  1.204        ad 	l->l_kpriority = true;
    248  1.244        ad 	sq = sleeptab_lookup(&sleeptab, l, &mp);
    249  1.244        ad 	sleepq_enter(sq, l, mp);
    250  1.204        ad 	sleepq_enqueue(sq, l, wmesg, &sleep_syncobj);
    251  1.188      yamt 	error = sleepq_block(timo, intr);
    252  1.174        ad 	if (mtx != NULL)
    253  1.174        ad 		mutex_enter(mtx);
    254   1.83   thorpej 
    255  1.174        ad 	return error;
    256  1.139        cl }
    257  1.139        cl 
    258   1.26       cgd /*
    259  1.174        ad  * OBSOLETE INTERFACE
    260  1.174        ad  *
    261  1.255     skrll  * Make all LWPs sleeping on the specified identifier runnable.
    262   1.26       cgd  */
    263   1.26       cgd void
    264  1.174        ad wakeup(wchan_t ident)
    265   1.26       cgd {
    266  1.174        ad 	sleepq_t *sq;
    267  1.244        ad 	kmutex_t *mp;
    268   1.83   thorpej 
    269  1.261     rmind 	if (__predict_false(cold))
    270  1.174        ad 		return;
    271   1.83   thorpej 
    272  1.244        ad 	sq = sleeptab_lookup(&sleeptab, ident, &mp);
    273  1.244        ad 	sleepq_wake(sq, ident, (u_int)-1, mp);
    274   1.63   thorpej }
    275   1.63   thorpej 
    276   1.63   thorpej /*
    277  1.255     skrll  * General yield call.  Puts the current LWP back on its run queue and
    278  1.117  gmcgarry  * performs a voluntary context switch.  Should only be called when the
    279  1.255     skrll  * current LWP explicitly requests it (eg sched_yield(2)).
    280  1.117  gmcgarry  */
    281  1.117  gmcgarry void
    282  1.117  gmcgarry yield(void)
    283  1.117  gmcgarry {
    284  1.122   thorpej 	struct lwp *l = curlwp;
    285  1.117  gmcgarry 
    286  1.174        ad 	KERNEL_UNLOCK_ALL(l, &l->l_biglocks);
    287  1.174        ad 	lwp_lock(l);
    288  1.217        ad 	KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_lwplock));
    289  1.188      yamt 	KASSERT(l->l_stat == LSONPROC);
    290  1.204        ad 	l->l_kpriority = false;
    291  1.188      yamt 	(void)mi_switch(l);
    292  1.174        ad 	KERNEL_LOCK(l->l_biglocks, l);
    293   1.69   thorpej }
    294   1.69   thorpej 
    295   1.69   thorpej /*
    296  1.255     skrll  * General preemption call.  Puts the current LWP back on its run queue
    297  1.156    rpaulo  * and performs an involuntary context switch.
    298   1.69   thorpej  */
    299   1.69   thorpej void
    300  1.174        ad preempt(void)
    301   1.69   thorpej {
    302  1.122   thorpej 	struct lwp *l = curlwp;
    303   1.69   thorpej 
    304  1.174        ad 	KERNEL_UNLOCK_ALL(l, &l->l_biglocks);
    305  1.174        ad 	lwp_lock(l);
    306  1.217        ad 	KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_lwplock));
    307  1.188      yamt 	KASSERT(l->l_stat == LSONPROC);
    308  1.204        ad 	l->l_kpriority = false;
    309  1.174        ad 	l->l_nivcsw++;
    310  1.188      yamt 	(void)mi_switch(l);
    311  1.174        ad 	KERNEL_LOCK(l->l_biglocks, l);
    312   1.69   thorpej }
    313   1.69   thorpej 
    314  1.234        ad /*
    315  1.234        ad  * Handle a request made by another agent to preempt the current LWP
    316  1.234        ad  * in-kernel.  Usually called when l_dopreempt may be non-zero.
    317  1.234        ad  *
    318  1.234        ad  * Character addresses for lockstat only.
    319  1.234        ad  */
    320  1.231        ad static char	in_critical_section;
    321  1.231        ad static char	kernel_lock_held;
    322  1.231        ad static char	is_softint;
    323  1.262      yamt static char	cpu_kpreempt_enter_fail;
    324  1.231        ad 
    325  1.231        ad bool
    326  1.231        ad kpreempt(uintptr_t where)
    327  1.231        ad {
    328  1.231        ad 	uintptr_t failed;
    329  1.231        ad 	lwp_t *l;
    330  1.264        ad 	int s, dop, lsflag;
    331  1.231        ad 
    332  1.231        ad 	l = curlwp;
    333  1.231        ad 	failed = 0;
    334  1.231        ad 	while ((dop = l->l_dopreempt) != 0) {
    335  1.231        ad 		if (l->l_stat != LSONPROC) {
    336  1.231        ad 			/*
    337  1.231        ad 			 * About to block (or die), let it happen.
    338  1.231        ad 			 * Doesn't really count as "preemption has
    339  1.231        ad 			 * been blocked", since we're going to
    340  1.231        ad 			 * context switch.
    341  1.231        ad 			 */
    342  1.231        ad 			l->l_dopreempt = 0;
    343  1.231        ad 			return true;
    344  1.231        ad 		}
    345  1.231        ad 		if (__predict_false((l->l_flag & LW_IDLE) != 0)) {
    346  1.231        ad 			/* Can't preempt idle loop, don't count as failure. */
    347  1.261     rmind 			l->l_dopreempt = 0;
    348  1.261     rmind 			return true;
    349  1.231        ad 		}
    350  1.231        ad 		if (__predict_false(l->l_nopreempt != 0)) {
    351  1.231        ad 			/* LWP holds preemption disabled, explicitly. */
    352  1.231        ad 			if ((dop & DOPREEMPT_COUNTED) == 0) {
    353  1.234        ad 				kpreempt_ev_crit.ev_count++;
    354  1.231        ad 			}
    355  1.231        ad 			failed = (uintptr_t)&in_critical_section;
    356  1.231        ad 			break;
    357  1.231        ad 		}
    358  1.231        ad 		if (__predict_false((l->l_pflag & LP_INTR) != 0)) {
    359  1.261     rmind 			/* Can't preempt soft interrupts yet. */
    360  1.261     rmind 			l->l_dopreempt = 0;
    361  1.261     rmind 			failed = (uintptr_t)&is_softint;
    362  1.261     rmind 			break;
    363  1.231        ad 		}
    364  1.231        ad 		s = splsched();
    365  1.231        ad 		if (__predict_false(l->l_blcnt != 0 ||
    366  1.231        ad 		    curcpu()->ci_biglock_wanted != NULL)) {
    367  1.231        ad 			/* Hold or want kernel_lock, code is not MT safe. */
    368  1.231        ad 			splx(s);
    369  1.231        ad 			if ((dop & DOPREEMPT_COUNTED) == 0) {
    370  1.234        ad 				kpreempt_ev_klock.ev_count++;
    371  1.231        ad 			}
    372  1.231        ad 			failed = (uintptr_t)&kernel_lock_held;
    373  1.231        ad 			break;
    374  1.231        ad 		}
    375  1.231        ad 		if (__predict_false(!cpu_kpreempt_enter(where, s))) {
    376  1.231        ad 			/*
    377  1.231        ad 			 * It may be that the IPL is too high.
    378  1.231        ad 			 * kpreempt_enter() can schedule an
    379  1.231        ad 			 * interrupt to retry later.
    380  1.231        ad 			 */
    381  1.231        ad 			splx(s);
    382  1.262      yamt 			failed = (uintptr_t)&cpu_kpreempt_enter_fail;
    383  1.231        ad 			break;
    384  1.231        ad 		}
    385  1.231        ad 		/* Do it! */
    386  1.231        ad 		if (__predict_true((dop & DOPREEMPT_COUNTED) == 0)) {
    387  1.234        ad 			kpreempt_ev_immed.ev_count++;
    388  1.231        ad 		}
    389  1.231        ad 		lwp_lock(l);
    390  1.231        ad 		mi_switch(l);
    391  1.231        ad 		l->l_nopreempt++;
    392  1.231        ad 		splx(s);
    393  1.231        ad 
    394  1.231        ad 		/* Take care of any MD cleanup. */
    395  1.231        ad 		cpu_kpreempt_exit(where);
    396  1.231        ad 		l->l_nopreempt--;
    397  1.231        ad 	}
    398  1.231        ad 
    399  1.264        ad 	if (__predict_true(!failed)) {
    400  1.264        ad 		return false;
    401  1.264        ad 	}
    402  1.264        ad 
    403  1.231        ad 	/* Record preemption failure for reporting via lockstat. */
    404  1.264        ad 	atomic_or_uint(&l->l_dopreempt, DOPREEMPT_COUNTED);
    405  1.264        ad 	lsflag = 0;
    406  1.264        ad 	LOCKSTAT_ENTER(lsflag);
    407  1.264        ad 	if (__predict_false(lsflag)) {
    408  1.264        ad 		if (where == 0) {
    409  1.264        ad 			where = (uintptr_t)__builtin_return_address(0);
    410  1.264        ad 		}
    411  1.264        ad 		/* Preemption is on, might recurse, so make it atomic. */
    412  1.264        ad 		if (atomic_cas_ptr_ni((void *)&l->l_pfailaddr, NULL,
    413  1.264        ad 		    (void *)where) == NULL) {
    414  1.264        ad 			LOCKSTAT_START_TIMER(lsflag, l->l_pfailtime);
    415  1.264        ad 			l->l_pfaillock = failed;
    416  1.231        ad 		}
    417  1.231        ad 	}
    418  1.264        ad 	LOCKSTAT_EXIT(lsflag);
    419  1.264        ad 	return true;
    420  1.231        ad }
    421  1.231        ad 
    422   1.69   thorpej /*
    423  1.231        ad  * Return true if preemption is explicitly disabled.
    424  1.230        ad  */
    425  1.231        ad bool
    426  1.231        ad kpreempt_disabled(void)
    427  1.231        ad {
    428  1.261     rmind 	const lwp_t *l = curlwp;
    429  1.231        ad 
    430  1.231        ad 	return l->l_nopreempt != 0 || l->l_stat == LSZOMB ||
    431  1.231        ad 	    (l->l_flag & LW_IDLE) != 0 || cpu_kpreempt_disabled();
    432  1.231        ad }
    433  1.230        ad 
    434  1.230        ad /*
    435  1.231        ad  * Disable kernel preemption.
    436  1.230        ad  */
    437  1.230        ad void
    438  1.231        ad kpreempt_disable(void)
    439  1.230        ad {
    440  1.230        ad 
    441  1.231        ad 	KPREEMPT_DISABLE(curlwp);
    442  1.230        ad }
    443  1.230        ad 
    444  1.230        ad /*
    445  1.231        ad  * Reenable kernel preemption.
    446  1.230        ad  */
    447  1.231        ad void
    448  1.231        ad kpreempt_enable(void)
    449  1.230        ad {
    450  1.230        ad 
    451  1.231        ad 	KPREEMPT_ENABLE(curlwp);
    452  1.230        ad }
    453  1.230        ad 
    454  1.230        ad /*
    455  1.188      yamt  * Compute the amount of time during which the current lwp was running.
    456  1.130   nathanw  *
    457  1.188      yamt  * - update l_rtime unless it's an idle lwp.
    458  1.188      yamt  */
    459  1.188      yamt 
    460  1.199        ad void
    461  1.212      yamt updatertime(lwp_t *l, const struct bintime *now)
    462  1.188      yamt {
    463  1.188      yamt 
    464  1.261     rmind 	if (__predict_false(l->l_flag & LW_IDLE))
    465  1.188      yamt 		return;
    466  1.188      yamt 
    467  1.212      yamt 	/* rtime += now - stime */
    468  1.212      yamt 	bintime_add(&l->l_rtime, now);
    469  1.212      yamt 	bintime_sub(&l->l_rtime, &l->l_stime);
    470  1.188      yamt }
    471  1.188      yamt 
    472  1.188      yamt /*
    473  1.245        ad  * Select next LWP from the current CPU to run..
    474  1.245        ad  */
    475  1.245        ad static inline lwp_t *
    476  1.245        ad nextlwp(struct cpu_info *ci, struct schedstate_percpu *spc)
    477  1.245        ad {
    478  1.245        ad 	lwp_t *newl;
    479  1.245        ad 
    480  1.245        ad 	/*
    481  1.245        ad 	 * Let sched_nextlwp() select the LWP to run the CPU next.
    482  1.245        ad 	 * If no LWP is runnable, select the idle LWP.
    483  1.245        ad 	 *
    484  1.245        ad 	 * Note that spc_lwplock might not necessary be held, and
    485  1.245        ad 	 * new thread would be unlocked after setting the LWP-lock.
    486  1.245        ad 	 */
    487  1.245        ad 	newl = sched_nextlwp();
    488  1.245        ad 	if (newl != NULL) {
    489  1.245        ad 		sched_dequeue(newl);
    490  1.245        ad 		KASSERT(lwp_locked(newl, spc->spc_mutex));
    491  1.274     rmind 		KASSERT(newl->l_cpu == ci);
    492  1.245        ad 		newl->l_stat = LSONPROC;
    493  1.248        ad 		newl->l_pflag |= LP_RUNNING;
    494  1.245        ad 		lwp_setlock(newl, spc->spc_lwplock);
    495  1.245        ad 	} else {
    496  1.245        ad 		newl = ci->ci_data.cpu_idlelwp;
    497  1.245        ad 		newl->l_stat = LSONPROC;
    498  1.248        ad 		newl->l_pflag |= LP_RUNNING;
    499  1.245        ad 	}
    500  1.261     rmind 
    501  1.245        ad 	/*
    502  1.245        ad 	 * Only clear want_resched if there are no pending (slow)
    503  1.245        ad 	 * software interrupts.
    504  1.245        ad 	 */
    505  1.245        ad 	ci->ci_want_resched = ci->ci_data.cpu_softints;
    506  1.245        ad 	spc->spc_flags &= ~SPCF_SWITCHCLEAR;
    507  1.245        ad 	spc->spc_curpriority = lwp_eprio(newl);
    508  1.245        ad 
    509  1.245        ad 	return newl;
    510  1.245        ad }
    511  1.245        ad 
    512  1.245        ad /*
    513  1.188      yamt  * The machine independent parts of context switch.
    514  1.188      yamt  *
    515  1.188      yamt  * Returns 1 if another LWP was actually run.
    516   1.26       cgd  */
    517  1.122   thorpej int
    518  1.199        ad mi_switch(lwp_t *l)
    519   1.26       cgd {
    520  1.246     rmind 	struct cpu_info *ci;
    521   1.76   thorpej 	struct schedstate_percpu *spc;
    522  1.188      yamt 	struct lwp *newl;
    523  1.174        ad 	int retval, oldspl;
    524  1.212      yamt 	struct bintime bt;
    525  1.199        ad 	bool returning;
    526   1.26       cgd 
    527  1.188      yamt 	KASSERT(lwp_locked(l, NULL));
    528  1.231        ad 	KASSERT(kpreempt_disabled());
    529  1.188      yamt 	LOCKDEBUG_BARRIER(l->l_mutex, 1);
    530  1.174        ad 
    531  1.174        ad 	kstack_check_magic(l);
    532   1.83   thorpej 
    533  1.212      yamt 	binuptime(&bt);
    534  1.199        ad 
    535  1.267      yamt 	KASSERT((l->l_pflag & LP_RUNNING) != 0);
    536  1.231        ad 	KASSERT(l->l_cpu == curcpu());
    537  1.196        ad 	ci = l->l_cpu;
    538  1.196        ad 	spc = &ci->ci_schedstate;
    539  1.199        ad 	returning = false;
    540  1.190        ad 	newl = NULL;
    541  1.190        ad 
    542  1.199        ad 	/*
    543  1.199        ad 	 * If we have been asked to switch to a specific LWP, then there
    544  1.199        ad 	 * is no need to inspect the run queues.  If a soft interrupt is
    545  1.199        ad 	 * blocking, then return to the interrupted thread without adjusting
    546  1.199        ad 	 * VM context or its start time: neither have been changed in order
    547  1.199        ad 	 * to take the interrupt.
    548  1.199        ad 	 */
    549  1.190        ad 	if (l->l_switchto != NULL) {
    550  1.204        ad 		if ((l->l_pflag & LP_INTR) != 0) {
    551  1.199        ad 			returning = true;
    552  1.199        ad 			softint_block(l);
    553  1.248        ad 			if ((l->l_pflag & LP_TIMEINTR) != 0)
    554  1.212      yamt 				updatertime(l, &bt);
    555  1.199        ad 		}
    556  1.190        ad 		newl = l->l_switchto;
    557  1.190        ad 		l->l_switchto = NULL;
    558  1.190        ad 	}
    559  1.204        ad #ifndef __HAVE_FAST_SOFTINTS
    560  1.204        ad 	else if (ci->ci_data.cpu_softints != 0) {
    561  1.204        ad 		/* There are pending soft interrupts, so pick one. */
    562  1.204        ad 		newl = softint_picklwp();
    563  1.204        ad 		newl->l_stat = LSONPROC;
    564  1.248        ad 		newl->l_pflag |= LP_RUNNING;
    565  1.204        ad 	}
    566  1.204        ad #endif	/* !__HAVE_FAST_SOFTINTS */
    567  1.190        ad 
    568  1.180       dsl 	/* Count time spent in current system call */
    569  1.199        ad 	if (!returning) {
    570  1.199        ad 		SYSCALL_TIME_SLEEP(l);
    571  1.180       dsl 
    572  1.199        ad 		/*
    573  1.199        ad 		 * XXXSMP If we are using h/w performance counters,
    574  1.199        ad 		 * save context.
    575  1.199        ad 		 */
    576  1.174        ad #if PERFCTRS
    577  1.199        ad 		if (PMC_ENABLED(l->l_proc)) {
    578  1.199        ad 			pmc_save_context(l->l_proc);
    579  1.199        ad 		}
    580  1.199        ad #endif
    581  1.212      yamt 		updatertime(l, &bt);
    582  1.174        ad 	}
    583  1.113  gmcgarry 
    584  1.246     rmind 	/* Lock the runqueue */
    585  1.246     rmind 	KASSERT(l->l_stat != LSRUN);
    586  1.246     rmind 	mutex_spin_enter(spc->spc_mutex);
    587  1.246     rmind 
    588  1.113  gmcgarry 	/*
    589  1.174        ad 	 * If on the CPU and we have gotten this far, then we must yield.
    590  1.113  gmcgarry 	 */
    591  1.246     rmind 	if (l->l_stat == LSONPROC && l != newl) {
    592  1.217        ad 		KASSERT(lwp_locked(l, spc->spc_lwplock));
    593  1.188      yamt 		if ((l->l_flag & LW_IDLE) == 0) {
    594  1.188      yamt 			l->l_stat = LSRUN;
    595  1.246     rmind 			lwp_setlock(l, spc->spc_mutex);
    596  1.246     rmind 			sched_enqueue(l, true);
    597  1.285     rmind 			/*
    598  1.285     rmind 			 * Handle migration.  Note that "migrating LWP" may
    599  1.285     rmind 			 * be reset here, if interrupt/preemption happens
    600  1.285     rmind 			 * early in idle LWP.
    601  1.285     rmind 			 */
    602  1.285     rmind 			if (l->l_target_cpu != NULL) {
    603  1.285     rmind 				KASSERT((l->l_pflag & LP_INTR) == 0);
    604  1.246     rmind 				spc->spc_migrating = l;
    605  1.216     rmind 			}
    606  1.246     rmind 		} else
    607  1.188      yamt 			l->l_stat = LSIDL;
    608  1.174        ad 	}
    609  1.174        ad 
    610  1.245        ad 	/* Pick new LWP to run. */
    611  1.190        ad 	if (newl == NULL) {
    612  1.245        ad 		newl = nextlwp(ci, spc);
    613  1.199        ad 	}
    614  1.199        ad 
    615  1.204        ad 	/* Items that must be updated with the CPU locked. */
    616  1.199        ad 	if (!returning) {
    617  1.204        ad 		/* Update the new LWP's start time. */
    618  1.212      yamt 		newl->l_stime = bt;
    619  1.204        ad 
    620  1.199        ad 		/*
    621  1.204        ad 		 * ci_curlwp changes when a fast soft interrupt occurs.
    622  1.204        ad 		 * We use cpu_onproc to keep track of which kernel or
    623  1.204        ad 		 * user thread is running 'underneath' the software
    624  1.204        ad 		 * interrupt.  This is important for time accounting,
    625  1.204        ad 		 * itimers and forcing user threads to preempt (aston).
    626  1.199        ad 		 */
    627  1.204        ad 		ci->ci_data.cpu_onproc = newl;
    628  1.188      yamt 	}
    629  1.188      yamt 
    630  1.241        ad 	/*
    631  1.241        ad 	 * Preemption related tasks.  Must be done with the current
    632  1.241        ad 	 * CPU locked.
    633  1.241        ad 	 */
    634  1.241        ad 	cpu_did_resched(l);
    635  1.231        ad 	l->l_dopreempt = 0;
    636  1.231        ad 	if (__predict_false(l->l_pfailaddr != 0)) {
    637  1.231        ad 		LOCKSTAT_FLAG(lsflag);
    638  1.231        ad 		LOCKSTAT_ENTER(lsflag);
    639  1.231        ad 		LOCKSTAT_STOP_TIMER(lsflag, l->l_pfailtime);
    640  1.231        ad 		LOCKSTAT_EVENT_RA(lsflag, l->l_pfaillock, LB_NOPREEMPT|LB_SPIN,
    641  1.231        ad 		    1, l->l_pfailtime, l->l_pfailaddr);
    642  1.231        ad 		LOCKSTAT_EXIT(lsflag);
    643  1.231        ad 		l->l_pfailtime = 0;
    644  1.231        ad 		l->l_pfaillock = 0;
    645  1.231        ad 		l->l_pfailaddr = 0;
    646  1.231        ad 	}
    647  1.231        ad 
    648  1.188      yamt 	if (l != newl) {
    649  1.188      yamt 		struct lwp *prevlwp;
    650  1.174        ad 
    651  1.209        ad 		/* Release all locks, but leave the current LWP locked */
    652  1.246     rmind 		if (l->l_mutex == spc->spc_mutex) {
    653  1.209        ad 			/*
    654  1.209        ad 			 * Drop spc_lwplock, if the current LWP has been moved
    655  1.209        ad 			 * to the run queue (it is now locked by spc_mutex).
    656  1.209        ad 			 */
    657  1.217        ad 			mutex_spin_exit(spc->spc_lwplock);
    658  1.188      yamt 		} else {
    659  1.209        ad 			/*
    660  1.209        ad 			 * Otherwise, drop the spc_mutex, we are done with the
    661  1.209        ad 			 * run queues.
    662  1.209        ad 			 */
    663  1.188      yamt 			mutex_spin_exit(spc->spc_mutex);
    664  1.188      yamt 		}
    665  1.188      yamt 
    666  1.209        ad 		/*
    667  1.253     skrll 		 * Mark that context switch is going to be performed
    668  1.209        ad 		 * for this LWP, to protect it from being switched
    669  1.209        ad 		 * to on another CPU.
    670  1.209        ad 		 */
    671  1.209        ad 		KASSERT(l->l_ctxswtch == 0);
    672  1.209        ad 		l->l_ctxswtch = 1;
    673  1.209        ad 		l->l_ncsw++;
    674  1.267      yamt 		KASSERT((l->l_pflag & LP_RUNNING) != 0);
    675  1.248        ad 		l->l_pflag &= ~LP_RUNNING;
    676  1.209        ad 
    677  1.209        ad 		/*
    678  1.209        ad 		 * Increase the count of spin-mutexes before the release
    679  1.209        ad 		 * of the last lock - we must remain at IPL_SCHED during
    680  1.209        ad 		 * the context switch.
    681  1.209        ad 		 */
    682  1.287      matt 		KASSERTMSG(ci->ci_mtx_count == -1,
    683  1.291       jym 		    "%s: cpu%u: ci_mtx_count (%d) != -1",
    684  1.291       jym 		     __func__, cpu_index(ci), ci->ci_mtx_count);
    685  1.209        ad 		oldspl = MUTEX_SPIN_OLDSPL(ci);
    686  1.209        ad 		ci->ci_mtx_count--;
    687  1.209        ad 		lwp_unlock(l);
    688  1.209        ad 
    689  1.218        ad 		/* Count the context switch on this CPU. */
    690  1.218        ad 		ci->ci_data.cpu_nswtch++;
    691  1.188      yamt 
    692  1.209        ad 		/* Update status for lwpctl, if present. */
    693  1.209        ad 		if (l->l_lwpctl != NULL)
    694  1.209        ad 			l->l_lwpctl->lc_curcpu = LWPCTL_CPU_NONE;
    695  1.209        ad 
    696  1.199        ad 		/*
    697  1.199        ad 		 * Save old VM context, unless a soft interrupt
    698  1.199        ad 		 * handler is blocking.
    699  1.199        ad 		 */
    700  1.199        ad 		if (!returning)
    701  1.199        ad 			pmap_deactivate(l);
    702  1.188      yamt 
    703  1.209        ad 		/*
    704  1.275     skrll 		 * We may need to spin-wait if 'newl' is still
    705  1.209        ad 		 * context switching on another CPU.
    706  1.209        ad 		 */
    707  1.261     rmind 		if (__predict_false(newl->l_ctxswtch != 0)) {
    708  1.209        ad 			u_int count;
    709  1.209        ad 			count = SPINLOCK_BACKOFF_MIN;
    710  1.209        ad 			while (newl->l_ctxswtch)
    711  1.209        ad 				SPINLOCK_BACKOFF(count);
    712  1.209        ad 		}
    713  1.207        ad 
    714  1.276    darran 		/*
    715  1.276    darran 		 * If DTrace has set the active vtime enum to anything
    716  1.276    darran 		 * other than INACTIVE (0), then it should have set the
    717  1.276    darran 		 * function to call.
    718  1.276    darran 		 */
    719  1.278    darran 		if (__predict_false(dtrace_vtime_active)) {
    720  1.276    darran 			(*dtrace_vtime_switch_func)(newl);
    721  1.276    darran 		}
    722  1.276    darran 
    723  1.188      yamt 		/* Switch to the new LWP.. */
    724  1.204        ad 		prevlwp = cpu_switchto(l, newl, returning);
    725  1.207        ad 		ci = curcpu();
    726  1.207        ad 
    727  1.188      yamt 		/*
    728  1.209        ad 		 * Switched away - we have new curlwp.
    729  1.209        ad 		 * Restore VM context and IPL.
    730  1.188      yamt 		 */
    731  1.209        ad 		pmap_activate(l);
    732  1.265     rmind 		uvm_emap_switch(l);
    733  1.288     rmind 		pcu_switchpoint(l);
    734  1.265     rmind 
    735  1.188      yamt 		if (prevlwp != NULL) {
    736  1.209        ad 			/* Normalize the count of the spin-mutexes */
    737  1.209        ad 			ci->ci_mtx_count++;
    738  1.209        ad 			/* Unmark the state of context switch */
    739  1.209        ad 			membar_exit();
    740  1.209        ad 			prevlwp->l_ctxswtch = 0;
    741  1.188      yamt 		}
    742  1.209        ad 
    743  1.209        ad 		/* Update status for lwpctl, if present. */
    744  1.219        ad 		if (l->l_lwpctl != NULL) {
    745  1.209        ad 			l->l_lwpctl->lc_curcpu = (int)cpu_index(ci);
    746  1.219        ad 			l->l_lwpctl->lc_pctr++;
    747  1.219        ad 		}
    748  1.174        ad 
    749  1.290  christos 		/* Note trip through cpu_switchto(). */
    750  1.290  christos 		pserialize_switchpoint();
    751  1.290  christos 
    752  1.231        ad 		KASSERT(l->l_cpu == ci);
    753  1.231        ad 		splx(oldspl);
    754  1.188      yamt 		retval = 1;
    755  1.188      yamt 	} else {
    756  1.188      yamt 		/* Nothing to do - just unlock and return. */
    757  1.246     rmind 		mutex_spin_exit(spc->spc_mutex);
    758  1.188      yamt 		lwp_unlock(l);
    759  1.122   thorpej 		retval = 0;
    760  1.122   thorpej 	}
    761  1.110    briggs 
    762  1.188      yamt 	KASSERT(l == curlwp);
    763  1.188      yamt 	KASSERT(l->l_stat == LSONPROC);
    764  1.188      yamt 
    765  1.110    briggs 	/*
    766  1.174        ad 	 * XXXSMP If we are using h/w performance counters, restore context.
    767  1.231        ad 	 * XXXSMP preemption problem.
    768   1.26       cgd 	 */
    769  1.114  gmcgarry #if PERFCTRS
    770  1.175  christos 	if (PMC_ENABLED(l->l_proc)) {
    771  1.175  christos 		pmc_restore_context(l->l_proc);
    772  1.166  christos 	}
    773  1.114  gmcgarry #endif
    774  1.180       dsl 	SYSCALL_TIME_WAKEUP(l);
    775  1.188      yamt 	LOCKDEBUG_BARRIER(NULL, 1);
    776  1.169      yamt 
    777  1.122   thorpej 	return retval;
    778   1.26       cgd }
    779   1.26       cgd 
    780   1.26       cgd /*
    781  1.245        ad  * The machine independent parts of context switch to oblivion.
    782  1.245        ad  * Does not return.  Call with the LWP unlocked.
    783  1.245        ad  */
    784  1.245        ad void
    785  1.245        ad lwp_exit_switchaway(lwp_t *l)
    786  1.245        ad {
    787  1.245        ad 	struct cpu_info *ci;
    788  1.245        ad 	struct lwp *newl;
    789  1.245        ad 	struct bintime bt;
    790  1.245        ad 
    791  1.245        ad 	ci = l->l_cpu;
    792  1.245        ad 
    793  1.245        ad 	KASSERT(kpreempt_disabled());
    794  1.245        ad 	KASSERT(l->l_stat == LSZOMB || l->l_stat == LSIDL);
    795  1.245        ad 	KASSERT(ci == curcpu());
    796  1.245        ad 	LOCKDEBUG_BARRIER(NULL, 0);
    797  1.245        ad 
    798  1.245        ad 	kstack_check_magic(l);
    799  1.245        ad 
    800  1.245        ad 	/* Count time spent in current system call */
    801  1.245        ad 	SYSCALL_TIME_SLEEP(l);
    802  1.245        ad 	binuptime(&bt);
    803  1.245        ad 	updatertime(l, &bt);
    804  1.245        ad 
    805  1.245        ad 	/* Must stay at IPL_SCHED even after releasing run queue lock. */
    806  1.245        ad 	(void)splsched();
    807  1.245        ad 
    808  1.245        ad 	/*
    809  1.245        ad 	 * Let sched_nextlwp() select the LWP to run the CPU next.
    810  1.245        ad 	 * If no LWP is runnable, select the idle LWP.
    811  1.245        ad 	 *
    812  1.245        ad 	 * Note that spc_lwplock might not necessary be held, and
    813  1.245        ad 	 * new thread would be unlocked after setting the LWP-lock.
    814  1.245        ad 	 */
    815  1.245        ad 	spc_lock(ci);
    816  1.245        ad #ifndef __HAVE_FAST_SOFTINTS
    817  1.245        ad 	if (ci->ci_data.cpu_softints != 0) {
    818  1.245        ad 		/* There are pending soft interrupts, so pick one. */
    819  1.245        ad 		newl = softint_picklwp();
    820  1.245        ad 		newl->l_stat = LSONPROC;
    821  1.248        ad 		newl->l_pflag |= LP_RUNNING;
    822  1.245        ad 	} else
    823  1.245        ad #endif	/* !__HAVE_FAST_SOFTINTS */
    824  1.245        ad 	{
    825  1.245        ad 		newl = nextlwp(ci, &ci->ci_schedstate);
    826  1.245        ad 	}
    827  1.245        ad 
    828  1.245        ad 	/* Update the new LWP's start time. */
    829  1.245        ad 	newl->l_stime = bt;
    830  1.248        ad 	l->l_pflag &= ~LP_RUNNING;
    831  1.245        ad 
    832  1.245        ad 	/*
    833  1.245        ad 	 * ci_curlwp changes when a fast soft interrupt occurs.
    834  1.245        ad 	 * We use cpu_onproc to keep track of which kernel or
    835  1.245        ad 	 * user thread is running 'underneath' the software
    836  1.245        ad 	 * interrupt.  This is important for time accounting,
    837  1.245        ad 	 * itimers and forcing user threads to preempt (aston).
    838  1.245        ad 	 */
    839  1.245        ad 	ci->ci_data.cpu_onproc = newl;
    840  1.245        ad 
    841  1.245        ad 	/*
    842  1.245        ad 	 * Preemption related tasks.  Must be done with the current
    843  1.245        ad 	 * CPU locked.
    844  1.245        ad 	 */
    845  1.245        ad 	cpu_did_resched(l);
    846  1.245        ad 
    847  1.245        ad 	/* Unlock the run queue. */
    848  1.245        ad 	spc_unlock(ci);
    849  1.245        ad 
    850  1.245        ad 	/* Count the context switch on this CPU. */
    851  1.245        ad 	ci->ci_data.cpu_nswtch++;
    852  1.245        ad 
    853  1.245        ad 	/* Update status for lwpctl, if present. */
    854  1.245        ad 	if (l->l_lwpctl != NULL)
    855  1.247        ad 		l->l_lwpctl->lc_curcpu = LWPCTL_CPU_EXITED;
    856  1.245        ad 
    857  1.245        ad 	/*
    858  1.275     skrll 	 * We may need to spin-wait if 'newl' is still
    859  1.245        ad 	 * context switching on another CPU.
    860  1.245        ad 	 */
    861  1.261     rmind 	if (__predict_false(newl->l_ctxswtch != 0)) {
    862  1.245        ad 		u_int count;
    863  1.245        ad 		count = SPINLOCK_BACKOFF_MIN;
    864  1.245        ad 		while (newl->l_ctxswtch)
    865  1.245        ad 			SPINLOCK_BACKOFF(count);
    866  1.245        ad 	}
    867  1.245        ad 
    868  1.279    darran 	/*
    869  1.279    darran 	 * If DTrace has set the active vtime enum to anything
    870  1.279    darran 	 * other than INACTIVE (0), then it should have set the
    871  1.279    darran 	 * function to call.
    872  1.279    darran 	 */
    873  1.279    darran 	if (__predict_false(dtrace_vtime_active)) {
    874  1.279    darran 		(*dtrace_vtime_switch_func)(newl);
    875  1.279    darran 	}
    876  1.276    darran 
    877  1.245        ad 	/* Switch to the new LWP.. */
    878  1.245        ad 	(void)cpu_switchto(NULL, newl, false);
    879  1.245        ad 
    880  1.251       uwe 	for (;;) continue;	/* XXX: convince gcc about "noreturn" */
    881  1.245        ad 	/* NOTREACHED */
    882  1.245        ad }
    883  1.245        ad 
    884  1.245        ad /*
    885  1.271     rmind  * setrunnable: change LWP state to be runnable, placing it on the run queue.
    886  1.174        ad  *
    887  1.174        ad  * Call with the process and LWP locked.  Will return with the LWP unlocked.
    888   1.26       cgd  */
    889   1.26       cgd void
    890  1.122   thorpej setrunnable(struct lwp *l)
    891   1.26       cgd {
    892  1.122   thorpej 	struct proc *p = l->l_proc;
    893  1.205        ad 	struct cpu_info *ci;
    894   1.26       cgd 
    895  1.188      yamt 	KASSERT((l->l_flag & LW_IDLE) == 0);
    896  1.229        ad 	KASSERT(mutex_owned(p->p_lock));
    897  1.183        ad 	KASSERT(lwp_locked(l, NULL));
    898  1.205        ad 	KASSERT(l->l_mutex != l->l_cpu->ci_schedstate.spc_mutex);
    899   1.83   thorpej 
    900  1.122   thorpej 	switch (l->l_stat) {
    901  1.122   thorpej 	case LSSTOP:
    902   1.33   mycroft 		/*
    903   1.33   mycroft 		 * If we're being traced (possibly because someone attached us
    904   1.33   mycroft 		 * while we were stopped), check for a signal from the debugger.
    905   1.33   mycroft 		 */
    906  1.256        ad 		if ((p->p_slflag & PSL_TRACED) != 0 && p->p_xstat != 0)
    907  1.174        ad 			signotify(l);
    908  1.174        ad 		p->p_nrlwps++;
    909   1.26       cgd 		break;
    910  1.174        ad 	case LSSUSPENDED:
    911  1.178     pavel 		l->l_flag &= ~LW_WSUSPEND;
    912  1.174        ad 		p->p_nrlwps++;
    913  1.192     rmind 		cv_broadcast(&p->p_lwpcv);
    914  1.122   thorpej 		break;
    915  1.174        ad 	case LSSLEEP:
    916  1.174        ad 		KASSERT(l->l_wchan != NULL);
    917   1.26       cgd 		break;
    918  1.174        ad 	default:
    919  1.174        ad 		panic("setrunnable: lwp %p state was %d", l, l->l_stat);
    920   1.26       cgd 	}
    921  1.139        cl 
    922  1.174        ad 	/*
    923  1.286     pooka 	 * If the LWP was sleeping, start it again.
    924  1.174        ad 	 */
    925  1.174        ad 	if (l->l_wchan != NULL) {
    926  1.174        ad 		l->l_stat = LSSLEEP;
    927  1.183        ad 		/* lwp_unsleep() will release the lock. */
    928  1.221        ad 		lwp_unsleep(l, true);
    929  1.174        ad 		return;
    930  1.174        ad 	}
    931  1.139        cl 
    932  1.174        ad 	/*
    933  1.174        ad 	 * If the LWP is still on the CPU, mark it as LSONPROC.  It may be
    934  1.174        ad 	 * about to call mi_switch(), in which case it will yield.
    935  1.174        ad 	 */
    936  1.248        ad 	if ((l->l_pflag & LP_RUNNING) != 0) {
    937  1.174        ad 		l->l_stat = LSONPROC;
    938  1.174        ad 		l->l_slptime = 0;
    939  1.174        ad 		lwp_unlock(l);
    940  1.174        ad 		return;
    941  1.174        ad 	}
    942  1.122   thorpej 
    943  1.174        ad 	/*
    944  1.205        ad 	 * Look for a CPU to run.
    945  1.205        ad 	 * Set the LWP runnable.
    946  1.174        ad 	 */
    947  1.205        ad 	ci = sched_takecpu(l);
    948  1.205        ad 	l->l_cpu = ci;
    949  1.236        ad 	spc_lock(ci);
    950  1.236        ad 	lwp_unlock_to(l, ci->ci_schedstate.spc_mutex);
    951  1.188      yamt 	sched_setrunnable(l);
    952  1.174        ad 	l->l_stat = LSRUN;
    953  1.122   thorpej 	l->l_slptime = 0;
    954  1.174        ad 
    955  1.271     rmind 	sched_enqueue(l, false);
    956  1.271     rmind 	resched_cpu(l);
    957  1.271     rmind 	lwp_unlock(l);
    958   1.26       cgd }
    959   1.26       cgd 
    960   1.26       cgd /*
    961  1.174        ad  * suspendsched:
    962  1.174        ad  *
    963  1.266      yamt  *	Convert all non-LW_SYSTEM LSSLEEP or LSRUN LWPs to LSSUSPENDED.
    964  1.174        ad  */
    965   1.94    bouyer void
    966  1.174        ad suspendsched(void)
    967   1.94    bouyer {
    968  1.174        ad 	CPU_INFO_ITERATOR cii;
    969  1.174        ad 	struct cpu_info *ci;
    970  1.122   thorpej 	struct lwp *l;
    971  1.174        ad 	struct proc *p;
    972   1.94    bouyer 
    973   1.94    bouyer 	/*
    974  1.174        ad 	 * We do this by process in order not to violate the locking rules.
    975   1.94    bouyer 	 */
    976  1.228        ad 	mutex_enter(proc_lock);
    977  1.174        ad 	PROCLIST_FOREACH(p, &allproc) {
    978  1.229        ad 		mutex_enter(p->p_lock);
    979  1.178     pavel 		if ((p->p_flag & PK_SYSTEM) != 0) {
    980  1.229        ad 			mutex_exit(p->p_lock);
    981   1.94    bouyer 			continue;
    982  1.174        ad 		}
    983  1.174        ad 
    984  1.174        ad 		p->p_stat = SSTOP;
    985  1.174        ad 
    986  1.174        ad 		LIST_FOREACH(l, &p->p_lwps, l_sibling) {
    987  1.174        ad 			if (l == curlwp)
    988  1.174        ad 				continue;
    989  1.174        ad 
    990  1.174        ad 			lwp_lock(l);
    991  1.122   thorpej 
    992   1.97     enami 			/*
    993  1.174        ad 			 * Set L_WREBOOT so that the LWP will suspend itself
    994  1.174        ad 			 * when it tries to return to user mode.  We want to
    995  1.174        ad 			 * try and get to get as many LWPs as possible to
    996  1.174        ad 			 * the user / kernel boundary, so that they will
    997  1.174        ad 			 * release any locks that they hold.
    998   1.97     enami 			 */
    999  1.178     pavel 			l->l_flag |= (LW_WREBOOT | LW_WSUSPEND);
   1000  1.174        ad 
   1001  1.174        ad 			if (l->l_stat == LSSLEEP &&
   1002  1.178     pavel 			    (l->l_flag & LW_SINTR) != 0) {
   1003  1.174        ad 				/* setrunnable() will release the lock. */
   1004  1.174        ad 				setrunnable(l);
   1005  1.174        ad 				continue;
   1006  1.174        ad 			}
   1007  1.174        ad 
   1008  1.174        ad 			lwp_unlock(l);
   1009   1.94    bouyer 		}
   1010  1.174        ad 
   1011  1.229        ad 		mutex_exit(p->p_lock);
   1012   1.94    bouyer 	}
   1013  1.228        ad 	mutex_exit(proc_lock);
   1014  1.174        ad 
   1015  1.174        ad 	/*
   1016  1.174        ad 	 * Kick all CPUs to make them preempt any LWPs running in user mode.
   1017  1.174        ad 	 * They'll trap into the kernel and suspend themselves in userret().
   1018  1.174        ad 	 */
   1019  1.204        ad 	for (CPU_INFO_FOREACH(cii, ci)) {
   1020  1.204        ad 		spc_lock(ci);
   1021  1.204        ad 		cpu_need_resched(ci, RESCHED_IMMED);
   1022  1.204        ad 		spc_unlock(ci);
   1023  1.204        ad 	}
   1024  1.174        ad }
   1025  1.174        ad 
   1026  1.174        ad /*
   1027  1.174        ad  * sched_unsleep:
   1028  1.174        ad  *
   1029  1.174        ad  *	The is called when the LWP has not been awoken normally but instead
   1030  1.174        ad  *	interrupted: for example, if the sleep timed out.  Because of this,
   1031  1.174        ad  *	it's not a valid action for running or idle LWPs.
   1032  1.174        ad  */
   1033  1.271     rmind static void
   1034  1.221        ad sched_unsleep(struct lwp *l, bool cleanup)
   1035  1.174        ad {
   1036  1.174        ad 
   1037  1.174        ad 	lwp_unlock(l);
   1038  1.174        ad 	panic("sched_unsleep");
   1039  1.174        ad }
   1040  1.174        ad 
   1041  1.250     rmind static void
   1042  1.188      yamt resched_cpu(struct lwp *l)
   1043  1.188      yamt {
   1044  1.274     rmind 	struct cpu_info *ci = l->l_cpu;
   1045  1.188      yamt 
   1046  1.250     rmind 	KASSERT(lwp_locked(l, NULL));
   1047  1.204        ad 	if (lwp_eprio(l) > ci->ci_schedstate.spc_curpriority)
   1048  1.188      yamt 		cpu_need_resched(ci, 0);
   1049  1.188      yamt }
   1050  1.188      yamt 
   1051  1.188      yamt static void
   1052  1.185      yamt sched_changepri(struct lwp *l, pri_t pri)
   1053  1.174        ad {
   1054  1.174        ad 
   1055  1.188      yamt 	KASSERT(lwp_locked(l, NULL));
   1056  1.174        ad 
   1057  1.271     rmind 	if (l->l_stat == LSRUN) {
   1058  1.204        ad 		KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_mutex));
   1059  1.204        ad 		sched_dequeue(l);
   1060  1.204        ad 		l->l_priority = pri;
   1061  1.204        ad 		sched_enqueue(l, false);
   1062  1.204        ad 	} else {
   1063  1.174        ad 		l->l_priority = pri;
   1064  1.157      yamt 	}
   1065  1.188      yamt 	resched_cpu(l);
   1066  1.184      yamt }
   1067  1.184      yamt 
   1068  1.188      yamt static void
   1069  1.185      yamt sched_lendpri(struct lwp *l, pri_t pri)
   1070  1.184      yamt {
   1071  1.184      yamt 
   1072  1.188      yamt 	KASSERT(lwp_locked(l, NULL));
   1073  1.184      yamt 
   1074  1.271     rmind 	if (l->l_stat == LSRUN) {
   1075  1.204        ad 		KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_mutex));
   1076  1.204        ad 		sched_dequeue(l);
   1077  1.204        ad 		l->l_inheritedprio = pri;
   1078  1.204        ad 		sched_enqueue(l, false);
   1079  1.204        ad 	} else {
   1080  1.184      yamt 		l->l_inheritedprio = pri;
   1081  1.184      yamt 	}
   1082  1.188      yamt 	resched_cpu(l);
   1083  1.184      yamt }
   1084  1.184      yamt 
   1085  1.184      yamt struct lwp *
   1086  1.184      yamt syncobj_noowner(wchan_t wchan)
   1087  1.184      yamt {
   1088  1.184      yamt 
   1089  1.184      yamt 	return NULL;
   1090  1.151      yamt }
   1091  1.151      yamt 
   1092  1.250     rmind /* Decay 95% of proc::p_pctcpu in 60 seconds, ccpu = exp(-1/20) */
   1093  1.281     rmind const fixpt_t ccpu = 0.95122942450071400909 * FSCALE;
   1094  1.281     rmind 
   1095  1.281     rmind /*
   1096  1.281     rmind  * Constants for averages over 1, 5 and 15 minutes when sampling at
   1097  1.281     rmind  * 5 second intervals.
   1098  1.281     rmind  */
   1099  1.281     rmind static const fixpt_t cexp[ ] = {
   1100  1.281     rmind 	0.9200444146293232 * FSCALE,	/* exp(-1/12) */
   1101  1.281     rmind 	0.9834714538216174 * FSCALE,	/* exp(-1/60) */
   1102  1.281     rmind 	0.9944598480048967 * FSCALE,	/* exp(-1/180) */
   1103  1.281     rmind };
   1104  1.134      matt 
   1105  1.134      matt /*
   1106  1.188      yamt  * sched_pstats:
   1107  1.188      yamt  *
   1108  1.281     rmind  * => Update process statistics and check CPU resource allocation.
   1109  1.281     rmind  * => Call scheduler-specific hook to eventually adjust LWP priorities.
   1110  1.281     rmind  * => Compute load average of a quantity on 1, 5 and 15 minute intervals.
   1111  1.130   nathanw  */
   1112  1.113  gmcgarry void
   1113  1.281     rmind sched_pstats(void)
   1114  1.113  gmcgarry {
   1115  1.281     rmind 	extern struct loadavg averunnable;
   1116  1.281     rmind 	struct loadavg *avg = &averunnable;
   1117  1.249     rmind 	const int clkhz = (stathz != 0 ? stathz : hz);
   1118  1.281     rmind 	static bool backwards = false;
   1119  1.281     rmind 	static u_int lavg_count = 0;
   1120  1.188      yamt 	struct proc *p;
   1121  1.281     rmind 	int nrun;
   1122  1.113  gmcgarry 
   1123  1.188      yamt 	sched_pstats_ticks++;
   1124  1.281     rmind 	if (++lavg_count >= 5) {
   1125  1.281     rmind 		lavg_count = 0;
   1126  1.281     rmind 		nrun = 0;
   1127  1.281     rmind 	}
   1128  1.228        ad 	mutex_enter(proc_lock);
   1129  1.188      yamt 	PROCLIST_FOREACH(p, &allproc) {
   1130  1.281     rmind 		struct lwp *l;
   1131  1.281     rmind 		struct rlimit *rlim;
   1132  1.296  dholland 		time_t runtm;
   1133  1.281     rmind 		int sig;
   1134  1.281     rmind 
   1135  1.271     rmind 		/* Increment sleep time (if sleeping), ignore overflow. */
   1136  1.229        ad 		mutex_enter(p->p_lock);
   1137  1.212      yamt 		runtm = p->p_rtime.sec;
   1138  1.188      yamt 		LIST_FOREACH(l, &p->p_lwps, l_sibling) {
   1139  1.281     rmind 			fixpt_t lpctcpu;
   1140  1.281     rmind 			u_int lcpticks;
   1141  1.281     rmind 
   1142  1.249     rmind 			if (__predict_false((l->l_flag & LW_IDLE) != 0))
   1143  1.188      yamt 				continue;
   1144  1.188      yamt 			lwp_lock(l);
   1145  1.212      yamt 			runtm += l->l_rtime.sec;
   1146  1.188      yamt 			l->l_swtime++;
   1147  1.242     rmind 			sched_lwp_stats(l);
   1148  1.281     rmind 
   1149  1.281     rmind 			/* For load average calculation. */
   1150  1.282     rmind 			if (__predict_false(lavg_count == 0) &&
   1151  1.282     rmind 			    (l->l_flag & (LW_SINTR | LW_SYSTEM)) == 0) {
   1152  1.281     rmind 				switch (l->l_stat) {
   1153  1.281     rmind 				case LSSLEEP:
   1154  1.281     rmind 					if (l->l_slptime > 1) {
   1155  1.281     rmind 						break;
   1156  1.281     rmind 					}
   1157  1.281     rmind 				case LSRUN:
   1158  1.281     rmind 				case LSONPROC:
   1159  1.281     rmind 				case LSIDL:
   1160  1.281     rmind 					nrun++;
   1161  1.281     rmind 				}
   1162  1.281     rmind 			}
   1163  1.282     rmind 			lwp_unlock(l);
   1164  1.282     rmind 
   1165  1.282     rmind 			l->l_pctcpu = (l->l_pctcpu * ccpu) >> FSHIFT;
   1166  1.282     rmind 			if (l->l_slptime != 0)
   1167  1.282     rmind 				continue;
   1168  1.282     rmind 
   1169  1.282     rmind 			lpctcpu = l->l_pctcpu;
   1170  1.282     rmind 			lcpticks = atomic_swap_uint(&l->l_cpticks, 0);
   1171  1.282     rmind 			lpctcpu += ((FSCALE - ccpu) *
   1172  1.282     rmind 			    (lcpticks * FSCALE / clkhz)) >> FSHIFT;
   1173  1.282     rmind 			l->l_pctcpu = lpctcpu;
   1174  1.188      yamt 		}
   1175  1.249     rmind 		/* Calculating p_pctcpu only for ps(1) */
   1176  1.188      yamt 		p->p_pctcpu = (p->p_pctcpu * ccpu) >> FSHIFT;
   1177  1.174        ad 
   1178  1.188      yamt 		/*
   1179  1.188      yamt 		 * Check if the process exceeds its CPU resource allocation.
   1180  1.293       apb 		 * If over the hard limit, kill it with SIGKILL.
   1181  1.293       apb 		 * If over the soft limit, send SIGXCPU and raise
   1182  1.293       apb 		 * the soft limit a little.
   1183  1.188      yamt 		 */
   1184  1.188      yamt 		rlim = &p->p_rlimit[RLIMIT_CPU];
   1185  1.188      yamt 		sig = 0;
   1186  1.249     rmind 		if (__predict_false(runtm >= rlim->rlim_cur)) {
   1187  1.293       apb 			if (runtm >= rlim->rlim_max) {
   1188  1.188      yamt 				sig = SIGKILL;
   1189  1.293       apb 				log(LOG_NOTICE, "pid %d is killed: %s\n",
   1190  1.293       apb 					p->p_pid, "exceeded RLIMIT_CPU");
   1191  1.293       apb 				uprintf("pid %d, command %s, is killed: %s\n",
   1192  1.293       apb 					p->p_pid, p->p_comm,
   1193  1.293       apb 					"exceeded RLIMIT_CPU");
   1194  1.293       apb 			} else {
   1195  1.188      yamt 				sig = SIGXCPU;
   1196  1.188      yamt 				if (rlim->rlim_cur < rlim->rlim_max)
   1197  1.188      yamt 					rlim->rlim_cur += 5;
   1198  1.188      yamt 			}
   1199  1.188      yamt 		}
   1200  1.229        ad 		mutex_exit(p->p_lock);
   1201  1.259     rmind 		if (__predict_false(runtm < 0)) {
   1202  1.260        ad 			if (!backwards) {
   1203  1.260        ad 				backwards = true;
   1204  1.294       apb 				printf("WARNING: negative runtime; "
   1205  1.260        ad 				    "monotonic clock has gone backwards\n");
   1206  1.260        ad 			}
   1207  1.259     rmind 		} else if (__predict_false(sig)) {
   1208  1.259     rmind 			KASSERT((p->p_flag & PK_SYSTEM) == 0);
   1209  1.188      yamt 			psignal(p, sig);
   1210  1.259     rmind 		}
   1211  1.174        ad 	}
   1212  1.228        ad 	mutex_exit(proc_lock);
   1213  1.281     rmind 
   1214  1.281     rmind 	/* Load average calculation. */
   1215  1.281     rmind 	if (__predict_false(lavg_count == 0)) {
   1216  1.281     rmind 		int i;
   1217  1.283    martin 		CTASSERT(__arraycount(cexp) == __arraycount(avg->ldavg));
   1218  1.281     rmind 		for (i = 0; i < __arraycount(cexp); i++) {
   1219  1.281     rmind 			avg->ldavg[i] = (cexp[i] * avg->ldavg[i] +
   1220  1.281     rmind 			    nrun * FSCALE * (FSCALE - cexp[i])) >> FSHIFT;
   1221  1.281     rmind 		}
   1222  1.281     rmind 	}
   1223  1.281     rmind 
   1224  1.281     rmind 	/* Lightning bolt. */
   1225  1.273     pooka 	cv_broadcast(&lbolt);
   1226  1.113  gmcgarry }
   1227