Home | History | Annotate | Line # | Download | only in kern
kern_synch.c revision 1.297.2.2
      1  1.297.2.2    bouyer /*	$NetBSD: kern_synch.c,v 1.297.2.2 2015/11/15 20:37:04 bouyer Exp $	*/
      2       1.63   thorpej 
      3       1.63   thorpej /*-
      4      1.260        ad  * Copyright (c) 1999, 2000, 2004, 2006, 2007, 2008, 2009
      5      1.260        ad  *    The NetBSD Foundation, Inc.
      6       1.63   thorpej  * All rights reserved.
      7       1.63   thorpej  *
      8       1.63   thorpej  * This code is derived from software contributed to The NetBSD Foundation
      9       1.63   thorpej  * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
     10      1.188      yamt  * NASA Ames Research Center, by Charles M. Hannum, Andrew Doran and
     11      1.188      yamt  * Daniel Sieger.
     12       1.63   thorpej  *
     13       1.63   thorpej  * Redistribution and use in source and binary forms, with or without
     14       1.63   thorpej  * modification, are permitted provided that the following conditions
     15       1.63   thorpej  * are met:
     16       1.63   thorpej  * 1. Redistributions of source code must retain the above copyright
     17       1.63   thorpej  *    notice, this list of conditions and the following disclaimer.
     18       1.63   thorpej  * 2. Redistributions in binary form must reproduce the above copyright
     19       1.63   thorpej  *    notice, this list of conditions and the following disclaimer in the
     20       1.63   thorpej  *    documentation and/or other materials provided with the distribution.
     21       1.63   thorpej  *
     22       1.63   thorpej  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     23       1.63   thorpej  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     24       1.63   thorpej  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     25       1.63   thorpej  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     26       1.63   thorpej  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     27       1.63   thorpej  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     28       1.63   thorpej  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     29       1.63   thorpej  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     30       1.63   thorpej  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     31       1.63   thorpej  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     32       1.63   thorpej  * POSSIBILITY OF SUCH DAMAGE.
     33       1.63   thorpej  */
     34       1.26       cgd 
     35       1.26       cgd /*-
     36       1.26       cgd  * Copyright (c) 1982, 1986, 1990, 1991, 1993
     37       1.26       cgd  *	The Regents of the University of California.  All rights reserved.
     38       1.26       cgd  * (c) UNIX System Laboratories, Inc.
     39       1.26       cgd  * All or some portions of this file are derived from material licensed
     40       1.26       cgd  * to the University of California by American Telephone and Telegraph
     41       1.26       cgd  * Co. or Unix System Laboratories, Inc. and are reproduced herein with
     42       1.26       cgd  * the permission of UNIX System Laboratories, Inc.
     43       1.26       cgd  *
     44       1.26       cgd  * Redistribution and use in source and binary forms, with or without
     45       1.26       cgd  * modification, are permitted provided that the following conditions
     46       1.26       cgd  * are met:
     47       1.26       cgd  * 1. Redistributions of source code must retain the above copyright
     48       1.26       cgd  *    notice, this list of conditions and the following disclaimer.
     49       1.26       cgd  * 2. Redistributions in binary form must reproduce the above copyright
     50       1.26       cgd  *    notice, this list of conditions and the following disclaimer in the
     51       1.26       cgd  *    documentation and/or other materials provided with the distribution.
     52      1.136       agc  * 3. Neither the name of the University nor the names of its contributors
     53       1.26       cgd  *    may be used to endorse or promote products derived from this software
     54       1.26       cgd  *    without specific prior written permission.
     55       1.26       cgd  *
     56       1.26       cgd  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
     57       1.26       cgd  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     58       1.26       cgd  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     59       1.26       cgd  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
     60       1.26       cgd  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     61       1.26       cgd  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
     62       1.26       cgd  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     63       1.26       cgd  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     64       1.26       cgd  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     65       1.26       cgd  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     66       1.26       cgd  * SUCH DAMAGE.
     67       1.26       cgd  *
     68       1.50      fvdl  *	@(#)kern_synch.c	8.9 (Berkeley) 5/19/95
     69       1.26       cgd  */
     70      1.106     lukem 
     71      1.106     lukem #include <sys/cdefs.h>
     72  1.297.2.2    bouyer __KERNEL_RCSID(0, "$NetBSD: kern_synch.c,v 1.297.2.2 2015/11/15 20:37:04 bouyer Exp $");
     73       1.48       mrg 
     74      1.109      yamt #include "opt_kstack.h"
     75      1.110    briggs #include "opt_perfctrs.h"
     76      1.252  wrstuden #include "opt_sa.h"
     77      1.277    darran #include "opt_dtrace.h"
     78       1.26       cgd 
     79      1.174        ad #define	__MUTEX_PRIVATE
     80      1.174        ad 
     81       1.26       cgd #include <sys/param.h>
     82       1.26       cgd #include <sys/systm.h>
     83       1.26       cgd #include <sys/proc.h>
     84       1.26       cgd #include <sys/kernel.h>
     85      1.111    briggs #if defined(PERFCTRS)
     86      1.110    briggs #include <sys/pmc.h>
     87      1.111    briggs #endif
     88      1.188      yamt #include <sys/cpu.h>
     89      1.290  christos #include <sys/pserialize.h>
     90       1.26       cgd #include <sys/resourcevar.h>
     91       1.55      ross #include <sys/sched.h>
     92      1.252  wrstuden #include <sys/sa.h>
     93      1.252  wrstuden #include <sys/savar.h>
     94      1.179       dsl #include <sys/syscall_stats.h>
     95      1.174        ad #include <sys/sleepq.h>
     96      1.174        ad #include <sys/lockdebug.h>
     97      1.190        ad #include <sys/evcnt.h>
     98      1.199        ad #include <sys/intr.h>
     99      1.207        ad #include <sys/lwpctl.h>
    100      1.209        ad #include <sys/atomic.h>
    101      1.215        ad #include <sys/simplelock.h>
    102      1.295     njoly #include <sys/syslog.h>
    103       1.47       mrg 
    104       1.47       mrg #include <uvm/uvm_extern.h>
    105       1.47       mrg 
    106      1.231        ad #include <dev/lockstat.h>
    107      1.231        ad 
    108      1.276    darran #include <sys/dtrace_bsd.h>
    109      1.279    darran int                             dtrace_vtime_active=0;
    110      1.276    darran dtrace_vtime_switch_func_t      dtrace_vtime_switch_func;
    111      1.276    darran 
    112      1.271     rmind static void	sched_unsleep(struct lwp *, bool);
    113      1.188      yamt static void	sched_changepri(struct lwp *, pri_t);
    114      1.188      yamt static void	sched_lendpri(struct lwp *, pri_t);
    115      1.250     rmind static void	resched_cpu(struct lwp *);
    116      1.122   thorpej 
    117      1.174        ad syncobj_t sleep_syncobj = {
    118      1.174        ad 	SOBJ_SLEEPQ_SORTED,
    119      1.174        ad 	sleepq_unsleep,
    120      1.184      yamt 	sleepq_changepri,
    121      1.184      yamt 	sleepq_lendpri,
    122      1.184      yamt 	syncobj_noowner,
    123      1.174        ad };
    124      1.174        ad 
    125      1.174        ad syncobj_t sched_syncobj = {
    126      1.174        ad 	SOBJ_SLEEPQ_SORTED,
    127      1.174        ad 	sched_unsleep,
    128      1.184      yamt 	sched_changepri,
    129      1.184      yamt 	sched_lendpri,
    130      1.184      yamt 	syncobj_noowner,
    131      1.174        ad };
    132      1.122   thorpej 
    133      1.289     rmind /* "Lightning bolt": once a second sleep address. */
    134      1.289     rmind kcondvar_t		lbolt			__cacheline_aligned;
    135      1.223        ad 
    136      1.289     rmind u_int			sched_pstats_ticks	__cacheline_aligned;
    137      1.289     rmind 
    138      1.289     rmind /* Preemption event counters. */
    139      1.289     rmind static struct evcnt	kpreempt_ev_crit	__cacheline_aligned;
    140      1.289     rmind static struct evcnt	kpreempt_ev_klock	__cacheline_aligned;
    141      1.289     rmind static struct evcnt	kpreempt_ev_immed	__cacheline_aligned;
    142      1.231        ad 
    143      1.231        ad /*
    144      1.174        ad  * During autoconfiguration or after a panic, a sleep will simply lower the
    145      1.174        ad  * priority briefly to allow interrupts, then return.  The priority to be
    146      1.174        ad  * used (safepri) is machine-dependent, thus this value is initialized and
    147      1.174        ad  * maintained in the machine-dependent layers.  This priority will typically
    148      1.174        ad  * be 0, or the lowest priority that is safe for use on the interrupt stack;
    149      1.174        ad  * it can be made higher to block network software interrupts after panics.
    150       1.26       cgd  */
    151      1.174        ad int	safepri;
    152       1.26       cgd 
    153      1.237     rmind void
    154      1.270      elad synch_init(void)
    155      1.237     rmind {
    156      1.237     rmind 
    157      1.237     rmind 	cv_init(&lbolt, "lbolt");
    158      1.237     rmind 
    159      1.239        ad 	evcnt_attach_dynamic(&kpreempt_ev_crit, EVCNT_TYPE_MISC, NULL,
    160      1.237     rmind 	   "kpreempt", "defer: critical section");
    161      1.239        ad 	evcnt_attach_dynamic(&kpreempt_ev_klock, EVCNT_TYPE_MISC, NULL,
    162      1.237     rmind 	   "kpreempt", "defer: kernel_lock");
    163      1.239        ad 	evcnt_attach_dynamic(&kpreempt_ev_immed, EVCNT_TYPE_MISC, NULL,
    164      1.237     rmind 	   "kpreempt", "immediate");
    165      1.237     rmind }
    166      1.237     rmind 
    167       1.26       cgd /*
    168      1.174        ad  * OBSOLETE INTERFACE
    169      1.174        ad  *
    170      1.255     skrll  * General sleep call.  Suspends the current LWP until a wakeup is
    171      1.255     skrll  * performed on the specified identifier.  The LWP will then be made
    172      1.174        ad  * runnable with the specified priority.  Sleeps at most timo/hz seconds (0
    173      1.174        ad  * means no timeout).  If pri includes PCATCH flag, signals are checked
    174       1.26       cgd  * before and after sleeping, else signals are not checked.  Returns 0 if
    175       1.26       cgd  * awakened, EWOULDBLOCK if the timeout expires.  If PCATCH is set and a
    176       1.26       cgd  * signal needs to be delivered, ERESTART is returned if the current system
    177       1.26       cgd  * call should be restarted if possible, and EINTR is returned if the system
    178       1.26       cgd  * call should be interrupted by the signal (return EINTR).
    179       1.26       cgd  */
    180       1.26       cgd int
    181      1.297     rmind tsleep(wchan_t ident, pri_t priority, const char *wmesg, int timo)
    182       1.26       cgd {
    183      1.122   thorpej 	struct lwp *l = curlwp;
    184      1.174        ad 	sleepq_t *sq;
    185      1.244        ad 	kmutex_t *mp;
    186       1.26       cgd 
    187      1.204        ad 	KASSERT((l->l_pflag & LP_INTR) == 0);
    188      1.272     pooka 	KASSERT(ident != &lbolt);
    189      1.204        ad 
    190      1.174        ad 	if (sleepq_dontsleep(l)) {
    191      1.174        ad 		(void)sleepq_abort(NULL, 0);
    192      1.174        ad 		return 0;
    193       1.26       cgd 	}
    194       1.78  sommerfe 
    195      1.204        ad 	l->l_kpriority = true;
    196      1.244        ad 	sq = sleeptab_lookup(&sleeptab, ident, &mp);
    197      1.244        ad 	sleepq_enter(sq, l, mp);
    198      1.204        ad 	sleepq_enqueue(sq, ident, wmesg, &sleep_syncobj);
    199      1.297     rmind 	return sleepq_block(timo, priority & PCATCH);
    200       1.26       cgd }
    201       1.26       cgd 
    202      1.187        ad int
    203      1.187        ad mtsleep(wchan_t ident, pri_t priority, const char *wmesg, int timo,
    204      1.187        ad 	kmutex_t *mtx)
    205      1.187        ad {
    206      1.187        ad 	struct lwp *l = curlwp;
    207      1.187        ad 	sleepq_t *sq;
    208      1.244        ad 	kmutex_t *mp;
    209      1.188      yamt 	int error;
    210      1.187        ad 
    211      1.204        ad 	KASSERT((l->l_pflag & LP_INTR) == 0);
    212      1.272     pooka 	KASSERT(ident != &lbolt);
    213      1.204        ad 
    214      1.187        ad 	if (sleepq_dontsleep(l)) {
    215      1.187        ad 		(void)sleepq_abort(mtx, (priority & PNORELOCK) != 0);
    216      1.187        ad 		return 0;
    217      1.187        ad 	}
    218      1.187        ad 
    219      1.204        ad 	l->l_kpriority = true;
    220      1.244        ad 	sq = sleeptab_lookup(&sleeptab, ident, &mp);
    221      1.244        ad 	sleepq_enter(sq, l, mp);
    222      1.204        ad 	sleepq_enqueue(sq, ident, wmesg, &sleep_syncobj);
    223      1.187        ad 	mutex_exit(mtx);
    224      1.188      yamt 	error = sleepq_block(timo, priority & PCATCH);
    225      1.187        ad 
    226      1.187        ad 	if ((priority & PNORELOCK) == 0)
    227      1.187        ad 		mutex_enter(mtx);
    228      1.297     rmind 
    229      1.187        ad 	return error;
    230      1.187        ad }
    231      1.187        ad 
    232       1.26       cgd /*
    233      1.174        ad  * General sleep call for situations where a wake-up is not expected.
    234       1.26       cgd  */
    235      1.174        ad int
    236      1.182   thorpej kpause(const char *wmesg, bool intr, int timo, kmutex_t *mtx)
    237       1.26       cgd {
    238      1.174        ad 	struct lwp *l = curlwp;
    239      1.244        ad 	kmutex_t *mp;
    240      1.174        ad 	sleepq_t *sq;
    241      1.174        ad 	int error;
    242       1.26       cgd 
    243      1.284     pooka 	KASSERT(!(timo == 0 && intr == false));
    244      1.284     pooka 
    245      1.174        ad 	if (sleepq_dontsleep(l))
    246      1.174        ad 		return sleepq_abort(NULL, 0);
    247       1.26       cgd 
    248      1.174        ad 	if (mtx != NULL)
    249      1.174        ad 		mutex_exit(mtx);
    250      1.204        ad 	l->l_kpriority = true;
    251      1.244        ad 	sq = sleeptab_lookup(&sleeptab, l, &mp);
    252      1.244        ad 	sleepq_enter(sq, l, mp);
    253      1.204        ad 	sleepq_enqueue(sq, l, wmesg, &sleep_syncobj);
    254      1.188      yamt 	error = sleepq_block(timo, intr);
    255      1.174        ad 	if (mtx != NULL)
    256      1.174        ad 		mutex_enter(mtx);
    257       1.83   thorpej 
    258      1.174        ad 	return error;
    259      1.139        cl }
    260      1.139        cl 
    261      1.252  wrstuden #ifdef KERN_SA
    262      1.252  wrstuden /*
    263      1.252  wrstuden  * sa_awaken:
    264      1.252  wrstuden  *
    265      1.252  wrstuden  *	We believe this lwp is an SA lwp. If it's yielding,
    266      1.252  wrstuden  * let it know it needs to wake up.
    267      1.252  wrstuden  *
    268      1.252  wrstuden  *	We are called and exit with the lwp locked. We are
    269      1.252  wrstuden  * called in the middle of wakeup operations, so we need
    270      1.252  wrstuden  * to not touch the locks at all.
    271      1.252  wrstuden  */
    272      1.252  wrstuden void
    273      1.252  wrstuden sa_awaken(struct lwp *l)
    274      1.252  wrstuden {
    275      1.252  wrstuden 	/* LOCK_ASSERT(lwp_locked(l, NULL)); */
    276      1.252  wrstuden 
    277      1.252  wrstuden 	if (l == l->l_savp->savp_lwp && l->l_flag & LW_SA_YIELD)
    278      1.252  wrstuden 		l->l_flag &= ~LW_SA_IDLE;
    279      1.252  wrstuden }
    280      1.252  wrstuden #endif /* KERN_SA */
    281      1.252  wrstuden 
    282       1.26       cgd /*
    283      1.174        ad  * OBSOLETE INTERFACE
    284      1.174        ad  *
    285      1.255     skrll  * Make all LWPs sleeping on the specified identifier runnable.
    286       1.26       cgd  */
    287       1.26       cgd void
    288      1.174        ad wakeup(wchan_t ident)
    289       1.26       cgd {
    290      1.174        ad 	sleepq_t *sq;
    291      1.244        ad 	kmutex_t *mp;
    292       1.83   thorpej 
    293      1.261     rmind 	if (__predict_false(cold))
    294      1.174        ad 		return;
    295       1.83   thorpej 
    296      1.244        ad 	sq = sleeptab_lookup(&sleeptab, ident, &mp);
    297      1.244        ad 	sleepq_wake(sq, ident, (u_int)-1, mp);
    298       1.63   thorpej }
    299       1.63   thorpej 
    300       1.63   thorpej /*
    301      1.255     skrll  * General yield call.  Puts the current LWP back on its run queue and
    302      1.117  gmcgarry  * performs a voluntary context switch.  Should only be called when the
    303      1.255     skrll  * current LWP explicitly requests it (eg sched_yield(2)).
    304      1.117  gmcgarry  */
    305      1.117  gmcgarry void
    306      1.117  gmcgarry yield(void)
    307      1.117  gmcgarry {
    308      1.122   thorpej 	struct lwp *l = curlwp;
    309      1.117  gmcgarry 
    310      1.174        ad 	KERNEL_UNLOCK_ALL(l, &l->l_biglocks);
    311      1.174        ad 	lwp_lock(l);
    312      1.217        ad 	KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_lwplock));
    313      1.188      yamt 	KASSERT(l->l_stat == LSONPROC);
    314      1.204        ad 	l->l_kpriority = false;
    315      1.188      yamt 	(void)mi_switch(l);
    316      1.174        ad 	KERNEL_LOCK(l->l_biglocks, l);
    317       1.69   thorpej }
    318       1.69   thorpej 
    319       1.69   thorpej /*
    320      1.255     skrll  * General preemption call.  Puts the current LWP back on its run queue
    321      1.156    rpaulo  * and performs an involuntary context switch.
    322       1.69   thorpej  */
    323       1.69   thorpej void
    324      1.174        ad preempt(void)
    325       1.69   thorpej {
    326      1.122   thorpej 	struct lwp *l = curlwp;
    327       1.69   thorpej 
    328      1.174        ad 	KERNEL_UNLOCK_ALL(l, &l->l_biglocks);
    329      1.174        ad 	lwp_lock(l);
    330      1.217        ad 	KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_lwplock));
    331      1.188      yamt 	KASSERT(l->l_stat == LSONPROC);
    332      1.204        ad 	l->l_kpriority = false;
    333      1.174        ad 	l->l_nivcsw++;
    334      1.188      yamt 	(void)mi_switch(l);
    335      1.174        ad 	KERNEL_LOCK(l->l_biglocks, l);
    336       1.69   thorpej }
    337       1.69   thorpej 
    338      1.234        ad /*
    339      1.234        ad  * Handle a request made by another agent to preempt the current LWP
    340      1.234        ad  * in-kernel.  Usually called when l_dopreempt may be non-zero.
    341      1.234        ad  *
    342      1.234        ad  * Character addresses for lockstat only.
    343      1.234        ad  */
    344      1.231        ad static char	in_critical_section;
    345      1.231        ad static char	kernel_lock_held;
    346      1.231        ad static char	is_softint;
    347      1.262      yamt static char	cpu_kpreempt_enter_fail;
    348      1.231        ad 
    349      1.231        ad bool
    350      1.231        ad kpreempt(uintptr_t where)
    351      1.231        ad {
    352      1.231        ad 	uintptr_t failed;
    353      1.231        ad 	lwp_t *l;
    354      1.264        ad 	int s, dop, lsflag;
    355      1.231        ad 
    356      1.231        ad 	l = curlwp;
    357      1.231        ad 	failed = 0;
    358      1.231        ad 	while ((dop = l->l_dopreempt) != 0) {
    359      1.231        ad 		if (l->l_stat != LSONPROC) {
    360      1.231        ad 			/*
    361      1.231        ad 			 * About to block (or die), let it happen.
    362      1.231        ad 			 * Doesn't really count as "preemption has
    363      1.231        ad 			 * been blocked", since we're going to
    364      1.231        ad 			 * context switch.
    365      1.231        ad 			 */
    366      1.231        ad 			l->l_dopreempt = 0;
    367      1.231        ad 			return true;
    368      1.231        ad 		}
    369      1.231        ad 		if (__predict_false((l->l_flag & LW_IDLE) != 0)) {
    370      1.231        ad 			/* Can't preempt idle loop, don't count as failure. */
    371      1.261     rmind 			l->l_dopreempt = 0;
    372      1.261     rmind 			return true;
    373      1.231        ad 		}
    374      1.231        ad 		if (__predict_false(l->l_nopreempt != 0)) {
    375      1.231        ad 			/* LWP holds preemption disabled, explicitly. */
    376      1.231        ad 			if ((dop & DOPREEMPT_COUNTED) == 0) {
    377      1.234        ad 				kpreempt_ev_crit.ev_count++;
    378      1.231        ad 			}
    379      1.231        ad 			failed = (uintptr_t)&in_critical_section;
    380      1.231        ad 			break;
    381      1.231        ad 		}
    382      1.231        ad 		if (__predict_false((l->l_pflag & LP_INTR) != 0)) {
    383      1.261     rmind 			/* Can't preempt soft interrupts yet. */
    384      1.261     rmind 			l->l_dopreempt = 0;
    385      1.261     rmind 			failed = (uintptr_t)&is_softint;
    386      1.261     rmind 			break;
    387      1.231        ad 		}
    388      1.231        ad 		s = splsched();
    389      1.231        ad 		if (__predict_false(l->l_blcnt != 0 ||
    390      1.231        ad 		    curcpu()->ci_biglock_wanted != NULL)) {
    391      1.231        ad 			/* Hold or want kernel_lock, code is not MT safe. */
    392      1.231        ad 			splx(s);
    393      1.231        ad 			if ((dop & DOPREEMPT_COUNTED) == 0) {
    394      1.234        ad 				kpreempt_ev_klock.ev_count++;
    395      1.231        ad 			}
    396      1.231        ad 			failed = (uintptr_t)&kernel_lock_held;
    397      1.231        ad 			break;
    398      1.231        ad 		}
    399      1.231        ad 		if (__predict_false(!cpu_kpreempt_enter(where, s))) {
    400      1.231        ad 			/*
    401      1.231        ad 			 * It may be that the IPL is too high.
    402      1.231        ad 			 * kpreempt_enter() can schedule an
    403      1.231        ad 			 * interrupt to retry later.
    404      1.231        ad 			 */
    405      1.231        ad 			splx(s);
    406      1.262      yamt 			failed = (uintptr_t)&cpu_kpreempt_enter_fail;
    407      1.231        ad 			break;
    408      1.231        ad 		}
    409      1.231        ad 		/* Do it! */
    410      1.231        ad 		if (__predict_true((dop & DOPREEMPT_COUNTED) == 0)) {
    411      1.234        ad 			kpreempt_ev_immed.ev_count++;
    412      1.231        ad 		}
    413      1.231        ad 		lwp_lock(l);
    414      1.231        ad 		mi_switch(l);
    415      1.231        ad 		l->l_nopreempt++;
    416      1.231        ad 		splx(s);
    417      1.231        ad 
    418      1.231        ad 		/* Take care of any MD cleanup. */
    419      1.231        ad 		cpu_kpreempt_exit(where);
    420      1.231        ad 		l->l_nopreempt--;
    421      1.231        ad 	}
    422      1.231        ad 
    423      1.264        ad 	if (__predict_true(!failed)) {
    424      1.264        ad 		return false;
    425      1.264        ad 	}
    426      1.264        ad 
    427      1.231        ad 	/* Record preemption failure for reporting via lockstat. */
    428      1.264        ad 	atomic_or_uint(&l->l_dopreempt, DOPREEMPT_COUNTED);
    429      1.264        ad 	lsflag = 0;
    430      1.264        ad 	LOCKSTAT_ENTER(lsflag);
    431      1.264        ad 	if (__predict_false(lsflag)) {
    432      1.264        ad 		if (where == 0) {
    433      1.264        ad 			where = (uintptr_t)__builtin_return_address(0);
    434      1.264        ad 		}
    435      1.264        ad 		/* Preemption is on, might recurse, so make it atomic. */
    436      1.264        ad 		if (atomic_cas_ptr_ni((void *)&l->l_pfailaddr, NULL,
    437      1.264        ad 		    (void *)where) == NULL) {
    438      1.264        ad 			LOCKSTAT_START_TIMER(lsflag, l->l_pfailtime);
    439      1.264        ad 			l->l_pfaillock = failed;
    440      1.231        ad 		}
    441      1.231        ad 	}
    442      1.264        ad 	LOCKSTAT_EXIT(lsflag);
    443      1.264        ad 	return true;
    444      1.231        ad }
    445      1.231        ad 
    446       1.69   thorpej /*
    447      1.231        ad  * Return true if preemption is explicitly disabled.
    448      1.230        ad  */
    449      1.231        ad bool
    450      1.231        ad kpreempt_disabled(void)
    451      1.231        ad {
    452      1.261     rmind 	const lwp_t *l = curlwp;
    453      1.231        ad 
    454      1.231        ad 	return l->l_nopreempt != 0 || l->l_stat == LSZOMB ||
    455      1.231        ad 	    (l->l_flag & LW_IDLE) != 0 || cpu_kpreempt_disabled();
    456      1.231        ad }
    457      1.230        ad 
    458      1.230        ad /*
    459      1.231        ad  * Disable kernel preemption.
    460      1.230        ad  */
    461      1.230        ad void
    462      1.231        ad kpreempt_disable(void)
    463      1.230        ad {
    464      1.230        ad 
    465      1.231        ad 	KPREEMPT_DISABLE(curlwp);
    466      1.230        ad }
    467      1.230        ad 
    468      1.230        ad /*
    469      1.231        ad  * Reenable kernel preemption.
    470      1.230        ad  */
    471      1.231        ad void
    472      1.231        ad kpreempt_enable(void)
    473      1.230        ad {
    474      1.230        ad 
    475      1.231        ad 	KPREEMPT_ENABLE(curlwp);
    476      1.230        ad }
    477      1.230        ad 
    478      1.230        ad /*
    479      1.188      yamt  * Compute the amount of time during which the current lwp was running.
    480      1.130   nathanw  *
    481      1.188      yamt  * - update l_rtime unless it's an idle lwp.
    482      1.188      yamt  */
    483      1.188      yamt 
    484      1.199        ad void
    485      1.212      yamt updatertime(lwp_t *l, const struct bintime *now)
    486      1.188      yamt {
    487      1.188      yamt 
    488      1.261     rmind 	if (__predict_false(l->l_flag & LW_IDLE))
    489      1.188      yamt 		return;
    490      1.188      yamt 
    491      1.212      yamt 	/* rtime += now - stime */
    492      1.212      yamt 	bintime_add(&l->l_rtime, now);
    493      1.212      yamt 	bintime_sub(&l->l_rtime, &l->l_stime);
    494      1.188      yamt }
    495      1.188      yamt 
    496      1.188      yamt /*
    497      1.245        ad  * Select next LWP from the current CPU to run..
    498      1.245        ad  */
    499      1.245        ad static inline lwp_t *
    500      1.245        ad nextlwp(struct cpu_info *ci, struct schedstate_percpu *spc)
    501      1.245        ad {
    502      1.245        ad 	lwp_t *newl;
    503      1.245        ad 
    504      1.245        ad 	/*
    505      1.245        ad 	 * Let sched_nextlwp() select the LWP to run the CPU next.
    506      1.245        ad 	 * If no LWP is runnable, select the idle LWP.
    507      1.245        ad 	 *
    508      1.245        ad 	 * Note that spc_lwplock might not necessary be held, and
    509      1.245        ad 	 * new thread would be unlocked after setting the LWP-lock.
    510      1.245        ad 	 */
    511      1.245        ad 	newl = sched_nextlwp();
    512      1.245        ad 	if (newl != NULL) {
    513      1.245        ad 		sched_dequeue(newl);
    514      1.245        ad 		KASSERT(lwp_locked(newl, spc->spc_mutex));
    515      1.274     rmind 		KASSERT(newl->l_cpu == ci);
    516      1.245        ad 		newl->l_stat = LSONPROC;
    517      1.248        ad 		newl->l_pflag |= LP_RUNNING;
    518      1.245        ad 		lwp_setlock(newl, spc->spc_lwplock);
    519      1.245        ad 	} else {
    520      1.245        ad 		newl = ci->ci_data.cpu_idlelwp;
    521      1.245        ad 		newl->l_stat = LSONPROC;
    522      1.248        ad 		newl->l_pflag |= LP_RUNNING;
    523      1.245        ad 	}
    524      1.261     rmind 
    525      1.245        ad 	/*
    526      1.245        ad 	 * Only clear want_resched if there are no pending (slow)
    527      1.245        ad 	 * software interrupts.
    528      1.245        ad 	 */
    529      1.245        ad 	ci->ci_want_resched = ci->ci_data.cpu_softints;
    530      1.245        ad 	spc->spc_flags &= ~SPCF_SWITCHCLEAR;
    531      1.245        ad 	spc->spc_curpriority = lwp_eprio(newl);
    532      1.245        ad 
    533      1.245        ad 	return newl;
    534      1.245        ad }
    535      1.245        ad 
    536      1.245        ad /*
    537      1.188      yamt  * The machine independent parts of context switch.
    538      1.188      yamt  *
    539      1.188      yamt  * Returns 1 if another LWP was actually run.
    540       1.26       cgd  */
    541      1.122   thorpej int
    542      1.199        ad mi_switch(lwp_t *l)
    543       1.26       cgd {
    544      1.246     rmind 	struct cpu_info *ci;
    545       1.76   thorpej 	struct schedstate_percpu *spc;
    546      1.188      yamt 	struct lwp *newl;
    547      1.174        ad 	int retval, oldspl;
    548      1.212      yamt 	struct bintime bt;
    549      1.199        ad 	bool returning;
    550       1.26       cgd 
    551      1.188      yamt 	KASSERT(lwp_locked(l, NULL));
    552      1.231        ad 	KASSERT(kpreempt_disabled());
    553      1.188      yamt 	LOCKDEBUG_BARRIER(l->l_mutex, 1);
    554      1.174        ad 
    555      1.174        ad 	kstack_check_magic(l);
    556       1.83   thorpej 
    557      1.212      yamt 	binuptime(&bt);
    558      1.199        ad 
    559      1.267      yamt 	KASSERT((l->l_pflag & LP_RUNNING) != 0);
    560      1.231        ad 	KASSERT(l->l_cpu == curcpu());
    561      1.196        ad 	ci = l->l_cpu;
    562      1.196        ad 	spc = &ci->ci_schedstate;
    563      1.199        ad 	returning = false;
    564      1.190        ad 	newl = NULL;
    565      1.190        ad 
    566      1.199        ad 	/*
    567      1.199        ad 	 * If we have been asked to switch to a specific LWP, then there
    568      1.199        ad 	 * is no need to inspect the run queues.  If a soft interrupt is
    569      1.199        ad 	 * blocking, then return to the interrupted thread without adjusting
    570      1.199        ad 	 * VM context or its start time: neither have been changed in order
    571      1.199        ad 	 * to take the interrupt.
    572      1.199        ad 	 */
    573      1.190        ad 	if (l->l_switchto != NULL) {
    574      1.204        ad 		if ((l->l_pflag & LP_INTR) != 0) {
    575      1.199        ad 			returning = true;
    576      1.199        ad 			softint_block(l);
    577      1.248        ad 			if ((l->l_pflag & LP_TIMEINTR) != 0)
    578      1.212      yamt 				updatertime(l, &bt);
    579      1.199        ad 		}
    580      1.190        ad 		newl = l->l_switchto;
    581      1.190        ad 		l->l_switchto = NULL;
    582      1.190        ad 	}
    583      1.204        ad #ifndef __HAVE_FAST_SOFTINTS
    584      1.204        ad 	else if (ci->ci_data.cpu_softints != 0) {
    585      1.204        ad 		/* There are pending soft interrupts, so pick one. */
    586      1.204        ad 		newl = softint_picklwp();
    587      1.204        ad 		newl->l_stat = LSONPROC;
    588      1.248        ad 		newl->l_pflag |= LP_RUNNING;
    589      1.204        ad 	}
    590      1.204        ad #endif	/* !__HAVE_FAST_SOFTINTS */
    591      1.190        ad 
    592      1.180       dsl 	/* Count time spent in current system call */
    593      1.199        ad 	if (!returning) {
    594      1.199        ad 		SYSCALL_TIME_SLEEP(l);
    595      1.180       dsl 
    596      1.199        ad 		/*
    597      1.199        ad 		 * XXXSMP If we are using h/w performance counters,
    598      1.199        ad 		 * save context.
    599      1.199        ad 		 */
    600      1.174        ad #if PERFCTRS
    601      1.199        ad 		if (PMC_ENABLED(l->l_proc)) {
    602      1.199        ad 			pmc_save_context(l->l_proc);
    603      1.199        ad 		}
    604      1.199        ad #endif
    605      1.212      yamt 		updatertime(l, &bt);
    606      1.174        ad 	}
    607      1.113  gmcgarry 
    608      1.246     rmind 	/* Lock the runqueue */
    609      1.246     rmind 	KASSERT(l->l_stat != LSRUN);
    610      1.246     rmind 	mutex_spin_enter(spc->spc_mutex);
    611      1.246     rmind 
    612      1.113  gmcgarry 	/*
    613      1.174        ad 	 * If on the CPU and we have gotten this far, then we must yield.
    614      1.113  gmcgarry 	 */
    615      1.246     rmind 	if (l->l_stat == LSONPROC && l != newl) {
    616      1.217        ad 		KASSERT(lwp_locked(l, spc->spc_lwplock));
    617      1.188      yamt 		if ((l->l_flag & LW_IDLE) == 0) {
    618      1.188      yamt 			l->l_stat = LSRUN;
    619      1.246     rmind 			lwp_setlock(l, spc->spc_mutex);
    620      1.246     rmind 			sched_enqueue(l, true);
    621      1.285     rmind 			/*
    622      1.285     rmind 			 * Handle migration.  Note that "migrating LWP" may
    623      1.285     rmind 			 * be reset here, if interrupt/preemption happens
    624      1.285     rmind 			 * early in idle LWP.
    625      1.285     rmind 			 */
    626      1.285     rmind 			if (l->l_target_cpu != NULL) {
    627      1.285     rmind 				KASSERT((l->l_pflag & LP_INTR) == 0);
    628      1.246     rmind 				spc->spc_migrating = l;
    629      1.216     rmind 			}
    630      1.246     rmind 		} else
    631      1.188      yamt 			l->l_stat = LSIDL;
    632      1.174        ad 	}
    633      1.174        ad 
    634      1.245        ad 	/* Pick new LWP to run. */
    635      1.190        ad 	if (newl == NULL) {
    636      1.245        ad 		newl = nextlwp(ci, spc);
    637      1.199        ad 	}
    638      1.199        ad 
    639      1.204        ad 	/* Items that must be updated with the CPU locked. */
    640      1.199        ad 	if (!returning) {
    641      1.204        ad 		/* Update the new LWP's start time. */
    642      1.212      yamt 		newl->l_stime = bt;
    643      1.204        ad 
    644      1.199        ad 		/*
    645      1.204        ad 		 * ci_curlwp changes when a fast soft interrupt occurs.
    646      1.204        ad 		 * We use cpu_onproc to keep track of which kernel or
    647      1.204        ad 		 * user thread is running 'underneath' the software
    648      1.204        ad 		 * interrupt.  This is important for time accounting,
    649      1.204        ad 		 * itimers and forcing user threads to preempt (aston).
    650      1.199        ad 		 */
    651      1.204        ad 		ci->ci_data.cpu_onproc = newl;
    652      1.188      yamt 	}
    653      1.188      yamt 
    654      1.241        ad 	/*
    655      1.241        ad 	 * Preemption related tasks.  Must be done with the current
    656      1.241        ad 	 * CPU locked.
    657      1.241        ad 	 */
    658      1.241        ad 	cpu_did_resched(l);
    659      1.231        ad 	l->l_dopreempt = 0;
    660      1.231        ad 	if (__predict_false(l->l_pfailaddr != 0)) {
    661      1.231        ad 		LOCKSTAT_FLAG(lsflag);
    662      1.231        ad 		LOCKSTAT_ENTER(lsflag);
    663      1.231        ad 		LOCKSTAT_STOP_TIMER(lsflag, l->l_pfailtime);
    664      1.231        ad 		LOCKSTAT_EVENT_RA(lsflag, l->l_pfaillock, LB_NOPREEMPT|LB_SPIN,
    665      1.231        ad 		    1, l->l_pfailtime, l->l_pfailaddr);
    666      1.231        ad 		LOCKSTAT_EXIT(lsflag);
    667      1.231        ad 		l->l_pfailtime = 0;
    668      1.231        ad 		l->l_pfaillock = 0;
    669      1.231        ad 		l->l_pfailaddr = 0;
    670      1.231        ad 	}
    671      1.231        ad 
    672      1.188      yamt 	if (l != newl) {
    673      1.188      yamt 		struct lwp *prevlwp;
    674      1.174        ad 
    675      1.209        ad 		/* Release all locks, but leave the current LWP locked */
    676      1.246     rmind 		if (l->l_mutex == spc->spc_mutex) {
    677      1.209        ad 			/*
    678      1.209        ad 			 * Drop spc_lwplock, if the current LWP has been moved
    679      1.209        ad 			 * to the run queue (it is now locked by spc_mutex).
    680      1.209        ad 			 */
    681      1.217        ad 			mutex_spin_exit(spc->spc_lwplock);
    682      1.188      yamt 		} else {
    683      1.209        ad 			/*
    684      1.209        ad 			 * Otherwise, drop the spc_mutex, we are done with the
    685      1.209        ad 			 * run queues.
    686      1.209        ad 			 */
    687      1.188      yamt 			mutex_spin_exit(spc->spc_mutex);
    688      1.188      yamt 		}
    689      1.188      yamt 
    690      1.209        ad 		/*
    691      1.253     skrll 		 * Mark that context switch is going to be performed
    692      1.209        ad 		 * for this LWP, to protect it from being switched
    693      1.209        ad 		 * to on another CPU.
    694      1.209        ad 		 */
    695      1.209        ad 		KASSERT(l->l_ctxswtch == 0);
    696      1.209        ad 		l->l_ctxswtch = 1;
    697      1.209        ad 		l->l_ncsw++;
    698      1.267      yamt 		KASSERT((l->l_pflag & LP_RUNNING) != 0);
    699      1.248        ad 		l->l_pflag &= ~LP_RUNNING;
    700      1.209        ad 
    701      1.209        ad 		/*
    702      1.209        ad 		 * Increase the count of spin-mutexes before the release
    703      1.209        ad 		 * of the last lock - we must remain at IPL_SCHED during
    704      1.209        ad 		 * the context switch.
    705      1.209        ad 		 */
    706      1.287      matt 		KASSERTMSG(ci->ci_mtx_count == -1,
    707      1.291       jym 		    "%s: cpu%u: ci_mtx_count (%d) != -1",
    708      1.291       jym 		     __func__, cpu_index(ci), ci->ci_mtx_count);
    709      1.209        ad 		oldspl = MUTEX_SPIN_OLDSPL(ci);
    710      1.209        ad 		ci->ci_mtx_count--;
    711      1.209        ad 		lwp_unlock(l);
    712      1.209        ad 
    713      1.218        ad 		/* Count the context switch on this CPU. */
    714      1.218        ad 		ci->ci_data.cpu_nswtch++;
    715      1.188      yamt 
    716      1.209        ad 		/* Update status for lwpctl, if present. */
    717      1.209        ad 		if (l->l_lwpctl != NULL)
    718      1.209        ad 			l->l_lwpctl->lc_curcpu = LWPCTL_CPU_NONE;
    719      1.209        ad 
    720      1.199        ad 		/*
    721      1.199        ad 		 * Save old VM context, unless a soft interrupt
    722      1.199        ad 		 * handler is blocking.
    723      1.199        ad 		 */
    724      1.199        ad 		if (!returning)
    725      1.199        ad 			pmap_deactivate(l);
    726      1.188      yamt 
    727      1.209        ad 		/*
    728      1.275     skrll 		 * We may need to spin-wait if 'newl' is still
    729      1.209        ad 		 * context switching on another CPU.
    730      1.209        ad 		 */
    731      1.261     rmind 		if (__predict_false(newl->l_ctxswtch != 0)) {
    732      1.209        ad 			u_int count;
    733      1.209        ad 			count = SPINLOCK_BACKOFF_MIN;
    734      1.209        ad 			while (newl->l_ctxswtch)
    735      1.209        ad 				SPINLOCK_BACKOFF(count);
    736      1.209        ad 		}
    737      1.207        ad 
    738      1.276    darran 		/*
    739      1.276    darran 		 * If DTrace has set the active vtime enum to anything
    740      1.276    darran 		 * other than INACTIVE (0), then it should have set the
    741      1.276    darran 		 * function to call.
    742      1.276    darran 		 */
    743      1.278    darran 		if (__predict_false(dtrace_vtime_active)) {
    744      1.276    darran 			(*dtrace_vtime_switch_func)(newl);
    745      1.276    darran 		}
    746      1.276    darran 
    747      1.188      yamt 		/* Switch to the new LWP.. */
    748      1.204        ad 		prevlwp = cpu_switchto(l, newl, returning);
    749      1.207        ad 		ci = curcpu();
    750      1.207        ad 
    751      1.188      yamt 		/*
    752      1.209        ad 		 * Switched away - we have new curlwp.
    753      1.209        ad 		 * Restore VM context and IPL.
    754      1.188      yamt 		 */
    755      1.209        ad 		pmap_activate(l);
    756      1.265     rmind 		uvm_emap_switch(l);
    757      1.288     rmind 		pcu_switchpoint(l);
    758      1.265     rmind 
    759      1.188      yamt 		if (prevlwp != NULL) {
    760      1.209        ad 			/* Normalize the count of the spin-mutexes */
    761      1.209        ad 			ci->ci_mtx_count++;
    762      1.209        ad 			/* Unmark the state of context switch */
    763      1.209        ad 			membar_exit();
    764      1.209        ad 			prevlwp->l_ctxswtch = 0;
    765      1.188      yamt 		}
    766      1.209        ad 
    767      1.209        ad 		/* Update status for lwpctl, if present. */
    768      1.219        ad 		if (l->l_lwpctl != NULL) {
    769      1.209        ad 			l->l_lwpctl->lc_curcpu = (int)cpu_index(ci);
    770      1.219        ad 			l->l_lwpctl->lc_pctr++;
    771      1.219        ad 		}
    772      1.174        ad 
    773      1.290  christos 		/* Note trip through cpu_switchto(). */
    774      1.290  christos 		pserialize_switchpoint();
    775      1.290  christos 
    776      1.231        ad 		KASSERT(l->l_cpu == ci);
    777      1.231        ad 		splx(oldspl);
    778      1.188      yamt 		retval = 1;
    779      1.188      yamt 	} else {
    780      1.188      yamt 		/* Nothing to do - just unlock and return. */
    781      1.246     rmind 		mutex_spin_exit(spc->spc_mutex);
    782      1.188      yamt 		lwp_unlock(l);
    783      1.122   thorpej 		retval = 0;
    784      1.122   thorpej 	}
    785      1.110    briggs 
    786      1.188      yamt 	KASSERT(l == curlwp);
    787      1.188      yamt 	KASSERT(l->l_stat == LSONPROC);
    788      1.188      yamt 
    789      1.110    briggs 	/*
    790      1.174        ad 	 * XXXSMP If we are using h/w performance counters, restore context.
    791      1.231        ad 	 * XXXSMP preemption problem.
    792       1.26       cgd 	 */
    793      1.114  gmcgarry #if PERFCTRS
    794      1.175  christos 	if (PMC_ENABLED(l->l_proc)) {
    795      1.175  christos 		pmc_restore_context(l->l_proc);
    796      1.166  christos 	}
    797      1.114  gmcgarry #endif
    798      1.180       dsl 	SYSCALL_TIME_WAKEUP(l);
    799      1.188      yamt 	LOCKDEBUG_BARRIER(NULL, 1);
    800      1.169      yamt 
    801      1.122   thorpej 	return retval;
    802       1.26       cgd }
    803       1.26       cgd 
    804       1.26       cgd /*
    805      1.245        ad  * The machine independent parts of context switch to oblivion.
    806      1.245        ad  * Does not return.  Call with the LWP unlocked.
    807      1.245        ad  */
    808      1.245        ad void
    809      1.245        ad lwp_exit_switchaway(lwp_t *l)
    810      1.245        ad {
    811      1.245        ad 	struct cpu_info *ci;
    812      1.245        ad 	struct lwp *newl;
    813      1.245        ad 	struct bintime bt;
    814      1.245        ad 
    815      1.245        ad 	ci = l->l_cpu;
    816      1.245        ad 
    817      1.245        ad 	KASSERT(kpreempt_disabled());
    818      1.245        ad 	KASSERT(l->l_stat == LSZOMB || l->l_stat == LSIDL);
    819      1.245        ad 	KASSERT(ci == curcpu());
    820      1.245        ad 	LOCKDEBUG_BARRIER(NULL, 0);
    821      1.245        ad 
    822      1.245        ad 	kstack_check_magic(l);
    823      1.245        ad 
    824      1.245        ad 	/* Count time spent in current system call */
    825      1.245        ad 	SYSCALL_TIME_SLEEP(l);
    826      1.245        ad 	binuptime(&bt);
    827      1.245        ad 	updatertime(l, &bt);
    828      1.245        ad 
    829      1.245        ad 	/* Must stay at IPL_SCHED even after releasing run queue lock. */
    830      1.245        ad 	(void)splsched();
    831      1.245        ad 
    832      1.245        ad 	/*
    833      1.245        ad 	 * Let sched_nextlwp() select the LWP to run the CPU next.
    834      1.245        ad 	 * If no LWP is runnable, select the idle LWP.
    835      1.245        ad 	 *
    836      1.245        ad 	 * Note that spc_lwplock might not necessary be held, and
    837      1.245        ad 	 * new thread would be unlocked after setting the LWP-lock.
    838      1.245        ad 	 */
    839      1.245        ad 	spc_lock(ci);
    840      1.245        ad #ifndef __HAVE_FAST_SOFTINTS
    841      1.245        ad 	if (ci->ci_data.cpu_softints != 0) {
    842      1.245        ad 		/* There are pending soft interrupts, so pick one. */
    843      1.245        ad 		newl = softint_picklwp();
    844      1.245        ad 		newl->l_stat = LSONPROC;
    845      1.248        ad 		newl->l_pflag |= LP_RUNNING;
    846      1.245        ad 	} else
    847      1.245        ad #endif	/* !__HAVE_FAST_SOFTINTS */
    848      1.245        ad 	{
    849      1.245        ad 		newl = nextlwp(ci, &ci->ci_schedstate);
    850      1.245        ad 	}
    851      1.245        ad 
    852      1.245        ad 	/* Update the new LWP's start time. */
    853      1.245        ad 	newl->l_stime = bt;
    854      1.248        ad 	l->l_pflag &= ~LP_RUNNING;
    855      1.245        ad 
    856      1.245        ad 	/*
    857      1.245        ad 	 * ci_curlwp changes when a fast soft interrupt occurs.
    858      1.245        ad 	 * We use cpu_onproc to keep track of which kernel or
    859      1.245        ad 	 * user thread is running 'underneath' the software
    860      1.245        ad 	 * interrupt.  This is important for time accounting,
    861      1.245        ad 	 * itimers and forcing user threads to preempt (aston).
    862      1.245        ad 	 */
    863      1.245        ad 	ci->ci_data.cpu_onproc = newl;
    864      1.245        ad 
    865      1.245        ad 	/*
    866      1.245        ad 	 * Preemption related tasks.  Must be done with the current
    867      1.245        ad 	 * CPU locked.
    868      1.245        ad 	 */
    869      1.245        ad 	cpu_did_resched(l);
    870      1.245        ad 
    871      1.245        ad 	/* Unlock the run queue. */
    872      1.245        ad 	spc_unlock(ci);
    873      1.245        ad 
    874      1.245        ad 	/* Count the context switch on this CPU. */
    875      1.245        ad 	ci->ci_data.cpu_nswtch++;
    876      1.245        ad 
    877      1.245        ad 	/* Update status for lwpctl, if present. */
    878      1.245        ad 	if (l->l_lwpctl != NULL)
    879      1.247        ad 		l->l_lwpctl->lc_curcpu = LWPCTL_CPU_EXITED;
    880      1.245        ad 
    881      1.245        ad 	/*
    882      1.275     skrll 	 * We may need to spin-wait if 'newl' is still
    883      1.245        ad 	 * context switching on another CPU.
    884      1.245        ad 	 */
    885      1.261     rmind 	if (__predict_false(newl->l_ctxswtch != 0)) {
    886      1.245        ad 		u_int count;
    887      1.245        ad 		count = SPINLOCK_BACKOFF_MIN;
    888      1.245        ad 		while (newl->l_ctxswtch)
    889      1.245        ad 			SPINLOCK_BACKOFF(count);
    890      1.245        ad 	}
    891      1.245        ad 
    892      1.279    darran 	/*
    893      1.279    darran 	 * If DTrace has set the active vtime enum to anything
    894      1.279    darran 	 * other than INACTIVE (0), then it should have set the
    895      1.279    darran 	 * function to call.
    896      1.279    darran 	 */
    897      1.279    darran 	if (__predict_false(dtrace_vtime_active)) {
    898      1.279    darran 		(*dtrace_vtime_switch_func)(newl);
    899      1.279    darran 	}
    900      1.276    darran 
    901      1.245        ad 	/* Switch to the new LWP.. */
    902      1.245        ad 	(void)cpu_switchto(NULL, newl, false);
    903      1.245        ad 
    904      1.251       uwe 	for (;;) continue;	/* XXX: convince gcc about "noreturn" */
    905      1.245        ad 	/* NOTREACHED */
    906      1.245        ad }
    907      1.245        ad 
    908      1.245        ad /*
    909      1.271     rmind  * setrunnable: change LWP state to be runnable, placing it on the run queue.
    910      1.174        ad  *
    911      1.174        ad  * Call with the process and LWP locked.  Will return with the LWP unlocked.
    912       1.26       cgd  */
    913       1.26       cgd void
    914      1.122   thorpej setrunnable(struct lwp *l)
    915       1.26       cgd {
    916      1.122   thorpej 	struct proc *p = l->l_proc;
    917      1.205        ad 	struct cpu_info *ci;
    918       1.26       cgd 
    919      1.188      yamt 	KASSERT((l->l_flag & LW_IDLE) == 0);
    920      1.229        ad 	KASSERT(mutex_owned(p->p_lock));
    921      1.183        ad 	KASSERT(lwp_locked(l, NULL));
    922      1.205        ad 	KASSERT(l->l_mutex != l->l_cpu->ci_schedstate.spc_mutex);
    923       1.83   thorpej 
    924      1.122   thorpej 	switch (l->l_stat) {
    925      1.122   thorpej 	case LSSTOP:
    926       1.33   mycroft 		/*
    927       1.33   mycroft 		 * If we're being traced (possibly because someone attached us
    928       1.33   mycroft 		 * while we were stopped), check for a signal from the debugger.
    929       1.33   mycroft 		 */
    930      1.256        ad 		if ((p->p_slflag & PSL_TRACED) != 0 && p->p_xstat != 0)
    931      1.174        ad 			signotify(l);
    932      1.174        ad 		p->p_nrlwps++;
    933       1.26       cgd 		break;
    934      1.174        ad 	case LSSUSPENDED:
    935      1.178     pavel 		l->l_flag &= ~LW_WSUSPEND;
    936      1.174        ad 		p->p_nrlwps++;
    937      1.192     rmind 		cv_broadcast(&p->p_lwpcv);
    938      1.122   thorpej 		break;
    939      1.174        ad 	case LSSLEEP:
    940      1.174        ad 		KASSERT(l->l_wchan != NULL);
    941       1.26       cgd 		break;
    942      1.174        ad 	default:
    943      1.174        ad 		panic("setrunnable: lwp %p state was %d", l, l->l_stat);
    944       1.26       cgd 	}
    945      1.139        cl 
    946      1.252  wrstuden #ifdef KERN_SA
    947      1.252  wrstuden 	if (l->l_proc->p_sa)
    948      1.252  wrstuden 		sa_awaken(l);
    949      1.252  wrstuden #endif /* KERN_SA */
    950      1.252  wrstuden 
    951      1.174        ad 	/*
    952      1.286     pooka 	 * If the LWP was sleeping, start it again.
    953      1.174        ad 	 */
    954      1.174        ad 	if (l->l_wchan != NULL) {
    955      1.174        ad 		l->l_stat = LSSLEEP;
    956      1.183        ad 		/* lwp_unsleep() will release the lock. */
    957      1.221        ad 		lwp_unsleep(l, true);
    958      1.174        ad 		return;
    959      1.174        ad 	}
    960      1.139        cl 
    961      1.174        ad 	/*
    962      1.174        ad 	 * If the LWP is still on the CPU, mark it as LSONPROC.  It may be
    963      1.174        ad 	 * about to call mi_switch(), in which case it will yield.
    964      1.174        ad 	 */
    965      1.248        ad 	if ((l->l_pflag & LP_RUNNING) != 0) {
    966      1.174        ad 		l->l_stat = LSONPROC;
    967      1.174        ad 		l->l_slptime = 0;
    968      1.174        ad 		lwp_unlock(l);
    969      1.174        ad 		return;
    970      1.174        ad 	}
    971      1.122   thorpej 
    972      1.174        ad 	/*
    973      1.205        ad 	 * Look for a CPU to run.
    974      1.205        ad 	 * Set the LWP runnable.
    975      1.174        ad 	 */
    976      1.205        ad 	ci = sched_takecpu(l);
    977      1.205        ad 	l->l_cpu = ci;
    978      1.236        ad 	spc_lock(ci);
    979      1.236        ad 	lwp_unlock_to(l, ci->ci_schedstate.spc_mutex);
    980      1.188      yamt 	sched_setrunnable(l);
    981      1.174        ad 	l->l_stat = LSRUN;
    982      1.122   thorpej 	l->l_slptime = 0;
    983      1.174        ad 
    984      1.271     rmind 	sched_enqueue(l, false);
    985      1.271     rmind 	resched_cpu(l);
    986      1.271     rmind 	lwp_unlock(l);
    987       1.26       cgd }
    988       1.26       cgd 
    989       1.26       cgd /*
    990      1.174        ad  * suspendsched:
    991      1.174        ad  *
    992      1.266      yamt  *	Convert all non-LW_SYSTEM LSSLEEP or LSRUN LWPs to LSSUSPENDED.
    993      1.174        ad  */
    994       1.94    bouyer void
    995      1.174        ad suspendsched(void)
    996       1.94    bouyer {
    997      1.174        ad 	CPU_INFO_ITERATOR cii;
    998      1.174        ad 	struct cpu_info *ci;
    999      1.122   thorpej 	struct lwp *l;
   1000      1.174        ad 	struct proc *p;
   1001       1.94    bouyer 
   1002       1.94    bouyer 	/*
   1003      1.174        ad 	 * We do this by process in order not to violate the locking rules.
   1004       1.94    bouyer 	 */
   1005      1.228        ad 	mutex_enter(proc_lock);
   1006      1.174        ad 	PROCLIST_FOREACH(p, &allproc) {
   1007      1.229        ad 		mutex_enter(p->p_lock);
   1008      1.178     pavel 		if ((p->p_flag & PK_SYSTEM) != 0) {
   1009      1.229        ad 			mutex_exit(p->p_lock);
   1010       1.94    bouyer 			continue;
   1011      1.174        ad 		}
   1012      1.174        ad 
   1013  1.297.2.2    bouyer 		if (p->p_stat != SSTOP) {
   1014  1.297.2.2    bouyer 			if (p->p_stat != SZOMB && p->p_stat != SDEAD) {
   1015  1.297.2.2    bouyer 				p->p_pptr->p_nstopchild++;
   1016  1.297.2.2    bouyer 				p->p_waited = 0;
   1017  1.297.2.2    bouyer 			}
   1018  1.297.2.2    bouyer 			p->p_stat = SSTOP;
   1019  1.297.2.2    bouyer 		}
   1020      1.174        ad 
   1021      1.174        ad 		LIST_FOREACH(l, &p->p_lwps, l_sibling) {
   1022      1.174        ad 			if (l == curlwp)
   1023      1.174        ad 				continue;
   1024      1.174        ad 
   1025      1.174        ad 			lwp_lock(l);
   1026      1.122   thorpej 
   1027       1.97     enami 			/*
   1028      1.174        ad 			 * Set L_WREBOOT so that the LWP will suspend itself
   1029      1.174        ad 			 * when it tries to return to user mode.  We want to
   1030      1.174        ad 			 * try and get to get as many LWPs as possible to
   1031      1.174        ad 			 * the user / kernel boundary, so that they will
   1032      1.174        ad 			 * release any locks that they hold.
   1033       1.97     enami 			 */
   1034      1.178     pavel 			l->l_flag |= (LW_WREBOOT | LW_WSUSPEND);
   1035      1.174        ad 
   1036      1.174        ad 			if (l->l_stat == LSSLEEP &&
   1037      1.178     pavel 			    (l->l_flag & LW_SINTR) != 0) {
   1038      1.174        ad 				/* setrunnable() will release the lock. */
   1039      1.174        ad 				setrunnable(l);
   1040      1.174        ad 				continue;
   1041      1.174        ad 			}
   1042      1.174        ad 
   1043      1.174        ad 			lwp_unlock(l);
   1044       1.94    bouyer 		}
   1045      1.174        ad 
   1046      1.229        ad 		mutex_exit(p->p_lock);
   1047       1.94    bouyer 	}
   1048      1.228        ad 	mutex_exit(proc_lock);
   1049      1.174        ad 
   1050      1.174        ad 	/*
   1051      1.174        ad 	 * Kick all CPUs to make them preempt any LWPs running in user mode.
   1052      1.174        ad 	 * They'll trap into the kernel and suspend themselves in userret().
   1053      1.174        ad 	 */
   1054      1.204        ad 	for (CPU_INFO_FOREACH(cii, ci)) {
   1055      1.204        ad 		spc_lock(ci);
   1056      1.204        ad 		cpu_need_resched(ci, RESCHED_IMMED);
   1057      1.204        ad 		spc_unlock(ci);
   1058      1.204        ad 	}
   1059      1.174        ad }
   1060      1.174        ad 
   1061      1.174        ad /*
   1062      1.174        ad  * sched_unsleep:
   1063      1.174        ad  *
   1064      1.174        ad  *	The is called when the LWP has not been awoken normally but instead
   1065      1.174        ad  *	interrupted: for example, if the sleep timed out.  Because of this,
   1066      1.174        ad  *	it's not a valid action for running or idle LWPs.
   1067      1.174        ad  */
   1068      1.271     rmind static void
   1069      1.221        ad sched_unsleep(struct lwp *l, bool cleanup)
   1070      1.174        ad {
   1071      1.174        ad 
   1072      1.174        ad 	lwp_unlock(l);
   1073      1.174        ad 	panic("sched_unsleep");
   1074      1.174        ad }
   1075      1.174        ad 
   1076      1.250     rmind static void
   1077      1.188      yamt resched_cpu(struct lwp *l)
   1078      1.188      yamt {
   1079      1.274     rmind 	struct cpu_info *ci = l->l_cpu;
   1080      1.188      yamt 
   1081      1.250     rmind 	KASSERT(lwp_locked(l, NULL));
   1082      1.204        ad 	if (lwp_eprio(l) > ci->ci_schedstate.spc_curpriority)
   1083      1.188      yamt 		cpu_need_resched(ci, 0);
   1084      1.188      yamt }
   1085      1.188      yamt 
   1086      1.188      yamt static void
   1087      1.185      yamt sched_changepri(struct lwp *l, pri_t pri)
   1088      1.174        ad {
   1089      1.174        ad 
   1090      1.188      yamt 	KASSERT(lwp_locked(l, NULL));
   1091      1.174        ad 
   1092      1.271     rmind 	if (l->l_stat == LSRUN) {
   1093      1.204        ad 		KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_mutex));
   1094      1.204        ad 		sched_dequeue(l);
   1095      1.204        ad 		l->l_priority = pri;
   1096      1.204        ad 		sched_enqueue(l, false);
   1097      1.204        ad 	} else {
   1098      1.174        ad 		l->l_priority = pri;
   1099      1.157      yamt 	}
   1100      1.188      yamt 	resched_cpu(l);
   1101      1.184      yamt }
   1102      1.184      yamt 
   1103      1.188      yamt static void
   1104      1.185      yamt sched_lendpri(struct lwp *l, pri_t pri)
   1105      1.184      yamt {
   1106      1.184      yamt 
   1107      1.188      yamt 	KASSERT(lwp_locked(l, NULL));
   1108      1.184      yamt 
   1109      1.271     rmind 	if (l->l_stat == LSRUN) {
   1110      1.204        ad 		KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_mutex));
   1111      1.204        ad 		sched_dequeue(l);
   1112      1.204        ad 		l->l_inheritedprio = pri;
   1113      1.204        ad 		sched_enqueue(l, false);
   1114      1.204        ad 	} else {
   1115      1.184      yamt 		l->l_inheritedprio = pri;
   1116      1.184      yamt 	}
   1117      1.188      yamt 	resched_cpu(l);
   1118      1.184      yamt }
   1119      1.184      yamt 
   1120      1.184      yamt struct lwp *
   1121      1.184      yamt syncobj_noowner(wchan_t wchan)
   1122      1.184      yamt {
   1123      1.184      yamt 
   1124      1.184      yamt 	return NULL;
   1125      1.151      yamt }
   1126      1.151      yamt 
   1127      1.250     rmind /* Decay 95% of proc::p_pctcpu in 60 seconds, ccpu = exp(-1/20) */
   1128      1.281     rmind const fixpt_t ccpu = 0.95122942450071400909 * FSCALE;
   1129      1.281     rmind 
   1130      1.281     rmind /*
   1131      1.281     rmind  * Constants for averages over 1, 5 and 15 minutes when sampling at
   1132      1.281     rmind  * 5 second intervals.
   1133      1.281     rmind  */
   1134      1.281     rmind static const fixpt_t cexp[ ] = {
   1135      1.281     rmind 	0.9200444146293232 * FSCALE,	/* exp(-1/12) */
   1136      1.281     rmind 	0.9834714538216174 * FSCALE,	/* exp(-1/60) */
   1137      1.281     rmind 	0.9944598480048967 * FSCALE,	/* exp(-1/180) */
   1138      1.281     rmind };
   1139      1.134      matt 
   1140      1.134      matt /*
   1141      1.188      yamt  * sched_pstats:
   1142      1.188      yamt  *
   1143      1.281     rmind  * => Update process statistics and check CPU resource allocation.
   1144      1.281     rmind  * => Call scheduler-specific hook to eventually adjust LWP priorities.
   1145      1.281     rmind  * => Compute load average of a quantity on 1, 5 and 15 minute intervals.
   1146      1.130   nathanw  */
   1147      1.113  gmcgarry void
   1148      1.281     rmind sched_pstats(void)
   1149      1.113  gmcgarry {
   1150      1.281     rmind 	extern struct loadavg averunnable;
   1151      1.281     rmind 	struct loadavg *avg = &averunnable;
   1152      1.249     rmind 	const int clkhz = (stathz != 0 ? stathz : hz);
   1153      1.281     rmind 	static bool backwards = false;
   1154      1.281     rmind 	static u_int lavg_count = 0;
   1155      1.188      yamt 	struct proc *p;
   1156      1.281     rmind 	int nrun;
   1157      1.113  gmcgarry 
   1158      1.188      yamt 	sched_pstats_ticks++;
   1159      1.281     rmind 	if (++lavg_count >= 5) {
   1160      1.281     rmind 		lavg_count = 0;
   1161      1.281     rmind 		nrun = 0;
   1162      1.281     rmind 	}
   1163      1.228        ad 	mutex_enter(proc_lock);
   1164      1.188      yamt 	PROCLIST_FOREACH(p, &allproc) {
   1165      1.281     rmind 		struct lwp *l;
   1166      1.281     rmind 		struct rlimit *rlim;
   1167      1.296  dholland 		time_t runtm;
   1168      1.281     rmind 		int sig;
   1169      1.281     rmind 
   1170      1.271     rmind 		/* Increment sleep time (if sleeping), ignore overflow. */
   1171      1.229        ad 		mutex_enter(p->p_lock);
   1172      1.212      yamt 		runtm = p->p_rtime.sec;
   1173      1.188      yamt 		LIST_FOREACH(l, &p->p_lwps, l_sibling) {
   1174      1.281     rmind 			fixpt_t lpctcpu;
   1175      1.281     rmind 			u_int lcpticks;
   1176      1.281     rmind 
   1177      1.249     rmind 			if (__predict_false((l->l_flag & LW_IDLE) != 0))
   1178      1.188      yamt 				continue;
   1179      1.188      yamt 			lwp_lock(l);
   1180      1.212      yamt 			runtm += l->l_rtime.sec;
   1181      1.188      yamt 			l->l_swtime++;
   1182      1.242     rmind 			sched_lwp_stats(l);
   1183      1.281     rmind 
   1184      1.281     rmind 			/* For load average calculation. */
   1185      1.282     rmind 			if (__predict_false(lavg_count == 0) &&
   1186      1.282     rmind 			    (l->l_flag & (LW_SINTR | LW_SYSTEM)) == 0) {
   1187      1.281     rmind 				switch (l->l_stat) {
   1188      1.281     rmind 				case LSSLEEP:
   1189      1.281     rmind 					if (l->l_slptime > 1) {
   1190      1.281     rmind 						break;
   1191      1.281     rmind 					}
   1192      1.281     rmind 				case LSRUN:
   1193      1.281     rmind 				case LSONPROC:
   1194      1.281     rmind 				case LSIDL:
   1195      1.281     rmind 					nrun++;
   1196      1.281     rmind 				}
   1197      1.281     rmind 			}
   1198      1.282     rmind 			lwp_unlock(l);
   1199      1.282     rmind 
   1200      1.282     rmind 			l->l_pctcpu = (l->l_pctcpu * ccpu) >> FSHIFT;
   1201      1.282     rmind 			if (l->l_slptime != 0)
   1202      1.282     rmind 				continue;
   1203      1.282     rmind 
   1204      1.282     rmind 			lpctcpu = l->l_pctcpu;
   1205      1.282     rmind 			lcpticks = atomic_swap_uint(&l->l_cpticks, 0);
   1206      1.282     rmind 			lpctcpu += ((FSCALE - ccpu) *
   1207      1.282     rmind 			    (lcpticks * FSCALE / clkhz)) >> FSHIFT;
   1208      1.282     rmind 			l->l_pctcpu = lpctcpu;
   1209      1.188      yamt 		}
   1210      1.249     rmind 		/* Calculating p_pctcpu only for ps(1) */
   1211      1.188      yamt 		p->p_pctcpu = (p->p_pctcpu * ccpu) >> FSHIFT;
   1212      1.174        ad 
   1213  1.297.2.1       riz 		if (__predict_false(runtm < 0)) {
   1214  1.297.2.1       riz 			if (!backwards) {
   1215  1.297.2.1       riz 				backwards = true;
   1216  1.297.2.1       riz 				printf("WARNING: negative runtime; "
   1217  1.297.2.1       riz 				    "monotonic clock has gone backwards\n");
   1218  1.297.2.1       riz 			}
   1219  1.297.2.1       riz 			mutex_exit(p->p_lock);
   1220  1.297.2.1       riz 			continue;
   1221  1.297.2.1       riz 		}
   1222  1.297.2.1       riz 
   1223      1.188      yamt 		/*
   1224      1.188      yamt 		 * Check if the process exceeds its CPU resource allocation.
   1225      1.293       apb 		 * If over the hard limit, kill it with SIGKILL.
   1226      1.293       apb 		 * If over the soft limit, send SIGXCPU and raise
   1227      1.293       apb 		 * the soft limit a little.
   1228      1.188      yamt 		 */
   1229      1.188      yamt 		rlim = &p->p_rlimit[RLIMIT_CPU];
   1230      1.188      yamt 		sig = 0;
   1231      1.249     rmind 		if (__predict_false(runtm >= rlim->rlim_cur)) {
   1232      1.293       apb 			if (runtm >= rlim->rlim_max) {
   1233      1.188      yamt 				sig = SIGKILL;
   1234      1.293       apb 				log(LOG_NOTICE, "pid %d is killed: %s\n",
   1235      1.293       apb 					p->p_pid, "exceeded RLIMIT_CPU");
   1236      1.293       apb 				uprintf("pid %d, command %s, is killed: %s\n",
   1237      1.293       apb 					p->p_pid, p->p_comm,
   1238      1.293       apb 					"exceeded RLIMIT_CPU");
   1239      1.293       apb 			} else {
   1240      1.188      yamt 				sig = SIGXCPU;
   1241      1.188      yamt 				if (rlim->rlim_cur < rlim->rlim_max)
   1242      1.188      yamt 					rlim->rlim_cur += 5;
   1243      1.188      yamt 			}
   1244      1.188      yamt 		}
   1245      1.229        ad 		mutex_exit(p->p_lock);
   1246  1.297.2.1       riz 		if (__predict_false(sig)) {
   1247      1.259     rmind 			KASSERT((p->p_flag & PK_SYSTEM) == 0);
   1248      1.188      yamt 			psignal(p, sig);
   1249      1.259     rmind 		}
   1250      1.174        ad 	}
   1251      1.228        ad 	mutex_exit(proc_lock);
   1252      1.281     rmind 
   1253      1.281     rmind 	/* Load average calculation. */
   1254      1.281     rmind 	if (__predict_false(lavg_count == 0)) {
   1255      1.281     rmind 		int i;
   1256      1.283    martin 		CTASSERT(__arraycount(cexp) == __arraycount(avg->ldavg));
   1257      1.281     rmind 		for (i = 0; i < __arraycount(cexp); i++) {
   1258      1.281     rmind 			avg->ldavg[i] = (cexp[i] * avg->ldavg[i] +
   1259      1.281     rmind 			    nrun * FSCALE * (FSCALE - cexp[i])) >> FSHIFT;
   1260      1.281     rmind 		}
   1261      1.281     rmind 	}
   1262      1.281     rmind 
   1263      1.281     rmind 	/* Lightning bolt. */
   1264      1.273     pooka 	cv_broadcast(&lbolt);
   1265      1.113  gmcgarry }
   1266