Home | History | Annotate | Line # | Download | only in kern
sched_4bsd.c revision 1.3
      1  1.3    ad /*	$NetBSD: sched_4bsd.c,v 1.3 2007/07/09 21:10:55 ad Exp $	*/
      2  1.2  yamt 
      3  1.2  yamt /*-
      4  1.2  yamt  * Copyright (c) 1999, 2000, 2004, 2006, 2007 The NetBSD Foundation, Inc.
      5  1.2  yamt  * All rights reserved.
      6  1.2  yamt  *
      7  1.2  yamt  * This code is derived from software contributed to The NetBSD Foundation
      8  1.2  yamt  * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
      9  1.2  yamt  * NASA Ames Research Center, by Charles M. Hannum, Andrew Doran, and
     10  1.2  yamt  * Daniel Sieger.
     11  1.2  yamt  *
     12  1.2  yamt  * Redistribution and use in source and binary forms, with or without
     13  1.2  yamt  * modification, are permitted provided that the following conditions
     14  1.2  yamt  * are met:
     15  1.2  yamt  * 1. Redistributions of source code must retain the above copyright
     16  1.2  yamt  *    notice, this list of conditions and the following disclaimer.
     17  1.2  yamt  * 2. Redistributions in binary form must reproduce the above copyright
     18  1.2  yamt  *    notice, this list of conditions and the following disclaimer in the
     19  1.2  yamt  *    documentation and/or other materials provided with the distribution.
     20  1.2  yamt  * 3. All advertising materials mentioning features or use of this software
     21  1.2  yamt  *    must display the following acknowledgement:
     22  1.2  yamt  *	This product includes software developed by the NetBSD
     23  1.2  yamt  *	Foundation, Inc. and its contributors.
     24  1.2  yamt  * 4. Neither the name of The NetBSD Foundation nor the names of its
     25  1.2  yamt  *    contributors may be used to endorse or promote products derived
     26  1.2  yamt  *    from this software without specific prior written permission.
     27  1.2  yamt  *
     28  1.2  yamt  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     29  1.2  yamt  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     30  1.2  yamt  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     31  1.2  yamt  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     32  1.2  yamt  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     33  1.2  yamt  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     34  1.2  yamt  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     35  1.2  yamt  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     36  1.2  yamt  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     37  1.2  yamt  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     38  1.2  yamt  * POSSIBILITY OF SUCH DAMAGE.
     39  1.2  yamt  */
     40  1.2  yamt 
     41  1.2  yamt /*-
     42  1.2  yamt  * Copyright (c) 1982, 1986, 1990, 1991, 1993
     43  1.2  yamt  *	The Regents of the University of California.  All rights reserved.
     44  1.2  yamt  * (c) UNIX System Laboratories, Inc.
     45  1.2  yamt  * All or some portions of this file are derived from material licensed
     46  1.2  yamt  * to the University of California by American Telephone and Telegraph
     47  1.2  yamt  * Co. or Unix System Laboratories, Inc. and are reproduced herein with
     48  1.2  yamt  * the permission of UNIX System Laboratories, Inc.
     49  1.2  yamt  *
     50  1.2  yamt  * Redistribution and use in source and binary forms, with or without
     51  1.2  yamt  * modification, are permitted provided that the following conditions
     52  1.2  yamt  * are met:
     53  1.2  yamt  * 1. Redistributions of source code must retain the above copyright
     54  1.2  yamt  *    notice, this list of conditions and the following disclaimer.
     55  1.2  yamt  * 2. Redistributions in binary form must reproduce the above copyright
     56  1.2  yamt  *    notice, this list of conditions and the following disclaimer in the
     57  1.2  yamt  *    documentation and/or other materials provided with the distribution.
     58  1.2  yamt  * 3. Neither the name of the University nor the names of its contributors
     59  1.2  yamt  *    may be used to endorse or promote products derived from this software
     60  1.2  yamt  *    without specific prior written permission.
     61  1.2  yamt  *
     62  1.2  yamt  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
     63  1.2  yamt  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     64  1.2  yamt  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     65  1.2  yamt  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
     66  1.2  yamt  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     67  1.2  yamt  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
     68  1.2  yamt  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     69  1.2  yamt  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     70  1.2  yamt  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     71  1.2  yamt  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     72  1.2  yamt  * SUCH DAMAGE.
     73  1.2  yamt  *
     74  1.2  yamt  *	@(#)kern_synch.c	8.9 (Berkeley) 5/19/95
     75  1.2  yamt  */
     76  1.2  yamt 
     77  1.2  yamt #include <sys/cdefs.h>
     78  1.3    ad __KERNEL_RCSID(0, "$NetBSD: sched_4bsd.c,v 1.3 2007/07/09 21:10:55 ad Exp $");
     79  1.2  yamt 
     80  1.2  yamt #include "opt_ddb.h"
     81  1.2  yamt #include "opt_lockdebug.h"
     82  1.2  yamt #include "opt_perfctrs.h"
     83  1.2  yamt 
     84  1.2  yamt #define	__MUTEX_PRIVATE
     85  1.2  yamt 
     86  1.2  yamt #include <sys/param.h>
     87  1.2  yamt #include <sys/systm.h>
     88  1.2  yamt #include <sys/callout.h>
     89  1.2  yamt #include <sys/cpu.h>
     90  1.2  yamt #include <sys/proc.h>
     91  1.2  yamt #include <sys/kernel.h>
     92  1.2  yamt #include <sys/signalvar.h>
     93  1.2  yamt #include <sys/resourcevar.h>
     94  1.2  yamt #include <sys/sched.h>
     95  1.2  yamt #include <sys/sysctl.h>
     96  1.2  yamt #include <sys/kauth.h>
     97  1.2  yamt #include <sys/lockdebug.h>
     98  1.2  yamt #include <sys/kmem.h>
     99  1.2  yamt 
    100  1.2  yamt #include <uvm/uvm_extern.h>
    101  1.2  yamt 
    102  1.2  yamt /*
    103  1.2  yamt  * Run queues.
    104  1.2  yamt  *
    105  1.2  yamt  * We have 32 run queues in descending priority of 0..31.  We maintain
    106  1.2  yamt  * a bitmask of non-empty queues in order speed up finding the first
    107  1.2  yamt  * runnable process.  The bitmask is maintained only by machine-dependent
    108  1.2  yamt  * code, allowing the most efficient instructions to be used to find the
    109  1.2  yamt  * first non-empty queue.
    110  1.2  yamt  */
    111  1.2  yamt 
    112  1.2  yamt #define	RUNQUE_NQS		32      /* number of runqueues */
    113  1.2  yamt #define	PPQ	(128 / RUNQUE_NQS)	/* priorities per queue */
    114  1.2  yamt 
    115  1.2  yamt typedef struct subqueue {
    116  1.2  yamt 	TAILQ_HEAD(, lwp) sq_queue;
    117  1.2  yamt } subqueue_t;
    118  1.2  yamt typedef struct runqueue {
    119  1.2  yamt 	subqueue_t rq_subqueues[RUNQUE_NQS];	/* run queues */
    120  1.2  yamt 	uint32_t rq_bitmap;	/* bitmap of non-empty queues */
    121  1.2  yamt } runqueue_t;
    122  1.2  yamt static runqueue_t global_queue;
    123  1.2  yamt 
    124  1.2  yamt static void updatepri(struct lwp *);
    125  1.2  yamt static void resetpriority(struct lwp *);
    126  1.2  yamt static void resetprocpriority(struct proc *);
    127  1.2  yamt 
    128  1.2  yamt extern unsigned int sched_pstats_ticks; /* defined in kern_synch.c */
    129  1.2  yamt 
    130  1.2  yamt /* The global scheduler state */
    131  1.2  yamt kmutex_t sched_mutex;
    132  1.2  yamt 
    133  1.2  yamt /* Number of hardclock ticks per sched_tick() */
    134  1.2  yamt int rrticks;
    135  1.2  yamt 
    136  1.2  yamt /*
    137  1.2  yamt  * Force switch among equal priority processes every 100ms.
    138  1.2  yamt  * Called from hardclock every hz/10 == rrticks hardclock ticks.
    139  1.2  yamt  */
    140  1.2  yamt /* ARGSUSED */
    141  1.2  yamt void
    142  1.2  yamt sched_tick(struct cpu_info *ci)
    143  1.2  yamt {
    144  1.2  yamt 	struct schedstate_percpu *spc = &ci->ci_schedstate;
    145  1.2  yamt 
    146  1.2  yamt 	spc->spc_ticks = rrticks;
    147  1.2  yamt 
    148  1.2  yamt 	if (!CURCPU_IDLE_P()) {
    149  1.2  yamt 		if (spc->spc_flags & SPCF_SEENRR) {
    150  1.2  yamt 			/*
    151  1.2  yamt 			 * The process has already been through a roundrobin
    152  1.2  yamt 			 * without switching and may be hogging the CPU.
    153  1.2  yamt 			 * Indicate that the process should yield.
    154  1.2  yamt 			 */
    155  1.2  yamt 			spc->spc_flags |= SPCF_SHOULDYIELD;
    156  1.2  yamt 		} else
    157  1.2  yamt 			spc->spc_flags |= SPCF_SEENRR;
    158  1.2  yamt 	}
    159  1.2  yamt 	cpu_need_resched(curcpu(), 0);
    160  1.2  yamt }
    161  1.2  yamt 
    162  1.2  yamt #define	NICE_WEIGHT 2			/* priorities per nice level */
    163  1.2  yamt 
    164  1.2  yamt #define	ESTCPU_SHIFT	11
    165  1.2  yamt #define	ESTCPU_MAX	((NICE_WEIGHT * PRIO_MAX - PPQ) << ESTCPU_SHIFT)
    166  1.2  yamt #define	ESTCPULIM(e)	min((e), ESTCPU_MAX)
    167  1.2  yamt 
    168  1.2  yamt /*
    169  1.2  yamt  * Constants for digital decay and forget:
    170  1.2  yamt  *	90% of (p_estcpu) usage in 5 * loadav time
    171  1.2  yamt  *	95% of (p_pctcpu) usage in 60 seconds (load insensitive)
    172  1.2  yamt  *          Note that, as ps(1) mentions, this can let percentages
    173  1.2  yamt  *          total over 100% (I've seen 137.9% for 3 processes).
    174  1.2  yamt  *
    175  1.2  yamt  * Note that hardclock updates p_estcpu and p_cpticks independently.
    176  1.2  yamt  *
    177  1.2  yamt  * We wish to decay away 90% of p_estcpu in (5 * loadavg) seconds.
    178  1.2  yamt  * That is, the system wants to compute a value of decay such
    179  1.2  yamt  * that the following for loop:
    180  1.2  yamt  * 	for (i = 0; i < (5 * loadavg); i++)
    181  1.2  yamt  * 		p_estcpu *= decay;
    182  1.2  yamt  * will compute
    183  1.2  yamt  * 	p_estcpu *= 0.1;
    184  1.2  yamt  * for all values of loadavg:
    185  1.2  yamt  *
    186  1.2  yamt  * Mathematically this loop can be expressed by saying:
    187  1.2  yamt  * 	decay ** (5 * loadavg) ~= .1
    188  1.2  yamt  *
    189  1.2  yamt  * The system computes decay as:
    190  1.2  yamt  * 	decay = (2 * loadavg) / (2 * loadavg + 1)
    191  1.2  yamt  *
    192  1.2  yamt  * We wish to prove that the system's computation of decay
    193  1.2  yamt  * will always fulfill the equation:
    194  1.2  yamt  * 	decay ** (5 * loadavg) ~= .1
    195  1.2  yamt  *
    196  1.2  yamt  * If we compute b as:
    197  1.2  yamt  * 	b = 2 * loadavg
    198  1.2  yamt  * then
    199  1.2  yamt  * 	decay = b / (b + 1)
    200  1.2  yamt  *
    201  1.2  yamt  * We now need to prove two things:
    202  1.2  yamt  *	1) Given factor ** (5 * loadavg) ~= .1, prove factor == b/(b+1)
    203  1.2  yamt  *	2) Given b/(b+1) ** power ~= .1, prove power == (5 * loadavg)
    204  1.2  yamt  *
    205  1.2  yamt  * Facts:
    206  1.2  yamt  *         For x close to zero, exp(x) =~ 1 + x, since
    207  1.2  yamt  *              exp(x) = 0! + x**1/1! + x**2/2! + ... .
    208  1.2  yamt  *              therefore exp(-1/b) =~ 1 - (1/b) = (b-1)/b.
    209  1.2  yamt  *         For x close to zero, ln(1+x) =~ x, since
    210  1.2  yamt  *              ln(1+x) = x - x**2/2 + x**3/3 - ...     -1 < x < 1
    211  1.2  yamt  *              therefore ln(b/(b+1)) = ln(1 - 1/(b+1)) =~ -1/(b+1).
    212  1.2  yamt  *         ln(.1) =~ -2.30
    213  1.2  yamt  *
    214  1.2  yamt  * Proof of (1):
    215  1.2  yamt  *    Solve (factor)**(power) =~ .1 given power (5*loadav):
    216  1.2  yamt  *	solving for factor,
    217  1.2  yamt  *      ln(factor) =~ (-2.30/5*loadav), or
    218  1.2  yamt  *      factor =~ exp(-1/((5/2.30)*loadav)) =~ exp(-1/(2*loadav)) =
    219  1.2  yamt  *          exp(-1/b) =~ (b-1)/b =~ b/(b+1).                    QED
    220  1.2  yamt  *
    221  1.2  yamt  * Proof of (2):
    222  1.2  yamt  *    Solve (factor)**(power) =~ .1 given factor == (b/(b+1)):
    223  1.2  yamt  *	solving for power,
    224  1.2  yamt  *      power*ln(b/(b+1)) =~ -2.30, or
    225  1.2  yamt  *      power =~ 2.3 * (b + 1) = 4.6*loadav + 2.3 =~ 5*loadav.  QED
    226  1.2  yamt  *
    227  1.2  yamt  * Actual power values for the implemented algorithm are as follows:
    228  1.2  yamt  *      loadav: 1       2       3       4
    229  1.2  yamt  *      power:  5.68    10.32   14.94   19.55
    230  1.2  yamt  */
    231  1.2  yamt 
    232  1.2  yamt /* calculations for digital decay to forget 90% of usage in 5*loadav sec */
    233  1.2  yamt #define	loadfactor(loadav)	(2 * (loadav))
    234  1.2  yamt 
    235  1.2  yamt static fixpt_t
    236  1.2  yamt decay_cpu(fixpt_t loadfac, fixpt_t estcpu)
    237  1.2  yamt {
    238  1.2  yamt 
    239  1.2  yamt 	if (estcpu == 0) {
    240  1.2  yamt 		return 0;
    241  1.2  yamt 	}
    242  1.2  yamt 
    243  1.2  yamt #if !defined(_LP64)
    244  1.2  yamt 	/* avoid 64bit arithmetics. */
    245  1.2  yamt #define	FIXPT_MAX ((fixpt_t)((UINTMAX_C(1) << sizeof(fixpt_t) * CHAR_BIT) - 1))
    246  1.2  yamt 	if (__predict_true(loadfac <= FIXPT_MAX / ESTCPU_MAX)) {
    247  1.2  yamt 		return estcpu * loadfac / (loadfac + FSCALE);
    248  1.2  yamt 	}
    249  1.2  yamt #endif /* !defined(_LP64) */
    250  1.2  yamt 
    251  1.2  yamt 	return (uint64_t)estcpu * loadfac / (loadfac + FSCALE);
    252  1.2  yamt }
    253  1.2  yamt 
    254  1.2  yamt /*
    255  1.2  yamt  * For all load averages >= 1 and max p_estcpu of (255 << ESTCPU_SHIFT),
    256  1.2  yamt  * sleeping for at least seven times the loadfactor will decay p_estcpu to
    257  1.2  yamt  * less than (1 << ESTCPU_SHIFT).
    258  1.2  yamt  *
    259  1.2  yamt  * note that our ESTCPU_MAX is actually much smaller than (255 << ESTCPU_SHIFT).
    260  1.2  yamt  */
    261  1.2  yamt static fixpt_t
    262  1.2  yamt decay_cpu_batch(fixpt_t loadfac, fixpt_t estcpu, unsigned int n)
    263  1.2  yamt {
    264  1.2  yamt 
    265  1.2  yamt 	if ((n << FSHIFT) >= 7 * loadfac) {
    266  1.2  yamt 		return 0;
    267  1.2  yamt 	}
    268  1.2  yamt 
    269  1.2  yamt 	while (estcpu != 0 && n > 1) {
    270  1.2  yamt 		estcpu = decay_cpu(loadfac, estcpu);
    271  1.2  yamt 		n--;
    272  1.2  yamt 	}
    273  1.2  yamt 
    274  1.2  yamt 	return estcpu;
    275  1.2  yamt }
    276  1.2  yamt 
    277  1.2  yamt /*
    278  1.2  yamt  * sched_pstats_hook:
    279  1.2  yamt  *
    280  1.2  yamt  * Periodically called from sched_pstats(); used to recalculate priorities.
    281  1.2  yamt  */
    282  1.2  yamt void
    283  1.2  yamt sched_pstats_hook(struct proc *p, int minslp)
    284  1.2  yamt {
    285  1.2  yamt 	struct lwp *l;
    286  1.2  yamt 	fixpt_t loadfac = loadfactor(averunnable.ldavg[0]);
    287  1.2  yamt 
    288  1.2  yamt 	/*
    289  1.2  yamt 	 * If the process has slept the entire second,
    290  1.2  yamt 	 * stop recalculating its priority until it wakes up.
    291  1.2  yamt 	 */
    292  1.2  yamt 	if (minslp <= 1) {
    293  1.2  yamt 		p->p_estcpu = decay_cpu(loadfac, p->p_estcpu);
    294  1.2  yamt 
    295  1.2  yamt 		LIST_FOREACH(l, &p->p_lwps, l_sibling) {
    296  1.2  yamt 			if ((l->l_flag & LW_IDLE) != 0)
    297  1.2  yamt 				continue;
    298  1.2  yamt 			lwp_lock(l);
    299  1.2  yamt 			if (l->l_slptime <= 1 && l->l_priority >= PUSER)
    300  1.2  yamt 				resetpriority(l);
    301  1.2  yamt 			lwp_unlock(l);
    302  1.2  yamt 		}
    303  1.2  yamt 	}
    304  1.2  yamt }
    305  1.2  yamt 
    306  1.2  yamt /*
    307  1.2  yamt  * Recalculate the priority of a process after it has slept for a while.
    308  1.2  yamt  */
    309  1.2  yamt static void
    310  1.2  yamt updatepri(struct lwp *l)
    311  1.2  yamt {
    312  1.2  yamt 	struct proc *p = l->l_proc;
    313  1.2  yamt 	fixpt_t loadfac;
    314  1.2  yamt 
    315  1.3    ad 	KASSERT(lwp_locked(l, NULL));
    316  1.2  yamt 	KASSERT(l->l_slptime > 1);
    317  1.2  yamt 
    318  1.2  yamt 	loadfac = loadfactor(averunnable.ldavg[0]);
    319  1.2  yamt 
    320  1.2  yamt 	l->l_slptime--; /* the first time was done in sched_pstats */
    321  1.2  yamt 	/* XXX NJWLWP */
    322  1.2  yamt 	/* XXXSMP occasionally unlocked, should be per-LWP */
    323  1.2  yamt 	p->p_estcpu = decay_cpu_batch(loadfac, p->p_estcpu, l->l_slptime);
    324  1.2  yamt 	resetpriority(l);
    325  1.2  yamt }
    326  1.2  yamt 
    327  1.2  yamt /*
    328  1.2  yamt  * On some architectures, it's faster to use a MSB ordering for the priorites
    329  1.2  yamt  * than the traditional LSB ordering.
    330  1.2  yamt  */
    331  1.2  yamt #define	RQMASK(n) (0x00000001 << (n))
    332  1.2  yamt 
    333  1.2  yamt /*
    334  1.2  yamt  * The primitives that manipulate the run queues.  whichqs tells which
    335  1.2  yamt  * of the 32 queues qs have processes in them.  sched_enqueue() puts processes
    336  1.2  yamt  * into queues, sched_dequeue removes them from queues.  The running process is
    337  1.2  yamt  * on no queue, other processes are on a queue related to p->p_priority,
    338  1.2  yamt  * divided by 4 actually to shrink the 0-127 range of priorities into the 32
    339  1.2  yamt  * available queues.
    340  1.2  yamt  */
    341  1.2  yamt #ifdef RQDEBUG
    342  1.2  yamt static void
    343  1.2  yamt runqueue_check(const runqueue_t *rq, int whichq, struct lwp *l)
    344  1.2  yamt {
    345  1.2  yamt 	const subqueue_t * const sq = &rq->rq_subqueues[whichq];
    346  1.2  yamt 	const uint32_t bitmap = rq->rq_bitmap;
    347  1.2  yamt 	struct lwp *l2;
    348  1.2  yamt 	int found = 0;
    349  1.2  yamt 	int die = 0;
    350  1.2  yamt 	int empty = 1;
    351  1.2  yamt 
    352  1.2  yamt 	TAILQ_FOREACH(l2, &sq->sq_queue, l_runq) {
    353  1.2  yamt 		if (l2->l_stat != LSRUN) {
    354  1.2  yamt 			printf("runqueue_check[%d]: lwp %p state (%d) "
    355  1.2  yamt 			    " != LSRUN\n", whichq, l2, l2->l_stat);
    356  1.2  yamt 		}
    357  1.2  yamt 		if (l2 == l)
    358  1.2  yamt 			found = 1;
    359  1.2  yamt 		empty = 0;
    360  1.2  yamt 	}
    361  1.2  yamt 	if (empty && (bitmap & RQMASK(whichq)) != 0) {
    362  1.2  yamt 		printf("runqueue_check[%d]: bit set for empty run-queue %p\n",
    363  1.2  yamt 		    whichq, rq);
    364  1.2  yamt 		die = 1;
    365  1.2  yamt 	} else if (!empty && (bitmap & RQMASK(whichq)) == 0) {
    366  1.2  yamt 		printf("runqueue_check[%d]: bit clear for non-empty "
    367  1.2  yamt 		    "run-queue %p\n", whichq, rq);
    368  1.2  yamt 		die = 1;
    369  1.2  yamt 	}
    370  1.2  yamt 	if (l != NULL && (bitmap & RQMASK(whichq)) == 0) {
    371  1.2  yamt 		printf("runqueue_check[%d]: bit clear for active lwp %p\n",
    372  1.2  yamt 		    whichq, l);
    373  1.2  yamt 		die = 1;
    374  1.2  yamt 	}
    375  1.2  yamt 	if (l != NULL && empty) {
    376  1.2  yamt 		printf("runqueue_check[%d]: empty run-queue %p with "
    377  1.2  yamt 		    "active lwp %p\n", whichq, rq, l);
    378  1.2  yamt 		die = 1;
    379  1.2  yamt 	}
    380  1.2  yamt 	if (l != NULL && !found) {
    381  1.2  yamt 		printf("runqueue_check[%d]: lwp %p not in runqueue %p!",
    382  1.2  yamt 		    whichq, l, rq);
    383  1.2  yamt 		die = 1;
    384  1.2  yamt 	}
    385  1.2  yamt 	if (die)
    386  1.2  yamt 		panic("runqueue_check: inconsistency found");
    387  1.2  yamt }
    388  1.2  yamt #else /* RQDEBUG */
    389  1.2  yamt #define	runqueue_check(a, b, c)	/* nothing */
    390  1.2  yamt #endif /* RQDEBUG */
    391  1.2  yamt 
    392  1.2  yamt static void
    393  1.2  yamt runqueue_init(runqueue_t *rq)
    394  1.2  yamt {
    395  1.2  yamt 	int i;
    396  1.2  yamt 
    397  1.2  yamt 	for (i = 0; i < RUNQUE_NQS; i++)
    398  1.2  yamt 		TAILQ_INIT(&rq->rq_subqueues[i].sq_queue);
    399  1.2  yamt }
    400  1.2  yamt 
    401  1.2  yamt static void
    402  1.2  yamt runqueue_enqueue(runqueue_t *rq, struct lwp *l)
    403  1.2  yamt {
    404  1.2  yamt 	subqueue_t *sq;
    405  1.2  yamt 	const int whichq = lwp_eprio(l) / PPQ;
    406  1.2  yamt 
    407  1.2  yamt 	KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_mutex));
    408  1.2  yamt 
    409  1.2  yamt 	runqueue_check(rq, whichq, NULL);
    410  1.2  yamt 	rq->rq_bitmap |= RQMASK(whichq);
    411  1.2  yamt 	sq = &rq->rq_subqueues[whichq];
    412  1.2  yamt 	TAILQ_INSERT_TAIL(&sq->sq_queue, l, l_runq);
    413  1.2  yamt 	runqueue_check(rq, whichq, l);
    414  1.2  yamt }
    415  1.2  yamt 
    416  1.2  yamt static void
    417  1.2  yamt runqueue_dequeue(runqueue_t *rq, struct lwp *l)
    418  1.2  yamt {
    419  1.2  yamt 	subqueue_t *sq;
    420  1.2  yamt 	const int whichq = lwp_eprio(l) / PPQ;
    421  1.2  yamt 
    422  1.2  yamt 	KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_mutex));
    423  1.2  yamt 
    424  1.2  yamt 	runqueue_check(rq, whichq, l);
    425  1.2  yamt 	KASSERT((rq->rq_bitmap & RQMASK(whichq)) != 0);
    426  1.2  yamt 	sq = &rq->rq_subqueues[whichq];
    427  1.2  yamt 	TAILQ_REMOVE(&sq->sq_queue, l, l_runq);
    428  1.2  yamt 	if (TAILQ_EMPTY(&sq->sq_queue))
    429  1.2  yamt 		rq->rq_bitmap &= ~RQMASK(whichq);
    430  1.2  yamt 	runqueue_check(rq, whichq, NULL);
    431  1.2  yamt }
    432  1.2  yamt 
    433  1.2  yamt static struct lwp *
    434  1.2  yamt runqueue_nextlwp(runqueue_t *rq)
    435  1.2  yamt {
    436  1.2  yamt 	const uint32_t bitmap = rq->rq_bitmap;
    437  1.2  yamt 	int whichq;
    438  1.2  yamt 
    439  1.2  yamt 	if (bitmap == 0) {
    440  1.2  yamt 		return NULL;
    441  1.2  yamt 	}
    442  1.2  yamt 	whichq = ffs(bitmap) - 1;
    443  1.2  yamt 	return TAILQ_FIRST(&rq->rq_subqueues[whichq].sq_queue);
    444  1.2  yamt }
    445  1.2  yamt 
    446  1.2  yamt #if defined(DDB)
    447  1.2  yamt static void
    448  1.2  yamt runqueue_print(const runqueue_t *rq, void (*pr)(const char *, ...))
    449  1.2  yamt {
    450  1.2  yamt 	const uint32_t bitmap = rq->rq_bitmap;
    451  1.2  yamt 	struct lwp *l;
    452  1.2  yamt 	int i, first;
    453  1.2  yamt 
    454  1.2  yamt 	for (i = 0; i < RUNQUE_NQS; i++) {
    455  1.2  yamt 		const subqueue_t *sq;
    456  1.2  yamt 		first = 1;
    457  1.2  yamt 		sq = &rq->rq_subqueues[i];
    458  1.2  yamt 		TAILQ_FOREACH(l, &sq->sq_queue, l_runq) {
    459  1.2  yamt 			if (first) {
    460  1.2  yamt 				(*pr)("%c%d",
    461  1.2  yamt 				    (bitmap & RQMASK(i)) ? ' ' : '!', i);
    462  1.2  yamt 				first = 0;
    463  1.2  yamt 			}
    464  1.2  yamt 			(*pr)("\t%d.%d (%s) pri=%d usrpri=%d\n",
    465  1.2  yamt 			    l->l_proc->p_pid,
    466  1.2  yamt 			    l->l_lid, l->l_proc->p_comm,
    467  1.2  yamt 			    (int)l->l_priority, (int)l->l_usrpri);
    468  1.2  yamt 		}
    469  1.2  yamt 	}
    470  1.2  yamt }
    471  1.2  yamt #endif /* defined(DDB) */
    472  1.2  yamt #undef RQMASK
    473  1.2  yamt 
    474  1.2  yamt /*
    475  1.2  yamt  * Initialize the (doubly-linked) run queues
    476  1.2  yamt  * to be empty.
    477  1.2  yamt  */
    478  1.2  yamt void
    479  1.2  yamt sched_rqinit()
    480  1.2  yamt {
    481  1.2  yamt 
    482  1.2  yamt 	runqueue_init(&global_queue);
    483  1.2  yamt 	mutex_init(&sched_mutex, MUTEX_SPIN, IPL_SCHED);
    484  1.2  yamt 	/* Initialize the lock pointer for lwp0 */
    485  1.2  yamt 	lwp0.l_mutex = &curcpu()->ci_schedstate.spc_lwplock;
    486  1.2  yamt }
    487  1.2  yamt 
    488  1.2  yamt void
    489  1.2  yamt sched_cpuattach(struct cpu_info *ci)
    490  1.2  yamt {
    491  1.2  yamt 	runqueue_t *rq;
    492  1.2  yamt 
    493  1.2  yamt 	ci->ci_schedstate.spc_mutex = &sched_mutex;
    494  1.2  yamt 	rq = kmem_zalloc(sizeof(*rq), KM_NOSLEEP);
    495  1.2  yamt 	runqueue_init(rq);
    496  1.2  yamt 	ci->ci_schedstate.spc_sched_info = rq;
    497  1.2  yamt }
    498  1.2  yamt 
    499  1.2  yamt void
    500  1.2  yamt sched_setup()
    501  1.2  yamt {
    502  1.2  yamt 
    503  1.2  yamt 	rrticks = hz / 10;
    504  1.2  yamt }
    505  1.2  yamt 
    506  1.2  yamt void
    507  1.2  yamt sched_setrunnable(struct lwp *l)
    508  1.2  yamt {
    509  1.2  yamt 
    510  1.2  yamt  	if (l->l_slptime > 1)
    511  1.2  yamt  		updatepri(l);
    512  1.2  yamt }
    513  1.2  yamt 
    514  1.2  yamt bool
    515  1.2  yamt sched_curcpu_runnable_p(void)
    516  1.2  yamt {
    517  1.2  yamt 	runqueue_t *rq = curcpu()->ci_schedstate.spc_sched_info;
    518  1.2  yamt 
    519  1.2  yamt 	return (global_queue.rq_bitmap | rq->rq_bitmap) != 0;
    520  1.2  yamt }
    521  1.2  yamt 
    522  1.2  yamt void
    523  1.2  yamt sched_nice(struct proc *chgp, int n)
    524  1.2  yamt {
    525  1.2  yamt 
    526  1.2  yamt 	chgp->p_nice = n;
    527  1.2  yamt 	(void)resetprocpriority(chgp);
    528  1.2  yamt }
    529  1.2  yamt 
    530  1.2  yamt /*
    531  1.2  yamt  * Compute the priority of a process when running in user mode.
    532  1.2  yamt  * Arrange to reschedule if the resulting priority is better
    533  1.2  yamt  * than that of the current process.
    534  1.2  yamt  */
    535  1.2  yamt static void
    536  1.2  yamt resetpriority(struct lwp *l)
    537  1.2  yamt {
    538  1.2  yamt 	unsigned int newpriority;
    539  1.2  yamt 	struct proc *p = l->l_proc;
    540  1.2  yamt 
    541  1.2  yamt 	/* XXXSMP LOCK_ASSERT(mutex_owned(&p->p_stmutex)); */
    542  1.2  yamt 	LOCK_ASSERT(lwp_locked(l, NULL));
    543  1.2  yamt 
    544  1.2  yamt 	if ((l->l_flag & LW_SYSTEM) != 0)
    545  1.2  yamt 		return;
    546  1.2  yamt 
    547  1.2  yamt 	newpriority = PUSER + (p->p_estcpu >> ESTCPU_SHIFT) +
    548  1.2  yamt 	    NICE_WEIGHT * (p->p_nice - NZERO);
    549  1.2  yamt 	newpriority = min(newpriority, MAXPRI);
    550  1.2  yamt 	lwp_changepri(l, newpriority);
    551  1.2  yamt }
    552  1.2  yamt 
    553  1.2  yamt /*
    554  1.2  yamt  * Recompute priority for all LWPs in a process.
    555  1.2  yamt  */
    556  1.2  yamt static void
    557  1.2  yamt resetprocpriority(struct proc *p)
    558  1.2  yamt {
    559  1.2  yamt 	struct lwp *l;
    560  1.2  yamt 
    561  1.3    ad 	KASSERT(mutex_owned(&p->p_stmutex));
    562  1.2  yamt 
    563  1.2  yamt 	LIST_FOREACH(l, &p->p_lwps, l_sibling) {
    564  1.2  yamt 		lwp_lock(l);
    565  1.2  yamt 		resetpriority(l);
    566  1.2  yamt 		lwp_unlock(l);
    567  1.2  yamt 	}
    568  1.2  yamt }
    569  1.2  yamt 
    570  1.2  yamt /*
    571  1.2  yamt  * We adjust the priority of the current process.  The priority of a process
    572  1.2  yamt  * gets worse as it accumulates CPU time.  The CPU usage estimator (p_estcpu)
    573  1.2  yamt  * is increased here.  The formula for computing priorities (in kern_synch.c)
    574  1.2  yamt  * will compute a different value each time p_estcpu increases. This can
    575  1.2  yamt  * cause a switch, but unless the priority crosses a PPQ boundary the actual
    576  1.2  yamt  * queue will not change.  The CPU usage estimator ramps up quite quickly
    577  1.2  yamt  * when the process is running (linearly), and decays away exponentially, at
    578  1.2  yamt  * a rate which is proportionally slower when the system is busy.  The basic
    579  1.2  yamt  * principle is that the system will 90% forget that the process used a lot
    580  1.2  yamt  * of CPU time in 5 * loadav seconds.  This causes the system to favor
    581  1.2  yamt  * processes which haven't run much recently, and to round-robin among other
    582  1.2  yamt  * processes.
    583  1.2  yamt  */
    584  1.2  yamt 
    585  1.2  yamt void
    586  1.2  yamt sched_schedclock(struct lwp *l)
    587  1.2  yamt {
    588  1.2  yamt 	struct proc *p = l->l_proc;
    589  1.2  yamt 
    590  1.2  yamt 	KASSERT(!CURCPU_IDLE_P());
    591  1.2  yamt 	mutex_spin_enter(&p->p_stmutex);
    592  1.2  yamt 	p->p_estcpu = ESTCPULIM(p->p_estcpu + (1 << ESTCPU_SHIFT));
    593  1.2  yamt 	lwp_lock(l);
    594  1.2  yamt 	resetpriority(l);
    595  1.2  yamt 	mutex_spin_exit(&p->p_stmutex);
    596  1.2  yamt 	if ((l->l_flag & LW_SYSTEM) == 0 && l->l_priority >= PUSER)
    597  1.2  yamt 		l->l_priority = l->l_usrpri;
    598  1.2  yamt 	lwp_unlock(l);
    599  1.2  yamt }
    600  1.2  yamt 
    601  1.2  yamt /*
    602  1.2  yamt  * sched_proc_fork:
    603  1.2  yamt  *
    604  1.2  yamt  *	Inherit the parent's scheduler history.
    605  1.2  yamt  */
    606  1.2  yamt void
    607  1.2  yamt sched_proc_fork(struct proc *parent, struct proc *child)
    608  1.2  yamt {
    609  1.2  yamt 
    610  1.3    ad 	KASSERT(mutex_owned(&parent->p_smutex));
    611  1.2  yamt 
    612  1.2  yamt 	child->p_estcpu = child->p_estcpu_inherited = parent->p_estcpu;
    613  1.2  yamt 	child->p_forktime = sched_pstats_ticks;
    614  1.2  yamt }
    615  1.2  yamt 
    616  1.2  yamt /*
    617  1.2  yamt  * sched_proc_exit:
    618  1.2  yamt  *
    619  1.2  yamt  *	Chargeback parents for the sins of their children.
    620  1.2  yamt  */
    621  1.2  yamt void
    622  1.2  yamt sched_proc_exit(struct proc *parent, struct proc *child)
    623  1.2  yamt {
    624  1.2  yamt 	fixpt_t loadfac = loadfactor(averunnable.ldavg[0]);
    625  1.2  yamt 	fixpt_t estcpu;
    626  1.2  yamt 
    627  1.2  yamt 	/* XXX Only if parent != init?? */
    628  1.2  yamt 
    629  1.2  yamt 	mutex_spin_enter(&parent->p_stmutex);
    630  1.2  yamt 	estcpu = decay_cpu_batch(loadfac, child->p_estcpu_inherited,
    631  1.2  yamt 	    sched_pstats_ticks - child->p_forktime);
    632  1.2  yamt 	if (child->p_estcpu > estcpu)
    633  1.2  yamt 		parent->p_estcpu =
    634  1.2  yamt 		    ESTCPULIM(parent->p_estcpu + child->p_estcpu - estcpu);
    635  1.2  yamt 	mutex_spin_exit(&parent->p_stmutex);
    636  1.2  yamt }
    637  1.2  yamt 
    638  1.2  yamt void
    639  1.2  yamt sched_enqueue(struct lwp *l, bool ctxswitch)
    640  1.2  yamt {
    641  1.2  yamt 
    642  1.2  yamt 	if ((l->l_flag & LW_BOUND) != 0)
    643  1.2  yamt 		runqueue_enqueue(l->l_cpu->ci_schedstate.spc_sched_info, l);
    644  1.2  yamt 	else
    645  1.2  yamt 		runqueue_enqueue(&global_queue, l);
    646  1.2  yamt }
    647  1.2  yamt 
    648  1.2  yamt /*
    649  1.2  yamt  * XXXSMP When LWP dispatch (cpu_switch()) is changed to use sched_dequeue(),
    650  1.2  yamt  * drop of the effective priority level from kernel to user needs to be
    651  1.2  yamt  * moved here from userret().  The assignment in userret() is currently
    652  1.2  yamt  * done unlocked.
    653  1.2  yamt  */
    654  1.2  yamt void
    655  1.2  yamt sched_dequeue(struct lwp *l)
    656  1.2  yamt {
    657  1.2  yamt 
    658  1.2  yamt 	if ((l->l_flag & LW_BOUND) != 0)
    659  1.2  yamt 		runqueue_dequeue(l->l_cpu->ci_schedstate.spc_sched_info, l);
    660  1.2  yamt 	else
    661  1.2  yamt 		runqueue_dequeue(&global_queue, l);
    662  1.2  yamt }
    663  1.2  yamt 
    664  1.2  yamt struct lwp *
    665  1.2  yamt sched_nextlwp(void)
    666  1.2  yamt {
    667  1.2  yamt 	lwp_t *l1, *l2;
    668  1.2  yamt 
    669  1.2  yamt 	/* For now, just pick the highest priority LWP. */
    670  1.2  yamt 	l1 = runqueue_nextlwp(curcpu()->ci_schedstate.spc_sched_info);
    671  1.2  yamt 	l2 = runqueue_nextlwp(&global_queue);
    672  1.2  yamt 
    673  1.2  yamt 	if (l1 == NULL)
    674  1.2  yamt 		return l2;
    675  1.2  yamt 	if (l2 == NULL)
    676  1.2  yamt 		return l1;
    677  1.2  yamt 	if (lwp_eprio(l2) < lwp_eprio(l1))
    678  1.2  yamt 		return l2;
    679  1.2  yamt 	else
    680  1.2  yamt 		return l1;
    681  1.2  yamt }
    682  1.2  yamt 
    683  1.2  yamt /* Dummy */
    684  1.2  yamt void
    685  1.2  yamt sched_lwp_fork(struct lwp *l)
    686  1.2  yamt {
    687  1.2  yamt 
    688  1.2  yamt }
    689  1.2  yamt 
    690  1.2  yamt void
    691  1.2  yamt sched_lwp_exit(struct lwp *l)
    692  1.2  yamt {
    693  1.2  yamt 
    694  1.2  yamt }
    695  1.2  yamt 
    696  1.2  yamt /* SysCtl */
    697  1.2  yamt 
    698  1.2  yamt SYSCTL_SETUP(sysctl_sched_setup, "sysctl kern.sched subtree setup")
    699  1.2  yamt {
    700  1.2  yamt 	const struct sysctlnode *node = NULL;
    701  1.2  yamt 
    702  1.2  yamt 	sysctl_createv(clog, 0, NULL, NULL,
    703  1.2  yamt 		CTLFLAG_PERMANENT,
    704  1.2  yamt 		CTLTYPE_NODE, "kern", NULL,
    705  1.2  yamt 		NULL, 0, NULL, 0,
    706  1.2  yamt 		CTL_KERN, CTL_EOL);
    707  1.2  yamt 	sysctl_createv(clog, 0, NULL, &node,
    708  1.2  yamt 		CTLFLAG_PERMANENT,
    709  1.2  yamt 		CTLTYPE_NODE, "sched",
    710  1.2  yamt 		SYSCTL_DESCR("Scheduler options"),
    711  1.2  yamt 		NULL, 0, NULL, 0,
    712  1.2  yamt 		CTL_KERN, CTL_CREATE, CTL_EOL);
    713  1.2  yamt 
    714  1.2  yamt 	if (node != NULL) {
    715  1.2  yamt 		sysctl_createv(clog, 0, &node, NULL,
    716  1.2  yamt 			CTLFLAG_PERMANENT,
    717  1.2  yamt 			CTLTYPE_STRING, "name", NULL,
    718  1.2  yamt 			NULL, 0, __UNCONST("4.4BSD"), 0,
    719  1.2  yamt 			CTL_CREATE, CTL_EOL);
    720  1.2  yamt 	}
    721  1.2  yamt }
    722  1.2  yamt 
    723  1.2  yamt #if defined(DDB)
    724  1.2  yamt void
    725  1.2  yamt sched_print_runqueue(void (*pr)(const char *, ...))
    726  1.2  yamt {
    727  1.2  yamt 
    728  1.2  yamt 	runqueue_print(&global_queue, pr);
    729  1.2  yamt }
    730  1.2  yamt #endif /* defined(DDB) */
    731