Home | History | Annotate | Line # | Download | only in kern
kern_sleepq.c revision 1.12.2.1
      1  1.12.2.1   matt /*	$NetBSD: kern_sleepq.c,v 1.12.2.1 2007/11/06 23:31:57 matt Exp $	*/
      2       1.2     ad 
      3       1.2     ad /*-
      4       1.2     ad  * Copyright (c) 2006, 2007 The NetBSD Foundation, Inc.
      5       1.2     ad  * All rights reserved.
      6       1.2     ad  *
      7       1.2     ad  * This code is derived from software contributed to The NetBSD Foundation
      8       1.2     ad  * by Andrew Doran.
      9       1.2     ad  *
     10       1.2     ad  * Redistribution and use in source and binary forms, with or without
     11       1.2     ad  * modification, are permitted provided that the following conditions
     12       1.2     ad  * are met:
     13       1.2     ad  * 1. Redistributions of source code must retain the above copyright
     14       1.2     ad  *    notice, this list of conditions and the following disclaimer.
     15       1.2     ad  * 2. Redistributions in binary form must reproduce the above copyright
     16       1.2     ad  *    notice, this list of conditions and the following disclaimer in the
     17       1.2     ad  *    documentation and/or other materials provided with the distribution.
     18       1.2     ad  * 3. All advertising materials mentioning features or use of this software
     19       1.2     ad  *    must display the following acknowledgement:
     20       1.2     ad  *	This product includes software developed by the NetBSD
     21       1.2     ad  *	Foundation, Inc. and its contributors.
     22       1.2     ad  * 4. Neither the name of The NetBSD Foundation nor the names of its
     23       1.2     ad  *    contributors may be used to endorse or promote products derived
     24       1.2     ad  *    from this software without specific prior written permission.
     25       1.2     ad  *
     26       1.2     ad  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     27       1.2     ad  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     28       1.2     ad  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     29       1.2     ad  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     30       1.2     ad  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     31       1.2     ad  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     32       1.2     ad  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     33       1.2     ad  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     34       1.2     ad  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     35       1.2     ad  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     36       1.2     ad  * POSSIBILITY OF SUCH DAMAGE.
     37       1.2     ad  */
     38       1.2     ad 
     39       1.2     ad /*
     40       1.2     ad  * Sleep queue implementation, used by turnstiles and general sleep/wakeup
     41       1.2     ad  * interfaces.
     42       1.2     ad  */
     43       1.2     ad 
     44       1.2     ad #include <sys/cdefs.h>
     45  1.12.2.1   matt __KERNEL_RCSID(0, "$NetBSD: kern_sleepq.c,v 1.12.2.1 2007/11/06 23:31:57 matt Exp $");
     46       1.2     ad 
     47       1.2     ad #include <sys/param.h>
     48       1.2     ad #include <sys/lock.h>
     49       1.2     ad #include <sys/kernel.h>
     50       1.9   yamt #include <sys/cpu.h>
     51       1.2     ad #include <sys/pool.h>
     52       1.2     ad #include <sys/proc.h>
     53       1.2     ad #include <sys/resourcevar.h>
     54       1.2     ad #include <sys/sched.h>
     55       1.2     ad #include <sys/systm.h>
     56       1.2     ad #include <sys/sleepq.h>
     57       1.2     ad #include <sys/ktrace.h>
     58       1.2     ad 
     59       1.4     ad #include <uvm/uvm_extern.h>
     60       1.4     ad 
     61       1.8     ad int	sleepq_sigtoerror(lwp_t *, int);
     62       1.2     ad 
     63       1.2     ad /* General purpose sleep table, used by ltsleep() and condition variables. */
     64       1.2     ad sleeptab_t	sleeptab;
     65       1.2     ad 
     66       1.2     ad /*
     67       1.2     ad  * sleeptab_init:
     68       1.2     ad  *
     69       1.2     ad  *	Initialize a sleep table.
     70       1.2     ad  */
     71       1.2     ad void
     72       1.2     ad sleeptab_init(sleeptab_t *st)
     73       1.2     ad {
     74       1.2     ad 	sleepq_t *sq;
     75       1.2     ad 	int i;
     76       1.2     ad 
     77       1.2     ad 	for (i = 0; i < SLEEPTAB_HASH_SIZE; i++) {
     78       1.2     ad 		sq = &st->st_queues[i].st_queue;
     79       1.2     ad 		mutex_init(&st->st_queues[i].st_mutex, MUTEX_SPIN, IPL_SCHED);
     80       1.2     ad 		sleepq_init(sq, &st->st_queues[i].st_mutex);
     81       1.2     ad 	}
     82       1.2     ad }
     83       1.2     ad 
     84       1.2     ad /*
     85       1.2     ad  * sleepq_init:
     86       1.2     ad  *
     87       1.2     ad  *	Prepare a sleep queue for use.
     88       1.2     ad  */
     89       1.2     ad void
     90       1.2     ad sleepq_init(sleepq_t *sq, kmutex_t *mtx)
     91       1.2     ad {
     92       1.2     ad 
     93       1.2     ad 	sq->sq_waiters = 0;
     94       1.2     ad 	sq->sq_mutex = mtx;
     95       1.2     ad 	TAILQ_INIT(&sq->sq_queue);
     96       1.2     ad }
     97       1.2     ad 
     98       1.2     ad /*
     99       1.2     ad  * sleepq_remove:
    100       1.2     ad  *
    101       1.2     ad  *	Remove an LWP from a sleep queue and wake it up.  Return non-zero if
    102       1.2     ad  *	the LWP is swapped out; if so the caller needs to awaken the swapper
    103       1.2     ad  *	to bring the LWP into memory.
    104       1.2     ad  */
    105       1.2     ad int
    106       1.8     ad sleepq_remove(sleepq_t *sq, lwp_t *l)
    107       1.2     ad {
    108       1.9   yamt 	struct schedstate_percpu *spc;
    109       1.2     ad 	struct cpu_info *ci;
    110  1.12.2.1   matt 	pri_t pri;
    111       1.2     ad 
    112       1.4     ad 	KASSERT(lwp_locked(l, sq->sq_mutex));
    113       1.2     ad 	KASSERT(sq->sq_waiters > 0);
    114       1.2     ad 
    115       1.2     ad 	sq->sq_waiters--;
    116       1.2     ad 	TAILQ_REMOVE(&sq->sq_queue, l, l_sleepchain);
    117       1.2     ad 
    118       1.2     ad #ifdef DIAGNOSTIC
    119       1.2     ad 	if (sq->sq_waiters == 0)
    120       1.2     ad 		KASSERT(TAILQ_FIRST(&sq->sq_queue) == NULL);
    121       1.2     ad 	else
    122       1.2     ad 		KASSERT(TAILQ_FIRST(&sq->sq_queue) != NULL);
    123       1.2     ad #endif
    124       1.2     ad 
    125       1.2     ad 	l->l_syncobj = &sched_syncobj;
    126       1.2     ad 	l->l_wchan = NULL;
    127       1.2     ad 	l->l_sleepq = NULL;
    128       1.5  pavel 	l->l_flag &= ~LW_SINTR;
    129       1.2     ad 
    130       1.9   yamt 	ci = l->l_cpu;
    131       1.9   yamt 	spc = &ci->ci_schedstate;
    132       1.9   yamt 
    133       1.2     ad 	/*
    134       1.2     ad 	 * If not sleeping, the LWP must have been suspended.  Let whoever
    135       1.2     ad 	 * holds it stopped set it running again.
    136       1.2     ad 	 */
    137       1.2     ad 	if (l->l_stat != LSSLEEP) {
    138  1.12.2.1   matt 		KASSERT(l->l_stat == LSSTOP || l->l_stat == LSSUSPENDED);
    139       1.9   yamt 		lwp_setlock(l, &spc->spc_lwplock);
    140       1.2     ad 		return 0;
    141       1.2     ad 	}
    142       1.2     ad 
    143       1.2     ad 	/*
    144       1.2     ad 	 * If the LWP is still on the CPU, mark it as LSONPROC.  It may be
    145       1.2     ad 	 * about to call mi_switch(), in which case it will yield.
    146       1.2     ad 	 */
    147       1.9   yamt 	if ((l->l_flag & LW_RUNNING) != 0) {
    148       1.2     ad 		l->l_stat = LSONPROC;
    149       1.2     ad 		l->l_slptime = 0;
    150       1.9   yamt 		lwp_setlock(l, &spc->spc_lwplock);
    151       1.2     ad 		return 0;
    152       1.2     ad 	}
    153       1.2     ad 
    154       1.2     ad 	/*
    155  1.12.2.1   matt 	 * Call the wake-up handler of scheduler.
    156  1.12.2.1   matt 	 * It might change the CPU for this thread.
    157  1.12.2.1   matt 	 */
    158  1.12.2.1   matt 	sched_wakeup(l);
    159  1.12.2.1   matt 	ci = l->l_cpu;
    160  1.12.2.1   matt 	spc = &ci->ci_schedstate;
    161  1.12.2.1   matt 
    162  1.12.2.1   matt 	/*
    163  1.12.2.1   matt 	 * Set it running.
    164       1.2     ad 	 */
    165       1.9   yamt 	spc_lock(ci);
    166       1.9   yamt 	lwp_setlock(l, spc->spc_mutex);
    167       1.9   yamt 	sched_setrunnable(l);
    168       1.2     ad 	l->l_stat = LSRUN;
    169       1.2     ad 	l->l_slptime = 0;
    170       1.5  pavel 	if ((l->l_flag & LW_INMEM) != 0) {
    171       1.9   yamt 		sched_enqueue(l, false);
    172  1.12.2.1   matt 		pri = lwp_eprio(l);
    173  1.12.2.1   matt 		/* XXX This test is not good enough! */
    174  1.12.2.1   matt 		if (pri > spc->spc_curpriority) {
    175  1.12.2.1   matt 			cpu_need_resched(ci,
    176  1.12.2.1   matt 			    (pri >= PRI_KERNEL ? RESCHED_IMMED : 0));
    177  1.12.2.1   matt 		}
    178       1.9   yamt 		spc_unlock(ci);
    179       1.2     ad 		return 0;
    180       1.2     ad 	}
    181       1.9   yamt 	spc_unlock(ci);
    182       1.2     ad 	return 1;
    183       1.2     ad }
    184       1.2     ad 
    185       1.2     ad /*
    186       1.2     ad  * sleepq_insert:
    187       1.2     ad  *
    188       1.2     ad  *	Insert an LWP into the sleep queue, optionally sorting by priority.
    189       1.2     ad  */
    190       1.2     ad inline void
    191       1.8     ad sleepq_insert(sleepq_t *sq, lwp_t *l, syncobj_t *sobj)
    192       1.2     ad {
    193       1.8     ad 	lwp_t *l2;
    194       1.6   yamt 	const int pri = lwp_eprio(l);
    195       1.2     ad 
    196       1.2     ad 	if ((sobj->sobj_flag & SOBJ_SLEEPQ_SORTED) != 0) {
    197       1.2     ad 		TAILQ_FOREACH(l2, &sq->sq_queue, l_sleepchain) {
    198  1.12.2.1   matt 			if (lwp_eprio(l2) < pri) {
    199       1.2     ad 				TAILQ_INSERT_BEFORE(l2, l, l_sleepchain);
    200       1.2     ad 				return;
    201       1.2     ad 			}
    202       1.2     ad 		}
    203       1.2     ad 	}
    204       1.2     ad 
    205  1.12.2.1   matt 	if ((sobj->sobj_flag & SOBJ_SLEEPQ_LIFO) != 0)
    206  1.12.2.1   matt 		TAILQ_INSERT_HEAD(&sq->sq_queue, l, l_sleepchain);
    207  1.12.2.1   matt 	else
    208  1.12.2.1   matt 		TAILQ_INSERT_TAIL(&sq->sq_queue, l, l_sleepchain);
    209       1.2     ad }
    210       1.2     ad 
    211       1.9   yamt /*
    212       1.9   yamt  * sleepq_enqueue:
    213       1.9   yamt  *
    214       1.9   yamt  *	Enter an LWP into the sleep queue and prepare for sleep.  The sleep
    215       1.9   yamt  *	queue must already be locked, and any interlock (such as the kernel
    216       1.9   yamt  *	lock) must have be released (see sleeptab_lookup(), sleepq_enter()).
    217       1.9   yamt  */
    218       1.2     ad void
    219  1.12.2.1   matt sleepq_enqueue(sleepq_t *sq, wchan_t wchan, const char *wmesg, syncobj_t *sobj)
    220       1.2     ad {
    221       1.8     ad 	lwp_t *l = curlwp;
    222       1.2     ad 
    223       1.4     ad 	KASSERT(mutex_owned(sq->sq_mutex));
    224       1.2     ad 	KASSERT(l->l_stat == LSONPROC);
    225       1.2     ad 	KASSERT(l->l_wchan == NULL && l->l_sleepq == NULL);
    226       1.2     ad 
    227       1.2     ad 	l->l_syncobj = sobj;
    228       1.2     ad 	l->l_wchan = wchan;
    229       1.2     ad 	l->l_sleepq = sq;
    230       1.2     ad 	l->l_wmesg = wmesg;
    231       1.2     ad 	l->l_slptime = 0;
    232       1.2     ad 	l->l_stat = LSSLEEP;
    233       1.2     ad 	l->l_sleeperr = 0;
    234       1.2     ad 
    235       1.2     ad 	sq->sq_waiters++;
    236       1.6   yamt 	sleepq_insert(sq, l, sobj);
    237  1.12.2.1   matt 	sched_slept(l);
    238       1.6   yamt }
    239       1.6   yamt 
    240       1.9   yamt /*
    241       1.9   yamt  * sleepq_block:
    242       1.9   yamt  *
    243       1.9   yamt  *	After any intermediate step such as releasing an interlock, switch.
    244       1.9   yamt  * 	sleepq_block() may return early under exceptional conditions, for
    245       1.9   yamt  * 	example if the LWP's containing process is exiting.
    246       1.9   yamt  */
    247       1.9   yamt int
    248       1.9   yamt sleepq_block(int timo, bool catch)
    249       1.6   yamt {
    250      1.10     ad 	int error = 0, sig;
    251       1.9   yamt 	struct proc *p;
    252       1.8     ad 	lwp_t *l = curlwp;
    253      1.11     ad 	bool early = false;
    254       1.2     ad 
    255      1.12     ad 	ktrcsw(1, 0);
    256       1.4     ad 
    257       1.2     ad 	/*
    258       1.2     ad 	 * If sleeping interruptably, check for pending signals, exits or
    259       1.2     ad 	 * core dump events.
    260       1.2     ad 	 */
    261       1.2     ad 	if (catch) {
    262       1.5  pavel 		l->l_flag |= LW_SINTR;
    263       1.5  pavel 		if ((l->l_flag & (LW_CANCELLED|LW_WEXIT|LW_WCORE)) != 0) {
    264       1.5  pavel 			l->l_flag &= ~LW_CANCELLED;
    265  1.12.2.1   matt 			error = EINTR;
    266  1.12.2.1   matt 			early = true;
    267  1.12.2.1   matt 		} else if ((l->l_flag & LW_PENDSIG) != 0 && sigispending(l, 0))
    268      1.11     ad 			early = true;
    269       1.2     ad 	}
    270       1.2     ad 
    271  1.12.2.1   matt 	if (early) {
    272  1.12.2.1   matt 		/* lwp_unsleep() will release the lock */
    273  1.12.2.1   matt 		lwp_unsleep(l);
    274  1.12.2.1   matt 	} else {
    275      1.11     ad 		if (timo)
    276  1.12.2.1   matt 			callout_schedule(&l->l_timeout_ch, timo);
    277      1.11     ad 		mi_switch(l);
    278      1.11     ad 
    279      1.11     ad 		/* The LWP and sleep queue are now unlocked. */
    280      1.11     ad 		if (timo) {
    281      1.11     ad 			/*
    282      1.11     ad 			 * Even if the callout appears to have fired, we need to
    283      1.11     ad 			 * stop it in order to synchronise with other CPUs.
    284      1.11     ad 			 */
    285  1.12.2.1   matt 			if (callout_stop(&l->l_timeout_ch))
    286      1.11     ad 				error = EWOULDBLOCK;
    287      1.11     ad 		}
    288       1.2     ad 	}
    289       1.2     ad 
    290       1.9   yamt 	if (catch && error == 0) {
    291       1.2     ad 		p = l->l_proc;
    292       1.5  pavel 		if ((l->l_flag & (LW_CANCELLED | LW_WEXIT | LW_WCORE)) != 0)
    293       1.2     ad 			error = EINTR;
    294       1.5  pavel 		else if ((l->l_flag & LW_PENDSIG) != 0) {
    295       1.2     ad 			mutex_enter(&p->p_smutex);
    296       1.2     ad 			if ((sig = issignal(l)) != 0)
    297       1.2     ad 				error = sleepq_sigtoerror(l, sig);
    298       1.2     ad 			mutex_exit(&p->p_smutex);
    299       1.2     ad 		}
    300       1.2     ad 	}
    301       1.2     ad 
    302      1.12     ad 	ktrcsw(0, 0);
    303       1.2     ad 
    304       1.2     ad 	KERNEL_LOCK(l->l_biglocks, l);
    305       1.2     ad 	return error;
    306       1.2     ad }
    307       1.2     ad 
    308       1.2     ad /*
    309       1.2     ad  * sleepq_wake:
    310       1.2     ad  *
    311       1.2     ad  *	Wake zero or more LWPs blocked on a single wait channel.
    312       1.2     ad  */
    313       1.8     ad lwp_t *
    314       1.2     ad sleepq_wake(sleepq_t *sq, wchan_t wchan, u_int expected)
    315       1.2     ad {
    316       1.8     ad 	lwp_t *l, *next;
    317       1.2     ad 	int swapin = 0;
    318       1.2     ad 
    319       1.4     ad 	KASSERT(mutex_owned(sq->sq_mutex));
    320       1.2     ad 
    321       1.2     ad 	for (l = TAILQ_FIRST(&sq->sq_queue); l != NULL; l = next) {
    322       1.2     ad 		KASSERT(l->l_sleepq == sq);
    323       1.2     ad 		next = TAILQ_NEXT(l, l_sleepchain);
    324       1.2     ad 		if (l->l_wchan != wchan)
    325       1.2     ad 			continue;
    326       1.2     ad 		swapin |= sleepq_remove(sq, l);
    327       1.2     ad 		if (--expected == 0)
    328       1.2     ad 			break;
    329       1.2     ad 	}
    330       1.2     ad 
    331       1.2     ad 	sleepq_unlock(sq);
    332       1.2     ad 
    333       1.2     ad 	/*
    334       1.2     ad 	 * If there are newly awakend threads that need to be swapped in,
    335       1.2     ad 	 * then kick the swapper into action.
    336       1.2     ad 	 */
    337       1.2     ad 	if (swapin)
    338       1.4     ad 		uvm_kick_scheduler();
    339       1.8     ad 
    340       1.8     ad 	return l;
    341       1.2     ad }
    342       1.2     ad 
    343       1.2     ad /*
    344       1.2     ad  * sleepq_unsleep:
    345       1.2     ad  *
    346       1.2     ad  *	Remove an LWP from its sleep queue and set it runnable again.
    347       1.2     ad  *	sleepq_unsleep() is called with the LWP's mutex held, and will
    348       1.2     ad  *	always release it.
    349       1.2     ad  */
    350       1.2     ad void
    351       1.8     ad sleepq_unsleep(lwp_t *l)
    352       1.2     ad {
    353       1.2     ad 	sleepq_t *sq = l->l_sleepq;
    354       1.2     ad 	int swapin;
    355       1.2     ad 
    356       1.4     ad 	KASSERT(lwp_locked(l, NULL));
    357       1.2     ad 	KASSERT(l->l_wchan != NULL);
    358       1.2     ad 	KASSERT(l->l_mutex == sq->sq_mutex);
    359       1.2     ad 
    360       1.2     ad 	swapin = sleepq_remove(sq, l);
    361       1.2     ad 	sleepq_unlock(sq);
    362       1.2     ad 
    363       1.2     ad 	if (swapin)
    364       1.4     ad 		uvm_kick_scheduler();
    365       1.2     ad }
    366       1.2     ad 
    367       1.2     ad /*
    368       1.2     ad  * sleepq_timeout:
    369       1.2     ad  *
    370       1.2     ad  *	Entered via the callout(9) subsystem to time out an LWP that is on a
    371       1.2     ad  *	sleep queue.
    372       1.2     ad  */
    373       1.2     ad void
    374       1.2     ad sleepq_timeout(void *arg)
    375       1.2     ad {
    376       1.8     ad 	lwp_t *l = arg;
    377       1.2     ad 
    378       1.2     ad 	/*
    379       1.2     ad 	 * Lock the LWP.  Assuming it's still on the sleep queue, its
    380       1.2     ad 	 * current mutex will also be the sleep queue mutex.
    381       1.2     ad 	 */
    382       1.2     ad 	lwp_lock(l);
    383       1.2     ad 
    384       1.2     ad 	if (l->l_wchan == NULL) {
    385       1.2     ad 		/* Somebody beat us to it. */
    386       1.2     ad 		lwp_unlock(l);
    387       1.2     ad 		return;
    388       1.2     ad 	}
    389       1.2     ad 
    390       1.2     ad 	lwp_unsleep(l);
    391       1.2     ad }
    392       1.2     ad 
    393       1.2     ad /*
    394       1.2     ad  * sleepq_sigtoerror:
    395       1.2     ad  *
    396       1.2     ad  *	Given a signal number, interpret and return an error code.
    397       1.2     ad  */
    398       1.2     ad int
    399       1.8     ad sleepq_sigtoerror(lwp_t *l, int sig)
    400       1.2     ad {
    401       1.2     ad 	struct proc *p = l->l_proc;
    402       1.2     ad 	int error;
    403       1.2     ad 
    404       1.4     ad 	KASSERT(mutex_owned(&p->p_smutex));
    405       1.2     ad 
    406       1.2     ad 	/*
    407       1.2     ad 	 * If this sleep was canceled, don't let the syscall restart.
    408       1.2     ad 	 */
    409       1.2     ad 	if ((SIGACTION(p, sig).sa_flags & SA_RESTART) == 0)
    410       1.2     ad 		error = EINTR;
    411       1.2     ad 	else
    412       1.2     ad 		error = ERESTART;
    413       1.2     ad 
    414       1.2     ad 	return error;
    415       1.2     ad }
    416       1.2     ad 
    417       1.2     ad /*
    418       1.2     ad  * sleepq_abort:
    419       1.2     ad  *
    420       1.2     ad  *	After a panic or during autoconfiguration, lower the interrupt
    421       1.2     ad  *	priority level to give pending interrupts a chance to run, and
    422       1.2     ad  *	then return.  Called if sleepq_dontsleep() returns non-zero, and
    423       1.2     ad  *	always returns zero.
    424       1.2     ad  */
    425       1.2     ad int
    426       1.2     ad sleepq_abort(kmutex_t *mtx, int unlock)
    427       1.2     ad {
    428       1.2     ad 	extern int safepri;
    429       1.2     ad 	int s;
    430       1.2     ad 
    431       1.2     ad 	s = splhigh();
    432       1.2     ad 	splx(safepri);
    433       1.2     ad 	splx(s);
    434       1.2     ad 	if (mtx != NULL && unlock != 0)
    435       1.2     ad 		mutex_exit(mtx);
    436       1.2     ad 
    437       1.2     ad 	return 0;
    438       1.2     ad }
    439       1.2     ad 
    440       1.2     ad /*
    441       1.2     ad  * sleepq_changepri:
    442       1.2     ad  *
    443       1.2     ad  *	Adjust the priority of an LWP residing on a sleepq.  This method
    444       1.2     ad  *	will only alter the user priority; the effective priority is
    445       1.2     ad  *	assumed to have been fixed at the time of insertion into the queue.
    446       1.2     ad  */
    447       1.2     ad void
    448       1.8     ad sleepq_changepri(lwp_t *l, pri_t pri)
    449       1.2     ad {
    450  1.12.2.1   matt 	sleepq_t *sq = l->l_sleepq;
    451  1.12.2.1   matt 	pri_t opri;
    452  1.12.2.1   matt 
    453  1.12.2.1   matt 	KASSERT(lwp_locked(l, sq->sq_mutex));
    454       1.2     ad 
    455  1.12.2.1   matt 	opri = lwp_eprio(l);
    456  1.12.2.1   matt 	l->l_priority = pri;
    457  1.12.2.1   matt 	if (lwp_eprio(l) != opri) {
    458  1.12.2.1   matt 		TAILQ_REMOVE(&sq->sq_queue, l, l_sleepchain);
    459  1.12.2.1   matt 		sleepq_insert(sq, l, l->l_syncobj);
    460  1.12.2.1   matt 	}
    461       1.2     ad }
    462       1.6   yamt 
    463       1.6   yamt void
    464       1.8     ad sleepq_lendpri(lwp_t *l, pri_t pri)
    465       1.6   yamt {
    466       1.6   yamt 	sleepq_t *sq = l->l_sleepq;
    467       1.7   yamt 	pri_t opri;
    468       1.6   yamt 
    469       1.6   yamt 	KASSERT(lwp_locked(l, sq->sq_mutex));
    470       1.6   yamt 
    471       1.6   yamt 	opri = lwp_eprio(l);
    472       1.6   yamt 	l->l_inheritedprio = pri;
    473       1.6   yamt 
    474       1.6   yamt 	if (lwp_eprio(l) != opri &&
    475       1.6   yamt 	    (l->l_syncobj->sobj_flag & SOBJ_SLEEPQ_SORTED) != 0) {
    476       1.6   yamt 		TAILQ_REMOVE(&sq->sq_queue, l, l_sleepchain);
    477       1.6   yamt 		sleepq_insert(sq, l, l->l_syncobj);
    478       1.6   yamt 	}
    479       1.6   yamt }
    480