Home | History | Annotate | Line # | Download | only in kern
kern_lwp.c revision 1.73
      1  1.73     rmind /*	$NetBSD: kern_lwp.c,v 1.73 2007/10/09 19:00:13 rmind Exp $	*/
      2   1.2   thorpej 
      3   1.2   thorpej /*-
      4  1.52        ad  * Copyright (c) 2001, 2006, 2007 The NetBSD Foundation, Inc.
      5   1.2   thorpej  * All rights reserved.
      6   1.2   thorpej  *
      7   1.2   thorpej  * This code is derived from software contributed to The NetBSD Foundation
      8  1.52        ad  * by Nathan J. Williams, and Andrew Doran.
      9   1.2   thorpej  *
     10   1.2   thorpej  * Redistribution and use in source and binary forms, with or without
     11   1.2   thorpej  * modification, are permitted provided that the following conditions
     12   1.2   thorpej  * are met:
     13   1.2   thorpej  * 1. Redistributions of source code must retain the above copyright
     14   1.2   thorpej  *    notice, this list of conditions and the following disclaimer.
     15   1.2   thorpej  * 2. Redistributions in binary form must reproduce the above copyright
     16   1.2   thorpej  *    notice, this list of conditions and the following disclaimer in the
     17   1.2   thorpej  *    documentation and/or other materials provided with the distribution.
     18   1.2   thorpej  * 3. All advertising materials mentioning features or use of this software
     19   1.2   thorpej  *    must display the following acknowledgement:
     20   1.2   thorpej  *        This product includes software developed by the NetBSD
     21   1.2   thorpej  *        Foundation, Inc. and its contributors.
     22   1.2   thorpej  * 4. Neither the name of The NetBSD Foundation nor the names of its
     23   1.2   thorpej  *    contributors may be used to endorse or promote products derived
     24   1.2   thorpej  *    from this software without specific prior written permission.
     25   1.2   thorpej  *
     26   1.2   thorpej  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     27   1.2   thorpej  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     28   1.2   thorpej  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     29   1.2   thorpej  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     30   1.2   thorpej  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     31   1.2   thorpej  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     32   1.2   thorpej  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     33   1.2   thorpej  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     34   1.2   thorpej  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     35   1.2   thorpej  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     36   1.2   thorpej  * POSSIBILITY OF SUCH DAMAGE.
     37   1.2   thorpej  */
     38   1.9     lukem 
     39  1.52        ad /*
     40  1.52        ad  * Overview
     41  1.52        ad  *
     42  1.66        ad  *	Lightweight processes (LWPs) are the basic unit or thread of
     43  1.52        ad  *	execution within the kernel.  The core state of an LWP is described
     44  1.66        ad  *	by "struct lwp", also known as lwp_t.
     45  1.52        ad  *
     46  1.52        ad  *	Each LWP is contained within a process (described by "struct proc"),
     47  1.52        ad  *	Every process contains at least one LWP, but may contain more.  The
     48  1.52        ad  *	process describes attributes shared among all of its LWPs such as a
     49  1.52        ad  *	private address space, global execution state (stopped, active,
     50  1.52        ad  *	zombie, ...), signal disposition and so on.  On a multiprocessor
     51  1.66        ad  *	machine, multiple LWPs be executing concurrently in the kernel.
     52  1.52        ad  *
     53  1.52        ad  * Execution states
     54  1.52        ad  *
     55  1.52        ad  *	At any given time, an LWP has overall state that is described by
     56  1.52        ad  *	lwp::l_stat.  The states are broken into two sets below.  The first
     57  1.52        ad  *	set is guaranteed to represent the absolute, current state of the
     58  1.52        ad  *	LWP:
     59  1.52        ad  *
     60  1.52        ad  * 	LSONPROC
     61  1.52        ad  *
     62  1.52        ad  * 		On processor: the LWP is executing on a CPU, either in the
     63  1.52        ad  * 		kernel or in user space.
     64  1.52        ad  *
     65  1.52        ad  * 	LSRUN
     66  1.52        ad  *
     67  1.52        ad  * 		Runnable: the LWP is parked on a run queue, and may soon be
     68  1.52        ad  * 		chosen to run by a idle processor, or by a processor that
     69  1.52        ad  * 		has been asked to preempt a currently runnning but lower
     70  1.52        ad  * 		priority LWP.  If the LWP is not swapped in (L_INMEM == 0)
     71  1.52        ad  *		then the LWP is not on a run queue, but may be soon.
     72  1.52        ad  *
     73  1.52        ad  * 	LSIDL
     74  1.52        ad  *
     75  1.66        ad  * 		Idle: the LWP has been created but has not yet executed,
     76  1.66        ad  *		or it has ceased executing a unit of work and is waiting
     77  1.66        ad  *		to be started again.
     78  1.52        ad  *
     79  1.52        ad  * 	LSSUSPENDED:
     80  1.52        ad  *
     81  1.52        ad  * 		Suspended: the LWP has had its execution suspended by
     82  1.52        ad  *		another LWP in the same process using the _lwp_suspend()
     83  1.52        ad  *		system call.  User-level LWPs also enter the suspended
     84  1.52        ad  *		state when the system is shutting down.
     85  1.52        ad  *
     86  1.52        ad  *	The second set represent a "statement of intent" on behalf of the
     87  1.52        ad  *	LWP.  The LWP may in fact be executing on a processor, may be
     88  1.66        ad  *	sleeping or idle. It is expected to take the necessary action to
     89  1.66        ad  *	stop executing or become "running" again within	a short timeframe.
     90  1.66        ad  *	The LW_RUNNING flag in lwp::l_flag indicates that an LWP is running.
     91  1.66        ad  *	Importantly, in indicates that its state is tied to a CPU.
     92  1.52        ad  *
     93  1.52        ad  * 	LSZOMB:
     94  1.52        ad  *
     95  1.66        ad  * 		Dead or dying: the LWP has released most of its resources
     96  1.66        ad  *		and is a) about to switch away into oblivion b) has already
     97  1.66        ad  *		switched away.  When it switches away, its few remaining
     98  1.66        ad  *		resources can be collected.
     99  1.52        ad  *
    100  1.52        ad  * 	LSSLEEP:
    101  1.52        ad  *
    102  1.52        ad  * 		Sleeping: the LWP has entered itself onto a sleep queue, and
    103  1.66        ad  * 		has switched away or will switch away shortly to allow other
    104  1.66        ad  *		LWPs to run on the CPU.
    105  1.52        ad  *
    106  1.52        ad  * 	LSSTOP:
    107  1.52        ad  *
    108  1.52        ad  * 		Stopped: the LWP has been stopped as a result of a job
    109  1.52        ad  * 		control signal, or as a result of the ptrace() interface.
    110  1.66        ad  *
    111  1.52        ad  * 		Stopped LWPs may run briefly within the kernel to handle
    112  1.52        ad  * 		signals that they receive, but will not return to user space
    113  1.52        ad  * 		until their process' state is changed away from stopped.
    114  1.66        ad  *
    115  1.52        ad  * 		Single LWPs within a process can not be set stopped
    116  1.52        ad  * 		selectively: all actions that can stop or continue LWPs
    117  1.52        ad  * 		occur at the process level.
    118  1.52        ad  *
    119  1.52        ad  * State transitions
    120  1.52        ad  *
    121  1.66        ad  *	Note that the LSSTOP state may only be set when returning to
    122  1.66        ad  *	user space in userret(), or when sleeping interruptably.  The
    123  1.66        ad  *	LSSUSPENDED state may only be set in userret().  Before setting
    124  1.66        ad  *	those states, we try to ensure that the LWPs will release all
    125  1.66        ad  *	locks that they hold, and at a minimum try to ensure that the
    126  1.66        ad  *	LWP can be set runnable again by a signal.
    127  1.52        ad  *
    128  1.52        ad  *	LWPs may transition states in the following ways:
    129  1.52        ad  *
    130  1.52        ad  *	 RUN -------> ONPROC		ONPROC -----> RUN
    131  1.52        ad  *	            > STOPPED			    > SLEEP
    132  1.52        ad  *	            > SUSPENDED			    > STOPPED
    133  1.52        ad  *						    > SUSPENDED
    134  1.52        ad  *						    > ZOMB
    135  1.52        ad  *
    136  1.52        ad  *	 STOPPED ---> RUN		SUSPENDED --> RUN
    137  1.52        ad  *	            > SLEEP			    > SLEEP
    138  1.52        ad  *
    139  1.52        ad  *	 SLEEP -----> ONPROC		IDL --------> RUN
    140  1.52        ad  *		    > RUN		            > SUSPENDED
    141  1.52        ad  *		    > STOPPED                       > STOPPED
    142  1.52        ad  *		    > SUSPENDED
    143  1.52        ad  *
    144  1.66        ad  *	Other state transitions are possible with kernel threads (eg
    145  1.66        ad  *	ONPROC -> IDL), but only happen under tightly controlled
    146  1.66        ad  *	circumstances the side effects are understood.
    147  1.66        ad  *
    148  1.52        ad  * Locking
    149  1.52        ad  *
    150  1.52        ad  *	The majority of fields in 'struct lwp' are covered by a single,
    151  1.66        ad  *	general spin lock pointed to by lwp::l_mutex.  The locks covering
    152  1.52        ad  *	each field are documented in sys/lwp.h.
    153  1.52        ad  *
    154  1.66        ad  *	State transitions must be made with the LWP's general lock held,
    155  1.66        ad  * 	and may cause the LWP's lock pointer to change. Manipulation of
    156  1.66        ad  *	the general lock is not performed directly, but through calls to
    157  1.66        ad  *	lwp_lock(), lwp_relock() and similar.
    158  1.52        ad  *
    159  1.52        ad  *	States and their associated locks:
    160  1.52        ad  *
    161  1.64      yamt  *	LSIDL, LSZOMB, LSONPROC:
    162  1.52        ad  *
    163  1.64      yamt  *		Always covered by spc_lwplock, which protects running LWPs.
    164  1.64      yamt  *		This is a per-CPU lock.
    165  1.52        ad  *
    166  1.64      yamt  *	LSRUN:
    167  1.52        ad  *
    168  1.64      yamt  *		Always covered by spc_mutex, which protects the run queues.
    169  1.64      yamt  *		This may be a per-CPU lock, depending on the scheduler.
    170  1.52        ad  *
    171  1.52        ad  *	LSSLEEP:
    172  1.52        ad  *
    173  1.66        ad  *		Covered by a lock associated with the sleep queue that the
    174  1.52        ad  *		LWP resides on, indirectly referenced by l_sleepq->sq_mutex.
    175  1.52        ad  *
    176  1.52        ad  *	LSSTOP, LSSUSPENDED:
    177  1.52        ad  *
    178  1.52        ad  *		If the LWP was previously sleeping (l_wchan != NULL), then
    179  1.66        ad  *		l_mutex references the sleep queue lock.  If the LWP was
    180  1.52        ad  *		runnable or on the CPU when halted, or has been removed from
    181  1.66        ad  *		the sleep queue since halted, then the lock is spc_lwplock.
    182  1.52        ad  *
    183  1.52        ad  *	The lock order is as follows:
    184  1.52        ad  *
    185  1.64      yamt  *		spc::spc_lwplock ->
    186  1.64      yamt  *		    sleepq_t::sq_mutex ->
    187  1.64      yamt  *			tschain_t::tc_mutex ->
    188  1.64      yamt  *			    spc::spc_mutex
    189  1.52        ad  *
    190  1.66        ad  *	Each process has an scheduler state lock (proc::p_smutex), and a
    191  1.52        ad  *	number of counters on LWPs and their states: p_nzlwps, p_nrlwps, and
    192  1.52        ad  *	so on.  When an LWP is to be entered into or removed from one of the
    193  1.52        ad  *	following states, p_mutex must be held and the process wide counters
    194  1.52        ad  *	adjusted:
    195  1.52        ad  *
    196  1.52        ad  *		LSIDL, LSZOMB, LSSTOP, LSSUSPENDED
    197  1.52        ad  *
    198  1.52        ad  *	Note that an LWP is considered running or likely to run soon if in
    199  1.52        ad  *	one of the following states.  This affects the value of p_nrlwps:
    200  1.52        ad  *
    201  1.52        ad  *		LSRUN, LSONPROC, LSSLEEP
    202  1.52        ad  *
    203  1.52        ad  *	p_smutex does not need to be held when transitioning among these
    204  1.52        ad  *	three states.
    205  1.52        ad  */
    206  1.52        ad 
    207   1.9     lukem #include <sys/cdefs.h>
    208  1.73     rmind __KERNEL_RCSID(0, "$NetBSD: kern_lwp.c,v 1.73 2007/10/09 19:00:13 rmind Exp $");
    209   1.8    martin 
    210   1.8    martin #include "opt_multiprocessor.h"
    211  1.52        ad #include "opt_lockdebug.h"
    212   1.2   thorpej 
    213  1.47   hannken #define _LWP_API_PRIVATE
    214  1.47   hannken 
    215   1.2   thorpej #include <sys/param.h>
    216   1.2   thorpej #include <sys/systm.h>
    217  1.64      yamt #include <sys/cpu.h>
    218   1.2   thorpej #include <sys/pool.h>
    219   1.2   thorpej #include <sys/proc.h>
    220   1.2   thorpej #include <sys/syscallargs.h>
    221  1.57       dsl #include <sys/syscall_stats.h>
    222  1.37        ad #include <sys/kauth.h>
    223  1.52        ad #include <sys/sleepq.h>
    224  1.52        ad #include <sys/lockdebug.h>
    225  1.52        ad #include <sys/kmem.h>
    226   1.2   thorpej 
    227   1.2   thorpej #include <uvm/uvm_extern.h>
    228   1.2   thorpej 
    229  1.52        ad struct lwplist	alllwp;
    230  1.52        ad 
    231  1.52        ad POOL_INIT(lwp_pool, sizeof(struct lwp), MIN_LWP_ALIGNMENT, 0, 0, "lwppl",
    232  1.62        ad     &pool_allocator_nointr, IPL_NONE);
    233  1.41   thorpej POOL_INIT(lwp_uc_pool, sizeof(ucontext_t), 0, 0, 0, "lwpucpl",
    234  1.62        ad     &pool_allocator_nointr, IPL_NONE);
    235  1.41   thorpej 
    236  1.41   thorpej static specificdata_domain_t lwp_specificdata_domain;
    237  1.41   thorpej 
    238   1.2   thorpej #define LWP_DEBUG
    239   1.2   thorpej 
    240   1.2   thorpej #ifdef LWP_DEBUG
    241   1.2   thorpej int lwp_debug = 0;
    242   1.2   thorpej #define DPRINTF(x) if (lwp_debug) printf x
    243   1.2   thorpej #else
    244   1.2   thorpej #define DPRINTF(x)
    245   1.2   thorpej #endif
    246  1.41   thorpej 
    247  1.41   thorpej void
    248  1.41   thorpej lwpinit(void)
    249  1.41   thorpej {
    250  1.41   thorpej 
    251  1.41   thorpej 	lwp_specificdata_domain = specificdata_domain_create();
    252  1.41   thorpej 	KASSERT(lwp_specificdata_domain != NULL);
    253  1.52        ad 	lwp_sys_init();
    254  1.41   thorpej }
    255  1.41   thorpej 
    256  1.52        ad /*
    257  1.52        ad  * Set an suspended.
    258  1.52        ad  *
    259  1.52        ad  * Must be called with p_smutex held, and the LWP locked.  Will unlock the
    260  1.52        ad  * LWP before return.
    261  1.52        ad  */
    262   1.2   thorpej int
    263  1.52        ad lwp_suspend(struct lwp *curl, struct lwp *t)
    264   1.2   thorpej {
    265  1.52        ad 	int error;
    266   1.2   thorpej 
    267  1.63        ad 	KASSERT(mutex_owned(&t->l_proc->p_smutex));
    268  1.63        ad 	KASSERT(lwp_locked(t, NULL));
    269  1.33       chs 
    270  1.52        ad 	KASSERT(curl != t || curl->l_stat == LSONPROC);
    271   1.2   thorpej 
    272  1.52        ad 	/*
    273  1.52        ad 	 * If the current LWP has been told to exit, we must not suspend anyone
    274  1.52        ad 	 * else or deadlock could occur.  We won't return to userspace.
    275   1.2   thorpej 	 */
    276  1.56     pavel 	if ((curl->l_stat & (LW_WEXIT | LW_WCORE)) != 0) {
    277  1.52        ad 		lwp_unlock(t);
    278  1.52        ad 		return (EDEADLK);
    279   1.2   thorpej 	}
    280   1.2   thorpej 
    281  1.52        ad 	error = 0;
    282   1.2   thorpej 
    283  1.52        ad 	switch (t->l_stat) {
    284  1.52        ad 	case LSRUN:
    285  1.52        ad 	case LSONPROC:
    286  1.56     pavel 		t->l_flag |= LW_WSUSPEND;
    287  1.52        ad 		lwp_need_userret(t);
    288  1.52        ad 		lwp_unlock(t);
    289  1.52        ad 		break;
    290   1.2   thorpej 
    291  1.52        ad 	case LSSLEEP:
    292  1.56     pavel 		t->l_flag |= LW_WSUSPEND;
    293   1.2   thorpej 
    294   1.2   thorpej 		/*
    295  1.52        ad 		 * Kick the LWP and try to get it to the kernel boundary
    296  1.52        ad 		 * so that it will release any locks that it holds.
    297  1.52        ad 		 * setrunnable() will release the lock.
    298   1.2   thorpej 		 */
    299  1.56     pavel 		if ((t->l_flag & LW_SINTR) != 0)
    300  1.52        ad 			setrunnable(t);
    301  1.52        ad 		else
    302  1.52        ad 			lwp_unlock(t);
    303  1.52        ad 		break;
    304   1.2   thorpej 
    305  1.52        ad 	case LSSUSPENDED:
    306  1.52        ad 		lwp_unlock(t);
    307  1.52        ad 		break;
    308  1.17      manu 
    309  1.52        ad 	case LSSTOP:
    310  1.56     pavel 		t->l_flag |= LW_WSUSPEND;
    311  1.52        ad 		setrunnable(t);
    312  1.52        ad 		break;
    313   1.2   thorpej 
    314  1.52        ad 	case LSIDL:
    315  1.52        ad 	case LSZOMB:
    316  1.52        ad 		error = EINTR; /* It's what Solaris does..... */
    317  1.52        ad 		lwp_unlock(t);
    318  1.52        ad 		break;
    319   1.2   thorpej 	}
    320   1.2   thorpej 
    321  1.69     rmind 	return (error);
    322   1.2   thorpej }
    323   1.2   thorpej 
    324  1.52        ad /*
    325  1.52        ad  * Restart a suspended LWP.
    326  1.52        ad  *
    327  1.52        ad  * Must be called with p_smutex held, and the LWP locked.  Will unlock the
    328  1.52        ad  * LWP before return.
    329  1.52        ad  */
    330   1.2   thorpej void
    331   1.2   thorpej lwp_continue(struct lwp *l)
    332   1.2   thorpej {
    333   1.2   thorpej 
    334  1.63        ad 	KASSERT(mutex_owned(&l->l_proc->p_smutex));
    335  1.63        ad 	KASSERT(lwp_locked(l, NULL));
    336  1.52        ad 
    337   1.2   thorpej 	DPRINTF(("lwp_continue of %d.%d (%s), state %d, wchan %p\n",
    338   1.2   thorpej 	    l->l_proc->p_pid, l->l_lid, l->l_proc->p_comm, l->l_stat,
    339   1.2   thorpej 	    l->l_wchan));
    340   1.2   thorpej 
    341  1.52        ad 	/* If rebooting or not suspended, then just bail out. */
    342  1.56     pavel 	if ((l->l_flag & LW_WREBOOT) != 0) {
    343  1.52        ad 		lwp_unlock(l);
    344   1.2   thorpej 		return;
    345  1.10      fvdl 	}
    346   1.2   thorpej 
    347  1.56     pavel 	l->l_flag &= ~LW_WSUSPEND;
    348   1.2   thorpej 
    349  1.52        ad 	if (l->l_stat != LSSUSPENDED) {
    350  1.52        ad 		lwp_unlock(l);
    351  1.52        ad 		return;
    352   1.2   thorpej 	}
    353   1.2   thorpej 
    354  1.52        ad 	/* setrunnable() will release the lock. */
    355  1.52        ad 	setrunnable(l);
    356   1.2   thorpej }
    357   1.2   thorpej 
    358  1.52        ad /*
    359  1.52        ad  * Wait for an LWP within the current process to exit.  If 'lid' is
    360  1.52        ad  * non-zero, we are waiting for a specific LWP.
    361  1.52        ad  *
    362  1.52        ad  * Must be called with p->p_smutex held.
    363  1.52        ad  */
    364   1.2   thorpej int
    365   1.2   thorpej lwp_wait1(struct lwp *l, lwpid_t lid, lwpid_t *departed, int flags)
    366   1.2   thorpej {
    367   1.2   thorpej 	struct proc *p = l->l_proc;
    368  1.52        ad 	struct lwp *l2;
    369  1.52        ad 	int nfound, error;
    370  1.63        ad 	lwpid_t curlid;
    371  1.63        ad 	bool exiting;
    372   1.2   thorpej 
    373   1.2   thorpej 	DPRINTF(("lwp_wait1: %d.%d waiting for %d.\n",
    374   1.2   thorpej 	    p->p_pid, l->l_lid, lid));
    375   1.2   thorpej 
    376  1.63        ad 	KASSERT(mutex_owned(&p->p_smutex));
    377  1.52        ad 
    378  1.52        ad 	p->p_nlwpwait++;
    379  1.63        ad 	l->l_waitingfor = lid;
    380  1.63        ad 	curlid = l->l_lid;
    381  1.63        ad 	exiting = ((flags & LWPWAIT_EXITCONTROL) != 0);
    382  1.52        ad 
    383  1.52        ad 	for (;;) {
    384  1.52        ad 		/*
    385  1.52        ad 		 * Avoid a race between exit1() and sigexit(): if the
    386  1.52        ad 		 * process is dumping core, then we need to bail out: call
    387  1.52        ad 		 * into lwp_userret() where we will be suspended until the
    388  1.52        ad 		 * deed is done.
    389  1.52        ad 		 */
    390  1.52        ad 		if ((p->p_sflag & PS_WCORE) != 0) {
    391  1.52        ad 			mutex_exit(&p->p_smutex);
    392  1.52        ad 			lwp_userret(l);
    393  1.52        ad #ifdef DIAGNOSTIC
    394  1.52        ad 			panic("lwp_wait1");
    395  1.52        ad #endif
    396  1.52        ad 			/* NOTREACHED */
    397  1.52        ad 		}
    398  1.52        ad 
    399  1.52        ad 		/*
    400  1.52        ad 		 * First off, drain any detached LWP that is waiting to be
    401  1.52        ad 		 * reaped.
    402  1.52        ad 		 */
    403  1.52        ad 		while ((l2 = p->p_zomblwp) != NULL) {
    404  1.52        ad 			p->p_zomblwp = NULL;
    405  1.63        ad 			lwp_free(l2, false, false);/* releases proc mutex */
    406  1.52        ad 			mutex_enter(&p->p_smutex);
    407  1.52        ad 		}
    408  1.52        ad 
    409  1.52        ad 		/*
    410  1.52        ad 		 * Now look for an LWP to collect.  If the whole process is
    411  1.52        ad 		 * exiting, count detached LWPs as eligible to be collected,
    412  1.52        ad 		 * but don't drain them here.
    413  1.52        ad 		 */
    414  1.52        ad 		nfound = 0;
    415  1.63        ad 		error = 0;
    416  1.52        ad 		LIST_FOREACH(l2, &p->p_lwps, l_sibling) {
    417  1.63        ad 			/*
    418  1.63        ad 			 * If a specific wait and the target is waiting on
    419  1.63        ad 			 * us, then avoid deadlock.  This also traps LWPs
    420  1.63        ad 			 * that try to wait on themselves.
    421  1.63        ad 			 *
    422  1.63        ad 			 * Note that this does not handle more complicated
    423  1.63        ad 			 * cycles, like: t1 -> t2 -> t3 -> t1.  The process
    424  1.63        ad 			 * can still be killed so it is not a major problem.
    425  1.63        ad 			 */
    426  1.63        ad 			if (l2->l_lid == lid && l2->l_waitingfor == curlid) {
    427  1.63        ad 				error = EDEADLK;
    428  1.63        ad 				break;
    429  1.63        ad 			}
    430  1.63        ad 			if (l2 == l)
    431  1.52        ad 				continue;
    432  1.52        ad 			if ((l2->l_prflag & LPR_DETACHED) != 0) {
    433  1.63        ad 				nfound += exiting;
    434  1.63        ad 				continue;
    435  1.63        ad 			}
    436  1.63        ad 			if (lid != 0) {
    437  1.63        ad 				if (l2->l_lid != lid)
    438  1.63        ad 					continue;
    439  1.63        ad 				/*
    440  1.63        ad 				 * Mark this LWP as the first waiter, if there
    441  1.63        ad 				 * is no other.
    442  1.63        ad 				 */
    443  1.63        ad 				if (l2->l_waiter == 0)
    444  1.63        ad 					l2->l_waiter = curlid;
    445  1.63        ad 			} else if (l2->l_waiter != 0) {
    446  1.63        ad 				/*
    447  1.63        ad 				 * It already has a waiter - so don't
    448  1.63        ad 				 * collect it.  If the waiter doesn't
    449  1.63        ad 				 * grab it we'll get another chance
    450  1.63        ad 				 * later.
    451  1.63        ad 				 */
    452  1.63        ad 				nfound++;
    453  1.52        ad 				continue;
    454  1.52        ad 			}
    455  1.52        ad 			nfound++;
    456   1.2   thorpej 
    457  1.52        ad 			/* No need to lock the LWP in order to see LSZOMB. */
    458  1.52        ad 			if (l2->l_stat != LSZOMB)
    459  1.52        ad 				continue;
    460   1.2   thorpej 
    461  1.63        ad 			/*
    462  1.63        ad 			 * We're no longer waiting.  Reset the "first waiter"
    463  1.63        ad 			 * pointer on the target, in case it was us.
    464  1.63        ad 			 */
    465  1.63        ad 			l->l_waitingfor = 0;
    466  1.63        ad 			l2->l_waiter = 0;
    467  1.63        ad 			p->p_nlwpwait--;
    468   1.2   thorpej 			if (departed)
    469   1.2   thorpej 				*departed = l2->l_lid;
    470  1.63        ad 
    471  1.63        ad 			/* lwp_free() releases the proc lock. */
    472  1.63        ad 			lwp_free(l2, false, false);
    473  1.52        ad 			mutex_enter(&p->p_smutex);
    474  1.52        ad 			return 0;
    475  1.52        ad 		}
    476   1.2   thorpej 
    477  1.63        ad 		if (error != 0)
    478  1.63        ad 			break;
    479  1.52        ad 		if (nfound == 0) {
    480  1.52        ad 			error = ESRCH;
    481  1.52        ad 			break;
    482  1.52        ad 		}
    483  1.63        ad 
    484  1.63        ad 		/*
    485  1.63        ad 		 * The kernel is careful to ensure that it can not deadlock
    486  1.63        ad 		 * when exiting - just keep waiting.
    487  1.63        ad 		 */
    488  1.63        ad 		if (exiting) {
    489  1.52        ad 			KASSERT(p->p_nlwps > 1);
    490  1.52        ad 			cv_wait(&p->p_lwpcv, &p->p_smutex);
    491  1.52        ad 			continue;
    492  1.52        ad 		}
    493  1.63        ad 
    494  1.63        ad 		/*
    495  1.63        ad 		 * If all other LWPs are waiting for exits or suspends
    496  1.63        ad 		 * and the supply of zombies and potential zombies is
    497  1.63        ad 		 * exhausted, then we are about to deadlock.
    498  1.63        ad 		 *
    499  1.63        ad 		 * If the process is exiting (and this LWP is not the one
    500  1.63        ad 		 * that is coordinating the exit) then bail out now.
    501  1.63        ad 		 */
    502  1.52        ad 		if ((p->p_sflag & PS_WEXIT) != 0 ||
    503  1.63        ad 		    p->p_nrlwps + p->p_nzlwps - p->p_ndlwps <= p->p_nlwpwait) {
    504  1.52        ad 			error = EDEADLK;
    505  1.52        ad 			break;
    506   1.2   thorpej 		}
    507  1.63        ad 
    508  1.63        ad 		/*
    509  1.63        ad 		 * Sit around and wait for something to happen.  We'll be
    510  1.63        ad 		 * awoken if any of the conditions examined change: if an
    511  1.63        ad 		 * LWP exits, is collected, or is detached.
    512  1.63        ad 		 */
    513  1.52        ad 		if ((error = cv_wait_sig(&p->p_lwpcv, &p->p_smutex)) != 0)
    514  1.52        ad 			break;
    515   1.2   thorpej 	}
    516   1.2   thorpej 
    517  1.63        ad 	/*
    518  1.63        ad 	 * We didn't find any LWPs to collect, we may have received a
    519  1.63        ad 	 * signal, or some other condition has caused us to bail out.
    520  1.63        ad 	 *
    521  1.63        ad 	 * If waiting on a specific LWP, clear the waiters marker: some
    522  1.63        ad 	 * other LWP may want it.  Then, kick all the remaining waiters
    523  1.63        ad 	 * so that they can re-check for zombies and for deadlock.
    524  1.63        ad 	 */
    525  1.63        ad 	if (lid != 0) {
    526  1.63        ad 		LIST_FOREACH(l2, &p->p_lwps, l_sibling) {
    527  1.63        ad 			if (l2->l_lid == lid) {
    528  1.63        ad 				if (l2->l_waiter == curlid)
    529  1.63        ad 					l2->l_waiter = 0;
    530  1.63        ad 				break;
    531  1.63        ad 			}
    532  1.63        ad 		}
    533  1.63        ad 	}
    534  1.52        ad 	p->p_nlwpwait--;
    535  1.63        ad 	l->l_waitingfor = 0;
    536  1.63        ad 	cv_broadcast(&p->p_lwpcv);
    537  1.63        ad 
    538  1.52        ad 	return error;
    539   1.2   thorpej }
    540   1.2   thorpej 
    541  1.52        ad /*
    542  1.52        ad  * Create a new LWP within process 'p2', using LWP 'l1' as a template.
    543  1.52        ad  * The new LWP is created in state LSIDL and must be set running,
    544  1.52        ad  * suspended, or stopped by the caller.
    545  1.52        ad  */
    546   1.2   thorpej int
    547  1.59   thorpej newlwp(struct lwp *l1, struct proc *p2, vaddr_t uaddr, bool inmem,
    548   1.2   thorpej     int flags, void *stack, size_t stacksize,
    549   1.2   thorpej     void (*func)(void *), void *arg, struct lwp **rnewlwpp)
    550   1.2   thorpej {
    551  1.52        ad 	struct lwp *l2, *isfree;
    552  1.52        ad 	turnstile_t *ts;
    553   1.2   thorpej 
    554  1.52        ad 	/*
    555  1.52        ad 	 * First off, reap any detached LWP waiting to be collected.
    556  1.52        ad 	 * We can re-use its LWP structure and turnstile.
    557  1.52        ad 	 */
    558  1.52        ad 	isfree = NULL;
    559  1.52        ad 	if (p2->p_zomblwp != NULL) {
    560  1.52        ad 		mutex_enter(&p2->p_smutex);
    561  1.52        ad 		if ((isfree = p2->p_zomblwp) != NULL) {
    562  1.52        ad 			p2->p_zomblwp = NULL;
    563  1.63        ad 			lwp_free(isfree, true, false);/* releases proc mutex */
    564  1.52        ad 		} else
    565  1.52        ad 			mutex_exit(&p2->p_smutex);
    566  1.52        ad 	}
    567  1.52        ad 	if (isfree == NULL) {
    568  1.52        ad 		l2 = pool_get(&lwp_pool, PR_WAITOK);
    569  1.52        ad 		memset(l2, 0, sizeof(*l2));
    570  1.52        ad 		l2->l_ts = pool_cache_get(&turnstile_cache, PR_WAITOK);
    571  1.60      yamt 		SLIST_INIT(&l2->l_pi_lenders);
    572  1.52        ad 	} else {
    573  1.52        ad 		l2 = isfree;
    574  1.52        ad 		ts = l2->l_ts;
    575  1.60      yamt 		KASSERT(l2->l_inheritedprio == MAXPRI);
    576  1.60      yamt 		KASSERT(SLIST_EMPTY(&l2->l_pi_lenders));
    577  1.52        ad 		memset(l2, 0, sizeof(*l2));
    578  1.52        ad 		l2->l_ts = ts;
    579  1.52        ad 	}
    580   1.2   thorpej 
    581   1.2   thorpej 	l2->l_stat = LSIDL;
    582   1.2   thorpej 	l2->l_proc = p2;
    583  1.52        ad 	l2->l_refcnt = 1;
    584  1.52        ad 	l2->l_priority = l1->l_priority;
    585  1.52        ad 	l2->l_usrpri = l1->l_usrpri;
    586  1.60      yamt 	l2->l_inheritedprio = MAXPRI;
    587  1.64      yamt 	l2->l_mutex = l1->l_cpu->ci_schedstate.spc_mutex;
    588  1.52        ad 	l2->l_cpu = l1->l_cpu;
    589  1.56     pavel 	l2->l_flag = inmem ? LW_INMEM : 0;
    590  1.41   thorpej 
    591  1.56     pavel 	if (p2->p_flag & PK_SYSTEM) {
    592  1.52        ad 		/*
    593  1.52        ad 		 * Mark it as a system process and not a candidate for
    594  1.52        ad 		 * swapping.
    595  1.52        ad 		 */
    596  1.56     pavel 		l2->l_flag |= LW_SYSTEM;
    597  1.73     rmind 	} else {
    598  1.73     rmind 		/* Look for a CPU to start */
    599  1.73     rmind 		l2->l_cpu = sched_takecpu(l2);
    600  1.73     rmind 		l2->l_mutex = l2->l_cpu->ci_schedstate.spc_mutex;
    601  1.52        ad 	}
    602   1.2   thorpej 
    603  1.73     rmind 	lwp_initspecific(l2);
    604  1.73     rmind 	sched_lwp_fork(l2);
    605  1.37        ad 	lwp_update_creds(l2);
    606  1.70        ad 	callout_init(&l2->l_timeout_ch, CALLOUT_MPSAFE);
    607  1.70        ad 	callout_setfunc(&l2->l_timeout_ch, sleepq_timeout, l2);
    608  1.65        ad 	mutex_init(&l2->l_swaplock, MUTEX_DEFAULT, IPL_NONE);
    609  1.52        ad 	cv_init(&l2->l_sigcv, "sigwait");
    610  1.52        ad 	l2->l_syncobj = &sched_syncobj;
    611   1.2   thorpej 
    612   1.2   thorpej 	if (rnewlwpp != NULL)
    613   1.2   thorpej 		*rnewlwpp = l2;
    614   1.2   thorpej 
    615  1.36      yamt 	l2->l_addr = UAREA_TO_USER(uaddr);
    616   1.2   thorpej 	uvm_lwp_fork(l1, l2, stack, stacksize, func,
    617   1.2   thorpej 	    (arg != NULL) ? arg : l2);
    618   1.2   thorpej 
    619  1.52        ad 	mutex_enter(&p2->p_smutex);
    620  1.52        ad 
    621  1.52        ad 	if ((flags & LWP_DETACHED) != 0) {
    622  1.52        ad 		l2->l_prflag = LPR_DETACHED;
    623  1.52        ad 		p2->p_ndlwps++;
    624  1.52        ad 	} else
    625  1.52        ad 		l2->l_prflag = 0;
    626  1.52        ad 
    627  1.52        ad 	l2->l_sigmask = l1->l_sigmask;
    628  1.52        ad 	CIRCLEQ_INIT(&l2->l_sigpend.sp_info);
    629  1.52        ad 	sigemptyset(&l2->l_sigpend.sp_set);
    630  1.52        ad 
    631  1.53      yamt 	p2->p_nlwpid++;
    632  1.53      yamt 	if (p2->p_nlwpid == 0)
    633  1.53      yamt 		p2->p_nlwpid++;
    634  1.53      yamt 	l2->l_lid = p2->p_nlwpid;
    635   1.2   thorpej 	LIST_INSERT_HEAD(&p2->p_lwps, l2, l_sibling);
    636   1.2   thorpej 	p2->p_nlwps++;
    637   1.2   thorpej 
    638  1.52        ad 	mutex_exit(&p2->p_smutex);
    639  1.52        ad 
    640  1.65        ad 	mutex_enter(&proclist_lock);
    641  1.52        ad 	mutex_enter(&proclist_mutex);
    642   1.2   thorpej 	LIST_INSERT_HEAD(&alllwp, l2, l_list);
    643  1.52        ad 	mutex_exit(&proclist_mutex);
    644  1.65        ad 	mutex_exit(&proclist_lock);
    645   1.2   thorpej 
    646  1.57       dsl 	SYSCALL_TIME_LWP_INIT(l2);
    647  1.57       dsl 
    648  1.16      manu 	if (p2->p_emul->e_lwp_fork)
    649  1.16      manu 		(*p2->p_emul->e_lwp_fork)(l1, l2);
    650  1.16      manu 
    651   1.2   thorpej 	return (0);
    652   1.2   thorpej }
    653   1.2   thorpej 
    654   1.2   thorpej /*
    655  1.64      yamt  * Called by MD code when a new LWP begins execution.  Must be called
    656  1.64      yamt  * with the previous LWP locked (so at splsched), or if there is no
    657  1.64      yamt  * previous LWP, at splsched.
    658  1.64      yamt  */
    659  1.64      yamt void
    660  1.64      yamt lwp_startup(struct lwp *prev, struct lwp *new)
    661  1.64      yamt {
    662  1.64      yamt 
    663  1.64      yamt 	if (prev != NULL) {
    664  1.64      yamt 		lwp_unlock(prev);
    665  1.64      yamt 	}
    666  1.64      yamt 	spl0();
    667  1.64      yamt 	pmap_activate(new);
    668  1.64      yamt 	LOCKDEBUG_BARRIER(NULL, 0);
    669  1.65        ad 	if ((new->l_pflag & LP_MPSAFE) == 0) {
    670  1.65        ad 		KERNEL_LOCK(1, new);
    671  1.65        ad 	}
    672  1.64      yamt }
    673  1.64      yamt 
    674  1.64      yamt /*
    675  1.65        ad  * Exit an LWP.
    676   1.2   thorpej  */
    677   1.2   thorpej void
    678   1.2   thorpej lwp_exit(struct lwp *l)
    679   1.2   thorpej {
    680   1.2   thorpej 	struct proc *p = l->l_proc;
    681  1.52        ad 	struct lwp *l2;
    682  1.65        ad 	bool current;
    683  1.65        ad 
    684  1.65        ad 	current = (l == curlwp);
    685   1.2   thorpej 
    686   1.2   thorpej 	DPRINTF(("lwp_exit: %d.%d exiting.\n", p->p_pid, l->l_lid));
    687  1.52        ad 	DPRINTF((" nlwps: %d nzlwps: %d\n", p->p_nlwps, p->p_nzlwps));
    688  1.65        ad 	KASSERT(current || l->l_stat == LSIDL);
    689   1.2   thorpej 
    690  1.52        ad 	/*
    691  1.52        ad 	 * Verify that we hold no locks other than the kernel lock.
    692  1.52        ad 	 */
    693  1.52        ad #ifdef MULTIPROCESSOR
    694  1.52        ad 	LOCKDEBUG_BARRIER(&kernel_lock, 0);
    695  1.52        ad #else
    696  1.52        ad 	LOCKDEBUG_BARRIER(NULL, 0);
    697  1.52        ad #endif
    698  1.16      manu 
    699   1.2   thorpej 	/*
    700  1.52        ad 	 * If we are the last live LWP in a process, we need to exit the
    701  1.52        ad 	 * entire process.  We do so with an exit status of zero, because
    702  1.52        ad 	 * it's a "controlled" exit, and because that's what Solaris does.
    703  1.52        ad 	 *
    704  1.52        ad 	 * We are not quite a zombie yet, but for accounting purposes we
    705  1.52        ad 	 * must increment the count of zombies here.
    706  1.45   thorpej 	 *
    707  1.45   thorpej 	 * Note: the last LWP's specificdata will be deleted here.
    708   1.2   thorpej 	 */
    709  1.52        ad 	mutex_enter(&p->p_smutex);
    710  1.52        ad 	if (p->p_nlwps - p->p_nzlwps == 1) {
    711  1.65        ad 		KASSERT(current == true);
    712   1.2   thorpej 		DPRINTF(("lwp_exit: %d.%d calling exit1()\n",
    713   1.2   thorpej 		    p->p_pid, l->l_lid));
    714   1.2   thorpej 		exit1(l, 0);
    715  1.19  jdolecek 		/* NOTREACHED */
    716   1.2   thorpej 	}
    717  1.52        ad 	p->p_nzlwps++;
    718  1.52        ad 	mutex_exit(&p->p_smutex);
    719  1.52        ad 
    720  1.52        ad 	if (p->p_emul->e_lwp_exit)
    721  1.52        ad 		(*p->p_emul->e_lwp_exit)(l);
    722   1.2   thorpej 
    723  1.45   thorpej 	/* Delete the specificdata while it's still safe to sleep. */
    724  1.45   thorpej 	specificdata_fini(lwp_specificdata_domain, &l->l_specdataref);
    725  1.45   thorpej 
    726  1.52        ad 	/*
    727  1.52        ad 	 * Release our cached credentials.
    728  1.52        ad 	 */
    729  1.37        ad 	kauth_cred_free(l->l_cred);
    730  1.70        ad 	callout_destroy(&l->l_timeout_ch);
    731  1.65        ad 
    732  1.65        ad 	/*
    733  1.65        ad 	 * While we can still block, mark the LWP as unswappable to
    734  1.65        ad 	 * prevent conflicts with the with the swapper.
    735  1.65        ad 	 */
    736  1.65        ad 	if (current)
    737  1.65        ad 		uvm_lwp_hold(l);
    738  1.37        ad 
    739  1.52        ad 	/*
    740  1.52        ad 	 * Remove the LWP from the global list.
    741  1.52        ad 	 */
    742  1.65        ad 	mutex_enter(&proclist_lock);
    743  1.52        ad 	mutex_enter(&proclist_mutex);
    744  1.52        ad 	LIST_REMOVE(l, l_list);
    745  1.52        ad 	mutex_exit(&proclist_mutex);
    746  1.65        ad 	mutex_exit(&proclist_lock);
    747  1.19  jdolecek 
    748  1.52        ad 	/*
    749  1.52        ad 	 * Get rid of all references to the LWP that others (e.g. procfs)
    750  1.52        ad 	 * may have, and mark the LWP as a zombie.  If the LWP is detached,
    751  1.52        ad 	 * mark it waiting for collection in the proc structure.  Note that
    752  1.52        ad 	 * before we can do that, we need to free any other dead, deatched
    753  1.52        ad 	 * LWP waiting to meet its maker.
    754  1.52        ad 	 *
    755  1.52        ad 	 * XXXSMP disable preemption.
    756  1.52        ad 	 */
    757  1.52        ad 	mutex_enter(&p->p_smutex);
    758  1.52        ad 	lwp_drainrefs(l);
    759  1.31      yamt 
    760  1.52        ad 	if ((l->l_prflag & LPR_DETACHED) != 0) {
    761  1.52        ad 		while ((l2 = p->p_zomblwp) != NULL) {
    762  1.52        ad 			p->p_zomblwp = NULL;
    763  1.63        ad 			lwp_free(l2, false, false);/* releases proc mutex */
    764  1.52        ad 			mutex_enter(&p->p_smutex);
    765  1.72        ad 			l->l_refcnt++;
    766  1.72        ad 			lwp_drainrefs(l);
    767  1.52        ad 		}
    768  1.52        ad 		p->p_zomblwp = l;
    769  1.52        ad 	}
    770  1.31      yamt 
    771  1.52        ad 	/*
    772  1.52        ad 	 * If we find a pending signal for the process and we have been
    773  1.52        ad 	 * asked to check for signals, then we loose: arrange to have
    774  1.52        ad 	 * all other LWPs in the process check for signals.
    775  1.52        ad 	 */
    776  1.56     pavel 	if ((l->l_flag & LW_PENDSIG) != 0 &&
    777  1.52        ad 	    firstsig(&p->p_sigpend.sp_set) != 0) {
    778  1.52        ad 		LIST_FOREACH(l2, &p->p_lwps, l_sibling) {
    779  1.52        ad 			lwp_lock(l2);
    780  1.56     pavel 			l2->l_flag |= LW_PENDSIG;
    781  1.52        ad 			lwp_unlock(l2);
    782  1.52        ad 		}
    783  1.31      yamt 	}
    784  1.31      yamt 
    785  1.52        ad 	lwp_lock(l);
    786  1.52        ad 	l->l_stat = LSZOMB;
    787  1.52        ad 	lwp_unlock(l);
    788   1.2   thorpej 	p->p_nrlwps--;
    789  1.52        ad 	cv_broadcast(&p->p_lwpcv);
    790  1.52        ad 	mutex_exit(&p->p_smutex);
    791  1.52        ad 
    792  1.52        ad 	/*
    793  1.52        ad 	 * We can no longer block.  At this point, lwp_free() may already
    794  1.52        ad 	 * be gunning for us.  On a multi-CPU system, we may be off p_lwps.
    795  1.52        ad 	 *
    796  1.52        ad 	 * Free MD LWP resources.
    797  1.52        ad 	 */
    798  1.52        ad #ifndef __NO_CPU_LWP_FREE
    799  1.52        ad 	cpu_lwp_free(l, 0);
    800  1.52        ad #endif
    801   1.2   thorpej 
    802  1.65        ad 	if (current) {
    803  1.65        ad 		pmap_deactivate(l);
    804  1.65        ad 
    805  1.65        ad 		/*
    806  1.65        ad 		 * Release the kernel lock, and switch away into
    807  1.65        ad 		 * oblivion.
    808  1.65        ad 		 */
    809  1.52        ad #ifdef notyet
    810  1.65        ad 		/* XXXSMP hold in lwp_userret() */
    811  1.65        ad 		KERNEL_UNLOCK_LAST(l);
    812  1.52        ad #else
    813  1.65        ad 		KERNEL_UNLOCK_ALL(l, NULL);
    814  1.52        ad #endif
    815  1.65        ad 		lwp_exit_switchaway(l);
    816  1.65        ad 	}
    817   1.2   thorpej }
    818   1.2   thorpej 
    819   1.2   thorpej void
    820  1.64      yamt lwp_exit_switchaway(struct lwp *l)
    821   1.2   thorpej {
    822  1.64      yamt 	struct cpu_info *ci;
    823  1.64      yamt 	struct lwp *idlelwp;
    824  1.64      yamt 
    825  1.64      yamt 	/* Unlocked, but is for statistics only. */
    826  1.64      yamt 	uvmexp.swtch++;
    827  1.64      yamt 
    828  1.64      yamt 	(void)splsched();
    829  1.64      yamt 	l->l_flag &= ~LW_RUNNING;
    830  1.64      yamt 	ci = curcpu();
    831  1.64      yamt 	idlelwp = ci->ci_data.cpu_idlelwp;
    832  1.64      yamt 	idlelwp->l_stat = LSONPROC;
    833  1.64      yamt 	cpu_switchto(NULL, idlelwp);
    834  1.52        ad }
    835  1.52        ad 
    836  1.52        ad /*
    837  1.52        ad  * Free a dead LWP's remaining resources.
    838  1.52        ad  *
    839  1.52        ad  * XXXLWP limits.
    840  1.52        ad  */
    841  1.52        ad void
    842  1.63        ad lwp_free(struct lwp *l, bool recycle, bool last)
    843  1.52        ad {
    844  1.52        ad 	struct proc *p = l->l_proc;
    845  1.52        ad 	ksiginfoq_t kq;
    846  1.52        ad 
    847  1.52        ad 	/*
    848  1.52        ad 	 * If this was not the last LWP in the process, then adjust
    849  1.52        ad 	 * counters and unlock.
    850  1.52        ad 	 */
    851  1.52        ad 	if (!last) {
    852  1.52        ad 		/*
    853  1.52        ad 		 * Add the LWP's run time to the process' base value.
    854  1.52        ad 		 * This needs to co-incide with coming off p_lwps.
    855  1.52        ad 		 */
    856  1.52        ad 		timeradd(&l->l_rtime, &p->p_rtime, &p->p_rtime);
    857  1.64      yamt 		p->p_pctcpu += l->l_pctcpu;
    858  1.52        ad 		LIST_REMOVE(l, l_sibling);
    859  1.52        ad 		p->p_nlwps--;
    860  1.52        ad 		p->p_nzlwps--;
    861  1.52        ad 		if ((l->l_prflag & LPR_DETACHED) != 0)
    862  1.52        ad 			p->p_ndlwps--;
    863  1.63        ad 
    864  1.63        ad 		/*
    865  1.63        ad 		 * Have any LWPs sleeping in lwp_wait() recheck for
    866  1.63        ad 		 * deadlock.
    867  1.63        ad 		 */
    868  1.63        ad 		cv_broadcast(&p->p_lwpcv);
    869  1.52        ad 		mutex_exit(&p->p_smutex);
    870  1.63        ad 	}
    871  1.52        ad 
    872  1.52        ad #ifdef MULTIPROCESSOR
    873  1.63        ad 	/*
    874  1.63        ad 	 * In the unlikely event that the LWP is still on the CPU,
    875  1.63        ad 	 * then spin until it has switched away.  We need to release
    876  1.63        ad 	 * all locks to avoid deadlock against interrupt handlers on
    877  1.63        ad 	 * the target CPU.
    878  1.63        ad 	 */
    879  1.64      yamt 	if ((l->l_flag & LW_RUNNING) != 0 || l->l_cpu->ci_curlwp == l) {
    880  1.63        ad 		int count;
    881  1.64      yamt 		(void)count; /* XXXgcc */
    882  1.63        ad 		KERNEL_UNLOCK_ALL(curlwp, &count);
    883  1.64      yamt 		while ((l->l_flag & LW_RUNNING) != 0 ||
    884  1.64      yamt 		    l->l_cpu->ci_curlwp == l)
    885  1.63        ad 			SPINLOCK_BACKOFF_HOOK;
    886  1.63        ad 		KERNEL_LOCK(count, curlwp);
    887  1.63        ad 	}
    888  1.52        ad #endif
    889  1.52        ad 
    890  1.52        ad 	/*
    891  1.52        ad 	 * Destroy the LWP's remaining signal information.
    892  1.52        ad 	 */
    893  1.52        ad 	ksiginfo_queue_init(&kq);
    894  1.52        ad 	sigclear(&l->l_sigpend, NULL, &kq);
    895  1.52        ad 	ksiginfo_queue_drain(&kq);
    896  1.52        ad 	cv_destroy(&l->l_sigcv);
    897  1.65        ad 	mutex_destroy(&l->l_swaplock);
    898   1.2   thorpej 
    899  1.19  jdolecek 	/*
    900  1.52        ad 	 * Free the LWP's turnstile and the LWP structure itself unless the
    901  1.64      yamt 	 * caller wants to recycle them.  Also, free the scheduler specific data.
    902  1.52        ad 	 *
    903  1.52        ad 	 * We can't return turnstile0 to the pool (it didn't come from it),
    904  1.52        ad 	 * so if it comes up just drop it quietly and move on.
    905  1.52        ad 	 *
    906  1.52        ad 	 * We don't recycle the VM resources at this time.
    907  1.19  jdolecek 	 */
    908  1.55        ad 	KERNEL_LOCK(1, curlwp);		/* XXXSMP */
    909  1.64      yamt 
    910  1.64      yamt 	sched_lwp_exit(l);
    911  1.64      yamt 
    912  1.52        ad 	if (!recycle && l->l_ts != &turnstile0)
    913  1.52        ad 		pool_cache_put(&turnstile_cache, l->l_ts);
    914  1.52        ad #ifndef __NO_CPU_LWP_FREE
    915  1.52        ad 	cpu_lwp_free2(l);
    916  1.52        ad #endif
    917  1.19  jdolecek 	uvm_lwp_exit(l);
    918  1.60      yamt 	KASSERT(SLIST_EMPTY(&l->l_pi_lenders));
    919  1.60      yamt 	KASSERT(l->l_inheritedprio == MAXPRI);
    920  1.52        ad 	if (!recycle)
    921  1.19  jdolecek 		pool_put(&lwp_pool, l);
    922  1.55        ad 	KERNEL_UNLOCK_ONE(curlwp);	/* XXXSMP */
    923   1.2   thorpej }
    924   1.2   thorpej 
    925   1.2   thorpej /*
    926   1.2   thorpej  * Pick a LWP to represent the process for those operations which
    927   1.2   thorpej  * want information about a "process" that is actually associated
    928   1.2   thorpej  * with a LWP.
    929  1.52        ad  *
    930  1.52        ad  * If 'locking' is false, no locking or lock checks are performed.
    931  1.52        ad  * This is intended for use by DDB.
    932  1.52        ad  *
    933  1.52        ad  * We don't bother locking the LWP here, since code that uses this
    934  1.52        ad  * interface is broken by design and an exact match is not required.
    935   1.2   thorpej  */
    936   1.2   thorpej struct lwp *
    937  1.52        ad proc_representative_lwp(struct proc *p, int *nrlwps, int locking)
    938   1.2   thorpej {
    939   1.2   thorpej 	struct lwp *l, *onproc, *running, *sleeping, *stopped, *suspended;
    940  1.27      matt 	struct lwp *signalled;
    941  1.52        ad 	int cnt;
    942  1.52        ad 
    943  1.52        ad 	if (locking) {
    944  1.63        ad 		KASSERT(mutex_owned(&p->p_smutex));
    945  1.52        ad 	}
    946   1.2   thorpej 
    947   1.2   thorpej 	/* Trivial case: only one LWP */
    948  1.52        ad 	if (p->p_nlwps == 1) {
    949  1.52        ad 		l = LIST_FIRST(&p->p_lwps);
    950  1.52        ad 		if (nrlwps)
    951  1.68       tnn 			*nrlwps = (l->l_stat == LSONPROC || l->l_stat == LSRUN);
    952  1.52        ad 		return l;
    953  1.52        ad 	}
    954   1.2   thorpej 
    955  1.52        ad 	cnt = 0;
    956   1.2   thorpej 	switch (p->p_stat) {
    957   1.2   thorpej 	case SSTOP:
    958   1.2   thorpej 	case SACTIVE:
    959   1.2   thorpej 		/* Pick the most live LWP */
    960   1.2   thorpej 		onproc = running = sleeping = stopped = suspended = NULL;
    961  1.27      matt 		signalled = NULL;
    962   1.2   thorpej 		LIST_FOREACH(l, &p->p_lwps, l_sibling) {
    963  1.64      yamt 			if ((l->l_flag & LW_IDLE) != 0) {
    964  1.64      yamt 				continue;
    965  1.64      yamt 			}
    966  1.27      matt 			if (l->l_lid == p->p_sigctx.ps_lwp)
    967  1.27      matt 				signalled = l;
    968   1.2   thorpej 			switch (l->l_stat) {
    969   1.2   thorpej 			case LSONPROC:
    970   1.2   thorpej 				onproc = l;
    971  1.52        ad 				cnt++;
    972   1.2   thorpej 				break;
    973   1.2   thorpej 			case LSRUN:
    974   1.2   thorpej 				running = l;
    975  1.52        ad 				cnt++;
    976   1.2   thorpej 				break;
    977   1.2   thorpej 			case LSSLEEP:
    978   1.2   thorpej 				sleeping = l;
    979   1.2   thorpej 				break;
    980   1.2   thorpej 			case LSSTOP:
    981   1.2   thorpej 				stopped = l;
    982   1.2   thorpej 				break;
    983   1.2   thorpej 			case LSSUSPENDED:
    984   1.2   thorpej 				suspended = l;
    985   1.2   thorpej 				break;
    986   1.2   thorpej 			}
    987   1.2   thorpej 		}
    988  1.52        ad 		if (nrlwps)
    989  1.52        ad 			*nrlwps = cnt;
    990  1.27      matt 		if (signalled)
    991  1.52        ad 			l = signalled;
    992  1.52        ad 		else if (onproc)
    993  1.52        ad 			l = onproc;
    994  1.52        ad 		else if (running)
    995  1.52        ad 			l = running;
    996  1.52        ad 		else if (sleeping)
    997  1.52        ad 			l = sleeping;
    998  1.52        ad 		else if (stopped)
    999  1.52        ad 			l = stopped;
   1000  1.52        ad 		else if (suspended)
   1001  1.52        ad 			l = suspended;
   1002  1.52        ad 		else
   1003  1.52        ad 			break;
   1004  1.52        ad 		return l;
   1005   1.2   thorpej #ifdef DIAGNOSTIC
   1006   1.2   thorpej 	case SIDL:
   1007  1.52        ad 	case SZOMB:
   1008  1.52        ad 	case SDYING:
   1009  1.52        ad 	case SDEAD:
   1010  1.52        ad 		if (locking)
   1011  1.52        ad 			mutex_exit(&p->p_smutex);
   1012   1.2   thorpej 		/* We have more than one LWP and we're in SIDL?
   1013   1.2   thorpej 		 * How'd that happen?
   1014   1.2   thorpej 		 */
   1015  1.52        ad 		panic("Too many LWPs in idle/dying process %d (%s) stat = %d",
   1016  1.52        ad 		    p->p_pid, p->p_comm, p->p_stat);
   1017  1.52        ad 		break;
   1018   1.2   thorpej 	default:
   1019  1.52        ad 		if (locking)
   1020  1.52        ad 			mutex_exit(&p->p_smutex);
   1021   1.2   thorpej 		panic("Process %d (%s) in unknown state %d",
   1022   1.2   thorpej 		    p->p_pid, p->p_comm, p->p_stat);
   1023   1.2   thorpej #endif
   1024   1.2   thorpej 	}
   1025   1.2   thorpej 
   1026  1.52        ad 	if (locking)
   1027  1.52        ad 		mutex_exit(&p->p_smutex);
   1028   1.2   thorpej 	panic("proc_representative_lwp: couldn't find a lwp for process"
   1029   1.2   thorpej 		" %d (%s)", p->p_pid, p->p_comm);
   1030   1.2   thorpej 	/* NOTREACHED */
   1031   1.2   thorpej 	return NULL;
   1032   1.2   thorpej }
   1033  1.37        ad 
   1034  1.37        ad /*
   1035  1.52        ad  * Look up a live LWP within the speicifed process, and return it locked.
   1036  1.52        ad  *
   1037  1.52        ad  * Must be called with p->p_smutex held.
   1038  1.52        ad  */
   1039  1.52        ad struct lwp *
   1040  1.52        ad lwp_find(struct proc *p, int id)
   1041  1.52        ad {
   1042  1.52        ad 	struct lwp *l;
   1043  1.52        ad 
   1044  1.63        ad 	KASSERT(mutex_owned(&p->p_smutex));
   1045  1.52        ad 
   1046  1.52        ad 	LIST_FOREACH(l, &p->p_lwps, l_sibling) {
   1047  1.52        ad 		if (l->l_lid == id)
   1048  1.52        ad 			break;
   1049  1.52        ad 	}
   1050  1.52        ad 
   1051  1.52        ad 	/*
   1052  1.52        ad 	 * No need to lock - all of these conditions will
   1053  1.52        ad 	 * be visible with the process level mutex held.
   1054  1.52        ad 	 */
   1055  1.52        ad 	if (l != NULL && (l->l_stat == LSIDL || l->l_stat == LSZOMB))
   1056  1.52        ad 		l = NULL;
   1057  1.52        ad 
   1058  1.52        ad 	return l;
   1059  1.52        ad }
   1060  1.52        ad 
   1061  1.52        ad /*
   1062  1.37        ad  * Update an LWP's cached credentials to mirror the process' master copy.
   1063  1.37        ad  *
   1064  1.37        ad  * This happens early in the syscall path, on user trap, and on LWP
   1065  1.37        ad  * creation.  A long-running LWP can also voluntarily choose to update
   1066  1.37        ad  * it's credentials by calling this routine.  This may be called from
   1067  1.37        ad  * LWP_CACHE_CREDS(), which checks l->l_cred != p->p_cred beforehand.
   1068  1.37        ad  */
   1069  1.37        ad void
   1070  1.37        ad lwp_update_creds(struct lwp *l)
   1071  1.37        ad {
   1072  1.37        ad 	kauth_cred_t oc;
   1073  1.37        ad 	struct proc *p;
   1074  1.37        ad 
   1075  1.37        ad 	p = l->l_proc;
   1076  1.37        ad 	oc = l->l_cred;
   1077  1.37        ad 
   1078  1.52        ad 	mutex_enter(&p->p_mutex);
   1079  1.37        ad 	kauth_cred_hold(p->p_cred);
   1080  1.37        ad 	l->l_cred = p->p_cred;
   1081  1.52        ad 	mutex_exit(&p->p_mutex);
   1082  1.52        ad 	if (oc != NULL) {
   1083  1.52        ad 		KERNEL_LOCK(1, l);	/* XXXSMP */
   1084  1.37        ad 		kauth_cred_free(oc);
   1085  1.52        ad 		KERNEL_UNLOCK_ONE(l);	/* XXXSMP */
   1086  1.52        ad 	}
   1087  1.52        ad }
   1088  1.52        ad 
   1089  1.52        ad /*
   1090  1.52        ad  * Verify that an LWP is locked, and optionally verify that the lock matches
   1091  1.52        ad  * one we specify.
   1092  1.52        ad  */
   1093  1.52        ad int
   1094  1.52        ad lwp_locked(struct lwp *l, kmutex_t *mtx)
   1095  1.52        ad {
   1096  1.52        ad 	kmutex_t *cur = l->l_mutex;
   1097  1.52        ad 
   1098  1.52        ad 	return mutex_owned(cur) && (mtx == cur || mtx == NULL);
   1099  1.52        ad }
   1100  1.52        ad 
   1101  1.52        ad /*
   1102  1.52        ad  * Lock an LWP.
   1103  1.52        ad  */
   1104  1.52        ad void
   1105  1.52        ad lwp_lock_retry(struct lwp *l, kmutex_t *old)
   1106  1.52        ad {
   1107  1.52        ad 
   1108  1.52        ad 	/*
   1109  1.52        ad 	 * XXXgcc ignoring kmutex_t * volatile on i386
   1110  1.52        ad 	 *
   1111  1.52        ad 	 * gcc version 4.1.2 20061021 prerelease (NetBSD nb1 20061021)
   1112  1.52        ad 	 */
   1113  1.52        ad #if 1
   1114  1.52        ad 	while (l->l_mutex != old) {
   1115  1.52        ad #else
   1116  1.52        ad 	for (;;) {
   1117  1.52        ad #endif
   1118  1.52        ad 		mutex_spin_exit(old);
   1119  1.52        ad 		old = l->l_mutex;
   1120  1.52        ad 		mutex_spin_enter(old);
   1121  1.52        ad 
   1122  1.52        ad 		/*
   1123  1.52        ad 		 * mutex_enter() will have posted a read barrier.  Re-test
   1124  1.52        ad 		 * l->l_mutex.  If it has changed, we need to try again.
   1125  1.52        ad 		 */
   1126  1.52        ad #if 1
   1127  1.52        ad 	}
   1128  1.52        ad #else
   1129  1.52        ad 	} while (__predict_false(l->l_mutex != old));
   1130  1.52        ad #endif
   1131  1.52        ad }
   1132  1.52        ad 
   1133  1.52        ad /*
   1134  1.52        ad  * Lend a new mutex to an LWP.  The old mutex must be held.
   1135  1.52        ad  */
   1136  1.52        ad void
   1137  1.52        ad lwp_setlock(struct lwp *l, kmutex_t *new)
   1138  1.52        ad {
   1139  1.52        ad 
   1140  1.63        ad 	KASSERT(mutex_owned(l->l_mutex));
   1141  1.52        ad 
   1142  1.52        ad 	mb_write();
   1143  1.52        ad 	l->l_mutex = new;
   1144  1.52        ad }
   1145  1.52        ad 
   1146  1.52        ad /*
   1147  1.52        ad  * Lend a new mutex to an LWP, and release the old mutex.  The old mutex
   1148  1.52        ad  * must be held.
   1149  1.52        ad  */
   1150  1.52        ad void
   1151  1.52        ad lwp_unlock_to(struct lwp *l, kmutex_t *new)
   1152  1.52        ad {
   1153  1.52        ad 	kmutex_t *old;
   1154  1.52        ad 
   1155  1.63        ad 	KASSERT(mutex_owned(l->l_mutex));
   1156  1.52        ad 
   1157  1.52        ad 	old = l->l_mutex;
   1158  1.52        ad 	mb_write();
   1159  1.52        ad 	l->l_mutex = new;
   1160  1.52        ad 	mutex_spin_exit(old);
   1161  1.52        ad }
   1162  1.52        ad 
   1163  1.52        ad /*
   1164  1.52        ad  * Acquire a new mutex, and donate it to an LWP.  The LWP must already be
   1165  1.52        ad  * locked.
   1166  1.52        ad  */
   1167  1.52        ad void
   1168  1.52        ad lwp_relock(struct lwp *l, kmutex_t *new)
   1169  1.52        ad {
   1170  1.52        ad 	kmutex_t *old;
   1171  1.52        ad 
   1172  1.63        ad 	KASSERT(mutex_owned(l->l_mutex));
   1173  1.52        ad 
   1174  1.52        ad 	old = l->l_mutex;
   1175  1.52        ad 	if (old != new) {
   1176  1.52        ad 		mutex_spin_enter(new);
   1177  1.52        ad 		l->l_mutex = new;
   1178  1.52        ad 		mutex_spin_exit(old);
   1179  1.52        ad 	}
   1180  1.52        ad }
   1181  1.52        ad 
   1182  1.60      yamt int
   1183  1.60      yamt lwp_trylock(struct lwp *l)
   1184  1.60      yamt {
   1185  1.60      yamt 	kmutex_t *old;
   1186  1.60      yamt 
   1187  1.60      yamt 	for (;;) {
   1188  1.60      yamt 		if (!mutex_tryenter(old = l->l_mutex))
   1189  1.60      yamt 			return 0;
   1190  1.60      yamt 		if (__predict_true(l->l_mutex == old))
   1191  1.60      yamt 			return 1;
   1192  1.60      yamt 		mutex_spin_exit(old);
   1193  1.60      yamt 	}
   1194  1.60      yamt }
   1195  1.60      yamt 
   1196  1.52        ad /*
   1197  1.56     pavel  * Handle exceptions for mi_userret().  Called if a member of LW_USERRET is
   1198  1.52        ad  * set.
   1199  1.52        ad  */
   1200  1.52        ad void
   1201  1.52        ad lwp_userret(struct lwp *l)
   1202  1.52        ad {
   1203  1.52        ad 	struct proc *p;
   1204  1.54        ad 	void (*hook)(void);
   1205  1.52        ad 	int sig;
   1206  1.52        ad 
   1207  1.52        ad 	p = l->l_proc;
   1208  1.52        ad 
   1209  1.52        ad 	/*
   1210  1.52        ad 	 * It should be safe to do this read unlocked on a multiprocessor
   1211  1.52        ad 	 * system..
   1212  1.52        ad 	 */
   1213  1.56     pavel 	while ((l->l_flag & LW_USERRET) != 0) {
   1214  1.52        ad 		/*
   1215  1.52        ad 		 * Process pending signals first, unless the process
   1216  1.61        ad 		 * is dumping core or exiting, where we will instead
   1217  1.61        ad 		 * enter the L_WSUSPEND case below.
   1218  1.52        ad 		 */
   1219  1.61        ad 		if ((l->l_flag & (LW_PENDSIG | LW_WCORE | LW_WEXIT)) ==
   1220  1.61        ad 		    LW_PENDSIG) {
   1221  1.52        ad 			KERNEL_LOCK(1, l);	/* XXXSMP pool_put() below */
   1222  1.52        ad 			mutex_enter(&p->p_smutex);
   1223  1.52        ad 			while ((sig = issignal(l)) != 0)
   1224  1.52        ad 				postsig(sig);
   1225  1.52        ad 			mutex_exit(&p->p_smutex);
   1226  1.52        ad 			KERNEL_UNLOCK_LAST(l);	/* XXXSMP */
   1227  1.52        ad 		}
   1228  1.52        ad 
   1229  1.52        ad 		/*
   1230  1.52        ad 		 * Core-dump or suspend pending.
   1231  1.52        ad 		 *
   1232  1.52        ad 		 * In case of core dump, suspend ourselves, so that the
   1233  1.52        ad 		 * kernel stack and therefore the userland registers saved
   1234  1.52        ad 		 * in the trapframe are around for coredump() to write them
   1235  1.52        ad 		 * out.  We issue a wakeup on p->p_lwpcv so that sigexit()
   1236  1.52        ad 		 * will write the core file out once all other LWPs are
   1237  1.52        ad 		 * suspended.
   1238  1.52        ad 		 */
   1239  1.56     pavel 		if ((l->l_flag & LW_WSUSPEND) != 0) {
   1240  1.52        ad 			mutex_enter(&p->p_smutex);
   1241  1.52        ad 			p->p_nrlwps--;
   1242  1.52        ad 			cv_broadcast(&p->p_lwpcv);
   1243  1.52        ad 			lwp_lock(l);
   1244  1.52        ad 			l->l_stat = LSSUSPENDED;
   1245  1.52        ad 			mutex_exit(&p->p_smutex);
   1246  1.64      yamt 			mi_switch(l);
   1247  1.52        ad 		}
   1248  1.52        ad 
   1249  1.52        ad 		/* Process is exiting. */
   1250  1.56     pavel 		if ((l->l_flag & LW_WEXIT) != 0) {
   1251  1.52        ad 			KERNEL_LOCK(1, l);
   1252  1.52        ad 			lwp_exit(l);
   1253  1.52        ad 			KASSERT(0);
   1254  1.52        ad 			/* NOTREACHED */
   1255  1.52        ad 		}
   1256  1.54        ad 
   1257  1.54        ad 		/* Call userret hook; used by Linux emulation. */
   1258  1.56     pavel 		if ((l->l_flag & LW_WUSERRET) != 0) {
   1259  1.54        ad 			lwp_lock(l);
   1260  1.56     pavel 			l->l_flag &= ~LW_WUSERRET;
   1261  1.54        ad 			lwp_unlock(l);
   1262  1.54        ad 			hook = p->p_userret;
   1263  1.54        ad 			p->p_userret = NULL;
   1264  1.54        ad 			(*hook)();
   1265  1.54        ad 		}
   1266  1.52        ad 	}
   1267  1.52        ad }
   1268  1.52        ad 
   1269  1.52        ad /*
   1270  1.52        ad  * Force an LWP to enter the kernel, to take a trip through lwp_userret().
   1271  1.52        ad  */
   1272  1.52        ad void
   1273  1.52        ad lwp_need_userret(struct lwp *l)
   1274  1.52        ad {
   1275  1.63        ad 	KASSERT(lwp_locked(l, NULL));
   1276  1.52        ad 
   1277  1.52        ad 	/*
   1278  1.52        ad 	 * Since the tests in lwp_userret() are done unlocked, make sure
   1279  1.52        ad 	 * that the condition will be seen before forcing the LWP to enter
   1280  1.52        ad 	 * kernel mode.
   1281  1.52        ad 	 */
   1282  1.52        ad 	mb_write();
   1283  1.52        ad 	cpu_signotify(l);
   1284  1.52        ad }
   1285  1.52        ad 
   1286  1.52        ad /*
   1287  1.52        ad  * Add one reference to an LWP.  This will prevent the LWP from
   1288  1.52        ad  * exiting, thus keep the lwp structure and PCB around to inspect.
   1289  1.52        ad  */
   1290  1.52        ad void
   1291  1.52        ad lwp_addref(struct lwp *l)
   1292  1.52        ad {
   1293  1.52        ad 
   1294  1.63        ad 	KASSERT(mutex_owned(&l->l_proc->p_smutex));
   1295  1.52        ad 	KASSERT(l->l_stat != LSZOMB);
   1296  1.52        ad 	KASSERT(l->l_refcnt != 0);
   1297  1.52        ad 
   1298  1.52        ad 	l->l_refcnt++;
   1299  1.52        ad }
   1300  1.52        ad 
   1301  1.52        ad /*
   1302  1.52        ad  * Remove one reference to an LWP.  If this is the last reference,
   1303  1.52        ad  * then we must finalize the LWP's death.
   1304  1.52        ad  */
   1305  1.52        ad void
   1306  1.52        ad lwp_delref(struct lwp *l)
   1307  1.52        ad {
   1308  1.52        ad 	struct proc *p = l->l_proc;
   1309  1.52        ad 
   1310  1.52        ad 	mutex_enter(&p->p_smutex);
   1311  1.72        ad 	KASSERT(l->l_stat != LSZOMB);
   1312  1.72        ad 	KASSERT(l->l_refcnt > 0);
   1313  1.52        ad 	if (--l->l_refcnt == 0)
   1314  1.52        ad 		cv_broadcast(&p->p_refcv);
   1315  1.52        ad 	mutex_exit(&p->p_smutex);
   1316  1.52        ad }
   1317  1.52        ad 
   1318  1.52        ad /*
   1319  1.52        ad  * Drain all references to the current LWP.
   1320  1.52        ad  */
   1321  1.52        ad void
   1322  1.52        ad lwp_drainrefs(struct lwp *l)
   1323  1.52        ad {
   1324  1.52        ad 	struct proc *p = l->l_proc;
   1325  1.52        ad 
   1326  1.63        ad 	KASSERT(mutex_owned(&p->p_smutex));
   1327  1.52        ad 	KASSERT(l->l_refcnt != 0);
   1328  1.52        ad 
   1329  1.52        ad 	l->l_refcnt--;
   1330  1.52        ad 	while (l->l_refcnt != 0)
   1331  1.52        ad 		cv_wait(&p->p_refcv, &p->p_smutex);
   1332  1.37        ad }
   1333  1.41   thorpej 
   1334  1.41   thorpej /*
   1335  1.41   thorpej  * lwp_specific_key_create --
   1336  1.41   thorpej  *	Create a key for subsystem lwp-specific data.
   1337  1.41   thorpej  */
   1338  1.41   thorpej int
   1339  1.41   thorpej lwp_specific_key_create(specificdata_key_t *keyp, specificdata_dtor_t dtor)
   1340  1.41   thorpej {
   1341  1.41   thorpej 
   1342  1.45   thorpej 	return (specificdata_key_create(lwp_specificdata_domain, keyp, dtor));
   1343  1.41   thorpej }
   1344  1.41   thorpej 
   1345  1.41   thorpej /*
   1346  1.41   thorpej  * lwp_specific_key_delete --
   1347  1.41   thorpej  *	Delete a key for subsystem lwp-specific data.
   1348  1.41   thorpej  */
   1349  1.41   thorpej void
   1350  1.41   thorpej lwp_specific_key_delete(specificdata_key_t key)
   1351  1.41   thorpej {
   1352  1.41   thorpej 
   1353  1.41   thorpej 	specificdata_key_delete(lwp_specificdata_domain, key);
   1354  1.41   thorpej }
   1355  1.41   thorpej 
   1356  1.45   thorpej /*
   1357  1.45   thorpej  * lwp_initspecific --
   1358  1.45   thorpej  *	Initialize an LWP's specificdata container.
   1359  1.45   thorpej  */
   1360  1.42  christos void
   1361  1.42  christos lwp_initspecific(struct lwp *l)
   1362  1.42  christos {
   1363  1.42  christos 	int error;
   1364  1.45   thorpej 
   1365  1.42  christos 	error = specificdata_init(lwp_specificdata_domain, &l->l_specdataref);
   1366  1.42  christos 	KASSERT(error == 0);
   1367  1.42  christos }
   1368  1.42  christos 
   1369  1.41   thorpej /*
   1370  1.45   thorpej  * lwp_finispecific --
   1371  1.45   thorpej  *	Finalize an LWP's specificdata container.
   1372  1.45   thorpej  */
   1373  1.45   thorpej void
   1374  1.45   thorpej lwp_finispecific(struct lwp *l)
   1375  1.45   thorpej {
   1376  1.45   thorpej 
   1377  1.45   thorpej 	specificdata_fini(lwp_specificdata_domain, &l->l_specdataref);
   1378  1.45   thorpej }
   1379  1.45   thorpej 
   1380  1.45   thorpej /*
   1381  1.41   thorpej  * lwp_getspecific --
   1382  1.41   thorpej  *	Return lwp-specific data corresponding to the specified key.
   1383  1.41   thorpej  *
   1384  1.41   thorpej  *	Note: LWP specific data is NOT INTERLOCKED.  An LWP should access
   1385  1.41   thorpej  *	only its OWN SPECIFIC DATA.  If it is necessary to access another
   1386  1.41   thorpej  *	LWP's specifc data, care must be taken to ensure that doing so
   1387  1.41   thorpej  *	would not cause internal data structure inconsistency (i.e. caller
   1388  1.41   thorpej  *	can guarantee that the target LWP is not inside an lwp_getspecific()
   1389  1.41   thorpej  *	or lwp_setspecific() call).
   1390  1.41   thorpej  */
   1391  1.41   thorpej void *
   1392  1.44   thorpej lwp_getspecific(specificdata_key_t key)
   1393  1.41   thorpej {
   1394  1.41   thorpej 
   1395  1.41   thorpej 	return (specificdata_getspecific_unlocked(lwp_specificdata_domain,
   1396  1.44   thorpej 						  &curlwp->l_specdataref, key));
   1397  1.41   thorpej }
   1398  1.41   thorpej 
   1399  1.47   hannken void *
   1400  1.47   hannken _lwp_getspecific_by_lwp(struct lwp *l, specificdata_key_t key)
   1401  1.47   hannken {
   1402  1.47   hannken 
   1403  1.47   hannken 	return (specificdata_getspecific_unlocked(lwp_specificdata_domain,
   1404  1.47   hannken 						  &l->l_specdataref, key));
   1405  1.47   hannken }
   1406  1.47   hannken 
   1407  1.41   thorpej /*
   1408  1.41   thorpej  * lwp_setspecific --
   1409  1.41   thorpej  *	Set lwp-specific data corresponding to the specified key.
   1410  1.41   thorpej  */
   1411  1.41   thorpej void
   1412  1.45   thorpej lwp_setspecific(specificdata_key_t key, void *data)
   1413  1.41   thorpej {
   1414  1.41   thorpej 
   1415  1.41   thorpej 	specificdata_setspecific(lwp_specificdata_domain,
   1416  1.44   thorpej 				 &curlwp->l_specdataref, key, data);
   1417  1.41   thorpej }
   1418