Home | History | Annotate | Line # | Download | only in kern
kern_lwp.c revision 1.125.2.3
      1  1.125.2.3     skrll /*	$NetBSD: kern_lwp.c,v 1.125.2.3 2009/04/28 07:36:59 skrll Exp $	*/
      2        1.2   thorpej 
      3        1.2   thorpej /*-
      4  1.125.2.2     skrll  * Copyright (c) 2001, 2006, 2007, 2008, 2009 The NetBSD Foundation, Inc.
      5        1.2   thorpej  * All rights reserved.
      6        1.2   thorpej  *
      7        1.2   thorpej  * This code is derived from software contributed to The NetBSD Foundation
      8       1.52        ad  * by Nathan J. Williams, and Andrew Doran.
      9        1.2   thorpej  *
     10        1.2   thorpej  * Redistribution and use in source and binary forms, with or without
     11        1.2   thorpej  * modification, are permitted provided that the following conditions
     12        1.2   thorpej  * are met:
     13        1.2   thorpej  * 1. Redistributions of source code must retain the above copyright
     14        1.2   thorpej  *    notice, this list of conditions and the following disclaimer.
     15        1.2   thorpej  * 2. Redistributions in binary form must reproduce the above copyright
     16        1.2   thorpej  *    notice, this list of conditions and the following disclaimer in the
     17        1.2   thorpej  *    documentation and/or other materials provided with the distribution.
     18        1.2   thorpej  *
     19        1.2   thorpej  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     20        1.2   thorpej  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     21        1.2   thorpej  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     22        1.2   thorpej  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     23        1.2   thorpej  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     24        1.2   thorpej  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     25        1.2   thorpej  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     26        1.2   thorpej  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     27        1.2   thorpej  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     28        1.2   thorpej  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     29        1.2   thorpej  * POSSIBILITY OF SUCH DAMAGE.
     30        1.2   thorpej  */
     31        1.9     lukem 
     32       1.52        ad /*
     33       1.52        ad  * Overview
     34       1.52        ad  *
     35       1.66        ad  *	Lightweight processes (LWPs) are the basic unit or thread of
     36       1.52        ad  *	execution within the kernel.  The core state of an LWP is described
     37       1.66        ad  *	by "struct lwp", also known as lwp_t.
     38       1.52        ad  *
     39       1.52        ad  *	Each LWP is contained within a process (described by "struct proc"),
     40       1.52        ad  *	Every process contains at least one LWP, but may contain more.  The
     41       1.52        ad  *	process describes attributes shared among all of its LWPs such as a
     42       1.52        ad  *	private address space, global execution state (stopped, active,
     43       1.52        ad  *	zombie, ...), signal disposition and so on.  On a multiprocessor
     44       1.66        ad  *	machine, multiple LWPs be executing concurrently in the kernel.
     45       1.52        ad  *
     46       1.52        ad  * Execution states
     47       1.52        ad  *
     48       1.52        ad  *	At any given time, an LWP has overall state that is described by
     49       1.52        ad  *	lwp::l_stat.  The states are broken into two sets below.  The first
     50       1.52        ad  *	set is guaranteed to represent the absolute, current state of the
     51       1.52        ad  *	LWP:
     52      1.101     rmind  *
     53      1.101     rmind  *	LSONPROC
     54      1.101     rmind  *
     55      1.101     rmind  *		On processor: the LWP is executing on a CPU, either in the
     56      1.101     rmind  *		kernel or in user space.
     57      1.101     rmind  *
     58      1.101     rmind  *	LSRUN
     59      1.101     rmind  *
     60      1.101     rmind  *		Runnable: the LWP is parked on a run queue, and may soon be
     61      1.101     rmind  *		chosen to run by an idle processor, or by a processor that
     62      1.101     rmind  *		has been asked to preempt a currently runnning but lower
     63      1.101     rmind  *		priority LWP.  If the LWP is not swapped in (LW_INMEM == 0)
     64       1.52        ad  *		then the LWP is not on a run queue, but may be soon.
     65      1.101     rmind  *
     66      1.101     rmind  *	LSIDL
     67      1.101     rmind  *
     68      1.101     rmind  *		Idle: the LWP has been created but has not yet executed,
     69       1.66        ad  *		or it has ceased executing a unit of work and is waiting
     70       1.66        ad  *		to be started again.
     71      1.101     rmind  *
     72      1.101     rmind  *	LSSUSPENDED:
     73      1.101     rmind  *
     74      1.101     rmind  *		Suspended: the LWP has had its execution suspended by
     75       1.52        ad  *		another LWP in the same process using the _lwp_suspend()
     76       1.52        ad  *		system call.  User-level LWPs also enter the suspended
     77       1.52        ad  *		state when the system is shutting down.
     78       1.52        ad  *
     79       1.52        ad  *	The second set represent a "statement of intent" on behalf of the
     80       1.52        ad  *	LWP.  The LWP may in fact be executing on a processor, may be
     81       1.66        ad  *	sleeping or idle. It is expected to take the necessary action to
     82      1.101     rmind  *	stop executing or become "running" again within a short timeframe.
     83      1.115        ad  *	The LP_RUNNING flag in lwp::l_pflag indicates that an LWP is running.
     84      1.101     rmind  *	Importantly, it indicates that its state is tied to a CPU.
     85      1.101     rmind  *
     86      1.101     rmind  *	LSZOMB:
     87      1.101     rmind  *
     88      1.101     rmind  *		Dead or dying: the LWP has released most of its resources
     89  1.125.2.3     skrll  *		and is about to switch away into oblivion, or has already
     90       1.66        ad  *		switched away.  When it switches away, its few remaining
     91       1.66        ad  *		resources can be collected.
     92      1.101     rmind  *
     93      1.101     rmind  *	LSSLEEP:
     94      1.101     rmind  *
     95      1.101     rmind  *		Sleeping: the LWP has entered itself onto a sleep queue, and
     96      1.101     rmind  *		has switched away or will switch away shortly to allow other
     97       1.66        ad  *		LWPs to run on the CPU.
     98      1.101     rmind  *
     99      1.101     rmind  *	LSSTOP:
    100      1.101     rmind  *
    101      1.101     rmind  *		Stopped: the LWP has been stopped as a result of a job
    102      1.101     rmind  *		control signal, or as a result of the ptrace() interface.
    103      1.101     rmind  *
    104      1.101     rmind  *		Stopped LWPs may run briefly within the kernel to handle
    105      1.101     rmind  *		signals that they receive, but will not return to user space
    106      1.101     rmind  *		until their process' state is changed away from stopped.
    107      1.101     rmind  *
    108      1.101     rmind  *		Single LWPs within a process can not be set stopped
    109      1.101     rmind  *		selectively: all actions that can stop or continue LWPs
    110      1.101     rmind  *		occur at the process level.
    111      1.101     rmind  *
    112       1.52        ad  * State transitions
    113       1.52        ad  *
    114       1.66        ad  *	Note that the LSSTOP state may only be set when returning to
    115       1.66        ad  *	user space in userret(), or when sleeping interruptably.  The
    116       1.66        ad  *	LSSUSPENDED state may only be set in userret().  Before setting
    117       1.66        ad  *	those states, we try to ensure that the LWPs will release all
    118       1.66        ad  *	locks that they hold, and at a minimum try to ensure that the
    119       1.66        ad  *	LWP can be set runnable again by a signal.
    120       1.52        ad  *
    121       1.52        ad  *	LWPs may transition states in the following ways:
    122       1.52        ad  *
    123       1.52        ad  *	 RUN -------> ONPROC		ONPROC -----> RUN
    124  1.125.2.3     skrll  *		    				    > SLEEP
    125  1.125.2.3     skrll  *		    				    > STOPPED
    126       1.52        ad  *						    > SUSPENDED
    127       1.52        ad  *						    > ZOMB
    128  1.125.2.3     skrll  *						    > IDL (special cases)
    129       1.52        ad  *
    130       1.52        ad  *	 STOPPED ---> RUN		SUSPENDED --> RUN
    131  1.125.2.3     skrll  *	            > SLEEP
    132       1.52        ad  *
    133       1.52        ad  *	 SLEEP -----> ONPROC		IDL --------> RUN
    134      1.101     rmind  *		    > RUN			    > SUSPENDED
    135      1.101     rmind  *		    > STOPPED			    > STOPPED
    136  1.125.2.3     skrll  *						    > ONPROC (special cases)
    137       1.52        ad  *
    138  1.125.2.3     skrll  *	Some state transitions are only possible with kernel threads (eg
    139  1.125.2.3     skrll  *	ONPROC -> IDL) and happen under tightly controlled circumstances
    140  1.125.2.3     skrll  *	free of unwanted side effects.
    141       1.66        ad  *
    142      1.114     rmind  * Migration
    143      1.114     rmind  *
    144      1.114     rmind  *	Migration of threads from one CPU to another could be performed
    145      1.114     rmind  *	internally by the scheduler via sched_takecpu() or sched_catchlwp()
    146      1.114     rmind  *	functions.  The universal lwp_migrate() function should be used for
    147      1.114     rmind  *	any other cases.  Subsystems in the kernel must be aware that CPU
    148      1.114     rmind  *	of LWP may change, while it is not locked.
    149      1.114     rmind  *
    150       1.52        ad  * Locking
    151       1.52        ad  *
    152       1.52        ad  *	The majority of fields in 'struct lwp' are covered by a single,
    153       1.66        ad  *	general spin lock pointed to by lwp::l_mutex.  The locks covering
    154       1.52        ad  *	each field are documented in sys/lwp.h.
    155       1.52        ad  *
    156       1.66        ad  *	State transitions must be made with the LWP's general lock held,
    157      1.101     rmind  *	and may cause the LWP's lock pointer to change. Manipulation of
    158       1.66        ad  *	the general lock is not performed directly, but through calls to
    159       1.66        ad  *	lwp_lock(), lwp_relock() and similar.
    160       1.52        ad  *
    161       1.52        ad  *	States and their associated locks:
    162       1.52        ad  *
    163       1.74     rmind  *	LSONPROC, LSZOMB:
    164       1.52        ad  *
    165       1.64      yamt  *		Always covered by spc_lwplock, which protects running LWPs.
    166  1.125.2.3     skrll  *		This is a per-CPU lock and matches lwp::l_cpu.
    167       1.52        ad  *
    168       1.74     rmind  *	LSIDL, LSRUN:
    169       1.52        ad  *
    170       1.64      yamt  *		Always covered by spc_mutex, which protects the run queues.
    171  1.125.2.3     skrll  *		This is a per-CPU lock and matches lwp::l_cpu.
    172       1.52        ad  *
    173       1.52        ad  *	LSSLEEP:
    174       1.52        ad  *
    175       1.66        ad  *		Covered by a lock associated with the sleep queue that the
    176  1.125.2.3     skrll  *		LWP resides on.  Matches lwp::l_sleepq::sq_mutex.
    177       1.52        ad  *
    178       1.52        ad  *	LSSTOP, LSSUSPENDED:
    179      1.101     rmind  *
    180       1.52        ad  *		If the LWP was previously sleeping (l_wchan != NULL), then
    181       1.66        ad  *		l_mutex references the sleep queue lock.  If the LWP was
    182       1.52        ad  *		runnable or on the CPU when halted, or has been removed from
    183       1.66        ad  *		the sleep queue since halted, then the lock is spc_lwplock.
    184       1.52        ad  *
    185       1.52        ad  *	The lock order is as follows:
    186       1.52        ad  *
    187       1.64      yamt  *		spc::spc_lwplock ->
    188      1.112        ad  *		    sleeptab::st_mutex ->
    189       1.64      yamt  *			tschain_t::tc_mutex ->
    190       1.64      yamt  *			    spc::spc_mutex
    191       1.52        ad  *
    192      1.103        ad  *	Each process has an scheduler state lock (proc::p_lock), and a
    193       1.52        ad  *	number of counters on LWPs and their states: p_nzlwps, p_nrlwps, and
    194       1.52        ad  *	so on.  When an LWP is to be entered into or removed from one of the
    195      1.103        ad  *	following states, p_lock must be held and the process wide counters
    196       1.52        ad  *	adjusted:
    197       1.52        ad  *
    198       1.52        ad  *		LSIDL, LSZOMB, LSSTOP, LSSUSPENDED
    199       1.52        ad  *
    200  1.125.2.3     skrll  *	(But not always for kernel threads.  There are some special cases
    201  1.125.2.3     skrll  *	as mentioned above.  See kern_softint.c.)
    202  1.125.2.3     skrll  *
    203       1.52        ad  *	Note that an LWP is considered running or likely to run soon if in
    204       1.52        ad  *	one of the following states.  This affects the value of p_nrlwps:
    205       1.52        ad  *
    206       1.52        ad  *		LSRUN, LSONPROC, LSSLEEP
    207       1.52        ad  *
    208      1.103        ad  *	p_lock does not need to be held when transitioning among these
    209  1.125.2.3     skrll  *	three states, hence p_lock is rarely taken for state transitions.
    210       1.52        ad  */
    211       1.52        ad 
    212        1.9     lukem #include <sys/cdefs.h>
    213  1.125.2.3     skrll __KERNEL_RCSID(0, "$NetBSD: kern_lwp.c,v 1.125.2.3 2009/04/28 07:36:59 skrll Exp $");
    214        1.8    martin 
    215       1.84      yamt #include "opt_ddb.h"
    216       1.52        ad #include "opt_lockdebug.h"
    217      1.124  wrstuden #include "opt_sa.h"
    218        1.2   thorpej 
    219       1.47   hannken #define _LWP_API_PRIVATE
    220       1.47   hannken 
    221        1.2   thorpej #include <sys/param.h>
    222        1.2   thorpej #include <sys/systm.h>
    223       1.64      yamt #include <sys/cpu.h>
    224        1.2   thorpej #include <sys/pool.h>
    225        1.2   thorpej #include <sys/proc.h>
    226      1.124  wrstuden #include <sys/sa.h>
    227      1.124  wrstuden #include <sys/savar.h>
    228        1.2   thorpej #include <sys/syscallargs.h>
    229       1.57       dsl #include <sys/syscall_stats.h>
    230       1.37        ad #include <sys/kauth.h>
    231       1.52        ad #include <sys/sleepq.h>
    232       1.85      yamt #include <sys/user.h>
    233       1.52        ad #include <sys/lockdebug.h>
    234       1.52        ad #include <sys/kmem.h>
    235       1.91     rmind #include <sys/pset.h>
    236       1.75        ad #include <sys/intr.h>
    237       1.78        ad #include <sys/lwpctl.h>
    238       1.81        ad #include <sys/atomic.h>
    239        1.2   thorpej 
    240        1.2   thorpej #include <uvm/uvm_extern.h>
    241       1.80     skrll #include <uvm/uvm_object.h>
    242        1.2   thorpej 
    243       1.77      matt struct lwplist	alllwp = LIST_HEAD_INITIALIZER(alllwp);
    244       1.52        ad 
    245       1.41   thorpej POOL_INIT(lwp_uc_pool, sizeof(ucontext_t), 0, 0, 0, "lwpucpl",
    246       1.62        ad     &pool_allocator_nointr, IPL_NONE);
    247       1.41   thorpej 
    248       1.87        ad static pool_cache_t lwp_cache;
    249       1.41   thorpej static specificdata_domain_t lwp_specificdata_domain;
    250       1.41   thorpej 
    251       1.41   thorpej void
    252       1.41   thorpej lwpinit(void)
    253       1.41   thorpej {
    254       1.41   thorpej 
    255       1.41   thorpej 	lwp_specificdata_domain = specificdata_domain_create();
    256       1.41   thorpej 	KASSERT(lwp_specificdata_domain != NULL);
    257       1.52        ad 	lwp_sys_init();
    258       1.87        ad 	lwp_cache = pool_cache_init(sizeof(lwp_t), MIN_LWP_ALIGNMENT, 0, 0,
    259       1.87        ad 	    "lwppl", NULL, IPL_NONE, NULL, NULL, NULL);
    260       1.41   thorpej }
    261       1.41   thorpej 
    262       1.52        ad /*
    263       1.52        ad  * Set an suspended.
    264       1.52        ad  *
    265      1.103        ad  * Must be called with p_lock held, and the LWP locked.  Will unlock the
    266       1.52        ad  * LWP before return.
    267       1.52        ad  */
    268        1.2   thorpej int
    269       1.52        ad lwp_suspend(struct lwp *curl, struct lwp *t)
    270        1.2   thorpej {
    271       1.52        ad 	int error;
    272        1.2   thorpej 
    273      1.103        ad 	KASSERT(mutex_owned(t->l_proc->p_lock));
    274       1.63        ad 	KASSERT(lwp_locked(t, NULL));
    275       1.33       chs 
    276       1.52        ad 	KASSERT(curl != t || curl->l_stat == LSONPROC);
    277        1.2   thorpej 
    278       1.52        ad 	/*
    279       1.52        ad 	 * If the current LWP has been told to exit, we must not suspend anyone
    280       1.52        ad 	 * else or deadlock could occur.  We won't return to userspace.
    281        1.2   thorpej 	 */
    282      1.109     rmind 	if ((curl->l_flag & (LW_WEXIT | LW_WCORE)) != 0) {
    283       1.52        ad 		lwp_unlock(t);
    284       1.52        ad 		return (EDEADLK);
    285        1.2   thorpej 	}
    286        1.2   thorpej 
    287       1.52        ad 	error = 0;
    288        1.2   thorpej 
    289       1.52        ad 	switch (t->l_stat) {
    290       1.52        ad 	case LSRUN:
    291       1.52        ad 	case LSONPROC:
    292       1.56     pavel 		t->l_flag |= LW_WSUSPEND;
    293       1.52        ad 		lwp_need_userret(t);
    294       1.52        ad 		lwp_unlock(t);
    295       1.52        ad 		break;
    296        1.2   thorpej 
    297       1.52        ad 	case LSSLEEP:
    298       1.56     pavel 		t->l_flag |= LW_WSUSPEND;
    299        1.2   thorpej 
    300        1.2   thorpej 		/*
    301       1.52        ad 		 * Kick the LWP and try to get it to the kernel boundary
    302       1.52        ad 		 * so that it will release any locks that it holds.
    303       1.52        ad 		 * setrunnable() will release the lock.
    304        1.2   thorpej 		 */
    305       1.56     pavel 		if ((t->l_flag & LW_SINTR) != 0)
    306       1.52        ad 			setrunnable(t);
    307       1.52        ad 		else
    308       1.52        ad 			lwp_unlock(t);
    309       1.52        ad 		break;
    310        1.2   thorpej 
    311       1.52        ad 	case LSSUSPENDED:
    312       1.52        ad 		lwp_unlock(t);
    313       1.52        ad 		break;
    314       1.17      manu 
    315       1.52        ad 	case LSSTOP:
    316       1.56     pavel 		t->l_flag |= LW_WSUSPEND;
    317       1.52        ad 		setrunnable(t);
    318       1.52        ad 		break;
    319        1.2   thorpej 
    320       1.52        ad 	case LSIDL:
    321       1.52        ad 	case LSZOMB:
    322       1.52        ad 		error = EINTR; /* It's what Solaris does..... */
    323       1.52        ad 		lwp_unlock(t);
    324       1.52        ad 		break;
    325        1.2   thorpej 	}
    326        1.2   thorpej 
    327       1.69     rmind 	return (error);
    328        1.2   thorpej }
    329        1.2   thorpej 
    330       1.52        ad /*
    331       1.52        ad  * Restart a suspended LWP.
    332       1.52        ad  *
    333      1.103        ad  * Must be called with p_lock held, and the LWP locked.  Will unlock the
    334       1.52        ad  * LWP before return.
    335       1.52        ad  */
    336        1.2   thorpej void
    337        1.2   thorpej lwp_continue(struct lwp *l)
    338        1.2   thorpej {
    339        1.2   thorpej 
    340      1.103        ad 	KASSERT(mutex_owned(l->l_proc->p_lock));
    341       1.63        ad 	KASSERT(lwp_locked(l, NULL));
    342       1.52        ad 
    343       1.52        ad 	/* If rebooting or not suspended, then just bail out. */
    344       1.56     pavel 	if ((l->l_flag & LW_WREBOOT) != 0) {
    345       1.52        ad 		lwp_unlock(l);
    346        1.2   thorpej 		return;
    347       1.10      fvdl 	}
    348        1.2   thorpej 
    349       1.56     pavel 	l->l_flag &= ~LW_WSUSPEND;
    350        1.2   thorpej 
    351       1.52        ad 	if (l->l_stat != LSSUSPENDED) {
    352       1.52        ad 		lwp_unlock(l);
    353       1.52        ad 		return;
    354        1.2   thorpej 	}
    355        1.2   thorpej 
    356       1.52        ad 	/* setrunnable() will release the lock. */
    357       1.52        ad 	setrunnable(l);
    358        1.2   thorpej }
    359        1.2   thorpej 
    360       1.52        ad /*
    361       1.52        ad  * Wait for an LWP within the current process to exit.  If 'lid' is
    362       1.52        ad  * non-zero, we are waiting for a specific LWP.
    363       1.52        ad  *
    364      1.103        ad  * Must be called with p->p_lock held.
    365       1.52        ad  */
    366        1.2   thorpej int
    367        1.2   thorpej lwp_wait1(struct lwp *l, lwpid_t lid, lwpid_t *departed, int flags)
    368        1.2   thorpej {
    369        1.2   thorpej 	struct proc *p = l->l_proc;
    370       1.52        ad 	struct lwp *l2;
    371       1.52        ad 	int nfound, error;
    372       1.63        ad 	lwpid_t curlid;
    373       1.63        ad 	bool exiting;
    374        1.2   thorpej 
    375      1.103        ad 	KASSERT(mutex_owned(p->p_lock));
    376       1.52        ad 
    377       1.52        ad 	p->p_nlwpwait++;
    378       1.63        ad 	l->l_waitingfor = lid;
    379       1.63        ad 	curlid = l->l_lid;
    380       1.63        ad 	exiting = ((flags & LWPWAIT_EXITCONTROL) != 0);
    381       1.52        ad 
    382       1.52        ad 	for (;;) {
    383       1.52        ad 		/*
    384       1.52        ad 		 * Avoid a race between exit1() and sigexit(): if the
    385       1.52        ad 		 * process is dumping core, then we need to bail out: call
    386       1.52        ad 		 * into lwp_userret() where we will be suspended until the
    387       1.52        ad 		 * deed is done.
    388       1.52        ad 		 */
    389       1.52        ad 		if ((p->p_sflag & PS_WCORE) != 0) {
    390      1.103        ad 			mutex_exit(p->p_lock);
    391       1.52        ad 			lwp_userret(l);
    392       1.52        ad #ifdef DIAGNOSTIC
    393       1.52        ad 			panic("lwp_wait1");
    394       1.52        ad #endif
    395       1.52        ad 			/* NOTREACHED */
    396       1.52        ad 		}
    397       1.52        ad 
    398       1.52        ad 		/*
    399       1.52        ad 		 * First off, drain any detached LWP that is waiting to be
    400       1.52        ad 		 * reaped.
    401       1.52        ad 		 */
    402       1.52        ad 		while ((l2 = p->p_zomblwp) != NULL) {
    403       1.52        ad 			p->p_zomblwp = NULL;
    404       1.63        ad 			lwp_free(l2, false, false);/* releases proc mutex */
    405      1.103        ad 			mutex_enter(p->p_lock);
    406       1.52        ad 		}
    407       1.52        ad 
    408       1.52        ad 		/*
    409       1.52        ad 		 * Now look for an LWP to collect.  If the whole process is
    410       1.52        ad 		 * exiting, count detached LWPs as eligible to be collected,
    411       1.52        ad 		 * but don't drain them here.
    412       1.52        ad 		 */
    413       1.52        ad 		nfound = 0;
    414       1.63        ad 		error = 0;
    415       1.52        ad 		LIST_FOREACH(l2, &p->p_lwps, l_sibling) {
    416       1.63        ad 			/*
    417       1.63        ad 			 * If a specific wait and the target is waiting on
    418       1.63        ad 			 * us, then avoid deadlock.  This also traps LWPs
    419       1.63        ad 			 * that try to wait on themselves.
    420       1.63        ad 			 *
    421       1.63        ad 			 * Note that this does not handle more complicated
    422       1.63        ad 			 * cycles, like: t1 -> t2 -> t3 -> t1.  The process
    423       1.63        ad 			 * can still be killed so it is not a major problem.
    424       1.63        ad 			 */
    425       1.63        ad 			if (l2->l_lid == lid && l2->l_waitingfor == curlid) {
    426       1.63        ad 				error = EDEADLK;
    427       1.63        ad 				break;
    428       1.63        ad 			}
    429       1.63        ad 			if (l2 == l)
    430       1.52        ad 				continue;
    431       1.52        ad 			if ((l2->l_prflag & LPR_DETACHED) != 0) {
    432       1.63        ad 				nfound += exiting;
    433       1.63        ad 				continue;
    434       1.63        ad 			}
    435       1.63        ad 			if (lid != 0) {
    436       1.63        ad 				if (l2->l_lid != lid)
    437       1.63        ad 					continue;
    438       1.63        ad 				/*
    439       1.63        ad 				 * Mark this LWP as the first waiter, if there
    440       1.63        ad 				 * is no other.
    441       1.63        ad 				 */
    442       1.63        ad 				if (l2->l_waiter == 0)
    443       1.63        ad 					l2->l_waiter = curlid;
    444       1.63        ad 			} else if (l2->l_waiter != 0) {
    445       1.63        ad 				/*
    446       1.63        ad 				 * It already has a waiter - so don't
    447       1.63        ad 				 * collect it.  If the waiter doesn't
    448       1.63        ad 				 * grab it we'll get another chance
    449       1.63        ad 				 * later.
    450       1.63        ad 				 */
    451       1.63        ad 				nfound++;
    452       1.52        ad 				continue;
    453       1.52        ad 			}
    454       1.52        ad 			nfound++;
    455        1.2   thorpej 
    456       1.52        ad 			/* No need to lock the LWP in order to see LSZOMB. */
    457       1.52        ad 			if (l2->l_stat != LSZOMB)
    458       1.52        ad 				continue;
    459        1.2   thorpej 
    460       1.63        ad 			/*
    461       1.63        ad 			 * We're no longer waiting.  Reset the "first waiter"
    462       1.63        ad 			 * pointer on the target, in case it was us.
    463       1.63        ad 			 */
    464       1.63        ad 			l->l_waitingfor = 0;
    465       1.63        ad 			l2->l_waiter = 0;
    466       1.63        ad 			p->p_nlwpwait--;
    467        1.2   thorpej 			if (departed)
    468        1.2   thorpej 				*departed = l2->l_lid;
    469       1.75        ad 			sched_lwp_collect(l2);
    470       1.63        ad 
    471       1.63        ad 			/* lwp_free() releases the proc lock. */
    472       1.63        ad 			lwp_free(l2, false, false);
    473      1.103        ad 			mutex_enter(p->p_lock);
    474       1.52        ad 			return 0;
    475       1.52        ad 		}
    476        1.2   thorpej 
    477       1.63        ad 		if (error != 0)
    478       1.63        ad 			break;
    479       1.52        ad 		if (nfound == 0) {
    480       1.52        ad 			error = ESRCH;
    481       1.52        ad 			break;
    482       1.52        ad 		}
    483       1.63        ad 
    484       1.63        ad 		/*
    485       1.63        ad 		 * The kernel is careful to ensure that it can not deadlock
    486       1.63        ad 		 * when exiting - just keep waiting.
    487       1.63        ad 		 */
    488       1.63        ad 		if (exiting) {
    489       1.52        ad 			KASSERT(p->p_nlwps > 1);
    490      1.103        ad 			cv_wait(&p->p_lwpcv, p->p_lock);
    491       1.52        ad 			continue;
    492       1.52        ad 		}
    493       1.63        ad 
    494       1.63        ad 		/*
    495       1.63        ad 		 * If all other LWPs are waiting for exits or suspends
    496       1.63        ad 		 * and the supply of zombies and potential zombies is
    497       1.63        ad 		 * exhausted, then we are about to deadlock.
    498       1.63        ad 		 *
    499       1.63        ad 		 * If the process is exiting (and this LWP is not the one
    500       1.63        ad 		 * that is coordinating the exit) then bail out now.
    501       1.63        ad 		 */
    502       1.52        ad 		if ((p->p_sflag & PS_WEXIT) != 0 ||
    503       1.63        ad 		    p->p_nrlwps + p->p_nzlwps - p->p_ndlwps <= p->p_nlwpwait) {
    504       1.52        ad 			error = EDEADLK;
    505       1.52        ad 			break;
    506        1.2   thorpej 		}
    507       1.63        ad 
    508       1.63        ad 		/*
    509       1.63        ad 		 * Sit around and wait for something to happen.  We'll be
    510       1.63        ad 		 * awoken if any of the conditions examined change: if an
    511       1.63        ad 		 * LWP exits, is collected, or is detached.
    512       1.63        ad 		 */
    513      1.103        ad 		if ((error = cv_wait_sig(&p->p_lwpcv, p->p_lock)) != 0)
    514       1.52        ad 			break;
    515        1.2   thorpej 	}
    516        1.2   thorpej 
    517       1.63        ad 	/*
    518       1.63        ad 	 * We didn't find any LWPs to collect, we may have received a
    519       1.63        ad 	 * signal, or some other condition has caused us to bail out.
    520       1.63        ad 	 *
    521       1.63        ad 	 * If waiting on a specific LWP, clear the waiters marker: some
    522       1.63        ad 	 * other LWP may want it.  Then, kick all the remaining waiters
    523       1.63        ad 	 * so that they can re-check for zombies and for deadlock.
    524       1.63        ad 	 */
    525       1.63        ad 	if (lid != 0) {
    526       1.63        ad 		LIST_FOREACH(l2, &p->p_lwps, l_sibling) {
    527       1.63        ad 			if (l2->l_lid == lid) {
    528       1.63        ad 				if (l2->l_waiter == curlid)
    529       1.63        ad 					l2->l_waiter = 0;
    530       1.63        ad 				break;
    531       1.63        ad 			}
    532       1.63        ad 		}
    533       1.63        ad 	}
    534       1.52        ad 	p->p_nlwpwait--;
    535       1.63        ad 	l->l_waitingfor = 0;
    536       1.63        ad 	cv_broadcast(&p->p_lwpcv);
    537       1.63        ad 
    538       1.52        ad 	return error;
    539        1.2   thorpej }
    540        1.2   thorpej 
    541       1.52        ad /*
    542       1.52        ad  * Create a new LWP within process 'p2', using LWP 'l1' as a template.
    543       1.52        ad  * The new LWP is created in state LSIDL and must be set running,
    544       1.52        ad  * suspended, or stopped by the caller.
    545       1.52        ad  */
    546        1.2   thorpej int
    547       1.75        ad lwp_create(lwp_t *l1, proc_t *p2, vaddr_t uaddr, bool inmem, int flags,
    548       1.75        ad 	   void *stack, size_t stacksize, void (*func)(void *), void *arg,
    549       1.75        ad 	   lwp_t **rnewlwpp, int sclass)
    550        1.2   thorpej {
    551       1.52        ad 	struct lwp *l2, *isfree;
    552       1.52        ad 	turnstile_t *ts;
    553        1.2   thorpej 
    554      1.107        ad 	KASSERT(l1 == curlwp || l1->l_proc == &proc0);
    555      1.107        ad 
    556       1.52        ad 	/*
    557       1.52        ad 	 * First off, reap any detached LWP waiting to be collected.
    558       1.52        ad 	 * We can re-use its LWP structure and turnstile.
    559       1.52        ad 	 */
    560       1.52        ad 	isfree = NULL;
    561       1.52        ad 	if (p2->p_zomblwp != NULL) {
    562      1.103        ad 		mutex_enter(p2->p_lock);
    563       1.52        ad 		if ((isfree = p2->p_zomblwp) != NULL) {
    564       1.52        ad 			p2->p_zomblwp = NULL;
    565       1.63        ad 			lwp_free(isfree, true, false);/* releases proc mutex */
    566       1.52        ad 		} else
    567      1.103        ad 			mutex_exit(p2->p_lock);
    568       1.52        ad 	}
    569       1.52        ad 	if (isfree == NULL) {
    570       1.87        ad 		l2 = pool_cache_get(lwp_cache, PR_WAITOK);
    571       1.52        ad 		memset(l2, 0, sizeof(*l2));
    572       1.76        ad 		l2->l_ts = pool_cache_get(turnstile_cache, PR_WAITOK);
    573       1.60      yamt 		SLIST_INIT(&l2->l_pi_lenders);
    574       1.52        ad 	} else {
    575       1.52        ad 		l2 = isfree;
    576       1.52        ad 		ts = l2->l_ts;
    577       1.75        ad 		KASSERT(l2->l_inheritedprio == -1);
    578       1.60      yamt 		KASSERT(SLIST_EMPTY(&l2->l_pi_lenders));
    579       1.52        ad 		memset(l2, 0, sizeof(*l2));
    580       1.52        ad 		l2->l_ts = ts;
    581       1.52        ad 	}
    582        1.2   thorpej 
    583        1.2   thorpej 	l2->l_stat = LSIDL;
    584        1.2   thorpej 	l2->l_proc = p2;
    585       1.52        ad 	l2->l_refcnt = 1;
    586       1.75        ad 	l2->l_class = sclass;
    587      1.116        ad 
    588      1.116        ad 	/*
    589      1.116        ad 	 * If vfork(), we want the LWP to run fast and on the same CPU
    590      1.116        ad 	 * as its parent, so that it can reuse the VM context and cache
    591      1.116        ad 	 * footprint on the local CPU.
    592      1.116        ad 	 */
    593      1.116        ad 	l2->l_kpriority = ((flags & LWP_VFORK) ? true : false);
    594       1.82        ad 	l2->l_kpribase = PRI_KERNEL;
    595       1.52        ad 	l2->l_priority = l1->l_priority;
    596       1.75        ad 	l2->l_inheritedprio = -1;
    597       1.56     pavel 	l2->l_flag = inmem ? LW_INMEM : 0;
    598       1.88        ad 	l2->l_pflag = LP_MPSAFE;
    599       1.97        ad 	l2->l_fd = p2->p_fd;
    600      1.110        ad 	TAILQ_INIT(&l2->l_ld_locks);
    601       1.41   thorpej 
    602       1.56     pavel 	if (p2->p_flag & PK_SYSTEM) {
    603       1.91     rmind 		/* Mark it as a system LWP and not a candidate for swapping */
    604       1.56     pavel 		l2->l_flag |= LW_SYSTEM;
    605       1.52        ad 	}
    606        1.2   thorpej 
    607      1.107        ad 	kpreempt_disable();
    608      1.107        ad 	l2->l_mutex = l1->l_cpu->ci_schedstate.spc_mutex;
    609      1.107        ad 	l2->l_cpu = l1->l_cpu;
    610      1.107        ad 	kpreempt_enable();
    611      1.107        ad 
    612       1.73     rmind 	lwp_initspecific(l2);
    613       1.75        ad 	sched_lwp_fork(l1, l2);
    614       1.37        ad 	lwp_update_creds(l2);
    615       1.70        ad 	callout_init(&l2->l_timeout_ch, CALLOUT_MPSAFE);
    616       1.70        ad 	callout_setfunc(&l2->l_timeout_ch, sleepq_timeout, l2);
    617       1.65        ad 	mutex_init(&l2->l_swaplock, MUTEX_DEFAULT, IPL_NONE);
    618       1.52        ad 	cv_init(&l2->l_sigcv, "sigwait");
    619       1.52        ad 	l2->l_syncobj = &sched_syncobj;
    620        1.2   thorpej 
    621        1.2   thorpej 	if (rnewlwpp != NULL)
    622        1.2   thorpej 		*rnewlwpp = l2;
    623        1.2   thorpej 
    624       1.36      yamt 	l2->l_addr = UAREA_TO_USER(uaddr);
    625        1.2   thorpej 	uvm_lwp_fork(l1, l2, stack, stacksize, func,
    626        1.2   thorpej 	    (arg != NULL) ? arg : l2);
    627        1.2   thorpej 
    628      1.103        ad 	mutex_enter(p2->p_lock);
    629       1.52        ad 
    630       1.52        ad 	if ((flags & LWP_DETACHED) != 0) {
    631       1.52        ad 		l2->l_prflag = LPR_DETACHED;
    632       1.52        ad 		p2->p_ndlwps++;
    633       1.52        ad 	} else
    634       1.52        ad 		l2->l_prflag = 0;
    635       1.52        ad 
    636       1.52        ad 	l2->l_sigmask = l1->l_sigmask;
    637       1.52        ad 	CIRCLEQ_INIT(&l2->l_sigpend.sp_info);
    638       1.52        ad 	sigemptyset(&l2->l_sigpend.sp_set);
    639       1.52        ad 
    640       1.53      yamt 	p2->p_nlwpid++;
    641       1.53      yamt 	if (p2->p_nlwpid == 0)
    642       1.53      yamt 		p2->p_nlwpid++;
    643       1.53      yamt 	l2->l_lid = p2->p_nlwpid;
    644        1.2   thorpej 	LIST_INSERT_HEAD(&p2->p_lwps, l2, l_sibling);
    645        1.2   thorpej 	p2->p_nlwps++;
    646        1.2   thorpej 
    647       1.91     rmind 	if ((p2->p_flag & PK_SYSTEM) == 0) {
    648       1.91     rmind 		/* Inherit an affinity */
    649      1.122     rmind 		if (l1->l_flag & LW_AFFINITY) {
    650  1.125.2.3     skrll 			/*
    651  1.125.2.3     skrll 			 * Note that we hold the state lock while inheriting
    652  1.125.2.3     skrll 			 * the affinity to avoid race with sched_setaffinity().
    653  1.125.2.3     skrll 			 */
    654  1.125.2.3     skrll 			lwp_lock(l1);
    655      1.122     rmind 			if (l1->l_flag & LW_AFFINITY) {
    656      1.122     rmind 				kcpuset_use(l1->l_affinity);
    657      1.122     rmind 				l2->l_affinity = l1->l_affinity;
    658      1.122     rmind 				l2->l_flag |= LW_AFFINITY;
    659      1.122     rmind 			}
    660  1.125.2.3     skrll 			lwp_unlock(l1);
    661      1.117  christos 		}
    662  1.125.2.3     skrll 		lwp_lock(l2);
    663  1.125.2.3     skrll 		/* Inherit a processor-set */
    664  1.125.2.3     skrll 		l2->l_psid = l1->l_psid;
    665       1.91     rmind 		/* Look for a CPU to start */
    666       1.91     rmind 		l2->l_cpu = sched_takecpu(l2);
    667       1.91     rmind 		lwp_unlock_to(l2, l2->l_cpu->ci_schedstate.spc_mutex);
    668       1.91     rmind 	}
    669  1.125.2.3     skrll 	mutex_exit(p2->p_lock);
    670  1.125.2.3     skrll 
    671  1.125.2.3     skrll 	mutex_enter(proc_lock);
    672  1.125.2.3     skrll 	LIST_INSERT_HEAD(&alllwp, l2, l_list);
    673  1.125.2.3     skrll 	mutex_exit(proc_lock);
    674       1.91     rmind 
    675       1.57       dsl 	SYSCALL_TIME_LWP_INIT(l2);
    676       1.57       dsl 
    677       1.16      manu 	if (p2->p_emul->e_lwp_fork)
    678       1.16      manu 		(*p2->p_emul->e_lwp_fork)(l1, l2);
    679       1.16      manu 
    680        1.2   thorpej 	return (0);
    681        1.2   thorpej }
    682        1.2   thorpej 
    683        1.2   thorpej /*
    684       1.64      yamt  * Called by MD code when a new LWP begins execution.  Must be called
    685       1.64      yamt  * with the previous LWP locked (so at splsched), or if there is no
    686       1.64      yamt  * previous LWP, at splsched.
    687       1.64      yamt  */
    688       1.64      yamt void
    689       1.64      yamt lwp_startup(struct lwp *prev, struct lwp *new)
    690       1.64      yamt {
    691       1.64      yamt 
    692      1.107        ad 	KASSERT(kpreempt_disabled());
    693       1.64      yamt 	if (prev != NULL) {
    694       1.81        ad 		/*
    695       1.81        ad 		 * Normalize the count of the spin-mutexes, it was
    696       1.81        ad 		 * increased in mi_switch().  Unmark the state of
    697       1.81        ad 		 * context switch - it is finished for previous LWP.
    698       1.81        ad 		 */
    699       1.81        ad 		curcpu()->ci_mtx_count++;
    700       1.81        ad 		membar_exit();
    701       1.81        ad 		prev->l_ctxswtch = 0;
    702       1.64      yamt 	}
    703      1.107        ad 	KPREEMPT_DISABLE(new);
    704      1.107        ad 	spl0();
    705      1.105        ad 	pmap_activate(new);
    706       1.64      yamt 	LOCKDEBUG_BARRIER(NULL, 0);
    707      1.107        ad 	KPREEMPT_ENABLE(new);
    708       1.65        ad 	if ((new->l_pflag & LP_MPSAFE) == 0) {
    709       1.65        ad 		KERNEL_LOCK(1, new);
    710       1.65        ad 	}
    711       1.64      yamt }
    712       1.64      yamt 
    713       1.64      yamt /*
    714       1.65        ad  * Exit an LWP.
    715        1.2   thorpej  */
    716        1.2   thorpej void
    717        1.2   thorpej lwp_exit(struct lwp *l)
    718        1.2   thorpej {
    719        1.2   thorpej 	struct proc *p = l->l_proc;
    720       1.52        ad 	struct lwp *l2;
    721       1.65        ad 	bool current;
    722       1.65        ad 
    723       1.65        ad 	current = (l == curlwp);
    724        1.2   thorpej 
    725      1.114     rmind 	KASSERT(current || (l->l_stat == LSIDL && l->l_target_cpu == NULL));
    726        1.2   thorpej 
    727       1.52        ad 	/*
    728       1.52        ad 	 * Verify that we hold no locks other than the kernel lock.
    729       1.52        ad 	 */
    730       1.52        ad 	LOCKDEBUG_BARRIER(&kernel_lock, 0);
    731       1.16      manu 
    732        1.2   thorpej 	/*
    733       1.52        ad 	 * If we are the last live LWP in a process, we need to exit the
    734       1.52        ad 	 * entire process.  We do so with an exit status of zero, because
    735       1.52        ad 	 * it's a "controlled" exit, and because that's what Solaris does.
    736       1.52        ad 	 *
    737       1.52        ad 	 * We are not quite a zombie yet, but for accounting purposes we
    738       1.52        ad 	 * must increment the count of zombies here.
    739       1.45   thorpej 	 *
    740       1.45   thorpej 	 * Note: the last LWP's specificdata will be deleted here.
    741        1.2   thorpej 	 */
    742      1.103        ad 	mutex_enter(p->p_lock);
    743       1.52        ad 	if (p->p_nlwps - p->p_nzlwps == 1) {
    744       1.65        ad 		KASSERT(current == true);
    745       1.88        ad 		/* XXXSMP kernel_lock not held */
    746        1.2   thorpej 		exit1(l, 0);
    747       1.19  jdolecek 		/* NOTREACHED */
    748        1.2   thorpej 	}
    749       1.52        ad 	p->p_nzlwps++;
    750      1.103        ad 	mutex_exit(p->p_lock);
    751       1.52        ad 
    752       1.52        ad 	if (p->p_emul->e_lwp_exit)
    753       1.52        ad 		(*p->p_emul->e_lwp_exit)(l);
    754        1.2   thorpej 
    755       1.45   thorpej 	/* Delete the specificdata while it's still safe to sleep. */
    756       1.45   thorpej 	specificdata_fini(lwp_specificdata_domain, &l->l_specdataref);
    757       1.45   thorpej 
    758       1.52        ad 	/*
    759       1.52        ad 	 * Release our cached credentials.
    760       1.52        ad 	 */
    761       1.37        ad 	kauth_cred_free(l->l_cred);
    762       1.70        ad 	callout_destroy(&l->l_timeout_ch);
    763       1.65        ad 
    764       1.65        ad 	/*
    765       1.65        ad 	 * While we can still block, mark the LWP as unswappable to
    766       1.65        ad 	 * prevent conflicts with the with the swapper.
    767       1.65        ad 	 */
    768       1.65        ad 	if (current)
    769       1.65        ad 		uvm_lwp_hold(l);
    770       1.37        ad 
    771       1.52        ad 	/*
    772       1.52        ad 	 * Remove the LWP from the global list.
    773       1.52        ad 	 */
    774      1.102        ad 	mutex_enter(proc_lock);
    775       1.52        ad 	LIST_REMOVE(l, l_list);
    776      1.102        ad 	mutex_exit(proc_lock);
    777       1.19  jdolecek 
    778       1.52        ad 	/*
    779       1.52        ad 	 * Get rid of all references to the LWP that others (e.g. procfs)
    780       1.52        ad 	 * may have, and mark the LWP as a zombie.  If the LWP is detached,
    781       1.52        ad 	 * mark it waiting for collection in the proc structure.  Note that
    782       1.52        ad 	 * before we can do that, we need to free any other dead, deatched
    783       1.52        ad 	 * LWP waiting to meet its maker.
    784       1.52        ad 	 */
    785      1.103        ad 	mutex_enter(p->p_lock);
    786       1.52        ad 	lwp_drainrefs(l);
    787       1.31      yamt 
    788       1.52        ad 	if ((l->l_prflag & LPR_DETACHED) != 0) {
    789       1.52        ad 		while ((l2 = p->p_zomblwp) != NULL) {
    790       1.52        ad 			p->p_zomblwp = NULL;
    791       1.63        ad 			lwp_free(l2, false, false);/* releases proc mutex */
    792      1.103        ad 			mutex_enter(p->p_lock);
    793       1.72        ad 			l->l_refcnt++;
    794       1.72        ad 			lwp_drainrefs(l);
    795       1.52        ad 		}
    796       1.52        ad 		p->p_zomblwp = l;
    797       1.52        ad 	}
    798       1.31      yamt 
    799       1.52        ad 	/*
    800       1.52        ad 	 * If we find a pending signal for the process and we have been
    801       1.52        ad 	 * asked to check for signals, then we loose: arrange to have
    802       1.52        ad 	 * all other LWPs in the process check for signals.
    803       1.52        ad 	 */
    804       1.56     pavel 	if ((l->l_flag & LW_PENDSIG) != 0 &&
    805       1.52        ad 	    firstsig(&p->p_sigpend.sp_set) != 0) {
    806       1.52        ad 		LIST_FOREACH(l2, &p->p_lwps, l_sibling) {
    807       1.52        ad 			lwp_lock(l2);
    808       1.56     pavel 			l2->l_flag |= LW_PENDSIG;
    809       1.52        ad 			lwp_unlock(l2);
    810       1.52        ad 		}
    811       1.31      yamt 	}
    812       1.31      yamt 
    813       1.52        ad 	lwp_lock(l);
    814       1.52        ad 	l->l_stat = LSZOMB;
    815       1.90        ad 	if (l->l_name != NULL)
    816       1.90        ad 		strcpy(l->l_name, "(zombie)");
    817  1.125.2.3     skrll 	if (l->l_flag & LW_AFFINITY) {
    818      1.122     rmind 		l->l_flag &= ~LW_AFFINITY;
    819  1.125.2.3     skrll 	} else {
    820  1.125.2.3     skrll 		KASSERT(l->l_affinity == NULL);
    821  1.125.2.3     skrll 	}
    822       1.52        ad 	lwp_unlock(l);
    823        1.2   thorpej 	p->p_nrlwps--;
    824       1.52        ad 	cv_broadcast(&p->p_lwpcv);
    825       1.78        ad 	if (l->l_lwpctl != NULL)
    826       1.78        ad 		l->l_lwpctl->lc_curcpu = LWPCTL_CPU_EXITED;
    827      1.103        ad 	mutex_exit(p->p_lock);
    828       1.52        ad 
    829      1.122     rmind 	/* Safe without lock since LWP is in zombie state */
    830      1.122     rmind 	if (l->l_affinity) {
    831      1.122     rmind 		kcpuset_unuse(l->l_affinity, NULL);
    832      1.122     rmind 		l->l_affinity = NULL;
    833      1.122     rmind 	}
    834      1.122     rmind 
    835       1.52        ad 	/*
    836       1.52        ad 	 * We can no longer block.  At this point, lwp_free() may already
    837       1.52        ad 	 * be gunning for us.  On a multi-CPU system, we may be off p_lwps.
    838       1.52        ad 	 *
    839       1.52        ad 	 * Free MD LWP resources.
    840       1.52        ad 	 */
    841       1.52        ad #ifndef __NO_CPU_LWP_FREE
    842       1.52        ad 	cpu_lwp_free(l, 0);
    843       1.52        ad #endif
    844        1.2   thorpej 
    845       1.65        ad 	if (current) {
    846       1.65        ad 		pmap_deactivate(l);
    847       1.65        ad 
    848       1.65        ad 		/*
    849       1.65        ad 		 * Release the kernel lock, and switch away into
    850       1.65        ad 		 * oblivion.
    851       1.65        ad 		 */
    852       1.52        ad #ifdef notyet
    853       1.65        ad 		/* XXXSMP hold in lwp_userret() */
    854       1.65        ad 		KERNEL_UNLOCK_LAST(l);
    855       1.52        ad #else
    856       1.65        ad 		KERNEL_UNLOCK_ALL(l, NULL);
    857       1.52        ad #endif
    858       1.65        ad 		lwp_exit_switchaway(l);
    859       1.65        ad 	}
    860        1.2   thorpej }
    861        1.2   thorpej 
    862       1.52        ad /*
    863       1.52        ad  * Free a dead LWP's remaining resources.
    864       1.52        ad  *
    865       1.52        ad  * XXXLWP limits.
    866       1.52        ad  */
    867       1.52        ad void
    868       1.63        ad lwp_free(struct lwp *l, bool recycle, bool last)
    869       1.52        ad {
    870       1.52        ad 	struct proc *p = l->l_proc;
    871      1.100        ad 	struct rusage *ru;
    872       1.52        ad 	ksiginfoq_t kq;
    873       1.52        ad 
    874       1.92      yamt 	KASSERT(l != curlwp);
    875       1.92      yamt 
    876       1.52        ad 	/*
    877       1.52        ad 	 * If this was not the last LWP in the process, then adjust
    878       1.52        ad 	 * counters and unlock.
    879       1.52        ad 	 */
    880       1.52        ad 	if (!last) {
    881       1.52        ad 		/*
    882       1.52        ad 		 * Add the LWP's run time to the process' base value.
    883       1.52        ad 		 * This needs to co-incide with coming off p_lwps.
    884       1.52        ad 		 */
    885       1.86      yamt 		bintime_add(&p->p_rtime, &l->l_rtime);
    886       1.64      yamt 		p->p_pctcpu += l->l_pctcpu;
    887      1.100        ad 		ru = &p->p_stats->p_ru;
    888      1.100        ad 		ruadd(ru, &l->l_ru);
    889      1.100        ad 		ru->ru_nvcsw += (l->l_ncsw - l->l_nivcsw);
    890      1.100        ad 		ru->ru_nivcsw += l->l_nivcsw;
    891       1.52        ad 		LIST_REMOVE(l, l_sibling);
    892       1.52        ad 		p->p_nlwps--;
    893       1.52        ad 		p->p_nzlwps--;
    894       1.52        ad 		if ((l->l_prflag & LPR_DETACHED) != 0)
    895       1.52        ad 			p->p_ndlwps--;
    896       1.63        ad 
    897       1.63        ad 		/*
    898       1.63        ad 		 * Have any LWPs sleeping in lwp_wait() recheck for
    899       1.63        ad 		 * deadlock.
    900       1.63        ad 		 */
    901       1.63        ad 		cv_broadcast(&p->p_lwpcv);
    902      1.103        ad 		mutex_exit(p->p_lock);
    903       1.63        ad 	}
    904       1.52        ad 
    905       1.52        ad #ifdef MULTIPROCESSOR
    906       1.63        ad 	/*
    907       1.63        ad 	 * In the unlikely event that the LWP is still on the CPU,
    908       1.63        ad 	 * then spin until it has switched away.  We need to release
    909       1.63        ad 	 * all locks to avoid deadlock against interrupt handlers on
    910       1.63        ad 	 * the target CPU.
    911       1.63        ad 	 */
    912      1.115        ad 	if ((l->l_pflag & LP_RUNNING) != 0 || l->l_cpu->ci_curlwp == l) {
    913       1.63        ad 		int count;
    914       1.64      yamt 		(void)count; /* XXXgcc */
    915       1.63        ad 		KERNEL_UNLOCK_ALL(curlwp, &count);
    916      1.115        ad 		while ((l->l_pflag & LP_RUNNING) != 0 ||
    917       1.64      yamt 		    l->l_cpu->ci_curlwp == l)
    918       1.63        ad 			SPINLOCK_BACKOFF_HOOK;
    919       1.63        ad 		KERNEL_LOCK(count, curlwp);
    920       1.63        ad 	}
    921       1.52        ad #endif
    922       1.52        ad 
    923       1.52        ad 	/*
    924       1.52        ad 	 * Destroy the LWP's remaining signal information.
    925       1.52        ad 	 */
    926       1.52        ad 	ksiginfo_queue_init(&kq);
    927       1.52        ad 	sigclear(&l->l_sigpend, NULL, &kq);
    928       1.52        ad 	ksiginfo_queue_drain(&kq);
    929       1.52        ad 	cv_destroy(&l->l_sigcv);
    930       1.65        ad 	mutex_destroy(&l->l_swaplock);
    931        1.2   thorpej 
    932       1.19  jdolecek 	/*
    933       1.52        ad 	 * Free the LWP's turnstile and the LWP structure itself unless the
    934       1.93      yamt 	 * caller wants to recycle them.  Also, free the scheduler specific
    935       1.93      yamt 	 * data.
    936       1.52        ad 	 *
    937       1.52        ad 	 * We can't return turnstile0 to the pool (it didn't come from it),
    938       1.52        ad 	 * so if it comes up just drop it quietly and move on.
    939       1.52        ad 	 *
    940       1.52        ad 	 * We don't recycle the VM resources at this time.
    941       1.19  jdolecek 	 */
    942       1.78        ad 	if (l->l_lwpctl != NULL)
    943       1.78        ad 		lwp_ctl_free(l);
    944       1.64      yamt 
    945       1.52        ad 	if (!recycle && l->l_ts != &turnstile0)
    946       1.76        ad 		pool_cache_put(turnstile_cache, l->l_ts);
    947       1.90        ad 	if (l->l_name != NULL)
    948       1.90        ad 		kmem_free(l->l_name, MAXCOMLEN);
    949       1.52        ad #ifndef __NO_CPU_LWP_FREE
    950       1.52        ad 	cpu_lwp_free2(l);
    951       1.52        ad #endif
    952       1.92      yamt 	KASSERT((l->l_flag & LW_INMEM) != 0);
    953       1.19  jdolecek 	uvm_lwp_exit(l);
    954       1.60      yamt 	KASSERT(SLIST_EMPTY(&l->l_pi_lenders));
    955       1.75        ad 	KASSERT(l->l_inheritedprio == -1);
    956       1.52        ad 	if (!recycle)
    957       1.87        ad 		pool_cache_put(lwp_cache, l);
    958        1.2   thorpej }
    959        1.2   thorpej 
    960        1.2   thorpej /*
    961       1.91     rmind  * Migrate the LWP to the another CPU.  Unlocks the LWP.
    962       1.91     rmind  */
    963       1.91     rmind void
    964      1.114     rmind lwp_migrate(lwp_t *l, struct cpu_info *tci)
    965       1.91     rmind {
    966      1.114     rmind 	struct schedstate_percpu *tspc;
    967      1.121     rmind 	int lstat = l->l_stat;
    968      1.121     rmind 
    969       1.91     rmind 	KASSERT(lwp_locked(l, NULL));
    970      1.114     rmind 	KASSERT(tci != NULL);
    971      1.114     rmind 
    972      1.121     rmind 	/* If LWP is still on the CPU, it must be handled like LSONPROC */
    973      1.121     rmind 	if ((l->l_pflag & LP_RUNNING) != 0) {
    974      1.121     rmind 		lstat = LSONPROC;
    975      1.121     rmind 	}
    976      1.121     rmind 
    977      1.114     rmind 	/*
    978      1.114     rmind 	 * The destination CPU could be changed while previous migration
    979      1.114     rmind 	 * was not finished.
    980      1.114     rmind 	 */
    981      1.121     rmind 	if (l->l_target_cpu != NULL) {
    982      1.114     rmind 		l->l_target_cpu = tci;
    983      1.114     rmind 		lwp_unlock(l);
    984      1.114     rmind 		return;
    985      1.114     rmind 	}
    986       1.91     rmind 
    987      1.114     rmind 	/* Nothing to do if trying to migrate to the same CPU */
    988      1.114     rmind 	if (l->l_cpu == tci) {
    989       1.91     rmind 		lwp_unlock(l);
    990       1.91     rmind 		return;
    991       1.91     rmind 	}
    992       1.91     rmind 
    993      1.114     rmind 	KASSERT(l->l_target_cpu == NULL);
    994      1.114     rmind 	tspc = &tci->ci_schedstate;
    995      1.121     rmind 	switch (lstat) {
    996       1.91     rmind 	case LSRUN:
    997       1.91     rmind 		if (l->l_flag & LW_INMEM) {
    998      1.114     rmind 			l->l_target_cpu = tci;
    999      1.114     rmind 			lwp_unlock(l);
   1000      1.114     rmind 			return;
   1001       1.91     rmind 		}
   1002       1.91     rmind 	case LSIDL:
   1003      1.114     rmind 		l->l_cpu = tci;
   1004      1.114     rmind 		lwp_unlock_to(l, tspc->spc_mutex);
   1005       1.91     rmind 		return;
   1006       1.91     rmind 	case LSSLEEP:
   1007      1.114     rmind 		l->l_cpu = tci;
   1008       1.91     rmind 		break;
   1009       1.91     rmind 	case LSSTOP:
   1010       1.91     rmind 	case LSSUSPENDED:
   1011      1.114     rmind 		l->l_cpu = tci;
   1012      1.114     rmind 		if (l->l_wchan == NULL) {
   1013      1.114     rmind 			lwp_unlock_to(l, tspc->spc_lwplock);
   1014      1.114     rmind 			return;
   1015       1.91     rmind 		}
   1016      1.114     rmind 		break;
   1017       1.91     rmind 	case LSONPROC:
   1018      1.114     rmind 		l->l_target_cpu = tci;
   1019      1.114     rmind 		spc_lock(l->l_cpu);
   1020      1.114     rmind 		cpu_need_resched(l->l_cpu, RESCHED_KPREEMPT);
   1021      1.114     rmind 		spc_unlock(l->l_cpu);
   1022       1.91     rmind 		break;
   1023       1.91     rmind 	}
   1024       1.91     rmind 	lwp_unlock(l);
   1025       1.91     rmind }
   1026       1.91     rmind 
   1027       1.91     rmind /*
   1028       1.94     rmind  * Find the LWP in the process.  Arguments may be zero, in such case,
   1029       1.94     rmind  * the calling process and first LWP in the list will be used.
   1030      1.103        ad  * On success - returns proc locked.
   1031       1.91     rmind  */
   1032       1.91     rmind struct lwp *
   1033       1.91     rmind lwp_find2(pid_t pid, lwpid_t lid)
   1034       1.91     rmind {
   1035       1.91     rmind 	proc_t *p;
   1036       1.91     rmind 	lwp_t *l;
   1037       1.91     rmind 
   1038       1.91     rmind 	/* Find the process */
   1039       1.94     rmind 	p = (pid == 0) ? curlwp->l_proc : p_find(pid, PFIND_UNLOCK_FAIL);
   1040       1.91     rmind 	if (p == NULL)
   1041       1.91     rmind 		return NULL;
   1042      1.103        ad 	mutex_enter(p->p_lock);
   1043       1.94     rmind 	if (pid != 0) {
   1044       1.94     rmind 		/* Case of p_find */
   1045      1.102        ad 		mutex_exit(proc_lock);
   1046       1.94     rmind 	}
   1047       1.91     rmind 
   1048       1.91     rmind 	/* Find the thread */
   1049       1.94     rmind 	l = (lid == 0) ? LIST_FIRST(&p->p_lwps) : lwp_find(p, lid);
   1050      1.103        ad 	if (l == NULL) {
   1051      1.103        ad 		mutex_exit(p->p_lock);
   1052      1.103        ad 	}
   1053       1.91     rmind 
   1054       1.91     rmind 	return l;
   1055       1.91     rmind }
   1056       1.91     rmind 
   1057       1.91     rmind /*
   1058       1.52        ad  * Look up a live LWP within the speicifed process, and return it locked.
   1059       1.52        ad  *
   1060      1.103        ad  * Must be called with p->p_lock held.
   1061       1.52        ad  */
   1062       1.52        ad struct lwp *
   1063       1.52        ad lwp_find(struct proc *p, int id)
   1064       1.52        ad {
   1065       1.52        ad 	struct lwp *l;
   1066       1.52        ad 
   1067      1.103        ad 	KASSERT(mutex_owned(p->p_lock));
   1068       1.52        ad 
   1069       1.52        ad 	LIST_FOREACH(l, &p->p_lwps, l_sibling) {
   1070       1.52        ad 		if (l->l_lid == id)
   1071       1.52        ad 			break;
   1072       1.52        ad 	}
   1073       1.52        ad 
   1074       1.52        ad 	/*
   1075       1.52        ad 	 * No need to lock - all of these conditions will
   1076       1.52        ad 	 * be visible with the process level mutex held.
   1077       1.52        ad 	 */
   1078       1.52        ad 	if (l != NULL && (l->l_stat == LSIDL || l->l_stat == LSZOMB))
   1079       1.52        ad 		l = NULL;
   1080       1.52        ad 
   1081       1.52        ad 	return l;
   1082       1.52        ad }
   1083       1.52        ad 
   1084       1.52        ad /*
   1085       1.37        ad  * Update an LWP's cached credentials to mirror the process' master copy.
   1086       1.37        ad  *
   1087       1.37        ad  * This happens early in the syscall path, on user trap, and on LWP
   1088       1.37        ad  * creation.  A long-running LWP can also voluntarily choose to update
   1089       1.37        ad  * it's credentials by calling this routine.  This may be called from
   1090       1.37        ad  * LWP_CACHE_CREDS(), which checks l->l_cred != p->p_cred beforehand.
   1091       1.37        ad  */
   1092       1.37        ad void
   1093       1.37        ad lwp_update_creds(struct lwp *l)
   1094       1.37        ad {
   1095       1.37        ad 	kauth_cred_t oc;
   1096       1.37        ad 	struct proc *p;
   1097       1.37        ad 
   1098       1.37        ad 	p = l->l_proc;
   1099       1.37        ad 	oc = l->l_cred;
   1100       1.37        ad 
   1101      1.103        ad 	mutex_enter(p->p_lock);
   1102       1.37        ad 	kauth_cred_hold(p->p_cred);
   1103       1.37        ad 	l->l_cred = p->p_cred;
   1104       1.98        ad 	l->l_prflag &= ~LPR_CRMOD;
   1105      1.103        ad 	mutex_exit(p->p_lock);
   1106       1.88        ad 	if (oc != NULL)
   1107       1.37        ad 		kauth_cred_free(oc);
   1108       1.52        ad }
   1109       1.52        ad 
   1110       1.52        ad /*
   1111       1.52        ad  * Verify that an LWP is locked, and optionally verify that the lock matches
   1112       1.52        ad  * one we specify.
   1113       1.52        ad  */
   1114       1.52        ad int
   1115       1.52        ad lwp_locked(struct lwp *l, kmutex_t *mtx)
   1116       1.52        ad {
   1117       1.52        ad 	kmutex_t *cur = l->l_mutex;
   1118       1.52        ad 
   1119       1.52        ad 	return mutex_owned(cur) && (mtx == cur || mtx == NULL);
   1120       1.52        ad }
   1121       1.52        ad 
   1122       1.52        ad /*
   1123       1.52        ad  * Lock an LWP.
   1124       1.52        ad  */
   1125      1.119        ad kmutex_t *
   1126       1.52        ad lwp_lock_retry(struct lwp *l, kmutex_t *old)
   1127       1.52        ad {
   1128       1.52        ad 
   1129       1.52        ad 	/*
   1130       1.52        ad 	 * XXXgcc ignoring kmutex_t * volatile on i386
   1131       1.52        ad 	 *
   1132       1.52        ad 	 * gcc version 4.1.2 20061021 prerelease (NetBSD nb1 20061021)
   1133       1.52        ad 	 */
   1134       1.52        ad #if 1
   1135       1.52        ad 	while (l->l_mutex != old) {
   1136       1.52        ad #else
   1137       1.52        ad 	for (;;) {
   1138       1.52        ad #endif
   1139       1.52        ad 		mutex_spin_exit(old);
   1140       1.52        ad 		old = l->l_mutex;
   1141       1.52        ad 		mutex_spin_enter(old);
   1142       1.52        ad 
   1143       1.52        ad 		/*
   1144       1.52        ad 		 * mutex_enter() will have posted a read barrier.  Re-test
   1145       1.52        ad 		 * l->l_mutex.  If it has changed, we need to try again.
   1146       1.52        ad 		 */
   1147       1.52        ad #if 1
   1148       1.52        ad 	}
   1149       1.52        ad #else
   1150       1.52        ad 	} while (__predict_false(l->l_mutex != old));
   1151       1.52        ad #endif
   1152      1.119        ad 
   1153      1.119        ad 	return old;
   1154       1.52        ad }
   1155       1.52        ad 
   1156       1.52        ad /*
   1157       1.52        ad  * Lend a new mutex to an LWP.  The old mutex must be held.
   1158       1.52        ad  */
   1159       1.52        ad void
   1160       1.52        ad lwp_setlock(struct lwp *l, kmutex_t *new)
   1161       1.52        ad {
   1162       1.52        ad 
   1163       1.63        ad 	KASSERT(mutex_owned(l->l_mutex));
   1164       1.52        ad 
   1165      1.107        ad 	membar_exit();
   1166       1.52        ad 	l->l_mutex = new;
   1167       1.52        ad }
   1168       1.52        ad 
   1169       1.52        ad /*
   1170       1.52        ad  * Lend a new mutex to an LWP, and release the old mutex.  The old mutex
   1171       1.52        ad  * must be held.
   1172       1.52        ad  */
   1173       1.52        ad void
   1174       1.52        ad lwp_unlock_to(struct lwp *l, kmutex_t *new)
   1175       1.52        ad {
   1176       1.52        ad 	kmutex_t *old;
   1177       1.52        ad 
   1178       1.63        ad 	KASSERT(mutex_owned(l->l_mutex));
   1179       1.52        ad 
   1180       1.52        ad 	old = l->l_mutex;
   1181      1.107        ad 	membar_exit();
   1182       1.52        ad 	l->l_mutex = new;
   1183       1.52        ad 	mutex_spin_exit(old);
   1184       1.52        ad }
   1185       1.52        ad 
   1186       1.52        ad /*
   1187       1.52        ad  * Acquire a new mutex, and donate it to an LWP.  The LWP must already be
   1188       1.52        ad  * locked.
   1189       1.52        ad  */
   1190       1.52        ad void
   1191       1.52        ad lwp_relock(struct lwp *l, kmutex_t *new)
   1192       1.52        ad {
   1193       1.52        ad 	kmutex_t *old;
   1194       1.52        ad 
   1195       1.63        ad 	KASSERT(mutex_owned(l->l_mutex));
   1196       1.52        ad 
   1197       1.52        ad 	old = l->l_mutex;
   1198       1.52        ad 	if (old != new) {
   1199       1.52        ad 		mutex_spin_enter(new);
   1200       1.52        ad 		l->l_mutex = new;
   1201       1.52        ad 		mutex_spin_exit(old);
   1202       1.52        ad 	}
   1203       1.52        ad }
   1204       1.52        ad 
   1205       1.60      yamt int
   1206       1.60      yamt lwp_trylock(struct lwp *l)
   1207       1.60      yamt {
   1208       1.60      yamt 	kmutex_t *old;
   1209       1.60      yamt 
   1210       1.60      yamt 	for (;;) {
   1211       1.60      yamt 		if (!mutex_tryenter(old = l->l_mutex))
   1212       1.60      yamt 			return 0;
   1213       1.60      yamt 		if (__predict_true(l->l_mutex == old))
   1214       1.60      yamt 			return 1;
   1215       1.60      yamt 		mutex_spin_exit(old);
   1216       1.60      yamt 	}
   1217       1.60      yamt }
   1218       1.60      yamt 
   1219       1.96        ad u_int
   1220       1.96        ad lwp_unsleep(lwp_t *l, bool cleanup)
   1221       1.96        ad {
   1222       1.96        ad 
   1223       1.96        ad 	KASSERT(mutex_owned(l->l_mutex));
   1224       1.96        ad 
   1225       1.96        ad 	return (*l->l_syncobj->sobj_unsleep)(l, cleanup);
   1226       1.96        ad }
   1227       1.96        ad 
   1228       1.96        ad 
   1229       1.52        ad /*
   1230       1.56     pavel  * Handle exceptions for mi_userret().  Called if a member of LW_USERRET is
   1231       1.52        ad  * set.
   1232       1.52        ad  */
   1233       1.52        ad void
   1234       1.52        ad lwp_userret(struct lwp *l)
   1235       1.52        ad {
   1236       1.52        ad 	struct proc *p;
   1237       1.54        ad 	void (*hook)(void);
   1238       1.52        ad 	int sig;
   1239       1.52        ad 
   1240      1.114     rmind 	KASSERT(l == curlwp);
   1241      1.114     rmind 	KASSERT(l->l_stat == LSONPROC);
   1242       1.52        ad 	p = l->l_proc;
   1243       1.52        ad 
   1244       1.75        ad #ifndef __HAVE_FAST_SOFTINTS
   1245       1.75        ad 	/* Run pending soft interrupts. */
   1246       1.75        ad 	if (l->l_cpu->ci_data.cpu_softints != 0)
   1247       1.75        ad 		softint_overlay();
   1248       1.75        ad #endif
   1249       1.75        ad 
   1250      1.125        ad #ifdef KERN_SA
   1251      1.125        ad 	/* Generate UNBLOCKED upcall if needed */
   1252      1.125        ad 	if (l->l_flag & LW_SA_BLOCKING) {
   1253      1.125        ad 		sa_unblock_userret(l);
   1254      1.125        ad 		/* NOTREACHED */
   1255      1.125        ad 	}
   1256      1.125        ad #endif
   1257      1.125        ad 
   1258       1.52        ad 	/*
   1259       1.52        ad 	 * It should be safe to do this read unlocked on a multiprocessor
   1260       1.52        ad 	 * system..
   1261  1.125.2.1     skrll 	 *
   1262  1.125.2.1     skrll 	 * LW_SA_UPCALL will be handled after the while() loop, so don't
   1263  1.125.2.1     skrll 	 * consider it now.
   1264       1.52        ad 	 */
   1265  1.125.2.1     skrll 	while ((l->l_flag & (LW_USERRET & ~(LW_SA_UPCALL))) != 0) {
   1266       1.52        ad 		/*
   1267       1.52        ad 		 * Process pending signals first, unless the process
   1268       1.61        ad 		 * is dumping core or exiting, where we will instead
   1269      1.101     rmind 		 * enter the LW_WSUSPEND case below.
   1270       1.52        ad 		 */
   1271       1.61        ad 		if ((l->l_flag & (LW_PENDSIG | LW_WCORE | LW_WEXIT)) ==
   1272       1.61        ad 		    LW_PENDSIG) {
   1273      1.103        ad 			mutex_enter(p->p_lock);
   1274       1.52        ad 			while ((sig = issignal(l)) != 0)
   1275       1.52        ad 				postsig(sig);
   1276      1.103        ad 			mutex_exit(p->p_lock);
   1277       1.52        ad 		}
   1278       1.52        ad 
   1279       1.52        ad 		/*
   1280       1.52        ad 		 * Core-dump or suspend pending.
   1281       1.52        ad 		 *
   1282       1.52        ad 		 * In case of core dump, suspend ourselves, so that the
   1283       1.52        ad 		 * kernel stack and therefore the userland registers saved
   1284       1.52        ad 		 * in the trapframe are around for coredump() to write them
   1285       1.52        ad 		 * out.  We issue a wakeup on p->p_lwpcv so that sigexit()
   1286       1.52        ad 		 * will write the core file out once all other LWPs are
   1287       1.52        ad 		 * suspended.
   1288       1.52        ad 		 */
   1289       1.56     pavel 		if ((l->l_flag & LW_WSUSPEND) != 0) {
   1290      1.103        ad 			mutex_enter(p->p_lock);
   1291       1.52        ad 			p->p_nrlwps--;
   1292       1.52        ad 			cv_broadcast(&p->p_lwpcv);
   1293       1.52        ad 			lwp_lock(l);
   1294       1.52        ad 			l->l_stat = LSSUSPENDED;
   1295      1.104        ad 			lwp_unlock(l);
   1296      1.103        ad 			mutex_exit(p->p_lock);
   1297      1.104        ad 			lwp_lock(l);
   1298       1.64      yamt 			mi_switch(l);
   1299       1.52        ad 		}
   1300       1.52        ad 
   1301       1.52        ad 		/* Process is exiting. */
   1302       1.56     pavel 		if ((l->l_flag & LW_WEXIT) != 0) {
   1303       1.52        ad 			lwp_exit(l);
   1304       1.52        ad 			KASSERT(0);
   1305       1.52        ad 			/* NOTREACHED */
   1306       1.52        ad 		}
   1307       1.54        ad 
   1308       1.54        ad 		/* Call userret hook; used by Linux emulation. */
   1309       1.56     pavel 		if ((l->l_flag & LW_WUSERRET) != 0) {
   1310       1.54        ad 			lwp_lock(l);
   1311       1.56     pavel 			l->l_flag &= ~LW_WUSERRET;
   1312       1.54        ad 			lwp_unlock(l);
   1313       1.54        ad 			hook = p->p_userret;
   1314       1.54        ad 			p->p_userret = NULL;
   1315       1.54        ad 			(*hook)();
   1316       1.54        ad 		}
   1317       1.52        ad 	}
   1318      1.124  wrstuden 
   1319      1.124  wrstuden #ifdef KERN_SA
   1320      1.124  wrstuden 	/*
   1321      1.124  wrstuden 	 * Timer events are handled specially.  We only try once to deliver
   1322      1.124  wrstuden 	 * pending timer upcalls; if if fails, we can try again on the next
   1323      1.124  wrstuden 	 * loop around.  If we need to re-enter lwp_userret(), MD code will
   1324      1.124  wrstuden 	 * bounce us back here through the trap path after we return.
   1325      1.124  wrstuden 	 */
   1326      1.124  wrstuden 	if (p->p_timerpend)
   1327      1.124  wrstuden 		timerupcall(l);
   1328      1.125        ad 	if (l->l_flag & LW_SA_UPCALL)
   1329      1.125        ad 		sa_upcall_userret(l);
   1330      1.124  wrstuden #endif /* KERN_SA */
   1331       1.52        ad }
   1332       1.52        ad 
   1333       1.52        ad /*
   1334       1.52        ad  * Force an LWP to enter the kernel, to take a trip through lwp_userret().
   1335       1.52        ad  */
   1336       1.52        ad void
   1337       1.52        ad lwp_need_userret(struct lwp *l)
   1338       1.52        ad {
   1339       1.63        ad 	KASSERT(lwp_locked(l, NULL));
   1340       1.52        ad 
   1341       1.52        ad 	/*
   1342       1.52        ad 	 * Since the tests in lwp_userret() are done unlocked, make sure
   1343       1.52        ad 	 * that the condition will be seen before forcing the LWP to enter
   1344       1.52        ad 	 * kernel mode.
   1345       1.52        ad 	 */
   1346       1.81        ad 	membar_producer();
   1347       1.52        ad 	cpu_signotify(l);
   1348       1.52        ad }
   1349       1.52        ad 
   1350       1.52        ad /*
   1351       1.52        ad  * Add one reference to an LWP.  This will prevent the LWP from
   1352       1.52        ad  * exiting, thus keep the lwp structure and PCB around to inspect.
   1353       1.52        ad  */
   1354       1.52        ad void
   1355       1.52        ad lwp_addref(struct lwp *l)
   1356       1.52        ad {
   1357       1.52        ad 
   1358      1.103        ad 	KASSERT(mutex_owned(l->l_proc->p_lock));
   1359       1.52        ad 	KASSERT(l->l_stat != LSZOMB);
   1360       1.52        ad 	KASSERT(l->l_refcnt != 0);
   1361       1.52        ad 
   1362       1.52        ad 	l->l_refcnt++;
   1363       1.52        ad }
   1364       1.52        ad 
   1365       1.52        ad /*
   1366       1.52        ad  * Remove one reference to an LWP.  If this is the last reference,
   1367       1.52        ad  * then we must finalize the LWP's death.
   1368       1.52        ad  */
   1369       1.52        ad void
   1370       1.52        ad lwp_delref(struct lwp *l)
   1371       1.52        ad {
   1372       1.52        ad 	struct proc *p = l->l_proc;
   1373       1.52        ad 
   1374      1.103        ad 	mutex_enter(p->p_lock);
   1375       1.72        ad 	KASSERT(l->l_stat != LSZOMB);
   1376       1.72        ad 	KASSERT(l->l_refcnt > 0);
   1377       1.52        ad 	if (--l->l_refcnt == 0)
   1378       1.76        ad 		cv_broadcast(&p->p_lwpcv);
   1379      1.103        ad 	mutex_exit(p->p_lock);
   1380       1.52        ad }
   1381       1.52        ad 
   1382       1.52        ad /*
   1383       1.52        ad  * Drain all references to the current LWP.
   1384       1.52        ad  */
   1385       1.52        ad void
   1386       1.52        ad lwp_drainrefs(struct lwp *l)
   1387       1.52        ad {
   1388       1.52        ad 	struct proc *p = l->l_proc;
   1389       1.52        ad 
   1390      1.103        ad 	KASSERT(mutex_owned(p->p_lock));
   1391       1.52        ad 	KASSERT(l->l_refcnt != 0);
   1392       1.52        ad 
   1393       1.52        ad 	l->l_refcnt--;
   1394       1.52        ad 	while (l->l_refcnt != 0)
   1395      1.103        ad 		cv_wait(&p->p_lwpcv, p->p_lock);
   1396       1.37        ad }
   1397       1.41   thorpej 
   1398       1.41   thorpej /*
   1399  1.125.2.2     skrll  * Return true if the specified LWP is 'alive'.  Only p->p_lock need
   1400  1.125.2.2     skrll  * be held.
   1401  1.125.2.2     skrll  */
   1402  1.125.2.2     skrll bool
   1403  1.125.2.2     skrll lwp_alive(lwp_t *l)
   1404  1.125.2.2     skrll {
   1405  1.125.2.2     skrll 
   1406  1.125.2.2     skrll 	KASSERT(mutex_owned(l->l_proc->p_lock));
   1407  1.125.2.2     skrll 
   1408  1.125.2.2     skrll 	switch (l->l_stat) {
   1409  1.125.2.2     skrll 	case LSSLEEP:
   1410  1.125.2.2     skrll 	case LSRUN:
   1411  1.125.2.2     skrll 	case LSONPROC:
   1412  1.125.2.2     skrll 	case LSSTOP:
   1413  1.125.2.2     skrll 	case LSSUSPENDED:
   1414  1.125.2.2     skrll 		return true;
   1415  1.125.2.2     skrll 	default:
   1416  1.125.2.2     skrll 		return false;
   1417  1.125.2.2     skrll 	}
   1418  1.125.2.2     skrll }
   1419  1.125.2.2     skrll 
   1420  1.125.2.2     skrll /*
   1421  1.125.2.2     skrll  * Return first live LWP in the process.
   1422  1.125.2.2     skrll  */
   1423  1.125.2.2     skrll lwp_t *
   1424  1.125.2.2     skrll lwp_find_first(proc_t *p)
   1425  1.125.2.2     skrll {
   1426  1.125.2.2     skrll 	lwp_t *l;
   1427  1.125.2.2     skrll 
   1428  1.125.2.2     skrll 	KASSERT(mutex_owned(p->p_lock));
   1429  1.125.2.2     skrll 
   1430  1.125.2.2     skrll 	LIST_FOREACH(l, &p->p_lwps, l_sibling) {
   1431  1.125.2.2     skrll 		if (lwp_alive(l)) {
   1432  1.125.2.2     skrll 			return l;
   1433  1.125.2.2     skrll 		}
   1434  1.125.2.2     skrll 	}
   1435  1.125.2.2     skrll 
   1436  1.125.2.2     skrll 	return NULL;
   1437  1.125.2.2     skrll }
   1438  1.125.2.2     skrll 
   1439  1.125.2.2     skrll /*
   1440       1.41   thorpej  * lwp_specific_key_create --
   1441       1.41   thorpej  *	Create a key for subsystem lwp-specific data.
   1442       1.41   thorpej  */
   1443       1.41   thorpej int
   1444       1.41   thorpej lwp_specific_key_create(specificdata_key_t *keyp, specificdata_dtor_t dtor)
   1445       1.41   thorpej {
   1446       1.41   thorpej 
   1447       1.45   thorpej 	return (specificdata_key_create(lwp_specificdata_domain, keyp, dtor));
   1448       1.41   thorpej }
   1449       1.41   thorpej 
   1450       1.41   thorpej /*
   1451       1.41   thorpej  * lwp_specific_key_delete --
   1452       1.41   thorpej  *	Delete a key for subsystem lwp-specific data.
   1453       1.41   thorpej  */
   1454       1.41   thorpej void
   1455       1.41   thorpej lwp_specific_key_delete(specificdata_key_t key)
   1456       1.41   thorpej {
   1457       1.41   thorpej 
   1458       1.41   thorpej 	specificdata_key_delete(lwp_specificdata_domain, key);
   1459       1.41   thorpej }
   1460       1.41   thorpej 
   1461       1.45   thorpej /*
   1462       1.45   thorpej  * lwp_initspecific --
   1463       1.45   thorpej  *	Initialize an LWP's specificdata container.
   1464       1.45   thorpej  */
   1465       1.42  christos void
   1466       1.42  christos lwp_initspecific(struct lwp *l)
   1467       1.42  christos {
   1468       1.42  christos 	int error;
   1469       1.45   thorpej 
   1470       1.42  christos 	error = specificdata_init(lwp_specificdata_domain, &l->l_specdataref);
   1471       1.42  christos 	KASSERT(error == 0);
   1472       1.42  christos }
   1473       1.42  christos 
   1474       1.41   thorpej /*
   1475       1.45   thorpej  * lwp_finispecific --
   1476       1.45   thorpej  *	Finalize an LWP's specificdata container.
   1477       1.45   thorpej  */
   1478       1.45   thorpej void
   1479       1.45   thorpej lwp_finispecific(struct lwp *l)
   1480       1.45   thorpej {
   1481       1.45   thorpej 
   1482       1.45   thorpej 	specificdata_fini(lwp_specificdata_domain, &l->l_specdataref);
   1483       1.45   thorpej }
   1484       1.45   thorpej 
   1485       1.45   thorpej /*
   1486       1.41   thorpej  * lwp_getspecific --
   1487       1.41   thorpej  *	Return lwp-specific data corresponding to the specified key.
   1488       1.41   thorpej  *
   1489       1.41   thorpej  *	Note: LWP specific data is NOT INTERLOCKED.  An LWP should access
   1490       1.41   thorpej  *	only its OWN SPECIFIC DATA.  If it is necessary to access another
   1491       1.41   thorpej  *	LWP's specifc data, care must be taken to ensure that doing so
   1492       1.41   thorpej  *	would not cause internal data structure inconsistency (i.e. caller
   1493       1.41   thorpej  *	can guarantee that the target LWP is not inside an lwp_getspecific()
   1494       1.41   thorpej  *	or lwp_setspecific() call).
   1495       1.41   thorpej  */
   1496       1.41   thorpej void *
   1497       1.44   thorpej lwp_getspecific(specificdata_key_t key)
   1498       1.41   thorpej {
   1499       1.41   thorpej 
   1500       1.41   thorpej 	return (specificdata_getspecific_unlocked(lwp_specificdata_domain,
   1501       1.44   thorpej 						  &curlwp->l_specdataref, key));
   1502       1.41   thorpej }
   1503       1.41   thorpej 
   1504       1.47   hannken void *
   1505       1.47   hannken _lwp_getspecific_by_lwp(struct lwp *l, specificdata_key_t key)
   1506       1.47   hannken {
   1507       1.47   hannken 
   1508       1.47   hannken 	return (specificdata_getspecific_unlocked(lwp_specificdata_domain,
   1509       1.47   hannken 						  &l->l_specdataref, key));
   1510       1.47   hannken }
   1511       1.47   hannken 
   1512       1.41   thorpej /*
   1513       1.41   thorpej  * lwp_setspecific --
   1514       1.41   thorpej  *	Set lwp-specific data corresponding to the specified key.
   1515       1.41   thorpej  */
   1516       1.41   thorpej void
   1517       1.45   thorpej lwp_setspecific(specificdata_key_t key, void *data)
   1518       1.41   thorpej {
   1519       1.41   thorpej 
   1520       1.41   thorpej 	specificdata_setspecific(lwp_specificdata_domain,
   1521       1.44   thorpej 				 &curlwp->l_specdataref, key, data);
   1522       1.41   thorpej }
   1523       1.78        ad 
   1524       1.78        ad /*
   1525       1.78        ad  * Allocate a new lwpctl structure for a user LWP.
   1526       1.78        ad  */
   1527       1.78        ad int
   1528       1.78        ad lwp_ctl_alloc(vaddr_t *uaddr)
   1529       1.78        ad {
   1530       1.78        ad 	lcproc_t *lp;
   1531       1.78        ad 	u_int bit, i, offset;
   1532       1.78        ad 	struct uvm_object *uao;
   1533       1.78        ad 	int error;
   1534       1.78        ad 	lcpage_t *lcp;
   1535       1.78        ad 	proc_t *p;
   1536       1.78        ad 	lwp_t *l;
   1537       1.78        ad 
   1538       1.78        ad 	l = curlwp;
   1539       1.78        ad 	p = l->l_proc;
   1540       1.78        ad 
   1541       1.81        ad 	if (l->l_lcpage != NULL) {
   1542       1.81        ad 		lcp = l->l_lcpage;
   1543       1.81        ad 		*uaddr = lcp->lcp_uaddr + (vaddr_t)l->l_lwpctl - lcp->lcp_kaddr;
   1544       1.78        ad 		return (EINVAL);
   1545       1.81        ad 	}
   1546       1.78        ad 
   1547       1.78        ad 	/* First time around, allocate header structure for the process. */
   1548       1.78        ad 	if ((lp = p->p_lwpctl) == NULL) {
   1549       1.78        ad 		lp = kmem_alloc(sizeof(*lp), KM_SLEEP);
   1550       1.78        ad 		mutex_init(&lp->lp_lock, MUTEX_DEFAULT, IPL_NONE);
   1551       1.78        ad 		lp->lp_uao = NULL;
   1552       1.78        ad 		TAILQ_INIT(&lp->lp_pages);
   1553      1.103        ad 		mutex_enter(p->p_lock);
   1554       1.78        ad 		if (p->p_lwpctl == NULL) {
   1555       1.78        ad 			p->p_lwpctl = lp;
   1556      1.103        ad 			mutex_exit(p->p_lock);
   1557       1.78        ad 		} else {
   1558      1.103        ad 			mutex_exit(p->p_lock);
   1559       1.78        ad 			mutex_destroy(&lp->lp_lock);
   1560       1.78        ad 			kmem_free(lp, sizeof(*lp));
   1561       1.78        ad 			lp = p->p_lwpctl;
   1562       1.78        ad 		}
   1563       1.78        ad 	}
   1564       1.78        ad 
   1565       1.78        ad  	/*
   1566       1.78        ad  	 * Set up an anonymous memory region to hold the shared pages.
   1567       1.78        ad  	 * Map them into the process' address space.  The user vmspace
   1568       1.78        ad  	 * gets the first reference on the UAO.
   1569       1.78        ad  	 */
   1570       1.78        ad 	mutex_enter(&lp->lp_lock);
   1571       1.78        ad 	if (lp->lp_uao == NULL) {
   1572       1.78        ad 		lp->lp_uao = uao_create(LWPCTL_UAREA_SZ, 0);
   1573       1.78        ad 		lp->lp_cur = 0;
   1574       1.78        ad 		lp->lp_max = LWPCTL_UAREA_SZ;
   1575       1.78        ad 		lp->lp_uva = p->p_emul->e_vm_default_addr(p,
   1576       1.78        ad 		     (vaddr_t)p->p_vmspace->vm_daddr, LWPCTL_UAREA_SZ);
   1577       1.78        ad 		error = uvm_map(&p->p_vmspace->vm_map, &lp->lp_uva,
   1578       1.78        ad 		    LWPCTL_UAREA_SZ, lp->lp_uao, 0, 0, UVM_MAPFLAG(UVM_PROT_RW,
   1579       1.78        ad 		    UVM_PROT_RW, UVM_INH_NONE, UVM_ADV_NORMAL, 0));
   1580       1.78        ad 		if (error != 0) {
   1581       1.78        ad 			uao_detach(lp->lp_uao);
   1582       1.78        ad 			lp->lp_uao = NULL;
   1583       1.78        ad 			mutex_exit(&lp->lp_lock);
   1584       1.78        ad 			return error;
   1585       1.78        ad 		}
   1586       1.78        ad 	}
   1587       1.78        ad 
   1588       1.78        ad 	/* Get a free block and allocate for this LWP. */
   1589       1.78        ad 	TAILQ_FOREACH(lcp, &lp->lp_pages, lcp_chain) {
   1590       1.78        ad 		if (lcp->lcp_nfree != 0)
   1591       1.78        ad 			break;
   1592       1.78        ad 	}
   1593       1.78        ad 	if (lcp == NULL) {
   1594       1.78        ad 		/* Nothing available - try to set up a free page. */
   1595       1.78        ad 		if (lp->lp_cur == lp->lp_max) {
   1596       1.78        ad 			mutex_exit(&lp->lp_lock);
   1597       1.78        ad 			return ENOMEM;
   1598       1.78        ad 		}
   1599       1.78        ad 		lcp = kmem_alloc(LWPCTL_LCPAGE_SZ, KM_SLEEP);
   1600       1.79      yamt 		if (lcp == NULL) {
   1601       1.79      yamt 			mutex_exit(&lp->lp_lock);
   1602       1.78        ad 			return ENOMEM;
   1603       1.79      yamt 		}
   1604       1.78        ad 		/*
   1605       1.78        ad 		 * Wire the next page down in kernel space.  Since this
   1606       1.78        ad 		 * is a new mapping, we must add a reference.
   1607       1.78        ad 		 */
   1608       1.78        ad 		uao = lp->lp_uao;
   1609       1.78        ad 		(*uao->pgops->pgo_reference)(uao);
   1610       1.99        ad 		lcp->lcp_kaddr = vm_map_min(kernel_map);
   1611       1.78        ad 		error = uvm_map(kernel_map, &lcp->lcp_kaddr, PAGE_SIZE,
   1612       1.78        ad 		    uao, lp->lp_cur, PAGE_SIZE,
   1613       1.78        ad 		    UVM_MAPFLAG(UVM_PROT_RW, UVM_PROT_RW,
   1614       1.78        ad 		    UVM_INH_NONE, UVM_ADV_RANDOM, 0));
   1615       1.78        ad 		if (error != 0) {
   1616       1.78        ad 			mutex_exit(&lp->lp_lock);
   1617       1.78        ad 			kmem_free(lcp, LWPCTL_LCPAGE_SZ);
   1618       1.78        ad 			(*uao->pgops->pgo_detach)(uao);
   1619       1.78        ad 			return error;
   1620       1.78        ad 		}
   1621       1.89      yamt 		error = uvm_map_pageable(kernel_map, lcp->lcp_kaddr,
   1622       1.89      yamt 		    lcp->lcp_kaddr + PAGE_SIZE, FALSE, 0);
   1623       1.89      yamt 		if (error != 0) {
   1624       1.89      yamt 			mutex_exit(&lp->lp_lock);
   1625       1.89      yamt 			uvm_unmap(kernel_map, lcp->lcp_kaddr,
   1626       1.89      yamt 			    lcp->lcp_kaddr + PAGE_SIZE);
   1627       1.89      yamt 			kmem_free(lcp, LWPCTL_LCPAGE_SZ);
   1628       1.89      yamt 			return error;
   1629       1.89      yamt 		}
   1630       1.78        ad 		/* Prepare the page descriptor and link into the list. */
   1631       1.78        ad 		lcp->lcp_uaddr = lp->lp_uva + lp->lp_cur;
   1632       1.78        ad 		lp->lp_cur += PAGE_SIZE;
   1633       1.78        ad 		lcp->lcp_nfree = LWPCTL_PER_PAGE;
   1634       1.78        ad 		lcp->lcp_rotor = 0;
   1635       1.78        ad 		memset(lcp->lcp_bitmap, 0xff, LWPCTL_BITMAP_SZ);
   1636       1.78        ad 		TAILQ_INSERT_HEAD(&lp->lp_pages, lcp, lcp_chain);
   1637       1.78        ad 	}
   1638       1.78        ad 	for (i = lcp->lcp_rotor; lcp->lcp_bitmap[i] == 0;) {
   1639       1.78        ad 		if (++i >= LWPCTL_BITMAP_ENTRIES)
   1640       1.78        ad 			i = 0;
   1641       1.78        ad 	}
   1642       1.78        ad 	bit = ffs(lcp->lcp_bitmap[i]) - 1;
   1643       1.78        ad 	lcp->lcp_bitmap[i] ^= (1 << bit);
   1644       1.78        ad 	lcp->lcp_rotor = i;
   1645       1.78        ad 	lcp->lcp_nfree--;
   1646       1.78        ad 	l->l_lcpage = lcp;
   1647       1.78        ad 	offset = (i << 5) + bit;
   1648       1.78        ad 	l->l_lwpctl = (lwpctl_t *)lcp->lcp_kaddr + offset;
   1649       1.78        ad 	*uaddr = lcp->lcp_uaddr + offset * sizeof(lwpctl_t);
   1650       1.78        ad 	mutex_exit(&lp->lp_lock);
   1651       1.78        ad 
   1652      1.107        ad 	KPREEMPT_DISABLE(l);
   1653      1.111        ad 	l->l_lwpctl->lc_curcpu = (int)curcpu()->ci_data.cpu_index;
   1654      1.107        ad 	KPREEMPT_ENABLE(l);
   1655       1.78        ad 
   1656       1.78        ad 	return 0;
   1657       1.78        ad }
   1658       1.78        ad 
   1659       1.78        ad /*
   1660       1.78        ad  * Free an lwpctl structure back to the per-process list.
   1661       1.78        ad  */
   1662       1.78        ad void
   1663       1.78        ad lwp_ctl_free(lwp_t *l)
   1664       1.78        ad {
   1665       1.78        ad 	lcproc_t *lp;
   1666       1.78        ad 	lcpage_t *lcp;
   1667       1.78        ad 	u_int map, offset;
   1668       1.78        ad 
   1669       1.78        ad 	lp = l->l_proc->p_lwpctl;
   1670       1.78        ad 	KASSERT(lp != NULL);
   1671       1.78        ad 
   1672       1.78        ad 	lcp = l->l_lcpage;
   1673       1.78        ad 	offset = (u_int)((lwpctl_t *)l->l_lwpctl - (lwpctl_t *)lcp->lcp_kaddr);
   1674       1.78        ad 	KASSERT(offset < LWPCTL_PER_PAGE);
   1675       1.78        ad 
   1676       1.78        ad 	mutex_enter(&lp->lp_lock);
   1677       1.78        ad 	lcp->lcp_nfree++;
   1678       1.78        ad 	map = offset >> 5;
   1679       1.78        ad 	lcp->lcp_bitmap[map] |= (1 << (offset & 31));
   1680       1.78        ad 	if (lcp->lcp_bitmap[lcp->lcp_rotor] == 0)
   1681       1.78        ad 		lcp->lcp_rotor = map;
   1682       1.78        ad 	if (TAILQ_FIRST(&lp->lp_pages)->lcp_nfree == 0) {
   1683       1.78        ad 		TAILQ_REMOVE(&lp->lp_pages, lcp, lcp_chain);
   1684       1.78        ad 		TAILQ_INSERT_HEAD(&lp->lp_pages, lcp, lcp_chain);
   1685       1.78        ad 	}
   1686       1.78        ad 	mutex_exit(&lp->lp_lock);
   1687       1.78        ad }
   1688       1.78        ad 
   1689       1.78        ad /*
   1690       1.78        ad  * Process is exiting; tear down lwpctl state.  This can only be safely
   1691       1.78        ad  * called by the last LWP in the process.
   1692       1.78        ad  */
   1693       1.78        ad void
   1694       1.78        ad lwp_ctl_exit(void)
   1695       1.78        ad {
   1696       1.78        ad 	lcpage_t *lcp, *next;
   1697       1.78        ad 	lcproc_t *lp;
   1698       1.78        ad 	proc_t *p;
   1699       1.78        ad 	lwp_t *l;
   1700       1.78        ad 
   1701       1.78        ad 	l = curlwp;
   1702       1.78        ad 	l->l_lwpctl = NULL;
   1703       1.95        ad 	l->l_lcpage = NULL;
   1704       1.78        ad 	p = l->l_proc;
   1705       1.78        ad 	lp = p->p_lwpctl;
   1706       1.78        ad 
   1707       1.78        ad 	KASSERT(lp != NULL);
   1708       1.78        ad 	KASSERT(p->p_nlwps == 1);
   1709       1.78        ad 
   1710       1.78        ad 	for (lcp = TAILQ_FIRST(&lp->lp_pages); lcp != NULL; lcp = next) {
   1711       1.78        ad 		next = TAILQ_NEXT(lcp, lcp_chain);
   1712       1.78        ad 		uvm_unmap(kernel_map, lcp->lcp_kaddr,
   1713       1.78        ad 		    lcp->lcp_kaddr + PAGE_SIZE);
   1714       1.78        ad 		kmem_free(lcp, LWPCTL_LCPAGE_SZ);
   1715       1.78        ad 	}
   1716       1.78        ad 
   1717       1.78        ad 	if (lp->lp_uao != NULL) {
   1718       1.78        ad 		uvm_unmap(&p->p_vmspace->vm_map, lp->lp_uva,
   1719       1.78        ad 		    lp->lp_uva + LWPCTL_UAREA_SZ);
   1720       1.78        ad 	}
   1721       1.78        ad 
   1722       1.78        ad 	mutex_destroy(&lp->lp_lock);
   1723       1.78        ad 	kmem_free(lp, sizeof(*lp));
   1724       1.78        ad 	p->p_lwpctl = NULL;
   1725       1.78        ad }
   1726       1.84      yamt 
   1727       1.84      yamt #if defined(DDB)
   1728       1.84      yamt void
   1729       1.84      yamt lwp_whatis(uintptr_t addr, void (*pr)(const char *, ...))
   1730       1.84      yamt {
   1731       1.84      yamt 	lwp_t *l;
   1732       1.84      yamt 
   1733       1.84      yamt 	LIST_FOREACH(l, &alllwp, l_list) {
   1734       1.84      yamt 		uintptr_t stack = (uintptr_t)KSTACK_LOWEST_ADDR(l);
   1735       1.84      yamt 
   1736       1.84      yamt 		if (addr < stack || stack + KSTACK_SIZE <= addr) {
   1737       1.84      yamt 			continue;
   1738       1.84      yamt 		}
   1739       1.84      yamt 		(*pr)("%p is %p+%zu, LWP %p's stack\n",
   1740       1.84      yamt 		    (void *)addr, (void *)stack,
   1741       1.84      yamt 		    (size_t)(addr - stack), l);
   1742       1.84      yamt 	}
   1743       1.84      yamt }
   1744       1.84      yamt #endif /* defined(DDB) */
   1745