Home | History | Annotate | Line # | Download | only in kern
kern_lwp.c revision 1.157
      1  1.157     rmind /*	$NetBSD: kern_lwp.c,v 1.157 2011/03/20 23:19:16 rmind Exp $	*/
      2    1.2   thorpej 
      3    1.2   thorpej /*-
      4  1.127        ad  * Copyright (c) 2001, 2006, 2007, 2008, 2009 The NetBSD Foundation, Inc.
      5    1.2   thorpej  * All rights reserved.
      6    1.2   thorpej  *
      7    1.2   thorpej  * This code is derived from software contributed to The NetBSD Foundation
      8   1.52        ad  * by Nathan J. Williams, and Andrew Doran.
      9    1.2   thorpej  *
     10    1.2   thorpej  * Redistribution and use in source and binary forms, with or without
     11    1.2   thorpej  * modification, are permitted provided that the following conditions
     12    1.2   thorpej  * are met:
     13    1.2   thorpej  * 1. Redistributions of source code must retain the above copyright
     14    1.2   thorpej  *    notice, this list of conditions and the following disclaimer.
     15    1.2   thorpej  * 2. Redistributions in binary form must reproduce the above copyright
     16    1.2   thorpej  *    notice, this list of conditions and the following disclaimer in the
     17    1.2   thorpej  *    documentation and/or other materials provided with the distribution.
     18    1.2   thorpej  *
     19    1.2   thorpej  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     20    1.2   thorpej  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     21    1.2   thorpej  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     22    1.2   thorpej  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     23    1.2   thorpej  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     24    1.2   thorpej  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     25    1.2   thorpej  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     26    1.2   thorpej  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     27    1.2   thorpej  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     28    1.2   thorpej  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     29    1.2   thorpej  * POSSIBILITY OF SUCH DAMAGE.
     30    1.2   thorpej  */
     31    1.9     lukem 
     32   1.52        ad /*
     33   1.52        ad  * Overview
     34   1.52        ad  *
     35   1.66        ad  *	Lightweight processes (LWPs) are the basic unit or thread of
     36   1.52        ad  *	execution within the kernel.  The core state of an LWP is described
     37   1.66        ad  *	by "struct lwp", also known as lwp_t.
     38   1.52        ad  *
     39   1.52        ad  *	Each LWP is contained within a process (described by "struct proc"),
     40   1.52        ad  *	Every process contains at least one LWP, but may contain more.  The
     41   1.52        ad  *	process describes attributes shared among all of its LWPs such as a
     42   1.52        ad  *	private address space, global execution state (stopped, active,
     43   1.52        ad  *	zombie, ...), signal disposition and so on.  On a multiprocessor
     44   1.66        ad  *	machine, multiple LWPs be executing concurrently in the kernel.
     45   1.52        ad  *
     46   1.52        ad  * Execution states
     47   1.52        ad  *
     48   1.52        ad  *	At any given time, an LWP has overall state that is described by
     49   1.52        ad  *	lwp::l_stat.  The states are broken into two sets below.  The first
     50   1.52        ad  *	set is guaranteed to represent the absolute, current state of the
     51   1.52        ad  *	LWP:
     52  1.101     rmind  *
     53  1.101     rmind  *	LSONPROC
     54  1.101     rmind  *
     55  1.101     rmind  *		On processor: the LWP is executing on a CPU, either in the
     56  1.101     rmind  *		kernel or in user space.
     57  1.101     rmind  *
     58  1.101     rmind  *	LSRUN
     59  1.101     rmind  *
     60  1.101     rmind  *		Runnable: the LWP is parked on a run queue, and may soon be
     61  1.101     rmind  *		chosen to run by an idle processor, or by a processor that
     62  1.101     rmind  *		has been asked to preempt a currently runnning but lower
     63  1.134     rmind  *		priority LWP.
     64  1.101     rmind  *
     65  1.101     rmind  *	LSIDL
     66  1.101     rmind  *
     67  1.101     rmind  *		Idle: the LWP has been created but has not yet executed,
     68   1.66        ad  *		or it has ceased executing a unit of work and is waiting
     69   1.66        ad  *		to be started again.
     70  1.101     rmind  *
     71  1.101     rmind  *	LSSUSPENDED:
     72  1.101     rmind  *
     73  1.101     rmind  *		Suspended: the LWP has had its execution suspended by
     74   1.52        ad  *		another LWP in the same process using the _lwp_suspend()
     75   1.52        ad  *		system call.  User-level LWPs also enter the suspended
     76   1.52        ad  *		state when the system is shutting down.
     77   1.52        ad  *
     78   1.52        ad  *	The second set represent a "statement of intent" on behalf of the
     79   1.52        ad  *	LWP.  The LWP may in fact be executing on a processor, may be
     80   1.66        ad  *	sleeping or idle. It is expected to take the necessary action to
     81  1.101     rmind  *	stop executing or become "running" again within a short timeframe.
     82  1.115        ad  *	The LP_RUNNING flag in lwp::l_pflag indicates that an LWP is running.
     83  1.101     rmind  *	Importantly, it indicates that its state is tied to a CPU.
     84  1.101     rmind  *
     85  1.101     rmind  *	LSZOMB:
     86  1.101     rmind  *
     87  1.101     rmind  *		Dead or dying: the LWP has released most of its resources
     88  1.129        ad  *		and is about to switch away into oblivion, or has already
     89   1.66        ad  *		switched away.  When it switches away, its few remaining
     90   1.66        ad  *		resources can be collected.
     91  1.101     rmind  *
     92  1.101     rmind  *	LSSLEEP:
     93  1.101     rmind  *
     94  1.101     rmind  *		Sleeping: the LWP has entered itself onto a sleep queue, and
     95  1.101     rmind  *		has switched away or will switch away shortly to allow other
     96   1.66        ad  *		LWPs to run on the CPU.
     97  1.101     rmind  *
     98  1.101     rmind  *	LSSTOP:
     99  1.101     rmind  *
    100  1.101     rmind  *		Stopped: the LWP has been stopped as a result of a job
    101  1.101     rmind  *		control signal, or as a result of the ptrace() interface.
    102  1.101     rmind  *
    103  1.101     rmind  *		Stopped LWPs may run briefly within the kernel to handle
    104  1.101     rmind  *		signals that they receive, but will not return to user space
    105  1.101     rmind  *		until their process' state is changed away from stopped.
    106  1.101     rmind  *
    107  1.101     rmind  *		Single LWPs within a process can not be set stopped
    108  1.101     rmind  *		selectively: all actions that can stop or continue LWPs
    109  1.101     rmind  *		occur at the process level.
    110  1.101     rmind  *
    111   1.52        ad  * State transitions
    112   1.52        ad  *
    113   1.66        ad  *	Note that the LSSTOP state may only be set when returning to
    114   1.66        ad  *	user space in userret(), or when sleeping interruptably.  The
    115   1.66        ad  *	LSSUSPENDED state may only be set in userret().  Before setting
    116   1.66        ad  *	those states, we try to ensure that the LWPs will release all
    117   1.66        ad  *	locks that they hold, and at a minimum try to ensure that the
    118   1.66        ad  *	LWP can be set runnable again by a signal.
    119   1.52        ad  *
    120   1.52        ad  *	LWPs may transition states in the following ways:
    121   1.52        ad  *
    122   1.52        ad  *	 RUN -------> ONPROC		ONPROC -----> RUN
    123  1.129        ad  *		    				    > SLEEP
    124  1.129        ad  *		    				    > STOPPED
    125   1.52        ad  *						    > SUSPENDED
    126   1.52        ad  *						    > ZOMB
    127  1.129        ad  *						    > IDL (special cases)
    128   1.52        ad  *
    129   1.52        ad  *	 STOPPED ---> RUN		SUSPENDED --> RUN
    130  1.129        ad  *	            > SLEEP
    131   1.52        ad  *
    132   1.52        ad  *	 SLEEP -----> ONPROC		IDL --------> RUN
    133  1.101     rmind  *		    > RUN			    > SUSPENDED
    134  1.101     rmind  *		    > STOPPED			    > STOPPED
    135  1.129        ad  *						    > ONPROC (special cases)
    136   1.52        ad  *
    137  1.129        ad  *	Some state transitions are only possible with kernel threads (eg
    138  1.129        ad  *	ONPROC -> IDL) and happen under tightly controlled circumstances
    139  1.129        ad  *	free of unwanted side effects.
    140   1.66        ad  *
    141  1.114     rmind  * Migration
    142  1.114     rmind  *
    143  1.114     rmind  *	Migration of threads from one CPU to another could be performed
    144  1.114     rmind  *	internally by the scheduler via sched_takecpu() or sched_catchlwp()
    145  1.114     rmind  *	functions.  The universal lwp_migrate() function should be used for
    146  1.114     rmind  *	any other cases.  Subsystems in the kernel must be aware that CPU
    147  1.114     rmind  *	of LWP may change, while it is not locked.
    148  1.114     rmind  *
    149   1.52        ad  * Locking
    150   1.52        ad  *
    151   1.52        ad  *	The majority of fields in 'struct lwp' are covered by a single,
    152   1.66        ad  *	general spin lock pointed to by lwp::l_mutex.  The locks covering
    153   1.52        ad  *	each field are documented in sys/lwp.h.
    154   1.52        ad  *
    155   1.66        ad  *	State transitions must be made with the LWP's general lock held,
    156  1.152     rmind  *	and may cause the LWP's lock pointer to change.  Manipulation of
    157   1.66        ad  *	the general lock is not performed directly, but through calls to
    158  1.152     rmind  *	lwp_lock(), lwp_unlock() and others.  It should be noted that the
    159  1.152     rmind  *	adaptive locks are not allowed to be released while the LWP's lock
    160  1.152     rmind  *	is being held (unlike for other spin-locks).
    161   1.52        ad  *
    162   1.52        ad  *	States and their associated locks:
    163   1.52        ad  *
    164   1.74     rmind  *	LSONPROC, LSZOMB:
    165   1.52        ad  *
    166   1.64      yamt  *		Always covered by spc_lwplock, which protects running LWPs.
    167  1.129        ad  *		This is a per-CPU lock and matches lwp::l_cpu.
    168   1.52        ad  *
    169   1.74     rmind  *	LSIDL, LSRUN:
    170   1.52        ad  *
    171   1.64      yamt  *		Always covered by spc_mutex, which protects the run queues.
    172  1.129        ad  *		This is a per-CPU lock and matches lwp::l_cpu.
    173   1.52        ad  *
    174   1.52        ad  *	LSSLEEP:
    175   1.52        ad  *
    176   1.66        ad  *		Covered by a lock associated with the sleep queue that the
    177  1.129        ad  *		LWP resides on.  Matches lwp::l_sleepq::sq_mutex.
    178   1.52        ad  *
    179   1.52        ad  *	LSSTOP, LSSUSPENDED:
    180  1.101     rmind  *
    181   1.52        ad  *		If the LWP was previously sleeping (l_wchan != NULL), then
    182   1.66        ad  *		l_mutex references the sleep queue lock.  If the LWP was
    183   1.52        ad  *		runnable or on the CPU when halted, or has been removed from
    184   1.66        ad  *		the sleep queue since halted, then the lock is spc_lwplock.
    185   1.52        ad  *
    186   1.52        ad  *	The lock order is as follows:
    187   1.52        ad  *
    188   1.64      yamt  *		spc::spc_lwplock ->
    189  1.112        ad  *		    sleeptab::st_mutex ->
    190   1.64      yamt  *			tschain_t::tc_mutex ->
    191   1.64      yamt  *			    spc::spc_mutex
    192   1.52        ad  *
    193  1.103        ad  *	Each process has an scheduler state lock (proc::p_lock), and a
    194   1.52        ad  *	number of counters on LWPs and their states: p_nzlwps, p_nrlwps, and
    195   1.52        ad  *	so on.  When an LWP is to be entered into or removed from one of the
    196  1.103        ad  *	following states, p_lock must be held and the process wide counters
    197   1.52        ad  *	adjusted:
    198   1.52        ad  *
    199   1.52        ad  *		LSIDL, LSZOMB, LSSTOP, LSSUSPENDED
    200   1.52        ad  *
    201  1.129        ad  *	(But not always for kernel threads.  There are some special cases
    202  1.129        ad  *	as mentioned above.  See kern_softint.c.)
    203  1.129        ad  *
    204   1.52        ad  *	Note that an LWP is considered running or likely to run soon if in
    205   1.52        ad  *	one of the following states.  This affects the value of p_nrlwps:
    206   1.52        ad  *
    207   1.52        ad  *		LSRUN, LSONPROC, LSSLEEP
    208   1.52        ad  *
    209  1.103        ad  *	p_lock does not need to be held when transitioning among these
    210  1.129        ad  *	three states, hence p_lock is rarely taken for state transitions.
    211   1.52        ad  */
    212   1.52        ad 
    213    1.9     lukem #include <sys/cdefs.h>
    214  1.157     rmind __KERNEL_RCSID(0, "$NetBSD: kern_lwp.c,v 1.157 2011/03/20 23:19:16 rmind Exp $");
    215    1.8    martin 
    216   1.84      yamt #include "opt_ddb.h"
    217   1.52        ad #include "opt_lockdebug.h"
    218  1.124  wrstuden #include "opt_sa.h"
    219  1.139    darran #include "opt_dtrace.h"
    220    1.2   thorpej 
    221   1.47   hannken #define _LWP_API_PRIVATE
    222   1.47   hannken 
    223    1.2   thorpej #include <sys/param.h>
    224    1.2   thorpej #include <sys/systm.h>
    225   1.64      yamt #include <sys/cpu.h>
    226    1.2   thorpej #include <sys/pool.h>
    227    1.2   thorpej #include <sys/proc.h>
    228  1.124  wrstuden #include <sys/sa.h>
    229  1.124  wrstuden #include <sys/savar.h>
    230    1.2   thorpej #include <sys/syscallargs.h>
    231   1.57       dsl #include <sys/syscall_stats.h>
    232   1.37        ad #include <sys/kauth.h>
    233   1.52        ad #include <sys/sleepq.h>
    234   1.52        ad #include <sys/lockdebug.h>
    235   1.52        ad #include <sys/kmem.h>
    236   1.91     rmind #include <sys/pset.h>
    237   1.75        ad #include <sys/intr.h>
    238   1.78        ad #include <sys/lwpctl.h>
    239   1.81        ad #include <sys/atomic.h>
    240  1.131        ad #include <sys/filedesc.h>
    241  1.138    darran #include <sys/dtrace_bsd.h>
    242  1.141    darran #include <sys/sdt.h>
    243  1.157     rmind #include <sys/xcall.h>
    244  1.138    darran 
    245    1.2   thorpej #include <uvm/uvm_extern.h>
    246   1.80     skrll #include <uvm/uvm_object.h>
    247    1.2   thorpej 
    248  1.152     rmind static pool_cache_t	lwp_cache	__read_mostly;
    249  1.152     rmind struct lwplist		alllwp		__cacheline_aligned;
    250   1.41   thorpej 
    251  1.157     rmind static void		lwp_dtor(void *, void *);
    252  1.157     rmind 
    253  1.141    darran /* DTrace proc provider probes */
    254  1.141    darran SDT_PROBE_DEFINE(proc,,,lwp_create,
    255  1.141    darran 	"struct lwp *", NULL,
    256  1.141    darran 	NULL, NULL, NULL, NULL,
    257  1.141    darran 	NULL, NULL, NULL, NULL);
    258  1.141    darran SDT_PROBE_DEFINE(proc,,,lwp_start,
    259  1.141    darran 	"struct lwp *", NULL,
    260  1.141    darran 	NULL, NULL, NULL, NULL,
    261  1.141    darran 	NULL, NULL, NULL, NULL);
    262  1.141    darran SDT_PROBE_DEFINE(proc,,,lwp_exit,
    263  1.141    darran 	"struct lwp *", NULL,
    264  1.141    darran 	NULL, NULL, NULL, NULL,
    265  1.141    darran 	NULL, NULL, NULL, NULL);
    266  1.141    darran 
    267  1.147     pooka struct turnstile turnstile0;
    268  1.147     pooka struct lwp lwp0 __aligned(MIN_LWP_ALIGNMENT) = {
    269  1.147     pooka #ifdef LWP0_CPU_INFO
    270  1.147     pooka 	.l_cpu = LWP0_CPU_INFO,
    271  1.147     pooka #endif
    272  1.154      matt #ifdef LWP0_MD_INITIALIZER
    273  1.154      matt 	.l_md = LWP0_MD_INITIALIZER,
    274  1.154      matt #endif
    275  1.147     pooka 	.l_proc = &proc0,
    276  1.147     pooka 	.l_lid = 1,
    277  1.147     pooka 	.l_flag = LW_SYSTEM,
    278  1.147     pooka 	.l_stat = LSONPROC,
    279  1.147     pooka 	.l_ts = &turnstile0,
    280  1.147     pooka 	.l_syncobj = &sched_syncobj,
    281  1.147     pooka 	.l_refcnt = 1,
    282  1.147     pooka 	.l_priority = PRI_USER + NPRI_USER - 1,
    283  1.147     pooka 	.l_inheritedprio = -1,
    284  1.147     pooka 	.l_class = SCHED_OTHER,
    285  1.147     pooka 	.l_psid = PS_NONE,
    286  1.147     pooka 	.l_pi_lenders = SLIST_HEAD_INITIALIZER(&lwp0.l_pi_lenders),
    287  1.147     pooka 	.l_name = __UNCONST("swapper"),
    288  1.147     pooka 	.l_fd = &filedesc0,
    289  1.147     pooka };
    290  1.147     pooka 
    291   1.41   thorpej void
    292   1.41   thorpej lwpinit(void)
    293   1.41   thorpej {
    294   1.41   thorpej 
    295  1.152     rmind 	LIST_INIT(&alllwp);
    296  1.144     pooka 	lwpinit_specificdata();
    297   1.52        ad 	lwp_sys_init();
    298   1.87        ad 	lwp_cache = pool_cache_init(sizeof(lwp_t), MIN_LWP_ALIGNMENT, 0, 0,
    299  1.157     rmind 	    "lwppl", NULL, IPL_NONE, NULL, lwp_dtor, NULL);
    300   1.41   thorpej }
    301   1.41   thorpej 
    302  1.147     pooka void
    303  1.147     pooka lwp0_init(void)
    304  1.147     pooka {
    305  1.147     pooka 	struct lwp *l = &lwp0;
    306  1.147     pooka 
    307  1.147     pooka 	KASSERT((void *)uvm_lwp_getuarea(l) != NULL);
    308  1.148     pooka 	KASSERT(l->l_lid == proc0.p_nlwpid);
    309  1.147     pooka 
    310  1.147     pooka 	LIST_INSERT_HEAD(&alllwp, l, l_list);
    311  1.147     pooka 
    312  1.147     pooka 	callout_init(&l->l_timeout_ch, CALLOUT_MPSAFE);
    313  1.147     pooka 	callout_setfunc(&l->l_timeout_ch, sleepq_timeout, l);
    314  1.147     pooka 	cv_init(&l->l_sigcv, "sigwait");
    315  1.147     pooka 
    316  1.147     pooka 	kauth_cred_hold(proc0.p_cred);
    317  1.147     pooka 	l->l_cred = proc0.p_cred;
    318  1.147     pooka 
    319  1.147     pooka 	lwp_initspecific(l);
    320  1.147     pooka 
    321  1.147     pooka 	SYSCALL_TIME_LWP_INIT(l);
    322  1.147     pooka }
    323  1.147     pooka 
    324  1.157     rmind static void
    325  1.157     rmind lwp_dtor(void *arg, void *obj)
    326  1.157     rmind {
    327  1.157     rmind 	lwp_t *l = obj;
    328  1.157     rmind 	uint64_t where;
    329  1.157     rmind 	(void)l;
    330  1.157     rmind 
    331  1.157     rmind 	/*
    332  1.157     rmind 	 * Provide a barrier to ensure that all mutex_oncpu() and rw_oncpu()
    333  1.157     rmind 	 * calls will exit before memory of LWP is returned to the pool, where
    334  1.157     rmind 	 * KVA of LWP structure might be freed and re-used for other purposes.
    335  1.157     rmind 	 * Kernel preemption is disabled around mutex_oncpu() and rw_oncpu()
    336  1.157     rmind 	 * callers, therefore cross-call to all CPUs will do the job.  Also,
    337  1.157     rmind 	 * the value of l->l_cpu must be still valid at this point.
    338  1.157     rmind 	 */
    339  1.157     rmind 	KASSERT(l->l_cpu != NULL);
    340  1.157     rmind 	where = xc_broadcast(0, (xcfunc_t)nullop, NULL, NULL);
    341  1.157     rmind 	xc_wait(where);
    342  1.157     rmind }
    343  1.157     rmind 
    344   1.52        ad /*
    345   1.52        ad  * Set an suspended.
    346   1.52        ad  *
    347  1.103        ad  * Must be called with p_lock held, and the LWP locked.  Will unlock the
    348   1.52        ad  * LWP before return.
    349   1.52        ad  */
    350    1.2   thorpej int
    351   1.52        ad lwp_suspend(struct lwp *curl, struct lwp *t)
    352    1.2   thorpej {
    353   1.52        ad 	int error;
    354    1.2   thorpej 
    355  1.103        ad 	KASSERT(mutex_owned(t->l_proc->p_lock));
    356   1.63        ad 	KASSERT(lwp_locked(t, NULL));
    357   1.33       chs 
    358   1.52        ad 	KASSERT(curl != t || curl->l_stat == LSONPROC);
    359    1.2   thorpej 
    360   1.52        ad 	/*
    361   1.52        ad 	 * If the current LWP has been told to exit, we must not suspend anyone
    362   1.52        ad 	 * else or deadlock could occur.  We won't return to userspace.
    363    1.2   thorpej 	 */
    364  1.109     rmind 	if ((curl->l_flag & (LW_WEXIT | LW_WCORE)) != 0) {
    365   1.52        ad 		lwp_unlock(t);
    366   1.52        ad 		return (EDEADLK);
    367    1.2   thorpej 	}
    368    1.2   thorpej 
    369   1.52        ad 	error = 0;
    370    1.2   thorpej 
    371   1.52        ad 	switch (t->l_stat) {
    372   1.52        ad 	case LSRUN:
    373   1.52        ad 	case LSONPROC:
    374   1.56     pavel 		t->l_flag |= LW_WSUSPEND;
    375   1.52        ad 		lwp_need_userret(t);
    376   1.52        ad 		lwp_unlock(t);
    377   1.52        ad 		break;
    378    1.2   thorpej 
    379   1.52        ad 	case LSSLEEP:
    380   1.56     pavel 		t->l_flag |= LW_WSUSPEND;
    381    1.2   thorpej 
    382    1.2   thorpej 		/*
    383   1.52        ad 		 * Kick the LWP and try to get it to the kernel boundary
    384   1.52        ad 		 * so that it will release any locks that it holds.
    385   1.52        ad 		 * setrunnable() will release the lock.
    386    1.2   thorpej 		 */
    387   1.56     pavel 		if ((t->l_flag & LW_SINTR) != 0)
    388   1.52        ad 			setrunnable(t);
    389   1.52        ad 		else
    390   1.52        ad 			lwp_unlock(t);
    391   1.52        ad 		break;
    392    1.2   thorpej 
    393   1.52        ad 	case LSSUSPENDED:
    394   1.52        ad 		lwp_unlock(t);
    395   1.52        ad 		break;
    396   1.17      manu 
    397   1.52        ad 	case LSSTOP:
    398   1.56     pavel 		t->l_flag |= LW_WSUSPEND;
    399   1.52        ad 		setrunnable(t);
    400   1.52        ad 		break;
    401    1.2   thorpej 
    402   1.52        ad 	case LSIDL:
    403   1.52        ad 	case LSZOMB:
    404   1.52        ad 		error = EINTR; /* It's what Solaris does..... */
    405   1.52        ad 		lwp_unlock(t);
    406   1.52        ad 		break;
    407    1.2   thorpej 	}
    408    1.2   thorpej 
    409   1.69     rmind 	return (error);
    410    1.2   thorpej }
    411    1.2   thorpej 
    412   1.52        ad /*
    413   1.52        ad  * Restart a suspended LWP.
    414   1.52        ad  *
    415  1.103        ad  * Must be called with p_lock held, and the LWP locked.  Will unlock the
    416   1.52        ad  * LWP before return.
    417   1.52        ad  */
    418    1.2   thorpej void
    419    1.2   thorpej lwp_continue(struct lwp *l)
    420    1.2   thorpej {
    421    1.2   thorpej 
    422  1.103        ad 	KASSERT(mutex_owned(l->l_proc->p_lock));
    423   1.63        ad 	KASSERT(lwp_locked(l, NULL));
    424   1.52        ad 
    425   1.52        ad 	/* If rebooting or not suspended, then just bail out. */
    426   1.56     pavel 	if ((l->l_flag & LW_WREBOOT) != 0) {
    427   1.52        ad 		lwp_unlock(l);
    428    1.2   thorpej 		return;
    429   1.10      fvdl 	}
    430    1.2   thorpej 
    431   1.56     pavel 	l->l_flag &= ~LW_WSUSPEND;
    432    1.2   thorpej 
    433   1.52        ad 	if (l->l_stat != LSSUSPENDED) {
    434   1.52        ad 		lwp_unlock(l);
    435   1.52        ad 		return;
    436    1.2   thorpej 	}
    437    1.2   thorpej 
    438   1.52        ad 	/* setrunnable() will release the lock. */
    439   1.52        ad 	setrunnable(l);
    440    1.2   thorpej }
    441    1.2   thorpej 
    442   1.52        ad /*
    443  1.142  christos  * Restart a stopped LWP.
    444  1.142  christos  *
    445  1.142  christos  * Must be called with p_lock held, and the LWP NOT locked.  Will unlock the
    446  1.142  christos  * LWP before return.
    447  1.142  christos  */
    448  1.142  christos void
    449  1.142  christos lwp_unstop(struct lwp *l)
    450  1.142  christos {
    451  1.142  christos 	struct proc *p = l->l_proc;
    452  1.142  christos 
    453  1.142  christos 	KASSERT(mutex_owned(proc_lock));
    454  1.142  christos 	KASSERT(mutex_owned(p->p_lock));
    455  1.142  christos 
    456  1.142  christos 	lwp_lock(l);
    457  1.142  christos 
    458  1.142  christos 	/* If not stopped, then just bail out. */
    459  1.142  christos 	if (l->l_stat != LSSTOP) {
    460  1.142  christos 		lwp_unlock(l);
    461  1.142  christos 		return;
    462  1.142  christos 	}
    463  1.142  christos 
    464  1.142  christos 	p->p_stat = SACTIVE;
    465  1.142  christos 	p->p_sflag &= ~PS_STOPPING;
    466  1.142  christos 
    467  1.142  christos 	if (!p->p_waited)
    468  1.142  christos 		p->p_pptr->p_nstopchild--;
    469  1.142  christos 
    470  1.142  christos 	if (l->l_wchan == NULL) {
    471  1.142  christos 		/* setrunnable() will release the lock. */
    472  1.142  christos 		setrunnable(l);
    473  1.142  christos 	} else {
    474  1.142  christos 		l->l_stat = LSSLEEP;
    475  1.142  christos 		p->p_nrlwps++;
    476  1.142  christos 		lwp_unlock(l);
    477  1.142  christos 	}
    478  1.142  christos }
    479  1.142  christos 
    480  1.142  christos /*
    481   1.52        ad  * Wait for an LWP within the current process to exit.  If 'lid' is
    482   1.52        ad  * non-zero, we are waiting for a specific LWP.
    483   1.52        ad  *
    484  1.103        ad  * Must be called with p->p_lock held.
    485   1.52        ad  */
    486    1.2   thorpej int
    487    1.2   thorpej lwp_wait1(struct lwp *l, lwpid_t lid, lwpid_t *departed, int flags)
    488    1.2   thorpej {
    489    1.2   thorpej 	struct proc *p = l->l_proc;
    490   1.52        ad 	struct lwp *l2;
    491   1.52        ad 	int nfound, error;
    492   1.63        ad 	lwpid_t curlid;
    493   1.63        ad 	bool exiting;
    494    1.2   thorpej 
    495  1.103        ad 	KASSERT(mutex_owned(p->p_lock));
    496   1.52        ad 
    497   1.52        ad 	p->p_nlwpwait++;
    498   1.63        ad 	l->l_waitingfor = lid;
    499   1.63        ad 	curlid = l->l_lid;
    500   1.63        ad 	exiting = ((flags & LWPWAIT_EXITCONTROL) != 0);
    501   1.52        ad 
    502   1.52        ad 	for (;;) {
    503   1.52        ad 		/*
    504   1.52        ad 		 * Avoid a race between exit1() and sigexit(): if the
    505   1.52        ad 		 * process is dumping core, then we need to bail out: call
    506   1.52        ad 		 * into lwp_userret() where we will be suspended until the
    507   1.52        ad 		 * deed is done.
    508   1.52        ad 		 */
    509   1.52        ad 		if ((p->p_sflag & PS_WCORE) != 0) {
    510  1.103        ad 			mutex_exit(p->p_lock);
    511   1.52        ad 			lwp_userret(l);
    512   1.52        ad #ifdef DIAGNOSTIC
    513   1.52        ad 			panic("lwp_wait1");
    514   1.52        ad #endif
    515   1.52        ad 			/* NOTREACHED */
    516   1.52        ad 		}
    517   1.52        ad 
    518   1.52        ad 		/*
    519   1.52        ad 		 * First off, drain any detached LWP that is waiting to be
    520   1.52        ad 		 * reaped.
    521   1.52        ad 		 */
    522   1.52        ad 		while ((l2 = p->p_zomblwp) != NULL) {
    523   1.52        ad 			p->p_zomblwp = NULL;
    524   1.63        ad 			lwp_free(l2, false, false);/* releases proc mutex */
    525  1.103        ad 			mutex_enter(p->p_lock);
    526   1.52        ad 		}
    527   1.52        ad 
    528   1.52        ad 		/*
    529   1.52        ad 		 * Now look for an LWP to collect.  If the whole process is
    530   1.52        ad 		 * exiting, count detached LWPs as eligible to be collected,
    531   1.52        ad 		 * but don't drain them here.
    532   1.52        ad 		 */
    533   1.52        ad 		nfound = 0;
    534   1.63        ad 		error = 0;
    535   1.52        ad 		LIST_FOREACH(l2, &p->p_lwps, l_sibling) {
    536   1.63        ad 			/*
    537   1.63        ad 			 * If a specific wait and the target is waiting on
    538   1.63        ad 			 * us, then avoid deadlock.  This also traps LWPs
    539   1.63        ad 			 * that try to wait on themselves.
    540   1.63        ad 			 *
    541   1.63        ad 			 * Note that this does not handle more complicated
    542   1.63        ad 			 * cycles, like: t1 -> t2 -> t3 -> t1.  The process
    543   1.63        ad 			 * can still be killed so it is not a major problem.
    544   1.63        ad 			 */
    545   1.63        ad 			if (l2->l_lid == lid && l2->l_waitingfor == curlid) {
    546   1.63        ad 				error = EDEADLK;
    547   1.63        ad 				break;
    548   1.63        ad 			}
    549   1.63        ad 			if (l2 == l)
    550   1.52        ad 				continue;
    551   1.52        ad 			if ((l2->l_prflag & LPR_DETACHED) != 0) {
    552   1.63        ad 				nfound += exiting;
    553   1.63        ad 				continue;
    554   1.63        ad 			}
    555   1.63        ad 			if (lid != 0) {
    556   1.63        ad 				if (l2->l_lid != lid)
    557   1.63        ad 					continue;
    558   1.63        ad 				/*
    559   1.63        ad 				 * Mark this LWP as the first waiter, if there
    560   1.63        ad 				 * is no other.
    561   1.63        ad 				 */
    562   1.63        ad 				if (l2->l_waiter == 0)
    563   1.63        ad 					l2->l_waiter = curlid;
    564   1.63        ad 			} else if (l2->l_waiter != 0) {
    565   1.63        ad 				/*
    566   1.63        ad 				 * It already has a waiter - so don't
    567   1.63        ad 				 * collect it.  If the waiter doesn't
    568   1.63        ad 				 * grab it we'll get another chance
    569   1.63        ad 				 * later.
    570   1.63        ad 				 */
    571   1.63        ad 				nfound++;
    572   1.52        ad 				continue;
    573   1.52        ad 			}
    574   1.52        ad 			nfound++;
    575    1.2   thorpej 
    576   1.52        ad 			/* No need to lock the LWP in order to see LSZOMB. */
    577   1.52        ad 			if (l2->l_stat != LSZOMB)
    578   1.52        ad 				continue;
    579    1.2   thorpej 
    580   1.63        ad 			/*
    581   1.63        ad 			 * We're no longer waiting.  Reset the "first waiter"
    582   1.63        ad 			 * pointer on the target, in case it was us.
    583   1.63        ad 			 */
    584   1.63        ad 			l->l_waitingfor = 0;
    585   1.63        ad 			l2->l_waiter = 0;
    586   1.63        ad 			p->p_nlwpwait--;
    587    1.2   thorpej 			if (departed)
    588    1.2   thorpej 				*departed = l2->l_lid;
    589   1.75        ad 			sched_lwp_collect(l2);
    590   1.63        ad 
    591   1.63        ad 			/* lwp_free() releases the proc lock. */
    592   1.63        ad 			lwp_free(l2, false, false);
    593  1.103        ad 			mutex_enter(p->p_lock);
    594   1.52        ad 			return 0;
    595   1.52        ad 		}
    596    1.2   thorpej 
    597   1.63        ad 		if (error != 0)
    598   1.63        ad 			break;
    599   1.52        ad 		if (nfound == 0) {
    600   1.52        ad 			error = ESRCH;
    601   1.52        ad 			break;
    602   1.52        ad 		}
    603   1.63        ad 
    604   1.63        ad 		/*
    605   1.63        ad 		 * The kernel is careful to ensure that it can not deadlock
    606   1.63        ad 		 * when exiting - just keep waiting.
    607   1.63        ad 		 */
    608   1.63        ad 		if (exiting) {
    609   1.52        ad 			KASSERT(p->p_nlwps > 1);
    610  1.103        ad 			cv_wait(&p->p_lwpcv, p->p_lock);
    611   1.52        ad 			continue;
    612   1.52        ad 		}
    613   1.63        ad 
    614   1.63        ad 		/*
    615   1.63        ad 		 * If all other LWPs are waiting for exits or suspends
    616   1.63        ad 		 * and the supply of zombies and potential zombies is
    617   1.63        ad 		 * exhausted, then we are about to deadlock.
    618   1.63        ad 		 *
    619   1.63        ad 		 * If the process is exiting (and this LWP is not the one
    620   1.63        ad 		 * that is coordinating the exit) then bail out now.
    621   1.63        ad 		 */
    622   1.52        ad 		if ((p->p_sflag & PS_WEXIT) != 0 ||
    623   1.63        ad 		    p->p_nrlwps + p->p_nzlwps - p->p_ndlwps <= p->p_nlwpwait) {
    624   1.52        ad 			error = EDEADLK;
    625   1.52        ad 			break;
    626    1.2   thorpej 		}
    627   1.63        ad 
    628   1.63        ad 		/*
    629   1.63        ad 		 * Sit around and wait for something to happen.  We'll be
    630   1.63        ad 		 * awoken if any of the conditions examined change: if an
    631   1.63        ad 		 * LWP exits, is collected, or is detached.
    632   1.63        ad 		 */
    633  1.103        ad 		if ((error = cv_wait_sig(&p->p_lwpcv, p->p_lock)) != 0)
    634   1.52        ad 			break;
    635    1.2   thorpej 	}
    636    1.2   thorpej 
    637   1.63        ad 	/*
    638   1.63        ad 	 * We didn't find any LWPs to collect, we may have received a
    639   1.63        ad 	 * signal, or some other condition has caused us to bail out.
    640   1.63        ad 	 *
    641   1.63        ad 	 * If waiting on a specific LWP, clear the waiters marker: some
    642   1.63        ad 	 * other LWP may want it.  Then, kick all the remaining waiters
    643   1.63        ad 	 * so that they can re-check for zombies and for deadlock.
    644   1.63        ad 	 */
    645   1.63        ad 	if (lid != 0) {
    646   1.63        ad 		LIST_FOREACH(l2, &p->p_lwps, l_sibling) {
    647   1.63        ad 			if (l2->l_lid == lid) {
    648   1.63        ad 				if (l2->l_waiter == curlid)
    649   1.63        ad 					l2->l_waiter = 0;
    650   1.63        ad 				break;
    651   1.63        ad 			}
    652   1.63        ad 		}
    653   1.63        ad 	}
    654   1.52        ad 	p->p_nlwpwait--;
    655   1.63        ad 	l->l_waitingfor = 0;
    656   1.63        ad 	cv_broadcast(&p->p_lwpcv);
    657   1.63        ad 
    658   1.52        ad 	return error;
    659    1.2   thorpej }
    660    1.2   thorpej 
    661   1.52        ad /*
    662   1.52        ad  * Create a new LWP within process 'p2', using LWP 'l1' as a template.
    663   1.52        ad  * The new LWP is created in state LSIDL and must be set running,
    664   1.52        ad  * suspended, or stopped by the caller.
    665   1.52        ad  */
    666    1.2   thorpej int
    667  1.134     rmind lwp_create(lwp_t *l1, proc_t *p2, vaddr_t uaddr, int flags,
    668   1.75        ad 	   void *stack, size_t stacksize, void (*func)(void *), void *arg,
    669   1.75        ad 	   lwp_t **rnewlwpp, int sclass)
    670    1.2   thorpej {
    671   1.52        ad 	struct lwp *l2, *isfree;
    672   1.52        ad 	turnstile_t *ts;
    673  1.151       chs 	lwpid_t lid;
    674    1.2   thorpej 
    675  1.107        ad 	KASSERT(l1 == curlwp || l1->l_proc == &proc0);
    676  1.107        ad 
    677   1.52        ad 	/*
    678   1.52        ad 	 * First off, reap any detached LWP waiting to be collected.
    679   1.52        ad 	 * We can re-use its LWP structure and turnstile.
    680   1.52        ad 	 */
    681   1.52        ad 	isfree = NULL;
    682   1.52        ad 	if (p2->p_zomblwp != NULL) {
    683  1.103        ad 		mutex_enter(p2->p_lock);
    684   1.52        ad 		if ((isfree = p2->p_zomblwp) != NULL) {
    685   1.52        ad 			p2->p_zomblwp = NULL;
    686   1.63        ad 			lwp_free(isfree, true, false);/* releases proc mutex */
    687   1.52        ad 		} else
    688  1.103        ad 			mutex_exit(p2->p_lock);
    689   1.52        ad 	}
    690   1.52        ad 	if (isfree == NULL) {
    691   1.87        ad 		l2 = pool_cache_get(lwp_cache, PR_WAITOK);
    692   1.52        ad 		memset(l2, 0, sizeof(*l2));
    693   1.76        ad 		l2->l_ts = pool_cache_get(turnstile_cache, PR_WAITOK);
    694   1.60      yamt 		SLIST_INIT(&l2->l_pi_lenders);
    695   1.52        ad 	} else {
    696   1.52        ad 		l2 = isfree;
    697   1.52        ad 		ts = l2->l_ts;
    698   1.75        ad 		KASSERT(l2->l_inheritedprio == -1);
    699   1.60      yamt 		KASSERT(SLIST_EMPTY(&l2->l_pi_lenders));
    700   1.52        ad 		memset(l2, 0, sizeof(*l2));
    701   1.52        ad 		l2->l_ts = ts;
    702   1.52        ad 	}
    703    1.2   thorpej 
    704    1.2   thorpej 	l2->l_stat = LSIDL;
    705    1.2   thorpej 	l2->l_proc = p2;
    706   1.52        ad 	l2->l_refcnt = 1;
    707   1.75        ad 	l2->l_class = sclass;
    708  1.116        ad 
    709  1.116        ad 	/*
    710  1.116        ad 	 * If vfork(), we want the LWP to run fast and on the same CPU
    711  1.116        ad 	 * as its parent, so that it can reuse the VM context and cache
    712  1.116        ad 	 * footprint on the local CPU.
    713  1.116        ad 	 */
    714  1.116        ad 	l2->l_kpriority = ((flags & LWP_VFORK) ? true : false);
    715   1.82        ad 	l2->l_kpribase = PRI_KERNEL;
    716   1.52        ad 	l2->l_priority = l1->l_priority;
    717   1.75        ad 	l2->l_inheritedprio = -1;
    718  1.134     rmind 	l2->l_flag = 0;
    719   1.88        ad 	l2->l_pflag = LP_MPSAFE;
    720  1.131        ad 	TAILQ_INIT(&l2->l_ld_locks);
    721  1.131        ad 
    722  1.131        ad 	/*
    723  1.156     pooka 	 * For vfork, borrow parent's lwpctl context if it exists.
    724  1.156     pooka 	 * This also causes us to return via lwp_userret.
    725  1.156     pooka 	 */
    726  1.156     pooka 	if (flags & LWP_VFORK && l1->l_lwpctl) {
    727  1.156     pooka 		l2->l_lwpctl = l1->l_lwpctl;
    728  1.156     pooka 		l2->l_flag |= LW_LWPCTL;
    729  1.156     pooka 	}
    730  1.156     pooka 
    731  1.156     pooka 	/*
    732  1.131        ad 	 * If not the first LWP in the process, grab a reference to the
    733  1.131        ad 	 * descriptor table.
    734  1.131        ad 	 */
    735   1.97        ad 	l2->l_fd = p2->p_fd;
    736  1.131        ad 	if (p2->p_nlwps != 0) {
    737  1.131        ad 		KASSERT(l1->l_proc == p2);
    738  1.136     rmind 		fd_hold(l2);
    739  1.131        ad 	} else {
    740  1.131        ad 		KASSERT(l1->l_proc != p2);
    741  1.131        ad 	}
    742   1.41   thorpej 
    743   1.56     pavel 	if (p2->p_flag & PK_SYSTEM) {
    744  1.134     rmind 		/* Mark it as a system LWP. */
    745   1.56     pavel 		l2->l_flag |= LW_SYSTEM;
    746   1.52        ad 	}
    747    1.2   thorpej 
    748  1.107        ad 	kpreempt_disable();
    749  1.107        ad 	l2->l_mutex = l1->l_cpu->ci_schedstate.spc_mutex;
    750  1.107        ad 	l2->l_cpu = l1->l_cpu;
    751  1.107        ad 	kpreempt_enable();
    752  1.107        ad 
    753  1.138    darran 	kdtrace_thread_ctor(NULL, l2);
    754   1.73     rmind 	lwp_initspecific(l2);
    755   1.75        ad 	sched_lwp_fork(l1, l2);
    756   1.37        ad 	lwp_update_creds(l2);
    757   1.70        ad 	callout_init(&l2->l_timeout_ch, CALLOUT_MPSAFE);
    758   1.70        ad 	callout_setfunc(&l2->l_timeout_ch, sleepq_timeout, l2);
    759   1.52        ad 	cv_init(&l2->l_sigcv, "sigwait");
    760   1.52        ad 	l2->l_syncobj = &sched_syncobj;
    761    1.2   thorpej 
    762    1.2   thorpej 	if (rnewlwpp != NULL)
    763    1.2   thorpej 		*rnewlwpp = l2;
    764    1.2   thorpej 
    765  1.137     rmind 	uvm_lwp_setuarea(l2, uaddr);
    766    1.2   thorpej 	uvm_lwp_fork(l1, l2, stack, stacksize, func,
    767    1.2   thorpej 	    (arg != NULL) ? arg : l2);
    768    1.2   thorpej 
    769  1.151       chs 	if ((flags & LWP_PIDLID) != 0) {
    770  1.151       chs 		lid = proc_alloc_pid(p2);
    771  1.151       chs 		l2->l_pflag |= LP_PIDLID;
    772  1.151       chs 	} else {
    773  1.151       chs 		lid = 0;
    774  1.151       chs 	}
    775  1.151       chs 
    776  1.103        ad 	mutex_enter(p2->p_lock);
    777   1.52        ad 
    778   1.52        ad 	if ((flags & LWP_DETACHED) != 0) {
    779   1.52        ad 		l2->l_prflag = LPR_DETACHED;
    780   1.52        ad 		p2->p_ndlwps++;
    781   1.52        ad 	} else
    782   1.52        ad 		l2->l_prflag = 0;
    783   1.52        ad 
    784   1.52        ad 	l2->l_sigmask = l1->l_sigmask;
    785   1.52        ad 	CIRCLEQ_INIT(&l2->l_sigpend.sp_info);
    786   1.52        ad 	sigemptyset(&l2->l_sigpend.sp_set);
    787   1.52        ad 
    788  1.151       chs 	if (lid == 0) {
    789   1.53      yamt 		p2->p_nlwpid++;
    790  1.151       chs 		if (p2->p_nlwpid == 0)
    791  1.151       chs 			p2->p_nlwpid++;
    792  1.151       chs 		lid = p2->p_nlwpid;
    793  1.151       chs 	}
    794  1.151       chs 	l2->l_lid = lid;
    795    1.2   thorpej 	LIST_INSERT_HEAD(&p2->p_lwps, l2, l_sibling);
    796    1.2   thorpej 	p2->p_nlwps++;
    797  1.149      yamt 	p2->p_nrlwps++;
    798    1.2   thorpej 
    799   1.91     rmind 	if ((p2->p_flag & PK_SYSTEM) == 0) {
    800   1.91     rmind 		/* Inherit an affinity */
    801  1.122     rmind 		if (l1->l_flag & LW_AFFINITY) {
    802  1.128     rmind 			/*
    803  1.128     rmind 			 * Note that we hold the state lock while inheriting
    804  1.128     rmind 			 * the affinity to avoid race with sched_setaffinity().
    805  1.128     rmind 			 */
    806  1.128     rmind 			lwp_lock(l1);
    807  1.122     rmind 			if (l1->l_flag & LW_AFFINITY) {
    808  1.122     rmind 				kcpuset_use(l1->l_affinity);
    809  1.122     rmind 				l2->l_affinity = l1->l_affinity;
    810  1.122     rmind 				l2->l_flag |= LW_AFFINITY;
    811  1.122     rmind 			}
    812  1.128     rmind 			lwp_unlock(l1);
    813  1.117  christos 		}
    814  1.128     rmind 		lwp_lock(l2);
    815  1.128     rmind 		/* Inherit a processor-set */
    816  1.128     rmind 		l2->l_psid = l1->l_psid;
    817   1.91     rmind 		/* Look for a CPU to start */
    818   1.91     rmind 		l2->l_cpu = sched_takecpu(l2);
    819   1.91     rmind 		lwp_unlock_to(l2, l2->l_cpu->ci_schedstate.spc_mutex);
    820   1.91     rmind 	}
    821  1.128     rmind 	mutex_exit(p2->p_lock);
    822  1.128     rmind 
    823  1.141    darran 	SDT_PROBE(proc,,,lwp_create, l2, 0,0,0,0);
    824  1.141    darran 
    825  1.128     rmind 	mutex_enter(proc_lock);
    826  1.128     rmind 	LIST_INSERT_HEAD(&alllwp, l2, l_list);
    827  1.128     rmind 	mutex_exit(proc_lock);
    828   1.91     rmind 
    829   1.57       dsl 	SYSCALL_TIME_LWP_INIT(l2);
    830   1.57       dsl 
    831   1.16      manu 	if (p2->p_emul->e_lwp_fork)
    832   1.16      manu 		(*p2->p_emul->e_lwp_fork)(l1, l2);
    833   1.16      manu 
    834    1.2   thorpej 	return (0);
    835    1.2   thorpej }
    836    1.2   thorpej 
    837    1.2   thorpej /*
    838   1.64      yamt  * Called by MD code when a new LWP begins execution.  Must be called
    839   1.64      yamt  * with the previous LWP locked (so at splsched), or if there is no
    840   1.64      yamt  * previous LWP, at splsched.
    841   1.64      yamt  */
    842   1.64      yamt void
    843   1.64      yamt lwp_startup(struct lwp *prev, struct lwp *new)
    844   1.64      yamt {
    845   1.64      yamt 
    846  1.141    darran 	SDT_PROBE(proc,,,lwp_start, new, 0,0,0,0);
    847  1.141    darran 
    848  1.107        ad 	KASSERT(kpreempt_disabled());
    849   1.64      yamt 	if (prev != NULL) {
    850   1.81        ad 		/*
    851   1.81        ad 		 * Normalize the count of the spin-mutexes, it was
    852   1.81        ad 		 * increased in mi_switch().  Unmark the state of
    853   1.81        ad 		 * context switch - it is finished for previous LWP.
    854   1.81        ad 		 */
    855   1.81        ad 		curcpu()->ci_mtx_count++;
    856   1.81        ad 		membar_exit();
    857   1.81        ad 		prev->l_ctxswtch = 0;
    858   1.64      yamt 	}
    859  1.107        ad 	KPREEMPT_DISABLE(new);
    860  1.107        ad 	spl0();
    861  1.105        ad 	pmap_activate(new);
    862   1.64      yamt 	LOCKDEBUG_BARRIER(NULL, 0);
    863  1.107        ad 	KPREEMPT_ENABLE(new);
    864   1.65        ad 	if ((new->l_pflag & LP_MPSAFE) == 0) {
    865   1.65        ad 		KERNEL_LOCK(1, new);
    866   1.65        ad 	}
    867   1.64      yamt }
    868   1.64      yamt 
    869   1.64      yamt /*
    870   1.65        ad  * Exit an LWP.
    871    1.2   thorpej  */
    872    1.2   thorpej void
    873    1.2   thorpej lwp_exit(struct lwp *l)
    874    1.2   thorpej {
    875    1.2   thorpej 	struct proc *p = l->l_proc;
    876   1.52        ad 	struct lwp *l2;
    877   1.65        ad 	bool current;
    878   1.65        ad 
    879   1.65        ad 	current = (l == curlwp);
    880    1.2   thorpej 
    881  1.114     rmind 	KASSERT(current || (l->l_stat == LSIDL && l->l_target_cpu == NULL));
    882  1.131        ad 	KASSERT(p == curproc);
    883    1.2   thorpej 
    884  1.141    darran 	SDT_PROBE(proc,,,lwp_exit, l, 0,0,0,0);
    885  1.141    darran 
    886   1.52        ad 	/*
    887   1.52        ad 	 * Verify that we hold no locks other than the kernel lock.
    888   1.52        ad 	 */
    889   1.52        ad 	LOCKDEBUG_BARRIER(&kernel_lock, 0);
    890   1.16      manu 
    891    1.2   thorpej 	/*
    892   1.52        ad 	 * If we are the last live LWP in a process, we need to exit the
    893   1.52        ad 	 * entire process.  We do so with an exit status of zero, because
    894   1.52        ad 	 * it's a "controlled" exit, and because that's what Solaris does.
    895   1.52        ad 	 *
    896   1.52        ad 	 * We are not quite a zombie yet, but for accounting purposes we
    897   1.52        ad 	 * must increment the count of zombies here.
    898   1.45   thorpej 	 *
    899   1.45   thorpej 	 * Note: the last LWP's specificdata will be deleted here.
    900    1.2   thorpej 	 */
    901  1.103        ad 	mutex_enter(p->p_lock);
    902   1.52        ad 	if (p->p_nlwps - p->p_nzlwps == 1) {
    903   1.65        ad 		KASSERT(current == true);
    904   1.88        ad 		/* XXXSMP kernel_lock not held */
    905    1.2   thorpej 		exit1(l, 0);
    906   1.19  jdolecek 		/* NOTREACHED */
    907    1.2   thorpej 	}
    908   1.52        ad 	p->p_nzlwps++;
    909  1.103        ad 	mutex_exit(p->p_lock);
    910   1.52        ad 
    911   1.52        ad 	if (p->p_emul->e_lwp_exit)
    912   1.52        ad 		(*p->p_emul->e_lwp_exit)(l);
    913    1.2   thorpej 
    914  1.131        ad 	/* Drop filedesc reference. */
    915  1.131        ad 	fd_free();
    916  1.131        ad 
    917   1.45   thorpej 	/* Delete the specificdata while it's still safe to sleep. */
    918  1.145     pooka 	lwp_finispecific(l);
    919   1.45   thorpej 
    920   1.52        ad 	/*
    921   1.52        ad 	 * Release our cached credentials.
    922   1.52        ad 	 */
    923   1.37        ad 	kauth_cred_free(l->l_cred);
    924   1.70        ad 	callout_destroy(&l->l_timeout_ch);
    925   1.65        ad 
    926   1.65        ad 	/*
    927   1.52        ad 	 * Remove the LWP from the global list.
    928  1.151       chs 	 * Free its LID from the PID namespace if needed.
    929   1.52        ad 	 */
    930  1.102        ad 	mutex_enter(proc_lock);
    931   1.52        ad 	LIST_REMOVE(l, l_list);
    932  1.151       chs 	if ((l->l_pflag & LP_PIDLID) != 0 && l->l_lid != p->p_pid) {
    933  1.151       chs 		proc_free_pid(l->l_lid);
    934  1.151       chs 	}
    935  1.102        ad 	mutex_exit(proc_lock);
    936   1.19  jdolecek 
    937   1.52        ad 	/*
    938   1.52        ad 	 * Get rid of all references to the LWP that others (e.g. procfs)
    939   1.52        ad 	 * may have, and mark the LWP as a zombie.  If the LWP is detached,
    940   1.52        ad 	 * mark it waiting for collection in the proc structure.  Note that
    941   1.52        ad 	 * before we can do that, we need to free any other dead, deatched
    942   1.52        ad 	 * LWP waiting to meet its maker.
    943   1.52        ad 	 */
    944  1.103        ad 	mutex_enter(p->p_lock);
    945   1.52        ad 	lwp_drainrefs(l);
    946   1.31      yamt 
    947   1.52        ad 	if ((l->l_prflag & LPR_DETACHED) != 0) {
    948   1.52        ad 		while ((l2 = p->p_zomblwp) != NULL) {
    949   1.52        ad 			p->p_zomblwp = NULL;
    950   1.63        ad 			lwp_free(l2, false, false);/* releases proc mutex */
    951  1.103        ad 			mutex_enter(p->p_lock);
    952   1.72        ad 			l->l_refcnt++;
    953   1.72        ad 			lwp_drainrefs(l);
    954   1.52        ad 		}
    955   1.52        ad 		p->p_zomblwp = l;
    956   1.52        ad 	}
    957   1.31      yamt 
    958   1.52        ad 	/*
    959   1.52        ad 	 * If we find a pending signal for the process and we have been
    960  1.151       chs 	 * asked to check for signals, then we lose: arrange to have
    961   1.52        ad 	 * all other LWPs in the process check for signals.
    962   1.52        ad 	 */
    963   1.56     pavel 	if ((l->l_flag & LW_PENDSIG) != 0 &&
    964   1.52        ad 	    firstsig(&p->p_sigpend.sp_set) != 0) {
    965   1.52        ad 		LIST_FOREACH(l2, &p->p_lwps, l_sibling) {
    966   1.52        ad 			lwp_lock(l2);
    967   1.56     pavel 			l2->l_flag |= LW_PENDSIG;
    968   1.52        ad 			lwp_unlock(l2);
    969   1.52        ad 		}
    970   1.31      yamt 	}
    971   1.31      yamt 
    972   1.52        ad 	lwp_lock(l);
    973   1.52        ad 	l->l_stat = LSZOMB;
    974   1.90        ad 	if (l->l_name != NULL)
    975   1.90        ad 		strcpy(l->l_name, "(zombie)");
    976  1.128     rmind 	if (l->l_flag & LW_AFFINITY) {
    977  1.122     rmind 		l->l_flag &= ~LW_AFFINITY;
    978  1.128     rmind 	} else {
    979  1.128     rmind 		KASSERT(l->l_affinity == NULL);
    980  1.128     rmind 	}
    981   1.52        ad 	lwp_unlock(l);
    982    1.2   thorpej 	p->p_nrlwps--;
    983   1.52        ad 	cv_broadcast(&p->p_lwpcv);
    984   1.78        ad 	if (l->l_lwpctl != NULL)
    985   1.78        ad 		l->l_lwpctl->lc_curcpu = LWPCTL_CPU_EXITED;
    986  1.103        ad 	mutex_exit(p->p_lock);
    987   1.52        ad 
    988  1.122     rmind 	/* Safe without lock since LWP is in zombie state */
    989  1.122     rmind 	if (l->l_affinity) {
    990  1.122     rmind 		kcpuset_unuse(l->l_affinity, NULL);
    991  1.122     rmind 		l->l_affinity = NULL;
    992  1.122     rmind 	}
    993  1.122     rmind 
    994   1.52        ad 	/*
    995   1.52        ad 	 * We can no longer block.  At this point, lwp_free() may already
    996   1.52        ad 	 * be gunning for us.  On a multi-CPU system, we may be off p_lwps.
    997   1.52        ad 	 *
    998   1.52        ad 	 * Free MD LWP resources.
    999   1.52        ad 	 */
   1000   1.52        ad 	cpu_lwp_free(l, 0);
   1001    1.2   thorpej 
   1002   1.65        ad 	if (current) {
   1003   1.65        ad 		pmap_deactivate(l);
   1004   1.65        ad 
   1005   1.65        ad 		/*
   1006   1.65        ad 		 * Release the kernel lock, and switch away into
   1007   1.65        ad 		 * oblivion.
   1008   1.65        ad 		 */
   1009   1.52        ad #ifdef notyet
   1010   1.65        ad 		/* XXXSMP hold in lwp_userret() */
   1011   1.65        ad 		KERNEL_UNLOCK_LAST(l);
   1012   1.52        ad #else
   1013   1.65        ad 		KERNEL_UNLOCK_ALL(l, NULL);
   1014   1.52        ad #endif
   1015   1.65        ad 		lwp_exit_switchaway(l);
   1016   1.65        ad 	}
   1017    1.2   thorpej }
   1018    1.2   thorpej 
   1019   1.52        ad /*
   1020   1.52        ad  * Free a dead LWP's remaining resources.
   1021   1.52        ad  *
   1022   1.52        ad  * XXXLWP limits.
   1023   1.52        ad  */
   1024   1.52        ad void
   1025   1.63        ad lwp_free(struct lwp *l, bool recycle, bool last)
   1026   1.52        ad {
   1027   1.52        ad 	struct proc *p = l->l_proc;
   1028  1.100        ad 	struct rusage *ru;
   1029   1.52        ad 	ksiginfoq_t kq;
   1030   1.52        ad 
   1031   1.92      yamt 	KASSERT(l != curlwp);
   1032   1.92      yamt 
   1033   1.52        ad 	/*
   1034   1.52        ad 	 * If this was not the last LWP in the process, then adjust
   1035   1.52        ad 	 * counters and unlock.
   1036   1.52        ad 	 */
   1037   1.52        ad 	if (!last) {
   1038   1.52        ad 		/*
   1039   1.52        ad 		 * Add the LWP's run time to the process' base value.
   1040   1.52        ad 		 * This needs to co-incide with coming off p_lwps.
   1041   1.52        ad 		 */
   1042   1.86      yamt 		bintime_add(&p->p_rtime, &l->l_rtime);
   1043   1.64      yamt 		p->p_pctcpu += l->l_pctcpu;
   1044  1.100        ad 		ru = &p->p_stats->p_ru;
   1045  1.100        ad 		ruadd(ru, &l->l_ru);
   1046  1.100        ad 		ru->ru_nvcsw += (l->l_ncsw - l->l_nivcsw);
   1047  1.100        ad 		ru->ru_nivcsw += l->l_nivcsw;
   1048   1.52        ad 		LIST_REMOVE(l, l_sibling);
   1049   1.52        ad 		p->p_nlwps--;
   1050   1.52        ad 		p->p_nzlwps--;
   1051   1.52        ad 		if ((l->l_prflag & LPR_DETACHED) != 0)
   1052   1.52        ad 			p->p_ndlwps--;
   1053   1.63        ad 
   1054   1.63        ad 		/*
   1055   1.63        ad 		 * Have any LWPs sleeping in lwp_wait() recheck for
   1056   1.63        ad 		 * deadlock.
   1057   1.63        ad 		 */
   1058   1.63        ad 		cv_broadcast(&p->p_lwpcv);
   1059  1.103        ad 		mutex_exit(p->p_lock);
   1060   1.63        ad 	}
   1061   1.52        ad 
   1062   1.52        ad #ifdef MULTIPROCESSOR
   1063   1.63        ad 	/*
   1064   1.63        ad 	 * In the unlikely event that the LWP is still on the CPU,
   1065   1.63        ad 	 * then spin until it has switched away.  We need to release
   1066   1.63        ad 	 * all locks to avoid deadlock against interrupt handlers on
   1067   1.63        ad 	 * the target CPU.
   1068   1.63        ad 	 */
   1069  1.115        ad 	if ((l->l_pflag & LP_RUNNING) != 0 || l->l_cpu->ci_curlwp == l) {
   1070   1.63        ad 		int count;
   1071   1.64      yamt 		(void)count; /* XXXgcc */
   1072   1.63        ad 		KERNEL_UNLOCK_ALL(curlwp, &count);
   1073  1.115        ad 		while ((l->l_pflag & LP_RUNNING) != 0 ||
   1074   1.64      yamt 		    l->l_cpu->ci_curlwp == l)
   1075   1.63        ad 			SPINLOCK_BACKOFF_HOOK;
   1076   1.63        ad 		KERNEL_LOCK(count, curlwp);
   1077   1.63        ad 	}
   1078   1.52        ad #endif
   1079   1.52        ad 
   1080   1.52        ad 	/*
   1081   1.52        ad 	 * Destroy the LWP's remaining signal information.
   1082   1.52        ad 	 */
   1083   1.52        ad 	ksiginfo_queue_init(&kq);
   1084   1.52        ad 	sigclear(&l->l_sigpend, NULL, &kq);
   1085   1.52        ad 	ksiginfo_queue_drain(&kq);
   1086   1.52        ad 	cv_destroy(&l->l_sigcv);
   1087    1.2   thorpej 
   1088   1.19  jdolecek 	/*
   1089   1.52        ad 	 * Free the LWP's turnstile and the LWP structure itself unless the
   1090   1.93      yamt 	 * caller wants to recycle them.  Also, free the scheduler specific
   1091   1.93      yamt 	 * data.
   1092   1.52        ad 	 *
   1093   1.52        ad 	 * We can't return turnstile0 to the pool (it didn't come from it),
   1094   1.52        ad 	 * so if it comes up just drop it quietly and move on.
   1095   1.52        ad 	 *
   1096   1.52        ad 	 * We don't recycle the VM resources at this time.
   1097   1.19  jdolecek 	 */
   1098   1.78        ad 	if (l->l_lwpctl != NULL)
   1099   1.78        ad 		lwp_ctl_free(l);
   1100   1.64      yamt 
   1101   1.52        ad 	if (!recycle && l->l_ts != &turnstile0)
   1102   1.76        ad 		pool_cache_put(turnstile_cache, l->l_ts);
   1103   1.90        ad 	if (l->l_name != NULL)
   1104   1.90        ad 		kmem_free(l->l_name, MAXCOMLEN);
   1105  1.135     rmind 
   1106   1.52        ad 	cpu_lwp_free2(l);
   1107   1.19  jdolecek 	uvm_lwp_exit(l);
   1108  1.134     rmind 
   1109   1.60      yamt 	KASSERT(SLIST_EMPTY(&l->l_pi_lenders));
   1110   1.75        ad 	KASSERT(l->l_inheritedprio == -1);
   1111  1.155      matt 	KASSERT(l->l_blcnt == 0);
   1112  1.138    darran 	kdtrace_thread_dtor(NULL, l);
   1113   1.52        ad 	if (!recycle)
   1114   1.87        ad 		pool_cache_put(lwp_cache, l);
   1115    1.2   thorpej }
   1116    1.2   thorpej 
   1117    1.2   thorpej /*
   1118   1.91     rmind  * Migrate the LWP to the another CPU.  Unlocks the LWP.
   1119   1.91     rmind  */
   1120   1.91     rmind void
   1121  1.114     rmind lwp_migrate(lwp_t *l, struct cpu_info *tci)
   1122   1.91     rmind {
   1123  1.114     rmind 	struct schedstate_percpu *tspc;
   1124  1.121     rmind 	int lstat = l->l_stat;
   1125  1.121     rmind 
   1126   1.91     rmind 	KASSERT(lwp_locked(l, NULL));
   1127  1.114     rmind 	KASSERT(tci != NULL);
   1128  1.114     rmind 
   1129  1.121     rmind 	/* If LWP is still on the CPU, it must be handled like LSONPROC */
   1130  1.121     rmind 	if ((l->l_pflag & LP_RUNNING) != 0) {
   1131  1.121     rmind 		lstat = LSONPROC;
   1132  1.121     rmind 	}
   1133  1.121     rmind 
   1134  1.114     rmind 	/*
   1135  1.114     rmind 	 * The destination CPU could be changed while previous migration
   1136  1.114     rmind 	 * was not finished.
   1137  1.114     rmind 	 */
   1138  1.121     rmind 	if (l->l_target_cpu != NULL) {
   1139  1.114     rmind 		l->l_target_cpu = tci;
   1140  1.114     rmind 		lwp_unlock(l);
   1141  1.114     rmind 		return;
   1142  1.114     rmind 	}
   1143   1.91     rmind 
   1144  1.114     rmind 	/* Nothing to do if trying to migrate to the same CPU */
   1145  1.114     rmind 	if (l->l_cpu == tci) {
   1146   1.91     rmind 		lwp_unlock(l);
   1147   1.91     rmind 		return;
   1148   1.91     rmind 	}
   1149   1.91     rmind 
   1150  1.114     rmind 	KASSERT(l->l_target_cpu == NULL);
   1151  1.114     rmind 	tspc = &tci->ci_schedstate;
   1152  1.121     rmind 	switch (lstat) {
   1153   1.91     rmind 	case LSRUN:
   1154  1.134     rmind 		l->l_target_cpu = tci;
   1155  1.134     rmind 		break;
   1156   1.91     rmind 	case LSIDL:
   1157  1.114     rmind 		l->l_cpu = tci;
   1158  1.114     rmind 		lwp_unlock_to(l, tspc->spc_mutex);
   1159   1.91     rmind 		return;
   1160   1.91     rmind 	case LSSLEEP:
   1161  1.114     rmind 		l->l_cpu = tci;
   1162   1.91     rmind 		break;
   1163   1.91     rmind 	case LSSTOP:
   1164   1.91     rmind 	case LSSUSPENDED:
   1165  1.114     rmind 		l->l_cpu = tci;
   1166  1.114     rmind 		if (l->l_wchan == NULL) {
   1167  1.114     rmind 			lwp_unlock_to(l, tspc->spc_lwplock);
   1168  1.114     rmind 			return;
   1169   1.91     rmind 		}
   1170  1.114     rmind 		break;
   1171   1.91     rmind 	case LSONPROC:
   1172  1.114     rmind 		l->l_target_cpu = tci;
   1173  1.114     rmind 		spc_lock(l->l_cpu);
   1174  1.114     rmind 		cpu_need_resched(l->l_cpu, RESCHED_KPREEMPT);
   1175  1.114     rmind 		spc_unlock(l->l_cpu);
   1176   1.91     rmind 		break;
   1177   1.91     rmind 	}
   1178   1.91     rmind 	lwp_unlock(l);
   1179   1.91     rmind }
   1180   1.91     rmind 
   1181   1.91     rmind /*
   1182   1.94     rmind  * Find the LWP in the process.  Arguments may be zero, in such case,
   1183   1.94     rmind  * the calling process and first LWP in the list will be used.
   1184  1.103        ad  * On success - returns proc locked.
   1185   1.91     rmind  */
   1186   1.91     rmind struct lwp *
   1187   1.91     rmind lwp_find2(pid_t pid, lwpid_t lid)
   1188   1.91     rmind {
   1189   1.91     rmind 	proc_t *p;
   1190   1.91     rmind 	lwp_t *l;
   1191   1.91     rmind 
   1192  1.150     rmind 	/* Find the process. */
   1193   1.94     rmind 	if (pid != 0) {
   1194  1.150     rmind 		mutex_enter(proc_lock);
   1195  1.150     rmind 		p = proc_find(pid);
   1196  1.150     rmind 		if (p == NULL) {
   1197  1.150     rmind 			mutex_exit(proc_lock);
   1198  1.150     rmind 			return NULL;
   1199  1.150     rmind 		}
   1200  1.150     rmind 		mutex_enter(p->p_lock);
   1201  1.102        ad 		mutex_exit(proc_lock);
   1202  1.150     rmind 	} else {
   1203  1.150     rmind 		p = curlwp->l_proc;
   1204  1.150     rmind 		mutex_enter(p->p_lock);
   1205  1.150     rmind 	}
   1206  1.150     rmind 	/* Find the thread. */
   1207  1.150     rmind 	if (lid != 0) {
   1208  1.150     rmind 		l = lwp_find(p, lid);
   1209  1.150     rmind 	} else {
   1210  1.150     rmind 		l = LIST_FIRST(&p->p_lwps);
   1211   1.94     rmind 	}
   1212  1.103        ad 	if (l == NULL) {
   1213  1.103        ad 		mutex_exit(p->p_lock);
   1214  1.103        ad 	}
   1215   1.91     rmind 	return l;
   1216   1.91     rmind }
   1217   1.91     rmind 
   1218   1.91     rmind /*
   1219  1.151       chs  * Look up a live LWP within the specified process, and return it locked.
   1220   1.52        ad  *
   1221  1.103        ad  * Must be called with p->p_lock held.
   1222   1.52        ad  */
   1223   1.52        ad struct lwp *
   1224  1.151       chs lwp_find(struct proc *p, lwpid_t id)
   1225   1.52        ad {
   1226   1.52        ad 	struct lwp *l;
   1227   1.52        ad 
   1228  1.103        ad 	KASSERT(mutex_owned(p->p_lock));
   1229   1.52        ad 
   1230   1.52        ad 	LIST_FOREACH(l, &p->p_lwps, l_sibling) {
   1231   1.52        ad 		if (l->l_lid == id)
   1232   1.52        ad 			break;
   1233   1.52        ad 	}
   1234   1.52        ad 
   1235   1.52        ad 	/*
   1236   1.52        ad 	 * No need to lock - all of these conditions will
   1237   1.52        ad 	 * be visible with the process level mutex held.
   1238   1.52        ad 	 */
   1239   1.52        ad 	if (l != NULL && (l->l_stat == LSIDL || l->l_stat == LSZOMB))
   1240   1.52        ad 		l = NULL;
   1241   1.52        ad 
   1242   1.52        ad 	return l;
   1243   1.52        ad }
   1244   1.52        ad 
   1245   1.52        ad /*
   1246   1.37        ad  * Update an LWP's cached credentials to mirror the process' master copy.
   1247   1.37        ad  *
   1248   1.37        ad  * This happens early in the syscall path, on user trap, and on LWP
   1249   1.37        ad  * creation.  A long-running LWP can also voluntarily choose to update
   1250   1.37        ad  * it's credentials by calling this routine.  This may be called from
   1251   1.37        ad  * LWP_CACHE_CREDS(), which checks l->l_cred != p->p_cred beforehand.
   1252   1.37        ad  */
   1253   1.37        ad void
   1254   1.37        ad lwp_update_creds(struct lwp *l)
   1255   1.37        ad {
   1256   1.37        ad 	kauth_cred_t oc;
   1257   1.37        ad 	struct proc *p;
   1258   1.37        ad 
   1259   1.37        ad 	p = l->l_proc;
   1260   1.37        ad 	oc = l->l_cred;
   1261   1.37        ad 
   1262  1.103        ad 	mutex_enter(p->p_lock);
   1263   1.37        ad 	kauth_cred_hold(p->p_cred);
   1264   1.37        ad 	l->l_cred = p->p_cred;
   1265   1.98        ad 	l->l_prflag &= ~LPR_CRMOD;
   1266  1.103        ad 	mutex_exit(p->p_lock);
   1267   1.88        ad 	if (oc != NULL)
   1268   1.37        ad 		kauth_cred_free(oc);
   1269   1.52        ad }
   1270   1.52        ad 
   1271   1.52        ad /*
   1272   1.52        ad  * Verify that an LWP is locked, and optionally verify that the lock matches
   1273   1.52        ad  * one we specify.
   1274   1.52        ad  */
   1275   1.52        ad int
   1276   1.52        ad lwp_locked(struct lwp *l, kmutex_t *mtx)
   1277   1.52        ad {
   1278   1.52        ad 	kmutex_t *cur = l->l_mutex;
   1279   1.52        ad 
   1280   1.52        ad 	return mutex_owned(cur) && (mtx == cur || mtx == NULL);
   1281   1.52        ad }
   1282   1.52        ad 
   1283   1.52        ad /*
   1284   1.52        ad  * Lend a new mutex to an LWP.  The old mutex must be held.
   1285   1.52        ad  */
   1286   1.52        ad void
   1287   1.52        ad lwp_setlock(struct lwp *l, kmutex_t *new)
   1288   1.52        ad {
   1289   1.52        ad 
   1290   1.63        ad 	KASSERT(mutex_owned(l->l_mutex));
   1291   1.52        ad 
   1292  1.107        ad 	membar_exit();
   1293   1.52        ad 	l->l_mutex = new;
   1294   1.52        ad }
   1295   1.52        ad 
   1296   1.52        ad /*
   1297   1.52        ad  * Lend a new mutex to an LWP, and release the old mutex.  The old mutex
   1298   1.52        ad  * must be held.
   1299   1.52        ad  */
   1300   1.52        ad void
   1301   1.52        ad lwp_unlock_to(struct lwp *l, kmutex_t *new)
   1302   1.52        ad {
   1303   1.52        ad 	kmutex_t *old;
   1304   1.52        ad 
   1305  1.152     rmind 	KASSERT(lwp_locked(l, NULL));
   1306   1.52        ad 
   1307   1.52        ad 	old = l->l_mutex;
   1308  1.107        ad 	membar_exit();
   1309   1.52        ad 	l->l_mutex = new;
   1310   1.52        ad 	mutex_spin_exit(old);
   1311   1.52        ad }
   1312   1.52        ad 
   1313   1.60      yamt int
   1314   1.60      yamt lwp_trylock(struct lwp *l)
   1315   1.60      yamt {
   1316   1.60      yamt 	kmutex_t *old;
   1317   1.60      yamt 
   1318   1.60      yamt 	for (;;) {
   1319   1.60      yamt 		if (!mutex_tryenter(old = l->l_mutex))
   1320   1.60      yamt 			return 0;
   1321   1.60      yamt 		if (__predict_true(l->l_mutex == old))
   1322   1.60      yamt 			return 1;
   1323   1.60      yamt 		mutex_spin_exit(old);
   1324   1.60      yamt 	}
   1325   1.60      yamt }
   1326   1.60      yamt 
   1327  1.134     rmind void
   1328   1.96        ad lwp_unsleep(lwp_t *l, bool cleanup)
   1329   1.96        ad {
   1330   1.96        ad 
   1331   1.96        ad 	KASSERT(mutex_owned(l->l_mutex));
   1332  1.134     rmind 	(*l->l_syncobj->sobj_unsleep)(l, cleanup);
   1333   1.96        ad }
   1334   1.96        ad 
   1335   1.52        ad /*
   1336   1.56     pavel  * Handle exceptions for mi_userret().  Called if a member of LW_USERRET is
   1337   1.52        ad  * set.
   1338   1.52        ad  */
   1339   1.52        ad void
   1340   1.52        ad lwp_userret(struct lwp *l)
   1341   1.52        ad {
   1342   1.52        ad 	struct proc *p;
   1343   1.52        ad 	int sig;
   1344   1.52        ad 
   1345  1.114     rmind 	KASSERT(l == curlwp);
   1346  1.114     rmind 	KASSERT(l->l_stat == LSONPROC);
   1347   1.52        ad 	p = l->l_proc;
   1348   1.52        ad 
   1349   1.75        ad #ifndef __HAVE_FAST_SOFTINTS
   1350   1.75        ad 	/* Run pending soft interrupts. */
   1351   1.75        ad 	if (l->l_cpu->ci_data.cpu_softints != 0)
   1352   1.75        ad 		softint_overlay();
   1353   1.75        ad #endif
   1354   1.75        ad 
   1355  1.125        ad #ifdef KERN_SA
   1356  1.125        ad 	/* Generate UNBLOCKED upcall if needed */
   1357  1.125        ad 	if (l->l_flag & LW_SA_BLOCKING) {
   1358  1.125        ad 		sa_unblock_userret(l);
   1359  1.125        ad 		/* NOTREACHED */
   1360  1.125        ad 	}
   1361  1.125        ad #endif
   1362  1.125        ad 
   1363   1.52        ad 	/*
   1364   1.52        ad 	 * It should be safe to do this read unlocked on a multiprocessor
   1365   1.52        ad 	 * system..
   1366  1.126  wrstuden 	 *
   1367  1.126  wrstuden 	 * LW_SA_UPCALL will be handled after the while() loop, so don't
   1368  1.126  wrstuden 	 * consider it now.
   1369   1.52        ad 	 */
   1370  1.126  wrstuden 	while ((l->l_flag & (LW_USERRET & ~(LW_SA_UPCALL))) != 0) {
   1371   1.52        ad 		/*
   1372   1.52        ad 		 * Process pending signals first, unless the process
   1373   1.61        ad 		 * is dumping core or exiting, where we will instead
   1374  1.101     rmind 		 * enter the LW_WSUSPEND case below.
   1375   1.52        ad 		 */
   1376   1.61        ad 		if ((l->l_flag & (LW_PENDSIG | LW_WCORE | LW_WEXIT)) ==
   1377   1.61        ad 		    LW_PENDSIG) {
   1378  1.103        ad 			mutex_enter(p->p_lock);
   1379   1.52        ad 			while ((sig = issignal(l)) != 0)
   1380   1.52        ad 				postsig(sig);
   1381  1.103        ad 			mutex_exit(p->p_lock);
   1382   1.52        ad 		}
   1383   1.52        ad 
   1384   1.52        ad 		/*
   1385   1.52        ad 		 * Core-dump or suspend pending.
   1386   1.52        ad 		 *
   1387   1.52        ad 		 * In case of core dump, suspend ourselves, so that the
   1388   1.52        ad 		 * kernel stack and therefore the userland registers saved
   1389   1.52        ad 		 * in the trapframe are around for coredump() to write them
   1390   1.52        ad 		 * out.  We issue a wakeup on p->p_lwpcv so that sigexit()
   1391   1.52        ad 		 * will write the core file out once all other LWPs are
   1392   1.52        ad 		 * suspended.
   1393   1.52        ad 		 */
   1394   1.56     pavel 		if ((l->l_flag & LW_WSUSPEND) != 0) {
   1395  1.103        ad 			mutex_enter(p->p_lock);
   1396   1.52        ad 			p->p_nrlwps--;
   1397   1.52        ad 			cv_broadcast(&p->p_lwpcv);
   1398   1.52        ad 			lwp_lock(l);
   1399   1.52        ad 			l->l_stat = LSSUSPENDED;
   1400  1.104        ad 			lwp_unlock(l);
   1401  1.103        ad 			mutex_exit(p->p_lock);
   1402  1.104        ad 			lwp_lock(l);
   1403   1.64      yamt 			mi_switch(l);
   1404   1.52        ad 		}
   1405   1.52        ad 
   1406   1.52        ad 		/* Process is exiting. */
   1407   1.56     pavel 		if ((l->l_flag & LW_WEXIT) != 0) {
   1408   1.52        ad 			lwp_exit(l);
   1409   1.52        ad 			KASSERT(0);
   1410   1.52        ad 			/* NOTREACHED */
   1411   1.52        ad 		}
   1412  1.156     pooka 
   1413  1.156     pooka 		/* update lwpctl processor (for vfork child_return) */
   1414  1.156     pooka 		if (l->l_flag & LW_LWPCTL) {
   1415  1.156     pooka 			lwp_lock(l);
   1416  1.156     pooka 			KASSERT(kpreempt_disabled());
   1417  1.156     pooka 			l->l_lwpctl->lc_curcpu = (int)cpu_index(l->l_cpu);
   1418  1.156     pooka 			l->l_lwpctl->lc_pctr++;
   1419  1.156     pooka 			l->l_flag &= ~LW_LWPCTL;
   1420  1.156     pooka 			lwp_unlock(l);
   1421  1.156     pooka 		}
   1422   1.52        ad 	}
   1423  1.124  wrstuden 
   1424  1.124  wrstuden #ifdef KERN_SA
   1425  1.124  wrstuden 	/*
   1426  1.124  wrstuden 	 * Timer events are handled specially.  We only try once to deliver
   1427  1.124  wrstuden 	 * pending timer upcalls; if if fails, we can try again on the next
   1428  1.124  wrstuden 	 * loop around.  If we need to re-enter lwp_userret(), MD code will
   1429  1.124  wrstuden 	 * bounce us back here through the trap path after we return.
   1430  1.124  wrstuden 	 */
   1431  1.124  wrstuden 	if (p->p_timerpend)
   1432  1.124  wrstuden 		timerupcall(l);
   1433  1.125        ad 	if (l->l_flag & LW_SA_UPCALL)
   1434  1.125        ad 		sa_upcall_userret(l);
   1435  1.124  wrstuden #endif /* KERN_SA */
   1436   1.52        ad }
   1437   1.52        ad 
   1438   1.52        ad /*
   1439   1.52        ad  * Force an LWP to enter the kernel, to take a trip through lwp_userret().
   1440   1.52        ad  */
   1441   1.52        ad void
   1442   1.52        ad lwp_need_userret(struct lwp *l)
   1443   1.52        ad {
   1444   1.63        ad 	KASSERT(lwp_locked(l, NULL));
   1445   1.52        ad 
   1446   1.52        ad 	/*
   1447   1.52        ad 	 * Since the tests in lwp_userret() are done unlocked, make sure
   1448   1.52        ad 	 * that the condition will be seen before forcing the LWP to enter
   1449   1.52        ad 	 * kernel mode.
   1450   1.52        ad 	 */
   1451   1.81        ad 	membar_producer();
   1452   1.52        ad 	cpu_signotify(l);
   1453   1.52        ad }
   1454   1.52        ad 
   1455   1.52        ad /*
   1456   1.52        ad  * Add one reference to an LWP.  This will prevent the LWP from
   1457   1.52        ad  * exiting, thus keep the lwp structure and PCB around to inspect.
   1458   1.52        ad  */
   1459   1.52        ad void
   1460   1.52        ad lwp_addref(struct lwp *l)
   1461   1.52        ad {
   1462   1.52        ad 
   1463  1.103        ad 	KASSERT(mutex_owned(l->l_proc->p_lock));
   1464   1.52        ad 	KASSERT(l->l_stat != LSZOMB);
   1465   1.52        ad 	KASSERT(l->l_refcnt != 0);
   1466   1.52        ad 
   1467   1.52        ad 	l->l_refcnt++;
   1468   1.52        ad }
   1469   1.52        ad 
   1470   1.52        ad /*
   1471   1.52        ad  * Remove one reference to an LWP.  If this is the last reference,
   1472   1.52        ad  * then we must finalize the LWP's death.
   1473   1.52        ad  */
   1474   1.52        ad void
   1475   1.52        ad lwp_delref(struct lwp *l)
   1476   1.52        ad {
   1477   1.52        ad 	struct proc *p = l->l_proc;
   1478   1.52        ad 
   1479  1.103        ad 	mutex_enter(p->p_lock);
   1480  1.142  christos 	lwp_delref2(l);
   1481  1.142  christos 	mutex_exit(p->p_lock);
   1482  1.142  christos }
   1483  1.142  christos 
   1484  1.142  christos /*
   1485  1.142  christos  * Remove one reference to an LWP.  If this is the last reference,
   1486  1.142  christos  * then we must finalize the LWP's death.  The proc mutex is held
   1487  1.142  christos  * on entry.
   1488  1.142  christos  */
   1489  1.142  christos void
   1490  1.142  christos lwp_delref2(struct lwp *l)
   1491  1.142  christos {
   1492  1.142  christos 	struct proc *p = l->l_proc;
   1493  1.142  christos 
   1494  1.142  christos 	KASSERT(mutex_owned(p->p_lock));
   1495   1.72        ad 	KASSERT(l->l_stat != LSZOMB);
   1496   1.72        ad 	KASSERT(l->l_refcnt > 0);
   1497   1.52        ad 	if (--l->l_refcnt == 0)
   1498   1.76        ad 		cv_broadcast(&p->p_lwpcv);
   1499   1.52        ad }
   1500   1.52        ad 
   1501   1.52        ad /*
   1502   1.52        ad  * Drain all references to the current LWP.
   1503   1.52        ad  */
   1504   1.52        ad void
   1505   1.52        ad lwp_drainrefs(struct lwp *l)
   1506   1.52        ad {
   1507   1.52        ad 	struct proc *p = l->l_proc;
   1508   1.52        ad 
   1509  1.103        ad 	KASSERT(mutex_owned(p->p_lock));
   1510   1.52        ad 	KASSERT(l->l_refcnt != 0);
   1511   1.52        ad 
   1512   1.52        ad 	l->l_refcnt--;
   1513   1.52        ad 	while (l->l_refcnt != 0)
   1514  1.103        ad 		cv_wait(&p->p_lwpcv, p->p_lock);
   1515   1.37        ad }
   1516   1.41   thorpej 
   1517   1.41   thorpej /*
   1518  1.127        ad  * Return true if the specified LWP is 'alive'.  Only p->p_lock need
   1519  1.127        ad  * be held.
   1520  1.127        ad  */
   1521  1.127        ad bool
   1522  1.127        ad lwp_alive(lwp_t *l)
   1523  1.127        ad {
   1524  1.127        ad 
   1525  1.127        ad 	KASSERT(mutex_owned(l->l_proc->p_lock));
   1526  1.127        ad 
   1527  1.127        ad 	switch (l->l_stat) {
   1528  1.127        ad 	case LSSLEEP:
   1529  1.127        ad 	case LSRUN:
   1530  1.127        ad 	case LSONPROC:
   1531  1.127        ad 	case LSSTOP:
   1532  1.127        ad 	case LSSUSPENDED:
   1533  1.127        ad 		return true;
   1534  1.127        ad 	default:
   1535  1.127        ad 		return false;
   1536  1.127        ad 	}
   1537  1.127        ad }
   1538  1.127        ad 
   1539  1.127        ad /*
   1540  1.127        ad  * Return first live LWP in the process.
   1541  1.127        ad  */
   1542  1.127        ad lwp_t *
   1543  1.127        ad lwp_find_first(proc_t *p)
   1544  1.127        ad {
   1545  1.127        ad 	lwp_t *l;
   1546  1.127        ad 
   1547  1.127        ad 	KASSERT(mutex_owned(p->p_lock));
   1548  1.127        ad 
   1549  1.127        ad 	LIST_FOREACH(l, &p->p_lwps, l_sibling) {
   1550  1.127        ad 		if (lwp_alive(l)) {
   1551  1.127        ad 			return l;
   1552  1.127        ad 		}
   1553  1.127        ad 	}
   1554  1.127        ad 
   1555  1.127        ad 	return NULL;
   1556  1.127        ad }
   1557  1.127        ad 
   1558  1.127        ad /*
   1559   1.78        ad  * Allocate a new lwpctl structure for a user LWP.
   1560   1.78        ad  */
   1561   1.78        ad int
   1562   1.78        ad lwp_ctl_alloc(vaddr_t *uaddr)
   1563   1.78        ad {
   1564   1.78        ad 	lcproc_t *lp;
   1565   1.78        ad 	u_int bit, i, offset;
   1566   1.78        ad 	struct uvm_object *uao;
   1567   1.78        ad 	int error;
   1568   1.78        ad 	lcpage_t *lcp;
   1569   1.78        ad 	proc_t *p;
   1570   1.78        ad 	lwp_t *l;
   1571   1.78        ad 
   1572   1.78        ad 	l = curlwp;
   1573   1.78        ad 	p = l->l_proc;
   1574   1.78        ad 
   1575  1.156     pooka 	/* don't allow a vforked process to create lwp ctls */
   1576  1.156     pooka 	if (p->p_lflag & PL_PPWAIT)
   1577  1.156     pooka 		return EBUSY;
   1578  1.156     pooka 
   1579   1.81        ad 	if (l->l_lcpage != NULL) {
   1580   1.81        ad 		lcp = l->l_lcpage;
   1581   1.81        ad 		*uaddr = lcp->lcp_uaddr + (vaddr_t)l->l_lwpctl - lcp->lcp_kaddr;
   1582  1.143     njoly 		return 0;
   1583   1.81        ad 	}
   1584   1.78        ad 
   1585   1.78        ad 	/* First time around, allocate header structure for the process. */
   1586   1.78        ad 	if ((lp = p->p_lwpctl) == NULL) {
   1587   1.78        ad 		lp = kmem_alloc(sizeof(*lp), KM_SLEEP);
   1588   1.78        ad 		mutex_init(&lp->lp_lock, MUTEX_DEFAULT, IPL_NONE);
   1589   1.78        ad 		lp->lp_uao = NULL;
   1590   1.78        ad 		TAILQ_INIT(&lp->lp_pages);
   1591  1.103        ad 		mutex_enter(p->p_lock);
   1592   1.78        ad 		if (p->p_lwpctl == NULL) {
   1593   1.78        ad 			p->p_lwpctl = lp;
   1594  1.103        ad 			mutex_exit(p->p_lock);
   1595   1.78        ad 		} else {
   1596  1.103        ad 			mutex_exit(p->p_lock);
   1597   1.78        ad 			mutex_destroy(&lp->lp_lock);
   1598   1.78        ad 			kmem_free(lp, sizeof(*lp));
   1599   1.78        ad 			lp = p->p_lwpctl;
   1600   1.78        ad 		}
   1601   1.78        ad 	}
   1602   1.78        ad 
   1603   1.78        ad  	/*
   1604   1.78        ad  	 * Set up an anonymous memory region to hold the shared pages.
   1605   1.78        ad  	 * Map them into the process' address space.  The user vmspace
   1606   1.78        ad  	 * gets the first reference on the UAO.
   1607   1.78        ad  	 */
   1608   1.78        ad 	mutex_enter(&lp->lp_lock);
   1609   1.78        ad 	if (lp->lp_uao == NULL) {
   1610   1.78        ad 		lp->lp_uao = uao_create(LWPCTL_UAREA_SZ, 0);
   1611   1.78        ad 		lp->lp_cur = 0;
   1612   1.78        ad 		lp->lp_max = LWPCTL_UAREA_SZ;
   1613   1.78        ad 		lp->lp_uva = p->p_emul->e_vm_default_addr(p,
   1614   1.78        ad 		     (vaddr_t)p->p_vmspace->vm_daddr, LWPCTL_UAREA_SZ);
   1615   1.78        ad 		error = uvm_map(&p->p_vmspace->vm_map, &lp->lp_uva,
   1616   1.78        ad 		    LWPCTL_UAREA_SZ, lp->lp_uao, 0, 0, UVM_MAPFLAG(UVM_PROT_RW,
   1617   1.78        ad 		    UVM_PROT_RW, UVM_INH_NONE, UVM_ADV_NORMAL, 0));
   1618   1.78        ad 		if (error != 0) {
   1619   1.78        ad 			uao_detach(lp->lp_uao);
   1620   1.78        ad 			lp->lp_uao = NULL;
   1621   1.78        ad 			mutex_exit(&lp->lp_lock);
   1622   1.78        ad 			return error;
   1623   1.78        ad 		}
   1624   1.78        ad 	}
   1625   1.78        ad 
   1626   1.78        ad 	/* Get a free block and allocate for this LWP. */
   1627   1.78        ad 	TAILQ_FOREACH(lcp, &lp->lp_pages, lcp_chain) {
   1628   1.78        ad 		if (lcp->lcp_nfree != 0)
   1629   1.78        ad 			break;
   1630   1.78        ad 	}
   1631   1.78        ad 	if (lcp == NULL) {
   1632   1.78        ad 		/* Nothing available - try to set up a free page. */
   1633   1.78        ad 		if (lp->lp_cur == lp->lp_max) {
   1634   1.78        ad 			mutex_exit(&lp->lp_lock);
   1635   1.78        ad 			return ENOMEM;
   1636   1.78        ad 		}
   1637   1.78        ad 		lcp = kmem_alloc(LWPCTL_LCPAGE_SZ, KM_SLEEP);
   1638   1.79      yamt 		if (lcp == NULL) {
   1639   1.79      yamt 			mutex_exit(&lp->lp_lock);
   1640   1.78        ad 			return ENOMEM;
   1641   1.79      yamt 		}
   1642   1.78        ad 		/*
   1643   1.78        ad 		 * Wire the next page down in kernel space.  Since this
   1644   1.78        ad 		 * is a new mapping, we must add a reference.
   1645   1.78        ad 		 */
   1646   1.78        ad 		uao = lp->lp_uao;
   1647   1.78        ad 		(*uao->pgops->pgo_reference)(uao);
   1648   1.99        ad 		lcp->lcp_kaddr = vm_map_min(kernel_map);
   1649   1.78        ad 		error = uvm_map(kernel_map, &lcp->lcp_kaddr, PAGE_SIZE,
   1650   1.78        ad 		    uao, lp->lp_cur, PAGE_SIZE,
   1651   1.78        ad 		    UVM_MAPFLAG(UVM_PROT_RW, UVM_PROT_RW,
   1652   1.78        ad 		    UVM_INH_NONE, UVM_ADV_RANDOM, 0));
   1653   1.78        ad 		if (error != 0) {
   1654   1.78        ad 			mutex_exit(&lp->lp_lock);
   1655   1.78        ad 			kmem_free(lcp, LWPCTL_LCPAGE_SZ);
   1656   1.78        ad 			(*uao->pgops->pgo_detach)(uao);
   1657   1.78        ad 			return error;
   1658   1.78        ad 		}
   1659   1.89      yamt 		error = uvm_map_pageable(kernel_map, lcp->lcp_kaddr,
   1660   1.89      yamt 		    lcp->lcp_kaddr + PAGE_SIZE, FALSE, 0);
   1661   1.89      yamt 		if (error != 0) {
   1662   1.89      yamt 			mutex_exit(&lp->lp_lock);
   1663   1.89      yamt 			uvm_unmap(kernel_map, lcp->lcp_kaddr,
   1664   1.89      yamt 			    lcp->lcp_kaddr + PAGE_SIZE);
   1665   1.89      yamt 			kmem_free(lcp, LWPCTL_LCPAGE_SZ);
   1666   1.89      yamt 			return error;
   1667   1.89      yamt 		}
   1668   1.78        ad 		/* Prepare the page descriptor and link into the list. */
   1669   1.78        ad 		lcp->lcp_uaddr = lp->lp_uva + lp->lp_cur;
   1670   1.78        ad 		lp->lp_cur += PAGE_SIZE;
   1671   1.78        ad 		lcp->lcp_nfree = LWPCTL_PER_PAGE;
   1672   1.78        ad 		lcp->lcp_rotor = 0;
   1673   1.78        ad 		memset(lcp->lcp_bitmap, 0xff, LWPCTL_BITMAP_SZ);
   1674   1.78        ad 		TAILQ_INSERT_HEAD(&lp->lp_pages, lcp, lcp_chain);
   1675   1.78        ad 	}
   1676   1.78        ad 	for (i = lcp->lcp_rotor; lcp->lcp_bitmap[i] == 0;) {
   1677   1.78        ad 		if (++i >= LWPCTL_BITMAP_ENTRIES)
   1678   1.78        ad 			i = 0;
   1679   1.78        ad 	}
   1680   1.78        ad 	bit = ffs(lcp->lcp_bitmap[i]) - 1;
   1681   1.78        ad 	lcp->lcp_bitmap[i] ^= (1 << bit);
   1682   1.78        ad 	lcp->lcp_rotor = i;
   1683   1.78        ad 	lcp->lcp_nfree--;
   1684   1.78        ad 	l->l_lcpage = lcp;
   1685   1.78        ad 	offset = (i << 5) + bit;
   1686   1.78        ad 	l->l_lwpctl = (lwpctl_t *)lcp->lcp_kaddr + offset;
   1687   1.78        ad 	*uaddr = lcp->lcp_uaddr + offset * sizeof(lwpctl_t);
   1688   1.78        ad 	mutex_exit(&lp->lp_lock);
   1689   1.78        ad 
   1690  1.107        ad 	KPREEMPT_DISABLE(l);
   1691  1.111        ad 	l->l_lwpctl->lc_curcpu = (int)curcpu()->ci_data.cpu_index;
   1692  1.107        ad 	KPREEMPT_ENABLE(l);
   1693   1.78        ad 
   1694   1.78        ad 	return 0;
   1695   1.78        ad }
   1696   1.78        ad 
   1697   1.78        ad /*
   1698   1.78        ad  * Free an lwpctl structure back to the per-process list.
   1699   1.78        ad  */
   1700   1.78        ad void
   1701   1.78        ad lwp_ctl_free(lwp_t *l)
   1702   1.78        ad {
   1703  1.156     pooka 	struct proc *p = l->l_proc;
   1704   1.78        ad 	lcproc_t *lp;
   1705   1.78        ad 	lcpage_t *lcp;
   1706   1.78        ad 	u_int map, offset;
   1707   1.78        ad 
   1708  1.156     pooka 	/* don't free a lwp context we borrowed for vfork */
   1709  1.156     pooka 	if (p->p_lflag & PL_PPWAIT) {
   1710  1.156     pooka 		l->l_lwpctl = NULL;
   1711  1.156     pooka 		return;
   1712  1.156     pooka 	}
   1713  1.156     pooka 
   1714  1.156     pooka 	lp = p->p_lwpctl;
   1715   1.78        ad 	KASSERT(lp != NULL);
   1716   1.78        ad 
   1717   1.78        ad 	lcp = l->l_lcpage;
   1718   1.78        ad 	offset = (u_int)((lwpctl_t *)l->l_lwpctl - (lwpctl_t *)lcp->lcp_kaddr);
   1719   1.78        ad 	KASSERT(offset < LWPCTL_PER_PAGE);
   1720   1.78        ad 
   1721   1.78        ad 	mutex_enter(&lp->lp_lock);
   1722   1.78        ad 	lcp->lcp_nfree++;
   1723   1.78        ad 	map = offset >> 5;
   1724   1.78        ad 	lcp->lcp_bitmap[map] |= (1 << (offset & 31));
   1725   1.78        ad 	if (lcp->lcp_bitmap[lcp->lcp_rotor] == 0)
   1726   1.78        ad 		lcp->lcp_rotor = map;
   1727   1.78        ad 	if (TAILQ_FIRST(&lp->lp_pages)->lcp_nfree == 0) {
   1728   1.78        ad 		TAILQ_REMOVE(&lp->lp_pages, lcp, lcp_chain);
   1729   1.78        ad 		TAILQ_INSERT_HEAD(&lp->lp_pages, lcp, lcp_chain);
   1730   1.78        ad 	}
   1731   1.78        ad 	mutex_exit(&lp->lp_lock);
   1732   1.78        ad }
   1733   1.78        ad 
   1734   1.78        ad /*
   1735   1.78        ad  * Process is exiting; tear down lwpctl state.  This can only be safely
   1736   1.78        ad  * called by the last LWP in the process.
   1737   1.78        ad  */
   1738   1.78        ad void
   1739   1.78        ad lwp_ctl_exit(void)
   1740   1.78        ad {
   1741   1.78        ad 	lcpage_t *lcp, *next;
   1742   1.78        ad 	lcproc_t *lp;
   1743   1.78        ad 	proc_t *p;
   1744   1.78        ad 	lwp_t *l;
   1745   1.78        ad 
   1746   1.78        ad 	l = curlwp;
   1747   1.78        ad 	l->l_lwpctl = NULL;
   1748   1.95        ad 	l->l_lcpage = NULL;
   1749   1.78        ad 	p = l->l_proc;
   1750   1.78        ad 	lp = p->p_lwpctl;
   1751   1.78        ad 
   1752   1.78        ad 	KASSERT(lp != NULL);
   1753   1.78        ad 	KASSERT(p->p_nlwps == 1);
   1754   1.78        ad 
   1755   1.78        ad 	for (lcp = TAILQ_FIRST(&lp->lp_pages); lcp != NULL; lcp = next) {
   1756   1.78        ad 		next = TAILQ_NEXT(lcp, lcp_chain);
   1757   1.78        ad 		uvm_unmap(kernel_map, lcp->lcp_kaddr,
   1758   1.78        ad 		    lcp->lcp_kaddr + PAGE_SIZE);
   1759   1.78        ad 		kmem_free(lcp, LWPCTL_LCPAGE_SZ);
   1760   1.78        ad 	}
   1761   1.78        ad 
   1762   1.78        ad 	if (lp->lp_uao != NULL) {
   1763   1.78        ad 		uvm_unmap(&p->p_vmspace->vm_map, lp->lp_uva,
   1764   1.78        ad 		    lp->lp_uva + LWPCTL_UAREA_SZ);
   1765   1.78        ad 	}
   1766   1.78        ad 
   1767   1.78        ad 	mutex_destroy(&lp->lp_lock);
   1768   1.78        ad 	kmem_free(lp, sizeof(*lp));
   1769   1.78        ad 	p->p_lwpctl = NULL;
   1770   1.78        ad }
   1771   1.84      yamt 
   1772  1.130        ad /*
   1773  1.130        ad  * Return the current LWP's "preemption counter".  Used to detect
   1774  1.130        ad  * preemption across operations that can tolerate preemption without
   1775  1.130        ad  * crashing, but which may generate incorrect results if preempted.
   1776  1.130        ad  */
   1777  1.130        ad uint64_t
   1778  1.130        ad lwp_pctr(void)
   1779  1.130        ad {
   1780  1.130        ad 
   1781  1.130        ad 	return curlwp->l_ncsw;
   1782  1.130        ad }
   1783  1.130        ad 
   1784  1.151       chs /*
   1785  1.151       chs  * Set an LWP's private data pointer.
   1786  1.151       chs  */
   1787  1.151       chs int
   1788  1.151       chs lwp_setprivate(struct lwp *l, void *ptr)
   1789  1.151       chs {
   1790  1.151       chs 	int error = 0;
   1791  1.151       chs 
   1792  1.151       chs 	l->l_private = ptr;
   1793  1.151       chs #ifdef __HAVE_CPU_LWP_SETPRIVATE
   1794  1.151       chs 	error = cpu_lwp_setprivate(l, ptr);
   1795  1.151       chs #endif
   1796  1.151       chs 	return error;
   1797  1.151       chs }
   1798  1.151       chs 
   1799   1.84      yamt #if defined(DDB)
   1800  1.153     rmind #include <machine/pcb.h>
   1801  1.153     rmind 
   1802   1.84      yamt void
   1803   1.84      yamt lwp_whatis(uintptr_t addr, void (*pr)(const char *, ...))
   1804   1.84      yamt {
   1805   1.84      yamt 	lwp_t *l;
   1806   1.84      yamt 
   1807   1.84      yamt 	LIST_FOREACH(l, &alllwp, l_list) {
   1808   1.84      yamt 		uintptr_t stack = (uintptr_t)KSTACK_LOWEST_ADDR(l);
   1809   1.84      yamt 
   1810   1.84      yamt 		if (addr < stack || stack + KSTACK_SIZE <= addr) {
   1811   1.84      yamt 			continue;
   1812   1.84      yamt 		}
   1813   1.84      yamt 		(*pr)("%p is %p+%zu, LWP %p's stack\n",
   1814   1.84      yamt 		    (void *)addr, (void *)stack,
   1815   1.84      yamt 		    (size_t)(addr - stack), l);
   1816   1.84      yamt 	}
   1817   1.84      yamt }
   1818   1.84      yamt #endif /* defined(DDB) */
   1819