Home | History | Annotate | Line # | Download | only in kern
kern_lwp.c revision 1.84
      1 /*	$NetBSD: kern_lwp.c,v 1.84 2007/12/13 02:45:10 yamt Exp $	*/
      2 
      3 /*-
      4  * Copyright (c) 2001, 2006, 2007 The NetBSD Foundation, Inc.
      5  * All rights reserved.
      6  *
      7  * This code is derived from software contributed to The NetBSD Foundation
      8  * by Nathan J. Williams, and Andrew Doran.
      9  *
     10  * Redistribution and use in source and binary forms, with or without
     11  * modification, are permitted provided that the following conditions
     12  * are met:
     13  * 1. Redistributions of source code must retain the above copyright
     14  *    notice, this list of conditions and the following disclaimer.
     15  * 2. Redistributions in binary form must reproduce the above copyright
     16  *    notice, this list of conditions and the following disclaimer in the
     17  *    documentation and/or other materials provided with the distribution.
     18  * 3. All advertising materials mentioning features or use of this software
     19  *    must display the following acknowledgement:
     20  *        This product includes software developed by the NetBSD
     21  *        Foundation, Inc. and its contributors.
     22  * 4. Neither the name of The NetBSD Foundation nor the names of its
     23  *    contributors may be used to endorse or promote products derived
     24  *    from this software without specific prior written permission.
     25  *
     26  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     27  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     28  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     29  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     30  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     31  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     32  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     33  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     34  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     35  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     36  * POSSIBILITY OF SUCH DAMAGE.
     37  */
     38 
     39 /*
     40  * Overview
     41  *
     42  *	Lightweight processes (LWPs) are the basic unit or thread of
     43  *	execution within the kernel.  The core state of an LWP is described
     44  *	by "struct lwp", also known as lwp_t.
     45  *
     46  *	Each LWP is contained within a process (described by "struct proc"),
     47  *	Every process contains at least one LWP, but may contain more.  The
     48  *	process describes attributes shared among all of its LWPs such as a
     49  *	private address space, global execution state (stopped, active,
     50  *	zombie, ...), signal disposition and so on.  On a multiprocessor
     51  *	machine, multiple LWPs be executing concurrently in the kernel.
     52  *
     53  * Execution states
     54  *
     55  *	At any given time, an LWP has overall state that is described by
     56  *	lwp::l_stat.  The states are broken into two sets below.  The first
     57  *	set is guaranteed to represent the absolute, current state of the
     58  *	LWP:
     59  *
     60  * 	LSONPROC
     61  *
     62  * 		On processor: the LWP is executing on a CPU, either in the
     63  * 		kernel or in user space.
     64  *
     65  * 	LSRUN
     66  *
     67  * 		Runnable: the LWP is parked on a run queue, and may soon be
     68  * 		chosen to run by a idle processor, or by a processor that
     69  * 		has been asked to preempt a currently runnning but lower
     70  * 		priority LWP.  If the LWP is not swapped in (L_INMEM == 0)
     71  *		then the LWP is not on a run queue, but may be soon.
     72  *
     73  * 	LSIDL
     74  *
     75  * 		Idle: the LWP has been created but has not yet executed,
     76  *		or it has ceased executing a unit of work and is waiting
     77  *		to be started again.
     78  *
     79  * 	LSSUSPENDED:
     80  *
     81  * 		Suspended: the LWP has had its execution suspended by
     82  *		another LWP in the same process using the _lwp_suspend()
     83  *		system call.  User-level LWPs also enter the suspended
     84  *		state when the system is shutting down.
     85  *
     86  *	The second set represent a "statement of intent" on behalf of the
     87  *	LWP.  The LWP may in fact be executing on a processor, may be
     88  *	sleeping or idle. It is expected to take the necessary action to
     89  *	stop executing or become "running" again within	a short timeframe.
     90  *	The LW_RUNNING flag in lwp::l_flag indicates that an LWP is running.
     91  *	Importantly, in indicates that its state is tied to a CPU.
     92  *
     93  * 	LSZOMB:
     94  *
     95  * 		Dead or dying: the LWP has released most of its resources
     96  *		and is a) about to switch away into oblivion b) has already
     97  *		switched away.  When it switches away, its few remaining
     98  *		resources can be collected.
     99  *
    100  * 	LSSLEEP:
    101  *
    102  * 		Sleeping: the LWP has entered itself onto a sleep queue, and
    103  * 		has switched away or will switch away shortly to allow other
    104  *		LWPs to run on the CPU.
    105  *
    106  * 	LSSTOP:
    107  *
    108  * 		Stopped: the LWP has been stopped as a result of a job
    109  * 		control signal, or as a result of the ptrace() interface.
    110  *
    111  * 		Stopped LWPs may run briefly within the kernel to handle
    112  * 		signals that they receive, but will not return to user space
    113  * 		until their process' state is changed away from stopped.
    114  *
    115  * 		Single LWPs within a process can not be set stopped
    116  * 		selectively: all actions that can stop or continue LWPs
    117  * 		occur at the process level.
    118  *
    119  * State transitions
    120  *
    121  *	Note that the LSSTOP state may only be set when returning to
    122  *	user space in userret(), or when sleeping interruptably.  The
    123  *	LSSUSPENDED state may only be set in userret().  Before setting
    124  *	those states, we try to ensure that the LWPs will release all
    125  *	locks that they hold, and at a minimum try to ensure that the
    126  *	LWP can be set runnable again by a signal.
    127  *
    128  *	LWPs may transition states in the following ways:
    129  *
    130  *	 RUN -------> ONPROC		ONPROC -----> RUN
    131  *	            > STOPPED			    > SLEEP
    132  *	            > SUSPENDED			    > STOPPED
    133  *						    > SUSPENDED
    134  *						    > ZOMB
    135  *
    136  *	 STOPPED ---> RUN		SUSPENDED --> RUN
    137  *	            > SLEEP			    > SLEEP
    138  *
    139  *	 SLEEP -----> ONPROC		IDL --------> RUN
    140  *		    > RUN		            > SUSPENDED
    141  *		    > STOPPED                       > STOPPED
    142  *		    > SUSPENDED
    143  *
    144  *	Other state transitions are possible with kernel threads (eg
    145  *	ONPROC -> IDL), but only happen under tightly controlled
    146  *	circumstances the side effects are understood.
    147  *
    148  * Locking
    149  *
    150  *	The majority of fields in 'struct lwp' are covered by a single,
    151  *	general spin lock pointed to by lwp::l_mutex.  The locks covering
    152  *	each field are documented in sys/lwp.h.
    153  *
    154  *	State transitions must be made with the LWP's general lock held,
    155  * 	and may cause the LWP's lock pointer to change. Manipulation of
    156  *	the general lock is not performed directly, but through calls to
    157  *	lwp_lock(), lwp_relock() and similar.
    158  *
    159  *	States and their associated locks:
    160  *
    161  *	LSONPROC, LSZOMB:
    162  *
    163  *		Always covered by spc_lwplock, which protects running LWPs.
    164  *		This is a per-CPU lock.
    165  *
    166  *	LSIDL, LSRUN:
    167  *
    168  *		Always covered by spc_mutex, which protects the run queues.
    169  *		This may be a per-CPU lock, depending on the scheduler.
    170  *
    171  *	LSSLEEP:
    172  *
    173  *		Covered by a lock associated with the sleep queue that the
    174  *		LWP resides on, indirectly referenced by l_sleepq->sq_mutex.
    175  *
    176  *	LSSTOP, LSSUSPENDED:
    177  *
    178  *		If the LWP was previously sleeping (l_wchan != NULL), then
    179  *		l_mutex references the sleep queue lock.  If the LWP was
    180  *		runnable or on the CPU when halted, or has been removed from
    181  *		the sleep queue since halted, then the lock is spc_lwplock.
    182  *
    183  *	The lock order is as follows:
    184  *
    185  *		spc::spc_lwplock ->
    186  *		    sleepq_t::sq_mutex ->
    187  *			tschain_t::tc_mutex ->
    188  *			    spc::spc_mutex
    189  *
    190  *	Each process has an scheduler state lock (proc::p_smutex), and a
    191  *	number of counters on LWPs and their states: p_nzlwps, p_nrlwps, and
    192  *	so on.  When an LWP is to be entered into or removed from one of the
    193  *	following states, p_mutex must be held and the process wide counters
    194  *	adjusted:
    195  *
    196  *		LSIDL, LSZOMB, LSSTOP, LSSUSPENDED
    197  *
    198  *	Note that an LWP is considered running or likely to run soon if in
    199  *	one of the following states.  This affects the value of p_nrlwps:
    200  *
    201  *		LSRUN, LSONPROC, LSSLEEP
    202  *
    203  *	p_smutex does not need to be held when transitioning among these
    204  *	three states.
    205  */
    206 
    207 #include <sys/cdefs.h>
    208 __KERNEL_RCSID(0, "$NetBSD: kern_lwp.c,v 1.84 2007/12/13 02:45:10 yamt Exp $");
    209 
    210 #include "opt_ddb.h"
    211 #include "opt_multiprocessor.h"
    212 #include "opt_lockdebug.h"
    213 
    214 #define _LWP_API_PRIVATE
    215 
    216 #include <sys/param.h>
    217 #include <sys/systm.h>
    218 #include <sys/cpu.h>
    219 #include <sys/pool.h>
    220 #include <sys/proc.h>
    221 #include <sys/syscallargs.h>
    222 #include <sys/syscall_stats.h>
    223 #include <sys/kauth.h>
    224 #include <sys/sleepq.h>
    225 #include <sys/lockdebug.h>
    226 #include <sys/kmem.h>
    227 #include <sys/intr.h>
    228 #include <sys/lwpctl.h>
    229 #include <sys/atomic.h>
    230 
    231 #include <uvm/uvm_extern.h>
    232 #include <uvm/uvm_object.h>
    233 
    234 struct lwplist	alllwp = LIST_HEAD_INITIALIZER(alllwp);
    235 
    236 POOL_INIT(lwp_pool, sizeof(struct lwp), MIN_LWP_ALIGNMENT, 0, 0, "lwppl",
    237     &pool_allocator_nointr, IPL_NONE);
    238 POOL_INIT(lwp_uc_pool, sizeof(ucontext_t), 0, 0, 0, "lwpucpl",
    239     &pool_allocator_nointr, IPL_NONE);
    240 
    241 static specificdata_domain_t lwp_specificdata_domain;
    242 
    243 void
    244 lwpinit(void)
    245 {
    246 
    247 	lwp_specificdata_domain = specificdata_domain_create();
    248 	KASSERT(lwp_specificdata_domain != NULL);
    249 	lwp_sys_init();
    250 }
    251 
    252 /*
    253  * Set an suspended.
    254  *
    255  * Must be called with p_smutex held, and the LWP locked.  Will unlock the
    256  * LWP before return.
    257  */
    258 int
    259 lwp_suspend(struct lwp *curl, struct lwp *t)
    260 {
    261 	int error;
    262 
    263 	KASSERT(mutex_owned(&t->l_proc->p_smutex));
    264 	KASSERT(lwp_locked(t, NULL));
    265 
    266 	KASSERT(curl != t || curl->l_stat == LSONPROC);
    267 
    268 	/*
    269 	 * If the current LWP has been told to exit, we must not suspend anyone
    270 	 * else or deadlock could occur.  We won't return to userspace.
    271 	 */
    272 	if ((curl->l_stat & (LW_WEXIT | LW_WCORE)) != 0) {
    273 		lwp_unlock(t);
    274 		return (EDEADLK);
    275 	}
    276 
    277 	error = 0;
    278 
    279 	switch (t->l_stat) {
    280 	case LSRUN:
    281 	case LSONPROC:
    282 		t->l_flag |= LW_WSUSPEND;
    283 		lwp_need_userret(t);
    284 		lwp_unlock(t);
    285 		break;
    286 
    287 	case LSSLEEP:
    288 		t->l_flag |= LW_WSUSPEND;
    289 
    290 		/*
    291 		 * Kick the LWP and try to get it to the kernel boundary
    292 		 * so that it will release any locks that it holds.
    293 		 * setrunnable() will release the lock.
    294 		 */
    295 		if ((t->l_flag & LW_SINTR) != 0)
    296 			setrunnable(t);
    297 		else
    298 			lwp_unlock(t);
    299 		break;
    300 
    301 	case LSSUSPENDED:
    302 		lwp_unlock(t);
    303 		break;
    304 
    305 	case LSSTOP:
    306 		t->l_flag |= LW_WSUSPEND;
    307 		setrunnable(t);
    308 		break;
    309 
    310 	case LSIDL:
    311 	case LSZOMB:
    312 		error = EINTR; /* It's what Solaris does..... */
    313 		lwp_unlock(t);
    314 		break;
    315 	}
    316 
    317 	return (error);
    318 }
    319 
    320 /*
    321  * Restart a suspended LWP.
    322  *
    323  * Must be called with p_smutex held, and the LWP locked.  Will unlock the
    324  * LWP before return.
    325  */
    326 void
    327 lwp_continue(struct lwp *l)
    328 {
    329 
    330 	KASSERT(mutex_owned(&l->l_proc->p_smutex));
    331 	KASSERT(lwp_locked(l, NULL));
    332 
    333 	/* If rebooting or not suspended, then just bail out. */
    334 	if ((l->l_flag & LW_WREBOOT) != 0) {
    335 		lwp_unlock(l);
    336 		return;
    337 	}
    338 
    339 	l->l_flag &= ~LW_WSUSPEND;
    340 
    341 	if (l->l_stat != LSSUSPENDED) {
    342 		lwp_unlock(l);
    343 		return;
    344 	}
    345 
    346 	/* setrunnable() will release the lock. */
    347 	setrunnable(l);
    348 }
    349 
    350 /*
    351  * Wait for an LWP within the current process to exit.  If 'lid' is
    352  * non-zero, we are waiting for a specific LWP.
    353  *
    354  * Must be called with p->p_smutex held.
    355  */
    356 int
    357 lwp_wait1(struct lwp *l, lwpid_t lid, lwpid_t *departed, int flags)
    358 {
    359 	struct proc *p = l->l_proc;
    360 	struct lwp *l2;
    361 	int nfound, error;
    362 	lwpid_t curlid;
    363 	bool exiting;
    364 
    365 	KASSERT(mutex_owned(&p->p_smutex));
    366 
    367 	p->p_nlwpwait++;
    368 	l->l_waitingfor = lid;
    369 	curlid = l->l_lid;
    370 	exiting = ((flags & LWPWAIT_EXITCONTROL) != 0);
    371 
    372 	for (;;) {
    373 		/*
    374 		 * Avoid a race between exit1() and sigexit(): if the
    375 		 * process is dumping core, then we need to bail out: call
    376 		 * into lwp_userret() where we will be suspended until the
    377 		 * deed is done.
    378 		 */
    379 		if ((p->p_sflag & PS_WCORE) != 0) {
    380 			mutex_exit(&p->p_smutex);
    381 			lwp_userret(l);
    382 #ifdef DIAGNOSTIC
    383 			panic("lwp_wait1");
    384 #endif
    385 			/* NOTREACHED */
    386 		}
    387 
    388 		/*
    389 		 * First off, drain any detached LWP that is waiting to be
    390 		 * reaped.
    391 		 */
    392 		while ((l2 = p->p_zomblwp) != NULL) {
    393 			p->p_zomblwp = NULL;
    394 			lwp_free(l2, false, false);/* releases proc mutex */
    395 			mutex_enter(&p->p_smutex);
    396 		}
    397 
    398 		/*
    399 		 * Now look for an LWP to collect.  If the whole process is
    400 		 * exiting, count detached LWPs as eligible to be collected,
    401 		 * but don't drain them here.
    402 		 */
    403 		nfound = 0;
    404 		error = 0;
    405 		LIST_FOREACH(l2, &p->p_lwps, l_sibling) {
    406 			/*
    407 			 * If a specific wait and the target is waiting on
    408 			 * us, then avoid deadlock.  This also traps LWPs
    409 			 * that try to wait on themselves.
    410 			 *
    411 			 * Note that this does not handle more complicated
    412 			 * cycles, like: t1 -> t2 -> t3 -> t1.  The process
    413 			 * can still be killed so it is not a major problem.
    414 			 */
    415 			if (l2->l_lid == lid && l2->l_waitingfor == curlid) {
    416 				error = EDEADLK;
    417 				break;
    418 			}
    419 			if (l2 == l)
    420 				continue;
    421 			if ((l2->l_prflag & LPR_DETACHED) != 0) {
    422 				nfound += exiting;
    423 				continue;
    424 			}
    425 			if (lid != 0) {
    426 				if (l2->l_lid != lid)
    427 					continue;
    428 				/*
    429 				 * Mark this LWP as the first waiter, if there
    430 				 * is no other.
    431 				 */
    432 				if (l2->l_waiter == 0)
    433 					l2->l_waiter = curlid;
    434 			} else if (l2->l_waiter != 0) {
    435 				/*
    436 				 * It already has a waiter - so don't
    437 				 * collect it.  If the waiter doesn't
    438 				 * grab it we'll get another chance
    439 				 * later.
    440 				 */
    441 				nfound++;
    442 				continue;
    443 			}
    444 			nfound++;
    445 
    446 			/* No need to lock the LWP in order to see LSZOMB. */
    447 			if (l2->l_stat != LSZOMB)
    448 				continue;
    449 
    450 			/*
    451 			 * We're no longer waiting.  Reset the "first waiter"
    452 			 * pointer on the target, in case it was us.
    453 			 */
    454 			l->l_waitingfor = 0;
    455 			l2->l_waiter = 0;
    456 			p->p_nlwpwait--;
    457 			if (departed)
    458 				*departed = l2->l_lid;
    459 			sched_lwp_collect(l2);
    460 
    461 			/* lwp_free() releases the proc lock. */
    462 			lwp_free(l2, false, false);
    463 			mutex_enter(&p->p_smutex);
    464 			return 0;
    465 		}
    466 
    467 		if (error != 0)
    468 			break;
    469 		if (nfound == 0) {
    470 			error = ESRCH;
    471 			break;
    472 		}
    473 
    474 		/*
    475 		 * The kernel is careful to ensure that it can not deadlock
    476 		 * when exiting - just keep waiting.
    477 		 */
    478 		if (exiting) {
    479 			KASSERT(p->p_nlwps > 1);
    480 			cv_wait(&p->p_lwpcv, &p->p_smutex);
    481 			continue;
    482 		}
    483 
    484 		/*
    485 		 * If all other LWPs are waiting for exits or suspends
    486 		 * and the supply of zombies and potential zombies is
    487 		 * exhausted, then we are about to deadlock.
    488 		 *
    489 		 * If the process is exiting (and this LWP is not the one
    490 		 * that is coordinating the exit) then bail out now.
    491 		 */
    492 		if ((p->p_sflag & PS_WEXIT) != 0 ||
    493 		    p->p_nrlwps + p->p_nzlwps - p->p_ndlwps <= p->p_nlwpwait) {
    494 			error = EDEADLK;
    495 			break;
    496 		}
    497 
    498 		/*
    499 		 * Sit around and wait for something to happen.  We'll be
    500 		 * awoken if any of the conditions examined change: if an
    501 		 * LWP exits, is collected, or is detached.
    502 		 */
    503 		if ((error = cv_wait_sig(&p->p_lwpcv, &p->p_smutex)) != 0)
    504 			break;
    505 	}
    506 
    507 	/*
    508 	 * We didn't find any LWPs to collect, we may have received a
    509 	 * signal, or some other condition has caused us to bail out.
    510 	 *
    511 	 * If waiting on a specific LWP, clear the waiters marker: some
    512 	 * other LWP may want it.  Then, kick all the remaining waiters
    513 	 * so that they can re-check for zombies and for deadlock.
    514 	 */
    515 	if (lid != 0) {
    516 		LIST_FOREACH(l2, &p->p_lwps, l_sibling) {
    517 			if (l2->l_lid == lid) {
    518 				if (l2->l_waiter == curlid)
    519 					l2->l_waiter = 0;
    520 				break;
    521 			}
    522 		}
    523 	}
    524 	p->p_nlwpwait--;
    525 	l->l_waitingfor = 0;
    526 	cv_broadcast(&p->p_lwpcv);
    527 
    528 	return error;
    529 }
    530 
    531 /*
    532  * Create a new LWP within process 'p2', using LWP 'l1' as a template.
    533  * The new LWP is created in state LSIDL and must be set running,
    534  * suspended, or stopped by the caller.
    535  */
    536 int
    537 lwp_create(lwp_t *l1, proc_t *p2, vaddr_t uaddr, bool inmem, int flags,
    538 	   void *stack, size_t stacksize, void (*func)(void *), void *arg,
    539 	   lwp_t **rnewlwpp, int sclass)
    540 {
    541 	struct lwp *l2, *isfree;
    542 	turnstile_t *ts;
    543 
    544 	/*
    545 	 * First off, reap any detached LWP waiting to be collected.
    546 	 * We can re-use its LWP structure and turnstile.
    547 	 */
    548 	isfree = NULL;
    549 	if (p2->p_zomblwp != NULL) {
    550 		mutex_enter(&p2->p_smutex);
    551 		if ((isfree = p2->p_zomblwp) != NULL) {
    552 			p2->p_zomblwp = NULL;
    553 			lwp_free(isfree, true, false);/* releases proc mutex */
    554 		} else
    555 			mutex_exit(&p2->p_smutex);
    556 	}
    557 	if (isfree == NULL) {
    558 		l2 = pool_get(&lwp_pool, PR_WAITOK);
    559 		memset(l2, 0, sizeof(*l2));
    560 		l2->l_ts = pool_cache_get(turnstile_cache, PR_WAITOK);
    561 		SLIST_INIT(&l2->l_pi_lenders);
    562 	} else {
    563 		l2 = isfree;
    564 		ts = l2->l_ts;
    565 		KASSERT(l2->l_inheritedprio == -1);
    566 		KASSERT(SLIST_EMPTY(&l2->l_pi_lenders));
    567 		memset(l2, 0, sizeof(*l2));
    568 		l2->l_ts = ts;
    569 	}
    570 
    571 	l2->l_stat = LSIDL;
    572 	l2->l_proc = p2;
    573 	l2->l_refcnt = 1;
    574 	l2->l_class = sclass;
    575 	l2->l_kpriority = l1->l_kpriority;
    576 	l2->l_kpribase = PRI_KERNEL;
    577 	l2->l_priority = l1->l_priority;
    578 	l2->l_inheritedprio = -1;
    579 	l2->l_mutex = l1->l_cpu->ci_schedstate.spc_mutex;
    580 	l2->l_cpu = l1->l_cpu;
    581 	l2->l_flag = inmem ? LW_INMEM : 0;
    582 
    583 	if (p2->p_flag & PK_SYSTEM) {
    584 		/*
    585 		 * Mark it as a system process and not a candidate for
    586 		 * swapping.
    587 		 */
    588 		l2->l_flag |= LW_SYSTEM;
    589 	} else {
    590 		/* Look for a CPU to start */
    591 		l2->l_cpu = sched_takecpu(l2);
    592 		l2->l_mutex = l2->l_cpu->ci_schedstate.spc_mutex;
    593 	}
    594 
    595 	lwp_initspecific(l2);
    596 	sched_lwp_fork(l1, l2);
    597 	lwp_update_creds(l2);
    598 	callout_init(&l2->l_timeout_ch, CALLOUT_MPSAFE);
    599 	callout_setfunc(&l2->l_timeout_ch, sleepq_timeout, l2);
    600 	mutex_init(&l2->l_swaplock, MUTEX_DEFAULT, IPL_NONE);
    601 	cv_init(&l2->l_sigcv, "sigwait");
    602 	l2->l_syncobj = &sched_syncobj;
    603 
    604 	if (rnewlwpp != NULL)
    605 		*rnewlwpp = l2;
    606 
    607 	l2->l_addr = UAREA_TO_USER(uaddr);
    608 	uvm_lwp_fork(l1, l2, stack, stacksize, func,
    609 	    (arg != NULL) ? arg : l2);
    610 
    611 	mutex_enter(&p2->p_smutex);
    612 
    613 	if ((flags & LWP_DETACHED) != 0) {
    614 		l2->l_prflag = LPR_DETACHED;
    615 		p2->p_ndlwps++;
    616 	} else
    617 		l2->l_prflag = 0;
    618 
    619 	l2->l_sigmask = l1->l_sigmask;
    620 	CIRCLEQ_INIT(&l2->l_sigpend.sp_info);
    621 	sigemptyset(&l2->l_sigpend.sp_set);
    622 
    623 	p2->p_nlwpid++;
    624 	if (p2->p_nlwpid == 0)
    625 		p2->p_nlwpid++;
    626 	l2->l_lid = p2->p_nlwpid;
    627 	LIST_INSERT_HEAD(&p2->p_lwps, l2, l_sibling);
    628 	p2->p_nlwps++;
    629 
    630 	mutex_exit(&p2->p_smutex);
    631 
    632 	mutex_enter(&proclist_lock);
    633 	LIST_INSERT_HEAD(&alllwp, l2, l_list);
    634 	mutex_exit(&proclist_lock);
    635 
    636 	SYSCALL_TIME_LWP_INIT(l2);
    637 
    638 	if (p2->p_emul->e_lwp_fork)
    639 		(*p2->p_emul->e_lwp_fork)(l1, l2);
    640 
    641 	return (0);
    642 }
    643 
    644 /*
    645  * Called by MD code when a new LWP begins execution.  Must be called
    646  * with the previous LWP locked (so at splsched), or if there is no
    647  * previous LWP, at splsched.
    648  */
    649 void
    650 lwp_startup(struct lwp *prev, struct lwp *new)
    651 {
    652 
    653 	if (prev != NULL) {
    654 		/*
    655 		 * Normalize the count of the spin-mutexes, it was
    656 		 * increased in mi_switch().  Unmark the state of
    657 		 * context switch - it is finished for previous LWP.
    658 		 */
    659 		curcpu()->ci_mtx_count++;
    660 		membar_exit();
    661 		prev->l_ctxswtch = 0;
    662 	}
    663 	spl0();
    664 	pmap_activate(new);
    665 	LOCKDEBUG_BARRIER(NULL, 0);
    666 	if ((new->l_pflag & LP_MPSAFE) == 0) {
    667 		KERNEL_LOCK(1, new);
    668 	}
    669 }
    670 
    671 /*
    672  * Exit an LWP.
    673  */
    674 void
    675 lwp_exit(struct lwp *l)
    676 {
    677 	struct proc *p = l->l_proc;
    678 	struct lwp *l2;
    679 	bool current;
    680 
    681 	current = (l == curlwp);
    682 
    683 	KASSERT(current || l->l_stat == LSIDL);
    684 
    685 	/*
    686 	 * Verify that we hold no locks other than the kernel lock.
    687 	 */
    688 #ifdef MULTIPROCESSOR
    689 	LOCKDEBUG_BARRIER(&kernel_lock, 0);
    690 #else
    691 	LOCKDEBUG_BARRIER(NULL, 0);
    692 #endif
    693 
    694 	/*
    695 	 * If we are the last live LWP in a process, we need to exit the
    696 	 * entire process.  We do so with an exit status of zero, because
    697 	 * it's a "controlled" exit, and because that's what Solaris does.
    698 	 *
    699 	 * We are not quite a zombie yet, but for accounting purposes we
    700 	 * must increment the count of zombies here.
    701 	 *
    702 	 * Note: the last LWP's specificdata will be deleted here.
    703 	 */
    704 	mutex_enter(&p->p_smutex);
    705 	if (p->p_nlwps - p->p_nzlwps == 1) {
    706 		KASSERT(current == true);
    707 		exit1(l, 0);
    708 		/* NOTREACHED */
    709 	}
    710 	p->p_nzlwps++;
    711 	mutex_exit(&p->p_smutex);
    712 
    713 	if (p->p_emul->e_lwp_exit)
    714 		(*p->p_emul->e_lwp_exit)(l);
    715 
    716 	/* Delete the specificdata while it's still safe to sleep. */
    717 	specificdata_fini(lwp_specificdata_domain, &l->l_specdataref);
    718 
    719 	/*
    720 	 * Release our cached credentials.
    721 	 */
    722 	kauth_cred_free(l->l_cred);
    723 	callout_destroy(&l->l_timeout_ch);
    724 
    725 	/*
    726 	 * While we can still block, mark the LWP as unswappable to
    727 	 * prevent conflicts with the with the swapper.
    728 	 */
    729 	if (current)
    730 		uvm_lwp_hold(l);
    731 
    732 	/*
    733 	 * Remove the LWP from the global list.
    734 	 */
    735 	mutex_enter(&proclist_lock);
    736 	mutex_enter(&proclist_mutex);
    737 	LIST_REMOVE(l, l_list);
    738 	mutex_exit(&proclist_mutex);
    739 	mutex_exit(&proclist_lock);
    740 
    741 	/*
    742 	 * Get rid of all references to the LWP that others (e.g. procfs)
    743 	 * may have, and mark the LWP as a zombie.  If the LWP is detached,
    744 	 * mark it waiting for collection in the proc structure.  Note that
    745 	 * before we can do that, we need to free any other dead, deatched
    746 	 * LWP waiting to meet its maker.
    747 	 *
    748 	 * XXXSMP disable preemption.
    749 	 */
    750 	mutex_enter(&p->p_smutex);
    751 	lwp_drainrefs(l);
    752 
    753 	if ((l->l_prflag & LPR_DETACHED) != 0) {
    754 		while ((l2 = p->p_zomblwp) != NULL) {
    755 			p->p_zomblwp = NULL;
    756 			lwp_free(l2, false, false);/* releases proc mutex */
    757 			mutex_enter(&p->p_smutex);
    758 			l->l_refcnt++;
    759 			lwp_drainrefs(l);
    760 		}
    761 		p->p_zomblwp = l;
    762 	}
    763 
    764 	/*
    765 	 * If we find a pending signal for the process and we have been
    766 	 * asked to check for signals, then we loose: arrange to have
    767 	 * all other LWPs in the process check for signals.
    768 	 */
    769 	if ((l->l_flag & LW_PENDSIG) != 0 &&
    770 	    firstsig(&p->p_sigpend.sp_set) != 0) {
    771 		LIST_FOREACH(l2, &p->p_lwps, l_sibling) {
    772 			lwp_lock(l2);
    773 			l2->l_flag |= LW_PENDSIG;
    774 			lwp_unlock(l2);
    775 		}
    776 	}
    777 
    778 	lwp_lock(l);
    779 	l->l_stat = LSZOMB;
    780 	lwp_unlock(l);
    781 	p->p_nrlwps--;
    782 	cv_broadcast(&p->p_lwpcv);
    783 	if (l->l_lwpctl != NULL)
    784 		l->l_lwpctl->lc_curcpu = LWPCTL_CPU_EXITED;
    785 	mutex_exit(&p->p_smutex);
    786 
    787 	/*
    788 	 * We can no longer block.  At this point, lwp_free() may already
    789 	 * be gunning for us.  On a multi-CPU system, we may be off p_lwps.
    790 	 *
    791 	 * Free MD LWP resources.
    792 	 */
    793 #ifndef __NO_CPU_LWP_FREE
    794 	cpu_lwp_free(l, 0);
    795 #endif
    796 
    797 	if (current) {
    798 		pmap_deactivate(l);
    799 
    800 		/*
    801 		 * Release the kernel lock, and switch away into
    802 		 * oblivion.
    803 		 */
    804 #ifdef notyet
    805 		/* XXXSMP hold in lwp_userret() */
    806 		KERNEL_UNLOCK_LAST(l);
    807 #else
    808 		KERNEL_UNLOCK_ALL(l, NULL);
    809 #endif
    810 		lwp_exit_switchaway(l);
    811 	}
    812 }
    813 
    814 void
    815 lwp_exit_switchaway(struct lwp *l)
    816 {
    817 	struct cpu_info *ci;
    818 	struct lwp *idlelwp;
    819 
    820 	/* Unlocked, but is for statistics only. */
    821 	uvmexp.swtch++;
    822 
    823 	(void)splsched();
    824 	l->l_flag &= ~LW_RUNNING;
    825 	ci = curcpu();
    826 	idlelwp = ci->ci_data.cpu_idlelwp;
    827 	idlelwp->l_stat = LSONPROC;
    828 
    829 	/*
    830 	 * cpu_onproc must be updated with the CPU locked, as
    831 	 * aston() may try to set a AST pending on the LWP (and
    832 	 * it does so with the CPU locked).  Otherwise, the LWP
    833 	 * may be destroyed before the AST can be set, leading
    834 	 * to a user-after-free.
    835 	 */
    836 	spc_lock(ci);
    837 	ci->ci_data.cpu_onproc = idlelwp;
    838 	spc_unlock(ci);
    839 	cpu_switchto(NULL, idlelwp, false);
    840 }
    841 
    842 /*
    843  * Free a dead LWP's remaining resources.
    844  *
    845  * XXXLWP limits.
    846  */
    847 void
    848 lwp_free(struct lwp *l, bool recycle, bool last)
    849 {
    850 	struct proc *p = l->l_proc;
    851 	ksiginfoq_t kq;
    852 
    853 	/*
    854 	 * If this was not the last LWP in the process, then adjust
    855 	 * counters and unlock.
    856 	 */
    857 	if (!last) {
    858 		/*
    859 		 * Add the LWP's run time to the process' base value.
    860 		 * This needs to co-incide with coming off p_lwps.
    861 		 */
    862 		timeradd(&l->l_rtime, &p->p_rtime, &p->p_rtime);
    863 		p->p_pctcpu += l->l_pctcpu;
    864 		LIST_REMOVE(l, l_sibling);
    865 		p->p_nlwps--;
    866 		p->p_nzlwps--;
    867 		if ((l->l_prflag & LPR_DETACHED) != 0)
    868 			p->p_ndlwps--;
    869 
    870 		/*
    871 		 * Have any LWPs sleeping in lwp_wait() recheck for
    872 		 * deadlock.
    873 		 */
    874 		cv_broadcast(&p->p_lwpcv);
    875 		mutex_exit(&p->p_smutex);
    876 	}
    877 
    878 #ifdef MULTIPROCESSOR
    879 	/*
    880 	 * In the unlikely event that the LWP is still on the CPU,
    881 	 * then spin until it has switched away.  We need to release
    882 	 * all locks to avoid deadlock against interrupt handlers on
    883 	 * the target CPU.
    884 	 */
    885 	if ((l->l_flag & LW_RUNNING) != 0 || l->l_cpu->ci_curlwp == l) {
    886 		int count;
    887 		(void)count; /* XXXgcc */
    888 		KERNEL_UNLOCK_ALL(curlwp, &count);
    889 		while ((l->l_flag & LW_RUNNING) != 0 ||
    890 		    l->l_cpu->ci_curlwp == l)
    891 			SPINLOCK_BACKOFF_HOOK;
    892 		KERNEL_LOCK(count, curlwp);
    893 	}
    894 #endif
    895 
    896 	/*
    897 	 * Destroy the LWP's remaining signal information.
    898 	 */
    899 	ksiginfo_queue_init(&kq);
    900 	sigclear(&l->l_sigpend, NULL, &kq);
    901 	ksiginfo_queue_drain(&kq);
    902 	cv_destroy(&l->l_sigcv);
    903 	mutex_destroy(&l->l_swaplock);
    904 
    905 	/*
    906 	 * Free the LWP's turnstile and the LWP structure itself unless the
    907 	 * caller wants to recycle them.  Also, free the scheduler specific data.
    908 	 *
    909 	 * We can't return turnstile0 to the pool (it didn't come from it),
    910 	 * so if it comes up just drop it quietly and move on.
    911 	 *
    912 	 * We don't recycle the VM resources at this time.
    913 	 */
    914 	KERNEL_LOCK(1, curlwp);		/* XXXSMP */
    915 
    916 	if (l->l_lwpctl != NULL)
    917 		lwp_ctl_free(l);
    918 	sched_lwp_exit(l);
    919 
    920 	if (!recycle && l->l_ts != &turnstile0)
    921 		pool_cache_put(turnstile_cache, l->l_ts);
    922 #ifndef __NO_CPU_LWP_FREE
    923 	cpu_lwp_free2(l);
    924 #endif
    925 	uvm_lwp_exit(l);
    926 	KASSERT(SLIST_EMPTY(&l->l_pi_lenders));
    927 	KASSERT(l->l_inheritedprio == -1);
    928 	if (!recycle)
    929 		pool_put(&lwp_pool, l);
    930 	KERNEL_UNLOCK_ONE(curlwp);	/* XXXSMP */
    931 }
    932 
    933 /*
    934  * Pick a LWP to represent the process for those operations which
    935  * want information about a "process" that is actually associated
    936  * with a LWP.
    937  *
    938  * If 'locking' is false, no locking or lock checks are performed.
    939  * This is intended for use by DDB.
    940  *
    941  * We don't bother locking the LWP here, since code that uses this
    942  * interface is broken by design and an exact match is not required.
    943  */
    944 struct lwp *
    945 proc_representative_lwp(struct proc *p, int *nrlwps, int locking)
    946 {
    947 	struct lwp *l, *onproc, *running, *sleeping, *stopped, *suspended;
    948 	struct lwp *signalled;
    949 	int cnt;
    950 
    951 	if (locking) {
    952 		KASSERT(mutex_owned(&p->p_smutex));
    953 	}
    954 
    955 	/* Trivial case: only one LWP */
    956 	if (p->p_nlwps == 1) {
    957 		l = LIST_FIRST(&p->p_lwps);
    958 		if (nrlwps)
    959 			*nrlwps = (l->l_stat == LSONPROC || l->l_stat == LSRUN);
    960 		return l;
    961 	}
    962 
    963 	cnt = 0;
    964 	switch (p->p_stat) {
    965 	case SSTOP:
    966 	case SACTIVE:
    967 		/* Pick the most live LWP */
    968 		onproc = running = sleeping = stopped = suspended = NULL;
    969 		signalled = NULL;
    970 		LIST_FOREACH(l, &p->p_lwps, l_sibling) {
    971 			if ((l->l_flag & LW_IDLE) != 0) {
    972 				continue;
    973 			}
    974 			if (l->l_lid == p->p_sigctx.ps_lwp)
    975 				signalled = l;
    976 			switch (l->l_stat) {
    977 			case LSONPROC:
    978 				onproc = l;
    979 				cnt++;
    980 				break;
    981 			case LSRUN:
    982 				running = l;
    983 				cnt++;
    984 				break;
    985 			case LSSLEEP:
    986 				sleeping = l;
    987 				break;
    988 			case LSSTOP:
    989 				stopped = l;
    990 				break;
    991 			case LSSUSPENDED:
    992 				suspended = l;
    993 				break;
    994 			}
    995 		}
    996 		if (nrlwps)
    997 			*nrlwps = cnt;
    998 		if (signalled)
    999 			l = signalled;
   1000 		else if (onproc)
   1001 			l = onproc;
   1002 		else if (running)
   1003 			l = running;
   1004 		else if (sleeping)
   1005 			l = sleeping;
   1006 		else if (stopped)
   1007 			l = stopped;
   1008 		else if (suspended)
   1009 			l = suspended;
   1010 		else
   1011 			break;
   1012 		return l;
   1013 #ifdef DIAGNOSTIC
   1014 	case SIDL:
   1015 	case SZOMB:
   1016 	case SDYING:
   1017 	case SDEAD:
   1018 		if (locking)
   1019 			mutex_exit(&p->p_smutex);
   1020 		/* We have more than one LWP and we're in SIDL?
   1021 		 * How'd that happen?
   1022 		 */
   1023 		panic("Too many LWPs in idle/dying process %d (%s) stat = %d",
   1024 		    p->p_pid, p->p_comm, p->p_stat);
   1025 		break;
   1026 	default:
   1027 		if (locking)
   1028 			mutex_exit(&p->p_smutex);
   1029 		panic("Process %d (%s) in unknown state %d",
   1030 		    p->p_pid, p->p_comm, p->p_stat);
   1031 #endif
   1032 	}
   1033 
   1034 	if (locking)
   1035 		mutex_exit(&p->p_smutex);
   1036 	panic("proc_representative_lwp: couldn't find a lwp for process"
   1037 		" %d (%s)", p->p_pid, p->p_comm);
   1038 	/* NOTREACHED */
   1039 	return NULL;
   1040 }
   1041 
   1042 /*
   1043  * Look up a live LWP within the speicifed process, and return it locked.
   1044  *
   1045  * Must be called with p->p_smutex held.
   1046  */
   1047 struct lwp *
   1048 lwp_find(struct proc *p, int id)
   1049 {
   1050 	struct lwp *l;
   1051 
   1052 	KASSERT(mutex_owned(&p->p_smutex));
   1053 
   1054 	LIST_FOREACH(l, &p->p_lwps, l_sibling) {
   1055 		if (l->l_lid == id)
   1056 			break;
   1057 	}
   1058 
   1059 	/*
   1060 	 * No need to lock - all of these conditions will
   1061 	 * be visible with the process level mutex held.
   1062 	 */
   1063 	if (l != NULL && (l->l_stat == LSIDL || l->l_stat == LSZOMB))
   1064 		l = NULL;
   1065 
   1066 	return l;
   1067 }
   1068 
   1069 /*
   1070  * Update an LWP's cached credentials to mirror the process' master copy.
   1071  *
   1072  * This happens early in the syscall path, on user trap, and on LWP
   1073  * creation.  A long-running LWP can also voluntarily choose to update
   1074  * it's credentials by calling this routine.  This may be called from
   1075  * LWP_CACHE_CREDS(), which checks l->l_cred != p->p_cred beforehand.
   1076  */
   1077 void
   1078 lwp_update_creds(struct lwp *l)
   1079 {
   1080 	kauth_cred_t oc;
   1081 	struct proc *p;
   1082 
   1083 	p = l->l_proc;
   1084 	oc = l->l_cred;
   1085 
   1086 	mutex_enter(&p->p_mutex);
   1087 	kauth_cred_hold(p->p_cred);
   1088 	l->l_cred = p->p_cred;
   1089 	mutex_exit(&p->p_mutex);
   1090 	if (oc != NULL) {
   1091 		KERNEL_LOCK(1, l);	/* XXXSMP */
   1092 		kauth_cred_free(oc);
   1093 		KERNEL_UNLOCK_ONE(l);	/* XXXSMP */
   1094 	}
   1095 }
   1096 
   1097 /*
   1098  * Verify that an LWP is locked, and optionally verify that the lock matches
   1099  * one we specify.
   1100  */
   1101 int
   1102 lwp_locked(struct lwp *l, kmutex_t *mtx)
   1103 {
   1104 	kmutex_t *cur = l->l_mutex;
   1105 
   1106 	return mutex_owned(cur) && (mtx == cur || mtx == NULL);
   1107 }
   1108 
   1109 /*
   1110  * Lock an LWP.
   1111  */
   1112 void
   1113 lwp_lock_retry(struct lwp *l, kmutex_t *old)
   1114 {
   1115 
   1116 	/*
   1117 	 * XXXgcc ignoring kmutex_t * volatile on i386
   1118 	 *
   1119 	 * gcc version 4.1.2 20061021 prerelease (NetBSD nb1 20061021)
   1120 	 */
   1121 #if 1
   1122 	while (l->l_mutex != old) {
   1123 #else
   1124 	for (;;) {
   1125 #endif
   1126 		mutex_spin_exit(old);
   1127 		old = l->l_mutex;
   1128 		mutex_spin_enter(old);
   1129 
   1130 		/*
   1131 		 * mutex_enter() will have posted a read barrier.  Re-test
   1132 		 * l->l_mutex.  If it has changed, we need to try again.
   1133 		 */
   1134 #if 1
   1135 	}
   1136 #else
   1137 	} while (__predict_false(l->l_mutex != old));
   1138 #endif
   1139 }
   1140 
   1141 /*
   1142  * Lend a new mutex to an LWP.  The old mutex must be held.
   1143  */
   1144 void
   1145 lwp_setlock(struct lwp *l, kmutex_t *new)
   1146 {
   1147 
   1148 	KASSERT(mutex_owned(l->l_mutex));
   1149 
   1150 	membar_producer();
   1151 	l->l_mutex = new;
   1152 }
   1153 
   1154 /*
   1155  * Lend a new mutex to an LWP, and release the old mutex.  The old mutex
   1156  * must be held.
   1157  */
   1158 void
   1159 lwp_unlock_to(struct lwp *l, kmutex_t *new)
   1160 {
   1161 	kmutex_t *old;
   1162 
   1163 	KASSERT(mutex_owned(l->l_mutex));
   1164 
   1165 	old = l->l_mutex;
   1166 	membar_producer();
   1167 	l->l_mutex = new;
   1168 	mutex_spin_exit(old);
   1169 }
   1170 
   1171 /*
   1172  * Acquire a new mutex, and donate it to an LWP.  The LWP must already be
   1173  * locked.
   1174  */
   1175 void
   1176 lwp_relock(struct lwp *l, kmutex_t *new)
   1177 {
   1178 	kmutex_t *old;
   1179 
   1180 	KASSERT(mutex_owned(l->l_mutex));
   1181 
   1182 	old = l->l_mutex;
   1183 	if (old != new) {
   1184 		mutex_spin_enter(new);
   1185 		l->l_mutex = new;
   1186 		mutex_spin_exit(old);
   1187 	}
   1188 }
   1189 
   1190 int
   1191 lwp_trylock(struct lwp *l)
   1192 {
   1193 	kmutex_t *old;
   1194 
   1195 	for (;;) {
   1196 		if (!mutex_tryenter(old = l->l_mutex))
   1197 			return 0;
   1198 		if (__predict_true(l->l_mutex == old))
   1199 			return 1;
   1200 		mutex_spin_exit(old);
   1201 	}
   1202 }
   1203 
   1204 /*
   1205  * Handle exceptions for mi_userret().  Called if a member of LW_USERRET is
   1206  * set.
   1207  */
   1208 void
   1209 lwp_userret(struct lwp *l)
   1210 {
   1211 	struct proc *p;
   1212 	void (*hook)(void);
   1213 	int sig;
   1214 
   1215 	p = l->l_proc;
   1216 
   1217 #ifndef __HAVE_FAST_SOFTINTS
   1218 	/* Run pending soft interrupts. */
   1219 	if (l->l_cpu->ci_data.cpu_softints != 0)
   1220 		softint_overlay();
   1221 #endif
   1222 
   1223 	/*
   1224 	 * It should be safe to do this read unlocked on a multiprocessor
   1225 	 * system..
   1226 	 */
   1227 	while ((l->l_flag & LW_USERRET) != 0) {
   1228 		/*
   1229 		 * Process pending signals first, unless the process
   1230 		 * is dumping core or exiting, where we will instead
   1231 		 * enter the L_WSUSPEND case below.
   1232 		 */
   1233 		if ((l->l_flag & (LW_PENDSIG | LW_WCORE | LW_WEXIT)) ==
   1234 		    LW_PENDSIG) {
   1235 			KERNEL_LOCK(1, l);	/* XXXSMP pool_put() below */
   1236 			mutex_enter(&p->p_smutex);
   1237 			while ((sig = issignal(l)) != 0)
   1238 				postsig(sig);
   1239 			mutex_exit(&p->p_smutex);
   1240 			KERNEL_UNLOCK_LAST(l);	/* XXXSMP */
   1241 		}
   1242 
   1243 		/*
   1244 		 * Core-dump or suspend pending.
   1245 		 *
   1246 		 * In case of core dump, suspend ourselves, so that the
   1247 		 * kernel stack and therefore the userland registers saved
   1248 		 * in the trapframe are around for coredump() to write them
   1249 		 * out.  We issue a wakeup on p->p_lwpcv so that sigexit()
   1250 		 * will write the core file out once all other LWPs are
   1251 		 * suspended.
   1252 		 */
   1253 		if ((l->l_flag & LW_WSUSPEND) != 0) {
   1254 			mutex_enter(&p->p_smutex);
   1255 			p->p_nrlwps--;
   1256 			cv_broadcast(&p->p_lwpcv);
   1257 			lwp_lock(l);
   1258 			l->l_stat = LSSUSPENDED;
   1259 			mutex_exit(&p->p_smutex);
   1260 			mi_switch(l);
   1261 		}
   1262 
   1263 		/* Process is exiting. */
   1264 		if ((l->l_flag & LW_WEXIT) != 0) {
   1265 			KERNEL_LOCK(1, l);
   1266 			lwp_exit(l);
   1267 			KASSERT(0);
   1268 			/* NOTREACHED */
   1269 		}
   1270 
   1271 		/* Call userret hook; used by Linux emulation. */
   1272 		if ((l->l_flag & LW_WUSERRET) != 0) {
   1273 			lwp_lock(l);
   1274 			l->l_flag &= ~LW_WUSERRET;
   1275 			lwp_unlock(l);
   1276 			hook = p->p_userret;
   1277 			p->p_userret = NULL;
   1278 			(*hook)();
   1279 		}
   1280 	}
   1281 }
   1282 
   1283 /*
   1284  * Force an LWP to enter the kernel, to take a trip through lwp_userret().
   1285  */
   1286 void
   1287 lwp_need_userret(struct lwp *l)
   1288 {
   1289 	KASSERT(lwp_locked(l, NULL));
   1290 
   1291 	/*
   1292 	 * Since the tests in lwp_userret() are done unlocked, make sure
   1293 	 * that the condition will be seen before forcing the LWP to enter
   1294 	 * kernel mode.
   1295 	 */
   1296 	membar_producer();
   1297 	cpu_signotify(l);
   1298 }
   1299 
   1300 /*
   1301  * Add one reference to an LWP.  This will prevent the LWP from
   1302  * exiting, thus keep the lwp structure and PCB around to inspect.
   1303  */
   1304 void
   1305 lwp_addref(struct lwp *l)
   1306 {
   1307 
   1308 	KASSERT(mutex_owned(&l->l_proc->p_smutex));
   1309 	KASSERT(l->l_stat != LSZOMB);
   1310 	KASSERT(l->l_refcnt != 0);
   1311 
   1312 	l->l_refcnt++;
   1313 }
   1314 
   1315 /*
   1316  * Remove one reference to an LWP.  If this is the last reference,
   1317  * then we must finalize the LWP's death.
   1318  */
   1319 void
   1320 lwp_delref(struct lwp *l)
   1321 {
   1322 	struct proc *p = l->l_proc;
   1323 
   1324 	mutex_enter(&p->p_smutex);
   1325 	KASSERT(l->l_stat != LSZOMB);
   1326 	KASSERT(l->l_refcnt > 0);
   1327 	if (--l->l_refcnt == 0)
   1328 		cv_broadcast(&p->p_lwpcv);
   1329 	mutex_exit(&p->p_smutex);
   1330 }
   1331 
   1332 /*
   1333  * Drain all references to the current LWP.
   1334  */
   1335 void
   1336 lwp_drainrefs(struct lwp *l)
   1337 {
   1338 	struct proc *p = l->l_proc;
   1339 
   1340 	KASSERT(mutex_owned(&p->p_smutex));
   1341 	KASSERT(l->l_refcnt != 0);
   1342 
   1343 	l->l_refcnt--;
   1344 	while (l->l_refcnt != 0)
   1345 		cv_wait(&p->p_lwpcv, &p->p_smutex);
   1346 }
   1347 
   1348 /*
   1349  * lwp_specific_key_create --
   1350  *	Create a key for subsystem lwp-specific data.
   1351  */
   1352 int
   1353 lwp_specific_key_create(specificdata_key_t *keyp, specificdata_dtor_t dtor)
   1354 {
   1355 
   1356 	return (specificdata_key_create(lwp_specificdata_domain, keyp, dtor));
   1357 }
   1358 
   1359 /*
   1360  * lwp_specific_key_delete --
   1361  *	Delete a key for subsystem lwp-specific data.
   1362  */
   1363 void
   1364 lwp_specific_key_delete(specificdata_key_t key)
   1365 {
   1366 
   1367 	specificdata_key_delete(lwp_specificdata_domain, key);
   1368 }
   1369 
   1370 /*
   1371  * lwp_initspecific --
   1372  *	Initialize an LWP's specificdata container.
   1373  */
   1374 void
   1375 lwp_initspecific(struct lwp *l)
   1376 {
   1377 	int error;
   1378 
   1379 	error = specificdata_init(lwp_specificdata_domain, &l->l_specdataref);
   1380 	KASSERT(error == 0);
   1381 }
   1382 
   1383 /*
   1384  * lwp_finispecific --
   1385  *	Finalize an LWP's specificdata container.
   1386  */
   1387 void
   1388 lwp_finispecific(struct lwp *l)
   1389 {
   1390 
   1391 	specificdata_fini(lwp_specificdata_domain, &l->l_specdataref);
   1392 }
   1393 
   1394 /*
   1395  * lwp_getspecific --
   1396  *	Return lwp-specific data corresponding to the specified key.
   1397  *
   1398  *	Note: LWP specific data is NOT INTERLOCKED.  An LWP should access
   1399  *	only its OWN SPECIFIC DATA.  If it is necessary to access another
   1400  *	LWP's specifc data, care must be taken to ensure that doing so
   1401  *	would not cause internal data structure inconsistency (i.e. caller
   1402  *	can guarantee that the target LWP is not inside an lwp_getspecific()
   1403  *	or lwp_setspecific() call).
   1404  */
   1405 void *
   1406 lwp_getspecific(specificdata_key_t key)
   1407 {
   1408 
   1409 	return (specificdata_getspecific_unlocked(lwp_specificdata_domain,
   1410 						  &curlwp->l_specdataref, key));
   1411 }
   1412 
   1413 void *
   1414 _lwp_getspecific_by_lwp(struct lwp *l, specificdata_key_t key)
   1415 {
   1416 
   1417 	return (specificdata_getspecific_unlocked(lwp_specificdata_domain,
   1418 						  &l->l_specdataref, key));
   1419 }
   1420 
   1421 /*
   1422  * lwp_setspecific --
   1423  *	Set lwp-specific data corresponding to the specified key.
   1424  */
   1425 void
   1426 lwp_setspecific(specificdata_key_t key, void *data)
   1427 {
   1428 
   1429 	specificdata_setspecific(lwp_specificdata_domain,
   1430 				 &curlwp->l_specdataref, key, data);
   1431 }
   1432 
   1433 /*
   1434  * Allocate a new lwpctl structure for a user LWP.
   1435  */
   1436 int
   1437 lwp_ctl_alloc(vaddr_t *uaddr)
   1438 {
   1439 	lcproc_t *lp;
   1440 	u_int bit, i, offset;
   1441 	struct uvm_object *uao;
   1442 	int error;
   1443 	lcpage_t *lcp;
   1444 	proc_t *p;
   1445 	lwp_t *l;
   1446 
   1447 	l = curlwp;
   1448 	p = l->l_proc;
   1449 
   1450 	if (l->l_lcpage != NULL) {
   1451 		lcp = l->l_lcpage;
   1452 		*uaddr = lcp->lcp_uaddr + (vaddr_t)l->l_lwpctl - lcp->lcp_kaddr;
   1453 		return (EINVAL);
   1454 	}
   1455 
   1456 	/* First time around, allocate header structure for the process. */
   1457 	if ((lp = p->p_lwpctl) == NULL) {
   1458 		lp = kmem_alloc(sizeof(*lp), KM_SLEEP);
   1459 		mutex_init(&lp->lp_lock, MUTEX_DEFAULT, IPL_NONE);
   1460 		lp->lp_uao = NULL;
   1461 		TAILQ_INIT(&lp->lp_pages);
   1462 		mutex_enter(&p->p_mutex);
   1463 		if (p->p_lwpctl == NULL) {
   1464 			p->p_lwpctl = lp;
   1465 			mutex_exit(&p->p_mutex);
   1466 		} else {
   1467 			mutex_exit(&p->p_mutex);
   1468 			mutex_destroy(&lp->lp_lock);
   1469 			kmem_free(lp, sizeof(*lp));
   1470 			lp = p->p_lwpctl;
   1471 		}
   1472 	}
   1473 
   1474  	/*
   1475  	 * Set up an anonymous memory region to hold the shared pages.
   1476  	 * Map them into the process' address space.  The user vmspace
   1477  	 * gets the first reference on the UAO.
   1478  	 */
   1479 	mutex_enter(&lp->lp_lock);
   1480 	if (lp->lp_uao == NULL) {
   1481 		lp->lp_uao = uao_create(LWPCTL_UAREA_SZ, 0);
   1482 		lp->lp_cur = 0;
   1483 		lp->lp_max = LWPCTL_UAREA_SZ;
   1484 		lp->lp_uva = p->p_emul->e_vm_default_addr(p,
   1485 		     (vaddr_t)p->p_vmspace->vm_daddr, LWPCTL_UAREA_SZ);
   1486 		error = uvm_map(&p->p_vmspace->vm_map, &lp->lp_uva,
   1487 		    LWPCTL_UAREA_SZ, lp->lp_uao, 0, 0, UVM_MAPFLAG(UVM_PROT_RW,
   1488 		    UVM_PROT_RW, UVM_INH_NONE, UVM_ADV_NORMAL, 0));
   1489 		if (error != 0) {
   1490 			uao_detach(lp->lp_uao);
   1491 			lp->lp_uao = NULL;
   1492 			mutex_exit(&lp->lp_lock);
   1493 			return error;
   1494 		}
   1495 	}
   1496 
   1497 	/* Get a free block and allocate for this LWP. */
   1498 	TAILQ_FOREACH(lcp, &lp->lp_pages, lcp_chain) {
   1499 		if (lcp->lcp_nfree != 0)
   1500 			break;
   1501 	}
   1502 	if (lcp == NULL) {
   1503 		/* Nothing available - try to set up a free page. */
   1504 		if (lp->lp_cur == lp->lp_max) {
   1505 			mutex_exit(&lp->lp_lock);
   1506 			return ENOMEM;
   1507 		}
   1508 		lcp = kmem_alloc(LWPCTL_LCPAGE_SZ, KM_SLEEP);
   1509 		if (lcp == NULL) {
   1510 			mutex_exit(&lp->lp_lock);
   1511 			return ENOMEM;
   1512 		}
   1513 		/*
   1514 		 * Wire the next page down in kernel space.  Since this
   1515 		 * is a new mapping, we must add a reference.
   1516 		 */
   1517 		uao = lp->lp_uao;
   1518 		(*uao->pgops->pgo_reference)(uao);
   1519 		error = uvm_map(kernel_map, &lcp->lcp_kaddr, PAGE_SIZE,
   1520 		    uao, lp->lp_cur, PAGE_SIZE,
   1521 		    UVM_MAPFLAG(UVM_PROT_RW, UVM_PROT_RW,
   1522 		    UVM_INH_NONE, UVM_ADV_RANDOM, 0));
   1523 		if (error == 0)
   1524 			error = uvm_map_pageable(kernel_map, lcp->lcp_kaddr,
   1525 			    lcp->lcp_kaddr + PAGE_SIZE, FALSE, 0);
   1526 		if (error != 0) {
   1527 			mutex_exit(&lp->lp_lock);
   1528 			kmem_free(lcp, LWPCTL_LCPAGE_SZ);
   1529 			(*uao->pgops->pgo_detach)(uao);
   1530 			return error;
   1531 		}
   1532 		/* Prepare the page descriptor and link into the list. */
   1533 		lcp->lcp_uaddr = lp->lp_uva + lp->lp_cur;
   1534 		lp->lp_cur += PAGE_SIZE;
   1535 		lcp->lcp_nfree = LWPCTL_PER_PAGE;
   1536 		lcp->lcp_rotor = 0;
   1537 		memset(lcp->lcp_bitmap, 0xff, LWPCTL_BITMAP_SZ);
   1538 		TAILQ_INSERT_HEAD(&lp->lp_pages, lcp, lcp_chain);
   1539 	}
   1540 	for (i = lcp->lcp_rotor; lcp->lcp_bitmap[i] == 0;) {
   1541 		if (++i >= LWPCTL_BITMAP_ENTRIES)
   1542 			i = 0;
   1543 	}
   1544 	bit = ffs(lcp->lcp_bitmap[i]) - 1;
   1545 	lcp->lcp_bitmap[i] ^= (1 << bit);
   1546 	lcp->lcp_rotor = i;
   1547 	lcp->lcp_nfree--;
   1548 	l->l_lcpage = lcp;
   1549 	offset = (i << 5) + bit;
   1550 	l->l_lwpctl = (lwpctl_t *)lcp->lcp_kaddr + offset;
   1551 	*uaddr = lcp->lcp_uaddr + offset * sizeof(lwpctl_t);
   1552 	mutex_exit(&lp->lp_lock);
   1553 
   1554 	l->l_lwpctl->lc_curcpu = (short)curcpu()->ci_data.cpu_index;
   1555 
   1556 	return 0;
   1557 }
   1558 
   1559 /*
   1560  * Free an lwpctl structure back to the per-process list.
   1561  */
   1562 void
   1563 lwp_ctl_free(lwp_t *l)
   1564 {
   1565 	lcproc_t *lp;
   1566 	lcpage_t *lcp;
   1567 	u_int map, offset;
   1568 
   1569 	lp = l->l_proc->p_lwpctl;
   1570 	KASSERT(lp != NULL);
   1571 
   1572 	lcp = l->l_lcpage;
   1573 	offset = (u_int)((lwpctl_t *)l->l_lwpctl - (lwpctl_t *)lcp->lcp_kaddr);
   1574 	KASSERT(offset < LWPCTL_PER_PAGE);
   1575 
   1576 	mutex_enter(&lp->lp_lock);
   1577 	lcp->lcp_nfree++;
   1578 	map = offset >> 5;
   1579 	lcp->lcp_bitmap[map] |= (1 << (offset & 31));
   1580 	if (lcp->lcp_bitmap[lcp->lcp_rotor] == 0)
   1581 		lcp->lcp_rotor = map;
   1582 	if (TAILQ_FIRST(&lp->lp_pages)->lcp_nfree == 0) {
   1583 		TAILQ_REMOVE(&lp->lp_pages, lcp, lcp_chain);
   1584 		TAILQ_INSERT_HEAD(&lp->lp_pages, lcp, lcp_chain);
   1585 	}
   1586 	mutex_exit(&lp->lp_lock);
   1587 }
   1588 
   1589 /*
   1590  * Process is exiting; tear down lwpctl state.  This can only be safely
   1591  * called by the last LWP in the process.
   1592  */
   1593 void
   1594 lwp_ctl_exit(void)
   1595 {
   1596 	lcpage_t *lcp, *next;
   1597 	lcproc_t *lp;
   1598 	proc_t *p;
   1599 	lwp_t *l;
   1600 
   1601 	l = curlwp;
   1602 	l->l_lwpctl = NULL;
   1603 	p = l->l_proc;
   1604 	lp = p->p_lwpctl;
   1605 
   1606 	KASSERT(lp != NULL);
   1607 	KASSERT(p->p_nlwps == 1);
   1608 
   1609 	for (lcp = TAILQ_FIRST(&lp->lp_pages); lcp != NULL; lcp = next) {
   1610 		next = TAILQ_NEXT(lcp, lcp_chain);
   1611 		uvm_unmap(kernel_map, lcp->lcp_kaddr,
   1612 		    lcp->lcp_kaddr + PAGE_SIZE);
   1613 		kmem_free(lcp, LWPCTL_LCPAGE_SZ);
   1614 	}
   1615 
   1616 	if (lp->lp_uao != NULL) {
   1617 		uvm_unmap(&p->p_vmspace->vm_map, lp->lp_uva,
   1618 		    lp->lp_uva + LWPCTL_UAREA_SZ);
   1619 	}
   1620 
   1621 	mutex_destroy(&lp->lp_lock);
   1622 	kmem_free(lp, sizeof(*lp));
   1623 	p->p_lwpctl = NULL;
   1624 }
   1625 
   1626 #if defined(DDB)
   1627 void
   1628 lwp_whatis(uintptr_t addr, void (*pr)(const char *, ...))
   1629 {
   1630 	lwp_t *l;
   1631 
   1632 	LIST_FOREACH(l, &alllwp, l_list) {
   1633 		uintptr_t stack = (uintptr_t)KSTACK_LOWEST_ADDR(l);
   1634 
   1635 		if (addr < stack || stack + KSTACK_SIZE <= addr) {
   1636 			continue;
   1637 		}
   1638 		(*pr)("%p is %p+%zu, LWP %p's stack\n",
   1639 		    (void *)addr, (void *)stack,
   1640 		    (size_t)(addr - stack), l);
   1641 	}
   1642 }
   1643 #endif /* defined(DDB) */
   1644