Home | History | Annotate | Line # | Download | only in kern
kern_lwp.c revision 1.55.2.11
      1 /*	$NetBSD: kern_lwp.c,v 1.55.2.11 2007/04/16 23:31:20 ad Exp $	*/
      2 
      3 /*-
      4  * Copyright (c) 2001, 2006, 2007 The NetBSD Foundation, Inc.
      5  * All rights reserved.
      6  *
      7  * This code is derived from software contributed to The NetBSD Foundation
      8  * by Nathan J. Williams, and Andrew Doran.
      9  *
     10  * Redistribution and use in source and binary forms, with or without
     11  * modification, are permitted provided that the following conditions
     12  * are met:
     13  * 1. Redistributions of source code must retain the above copyright
     14  *    notice, this list of conditions and the following disclaimer.
     15  * 2. Redistributions in binary form must reproduce the above copyright
     16  *    notice, this list of conditions and the following disclaimer in the
     17  *    documentation and/or other materials provided with the distribution.
     18  * 3. All advertising materials mentioning features or use of this software
     19  *    must display the following acknowledgement:
     20  *        This product includes software developed by the NetBSD
     21  *        Foundation, Inc. and its contributors.
     22  * 4. Neither the name of The NetBSD Foundation nor the names of its
     23  *    contributors may be used to endorse or promote products derived
     24  *    from this software without specific prior written permission.
     25  *
     26  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     27  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     28  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     29  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     30  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     31  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     32  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     33  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     34  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     35  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     36  * POSSIBILITY OF SUCH DAMAGE.
     37  */
     38 
     39 /*
     40  * Overview
     41  *
     42  *	Lightweight processes (LWPs) are the basic unit (or thread) of
     43  *	execution within the kernel.  The core state of an LWP is described
     44  *	by "struct lwp".
     45  *
     46  *	Each LWP is contained within a process (described by "struct proc"),
     47  *	Every process contains at least one LWP, but may contain more.  The
     48  *	process describes attributes shared among all of its LWPs such as a
     49  *	private address space, global execution state (stopped, active,
     50  *	zombie, ...), signal disposition and so on.  On a multiprocessor
     51  *	machine, multiple LWPs be executing in kernel simultaneously.
     52  *
     53  *	Note that LWPs differ from kernel threads (kthreads) in that kernel
     54  *	threads are distinct processes (system processes) with no user space
     55  *	component, which themselves may contain one or more LWPs.
     56  *
     57  * Execution states
     58  *
     59  *	At any given time, an LWP has overall state that is described by
     60  *	lwp::l_stat.  The states are broken into two sets below.  The first
     61  *	set is guaranteed to represent the absolute, current state of the
     62  *	LWP:
     63  *
     64  * 	LSONPROC
     65  *
     66  * 		On processor: the LWP is executing on a CPU, either in the
     67  * 		kernel or in user space.
     68  *
     69  * 	LSRUN
     70  *
     71  * 		Runnable: the LWP is parked on a run queue, and may soon be
     72  * 		chosen to run by a idle processor, or by a processor that
     73  * 		has been asked to preempt a currently runnning but lower
     74  * 		priority LWP.  If the LWP is not swapped in (L_INMEM == 0)
     75  *		then the LWP is not on a run queue, but may be soon.
     76  *
     77  * 	LSIDL
     78  *
     79  * 		Idle: the LWP has been created but has not yet executed.
     80  * 		Whoever created the new LWP can be expected to set it to
     81  * 		another state shortly.
     82  *
     83  * 	LSSUSPENDED:
     84  *
     85  * 		Suspended: the LWP has had its execution suspended by
     86  *		another LWP in the same process using the _lwp_suspend()
     87  *		system call.  User-level LWPs also enter the suspended
     88  *		state when the system is shutting down.
     89  *
     90  *	The second set represent a "statement of intent" on behalf of the
     91  *	LWP.  The LWP may in fact be executing on a processor, may be
     92  *	sleeping, idle, or on a run queue. It is expected to take the
     93  *	necessary action to stop executing or become "running" again within
     94  *	a short timeframe.
     95  *
     96  * 	LSZOMB:
     97  *
     98  * 		Dead: the LWP has released most of its resources and is
     99  * 		about to switch away into oblivion.  When it switches away,
    100  * 		its few remaining resources will be collected.
    101  *
    102  * 	LSSLEEP:
    103  *
    104  * 		Sleeping: the LWP has entered itself onto a sleep queue, and
    105  * 		will switch away shortly to allow other LWPs to run on the
    106  * 		CPU.
    107  *
    108  * 	LSSTOP:
    109  *
    110  * 		Stopped: the LWP has been stopped as a result of a job
    111  * 		control signal, or as a result of the ptrace() interface.
    112  * 		Stopped LWPs may run briefly within the kernel to handle
    113  * 		signals that they receive, but will not return to user space
    114  * 		until their process' state is changed away from stopped.
    115  * 		Single LWPs within a process can not be set stopped
    116  * 		selectively: all actions that can stop or continue LWPs
    117  * 		occur at the process level.
    118  *
    119  * State transitions
    120  *
    121  *	Note that the LSSTOP and LSSUSPENDED states may only be set
    122  *	when returning to user space in userret(), or when sleeping
    123  *	interruptably.  Before setting those states, we try to ensure
    124  *	that the LWPs will release all kernel locks that they hold,
    125  *	and at a minimum try to ensure that the LWP can be set runnable
    126  *	again by a signal.
    127  *
    128  *	LWPs may transition states in the following ways:
    129  *
    130  *	 RUN -------> ONPROC		ONPROC -----> RUN
    131  *	            > STOPPED			    > SLEEP
    132  *	            > SUSPENDED			    > STOPPED
    133  *						    > SUSPENDED
    134  *						    > ZOMB
    135  *
    136  *	 STOPPED ---> RUN		SUSPENDED --> RUN
    137  *	            > SLEEP			    > SLEEP
    138  *
    139  *	 SLEEP -----> ONPROC		IDL --------> RUN
    140  *		    > RUN		            > SUSPENDED
    141  *		    > STOPPED                       > STOPPED
    142  *		    > SUSPENDED
    143  *
    144  * Locking
    145  *
    146  *	The majority of fields in 'struct lwp' are covered by a single,
    147  *	general spin mutex pointed to by lwp::l_mutex.  The locks covering
    148  *	each field are documented in sys/lwp.h.
    149  *
    150  *	State transitions must be made with the LWP's general lock held.  In
    151  *	a multiprocessor kernel, state transitions may cause the LWP's lock
    152  *	pointer to change.  On uniprocessor kernels, most scheduler and
    153  *	synchronisation objects such as sleep queues and LWPs are protected
    154  *	by only one mutex (spc_mutex on single CPU).  In this case, LWPs' lock
    155  *	pointers will never change and will always reference spc_mutex.
    156  *	Please note that in a multiprocessor kernel each CPU has own spc_mutex.
    157  *
    158  *	Where spc_mutex is noted, it refers to l->l_cpu->ci_schedstate.spc_mutex
    159 
    160  *
    161  *	Manipulation of the general lock is not performed directly, but
    162  *	through calls to lwp_lock(), lwp_relock() and similar.
    163  *
    164  *	States and their associated locks:
    165  *
    166  *	LSIDL, LSZOMB
    167  *
    168  *		Always covered by spc_mutex.
    169  *
    170  *	LSONPROC, LSRUN:
    171  *
    172  *		Always covered by spc_mutex, which protects the run queues
    173  *		and other miscellaneous items.  If the scheduler is changed
    174  *		to use per-CPU run queues, this may become a per-CPU mutex.
    175  *
    176  *	LSSLEEP:
    177  *
    178  *		Covered by a mutex associated with the sleep queue that the
    179  *		LWP resides on, indirectly referenced by l_sleepq->sq_mutex.
    180  *
    181  *	LSSTOP, LSSUSPENDED:
    182  *
    183  *		If the LWP was previously sleeping (l_wchan != NULL), then
    184  *		l_mutex references the sleep queue mutex.  If the LWP was
    185  *		runnable or on the CPU when halted, or has been removed from
    186  *		the sleep queue since halted, then the mutex is spc_mutex.
    187  *
    188  *	The lock order is as follows:
    189  *
    190  *		sleepq_t::sq_mutex  |---> spc_mutex
    191  *		tschain_t::tc_mutex |
    192  *
    193  *	Each process has an scheduler state mutex (proc::p_smutex), and a
    194  *	number of counters on LWPs and their states: p_nzlwps, p_nrlwps, and
    195  *	so on.  When an LWP is to be entered into or removed from one of the
    196  *	following states, p_mutex must be held and the process wide counters
    197  *	adjusted:
    198  *
    199  *		LSIDL, LSZOMB, LSSTOP, LSSUSPENDED
    200  *
    201  *	Note that an LWP is considered running or likely to run soon if in
    202  *	one of the following states.  This affects the value of p_nrlwps:
    203  *
    204  *		LSRUN, LSONPROC, LSSLEEP
    205  *
    206  *	p_smutex does not need to be held when transitioning among these
    207  *	three states.
    208  */
    209 
    210 #include <sys/cdefs.h>
    211 __KERNEL_RCSID(0, "$NetBSD: kern_lwp.c,v 1.55.2.11 2007/04/16 23:31:20 ad Exp $");
    212 
    213 #include "opt_multiprocessor.h"
    214 #include "opt_lockdebug.h"
    215 
    216 #define _LWP_API_PRIVATE
    217 
    218 #include <sys/param.h>
    219 #include <sys/systm.h>
    220 #include <sys/cpu.h>
    221 #include <sys/pool.h>
    222 #include <sys/proc.h>
    223 #include <sys/syscallargs.h>
    224 #include <sys/syscall_stats.h>
    225 #include <sys/kauth.h>
    226 #include <sys/sleepq.h>
    227 #include <sys/lockdebug.h>
    228 #include <sys/kmem.h>
    229 
    230 #include <uvm/uvm_extern.h>
    231 
    232 struct lwplist	alllwp;
    233 
    234 POOL_INIT(lwp_pool, sizeof(struct lwp), MIN_LWP_ALIGNMENT, 0, 0, "lwppl",
    235     &pool_allocator_nointr, IPL_NONE);
    236 POOL_INIT(lwp_uc_pool, sizeof(ucontext_t), 0, 0, 0, "lwpucpl",
    237     &pool_allocator_nointr, IPL_NONE);
    238 
    239 static specificdata_domain_t lwp_specificdata_domain;
    240 
    241 #define LWP_DEBUG
    242 
    243 #ifdef LWP_DEBUG
    244 int lwp_debug = 0;
    245 #define DPRINTF(x) if (lwp_debug) printf x
    246 #else
    247 #define DPRINTF(x)
    248 #endif
    249 
    250 void
    251 lwpinit(void)
    252 {
    253 
    254 	lwp_specificdata_domain = specificdata_domain_create();
    255 	KASSERT(lwp_specificdata_domain != NULL);
    256 	lwp_sys_init();
    257 }
    258 
    259 /*
    260  * Set an suspended.
    261  *
    262  * Must be called with p_smutex held, and the LWP locked.  Will unlock the
    263  * LWP before return.
    264  */
    265 int
    266 lwp_suspend(struct lwp *curl, struct lwp *t)
    267 {
    268 	int error;
    269 
    270 	KASSERT(mutex_owned(&t->l_proc->p_smutex));
    271 	KASSERT(lwp_locked(t, NULL));
    272 
    273 	KASSERT(curl != t || curl->l_stat == LSONPROC);
    274 
    275 	/*
    276 	 * If the current LWP has been told to exit, we must not suspend anyone
    277 	 * else or deadlock could occur.  We won't return to userspace.
    278 	 */
    279 	if ((curl->l_stat & (LW_WEXIT | LW_WCORE)) != 0) {
    280 		lwp_unlock(t);
    281 		return (EDEADLK);
    282 	}
    283 
    284 	error = 0;
    285 
    286 	switch (t->l_stat) {
    287 	case LSRUN:
    288 	case LSONPROC:
    289 		t->l_flag |= LW_WSUSPEND;
    290 		lwp_need_userret(t);
    291 		lwp_unlock(t);
    292 		break;
    293 
    294 	case LSSLEEP:
    295 		t->l_flag |= LW_WSUSPEND;
    296 
    297 		/*
    298 		 * Kick the LWP and try to get it to the kernel boundary
    299 		 * so that it will release any locks that it holds.
    300 		 * setrunnable() will release the lock.
    301 		 */
    302 		if ((t->l_flag & LW_SINTR) != 0)
    303 			setrunnable(t);
    304 		else
    305 			lwp_unlock(t);
    306 		break;
    307 
    308 	case LSSUSPENDED:
    309 		lwp_unlock(t);
    310 		break;
    311 
    312 	case LSSTOP:
    313 		t->l_flag |= LW_WSUSPEND;
    314 		setrunnable(t);
    315 		break;
    316 
    317 	case LSIDL:
    318 	case LSZOMB:
    319 		error = EINTR; /* It's what Solaris does..... */
    320 		lwp_unlock(t);
    321 		break;
    322 	}
    323 
    324 	/*
    325 	 * XXXLWP Wait for:
    326 	 *
    327 	 * o process exiting
    328 	 * o target LWP suspended
    329 	 * o target LWP not suspended and L_WSUSPEND clear
    330 	 * o target LWP exited
    331 	 */
    332 
    333 	 return (error);
    334 }
    335 
    336 /*
    337  * Restart a suspended LWP.
    338  *
    339  * Must be called with p_smutex held, and the LWP locked.  Will unlock the
    340  * LWP before return.
    341  */
    342 void
    343 lwp_continue(struct lwp *l)
    344 {
    345 
    346 	KASSERT(mutex_owned(&l->l_proc->p_smutex));
    347 	KASSERT(lwp_locked(l, NULL));
    348 
    349 	DPRINTF(("lwp_continue of %d.%d (%s), state %d, wchan %p\n",
    350 	    l->l_proc->p_pid, l->l_lid, l->l_proc->p_comm, l->l_stat,
    351 	    l->l_wchan));
    352 
    353 	/* If rebooting or not suspended, then just bail out. */
    354 	if ((l->l_flag & LW_WREBOOT) != 0) {
    355 		lwp_unlock(l);
    356 		return;
    357 	}
    358 
    359 	l->l_flag &= ~LW_WSUSPEND;
    360 
    361 	if (l->l_stat != LSSUSPENDED) {
    362 		lwp_unlock(l);
    363 		return;
    364 	}
    365 
    366 	/* setrunnable() will release the lock. */
    367 	setrunnable(l);
    368 }
    369 
    370 /*
    371  * Wait for an LWP within the current process to exit.  If 'lid' is
    372  * non-zero, we are waiting for a specific LWP.
    373  *
    374  * Must be called with p->p_smutex held.
    375  */
    376 int
    377 lwp_wait1(struct lwp *l, lwpid_t lid, lwpid_t *departed, int flags)
    378 {
    379 	struct proc *p = l->l_proc;
    380 	struct lwp *l2;
    381 	int nfound, error;
    382 	lwpid_t curlid;
    383 	bool exiting;
    384 
    385 	DPRINTF(("lwp_wait1: %d.%d waiting for %d.\n",
    386 	    p->p_pid, l->l_lid, lid));
    387 
    388 	KASSERT(mutex_owned(&p->p_smutex));
    389 
    390 	p->p_nlwpwait++;
    391 	l->l_waitingfor = lid;
    392 	curlid = l->l_lid;
    393 	exiting = ((flags & LWPWAIT_EXITCONTROL) != 0);
    394 
    395 	for (;;) {
    396 		/*
    397 		 * Avoid a race between exit1() and sigexit(): if the
    398 		 * process is dumping core, then we need to bail out: call
    399 		 * into lwp_userret() where we will be suspended until the
    400 		 * deed is done.
    401 		 */
    402 		if ((p->p_sflag & PS_WCORE) != 0) {
    403 			mutex_exit(&p->p_smutex);
    404 			lwp_userret(l);
    405 #ifdef DIAGNOSTIC
    406 			panic("lwp_wait1");
    407 #endif
    408 			/* NOTREACHED */
    409 		}
    410 
    411 		/*
    412 		 * First off, drain any detached LWP that is waiting to be
    413 		 * reaped.
    414 		 */
    415 		while ((l2 = p->p_zomblwp) != NULL) {
    416 			p->p_zomblwp = NULL;
    417 			lwp_free(l2, false, false);/* releases proc mutex */
    418 			mutex_enter(&p->p_smutex);
    419 		}
    420 
    421 		/*
    422 		 * Now look for an LWP to collect.  If the whole process is
    423 		 * exiting, count detached LWPs as eligible to be collected,
    424 		 * but don't drain them here.
    425 		 */
    426 		nfound = 0;
    427 		error = 0;
    428 		LIST_FOREACH(l2, &p->p_lwps, l_sibling) {
    429 			/*
    430 			 * If a specific wait and the target is waiting on
    431 			 * us, then avoid deadlock.  This also traps LWPs
    432 			 * that try to wait on themselves.
    433 			 *
    434 			 * Note that this does not handle more complicated
    435 			 * cycles, like: t1 -> t2 -> t3 -> t1.  The process
    436 			 * can still be killed so it is not a major problem.
    437 			 */
    438 			if (l2->l_lid == lid && l2->l_waitingfor == curlid) {
    439 				error = EDEADLK;
    440 				break;
    441 			}
    442 			if (l2 == l)
    443 				continue;
    444 			if ((l2->l_prflag & LPR_DETACHED) != 0) {
    445 				nfound += exiting;
    446 				continue;
    447 			}
    448 			if (lid != 0) {
    449 				if (l2->l_lid != lid)
    450 					continue;
    451 				/*
    452 				 * Mark this LWP as the first waiter, if there
    453 				 * is no other.
    454 				 */
    455 				if (l2->l_waiter == 0)
    456 					l2->l_waiter = curlid;
    457 			} else if (l2->l_waiter != 0) {
    458 				/*
    459 				 * It already has a waiter - so don't
    460 				 * collect it.  If the waiter doesn't
    461 				 * grab it we'll get another chance
    462 				 * later.
    463 				 */
    464 				nfound++;
    465 				continue;
    466 			}
    467 			nfound++;
    468 
    469 			/* No need to lock the LWP in order to see LSZOMB. */
    470 			if (l2->l_stat != LSZOMB)
    471 				continue;
    472 
    473 			/*
    474 			 * We're no longer waiting.  Reset the "first waiter"
    475 			 * pointer on the target, in case it was us.
    476 			 */
    477 			l->l_waitingfor = 0;
    478 			l2->l_waiter = 0;
    479 			p->p_nlwpwait--;
    480 			if (departed)
    481 				*departed = l2->l_lid;
    482 
    483 			/* lwp_free() releases the proc lock. */
    484 			lwp_free(l2, false, false);
    485 			mutex_enter(&p->p_smutex);
    486 			return 0;
    487 		}
    488 
    489 		if (error != 0)
    490 			break;
    491 		if (nfound == 0) {
    492 			error = ESRCH;
    493 			break;
    494 		}
    495 
    496 		/*
    497 		 * The kernel is careful to ensure that it can not deadlock
    498 		 * when exiting - just keep waiting.
    499 		 */
    500 		if (exiting) {
    501 			KASSERT(p->p_nlwps > 1);
    502 			cv_wait(&p->p_lwpcv, &p->p_smutex);
    503 			continue;
    504 		}
    505 
    506 		/*
    507 		 * If all other LWPs are waiting for exits or suspends
    508 		 * and the supply of zombies and potential zombies is
    509 		 * exhausted, then we are about to deadlock.
    510 		 *
    511 		 * If the process is exiting (and this LWP is not the one
    512 		 * that is coordinating the exit) then bail out now.
    513 		 */
    514 		if ((p->p_sflag & PS_WEXIT) != 0 ||
    515 		    p->p_nrlwps + p->p_nzlwps - p->p_ndlwps <= p->p_nlwpwait) {
    516 			error = EDEADLK;
    517 			break;
    518 		}
    519 
    520 		/*
    521 		 * Sit around and wait for something to happen.  We'll be
    522 		 * awoken if any of the conditions examined change: if an
    523 		 * LWP exits, is collected, or is detached.
    524 		 */
    525 		if ((error = cv_wait_sig(&p->p_lwpcv, &p->p_smutex)) != 0)
    526 			break;
    527 	}
    528 
    529 	/*
    530 	 * We didn't find any LWPs to collect, we may have received a
    531 	 * signal, or some other condition has caused us to bail out.
    532 	 *
    533 	 * If waiting on a specific LWP, clear the waiters marker: some
    534 	 * other LWP may want it.  Then, kick all the remaining waiters
    535 	 * so that they can re-check for zombies and for deadlock.
    536 	 */
    537 	if (lid != 0) {
    538 		LIST_FOREACH(l2, &p->p_lwps, l_sibling) {
    539 			if (l2->l_lid == lid) {
    540 				if (l2->l_waiter == curlid)
    541 					l2->l_waiter = 0;
    542 				break;
    543 			}
    544 		}
    545 	}
    546 	p->p_nlwpwait--;
    547 	l->l_waitingfor = 0;
    548 	cv_broadcast(&p->p_lwpcv);
    549 
    550 	return error;
    551 }
    552 
    553 /*
    554  * Create a new LWP within process 'p2', using LWP 'l1' as a template.
    555  * The new LWP is created in state LSIDL and must be set running,
    556  * suspended, or stopped by the caller.
    557  */
    558 int
    559 newlwp(struct lwp *l1, struct proc *p2, vaddr_t uaddr, bool inmem,
    560     int flags, void *stack, size_t stacksize,
    561     void (*func)(void *), void *arg, struct lwp **rnewlwpp)
    562 {
    563 	struct lwp *l2, *isfree;
    564 	turnstile_t *ts;
    565 
    566 	/*
    567 	 * First off, reap any detached LWP waiting to be collected.
    568 	 * We can re-use its LWP structure and turnstile.
    569 	 */
    570 	isfree = NULL;
    571 	if (p2->p_zomblwp != NULL) {
    572 		mutex_enter(&p2->p_smutex);
    573 		if ((isfree = p2->p_zomblwp) != NULL) {
    574 			p2->p_zomblwp = NULL;
    575 			lwp_free(isfree, true, false);/* releases proc mutex */
    576 		} else
    577 			mutex_exit(&p2->p_smutex);
    578 	}
    579 	if (isfree == NULL) {
    580 		l2 = pool_get(&lwp_pool, PR_WAITOK);
    581 		memset(l2, 0, sizeof(*l2));
    582 		l2->l_ts = pool_cache_get(&turnstile_cache, PR_WAITOK);
    583 		SLIST_INIT(&l2->l_pi_lenders);
    584 	} else {
    585 		l2 = isfree;
    586 		ts = l2->l_ts;
    587 		KASSERT(l2->l_inheritedprio == MAXPRI);
    588 		KASSERT(SLIST_EMPTY(&l2->l_pi_lenders));
    589 		memset(l2, 0, sizeof(*l2));
    590 		l2->l_ts = ts;
    591 	}
    592 
    593 	l2->l_stat = LSIDL;
    594 	l2->l_proc = p2;
    595 	l2->l_refcnt = 1;
    596 	l2->l_priority = l1->l_priority;
    597 	l2->l_usrpri = l1->l_usrpri;
    598 	l2->l_inheritedprio = MAXPRI;
    599 	l2->l_mutex = l1->l_cpu->ci_schedstate.spc_mutex;
    600 	l2->l_cpu = l1->l_cpu;
    601 	l2->l_flag = inmem ? LW_INMEM : 0;
    602 	lwp_initspecific(l2);
    603 	sched_lwp_fork(l2);
    604 
    605 	if (p2->p_flag & PK_SYSTEM) {
    606 		/*
    607 		 * Mark it as a system process and not a candidate for
    608 		 * swapping.
    609 		 */
    610 		l2->l_flag |= LW_SYSTEM;
    611 	}
    612 
    613 	lwp_update_creds(l2);
    614 	callout_init(&l2->l_tsleep_ch);
    615 	cv_init(&l2->l_sigcv, "sigwait");
    616 	l2->l_syncobj = &sched_syncobj;
    617 
    618 	if (rnewlwpp != NULL)
    619 		*rnewlwpp = l2;
    620 
    621 	l2->l_addr = UAREA_TO_USER(uaddr);
    622 	uvm_lwp_fork(l1, l2, stack, stacksize, func,
    623 	    (arg != NULL) ? arg : l2);
    624 
    625 	mutex_enter(&p2->p_smutex);
    626 
    627 	if ((flags & LWP_DETACHED) != 0) {
    628 		l2->l_prflag = LPR_DETACHED;
    629 		p2->p_ndlwps++;
    630 	} else
    631 		l2->l_prflag = 0;
    632 
    633 	l2->l_sigmask = l1->l_sigmask;
    634 	CIRCLEQ_INIT(&l2->l_sigpend.sp_info);
    635 	sigemptyset(&l2->l_sigpend.sp_set);
    636 
    637 	p2->p_nlwpid++;
    638 	if (p2->p_nlwpid == 0)
    639 		p2->p_nlwpid++;
    640 	l2->l_lid = p2->p_nlwpid;
    641 	LIST_INSERT_HEAD(&p2->p_lwps, l2, l_sibling);
    642 	p2->p_nlwps++;
    643 
    644 	mutex_exit(&p2->p_smutex);
    645 
    646 	mutex_enter(&proclist_mutex);
    647 	LIST_INSERT_HEAD(&alllwp, l2, l_list);
    648 	mutex_exit(&proclist_mutex);
    649 
    650 	SYSCALL_TIME_LWP_INIT(l2);
    651 
    652 	if (p2->p_emul->e_lwp_fork)
    653 		(*p2->p_emul->e_lwp_fork)(l1, l2);
    654 
    655 	return (0);
    656 }
    657 
    658 /*
    659  * Called by MD code when a new LWP begins execution.  Must be called
    660  * with the previous LWP locked (so at splsched), or if there is no
    661  * previous LWP, at splsched.
    662  */
    663 void
    664 lwp_startup(struct lwp *prev, struct lwp *new)
    665 {
    666 
    667 	curlwp = new;
    668 	if (prev != NULL) {
    669 		lwp_unlock(prev);
    670 	}
    671 	spl0();
    672 	pmap_activate(new);
    673 	LOCKDEBUG_BARRIER(NULL, 0);
    674 	KERNEL_LOCK(1, new);
    675 }
    676 
    677 /*
    678  * Quit the process.
    679  * this can only be used meaningfully if you're willing to switch away.
    680  * Calling with l != curlwp would be weird.
    681  */
    682 void
    683 lwp_exit(struct lwp *l)
    684 {
    685 	struct proc *p = l->l_proc;
    686 	struct lwp *l2;
    687 
    688 	DPRINTF(("lwp_exit: %d.%d exiting.\n", p->p_pid, l->l_lid));
    689 	DPRINTF((" nlwps: %d nzlwps: %d\n", p->p_nlwps, p->p_nzlwps));
    690 
    691 	/*
    692 	 * Verify that we hold no locks other than the kernel lock.
    693 	 */
    694 #ifdef MULTIPROCESSOR
    695 	LOCKDEBUG_BARRIER(&kernel_lock, 0);
    696 #else
    697 	LOCKDEBUG_BARRIER(NULL, 0);
    698 #endif
    699 
    700 	/*
    701 	 * If we are the last live LWP in a process, we need to exit the
    702 	 * entire process.  We do so with an exit status of zero, because
    703 	 * it's a "controlled" exit, and because that's what Solaris does.
    704 	 *
    705 	 * We are not quite a zombie yet, but for accounting purposes we
    706 	 * must increment the count of zombies here.
    707 	 *
    708 	 * Note: the last LWP's specificdata will be deleted here.
    709 	 */
    710 	mutex_enter(&p->p_smutex);
    711 	if (p->p_nlwps - p->p_nzlwps == 1) {
    712 		DPRINTF(("lwp_exit: %d.%d calling exit1()\n",
    713 		    p->p_pid, l->l_lid));
    714 		exit1(l, 0);
    715 		/* NOTREACHED */
    716 	}
    717 	p->p_nzlwps++;
    718 	mutex_exit(&p->p_smutex);
    719 
    720 	if (p->p_emul->e_lwp_exit)
    721 		(*p->p_emul->e_lwp_exit)(l);
    722 
    723 	/* Delete the specificdata while it's still safe to sleep. */
    724 	specificdata_fini(lwp_specificdata_domain, &l->l_specdataref);
    725 
    726 	/*
    727 	 * Release our cached credentials.
    728 	 */
    729 	kauth_cred_free(l->l_cred);
    730 
    731 	/*
    732 	 * Remove the LWP from the global list.
    733 	 */
    734 	mutex_enter(&proclist_mutex);
    735 	LIST_REMOVE(l, l_list);
    736 	mutex_exit(&proclist_mutex);
    737 
    738 	/*
    739 	 * Get rid of all references to the LWP that others (e.g. procfs)
    740 	 * may have, and mark the LWP as a zombie.  If the LWP is detached,
    741 	 * mark it waiting for collection in the proc structure.  Note that
    742 	 * before we can do that, we need to free any other dead, deatched
    743 	 * LWP waiting to meet its maker.
    744 	 *
    745 	 * XXXSMP disable preemption.
    746 	 */
    747 	mutex_enter(&p->p_smutex);
    748 	lwp_drainrefs(l);
    749 
    750 	if ((l->l_prflag & LPR_DETACHED) != 0) {
    751 		while ((l2 = p->p_zomblwp) != NULL) {
    752 			p->p_zomblwp = NULL;
    753 			lwp_free(l2, false, false);/* releases proc mutex */
    754 			mutex_enter(&p->p_smutex);
    755 		}
    756 		p->p_zomblwp = l;
    757 	}
    758 
    759 	/*
    760 	 * If we find a pending signal for the process and we have been
    761 	 * asked to check for signals, then we loose: arrange to have
    762 	 * all other LWPs in the process check for signals.
    763 	 */
    764 	if ((l->l_flag & LW_PENDSIG) != 0 &&
    765 	    firstsig(&p->p_sigpend.sp_set) != 0) {
    766 		LIST_FOREACH(l2, &p->p_lwps, l_sibling) {
    767 			lwp_lock(l2);
    768 			l2->l_flag |= LW_PENDSIG;
    769 			lwp_unlock(l2);
    770 		}
    771 	}
    772 
    773 	lwp_lock(l);
    774 	l->l_stat = LSZOMB;
    775 	lwp_unlock(l);
    776 	p->p_nrlwps--;
    777 	cv_broadcast(&p->p_lwpcv);
    778 	mutex_exit(&p->p_smutex);
    779 
    780 	/*
    781 	 * We can no longer block.  At this point, lwp_free() may already
    782 	 * be gunning for us.  On a multi-CPU system, we may be off p_lwps.
    783 	 *
    784 	 * Free MD LWP resources.
    785 	 */
    786 #ifndef __NO_CPU_LWP_FREE
    787 	cpu_lwp_free(l, 0);
    788 #endif
    789 	pmap_deactivate(l);
    790 
    791 	/*
    792 	 * Release the kernel lock, signal another LWP to collect us,
    793 	 * and switch away into oblivion.
    794 	 */
    795 #ifdef notyet
    796 	/* XXXSMP hold in lwp_userret() */
    797 	KERNEL_UNLOCK_LAST(l);
    798 #else
    799 	KERNEL_UNLOCK_ALL(l, NULL);
    800 #endif
    801 
    802 	lwp_exit_switchaway(l);
    803 }
    804 
    805 void
    806 lwp_exit_switchaway(struct lwp *l)
    807 {
    808 	struct cpu_info *ci;
    809 	struct lwp *idlelwp;
    810 
    811 	/* Unlocked, but is for statistics only. */
    812 	uvmexp.swtch++;
    813 
    814 	(void)splsched();
    815 	ci = curcpu();
    816 	idlelwp = ci->ci_data.cpu_idlelwp;
    817 	idlelwp->l_stat = LSONPROC;
    818 	cpu_switchto(NULL, idlelwp);
    819 }
    820 
    821 /*
    822  * Free a dead LWP's remaining resources.
    823  *
    824  * XXXLWP limits.
    825  */
    826 void
    827 lwp_free(struct lwp *l, bool recycle, bool last)
    828 {
    829 	struct proc *p = l->l_proc;
    830 	ksiginfoq_t kq;
    831 
    832 	/*
    833 	 * If this was not the last LWP in the process, then adjust
    834 	 * counters and unlock.
    835 	 */
    836 	if (!last) {
    837 		/*
    838 		 * Add the LWP's run time to the process' base value.
    839 		 * This needs to co-incide with coming off p_lwps.
    840 		 */
    841 		timeradd(&l->l_rtime, &p->p_rtime, &p->p_rtime);
    842 		LIST_REMOVE(l, l_sibling);
    843 		p->p_nlwps--;
    844 		p->p_nzlwps--;
    845 		if ((l->l_prflag & LPR_DETACHED) != 0)
    846 			p->p_ndlwps--;
    847 
    848 		/*
    849 		 * Have any LWPs sleeping in lwp_wait() recheck for
    850 		 * deadlock.
    851 		 */
    852 		cv_broadcast(&p->p_lwpcv);
    853 		mutex_exit(&p->p_smutex);
    854 	}
    855 
    856 #ifdef MULTIPROCESSOR
    857 	/*
    858 	 * In the unlikely event that the LWP is still on the CPU,
    859 	 * then spin until it has switched away.  We need to release
    860 	 * all locks to avoid deadlock against interrupt handlers on
    861 	 * the target CPU.
    862 	 */
    863 	if (l->l_cpu->ci_curlwp == l) {
    864 		int count;
    865 		(void)count; /* XXXgcc */
    866 		KERNEL_UNLOCK_ALL(curlwp, &count);
    867 		while (l->l_cpu->ci_curlwp == l)
    868 			SPINLOCK_BACKOFF_HOOK;
    869 		KERNEL_LOCK(count, curlwp);
    870 	}
    871 #endif
    872 
    873 	/*
    874 	 * Destroy the LWP's remaining signal information.
    875 	 */
    876 	ksiginfo_queue_init(&kq);
    877 	sigclear(&l->l_sigpend, NULL, &kq);
    878 	ksiginfo_queue_drain(&kq);
    879 	cv_destroy(&l->l_sigcv);
    880 
    881 	/*
    882 	 * Free the LWP's turnstile and the LWP structure itself unless the
    883 	 * caller wants to recycle them.  Also, free the scheduler specific data.
    884 	 *
    885 	 * We can't return turnstile0 to the pool (it didn't come from it),
    886 	 * so if it comes up just drop it quietly and move on.
    887 	 *
    888 	 * We don't recycle the VM resources at this time.
    889 	 */
    890 	KERNEL_LOCK(1, curlwp);		/* XXXSMP */
    891 
    892 	sched_lwp_exit(l);
    893 
    894 	if (!recycle && l->l_ts != &turnstile0)
    895 		pool_cache_put(&turnstile_cache, l->l_ts);
    896 #ifndef __NO_CPU_LWP_FREE
    897 	cpu_lwp_free2(l);
    898 #endif
    899 	uvm_lwp_exit(l);
    900 	KASSERT(SLIST_EMPTY(&l->l_pi_lenders));
    901 	KASSERT(l->l_inheritedprio == MAXPRI);
    902 	if (!recycle)
    903 		pool_put(&lwp_pool, l);
    904 	KERNEL_UNLOCK_ONE(curlwp);	/* XXXSMP */
    905 }
    906 
    907 /*
    908  * Pick a LWP to represent the process for those operations which
    909  * want information about a "process" that is actually associated
    910  * with a LWP.
    911  *
    912  * If 'locking' is false, no locking or lock checks are performed.
    913  * This is intended for use by DDB.
    914  *
    915  * We don't bother locking the LWP here, since code that uses this
    916  * interface is broken by design and an exact match is not required.
    917  */
    918 struct lwp *
    919 proc_representative_lwp(struct proc *p, int *nrlwps, int locking)
    920 {
    921 	struct lwp *l, *onproc, *running, *sleeping, *stopped, *suspended;
    922 	struct lwp *signalled;
    923 	int cnt;
    924 
    925 	if (locking) {
    926 		KASSERT(mutex_owned(&p->p_smutex));
    927 	}
    928 
    929 	/* Trivial case: only one LWP */
    930 	if (p->p_nlwps == 1) {
    931 		l = LIST_FIRST(&p->p_lwps);
    932 		if (nrlwps)
    933 			*nrlwps = (l->l_stat == LSONPROC || LSRUN);
    934 		return l;
    935 	}
    936 
    937 	cnt = 0;
    938 	switch (p->p_stat) {
    939 	case SSTOP:
    940 	case SACTIVE:
    941 		/* Pick the most live LWP */
    942 		onproc = running = sleeping = stopped = suspended = NULL;
    943 		signalled = NULL;
    944 		LIST_FOREACH(l, &p->p_lwps, l_sibling) {
    945 			if ((l->l_flag & LW_IDLE) != 0) {
    946 				continue;
    947 			}
    948 			if (l->l_lid == p->p_sigctx.ps_lwp)
    949 				signalled = l;
    950 			switch (l->l_stat) {
    951 			case LSONPROC:
    952 				onproc = l;
    953 				cnt++;
    954 				break;
    955 			case LSRUN:
    956 				running = l;
    957 				cnt++;
    958 				break;
    959 			case LSSLEEP:
    960 				sleeping = l;
    961 				break;
    962 			case LSSTOP:
    963 				stopped = l;
    964 				break;
    965 			case LSSUSPENDED:
    966 				suspended = l;
    967 				break;
    968 			}
    969 		}
    970 		if (nrlwps)
    971 			*nrlwps = cnt;
    972 		if (signalled)
    973 			l = signalled;
    974 		else if (onproc)
    975 			l = onproc;
    976 		else if (running)
    977 			l = running;
    978 		else if (sleeping)
    979 			l = sleeping;
    980 		else if (stopped)
    981 			l = stopped;
    982 		else if (suspended)
    983 			l = suspended;
    984 		else
    985 			break;
    986 		return l;
    987 		if (nrlwps)
    988 			*nrlwps = 0;
    989 		l = LIST_FIRST(&p->p_lwps);
    990 		return l;
    991 #ifdef DIAGNOSTIC
    992 	case SIDL:
    993 	case SZOMB:
    994 	case SDYING:
    995 	case SDEAD:
    996 		if (locking)
    997 			mutex_exit(&p->p_smutex);
    998 		/* We have more than one LWP and we're in SIDL?
    999 		 * How'd that happen?
   1000 		 */
   1001 		panic("Too many LWPs in idle/dying process %d (%s) stat = %d",
   1002 		    p->p_pid, p->p_comm, p->p_stat);
   1003 		break;
   1004 	default:
   1005 		if (locking)
   1006 			mutex_exit(&p->p_smutex);
   1007 		panic("Process %d (%s) in unknown state %d",
   1008 		    p->p_pid, p->p_comm, p->p_stat);
   1009 #endif
   1010 	}
   1011 
   1012 	if (locking)
   1013 		mutex_exit(&p->p_smutex);
   1014 	panic("proc_representative_lwp: couldn't find a lwp for process"
   1015 		" %d (%s)", p->p_pid, p->p_comm);
   1016 	/* NOTREACHED */
   1017 	return NULL;
   1018 }
   1019 
   1020 /*
   1021  * Look up a live LWP within the speicifed process, and return it locked.
   1022  *
   1023  * Must be called with p->p_smutex held.
   1024  */
   1025 struct lwp *
   1026 lwp_find(struct proc *p, int id)
   1027 {
   1028 	struct lwp *l;
   1029 
   1030 	KASSERT(mutex_owned(&p->p_smutex));
   1031 
   1032 	LIST_FOREACH(l, &p->p_lwps, l_sibling) {
   1033 		if (l->l_lid == id)
   1034 			break;
   1035 	}
   1036 
   1037 	/*
   1038 	 * No need to lock - all of these conditions will
   1039 	 * be visible with the process level mutex held.
   1040 	 */
   1041 	if (l != NULL && (l->l_stat == LSIDL || l->l_stat == LSZOMB))
   1042 		l = NULL;
   1043 
   1044 	return l;
   1045 }
   1046 
   1047 /*
   1048  * Update an LWP's cached credentials to mirror the process' master copy.
   1049  *
   1050  * This happens early in the syscall path, on user trap, and on LWP
   1051  * creation.  A long-running LWP can also voluntarily choose to update
   1052  * it's credentials by calling this routine.  This may be called from
   1053  * LWP_CACHE_CREDS(), which checks l->l_cred != p->p_cred beforehand.
   1054  */
   1055 void
   1056 lwp_update_creds(struct lwp *l)
   1057 {
   1058 	kauth_cred_t oc;
   1059 	struct proc *p;
   1060 
   1061 	p = l->l_proc;
   1062 	oc = l->l_cred;
   1063 
   1064 	mutex_enter(&p->p_mutex);
   1065 	kauth_cred_hold(p->p_cred);
   1066 	l->l_cred = p->p_cred;
   1067 	mutex_exit(&p->p_mutex);
   1068 	if (oc != NULL) {
   1069 		KERNEL_LOCK(1, l);	/* XXXSMP */
   1070 		kauth_cred_free(oc);
   1071 		KERNEL_UNLOCK_ONE(l);	/* XXXSMP */
   1072 	}
   1073 }
   1074 
   1075 /*
   1076  * Verify that an LWP is locked, and optionally verify that the lock matches
   1077  * one we specify.
   1078  */
   1079 int
   1080 lwp_locked(struct lwp *l, kmutex_t *mtx)
   1081 {
   1082 	kmutex_t *cur = l->l_mutex;
   1083 
   1084 	return mutex_owned(cur) && (mtx == cur || mtx == NULL);
   1085 }
   1086 
   1087 /*
   1088  * Lock an LWP.
   1089  */
   1090 void
   1091 lwp_lock_retry(struct lwp *l, kmutex_t *old)
   1092 {
   1093 
   1094 	/*
   1095 	 * XXXgcc ignoring kmutex_t * volatile on i386
   1096 	 *
   1097 	 * gcc version 4.1.2 20061021 prerelease (NetBSD nb1 20061021)
   1098 	 */
   1099 #if 1
   1100 	while (l->l_mutex != old) {
   1101 #else
   1102 	for (;;) {
   1103 #endif
   1104 		mutex_spin_exit(old);
   1105 		old = l->l_mutex;
   1106 		mutex_spin_enter(old);
   1107 
   1108 		/*
   1109 		 * mutex_enter() will have posted a read barrier.  Re-test
   1110 		 * l->l_mutex.  If it has changed, we need to try again.
   1111 		 */
   1112 #if 1
   1113 	}
   1114 #else
   1115 	} while (__predict_false(l->l_mutex != old));
   1116 #endif
   1117 }
   1118 
   1119 /*
   1120  * Lend a new mutex to an LWP.  The old mutex must be held.
   1121  */
   1122 void
   1123 lwp_setlock(struct lwp *l, kmutex_t *new)
   1124 {
   1125 
   1126 	KASSERT(mutex_owned(l->l_mutex));
   1127 
   1128 	mb_write();
   1129 	l->l_mutex = new;
   1130 }
   1131 
   1132 /*
   1133  * Lend a new mutex to an LWP, and release the old mutex.  The old mutex
   1134  * must be held.
   1135  */
   1136 void
   1137 lwp_unlock_to(struct lwp *l, kmutex_t *new)
   1138 {
   1139 	kmutex_t *old;
   1140 
   1141 	KASSERT(mutex_owned(l->l_mutex));
   1142 
   1143 	old = l->l_mutex;
   1144 	mb_write();
   1145 	l->l_mutex = new;
   1146 	mutex_spin_exit(old);
   1147 }
   1148 
   1149 /*
   1150  * Acquire a new mutex, and donate it to an LWP.  The LWP must already be
   1151  * locked.
   1152  */
   1153 void
   1154 lwp_relock(struct lwp *l, kmutex_t *new)
   1155 {
   1156 	kmutex_t *old;
   1157 
   1158 	KASSERT(mutex_owned(l->l_mutex));
   1159 
   1160 	old = l->l_mutex;
   1161 	if (old != new) {
   1162 		mutex_spin_enter(new);
   1163 		l->l_mutex = new;
   1164 		mutex_spin_exit(old);
   1165 	}
   1166 }
   1167 
   1168 int
   1169 lwp_trylock(struct lwp *l)
   1170 {
   1171 	kmutex_t *old;
   1172 
   1173 	for (;;) {
   1174 		if (!mutex_tryenter(old = l->l_mutex))
   1175 			return 0;
   1176 		if (__predict_true(l->l_mutex == old))
   1177 			return 1;
   1178 		mutex_spin_exit(old);
   1179 	}
   1180 }
   1181 
   1182 /*
   1183  * Handle exceptions for mi_userret().  Called if a member of LW_USERRET is
   1184  * set.
   1185  */
   1186 void
   1187 lwp_userret(struct lwp *l)
   1188 {
   1189 	struct proc *p;
   1190 	void (*hook)(void);
   1191 	int sig;
   1192 
   1193 	p = l->l_proc;
   1194 
   1195 	/*
   1196 	 * It should be safe to do this read unlocked on a multiprocessor
   1197 	 * system..
   1198 	 */
   1199 	while ((l->l_flag & LW_USERRET) != 0) {
   1200 		/*
   1201 		 * Process pending signals first, unless the process
   1202 		 * is dumping core or exiting, where we will instead
   1203 		 * enter the L_WSUSPEND case below.
   1204 		 */
   1205 		if ((l->l_flag & (LW_PENDSIG | LW_WCORE | LW_WEXIT)) ==
   1206 		    LW_PENDSIG) {
   1207 			KERNEL_LOCK(1, l);	/* XXXSMP pool_put() below */
   1208 			mutex_enter(&p->p_smutex);
   1209 			while ((sig = issignal(l)) != 0)
   1210 				postsig(sig);
   1211 			mutex_exit(&p->p_smutex);
   1212 			KERNEL_UNLOCK_LAST(l);	/* XXXSMP */
   1213 		}
   1214 
   1215 		/*
   1216 		 * Core-dump or suspend pending.
   1217 		 *
   1218 		 * In case of core dump, suspend ourselves, so that the
   1219 		 * kernel stack and therefore the userland registers saved
   1220 		 * in the trapframe are around for coredump() to write them
   1221 		 * out.  We issue a wakeup on p->p_lwpcv so that sigexit()
   1222 		 * will write the core file out once all other LWPs are
   1223 		 * suspended.
   1224 		 */
   1225 		if ((l->l_flag & LW_WSUSPEND) != 0) {
   1226 			mutex_enter(&p->p_smutex);
   1227 			p->p_nrlwps--;
   1228 			cv_broadcast(&p->p_lwpcv);
   1229 			lwp_lock(l);
   1230 			l->l_stat = LSSUSPENDED;
   1231 			mutex_exit(&p->p_smutex);
   1232 			mi_switch(l);
   1233 		}
   1234 
   1235 		/* Process is exiting. */
   1236 		if ((l->l_flag & LW_WEXIT) != 0) {
   1237 			KERNEL_LOCK(1, l);
   1238 			lwp_exit(l);
   1239 			KASSERT(0);
   1240 			/* NOTREACHED */
   1241 		}
   1242 
   1243 		/* Call userret hook; used by Linux emulation. */
   1244 		if ((l->l_flag & LW_WUSERRET) != 0) {
   1245 			lwp_lock(l);
   1246 			l->l_flag &= ~LW_WUSERRET;
   1247 			lwp_unlock(l);
   1248 			hook = p->p_userret;
   1249 			p->p_userret = NULL;
   1250 			(*hook)();
   1251 		}
   1252 	}
   1253 }
   1254 
   1255 /*
   1256  * Force an LWP to enter the kernel, to take a trip through lwp_userret().
   1257  */
   1258 void
   1259 lwp_need_userret(struct lwp *l)
   1260 {
   1261 	KASSERT(lwp_locked(l, NULL));
   1262 
   1263 	/*
   1264 	 * Since the tests in lwp_userret() are done unlocked, make sure
   1265 	 * that the condition will be seen before forcing the LWP to enter
   1266 	 * kernel mode.
   1267 	 */
   1268 	mb_write();
   1269 	cpu_signotify(l);
   1270 }
   1271 
   1272 /*
   1273  * Add one reference to an LWP.  This will prevent the LWP from
   1274  * exiting, thus keep the lwp structure and PCB around to inspect.
   1275  */
   1276 void
   1277 lwp_addref(struct lwp *l)
   1278 {
   1279 
   1280 	KASSERT(mutex_owned(&l->l_proc->p_smutex));
   1281 	KASSERT(l->l_stat != LSZOMB);
   1282 	KASSERT(l->l_refcnt != 0);
   1283 
   1284 	l->l_refcnt++;
   1285 }
   1286 
   1287 /*
   1288  * Remove one reference to an LWP.  If this is the last reference,
   1289  * then we must finalize the LWP's death.
   1290  */
   1291 void
   1292 lwp_delref(struct lwp *l)
   1293 {
   1294 	struct proc *p = l->l_proc;
   1295 
   1296 	mutex_enter(&p->p_smutex);
   1297 	if (--l->l_refcnt == 0)
   1298 		cv_broadcast(&p->p_refcv);
   1299 	mutex_exit(&p->p_smutex);
   1300 }
   1301 
   1302 /*
   1303  * Drain all references to the current LWP.
   1304  */
   1305 void
   1306 lwp_drainrefs(struct lwp *l)
   1307 {
   1308 	struct proc *p = l->l_proc;
   1309 
   1310 	KASSERT(mutex_owned(&p->p_smutex));
   1311 	KASSERT(l->l_refcnt != 0);
   1312 
   1313 	l->l_refcnt--;
   1314 	while (l->l_refcnt != 0)
   1315 		cv_wait(&p->p_refcv, &p->p_smutex);
   1316 }
   1317 
   1318 /*
   1319  * lwp_specific_key_create --
   1320  *	Create a key for subsystem lwp-specific data.
   1321  */
   1322 int
   1323 lwp_specific_key_create(specificdata_key_t *keyp, specificdata_dtor_t dtor)
   1324 {
   1325 
   1326 	return (specificdata_key_create(lwp_specificdata_domain, keyp, dtor));
   1327 }
   1328 
   1329 /*
   1330  * lwp_specific_key_delete --
   1331  *	Delete a key for subsystem lwp-specific data.
   1332  */
   1333 void
   1334 lwp_specific_key_delete(specificdata_key_t key)
   1335 {
   1336 
   1337 	specificdata_key_delete(lwp_specificdata_domain, key);
   1338 }
   1339 
   1340 /*
   1341  * lwp_initspecific --
   1342  *	Initialize an LWP's specificdata container.
   1343  */
   1344 void
   1345 lwp_initspecific(struct lwp *l)
   1346 {
   1347 	int error;
   1348 
   1349 	error = specificdata_init(lwp_specificdata_domain, &l->l_specdataref);
   1350 	KASSERT(error == 0);
   1351 }
   1352 
   1353 /*
   1354  * lwp_finispecific --
   1355  *	Finalize an LWP's specificdata container.
   1356  */
   1357 void
   1358 lwp_finispecific(struct lwp *l)
   1359 {
   1360 
   1361 	specificdata_fini(lwp_specificdata_domain, &l->l_specdataref);
   1362 }
   1363 
   1364 /*
   1365  * lwp_getspecific --
   1366  *	Return lwp-specific data corresponding to the specified key.
   1367  *
   1368  *	Note: LWP specific data is NOT INTERLOCKED.  An LWP should access
   1369  *	only its OWN SPECIFIC DATA.  If it is necessary to access another
   1370  *	LWP's specifc data, care must be taken to ensure that doing so
   1371  *	would not cause internal data structure inconsistency (i.e. caller
   1372  *	can guarantee that the target LWP is not inside an lwp_getspecific()
   1373  *	or lwp_setspecific() call).
   1374  */
   1375 void *
   1376 lwp_getspecific(specificdata_key_t key)
   1377 {
   1378 
   1379 	return (specificdata_getspecific_unlocked(lwp_specificdata_domain,
   1380 						  &curlwp->l_specdataref, key));
   1381 }
   1382 
   1383 void *
   1384 _lwp_getspecific_by_lwp(struct lwp *l, specificdata_key_t key)
   1385 {
   1386 
   1387 	return (specificdata_getspecific_unlocked(lwp_specificdata_domain,
   1388 						  &l->l_specdataref, key));
   1389 }
   1390 
   1391 /*
   1392  * lwp_setspecific --
   1393  *	Set lwp-specific data corresponding to the specified key.
   1394  */
   1395 void
   1396 lwp_setspecific(specificdata_key_t key, void *data)
   1397 {
   1398 
   1399 	specificdata_setspecific(lwp_specificdata_domain,
   1400 				 &curlwp->l_specdataref, key, data);
   1401 }
   1402