Home | History | Annotate | Line # | Download | only in kern
kern_lwp.c revision 1.263
      1 /*	$NetBSD: kern_lwp.c,v 1.263 2023/10/04 22:17:09 ad Exp $	*/
      2 
      3 /*-
      4  * Copyright (c) 2001, 2006, 2007, 2008, 2009, 2019, 2020, 2023
      5  *     The NetBSD Foundation, Inc.
      6  * All rights reserved.
      7  *
      8  * This code is derived from software contributed to The NetBSD Foundation
      9  * by Nathan J. Williams, and Andrew Doran.
     10  *
     11  * Redistribution and use in source and binary forms, with or without
     12  * modification, are permitted provided that the following conditions
     13  * are met:
     14  * 1. Redistributions of source code must retain the above copyright
     15  *    notice, this list of conditions and the following disclaimer.
     16  * 2. Redistributions in binary form must reproduce the above copyright
     17  *    notice, this list of conditions and the following disclaimer in the
     18  *    documentation and/or other materials provided with the distribution.
     19  *
     20  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     21  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     22  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     23  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     24  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     25  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     26  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     27  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     28  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     29  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     30  * POSSIBILITY OF SUCH DAMAGE.
     31  */
     32 
     33 /*
     34  * Overview
     35  *
     36  *	Lightweight processes (LWPs) are the basic unit or thread of
     37  *	execution within the kernel.  The core state of an LWP is described
     38  *	by "struct lwp", also known as lwp_t.
     39  *
     40  *	Each LWP is contained within a process (described by "struct proc"),
     41  *	Every process contains at least one LWP, but may contain more.  The
     42  *	process describes attributes shared among all of its LWPs such as a
     43  *	private address space, global execution state (stopped, active,
     44  *	zombie, ...), signal disposition and so on.  On a multiprocessor
     45  *	machine, multiple LWPs be executing concurrently in the kernel.
     46  *
     47  * Execution states
     48  *
     49  *	At any given time, an LWP has overall state that is described by
     50  *	lwp::l_stat.  The states are broken into two sets below.  The first
     51  *	set is guaranteed to represent the absolute, current state of the
     52  *	LWP:
     53  *
     54  *	LSONPROC
     55  *
     56  *		On processor: the LWP is executing on a CPU, either in the
     57  *		kernel or in user space.
     58  *
     59  *	LSRUN
     60  *
     61  *		Runnable: the LWP is parked on a run queue, and may soon be
     62  *		chosen to run by an idle processor, or by a processor that
     63  *		has been asked to preempt a currently runnning but lower
     64  *		priority LWP.
     65  *
     66  *	LSIDL
     67  *
     68  *		Idle: the LWP has been created but has not yet executed, or
     69  *		it has ceased executing a unit of work and is waiting to be
     70  *		started again.  This state exists so that the LWP can occupy
     71  *		a slot in the process & PID table, but without having to
     72  *		worry about being touched; lookups of the LWP by ID will
     73  *		fail while in this state.  The LWP will become visible for
     74  *		lookup once its state transitions further.  Some special
     75  *		kernel threads also (ab)use this state to indicate that they
     76  *		are idle (soft interrupts and idle LWPs).
     77  *
     78  *	LSSUSPENDED:
     79  *
     80  *		Suspended: the LWP has had its execution suspended by
     81  *		another LWP in the same process using the _lwp_suspend()
     82  *		system call.  User-level LWPs also enter the suspended
     83  *		state when the system is shutting down.
     84  *
     85  *	The second set represent a "statement of intent" on behalf of the
     86  *	LWP.  The LWP may in fact be executing on a processor, may be
     87  *	sleeping or idle. It is expected to take the necessary action to
     88  *	stop executing or become "running" again within a short timeframe.
     89  *	The LP_RUNNING flag in lwp::l_pflag indicates that an LWP is running.
     90  *	Importantly, it indicates that its state is tied to a CPU.
     91  *
     92  *	LSZOMB:
     93  *
     94  *		Dead or dying: the LWP has released most of its resources
     95  *		and is about to switch away into oblivion, or has already
     96  *		switched away.  When it switches away, its few remaining
     97  *		resources can be collected.
     98  *
     99  *	LSSLEEP:
    100  *
    101  *		Sleeping: the LWP has entered itself onto a sleep queue, and
    102  *		has switched away or will switch away shortly to allow other
    103  *		LWPs to run on the CPU.
    104  *
    105  *	LSSTOP:
    106  *
    107  *		Stopped: the LWP has been stopped as a result of a job
    108  *		control signal, or as a result of the ptrace() interface.
    109  *
    110  *		Stopped LWPs may run briefly within the kernel to handle
    111  *		signals that they receive, but will not return to user space
    112  *		until their process' state is changed away from stopped.
    113  *
    114  *		Single LWPs within a process can not be set stopped
    115  *		selectively: all actions that can stop or continue LWPs
    116  *		occur at the process level.
    117  *
    118  * State transitions
    119  *
    120  *	Note that the LSSTOP state may only be set when returning to
    121  *	user space in userret(), or when sleeping interruptably.  The
    122  *	LSSUSPENDED state may only be set in userret().  Before setting
    123  *	those states, we try to ensure that the LWPs will release all
    124  *	locks that they hold, and at a minimum try to ensure that the
    125  *	LWP can be set runnable again by a signal.
    126  *
    127  *	LWPs may transition states in the following ways:
    128  *
    129  *	 RUN -------> ONPROC		ONPROC -----> RUN
    130  *		    				    > SLEEP
    131  *		    				    > STOPPED
    132  *						    > SUSPENDED
    133  *						    > ZOMB
    134  *						    > IDL (special cases)
    135  *
    136  *	 STOPPED ---> RUN		SUSPENDED --> RUN
    137  *	            > SLEEP
    138  *
    139  *	 SLEEP -----> ONPROC		IDL --------> RUN
    140  *		    > RUN			    > SUSPENDED
    141  *		    > STOPPED			    > STOPPED
    142  *						    > ONPROC (special cases)
    143  *
    144  *	Some state transitions are only possible with kernel threads (eg
    145  *	ONPROC -> IDL) and happen under tightly controlled circumstances
    146  *	free of unwanted side effects.
    147  *
    148  * Migration
    149  *
    150  *	Migration of threads from one CPU to another could be performed
    151  *	internally by the scheduler via sched_takecpu() or sched_catchlwp()
    152  *	functions.  The universal lwp_migrate() function should be used for
    153  *	any other cases.  Subsystems in the kernel must be aware that CPU
    154  *	of LWP may change, while it is not locked.
    155  *
    156  * Locking
    157  *
    158  *	The majority of fields in 'struct lwp' are covered by a single,
    159  *	general spin lock pointed to by lwp::l_mutex.  The locks covering
    160  *	each field are documented in sys/lwp.h.
    161  *
    162  *	State transitions must be made with the LWP's general lock held,
    163  *	and may cause the LWP's lock pointer to change.  Manipulation of
    164  *	the general lock is not performed directly, but through calls to
    165  *	lwp_lock(), lwp_unlock() and others.  It should be noted that the
    166  *	adaptive locks are not allowed to be released while the LWP's lock
    167  *	is being held (unlike for other spin-locks).
    168  *
    169  *	States and their associated locks:
    170  *
    171  *	LSIDL, LSONPROC, LSZOMB, LSSUPENDED:
    172  *
    173  *		Always covered by spc_lwplock, which protects LWPs not
    174  *		associated with any other sync object.  This is a per-CPU
    175  *		lock and matches lwp::l_cpu.
    176  *
    177  *	LSRUN:
    178  *
    179  *		Always covered by spc_mutex, which protects the run queues.
    180  *		This is a per-CPU lock and matches lwp::l_cpu.
    181  *
    182  *	LSSLEEP:
    183  *
    184  *		Covered by a lock associated with the sleep queue (sometimes
    185  *		a turnstile sleep queue) that the LWP resides on.  This can
    186  *		be spc_lwplock for SOBJ_SLEEPQ_NULL (an "untracked" sleep).
    187  *
    188  *	LSSTOP:
    189  *
    190  *		If the LWP was previously sleeping (l_wchan != NULL), then
    191  *		l_mutex references the sleep queue lock.  If the LWP was
    192  *		runnable or on the CPU when halted, or has been removed from
    193  *		the sleep queue since halted, then the lock is spc_lwplock.
    194  *
    195  *	The lock order is as follows:
    196  *
    197  *		sleepq -> turnstile -> spc_lwplock -> spc_mutex
    198  *
    199  *	Each process has a scheduler state lock (proc::p_lock), and a
    200  *	number of counters on LWPs and their states: p_nzlwps, p_nrlwps, and
    201  *	so on.  When an LWP is to be entered into or removed from one of the
    202  *	following states, p_lock must be held and the process wide counters
    203  *	adjusted:
    204  *
    205  *		LSIDL, LSZOMB, LSSTOP, LSSUSPENDED
    206  *
    207  *	(But not always for kernel threads.  There are some special cases
    208  *	as mentioned above: soft interrupts, and the idle loops.)
    209  *
    210  *	Note that an LWP is considered running or likely to run soon if in
    211  *	one of the following states.  This affects the value of p_nrlwps:
    212  *
    213  *		LSRUN, LSONPROC, LSSLEEP
    214  *
    215  *	p_lock does not need to be held when transitioning among these
    216  *	three states, hence p_lock is rarely taken for state transitions.
    217  */
    218 
    219 #include <sys/cdefs.h>
    220 __KERNEL_RCSID(0, "$NetBSD: kern_lwp.c,v 1.263 2023/10/04 22:17:09 ad Exp $");
    221 
    222 #include "opt_ddb.h"
    223 #include "opt_lockdebug.h"
    224 #include "opt_dtrace.h"
    225 
    226 #define _LWP_API_PRIVATE
    227 
    228 #include <sys/param.h>
    229 #include <sys/systm.h>
    230 #include <sys/cpu.h>
    231 #include <sys/pool.h>
    232 #include <sys/proc.h>
    233 #include <sys/syscallargs.h>
    234 #include <sys/syscall_stats.h>
    235 #include <sys/kauth.h>
    236 #include <sys/sleepq.h>
    237 #include <sys/lockdebug.h>
    238 #include <sys/kmem.h>
    239 #include <sys/pset.h>
    240 #include <sys/intr.h>
    241 #include <sys/lwpctl.h>
    242 #include <sys/atomic.h>
    243 #include <sys/filedesc.h>
    244 #include <sys/fstrans.h>
    245 #include <sys/dtrace_bsd.h>
    246 #include <sys/sdt.h>
    247 #include <sys/ptrace.h>
    248 #include <sys/xcall.h>
    249 #include <sys/uidinfo.h>
    250 #include <sys/sysctl.h>
    251 #include <sys/psref.h>
    252 #include <sys/msan.h>
    253 #include <sys/kcov.h>
    254 #include <sys/cprng.h>
    255 #include <sys/futex.h>
    256 
    257 #include <uvm/uvm_extern.h>
    258 #include <uvm/uvm_object.h>
    259 
    260 static pool_cache_t	lwp_cache	__read_mostly;
    261 struct lwplist		alllwp		__cacheline_aligned;
    262 
    263 static int		lwp_ctor(void *, void *, int);
    264 static void		lwp_dtor(void *, void *);
    265 
    266 /* DTrace proc provider probes */
    267 SDT_PROVIDER_DEFINE(proc);
    268 
    269 SDT_PROBE_DEFINE1(proc, kernel, , lwp__create, "struct lwp *");
    270 SDT_PROBE_DEFINE1(proc, kernel, , lwp__start, "struct lwp *");
    271 SDT_PROBE_DEFINE1(proc, kernel, , lwp__exit, "struct lwp *");
    272 
    273 struct turnstile turnstile0 __cacheline_aligned;
    274 struct lwp lwp0 __aligned(MIN_LWP_ALIGNMENT) = {
    275 #ifdef LWP0_CPU_INFO
    276 	.l_cpu = LWP0_CPU_INFO,
    277 #endif
    278 #ifdef LWP0_MD_INITIALIZER
    279 	.l_md = LWP0_MD_INITIALIZER,
    280 #endif
    281 	.l_proc = &proc0,
    282 	.l_lid = 0,		/* we own proc0's slot in the pid table */
    283 	.l_flag = LW_SYSTEM,
    284 	.l_stat = LSONPROC,
    285 	.l_ts = &turnstile0,
    286 	.l_syncobj = &sched_syncobj,
    287 	.l_refcnt = 0,
    288 	.l_priority = PRI_USER + NPRI_USER - 1,
    289 	.l_inheritedprio = -1,
    290 	.l_class = SCHED_OTHER,
    291 	.l_psid = PS_NONE,
    292 	.l_pi_lenders = SLIST_HEAD_INITIALIZER(&lwp0.l_pi_lenders),
    293 	.l_name = __UNCONST("swapper"),
    294 	.l_fd = &filedesc0,
    295 };
    296 
    297 static int
    298 lwp_maxlwp(void)
    299 {
    300 	/* Assume 1 LWP per 1MiB. */
    301 	uint64_t lwps_per = ctob(physmem) / (1024 * 1024);
    302 
    303 	return MAX(MIN(MAXMAXLWP, lwps_per), MAXLWP);
    304 }
    305 
    306 static int sysctl_kern_maxlwp(SYSCTLFN_PROTO);
    307 
    308 /*
    309  * sysctl helper routine for kern.maxlwp. Ensures that the new
    310  * values are not too low or too high.
    311  */
    312 static int
    313 sysctl_kern_maxlwp(SYSCTLFN_ARGS)
    314 {
    315 	int error, nmaxlwp;
    316 	struct sysctlnode node;
    317 
    318 	nmaxlwp = maxlwp;
    319 	node = *rnode;
    320 	node.sysctl_data = &nmaxlwp;
    321 	error = sysctl_lookup(SYSCTLFN_CALL(&node));
    322 	if (error || newp == NULL)
    323 		return error;
    324 
    325 	if (nmaxlwp < 0 || nmaxlwp >= MAXMAXLWP)
    326 		return EINVAL;
    327 	if (nmaxlwp > lwp_maxlwp())
    328 		return EINVAL;
    329 	maxlwp = nmaxlwp;
    330 
    331 	return 0;
    332 }
    333 
    334 static void
    335 sysctl_kern_lwp_setup(void)
    336 {
    337 	sysctl_createv(NULL, 0, NULL, NULL,
    338 		       CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
    339 		       CTLTYPE_INT, "maxlwp",
    340 		       SYSCTL_DESCR("Maximum number of simultaneous threads"),
    341 		       sysctl_kern_maxlwp, 0, NULL, 0,
    342 		       CTL_KERN, CTL_CREATE, CTL_EOL);
    343 }
    344 
    345 void
    346 lwpinit(void)
    347 {
    348 
    349 	LIST_INIT(&alllwp);
    350 	lwpinit_specificdata();
    351 	/*
    352 	 * Provide a barrier to ensure that all mutex_oncpu() and rw_oncpu()
    353 	 * calls will exit before memory of LWPs is returned to the pool, where
    354 	 * KVA of LWP structure might be freed and re-used for other purposes.
    355 	 * Kernel preemption is disabled around mutex_oncpu() and rw_oncpu()
    356 	 * callers, therefore a regular passive serialization barrier will
    357 	 * do the job.
    358 	 */
    359 	lwp_cache = pool_cache_init(sizeof(lwp_t), MIN_LWP_ALIGNMENT, 0,
    360 	    PR_PSERIALIZE, "lwppl", NULL, IPL_NONE, lwp_ctor, lwp_dtor, NULL);
    361 
    362 	maxlwp = lwp_maxlwp();
    363 	sysctl_kern_lwp_setup();
    364 }
    365 
    366 void
    367 lwp0_init(void)
    368 {
    369 	struct lwp *l = &lwp0;
    370 
    371 	KASSERT((void *)uvm_lwp_getuarea(l) != NULL);
    372 
    373 	LIST_INSERT_HEAD(&alllwp, l, l_list);
    374 
    375 	callout_init(&l->l_timeout_ch, CALLOUT_MPSAFE);
    376 	callout_setfunc(&l->l_timeout_ch, sleepq_timeout, l);
    377 	cv_init(&l->l_sigcv, "sigwait");
    378 	cv_init(&l->l_waitcv, "vfork");
    379 
    380 	l->l_cred = kauth_cred_hold(proc0.p_cred);
    381 
    382 	kdtrace_thread_ctor(NULL, l);
    383 	lwp_initspecific(l);
    384 
    385 	SYSCALL_TIME_LWP_INIT(l);
    386 }
    387 
    388 /*
    389  * Initialize the non-zeroed portion of an lwp_t.
    390  */
    391 static int
    392 lwp_ctor(void *arg, void *obj, int flags)
    393 {
    394 	lwp_t *l = obj;
    395 
    396 	l->l_stat = LSIDL;
    397 	l->l_cpu = curcpu();
    398 	l->l_mutex = l->l_cpu->ci_schedstate.spc_lwplock;
    399 	l->l_ts = kmem_alloc(sizeof(*l->l_ts), flags == PR_WAITOK ?
    400 	    KM_SLEEP : KM_NOSLEEP);
    401 
    402 	if (l->l_ts == NULL) {
    403 		return ENOMEM;
    404 	} else {
    405 		turnstile_ctor(l->l_ts);
    406 		return 0;
    407 	}
    408 }
    409 
    410 static void
    411 lwp_dtor(void *arg, void *obj)
    412 {
    413 	lwp_t *l = obj;
    414 
    415 	/*
    416 	 * The value of l->l_cpu must still be valid at this point.
    417 	 */
    418 	KASSERT(l->l_cpu != NULL);
    419 
    420 	/*
    421 	 * We can't return turnstile0 to the pool (it didn't come from it),
    422 	 * so if it comes up just drop it quietly and move on.
    423 	 */
    424 	if (l->l_ts != &turnstile0)
    425 		kmem_free(l->l_ts, sizeof(*l->l_ts));
    426 }
    427 
    428 /*
    429  * Set an LWP suspended.
    430  *
    431  * Must be called with p_lock held, and the LWP locked.  Will unlock the
    432  * LWP before return.
    433  */
    434 int
    435 lwp_suspend(struct lwp *curl, struct lwp *t)
    436 {
    437 	int error;
    438 
    439 	KASSERT(mutex_owned(t->l_proc->p_lock));
    440 	KASSERT(lwp_locked(t, NULL));
    441 
    442 	KASSERT(curl != t || curl->l_stat == LSONPROC);
    443 
    444 	/*
    445 	 * If the current LWP has been told to exit, we must not suspend anyone
    446 	 * else or deadlock could occur.  We won't return to userspace.
    447 	 */
    448 	if ((curl->l_flag & (LW_WEXIT | LW_WCORE)) != 0) {
    449 		lwp_unlock(t);
    450 		return (EDEADLK);
    451 	}
    452 
    453 	if ((t->l_flag & LW_DBGSUSPEND) != 0) {
    454 		lwp_unlock(t);
    455 		return 0;
    456 	}
    457 
    458 	error = 0;
    459 
    460 	switch (t->l_stat) {
    461 	case LSRUN:
    462 	case LSONPROC:
    463 		t->l_flag |= LW_WSUSPEND;
    464 		lwp_need_userret(t);
    465 		lwp_unlock(t);
    466 		break;
    467 
    468 	case LSSLEEP:
    469 		t->l_flag |= LW_WSUSPEND;
    470 		lwp_need_userret(t);
    471 
    472 		/*
    473 		 * Kick the LWP and try to get it to the kernel boundary
    474 		 * so that it will release any locks that it holds.
    475 		 * setrunnable() will release the lock.
    476 		 */
    477 		if ((t->l_flag & LW_SINTR) != 0)
    478 			setrunnable(t);
    479 		else
    480 			lwp_unlock(t);
    481 		break;
    482 
    483 	case LSSUSPENDED:
    484 		lwp_unlock(t);
    485 		break;
    486 
    487 	case LSSTOP:
    488 		t->l_flag |= LW_WSUSPEND;
    489 		lwp_need_userret(t);
    490 		setrunnable(t);
    491 		break;
    492 
    493 	case LSIDL:
    494 	case LSZOMB:
    495 		error = EINTR; /* It's what Solaris does..... */
    496 		lwp_unlock(t);
    497 		break;
    498 	}
    499 
    500 	return (error);
    501 }
    502 
    503 /*
    504  * Restart a suspended LWP.
    505  *
    506  * Must be called with p_lock held, and the LWP locked.  Will unlock the
    507  * LWP before return.
    508  */
    509 void
    510 lwp_continue(struct lwp *l)
    511 {
    512 
    513 	KASSERT(mutex_owned(l->l_proc->p_lock));
    514 	KASSERT(lwp_locked(l, NULL));
    515 
    516 	/* If rebooting or not suspended, then just bail out. */
    517 	if ((l->l_flag & LW_WREBOOT) != 0) {
    518 		lwp_unlock(l);
    519 		return;
    520 	}
    521 
    522 	l->l_flag &= ~LW_WSUSPEND;
    523 
    524 	if (l->l_stat != LSSUSPENDED || (l->l_flag & LW_DBGSUSPEND) != 0) {
    525 		lwp_unlock(l);
    526 		return;
    527 	}
    528 
    529 	/* setrunnable() will release the lock. */
    530 	setrunnable(l);
    531 }
    532 
    533 /*
    534  * Restart a stopped LWP.
    535  *
    536  * Must be called with p_lock held, and the LWP NOT locked.  Will unlock the
    537  * LWP before return.
    538  */
    539 void
    540 lwp_unstop(struct lwp *l)
    541 {
    542 	struct proc *p = l->l_proc;
    543 
    544 	KASSERT(mutex_owned(&proc_lock));
    545 	KASSERT(mutex_owned(p->p_lock));
    546 
    547 	lwp_lock(l);
    548 
    549 	KASSERT((l->l_flag & LW_DBGSUSPEND) == 0);
    550 
    551 	/* If not stopped, then just bail out. */
    552 	if (l->l_stat != LSSTOP) {
    553 		lwp_unlock(l);
    554 		return;
    555 	}
    556 
    557 	p->p_stat = SACTIVE;
    558 	p->p_sflag &= ~PS_STOPPING;
    559 
    560 	if (!p->p_waited)
    561 		p->p_pptr->p_nstopchild--;
    562 
    563 	if (l->l_wchan == NULL) {
    564 		/* setrunnable() will release the lock. */
    565 		setrunnable(l);
    566 	} else if (p->p_xsig && (l->l_flag & LW_SINTR) != 0) {
    567 		/* setrunnable() so we can receive the signal */
    568 		setrunnable(l);
    569 	} else {
    570 		l->l_stat = LSSLEEP;
    571 		p->p_nrlwps++;
    572 		lwp_unlock(l);
    573 	}
    574 }
    575 
    576 /*
    577  * Wait for an LWP within the current process to exit.  If 'lid' is
    578  * non-zero, we are waiting for a specific LWP.
    579  *
    580  * Must be called with p->p_lock held.
    581  */
    582 int
    583 lwp_wait(struct lwp *l, lwpid_t lid, lwpid_t *departed, bool exiting)
    584 {
    585 	const lwpid_t curlid = l->l_lid;
    586 	proc_t *p = l->l_proc;
    587 	lwp_t *l2, *next;
    588 	int error;
    589 
    590 	KASSERT(mutex_owned(p->p_lock));
    591 
    592 	p->p_nlwpwait++;
    593 	l->l_waitingfor = lid;
    594 
    595 	for (;;) {
    596 		int nfound;
    597 
    598 		/*
    599 		 * Avoid a race between exit1() and sigexit(): if the
    600 		 * process is dumping core, then we need to bail out: call
    601 		 * into lwp_userret() where we will be suspended until the
    602 		 * deed is done.
    603 		 */
    604 		if ((p->p_sflag & PS_WCORE) != 0) {
    605 			mutex_exit(p->p_lock);
    606 			lwp_userret(l);
    607 			KASSERT(false);
    608 		}
    609 
    610 		/*
    611 		 * First off, drain any detached LWP that is waiting to be
    612 		 * reaped.
    613 		 */
    614 		if ((l2 = p->p_zomblwp) != NULL) {
    615 			p->p_zomblwp = NULL;
    616 			lwp_free(l2, false, false);/* releases proc mutex */
    617 			mutex_enter(p->p_lock);
    618 			continue;
    619 		}
    620 
    621 		/*
    622 		 * Now look for an LWP to collect.  If the whole process is
    623 		 * exiting, count detached LWPs as eligible to be collected,
    624 		 * but don't drain them here.
    625 		 */
    626 		nfound = 0;
    627 		error = 0;
    628 
    629 		/*
    630 		 * If given a specific LID, go via pid_table and make sure
    631 		 * it's not detached.
    632 		 */
    633 		if (lid != 0) {
    634 			l2 = proc_find_lwp(p, lid);
    635 			if (l2 == NULL) {
    636 				error = ESRCH;
    637 				break;
    638 			}
    639 			KASSERT(l2->l_lid == lid);
    640 			if ((l2->l_prflag & LPR_DETACHED) != 0) {
    641 				error = EINVAL;
    642 				break;
    643 			}
    644 		} else {
    645 			l2 = LIST_FIRST(&p->p_lwps);
    646 		}
    647 		for (; l2 != NULL; l2 = next) {
    648 			next = (lid != 0 ? NULL : LIST_NEXT(l2, l_sibling));
    649 
    650 			/*
    651 			 * If a specific wait and the target is waiting on
    652 			 * us, then avoid deadlock.  This also traps LWPs
    653 			 * that try to wait on themselves.
    654 			 *
    655 			 * Note that this does not handle more complicated
    656 			 * cycles, like: t1 -> t2 -> t3 -> t1.  The process
    657 			 * can still be killed so it is not a major problem.
    658 			 */
    659 			if (l2->l_lid == lid && l2->l_waitingfor == curlid) {
    660 				error = EDEADLK;
    661 				break;
    662 			}
    663 			if (l2 == l)
    664 				continue;
    665 			if ((l2->l_prflag & LPR_DETACHED) != 0) {
    666 				nfound += exiting;
    667 				continue;
    668 			}
    669 			if (lid != 0) {
    670 				/*
    671 				 * Mark this LWP as the first waiter, if there
    672 				 * is no other.
    673 				 */
    674 				if (l2->l_waiter == 0)
    675 					l2->l_waiter = curlid;
    676 			} else if (l2->l_waiter != 0) {
    677 				/*
    678 				 * It already has a waiter - so don't
    679 				 * collect it.  If the waiter doesn't
    680 				 * grab it we'll get another chance
    681 				 * later.
    682 				 */
    683 				nfound++;
    684 				continue;
    685 			}
    686 			nfound++;
    687 
    688 			/* No need to lock the LWP in order to see LSZOMB. */
    689 			if (l2->l_stat != LSZOMB)
    690 				continue;
    691 
    692 			/*
    693 			 * We're no longer waiting.  Reset the "first waiter"
    694 			 * pointer on the target, in case it was us.
    695 			 */
    696 			l->l_waitingfor = 0;
    697 			l2->l_waiter = 0;
    698 			p->p_nlwpwait--;
    699 			if (departed)
    700 				*departed = l2->l_lid;
    701 			sched_lwp_collect(l2);
    702 
    703 			/* lwp_free() releases the proc lock. */
    704 			lwp_free(l2, false, false);
    705 			mutex_enter(p->p_lock);
    706 			return 0;
    707 		}
    708 
    709 		if (error != 0)
    710 			break;
    711 		if (nfound == 0) {
    712 			error = ESRCH;
    713 			break;
    714 		}
    715 
    716 		/*
    717 		 * Note: since the lock will be dropped, need to restart on
    718 		 * wakeup to run all LWPs again, e.g. there may be new LWPs.
    719 		 */
    720 		if (exiting) {
    721 			KASSERT(p->p_nlwps > 1);
    722 			error = cv_timedwait(&p->p_lwpcv, p->p_lock, 1);
    723 			break;
    724 		}
    725 
    726 		/*
    727 		 * Break out if all LWPs are in _lwp_wait().  There are
    728 		 * other ways to hang the process with _lwp_wait(), but the
    729 		 * sleep is interruptable so little point checking for them.
    730 		 */
    731 		if (p->p_nlwpwait == p->p_nlwps) {
    732 			error = EDEADLK;
    733 			break;
    734 		}
    735 
    736 		/*
    737 		 * Sit around and wait for something to happen.  We'll be
    738 		 * awoken if any of the conditions examined change: if an
    739 		 * LWP exits, is collected, or is detached.
    740 		 */
    741 		if ((error = cv_wait_sig(&p->p_lwpcv, p->p_lock)) != 0)
    742 			break;
    743 	}
    744 
    745 	/*
    746 	 * We didn't find any LWPs to collect, we may have received a
    747 	 * signal, or some other condition has caused us to bail out.
    748 	 *
    749 	 * If waiting on a specific LWP, clear the waiters marker: some
    750 	 * other LWP may want it.  Then, kick all the remaining waiters
    751 	 * so that they can re-check for zombies and for deadlock.
    752 	 */
    753 	if (lid != 0) {
    754 		l2 = proc_find_lwp(p, lid);
    755 		KASSERT(l2 == NULL || l2->l_lid == lid);
    756 
    757 		if (l2 != NULL && l2->l_waiter == curlid)
    758 			l2->l_waiter = 0;
    759 	}
    760 	p->p_nlwpwait--;
    761 	l->l_waitingfor = 0;
    762 	cv_broadcast(&p->p_lwpcv);
    763 
    764 	return error;
    765 }
    766 
    767 /*
    768  * Create a new LWP within process 'p2', using LWP 'l1' as a template.
    769  * The new LWP is created in state LSIDL and must be set running,
    770  * suspended, or stopped by the caller.
    771  */
    772 int
    773 lwp_create(lwp_t *l1, proc_t *p2, vaddr_t uaddr, int flags,
    774     void *stack, size_t stacksize, void (*func)(void *), void *arg,
    775     lwp_t **rnewlwpp, int sclass, const sigset_t *sigmask,
    776     const stack_t *sigstk)
    777 {
    778 	struct lwp *l2;
    779 
    780 	KASSERT(l1 == curlwp || l1->l_proc == &proc0);
    781 
    782 	/*
    783 	 * Enforce limits, excluding the first lwp and kthreads.  We must
    784 	 * use the process credentials here when adjusting the limit, as
    785 	 * they are what's tied to the accounting entity.  However for
    786 	 * authorizing the action, we'll use the LWP's credentials.
    787 	 */
    788 	mutex_enter(p2->p_lock);
    789 	if (p2->p_nlwps != 0 && p2 != &proc0) {
    790 		uid_t uid = kauth_cred_getuid(p2->p_cred);
    791 		int count = chglwpcnt(uid, 1);
    792 		if (__predict_false(count >
    793 		    p2->p_rlimit[RLIMIT_NTHR].rlim_cur)) {
    794 			if (kauth_authorize_process(l1->l_cred,
    795 			    KAUTH_PROCESS_RLIMIT, p2,
    796 			    KAUTH_ARG(KAUTH_REQ_PROCESS_RLIMIT_BYPASS),
    797 			    &p2->p_rlimit[RLIMIT_NTHR], KAUTH_ARG(RLIMIT_NTHR))
    798 			    != 0) {
    799 				(void)chglwpcnt(uid, -1);
    800 				mutex_exit(p2->p_lock);
    801 				return EAGAIN;
    802 			}
    803 		}
    804 	}
    805 
    806 	/*
    807 	 * First off, reap any detached LWP waiting to be collected.
    808 	 * We can re-use its LWP structure and turnstile.
    809 	 */
    810 	if ((l2 = p2->p_zomblwp) != NULL) {
    811 		p2->p_zomblwp = NULL;
    812 		lwp_free(l2, true, false);
    813 		/* p2 now unlocked by lwp_free() */
    814 		KASSERT(l2->l_ts != NULL);
    815 		KASSERT(l2->l_inheritedprio == -1);
    816 		KASSERT(SLIST_EMPTY(&l2->l_pi_lenders));
    817 		memset(&l2->l_startzero, 0, sizeof(*l2) -
    818 		    offsetof(lwp_t, l_startzero));
    819 	} else {
    820 		mutex_exit(p2->p_lock);
    821 		l2 = pool_cache_get(lwp_cache, PR_WAITOK);
    822 		memset(&l2->l_startzero, 0, sizeof(*l2) -
    823 		    offsetof(lwp_t, l_startzero));
    824 		SLIST_INIT(&l2->l_pi_lenders);
    825 	}
    826 
    827 	/*
    828 	 * Because of lockless lookup via pid_table, the LWP can be locked
    829 	 * and inspected briefly even after it's freed, so a few fields are
    830 	 * kept stable.
    831 	 */
    832 	KASSERT(l2->l_stat == LSIDL);
    833 	KASSERT(l2->l_cpu != NULL);
    834 	KASSERT(l2->l_ts != NULL);
    835 	KASSERT(l2->l_mutex == l2->l_cpu->ci_schedstate.spc_lwplock);
    836 
    837 	l2->l_proc = p2;
    838 	l2->l_refcnt = 0;
    839 	l2->l_class = sclass;
    840 
    841 	/*
    842 	 * Allocate a process ID for this LWP.  We need to do this now
    843 	 * while we can still unwind if it fails.  Because we're marked
    844 	 * as LSIDL, no lookups by the ID will succeed.
    845 	 *
    846 	 * N.B. this will always succeed for the first LWP in a process,
    847 	 * because proc_alloc_lwpid() will usurp the slot.  Also note
    848 	 * that l2->l_proc MUST be valid so that lookups of the proc
    849 	 * will succeed, even if the LWP itself is not visible.
    850 	 */
    851 	if (__predict_false(proc_alloc_lwpid(p2, l2) == -1)) {
    852 		pool_cache_put(lwp_cache, l2);
    853 		return EAGAIN;
    854 	}
    855 
    856 	/*
    857 	 * If vfork(), we want the LWP to run fast and on the same CPU
    858 	 * as its parent, so that it can reuse the VM context and cache
    859 	 * footprint on the local CPU.
    860 	 */
    861 	l2->l_boostpri = ((flags & LWP_VFORK) ? PRI_KERNEL : PRI_USER);
    862  	l2->l_priority = l1->l_priority;
    863 	l2->l_inheritedprio = -1;
    864 	l2->l_protectprio = -1;
    865 	l2->l_auxprio = -1;
    866 	l2->l_flag = 0;
    867 	l2->l_pflag = LP_MPSAFE;
    868 	TAILQ_INIT(&l2->l_ld_locks);
    869 	l2->l_psrefs = 0;
    870 	kmsan_lwp_alloc(l2);
    871 
    872 	/*
    873 	 * For vfork, borrow parent's lwpctl context if it exists.
    874 	 * This also causes us to return via lwp_userret.
    875 	 */
    876 	if (flags & LWP_VFORK && l1->l_lwpctl) {
    877 		l2->l_lwpctl = l1->l_lwpctl;
    878 		l2->l_flag |= LW_LWPCTL;
    879 	}
    880 
    881 	/*
    882 	 * If not the first LWP in the process, grab a reference to the
    883 	 * descriptor table.
    884 	 */
    885 	l2->l_fd = p2->p_fd;
    886 	if (p2->p_nlwps != 0) {
    887 		KASSERT(l1->l_proc == p2);
    888 		fd_hold(l2);
    889 	} else {
    890 		KASSERT(l1->l_proc != p2);
    891 	}
    892 
    893 	if (p2->p_flag & PK_SYSTEM) {
    894 		/* Mark it as a system LWP. */
    895 		l2->l_flag |= LW_SYSTEM;
    896 	}
    897 
    898 	kdtrace_thread_ctor(NULL, l2);
    899 	lwp_initspecific(l2);
    900 	sched_lwp_fork(l1, l2);
    901 	callout_init(&l2->l_timeout_ch, CALLOUT_MPSAFE);
    902 	callout_setfunc(&l2->l_timeout_ch, sleepq_timeout, l2);
    903 	cv_init(&l2->l_sigcv, "sigwait");
    904 	cv_init(&l2->l_waitcv, "vfork");
    905 	l2->l_syncobj = &sched_syncobj;
    906 	PSREF_DEBUG_INIT_LWP(l2);
    907 
    908 	if (rnewlwpp != NULL)
    909 		*rnewlwpp = l2;
    910 
    911 	/*
    912 	 * PCU state needs to be saved before calling uvm_lwp_fork() so that
    913 	 * the MD cpu_lwp_fork() can copy the saved state to the new LWP.
    914 	 */
    915 	pcu_save_all(l1);
    916 #if PCU_UNIT_COUNT > 0
    917 	l2->l_pcu_valid = l1->l_pcu_valid;
    918 #endif
    919 
    920 	uvm_lwp_setuarea(l2, uaddr);
    921 	uvm_lwp_fork(l1, l2, stack, stacksize, func, (arg != NULL) ? arg : l2);
    922 
    923 	mutex_enter(p2->p_lock);
    924 	l2->l_cred = kauth_cred_hold(p2->p_cred);
    925 	if ((flags & LWP_DETACHED) != 0) {
    926 		l2->l_prflag = LPR_DETACHED;
    927 		p2->p_ndlwps++;
    928 	} else
    929 		l2->l_prflag = 0;
    930 
    931 	if (l1->l_proc == p2) {
    932 		/*
    933 		 * These flags are set while p_lock is held.  Copy with
    934 		 * p_lock held too, so the LWP doesn't sneak into the
    935 		 * process without them being set.
    936 		 */
    937 		l2->l_flag |= (l1->l_flag & (LW_WEXIT | LW_WREBOOT | LW_WCORE));
    938 	} else {
    939 		/* fork(): pending core/exit doesn't apply to child. */
    940 		l2->l_flag |= (l1->l_flag & LW_WREBOOT);
    941 	}
    942 
    943 	l2->l_sigstk = *sigstk;
    944 	l2->l_sigmask = *sigmask;
    945 	TAILQ_INIT(&l2->l_sigpend.sp_info);
    946 	sigemptyset(&l2->l_sigpend.sp_set);
    947 	LIST_INSERT_HEAD(&p2->p_lwps, l2, l_sibling);
    948 	p2->p_nlwps++;
    949 	p2->p_nrlwps++;
    950 
    951 	KASSERT(l2->l_affinity == NULL);
    952 
    953 	/* Inherit the affinity mask. */
    954 	if (l1->l_affinity) {
    955 		/*
    956 		 * Note that we hold the state lock while inheriting
    957 		 * the affinity to avoid race with sched_setaffinity().
    958 		 */
    959 		lwp_lock(l1);
    960 		if (l1->l_affinity) {
    961 			kcpuset_use(l1->l_affinity);
    962 			l2->l_affinity = l1->l_affinity;
    963 		}
    964 		lwp_unlock(l1);
    965 	}
    966 
    967 	/* Ensure a trip through lwp_userret() if needed. */
    968 	if ((l2->l_flag & LW_USERRET) != 0) {
    969 		lwp_need_userret(l2);
    970 	}
    971 
    972 	/* This marks the end of the "must be atomic" section. */
    973 	mutex_exit(p2->p_lock);
    974 
    975 	SDT_PROBE(proc, kernel, , lwp__create, l2, 0, 0, 0, 0);
    976 
    977 	mutex_enter(&proc_lock);
    978 	LIST_INSERT_HEAD(&alllwp, l2, l_list);
    979 	/* Inherit a processor-set */
    980 	l2->l_psid = l1->l_psid;
    981 	mutex_exit(&proc_lock);
    982 
    983 	SYSCALL_TIME_LWP_INIT(l2);
    984 
    985 	if (p2->p_emul->e_lwp_fork)
    986 		(*p2->p_emul->e_lwp_fork)(l1, l2);
    987 
    988 	return (0);
    989 }
    990 
    991 /*
    992  * Set a new LWP running.  If the process is stopping, then the LWP is
    993  * created stopped.
    994  */
    995 void
    996 lwp_start(lwp_t *l, int flags)
    997 {
    998 	proc_t *p = l->l_proc;
    999 
   1000 	mutex_enter(p->p_lock);
   1001 	lwp_lock(l);
   1002 	KASSERT(l->l_stat == LSIDL);
   1003 	if ((flags & LWP_SUSPENDED) != 0) {
   1004 		/* It'll suspend itself in lwp_userret(). */
   1005 		l->l_flag |= LW_WSUSPEND;
   1006 		lwp_need_userret(l);
   1007 	}
   1008 	if (p->p_stat == SSTOP || (p->p_sflag & PS_STOPPING) != 0) {
   1009 		KASSERT(l->l_wchan == NULL);
   1010 	    	l->l_stat = LSSTOP;
   1011 		p->p_nrlwps--;
   1012 		lwp_unlock(l);
   1013 	} else {
   1014 		setrunnable(l);
   1015 		/* LWP now unlocked */
   1016 	}
   1017 	mutex_exit(p->p_lock);
   1018 }
   1019 
   1020 /*
   1021  * Called by MD code when a new LWP begins execution.  Must be called
   1022  * with the previous LWP locked (so at splsched), or if there is no
   1023  * previous LWP, at splsched.
   1024  */
   1025 void
   1026 lwp_startup(struct lwp *prev, struct lwp *new_lwp)
   1027 {
   1028 	kmutex_t *lock;
   1029 
   1030 	KASSERTMSG(new_lwp == curlwp, "l %p curlwp %p prevlwp %p", new_lwp, curlwp, prev);
   1031 	KASSERT(kpreempt_disabled());
   1032 	KASSERT(prev != NULL);
   1033 	KASSERT((prev->l_pflag & LP_RUNNING) != 0);
   1034 	KASSERT(curcpu()->ci_mtx_count == -2);
   1035 
   1036 	/*
   1037 	 * Immediately mark the previous LWP as no longer running and
   1038 	 * unlock (to keep lock wait times short as possible).  If a
   1039 	 * zombie, don't touch after clearing LP_RUNNING as it could be
   1040 	 * reaped by another CPU.  Use atomic_store_release to ensure
   1041 	 * this -- matches atomic_load_acquire in lwp_free.
   1042 	 */
   1043 	lock = prev->l_mutex;
   1044 	if (__predict_false(prev->l_stat == LSZOMB)) {
   1045 		atomic_store_release(&prev->l_pflag,
   1046 		    prev->l_pflag & ~LP_RUNNING);
   1047 	} else {
   1048 		prev->l_pflag &= ~LP_RUNNING;
   1049 	}
   1050 	mutex_spin_exit(lock);
   1051 
   1052 	/* Correct spin mutex count after mi_switch(). */
   1053 	curcpu()->ci_mtx_count = 0;
   1054 
   1055 	/* Install new VM context. */
   1056 	if (__predict_true(new_lwp->l_proc->p_vmspace)) {
   1057 		pmap_activate(new_lwp);
   1058 	}
   1059 
   1060 	/* We remain at IPL_SCHED from mi_switch() - reset it. */
   1061 	spl0();
   1062 
   1063 	LOCKDEBUG_BARRIER(NULL, 0);
   1064 	SDT_PROBE(proc, kernel, , lwp__start, new_lwp, 0, 0, 0, 0);
   1065 
   1066 	/* For kthreads, acquire kernel lock if not MPSAFE. */
   1067 	if (__predict_false((new_lwp->l_pflag & LP_MPSAFE) == 0)) {
   1068 		KERNEL_LOCK(1, new_lwp);
   1069 	}
   1070 }
   1071 
   1072 /*
   1073  * Exit an LWP.
   1074  *
   1075  * *** WARNING *** This can be called with (l != curlwp) in error paths.
   1076  */
   1077 void
   1078 lwp_exit(struct lwp *l)
   1079 {
   1080 	struct proc *p = l->l_proc;
   1081 	struct lwp *l2;
   1082 	bool current;
   1083 
   1084 	current = (l == curlwp);
   1085 
   1086 	KASSERT(current || l->l_stat == LSIDL);
   1087 	KASSERT(current || l->l_target_cpu == NULL);
   1088 	KASSERT(p == curproc);
   1089 
   1090 	SDT_PROBE(proc, kernel, , lwp__exit, l, 0, 0, 0, 0);
   1091 
   1092 	/* Verify that we hold no locks; for DIAGNOSTIC check kernel_lock. */
   1093 	LOCKDEBUG_BARRIER(NULL, 0);
   1094 	KASSERTMSG(curcpu()->ci_biglock_count == 0, "kernel_lock leaked");
   1095 
   1096 	/*
   1097 	 * If we are the last live LWP in a process, we need to exit the
   1098 	 * entire process.  We do so with an exit status of zero, because
   1099 	 * it's a "controlled" exit, and because that's what Solaris does.
   1100 	 *
   1101 	 * We are not quite a zombie yet, but for accounting purposes we
   1102 	 * must increment the count of zombies here.
   1103 	 *
   1104 	 * Note: the last LWP's specificdata will be deleted here.
   1105 	 */
   1106 	mutex_enter(p->p_lock);
   1107 	if (p->p_nlwps - p->p_nzlwps == 1) {
   1108 		KASSERT(current == true);
   1109 		KASSERT(p != &proc0);
   1110 		exit1(l, 0, 0);
   1111 		/* NOTREACHED */
   1112 	}
   1113 	p->p_nzlwps++;
   1114 
   1115 	/*
   1116 	 * Perform any required thread cleanup.  Do this early so
   1117 	 * anyone wanting to look us up with lwp_getref_lwpid() will
   1118 	 * fail to find us before we become a zombie.
   1119 	 *
   1120 	 * N.B. this will unlock p->p_lock on our behalf.
   1121 	 */
   1122 	lwp_thread_cleanup(l);
   1123 
   1124 	if (p->p_emul->e_lwp_exit)
   1125 		(*p->p_emul->e_lwp_exit)(l);
   1126 
   1127 	/* Drop filedesc reference. */
   1128 	fd_free();
   1129 
   1130 	/* Release fstrans private data. */
   1131 	fstrans_lwp_dtor(l);
   1132 
   1133 	/* Delete the specificdata while it's still safe to sleep. */
   1134 	lwp_finispecific(l);
   1135 
   1136 	/*
   1137 	 * Release our cached credentials.
   1138 	 */
   1139 	kauth_cred_free(l->l_cred);
   1140 	callout_destroy(&l->l_timeout_ch);
   1141 
   1142 	/*
   1143 	 * If traced, report LWP exit event to the debugger.
   1144 	 *
   1145 	 * Remove the LWP from the global list.
   1146 	 * Free its LID from the PID namespace if needed.
   1147 	 */
   1148 	mutex_enter(&proc_lock);
   1149 
   1150 	if ((p->p_slflag & (PSL_TRACED|PSL_TRACELWP_EXIT)) ==
   1151 	    (PSL_TRACED|PSL_TRACELWP_EXIT)) {
   1152 		mutex_enter(p->p_lock);
   1153 		if (ISSET(p->p_sflag, PS_WEXIT)) {
   1154 			mutex_exit(p->p_lock);
   1155 			/*
   1156 			 * We are exiting, bail out without informing parent
   1157 			 * about a terminating LWP as it would deadlock.
   1158 			 */
   1159 		} else {
   1160 			eventswitch(TRAP_LWP, PTRACE_LWP_EXIT, l->l_lid);
   1161 			mutex_enter(&proc_lock);
   1162 		}
   1163 	}
   1164 
   1165 	LIST_REMOVE(l, l_list);
   1166 	mutex_exit(&proc_lock);
   1167 
   1168 	/*
   1169 	 * Get rid of all references to the LWP that others (e.g. procfs)
   1170 	 * may have, and mark the LWP as a zombie.  If the LWP is detached,
   1171 	 * mark it waiting for collection in the proc structure.  Note that
   1172 	 * before we can do that, we need to free any other dead, deatched
   1173 	 * LWP waiting to meet its maker.
   1174 	 *
   1175 	 * All conditions need to be observed upon under the same hold of
   1176 	 * p_lock, because if the lock is dropped any of them can change.
   1177 	 */
   1178 	mutex_enter(p->p_lock);
   1179 	for (;;) {
   1180 		if (lwp_drainrefs(l))
   1181 			continue;
   1182 		if ((l->l_prflag & LPR_DETACHED) != 0) {
   1183 			if ((l2 = p->p_zomblwp) != NULL) {
   1184 				p->p_zomblwp = NULL;
   1185 				lwp_free(l2, false, false);
   1186 				/* proc now unlocked */
   1187 				mutex_enter(p->p_lock);
   1188 				continue;
   1189 			}
   1190 			p->p_zomblwp = l;
   1191 		}
   1192 		break;
   1193 	}
   1194 
   1195 	/*
   1196 	 * If we find a pending signal for the process and we have been
   1197 	 * asked to check for signals, then we lose: arrange to have
   1198 	 * all other LWPs in the process check for signals.
   1199 	 */
   1200 	if ((l->l_flag & LW_PENDSIG) != 0 &&
   1201 	    firstsig(&p->p_sigpend.sp_set) != 0) {
   1202 		LIST_FOREACH(l2, &p->p_lwps, l_sibling) {
   1203 			lwp_lock(l2);
   1204 			signotify(l2);
   1205 			lwp_unlock(l2);
   1206 		}
   1207 	}
   1208 
   1209 	/*
   1210 	 * Release any PCU resources before becoming a zombie.
   1211 	 */
   1212 	pcu_discard_all(l);
   1213 
   1214 	lwp_lock(l);
   1215 	l->l_stat = LSZOMB;
   1216 	if (l->l_name != NULL) {
   1217 		strcpy(l->l_name, "(zombie)");
   1218 	}
   1219 	lwp_unlock(l);
   1220 	p->p_nrlwps--;
   1221 	if (l->l_lwpctl != NULL)
   1222 		l->l_lwpctl->lc_curcpu = LWPCTL_CPU_EXITED;
   1223 	mutex_exit(p->p_lock);
   1224 	cv_broadcast(&p->p_lwpcv);
   1225 
   1226 	/*
   1227 	 * We can no longer block.  At this point, lwp_free() may already
   1228 	 * be gunning for us.  On a multi-CPU system, we may be off p_lwps.
   1229 	 *
   1230 	 * Free MD LWP resources.
   1231 	 */
   1232 	cpu_lwp_free(l, 0);
   1233 
   1234 	if (current) {
   1235 		/* Switch away into oblivion. */
   1236 		lwp_lock(l);
   1237 		spc_lock(l->l_cpu);
   1238 		mi_switch(l);
   1239 		panic("lwp_exit");
   1240 	}
   1241 }
   1242 
   1243 /*
   1244  * Free a dead LWP's remaining resources.
   1245  *
   1246  * XXXLWP limits.
   1247  */
   1248 void
   1249 lwp_free(struct lwp *l, bool recycle, bool last)
   1250 {
   1251 	struct proc *p = l->l_proc;
   1252 	struct rusage *ru;
   1253 	ksiginfoq_t kq;
   1254 
   1255 	KASSERT(l != curlwp);
   1256 	KASSERT(last || mutex_owned(p->p_lock));
   1257 
   1258 	/*
   1259 	 * We use the process credentials instead of the lwp credentials here
   1260 	 * because the lwp credentials maybe cached (just after a setuid call)
   1261 	 * and we don't want pay for syncing, since the lwp is going away
   1262 	 * anyway
   1263 	 */
   1264 	if (p != &proc0 && p->p_nlwps != 1)
   1265 		(void)chglwpcnt(kauth_cred_getuid(p->p_cred), -1);
   1266 
   1267 	/*
   1268 	 * In the unlikely event that the LWP is still on the CPU,
   1269 	 * then spin until it has switched away.
   1270 	 *
   1271 	 * atomic_load_acquire matches atomic_store_release in
   1272 	 * lwp_startup and mi_switch.
   1273 	 */
   1274 	while (__predict_false((atomic_load_acquire(&l->l_pflag) & LP_RUNNING)
   1275 		!= 0)) {
   1276 		SPINLOCK_BACKOFF_HOOK;
   1277 	}
   1278 
   1279 	/*
   1280 	 * Now that the LWP's known off the CPU, reset its state back to
   1281 	 * LSIDL, which defeats anything that might have gotten a hold on
   1282 	 * the LWP via pid_table before the ID was freed.  It's important
   1283 	 * to do this with both the LWP locked and p_lock held.
   1284 	 *
   1285 	 * Also reset the CPU and lock pointer back to curcpu(), since the
   1286 	 * LWP will in all likelyhood be cached with the current CPU in
   1287 	 * lwp_cache when we free it and later allocated from there again
   1288 	 * (avoid incidental lock contention).
   1289 	 */
   1290 	lwp_lock(l);
   1291 	l->l_stat = LSIDL;
   1292 	l->l_cpu = curcpu();
   1293 	lwp_unlock_to(l, l->l_cpu->ci_schedstate.spc_lwplock);
   1294 
   1295 	/*
   1296 	 * If this was not the last LWP in the process, then adjust counters
   1297 	 * and unlock.  This is done differently for the last LWP in exit1().
   1298 	 */
   1299 	if (!last) {
   1300 		/*
   1301 		 * Add the LWP's run time to the process' base value.
   1302 		 * This needs to co-incide with coming off p_lwps.
   1303 		 */
   1304 		bintime_add(&p->p_rtime, &l->l_rtime);
   1305 		p->p_pctcpu += l->l_pctcpu;
   1306 		ru = &p->p_stats->p_ru;
   1307 		ruadd(ru, &l->l_ru);
   1308 		LIST_REMOVE(l, l_sibling);
   1309 		p->p_nlwps--;
   1310 		p->p_nzlwps--;
   1311 		if ((l->l_prflag & LPR_DETACHED) != 0)
   1312 			p->p_ndlwps--;
   1313 		mutex_exit(p->p_lock);
   1314 
   1315 		/*
   1316 		 * Have any LWPs sleeping in lwp_wait() recheck for
   1317 		 * deadlock.
   1318 		 */
   1319 		cv_broadcast(&p->p_lwpcv);
   1320 
   1321 		/* Free the LWP ID. */
   1322 		mutex_enter(&proc_lock);
   1323 		proc_free_lwpid(p, l->l_lid);
   1324 		mutex_exit(&proc_lock);
   1325 	}
   1326 
   1327 	/*
   1328 	 * Destroy the LWP's remaining signal information.
   1329 	 */
   1330 	ksiginfo_queue_init(&kq);
   1331 	sigclear(&l->l_sigpend, NULL, &kq);
   1332 	ksiginfo_queue_drain(&kq);
   1333 	cv_destroy(&l->l_sigcv);
   1334 	cv_destroy(&l->l_waitcv);
   1335 
   1336 	/*
   1337 	 * Free lwpctl structure and affinity.
   1338 	 */
   1339 	if (l->l_lwpctl) {
   1340 		lwp_ctl_free(l);
   1341 	}
   1342 	if (l->l_affinity) {
   1343 		kcpuset_unuse(l->l_affinity, NULL);
   1344 		l->l_affinity = NULL;
   1345 	}
   1346 
   1347 	/*
   1348 	 * Free remaining data structures and the LWP itself unless the
   1349 	 * caller wants to recycle.
   1350 	 */
   1351 	if (l->l_name != NULL)
   1352 		kmem_free(l->l_name, MAXCOMLEN);
   1353 
   1354 	kmsan_lwp_free(l);
   1355 	kcov_lwp_free(l);
   1356 	cpu_lwp_free2(l);
   1357 	uvm_lwp_exit(l);
   1358 
   1359 	KASSERT(SLIST_EMPTY(&l->l_pi_lenders));
   1360 	KASSERT(l->l_inheritedprio == -1);
   1361 	KASSERT(l->l_blcnt == 0);
   1362 	kdtrace_thread_dtor(NULL, l);
   1363 	if (!recycle)
   1364 		pool_cache_put(lwp_cache, l);
   1365 }
   1366 
   1367 /*
   1368  * Migrate the LWP to the another CPU.  Unlocks the LWP.
   1369  */
   1370 void
   1371 lwp_migrate(lwp_t *l, struct cpu_info *tci)
   1372 {
   1373 	struct schedstate_percpu *tspc;
   1374 	int lstat = l->l_stat;
   1375 
   1376 	KASSERT(lwp_locked(l, NULL));
   1377 	KASSERT(tci != NULL);
   1378 
   1379 	/* If LWP is still on the CPU, it must be handled like LSONPROC */
   1380 	if ((l->l_pflag & LP_RUNNING) != 0) {
   1381 		lstat = LSONPROC;
   1382 	}
   1383 
   1384 	/*
   1385 	 * The destination CPU could be changed while previous migration
   1386 	 * was not finished.
   1387 	 */
   1388 	if (l->l_target_cpu != NULL) {
   1389 		l->l_target_cpu = tci;
   1390 		lwp_unlock(l);
   1391 		return;
   1392 	}
   1393 
   1394 	/* Nothing to do if trying to migrate to the same CPU */
   1395 	if (l->l_cpu == tci) {
   1396 		lwp_unlock(l);
   1397 		return;
   1398 	}
   1399 
   1400 	KASSERT(l->l_target_cpu == NULL);
   1401 	tspc = &tci->ci_schedstate;
   1402 	switch (lstat) {
   1403 	case LSRUN:
   1404 		l->l_target_cpu = tci;
   1405 		break;
   1406 	case LSSLEEP:
   1407 		l->l_cpu = tci;
   1408 		break;
   1409 	case LSIDL:
   1410 	case LSSTOP:
   1411 	case LSSUSPENDED:
   1412 		l->l_cpu = tci;
   1413 		if (l->l_wchan == NULL) {
   1414 			lwp_unlock_to(l, tspc->spc_lwplock);
   1415 			return;
   1416 		}
   1417 		break;
   1418 	case LSONPROC:
   1419 		l->l_target_cpu = tci;
   1420 		spc_lock(l->l_cpu);
   1421 		sched_resched_cpu(l->l_cpu, PRI_USER_RT, true);
   1422 		/* spc now unlocked */
   1423 		break;
   1424 	}
   1425 	lwp_unlock(l);
   1426 }
   1427 
   1428 #define	lwp_find_exclude(l)					\
   1429 	((l)->l_stat == LSIDL || (l)->l_stat == LSZOMB)
   1430 
   1431 /*
   1432  * Find the LWP in the process.  Arguments may be zero, in such case,
   1433  * the calling process and first LWP in the list will be used.
   1434  * On success - returns proc locked.
   1435  *
   1436  * => pid == 0 -> look in curproc.
   1437  * => pid == -1 -> match any proc.
   1438  * => otherwise look up the proc.
   1439  *
   1440  * => lid == 0 -> first LWP in the proc
   1441  * => otherwise specific LWP
   1442  */
   1443 struct lwp *
   1444 lwp_find2(pid_t pid, lwpid_t lid)
   1445 {
   1446 	proc_t *p;
   1447 	lwp_t *l;
   1448 
   1449 	/* First LWP of specified proc. */
   1450 	if (lid == 0) {
   1451 		switch (pid) {
   1452 		case -1:
   1453 			/* No lookup keys. */
   1454 			return NULL;
   1455 		case 0:
   1456 			p = curproc;
   1457 			mutex_enter(p->p_lock);
   1458 			break;
   1459 		default:
   1460 			mutex_enter(&proc_lock);
   1461 			p = proc_find(pid);
   1462 			if (__predict_false(p == NULL)) {
   1463 				mutex_exit(&proc_lock);
   1464 				return NULL;
   1465 			}
   1466 			mutex_enter(p->p_lock);
   1467 			mutex_exit(&proc_lock);
   1468 			break;
   1469 		}
   1470 		LIST_FOREACH(l, &p->p_lwps, l_sibling) {
   1471 			if (__predict_true(!lwp_find_exclude(l)))
   1472 				break;
   1473 		}
   1474 		goto out;
   1475 	}
   1476 
   1477 	l = proc_find_lwp_acquire_proc(lid, &p);
   1478 	if (l == NULL)
   1479 		return NULL;
   1480 	KASSERT(p != NULL);
   1481 	KASSERT(mutex_owned(p->p_lock));
   1482 
   1483 	if (__predict_false(lwp_find_exclude(l))) {
   1484 		l = NULL;
   1485 		goto out;
   1486 	}
   1487 
   1488 	/* Apply proc filter, if applicable. */
   1489 	switch (pid) {
   1490 	case -1:
   1491 		/* Match anything. */
   1492 		break;
   1493 	case 0:
   1494 		if (p != curproc)
   1495 			l = NULL;
   1496 		break;
   1497 	default:
   1498 		if (p->p_pid != pid)
   1499 			l = NULL;
   1500 		break;
   1501 	}
   1502 
   1503  out:
   1504 	if (__predict_false(l == NULL)) {
   1505 		mutex_exit(p->p_lock);
   1506 	}
   1507 	return l;
   1508 }
   1509 
   1510 /*
   1511  * Look up a live LWP within the specified process.
   1512  *
   1513  * Must be called with p->p_lock held (as it looks at the radix tree,
   1514  * and also wants to exclude idle and zombie LWPs).
   1515  */
   1516 struct lwp *
   1517 lwp_find(struct proc *p, lwpid_t id)
   1518 {
   1519 	struct lwp *l;
   1520 
   1521 	KASSERT(mutex_owned(p->p_lock));
   1522 
   1523 	l = proc_find_lwp(p, id);
   1524 	KASSERT(l == NULL || l->l_lid == id);
   1525 
   1526 	/*
   1527 	 * No need to lock - all of these conditions will
   1528 	 * be visible with the process level mutex held.
   1529 	 */
   1530 	if (__predict_false(l != NULL && lwp_find_exclude(l)))
   1531 		l = NULL;
   1532 
   1533 	return l;
   1534 }
   1535 
   1536 /*
   1537  * Update an LWP's cached credentials to mirror the process' master copy.
   1538  *
   1539  * This happens early in the syscall path, on user trap, and on LWP
   1540  * creation.  A long-running LWP can also voluntarily choose to update
   1541  * its credentials by calling this routine.  This may be called from
   1542  * LWP_CACHE_CREDS(), which checks l->l_prflag & LPR_CRMOD beforehand.
   1543  */
   1544 void
   1545 lwp_update_creds(struct lwp *l)
   1546 {
   1547 	kauth_cred_t oc;
   1548 	struct proc *p;
   1549 
   1550 	p = l->l_proc;
   1551 	oc = l->l_cred;
   1552 
   1553 	mutex_enter(p->p_lock);
   1554 	kauth_cred_hold(p->p_cred);
   1555 	l->l_cred = p->p_cred;
   1556 	l->l_prflag &= ~LPR_CRMOD;
   1557 	mutex_exit(p->p_lock);
   1558 	if (oc != NULL)
   1559 		kauth_cred_free(oc);
   1560 }
   1561 
   1562 /*
   1563  * Verify that an LWP is locked, and optionally verify that the lock matches
   1564  * one we specify.
   1565  */
   1566 int
   1567 lwp_locked(struct lwp *l, kmutex_t *mtx)
   1568 {
   1569 	kmutex_t *cur = l->l_mutex;
   1570 
   1571 	return mutex_owned(cur) && (mtx == cur || mtx == NULL);
   1572 }
   1573 
   1574 /*
   1575  * Lend a new mutex to an LWP.  The old mutex must be held.
   1576  */
   1577 kmutex_t *
   1578 lwp_setlock(struct lwp *l, kmutex_t *mtx)
   1579 {
   1580 	kmutex_t *oldmtx = l->l_mutex;
   1581 
   1582 	KASSERT(mutex_owned(oldmtx));
   1583 
   1584 	atomic_store_release(&l->l_mutex, mtx);
   1585 	return oldmtx;
   1586 }
   1587 
   1588 /*
   1589  * Lend a new mutex to an LWP, and release the old mutex.  The old mutex
   1590  * must be held.
   1591  */
   1592 void
   1593 lwp_unlock_to(struct lwp *l, kmutex_t *mtx)
   1594 {
   1595 	kmutex_t *old;
   1596 
   1597 	KASSERT(lwp_locked(l, NULL));
   1598 
   1599 	old = l->l_mutex;
   1600 	atomic_store_release(&l->l_mutex, mtx);
   1601 	mutex_spin_exit(old);
   1602 }
   1603 
   1604 int
   1605 lwp_trylock(struct lwp *l)
   1606 {
   1607 	kmutex_t *old;
   1608 
   1609 	for (;;) {
   1610 		if (!mutex_tryenter(old = atomic_load_consume(&l->l_mutex)))
   1611 			return 0;
   1612 		if (__predict_true(atomic_load_relaxed(&l->l_mutex) == old))
   1613 			return 1;
   1614 		mutex_spin_exit(old);
   1615 	}
   1616 }
   1617 
   1618 void
   1619 lwp_unsleep(lwp_t *l, bool unlock)
   1620 {
   1621 
   1622 	KASSERT(mutex_owned(l->l_mutex));
   1623 	(*l->l_syncobj->sobj_unsleep)(l, unlock);
   1624 }
   1625 
   1626 /*
   1627  * Lock an LWP.
   1628  */
   1629 void
   1630 lwp_lock(lwp_t *l)
   1631 {
   1632 	kmutex_t *old = atomic_load_consume(&l->l_mutex);
   1633 
   1634 	/*
   1635 	 * Note: mutex_spin_enter() will have posted a read barrier.
   1636 	 * Re-test l->l_mutex.  If it has changed, we need to try again.
   1637 	 */
   1638 	mutex_spin_enter(old);
   1639 	while (__predict_false(atomic_load_relaxed(&l->l_mutex) != old)) {
   1640 		mutex_spin_exit(old);
   1641 		old = atomic_load_consume(&l->l_mutex);
   1642 		mutex_spin_enter(old);
   1643 	}
   1644 }
   1645 
   1646 /*
   1647  * Unlock an LWP.
   1648  */
   1649 void
   1650 lwp_unlock(lwp_t *l)
   1651 {
   1652 
   1653 	mutex_spin_exit(l->l_mutex);
   1654 }
   1655 
   1656 void
   1657 lwp_changepri(lwp_t *l, pri_t pri)
   1658 {
   1659 
   1660 	KASSERT(mutex_owned(l->l_mutex));
   1661 
   1662 	if (l->l_priority == pri)
   1663 		return;
   1664 
   1665 	(*l->l_syncobj->sobj_changepri)(l, pri);
   1666 	KASSERT(l->l_priority == pri);
   1667 }
   1668 
   1669 void
   1670 lwp_lendpri(lwp_t *l, pri_t pri)
   1671 {
   1672 	KASSERT(mutex_owned(l->l_mutex));
   1673 
   1674 	(*l->l_syncobj->sobj_lendpri)(l, pri);
   1675 	KASSERT(l->l_inheritedprio == pri);
   1676 }
   1677 
   1678 pri_t
   1679 lwp_eprio(lwp_t *l)
   1680 {
   1681 	pri_t pri = l->l_priority;
   1682 
   1683 	KASSERT(mutex_owned(l->l_mutex));
   1684 
   1685 	/*
   1686 	 * Timeshared/user LWPs get a temporary priority boost for blocking
   1687 	 * in kernel.  This is key to good interactive response on a loaded
   1688 	 * system: without it, things will seem very sluggish to the user.
   1689 	 *
   1690 	 * The function of the boost is to get the LWP onto a CPU and
   1691 	 * running quickly.  Once that happens the LWP loses the priority
   1692 	 * boost and could be preempted very quickly by another LWP but that
   1693 	 * won't happen often enough to be a annoyance.
   1694 	 */
   1695 	if (pri <= MAXPRI_USER && l->l_boostpri > MAXPRI_USER)
   1696 		pri = (pri >> 1) + l->l_boostpri;
   1697 
   1698 	return MAX(l->l_auxprio, pri);
   1699 }
   1700 
   1701 /*
   1702  * Handle exceptions for mi_userret().  Called if a member of LW_USERRET is
   1703  * set or a preemption is required.
   1704  */
   1705 void
   1706 lwp_userret(struct lwp *l)
   1707 {
   1708 	struct proc *p;
   1709 	int sig, f;
   1710 
   1711 	KASSERT(l == curlwp);
   1712 	KASSERT(l->l_stat == LSONPROC);
   1713 	p = l->l_proc;
   1714 
   1715 	for (;;) {
   1716 		/*
   1717 		 * This is the main location that user preemptions are
   1718 		 * processed.
   1719 		 */
   1720 		preempt_point();
   1721 
   1722 		/*
   1723 		 * It is safe to do this unlocked and without raised SPL,
   1724 		 * since whenever a flag of interest is added to l_flag the
   1725 		 * LWP will take an AST and come down this path again.  If a
   1726 		 * remote CPU posts the AST, it will be done with an IPI
   1727 		 * (strongly synchronising).
   1728 		 */
   1729 		if ((f = atomic_load_relaxed(&l->l_flag) & LW_USERRET) == 0) {
   1730 			return;
   1731 		}
   1732 
   1733 		/*
   1734 		 * Process pending signals first, unless the process
   1735 		 * is dumping core or exiting, where we will instead
   1736 		 * enter the LW_WSUSPEND case below.
   1737 		 */
   1738 		if ((f & (LW_PENDSIG | LW_WCORE | LW_WEXIT)) == LW_PENDSIG) {
   1739 			mutex_enter(p->p_lock);
   1740 			while ((sig = issignal(l)) != 0)
   1741 				postsig(sig);
   1742 			mutex_exit(p->p_lock);
   1743 			continue;
   1744 		}
   1745 
   1746 		/*
   1747 		 * Core-dump or suspend pending.
   1748 		 *
   1749 		 * In case of core dump, suspend ourselves, so that the kernel
   1750 		 * stack and therefore the userland registers saved in the
   1751 		 * trapframe are around for coredump() to write them out.
   1752 		 * We also need to save any PCU resources that we have so that
   1753 		 * they accessible for coredump().  We issue a wakeup on
   1754 		 * p->p_lwpcv so that sigexit() will write the core file out
   1755 		 * once all other LWPs are suspended.
   1756 		 */
   1757 		if ((f & LW_WSUSPEND) != 0) {
   1758 			pcu_save_all(l);
   1759 			mutex_enter(p->p_lock);
   1760 			p->p_nrlwps--;
   1761 			lwp_lock(l);
   1762 			l->l_stat = LSSUSPENDED;
   1763 			lwp_unlock(l);
   1764 			mutex_exit(p->p_lock);
   1765 			cv_broadcast(&p->p_lwpcv);
   1766 			lwp_lock(l);
   1767 			spc_lock(l->l_cpu);
   1768 			mi_switch(l);
   1769 			continue;
   1770 		}
   1771 
   1772 		/*
   1773 		 * Process is exiting.  The core dump and signal cases must
   1774 		 * be handled first.
   1775 		 */
   1776 		if ((f & LW_WEXIT) != 0) {
   1777 			lwp_exit(l);
   1778 			KASSERT(0);
   1779 			/* NOTREACHED */
   1780 		}
   1781 
   1782 		/*
   1783 		 * Update lwpctl processor (for vfork child_return).
   1784 		 */
   1785 		if ((f & LW_LWPCTL) != 0) {
   1786 			lwp_lock(l);
   1787 			KASSERT(kpreempt_disabled());
   1788 			l->l_lwpctl->lc_curcpu = (int)cpu_index(l->l_cpu);
   1789 			l->l_lwpctl->lc_pctr++;
   1790 			l->l_flag &= ~LW_LWPCTL;
   1791 			lwp_unlock(l);
   1792 			continue;
   1793 		}
   1794 	}
   1795 }
   1796 
   1797 /*
   1798  * Force an LWP to enter the kernel, to take a trip through lwp_userret().
   1799  */
   1800 void
   1801 lwp_need_userret(struct lwp *l)
   1802 {
   1803 
   1804 	KASSERT(!cpu_intr_p());
   1805 	KASSERT(lwp_locked(l, NULL) || l->l_stat == LSIDL);
   1806 
   1807 	/*
   1808 	 * If the LWP is in any state other than LSONPROC, we know that it
   1809 	 * is executing in-kernel and will hit userret() on the way out.
   1810 	 *
   1811 	 * If the LWP is curlwp, then we know we'll be back out to userspace
   1812 	 * soon (can't be called from a hardware interrupt here).
   1813 	 *
   1814 	 * Otherwise, we can't be sure what the LWP is doing, so first make
   1815 	 * sure the update to l_flag will be globally visible, and then
   1816 	 * force the LWP to take a trip through trap() where it will do
   1817 	 * userret().
   1818 	 */
   1819 	if (l->l_stat == LSONPROC && l != curlwp) {
   1820 		membar_producer();
   1821 		cpu_signotify(l);
   1822 	}
   1823 }
   1824 
   1825 /*
   1826  * Add one reference to an LWP.  This will prevent the LWP from
   1827  * exiting, thus keep the lwp structure and PCB around to inspect.
   1828  */
   1829 void
   1830 lwp_addref(struct lwp *l)
   1831 {
   1832 	KASSERT(mutex_owned(l->l_proc->p_lock));
   1833 	KASSERT(l->l_stat != LSZOMB);
   1834 	l->l_refcnt++;
   1835 }
   1836 
   1837 /*
   1838  * Remove one reference to an LWP.  If this is the last reference,
   1839  * then we must finalize the LWP's death.
   1840  */
   1841 void
   1842 lwp_delref(struct lwp *l)
   1843 {
   1844 	struct proc *p = l->l_proc;
   1845 
   1846 	mutex_enter(p->p_lock);
   1847 	lwp_delref2(l);
   1848 	mutex_exit(p->p_lock);
   1849 }
   1850 
   1851 /*
   1852  * Remove one reference to an LWP.  If this is the last reference,
   1853  * then we must finalize the LWP's death.  The proc mutex is held
   1854  * on entry.
   1855  */
   1856 void
   1857 lwp_delref2(struct lwp *l)
   1858 {
   1859 	struct proc *p = l->l_proc;
   1860 
   1861 	KASSERT(mutex_owned(p->p_lock));
   1862 	KASSERT(l->l_stat != LSZOMB);
   1863 	KASSERT(l->l_refcnt > 0);
   1864 
   1865 	if (--l->l_refcnt == 0)
   1866 		cv_broadcast(&p->p_lwpcv);
   1867 }
   1868 
   1869 /*
   1870  * Drain all references to the current LWP.  Returns true if
   1871  * we blocked.
   1872  */
   1873 bool
   1874 lwp_drainrefs(struct lwp *l)
   1875 {
   1876 	struct proc *p = l->l_proc;
   1877 	bool rv = false;
   1878 
   1879 	KASSERT(mutex_owned(p->p_lock));
   1880 
   1881 	l->l_prflag |= LPR_DRAINING;
   1882 
   1883 	while (l->l_refcnt > 0) {
   1884 		rv = true;
   1885 		cv_wait(&p->p_lwpcv, p->p_lock);
   1886 	}
   1887 	return rv;
   1888 }
   1889 
   1890 /*
   1891  * Return true if the specified LWP is 'alive'.  Only p->p_lock need
   1892  * be held.
   1893  */
   1894 bool
   1895 lwp_alive(lwp_t *l)
   1896 {
   1897 
   1898 	KASSERT(mutex_owned(l->l_proc->p_lock));
   1899 
   1900 	switch (l->l_stat) {
   1901 	case LSSLEEP:
   1902 	case LSRUN:
   1903 	case LSONPROC:
   1904 	case LSSTOP:
   1905 	case LSSUSPENDED:
   1906 		return true;
   1907 	default:
   1908 		return false;
   1909 	}
   1910 }
   1911 
   1912 /*
   1913  * Return first live LWP in the process.
   1914  */
   1915 lwp_t *
   1916 lwp_find_first(proc_t *p)
   1917 {
   1918 	lwp_t *l;
   1919 
   1920 	KASSERT(mutex_owned(p->p_lock));
   1921 
   1922 	LIST_FOREACH(l, &p->p_lwps, l_sibling) {
   1923 		if (lwp_alive(l)) {
   1924 			return l;
   1925 		}
   1926 	}
   1927 
   1928 	return NULL;
   1929 }
   1930 
   1931 /*
   1932  * Allocate a new lwpctl structure for a user LWP.
   1933  */
   1934 int
   1935 lwp_ctl_alloc(vaddr_t *uaddr)
   1936 {
   1937 	lcproc_t *lp;
   1938 	u_int bit, i, offset;
   1939 	struct uvm_object *uao;
   1940 	int error;
   1941 	lcpage_t *lcp;
   1942 	proc_t *p;
   1943 	lwp_t *l;
   1944 
   1945 	l = curlwp;
   1946 	p = l->l_proc;
   1947 
   1948 	/* don't allow a vforked process to create lwp ctls */
   1949 	if (p->p_lflag & PL_PPWAIT)
   1950 		return EBUSY;
   1951 
   1952 	if (l->l_lcpage != NULL) {
   1953 		lcp = l->l_lcpage;
   1954 		*uaddr = lcp->lcp_uaddr + (vaddr_t)l->l_lwpctl - lcp->lcp_kaddr;
   1955 		return 0;
   1956 	}
   1957 
   1958 	/* First time around, allocate header structure for the process. */
   1959 	if ((lp = p->p_lwpctl) == NULL) {
   1960 		lp = kmem_alloc(sizeof(*lp), KM_SLEEP);
   1961 		mutex_init(&lp->lp_lock, MUTEX_DEFAULT, IPL_NONE);
   1962 		lp->lp_uao = NULL;
   1963 		TAILQ_INIT(&lp->lp_pages);
   1964 		mutex_enter(p->p_lock);
   1965 		if (p->p_lwpctl == NULL) {
   1966 			p->p_lwpctl = lp;
   1967 			mutex_exit(p->p_lock);
   1968 		} else {
   1969 			mutex_exit(p->p_lock);
   1970 			mutex_destroy(&lp->lp_lock);
   1971 			kmem_free(lp, sizeof(*lp));
   1972 			lp = p->p_lwpctl;
   1973 		}
   1974 	}
   1975 
   1976  	/*
   1977  	 * Set up an anonymous memory region to hold the shared pages.
   1978  	 * Map them into the process' address space.  The user vmspace
   1979  	 * gets the first reference on the UAO.
   1980  	 */
   1981 	mutex_enter(&lp->lp_lock);
   1982 	if (lp->lp_uao == NULL) {
   1983 		lp->lp_uao = uao_create(LWPCTL_UAREA_SZ, 0);
   1984 		lp->lp_cur = 0;
   1985 		lp->lp_max = LWPCTL_UAREA_SZ;
   1986 		lp->lp_uva = p->p_emul->e_vm_default_addr(p,
   1987 		     (vaddr_t)p->p_vmspace->vm_daddr, LWPCTL_UAREA_SZ,
   1988 		     p->p_vmspace->vm_map.flags & VM_MAP_TOPDOWN);
   1989 		error = uvm_map(&p->p_vmspace->vm_map, &lp->lp_uva,
   1990 		    LWPCTL_UAREA_SZ, lp->lp_uao, 0, 0, UVM_MAPFLAG(UVM_PROT_RW,
   1991 		    UVM_PROT_RW, UVM_INH_NONE, UVM_ADV_NORMAL, 0));
   1992 		if (error != 0) {
   1993 			uao_detach(lp->lp_uao);
   1994 			lp->lp_uao = NULL;
   1995 			mutex_exit(&lp->lp_lock);
   1996 			return error;
   1997 		}
   1998 	}
   1999 
   2000 	/* Get a free block and allocate for this LWP. */
   2001 	TAILQ_FOREACH(lcp, &lp->lp_pages, lcp_chain) {
   2002 		if (lcp->lcp_nfree != 0)
   2003 			break;
   2004 	}
   2005 	if (lcp == NULL) {
   2006 		/* Nothing available - try to set up a free page. */
   2007 		if (lp->lp_cur == lp->lp_max) {
   2008 			mutex_exit(&lp->lp_lock);
   2009 			return ENOMEM;
   2010 		}
   2011 		lcp = kmem_alloc(LWPCTL_LCPAGE_SZ, KM_SLEEP);
   2012 
   2013 		/*
   2014 		 * Wire the next page down in kernel space.  Since this
   2015 		 * is a new mapping, we must add a reference.
   2016 		 */
   2017 		uao = lp->lp_uao;
   2018 		(*uao->pgops->pgo_reference)(uao);
   2019 		lcp->lcp_kaddr = vm_map_min(kernel_map);
   2020 		error = uvm_map(kernel_map, &lcp->lcp_kaddr, PAGE_SIZE,
   2021 		    uao, lp->lp_cur, PAGE_SIZE,
   2022 		    UVM_MAPFLAG(UVM_PROT_RW, UVM_PROT_RW,
   2023 		    UVM_INH_NONE, UVM_ADV_RANDOM, 0));
   2024 		if (error != 0) {
   2025 			mutex_exit(&lp->lp_lock);
   2026 			kmem_free(lcp, LWPCTL_LCPAGE_SZ);
   2027 			(*uao->pgops->pgo_detach)(uao);
   2028 			return error;
   2029 		}
   2030 		error = uvm_map_pageable(kernel_map, lcp->lcp_kaddr,
   2031 		    lcp->lcp_kaddr + PAGE_SIZE, FALSE, 0);
   2032 		if (error != 0) {
   2033 			mutex_exit(&lp->lp_lock);
   2034 			uvm_unmap(kernel_map, lcp->lcp_kaddr,
   2035 			    lcp->lcp_kaddr + PAGE_SIZE);
   2036 			kmem_free(lcp, LWPCTL_LCPAGE_SZ);
   2037 			return error;
   2038 		}
   2039 		/* Prepare the page descriptor and link into the list. */
   2040 		lcp->lcp_uaddr = lp->lp_uva + lp->lp_cur;
   2041 		lp->lp_cur += PAGE_SIZE;
   2042 		lcp->lcp_nfree = LWPCTL_PER_PAGE;
   2043 		lcp->lcp_rotor = 0;
   2044 		memset(lcp->lcp_bitmap, 0xff, LWPCTL_BITMAP_SZ);
   2045 		TAILQ_INSERT_HEAD(&lp->lp_pages, lcp, lcp_chain);
   2046 	}
   2047 	for (i = lcp->lcp_rotor; lcp->lcp_bitmap[i] == 0;) {
   2048 		if (++i >= LWPCTL_BITMAP_ENTRIES)
   2049 			i = 0;
   2050 	}
   2051 	bit = ffs(lcp->lcp_bitmap[i]) - 1;
   2052 	lcp->lcp_bitmap[i] ^= (1U << bit);
   2053 	lcp->lcp_rotor = i;
   2054 	lcp->lcp_nfree--;
   2055 	l->l_lcpage = lcp;
   2056 	offset = (i << 5) + bit;
   2057 	l->l_lwpctl = (lwpctl_t *)lcp->lcp_kaddr + offset;
   2058 	*uaddr = lcp->lcp_uaddr + offset * sizeof(lwpctl_t);
   2059 	mutex_exit(&lp->lp_lock);
   2060 
   2061 	KPREEMPT_DISABLE(l);
   2062 	l->l_lwpctl->lc_curcpu = (int)cpu_index(curcpu());
   2063 	KPREEMPT_ENABLE(l);
   2064 
   2065 	return 0;
   2066 }
   2067 
   2068 /*
   2069  * Free an lwpctl structure back to the per-process list.
   2070  */
   2071 void
   2072 lwp_ctl_free(lwp_t *l)
   2073 {
   2074 	struct proc *p = l->l_proc;
   2075 	lcproc_t *lp;
   2076 	lcpage_t *lcp;
   2077 	u_int map, offset;
   2078 
   2079 	/* don't free a lwp context we borrowed for vfork */
   2080 	if (p->p_lflag & PL_PPWAIT) {
   2081 		l->l_lwpctl = NULL;
   2082 		return;
   2083 	}
   2084 
   2085 	lp = p->p_lwpctl;
   2086 	KASSERT(lp != NULL);
   2087 
   2088 	lcp = l->l_lcpage;
   2089 	offset = (u_int)((lwpctl_t *)l->l_lwpctl - (lwpctl_t *)lcp->lcp_kaddr);
   2090 	KASSERT(offset < LWPCTL_PER_PAGE);
   2091 
   2092 	mutex_enter(&lp->lp_lock);
   2093 	lcp->lcp_nfree++;
   2094 	map = offset >> 5;
   2095 	lcp->lcp_bitmap[map] |= (1U << (offset & 31));
   2096 	if (lcp->lcp_bitmap[lcp->lcp_rotor] == 0)
   2097 		lcp->lcp_rotor = map;
   2098 	if (TAILQ_FIRST(&lp->lp_pages)->lcp_nfree == 0) {
   2099 		TAILQ_REMOVE(&lp->lp_pages, lcp, lcp_chain);
   2100 		TAILQ_INSERT_HEAD(&lp->lp_pages, lcp, lcp_chain);
   2101 	}
   2102 	mutex_exit(&lp->lp_lock);
   2103 }
   2104 
   2105 /*
   2106  * Process is exiting; tear down lwpctl state.  This can only be safely
   2107  * called by the last LWP in the process.
   2108  */
   2109 void
   2110 lwp_ctl_exit(void)
   2111 {
   2112 	lcpage_t *lcp, *next;
   2113 	lcproc_t *lp;
   2114 	proc_t *p;
   2115 	lwp_t *l;
   2116 
   2117 	l = curlwp;
   2118 	l->l_lwpctl = NULL;
   2119 	l->l_lcpage = NULL;
   2120 	p = l->l_proc;
   2121 	lp = p->p_lwpctl;
   2122 
   2123 	KASSERT(lp != NULL);
   2124 	KASSERT(p->p_nlwps == 1);
   2125 
   2126 	for (lcp = TAILQ_FIRST(&lp->lp_pages); lcp != NULL; lcp = next) {
   2127 		next = TAILQ_NEXT(lcp, lcp_chain);
   2128 		uvm_unmap(kernel_map, lcp->lcp_kaddr,
   2129 		    lcp->lcp_kaddr + PAGE_SIZE);
   2130 		kmem_free(lcp, LWPCTL_LCPAGE_SZ);
   2131 	}
   2132 
   2133 	if (lp->lp_uao != NULL) {
   2134 		uvm_unmap(&p->p_vmspace->vm_map, lp->lp_uva,
   2135 		    lp->lp_uva + LWPCTL_UAREA_SZ);
   2136 	}
   2137 
   2138 	mutex_destroy(&lp->lp_lock);
   2139 	kmem_free(lp, sizeof(*lp));
   2140 	p->p_lwpctl = NULL;
   2141 }
   2142 
   2143 /*
   2144  * Return the current LWP's "preemption counter".  Used to detect
   2145  * preemption across operations that can tolerate preemption without
   2146  * crashing, but which may generate incorrect results if preempted.
   2147  */
   2148 long
   2149 lwp_pctr(void)
   2150 {
   2151 
   2152 	return curlwp->l_ru.ru_nvcsw + curlwp->l_ru.ru_nivcsw;
   2153 }
   2154 
   2155 /*
   2156  * Set an LWP's private data pointer.
   2157  */
   2158 int
   2159 lwp_setprivate(struct lwp *l, void *ptr)
   2160 {
   2161 	int error = 0;
   2162 
   2163 	l->l_private = ptr;
   2164 #ifdef __HAVE_CPU_LWP_SETPRIVATE
   2165 	error = cpu_lwp_setprivate(l, ptr);
   2166 #endif
   2167 	return error;
   2168 }
   2169 
   2170 /*
   2171  * Perform any thread-related cleanup on LWP exit.
   2172  * N.B. l->l_proc->p_lock must be HELD on entry but will
   2173  * be released before returning!
   2174  */
   2175 void
   2176 lwp_thread_cleanup(struct lwp *l)
   2177 {
   2178 
   2179 	KASSERT(mutex_owned(l->l_proc->p_lock));
   2180 	mutex_exit(l->l_proc->p_lock);
   2181 
   2182 	/*
   2183 	 * If the LWP has robust futexes, release them all
   2184 	 * now.
   2185 	 */
   2186 	if (__predict_false(l->l_robust_head != 0)) {
   2187 		futex_release_all_lwp(l);
   2188 	}
   2189 }
   2190 
   2191 #if defined(DDB)
   2192 #include <machine/pcb.h>
   2193 
   2194 void
   2195 lwp_whatis(uintptr_t addr, void (*pr)(const char *, ...))
   2196 {
   2197 	lwp_t *l;
   2198 
   2199 	LIST_FOREACH(l, &alllwp, l_list) {
   2200 		uintptr_t stack = (uintptr_t)KSTACK_LOWEST_ADDR(l);
   2201 
   2202 		if (addr < stack || stack + KSTACK_SIZE <= addr) {
   2203 			continue;
   2204 		}
   2205 		(*pr)("%p is %p+%zu, LWP %p's stack\n",
   2206 		    (void *)addr, (void *)stack,
   2207 		    (size_t)(addr - stack), l);
   2208 	}
   2209 }
   2210 #endif /* defined(DDB) */
   2211