Home | History | Annotate | Line # | Download | only in kern
kern_lwp.c revision 1.243.12.1
      1  1.243.12.1   thorpej /*	$NetBSD: kern_lwp.c,v 1.243.12.1 2021/11/03 14:49:21 thorpej Exp $	*/
      2         1.2   thorpej 
      3         1.2   thorpej /*-
      4       1.220        ad  * Copyright (c) 2001, 2006, 2007, 2008, 2009, 2019, 2020
      5       1.220        ad  *     The NetBSD Foundation, Inc.
      6         1.2   thorpej  * All rights reserved.
      7         1.2   thorpej  *
      8         1.2   thorpej  * This code is derived from software contributed to The NetBSD Foundation
      9        1.52        ad  * by Nathan J. Williams, and Andrew Doran.
     10         1.2   thorpej  *
     11         1.2   thorpej  * Redistribution and use in source and binary forms, with or without
     12         1.2   thorpej  * modification, are permitted provided that the following conditions
     13         1.2   thorpej  * are met:
     14         1.2   thorpej  * 1. Redistributions of source code must retain the above copyright
     15         1.2   thorpej  *    notice, this list of conditions and the following disclaimer.
     16         1.2   thorpej  * 2. Redistributions in binary form must reproduce the above copyright
     17         1.2   thorpej  *    notice, this list of conditions and the following disclaimer in the
     18         1.2   thorpej  *    documentation and/or other materials provided with the distribution.
     19         1.2   thorpej  *
     20         1.2   thorpej  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     21         1.2   thorpej  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     22         1.2   thorpej  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     23         1.2   thorpej  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     24         1.2   thorpej  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     25         1.2   thorpej  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     26         1.2   thorpej  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     27         1.2   thorpej  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     28         1.2   thorpej  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     29         1.2   thorpej  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     30         1.2   thorpej  * POSSIBILITY OF SUCH DAMAGE.
     31         1.2   thorpej  */
     32         1.9     lukem 
     33        1.52        ad /*
     34        1.52        ad  * Overview
     35        1.52        ad  *
     36        1.66        ad  *	Lightweight processes (LWPs) are the basic unit or thread of
     37        1.52        ad  *	execution within the kernel.  The core state of an LWP is described
     38        1.66        ad  *	by "struct lwp", also known as lwp_t.
     39        1.52        ad  *
     40        1.52        ad  *	Each LWP is contained within a process (described by "struct proc"),
     41        1.52        ad  *	Every process contains at least one LWP, but may contain more.  The
     42        1.52        ad  *	process describes attributes shared among all of its LWPs such as a
     43        1.52        ad  *	private address space, global execution state (stopped, active,
     44        1.52        ad  *	zombie, ...), signal disposition and so on.  On a multiprocessor
     45        1.66        ad  *	machine, multiple LWPs be executing concurrently in the kernel.
     46        1.52        ad  *
     47        1.52        ad  * Execution states
     48        1.52        ad  *
     49        1.52        ad  *	At any given time, an LWP has overall state that is described by
     50        1.52        ad  *	lwp::l_stat.  The states are broken into two sets below.  The first
     51        1.52        ad  *	set is guaranteed to represent the absolute, current state of the
     52        1.52        ad  *	LWP:
     53       1.101     rmind  *
     54       1.101     rmind  *	LSONPROC
     55       1.101     rmind  *
     56       1.101     rmind  *		On processor: the LWP is executing on a CPU, either in the
     57       1.101     rmind  *		kernel or in user space.
     58       1.101     rmind  *
     59       1.101     rmind  *	LSRUN
     60       1.101     rmind  *
     61       1.101     rmind  *		Runnable: the LWP is parked on a run queue, and may soon be
     62       1.101     rmind  *		chosen to run by an idle processor, or by a processor that
     63       1.101     rmind  *		has been asked to preempt a currently runnning but lower
     64       1.134     rmind  *		priority LWP.
     65       1.101     rmind  *
     66       1.101     rmind  *	LSIDL
     67       1.101     rmind  *
     68       1.238        ad  *		Idle: the LWP has been created but has not yet executed, or
     69       1.238        ad  *		it has ceased executing a unit of work and is waiting to be
     70       1.238        ad  *		started again.  This state exists so that the LWP can occupy
     71       1.238        ad  *		a slot in the process & PID table, but without having to
     72       1.238        ad  *		worry about being touched; lookups of the LWP by ID will
     73       1.238        ad  *		fail while in this state.  The LWP will become visible for
     74       1.238        ad  *		lookup once its state transitions further.  Some special
     75       1.238        ad  *		kernel threads also (ab)use this state to indicate that they
     76       1.238        ad  *		are idle (soft interrupts and idle LWPs).
     77       1.101     rmind  *
     78       1.101     rmind  *	LSSUSPENDED:
     79       1.101     rmind  *
     80       1.101     rmind  *		Suspended: the LWP has had its execution suspended by
     81        1.52        ad  *		another LWP in the same process using the _lwp_suspend()
     82        1.52        ad  *		system call.  User-level LWPs also enter the suspended
     83        1.52        ad  *		state when the system is shutting down.
     84        1.52        ad  *
     85        1.52        ad  *	The second set represent a "statement of intent" on behalf of the
     86        1.52        ad  *	LWP.  The LWP may in fact be executing on a processor, may be
     87        1.66        ad  *	sleeping or idle. It is expected to take the necessary action to
     88       1.101     rmind  *	stop executing or become "running" again within a short timeframe.
     89       1.227        ad  *	The LP_RUNNING flag in lwp::l_pflag indicates that an LWP is running.
     90       1.101     rmind  *	Importantly, it indicates that its state is tied to a CPU.
     91       1.101     rmind  *
     92       1.101     rmind  *	LSZOMB:
     93       1.101     rmind  *
     94       1.101     rmind  *		Dead or dying: the LWP has released most of its resources
     95       1.129        ad  *		and is about to switch away into oblivion, or has already
     96        1.66        ad  *		switched away.  When it switches away, its few remaining
     97        1.66        ad  *		resources can be collected.
     98       1.101     rmind  *
     99       1.101     rmind  *	LSSLEEP:
    100       1.101     rmind  *
    101       1.101     rmind  *		Sleeping: the LWP has entered itself onto a sleep queue, and
    102       1.101     rmind  *		has switched away or will switch away shortly to allow other
    103        1.66        ad  *		LWPs to run on the CPU.
    104       1.101     rmind  *
    105       1.101     rmind  *	LSSTOP:
    106       1.101     rmind  *
    107       1.101     rmind  *		Stopped: the LWP has been stopped as a result of a job
    108       1.101     rmind  *		control signal, or as a result of the ptrace() interface.
    109       1.101     rmind  *
    110       1.101     rmind  *		Stopped LWPs may run briefly within the kernel to handle
    111       1.101     rmind  *		signals that they receive, but will not return to user space
    112       1.101     rmind  *		until their process' state is changed away from stopped.
    113       1.101     rmind  *
    114       1.101     rmind  *		Single LWPs within a process can not be set stopped
    115       1.101     rmind  *		selectively: all actions that can stop or continue LWPs
    116       1.101     rmind  *		occur at the process level.
    117       1.101     rmind  *
    118        1.52        ad  * State transitions
    119        1.52        ad  *
    120        1.66        ad  *	Note that the LSSTOP state may only be set when returning to
    121        1.66        ad  *	user space in userret(), or when sleeping interruptably.  The
    122        1.66        ad  *	LSSUSPENDED state may only be set in userret().  Before setting
    123        1.66        ad  *	those states, we try to ensure that the LWPs will release all
    124        1.66        ad  *	locks that they hold, and at a minimum try to ensure that the
    125        1.66        ad  *	LWP can be set runnable again by a signal.
    126        1.52        ad  *
    127        1.52        ad  *	LWPs may transition states in the following ways:
    128        1.52        ad  *
    129        1.52        ad  *	 RUN -------> ONPROC		ONPROC -----> RUN
    130       1.129        ad  *		    				    > SLEEP
    131       1.129        ad  *		    				    > STOPPED
    132        1.52        ad  *						    > SUSPENDED
    133        1.52        ad  *						    > ZOMB
    134       1.129        ad  *						    > IDL (special cases)
    135        1.52        ad  *
    136        1.52        ad  *	 STOPPED ---> RUN		SUSPENDED --> RUN
    137       1.129        ad  *	            > SLEEP
    138        1.52        ad  *
    139        1.52        ad  *	 SLEEP -----> ONPROC		IDL --------> RUN
    140       1.101     rmind  *		    > RUN			    > SUSPENDED
    141       1.101     rmind  *		    > STOPPED			    > STOPPED
    142       1.129        ad  *						    > ONPROC (special cases)
    143        1.52        ad  *
    144       1.129        ad  *	Some state transitions are only possible with kernel threads (eg
    145       1.129        ad  *	ONPROC -> IDL) and happen under tightly controlled circumstances
    146       1.129        ad  *	free of unwanted side effects.
    147        1.66        ad  *
    148       1.114     rmind  * Migration
    149       1.114     rmind  *
    150       1.114     rmind  *	Migration of threads from one CPU to another could be performed
    151       1.114     rmind  *	internally by the scheduler via sched_takecpu() or sched_catchlwp()
    152       1.114     rmind  *	functions.  The universal lwp_migrate() function should be used for
    153       1.114     rmind  *	any other cases.  Subsystems in the kernel must be aware that CPU
    154       1.114     rmind  *	of LWP may change, while it is not locked.
    155       1.114     rmind  *
    156        1.52        ad  * Locking
    157        1.52        ad  *
    158        1.52        ad  *	The majority of fields in 'struct lwp' are covered by a single,
    159        1.66        ad  *	general spin lock pointed to by lwp::l_mutex.  The locks covering
    160        1.52        ad  *	each field are documented in sys/lwp.h.
    161        1.52        ad  *
    162        1.66        ad  *	State transitions must be made with the LWP's general lock held,
    163       1.152     rmind  *	and may cause the LWP's lock pointer to change.  Manipulation of
    164        1.66        ad  *	the general lock is not performed directly, but through calls to
    165       1.152     rmind  *	lwp_lock(), lwp_unlock() and others.  It should be noted that the
    166       1.152     rmind  *	adaptive locks are not allowed to be released while the LWP's lock
    167       1.152     rmind  *	is being held (unlike for other spin-locks).
    168        1.52        ad  *
    169        1.52        ad  *	States and their associated locks:
    170        1.52        ad  *
    171       1.212        ad  *	LSIDL, LSONPROC, LSZOMB, LSSUPENDED:
    172        1.52        ad  *
    173       1.212        ad  *		Always covered by spc_lwplock, which protects LWPs not
    174       1.212        ad  *		associated with any other sync object.  This is a per-CPU
    175       1.212        ad  *		lock and matches lwp::l_cpu.
    176        1.52        ad  *
    177       1.212        ad  *	LSRUN:
    178        1.52        ad  *
    179        1.64      yamt  *		Always covered by spc_mutex, which protects the run queues.
    180       1.129        ad  *		This is a per-CPU lock and matches lwp::l_cpu.
    181        1.52        ad  *
    182        1.52        ad  *	LSSLEEP:
    183        1.52        ad  *
    184       1.212        ad  *		Covered by a lock associated with the sleep queue (sometimes
    185       1.221        ad  *		a turnstile sleep queue) that the LWP resides on.  This can
    186       1.221        ad  *		be spc_lwplock for SOBJ_SLEEPQ_NULL (an "untracked" sleep).
    187        1.52        ad  *
    188       1.212        ad  *	LSSTOP:
    189       1.101     rmind  *
    190        1.52        ad  *		If the LWP was previously sleeping (l_wchan != NULL), then
    191        1.66        ad  *		l_mutex references the sleep queue lock.  If the LWP was
    192        1.52        ad  *		runnable or on the CPU when halted, or has been removed from
    193        1.66        ad  *		the sleep queue since halted, then the lock is spc_lwplock.
    194        1.52        ad  *
    195        1.52        ad  *	The lock order is as follows:
    196        1.52        ad  *
    197       1.212        ad  *		sleepq -> turnstile -> spc_lwplock -> spc_mutex
    198        1.52        ad  *
    199       1.243     skrll  *	Each process has a scheduler state lock (proc::p_lock), and a
    200        1.52        ad  *	number of counters on LWPs and their states: p_nzlwps, p_nrlwps, and
    201        1.52        ad  *	so on.  When an LWP is to be entered into or removed from one of the
    202       1.103        ad  *	following states, p_lock must be held and the process wide counters
    203        1.52        ad  *	adjusted:
    204        1.52        ad  *
    205        1.52        ad  *		LSIDL, LSZOMB, LSSTOP, LSSUSPENDED
    206        1.52        ad  *
    207       1.129        ad  *	(But not always for kernel threads.  There are some special cases
    208       1.212        ad  *	as mentioned above: soft interrupts, and the idle loops.)
    209       1.129        ad  *
    210        1.52        ad  *	Note that an LWP is considered running or likely to run soon if in
    211        1.52        ad  *	one of the following states.  This affects the value of p_nrlwps:
    212        1.52        ad  *
    213        1.52        ad  *		LSRUN, LSONPROC, LSSLEEP
    214        1.52        ad  *
    215       1.103        ad  *	p_lock does not need to be held when transitioning among these
    216       1.129        ad  *	three states, hence p_lock is rarely taken for state transitions.
    217        1.52        ad  */
    218        1.52        ad 
    219         1.9     lukem #include <sys/cdefs.h>
    220  1.243.12.1   thorpej __KERNEL_RCSID(0, "$NetBSD: kern_lwp.c,v 1.243.12.1 2021/11/03 14:49:21 thorpej Exp $");
    221         1.8    martin 
    222        1.84      yamt #include "opt_ddb.h"
    223        1.52        ad #include "opt_lockdebug.h"
    224       1.139    darran #include "opt_dtrace.h"
    225         1.2   thorpej 
    226        1.47   hannken #define _LWP_API_PRIVATE
    227        1.47   hannken 
    228         1.2   thorpej #include <sys/param.h>
    229         1.2   thorpej #include <sys/systm.h>
    230        1.64      yamt #include <sys/cpu.h>
    231         1.2   thorpej #include <sys/pool.h>
    232         1.2   thorpej #include <sys/proc.h>
    233         1.2   thorpej #include <sys/syscallargs.h>
    234        1.57       dsl #include <sys/syscall_stats.h>
    235        1.37        ad #include <sys/kauth.h>
    236        1.52        ad #include <sys/sleepq.h>
    237        1.52        ad #include <sys/lockdebug.h>
    238        1.52        ad #include <sys/kmem.h>
    239        1.91     rmind #include <sys/pset.h>
    240        1.75        ad #include <sys/intr.h>
    241        1.78        ad #include <sys/lwpctl.h>
    242        1.81        ad #include <sys/atomic.h>
    243       1.131        ad #include <sys/filedesc.h>
    244       1.196   hannken #include <sys/fstrans.h>
    245       1.138    darran #include <sys/dtrace_bsd.h>
    246       1.141    darran #include <sys/sdt.h>
    247       1.203     kamil #include <sys/ptrace.h>
    248       1.157     rmind #include <sys/xcall.h>
    249       1.169  christos #include <sys/uidinfo.h>
    250       1.169  christos #include <sys/sysctl.h>
    251       1.201     ozaki #include <sys/psref.h>
    252       1.208      maxv #include <sys/msan.h>
    253       1.232      maxv #include <sys/kcov.h>
    254       1.233   thorpej #include <sys/cprng.h>
    255       1.236   thorpej #include <sys/futex.h>
    256       1.138    darran 
    257         1.2   thorpej #include <uvm/uvm_extern.h>
    258        1.80     skrll #include <uvm/uvm_object.h>
    259         1.2   thorpej 
    260       1.152     rmind static pool_cache_t	lwp_cache	__read_mostly;
    261       1.152     rmind struct lwplist		alllwp		__cacheline_aligned;
    262        1.41   thorpej 
    263       1.238        ad static int		lwp_ctor(void *, void *, int);
    264       1.157     rmind static void		lwp_dtor(void *, void *);
    265       1.157     rmind 
    266       1.141    darran /* DTrace proc provider probes */
    267       1.180  christos SDT_PROVIDER_DEFINE(proc);
    268       1.180  christos 
    269       1.180  christos SDT_PROBE_DEFINE1(proc, kernel, , lwp__create, "struct lwp *");
    270       1.180  christos SDT_PROBE_DEFINE1(proc, kernel, , lwp__start, "struct lwp *");
    271       1.180  christos SDT_PROBE_DEFINE1(proc, kernel, , lwp__exit, "struct lwp *");
    272       1.141    darran 
    273       1.213        ad struct turnstile turnstile0 __cacheline_aligned;
    274       1.147     pooka struct lwp lwp0 __aligned(MIN_LWP_ALIGNMENT) = {
    275       1.147     pooka #ifdef LWP0_CPU_INFO
    276       1.147     pooka 	.l_cpu = LWP0_CPU_INFO,
    277       1.147     pooka #endif
    278       1.154      matt #ifdef LWP0_MD_INITIALIZER
    279       1.154      matt 	.l_md = LWP0_MD_INITIALIZER,
    280       1.154      matt #endif
    281       1.147     pooka 	.l_proc = &proc0,
    282       1.235   thorpej 	.l_lid = 0,		/* we own proc0's slot in the pid table */
    283       1.147     pooka 	.l_flag = LW_SYSTEM,
    284       1.147     pooka 	.l_stat = LSONPROC,
    285       1.147     pooka 	.l_ts = &turnstile0,
    286       1.147     pooka 	.l_syncobj = &sched_syncobj,
    287       1.231        ad 	.l_refcnt = 0,
    288       1.147     pooka 	.l_priority = PRI_USER + NPRI_USER - 1,
    289       1.147     pooka 	.l_inheritedprio = -1,
    290       1.147     pooka 	.l_class = SCHED_OTHER,
    291       1.147     pooka 	.l_psid = PS_NONE,
    292       1.147     pooka 	.l_pi_lenders = SLIST_HEAD_INITIALIZER(&lwp0.l_pi_lenders),
    293       1.147     pooka 	.l_name = __UNCONST("swapper"),
    294       1.147     pooka 	.l_fd = &filedesc0,
    295       1.147     pooka };
    296       1.147     pooka 
    297       1.169  christos static int sysctl_kern_maxlwp(SYSCTLFN_PROTO);
    298       1.169  christos 
    299       1.169  christos /*
    300       1.169  christos  * sysctl helper routine for kern.maxlwp. Ensures that the new
    301       1.169  christos  * values are not too low or too high.
    302       1.169  christos  */
    303       1.169  christos static int
    304       1.169  christos sysctl_kern_maxlwp(SYSCTLFN_ARGS)
    305       1.169  christos {
    306       1.169  christos 	int error, nmaxlwp;
    307       1.169  christos 	struct sysctlnode node;
    308       1.169  christos 
    309       1.169  christos 	nmaxlwp = maxlwp;
    310       1.169  christos 	node = *rnode;
    311       1.169  christos 	node.sysctl_data = &nmaxlwp;
    312       1.169  christos 	error = sysctl_lookup(SYSCTLFN_CALL(&node));
    313       1.169  christos 	if (error || newp == NULL)
    314       1.169  christos 		return error;
    315       1.169  christos 
    316       1.169  christos 	if (nmaxlwp < 0 || nmaxlwp >= 65536)
    317       1.169  christos 		return EINVAL;
    318       1.169  christos 	if (nmaxlwp > cpu_maxlwp())
    319       1.169  christos 		return EINVAL;
    320       1.169  christos 	maxlwp = nmaxlwp;
    321       1.169  christos 
    322       1.169  christos 	return 0;
    323       1.169  christos }
    324       1.169  christos 
    325       1.169  christos static void
    326       1.169  christos sysctl_kern_lwp_setup(void)
    327       1.169  christos {
    328       1.242      maxv 	sysctl_createv(NULL, 0, NULL, NULL,
    329       1.169  christos 		       CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
    330       1.169  christos 		       CTLTYPE_INT, "maxlwp",
    331       1.169  christos 		       SYSCTL_DESCR("Maximum number of simultaneous threads"),
    332       1.169  christos 		       sysctl_kern_maxlwp, 0, NULL, 0,
    333       1.169  christos 		       CTL_KERN, CTL_CREATE, CTL_EOL);
    334       1.169  christos }
    335       1.169  christos 
    336        1.41   thorpej void
    337        1.41   thorpej lwpinit(void)
    338        1.41   thorpej {
    339        1.41   thorpej 
    340       1.152     rmind 	LIST_INIT(&alllwp);
    341       1.144     pooka 	lwpinit_specificdata();
    342        1.87        ad 	lwp_cache = pool_cache_init(sizeof(lwp_t), MIN_LWP_ALIGNMENT, 0, 0,
    343       1.238        ad 	    "lwppl", NULL, IPL_NONE, lwp_ctor, lwp_dtor, NULL);
    344       1.169  christos 
    345       1.169  christos 	maxlwp = cpu_maxlwp();
    346       1.169  christos 	sysctl_kern_lwp_setup();
    347        1.41   thorpej }
    348        1.41   thorpej 
    349       1.147     pooka void
    350       1.147     pooka lwp0_init(void)
    351       1.147     pooka {
    352       1.147     pooka 	struct lwp *l = &lwp0;
    353       1.147     pooka 
    354       1.147     pooka 	KASSERT((void *)uvm_lwp_getuarea(l) != NULL);
    355       1.147     pooka 
    356       1.147     pooka 	LIST_INSERT_HEAD(&alllwp, l, l_list);
    357       1.147     pooka 
    358       1.147     pooka 	callout_init(&l->l_timeout_ch, CALLOUT_MPSAFE);
    359       1.147     pooka 	callout_setfunc(&l->l_timeout_ch, sleepq_timeout, l);
    360       1.147     pooka 	cv_init(&l->l_sigcv, "sigwait");
    361       1.171     rmind 	cv_init(&l->l_waitcv, "vfork");
    362       1.147     pooka 
    363       1.147     pooka 	kauth_cred_hold(proc0.p_cred);
    364       1.147     pooka 	l->l_cred = proc0.p_cred;
    365       1.147     pooka 
    366       1.164      yamt 	kdtrace_thread_ctor(NULL, l);
    367       1.147     pooka 	lwp_initspecific(l);
    368       1.147     pooka 
    369       1.147     pooka 	SYSCALL_TIME_LWP_INIT(l);
    370       1.147     pooka }
    371       1.147     pooka 
    372       1.238        ad /*
    373       1.238        ad  * Initialize the non-zeroed portion of an lwp_t.
    374       1.238        ad  */
    375       1.238        ad static int
    376       1.238        ad lwp_ctor(void *arg, void *obj, int flags)
    377       1.238        ad {
    378       1.238        ad 	lwp_t *l = obj;
    379       1.238        ad 
    380       1.238        ad 	l->l_stat = LSIDL;
    381       1.238        ad 	l->l_cpu = curcpu();
    382       1.238        ad 	l->l_mutex = l->l_cpu->ci_schedstate.spc_lwplock;
    383       1.238        ad 	l->l_ts = pool_get(&turnstile_pool, flags);
    384       1.238        ad 
    385       1.238        ad 	if (l->l_ts == NULL) {
    386       1.238        ad 		return ENOMEM;
    387       1.238        ad 	} else {
    388       1.238        ad 		turnstile_ctor(l->l_ts);
    389       1.238        ad 		return 0;
    390       1.238        ad 	}
    391       1.238        ad }
    392       1.238        ad 
    393       1.157     rmind static void
    394       1.157     rmind lwp_dtor(void *arg, void *obj)
    395       1.157     rmind {
    396       1.157     rmind 	lwp_t *l = obj;
    397       1.157     rmind 	(void)l;
    398       1.157     rmind 
    399       1.157     rmind 	/*
    400       1.157     rmind 	 * Provide a barrier to ensure that all mutex_oncpu() and rw_oncpu()
    401       1.157     rmind 	 * calls will exit before memory of LWP is returned to the pool, where
    402       1.157     rmind 	 * KVA of LWP structure might be freed and re-used for other purposes.
    403       1.157     rmind 	 * Kernel preemption is disabled around mutex_oncpu() and rw_oncpu()
    404       1.157     rmind 	 * callers, therefore cross-call to all CPUs will do the job.  Also,
    405       1.157     rmind 	 * the value of l->l_cpu must be still valid at this point.
    406       1.238        ad 	 *
    407       1.238        ad 	 * XXX should use epoch based reclamation.
    408       1.157     rmind 	 */
    409       1.157     rmind 	KASSERT(l->l_cpu != NULL);
    410       1.205       uwe 	xc_barrier(0);
    411       1.238        ad 
    412       1.238        ad 	/*
    413       1.238        ad 	 * We can't return turnstile0 to the pool (it didn't come from it),
    414       1.238        ad 	 * so if it comes up just drop it quietly and move on.
    415       1.238        ad 	 */
    416       1.238        ad 	if (l->l_ts != &turnstile0)
    417       1.238        ad 		pool_put(&turnstile_pool, l->l_ts);
    418       1.157     rmind }
    419       1.157     rmind 
    420        1.52        ad /*
    421       1.238        ad  * Set an LWP suspended.
    422        1.52        ad  *
    423       1.103        ad  * Must be called with p_lock held, and the LWP locked.  Will unlock the
    424        1.52        ad  * LWP before return.
    425        1.52        ad  */
    426         1.2   thorpej int
    427        1.52        ad lwp_suspend(struct lwp *curl, struct lwp *t)
    428         1.2   thorpej {
    429        1.52        ad 	int error;
    430         1.2   thorpej 
    431       1.103        ad 	KASSERT(mutex_owned(t->l_proc->p_lock));
    432        1.63        ad 	KASSERT(lwp_locked(t, NULL));
    433        1.33       chs 
    434        1.52        ad 	KASSERT(curl != t || curl->l_stat == LSONPROC);
    435         1.2   thorpej 
    436        1.52        ad 	/*
    437        1.52        ad 	 * If the current LWP has been told to exit, we must not suspend anyone
    438        1.52        ad 	 * else or deadlock could occur.  We won't return to userspace.
    439         1.2   thorpej 	 */
    440       1.109     rmind 	if ((curl->l_flag & (LW_WEXIT | LW_WCORE)) != 0) {
    441        1.52        ad 		lwp_unlock(t);
    442        1.52        ad 		return (EDEADLK);
    443         1.2   thorpej 	}
    444         1.2   thorpej 
    445       1.204     kamil 	if ((t->l_flag & LW_DBGSUSPEND) != 0) {
    446       1.204     kamil 		lwp_unlock(t);
    447       1.204     kamil 		return 0;
    448       1.204     kamil 	}
    449       1.204     kamil 
    450        1.52        ad 	error = 0;
    451         1.2   thorpej 
    452        1.52        ad 	switch (t->l_stat) {
    453        1.52        ad 	case LSRUN:
    454        1.52        ad 	case LSONPROC:
    455        1.56     pavel 		t->l_flag |= LW_WSUSPEND;
    456        1.52        ad 		lwp_need_userret(t);
    457        1.52        ad 		lwp_unlock(t);
    458        1.52        ad 		break;
    459         1.2   thorpej 
    460        1.52        ad 	case LSSLEEP:
    461        1.56     pavel 		t->l_flag |= LW_WSUSPEND;
    462         1.2   thorpej 
    463         1.2   thorpej 		/*
    464        1.52        ad 		 * Kick the LWP and try to get it to the kernel boundary
    465        1.52        ad 		 * so that it will release any locks that it holds.
    466        1.52        ad 		 * setrunnable() will release the lock.
    467         1.2   thorpej 		 */
    468        1.56     pavel 		if ((t->l_flag & LW_SINTR) != 0)
    469        1.52        ad 			setrunnable(t);
    470        1.52        ad 		else
    471        1.52        ad 			lwp_unlock(t);
    472        1.52        ad 		break;
    473         1.2   thorpej 
    474        1.52        ad 	case LSSUSPENDED:
    475        1.52        ad 		lwp_unlock(t);
    476        1.52        ad 		break;
    477        1.17      manu 
    478        1.52        ad 	case LSSTOP:
    479        1.56     pavel 		t->l_flag |= LW_WSUSPEND;
    480        1.52        ad 		setrunnable(t);
    481        1.52        ad 		break;
    482         1.2   thorpej 
    483        1.52        ad 	case LSIDL:
    484        1.52        ad 	case LSZOMB:
    485        1.52        ad 		error = EINTR; /* It's what Solaris does..... */
    486        1.52        ad 		lwp_unlock(t);
    487        1.52        ad 		break;
    488         1.2   thorpej 	}
    489         1.2   thorpej 
    490        1.69     rmind 	return (error);
    491         1.2   thorpej }
    492         1.2   thorpej 
    493        1.52        ad /*
    494        1.52        ad  * Restart a suspended LWP.
    495        1.52        ad  *
    496       1.103        ad  * Must be called with p_lock held, and the LWP locked.  Will unlock the
    497        1.52        ad  * LWP before return.
    498        1.52        ad  */
    499         1.2   thorpej void
    500         1.2   thorpej lwp_continue(struct lwp *l)
    501         1.2   thorpej {
    502         1.2   thorpej 
    503       1.103        ad 	KASSERT(mutex_owned(l->l_proc->p_lock));
    504        1.63        ad 	KASSERT(lwp_locked(l, NULL));
    505        1.52        ad 
    506        1.52        ad 	/* If rebooting or not suspended, then just bail out. */
    507        1.56     pavel 	if ((l->l_flag & LW_WREBOOT) != 0) {
    508        1.52        ad 		lwp_unlock(l);
    509         1.2   thorpej 		return;
    510        1.10      fvdl 	}
    511         1.2   thorpej 
    512        1.56     pavel 	l->l_flag &= ~LW_WSUSPEND;
    513         1.2   thorpej 
    514       1.204     kamil 	if (l->l_stat != LSSUSPENDED || (l->l_flag & LW_DBGSUSPEND) != 0) {
    515        1.52        ad 		lwp_unlock(l);
    516        1.52        ad 		return;
    517         1.2   thorpej 	}
    518         1.2   thorpej 
    519        1.52        ad 	/* setrunnable() will release the lock. */
    520        1.52        ad 	setrunnable(l);
    521         1.2   thorpej }
    522         1.2   thorpej 
    523        1.52        ad /*
    524       1.142  christos  * Restart a stopped LWP.
    525       1.142  christos  *
    526       1.142  christos  * Must be called with p_lock held, and the LWP NOT locked.  Will unlock the
    527       1.142  christos  * LWP before return.
    528       1.142  christos  */
    529       1.142  christos void
    530       1.142  christos lwp_unstop(struct lwp *l)
    531       1.142  christos {
    532       1.142  christos 	struct proc *p = l->l_proc;
    533       1.167     rmind 
    534       1.239        ad 	KASSERT(mutex_owned(&proc_lock));
    535       1.142  christos 	KASSERT(mutex_owned(p->p_lock));
    536       1.142  christos 
    537       1.142  christos 	lwp_lock(l);
    538       1.142  christos 
    539       1.204     kamil 	KASSERT((l->l_flag & LW_DBGSUSPEND) == 0);
    540       1.204     kamil 
    541       1.142  christos 	/* If not stopped, then just bail out. */
    542       1.142  christos 	if (l->l_stat != LSSTOP) {
    543       1.142  christos 		lwp_unlock(l);
    544       1.142  christos 		return;
    545       1.142  christos 	}
    546       1.142  christos 
    547       1.142  christos 	p->p_stat = SACTIVE;
    548       1.142  christos 	p->p_sflag &= ~PS_STOPPING;
    549       1.142  christos 
    550       1.142  christos 	if (!p->p_waited)
    551       1.142  christos 		p->p_pptr->p_nstopchild--;
    552       1.142  christos 
    553       1.142  christos 	if (l->l_wchan == NULL) {
    554       1.142  christos 		/* setrunnable() will release the lock. */
    555       1.142  christos 		setrunnable(l);
    556       1.183  christos 	} else if (p->p_xsig && (l->l_flag & LW_SINTR) != 0) {
    557       1.163  christos 		/* setrunnable() so we can receive the signal */
    558       1.163  christos 		setrunnable(l);
    559       1.142  christos 	} else {
    560       1.142  christos 		l->l_stat = LSSLEEP;
    561       1.142  christos 		p->p_nrlwps++;
    562       1.142  christos 		lwp_unlock(l);
    563       1.142  christos 	}
    564       1.142  christos }
    565       1.142  christos 
    566       1.142  christos /*
    567        1.52        ad  * Wait for an LWP within the current process to exit.  If 'lid' is
    568        1.52        ad  * non-zero, we are waiting for a specific LWP.
    569        1.52        ad  *
    570       1.103        ad  * Must be called with p->p_lock held.
    571        1.52        ad  */
    572         1.2   thorpej int
    573       1.173     rmind lwp_wait(struct lwp *l, lwpid_t lid, lwpid_t *departed, bool exiting)
    574         1.2   thorpej {
    575       1.173     rmind 	const lwpid_t curlid = l->l_lid;
    576       1.173     rmind 	proc_t *p = l->l_proc;
    577       1.223        ad 	lwp_t *l2, *next;
    578       1.173     rmind 	int error;
    579         1.2   thorpej 
    580       1.103        ad 	KASSERT(mutex_owned(p->p_lock));
    581        1.52        ad 
    582        1.52        ad 	p->p_nlwpwait++;
    583        1.63        ad 	l->l_waitingfor = lid;
    584        1.52        ad 
    585        1.52        ad 	for (;;) {
    586       1.173     rmind 		int nfound;
    587       1.173     rmind 
    588        1.52        ad 		/*
    589        1.52        ad 		 * Avoid a race between exit1() and sigexit(): if the
    590        1.52        ad 		 * process is dumping core, then we need to bail out: call
    591        1.52        ad 		 * into lwp_userret() where we will be suspended until the
    592        1.52        ad 		 * deed is done.
    593        1.52        ad 		 */
    594        1.52        ad 		if ((p->p_sflag & PS_WCORE) != 0) {
    595       1.103        ad 			mutex_exit(p->p_lock);
    596        1.52        ad 			lwp_userret(l);
    597       1.173     rmind 			KASSERT(false);
    598        1.52        ad 		}
    599        1.52        ad 
    600        1.52        ad 		/*
    601        1.52        ad 		 * First off, drain any detached LWP that is waiting to be
    602        1.52        ad 		 * reaped.
    603        1.52        ad 		 */
    604        1.52        ad 		while ((l2 = p->p_zomblwp) != NULL) {
    605        1.52        ad 			p->p_zomblwp = NULL;
    606        1.63        ad 			lwp_free(l2, false, false);/* releases proc mutex */
    607       1.103        ad 			mutex_enter(p->p_lock);
    608        1.52        ad 		}
    609        1.52        ad 
    610        1.52        ad 		/*
    611        1.52        ad 		 * Now look for an LWP to collect.  If the whole process is
    612        1.52        ad 		 * exiting, count detached LWPs as eligible to be collected,
    613        1.52        ad 		 * but don't drain them here.
    614        1.52        ad 		 */
    615        1.52        ad 		nfound = 0;
    616        1.63        ad 		error = 0;
    617       1.223        ad 
    618       1.223        ad 		/*
    619       1.238        ad 		 * If given a specific LID, go via pid_table and make sure
    620       1.223        ad 		 * it's not detached.
    621       1.223        ad 		 */
    622       1.223        ad 		if (lid != 0) {
    623       1.235   thorpej 			l2 = proc_find_lwp(p, lid);
    624       1.223        ad 			if (l2 == NULL) {
    625       1.223        ad 				error = ESRCH;
    626       1.223        ad 				break;
    627       1.223        ad 			}
    628       1.223        ad 			KASSERT(l2->l_lid == lid);
    629       1.223        ad 			if ((l2->l_prflag & LPR_DETACHED) != 0) {
    630       1.223        ad 				error = EINVAL;
    631       1.223        ad 				break;
    632       1.223        ad 			}
    633       1.223        ad 		} else {
    634       1.223        ad 			l2 = LIST_FIRST(&p->p_lwps);
    635       1.223        ad 		}
    636       1.223        ad 		for (; l2 != NULL; l2 = next) {
    637       1.223        ad 			next = (lid != 0 ? NULL : LIST_NEXT(l2, l_sibling));
    638       1.223        ad 
    639        1.63        ad 			/*
    640        1.63        ad 			 * If a specific wait and the target is waiting on
    641        1.63        ad 			 * us, then avoid deadlock.  This also traps LWPs
    642        1.63        ad 			 * that try to wait on themselves.
    643        1.63        ad 			 *
    644        1.63        ad 			 * Note that this does not handle more complicated
    645        1.63        ad 			 * cycles, like: t1 -> t2 -> t3 -> t1.  The process
    646        1.63        ad 			 * can still be killed so it is not a major problem.
    647        1.63        ad 			 */
    648        1.63        ad 			if (l2->l_lid == lid && l2->l_waitingfor == curlid) {
    649        1.63        ad 				error = EDEADLK;
    650        1.63        ad 				break;
    651        1.63        ad 			}
    652        1.63        ad 			if (l2 == l)
    653        1.52        ad 				continue;
    654        1.52        ad 			if ((l2->l_prflag & LPR_DETACHED) != 0) {
    655        1.63        ad 				nfound += exiting;
    656        1.63        ad 				continue;
    657        1.63        ad 			}
    658        1.63        ad 			if (lid != 0) {
    659        1.63        ad 				/*
    660        1.63        ad 				 * Mark this LWP as the first waiter, if there
    661        1.63        ad 				 * is no other.
    662        1.63        ad 				 */
    663        1.63        ad 				if (l2->l_waiter == 0)
    664        1.63        ad 					l2->l_waiter = curlid;
    665        1.63        ad 			} else if (l2->l_waiter != 0) {
    666        1.63        ad 				/*
    667        1.63        ad 				 * It already has a waiter - so don't
    668        1.63        ad 				 * collect it.  If the waiter doesn't
    669        1.63        ad 				 * grab it we'll get another chance
    670        1.63        ad 				 * later.
    671        1.63        ad 				 */
    672        1.63        ad 				nfound++;
    673        1.52        ad 				continue;
    674        1.52        ad 			}
    675        1.52        ad 			nfound++;
    676         1.2   thorpej 
    677        1.52        ad 			/* No need to lock the LWP in order to see LSZOMB. */
    678        1.52        ad 			if (l2->l_stat != LSZOMB)
    679        1.52        ad 				continue;
    680         1.2   thorpej 
    681        1.63        ad 			/*
    682        1.63        ad 			 * We're no longer waiting.  Reset the "first waiter"
    683        1.63        ad 			 * pointer on the target, in case it was us.
    684        1.63        ad 			 */
    685        1.63        ad 			l->l_waitingfor = 0;
    686        1.63        ad 			l2->l_waiter = 0;
    687        1.63        ad 			p->p_nlwpwait--;
    688         1.2   thorpej 			if (departed)
    689         1.2   thorpej 				*departed = l2->l_lid;
    690        1.75        ad 			sched_lwp_collect(l2);
    691        1.63        ad 
    692        1.63        ad 			/* lwp_free() releases the proc lock. */
    693        1.63        ad 			lwp_free(l2, false, false);
    694       1.103        ad 			mutex_enter(p->p_lock);
    695        1.52        ad 			return 0;
    696        1.52        ad 		}
    697         1.2   thorpej 
    698        1.63        ad 		if (error != 0)
    699        1.63        ad 			break;
    700        1.52        ad 		if (nfound == 0) {
    701        1.52        ad 			error = ESRCH;
    702        1.52        ad 			break;
    703        1.52        ad 		}
    704        1.63        ad 
    705        1.63        ad 		/*
    706       1.173     rmind 		 * Note: since the lock will be dropped, need to restart on
    707       1.173     rmind 		 * wakeup to run all LWPs again, e.g. there may be new LWPs.
    708        1.63        ad 		 */
    709        1.63        ad 		if (exiting) {
    710        1.52        ad 			KASSERT(p->p_nlwps > 1);
    711       1.222        ad 			error = cv_timedwait(&p->p_lwpcv, p->p_lock, 1);
    712       1.173     rmind 			break;
    713        1.52        ad 		}
    714        1.63        ad 
    715        1.63        ad 		/*
    716       1.234        ad 		 * Break out if all LWPs are in _lwp_wait().  There are
    717       1.234        ad 		 * other ways to hang the process with _lwp_wait(), but the
    718       1.234        ad 		 * sleep is interruptable so little point checking for them.
    719        1.63        ad 		 */
    720       1.234        ad 		if (p->p_nlwpwait == p->p_nlwps) {
    721        1.52        ad 			error = EDEADLK;
    722        1.52        ad 			break;
    723         1.2   thorpej 		}
    724        1.63        ad 
    725        1.63        ad 		/*
    726        1.63        ad 		 * Sit around and wait for something to happen.  We'll be
    727        1.63        ad 		 * awoken if any of the conditions examined change: if an
    728        1.63        ad 		 * LWP exits, is collected, or is detached.
    729        1.63        ad 		 */
    730       1.103        ad 		if ((error = cv_wait_sig(&p->p_lwpcv, p->p_lock)) != 0)
    731        1.52        ad 			break;
    732         1.2   thorpej 	}
    733         1.2   thorpej 
    734        1.63        ad 	/*
    735        1.63        ad 	 * We didn't find any LWPs to collect, we may have received a
    736        1.63        ad 	 * signal, or some other condition has caused us to bail out.
    737        1.63        ad 	 *
    738        1.63        ad 	 * If waiting on a specific LWP, clear the waiters marker: some
    739        1.63        ad 	 * other LWP may want it.  Then, kick all the remaining waiters
    740        1.63        ad 	 * so that they can re-check for zombies and for deadlock.
    741        1.63        ad 	 */
    742        1.63        ad 	if (lid != 0) {
    743       1.235   thorpej 		l2 = proc_find_lwp(p, lid);
    744       1.223        ad 		KASSERT(l2 == NULL || l2->l_lid == lid);
    745       1.223        ad 
    746       1.223        ad 		if (l2 != NULL && l2->l_waiter == curlid)
    747       1.223        ad 			l2->l_waiter = 0;
    748        1.63        ad 	}
    749        1.52        ad 	p->p_nlwpwait--;
    750        1.63        ad 	l->l_waitingfor = 0;
    751        1.63        ad 	cv_broadcast(&p->p_lwpcv);
    752        1.63        ad 
    753        1.52        ad 	return error;
    754         1.2   thorpej }
    755         1.2   thorpej 
    756       1.223        ad /*
    757        1.52        ad  * Create a new LWP within process 'p2', using LWP 'l1' as a template.
    758        1.52        ad  * The new LWP is created in state LSIDL and must be set running,
    759        1.52        ad  * suspended, or stopped by the caller.
    760        1.52        ad  */
    761         1.2   thorpej int
    762       1.134     rmind lwp_create(lwp_t *l1, proc_t *p2, vaddr_t uaddr, int flags,
    763       1.188  christos     void *stack, size_t stacksize, void (*func)(void *), void *arg,
    764       1.188  christos     lwp_t **rnewlwpp, int sclass, const sigset_t *sigmask,
    765       1.188  christos     const stack_t *sigstk)
    766         1.2   thorpej {
    767       1.215        ad 	struct lwp *l2;
    768         1.2   thorpej 
    769       1.107        ad 	KASSERT(l1 == curlwp || l1->l_proc == &proc0);
    770       1.107        ad 
    771        1.52        ad 	/*
    772       1.215        ad 	 * Enforce limits, excluding the first lwp and kthreads.  We must
    773       1.215        ad 	 * use the process credentials here when adjusting the limit, as
    774       1.215        ad 	 * they are what's tied to the accounting entity.  However for
    775       1.215        ad 	 * authorizing the action, we'll use the LWP's credentials.
    776       1.169  christos 	 */
    777       1.215        ad 	mutex_enter(p2->p_lock);
    778       1.169  christos 	if (p2->p_nlwps != 0 && p2 != &proc0) {
    779       1.215        ad 		uid_t uid = kauth_cred_getuid(p2->p_cred);
    780       1.169  christos 		int count = chglwpcnt(uid, 1);
    781       1.169  christos 		if (__predict_false(count >
    782       1.169  christos 		    p2->p_rlimit[RLIMIT_NTHR].rlim_cur)) {
    783       1.169  christos 			if (kauth_authorize_process(l1->l_cred,
    784       1.169  christos 			    KAUTH_PROCESS_RLIMIT, p2,
    785       1.169  christos 			    KAUTH_ARG(KAUTH_REQ_PROCESS_RLIMIT_BYPASS),
    786       1.169  christos 			    &p2->p_rlimit[RLIMIT_NTHR], KAUTH_ARG(RLIMIT_NTHR))
    787       1.169  christos 			    != 0) {
    788       1.170  christos 				(void)chglwpcnt(uid, -1);
    789       1.215        ad 				mutex_exit(p2->p_lock);
    790       1.170  christos 				return EAGAIN;
    791       1.169  christos 			}
    792       1.169  christos 		}
    793       1.169  christos 	}
    794       1.169  christos 
    795       1.169  christos 	/*
    796        1.52        ad 	 * First off, reap any detached LWP waiting to be collected.
    797        1.52        ad 	 * We can re-use its LWP structure and turnstile.
    798        1.52        ad 	 */
    799       1.215        ad 	if ((l2 = p2->p_zomblwp) != NULL) {
    800       1.215        ad 		p2->p_zomblwp = NULL;
    801       1.215        ad 		lwp_free(l2, true, false);
    802       1.215        ad 		/* p2 now unlocked by lwp_free() */
    803       1.238        ad 		KASSERT(l2->l_ts != NULL);
    804        1.75        ad 		KASSERT(l2->l_inheritedprio == -1);
    805        1.60      yamt 		KASSERT(SLIST_EMPTY(&l2->l_pi_lenders));
    806       1.238        ad 		memset(&l2->l_startzero, 0, sizeof(*l2) -
    807       1.238        ad 		    offsetof(lwp_t, l_startzero));
    808       1.215        ad 	} else {
    809       1.215        ad 		mutex_exit(p2->p_lock);
    810       1.215        ad 		l2 = pool_cache_get(lwp_cache, PR_WAITOK);
    811       1.238        ad 		memset(&l2->l_startzero, 0, sizeof(*l2) -
    812       1.238        ad 		    offsetof(lwp_t, l_startzero));
    813       1.215        ad 		SLIST_INIT(&l2->l_pi_lenders);
    814        1.52        ad 	}
    815         1.2   thorpej 
    816       1.238        ad 	/*
    817       1.238        ad 	 * Because of lockless lookup via pid_table, the LWP can be locked
    818       1.238        ad 	 * and inspected briefly even after it's freed, so a few fields are
    819       1.238        ad 	 * kept stable.
    820       1.238        ad 	 */
    821       1.238        ad 	KASSERT(l2->l_stat == LSIDL);
    822       1.238        ad 	KASSERT(l2->l_cpu != NULL);
    823       1.238        ad 	KASSERT(l2->l_ts != NULL);
    824       1.238        ad 	KASSERT(l2->l_mutex == l2->l_cpu->ci_schedstate.spc_lwplock);
    825       1.238        ad 
    826         1.2   thorpej 	l2->l_proc = p2;
    827       1.231        ad 	l2->l_refcnt = 0;
    828        1.75        ad 	l2->l_class = sclass;
    829       1.116        ad 
    830       1.116        ad 	/*
    831       1.235   thorpej 	 * Allocate a process ID for this LWP.  We need to do this now
    832       1.235   thorpej 	 * while we can still unwind if it fails.  Beacuse we're marked
    833       1.238        ad 	 * as LSIDL, no lookups by the ID will succeed.
    834       1.235   thorpej 	 *
    835       1.235   thorpej 	 * N.B. this will always succeed for the first LWP in a process,
    836       1.235   thorpej 	 * because proc_alloc_lwpid() will usurp the slot.  Also note
    837       1.235   thorpej 	 * that l2->l_proc MUST be valid so that lookups of the proc
    838       1.235   thorpej 	 * will succeed, even if the LWP itself is not visible.
    839       1.235   thorpej 	 */
    840       1.235   thorpej 	if (__predict_false(proc_alloc_lwpid(p2, l2) == -1)) {
    841       1.235   thorpej 		pool_cache_put(lwp_cache, l2);
    842       1.235   thorpej 		return EAGAIN;
    843       1.235   thorpej 	}
    844       1.235   thorpej 
    845       1.235   thorpej 	/*
    846       1.116        ad 	 * If vfork(), we want the LWP to run fast and on the same CPU
    847       1.116        ad 	 * as its parent, so that it can reuse the VM context and cache
    848       1.116        ad 	 * footprint on the local CPU.
    849       1.116        ad 	 */
    850       1.116        ad 	l2->l_kpriority = ((flags & LWP_VFORK) ? true : false);
    851        1.82        ad 	l2->l_kpribase = PRI_KERNEL;
    852        1.52        ad 	l2->l_priority = l1->l_priority;
    853        1.75        ad 	l2->l_inheritedprio = -1;
    854       1.185  christos 	l2->l_protectprio = -1;
    855       1.185  christos 	l2->l_auxprio = -1;
    856       1.222        ad 	l2->l_flag = 0;
    857        1.88        ad 	l2->l_pflag = LP_MPSAFE;
    858       1.131        ad 	TAILQ_INIT(&l2->l_ld_locks);
    859       1.197     ozaki 	l2->l_psrefs = 0;
    860       1.208      maxv 	kmsan_lwp_alloc(l2);
    861       1.131        ad 
    862       1.131        ad 	/*
    863       1.156     pooka 	 * For vfork, borrow parent's lwpctl context if it exists.
    864       1.156     pooka 	 * This also causes us to return via lwp_userret.
    865       1.156     pooka 	 */
    866       1.156     pooka 	if (flags & LWP_VFORK && l1->l_lwpctl) {
    867       1.156     pooka 		l2->l_lwpctl = l1->l_lwpctl;
    868       1.156     pooka 		l2->l_flag |= LW_LWPCTL;
    869       1.156     pooka 	}
    870       1.156     pooka 
    871       1.156     pooka 	/*
    872       1.131        ad 	 * If not the first LWP in the process, grab a reference to the
    873       1.131        ad 	 * descriptor table.
    874       1.131        ad 	 */
    875        1.97        ad 	l2->l_fd = p2->p_fd;
    876       1.131        ad 	if (p2->p_nlwps != 0) {
    877       1.131        ad 		KASSERT(l1->l_proc == p2);
    878       1.136     rmind 		fd_hold(l2);
    879       1.131        ad 	} else {
    880       1.131        ad 		KASSERT(l1->l_proc != p2);
    881       1.131        ad 	}
    882        1.41   thorpej 
    883        1.56     pavel 	if (p2->p_flag & PK_SYSTEM) {
    884       1.134     rmind 		/* Mark it as a system LWP. */
    885        1.56     pavel 		l2->l_flag |= LW_SYSTEM;
    886        1.52        ad 	}
    887         1.2   thorpej 
    888       1.138    darran 	kdtrace_thread_ctor(NULL, l2);
    889        1.73     rmind 	lwp_initspecific(l2);
    890        1.75        ad 	sched_lwp_fork(l1, l2);
    891        1.37        ad 	lwp_update_creds(l2);
    892        1.70        ad 	callout_init(&l2->l_timeout_ch, CALLOUT_MPSAFE);
    893        1.70        ad 	callout_setfunc(&l2->l_timeout_ch, sleepq_timeout, l2);
    894        1.52        ad 	cv_init(&l2->l_sigcv, "sigwait");
    895       1.171     rmind 	cv_init(&l2->l_waitcv, "vfork");
    896        1.52        ad 	l2->l_syncobj = &sched_syncobj;
    897       1.201     ozaki 	PSREF_DEBUG_INIT_LWP(l2);
    898         1.2   thorpej 
    899         1.2   thorpej 	if (rnewlwpp != NULL)
    900         1.2   thorpej 		*rnewlwpp = l2;
    901         1.2   thorpej 
    902       1.158      matt 	/*
    903       1.158      matt 	 * PCU state needs to be saved before calling uvm_lwp_fork() so that
    904       1.158      matt 	 * the MD cpu_lwp_fork() can copy the saved state to the new LWP.
    905       1.158      matt 	 */
    906       1.158      matt 	pcu_save_all(l1);
    907       1.225    dogcow #if PCU_UNIT_COUNT > 0
    908       1.224  riastrad 	l2->l_pcu_valid = l1->l_pcu_valid;
    909       1.225    dogcow #endif
    910       1.158      matt 
    911       1.137     rmind 	uvm_lwp_setuarea(l2, uaddr);
    912       1.190     skrll 	uvm_lwp_fork(l1, l2, stack, stacksize, func, (arg != NULL) ? arg : l2);
    913         1.2   thorpej 
    914       1.235   thorpej 	mutex_enter(p2->p_lock);
    915        1.52        ad 	if ((flags & LWP_DETACHED) != 0) {
    916        1.52        ad 		l2->l_prflag = LPR_DETACHED;
    917        1.52        ad 		p2->p_ndlwps++;
    918        1.52        ad 	} else
    919        1.52        ad 		l2->l_prflag = 0;
    920        1.52        ad 
    921       1.223        ad 	if (l1->l_proc == p2) {
    922       1.223        ad 		/*
    923       1.223        ad 		 * These flags are set while p_lock is held.  Copy with
    924       1.223        ad 		 * p_lock held too, so the LWP doesn't sneak into the
    925       1.223        ad 		 * process without them being set.
    926       1.223        ad 		 */
    927       1.222        ad 		l2->l_flag |= (l1->l_flag & (LW_WEXIT | LW_WREBOOT | LW_WCORE));
    928       1.223        ad 	} else {
    929       1.223        ad 		/* fork(): pending core/exit doesn't apply to child. */
    930       1.222        ad 		l2->l_flag |= (l1->l_flag & LW_WREBOOT);
    931       1.223        ad 	}
    932       1.222        ad 
    933       1.188  christos 	l2->l_sigstk = *sigstk;
    934       1.188  christos 	l2->l_sigmask = *sigmask;
    935       1.176  christos 	TAILQ_INIT(&l2->l_sigpend.sp_info);
    936        1.52        ad 	sigemptyset(&l2->l_sigpend.sp_set);
    937       1.174       dsl 	LIST_INSERT_HEAD(&p2->p_lwps, l2, l_sibling);
    938         1.2   thorpej 	p2->p_nlwps++;
    939       1.149      yamt 	p2->p_nrlwps++;
    940         1.2   thorpej 
    941       1.162     rmind 	KASSERT(l2->l_affinity == NULL);
    942       1.162     rmind 
    943       1.210        ad 	/* Inherit the affinity mask. */
    944       1.210        ad 	if (l1->l_affinity) {
    945       1.210        ad 		/*
    946       1.210        ad 		 * Note that we hold the state lock while inheriting
    947       1.210        ad 		 * the affinity to avoid race with sched_setaffinity().
    948       1.210        ad 		 */
    949       1.210        ad 		lwp_lock(l1);
    950       1.162     rmind 		if (l1->l_affinity) {
    951       1.210        ad 			kcpuset_use(l1->l_affinity);
    952       1.210        ad 			l2->l_affinity = l1->l_affinity;
    953       1.117  christos 		}
    954       1.210        ad 		lwp_unlock(l1);
    955        1.91     rmind 	}
    956       1.223        ad 
    957       1.223        ad 	/* This marks the end of the "must be atomic" section. */
    958       1.128     rmind 	mutex_exit(p2->p_lock);
    959       1.128     rmind 
    960       1.180  christos 	SDT_PROBE(proc, kernel, , lwp__create, l2, 0, 0, 0, 0);
    961       1.141    darran 
    962       1.239        ad 	mutex_enter(&proc_lock);
    963       1.128     rmind 	LIST_INSERT_HEAD(&alllwp, l2, l_list);
    964       1.210        ad 	/* Inherit a processor-set */
    965       1.210        ad 	l2->l_psid = l1->l_psid;
    966       1.239        ad 	mutex_exit(&proc_lock);
    967        1.91     rmind 
    968        1.57       dsl 	SYSCALL_TIME_LWP_INIT(l2);
    969        1.57       dsl 
    970        1.16      manu 	if (p2->p_emul->e_lwp_fork)
    971        1.16      manu 		(*p2->p_emul->e_lwp_fork)(l1, l2);
    972        1.16      manu 
    973         1.2   thorpej 	return (0);
    974         1.2   thorpej }
    975         1.2   thorpej 
    976         1.2   thorpej /*
    977       1.212        ad  * Set a new LWP running.  If the process is stopping, then the LWP is
    978       1.212        ad  * created stopped.
    979       1.212        ad  */
    980       1.212        ad void
    981       1.212        ad lwp_start(lwp_t *l, int flags)
    982       1.212        ad {
    983       1.212        ad 	proc_t *p = l->l_proc;
    984       1.212        ad 
    985       1.212        ad 	mutex_enter(p->p_lock);
    986       1.212        ad 	lwp_lock(l);
    987       1.212        ad 	KASSERT(l->l_stat == LSIDL);
    988       1.212        ad 	if ((flags & LWP_SUSPENDED) != 0) {
    989       1.212        ad 		/* It'll suspend itself in lwp_userret(). */
    990       1.212        ad 		l->l_flag |= LW_WSUSPEND;
    991       1.212        ad 	}
    992       1.212        ad 	if (p->p_stat == SSTOP || (p->p_sflag & PS_STOPPING) != 0) {
    993       1.212        ad 		KASSERT(l->l_wchan == NULL);
    994       1.212        ad 	    	l->l_stat = LSSTOP;
    995       1.212        ad 		p->p_nrlwps--;
    996       1.212        ad 		lwp_unlock(l);
    997       1.212        ad 	} else {
    998       1.212        ad 		setrunnable(l);
    999       1.212        ad 		/* LWP now unlocked */
   1000       1.212        ad 	}
   1001       1.212        ad 	mutex_exit(p->p_lock);
   1002       1.212        ad }
   1003       1.212        ad 
   1004       1.212        ad /*
   1005        1.64      yamt  * Called by MD code when a new LWP begins execution.  Must be called
   1006        1.64      yamt  * with the previous LWP locked (so at splsched), or if there is no
   1007        1.64      yamt  * previous LWP, at splsched.
   1008        1.64      yamt  */
   1009        1.64      yamt void
   1010       1.178      matt lwp_startup(struct lwp *prev, struct lwp *new_lwp)
   1011        1.64      yamt {
   1012       1.227        ad 	kmutex_t *lock;
   1013       1.218        ad 
   1014       1.178      matt 	KASSERTMSG(new_lwp == curlwp, "l %p curlwp %p prevlwp %p", new_lwp, curlwp, prev);
   1015       1.218        ad 	KASSERT(kpreempt_disabled());
   1016       1.218        ad 	KASSERT(prev != NULL);
   1017       1.227        ad 	KASSERT((prev->l_pflag & LP_RUNNING) != 0);
   1018       1.218        ad 	KASSERT(curcpu()->ci_mtx_count == -2);
   1019       1.218        ad 
   1020       1.227        ad 	/*
   1021       1.227        ad 	 * Immediately mark the previous LWP as no longer running and unlock
   1022       1.227        ad 	 * (to keep lock wait times short as possible).  If a zombie, don't
   1023       1.227        ad 	 * touch after clearing LP_RUNNING as it could be reaped by another
   1024       1.227        ad 	 * CPU.  Issue a memory barrier to ensure this.
   1025       1.227        ad 	 */
   1026       1.227        ad 	lock = prev->l_mutex;
   1027       1.227        ad 	if (__predict_false(prev->l_stat == LSZOMB)) {
   1028       1.227        ad 		membar_sync();
   1029       1.227        ad 	}
   1030       1.227        ad 	prev->l_pflag &= ~LP_RUNNING;
   1031       1.227        ad 	mutex_spin_exit(lock);
   1032        1.64      yamt 
   1033       1.218        ad 	/* Correct spin mutex count after mi_switch(). */
   1034       1.218        ad 	curcpu()->ci_mtx_count = 0;
   1035       1.141    darran 
   1036       1.218        ad 	/* Install new VM context. */
   1037       1.218        ad 	if (__predict_true(new_lwp->l_proc->p_vmspace)) {
   1038       1.218        ad 		pmap_activate(new_lwp);
   1039        1.64      yamt 	}
   1040       1.218        ad 
   1041       1.218        ad 	/* We remain at IPL_SCHED from mi_switch() - reset it. */
   1042       1.181     skrll 	spl0();
   1043       1.161  christos 
   1044        1.64      yamt 	LOCKDEBUG_BARRIER(NULL, 0);
   1045       1.218        ad 	SDT_PROBE(proc, kernel, , lwp__start, new_lwp, 0, 0, 0, 0);
   1046       1.218        ad 
   1047       1.218        ad 	/* For kthreads, acquire kernel lock if not MPSAFE. */
   1048       1.218        ad 	if (__predict_false((new_lwp->l_pflag & LP_MPSAFE) == 0)) {
   1049       1.178      matt 		KERNEL_LOCK(1, new_lwp);
   1050        1.65        ad 	}
   1051        1.64      yamt }
   1052        1.64      yamt 
   1053        1.64      yamt /*
   1054        1.65        ad  * Exit an LWP.
   1055       1.241        ad  *
   1056       1.241        ad  * *** WARNING *** This can be called with (l != curlwp) in error paths.
   1057         1.2   thorpej  */
   1058         1.2   thorpej void
   1059         1.2   thorpej lwp_exit(struct lwp *l)
   1060         1.2   thorpej {
   1061         1.2   thorpej 	struct proc *p = l->l_proc;
   1062        1.52        ad 	struct lwp *l2;
   1063        1.65        ad 	bool current;
   1064        1.65        ad 
   1065        1.65        ad 	current = (l == curlwp);
   1066         1.2   thorpej 
   1067       1.114     rmind 	KASSERT(current || (l->l_stat == LSIDL && l->l_target_cpu == NULL));
   1068       1.131        ad 	KASSERT(p == curproc);
   1069         1.2   thorpej 
   1070       1.180  christos 	SDT_PROBE(proc, kernel, , lwp__exit, l, 0, 0, 0, 0);
   1071       1.141    darran 
   1072       1.220        ad 	/* Verify that we hold no locks; for DIAGNOSTIC check kernel_lock. */
   1073       1.218        ad 	LOCKDEBUG_BARRIER(NULL, 0);
   1074       1.220        ad 	KASSERTMSG(curcpu()->ci_biglock_count == 0, "kernel_lock leaked");
   1075        1.16      manu 
   1076         1.2   thorpej 	/*
   1077        1.52        ad 	 * If we are the last live LWP in a process, we need to exit the
   1078        1.52        ad 	 * entire process.  We do so with an exit status of zero, because
   1079        1.52        ad 	 * it's a "controlled" exit, and because that's what Solaris does.
   1080        1.52        ad 	 *
   1081        1.52        ad 	 * We are not quite a zombie yet, but for accounting purposes we
   1082        1.52        ad 	 * must increment the count of zombies here.
   1083        1.45   thorpej 	 *
   1084        1.45   thorpej 	 * Note: the last LWP's specificdata will be deleted here.
   1085         1.2   thorpej 	 */
   1086       1.103        ad 	mutex_enter(p->p_lock);
   1087        1.52        ad 	if (p->p_nlwps - p->p_nzlwps == 1) {
   1088        1.65        ad 		KASSERT(current == true);
   1089       1.172      matt 		KASSERT(p != &proc0);
   1090       1.184  christos 		exit1(l, 0, 0);
   1091        1.19  jdolecek 		/* NOTREACHED */
   1092         1.2   thorpej 	}
   1093        1.52        ad 	p->p_nzlwps++;
   1094       1.233   thorpej 
   1095       1.233   thorpej 	/*
   1096       1.233   thorpej 	 * Perform any required thread cleanup.  Do this early so
   1097       1.235   thorpej 	 * anyone wanting to look us up with lwp_getref_lwpid() will
   1098       1.235   thorpej 	 * fail to find us before we become a zombie.
   1099       1.233   thorpej 	 *
   1100       1.233   thorpej 	 * N.B. this will unlock p->p_lock on our behalf.
   1101       1.233   thorpej 	 */
   1102       1.233   thorpej 	lwp_thread_cleanup(l);
   1103        1.52        ad 
   1104        1.52        ad 	if (p->p_emul->e_lwp_exit)
   1105        1.52        ad 		(*p->p_emul->e_lwp_exit)(l);
   1106         1.2   thorpej 
   1107       1.131        ad 	/* Drop filedesc reference. */
   1108       1.131        ad 	fd_free();
   1109       1.131        ad 
   1110       1.196   hannken 	/* Release fstrans private data. */
   1111       1.196   hannken 	fstrans_lwp_dtor(l);
   1112       1.196   hannken 
   1113        1.45   thorpej 	/* Delete the specificdata while it's still safe to sleep. */
   1114       1.145     pooka 	lwp_finispecific(l);
   1115        1.45   thorpej 
   1116        1.52        ad 	/*
   1117        1.52        ad 	 * Release our cached credentials.
   1118        1.52        ad 	 */
   1119        1.37        ad 	kauth_cred_free(l->l_cred);
   1120        1.70        ad 	callout_destroy(&l->l_timeout_ch);
   1121        1.65        ad 
   1122        1.65        ad 	/*
   1123       1.198     kamil 	 * If traced, report LWP exit event to the debugger.
   1124       1.198     kamil 	 *
   1125        1.52        ad 	 * Remove the LWP from the global list.
   1126       1.151       chs 	 * Free its LID from the PID namespace if needed.
   1127        1.52        ad 	 */
   1128       1.239        ad 	mutex_enter(&proc_lock);
   1129       1.198     kamil 
   1130       1.199     kamil 	if ((p->p_slflag & (PSL_TRACED|PSL_TRACELWP_EXIT)) ==
   1131       1.198     kamil 	    (PSL_TRACED|PSL_TRACELWP_EXIT)) {
   1132       1.198     kamil 		mutex_enter(p->p_lock);
   1133       1.202     kamil 		if (ISSET(p->p_sflag, PS_WEXIT)) {
   1134       1.202     kamil 			mutex_exit(p->p_lock);
   1135       1.202     kamil 			/*
   1136       1.202     kamil 			 * We are exiting, bail out without informing parent
   1137       1.202     kamil 			 * about a terminating LWP as it would deadlock.
   1138       1.202     kamil 			 */
   1139       1.202     kamil 		} else {
   1140       1.203     kamil 			eventswitch(TRAP_LWP, PTRACE_LWP_EXIT, l->l_lid);
   1141       1.239        ad 			mutex_enter(&proc_lock);
   1142       1.202     kamil 		}
   1143       1.198     kamil 	}
   1144       1.198     kamil 
   1145        1.52        ad 	LIST_REMOVE(l, l_list);
   1146       1.239        ad 	mutex_exit(&proc_lock);
   1147        1.19  jdolecek 
   1148        1.52        ad 	/*
   1149        1.52        ad 	 * Get rid of all references to the LWP that others (e.g. procfs)
   1150        1.52        ad 	 * may have, and mark the LWP as a zombie.  If the LWP is detached,
   1151        1.52        ad 	 * mark it waiting for collection in the proc structure.  Note that
   1152        1.52        ad 	 * before we can do that, we need to free any other dead, deatched
   1153        1.52        ad 	 * LWP waiting to meet its maker.
   1154       1.231        ad 	 *
   1155       1.231        ad 	 * All conditions need to be observed upon under the same hold of
   1156       1.231        ad 	 * p_lock, because if the lock is dropped any of them can change.
   1157        1.52        ad 	 */
   1158       1.103        ad 	mutex_enter(p->p_lock);
   1159       1.231        ad 	for (;;) {
   1160       1.233   thorpej 		if (lwp_drainrefs(l))
   1161       1.231        ad 			continue;
   1162       1.231        ad 		if ((l->l_prflag & LPR_DETACHED) != 0) {
   1163       1.231        ad 			if ((l2 = p->p_zomblwp) != NULL) {
   1164       1.231        ad 				p->p_zomblwp = NULL;
   1165       1.231        ad 				lwp_free(l2, false, false);
   1166       1.231        ad 				/* proc now unlocked */
   1167       1.231        ad 				mutex_enter(p->p_lock);
   1168       1.231        ad 				continue;
   1169       1.231        ad 			}
   1170       1.231        ad 			p->p_zomblwp = l;
   1171        1.52        ad 		}
   1172       1.231        ad 		break;
   1173        1.52        ad 	}
   1174        1.31      yamt 
   1175        1.52        ad 	/*
   1176        1.52        ad 	 * If we find a pending signal for the process and we have been
   1177       1.151       chs 	 * asked to check for signals, then we lose: arrange to have
   1178        1.52        ad 	 * all other LWPs in the process check for signals.
   1179        1.52        ad 	 */
   1180        1.56     pavel 	if ((l->l_flag & LW_PENDSIG) != 0 &&
   1181        1.52        ad 	    firstsig(&p->p_sigpend.sp_set) != 0) {
   1182        1.52        ad 		LIST_FOREACH(l2, &p->p_lwps, l_sibling) {
   1183        1.52        ad 			lwp_lock(l2);
   1184       1.209        ad 			signotify(l2);
   1185        1.52        ad 			lwp_unlock(l2);
   1186        1.52        ad 		}
   1187        1.31      yamt 	}
   1188        1.31      yamt 
   1189       1.158      matt 	/*
   1190       1.158      matt 	 * Release any PCU resources before becoming a zombie.
   1191       1.158      matt 	 */
   1192       1.158      matt 	pcu_discard_all(l);
   1193       1.158      matt 
   1194        1.52        ad 	lwp_lock(l);
   1195        1.52        ad 	l->l_stat = LSZOMB;
   1196       1.162     rmind 	if (l->l_name != NULL) {
   1197        1.90        ad 		strcpy(l->l_name, "(zombie)");
   1198       1.128     rmind 	}
   1199        1.52        ad 	lwp_unlock(l);
   1200         1.2   thorpej 	p->p_nrlwps--;
   1201        1.52        ad 	cv_broadcast(&p->p_lwpcv);
   1202        1.78        ad 	if (l->l_lwpctl != NULL)
   1203        1.78        ad 		l->l_lwpctl->lc_curcpu = LWPCTL_CPU_EXITED;
   1204       1.103        ad 	mutex_exit(p->p_lock);
   1205        1.52        ad 
   1206        1.52        ad 	/*
   1207        1.52        ad 	 * We can no longer block.  At this point, lwp_free() may already
   1208        1.52        ad 	 * be gunning for us.  On a multi-CPU system, we may be off p_lwps.
   1209        1.52        ad 	 *
   1210        1.52        ad 	 * Free MD LWP resources.
   1211        1.52        ad 	 */
   1212        1.52        ad 	cpu_lwp_free(l, 0);
   1213         1.2   thorpej 
   1214        1.65        ad 	if (current) {
   1215       1.218        ad 		/* Switch away into oblivion. */
   1216       1.218        ad 		lwp_lock(l);
   1217       1.218        ad 		spc_lock(l->l_cpu);
   1218       1.218        ad 		mi_switch(l);
   1219       1.218        ad 		panic("lwp_exit");
   1220        1.65        ad 	}
   1221         1.2   thorpej }
   1222         1.2   thorpej 
   1223        1.52        ad /*
   1224        1.52        ad  * Free a dead LWP's remaining resources.
   1225        1.52        ad  *
   1226        1.52        ad  * XXXLWP limits.
   1227        1.52        ad  */
   1228        1.52        ad void
   1229        1.63        ad lwp_free(struct lwp *l, bool recycle, bool last)
   1230        1.52        ad {
   1231        1.52        ad 	struct proc *p = l->l_proc;
   1232       1.100        ad 	struct rusage *ru;
   1233        1.52        ad 	ksiginfoq_t kq;
   1234        1.52        ad 
   1235        1.92      yamt 	KASSERT(l != curlwp);
   1236       1.160      yamt 	KASSERT(last || mutex_owned(p->p_lock));
   1237        1.92      yamt 
   1238       1.177  christos 	/*
   1239       1.177  christos 	 * We use the process credentials instead of the lwp credentials here
   1240       1.177  christos 	 * because the lwp credentials maybe cached (just after a setuid call)
   1241       1.177  christos 	 * and we don't want pay for syncing, since the lwp is going away
   1242       1.177  christos 	 * anyway
   1243       1.177  christos 	 */
   1244       1.169  christos 	if (p != &proc0 && p->p_nlwps != 1)
   1245       1.177  christos 		(void)chglwpcnt(kauth_cred_getuid(p->p_cred), -1);
   1246       1.218        ad 
   1247        1.52        ad 	/*
   1248       1.238        ad 	 * In the unlikely event that the LWP is still on the CPU,
   1249       1.238        ad 	 * then spin until it has switched away.
   1250       1.238        ad 	 */
   1251       1.238        ad 	membar_consumer();
   1252       1.238        ad 	while (__predict_false((l->l_pflag & LP_RUNNING) != 0)) {
   1253       1.238        ad 		SPINLOCK_BACKOFF_HOOK;
   1254       1.238        ad 	}
   1255       1.238        ad 
   1256       1.238        ad 	/*
   1257       1.238        ad 	 * Now that the LWP's known off the CPU, reset its state back to
   1258       1.238        ad 	 * LSIDL, which defeats anything that might have gotten a hold on
   1259       1.238        ad 	 * the LWP via pid_table before the ID was freed.  It's important
   1260       1.238        ad 	 * to do this with both the LWP locked and p_lock held.
   1261       1.238        ad 	 *
   1262       1.238        ad 	 * Also reset the CPU and lock pointer back to curcpu(), since the
   1263       1.238        ad 	 * LWP will in all likelyhood be cached with the current CPU in
   1264       1.238        ad 	 * lwp_cache when we free it and later allocated from there again
   1265       1.238        ad 	 * (avoid incidental lock contention).
   1266       1.238        ad 	 */
   1267       1.238        ad 	lwp_lock(l);
   1268       1.238        ad 	l->l_stat = LSIDL;
   1269       1.238        ad 	l->l_cpu = curcpu();
   1270       1.238        ad 	lwp_unlock_to(l, l->l_cpu->ci_schedstate.spc_lwplock);
   1271       1.238        ad 
   1272       1.238        ad 	/*
   1273       1.223        ad 	 * If this was not the last LWP in the process, then adjust counters
   1274       1.223        ad 	 * and unlock.  This is done differently for the last LWP in exit1().
   1275        1.52        ad 	 */
   1276        1.52        ad 	if (!last) {
   1277        1.52        ad 		/*
   1278        1.52        ad 		 * Add the LWP's run time to the process' base value.
   1279        1.52        ad 		 * This needs to co-incide with coming off p_lwps.
   1280        1.52        ad 		 */
   1281        1.86      yamt 		bintime_add(&p->p_rtime, &l->l_rtime);
   1282        1.64      yamt 		p->p_pctcpu += l->l_pctcpu;
   1283       1.100        ad 		ru = &p->p_stats->p_ru;
   1284       1.100        ad 		ruadd(ru, &l->l_ru);
   1285       1.100        ad 		ru->ru_nvcsw += (l->l_ncsw - l->l_nivcsw);
   1286       1.100        ad 		ru->ru_nivcsw += l->l_nivcsw;
   1287        1.52        ad 		LIST_REMOVE(l, l_sibling);
   1288        1.52        ad 		p->p_nlwps--;
   1289        1.52        ad 		p->p_nzlwps--;
   1290        1.52        ad 		if ((l->l_prflag & LPR_DETACHED) != 0)
   1291        1.52        ad 			p->p_ndlwps--;
   1292        1.63        ad 
   1293        1.63        ad 		/*
   1294        1.63        ad 		 * Have any LWPs sleeping in lwp_wait() recheck for
   1295        1.63        ad 		 * deadlock.
   1296        1.63        ad 		 */
   1297        1.63        ad 		cv_broadcast(&p->p_lwpcv);
   1298       1.103        ad 		mutex_exit(p->p_lock);
   1299        1.52        ad 
   1300       1.238        ad 		/* Free the LWP ID. */
   1301       1.239        ad 		mutex_enter(&proc_lock);
   1302       1.238        ad 		proc_free_lwpid(p, l->l_lid);
   1303       1.239        ad 		mutex_exit(&proc_lock);
   1304        1.63        ad 	}
   1305        1.52        ad 
   1306        1.52        ad 	/*
   1307        1.52        ad 	 * Destroy the LWP's remaining signal information.
   1308        1.52        ad 	 */
   1309        1.52        ad 	ksiginfo_queue_init(&kq);
   1310        1.52        ad 	sigclear(&l->l_sigpend, NULL, &kq);
   1311        1.52        ad 	ksiginfo_queue_drain(&kq);
   1312        1.52        ad 	cv_destroy(&l->l_sigcv);
   1313       1.171     rmind 	cv_destroy(&l->l_waitcv);
   1314         1.2   thorpej 
   1315        1.19  jdolecek 	/*
   1316       1.162     rmind 	 * Free lwpctl structure and affinity.
   1317       1.162     rmind 	 */
   1318       1.162     rmind 	if (l->l_lwpctl) {
   1319       1.162     rmind 		lwp_ctl_free(l);
   1320       1.162     rmind 	}
   1321       1.162     rmind 	if (l->l_affinity) {
   1322       1.162     rmind 		kcpuset_unuse(l->l_affinity, NULL);
   1323       1.162     rmind 		l->l_affinity = NULL;
   1324       1.162     rmind 	}
   1325       1.162     rmind 
   1326       1.162     rmind 	/*
   1327       1.238        ad 	 * Free remaining data structures and the LWP itself unless the
   1328       1.238        ad 	 * caller wants to recycle.
   1329        1.19  jdolecek 	 */
   1330        1.90        ad 	if (l->l_name != NULL)
   1331        1.90        ad 		kmem_free(l->l_name, MAXCOMLEN);
   1332       1.135     rmind 
   1333       1.208      maxv 	kmsan_lwp_free(l);
   1334       1.232      maxv 	kcov_lwp_free(l);
   1335        1.52        ad 	cpu_lwp_free2(l);
   1336        1.19  jdolecek 	uvm_lwp_exit(l);
   1337       1.134     rmind 
   1338        1.60      yamt 	KASSERT(SLIST_EMPTY(&l->l_pi_lenders));
   1339        1.75        ad 	KASSERT(l->l_inheritedprio == -1);
   1340       1.155      matt 	KASSERT(l->l_blcnt == 0);
   1341       1.138    darran 	kdtrace_thread_dtor(NULL, l);
   1342        1.52        ad 	if (!recycle)
   1343        1.87        ad 		pool_cache_put(lwp_cache, l);
   1344         1.2   thorpej }
   1345         1.2   thorpej 
   1346         1.2   thorpej /*
   1347        1.91     rmind  * Migrate the LWP to the another CPU.  Unlocks the LWP.
   1348        1.91     rmind  */
   1349        1.91     rmind void
   1350       1.114     rmind lwp_migrate(lwp_t *l, struct cpu_info *tci)
   1351        1.91     rmind {
   1352       1.114     rmind 	struct schedstate_percpu *tspc;
   1353       1.121     rmind 	int lstat = l->l_stat;
   1354       1.121     rmind 
   1355        1.91     rmind 	KASSERT(lwp_locked(l, NULL));
   1356       1.114     rmind 	KASSERT(tci != NULL);
   1357       1.114     rmind 
   1358       1.121     rmind 	/* If LWP is still on the CPU, it must be handled like LSONPROC */
   1359       1.227        ad 	if ((l->l_pflag & LP_RUNNING) != 0) {
   1360       1.121     rmind 		lstat = LSONPROC;
   1361       1.121     rmind 	}
   1362       1.121     rmind 
   1363       1.114     rmind 	/*
   1364       1.114     rmind 	 * The destination CPU could be changed while previous migration
   1365       1.114     rmind 	 * was not finished.
   1366       1.114     rmind 	 */
   1367       1.121     rmind 	if (l->l_target_cpu != NULL) {
   1368       1.114     rmind 		l->l_target_cpu = tci;
   1369       1.114     rmind 		lwp_unlock(l);
   1370       1.114     rmind 		return;
   1371       1.114     rmind 	}
   1372        1.91     rmind 
   1373       1.114     rmind 	/* Nothing to do if trying to migrate to the same CPU */
   1374       1.114     rmind 	if (l->l_cpu == tci) {
   1375        1.91     rmind 		lwp_unlock(l);
   1376        1.91     rmind 		return;
   1377        1.91     rmind 	}
   1378        1.91     rmind 
   1379       1.114     rmind 	KASSERT(l->l_target_cpu == NULL);
   1380       1.114     rmind 	tspc = &tci->ci_schedstate;
   1381       1.121     rmind 	switch (lstat) {
   1382        1.91     rmind 	case LSRUN:
   1383       1.134     rmind 		l->l_target_cpu = tci;
   1384       1.134     rmind 		break;
   1385        1.91     rmind 	case LSSLEEP:
   1386       1.114     rmind 		l->l_cpu = tci;
   1387        1.91     rmind 		break;
   1388       1.212        ad 	case LSIDL:
   1389        1.91     rmind 	case LSSTOP:
   1390        1.91     rmind 	case LSSUSPENDED:
   1391       1.114     rmind 		l->l_cpu = tci;
   1392       1.114     rmind 		if (l->l_wchan == NULL) {
   1393       1.114     rmind 			lwp_unlock_to(l, tspc->spc_lwplock);
   1394       1.114     rmind 			return;
   1395        1.91     rmind 		}
   1396       1.114     rmind 		break;
   1397        1.91     rmind 	case LSONPROC:
   1398       1.114     rmind 		l->l_target_cpu = tci;
   1399       1.114     rmind 		spc_lock(l->l_cpu);
   1400       1.212        ad 		sched_resched_cpu(l->l_cpu, PRI_USER_RT, true);
   1401       1.212        ad 		/* spc now unlocked */
   1402        1.91     rmind 		break;
   1403        1.91     rmind 	}
   1404        1.91     rmind 	lwp_unlock(l);
   1405        1.91     rmind }
   1406        1.91     rmind 
   1407       1.237   thorpej #define	lwp_find_exclude(l)					\
   1408       1.237   thorpej 	((l)->l_stat == LSIDL || (l)->l_stat == LSZOMB)
   1409       1.237   thorpej 
   1410        1.91     rmind /*
   1411        1.94     rmind  * Find the LWP in the process.  Arguments may be zero, in such case,
   1412        1.94     rmind  * the calling process and first LWP in the list will be used.
   1413       1.103        ad  * On success - returns proc locked.
   1414       1.237   thorpej  *
   1415       1.237   thorpej  * => pid == 0 -> look in curproc.
   1416       1.237   thorpej  * => pid == -1 -> match any proc.
   1417       1.237   thorpej  * => otherwise look up the proc.
   1418       1.237   thorpej  *
   1419       1.237   thorpej  * => lid == 0 -> first LWP in the proc
   1420       1.237   thorpej  * => otherwise specific LWP
   1421        1.91     rmind  */
   1422        1.91     rmind struct lwp *
   1423        1.91     rmind lwp_find2(pid_t pid, lwpid_t lid)
   1424        1.91     rmind {
   1425        1.91     rmind 	proc_t *p;
   1426        1.91     rmind 	lwp_t *l;
   1427        1.91     rmind 
   1428       1.237   thorpej 	/* First LWP of specified proc. */
   1429       1.237   thorpej 	if (lid == 0) {
   1430       1.237   thorpej 		switch (pid) {
   1431       1.237   thorpej 		case -1:
   1432       1.237   thorpej 			/* No lookup keys. */
   1433       1.237   thorpej 			return NULL;
   1434       1.237   thorpej 		case 0:
   1435       1.237   thorpej 			p = curproc;
   1436       1.237   thorpej 			mutex_enter(p->p_lock);
   1437       1.237   thorpej 			break;
   1438       1.237   thorpej 		default:
   1439       1.239        ad 			mutex_enter(&proc_lock);
   1440       1.237   thorpej 			p = proc_find(pid);
   1441       1.237   thorpej 			if (__predict_false(p == NULL)) {
   1442       1.239        ad 				mutex_exit(&proc_lock);
   1443       1.237   thorpej 				return NULL;
   1444       1.237   thorpej 			}
   1445       1.237   thorpej 			mutex_enter(p->p_lock);
   1446       1.239        ad 			mutex_exit(&proc_lock);
   1447       1.237   thorpej 			break;
   1448       1.237   thorpej 		}
   1449       1.237   thorpej 		LIST_FOREACH(l, &p->p_lwps, l_sibling) {
   1450       1.237   thorpej 			if (__predict_true(!lwp_find_exclude(l)))
   1451       1.237   thorpej 				break;
   1452       1.150     rmind 		}
   1453       1.237   thorpej 		goto out;
   1454       1.237   thorpej 	}
   1455       1.237   thorpej 
   1456       1.237   thorpej 	l = proc_find_lwp_acquire_proc(lid, &p);
   1457       1.237   thorpej 	if (l == NULL)
   1458       1.237   thorpej 		return NULL;
   1459       1.237   thorpej 	KASSERT(p != NULL);
   1460       1.237   thorpej 	KASSERT(mutex_owned(p->p_lock));
   1461       1.237   thorpej 
   1462       1.237   thorpej 	if (__predict_false(lwp_find_exclude(l))) {
   1463       1.237   thorpej 		l = NULL;
   1464       1.237   thorpej 		goto out;
   1465       1.150     rmind 	}
   1466       1.237   thorpej 
   1467       1.237   thorpej 	/* Apply proc filter, if applicable. */
   1468       1.237   thorpej 	switch (pid) {
   1469       1.237   thorpej 	case -1:
   1470       1.237   thorpej 		/* Match anything. */
   1471       1.237   thorpej 		break;
   1472       1.237   thorpej 	case 0:
   1473       1.237   thorpej 		if (p != curproc)
   1474       1.237   thorpej 			l = NULL;
   1475       1.237   thorpej 		break;
   1476       1.237   thorpej 	default:
   1477       1.237   thorpej 		if (p->p_pid != pid)
   1478       1.237   thorpej 			l = NULL;
   1479       1.237   thorpej 		break;
   1480        1.94     rmind 	}
   1481       1.237   thorpej 
   1482       1.237   thorpej  out:
   1483       1.237   thorpej 	if (__predict_false(l == NULL)) {
   1484       1.103        ad 		mutex_exit(p->p_lock);
   1485       1.103        ad 	}
   1486        1.91     rmind 	return l;
   1487        1.91     rmind }
   1488        1.91     rmind 
   1489        1.91     rmind /*
   1490       1.168      yamt  * Look up a live LWP within the specified process.
   1491        1.52        ad  *
   1492       1.223        ad  * Must be called with p->p_lock held (as it looks at the radix tree,
   1493       1.223        ad  * and also wants to exclude idle and zombie LWPs).
   1494        1.52        ad  */
   1495        1.52        ad struct lwp *
   1496       1.151       chs lwp_find(struct proc *p, lwpid_t id)
   1497        1.52        ad {
   1498        1.52        ad 	struct lwp *l;
   1499        1.52        ad 
   1500       1.103        ad 	KASSERT(mutex_owned(p->p_lock));
   1501        1.52        ad 
   1502       1.235   thorpej 	l = proc_find_lwp(p, id);
   1503       1.223        ad 	KASSERT(l == NULL || l->l_lid == id);
   1504        1.52        ad 
   1505        1.52        ad 	/*
   1506        1.52        ad 	 * No need to lock - all of these conditions will
   1507        1.52        ad 	 * be visible with the process level mutex held.
   1508        1.52        ad 	 */
   1509       1.237   thorpej 	if (__predict_false(l != NULL && lwp_find_exclude(l)))
   1510        1.52        ad 		l = NULL;
   1511        1.52        ad 
   1512        1.52        ad 	return l;
   1513        1.52        ad }
   1514        1.52        ad 
   1515        1.52        ad /*
   1516        1.37        ad  * Update an LWP's cached credentials to mirror the process' master copy.
   1517        1.37        ad  *
   1518        1.37        ad  * This happens early in the syscall path, on user trap, and on LWP
   1519        1.37        ad  * creation.  A long-running LWP can also voluntarily choose to update
   1520       1.179       snj  * its credentials by calling this routine.  This may be called from
   1521        1.37        ad  * LWP_CACHE_CREDS(), which checks l->l_cred != p->p_cred beforehand.
   1522        1.37        ad  */
   1523        1.37        ad void
   1524        1.37        ad lwp_update_creds(struct lwp *l)
   1525        1.37        ad {
   1526        1.37        ad 	kauth_cred_t oc;
   1527        1.37        ad 	struct proc *p;
   1528        1.37        ad 
   1529        1.37        ad 	p = l->l_proc;
   1530        1.37        ad 	oc = l->l_cred;
   1531        1.37        ad 
   1532       1.103        ad 	mutex_enter(p->p_lock);
   1533        1.37        ad 	kauth_cred_hold(p->p_cred);
   1534        1.37        ad 	l->l_cred = p->p_cred;
   1535        1.98        ad 	l->l_prflag &= ~LPR_CRMOD;
   1536       1.103        ad 	mutex_exit(p->p_lock);
   1537        1.88        ad 	if (oc != NULL)
   1538        1.37        ad 		kauth_cred_free(oc);
   1539        1.52        ad }
   1540        1.52        ad 
   1541        1.52        ad /*
   1542        1.52        ad  * Verify that an LWP is locked, and optionally verify that the lock matches
   1543        1.52        ad  * one we specify.
   1544        1.52        ad  */
   1545        1.52        ad int
   1546        1.52        ad lwp_locked(struct lwp *l, kmutex_t *mtx)
   1547        1.52        ad {
   1548        1.52        ad 	kmutex_t *cur = l->l_mutex;
   1549        1.52        ad 
   1550        1.52        ad 	return mutex_owned(cur) && (mtx == cur || mtx == NULL);
   1551        1.52        ad }
   1552        1.52        ad 
   1553        1.52        ad /*
   1554        1.52        ad  * Lend a new mutex to an LWP.  The old mutex must be held.
   1555        1.52        ad  */
   1556       1.211        ad kmutex_t *
   1557       1.178      matt lwp_setlock(struct lwp *l, kmutex_t *mtx)
   1558        1.52        ad {
   1559       1.211        ad 	kmutex_t *oldmtx = l->l_mutex;
   1560        1.52        ad 
   1561       1.211        ad 	KASSERT(mutex_owned(oldmtx));
   1562        1.52        ad 
   1563       1.107        ad 	membar_exit();
   1564       1.178      matt 	l->l_mutex = mtx;
   1565       1.211        ad 	return oldmtx;
   1566        1.52        ad }
   1567        1.52        ad 
   1568        1.52        ad /*
   1569        1.52        ad  * Lend a new mutex to an LWP, and release the old mutex.  The old mutex
   1570        1.52        ad  * must be held.
   1571        1.52        ad  */
   1572        1.52        ad void
   1573       1.178      matt lwp_unlock_to(struct lwp *l, kmutex_t *mtx)
   1574        1.52        ad {
   1575        1.52        ad 	kmutex_t *old;
   1576        1.52        ad 
   1577       1.152     rmind 	KASSERT(lwp_locked(l, NULL));
   1578        1.52        ad 
   1579        1.52        ad 	old = l->l_mutex;
   1580       1.107        ad 	membar_exit();
   1581       1.178      matt 	l->l_mutex = mtx;
   1582        1.52        ad 	mutex_spin_exit(old);
   1583        1.52        ad }
   1584        1.52        ad 
   1585        1.60      yamt int
   1586        1.60      yamt lwp_trylock(struct lwp *l)
   1587        1.60      yamt {
   1588        1.60      yamt 	kmutex_t *old;
   1589        1.60      yamt 
   1590        1.60      yamt 	for (;;) {
   1591        1.60      yamt 		if (!mutex_tryenter(old = l->l_mutex))
   1592        1.60      yamt 			return 0;
   1593        1.60      yamt 		if (__predict_true(l->l_mutex == old))
   1594        1.60      yamt 			return 1;
   1595        1.60      yamt 		mutex_spin_exit(old);
   1596        1.60      yamt 	}
   1597        1.60      yamt }
   1598        1.60      yamt 
   1599       1.134     rmind void
   1600       1.211        ad lwp_unsleep(lwp_t *l, bool unlock)
   1601        1.96        ad {
   1602        1.96        ad 
   1603        1.96        ad 	KASSERT(mutex_owned(l->l_mutex));
   1604       1.211        ad 	(*l->l_syncobj->sobj_unsleep)(l, unlock);
   1605        1.96        ad }
   1606        1.96        ad 
   1607        1.52        ad /*
   1608        1.56     pavel  * Handle exceptions for mi_userret().  Called if a member of LW_USERRET is
   1609        1.52        ad  * set.
   1610        1.52        ad  */
   1611        1.52        ad void
   1612        1.52        ad lwp_userret(struct lwp *l)
   1613        1.52        ad {
   1614        1.52        ad 	struct proc *p;
   1615        1.52        ad 	int sig;
   1616        1.52        ad 
   1617       1.114     rmind 	KASSERT(l == curlwp);
   1618       1.114     rmind 	KASSERT(l->l_stat == LSONPROC);
   1619        1.52        ad 	p = l->l_proc;
   1620        1.52        ad 
   1621        1.52        ad 	/*
   1622       1.167     rmind 	 * It is safe to do this read unlocked on a MP system..
   1623        1.52        ad 	 */
   1624       1.167     rmind 	while ((l->l_flag & LW_USERRET) != 0) {
   1625        1.52        ad 		/*
   1626        1.52        ad 		 * Process pending signals first, unless the process
   1627        1.61        ad 		 * is dumping core or exiting, where we will instead
   1628       1.101     rmind 		 * enter the LW_WSUSPEND case below.
   1629        1.52        ad 		 */
   1630        1.61        ad 		if ((l->l_flag & (LW_PENDSIG | LW_WCORE | LW_WEXIT)) ==
   1631        1.61        ad 		    LW_PENDSIG) {
   1632       1.103        ad 			mutex_enter(p->p_lock);
   1633        1.52        ad 			while ((sig = issignal(l)) != 0)
   1634        1.52        ad 				postsig(sig);
   1635       1.103        ad 			mutex_exit(p->p_lock);
   1636        1.52        ad 		}
   1637        1.52        ad 
   1638        1.52        ad 		/*
   1639        1.52        ad 		 * Core-dump or suspend pending.
   1640        1.52        ad 		 *
   1641       1.159      matt 		 * In case of core dump, suspend ourselves, so that the kernel
   1642       1.159      matt 		 * stack and therefore the userland registers saved in the
   1643       1.159      matt 		 * trapframe are around for coredump() to write them out.
   1644       1.159      matt 		 * We also need to save any PCU resources that we have so that
   1645       1.159      matt 		 * they accessible for coredump().  We issue a wakeup on
   1646       1.159      matt 		 * p->p_lwpcv so that sigexit() will write the core file out
   1647       1.159      matt 		 * once all other LWPs are suspended.
   1648        1.52        ad 		 */
   1649        1.56     pavel 		if ((l->l_flag & LW_WSUSPEND) != 0) {
   1650       1.159      matt 			pcu_save_all(l);
   1651       1.103        ad 			mutex_enter(p->p_lock);
   1652        1.52        ad 			p->p_nrlwps--;
   1653        1.52        ad 			cv_broadcast(&p->p_lwpcv);
   1654        1.52        ad 			lwp_lock(l);
   1655        1.52        ad 			l->l_stat = LSSUSPENDED;
   1656       1.104        ad 			lwp_unlock(l);
   1657       1.103        ad 			mutex_exit(p->p_lock);
   1658       1.104        ad 			lwp_lock(l);
   1659       1.217        ad 			spc_lock(l->l_cpu);
   1660        1.64      yamt 			mi_switch(l);
   1661        1.52        ad 		}
   1662        1.52        ad 
   1663        1.52        ad 		/* Process is exiting. */
   1664        1.56     pavel 		if ((l->l_flag & LW_WEXIT) != 0) {
   1665        1.52        ad 			lwp_exit(l);
   1666        1.52        ad 			KASSERT(0);
   1667        1.52        ad 			/* NOTREACHED */
   1668        1.52        ad 		}
   1669       1.156     pooka 
   1670       1.156     pooka 		/* update lwpctl processor (for vfork child_return) */
   1671       1.156     pooka 		if (l->l_flag & LW_LWPCTL) {
   1672       1.156     pooka 			lwp_lock(l);
   1673       1.156     pooka 			KASSERT(kpreempt_disabled());
   1674       1.156     pooka 			l->l_lwpctl->lc_curcpu = (int)cpu_index(l->l_cpu);
   1675       1.156     pooka 			l->l_lwpctl->lc_pctr++;
   1676       1.156     pooka 			l->l_flag &= ~LW_LWPCTL;
   1677       1.156     pooka 			lwp_unlock(l);
   1678       1.156     pooka 		}
   1679        1.52        ad 	}
   1680        1.52        ad }
   1681        1.52        ad 
   1682        1.52        ad /*
   1683        1.52        ad  * Force an LWP to enter the kernel, to take a trip through lwp_userret().
   1684        1.52        ad  */
   1685        1.52        ad void
   1686        1.52        ad lwp_need_userret(struct lwp *l)
   1687        1.52        ad {
   1688       1.209        ad 
   1689       1.209        ad 	KASSERT(!cpu_intr_p());
   1690        1.63        ad 	KASSERT(lwp_locked(l, NULL));
   1691        1.52        ad 
   1692        1.52        ad 	/*
   1693       1.209        ad 	 * If the LWP is in any state other than LSONPROC, we know that it
   1694       1.209        ad 	 * is executing in-kernel and will hit userret() on the way out.
   1695       1.209        ad 	 *
   1696       1.209        ad 	 * If the LWP is curlwp, then we know we'll be back out to userspace
   1697       1.209        ad 	 * soon (can't be called from a hardware interrupt here).
   1698       1.209        ad 	 *
   1699       1.209        ad 	 * Otherwise, we can't be sure what the LWP is doing, so first make
   1700       1.209        ad 	 * sure the update to l_flag will be globally visible, and then
   1701       1.209        ad 	 * force the LWP to take a trip through trap() where it will do
   1702       1.209        ad 	 * userret().
   1703       1.209        ad 	 */
   1704       1.209        ad 	if (l->l_stat == LSONPROC && l != curlwp) {
   1705       1.209        ad 		membar_producer();
   1706       1.209        ad 		cpu_signotify(l);
   1707       1.209        ad 	}
   1708        1.52        ad }
   1709        1.52        ad 
   1710        1.52        ad /*
   1711        1.52        ad  * Add one reference to an LWP.  This will prevent the LWP from
   1712        1.52        ad  * exiting, thus keep the lwp structure and PCB around to inspect.
   1713        1.52        ad  */
   1714        1.52        ad void
   1715        1.52        ad lwp_addref(struct lwp *l)
   1716        1.52        ad {
   1717       1.103        ad 	KASSERT(mutex_owned(l->l_proc->p_lock));
   1718       1.237   thorpej 	KASSERT(l->l_stat != LSZOMB);
   1719       1.237   thorpej 	l->l_refcnt++;
   1720        1.52        ad }
   1721        1.52        ad 
   1722        1.52        ad /*
   1723        1.52        ad  * Remove one reference to an LWP.  If this is the last reference,
   1724        1.52        ad  * then we must finalize the LWP's death.
   1725        1.52        ad  */
   1726        1.52        ad void
   1727        1.52        ad lwp_delref(struct lwp *l)
   1728        1.52        ad {
   1729        1.52        ad 	struct proc *p = l->l_proc;
   1730        1.52        ad 
   1731       1.103        ad 	mutex_enter(p->p_lock);
   1732       1.142  christos 	lwp_delref2(l);
   1733       1.142  christos 	mutex_exit(p->p_lock);
   1734       1.142  christos }
   1735       1.142  christos 
   1736       1.142  christos /*
   1737       1.142  christos  * Remove one reference to an LWP.  If this is the last reference,
   1738       1.142  christos  * then we must finalize the LWP's death.  The proc mutex is held
   1739       1.142  christos  * on entry.
   1740       1.142  christos  */
   1741       1.142  christos void
   1742       1.142  christos lwp_delref2(struct lwp *l)
   1743       1.142  christos {
   1744       1.142  christos 	struct proc *p = l->l_proc;
   1745       1.142  christos 
   1746       1.142  christos 	KASSERT(mutex_owned(p->p_lock));
   1747        1.72        ad 	KASSERT(l->l_stat != LSZOMB);
   1748       1.237   thorpej 	KASSERT(l->l_refcnt > 0);
   1749       1.231        ad 
   1750       1.237   thorpej 	if (--l->l_refcnt == 0)
   1751        1.76        ad 		cv_broadcast(&p->p_lwpcv);
   1752        1.52        ad }
   1753        1.52        ad 
   1754        1.52        ad /*
   1755       1.233   thorpej  * Drain all references to the current LWP.  Returns true if
   1756       1.233   thorpej  * we blocked.
   1757        1.52        ad  */
   1758       1.233   thorpej bool
   1759        1.52        ad lwp_drainrefs(struct lwp *l)
   1760        1.52        ad {
   1761        1.52        ad 	struct proc *p = l->l_proc;
   1762       1.233   thorpej 	bool rv = false;
   1763        1.52        ad 
   1764       1.103        ad 	KASSERT(mutex_owned(p->p_lock));
   1765        1.52        ad 
   1766       1.233   thorpej 	l->l_prflag |= LPR_DRAINING;
   1767       1.233   thorpej 
   1768       1.237   thorpej 	while (l->l_refcnt > 0) {
   1769       1.233   thorpej 		rv = true;
   1770       1.103        ad 		cv_wait(&p->p_lwpcv, p->p_lock);
   1771       1.233   thorpej 	}
   1772       1.233   thorpej 	return rv;
   1773        1.37        ad }
   1774        1.41   thorpej 
   1775        1.41   thorpej /*
   1776       1.127        ad  * Return true if the specified LWP is 'alive'.  Only p->p_lock need
   1777       1.127        ad  * be held.
   1778       1.127        ad  */
   1779       1.127        ad bool
   1780       1.127        ad lwp_alive(lwp_t *l)
   1781       1.127        ad {
   1782       1.127        ad 
   1783       1.127        ad 	KASSERT(mutex_owned(l->l_proc->p_lock));
   1784       1.127        ad 
   1785       1.127        ad 	switch (l->l_stat) {
   1786       1.127        ad 	case LSSLEEP:
   1787       1.127        ad 	case LSRUN:
   1788       1.127        ad 	case LSONPROC:
   1789       1.127        ad 	case LSSTOP:
   1790       1.127        ad 	case LSSUSPENDED:
   1791       1.127        ad 		return true;
   1792       1.127        ad 	default:
   1793       1.127        ad 		return false;
   1794       1.127        ad 	}
   1795       1.127        ad }
   1796       1.127        ad 
   1797       1.127        ad /*
   1798       1.127        ad  * Return first live LWP in the process.
   1799       1.127        ad  */
   1800       1.127        ad lwp_t *
   1801       1.127        ad lwp_find_first(proc_t *p)
   1802       1.127        ad {
   1803       1.127        ad 	lwp_t *l;
   1804       1.127        ad 
   1805       1.127        ad 	KASSERT(mutex_owned(p->p_lock));
   1806       1.127        ad 
   1807       1.127        ad 	LIST_FOREACH(l, &p->p_lwps, l_sibling) {
   1808       1.127        ad 		if (lwp_alive(l)) {
   1809       1.127        ad 			return l;
   1810       1.127        ad 		}
   1811       1.127        ad 	}
   1812       1.127        ad 
   1813       1.127        ad 	return NULL;
   1814       1.127        ad }
   1815       1.127        ad 
   1816       1.127        ad /*
   1817        1.78        ad  * Allocate a new lwpctl structure for a user LWP.
   1818        1.78        ad  */
   1819        1.78        ad int
   1820        1.78        ad lwp_ctl_alloc(vaddr_t *uaddr)
   1821        1.78        ad {
   1822        1.78        ad 	lcproc_t *lp;
   1823        1.78        ad 	u_int bit, i, offset;
   1824        1.78        ad 	struct uvm_object *uao;
   1825        1.78        ad 	int error;
   1826        1.78        ad 	lcpage_t *lcp;
   1827        1.78        ad 	proc_t *p;
   1828        1.78        ad 	lwp_t *l;
   1829        1.78        ad 
   1830        1.78        ad 	l = curlwp;
   1831        1.78        ad 	p = l->l_proc;
   1832        1.78        ad 
   1833       1.156     pooka 	/* don't allow a vforked process to create lwp ctls */
   1834       1.156     pooka 	if (p->p_lflag & PL_PPWAIT)
   1835       1.156     pooka 		return EBUSY;
   1836       1.156     pooka 
   1837        1.81        ad 	if (l->l_lcpage != NULL) {
   1838        1.81        ad 		lcp = l->l_lcpage;
   1839        1.81        ad 		*uaddr = lcp->lcp_uaddr + (vaddr_t)l->l_lwpctl - lcp->lcp_kaddr;
   1840       1.143     njoly 		return 0;
   1841        1.81        ad 	}
   1842        1.78        ad 
   1843        1.78        ad 	/* First time around, allocate header structure for the process. */
   1844        1.78        ad 	if ((lp = p->p_lwpctl) == NULL) {
   1845        1.78        ad 		lp = kmem_alloc(sizeof(*lp), KM_SLEEP);
   1846        1.78        ad 		mutex_init(&lp->lp_lock, MUTEX_DEFAULT, IPL_NONE);
   1847        1.78        ad 		lp->lp_uao = NULL;
   1848        1.78        ad 		TAILQ_INIT(&lp->lp_pages);
   1849       1.103        ad 		mutex_enter(p->p_lock);
   1850        1.78        ad 		if (p->p_lwpctl == NULL) {
   1851        1.78        ad 			p->p_lwpctl = lp;
   1852       1.103        ad 			mutex_exit(p->p_lock);
   1853        1.78        ad 		} else {
   1854       1.103        ad 			mutex_exit(p->p_lock);
   1855        1.78        ad 			mutex_destroy(&lp->lp_lock);
   1856        1.78        ad 			kmem_free(lp, sizeof(*lp));
   1857        1.78        ad 			lp = p->p_lwpctl;
   1858        1.78        ad 		}
   1859        1.78        ad 	}
   1860        1.78        ad 
   1861        1.78        ad  	/*
   1862        1.78        ad  	 * Set up an anonymous memory region to hold the shared pages.
   1863        1.78        ad  	 * Map them into the process' address space.  The user vmspace
   1864        1.78        ad  	 * gets the first reference on the UAO.
   1865        1.78        ad  	 */
   1866        1.78        ad 	mutex_enter(&lp->lp_lock);
   1867        1.78        ad 	if (lp->lp_uao == NULL) {
   1868        1.78        ad 		lp->lp_uao = uao_create(LWPCTL_UAREA_SZ, 0);
   1869        1.78        ad 		lp->lp_cur = 0;
   1870        1.78        ad 		lp->lp_max = LWPCTL_UAREA_SZ;
   1871        1.78        ad 		lp->lp_uva = p->p_emul->e_vm_default_addr(p,
   1872       1.182    martin 		     (vaddr_t)p->p_vmspace->vm_daddr, LWPCTL_UAREA_SZ,
   1873       1.182    martin 		     p->p_vmspace->vm_map.flags & VM_MAP_TOPDOWN);
   1874        1.78        ad 		error = uvm_map(&p->p_vmspace->vm_map, &lp->lp_uva,
   1875        1.78        ad 		    LWPCTL_UAREA_SZ, lp->lp_uao, 0, 0, UVM_MAPFLAG(UVM_PROT_RW,
   1876        1.78        ad 		    UVM_PROT_RW, UVM_INH_NONE, UVM_ADV_NORMAL, 0));
   1877        1.78        ad 		if (error != 0) {
   1878        1.78        ad 			uao_detach(lp->lp_uao);
   1879        1.78        ad 			lp->lp_uao = NULL;
   1880        1.78        ad 			mutex_exit(&lp->lp_lock);
   1881        1.78        ad 			return error;
   1882        1.78        ad 		}
   1883        1.78        ad 	}
   1884        1.78        ad 
   1885        1.78        ad 	/* Get a free block and allocate for this LWP. */
   1886        1.78        ad 	TAILQ_FOREACH(lcp, &lp->lp_pages, lcp_chain) {
   1887        1.78        ad 		if (lcp->lcp_nfree != 0)
   1888        1.78        ad 			break;
   1889        1.78        ad 	}
   1890        1.78        ad 	if (lcp == NULL) {
   1891        1.78        ad 		/* Nothing available - try to set up a free page. */
   1892        1.78        ad 		if (lp->lp_cur == lp->lp_max) {
   1893        1.78        ad 			mutex_exit(&lp->lp_lock);
   1894        1.78        ad 			return ENOMEM;
   1895        1.78        ad 		}
   1896        1.78        ad 		lcp = kmem_alloc(LWPCTL_LCPAGE_SZ, KM_SLEEP);
   1897       1.189       chs 
   1898        1.78        ad 		/*
   1899        1.78        ad 		 * Wire the next page down in kernel space.  Since this
   1900        1.78        ad 		 * is a new mapping, we must add a reference.
   1901        1.78        ad 		 */
   1902        1.78        ad 		uao = lp->lp_uao;
   1903        1.78        ad 		(*uao->pgops->pgo_reference)(uao);
   1904        1.99        ad 		lcp->lcp_kaddr = vm_map_min(kernel_map);
   1905        1.78        ad 		error = uvm_map(kernel_map, &lcp->lcp_kaddr, PAGE_SIZE,
   1906        1.78        ad 		    uao, lp->lp_cur, PAGE_SIZE,
   1907        1.78        ad 		    UVM_MAPFLAG(UVM_PROT_RW, UVM_PROT_RW,
   1908        1.78        ad 		    UVM_INH_NONE, UVM_ADV_RANDOM, 0));
   1909        1.78        ad 		if (error != 0) {
   1910        1.78        ad 			mutex_exit(&lp->lp_lock);
   1911        1.78        ad 			kmem_free(lcp, LWPCTL_LCPAGE_SZ);
   1912        1.78        ad 			(*uao->pgops->pgo_detach)(uao);
   1913        1.78        ad 			return error;
   1914        1.78        ad 		}
   1915        1.89      yamt 		error = uvm_map_pageable(kernel_map, lcp->lcp_kaddr,
   1916        1.89      yamt 		    lcp->lcp_kaddr + PAGE_SIZE, FALSE, 0);
   1917        1.89      yamt 		if (error != 0) {
   1918        1.89      yamt 			mutex_exit(&lp->lp_lock);
   1919        1.89      yamt 			uvm_unmap(kernel_map, lcp->lcp_kaddr,
   1920        1.89      yamt 			    lcp->lcp_kaddr + PAGE_SIZE);
   1921        1.89      yamt 			kmem_free(lcp, LWPCTL_LCPAGE_SZ);
   1922        1.89      yamt 			return error;
   1923        1.89      yamt 		}
   1924        1.78        ad 		/* Prepare the page descriptor and link into the list. */
   1925        1.78        ad 		lcp->lcp_uaddr = lp->lp_uva + lp->lp_cur;
   1926        1.78        ad 		lp->lp_cur += PAGE_SIZE;
   1927        1.78        ad 		lcp->lcp_nfree = LWPCTL_PER_PAGE;
   1928        1.78        ad 		lcp->lcp_rotor = 0;
   1929        1.78        ad 		memset(lcp->lcp_bitmap, 0xff, LWPCTL_BITMAP_SZ);
   1930        1.78        ad 		TAILQ_INSERT_HEAD(&lp->lp_pages, lcp, lcp_chain);
   1931        1.78        ad 	}
   1932        1.78        ad 	for (i = lcp->lcp_rotor; lcp->lcp_bitmap[i] == 0;) {
   1933        1.78        ad 		if (++i >= LWPCTL_BITMAP_ENTRIES)
   1934        1.78        ad 			i = 0;
   1935        1.78        ad 	}
   1936        1.78        ad 	bit = ffs(lcp->lcp_bitmap[i]) - 1;
   1937       1.193     kamil 	lcp->lcp_bitmap[i] ^= (1U << bit);
   1938        1.78        ad 	lcp->lcp_rotor = i;
   1939        1.78        ad 	lcp->lcp_nfree--;
   1940        1.78        ad 	l->l_lcpage = lcp;
   1941        1.78        ad 	offset = (i << 5) + bit;
   1942        1.78        ad 	l->l_lwpctl = (lwpctl_t *)lcp->lcp_kaddr + offset;
   1943        1.78        ad 	*uaddr = lcp->lcp_uaddr + offset * sizeof(lwpctl_t);
   1944        1.78        ad 	mutex_exit(&lp->lp_lock);
   1945        1.78        ad 
   1946       1.107        ad 	KPREEMPT_DISABLE(l);
   1947       1.195     skrll 	l->l_lwpctl->lc_curcpu = (int)cpu_index(curcpu());
   1948       1.107        ad 	KPREEMPT_ENABLE(l);
   1949        1.78        ad 
   1950        1.78        ad 	return 0;
   1951        1.78        ad }
   1952        1.78        ad 
   1953        1.78        ad /*
   1954        1.78        ad  * Free an lwpctl structure back to the per-process list.
   1955        1.78        ad  */
   1956        1.78        ad void
   1957        1.78        ad lwp_ctl_free(lwp_t *l)
   1958        1.78        ad {
   1959       1.156     pooka 	struct proc *p = l->l_proc;
   1960        1.78        ad 	lcproc_t *lp;
   1961        1.78        ad 	lcpage_t *lcp;
   1962        1.78        ad 	u_int map, offset;
   1963        1.78        ad 
   1964       1.156     pooka 	/* don't free a lwp context we borrowed for vfork */
   1965       1.156     pooka 	if (p->p_lflag & PL_PPWAIT) {
   1966       1.156     pooka 		l->l_lwpctl = NULL;
   1967       1.156     pooka 		return;
   1968       1.156     pooka 	}
   1969       1.156     pooka 
   1970       1.156     pooka 	lp = p->p_lwpctl;
   1971        1.78        ad 	KASSERT(lp != NULL);
   1972        1.78        ad 
   1973        1.78        ad 	lcp = l->l_lcpage;
   1974        1.78        ad 	offset = (u_int)((lwpctl_t *)l->l_lwpctl - (lwpctl_t *)lcp->lcp_kaddr);
   1975        1.78        ad 	KASSERT(offset < LWPCTL_PER_PAGE);
   1976        1.78        ad 
   1977        1.78        ad 	mutex_enter(&lp->lp_lock);
   1978        1.78        ad 	lcp->lcp_nfree++;
   1979        1.78        ad 	map = offset >> 5;
   1980       1.194     kamil 	lcp->lcp_bitmap[map] |= (1U << (offset & 31));
   1981        1.78        ad 	if (lcp->lcp_bitmap[lcp->lcp_rotor] == 0)
   1982        1.78        ad 		lcp->lcp_rotor = map;
   1983        1.78        ad 	if (TAILQ_FIRST(&lp->lp_pages)->lcp_nfree == 0) {
   1984        1.78        ad 		TAILQ_REMOVE(&lp->lp_pages, lcp, lcp_chain);
   1985        1.78        ad 		TAILQ_INSERT_HEAD(&lp->lp_pages, lcp, lcp_chain);
   1986        1.78        ad 	}
   1987        1.78        ad 	mutex_exit(&lp->lp_lock);
   1988        1.78        ad }
   1989        1.78        ad 
   1990        1.78        ad /*
   1991        1.78        ad  * Process is exiting; tear down lwpctl state.  This can only be safely
   1992        1.78        ad  * called by the last LWP in the process.
   1993        1.78        ad  */
   1994        1.78        ad void
   1995        1.78        ad lwp_ctl_exit(void)
   1996        1.78        ad {
   1997        1.78        ad 	lcpage_t *lcp, *next;
   1998        1.78        ad 	lcproc_t *lp;
   1999        1.78        ad 	proc_t *p;
   2000        1.78        ad 	lwp_t *l;
   2001        1.78        ad 
   2002        1.78        ad 	l = curlwp;
   2003        1.78        ad 	l->l_lwpctl = NULL;
   2004        1.95        ad 	l->l_lcpage = NULL;
   2005        1.78        ad 	p = l->l_proc;
   2006        1.78        ad 	lp = p->p_lwpctl;
   2007        1.78        ad 
   2008        1.78        ad 	KASSERT(lp != NULL);
   2009        1.78        ad 	KASSERT(p->p_nlwps == 1);
   2010        1.78        ad 
   2011        1.78        ad 	for (lcp = TAILQ_FIRST(&lp->lp_pages); lcp != NULL; lcp = next) {
   2012        1.78        ad 		next = TAILQ_NEXT(lcp, lcp_chain);
   2013        1.78        ad 		uvm_unmap(kernel_map, lcp->lcp_kaddr,
   2014        1.78        ad 		    lcp->lcp_kaddr + PAGE_SIZE);
   2015        1.78        ad 		kmem_free(lcp, LWPCTL_LCPAGE_SZ);
   2016        1.78        ad 	}
   2017        1.78        ad 
   2018        1.78        ad 	if (lp->lp_uao != NULL) {
   2019        1.78        ad 		uvm_unmap(&p->p_vmspace->vm_map, lp->lp_uva,
   2020        1.78        ad 		    lp->lp_uva + LWPCTL_UAREA_SZ);
   2021        1.78        ad 	}
   2022        1.78        ad 
   2023        1.78        ad 	mutex_destroy(&lp->lp_lock);
   2024        1.78        ad 	kmem_free(lp, sizeof(*lp));
   2025        1.78        ad 	p->p_lwpctl = NULL;
   2026        1.78        ad }
   2027        1.84      yamt 
   2028       1.130        ad /*
   2029       1.130        ad  * Return the current LWP's "preemption counter".  Used to detect
   2030       1.130        ad  * preemption across operations that can tolerate preemption without
   2031       1.130        ad  * crashing, but which may generate incorrect results if preempted.
   2032       1.130        ad  */
   2033       1.130        ad uint64_t
   2034       1.130        ad lwp_pctr(void)
   2035       1.130        ad {
   2036       1.130        ad 
   2037       1.130        ad 	return curlwp->l_ncsw;
   2038       1.130        ad }
   2039       1.130        ad 
   2040       1.151       chs /*
   2041       1.151       chs  * Set an LWP's private data pointer.
   2042       1.151       chs  */
   2043       1.151       chs int
   2044       1.151       chs lwp_setprivate(struct lwp *l, void *ptr)
   2045       1.151       chs {
   2046       1.151       chs 	int error = 0;
   2047       1.151       chs 
   2048       1.151       chs 	l->l_private = ptr;
   2049       1.151       chs #ifdef __HAVE_CPU_LWP_SETPRIVATE
   2050       1.151       chs 	error = cpu_lwp_setprivate(l, ptr);
   2051       1.151       chs #endif
   2052       1.151       chs 	return error;
   2053       1.151       chs }
   2054       1.151       chs 
   2055       1.233   thorpej /*
   2056       1.233   thorpej  * Perform any thread-related cleanup on LWP exit.
   2057       1.233   thorpej  * N.B. l->l_proc->p_lock must be HELD on entry but will
   2058       1.233   thorpej  * be released before returning!
   2059       1.233   thorpej  */
   2060       1.233   thorpej void
   2061       1.233   thorpej lwp_thread_cleanup(struct lwp *l)
   2062       1.233   thorpej {
   2063       1.233   thorpej 
   2064       1.233   thorpej 	KASSERT(mutex_owned(l->l_proc->p_lock));
   2065       1.235   thorpej 	mutex_exit(l->l_proc->p_lock);
   2066       1.236   thorpej 
   2067       1.236   thorpej 	/*
   2068       1.236   thorpej 	 * If the LWP has robust futexes, release them all
   2069       1.236   thorpej 	 * now.
   2070       1.236   thorpej 	 */
   2071       1.236   thorpej 	if (__predict_false(l->l_robust_head != 0)) {
   2072  1.243.12.1   thorpej 		futex_release_all_lwp(l);
   2073       1.236   thorpej 	}
   2074       1.233   thorpej }
   2075       1.233   thorpej 
   2076        1.84      yamt #if defined(DDB)
   2077       1.153     rmind #include <machine/pcb.h>
   2078       1.153     rmind 
   2079        1.84      yamt void
   2080        1.84      yamt lwp_whatis(uintptr_t addr, void (*pr)(const char *, ...))
   2081        1.84      yamt {
   2082        1.84      yamt 	lwp_t *l;
   2083        1.84      yamt 
   2084        1.84      yamt 	LIST_FOREACH(l, &alllwp, l_list) {
   2085        1.84      yamt 		uintptr_t stack = (uintptr_t)KSTACK_LOWEST_ADDR(l);
   2086        1.84      yamt 
   2087        1.84      yamt 		if (addr < stack || stack + KSTACK_SIZE <= addr) {
   2088        1.84      yamt 			continue;
   2089        1.84      yamt 		}
   2090        1.84      yamt 		(*pr)("%p is %p+%zu, LWP %p's stack\n",
   2091        1.84      yamt 		    (void *)addr, (void *)stack,
   2092        1.84      yamt 		    (size_t)(addr - stack), l);
   2093        1.84      yamt 	}
   2094        1.84      yamt }
   2095        1.84      yamt #endif /* defined(DDB) */
   2096