kern_lwp.c revision 1.233.2.1 1 /* $NetBSD: kern_lwp.c,v 1.233.2.1 2020/04/20 11:29:10 bouyer Exp $ */
2
3 /*-
4 * Copyright (c) 2001, 2006, 2007, 2008, 2009, 2019, 2020
5 * The NetBSD Foundation, Inc.
6 * All rights reserved.
7 *
8 * This code is derived from software contributed to The NetBSD Foundation
9 * by Nathan J. Williams, and Andrew Doran.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 * POSSIBILITY OF SUCH DAMAGE.
31 */
32
33 /*
34 * Overview
35 *
36 * Lightweight processes (LWPs) are the basic unit or thread of
37 * execution within the kernel. The core state of an LWP is described
38 * by "struct lwp", also known as lwp_t.
39 *
40 * Each LWP is contained within a process (described by "struct proc"),
41 * Every process contains at least one LWP, but may contain more. The
42 * process describes attributes shared among all of its LWPs such as a
43 * private address space, global execution state (stopped, active,
44 * zombie, ...), signal disposition and so on. On a multiprocessor
45 * machine, multiple LWPs be executing concurrently in the kernel.
46 *
47 * Execution states
48 *
49 * At any given time, an LWP has overall state that is described by
50 * lwp::l_stat. The states are broken into two sets below. The first
51 * set is guaranteed to represent the absolute, current state of the
52 * LWP:
53 *
54 * LSONPROC
55 *
56 * On processor: the LWP is executing on a CPU, either in the
57 * kernel or in user space.
58 *
59 * LSRUN
60 *
61 * Runnable: the LWP is parked on a run queue, and may soon be
62 * chosen to run by an idle processor, or by a processor that
63 * has been asked to preempt a currently runnning but lower
64 * priority LWP.
65 *
66 * LSIDL
67 *
68 * Idle: the LWP has been created but has not yet executed,
69 * or it has ceased executing a unit of work and is waiting
70 * to be started again.
71 *
72 * LSSUSPENDED:
73 *
74 * Suspended: the LWP has had its execution suspended by
75 * another LWP in the same process using the _lwp_suspend()
76 * system call. User-level LWPs also enter the suspended
77 * state when the system is shutting down.
78 *
79 * The second set represent a "statement of intent" on behalf of the
80 * LWP. The LWP may in fact be executing on a processor, may be
81 * sleeping or idle. It is expected to take the necessary action to
82 * stop executing or become "running" again within a short timeframe.
83 * The LP_RUNNING flag in lwp::l_pflag indicates that an LWP is running.
84 * Importantly, it indicates that its state is tied to a CPU.
85 *
86 * LSZOMB:
87 *
88 * Dead or dying: the LWP has released most of its resources
89 * and is about to switch away into oblivion, or has already
90 * switched away. When it switches away, its few remaining
91 * resources can be collected.
92 *
93 * LSSLEEP:
94 *
95 * Sleeping: the LWP has entered itself onto a sleep queue, and
96 * has switched away or will switch away shortly to allow other
97 * LWPs to run on the CPU.
98 *
99 * LSSTOP:
100 *
101 * Stopped: the LWP has been stopped as a result of a job
102 * control signal, or as a result of the ptrace() interface.
103 *
104 * Stopped LWPs may run briefly within the kernel to handle
105 * signals that they receive, but will not return to user space
106 * until their process' state is changed away from stopped.
107 *
108 * Single LWPs within a process can not be set stopped
109 * selectively: all actions that can stop or continue LWPs
110 * occur at the process level.
111 *
112 * State transitions
113 *
114 * Note that the LSSTOP state may only be set when returning to
115 * user space in userret(), or when sleeping interruptably. The
116 * LSSUSPENDED state may only be set in userret(). Before setting
117 * those states, we try to ensure that the LWPs will release all
118 * locks that they hold, and at a minimum try to ensure that the
119 * LWP can be set runnable again by a signal.
120 *
121 * LWPs may transition states in the following ways:
122 *
123 * RUN -------> ONPROC ONPROC -----> RUN
124 * > SLEEP
125 * > STOPPED
126 * > SUSPENDED
127 * > ZOMB
128 * > IDL (special cases)
129 *
130 * STOPPED ---> RUN SUSPENDED --> RUN
131 * > SLEEP
132 *
133 * SLEEP -----> ONPROC IDL --------> RUN
134 * > RUN > SUSPENDED
135 * > STOPPED > STOPPED
136 * > ONPROC (special cases)
137 *
138 * Some state transitions are only possible with kernel threads (eg
139 * ONPROC -> IDL) and happen under tightly controlled circumstances
140 * free of unwanted side effects.
141 *
142 * Migration
143 *
144 * Migration of threads from one CPU to another could be performed
145 * internally by the scheduler via sched_takecpu() or sched_catchlwp()
146 * functions. The universal lwp_migrate() function should be used for
147 * any other cases. Subsystems in the kernel must be aware that CPU
148 * of LWP may change, while it is not locked.
149 *
150 * Locking
151 *
152 * The majority of fields in 'struct lwp' are covered by a single,
153 * general spin lock pointed to by lwp::l_mutex. The locks covering
154 * each field are documented in sys/lwp.h.
155 *
156 * State transitions must be made with the LWP's general lock held,
157 * and may cause the LWP's lock pointer to change. Manipulation of
158 * the general lock is not performed directly, but through calls to
159 * lwp_lock(), lwp_unlock() and others. It should be noted that the
160 * adaptive locks are not allowed to be released while the LWP's lock
161 * is being held (unlike for other spin-locks).
162 *
163 * States and their associated locks:
164 *
165 * LSIDL, LSONPROC, LSZOMB, LSSUPENDED:
166 *
167 * Always covered by spc_lwplock, which protects LWPs not
168 * associated with any other sync object. This is a per-CPU
169 * lock and matches lwp::l_cpu.
170 *
171 * LSRUN:
172 *
173 * Always covered by spc_mutex, which protects the run queues.
174 * This is a per-CPU lock and matches lwp::l_cpu.
175 *
176 * LSSLEEP:
177 *
178 * Covered by a lock associated with the sleep queue (sometimes
179 * a turnstile sleep queue) that the LWP resides on. This can
180 * be spc_lwplock for SOBJ_SLEEPQ_NULL (an "untracked" sleep).
181 *
182 * LSSTOP:
183 *
184 * If the LWP was previously sleeping (l_wchan != NULL), then
185 * l_mutex references the sleep queue lock. If the LWP was
186 * runnable or on the CPU when halted, or has been removed from
187 * the sleep queue since halted, then the lock is spc_lwplock.
188 *
189 * The lock order is as follows:
190 *
191 * sleepq -> turnstile -> spc_lwplock -> spc_mutex
192 *
193 * Each process has an scheduler state lock (proc::p_lock), and a
194 * number of counters on LWPs and their states: p_nzlwps, p_nrlwps, and
195 * so on. When an LWP is to be entered into or removed from one of the
196 * following states, p_lock must be held and the process wide counters
197 * adjusted:
198 *
199 * LSIDL, LSZOMB, LSSTOP, LSSUSPENDED
200 *
201 * (But not always for kernel threads. There are some special cases
202 * as mentioned above: soft interrupts, and the idle loops.)
203 *
204 * Note that an LWP is considered running or likely to run soon if in
205 * one of the following states. This affects the value of p_nrlwps:
206 *
207 * LSRUN, LSONPROC, LSSLEEP
208 *
209 * p_lock does not need to be held when transitioning among these
210 * three states, hence p_lock is rarely taken for state transitions.
211 */
212
213 #include <sys/cdefs.h>
214 __KERNEL_RCSID(0, "$NetBSD: kern_lwp.c,v 1.233.2.1 2020/04/20 11:29:10 bouyer Exp $");
215
216 #include "opt_ddb.h"
217 #include "opt_lockdebug.h"
218 #include "opt_dtrace.h"
219
220 #define _LWP_API_PRIVATE
221
222 #include <sys/param.h>
223 #include <sys/systm.h>
224 #include <sys/cpu.h>
225 #include <sys/pool.h>
226 #include <sys/proc.h>
227 #include <sys/syscallargs.h>
228 #include <sys/syscall_stats.h>
229 #include <sys/kauth.h>
230 #include <sys/sleepq.h>
231 #include <sys/lockdebug.h>
232 #include <sys/kmem.h>
233 #include <sys/pset.h>
234 #include <sys/intr.h>
235 #include <sys/lwpctl.h>
236 #include <sys/atomic.h>
237 #include <sys/filedesc.h>
238 #include <sys/fstrans.h>
239 #include <sys/dtrace_bsd.h>
240 #include <sys/sdt.h>
241 #include <sys/ptrace.h>
242 #include <sys/xcall.h>
243 #include <sys/uidinfo.h>
244 #include <sys/sysctl.h>
245 #include <sys/psref.h>
246 #include <sys/msan.h>
247 #include <sys/kcov.h>
248 #include <sys/thmap.h>
249 #include <sys/cprng.h>
250
251 #include <uvm/uvm_extern.h>
252 #include <uvm/uvm_object.h>
253
254 static pool_cache_t lwp_cache __read_mostly;
255 struct lwplist alllwp __cacheline_aligned;
256
257 /*
258 * Lookups by global thread ID operate outside of the normal LWP
259 * locking protocol.
260 *
261 * We are using a thmap, which internally can perform lookups lock-free.
262 * However, we still need to serialize lookups against LWP exit. We
263 * achieve this as follows:
264 *
265 * => Assignment of TID is performed lazily by the LWP itself, when it
266 * is first requested. Insertion into the thmap is done completely
267 * lock-free (other than the internal locking performed by thmap itself).
268 * Once the TID is published in the map, the l___tid field in the LWP
269 * is protected by p_lock.
270 *
271 * => When we look up an LWP in the thmap, we take lwp_threadid_lock as
272 * a READER. While still holding the lock, we add a reference to
273 * the LWP (using atomics). After adding the reference, we drop the
274 * lwp_threadid_lock. We now take p_lock and check the state of the
275 * LWP. If the LWP is draining its references or if the l___tid field
276 * has been invalidated, we drop the reference we took and return NULL.
277 * Otherwise, the lookup has succeeded and the LWP is returned with a
278 * reference count that the caller is responsible for dropping.
279 *
280 * => When a LWP is exiting it releases its TID. While holding the
281 * p_lock, the entry is deleted from the thmap and the l___tid field
282 * invalidated. Once the field is invalidated, p_lock is released.
283 * It is done in this sequence because the l___tid field is used as
284 * the lookup key storage in the thmap in order to conserve memory.
285 * Even if a lookup races with this process and succeeds only to have
286 * the TID invalidated, it's OK because it also results in a reference
287 * that will be drained later.
288 *
289 * => Deleting a node also requires GC of now-unused thmap nodes. The
290 * serialization point between stage_gc and gc is performed by simply
291 * taking the lwp_threadid_lock as a WRITER and immediately releasing
292 * it. By doing this, we know that any busy readers will have drained.
293 *
294 * => When a LWP is exiting, it also drains off any references being
295 * held by others. However, the reference in the lookup path is taken
296 * outside the normal locking protocol. There needs to be additional
297 * serialization so that EITHER lwp_drainrefs() sees the incremented
298 * reference count so that it knows to wait, OR lwp_getref_tid() sees
299 * that the LWP is waiting to drain and thus drops the reference
300 * immediately. This is achieved by taking lwp_threadid_lock as a
301 * WRITER when setting LPR_DRAINING. Note the locking order:
302 *
303 * p_lock -> lwp_threadid_lock
304 *
305 * Note that this scheme could easily use pserialize(9) in place of the
306 * lwp_threadid_lock rwlock lock. However, this would require placing a
307 * pserialize_perform() call in the LWP exit path, which is arguably more
308 * expensive than briefly taking a global lock that should be relatively
309 * uncontended. This issue can be revisited if the rwlock proves to be
310 * a performance problem.
311 */
312 static krwlock_t lwp_threadid_lock __cacheline_aligned;
313 static thmap_t * lwp_threadid_map __read_mostly;
314
315 static void lwp_dtor(void *, void *);
316
317 /* DTrace proc provider probes */
318 SDT_PROVIDER_DEFINE(proc);
319
320 SDT_PROBE_DEFINE1(proc, kernel, , lwp__create, "struct lwp *");
321 SDT_PROBE_DEFINE1(proc, kernel, , lwp__start, "struct lwp *");
322 SDT_PROBE_DEFINE1(proc, kernel, , lwp__exit, "struct lwp *");
323
324 struct turnstile turnstile0 __cacheline_aligned;
325 struct lwp lwp0 __aligned(MIN_LWP_ALIGNMENT) = {
326 #ifdef LWP0_CPU_INFO
327 .l_cpu = LWP0_CPU_INFO,
328 #endif
329 #ifdef LWP0_MD_INITIALIZER
330 .l_md = LWP0_MD_INITIALIZER,
331 #endif
332 .l_proc = &proc0,
333 .l_lid = 1,
334 .l_flag = LW_SYSTEM,
335 .l_stat = LSONPROC,
336 .l_ts = &turnstile0,
337 .l_syncobj = &sched_syncobj,
338 .l_refcnt = 0,
339 .l_priority = PRI_USER + NPRI_USER - 1,
340 .l_inheritedprio = -1,
341 .l_class = SCHED_OTHER,
342 .l_psid = PS_NONE,
343 .l_pi_lenders = SLIST_HEAD_INITIALIZER(&lwp0.l_pi_lenders),
344 .l_name = __UNCONST("swapper"),
345 .l_fd = &filedesc0,
346 };
347
348 static void lwp_threadid_init(void);
349 static int sysctl_kern_maxlwp(SYSCTLFN_PROTO);
350
351 /*
352 * sysctl helper routine for kern.maxlwp. Ensures that the new
353 * values are not too low or too high.
354 */
355 static int
356 sysctl_kern_maxlwp(SYSCTLFN_ARGS)
357 {
358 int error, nmaxlwp;
359 struct sysctlnode node;
360
361 nmaxlwp = maxlwp;
362 node = *rnode;
363 node.sysctl_data = &nmaxlwp;
364 error = sysctl_lookup(SYSCTLFN_CALL(&node));
365 if (error || newp == NULL)
366 return error;
367
368 if (nmaxlwp < 0 || nmaxlwp >= 65536)
369 return EINVAL;
370 if (nmaxlwp > cpu_maxlwp())
371 return EINVAL;
372 maxlwp = nmaxlwp;
373
374 return 0;
375 }
376
377 static void
378 sysctl_kern_lwp_setup(void)
379 {
380 struct sysctllog *clog = NULL;
381
382 sysctl_createv(&clog, 0, NULL, NULL,
383 CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
384 CTLTYPE_INT, "maxlwp",
385 SYSCTL_DESCR("Maximum number of simultaneous threads"),
386 sysctl_kern_maxlwp, 0, NULL, 0,
387 CTL_KERN, CTL_CREATE, CTL_EOL);
388 }
389
390 void
391 lwpinit(void)
392 {
393
394 LIST_INIT(&alllwp);
395 lwpinit_specificdata();
396 lwp_cache = pool_cache_init(sizeof(lwp_t), MIN_LWP_ALIGNMENT, 0, 0,
397 "lwppl", NULL, IPL_NONE, NULL, lwp_dtor, NULL);
398
399 maxlwp = cpu_maxlwp();
400 sysctl_kern_lwp_setup();
401 lwp_threadid_init();
402 }
403
404 void
405 lwp0_init(void)
406 {
407 struct lwp *l = &lwp0;
408
409 KASSERT((void *)uvm_lwp_getuarea(l) != NULL);
410 KASSERT(l->l_lid == proc0.p_nlwpid);
411
412 LIST_INSERT_HEAD(&alllwp, l, l_list);
413
414 callout_init(&l->l_timeout_ch, CALLOUT_MPSAFE);
415 callout_setfunc(&l->l_timeout_ch, sleepq_timeout, l);
416 cv_init(&l->l_sigcv, "sigwait");
417 cv_init(&l->l_waitcv, "vfork");
418
419 kauth_cred_hold(proc0.p_cred);
420 l->l_cred = proc0.p_cred;
421
422 kdtrace_thread_ctor(NULL, l);
423 lwp_initspecific(l);
424
425 SYSCALL_TIME_LWP_INIT(l);
426 }
427
428 static void
429 lwp_dtor(void *arg, void *obj)
430 {
431 lwp_t *l = obj;
432 (void)l;
433
434 /*
435 * Provide a barrier to ensure that all mutex_oncpu() and rw_oncpu()
436 * calls will exit before memory of LWP is returned to the pool, where
437 * KVA of LWP structure might be freed and re-used for other purposes.
438 * Kernel preemption is disabled around mutex_oncpu() and rw_oncpu()
439 * callers, therefore cross-call to all CPUs will do the job. Also,
440 * the value of l->l_cpu must be still valid at this point.
441 */
442 KASSERT(l->l_cpu != NULL);
443 xc_barrier(0);
444 }
445
446 /*
447 * Set an suspended.
448 *
449 * Must be called with p_lock held, and the LWP locked. Will unlock the
450 * LWP before return.
451 */
452 int
453 lwp_suspend(struct lwp *curl, struct lwp *t)
454 {
455 int error;
456
457 KASSERT(mutex_owned(t->l_proc->p_lock));
458 KASSERT(lwp_locked(t, NULL));
459
460 KASSERT(curl != t || curl->l_stat == LSONPROC);
461
462 /*
463 * If the current LWP has been told to exit, we must not suspend anyone
464 * else or deadlock could occur. We won't return to userspace.
465 */
466 if ((curl->l_flag & (LW_WEXIT | LW_WCORE)) != 0) {
467 lwp_unlock(t);
468 return (EDEADLK);
469 }
470
471 if ((t->l_flag & LW_DBGSUSPEND) != 0) {
472 lwp_unlock(t);
473 return 0;
474 }
475
476 error = 0;
477
478 switch (t->l_stat) {
479 case LSRUN:
480 case LSONPROC:
481 t->l_flag |= LW_WSUSPEND;
482 lwp_need_userret(t);
483 lwp_unlock(t);
484 break;
485
486 case LSSLEEP:
487 t->l_flag |= LW_WSUSPEND;
488
489 /*
490 * Kick the LWP and try to get it to the kernel boundary
491 * so that it will release any locks that it holds.
492 * setrunnable() will release the lock.
493 */
494 if ((t->l_flag & LW_SINTR) != 0)
495 setrunnable(t);
496 else
497 lwp_unlock(t);
498 break;
499
500 case LSSUSPENDED:
501 lwp_unlock(t);
502 break;
503
504 case LSSTOP:
505 t->l_flag |= LW_WSUSPEND;
506 setrunnable(t);
507 break;
508
509 case LSIDL:
510 case LSZOMB:
511 error = EINTR; /* It's what Solaris does..... */
512 lwp_unlock(t);
513 break;
514 }
515
516 return (error);
517 }
518
519 /*
520 * Restart a suspended LWP.
521 *
522 * Must be called with p_lock held, and the LWP locked. Will unlock the
523 * LWP before return.
524 */
525 void
526 lwp_continue(struct lwp *l)
527 {
528
529 KASSERT(mutex_owned(l->l_proc->p_lock));
530 KASSERT(lwp_locked(l, NULL));
531
532 /* If rebooting or not suspended, then just bail out. */
533 if ((l->l_flag & LW_WREBOOT) != 0) {
534 lwp_unlock(l);
535 return;
536 }
537
538 l->l_flag &= ~LW_WSUSPEND;
539
540 if (l->l_stat != LSSUSPENDED || (l->l_flag & LW_DBGSUSPEND) != 0) {
541 lwp_unlock(l);
542 return;
543 }
544
545 /* setrunnable() will release the lock. */
546 setrunnable(l);
547 }
548
549 /*
550 * Restart a stopped LWP.
551 *
552 * Must be called with p_lock held, and the LWP NOT locked. Will unlock the
553 * LWP before return.
554 */
555 void
556 lwp_unstop(struct lwp *l)
557 {
558 struct proc *p = l->l_proc;
559
560 KASSERT(mutex_owned(proc_lock));
561 KASSERT(mutex_owned(p->p_lock));
562
563 lwp_lock(l);
564
565 KASSERT((l->l_flag & LW_DBGSUSPEND) == 0);
566
567 /* If not stopped, then just bail out. */
568 if (l->l_stat != LSSTOP) {
569 lwp_unlock(l);
570 return;
571 }
572
573 p->p_stat = SACTIVE;
574 p->p_sflag &= ~PS_STOPPING;
575
576 if (!p->p_waited)
577 p->p_pptr->p_nstopchild--;
578
579 if (l->l_wchan == NULL) {
580 /* setrunnable() will release the lock. */
581 setrunnable(l);
582 } else if (p->p_xsig && (l->l_flag & LW_SINTR) != 0) {
583 /* setrunnable() so we can receive the signal */
584 setrunnable(l);
585 } else {
586 l->l_stat = LSSLEEP;
587 p->p_nrlwps++;
588 lwp_unlock(l);
589 }
590 }
591
592 /*
593 * Wait for an LWP within the current process to exit. If 'lid' is
594 * non-zero, we are waiting for a specific LWP.
595 *
596 * Must be called with p->p_lock held.
597 */
598 int
599 lwp_wait(struct lwp *l, lwpid_t lid, lwpid_t *departed, bool exiting)
600 {
601 const lwpid_t curlid = l->l_lid;
602 proc_t *p = l->l_proc;
603 lwp_t *l2, *next;
604 int error;
605
606 KASSERT(mutex_owned(p->p_lock));
607
608 p->p_nlwpwait++;
609 l->l_waitingfor = lid;
610
611 for (;;) {
612 int nfound;
613
614 /*
615 * Avoid a race between exit1() and sigexit(): if the
616 * process is dumping core, then we need to bail out: call
617 * into lwp_userret() where we will be suspended until the
618 * deed is done.
619 */
620 if ((p->p_sflag & PS_WCORE) != 0) {
621 mutex_exit(p->p_lock);
622 lwp_userret(l);
623 KASSERT(false);
624 }
625
626 /*
627 * First off, drain any detached LWP that is waiting to be
628 * reaped.
629 */
630 while ((l2 = p->p_zomblwp) != NULL) {
631 p->p_zomblwp = NULL;
632 lwp_free(l2, false, false);/* releases proc mutex */
633 mutex_enter(p->p_lock);
634 }
635
636 /*
637 * Now look for an LWP to collect. If the whole process is
638 * exiting, count detached LWPs as eligible to be collected,
639 * but don't drain them here.
640 */
641 nfound = 0;
642 error = 0;
643
644 /*
645 * If given a specific LID, go via the tree and make sure
646 * it's not detached.
647 */
648 if (lid != 0) {
649 l2 = radix_tree_lookup_node(&p->p_lwptree,
650 (uint64_t)(lid - 1));
651 if (l2 == NULL) {
652 error = ESRCH;
653 break;
654 }
655 KASSERT(l2->l_lid == lid);
656 if ((l2->l_prflag & LPR_DETACHED) != 0) {
657 error = EINVAL;
658 break;
659 }
660 } else {
661 l2 = LIST_FIRST(&p->p_lwps);
662 }
663 for (; l2 != NULL; l2 = next) {
664 next = (lid != 0 ? NULL : LIST_NEXT(l2, l_sibling));
665
666 /*
667 * If a specific wait and the target is waiting on
668 * us, then avoid deadlock. This also traps LWPs
669 * that try to wait on themselves.
670 *
671 * Note that this does not handle more complicated
672 * cycles, like: t1 -> t2 -> t3 -> t1. The process
673 * can still be killed so it is not a major problem.
674 */
675 if (l2->l_lid == lid && l2->l_waitingfor == curlid) {
676 error = EDEADLK;
677 break;
678 }
679 if (l2 == l)
680 continue;
681 if ((l2->l_prflag & LPR_DETACHED) != 0) {
682 nfound += exiting;
683 continue;
684 }
685 if (lid != 0) {
686 /*
687 * Mark this LWP as the first waiter, if there
688 * is no other.
689 */
690 if (l2->l_waiter == 0)
691 l2->l_waiter = curlid;
692 } else if (l2->l_waiter != 0) {
693 /*
694 * It already has a waiter - so don't
695 * collect it. If the waiter doesn't
696 * grab it we'll get another chance
697 * later.
698 */
699 nfound++;
700 continue;
701 }
702 nfound++;
703
704 /* No need to lock the LWP in order to see LSZOMB. */
705 if (l2->l_stat != LSZOMB)
706 continue;
707
708 /*
709 * We're no longer waiting. Reset the "first waiter"
710 * pointer on the target, in case it was us.
711 */
712 l->l_waitingfor = 0;
713 l2->l_waiter = 0;
714 p->p_nlwpwait--;
715 if (departed)
716 *departed = l2->l_lid;
717 sched_lwp_collect(l2);
718
719 /* lwp_free() releases the proc lock. */
720 lwp_free(l2, false, false);
721 mutex_enter(p->p_lock);
722 return 0;
723 }
724
725 if (error != 0)
726 break;
727 if (nfound == 0) {
728 error = ESRCH;
729 break;
730 }
731
732 /*
733 * Note: since the lock will be dropped, need to restart on
734 * wakeup to run all LWPs again, e.g. there may be new LWPs.
735 */
736 if (exiting) {
737 KASSERT(p->p_nlwps > 1);
738 error = cv_timedwait(&p->p_lwpcv, p->p_lock, 1);
739 break;
740 }
741
742 /*
743 * Break out if all LWPs are in _lwp_wait(). There are
744 * other ways to hang the process with _lwp_wait(), but the
745 * sleep is interruptable so little point checking for them.
746 */
747 if (p->p_nlwpwait == p->p_nlwps) {
748 error = EDEADLK;
749 break;
750 }
751
752 /*
753 * Sit around and wait for something to happen. We'll be
754 * awoken if any of the conditions examined change: if an
755 * LWP exits, is collected, or is detached.
756 */
757 if ((error = cv_wait_sig(&p->p_lwpcv, p->p_lock)) != 0)
758 break;
759 }
760
761 /*
762 * We didn't find any LWPs to collect, we may have received a
763 * signal, or some other condition has caused us to bail out.
764 *
765 * If waiting on a specific LWP, clear the waiters marker: some
766 * other LWP may want it. Then, kick all the remaining waiters
767 * so that they can re-check for zombies and for deadlock.
768 */
769 if (lid != 0) {
770 l2 = radix_tree_lookup_node(&p->p_lwptree,
771 (uint64_t)(lid - 1));
772 KASSERT(l2 == NULL || l2->l_lid == lid);
773
774 if (l2 != NULL && l2->l_waiter == curlid)
775 l2->l_waiter = 0;
776 }
777 p->p_nlwpwait--;
778 l->l_waitingfor = 0;
779 cv_broadcast(&p->p_lwpcv);
780
781 return error;
782 }
783
784 /*
785 * Find an unused LID for a new LWP.
786 */
787 static lwpid_t
788 lwp_find_free_lid(struct proc *p)
789 {
790 struct lwp *gang[32];
791 lwpid_t lid;
792 unsigned n;
793
794 KASSERT(mutex_owned(p->p_lock));
795 KASSERT(p->p_nlwpid > 0);
796
797 /*
798 * Scoot forward through the tree in blocks of LIDs doing gang
799 * lookup with dense=true, meaning the lookup will terminate the
800 * instant a hole is encountered. Most of the time the first entry
801 * (p->p_nlwpid) is free and the lookup fails fast.
802 */
803 for (lid = p->p_nlwpid;;) {
804 n = radix_tree_gang_lookup_node(&p->p_lwptree, lid - 1,
805 (void **)gang, __arraycount(gang), true);
806 if (n == 0) {
807 /* Start point was empty. */
808 break;
809 }
810 KASSERT(gang[0]->l_lid == lid);
811 lid = gang[n - 1]->l_lid + 1;
812 if (n < __arraycount(gang)) {
813 /* Scan encountered a hole. */
814 break;
815 }
816 }
817
818 return (lwpid_t)lid;
819 }
820
821 /*
822 * Create a new LWP within process 'p2', using LWP 'l1' as a template.
823 * The new LWP is created in state LSIDL and must be set running,
824 * suspended, or stopped by the caller.
825 */
826 int
827 lwp_create(lwp_t *l1, proc_t *p2, vaddr_t uaddr, int flags,
828 void *stack, size_t stacksize, void (*func)(void *), void *arg,
829 lwp_t **rnewlwpp, int sclass, const sigset_t *sigmask,
830 const stack_t *sigstk)
831 {
832 struct lwp *l2;
833 turnstile_t *ts;
834 lwpid_t lid;
835
836 KASSERT(l1 == curlwp || l1->l_proc == &proc0);
837
838 /*
839 * Enforce limits, excluding the first lwp and kthreads. We must
840 * use the process credentials here when adjusting the limit, as
841 * they are what's tied to the accounting entity. However for
842 * authorizing the action, we'll use the LWP's credentials.
843 */
844 mutex_enter(p2->p_lock);
845 if (p2->p_nlwps != 0 && p2 != &proc0) {
846 uid_t uid = kauth_cred_getuid(p2->p_cred);
847 int count = chglwpcnt(uid, 1);
848 if (__predict_false(count >
849 p2->p_rlimit[RLIMIT_NTHR].rlim_cur)) {
850 if (kauth_authorize_process(l1->l_cred,
851 KAUTH_PROCESS_RLIMIT, p2,
852 KAUTH_ARG(KAUTH_REQ_PROCESS_RLIMIT_BYPASS),
853 &p2->p_rlimit[RLIMIT_NTHR], KAUTH_ARG(RLIMIT_NTHR))
854 != 0) {
855 (void)chglwpcnt(uid, -1);
856 mutex_exit(p2->p_lock);
857 return EAGAIN;
858 }
859 }
860 }
861
862 /*
863 * First off, reap any detached LWP waiting to be collected.
864 * We can re-use its LWP structure and turnstile.
865 */
866 if ((l2 = p2->p_zomblwp) != NULL) {
867 p2->p_zomblwp = NULL;
868 lwp_free(l2, true, false);
869 /* p2 now unlocked by lwp_free() */
870 ts = l2->l_ts;
871 KASSERT(l2->l_inheritedprio == -1);
872 KASSERT(SLIST_EMPTY(&l2->l_pi_lenders));
873 memset(l2, 0, sizeof(*l2));
874 l2->l_ts = ts;
875 } else {
876 mutex_exit(p2->p_lock);
877 l2 = pool_cache_get(lwp_cache, PR_WAITOK);
878 memset(l2, 0, sizeof(*l2));
879 l2->l_ts = pool_cache_get(turnstile_cache, PR_WAITOK);
880 SLIST_INIT(&l2->l_pi_lenders);
881 }
882
883 l2->l_stat = LSIDL;
884 l2->l_proc = p2;
885 l2->l_refcnt = 0;
886 l2->l_class = sclass;
887
888 /*
889 * If vfork(), we want the LWP to run fast and on the same CPU
890 * as its parent, so that it can reuse the VM context and cache
891 * footprint on the local CPU.
892 */
893 l2->l_kpriority = ((flags & LWP_VFORK) ? true : false);
894 l2->l_kpribase = PRI_KERNEL;
895 l2->l_priority = l1->l_priority;
896 l2->l_inheritedprio = -1;
897 l2->l_protectprio = -1;
898 l2->l_auxprio = -1;
899 l2->l_flag = 0;
900 l2->l_pflag = LP_MPSAFE;
901 TAILQ_INIT(&l2->l_ld_locks);
902 l2->l_psrefs = 0;
903 kmsan_lwp_alloc(l2);
904
905 /*
906 * For vfork, borrow parent's lwpctl context if it exists.
907 * This also causes us to return via lwp_userret.
908 */
909 if (flags & LWP_VFORK && l1->l_lwpctl) {
910 l2->l_lwpctl = l1->l_lwpctl;
911 l2->l_flag |= LW_LWPCTL;
912 }
913
914 /*
915 * If not the first LWP in the process, grab a reference to the
916 * descriptor table.
917 */
918 l2->l_fd = p2->p_fd;
919 if (p2->p_nlwps != 0) {
920 KASSERT(l1->l_proc == p2);
921 fd_hold(l2);
922 } else {
923 KASSERT(l1->l_proc != p2);
924 }
925
926 if (p2->p_flag & PK_SYSTEM) {
927 /* Mark it as a system LWP. */
928 l2->l_flag |= LW_SYSTEM;
929 }
930
931 kpreempt_disable();
932 l2->l_mutex = l1->l_cpu->ci_schedstate.spc_lwplock;
933 l2->l_cpu = l1->l_cpu;
934 kpreempt_enable();
935
936 kdtrace_thread_ctor(NULL, l2);
937 lwp_initspecific(l2);
938 sched_lwp_fork(l1, l2);
939 lwp_update_creds(l2);
940 callout_init(&l2->l_timeout_ch, CALLOUT_MPSAFE);
941 callout_setfunc(&l2->l_timeout_ch, sleepq_timeout, l2);
942 cv_init(&l2->l_sigcv, "sigwait");
943 cv_init(&l2->l_waitcv, "vfork");
944 l2->l_syncobj = &sched_syncobj;
945 PSREF_DEBUG_INIT_LWP(l2);
946
947 if (rnewlwpp != NULL)
948 *rnewlwpp = l2;
949
950 /*
951 * PCU state needs to be saved before calling uvm_lwp_fork() so that
952 * the MD cpu_lwp_fork() can copy the saved state to the new LWP.
953 */
954 pcu_save_all(l1);
955 #if PCU_UNIT_COUNT > 0
956 l2->l_pcu_valid = l1->l_pcu_valid;
957 #endif
958
959 uvm_lwp_setuarea(l2, uaddr);
960 uvm_lwp_fork(l1, l2, stack, stacksize, func, (arg != NULL) ? arg : l2);
961
962 if ((flags & LWP_PIDLID) != 0) {
963 /* Linux threads: use a PID. */
964 lid = proc_alloc_pid(p2);
965 l2->l_pflag |= LP_PIDLID;
966 } else if (p2->p_nlwps == 0) {
967 /*
968 * First LWP in process. Copy the parent's LID to avoid
969 * causing problems for fork() + threads. Don't give
970 * subsequent threads the distinction of using LID 1.
971 */
972 lid = l1->l_lid;
973 p2->p_nlwpid = 2;
974 } else {
975 /* Scan the radix tree for a free LID. */
976 lid = 0;
977 }
978
979 /*
980 * Allocate LID if needed, and insert into the radix tree. The
981 * first LWP in most processes has a LID of 1. It turns out that if
982 * you insert an item with a key of zero to a radixtree, it's stored
983 * directly in the root (p_lwptree) and no extra memory is
984 * allocated. We therefore always subtract 1 from the LID, which
985 * means no memory is allocated for the tree unless the program is
986 * using threads. NB: the allocation and insert must take place
987 * under the same hold of p_lock.
988 */
989 mutex_enter(p2->p_lock);
990 for (;;) {
991 int error;
992
993 l2->l_lid = (lid == 0 ? lwp_find_free_lid(p2) : lid);
994
995 rw_enter(&p2->p_treelock, RW_WRITER);
996 error = radix_tree_insert_node(&p2->p_lwptree,
997 (uint64_t)(l2->l_lid - 1), l2);
998 rw_exit(&p2->p_treelock);
999
1000 if (__predict_true(error == 0)) {
1001 if (lid == 0)
1002 p2->p_nlwpid = l2->l_lid + 1;
1003 break;
1004 }
1005
1006 KASSERT(error == ENOMEM);
1007 mutex_exit(p2->p_lock);
1008 radix_tree_await_memory();
1009 mutex_enter(p2->p_lock);
1010 }
1011
1012 if ((flags & LWP_DETACHED) != 0) {
1013 l2->l_prflag = LPR_DETACHED;
1014 p2->p_ndlwps++;
1015 } else
1016 l2->l_prflag = 0;
1017
1018 if (l1->l_proc == p2) {
1019 /*
1020 * These flags are set while p_lock is held. Copy with
1021 * p_lock held too, so the LWP doesn't sneak into the
1022 * process without them being set.
1023 */
1024 l2->l_flag |= (l1->l_flag & (LW_WEXIT | LW_WREBOOT | LW_WCORE));
1025 } else {
1026 /* fork(): pending core/exit doesn't apply to child. */
1027 l2->l_flag |= (l1->l_flag & LW_WREBOOT);
1028 }
1029
1030 l2->l_sigstk = *sigstk;
1031 l2->l_sigmask = *sigmask;
1032 TAILQ_INIT(&l2->l_sigpend.sp_info);
1033 sigemptyset(&l2->l_sigpend.sp_set);
1034 LIST_INSERT_HEAD(&p2->p_lwps, l2, l_sibling);
1035 p2->p_nlwps++;
1036 p2->p_nrlwps++;
1037
1038 KASSERT(l2->l_affinity == NULL);
1039
1040 /* Inherit the affinity mask. */
1041 if (l1->l_affinity) {
1042 /*
1043 * Note that we hold the state lock while inheriting
1044 * the affinity to avoid race with sched_setaffinity().
1045 */
1046 lwp_lock(l1);
1047 if (l1->l_affinity) {
1048 kcpuset_use(l1->l_affinity);
1049 l2->l_affinity = l1->l_affinity;
1050 }
1051 lwp_unlock(l1);
1052 }
1053
1054 /* This marks the end of the "must be atomic" section. */
1055 mutex_exit(p2->p_lock);
1056
1057 SDT_PROBE(proc, kernel, , lwp__create, l2, 0, 0, 0, 0);
1058
1059 mutex_enter(proc_lock);
1060 LIST_INSERT_HEAD(&alllwp, l2, l_list);
1061 /* Inherit a processor-set */
1062 l2->l_psid = l1->l_psid;
1063 mutex_exit(proc_lock);
1064
1065 SYSCALL_TIME_LWP_INIT(l2);
1066
1067 if (p2->p_emul->e_lwp_fork)
1068 (*p2->p_emul->e_lwp_fork)(l1, l2);
1069
1070 return (0);
1071 }
1072
1073 /*
1074 * Set a new LWP running. If the process is stopping, then the LWP is
1075 * created stopped.
1076 */
1077 void
1078 lwp_start(lwp_t *l, int flags)
1079 {
1080 proc_t *p = l->l_proc;
1081
1082 mutex_enter(p->p_lock);
1083 lwp_lock(l);
1084 KASSERT(l->l_stat == LSIDL);
1085 if ((flags & LWP_SUSPENDED) != 0) {
1086 /* It'll suspend itself in lwp_userret(). */
1087 l->l_flag |= LW_WSUSPEND;
1088 }
1089 if (p->p_stat == SSTOP || (p->p_sflag & PS_STOPPING) != 0) {
1090 KASSERT(l->l_wchan == NULL);
1091 l->l_stat = LSSTOP;
1092 p->p_nrlwps--;
1093 lwp_unlock(l);
1094 } else {
1095 setrunnable(l);
1096 /* LWP now unlocked */
1097 }
1098 mutex_exit(p->p_lock);
1099 }
1100
1101 /*
1102 * Called by MD code when a new LWP begins execution. Must be called
1103 * with the previous LWP locked (so at splsched), or if there is no
1104 * previous LWP, at splsched.
1105 */
1106 void
1107 lwp_startup(struct lwp *prev, struct lwp *new_lwp)
1108 {
1109 kmutex_t *lock;
1110
1111 KASSERTMSG(new_lwp == curlwp, "l %p curlwp %p prevlwp %p", new_lwp, curlwp, prev);
1112 KASSERT(kpreempt_disabled());
1113 KASSERT(prev != NULL);
1114 KASSERT((prev->l_pflag & LP_RUNNING) != 0);
1115 KASSERT(curcpu()->ci_mtx_count == -2);
1116
1117 /*
1118 * Immediately mark the previous LWP as no longer running and unlock
1119 * (to keep lock wait times short as possible). If a zombie, don't
1120 * touch after clearing LP_RUNNING as it could be reaped by another
1121 * CPU. Issue a memory barrier to ensure this.
1122 */
1123 lock = prev->l_mutex;
1124 if (__predict_false(prev->l_stat == LSZOMB)) {
1125 membar_sync();
1126 }
1127 prev->l_pflag &= ~LP_RUNNING;
1128 mutex_spin_exit(lock);
1129
1130 /* Correct spin mutex count after mi_switch(). */
1131 curcpu()->ci_mtx_count = 0;
1132
1133 /* Install new VM context. */
1134 if (__predict_true(new_lwp->l_proc->p_vmspace)) {
1135 pmap_activate(new_lwp);
1136 }
1137
1138 /* We remain at IPL_SCHED from mi_switch() - reset it. */
1139 spl0();
1140
1141 LOCKDEBUG_BARRIER(NULL, 0);
1142 SDT_PROBE(proc, kernel, , lwp__start, new_lwp, 0, 0, 0, 0);
1143
1144 /* For kthreads, acquire kernel lock if not MPSAFE. */
1145 if (__predict_false((new_lwp->l_pflag & LP_MPSAFE) == 0)) {
1146 KERNEL_LOCK(1, new_lwp);
1147 }
1148 }
1149
1150 /*
1151 * Exit an LWP.
1152 */
1153 void
1154 lwp_exit(struct lwp *l)
1155 {
1156 struct proc *p = l->l_proc;
1157 struct lwp *l2;
1158 bool current;
1159
1160 current = (l == curlwp);
1161
1162 KASSERT(current || (l->l_stat == LSIDL && l->l_target_cpu == NULL));
1163 KASSERT(p == curproc);
1164
1165 SDT_PROBE(proc, kernel, , lwp__exit, l, 0, 0, 0, 0);
1166
1167 /* Verify that we hold no locks; for DIAGNOSTIC check kernel_lock. */
1168 LOCKDEBUG_BARRIER(NULL, 0);
1169 KASSERTMSG(curcpu()->ci_biglock_count == 0, "kernel_lock leaked");
1170
1171 /*
1172 * If we are the last live LWP in a process, we need to exit the
1173 * entire process. We do so with an exit status of zero, because
1174 * it's a "controlled" exit, and because that's what Solaris does.
1175 *
1176 * We are not quite a zombie yet, but for accounting purposes we
1177 * must increment the count of zombies here.
1178 *
1179 * Note: the last LWP's specificdata will be deleted here.
1180 */
1181 mutex_enter(p->p_lock);
1182 if (p->p_nlwps - p->p_nzlwps == 1) {
1183 KASSERT(current == true);
1184 KASSERT(p != &proc0);
1185 exit1(l, 0, 0);
1186 /* NOTREACHED */
1187 }
1188 p->p_nzlwps++;
1189
1190 /*
1191 * Perform any required thread cleanup. Do this early so
1192 * anyone wanting to look us up by our global thread ID
1193 * will fail to find us.
1194 *
1195 * N.B. this will unlock p->p_lock on our behalf.
1196 */
1197 lwp_thread_cleanup(l);
1198
1199 if (p->p_emul->e_lwp_exit)
1200 (*p->p_emul->e_lwp_exit)(l);
1201
1202 /* Drop filedesc reference. */
1203 fd_free();
1204
1205 /* Release fstrans private data. */
1206 fstrans_lwp_dtor(l);
1207
1208 /* Delete the specificdata while it's still safe to sleep. */
1209 lwp_finispecific(l);
1210
1211 /*
1212 * Release our cached credentials.
1213 */
1214 kauth_cred_free(l->l_cred);
1215 callout_destroy(&l->l_timeout_ch);
1216
1217 /*
1218 * If traced, report LWP exit event to the debugger.
1219 *
1220 * Remove the LWP from the global list.
1221 * Free its LID from the PID namespace if needed.
1222 */
1223 mutex_enter(proc_lock);
1224
1225 if ((p->p_slflag & (PSL_TRACED|PSL_TRACELWP_EXIT)) ==
1226 (PSL_TRACED|PSL_TRACELWP_EXIT)) {
1227 mutex_enter(p->p_lock);
1228 if (ISSET(p->p_sflag, PS_WEXIT)) {
1229 mutex_exit(p->p_lock);
1230 /*
1231 * We are exiting, bail out without informing parent
1232 * about a terminating LWP as it would deadlock.
1233 */
1234 } else {
1235 eventswitch(TRAP_LWP, PTRACE_LWP_EXIT, l->l_lid);
1236 mutex_enter(proc_lock);
1237 }
1238 }
1239
1240 LIST_REMOVE(l, l_list);
1241 if ((l->l_pflag & LP_PIDLID) != 0 && l->l_lid != p->p_pid) {
1242 proc_free_pid(l->l_lid);
1243 }
1244 mutex_exit(proc_lock);
1245
1246 /*
1247 * Get rid of all references to the LWP that others (e.g. procfs)
1248 * may have, and mark the LWP as a zombie. If the LWP is detached,
1249 * mark it waiting for collection in the proc structure. Note that
1250 * before we can do that, we need to free any other dead, deatched
1251 * LWP waiting to meet its maker.
1252 *
1253 * All conditions need to be observed upon under the same hold of
1254 * p_lock, because if the lock is dropped any of them can change.
1255 */
1256 mutex_enter(p->p_lock);
1257 for (;;) {
1258 if (lwp_drainrefs(l))
1259 continue;
1260 if ((l->l_prflag & LPR_DETACHED) != 0) {
1261 if ((l2 = p->p_zomblwp) != NULL) {
1262 p->p_zomblwp = NULL;
1263 lwp_free(l2, false, false);
1264 /* proc now unlocked */
1265 mutex_enter(p->p_lock);
1266 continue;
1267 }
1268 p->p_zomblwp = l;
1269 }
1270 break;
1271 }
1272
1273 /*
1274 * If we find a pending signal for the process and we have been
1275 * asked to check for signals, then we lose: arrange to have
1276 * all other LWPs in the process check for signals.
1277 */
1278 if ((l->l_flag & LW_PENDSIG) != 0 &&
1279 firstsig(&p->p_sigpend.sp_set) != 0) {
1280 LIST_FOREACH(l2, &p->p_lwps, l_sibling) {
1281 lwp_lock(l2);
1282 signotify(l2);
1283 lwp_unlock(l2);
1284 }
1285 }
1286
1287 /*
1288 * Release any PCU resources before becoming a zombie.
1289 */
1290 pcu_discard_all(l);
1291
1292 lwp_lock(l);
1293 l->l_stat = LSZOMB;
1294 if (l->l_name != NULL) {
1295 strcpy(l->l_name, "(zombie)");
1296 }
1297 lwp_unlock(l);
1298 p->p_nrlwps--;
1299 cv_broadcast(&p->p_lwpcv);
1300 if (l->l_lwpctl != NULL)
1301 l->l_lwpctl->lc_curcpu = LWPCTL_CPU_EXITED;
1302 mutex_exit(p->p_lock);
1303
1304 /*
1305 * We can no longer block. At this point, lwp_free() may already
1306 * be gunning for us. On a multi-CPU system, we may be off p_lwps.
1307 *
1308 * Free MD LWP resources.
1309 */
1310 cpu_lwp_free(l, 0);
1311
1312 if (current) {
1313 /* Switch away into oblivion. */
1314 lwp_lock(l);
1315 spc_lock(l->l_cpu);
1316 mi_switch(l);
1317 panic("lwp_exit");
1318 }
1319 }
1320
1321 /*
1322 * Free a dead LWP's remaining resources.
1323 *
1324 * XXXLWP limits.
1325 */
1326 void
1327 lwp_free(struct lwp *l, bool recycle, bool last)
1328 {
1329 struct proc *p = l->l_proc;
1330 struct rusage *ru;
1331 struct lwp *l2 __diagused;
1332 ksiginfoq_t kq;
1333
1334 KASSERT(l != curlwp);
1335 KASSERT(last || mutex_owned(p->p_lock));
1336
1337 /*
1338 * We use the process credentials instead of the lwp credentials here
1339 * because the lwp credentials maybe cached (just after a setuid call)
1340 * and we don't want pay for syncing, since the lwp is going away
1341 * anyway
1342 */
1343 if (p != &proc0 && p->p_nlwps != 1)
1344 (void)chglwpcnt(kauth_cred_getuid(p->p_cred), -1);
1345
1346 /*
1347 * If this was not the last LWP in the process, then adjust counters
1348 * and unlock. This is done differently for the last LWP in exit1().
1349 */
1350 if (!last) {
1351 /*
1352 * Add the LWP's run time to the process' base value.
1353 * This needs to co-incide with coming off p_lwps.
1354 */
1355 bintime_add(&p->p_rtime, &l->l_rtime);
1356 p->p_pctcpu += l->l_pctcpu;
1357 ru = &p->p_stats->p_ru;
1358 ruadd(ru, &l->l_ru);
1359 ru->ru_nvcsw += (l->l_ncsw - l->l_nivcsw);
1360 ru->ru_nivcsw += l->l_nivcsw;
1361 LIST_REMOVE(l, l_sibling);
1362 p->p_nlwps--;
1363 p->p_nzlwps--;
1364 if ((l->l_prflag & LPR_DETACHED) != 0)
1365 p->p_ndlwps--;
1366
1367 /* Make note of the LID being free, and remove from tree. */
1368 if (l->l_lid < p->p_nlwpid)
1369 p->p_nlwpid = l->l_lid;
1370 rw_enter(&p->p_treelock, RW_WRITER);
1371 l2 = radix_tree_remove_node(&p->p_lwptree,
1372 (uint64_t)(l->l_lid - 1));
1373 KASSERT(l2 == l);
1374 rw_exit(&p->p_treelock);
1375
1376 /*
1377 * Have any LWPs sleeping in lwp_wait() recheck for
1378 * deadlock.
1379 */
1380 cv_broadcast(&p->p_lwpcv);
1381 mutex_exit(p->p_lock);
1382 }
1383
1384 /*
1385 * In the unlikely event that the LWP is still on the CPU,
1386 * then spin until it has switched away.
1387 */
1388 membar_consumer();
1389 while (__predict_false((l->l_pflag & LP_RUNNING) != 0)) {
1390 SPINLOCK_BACKOFF_HOOK;
1391 }
1392
1393 /*
1394 * Destroy the LWP's remaining signal information.
1395 */
1396 ksiginfo_queue_init(&kq);
1397 sigclear(&l->l_sigpend, NULL, &kq);
1398 ksiginfo_queue_drain(&kq);
1399 cv_destroy(&l->l_sigcv);
1400 cv_destroy(&l->l_waitcv);
1401
1402 /*
1403 * Free lwpctl structure and affinity.
1404 */
1405 if (l->l_lwpctl) {
1406 lwp_ctl_free(l);
1407 }
1408 if (l->l_affinity) {
1409 kcpuset_unuse(l->l_affinity, NULL);
1410 l->l_affinity = NULL;
1411 }
1412
1413 /*
1414 * Free the LWP's turnstile and the LWP structure itself unless the
1415 * caller wants to recycle them. Also, free the scheduler specific
1416 * data.
1417 *
1418 * We can't return turnstile0 to the pool (it didn't come from it),
1419 * so if it comes up just drop it quietly and move on.
1420 *
1421 * We don't recycle the VM resources at this time.
1422 */
1423
1424 if (!recycle && l->l_ts != &turnstile0)
1425 pool_cache_put(turnstile_cache, l->l_ts);
1426 if (l->l_name != NULL)
1427 kmem_free(l->l_name, MAXCOMLEN);
1428
1429 kmsan_lwp_free(l);
1430 kcov_lwp_free(l);
1431 cpu_lwp_free2(l);
1432 uvm_lwp_exit(l);
1433
1434 KASSERT(SLIST_EMPTY(&l->l_pi_lenders));
1435 KASSERT(l->l_inheritedprio == -1);
1436 KASSERT(l->l_blcnt == 0);
1437 kdtrace_thread_dtor(NULL, l);
1438 if (!recycle)
1439 pool_cache_put(lwp_cache, l);
1440 }
1441
1442 /*
1443 * Migrate the LWP to the another CPU. Unlocks the LWP.
1444 */
1445 void
1446 lwp_migrate(lwp_t *l, struct cpu_info *tci)
1447 {
1448 struct schedstate_percpu *tspc;
1449 int lstat = l->l_stat;
1450
1451 KASSERT(lwp_locked(l, NULL));
1452 KASSERT(tci != NULL);
1453
1454 /* If LWP is still on the CPU, it must be handled like LSONPROC */
1455 if ((l->l_pflag & LP_RUNNING) != 0) {
1456 lstat = LSONPROC;
1457 }
1458
1459 /*
1460 * The destination CPU could be changed while previous migration
1461 * was not finished.
1462 */
1463 if (l->l_target_cpu != NULL) {
1464 l->l_target_cpu = tci;
1465 lwp_unlock(l);
1466 return;
1467 }
1468
1469 /* Nothing to do if trying to migrate to the same CPU */
1470 if (l->l_cpu == tci) {
1471 lwp_unlock(l);
1472 return;
1473 }
1474
1475 KASSERT(l->l_target_cpu == NULL);
1476 tspc = &tci->ci_schedstate;
1477 switch (lstat) {
1478 case LSRUN:
1479 l->l_target_cpu = tci;
1480 break;
1481 case LSSLEEP:
1482 l->l_cpu = tci;
1483 break;
1484 case LSIDL:
1485 case LSSTOP:
1486 case LSSUSPENDED:
1487 l->l_cpu = tci;
1488 if (l->l_wchan == NULL) {
1489 lwp_unlock_to(l, tspc->spc_lwplock);
1490 return;
1491 }
1492 break;
1493 case LSONPROC:
1494 l->l_target_cpu = tci;
1495 spc_lock(l->l_cpu);
1496 sched_resched_cpu(l->l_cpu, PRI_USER_RT, true);
1497 /* spc now unlocked */
1498 break;
1499 }
1500 lwp_unlock(l);
1501 }
1502
1503 /*
1504 * Find the LWP in the process. Arguments may be zero, in such case,
1505 * the calling process and first LWP in the list will be used.
1506 * On success - returns proc locked.
1507 */
1508 struct lwp *
1509 lwp_find2(pid_t pid, lwpid_t lid)
1510 {
1511 proc_t *p;
1512 lwp_t *l;
1513
1514 /* Find the process. */
1515 if (pid != 0) {
1516 mutex_enter(proc_lock);
1517 p = proc_find(pid);
1518 if (p == NULL) {
1519 mutex_exit(proc_lock);
1520 return NULL;
1521 }
1522 mutex_enter(p->p_lock);
1523 mutex_exit(proc_lock);
1524 } else {
1525 p = curlwp->l_proc;
1526 mutex_enter(p->p_lock);
1527 }
1528 /* Find the thread. */
1529 if (lid != 0) {
1530 l = lwp_find(p, lid);
1531 } else {
1532 l = LIST_FIRST(&p->p_lwps);
1533 }
1534 if (l == NULL) {
1535 mutex_exit(p->p_lock);
1536 }
1537 return l;
1538 }
1539
1540 /*
1541 * Look up a live LWP within the specified process.
1542 *
1543 * Must be called with p->p_lock held (as it looks at the radix tree,
1544 * and also wants to exclude idle and zombie LWPs).
1545 */
1546 struct lwp *
1547 lwp_find(struct proc *p, lwpid_t id)
1548 {
1549 struct lwp *l;
1550
1551 KASSERT(mutex_owned(p->p_lock));
1552
1553 l = radix_tree_lookup_node(&p->p_lwptree, (uint64_t)(id - 1));
1554 KASSERT(l == NULL || l->l_lid == id);
1555
1556 /*
1557 * No need to lock - all of these conditions will
1558 * be visible with the process level mutex held.
1559 */
1560 if (l != NULL && (l->l_stat == LSIDL || l->l_stat == LSZOMB))
1561 l = NULL;
1562
1563 return l;
1564 }
1565
1566 /*
1567 * Update an LWP's cached credentials to mirror the process' master copy.
1568 *
1569 * This happens early in the syscall path, on user trap, and on LWP
1570 * creation. A long-running LWP can also voluntarily choose to update
1571 * its credentials by calling this routine. This may be called from
1572 * LWP_CACHE_CREDS(), which checks l->l_cred != p->p_cred beforehand.
1573 */
1574 void
1575 lwp_update_creds(struct lwp *l)
1576 {
1577 kauth_cred_t oc;
1578 struct proc *p;
1579
1580 p = l->l_proc;
1581 oc = l->l_cred;
1582
1583 mutex_enter(p->p_lock);
1584 kauth_cred_hold(p->p_cred);
1585 l->l_cred = p->p_cred;
1586 l->l_prflag &= ~LPR_CRMOD;
1587 mutex_exit(p->p_lock);
1588 if (oc != NULL)
1589 kauth_cred_free(oc);
1590 }
1591
1592 /*
1593 * Verify that an LWP is locked, and optionally verify that the lock matches
1594 * one we specify.
1595 */
1596 int
1597 lwp_locked(struct lwp *l, kmutex_t *mtx)
1598 {
1599 kmutex_t *cur = l->l_mutex;
1600
1601 return mutex_owned(cur) && (mtx == cur || mtx == NULL);
1602 }
1603
1604 /*
1605 * Lend a new mutex to an LWP. The old mutex must be held.
1606 */
1607 kmutex_t *
1608 lwp_setlock(struct lwp *l, kmutex_t *mtx)
1609 {
1610 kmutex_t *oldmtx = l->l_mutex;
1611
1612 KASSERT(mutex_owned(oldmtx));
1613
1614 membar_exit();
1615 l->l_mutex = mtx;
1616 return oldmtx;
1617 }
1618
1619 /*
1620 * Lend a new mutex to an LWP, and release the old mutex. The old mutex
1621 * must be held.
1622 */
1623 void
1624 lwp_unlock_to(struct lwp *l, kmutex_t *mtx)
1625 {
1626 kmutex_t *old;
1627
1628 KASSERT(lwp_locked(l, NULL));
1629
1630 old = l->l_mutex;
1631 membar_exit();
1632 l->l_mutex = mtx;
1633 mutex_spin_exit(old);
1634 }
1635
1636 int
1637 lwp_trylock(struct lwp *l)
1638 {
1639 kmutex_t *old;
1640
1641 for (;;) {
1642 if (!mutex_tryenter(old = l->l_mutex))
1643 return 0;
1644 if (__predict_true(l->l_mutex == old))
1645 return 1;
1646 mutex_spin_exit(old);
1647 }
1648 }
1649
1650 void
1651 lwp_unsleep(lwp_t *l, bool unlock)
1652 {
1653
1654 KASSERT(mutex_owned(l->l_mutex));
1655 (*l->l_syncobj->sobj_unsleep)(l, unlock);
1656 }
1657
1658 /*
1659 * Handle exceptions for mi_userret(). Called if a member of LW_USERRET is
1660 * set.
1661 */
1662 void
1663 lwp_userret(struct lwp *l)
1664 {
1665 struct proc *p;
1666 int sig;
1667
1668 KASSERT(l == curlwp);
1669 KASSERT(l->l_stat == LSONPROC);
1670 p = l->l_proc;
1671
1672 /*
1673 * It is safe to do this read unlocked on a MP system..
1674 */
1675 while ((l->l_flag & LW_USERRET) != 0) {
1676 /*
1677 * Process pending signals first, unless the process
1678 * is dumping core or exiting, where we will instead
1679 * enter the LW_WSUSPEND case below.
1680 */
1681 if ((l->l_flag & (LW_PENDSIG | LW_WCORE | LW_WEXIT)) ==
1682 LW_PENDSIG) {
1683 mutex_enter(p->p_lock);
1684 while ((sig = issignal(l)) != 0)
1685 postsig(sig);
1686 mutex_exit(p->p_lock);
1687 }
1688
1689 /*
1690 * Core-dump or suspend pending.
1691 *
1692 * In case of core dump, suspend ourselves, so that the kernel
1693 * stack and therefore the userland registers saved in the
1694 * trapframe are around for coredump() to write them out.
1695 * We also need to save any PCU resources that we have so that
1696 * they accessible for coredump(). We issue a wakeup on
1697 * p->p_lwpcv so that sigexit() will write the core file out
1698 * once all other LWPs are suspended.
1699 */
1700 if ((l->l_flag & LW_WSUSPEND) != 0) {
1701 pcu_save_all(l);
1702 mutex_enter(p->p_lock);
1703 p->p_nrlwps--;
1704 cv_broadcast(&p->p_lwpcv);
1705 lwp_lock(l);
1706 l->l_stat = LSSUSPENDED;
1707 lwp_unlock(l);
1708 mutex_exit(p->p_lock);
1709 lwp_lock(l);
1710 spc_lock(l->l_cpu);
1711 mi_switch(l);
1712 }
1713
1714 /* Process is exiting. */
1715 if ((l->l_flag & LW_WEXIT) != 0) {
1716 lwp_exit(l);
1717 KASSERT(0);
1718 /* NOTREACHED */
1719 }
1720
1721 /* update lwpctl processor (for vfork child_return) */
1722 if (l->l_flag & LW_LWPCTL) {
1723 lwp_lock(l);
1724 KASSERT(kpreempt_disabled());
1725 l->l_lwpctl->lc_curcpu = (int)cpu_index(l->l_cpu);
1726 l->l_lwpctl->lc_pctr++;
1727 l->l_flag &= ~LW_LWPCTL;
1728 lwp_unlock(l);
1729 }
1730 }
1731 }
1732
1733 /*
1734 * Force an LWP to enter the kernel, to take a trip through lwp_userret().
1735 */
1736 void
1737 lwp_need_userret(struct lwp *l)
1738 {
1739
1740 KASSERT(!cpu_intr_p());
1741 KASSERT(lwp_locked(l, NULL));
1742
1743 /*
1744 * If the LWP is in any state other than LSONPROC, we know that it
1745 * is executing in-kernel and will hit userret() on the way out.
1746 *
1747 * If the LWP is curlwp, then we know we'll be back out to userspace
1748 * soon (can't be called from a hardware interrupt here).
1749 *
1750 * Otherwise, we can't be sure what the LWP is doing, so first make
1751 * sure the update to l_flag will be globally visible, and then
1752 * force the LWP to take a trip through trap() where it will do
1753 * userret().
1754 */
1755 if (l->l_stat == LSONPROC && l != curlwp) {
1756 membar_producer();
1757 cpu_signotify(l);
1758 }
1759 }
1760
1761 /*
1762 * Add one reference to an LWP. Interlocked against lwp_drainrefs()
1763 * either by holding the proc's lock or by holding lwp_threadid_lock.
1764 */
1765 static void
1766 lwp_addref2(struct lwp *l)
1767 {
1768
1769 KASSERT(l->l_stat != LSZOMB);
1770
1771 atomic_inc_uint(&l->l_refcnt);
1772 }
1773
1774 /*
1775 * Add one reference to an LWP. This will prevent the LWP from
1776 * exiting, thus keep the lwp structure and PCB around to inspect.
1777 */
1778 void
1779 lwp_addref(struct lwp *l)
1780 {
1781
1782 KASSERT(mutex_owned(l->l_proc->p_lock));
1783 lwp_addref2(l);
1784 }
1785
1786 /*
1787 * Remove one reference to an LWP. If this is the last reference,
1788 * then we must finalize the LWP's death.
1789 */
1790 void
1791 lwp_delref(struct lwp *l)
1792 {
1793 struct proc *p = l->l_proc;
1794
1795 mutex_enter(p->p_lock);
1796 lwp_delref2(l);
1797 mutex_exit(p->p_lock);
1798 }
1799
1800 /*
1801 * Remove one reference to an LWP. If this is the last reference,
1802 * then we must finalize the LWP's death. The proc mutex is held
1803 * on entry.
1804 */
1805 void
1806 lwp_delref2(struct lwp *l)
1807 {
1808 struct proc *p = l->l_proc;
1809
1810 KASSERT(mutex_owned(p->p_lock));
1811 KASSERT(l->l_stat != LSZOMB);
1812 KASSERT(atomic_load_relaxed(&l->l_refcnt) > 0);
1813
1814 if (atomic_dec_uint_nv(&l->l_refcnt) == 0)
1815 cv_broadcast(&p->p_lwpcv);
1816 }
1817
1818 /*
1819 * Drain all references to the current LWP. Returns true if
1820 * we blocked.
1821 */
1822 bool
1823 lwp_drainrefs(struct lwp *l)
1824 {
1825 struct proc *p = l->l_proc;
1826 bool rv = false;
1827
1828 KASSERT(mutex_owned(p->p_lock));
1829
1830 /*
1831 * Lookups in the lwp_threadid_map hold lwp_threadid_lock
1832 * as a reader, increase l_refcnt, release it, and then
1833 * acquire p_lock to check for LPR_DRAINING. By taking
1834 * lwp_threadid_lock as a writer here we ensure that either
1835 * we see the increase in l_refcnt or that they see LPR_DRAINING.
1836 */
1837 rw_enter(&lwp_threadid_lock, RW_WRITER);
1838 l->l_prflag |= LPR_DRAINING;
1839 rw_exit(&lwp_threadid_lock);
1840
1841 while (atomic_load_relaxed(&l->l_refcnt) > 0) {
1842 rv = true;
1843 cv_wait(&p->p_lwpcv, p->p_lock);
1844 }
1845 return rv;
1846 }
1847
1848 /*
1849 * Return true if the specified LWP is 'alive'. Only p->p_lock need
1850 * be held.
1851 */
1852 bool
1853 lwp_alive(lwp_t *l)
1854 {
1855
1856 KASSERT(mutex_owned(l->l_proc->p_lock));
1857
1858 switch (l->l_stat) {
1859 case LSSLEEP:
1860 case LSRUN:
1861 case LSONPROC:
1862 case LSSTOP:
1863 case LSSUSPENDED:
1864 return true;
1865 default:
1866 return false;
1867 }
1868 }
1869
1870 /*
1871 * Return first live LWP in the process.
1872 */
1873 lwp_t *
1874 lwp_find_first(proc_t *p)
1875 {
1876 lwp_t *l;
1877
1878 KASSERT(mutex_owned(p->p_lock));
1879
1880 LIST_FOREACH(l, &p->p_lwps, l_sibling) {
1881 if (lwp_alive(l)) {
1882 return l;
1883 }
1884 }
1885
1886 return NULL;
1887 }
1888
1889 /*
1890 * Allocate a new lwpctl structure for a user LWP.
1891 */
1892 int
1893 lwp_ctl_alloc(vaddr_t *uaddr)
1894 {
1895 lcproc_t *lp;
1896 u_int bit, i, offset;
1897 struct uvm_object *uao;
1898 int error;
1899 lcpage_t *lcp;
1900 proc_t *p;
1901 lwp_t *l;
1902
1903 l = curlwp;
1904 p = l->l_proc;
1905
1906 /* don't allow a vforked process to create lwp ctls */
1907 if (p->p_lflag & PL_PPWAIT)
1908 return EBUSY;
1909
1910 if (l->l_lcpage != NULL) {
1911 lcp = l->l_lcpage;
1912 *uaddr = lcp->lcp_uaddr + (vaddr_t)l->l_lwpctl - lcp->lcp_kaddr;
1913 return 0;
1914 }
1915
1916 /* First time around, allocate header structure for the process. */
1917 if ((lp = p->p_lwpctl) == NULL) {
1918 lp = kmem_alloc(sizeof(*lp), KM_SLEEP);
1919 mutex_init(&lp->lp_lock, MUTEX_DEFAULT, IPL_NONE);
1920 lp->lp_uao = NULL;
1921 TAILQ_INIT(&lp->lp_pages);
1922 mutex_enter(p->p_lock);
1923 if (p->p_lwpctl == NULL) {
1924 p->p_lwpctl = lp;
1925 mutex_exit(p->p_lock);
1926 } else {
1927 mutex_exit(p->p_lock);
1928 mutex_destroy(&lp->lp_lock);
1929 kmem_free(lp, sizeof(*lp));
1930 lp = p->p_lwpctl;
1931 }
1932 }
1933
1934 /*
1935 * Set up an anonymous memory region to hold the shared pages.
1936 * Map them into the process' address space. The user vmspace
1937 * gets the first reference on the UAO.
1938 */
1939 mutex_enter(&lp->lp_lock);
1940 if (lp->lp_uao == NULL) {
1941 lp->lp_uao = uao_create(LWPCTL_UAREA_SZ, 0);
1942 lp->lp_cur = 0;
1943 lp->lp_max = LWPCTL_UAREA_SZ;
1944 lp->lp_uva = p->p_emul->e_vm_default_addr(p,
1945 (vaddr_t)p->p_vmspace->vm_daddr, LWPCTL_UAREA_SZ,
1946 p->p_vmspace->vm_map.flags & VM_MAP_TOPDOWN);
1947 error = uvm_map(&p->p_vmspace->vm_map, &lp->lp_uva,
1948 LWPCTL_UAREA_SZ, lp->lp_uao, 0, 0, UVM_MAPFLAG(UVM_PROT_RW,
1949 UVM_PROT_RW, UVM_INH_NONE, UVM_ADV_NORMAL, 0));
1950 if (error != 0) {
1951 uao_detach(lp->lp_uao);
1952 lp->lp_uao = NULL;
1953 mutex_exit(&lp->lp_lock);
1954 return error;
1955 }
1956 }
1957
1958 /* Get a free block and allocate for this LWP. */
1959 TAILQ_FOREACH(lcp, &lp->lp_pages, lcp_chain) {
1960 if (lcp->lcp_nfree != 0)
1961 break;
1962 }
1963 if (lcp == NULL) {
1964 /* Nothing available - try to set up a free page. */
1965 if (lp->lp_cur == lp->lp_max) {
1966 mutex_exit(&lp->lp_lock);
1967 return ENOMEM;
1968 }
1969 lcp = kmem_alloc(LWPCTL_LCPAGE_SZ, KM_SLEEP);
1970
1971 /*
1972 * Wire the next page down in kernel space. Since this
1973 * is a new mapping, we must add a reference.
1974 */
1975 uao = lp->lp_uao;
1976 (*uao->pgops->pgo_reference)(uao);
1977 lcp->lcp_kaddr = vm_map_min(kernel_map);
1978 error = uvm_map(kernel_map, &lcp->lcp_kaddr, PAGE_SIZE,
1979 uao, lp->lp_cur, PAGE_SIZE,
1980 UVM_MAPFLAG(UVM_PROT_RW, UVM_PROT_RW,
1981 UVM_INH_NONE, UVM_ADV_RANDOM, 0));
1982 if (error != 0) {
1983 mutex_exit(&lp->lp_lock);
1984 kmem_free(lcp, LWPCTL_LCPAGE_SZ);
1985 (*uao->pgops->pgo_detach)(uao);
1986 return error;
1987 }
1988 error = uvm_map_pageable(kernel_map, lcp->lcp_kaddr,
1989 lcp->lcp_kaddr + PAGE_SIZE, FALSE, 0);
1990 if (error != 0) {
1991 mutex_exit(&lp->lp_lock);
1992 uvm_unmap(kernel_map, lcp->lcp_kaddr,
1993 lcp->lcp_kaddr + PAGE_SIZE);
1994 kmem_free(lcp, LWPCTL_LCPAGE_SZ);
1995 return error;
1996 }
1997 /* Prepare the page descriptor and link into the list. */
1998 lcp->lcp_uaddr = lp->lp_uva + lp->lp_cur;
1999 lp->lp_cur += PAGE_SIZE;
2000 lcp->lcp_nfree = LWPCTL_PER_PAGE;
2001 lcp->lcp_rotor = 0;
2002 memset(lcp->lcp_bitmap, 0xff, LWPCTL_BITMAP_SZ);
2003 TAILQ_INSERT_HEAD(&lp->lp_pages, lcp, lcp_chain);
2004 }
2005 for (i = lcp->lcp_rotor; lcp->lcp_bitmap[i] == 0;) {
2006 if (++i >= LWPCTL_BITMAP_ENTRIES)
2007 i = 0;
2008 }
2009 bit = ffs(lcp->lcp_bitmap[i]) - 1;
2010 lcp->lcp_bitmap[i] ^= (1U << bit);
2011 lcp->lcp_rotor = i;
2012 lcp->lcp_nfree--;
2013 l->l_lcpage = lcp;
2014 offset = (i << 5) + bit;
2015 l->l_lwpctl = (lwpctl_t *)lcp->lcp_kaddr + offset;
2016 *uaddr = lcp->lcp_uaddr + offset * sizeof(lwpctl_t);
2017 mutex_exit(&lp->lp_lock);
2018
2019 KPREEMPT_DISABLE(l);
2020 l->l_lwpctl->lc_curcpu = (int)cpu_index(curcpu());
2021 KPREEMPT_ENABLE(l);
2022
2023 return 0;
2024 }
2025
2026 /*
2027 * Free an lwpctl structure back to the per-process list.
2028 */
2029 void
2030 lwp_ctl_free(lwp_t *l)
2031 {
2032 struct proc *p = l->l_proc;
2033 lcproc_t *lp;
2034 lcpage_t *lcp;
2035 u_int map, offset;
2036
2037 /* don't free a lwp context we borrowed for vfork */
2038 if (p->p_lflag & PL_PPWAIT) {
2039 l->l_lwpctl = NULL;
2040 return;
2041 }
2042
2043 lp = p->p_lwpctl;
2044 KASSERT(lp != NULL);
2045
2046 lcp = l->l_lcpage;
2047 offset = (u_int)((lwpctl_t *)l->l_lwpctl - (lwpctl_t *)lcp->lcp_kaddr);
2048 KASSERT(offset < LWPCTL_PER_PAGE);
2049
2050 mutex_enter(&lp->lp_lock);
2051 lcp->lcp_nfree++;
2052 map = offset >> 5;
2053 lcp->lcp_bitmap[map] |= (1U << (offset & 31));
2054 if (lcp->lcp_bitmap[lcp->lcp_rotor] == 0)
2055 lcp->lcp_rotor = map;
2056 if (TAILQ_FIRST(&lp->lp_pages)->lcp_nfree == 0) {
2057 TAILQ_REMOVE(&lp->lp_pages, lcp, lcp_chain);
2058 TAILQ_INSERT_HEAD(&lp->lp_pages, lcp, lcp_chain);
2059 }
2060 mutex_exit(&lp->lp_lock);
2061 }
2062
2063 /*
2064 * Process is exiting; tear down lwpctl state. This can only be safely
2065 * called by the last LWP in the process.
2066 */
2067 void
2068 lwp_ctl_exit(void)
2069 {
2070 lcpage_t *lcp, *next;
2071 lcproc_t *lp;
2072 proc_t *p;
2073 lwp_t *l;
2074
2075 l = curlwp;
2076 l->l_lwpctl = NULL;
2077 l->l_lcpage = NULL;
2078 p = l->l_proc;
2079 lp = p->p_lwpctl;
2080
2081 KASSERT(lp != NULL);
2082 KASSERT(p->p_nlwps == 1);
2083
2084 for (lcp = TAILQ_FIRST(&lp->lp_pages); lcp != NULL; lcp = next) {
2085 next = TAILQ_NEXT(lcp, lcp_chain);
2086 uvm_unmap(kernel_map, lcp->lcp_kaddr,
2087 lcp->lcp_kaddr + PAGE_SIZE);
2088 kmem_free(lcp, LWPCTL_LCPAGE_SZ);
2089 }
2090
2091 if (lp->lp_uao != NULL) {
2092 uvm_unmap(&p->p_vmspace->vm_map, lp->lp_uva,
2093 lp->lp_uva + LWPCTL_UAREA_SZ);
2094 }
2095
2096 mutex_destroy(&lp->lp_lock);
2097 kmem_free(lp, sizeof(*lp));
2098 p->p_lwpctl = NULL;
2099 }
2100
2101 /*
2102 * Return the current LWP's "preemption counter". Used to detect
2103 * preemption across operations that can tolerate preemption without
2104 * crashing, but which may generate incorrect results if preempted.
2105 */
2106 uint64_t
2107 lwp_pctr(void)
2108 {
2109
2110 return curlwp->l_ncsw;
2111 }
2112
2113 /*
2114 * Set an LWP's private data pointer.
2115 */
2116 int
2117 lwp_setprivate(struct lwp *l, void *ptr)
2118 {
2119 int error = 0;
2120
2121 l->l_private = ptr;
2122 #ifdef __HAVE_CPU_LWP_SETPRIVATE
2123 error = cpu_lwp_setprivate(l, ptr);
2124 #endif
2125 return error;
2126 }
2127
2128 /*
2129 * Renumber the first and only LWP in a process on exec() or fork().
2130 * Don't bother with p_treelock here as this is the only live LWP in
2131 * the proc right now.
2132 */
2133 void
2134 lwp_renumber(lwp_t *l, lwpid_t lid)
2135 {
2136 lwp_t *l2 __diagused;
2137 proc_t *p = l->l_proc;
2138 int error;
2139
2140 KASSERT(p->p_nlwps == 1);
2141
2142 while (l->l_lid != lid) {
2143 mutex_enter(p->p_lock);
2144 error = radix_tree_insert_node(&p->p_lwptree, lid - 1, l);
2145 if (error == 0) {
2146 l2 = radix_tree_remove_node(&p->p_lwptree,
2147 (uint64_t)(l->l_lid - 1));
2148 KASSERT(l2 == l);
2149 p->p_nlwpid = lid + 1;
2150 l->l_lid = lid;
2151 }
2152 mutex_exit(p->p_lock);
2153
2154 if (error == 0)
2155 break;
2156
2157 KASSERT(error == ENOMEM);
2158 radix_tree_await_memory();
2159 }
2160 }
2161
2162 #define LWP_TID_MASK 0x3fffffff /* placeholder */
2163
2164 static void
2165 lwp_threadid_init(void)
2166 {
2167 rw_init(&lwp_threadid_lock);
2168 lwp_threadid_map = thmap_create(0, NULL, THMAP_NOCOPY);
2169 }
2170
2171 static void
2172 lwp_threadid_alloc(struct lwp * const l)
2173 {
2174
2175 KASSERT(l == curlwp);
2176 KASSERT(l->l___tid == 0);
2177
2178 for (;;) {
2179 l->l___tid = cprng_fast32() & LWP_TID_MASK;
2180 if (l->l___tid != 0 &&
2181 /*
2182 * There is no need to take the lwp_threadid_lock
2183 * while inserting into the map: internally, the
2184 * map is already concurrency-safe, and the lock
2185 * is only needed to serialize removal with respect
2186 * to lookup.
2187 */
2188 thmap_put(lwp_threadid_map,
2189 &l->l___tid, sizeof(l->l___tid), l) == l) {
2190 /* claimed! */
2191 return;
2192 }
2193 preempt_point();
2194 }
2195 }
2196
2197 static inline void
2198 lwp_threadid_gc_serialize(void)
2199 {
2200
2201 /*
2202 * By acquiring the lock as a writer, we will know that
2203 * all of the existing readers have drained away and thus
2204 * the GC is safe.
2205 */
2206 rw_enter(&lwp_threadid_lock, RW_WRITER);
2207 rw_exit(&lwp_threadid_lock);
2208 }
2209
2210 static void
2211 lwp_threadid_free(struct lwp * const l)
2212 {
2213
2214 KASSERT(l == curlwp);
2215 KASSERT(l->l___tid != 0);
2216
2217 /*
2218 * Ensure that anyone who finds this entry in the lock-free lookup
2219 * path sees that the key has been deleted by serialzing with the
2220 * examination of l___tid.
2221 *
2222 * N.B. l___tid field must be zapped *after* deleting from the map
2223 * because that field is being used as the key storage by thmap.
2224 */
2225 KASSERT(mutex_owned(l->l_proc->p_lock));
2226 struct lwp * const ldiag __diagused = thmap_del(lwp_threadid_map,
2227 &l->l___tid, sizeof(l->l___tid));
2228 l->l___tid = 0;
2229 mutex_exit(l->l_proc->p_lock);
2230
2231 KASSERT(l == ldiag);
2232
2233 void * const gc_ref = thmap_stage_gc(lwp_threadid_map);
2234 lwp_threadid_gc_serialize();
2235 thmap_gc(lwp_threadid_map, gc_ref);
2236 }
2237
2238 /*
2239 * Return the current LWP's global thread ID. Only the current LWP
2240 * should ever use this value, unless it is guaranteed that the LWP
2241 * is paused (and then it should be accessed directly, rather than
2242 * by this accessor).
2243 */
2244 lwpid_t
2245 lwp_gettid(void)
2246 {
2247 struct lwp * const l = curlwp;
2248
2249 if (l->l___tid == 0)
2250 lwp_threadid_alloc(l);
2251
2252 return l->l___tid;
2253 }
2254
2255 /*
2256 * Lookup an LWP by global thread ID. Care must be taken because
2257 * callers of this are operating outside the normal locking protocol.
2258 * We return the LWP with an additional reference that must be dropped
2259 * with lwp_delref().
2260 */
2261 struct lwp *
2262 lwp_getref_tid(lwpid_t tid)
2263 {
2264 struct lwp *l, *rv;
2265
2266 rw_enter(&lwp_threadid_lock, RW_READER);
2267 l = thmap_get(lwp_threadid_map, &tid, sizeof(&tid));
2268 if (__predict_false(l == NULL)) {
2269 rw_exit(&lwp_threadid_lock);
2270 return NULL;
2271 }
2272
2273 /*
2274 * Acquire a reference on the lwp. It is safe to do this unlocked
2275 * here because lwp_drainrefs() serializes with us by taking the
2276 * lwp_threadid_lock as a writer.
2277 */
2278 lwp_addref2(l);
2279 rw_exit(&lwp_threadid_lock);
2280
2281 /*
2282 * Now verify that our reference is valid.
2283 */
2284 mutex_enter(l->l_proc->p_lock);
2285 if (__predict_false((l->l_prflag & LPR_DRAINING) != 0 ||
2286 l->l___tid == 0)) {
2287 lwp_delref2(l);
2288 rv = NULL;
2289 } else {
2290 rv = l;
2291 }
2292 mutex_exit(l->l_proc->p_lock);
2293
2294 return rv;
2295 }
2296
2297 /*
2298 * Perform any thread-related cleanup on LWP exit.
2299 * N.B. l->l_proc->p_lock must be HELD on entry but will
2300 * be released before returning!
2301 */
2302 void
2303 lwp_thread_cleanup(struct lwp *l)
2304 {
2305 KASSERT(l == curlwp);
2306 const lwpid_t tid = l->l___tid;
2307
2308 KASSERT(mutex_owned(l->l_proc->p_lock));
2309
2310 if (__predict_false(tid != 0)) {
2311 /*
2312 * Drop our thread ID. This will also unlock
2313 * our proc.
2314 */
2315 lwp_threadid_free(l);
2316 } else {
2317 /*
2318 * No thread cleanup was required; just unlock
2319 * the proc.
2320 */
2321 mutex_exit(l->l_proc->p_lock);
2322 }
2323 }
2324
2325 #if defined(DDB)
2326 #include <machine/pcb.h>
2327
2328 void
2329 lwp_whatis(uintptr_t addr, void (*pr)(const char *, ...))
2330 {
2331 lwp_t *l;
2332
2333 LIST_FOREACH(l, &alllwp, l_list) {
2334 uintptr_t stack = (uintptr_t)KSTACK_LOWEST_ADDR(l);
2335
2336 if (addr < stack || stack + KSTACK_SIZE <= addr) {
2337 continue;
2338 }
2339 (*pr)("%p is %p+%zu, LWP %p's stack\n",
2340 (void *)addr, (void *)stack,
2341 (size_t)(addr - stack), l);
2342 }
2343 }
2344 #endif /* defined(DDB) */
2345