kern_lwp.c revision 1.263 1 /* $NetBSD: kern_lwp.c,v 1.263 2023/10/04 22:17:09 ad Exp $ */
2
3 /*-
4 * Copyright (c) 2001, 2006, 2007, 2008, 2009, 2019, 2020, 2023
5 * The NetBSD Foundation, Inc.
6 * All rights reserved.
7 *
8 * This code is derived from software contributed to The NetBSD Foundation
9 * by Nathan J. Williams, and Andrew Doran.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 * POSSIBILITY OF SUCH DAMAGE.
31 */
32
33 /*
34 * Overview
35 *
36 * Lightweight processes (LWPs) are the basic unit or thread of
37 * execution within the kernel. The core state of an LWP is described
38 * by "struct lwp", also known as lwp_t.
39 *
40 * Each LWP is contained within a process (described by "struct proc"),
41 * Every process contains at least one LWP, but may contain more. The
42 * process describes attributes shared among all of its LWPs such as a
43 * private address space, global execution state (stopped, active,
44 * zombie, ...), signal disposition and so on. On a multiprocessor
45 * machine, multiple LWPs be executing concurrently in the kernel.
46 *
47 * Execution states
48 *
49 * At any given time, an LWP has overall state that is described by
50 * lwp::l_stat. The states are broken into two sets below. The first
51 * set is guaranteed to represent the absolute, current state of the
52 * LWP:
53 *
54 * LSONPROC
55 *
56 * On processor: the LWP is executing on a CPU, either in the
57 * kernel or in user space.
58 *
59 * LSRUN
60 *
61 * Runnable: the LWP is parked on a run queue, and may soon be
62 * chosen to run by an idle processor, or by a processor that
63 * has been asked to preempt a currently runnning but lower
64 * priority LWP.
65 *
66 * LSIDL
67 *
68 * Idle: the LWP has been created but has not yet executed, or
69 * it has ceased executing a unit of work and is waiting to be
70 * started again. This state exists so that the LWP can occupy
71 * a slot in the process & PID table, but without having to
72 * worry about being touched; lookups of the LWP by ID will
73 * fail while in this state. The LWP will become visible for
74 * lookup once its state transitions further. Some special
75 * kernel threads also (ab)use this state to indicate that they
76 * are idle (soft interrupts and idle LWPs).
77 *
78 * LSSUSPENDED:
79 *
80 * Suspended: the LWP has had its execution suspended by
81 * another LWP in the same process using the _lwp_suspend()
82 * system call. User-level LWPs also enter the suspended
83 * state when the system is shutting down.
84 *
85 * The second set represent a "statement of intent" on behalf of the
86 * LWP. The LWP may in fact be executing on a processor, may be
87 * sleeping or idle. It is expected to take the necessary action to
88 * stop executing or become "running" again within a short timeframe.
89 * The LP_RUNNING flag in lwp::l_pflag indicates that an LWP is running.
90 * Importantly, it indicates that its state is tied to a CPU.
91 *
92 * LSZOMB:
93 *
94 * Dead or dying: the LWP has released most of its resources
95 * and is about to switch away into oblivion, or has already
96 * switched away. When it switches away, its few remaining
97 * resources can be collected.
98 *
99 * LSSLEEP:
100 *
101 * Sleeping: the LWP has entered itself onto a sleep queue, and
102 * has switched away or will switch away shortly to allow other
103 * LWPs to run on the CPU.
104 *
105 * LSSTOP:
106 *
107 * Stopped: the LWP has been stopped as a result of a job
108 * control signal, or as a result of the ptrace() interface.
109 *
110 * Stopped LWPs may run briefly within the kernel to handle
111 * signals that they receive, but will not return to user space
112 * until their process' state is changed away from stopped.
113 *
114 * Single LWPs within a process can not be set stopped
115 * selectively: all actions that can stop or continue LWPs
116 * occur at the process level.
117 *
118 * State transitions
119 *
120 * Note that the LSSTOP state may only be set when returning to
121 * user space in userret(), or when sleeping interruptably. The
122 * LSSUSPENDED state may only be set in userret(). Before setting
123 * those states, we try to ensure that the LWPs will release all
124 * locks that they hold, and at a minimum try to ensure that the
125 * LWP can be set runnable again by a signal.
126 *
127 * LWPs may transition states in the following ways:
128 *
129 * RUN -------> ONPROC ONPROC -----> RUN
130 * > SLEEP
131 * > STOPPED
132 * > SUSPENDED
133 * > ZOMB
134 * > IDL (special cases)
135 *
136 * STOPPED ---> RUN SUSPENDED --> RUN
137 * > SLEEP
138 *
139 * SLEEP -----> ONPROC IDL --------> RUN
140 * > RUN > SUSPENDED
141 * > STOPPED > STOPPED
142 * > ONPROC (special cases)
143 *
144 * Some state transitions are only possible with kernel threads (eg
145 * ONPROC -> IDL) and happen under tightly controlled circumstances
146 * free of unwanted side effects.
147 *
148 * Migration
149 *
150 * Migration of threads from one CPU to another could be performed
151 * internally by the scheduler via sched_takecpu() or sched_catchlwp()
152 * functions. The universal lwp_migrate() function should be used for
153 * any other cases. Subsystems in the kernel must be aware that CPU
154 * of LWP may change, while it is not locked.
155 *
156 * Locking
157 *
158 * The majority of fields in 'struct lwp' are covered by a single,
159 * general spin lock pointed to by lwp::l_mutex. The locks covering
160 * each field are documented in sys/lwp.h.
161 *
162 * State transitions must be made with the LWP's general lock held,
163 * and may cause the LWP's lock pointer to change. Manipulation of
164 * the general lock is not performed directly, but through calls to
165 * lwp_lock(), lwp_unlock() and others. It should be noted that the
166 * adaptive locks are not allowed to be released while the LWP's lock
167 * is being held (unlike for other spin-locks).
168 *
169 * States and their associated locks:
170 *
171 * LSIDL, LSONPROC, LSZOMB, LSSUPENDED:
172 *
173 * Always covered by spc_lwplock, which protects LWPs not
174 * associated with any other sync object. This is a per-CPU
175 * lock and matches lwp::l_cpu.
176 *
177 * LSRUN:
178 *
179 * Always covered by spc_mutex, which protects the run queues.
180 * This is a per-CPU lock and matches lwp::l_cpu.
181 *
182 * LSSLEEP:
183 *
184 * Covered by a lock associated with the sleep queue (sometimes
185 * a turnstile sleep queue) that the LWP resides on. This can
186 * be spc_lwplock for SOBJ_SLEEPQ_NULL (an "untracked" sleep).
187 *
188 * LSSTOP:
189 *
190 * If the LWP was previously sleeping (l_wchan != NULL), then
191 * l_mutex references the sleep queue lock. If the LWP was
192 * runnable or on the CPU when halted, or has been removed from
193 * the sleep queue since halted, then the lock is spc_lwplock.
194 *
195 * The lock order is as follows:
196 *
197 * sleepq -> turnstile -> spc_lwplock -> spc_mutex
198 *
199 * Each process has a scheduler state lock (proc::p_lock), and a
200 * number of counters on LWPs and their states: p_nzlwps, p_nrlwps, and
201 * so on. When an LWP is to be entered into or removed from one of the
202 * following states, p_lock must be held and the process wide counters
203 * adjusted:
204 *
205 * LSIDL, LSZOMB, LSSTOP, LSSUSPENDED
206 *
207 * (But not always for kernel threads. There are some special cases
208 * as mentioned above: soft interrupts, and the idle loops.)
209 *
210 * Note that an LWP is considered running or likely to run soon if in
211 * one of the following states. This affects the value of p_nrlwps:
212 *
213 * LSRUN, LSONPROC, LSSLEEP
214 *
215 * p_lock does not need to be held when transitioning among these
216 * three states, hence p_lock is rarely taken for state transitions.
217 */
218
219 #include <sys/cdefs.h>
220 __KERNEL_RCSID(0, "$NetBSD: kern_lwp.c,v 1.263 2023/10/04 22:17:09 ad Exp $");
221
222 #include "opt_ddb.h"
223 #include "opt_lockdebug.h"
224 #include "opt_dtrace.h"
225
226 #define _LWP_API_PRIVATE
227
228 #include <sys/param.h>
229 #include <sys/systm.h>
230 #include <sys/cpu.h>
231 #include <sys/pool.h>
232 #include <sys/proc.h>
233 #include <sys/syscallargs.h>
234 #include <sys/syscall_stats.h>
235 #include <sys/kauth.h>
236 #include <sys/sleepq.h>
237 #include <sys/lockdebug.h>
238 #include <sys/kmem.h>
239 #include <sys/pset.h>
240 #include <sys/intr.h>
241 #include <sys/lwpctl.h>
242 #include <sys/atomic.h>
243 #include <sys/filedesc.h>
244 #include <sys/fstrans.h>
245 #include <sys/dtrace_bsd.h>
246 #include <sys/sdt.h>
247 #include <sys/ptrace.h>
248 #include <sys/xcall.h>
249 #include <sys/uidinfo.h>
250 #include <sys/sysctl.h>
251 #include <sys/psref.h>
252 #include <sys/msan.h>
253 #include <sys/kcov.h>
254 #include <sys/cprng.h>
255 #include <sys/futex.h>
256
257 #include <uvm/uvm_extern.h>
258 #include <uvm/uvm_object.h>
259
260 static pool_cache_t lwp_cache __read_mostly;
261 struct lwplist alllwp __cacheline_aligned;
262
263 static int lwp_ctor(void *, void *, int);
264 static void lwp_dtor(void *, void *);
265
266 /* DTrace proc provider probes */
267 SDT_PROVIDER_DEFINE(proc);
268
269 SDT_PROBE_DEFINE1(proc, kernel, , lwp__create, "struct lwp *");
270 SDT_PROBE_DEFINE1(proc, kernel, , lwp__start, "struct lwp *");
271 SDT_PROBE_DEFINE1(proc, kernel, , lwp__exit, "struct lwp *");
272
273 struct turnstile turnstile0 __cacheline_aligned;
274 struct lwp lwp0 __aligned(MIN_LWP_ALIGNMENT) = {
275 #ifdef LWP0_CPU_INFO
276 .l_cpu = LWP0_CPU_INFO,
277 #endif
278 #ifdef LWP0_MD_INITIALIZER
279 .l_md = LWP0_MD_INITIALIZER,
280 #endif
281 .l_proc = &proc0,
282 .l_lid = 0, /* we own proc0's slot in the pid table */
283 .l_flag = LW_SYSTEM,
284 .l_stat = LSONPROC,
285 .l_ts = &turnstile0,
286 .l_syncobj = &sched_syncobj,
287 .l_refcnt = 0,
288 .l_priority = PRI_USER + NPRI_USER - 1,
289 .l_inheritedprio = -1,
290 .l_class = SCHED_OTHER,
291 .l_psid = PS_NONE,
292 .l_pi_lenders = SLIST_HEAD_INITIALIZER(&lwp0.l_pi_lenders),
293 .l_name = __UNCONST("swapper"),
294 .l_fd = &filedesc0,
295 };
296
297 static int
298 lwp_maxlwp(void)
299 {
300 /* Assume 1 LWP per 1MiB. */
301 uint64_t lwps_per = ctob(physmem) / (1024 * 1024);
302
303 return MAX(MIN(MAXMAXLWP, lwps_per), MAXLWP);
304 }
305
306 static int sysctl_kern_maxlwp(SYSCTLFN_PROTO);
307
308 /*
309 * sysctl helper routine for kern.maxlwp. Ensures that the new
310 * values are not too low or too high.
311 */
312 static int
313 sysctl_kern_maxlwp(SYSCTLFN_ARGS)
314 {
315 int error, nmaxlwp;
316 struct sysctlnode node;
317
318 nmaxlwp = maxlwp;
319 node = *rnode;
320 node.sysctl_data = &nmaxlwp;
321 error = sysctl_lookup(SYSCTLFN_CALL(&node));
322 if (error || newp == NULL)
323 return error;
324
325 if (nmaxlwp < 0 || nmaxlwp >= MAXMAXLWP)
326 return EINVAL;
327 if (nmaxlwp > lwp_maxlwp())
328 return EINVAL;
329 maxlwp = nmaxlwp;
330
331 return 0;
332 }
333
334 static void
335 sysctl_kern_lwp_setup(void)
336 {
337 sysctl_createv(NULL, 0, NULL, NULL,
338 CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
339 CTLTYPE_INT, "maxlwp",
340 SYSCTL_DESCR("Maximum number of simultaneous threads"),
341 sysctl_kern_maxlwp, 0, NULL, 0,
342 CTL_KERN, CTL_CREATE, CTL_EOL);
343 }
344
345 void
346 lwpinit(void)
347 {
348
349 LIST_INIT(&alllwp);
350 lwpinit_specificdata();
351 /*
352 * Provide a barrier to ensure that all mutex_oncpu() and rw_oncpu()
353 * calls will exit before memory of LWPs is returned to the pool, where
354 * KVA of LWP structure might be freed and re-used for other purposes.
355 * Kernel preemption is disabled around mutex_oncpu() and rw_oncpu()
356 * callers, therefore a regular passive serialization barrier will
357 * do the job.
358 */
359 lwp_cache = pool_cache_init(sizeof(lwp_t), MIN_LWP_ALIGNMENT, 0,
360 PR_PSERIALIZE, "lwppl", NULL, IPL_NONE, lwp_ctor, lwp_dtor, NULL);
361
362 maxlwp = lwp_maxlwp();
363 sysctl_kern_lwp_setup();
364 }
365
366 void
367 lwp0_init(void)
368 {
369 struct lwp *l = &lwp0;
370
371 KASSERT((void *)uvm_lwp_getuarea(l) != NULL);
372
373 LIST_INSERT_HEAD(&alllwp, l, l_list);
374
375 callout_init(&l->l_timeout_ch, CALLOUT_MPSAFE);
376 callout_setfunc(&l->l_timeout_ch, sleepq_timeout, l);
377 cv_init(&l->l_sigcv, "sigwait");
378 cv_init(&l->l_waitcv, "vfork");
379
380 l->l_cred = kauth_cred_hold(proc0.p_cred);
381
382 kdtrace_thread_ctor(NULL, l);
383 lwp_initspecific(l);
384
385 SYSCALL_TIME_LWP_INIT(l);
386 }
387
388 /*
389 * Initialize the non-zeroed portion of an lwp_t.
390 */
391 static int
392 lwp_ctor(void *arg, void *obj, int flags)
393 {
394 lwp_t *l = obj;
395
396 l->l_stat = LSIDL;
397 l->l_cpu = curcpu();
398 l->l_mutex = l->l_cpu->ci_schedstate.spc_lwplock;
399 l->l_ts = kmem_alloc(sizeof(*l->l_ts), flags == PR_WAITOK ?
400 KM_SLEEP : KM_NOSLEEP);
401
402 if (l->l_ts == NULL) {
403 return ENOMEM;
404 } else {
405 turnstile_ctor(l->l_ts);
406 return 0;
407 }
408 }
409
410 static void
411 lwp_dtor(void *arg, void *obj)
412 {
413 lwp_t *l = obj;
414
415 /*
416 * The value of l->l_cpu must still be valid at this point.
417 */
418 KASSERT(l->l_cpu != NULL);
419
420 /*
421 * We can't return turnstile0 to the pool (it didn't come from it),
422 * so if it comes up just drop it quietly and move on.
423 */
424 if (l->l_ts != &turnstile0)
425 kmem_free(l->l_ts, sizeof(*l->l_ts));
426 }
427
428 /*
429 * Set an LWP suspended.
430 *
431 * Must be called with p_lock held, and the LWP locked. Will unlock the
432 * LWP before return.
433 */
434 int
435 lwp_suspend(struct lwp *curl, struct lwp *t)
436 {
437 int error;
438
439 KASSERT(mutex_owned(t->l_proc->p_lock));
440 KASSERT(lwp_locked(t, NULL));
441
442 KASSERT(curl != t || curl->l_stat == LSONPROC);
443
444 /*
445 * If the current LWP has been told to exit, we must not suspend anyone
446 * else or deadlock could occur. We won't return to userspace.
447 */
448 if ((curl->l_flag & (LW_WEXIT | LW_WCORE)) != 0) {
449 lwp_unlock(t);
450 return (EDEADLK);
451 }
452
453 if ((t->l_flag & LW_DBGSUSPEND) != 0) {
454 lwp_unlock(t);
455 return 0;
456 }
457
458 error = 0;
459
460 switch (t->l_stat) {
461 case LSRUN:
462 case LSONPROC:
463 t->l_flag |= LW_WSUSPEND;
464 lwp_need_userret(t);
465 lwp_unlock(t);
466 break;
467
468 case LSSLEEP:
469 t->l_flag |= LW_WSUSPEND;
470 lwp_need_userret(t);
471
472 /*
473 * Kick the LWP and try to get it to the kernel boundary
474 * so that it will release any locks that it holds.
475 * setrunnable() will release the lock.
476 */
477 if ((t->l_flag & LW_SINTR) != 0)
478 setrunnable(t);
479 else
480 lwp_unlock(t);
481 break;
482
483 case LSSUSPENDED:
484 lwp_unlock(t);
485 break;
486
487 case LSSTOP:
488 t->l_flag |= LW_WSUSPEND;
489 lwp_need_userret(t);
490 setrunnable(t);
491 break;
492
493 case LSIDL:
494 case LSZOMB:
495 error = EINTR; /* It's what Solaris does..... */
496 lwp_unlock(t);
497 break;
498 }
499
500 return (error);
501 }
502
503 /*
504 * Restart a suspended LWP.
505 *
506 * Must be called with p_lock held, and the LWP locked. Will unlock the
507 * LWP before return.
508 */
509 void
510 lwp_continue(struct lwp *l)
511 {
512
513 KASSERT(mutex_owned(l->l_proc->p_lock));
514 KASSERT(lwp_locked(l, NULL));
515
516 /* If rebooting or not suspended, then just bail out. */
517 if ((l->l_flag & LW_WREBOOT) != 0) {
518 lwp_unlock(l);
519 return;
520 }
521
522 l->l_flag &= ~LW_WSUSPEND;
523
524 if (l->l_stat != LSSUSPENDED || (l->l_flag & LW_DBGSUSPEND) != 0) {
525 lwp_unlock(l);
526 return;
527 }
528
529 /* setrunnable() will release the lock. */
530 setrunnable(l);
531 }
532
533 /*
534 * Restart a stopped LWP.
535 *
536 * Must be called with p_lock held, and the LWP NOT locked. Will unlock the
537 * LWP before return.
538 */
539 void
540 lwp_unstop(struct lwp *l)
541 {
542 struct proc *p = l->l_proc;
543
544 KASSERT(mutex_owned(&proc_lock));
545 KASSERT(mutex_owned(p->p_lock));
546
547 lwp_lock(l);
548
549 KASSERT((l->l_flag & LW_DBGSUSPEND) == 0);
550
551 /* If not stopped, then just bail out. */
552 if (l->l_stat != LSSTOP) {
553 lwp_unlock(l);
554 return;
555 }
556
557 p->p_stat = SACTIVE;
558 p->p_sflag &= ~PS_STOPPING;
559
560 if (!p->p_waited)
561 p->p_pptr->p_nstopchild--;
562
563 if (l->l_wchan == NULL) {
564 /* setrunnable() will release the lock. */
565 setrunnable(l);
566 } else if (p->p_xsig && (l->l_flag & LW_SINTR) != 0) {
567 /* setrunnable() so we can receive the signal */
568 setrunnable(l);
569 } else {
570 l->l_stat = LSSLEEP;
571 p->p_nrlwps++;
572 lwp_unlock(l);
573 }
574 }
575
576 /*
577 * Wait for an LWP within the current process to exit. If 'lid' is
578 * non-zero, we are waiting for a specific LWP.
579 *
580 * Must be called with p->p_lock held.
581 */
582 int
583 lwp_wait(struct lwp *l, lwpid_t lid, lwpid_t *departed, bool exiting)
584 {
585 const lwpid_t curlid = l->l_lid;
586 proc_t *p = l->l_proc;
587 lwp_t *l2, *next;
588 int error;
589
590 KASSERT(mutex_owned(p->p_lock));
591
592 p->p_nlwpwait++;
593 l->l_waitingfor = lid;
594
595 for (;;) {
596 int nfound;
597
598 /*
599 * Avoid a race between exit1() and sigexit(): if the
600 * process is dumping core, then we need to bail out: call
601 * into lwp_userret() where we will be suspended until the
602 * deed is done.
603 */
604 if ((p->p_sflag & PS_WCORE) != 0) {
605 mutex_exit(p->p_lock);
606 lwp_userret(l);
607 KASSERT(false);
608 }
609
610 /*
611 * First off, drain any detached LWP that is waiting to be
612 * reaped.
613 */
614 if ((l2 = p->p_zomblwp) != NULL) {
615 p->p_zomblwp = NULL;
616 lwp_free(l2, false, false);/* releases proc mutex */
617 mutex_enter(p->p_lock);
618 continue;
619 }
620
621 /*
622 * Now look for an LWP to collect. If the whole process is
623 * exiting, count detached LWPs as eligible to be collected,
624 * but don't drain them here.
625 */
626 nfound = 0;
627 error = 0;
628
629 /*
630 * If given a specific LID, go via pid_table and make sure
631 * it's not detached.
632 */
633 if (lid != 0) {
634 l2 = proc_find_lwp(p, lid);
635 if (l2 == NULL) {
636 error = ESRCH;
637 break;
638 }
639 KASSERT(l2->l_lid == lid);
640 if ((l2->l_prflag & LPR_DETACHED) != 0) {
641 error = EINVAL;
642 break;
643 }
644 } else {
645 l2 = LIST_FIRST(&p->p_lwps);
646 }
647 for (; l2 != NULL; l2 = next) {
648 next = (lid != 0 ? NULL : LIST_NEXT(l2, l_sibling));
649
650 /*
651 * If a specific wait and the target is waiting on
652 * us, then avoid deadlock. This also traps LWPs
653 * that try to wait on themselves.
654 *
655 * Note that this does not handle more complicated
656 * cycles, like: t1 -> t2 -> t3 -> t1. The process
657 * can still be killed so it is not a major problem.
658 */
659 if (l2->l_lid == lid && l2->l_waitingfor == curlid) {
660 error = EDEADLK;
661 break;
662 }
663 if (l2 == l)
664 continue;
665 if ((l2->l_prflag & LPR_DETACHED) != 0) {
666 nfound += exiting;
667 continue;
668 }
669 if (lid != 0) {
670 /*
671 * Mark this LWP as the first waiter, if there
672 * is no other.
673 */
674 if (l2->l_waiter == 0)
675 l2->l_waiter = curlid;
676 } else if (l2->l_waiter != 0) {
677 /*
678 * It already has a waiter - so don't
679 * collect it. If the waiter doesn't
680 * grab it we'll get another chance
681 * later.
682 */
683 nfound++;
684 continue;
685 }
686 nfound++;
687
688 /* No need to lock the LWP in order to see LSZOMB. */
689 if (l2->l_stat != LSZOMB)
690 continue;
691
692 /*
693 * We're no longer waiting. Reset the "first waiter"
694 * pointer on the target, in case it was us.
695 */
696 l->l_waitingfor = 0;
697 l2->l_waiter = 0;
698 p->p_nlwpwait--;
699 if (departed)
700 *departed = l2->l_lid;
701 sched_lwp_collect(l2);
702
703 /* lwp_free() releases the proc lock. */
704 lwp_free(l2, false, false);
705 mutex_enter(p->p_lock);
706 return 0;
707 }
708
709 if (error != 0)
710 break;
711 if (nfound == 0) {
712 error = ESRCH;
713 break;
714 }
715
716 /*
717 * Note: since the lock will be dropped, need to restart on
718 * wakeup to run all LWPs again, e.g. there may be new LWPs.
719 */
720 if (exiting) {
721 KASSERT(p->p_nlwps > 1);
722 error = cv_timedwait(&p->p_lwpcv, p->p_lock, 1);
723 break;
724 }
725
726 /*
727 * Break out if all LWPs are in _lwp_wait(). There are
728 * other ways to hang the process with _lwp_wait(), but the
729 * sleep is interruptable so little point checking for them.
730 */
731 if (p->p_nlwpwait == p->p_nlwps) {
732 error = EDEADLK;
733 break;
734 }
735
736 /*
737 * Sit around and wait for something to happen. We'll be
738 * awoken if any of the conditions examined change: if an
739 * LWP exits, is collected, or is detached.
740 */
741 if ((error = cv_wait_sig(&p->p_lwpcv, p->p_lock)) != 0)
742 break;
743 }
744
745 /*
746 * We didn't find any LWPs to collect, we may have received a
747 * signal, or some other condition has caused us to bail out.
748 *
749 * If waiting on a specific LWP, clear the waiters marker: some
750 * other LWP may want it. Then, kick all the remaining waiters
751 * so that they can re-check for zombies and for deadlock.
752 */
753 if (lid != 0) {
754 l2 = proc_find_lwp(p, lid);
755 KASSERT(l2 == NULL || l2->l_lid == lid);
756
757 if (l2 != NULL && l2->l_waiter == curlid)
758 l2->l_waiter = 0;
759 }
760 p->p_nlwpwait--;
761 l->l_waitingfor = 0;
762 cv_broadcast(&p->p_lwpcv);
763
764 return error;
765 }
766
767 /*
768 * Create a new LWP within process 'p2', using LWP 'l1' as a template.
769 * The new LWP is created in state LSIDL and must be set running,
770 * suspended, or stopped by the caller.
771 */
772 int
773 lwp_create(lwp_t *l1, proc_t *p2, vaddr_t uaddr, int flags,
774 void *stack, size_t stacksize, void (*func)(void *), void *arg,
775 lwp_t **rnewlwpp, int sclass, const sigset_t *sigmask,
776 const stack_t *sigstk)
777 {
778 struct lwp *l2;
779
780 KASSERT(l1 == curlwp || l1->l_proc == &proc0);
781
782 /*
783 * Enforce limits, excluding the first lwp and kthreads. We must
784 * use the process credentials here when adjusting the limit, as
785 * they are what's tied to the accounting entity. However for
786 * authorizing the action, we'll use the LWP's credentials.
787 */
788 mutex_enter(p2->p_lock);
789 if (p2->p_nlwps != 0 && p2 != &proc0) {
790 uid_t uid = kauth_cred_getuid(p2->p_cred);
791 int count = chglwpcnt(uid, 1);
792 if (__predict_false(count >
793 p2->p_rlimit[RLIMIT_NTHR].rlim_cur)) {
794 if (kauth_authorize_process(l1->l_cred,
795 KAUTH_PROCESS_RLIMIT, p2,
796 KAUTH_ARG(KAUTH_REQ_PROCESS_RLIMIT_BYPASS),
797 &p2->p_rlimit[RLIMIT_NTHR], KAUTH_ARG(RLIMIT_NTHR))
798 != 0) {
799 (void)chglwpcnt(uid, -1);
800 mutex_exit(p2->p_lock);
801 return EAGAIN;
802 }
803 }
804 }
805
806 /*
807 * First off, reap any detached LWP waiting to be collected.
808 * We can re-use its LWP structure and turnstile.
809 */
810 if ((l2 = p2->p_zomblwp) != NULL) {
811 p2->p_zomblwp = NULL;
812 lwp_free(l2, true, false);
813 /* p2 now unlocked by lwp_free() */
814 KASSERT(l2->l_ts != NULL);
815 KASSERT(l2->l_inheritedprio == -1);
816 KASSERT(SLIST_EMPTY(&l2->l_pi_lenders));
817 memset(&l2->l_startzero, 0, sizeof(*l2) -
818 offsetof(lwp_t, l_startzero));
819 } else {
820 mutex_exit(p2->p_lock);
821 l2 = pool_cache_get(lwp_cache, PR_WAITOK);
822 memset(&l2->l_startzero, 0, sizeof(*l2) -
823 offsetof(lwp_t, l_startzero));
824 SLIST_INIT(&l2->l_pi_lenders);
825 }
826
827 /*
828 * Because of lockless lookup via pid_table, the LWP can be locked
829 * and inspected briefly even after it's freed, so a few fields are
830 * kept stable.
831 */
832 KASSERT(l2->l_stat == LSIDL);
833 KASSERT(l2->l_cpu != NULL);
834 KASSERT(l2->l_ts != NULL);
835 KASSERT(l2->l_mutex == l2->l_cpu->ci_schedstate.spc_lwplock);
836
837 l2->l_proc = p2;
838 l2->l_refcnt = 0;
839 l2->l_class = sclass;
840
841 /*
842 * Allocate a process ID for this LWP. We need to do this now
843 * while we can still unwind if it fails. Because we're marked
844 * as LSIDL, no lookups by the ID will succeed.
845 *
846 * N.B. this will always succeed for the first LWP in a process,
847 * because proc_alloc_lwpid() will usurp the slot. Also note
848 * that l2->l_proc MUST be valid so that lookups of the proc
849 * will succeed, even if the LWP itself is not visible.
850 */
851 if (__predict_false(proc_alloc_lwpid(p2, l2) == -1)) {
852 pool_cache_put(lwp_cache, l2);
853 return EAGAIN;
854 }
855
856 /*
857 * If vfork(), we want the LWP to run fast and on the same CPU
858 * as its parent, so that it can reuse the VM context and cache
859 * footprint on the local CPU.
860 */
861 l2->l_boostpri = ((flags & LWP_VFORK) ? PRI_KERNEL : PRI_USER);
862 l2->l_priority = l1->l_priority;
863 l2->l_inheritedprio = -1;
864 l2->l_protectprio = -1;
865 l2->l_auxprio = -1;
866 l2->l_flag = 0;
867 l2->l_pflag = LP_MPSAFE;
868 TAILQ_INIT(&l2->l_ld_locks);
869 l2->l_psrefs = 0;
870 kmsan_lwp_alloc(l2);
871
872 /*
873 * For vfork, borrow parent's lwpctl context if it exists.
874 * This also causes us to return via lwp_userret.
875 */
876 if (flags & LWP_VFORK && l1->l_lwpctl) {
877 l2->l_lwpctl = l1->l_lwpctl;
878 l2->l_flag |= LW_LWPCTL;
879 }
880
881 /*
882 * If not the first LWP in the process, grab a reference to the
883 * descriptor table.
884 */
885 l2->l_fd = p2->p_fd;
886 if (p2->p_nlwps != 0) {
887 KASSERT(l1->l_proc == p2);
888 fd_hold(l2);
889 } else {
890 KASSERT(l1->l_proc != p2);
891 }
892
893 if (p2->p_flag & PK_SYSTEM) {
894 /* Mark it as a system LWP. */
895 l2->l_flag |= LW_SYSTEM;
896 }
897
898 kdtrace_thread_ctor(NULL, l2);
899 lwp_initspecific(l2);
900 sched_lwp_fork(l1, l2);
901 callout_init(&l2->l_timeout_ch, CALLOUT_MPSAFE);
902 callout_setfunc(&l2->l_timeout_ch, sleepq_timeout, l2);
903 cv_init(&l2->l_sigcv, "sigwait");
904 cv_init(&l2->l_waitcv, "vfork");
905 l2->l_syncobj = &sched_syncobj;
906 PSREF_DEBUG_INIT_LWP(l2);
907
908 if (rnewlwpp != NULL)
909 *rnewlwpp = l2;
910
911 /*
912 * PCU state needs to be saved before calling uvm_lwp_fork() so that
913 * the MD cpu_lwp_fork() can copy the saved state to the new LWP.
914 */
915 pcu_save_all(l1);
916 #if PCU_UNIT_COUNT > 0
917 l2->l_pcu_valid = l1->l_pcu_valid;
918 #endif
919
920 uvm_lwp_setuarea(l2, uaddr);
921 uvm_lwp_fork(l1, l2, stack, stacksize, func, (arg != NULL) ? arg : l2);
922
923 mutex_enter(p2->p_lock);
924 l2->l_cred = kauth_cred_hold(p2->p_cred);
925 if ((flags & LWP_DETACHED) != 0) {
926 l2->l_prflag = LPR_DETACHED;
927 p2->p_ndlwps++;
928 } else
929 l2->l_prflag = 0;
930
931 if (l1->l_proc == p2) {
932 /*
933 * These flags are set while p_lock is held. Copy with
934 * p_lock held too, so the LWP doesn't sneak into the
935 * process without them being set.
936 */
937 l2->l_flag |= (l1->l_flag & (LW_WEXIT | LW_WREBOOT | LW_WCORE));
938 } else {
939 /* fork(): pending core/exit doesn't apply to child. */
940 l2->l_flag |= (l1->l_flag & LW_WREBOOT);
941 }
942
943 l2->l_sigstk = *sigstk;
944 l2->l_sigmask = *sigmask;
945 TAILQ_INIT(&l2->l_sigpend.sp_info);
946 sigemptyset(&l2->l_sigpend.sp_set);
947 LIST_INSERT_HEAD(&p2->p_lwps, l2, l_sibling);
948 p2->p_nlwps++;
949 p2->p_nrlwps++;
950
951 KASSERT(l2->l_affinity == NULL);
952
953 /* Inherit the affinity mask. */
954 if (l1->l_affinity) {
955 /*
956 * Note that we hold the state lock while inheriting
957 * the affinity to avoid race with sched_setaffinity().
958 */
959 lwp_lock(l1);
960 if (l1->l_affinity) {
961 kcpuset_use(l1->l_affinity);
962 l2->l_affinity = l1->l_affinity;
963 }
964 lwp_unlock(l1);
965 }
966
967 /* Ensure a trip through lwp_userret() if needed. */
968 if ((l2->l_flag & LW_USERRET) != 0) {
969 lwp_need_userret(l2);
970 }
971
972 /* This marks the end of the "must be atomic" section. */
973 mutex_exit(p2->p_lock);
974
975 SDT_PROBE(proc, kernel, , lwp__create, l2, 0, 0, 0, 0);
976
977 mutex_enter(&proc_lock);
978 LIST_INSERT_HEAD(&alllwp, l2, l_list);
979 /* Inherit a processor-set */
980 l2->l_psid = l1->l_psid;
981 mutex_exit(&proc_lock);
982
983 SYSCALL_TIME_LWP_INIT(l2);
984
985 if (p2->p_emul->e_lwp_fork)
986 (*p2->p_emul->e_lwp_fork)(l1, l2);
987
988 return (0);
989 }
990
991 /*
992 * Set a new LWP running. If the process is stopping, then the LWP is
993 * created stopped.
994 */
995 void
996 lwp_start(lwp_t *l, int flags)
997 {
998 proc_t *p = l->l_proc;
999
1000 mutex_enter(p->p_lock);
1001 lwp_lock(l);
1002 KASSERT(l->l_stat == LSIDL);
1003 if ((flags & LWP_SUSPENDED) != 0) {
1004 /* It'll suspend itself in lwp_userret(). */
1005 l->l_flag |= LW_WSUSPEND;
1006 lwp_need_userret(l);
1007 }
1008 if (p->p_stat == SSTOP || (p->p_sflag & PS_STOPPING) != 0) {
1009 KASSERT(l->l_wchan == NULL);
1010 l->l_stat = LSSTOP;
1011 p->p_nrlwps--;
1012 lwp_unlock(l);
1013 } else {
1014 setrunnable(l);
1015 /* LWP now unlocked */
1016 }
1017 mutex_exit(p->p_lock);
1018 }
1019
1020 /*
1021 * Called by MD code when a new LWP begins execution. Must be called
1022 * with the previous LWP locked (so at splsched), or if there is no
1023 * previous LWP, at splsched.
1024 */
1025 void
1026 lwp_startup(struct lwp *prev, struct lwp *new_lwp)
1027 {
1028 kmutex_t *lock;
1029
1030 KASSERTMSG(new_lwp == curlwp, "l %p curlwp %p prevlwp %p", new_lwp, curlwp, prev);
1031 KASSERT(kpreempt_disabled());
1032 KASSERT(prev != NULL);
1033 KASSERT((prev->l_pflag & LP_RUNNING) != 0);
1034 KASSERT(curcpu()->ci_mtx_count == -2);
1035
1036 /*
1037 * Immediately mark the previous LWP as no longer running and
1038 * unlock (to keep lock wait times short as possible). If a
1039 * zombie, don't touch after clearing LP_RUNNING as it could be
1040 * reaped by another CPU. Use atomic_store_release to ensure
1041 * this -- matches atomic_load_acquire in lwp_free.
1042 */
1043 lock = prev->l_mutex;
1044 if (__predict_false(prev->l_stat == LSZOMB)) {
1045 atomic_store_release(&prev->l_pflag,
1046 prev->l_pflag & ~LP_RUNNING);
1047 } else {
1048 prev->l_pflag &= ~LP_RUNNING;
1049 }
1050 mutex_spin_exit(lock);
1051
1052 /* Correct spin mutex count after mi_switch(). */
1053 curcpu()->ci_mtx_count = 0;
1054
1055 /* Install new VM context. */
1056 if (__predict_true(new_lwp->l_proc->p_vmspace)) {
1057 pmap_activate(new_lwp);
1058 }
1059
1060 /* We remain at IPL_SCHED from mi_switch() - reset it. */
1061 spl0();
1062
1063 LOCKDEBUG_BARRIER(NULL, 0);
1064 SDT_PROBE(proc, kernel, , lwp__start, new_lwp, 0, 0, 0, 0);
1065
1066 /* For kthreads, acquire kernel lock if not MPSAFE. */
1067 if (__predict_false((new_lwp->l_pflag & LP_MPSAFE) == 0)) {
1068 KERNEL_LOCK(1, new_lwp);
1069 }
1070 }
1071
1072 /*
1073 * Exit an LWP.
1074 *
1075 * *** WARNING *** This can be called with (l != curlwp) in error paths.
1076 */
1077 void
1078 lwp_exit(struct lwp *l)
1079 {
1080 struct proc *p = l->l_proc;
1081 struct lwp *l2;
1082 bool current;
1083
1084 current = (l == curlwp);
1085
1086 KASSERT(current || l->l_stat == LSIDL);
1087 KASSERT(current || l->l_target_cpu == NULL);
1088 KASSERT(p == curproc);
1089
1090 SDT_PROBE(proc, kernel, , lwp__exit, l, 0, 0, 0, 0);
1091
1092 /* Verify that we hold no locks; for DIAGNOSTIC check kernel_lock. */
1093 LOCKDEBUG_BARRIER(NULL, 0);
1094 KASSERTMSG(curcpu()->ci_biglock_count == 0, "kernel_lock leaked");
1095
1096 /*
1097 * If we are the last live LWP in a process, we need to exit the
1098 * entire process. We do so with an exit status of zero, because
1099 * it's a "controlled" exit, and because that's what Solaris does.
1100 *
1101 * We are not quite a zombie yet, but for accounting purposes we
1102 * must increment the count of zombies here.
1103 *
1104 * Note: the last LWP's specificdata will be deleted here.
1105 */
1106 mutex_enter(p->p_lock);
1107 if (p->p_nlwps - p->p_nzlwps == 1) {
1108 KASSERT(current == true);
1109 KASSERT(p != &proc0);
1110 exit1(l, 0, 0);
1111 /* NOTREACHED */
1112 }
1113 p->p_nzlwps++;
1114
1115 /*
1116 * Perform any required thread cleanup. Do this early so
1117 * anyone wanting to look us up with lwp_getref_lwpid() will
1118 * fail to find us before we become a zombie.
1119 *
1120 * N.B. this will unlock p->p_lock on our behalf.
1121 */
1122 lwp_thread_cleanup(l);
1123
1124 if (p->p_emul->e_lwp_exit)
1125 (*p->p_emul->e_lwp_exit)(l);
1126
1127 /* Drop filedesc reference. */
1128 fd_free();
1129
1130 /* Release fstrans private data. */
1131 fstrans_lwp_dtor(l);
1132
1133 /* Delete the specificdata while it's still safe to sleep. */
1134 lwp_finispecific(l);
1135
1136 /*
1137 * Release our cached credentials.
1138 */
1139 kauth_cred_free(l->l_cred);
1140 callout_destroy(&l->l_timeout_ch);
1141
1142 /*
1143 * If traced, report LWP exit event to the debugger.
1144 *
1145 * Remove the LWP from the global list.
1146 * Free its LID from the PID namespace if needed.
1147 */
1148 mutex_enter(&proc_lock);
1149
1150 if ((p->p_slflag & (PSL_TRACED|PSL_TRACELWP_EXIT)) ==
1151 (PSL_TRACED|PSL_TRACELWP_EXIT)) {
1152 mutex_enter(p->p_lock);
1153 if (ISSET(p->p_sflag, PS_WEXIT)) {
1154 mutex_exit(p->p_lock);
1155 /*
1156 * We are exiting, bail out without informing parent
1157 * about a terminating LWP as it would deadlock.
1158 */
1159 } else {
1160 eventswitch(TRAP_LWP, PTRACE_LWP_EXIT, l->l_lid);
1161 mutex_enter(&proc_lock);
1162 }
1163 }
1164
1165 LIST_REMOVE(l, l_list);
1166 mutex_exit(&proc_lock);
1167
1168 /*
1169 * Get rid of all references to the LWP that others (e.g. procfs)
1170 * may have, and mark the LWP as a zombie. If the LWP is detached,
1171 * mark it waiting for collection in the proc structure. Note that
1172 * before we can do that, we need to free any other dead, deatched
1173 * LWP waiting to meet its maker.
1174 *
1175 * All conditions need to be observed upon under the same hold of
1176 * p_lock, because if the lock is dropped any of them can change.
1177 */
1178 mutex_enter(p->p_lock);
1179 for (;;) {
1180 if (lwp_drainrefs(l))
1181 continue;
1182 if ((l->l_prflag & LPR_DETACHED) != 0) {
1183 if ((l2 = p->p_zomblwp) != NULL) {
1184 p->p_zomblwp = NULL;
1185 lwp_free(l2, false, false);
1186 /* proc now unlocked */
1187 mutex_enter(p->p_lock);
1188 continue;
1189 }
1190 p->p_zomblwp = l;
1191 }
1192 break;
1193 }
1194
1195 /*
1196 * If we find a pending signal for the process and we have been
1197 * asked to check for signals, then we lose: arrange to have
1198 * all other LWPs in the process check for signals.
1199 */
1200 if ((l->l_flag & LW_PENDSIG) != 0 &&
1201 firstsig(&p->p_sigpend.sp_set) != 0) {
1202 LIST_FOREACH(l2, &p->p_lwps, l_sibling) {
1203 lwp_lock(l2);
1204 signotify(l2);
1205 lwp_unlock(l2);
1206 }
1207 }
1208
1209 /*
1210 * Release any PCU resources before becoming a zombie.
1211 */
1212 pcu_discard_all(l);
1213
1214 lwp_lock(l);
1215 l->l_stat = LSZOMB;
1216 if (l->l_name != NULL) {
1217 strcpy(l->l_name, "(zombie)");
1218 }
1219 lwp_unlock(l);
1220 p->p_nrlwps--;
1221 if (l->l_lwpctl != NULL)
1222 l->l_lwpctl->lc_curcpu = LWPCTL_CPU_EXITED;
1223 mutex_exit(p->p_lock);
1224 cv_broadcast(&p->p_lwpcv);
1225
1226 /*
1227 * We can no longer block. At this point, lwp_free() may already
1228 * be gunning for us. On a multi-CPU system, we may be off p_lwps.
1229 *
1230 * Free MD LWP resources.
1231 */
1232 cpu_lwp_free(l, 0);
1233
1234 if (current) {
1235 /* Switch away into oblivion. */
1236 lwp_lock(l);
1237 spc_lock(l->l_cpu);
1238 mi_switch(l);
1239 panic("lwp_exit");
1240 }
1241 }
1242
1243 /*
1244 * Free a dead LWP's remaining resources.
1245 *
1246 * XXXLWP limits.
1247 */
1248 void
1249 lwp_free(struct lwp *l, bool recycle, bool last)
1250 {
1251 struct proc *p = l->l_proc;
1252 struct rusage *ru;
1253 ksiginfoq_t kq;
1254
1255 KASSERT(l != curlwp);
1256 KASSERT(last || mutex_owned(p->p_lock));
1257
1258 /*
1259 * We use the process credentials instead of the lwp credentials here
1260 * because the lwp credentials maybe cached (just after a setuid call)
1261 * and we don't want pay for syncing, since the lwp is going away
1262 * anyway
1263 */
1264 if (p != &proc0 && p->p_nlwps != 1)
1265 (void)chglwpcnt(kauth_cred_getuid(p->p_cred), -1);
1266
1267 /*
1268 * In the unlikely event that the LWP is still on the CPU,
1269 * then spin until it has switched away.
1270 *
1271 * atomic_load_acquire matches atomic_store_release in
1272 * lwp_startup and mi_switch.
1273 */
1274 while (__predict_false((atomic_load_acquire(&l->l_pflag) & LP_RUNNING)
1275 != 0)) {
1276 SPINLOCK_BACKOFF_HOOK;
1277 }
1278
1279 /*
1280 * Now that the LWP's known off the CPU, reset its state back to
1281 * LSIDL, which defeats anything that might have gotten a hold on
1282 * the LWP via pid_table before the ID was freed. It's important
1283 * to do this with both the LWP locked and p_lock held.
1284 *
1285 * Also reset the CPU and lock pointer back to curcpu(), since the
1286 * LWP will in all likelyhood be cached with the current CPU in
1287 * lwp_cache when we free it and later allocated from there again
1288 * (avoid incidental lock contention).
1289 */
1290 lwp_lock(l);
1291 l->l_stat = LSIDL;
1292 l->l_cpu = curcpu();
1293 lwp_unlock_to(l, l->l_cpu->ci_schedstate.spc_lwplock);
1294
1295 /*
1296 * If this was not the last LWP in the process, then adjust counters
1297 * and unlock. This is done differently for the last LWP in exit1().
1298 */
1299 if (!last) {
1300 /*
1301 * Add the LWP's run time to the process' base value.
1302 * This needs to co-incide with coming off p_lwps.
1303 */
1304 bintime_add(&p->p_rtime, &l->l_rtime);
1305 p->p_pctcpu += l->l_pctcpu;
1306 ru = &p->p_stats->p_ru;
1307 ruadd(ru, &l->l_ru);
1308 LIST_REMOVE(l, l_sibling);
1309 p->p_nlwps--;
1310 p->p_nzlwps--;
1311 if ((l->l_prflag & LPR_DETACHED) != 0)
1312 p->p_ndlwps--;
1313 mutex_exit(p->p_lock);
1314
1315 /*
1316 * Have any LWPs sleeping in lwp_wait() recheck for
1317 * deadlock.
1318 */
1319 cv_broadcast(&p->p_lwpcv);
1320
1321 /* Free the LWP ID. */
1322 mutex_enter(&proc_lock);
1323 proc_free_lwpid(p, l->l_lid);
1324 mutex_exit(&proc_lock);
1325 }
1326
1327 /*
1328 * Destroy the LWP's remaining signal information.
1329 */
1330 ksiginfo_queue_init(&kq);
1331 sigclear(&l->l_sigpend, NULL, &kq);
1332 ksiginfo_queue_drain(&kq);
1333 cv_destroy(&l->l_sigcv);
1334 cv_destroy(&l->l_waitcv);
1335
1336 /*
1337 * Free lwpctl structure and affinity.
1338 */
1339 if (l->l_lwpctl) {
1340 lwp_ctl_free(l);
1341 }
1342 if (l->l_affinity) {
1343 kcpuset_unuse(l->l_affinity, NULL);
1344 l->l_affinity = NULL;
1345 }
1346
1347 /*
1348 * Free remaining data structures and the LWP itself unless the
1349 * caller wants to recycle.
1350 */
1351 if (l->l_name != NULL)
1352 kmem_free(l->l_name, MAXCOMLEN);
1353
1354 kmsan_lwp_free(l);
1355 kcov_lwp_free(l);
1356 cpu_lwp_free2(l);
1357 uvm_lwp_exit(l);
1358
1359 KASSERT(SLIST_EMPTY(&l->l_pi_lenders));
1360 KASSERT(l->l_inheritedprio == -1);
1361 KASSERT(l->l_blcnt == 0);
1362 kdtrace_thread_dtor(NULL, l);
1363 if (!recycle)
1364 pool_cache_put(lwp_cache, l);
1365 }
1366
1367 /*
1368 * Migrate the LWP to the another CPU. Unlocks the LWP.
1369 */
1370 void
1371 lwp_migrate(lwp_t *l, struct cpu_info *tci)
1372 {
1373 struct schedstate_percpu *tspc;
1374 int lstat = l->l_stat;
1375
1376 KASSERT(lwp_locked(l, NULL));
1377 KASSERT(tci != NULL);
1378
1379 /* If LWP is still on the CPU, it must be handled like LSONPROC */
1380 if ((l->l_pflag & LP_RUNNING) != 0) {
1381 lstat = LSONPROC;
1382 }
1383
1384 /*
1385 * The destination CPU could be changed while previous migration
1386 * was not finished.
1387 */
1388 if (l->l_target_cpu != NULL) {
1389 l->l_target_cpu = tci;
1390 lwp_unlock(l);
1391 return;
1392 }
1393
1394 /* Nothing to do if trying to migrate to the same CPU */
1395 if (l->l_cpu == tci) {
1396 lwp_unlock(l);
1397 return;
1398 }
1399
1400 KASSERT(l->l_target_cpu == NULL);
1401 tspc = &tci->ci_schedstate;
1402 switch (lstat) {
1403 case LSRUN:
1404 l->l_target_cpu = tci;
1405 break;
1406 case LSSLEEP:
1407 l->l_cpu = tci;
1408 break;
1409 case LSIDL:
1410 case LSSTOP:
1411 case LSSUSPENDED:
1412 l->l_cpu = tci;
1413 if (l->l_wchan == NULL) {
1414 lwp_unlock_to(l, tspc->spc_lwplock);
1415 return;
1416 }
1417 break;
1418 case LSONPROC:
1419 l->l_target_cpu = tci;
1420 spc_lock(l->l_cpu);
1421 sched_resched_cpu(l->l_cpu, PRI_USER_RT, true);
1422 /* spc now unlocked */
1423 break;
1424 }
1425 lwp_unlock(l);
1426 }
1427
1428 #define lwp_find_exclude(l) \
1429 ((l)->l_stat == LSIDL || (l)->l_stat == LSZOMB)
1430
1431 /*
1432 * Find the LWP in the process. Arguments may be zero, in such case,
1433 * the calling process and first LWP in the list will be used.
1434 * On success - returns proc locked.
1435 *
1436 * => pid == 0 -> look in curproc.
1437 * => pid == -1 -> match any proc.
1438 * => otherwise look up the proc.
1439 *
1440 * => lid == 0 -> first LWP in the proc
1441 * => otherwise specific LWP
1442 */
1443 struct lwp *
1444 lwp_find2(pid_t pid, lwpid_t lid)
1445 {
1446 proc_t *p;
1447 lwp_t *l;
1448
1449 /* First LWP of specified proc. */
1450 if (lid == 0) {
1451 switch (pid) {
1452 case -1:
1453 /* No lookup keys. */
1454 return NULL;
1455 case 0:
1456 p = curproc;
1457 mutex_enter(p->p_lock);
1458 break;
1459 default:
1460 mutex_enter(&proc_lock);
1461 p = proc_find(pid);
1462 if (__predict_false(p == NULL)) {
1463 mutex_exit(&proc_lock);
1464 return NULL;
1465 }
1466 mutex_enter(p->p_lock);
1467 mutex_exit(&proc_lock);
1468 break;
1469 }
1470 LIST_FOREACH(l, &p->p_lwps, l_sibling) {
1471 if (__predict_true(!lwp_find_exclude(l)))
1472 break;
1473 }
1474 goto out;
1475 }
1476
1477 l = proc_find_lwp_acquire_proc(lid, &p);
1478 if (l == NULL)
1479 return NULL;
1480 KASSERT(p != NULL);
1481 KASSERT(mutex_owned(p->p_lock));
1482
1483 if (__predict_false(lwp_find_exclude(l))) {
1484 l = NULL;
1485 goto out;
1486 }
1487
1488 /* Apply proc filter, if applicable. */
1489 switch (pid) {
1490 case -1:
1491 /* Match anything. */
1492 break;
1493 case 0:
1494 if (p != curproc)
1495 l = NULL;
1496 break;
1497 default:
1498 if (p->p_pid != pid)
1499 l = NULL;
1500 break;
1501 }
1502
1503 out:
1504 if (__predict_false(l == NULL)) {
1505 mutex_exit(p->p_lock);
1506 }
1507 return l;
1508 }
1509
1510 /*
1511 * Look up a live LWP within the specified process.
1512 *
1513 * Must be called with p->p_lock held (as it looks at the radix tree,
1514 * and also wants to exclude idle and zombie LWPs).
1515 */
1516 struct lwp *
1517 lwp_find(struct proc *p, lwpid_t id)
1518 {
1519 struct lwp *l;
1520
1521 KASSERT(mutex_owned(p->p_lock));
1522
1523 l = proc_find_lwp(p, id);
1524 KASSERT(l == NULL || l->l_lid == id);
1525
1526 /*
1527 * No need to lock - all of these conditions will
1528 * be visible with the process level mutex held.
1529 */
1530 if (__predict_false(l != NULL && lwp_find_exclude(l)))
1531 l = NULL;
1532
1533 return l;
1534 }
1535
1536 /*
1537 * Update an LWP's cached credentials to mirror the process' master copy.
1538 *
1539 * This happens early in the syscall path, on user trap, and on LWP
1540 * creation. A long-running LWP can also voluntarily choose to update
1541 * its credentials by calling this routine. This may be called from
1542 * LWP_CACHE_CREDS(), which checks l->l_prflag & LPR_CRMOD beforehand.
1543 */
1544 void
1545 lwp_update_creds(struct lwp *l)
1546 {
1547 kauth_cred_t oc;
1548 struct proc *p;
1549
1550 p = l->l_proc;
1551 oc = l->l_cred;
1552
1553 mutex_enter(p->p_lock);
1554 kauth_cred_hold(p->p_cred);
1555 l->l_cred = p->p_cred;
1556 l->l_prflag &= ~LPR_CRMOD;
1557 mutex_exit(p->p_lock);
1558 if (oc != NULL)
1559 kauth_cred_free(oc);
1560 }
1561
1562 /*
1563 * Verify that an LWP is locked, and optionally verify that the lock matches
1564 * one we specify.
1565 */
1566 int
1567 lwp_locked(struct lwp *l, kmutex_t *mtx)
1568 {
1569 kmutex_t *cur = l->l_mutex;
1570
1571 return mutex_owned(cur) && (mtx == cur || mtx == NULL);
1572 }
1573
1574 /*
1575 * Lend a new mutex to an LWP. The old mutex must be held.
1576 */
1577 kmutex_t *
1578 lwp_setlock(struct lwp *l, kmutex_t *mtx)
1579 {
1580 kmutex_t *oldmtx = l->l_mutex;
1581
1582 KASSERT(mutex_owned(oldmtx));
1583
1584 atomic_store_release(&l->l_mutex, mtx);
1585 return oldmtx;
1586 }
1587
1588 /*
1589 * Lend a new mutex to an LWP, and release the old mutex. The old mutex
1590 * must be held.
1591 */
1592 void
1593 lwp_unlock_to(struct lwp *l, kmutex_t *mtx)
1594 {
1595 kmutex_t *old;
1596
1597 KASSERT(lwp_locked(l, NULL));
1598
1599 old = l->l_mutex;
1600 atomic_store_release(&l->l_mutex, mtx);
1601 mutex_spin_exit(old);
1602 }
1603
1604 int
1605 lwp_trylock(struct lwp *l)
1606 {
1607 kmutex_t *old;
1608
1609 for (;;) {
1610 if (!mutex_tryenter(old = atomic_load_consume(&l->l_mutex)))
1611 return 0;
1612 if (__predict_true(atomic_load_relaxed(&l->l_mutex) == old))
1613 return 1;
1614 mutex_spin_exit(old);
1615 }
1616 }
1617
1618 void
1619 lwp_unsleep(lwp_t *l, bool unlock)
1620 {
1621
1622 KASSERT(mutex_owned(l->l_mutex));
1623 (*l->l_syncobj->sobj_unsleep)(l, unlock);
1624 }
1625
1626 /*
1627 * Lock an LWP.
1628 */
1629 void
1630 lwp_lock(lwp_t *l)
1631 {
1632 kmutex_t *old = atomic_load_consume(&l->l_mutex);
1633
1634 /*
1635 * Note: mutex_spin_enter() will have posted a read barrier.
1636 * Re-test l->l_mutex. If it has changed, we need to try again.
1637 */
1638 mutex_spin_enter(old);
1639 while (__predict_false(atomic_load_relaxed(&l->l_mutex) != old)) {
1640 mutex_spin_exit(old);
1641 old = atomic_load_consume(&l->l_mutex);
1642 mutex_spin_enter(old);
1643 }
1644 }
1645
1646 /*
1647 * Unlock an LWP.
1648 */
1649 void
1650 lwp_unlock(lwp_t *l)
1651 {
1652
1653 mutex_spin_exit(l->l_mutex);
1654 }
1655
1656 void
1657 lwp_changepri(lwp_t *l, pri_t pri)
1658 {
1659
1660 KASSERT(mutex_owned(l->l_mutex));
1661
1662 if (l->l_priority == pri)
1663 return;
1664
1665 (*l->l_syncobj->sobj_changepri)(l, pri);
1666 KASSERT(l->l_priority == pri);
1667 }
1668
1669 void
1670 lwp_lendpri(lwp_t *l, pri_t pri)
1671 {
1672 KASSERT(mutex_owned(l->l_mutex));
1673
1674 (*l->l_syncobj->sobj_lendpri)(l, pri);
1675 KASSERT(l->l_inheritedprio == pri);
1676 }
1677
1678 pri_t
1679 lwp_eprio(lwp_t *l)
1680 {
1681 pri_t pri = l->l_priority;
1682
1683 KASSERT(mutex_owned(l->l_mutex));
1684
1685 /*
1686 * Timeshared/user LWPs get a temporary priority boost for blocking
1687 * in kernel. This is key to good interactive response on a loaded
1688 * system: without it, things will seem very sluggish to the user.
1689 *
1690 * The function of the boost is to get the LWP onto a CPU and
1691 * running quickly. Once that happens the LWP loses the priority
1692 * boost and could be preempted very quickly by another LWP but that
1693 * won't happen often enough to be a annoyance.
1694 */
1695 if (pri <= MAXPRI_USER && l->l_boostpri > MAXPRI_USER)
1696 pri = (pri >> 1) + l->l_boostpri;
1697
1698 return MAX(l->l_auxprio, pri);
1699 }
1700
1701 /*
1702 * Handle exceptions for mi_userret(). Called if a member of LW_USERRET is
1703 * set or a preemption is required.
1704 */
1705 void
1706 lwp_userret(struct lwp *l)
1707 {
1708 struct proc *p;
1709 int sig, f;
1710
1711 KASSERT(l == curlwp);
1712 KASSERT(l->l_stat == LSONPROC);
1713 p = l->l_proc;
1714
1715 for (;;) {
1716 /*
1717 * This is the main location that user preemptions are
1718 * processed.
1719 */
1720 preempt_point();
1721
1722 /*
1723 * It is safe to do this unlocked and without raised SPL,
1724 * since whenever a flag of interest is added to l_flag the
1725 * LWP will take an AST and come down this path again. If a
1726 * remote CPU posts the AST, it will be done with an IPI
1727 * (strongly synchronising).
1728 */
1729 if ((f = atomic_load_relaxed(&l->l_flag) & LW_USERRET) == 0) {
1730 return;
1731 }
1732
1733 /*
1734 * Process pending signals first, unless the process
1735 * is dumping core or exiting, where we will instead
1736 * enter the LW_WSUSPEND case below.
1737 */
1738 if ((f & (LW_PENDSIG | LW_WCORE | LW_WEXIT)) == LW_PENDSIG) {
1739 mutex_enter(p->p_lock);
1740 while ((sig = issignal(l)) != 0)
1741 postsig(sig);
1742 mutex_exit(p->p_lock);
1743 continue;
1744 }
1745
1746 /*
1747 * Core-dump or suspend pending.
1748 *
1749 * In case of core dump, suspend ourselves, so that the kernel
1750 * stack and therefore the userland registers saved in the
1751 * trapframe are around for coredump() to write them out.
1752 * We also need to save any PCU resources that we have so that
1753 * they accessible for coredump(). We issue a wakeup on
1754 * p->p_lwpcv so that sigexit() will write the core file out
1755 * once all other LWPs are suspended.
1756 */
1757 if ((f & LW_WSUSPEND) != 0) {
1758 pcu_save_all(l);
1759 mutex_enter(p->p_lock);
1760 p->p_nrlwps--;
1761 lwp_lock(l);
1762 l->l_stat = LSSUSPENDED;
1763 lwp_unlock(l);
1764 mutex_exit(p->p_lock);
1765 cv_broadcast(&p->p_lwpcv);
1766 lwp_lock(l);
1767 spc_lock(l->l_cpu);
1768 mi_switch(l);
1769 continue;
1770 }
1771
1772 /*
1773 * Process is exiting. The core dump and signal cases must
1774 * be handled first.
1775 */
1776 if ((f & LW_WEXIT) != 0) {
1777 lwp_exit(l);
1778 KASSERT(0);
1779 /* NOTREACHED */
1780 }
1781
1782 /*
1783 * Update lwpctl processor (for vfork child_return).
1784 */
1785 if ((f & LW_LWPCTL) != 0) {
1786 lwp_lock(l);
1787 KASSERT(kpreempt_disabled());
1788 l->l_lwpctl->lc_curcpu = (int)cpu_index(l->l_cpu);
1789 l->l_lwpctl->lc_pctr++;
1790 l->l_flag &= ~LW_LWPCTL;
1791 lwp_unlock(l);
1792 continue;
1793 }
1794 }
1795 }
1796
1797 /*
1798 * Force an LWP to enter the kernel, to take a trip through lwp_userret().
1799 */
1800 void
1801 lwp_need_userret(struct lwp *l)
1802 {
1803
1804 KASSERT(!cpu_intr_p());
1805 KASSERT(lwp_locked(l, NULL) || l->l_stat == LSIDL);
1806
1807 /*
1808 * If the LWP is in any state other than LSONPROC, we know that it
1809 * is executing in-kernel and will hit userret() on the way out.
1810 *
1811 * If the LWP is curlwp, then we know we'll be back out to userspace
1812 * soon (can't be called from a hardware interrupt here).
1813 *
1814 * Otherwise, we can't be sure what the LWP is doing, so first make
1815 * sure the update to l_flag will be globally visible, and then
1816 * force the LWP to take a trip through trap() where it will do
1817 * userret().
1818 */
1819 if (l->l_stat == LSONPROC && l != curlwp) {
1820 membar_producer();
1821 cpu_signotify(l);
1822 }
1823 }
1824
1825 /*
1826 * Add one reference to an LWP. This will prevent the LWP from
1827 * exiting, thus keep the lwp structure and PCB around to inspect.
1828 */
1829 void
1830 lwp_addref(struct lwp *l)
1831 {
1832 KASSERT(mutex_owned(l->l_proc->p_lock));
1833 KASSERT(l->l_stat != LSZOMB);
1834 l->l_refcnt++;
1835 }
1836
1837 /*
1838 * Remove one reference to an LWP. If this is the last reference,
1839 * then we must finalize the LWP's death.
1840 */
1841 void
1842 lwp_delref(struct lwp *l)
1843 {
1844 struct proc *p = l->l_proc;
1845
1846 mutex_enter(p->p_lock);
1847 lwp_delref2(l);
1848 mutex_exit(p->p_lock);
1849 }
1850
1851 /*
1852 * Remove one reference to an LWP. If this is the last reference,
1853 * then we must finalize the LWP's death. The proc mutex is held
1854 * on entry.
1855 */
1856 void
1857 lwp_delref2(struct lwp *l)
1858 {
1859 struct proc *p = l->l_proc;
1860
1861 KASSERT(mutex_owned(p->p_lock));
1862 KASSERT(l->l_stat != LSZOMB);
1863 KASSERT(l->l_refcnt > 0);
1864
1865 if (--l->l_refcnt == 0)
1866 cv_broadcast(&p->p_lwpcv);
1867 }
1868
1869 /*
1870 * Drain all references to the current LWP. Returns true if
1871 * we blocked.
1872 */
1873 bool
1874 lwp_drainrefs(struct lwp *l)
1875 {
1876 struct proc *p = l->l_proc;
1877 bool rv = false;
1878
1879 KASSERT(mutex_owned(p->p_lock));
1880
1881 l->l_prflag |= LPR_DRAINING;
1882
1883 while (l->l_refcnt > 0) {
1884 rv = true;
1885 cv_wait(&p->p_lwpcv, p->p_lock);
1886 }
1887 return rv;
1888 }
1889
1890 /*
1891 * Return true if the specified LWP is 'alive'. Only p->p_lock need
1892 * be held.
1893 */
1894 bool
1895 lwp_alive(lwp_t *l)
1896 {
1897
1898 KASSERT(mutex_owned(l->l_proc->p_lock));
1899
1900 switch (l->l_stat) {
1901 case LSSLEEP:
1902 case LSRUN:
1903 case LSONPROC:
1904 case LSSTOP:
1905 case LSSUSPENDED:
1906 return true;
1907 default:
1908 return false;
1909 }
1910 }
1911
1912 /*
1913 * Return first live LWP in the process.
1914 */
1915 lwp_t *
1916 lwp_find_first(proc_t *p)
1917 {
1918 lwp_t *l;
1919
1920 KASSERT(mutex_owned(p->p_lock));
1921
1922 LIST_FOREACH(l, &p->p_lwps, l_sibling) {
1923 if (lwp_alive(l)) {
1924 return l;
1925 }
1926 }
1927
1928 return NULL;
1929 }
1930
1931 /*
1932 * Allocate a new lwpctl structure for a user LWP.
1933 */
1934 int
1935 lwp_ctl_alloc(vaddr_t *uaddr)
1936 {
1937 lcproc_t *lp;
1938 u_int bit, i, offset;
1939 struct uvm_object *uao;
1940 int error;
1941 lcpage_t *lcp;
1942 proc_t *p;
1943 lwp_t *l;
1944
1945 l = curlwp;
1946 p = l->l_proc;
1947
1948 /* don't allow a vforked process to create lwp ctls */
1949 if (p->p_lflag & PL_PPWAIT)
1950 return EBUSY;
1951
1952 if (l->l_lcpage != NULL) {
1953 lcp = l->l_lcpage;
1954 *uaddr = lcp->lcp_uaddr + (vaddr_t)l->l_lwpctl - lcp->lcp_kaddr;
1955 return 0;
1956 }
1957
1958 /* First time around, allocate header structure for the process. */
1959 if ((lp = p->p_lwpctl) == NULL) {
1960 lp = kmem_alloc(sizeof(*lp), KM_SLEEP);
1961 mutex_init(&lp->lp_lock, MUTEX_DEFAULT, IPL_NONE);
1962 lp->lp_uao = NULL;
1963 TAILQ_INIT(&lp->lp_pages);
1964 mutex_enter(p->p_lock);
1965 if (p->p_lwpctl == NULL) {
1966 p->p_lwpctl = lp;
1967 mutex_exit(p->p_lock);
1968 } else {
1969 mutex_exit(p->p_lock);
1970 mutex_destroy(&lp->lp_lock);
1971 kmem_free(lp, sizeof(*lp));
1972 lp = p->p_lwpctl;
1973 }
1974 }
1975
1976 /*
1977 * Set up an anonymous memory region to hold the shared pages.
1978 * Map them into the process' address space. The user vmspace
1979 * gets the first reference on the UAO.
1980 */
1981 mutex_enter(&lp->lp_lock);
1982 if (lp->lp_uao == NULL) {
1983 lp->lp_uao = uao_create(LWPCTL_UAREA_SZ, 0);
1984 lp->lp_cur = 0;
1985 lp->lp_max = LWPCTL_UAREA_SZ;
1986 lp->lp_uva = p->p_emul->e_vm_default_addr(p,
1987 (vaddr_t)p->p_vmspace->vm_daddr, LWPCTL_UAREA_SZ,
1988 p->p_vmspace->vm_map.flags & VM_MAP_TOPDOWN);
1989 error = uvm_map(&p->p_vmspace->vm_map, &lp->lp_uva,
1990 LWPCTL_UAREA_SZ, lp->lp_uao, 0, 0, UVM_MAPFLAG(UVM_PROT_RW,
1991 UVM_PROT_RW, UVM_INH_NONE, UVM_ADV_NORMAL, 0));
1992 if (error != 0) {
1993 uao_detach(lp->lp_uao);
1994 lp->lp_uao = NULL;
1995 mutex_exit(&lp->lp_lock);
1996 return error;
1997 }
1998 }
1999
2000 /* Get a free block and allocate for this LWP. */
2001 TAILQ_FOREACH(lcp, &lp->lp_pages, lcp_chain) {
2002 if (lcp->lcp_nfree != 0)
2003 break;
2004 }
2005 if (lcp == NULL) {
2006 /* Nothing available - try to set up a free page. */
2007 if (lp->lp_cur == lp->lp_max) {
2008 mutex_exit(&lp->lp_lock);
2009 return ENOMEM;
2010 }
2011 lcp = kmem_alloc(LWPCTL_LCPAGE_SZ, KM_SLEEP);
2012
2013 /*
2014 * Wire the next page down in kernel space. Since this
2015 * is a new mapping, we must add a reference.
2016 */
2017 uao = lp->lp_uao;
2018 (*uao->pgops->pgo_reference)(uao);
2019 lcp->lcp_kaddr = vm_map_min(kernel_map);
2020 error = uvm_map(kernel_map, &lcp->lcp_kaddr, PAGE_SIZE,
2021 uao, lp->lp_cur, PAGE_SIZE,
2022 UVM_MAPFLAG(UVM_PROT_RW, UVM_PROT_RW,
2023 UVM_INH_NONE, UVM_ADV_RANDOM, 0));
2024 if (error != 0) {
2025 mutex_exit(&lp->lp_lock);
2026 kmem_free(lcp, LWPCTL_LCPAGE_SZ);
2027 (*uao->pgops->pgo_detach)(uao);
2028 return error;
2029 }
2030 error = uvm_map_pageable(kernel_map, lcp->lcp_kaddr,
2031 lcp->lcp_kaddr + PAGE_SIZE, FALSE, 0);
2032 if (error != 0) {
2033 mutex_exit(&lp->lp_lock);
2034 uvm_unmap(kernel_map, lcp->lcp_kaddr,
2035 lcp->lcp_kaddr + PAGE_SIZE);
2036 kmem_free(lcp, LWPCTL_LCPAGE_SZ);
2037 return error;
2038 }
2039 /* Prepare the page descriptor and link into the list. */
2040 lcp->lcp_uaddr = lp->lp_uva + lp->lp_cur;
2041 lp->lp_cur += PAGE_SIZE;
2042 lcp->lcp_nfree = LWPCTL_PER_PAGE;
2043 lcp->lcp_rotor = 0;
2044 memset(lcp->lcp_bitmap, 0xff, LWPCTL_BITMAP_SZ);
2045 TAILQ_INSERT_HEAD(&lp->lp_pages, lcp, lcp_chain);
2046 }
2047 for (i = lcp->lcp_rotor; lcp->lcp_bitmap[i] == 0;) {
2048 if (++i >= LWPCTL_BITMAP_ENTRIES)
2049 i = 0;
2050 }
2051 bit = ffs(lcp->lcp_bitmap[i]) - 1;
2052 lcp->lcp_bitmap[i] ^= (1U << bit);
2053 lcp->lcp_rotor = i;
2054 lcp->lcp_nfree--;
2055 l->l_lcpage = lcp;
2056 offset = (i << 5) + bit;
2057 l->l_lwpctl = (lwpctl_t *)lcp->lcp_kaddr + offset;
2058 *uaddr = lcp->lcp_uaddr + offset * sizeof(lwpctl_t);
2059 mutex_exit(&lp->lp_lock);
2060
2061 KPREEMPT_DISABLE(l);
2062 l->l_lwpctl->lc_curcpu = (int)cpu_index(curcpu());
2063 KPREEMPT_ENABLE(l);
2064
2065 return 0;
2066 }
2067
2068 /*
2069 * Free an lwpctl structure back to the per-process list.
2070 */
2071 void
2072 lwp_ctl_free(lwp_t *l)
2073 {
2074 struct proc *p = l->l_proc;
2075 lcproc_t *lp;
2076 lcpage_t *lcp;
2077 u_int map, offset;
2078
2079 /* don't free a lwp context we borrowed for vfork */
2080 if (p->p_lflag & PL_PPWAIT) {
2081 l->l_lwpctl = NULL;
2082 return;
2083 }
2084
2085 lp = p->p_lwpctl;
2086 KASSERT(lp != NULL);
2087
2088 lcp = l->l_lcpage;
2089 offset = (u_int)((lwpctl_t *)l->l_lwpctl - (lwpctl_t *)lcp->lcp_kaddr);
2090 KASSERT(offset < LWPCTL_PER_PAGE);
2091
2092 mutex_enter(&lp->lp_lock);
2093 lcp->lcp_nfree++;
2094 map = offset >> 5;
2095 lcp->lcp_bitmap[map] |= (1U << (offset & 31));
2096 if (lcp->lcp_bitmap[lcp->lcp_rotor] == 0)
2097 lcp->lcp_rotor = map;
2098 if (TAILQ_FIRST(&lp->lp_pages)->lcp_nfree == 0) {
2099 TAILQ_REMOVE(&lp->lp_pages, lcp, lcp_chain);
2100 TAILQ_INSERT_HEAD(&lp->lp_pages, lcp, lcp_chain);
2101 }
2102 mutex_exit(&lp->lp_lock);
2103 }
2104
2105 /*
2106 * Process is exiting; tear down lwpctl state. This can only be safely
2107 * called by the last LWP in the process.
2108 */
2109 void
2110 lwp_ctl_exit(void)
2111 {
2112 lcpage_t *lcp, *next;
2113 lcproc_t *lp;
2114 proc_t *p;
2115 lwp_t *l;
2116
2117 l = curlwp;
2118 l->l_lwpctl = NULL;
2119 l->l_lcpage = NULL;
2120 p = l->l_proc;
2121 lp = p->p_lwpctl;
2122
2123 KASSERT(lp != NULL);
2124 KASSERT(p->p_nlwps == 1);
2125
2126 for (lcp = TAILQ_FIRST(&lp->lp_pages); lcp != NULL; lcp = next) {
2127 next = TAILQ_NEXT(lcp, lcp_chain);
2128 uvm_unmap(kernel_map, lcp->lcp_kaddr,
2129 lcp->lcp_kaddr + PAGE_SIZE);
2130 kmem_free(lcp, LWPCTL_LCPAGE_SZ);
2131 }
2132
2133 if (lp->lp_uao != NULL) {
2134 uvm_unmap(&p->p_vmspace->vm_map, lp->lp_uva,
2135 lp->lp_uva + LWPCTL_UAREA_SZ);
2136 }
2137
2138 mutex_destroy(&lp->lp_lock);
2139 kmem_free(lp, sizeof(*lp));
2140 p->p_lwpctl = NULL;
2141 }
2142
2143 /*
2144 * Return the current LWP's "preemption counter". Used to detect
2145 * preemption across operations that can tolerate preemption without
2146 * crashing, but which may generate incorrect results if preempted.
2147 */
2148 long
2149 lwp_pctr(void)
2150 {
2151
2152 return curlwp->l_ru.ru_nvcsw + curlwp->l_ru.ru_nivcsw;
2153 }
2154
2155 /*
2156 * Set an LWP's private data pointer.
2157 */
2158 int
2159 lwp_setprivate(struct lwp *l, void *ptr)
2160 {
2161 int error = 0;
2162
2163 l->l_private = ptr;
2164 #ifdef __HAVE_CPU_LWP_SETPRIVATE
2165 error = cpu_lwp_setprivate(l, ptr);
2166 #endif
2167 return error;
2168 }
2169
2170 /*
2171 * Perform any thread-related cleanup on LWP exit.
2172 * N.B. l->l_proc->p_lock must be HELD on entry but will
2173 * be released before returning!
2174 */
2175 void
2176 lwp_thread_cleanup(struct lwp *l)
2177 {
2178
2179 KASSERT(mutex_owned(l->l_proc->p_lock));
2180 mutex_exit(l->l_proc->p_lock);
2181
2182 /*
2183 * If the LWP has robust futexes, release them all
2184 * now.
2185 */
2186 if (__predict_false(l->l_robust_head != 0)) {
2187 futex_release_all_lwp(l);
2188 }
2189 }
2190
2191 #if defined(DDB)
2192 #include <machine/pcb.h>
2193
2194 void
2195 lwp_whatis(uintptr_t addr, void (*pr)(const char *, ...))
2196 {
2197 lwp_t *l;
2198
2199 LIST_FOREACH(l, &alllwp, l_list) {
2200 uintptr_t stack = (uintptr_t)KSTACK_LOWEST_ADDR(l);
2201
2202 if (addr < stack || stack + KSTACK_SIZE <= addr) {
2203 continue;
2204 }
2205 (*pr)("%p is %p+%zu, LWP %p's stack\n",
2206 (void *)addr, (void *)stack,
2207 (size_t)(addr - stack), l);
2208 }
2209 }
2210 #endif /* defined(DDB) */
2211