kern_lwp.c revision 1.202.2.2 1 /* $NetBSD: kern_lwp.c,v 1.202.2.2 2019/10/15 19:01:06 martin Exp $ */
2
3 /*-
4 * Copyright (c) 2001, 2006, 2007, 2008, 2009 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Nathan J. Williams, and Andrew Doran.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32 /*
33 * Overview
34 *
35 * Lightweight processes (LWPs) are the basic unit or thread of
36 * execution within the kernel. The core state of an LWP is described
37 * by "struct lwp", also known as lwp_t.
38 *
39 * Each LWP is contained within a process (described by "struct proc"),
40 * Every process contains at least one LWP, but may contain more. The
41 * process describes attributes shared among all of its LWPs such as a
42 * private address space, global execution state (stopped, active,
43 * zombie, ...), signal disposition and so on. On a multiprocessor
44 * machine, multiple LWPs be executing concurrently in the kernel.
45 *
46 * Execution states
47 *
48 * At any given time, an LWP has overall state that is described by
49 * lwp::l_stat. The states are broken into two sets below. The first
50 * set is guaranteed to represent the absolute, current state of the
51 * LWP:
52 *
53 * LSONPROC
54 *
55 * On processor: the LWP is executing on a CPU, either in the
56 * kernel or in user space.
57 *
58 * LSRUN
59 *
60 * Runnable: the LWP is parked on a run queue, and may soon be
61 * chosen to run by an idle processor, or by a processor that
62 * has been asked to preempt a currently runnning but lower
63 * priority LWP.
64 *
65 * LSIDL
66 *
67 * Idle: the LWP has been created but has not yet executed,
68 * or it has ceased executing a unit of work and is waiting
69 * to be started again.
70 *
71 * LSSUSPENDED:
72 *
73 * Suspended: the LWP has had its execution suspended by
74 * another LWP in the same process using the _lwp_suspend()
75 * system call. User-level LWPs also enter the suspended
76 * state when the system is shutting down.
77 *
78 * The second set represent a "statement of intent" on behalf of the
79 * LWP. The LWP may in fact be executing on a processor, may be
80 * sleeping or idle. It is expected to take the necessary action to
81 * stop executing or become "running" again within a short timeframe.
82 * The LP_RUNNING flag in lwp::l_pflag indicates that an LWP is running.
83 * Importantly, it indicates that its state is tied to a CPU.
84 *
85 * LSZOMB:
86 *
87 * Dead or dying: the LWP has released most of its resources
88 * and is about to switch away into oblivion, or has already
89 * switched away. When it switches away, its few remaining
90 * resources can be collected.
91 *
92 * LSSLEEP:
93 *
94 * Sleeping: the LWP has entered itself onto a sleep queue, and
95 * has switched away or will switch away shortly to allow other
96 * LWPs to run on the CPU.
97 *
98 * LSSTOP:
99 *
100 * Stopped: the LWP has been stopped as a result of a job
101 * control signal, or as a result of the ptrace() interface.
102 *
103 * Stopped LWPs may run briefly within the kernel to handle
104 * signals that they receive, but will not return to user space
105 * until their process' state is changed away from stopped.
106 *
107 * Single LWPs within a process can not be set stopped
108 * selectively: all actions that can stop or continue LWPs
109 * occur at the process level.
110 *
111 * State transitions
112 *
113 * Note that the LSSTOP state may only be set when returning to
114 * user space in userret(), or when sleeping interruptably. The
115 * LSSUSPENDED state may only be set in userret(). Before setting
116 * those states, we try to ensure that the LWPs will release all
117 * locks that they hold, and at a minimum try to ensure that the
118 * LWP can be set runnable again by a signal.
119 *
120 * LWPs may transition states in the following ways:
121 *
122 * RUN -------> ONPROC ONPROC -----> RUN
123 * > SLEEP
124 * > STOPPED
125 * > SUSPENDED
126 * > ZOMB
127 * > IDL (special cases)
128 *
129 * STOPPED ---> RUN SUSPENDED --> RUN
130 * > SLEEP
131 *
132 * SLEEP -----> ONPROC IDL --------> RUN
133 * > RUN > SUSPENDED
134 * > STOPPED > STOPPED
135 * > ONPROC (special cases)
136 *
137 * Some state transitions are only possible with kernel threads (eg
138 * ONPROC -> IDL) and happen under tightly controlled circumstances
139 * free of unwanted side effects.
140 *
141 * Migration
142 *
143 * Migration of threads from one CPU to another could be performed
144 * internally by the scheduler via sched_takecpu() or sched_catchlwp()
145 * functions. The universal lwp_migrate() function should be used for
146 * any other cases. Subsystems in the kernel must be aware that CPU
147 * of LWP may change, while it is not locked.
148 *
149 * Locking
150 *
151 * The majority of fields in 'struct lwp' are covered by a single,
152 * general spin lock pointed to by lwp::l_mutex. The locks covering
153 * each field are documented in sys/lwp.h.
154 *
155 * State transitions must be made with the LWP's general lock held,
156 * and may cause the LWP's lock pointer to change. Manipulation of
157 * the general lock is not performed directly, but through calls to
158 * lwp_lock(), lwp_unlock() and others. It should be noted that the
159 * adaptive locks are not allowed to be released while the LWP's lock
160 * is being held (unlike for other spin-locks).
161 *
162 * States and their associated locks:
163 *
164 * LSONPROC, LSZOMB:
165 *
166 * Always covered by spc_lwplock, which protects running LWPs.
167 * This is a per-CPU lock and matches lwp::l_cpu.
168 *
169 * LSIDL, LSRUN:
170 *
171 * Always covered by spc_mutex, which protects the run queues.
172 * This is a per-CPU lock and matches lwp::l_cpu.
173 *
174 * LSSLEEP:
175 *
176 * Covered by a lock associated with the sleep queue that the
177 * LWP resides on. Matches lwp::l_sleepq::sq_mutex.
178 *
179 * LSSTOP, LSSUSPENDED:
180 *
181 * If the LWP was previously sleeping (l_wchan != NULL), then
182 * l_mutex references the sleep queue lock. If the LWP was
183 * runnable or on the CPU when halted, or has been removed from
184 * the sleep queue since halted, then the lock is spc_lwplock.
185 *
186 * The lock order is as follows:
187 *
188 * spc::spc_lwplock ->
189 * sleeptab::st_mutex ->
190 * tschain_t::tc_mutex ->
191 * spc::spc_mutex
192 *
193 * Each process has an scheduler state lock (proc::p_lock), and a
194 * number of counters on LWPs and their states: p_nzlwps, p_nrlwps, and
195 * so on. When an LWP is to be entered into or removed from one of the
196 * following states, p_lock must be held and the process wide counters
197 * adjusted:
198 *
199 * LSIDL, LSZOMB, LSSTOP, LSSUSPENDED
200 *
201 * (But not always for kernel threads. There are some special cases
202 * as mentioned above. See kern_softint.c.)
203 *
204 * Note that an LWP is considered running or likely to run soon if in
205 * one of the following states. This affects the value of p_nrlwps:
206 *
207 * LSRUN, LSONPROC, LSSLEEP
208 *
209 * p_lock does not need to be held when transitioning among these
210 * three states, hence p_lock is rarely taken for state transitions.
211 */
212
213 #include <sys/cdefs.h>
214 __KERNEL_RCSID(0, "$NetBSD: kern_lwp.c,v 1.202.2.2 2019/10/15 19:01:06 martin Exp $");
215
216 #include "opt_ddb.h"
217 #include "opt_lockdebug.h"
218 #include "opt_dtrace.h"
219
220 #define _LWP_API_PRIVATE
221
222 #include <sys/param.h>
223 #include <sys/systm.h>
224 #include <sys/cpu.h>
225 #include <sys/pool.h>
226 #include <sys/proc.h>
227 #include <sys/syscallargs.h>
228 #include <sys/syscall_stats.h>
229 #include <sys/kauth.h>
230 #include <sys/pserialize.h>
231 #include <sys/sleepq.h>
232 #include <sys/lockdebug.h>
233 #include <sys/kmem.h>
234 #include <sys/pset.h>
235 #include <sys/intr.h>
236 #include <sys/lwpctl.h>
237 #include <sys/atomic.h>
238 #include <sys/filedesc.h>
239 #include <sys/fstrans.h>
240 #include <sys/dtrace_bsd.h>
241 #include <sys/sdt.h>
242 #include <sys/ptrace.h>
243 #include <sys/xcall.h>
244 #include <sys/uidinfo.h>
245 #include <sys/sysctl.h>
246 #include <sys/psref.h>
247
248 #include <uvm/uvm_extern.h>
249 #include <uvm/uvm_object.h>
250
251 static pool_cache_t lwp_cache __read_mostly;
252 struct lwplist alllwp __cacheline_aligned;
253
254 static void lwp_dtor(void *, void *);
255
256 /* DTrace proc provider probes */
257 SDT_PROVIDER_DEFINE(proc);
258
259 SDT_PROBE_DEFINE1(proc, kernel, , lwp__create, "struct lwp *");
260 SDT_PROBE_DEFINE1(proc, kernel, , lwp__start, "struct lwp *");
261 SDT_PROBE_DEFINE1(proc, kernel, , lwp__exit, "struct lwp *");
262
263 struct turnstile turnstile0;
264 struct lwp lwp0 __aligned(MIN_LWP_ALIGNMENT) = {
265 #ifdef LWP0_CPU_INFO
266 .l_cpu = LWP0_CPU_INFO,
267 #endif
268 #ifdef LWP0_MD_INITIALIZER
269 .l_md = LWP0_MD_INITIALIZER,
270 #endif
271 .l_proc = &proc0,
272 .l_lid = 1,
273 .l_flag = LW_SYSTEM,
274 .l_stat = LSONPROC,
275 .l_ts = &turnstile0,
276 .l_syncobj = &sched_syncobj,
277 .l_refcnt = 1,
278 .l_priority = PRI_USER + NPRI_USER - 1,
279 .l_inheritedprio = -1,
280 .l_class = SCHED_OTHER,
281 .l_psid = PS_NONE,
282 .l_pi_lenders = SLIST_HEAD_INITIALIZER(&lwp0.l_pi_lenders),
283 .l_name = __UNCONST("swapper"),
284 .l_fd = &filedesc0,
285 };
286
287 static int sysctl_kern_maxlwp(SYSCTLFN_PROTO);
288
289 /*
290 * sysctl helper routine for kern.maxlwp. Ensures that the new
291 * values are not too low or too high.
292 */
293 static int
294 sysctl_kern_maxlwp(SYSCTLFN_ARGS)
295 {
296 int error, nmaxlwp;
297 struct sysctlnode node;
298
299 nmaxlwp = maxlwp;
300 node = *rnode;
301 node.sysctl_data = &nmaxlwp;
302 error = sysctl_lookup(SYSCTLFN_CALL(&node));
303 if (error || newp == NULL)
304 return error;
305
306 if (nmaxlwp < 0 || nmaxlwp >= 65536)
307 return EINVAL;
308 if (nmaxlwp > cpu_maxlwp())
309 return EINVAL;
310 maxlwp = nmaxlwp;
311
312 return 0;
313 }
314
315 static void
316 sysctl_kern_lwp_setup(void)
317 {
318 struct sysctllog *clog = NULL;
319
320 sysctl_createv(&clog, 0, NULL, NULL,
321 CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
322 CTLTYPE_INT, "maxlwp",
323 SYSCTL_DESCR("Maximum number of simultaneous threads"),
324 sysctl_kern_maxlwp, 0, NULL, 0,
325 CTL_KERN, CTL_CREATE, CTL_EOL);
326 }
327
328 void
329 lwpinit(void)
330 {
331
332 LIST_INIT(&alllwp);
333 lwpinit_specificdata();
334 lwp_sys_init();
335 lwp_cache = pool_cache_init(sizeof(lwp_t), MIN_LWP_ALIGNMENT, 0, 0,
336 "lwppl", NULL, IPL_NONE, NULL, lwp_dtor, NULL);
337
338 maxlwp = cpu_maxlwp();
339 sysctl_kern_lwp_setup();
340 }
341
342 void
343 lwp0_init(void)
344 {
345 struct lwp *l = &lwp0;
346
347 KASSERT((void *)uvm_lwp_getuarea(l) != NULL);
348 KASSERT(l->l_lid == proc0.p_nlwpid);
349
350 LIST_INSERT_HEAD(&alllwp, l, l_list);
351
352 callout_init(&l->l_timeout_ch, CALLOUT_MPSAFE);
353 callout_setfunc(&l->l_timeout_ch, sleepq_timeout, l);
354 cv_init(&l->l_sigcv, "sigwait");
355 cv_init(&l->l_waitcv, "vfork");
356
357 kauth_cred_hold(proc0.p_cred);
358 l->l_cred = proc0.p_cred;
359
360 kdtrace_thread_ctor(NULL, l);
361 lwp_initspecific(l);
362
363 SYSCALL_TIME_LWP_INIT(l);
364 }
365
366 static void
367 lwp_dtor(void *arg, void *obj)
368 {
369 lwp_t *l = obj;
370 uint64_t where;
371 (void)l;
372
373 /*
374 * Provide a barrier to ensure that all mutex_oncpu() and rw_oncpu()
375 * calls will exit before memory of LWP is returned to the pool, where
376 * KVA of LWP structure might be freed and re-used for other purposes.
377 * Kernel preemption is disabled around mutex_oncpu() and rw_oncpu()
378 * callers, therefore cross-call to all CPUs will do the job. Also,
379 * the value of l->l_cpu must be still valid at this point.
380 */
381 KASSERT(l->l_cpu != NULL);
382 where = xc_broadcast(0, (xcfunc_t)nullop, NULL, NULL);
383 xc_wait(where);
384 }
385
386 /*
387 * Set an suspended.
388 *
389 * Must be called with p_lock held, and the LWP locked. Will unlock the
390 * LWP before return.
391 */
392 int
393 lwp_suspend(struct lwp *curl, struct lwp *t)
394 {
395 int error;
396
397 KASSERT(mutex_owned(t->l_proc->p_lock));
398 KASSERT(lwp_locked(t, NULL));
399
400 KASSERT(curl != t || curl->l_stat == LSONPROC);
401
402 /*
403 * If the current LWP has been told to exit, we must not suspend anyone
404 * else or deadlock could occur. We won't return to userspace.
405 */
406 if ((curl->l_flag & (LW_WEXIT | LW_WCORE)) != 0) {
407 lwp_unlock(t);
408 return (EDEADLK);
409 }
410
411 if ((t->l_flag & LW_DBGSUSPEND) != 0) {
412 lwp_unlock(t);
413 return 0;
414 }
415
416 error = 0;
417
418 switch (t->l_stat) {
419 case LSRUN:
420 case LSONPROC:
421 t->l_flag |= LW_WSUSPEND;
422 lwp_need_userret(t);
423 lwp_unlock(t);
424 break;
425
426 case LSSLEEP:
427 t->l_flag |= LW_WSUSPEND;
428
429 /*
430 * Kick the LWP and try to get it to the kernel boundary
431 * so that it will release any locks that it holds.
432 * setrunnable() will release the lock.
433 */
434 if ((t->l_flag & LW_SINTR) != 0)
435 setrunnable(t);
436 else
437 lwp_unlock(t);
438 break;
439
440 case LSSUSPENDED:
441 lwp_unlock(t);
442 break;
443
444 case LSSTOP:
445 t->l_flag |= LW_WSUSPEND;
446 setrunnable(t);
447 break;
448
449 case LSIDL:
450 case LSZOMB:
451 error = EINTR; /* It's what Solaris does..... */
452 lwp_unlock(t);
453 break;
454 }
455
456 return (error);
457 }
458
459 /*
460 * Restart a suspended LWP.
461 *
462 * Must be called with p_lock held, and the LWP locked. Will unlock the
463 * LWP before return.
464 */
465 void
466 lwp_continue(struct lwp *l)
467 {
468
469 KASSERT(mutex_owned(l->l_proc->p_lock));
470 KASSERT(lwp_locked(l, NULL));
471
472 /* If rebooting or not suspended, then just bail out. */
473 if ((l->l_flag & LW_WREBOOT) != 0) {
474 lwp_unlock(l);
475 return;
476 }
477
478 l->l_flag &= ~LW_WSUSPEND;
479
480 if (l->l_stat != LSSUSPENDED || (l->l_flag & LW_DBGSUSPEND) != 0) {
481 lwp_unlock(l);
482 return;
483 }
484
485 /* setrunnable() will release the lock. */
486 setrunnable(l);
487 }
488
489 /*
490 * Restart a stopped LWP.
491 *
492 * Must be called with p_lock held, and the LWP NOT locked. Will unlock the
493 * LWP before return.
494 */
495 void
496 lwp_unstop(struct lwp *l)
497 {
498 struct proc *p = l->l_proc;
499
500 KASSERT(mutex_owned(proc_lock));
501 KASSERT(mutex_owned(p->p_lock));
502
503 lwp_lock(l);
504
505 KASSERT((l->l_flag & LW_DBGSUSPEND) == 0);
506
507 /* If not stopped, then just bail out. */
508 if (l->l_stat != LSSTOP) {
509 lwp_unlock(l);
510 return;
511 }
512
513 p->p_stat = SACTIVE;
514 p->p_sflag &= ~PS_STOPPING;
515
516 if (!p->p_waited)
517 p->p_pptr->p_nstopchild--;
518
519 if (l->l_wchan == NULL) {
520 /* setrunnable() will release the lock. */
521 setrunnable(l);
522 } else if (p->p_xsig && (l->l_flag & LW_SINTR) != 0) {
523 /* setrunnable() so we can receive the signal */
524 setrunnable(l);
525 } else {
526 l->l_stat = LSSLEEP;
527 p->p_nrlwps++;
528 lwp_unlock(l);
529 }
530 }
531
532 /*
533 * Wait for an LWP within the current process to exit. If 'lid' is
534 * non-zero, we are waiting for a specific LWP.
535 *
536 * Must be called with p->p_lock held.
537 */
538 int
539 lwp_wait(struct lwp *l, lwpid_t lid, lwpid_t *departed, bool exiting)
540 {
541 const lwpid_t curlid = l->l_lid;
542 proc_t *p = l->l_proc;
543 lwp_t *l2;
544 int error;
545
546 KASSERT(mutex_owned(p->p_lock));
547
548 p->p_nlwpwait++;
549 l->l_waitingfor = lid;
550
551 for (;;) {
552 int nfound;
553
554 /*
555 * Avoid a race between exit1() and sigexit(): if the
556 * process is dumping core, then we need to bail out: call
557 * into lwp_userret() where we will be suspended until the
558 * deed is done.
559 */
560 if ((p->p_sflag & PS_WCORE) != 0) {
561 mutex_exit(p->p_lock);
562 lwp_userret(l);
563 KASSERT(false);
564 }
565
566 /*
567 * First off, drain any detached LWP that is waiting to be
568 * reaped.
569 */
570 while ((l2 = p->p_zomblwp) != NULL) {
571 p->p_zomblwp = NULL;
572 lwp_free(l2, false, false);/* releases proc mutex */
573 mutex_enter(p->p_lock);
574 }
575
576 /*
577 * Now look for an LWP to collect. If the whole process is
578 * exiting, count detached LWPs as eligible to be collected,
579 * but don't drain them here.
580 */
581 nfound = 0;
582 error = 0;
583 LIST_FOREACH(l2, &p->p_lwps, l_sibling) {
584 /*
585 * If a specific wait and the target is waiting on
586 * us, then avoid deadlock. This also traps LWPs
587 * that try to wait on themselves.
588 *
589 * Note that this does not handle more complicated
590 * cycles, like: t1 -> t2 -> t3 -> t1. The process
591 * can still be killed so it is not a major problem.
592 */
593 if (l2->l_lid == lid && l2->l_waitingfor == curlid) {
594 error = EDEADLK;
595 break;
596 }
597 if (l2 == l)
598 continue;
599 if ((l2->l_prflag & LPR_DETACHED) != 0) {
600 nfound += exiting;
601 continue;
602 }
603 if (lid != 0) {
604 if (l2->l_lid != lid)
605 continue;
606 /*
607 * Mark this LWP as the first waiter, if there
608 * is no other.
609 */
610 if (l2->l_waiter == 0)
611 l2->l_waiter = curlid;
612 } else if (l2->l_waiter != 0) {
613 /*
614 * It already has a waiter - so don't
615 * collect it. If the waiter doesn't
616 * grab it we'll get another chance
617 * later.
618 */
619 nfound++;
620 continue;
621 }
622 nfound++;
623
624 /* No need to lock the LWP in order to see LSZOMB. */
625 if (l2->l_stat != LSZOMB)
626 continue;
627
628 /*
629 * We're no longer waiting. Reset the "first waiter"
630 * pointer on the target, in case it was us.
631 */
632 l->l_waitingfor = 0;
633 l2->l_waiter = 0;
634 p->p_nlwpwait--;
635 if (departed)
636 *departed = l2->l_lid;
637 sched_lwp_collect(l2);
638
639 /* lwp_free() releases the proc lock. */
640 lwp_free(l2, false, false);
641 mutex_enter(p->p_lock);
642 return 0;
643 }
644
645 if (error != 0)
646 break;
647 if (nfound == 0) {
648 error = ESRCH;
649 break;
650 }
651
652 /*
653 * Note: since the lock will be dropped, need to restart on
654 * wakeup to run all LWPs again, e.g. there may be new LWPs.
655 */
656 if (exiting) {
657 KASSERT(p->p_nlwps > 1);
658 cv_wait(&p->p_lwpcv, p->p_lock);
659 error = EAGAIN;
660 break;
661 }
662
663 /*
664 * If all other LWPs are waiting for exits or suspends
665 * and the supply of zombies and potential zombies is
666 * exhausted, then we are about to deadlock.
667 *
668 * If the process is exiting (and this LWP is not the one
669 * that is coordinating the exit) then bail out now.
670 */
671 if ((p->p_sflag & PS_WEXIT) != 0 ||
672 p->p_nrlwps + p->p_nzlwps - p->p_ndlwps <= p->p_nlwpwait) {
673 error = EDEADLK;
674 break;
675 }
676
677 /*
678 * Sit around and wait for something to happen. We'll be
679 * awoken if any of the conditions examined change: if an
680 * LWP exits, is collected, or is detached.
681 */
682 if ((error = cv_wait_sig(&p->p_lwpcv, p->p_lock)) != 0)
683 break;
684 }
685
686 /*
687 * We didn't find any LWPs to collect, we may have received a
688 * signal, or some other condition has caused us to bail out.
689 *
690 * If waiting on a specific LWP, clear the waiters marker: some
691 * other LWP may want it. Then, kick all the remaining waiters
692 * so that they can re-check for zombies and for deadlock.
693 */
694 if (lid != 0) {
695 LIST_FOREACH(l2, &p->p_lwps, l_sibling) {
696 if (l2->l_lid == lid) {
697 if (l2->l_waiter == curlid)
698 l2->l_waiter = 0;
699 break;
700 }
701 }
702 }
703 p->p_nlwpwait--;
704 l->l_waitingfor = 0;
705 cv_broadcast(&p->p_lwpcv);
706
707 return error;
708 }
709
710 static lwpid_t
711 lwp_find_free_lid(lwpid_t try_lid, lwp_t * new_lwp, proc_t *p)
712 {
713 #define LID_SCAN (1u << 31)
714 lwp_t *scan, *free_before;
715 lwpid_t nxt_lid;
716
717 /*
718 * We want the first unused lid greater than or equal to
719 * try_lid (modulo 2^31).
720 * (If nothing else ld.elf_so doesn't want lwpid with the top bit set.)
721 * We must not return 0, and avoiding 'LID_SCAN - 1' makes
722 * the outer test easier.
723 * This would be much easier if the list were sorted in
724 * increasing order.
725 * The list is kept sorted in decreasing order.
726 * This code is only used after a process has generated 2^31 lwp.
727 *
728 * Code assumes it can always find an id.
729 */
730
731 try_lid &= LID_SCAN - 1;
732 if (try_lid <= 1)
733 try_lid = 2;
734
735 free_before = NULL;
736 nxt_lid = LID_SCAN - 1;
737 LIST_FOREACH(scan, &p->p_lwps, l_sibling) {
738 if (scan->l_lid != nxt_lid) {
739 /* There are available lid before this entry */
740 free_before = scan;
741 if (try_lid > scan->l_lid)
742 break;
743 }
744 if (try_lid == scan->l_lid) {
745 /* The ideal lid is busy, take a higher one */
746 if (free_before != NULL) {
747 try_lid = free_before->l_lid + 1;
748 break;
749 }
750 /* No higher ones, reuse low numbers */
751 try_lid = 2;
752 }
753
754 nxt_lid = scan->l_lid - 1;
755 if (LIST_NEXT(scan, l_sibling) == NULL) {
756 /* The value we have is lower than any existing lwp */
757 LIST_INSERT_AFTER(scan, new_lwp, l_sibling);
758 return try_lid;
759 }
760 }
761
762 LIST_INSERT_BEFORE(free_before, new_lwp, l_sibling);
763 return try_lid;
764 }
765
766 /*
767 * Create a new LWP within process 'p2', using LWP 'l1' as a template.
768 * The new LWP is created in state LSIDL and must be set running,
769 * suspended, or stopped by the caller.
770 */
771 int
772 lwp_create(lwp_t *l1, proc_t *p2, vaddr_t uaddr, int flags,
773 void *stack, size_t stacksize, void (*func)(void *), void *arg,
774 lwp_t **rnewlwpp, int sclass, const sigset_t *sigmask,
775 const stack_t *sigstk)
776 {
777 struct lwp *l2, *isfree;
778 turnstile_t *ts;
779 lwpid_t lid;
780
781 KASSERT(l1 == curlwp || l1->l_proc == &proc0);
782
783 /*
784 * Enforce limits, excluding the first lwp and kthreads.
785 */
786 if (p2->p_nlwps != 0 && p2 != &proc0) {
787 uid_t uid = kauth_cred_getuid(l1->l_cred);
788 int count = chglwpcnt(uid, 1);
789 if (__predict_false(count >
790 p2->p_rlimit[RLIMIT_NTHR].rlim_cur)) {
791 if (kauth_authorize_process(l1->l_cred,
792 KAUTH_PROCESS_RLIMIT, p2,
793 KAUTH_ARG(KAUTH_REQ_PROCESS_RLIMIT_BYPASS),
794 &p2->p_rlimit[RLIMIT_NTHR], KAUTH_ARG(RLIMIT_NTHR))
795 != 0) {
796 (void)chglwpcnt(uid, -1);
797 return EAGAIN;
798 }
799 }
800 }
801
802 /*
803 * First off, reap any detached LWP waiting to be collected.
804 * We can re-use its LWP structure and turnstile.
805 */
806 isfree = NULL;
807 if (p2->p_zomblwp != NULL) {
808 mutex_enter(p2->p_lock);
809 if ((isfree = p2->p_zomblwp) != NULL) {
810 p2->p_zomblwp = NULL;
811 lwp_free(isfree, true, false);/* releases proc mutex */
812 } else
813 mutex_exit(p2->p_lock);
814 }
815 if (isfree == NULL) {
816 l2 = pool_cache_get(lwp_cache, PR_WAITOK);
817 memset(l2, 0, sizeof(*l2));
818 l2->l_ts = pool_cache_get(turnstile_cache, PR_WAITOK);
819 SLIST_INIT(&l2->l_pi_lenders);
820 } else {
821 l2 = isfree;
822 ts = l2->l_ts;
823 KASSERT(l2->l_inheritedprio == -1);
824 KASSERT(SLIST_EMPTY(&l2->l_pi_lenders));
825 memset(l2, 0, sizeof(*l2));
826 l2->l_ts = ts;
827 }
828
829 l2->l_stat = LSIDL;
830 l2->l_proc = p2;
831 l2->l_refcnt = 1;
832 l2->l_class = sclass;
833
834 /*
835 * If vfork(), we want the LWP to run fast and on the same CPU
836 * as its parent, so that it can reuse the VM context and cache
837 * footprint on the local CPU.
838 */
839 l2->l_kpriority = ((flags & LWP_VFORK) ? true : false);
840 l2->l_kpribase = PRI_KERNEL;
841 l2->l_priority = l1->l_priority;
842 l2->l_inheritedprio = -1;
843 l2->l_protectprio = -1;
844 l2->l_auxprio = -1;
845 l2->l_flag = 0;
846 l2->l_pflag = LP_MPSAFE;
847 TAILQ_INIT(&l2->l_ld_locks);
848 l2->l_psrefs = 0;
849
850 /*
851 * For vfork, borrow parent's lwpctl context if it exists.
852 * This also causes us to return via lwp_userret.
853 */
854 if (flags & LWP_VFORK && l1->l_lwpctl) {
855 l2->l_lwpctl = l1->l_lwpctl;
856 l2->l_flag |= LW_LWPCTL;
857 }
858
859 /*
860 * If not the first LWP in the process, grab a reference to the
861 * descriptor table.
862 */
863 l2->l_fd = p2->p_fd;
864 if (p2->p_nlwps != 0) {
865 KASSERT(l1->l_proc == p2);
866 fd_hold(l2);
867 } else {
868 KASSERT(l1->l_proc != p2);
869 }
870
871 if (p2->p_flag & PK_SYSTEM) {
872 /* Mark it as a system LWP. */
873 l2->l_flag |= LW_SYSTEM;
874 }
875
876 kpreempt_disable();
877 l2->l_mutex = l1->l_cpu->ci_schedstate.spc_mutex;
878 l2->l_cpu = l1->l_cpu;
879 kpreempt_enable();
880
881 kdtrace_thread_ctor(NULL, l2);
882 lwp_initspecific(l2);
883 sched_lwp_fork(l1, l2);
884 lwp_update_creds(l2);
885 callout_init(&l2->l_timeout_ch, CALLOUT_MPSAFE);
886 callout_setfunc(&l2->l_timeout_ch, sleepq_timeout, l2);
887 cv_init(&l2->l_sigcv, "sigwait");
888 cv_init(&l2->l_waitcv, "vfork");
889 l2->l_syncobj = &sched_syncobj;
890 PSREF_DEBUG_INIT_LWP(l2);
891
892 if (rnewlwpp != NULL)
893 *rnewlwpp = l2;
894
895 /*
896 * PCU state needs to be saved before calling uvm_lwp_fork() so that
897 * the MD cpu_lwp_fork() can copy the saved state to the new LWP.
898 */
899 pcu_save_all(l1);
900
901 uvm_lwp_setuarea(l2, uaddr);
902 uvm_lwp_fork(l1, l2, stack, stacksize, func, (arg != NULL) ? arg : l2);
903
904 if ((flags & LWP_PIDLID) != 0) {
905 lid = proc_alloc_pid(p2);
906 l2->l_pflag |= LP_PIDLID;
907 } else {
908 lid = 0;
909 }
910
911 mutex_enter(p2->p_lock);
912
913 if ((flags & LWP_DETACHED) != 0) {
914 l2->l_prflag = LPR_DETACHED;
915 p2->p_ndlwps++;
916 } else
917 l2->l_prflag = 0;
918
919 l2->l_sigstk = *sigstk;
920 l2->l_sigmask = *sigmask;
921 TAILQ_INIT(&l2->l_sigpend.sp_info);
922 sigemptyset(&l2->l_sigpend.sp_set);
923
924 if (__predict_true(lid == 0)) {
925 /*
926 * XXX: l_lid are expected to be unique (for a process)
927 * if LWP_PIDLID is sometimes set this won't be true.
928 * Once 2^31 threads have been allocated we have to
929 * scan to ensure we allocate a unique value.
930 */
931 lid = ++p2->p_nlwpid;
932 if (__predict_false(lid & LID_SCAN)) {
933 lid = lwp_find_free_lid(lid, l2, p2);
934 p2->p_nlwpid = lid | LID_SCAN;
935 /* l2 as been inserted into p_lwps in order */
936 goto skip_insert;
937 }
938 p2->p_nlwpid = lid;
939 }
940 LIST_INSERT_HEAD(&p2->p_lwps, l2, l_sibling);
941 skip_insert:
942 l2->l_lid = lid;
943 p2->p_nlwps++;
944 p2->p_nrlwps++;
945
946 KASSERT(l2->l_affinity == NULL);
947
948 if ((p2->p_flag & PK_SYSTEM) == 0) {
949 /* Inherit the affinity mask. */
950 if (l1->l_affinity) {
951 /*
952 * Note that we hold the state lock while inheriting
953 * the affinity to avoid race with sched_setaffinity().
954 */
955 lwp_lock(l1);
956 if (l1->l_affinity) {
957 kcpuset_use(l1->l_affinity);
958 l2->l_affinity = l1->l_affinity;
959 }
960 lwp_unlock(l1);
961 }
962 lwp_lock(l2);
963 /* Inherit a processor-set */
964 l2->l_psid = l1->l_psid;
965 /* Look for a CPU to start */
966 l2->l_cpu = sched_takecpu(l2);
967 lwp_unlock_to(l2, l2->l_cpu->ci_schedstate.spc_mutex);
968 }
969 mutex_exit(p2->p_lock);
970
971 SDT_PROBE(proc, kernel, , lwp__create, l2, 0, 0, 0, 0);
972
973 mutex_enter(proc_lock);
974 LIST_INSERT_HEAD(&alllwp, l2, l_list);
975 mutex_exit(proc_lock);
976
977 SYSCALL_TIME_LWP_INIT(l2);
978
979 if (p2->p_emul->e_lwp_fork)
980 (*p2->p_emul->e_lwp_fork)(l1, l2);
981
982 return (0);
983 }
984
985 /*
986 * Called by MD code when a new LWP begins execution. Must be called
987 * with the previous LWP locked (so at splsched), or if there is no
988 * previous LWP, at splsched.
989 */
990 void
991 lwp_startup(struct lwp *prev, struct lwp *new_lwp)
992 {
993 KASSERTMSG(new_lwp == curlwp, "l %p curlwp %p prevlwp %p", new_lwp, curlwp, prev);
994
995 SDT_PROBE(proc, kernel, , lwp__start, new_lwp, 0, 0, 0, 0);
996
997 KASSERT(kpreempt_disabled());
998 if (prev != NULL) {
999 /*
1000 * Normalize the count of the spin-mutexes, it was
1001 * increased in mi_switch(). Unmark the state of
1002 * context switch - it is finished for previous LWP.
1003 */
1004 curcpu()->ci_mtx_count++;
1005 membar_exit();
1006 prev->l_ctxswtch = 0;
1007 }
1008 KPREEMPT_DISABLE(new_lwp);
1009 if (__predict_true(new_lwp->l_proc->p_vmspace))
1010 pmap_activate(new_lwp);
1011 spl0();
1012
1013 /* Note trip through cpu_switchto(). */
1014 pserialize_switchpoint();
1015
1016 LOCKDEBUG_BARRIER(NULL, 0);
1017 KPREEMPT_ENABLE(new_lwp);
1018 if ((new_lwp->l_pflag & LP_MPSAFE) == 0) {
1019 KERNEL_LOCK(1, new_lwp);
1020 }
1021 }
1022
1023 /*
1024 * Exit an LWP.
1025 */
1026 void
1027 lwp_exit(struct lwp *l)
1028 {
1029 struct proc *p = l->l_proc;
1030 struct lwp *l2;
1031 bool current;
1032
1033 current = (l == curlwp);
1034
1035 KASSERT(current || (l->l_stat == LSIDL && l->l_target_cpu == NULL));
1036 KASSERT(p == curproc);
1037
1038 SDT_PROBE(proc, kernel, , lwp__exit, l, 0, 0, 0, 0);
1039
1040 /*
1041 * Verify that we hold no locks other than the kernel lock.
1042 */
1043 LOCKDEBUG_BARRIER(&kernel_lock, 0);
1044
1045 /*
1046 * If we are the last live LWP in a process, we need to exit the
1047 * entire process. We do so with an exit status of zero, because
1048 * it's a "controlled" exit, and because that's what Solaris does.
1049 *
1050 * We are not quite a zombie yet, but for accounting purposes we
1051 * must increment the count of zombies here.
1052 *
1053 * Note: the last LWP's specificdata will be deleted here.
1054 */
1055 mutex_enter(p->p_lock);
1056 if (p->p_nlwps - p->p_nzlwps == 1) {
1057 KASSERT(current == true);
1058 KASSERT(p != &proc0);
1059 /* XXXSMP kernel_lock not held */
1060 exit1(l, 0, 0);
1061 /* NOTREACHED */
1062 }
1063 p->p_nzlwps++;
1064 mutex_exit(p->p_lock);
1065
1066 if (p->p_emul->e_lwp_exit)
1067 (*p->p_emul->e_lwp_exit)(l);
1068
1069 /* Drop filedesc reference. */
1070 fd_free();
1071
1072 /* Release fstrans private data. */
1073 fstrans_lwp_dtor(l);
1074
1075 /* Delete the specificdata while it's still safe to sleep. */
1076 lwp_finispecific(l);
1077
1078 /*
1079 * Release our cached credentials.
1080 */
1081 kauth_cred_free(l->l_cred);
1082 callout_destroy(&l->l_timeout_ch);
1083
1084 /*
1085 * If traced, report LWP exit event to the debugger.
1086 *
1087 * Remove the LWP from the global list.
1088 * Free its LID from the PID namespace if needed.
1089 */
1090 mutex_enter(proc_lock);
1091
1092 if ((p->p_slflag & (PSL_TRACED|PSL_TRACELWP_EXIT)) ==
1093 (PSL_TRACED|PSL_TRACELWP_EXIT)) {
1094 mutex_enter(p->p_lock);
1095 if (ISSET(p->p_sflag, PS_WEXIT)) {
1096 mutex_exit(p->p_lock);
1097 /*
1098 * We are exiting, bail out without informing parent
1099 * about a terminating LWP as it would deadlock.
1100 */
1101 } else {
1102 eventswitch(TRAP_LWP, PTRACE_LWP_EXIT, l->l_lid);
1103 mutex_enter(proc_lock);
1104 }
1105 }
1106
1107 LIST_REMOVE(l, l_list);
1108 if ((l->l_pflag & LP_PIDLID) != 0 && l->l_lid != p->p_pid) {
1109 proc_free_pid(l->l_lid);
1110 }
1111 mutex_exit(proc_lock);
1112
1113 /*
1114 * Get rid of all references to the LWP that others (e.g. procfs)
1115 * may have, and mark the LWP as a zombie. If the LWP is detached,
1116 * mark it waiting for collection in the proc structure. Note that
1117 * before we can do that, we need to free any other dead, deatched
1118 * LWP waiting to meet its maker.
1119 */
1120 mutex_enter(p->p_lock);
1121 lwp_drainrefs(l);
1122
1123 if ((l->l_prflag & LPR_DETACHED) != 0) {
1124 while ((l2 = p->p_zomblwp) != NULL) {
1125 p->p_zomblwp = NULL;
1126 lwp_free(l2, false, false);/* releases proc mutex */
1127 mutex_enter(p->p_lock);
1128 l->l_refcnt++;
1129 lwp_drainrefs(l);
1130 }
1131 p->p_zomblwp = l;
1132 }
1133
1134 /*
1135 * If we find a pending signal for the process and we have been
1136 * asked to check for signals, then we lose: arrange to have
1137 * all other LWPs in the process check for signals.
1138 */
1139 if ((l->l_flag & LW_PENDSIG) != 0 &&
1140 firstsig(&p->p_sigpend.sp_set) != 0) {
1141 LIST_FOREACH(l2, &p->p_lwps, l_sibling) {
1142 lwp_lock(l2);
1143 l2->l_flag |= LW_PENDSIG;
1144 lwp_unlock(l2);
1145 }
1146 }
1147
1148 /*
1149 * Release any PCU resources before becoming a zombie.
1150 */
1151 pcu_discard_all(l);
1152
1153 lwp_lock(l);
1154 l->l_stat = LSZOMB;
1155 if (l->l_name != NULL) {
1156 strcpy(l->l_name, "(zombie)");
1157 }
1158 lwp_unlock(l);
1159 p->p_nrlwps--;
1160 cv_broadcast(&p->p_lwpcv);
1161 if (l->l_lwpctl != NULL)
1162 l->l_lwpctl->lc_curcpu = LWPCTL_CPU_EXITED;
1163 mutex_exit(p->p_lock);
1164
1165 /*
1166 * We can no longer block. At this point, lwp_free() may already
1167 * be gunning for us. On a multi-CPU system, we may be off p_lwps.
1168 *
1169 * Free MD LWP resources.
1170 */
1171 cpu_lwp_free(l, 0);
1172
1173 if (current) {
1174 pmap_deactivate(l);
1175
1176 /*
1177 * Release the kernel lock, and switch away into
1178 * oblivion.
1179 */
1180 #ifdef notyet
1181 /* XXXSMP hold in lwp_userret() */
1182 KERNEL_UNLOCK_LAST(l);
1183 #else
1184 KERNEL_UNLOCK_ALL(l, NULL);
1185 #endif
1186 lwp_exit_switchaway(l);
1187 }
1188 }
1189
1190 /*
1191 * Free a dead LWP's remaining resources.
1192 *
1193 * XXXLWP limits.
1194 */
1195 void
1196 lwp_free(struct lwp *l, bool recycle, bool last)
1197 {
1198 struct proc *p = l->l_proc;
1199 struct rusage *ru;
1200 ksiginfoq_t kq;
1201
1202 KASSERT(l != curlwp);
1203 KASSERT(last || mutex_owned(p->p_lock));
1204
1205 /*
1206 * We use the process credentials instead of the lwp credentials here
1207 * because the lwp credentials maybe cached (just after a setuid call)
1208 * and we don't want pay for syncing, since the lwp is going away
1209 * anyway
1210 */
1211 if (p != &proc0 && p->p_nlwps != 1)
1212 (void)chglwpcnt(kauth_cred_getuid(p->p_cred), -1);
1213 /*
1214 * If this was not the last LWP in the process, then adjust
1215 * counters and unlock.
1216 */
1217 if (!last) {
1218 /*
1219 * Add the LWP's run time to the process' base value.
1220 * This needs to co-incide with coming off p_lwps.
1221 */
1222 bintime_add(&p->p_rtime, &l->l_rtime);
1223 p->p_pctcpu += l->l_pctcpu;
1224 ru = &p->p_stats->p_ru;
1225 ruadd(ru, &l->l_ru);
1226 ru->ru_nvcsw += (l->l_ncsw - l->l_nivcsw);
1227 ru->ru_nivcsw += l->l_nivcsw;
1228 LIST_REMOVE(l, l_sibling);
1229 p->p_nlwps--;
1230 p->p_nzlwps--;
1231 if ((l->l_prflag & LPR_DETACHED) != 0)
1232 p->p_ndlwps--;
1233
1234 /*
1235 * Have any LWPs sleeping in lwp_wait() recheck for
1236 * deadlock.
1237 */
1238 cv_broadcast(&p->p_lwpcv);
1239 mutex_exit(p->p_lock);
1240 }
1241
1242 #ifdef MULTIPROCESSOR
1243 /*
1244 * In the unlikely event that the LWP is still on the CPU,
1245 * then spin until it has switched away. We need to release
1246 * all locks to avoid deadlock against interrupt handlers on
1247 * the target CPU.
1248 */
1249 if ((l->l_pflag & LP_RUNNING) != 0 || l->l_cpu->ci_curlwp == l) {
1250 int count;
1251 (void)count; /* XXXgcc */
1252 KERNEL_UNLOCK_ALL(curlwp, &count);
1253 while ((l->l_pflag & LP_RUNNING) != 0 ||
1254 l->l_cpu->ci_curlwp == l)
1255 SPINLOCK_BACKOFF_HOOK;
1256 KERNEL_LOCK(count, curlwp);
1257 }
1258 #endif
1259
1260 /*
1261 * Destroy the LWP's remaining signal information.
1262 */
1263 ksiginfo_queue_init(&kq);
1264 sigclear(&l->l_sigpend, NULL, &kq);
1265 ksiginfo_queue_drain(&kq);
1266 cv_destroy(&l->l_sigcv);
1267 cv_destroy(&l->l_waitcv);
1268
1269 /*
1270 * Free lwpctl structure and affinity.
1271 */
1272 if (l->l_lwpctl) {
1273 lwp_ctl_free(l);
1274 }
1275 if (l->l_affinity) {
1276 kcpuset_unuse(l->l_affinity, NULL);
1277 l->l_affinity = NULL;
1278 }
1279
1280 /*
1281 * Free the LWP's turnstile and the LWP structure itself unless the
1282 * caller wants to recycle them. Also, free the scheduler specific
1283 * data.
1284 *
1285 * We can't return turnstile0 to the pool (it didn't come from it),
1286 * so if it comes up just drop it quietly and move on.
1287 *
1288 * We don't recycle the VM resources at this time.
1289 */
1290
1291 if (!recycle && l->l_ts != &turnstile0)
1292 pool_cache_put(turnstile_cache, l->l_ts);
1293 if (l->l_name != NULL)
1294 kmem_free(l->l_name, MAXCOMLEN);
1295
1296 cpu_lwp_free2(l);
1297 uvm_lwp_exit(l);
1298
1299 KASSERT(SLIST_EMPTY(&l->l_pi_lenders));
1300 KASSERT(l->l_inheritedprio == -1);
1301 KASSERT(l->l_blcnt == 0);
1302 kdtrace_thread_dtor(NULL, l);
1303 if (!recycle)
1304 pool_cache_put(lwp_cache, l);
1305 }
1306
1307 /*
1308 * Migrate the LWP to the another CPU. Unlocks the LWP.
1309 */
1310 void
1311 lwp_migrate(lwp_t *l, struct cpu_info *tci)
1312 {
1313 struct schedstate_percpu *tspc;
1314 int lstat = l->l_stat;
1315
1316 KASSERT(lwp_locked(l, NULL));
1317 KASSERT(tci != NULL);
1318
1319 /* If LWP is still on the CPU, it must be handled like LSONPROC */
1320 if ((l->l_pflag & LP_RUNNING) != 0) {
1321 lstat = LSONPROC;
1322 }
1323
1324 /*
1325 * The destination CPU could be changed while previous migration
1326 * was not finished.
1327 */
1328 if (l->l_target_cpu != NULL) {
1329 l->l_target_cpu = tci;
1330 lwp_unlock(l);
1331 return;
1332 }
1333
1334 /* Nothing to do if trying to migrate to the same CPU */
1335 if (l->l_cpu == tci) {
1336 lwp_unlock(l);
1337 return;
1338 }
1339
1340 KASSERT(l->l_target_cpu == NULL);
1341 tspc = &tci->ci_schedstate;
1342 switch (lstat) {
1343 case LSRUN:
1344 l->l_target_cpu = tci;
1345 break;
1346 case LSIDL:
1347 l->l_cpu = tci;
1348 lwp_unlock_to(l, tspc->spc_mutex);
1349 return;
1350 case LSSLEEP:
1351 l->l_cpu = tci;
1352 break;
1353 case LSSTOP:
1354 case LSSUSPENDED:
1355 l->l_cpu = tci;
1356 if (l->l_wchan == NULL) {
1357 lwp_unlock_to(l, tspc->spc_lwplock);
1358 return;
1359 }
1360 break;
1361 case LSONPROC:
1362 l->l_target_cpu = tci;
1363 spc_lock(l->l_cpu);
1364 cpu_need_resched(l->l_cpu, RESCHED_KPREEMPT);
1365 spc_unlock(l->l_cpu);
1366 break;
1367 }
1368 lwp_unlock(l);
1369 }
1370
1371 /*
1372 * Find the LWP in the process. Arguments may be zero, in such case,
1373 * the calling process and first LWP in the list will be used.
1374 * On success - returns proc locked.
1375 */
1376 struct lwp *
1377 lwp_find2(pid_t pid, lwpid_t lid)
1378 {
1379 proc_t *p;
1380 lwp_t *l;
1381
1382 /* Find the process. */
1383 if (pid != 0) {
1384 mutex_enter(proc_lock);
1385 p = proc_find(pid);
1386 if (p == NULL) {
1387 mutex_exit(proc_lock);
1388 return NULL;
1389 }
1390 mutex_enter(p->p_lock);
1391 mutex_exit(proc_lock);
1392 } else {
1393 p = curlwp->l_proc;
1394 mutex_enter(p->p_lock);
1395 }
1396 /* Find the thread. */
1397 if (lid != 0) {
1398 l = lwp_find(p, lid);
1399 } else {
1400 l = LIST_FIRST(&p->p_lwps);
1401 }
1402 if (l == NULL) {
1403 mutex_exit(p->p_lock);
1404 }
1405 return l;
1406 }
1407
1408 /*
1409 * Look up a live LWP within the specified process.
1410 *
1411 * Must be called with p->p_lock held.
1412 */
1413 struct lwp *
1414 lwp_find(struct proc *p, lwpid_t id)
1415 {
1416 struct lwp *l;
1417
1418 KASSERT(mutex_owned(p->p_lock));
1419
1420 LIST_FOREACH(l, &p->p_lwps, l_sibling) {
1421 if (l->l_lid == id)
1422 break;
1423 }
1424
1425 /*
1426 * No need to lock - all of these conditions will
1427 * be visible with the process level mutex held.
1428 */
1429 if (l != NULL && (l->l_stat == LSIDL || l->l_stat == LSZOMB))
1430 l = NULL;
1431
1432 return l;
1433 }
1434
1435 /*
1436 * Update an LWP's cached credentials to mirror the process' master copy.
1437 *
1438 * This happens early in the syscall path, on user trap, and on LWP
1439 * creation. A long-running LWP can also voluntarily choose to update
1440 * its credentials by calling this routine. This may be called from
1441 * LWP_CACHE_CREDS(), which checks l->l_cred != p->p_cred beforehand.
1442 */
1443 void
1444 lwp_update_creds(struct lwp *l)
1445 {
1446 kauth_cred_t oc;
1447 struct proc *p;
1448
1449 p = l->l_proc;
1450 oc = l->l_cred;
1451
1452 mutex_enter(p->p_lock);
1453 kauth_cred_hold(p->p_cred);
1454 l->l_cred = p->p_cred;
1455 l->l_prflag &= ~LPR_CRMOD;
1456 mutex_exit(p->p_lock);
1457 if (oc != NULL)
1458 kauth_cred_free(oc);
1459 }
1460
1461 /*
1462 * Verify that an LWP is locked, and optionally verify that the lock matches
1463 * one we specify.
1464 */
1465 int
1466 lwp_locked(struct lwp *l, kmutex_t *mtx)
1467 {
1468 kmutex_t *cur = l->l_mutex;
1469
1470 return mutex_owned(cur) && (mtx == cur || mtx == NULL);
1471 }
1472
1473 /*
1474 * Lend a new mutex to an LWP. The old mutex must be held.
1475 */
1476 void
1477 lwp_setlock(struct lwp *l, kmutex_t *mtx)
1478 {
1479
1480 KASSERT(mutex_owned(l->l_mutex));
1481
1482 membar_exit();
1483 l->l_mutex = mtx;
1484 }
1485
1486 /*
1487 * Lend a new mutex to an LWP, and release the old mutex. The old mutex
1488 * must be held.
1489 */
1490 void
1491 lwp_unlock_to(struct lwp *l, kmutex_t *mtx)
1492 {
1493 kmutex_t *old;
1494
1495 KASSERT(lwp_locked(l, NULL));
1496
1497 old = l->l_mutex;
1498 membar_exit();
1499 l->l_mutex = mtx;
1500 mutex_spin_exit(old);
1501 }
1502
1503 int
1504 lwp_trylock(struct lwp *l)
1505 {
1506 kmutex_t *old;
1507
1508 for (;;) {
1509 if (!mutex_tryenter(old = l->l_mutex))
1510 return 0;
1511 if (__predict_true(l->l_mutex == old))
1512 return 1;
1513 mutex_spin_exit(old);
1514 }
1515 }
1516
1517 void
1518 lwp_unsleep(lwp_t *l, bool cleanup)
1519 {
1520
1521 KASSERT(mutex_owned(l->l_mutex));
1522 (*l->l_syncobj->sobj_unsleep)(l, cleanup);
1523 }
1524
1525 /*
1526 * Handle exceptions for mi_userret(). Called if a member of LW_USERRET is
1527 * set.
1528 */
1529 void
1530 lwp_userret(struct lwp *l)
1531 {
1532 struct proc *p;
1533 int sig;
1534
1535 KASSERT(l == curlwp);
1536 KASSERT(l->l_stat == LSONPROC);
1537 p = l->l_proc;
1538
1539 #ifndef __HAVE_FAST_SOFTINTS
1540 /* Run pending soft interrupts. */
1541 if (l->l_cpu->ci_data.cpu_softints != 0)
1542 softint_overlay();
1543 #endif
1544
1545 /*
1546 * It is safe to do this read unlocked on a MP system..
1547 */
1548 while ((l->l_flag & LW_USERRET) != 0) {
1549 /*
1550 * Process pending signals first, unless the process
1551 * is dumping core or exiting, where we will instead
1552 * enter the LW_WSUSPEND case below.
1553 */
1554 if ((l->l_flag & (LW_PENDSIG | LW_WCORE | LW_WEXIT)) ==
1555 LW_PENDSIG) {
1556 mutex_enter(p->p_lock);
1557 while ((sig = issignal(l)) != 0)
1558 postsig(sig);
1559 mutex_exit(p->p_lock);
1560 }
1561
1562 /*
1563 * Core-dump or suspend pending.
1564 *
1565 * In case of core dump, suspend ourselves, so that the kernel
1566 * stack and therefore the userland registers saved in the
1567 * trapframe are around for coredump() to write them out.
1568 * We also need to save any PCU resources that we have so that
1569 * they accessible for coredump(). We issue a wakeup on
1570 * p->p_lwpcv so that sigexit() will write the core file out
1571 * once all other LWPs are suspended.
1572 */
1573 if ((l->l_flag & LW_WSUSPEND) != 0) {
1574 pcu_save_all(l);
1575 mutex_enter(p->p_lock);
1576 p->p_nrlwps--;
1577 cv_broadcast(&p->p_lwpcv);
1578 lwp_lock(l);
1579 l->l_stat = LSSUSPENDED;
1580 lwp_unlock(l);
1581 mutex_exit(p->p_lock);
1582 lwp_lock(l);
1583 mi_switch(l);
1584 }
1585
1586 /* Process is exiting. */
1587 if ((l->l_flag & LW_WEXIT) != 0) {
1588 lwp_exit(l);
1589 KASSERT(0);
1590 /* NOTREACHED */
1591 }
1592
1593 /* update lwpctl processor (for vfork child_return) */
1594 if (l->l_flag & LW_LWPCTL) {
1595 lwp_lock(l);
1596 KASSERT(kpreempt_disabled());
1597 l->l_lwpctl->lc_curcpu = (int)cpu_index(l->l_cpu);
1598 l->l_lwpctl->lc_pctr++;
1599 l->l_flag &= ~LW_LWPCTL;
1600 lwp_unlock(l);
1601 }
1602 }
1603 }
1604
1605 /*
1606 * Force an LWP to enter the kernel, to take a trip through lwp_userret().
1607 */
1608 void
1609 lwp_need_userret(struct lwp *l)
1610 {
1611 KASSERT(lwp_locked(l, NULL));
1612
1613 /*
1614 * Since the tests in lwp_userret() are done unlocked, make sure
1615 * that the condition will be seen before forcing the LWP to enter
1616 * kernel mode.
1617 */
1618 membar_producer();
1619 cpu_signotify(l);
1620 }
1621
1622 /*
1623 * Add one reference to an LWP. This will prevent the LWP from
1624 * exiting, thus keep the lwp structure and PCB around to inspect.
1625 */
1626 void
1627 lwp_addref(struct lwp *l)
1628 {
1629
1630 KASSERT(mutex_owned(l->l_proc->p_lock));
1631 KASSERT(l->l_stat != LSZOMB);
1632 KASSERT(l->l_refcnt != 0);
1633
1634 l->l_refcnt++;
1635 }
1636
1637 /*
1638 * Remove one reference to an LWP. If this is the last reference,
1639 * then we must finalize the LWP's death.
1640 */
1641 void
1642 lwp_delref(struct lwp *l)
1643 {
1644 struct proc *p = l->l_proc;
1645
1646 mutex_enter(p->p_lock);
1647 lwp_delref2(l);
1648 mutex_exit(p->p_lock);
1649 }
1650
1651 /*
1652 * Remove one reference to an LWP. If this is the last reference,
1653 * then we must finalize the LWP's death. The proc mutex is held
1654 * on entry.
1655 */
1656 void
1657 lwp_delref2(struct lwp *l)
1658 {
1659 struct proc *p = l->l_proc;
1660
1661 KASSERT(mutex_owned(p->p_lock));
1662 KASSERT(l->l_stat != LSZOMB);
1663 KASSERT(l->l_refcnt > 0);
1664 if (--l->l_refcnt == 0)
1665 cv_broadcast(&p->p_lwpcv);
1666 }
1667
1668 /*
1669 * Drain all references to the current LWP.
1670 */
1671 void
1672 lwp_drainrefs(struct lwp *l)
1673 {
1674 struct proc *p = l->l_proc;
1675
1676 KASSERT(mutex_owned(p->p_lock));
1677 KASSERT(l->l_refcnt != 0);
1678
1679 l->l_refcnt--;
1680 while (l->l_refcnt != 0)
1681 cv_wait(&p->p_lwpcv, p->p_lock);
1682 }
1683
1684 /*
1685 * Return true if the specified LWP is 'alive'. Only p->p_lock need
1686 * be held.
1687 */
1688 bool
1689 lwp_alive(lwp_t *l)
1690 {
1691
1692 KASSERT(mutex_owned(l->l_proc->p_lock));
1693
1694 switch (l->l_stat) {
1695 case LSSLEEP:
1696 case LSRUN:
1697 case LSONPROC:
1698 case LSSTOP:
1699 case LSSUSPENDED:
1700 return true;
1701 default:
1702 return false;
1703 }
1704 }
1705
1706 /*
1707 * Return first live LWP in the process.
1708 */
1709 lwp_t *
1710 lwp_find_first(proc_t *p)
1711 {
1712 lwp_t *l;
1713
1714 KASSERT(mutex_owned(p->p_lock));
1715
1716 LIST_FOREACH(l, &p->p_lwps, l_sibling) {
1717 if (lwp_alive(l)) {
1718 return l;
1719 }
1720 }
1721
1722 return NULL;
1723 }
1724
1725 /*
1726 * Allocate a new lwpctl structure for a user LWP.
1727 */
1728 int
1729 lwp_ctl_alloc(vaddr_t *uaddr)
1730 {
1731 lcproc_t *lp;
1732 u_int bit, i, offset;
1733 struct uvm_object *uao;
1734 int error;
1735 lcpage_t *lcp;
1736 proc_t *p;
1737 lwp_t *l;
1738
1739 l = curlwp;
1740 p = l->l_proc;
1741
1742 /* don't allow a vforked process to create lwp ctls */
1743 if (p->p_lflag & PL_PPWAIT)
1744 return EBUSY;
1745
1746 if (l->l_lcpage != NULL) {
1747 lcp = l->l_lcpage;
1748 *uaddr = lcp->lcp_uaddr + (vaddr_t)l->l_lwpctl - lcp->lcp_kaddr;
1749 return 0;
1750 }
1751
1752 /* First time around, allocate header structure for the process. */
1753 if ((lp = p->p_lwpctl) == NULL) {
1754 lp = kmem_alloc(sizeof(*lp), KM_SLEEP);
1755 mutex_init(&lp->lp_lock, MUTEX_DEFAULT, IPL_NONE);
1756 lp->lp_uao = NULL;
1757 TAILQ_INIT(&lp->lp_pages);
1758 mutex_enter(p->p_lock);
1759 if (p->p_lwpctl == NULL) {
1760 p->p_lwpctl = lp;
1761 mutex_exit(p->p_lock);
1762 } else {
1763 mutex_exit(p->p_lock);
1764 mutex_destroy(&lp->lp_lock);
1765 kmem_free(lp, sizeof(*lp));
1766 lp = p->p_lwpctl;
1767 }
1768 }
1769
1770 /*
1771 * Set up an anonymous memory region to hold the shared pages.
1772 * Map them into the process' address space. The user vmspace
1773 * gets the first reference on the UAO.
1774 */
1775 mutex_enter(&lp->lp_lock);
1776 if (lp->lp_uao == NULL) {
1777 lp->lp_uao = uao_create(LWPCTL_UAREA_SZ, 0);
1778 lp->lp_cur = 0;
1779 lp->lp_max = LWPCTL_UAREA_SZ;
1780 lp->lp_uva = p->p_emul->e_vm_default_addr(p,
1781 (vaddr_t)p->p_vmspace->vm_daddr, LWPCTL_UAREA_SZ,
1782 p->p_vmspace->vm_map.flags & VM_MAP_TOPDOWN);
1783 error = uvm_map(&p->p_vmspace->vm_map, &lp->lp_uva,
1784 LWPCTL_UAREA_SZ, lp->lp_uao, 0, 0, UVM_MAPFLAG(UVM_PROT_RW,
1785 UVM_PROT_RW, UVM_INH_NONE, UVM_ADV_NORMAL, 0));
1786 if (error != 0) {
1787 uao_detach(lp->lp_uao);
1788 lp->lp_uao = NULL;
1789 mutex_exit(&lp->lp_lock);
1790 return error;
1791 }
1792 }
1793
1794 /* Get a free block and allocate for this LWP. */
1795 TAILQ_FOREACH(lcp, &lp->lp_pages, lcp_chain) {
1796 if (lcp->lcp_nfree != 0)
1797 break;
1798 }
1799 if (lcp == NULL) {
1800 /* Nothing available - try to set up a free page. */
1801 if (lp->lp_cur == lp->lp_max) {
1802 mutex_exit(&lp->lp_lock);
1803 return ENOMEM;
1804 }
1805 lcp = kmem_alloc(LWPCTL_LCPAGE_SZ, KM_SLEEP);
1806
1807 /*
1808 * Wire the next page down in kernel space. Since this
1809 * is a new mapping, we must add a reference.
1810 */
1811 uao = lp->lp_uao;
1812 (*uao->pgops->pgo_reference)(uao);
1813 lcp->lcp_kaddr = vm_map_min(kernel_map);
1814 error = uvm_map(kernel_map, &lcp->lcp_kaddr, PAGE_SIZE,
1815 uao, lp->lp_cur, PAGE_SIZE,
1816 UVM_MAPFLAG(UVM_PROT_RW, UVM_PROT_RW,
1817 UVM_INH_NONE, UVM_ADV_RANDOM, 0));
1818 if (error != 0) {
1819 mutex_exit(&lp->lp_lock);
1820 kmem_free(lcp, LWPCTL_LCPAGE_SZ);
1821 (*uao->pgops->pgo_detach)(uao);
1822 return error;
1823 }
1824 error = uvm_map_pageable(kernel_map, lcp->lcp_kaddr,
1825 lcp->lcp_kaddr + PAGE_SIZE, FALSE, 0);
1826 if (error != 0) {
1827 mutex_exit(&lp->lp_lock);
1828 uvm_unmap(kernel_map, lcp->lcp_kaddr,
1829 lcp->lcp_kaddr + PAGE_SIZE);
1830 kmem_free(lcp, LWPCTL_LCPAGE_SZ);
1831 return error;
1832 }
1833 /* Prepare the page descriptor and link into the list. */
1834 lcp->lcp_uaddr = lp->lp_uva + lp->lp_cur;
1835 lp->lp_cur += PAGE_SIZE;
1836 lcp->lcp_nfree = LWPCTL_PER_PAGE;
1837 lcp->lcp_rotor = 0;
1838 memset(lcp->lcp_bitmap, 0xff, LWPCTL_BITMAP_SZ);
1839 TAILQ_INSERT_HEAD(&lp->lp_pages, lcp, lcp_chain);
1840 }
1841 for (i = lcp->lcp_rotor; lcp->lcp_bitmap[i] == 0;) {
1842 if (++i >= LWPCTL_BITMAP_ENTRIES)
1843 i = 0;
1844 }
1845 bit = ffs(lcp->lcp_bitmap[i]) - 1;
1846 lcp->lcp_bitmap[i] ^= (1U << bit);
1847 lcp->lcp_rotor = i;
1848 lcp->lcp_nfree--;
1849 l->l_lcpage = lcp;
1850 offset = (i << 5) + bit;
1851 l->l_lwpctl = (lwpctl_t *)lcp->lcp_kaddr + offset;
1852 *uaddr = lcp->lcp_uaddr + offset * sizeof(lwpctl_t);
1853 mutex_exit(&lp->lp_lock);
1854
1855 KPREEMPT_DISABLE(l);
1856 l->l_lwpctl->lc_curcpu = (int)cpu_index(curcpu());
1857 KPREEMPT_ENABLE(l);
1858
1859 return 0;
1860 }
1861
1862 /*
1863 * Free an lwpctl structure back to the per-process list.
1864 */
1865 void
1866 lwp_ctl_free(lwp_t *l)
1867 {
1868 struct proc *p = l->l_proc;
1869 lcproc_t *lp;
1870 lcpage_t *lcp;
1871 u_int map, offset;
1872
1873 /* don't free a lwp context we borrowed for vfork */
1874 if (p->p_lflag & PL_PPWAIT) {
1875 l->l_lwpctl = NULL;
1876 return;
1877 }
1878
1879 lp = p->p_lwpctl;
1880 KASSERT(lp != NULL);
1881
1882 lcp = l->l_lcpage;
1883 offset = (u_int)((lwpctl_t *)l->l_lwpctl - (lwpctl_t *)lcp->lcp_kaddr);
1884 KASSERT(offset < LWPCTL_PER_PAGE);
1885
1886 mutex_enter(&lp->lp_lock);
1887 lcp->lcp_nfree++;
1888 map = offset >> 5;
1889 lcp->lcp_bitmap[map] |= (1U << (offset & 31));
1890 if (lcp->lcp_bitmap[lcp->lcp_rotor] == 0)
1891 lcp->lcp_rotor = map;
1892 if (TAILQ_FIRST(&lp->lp_pages)->lcp_nfree == 0) {
1893 TAILQ_REMOVE(&lp->lp_pages, lcp, lcp_chain);
1894 TAILQ_INSERT_HEAD(&lp->lp_pages, lcp, lcp_chain);
1895 }
1896 mutex_exit(&lp->lp_lock);
1897 }
1898
1899 /*
1900 * Process is exiting; tear down lwpctl state. This can only be safely
1901 * called by the last LWP in the process.
1902 */
1903 void
1904 lwp_ctl_exit(void)
1905 {
1906 lcpage_t *lcp, *next;
1907 lcproc_t *lp;
1908 proc_t *p;
1909 lwp_t *l;
1910
1911 l = curlwp;
1912 l->l_lwpctl = NULL;
1913 l->l_lcpage = NULL;
1914 p = l->l_proc;
1915 lp = p->p_lwpctl;
1916
1917 KASSERT(lp != NULL);
1918 KASSERT(p->p_nlwps == 1);
1919
1920 for (lcp = TAILQ_FIRST(&lp->lp_pages); lcp != NULL; lcp = next) {
1921 next = TAILQ_NEXT(lcp, lcp_chain);
1922 uvm_unmap(kernel_map, lcp->lcp_kaddr,
1923 lcp->lcp_kaddr + PAGE_SIZE);
1924 kmem_free(lcp, LWPCTL_LCPAGE_SZ);
1925 }
1926
1927 if (lp->lp_uao != NULL) {
1928 uvm_unmap(&p->p_vmspace->vm_map, lp->lp_uva,
1929 lp->lp_uva + LWPCTL_UAREA_SZ);
1930 }
1931
1932 mutex_destroy(&lp->lp_lock);
1933 kmem_free(lp, sizeof(*lp));
1934 p->p_lwpctl = NULL;
1935 }
1936
1937 /*
1938 * Return the current LWP's "preemption counter". Used to detect
1939 * preemption across operations that can tolerate preemption without
1940 * crashing, but which may generate incorrect results if preempted.
1941 */
1942 uint64_t
1943 lwp_pctr(void)
1944 {
1945
1946 return curlwp->l_ncsw;
1947 }
1948
1949 /*
1950 * Set an LWP's private data pointer.
1951 */
1952 int
1953 lwp_setprivate(struct lwp *l, void *ptr)
1954 {
1955 int error = 0;
1956
1957 l->l_private = ptr;
1958 #ifdef __HAVE_CPU_LWP_SETPRIVATE
1959 error = cpu_lwp_setprivate(l, ptr);
1960 #endif
1961 return error;
1962 }
1963
1964 #if defined(DDB)
1965 #include <machine/pcb.h>
1966
1967 void
1968 lwp_whatis(uintptr_t addr, void (*pr)(const char *, ...))
1969 {
1970 lwp_t *l;
1971
1972 LIST_FOREACH(l, &alllwp, l_list) {
1973 uintptr_t stack = (uintptr_t)KSTACK_LOWEST_ADDR(l);
1974
1975 if (addr < stack || stack + KSTACK_SIZE <= addr) {
1976 continue;
1977 }
1978 (*pr)("%p is %p+%zu, LWP %p's stack\n",
1979 (void *)addr, (void *)stack,
1980 (size_t)(addr - stack), l);
1981 }
1982 }
1983 #endif /* defined(DDB) */
1984