kern_lwp.c revision 1.171 1 /* $NetBSD: kern_lwp.c,v 1.171 2012/07/22 22:40:19 rmind Exp $ */
2
3 /*-
4 * Copyright (c) 2001, 2006, 2007, 2008, 2009 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Nathan J. Williams, and Andrew Doran.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32 /*
33 * Overview
34 *
35 * Lightweight processes (LWPs) are the basic unit or thread of
36 * execution within the kernel. The core state of an LWP is described
37 * by "struct lwp", also known as lwp_t.
38 *
39 * Each LWP is contained within a process (described by "struct proc"),
40 * Every process contains at least one LWP, but may contain more. The
41 * process describes attributes shared among all of its LWPs such as a
42 * private address space, global execution state (stopped, active,
43 * zombie, ...), signal disposition and so on. On a multiprocessor
44 * machine, multiple LWPs be executing concurrently in the kernel.
45 *
46 * Execution states
47 *
48 * At any given time, an LWP has overall state that is described by
49 * lwp::l_stat. The states are broken into two sets below. The first
50 * set is guaranteed to represent the absolute, current state of the
51 * LWP:
52 *
53 * LSONPROC
54 *
55 * On processor: the LWP is executing on a CPU, either in the
56 * kernel or in user space.
57 *
58 * LSRUN
59 *
60 * Runnable: the LWP is parked on a run queue, and may soon be
61 * chosen to run by an idle processor, or by a processor that
62 * has been asked to preempt a currently runnning but lower
63 * priority LWP.
64 *
65 * LSIDL
66 *
67 * Idle: the LWP has been created but has not yet executed,
68 * or it has ceased executing a unit of work and is waiting
69 * to be started again.
70 *
71 * LSSUSPENDED:
72 *
73 * Suspended: the LWP has had its execution suspended by
74 * another LWP in the same process using the _lwp_suspend()
75 * system call. User-level LWPs also enter the suspended
76 * state when the system is shutting down.
77 *
78 * The second set represent a "statement of intent" on behalf of the
79 * LWP. The LWP may in fact be executing on a processor, may be
80 * sleeping or idle. It is expected to take the necessary action to
81 * stop executing or become "running" again within a short timeframe.
82 * The LP_RUNNING flag in lwp::l_pflag indicates that an LWP is running.
83 * Importantly, it indicates that its state is tied to a CPU.
84 *
85 * LSZOMB:
86 *
87 * Dead or dying: the LWP has released most of its resources
88 * and is about to switch away into oblivion, or has already
89 * switched away. When it switches away, its few remaining
90 * resources can be collected.
91 *
92 * LSSLEEP:
93 *
94 * Sleeping: the LWP has entered itself onto a sleep queue, and
95 * has switched away or will switch away shortly to allow other
96 * LWPs to run on the CPU.
97 *
98 * LSSTOP:
99 *
100 * Stopped: the LWP has been stopped as a result of a job
101 * control signal, or as a result of the ptrace() interface.
102 *
103 * Stopped LWPs may run briefly within the kernel to handle
104 * signals that they receive, but will not return to user space
105 * until their process' state is changed away from stopped.
106 *
107 * Single LWPs within a process can not be set stopped
108 * selectively: all actions that can stop or continue LWPs
109 * occur at the process level.
110 *
111 * State transitions
112 *
113 * Note that the LSSTOP state may only be set when returning to
114 * user space in userret(), or when sleeping interruptably. The
115 * LSSUSPENDED state may only be set in userret(). Before setting
116 * those states, we try to ensure that the LWPs will release all
117 * locks that they hold, and at a minimum try to ensure that the
118 * LWP can be set runnable again by a signal.
119 *
120 * LWPs may transition states in the following ways:
121 *
122 * RUN -------> ONPROC ONPROC -----> RUN
123 * > SLEEP
124 * > STOPPED
125 * > SUSPENDED
126 * > ZOMB
127 * > IDL (special cases)
128 *
129 * STOPPED ---> RUN SUSPENDED --> RUN
130 * > SLEEP
131 *
132 * SLEEP -----> ONPROC IDL --------> RUN
133 * > RUN > SUSPENDED
134 * > STOPPED > STOPPED
135 * > ONPROC (special cases)
136 *
137 * Some state transitions are only possible with kernel threads (eg
138 * ONPROC -> IDL) and happen under tightly controlled circumstances
139 * free of unwanted side effects.
140 *
141 * Migration
142 *
143 * Migration of threads from one CPU to another could be performed
144 * internally by the scheduler via sched_takecpu() or sched_catchlwp()
145 * functions. The universal lwp_migrate() function should be used for
146 * any other cases. Subsystems in the kernel must be aware that CPU
147 * of LWP may change, while it is not locked.
148 *
149 * Locking
150 *
151 * The majority of fields in 'struct lwp' are covered by a single,
152 * general spin lock pointed to by lwp::l_mutex. The locks covering
153 * each field are documented in sys/lwp.h.
154 *
155 * State transitions must be made with the LWP's general lock held,
156 * and may cause the LWP's lock pointer to change. Manipulation of
157 * the general lock is not performed directly, but through calls to
158 * lwp_lock(), lwp_unlock() and others. It should be noted that the
159 * adaptive locks are not allowed to be released while the LWP's lock
160 * is being held (unlike for other spin-locks).
161 *
162 * States and their associated locks:
163 *
164 * LSONPROC, LSZOMB:
165 *
166 * Always covered by spc_lwplock, which protects running LWPs.
167 * This is a per-CPU lock and matches lwp::l_cpu.
168 *
169 * LSIDL, LSRUN:
170 *
171 * Always covered by spc_mutex, which protects the run queues.
172 * This is a per-CPU lock and matches lwp::l_cpu.
173 *
174 * LSSLEEP:
175 *
176 * Covered by a lock associated with the sleep queue that the
177 * LWP resides on. Matches lwp::l_sleepq::sq_mutex.
178 *
179 * LSSTOP, LSSUSPENDED:
180 *
181 * If the LWP was previously sleeping (l_wchan != NULL), then
182 * l_mutex references the sleep queue lock. If the LWP was
183 * runnable or on the CPU when halted, or has been removed from
184 * the sleep queue since halted, then the lock is spc_lwplock.
185 *
186 * The lock order is as follows:
187 *
188 * spc::spc_lwplock ->
189 * sleeptab::st_mutex ->
190 * tschain_t::tc_mutex ->
191 * spc::spc_mutex
192 *
193 * Each process has an scheduler state lock (proc::p_lock), and a
194 * number of counters on LWPs and their states: p_nzlwps, p_nrlwps, and
195 * so on. When an LWP is to be entered into or removed from one of the
196 * following states, p_lock must be held and the process wide counters
197 * adjusted:
198 *
199 * LSIDL, LSZOMB, LSSTOP, LSSUSPENDED
200 *
201 * (But not always for kernel threads. There are some special cases
202 * as mentioned above. See kern_softint.c.)
203 *
204 * Note that an LWP is considered running or likely to run soon if in
205 * one of the following states. This affects the value of p_nrlwps:
206 *
207 * LSRUN, LSONPROC, LSSLEEP
208 *
209 * p_lock does not need to be held when transitioning among these
210 * three states, hence p_lock is rarely taken for state transitions.
211 */
212
213 #include <sys/cdefs.h>
214 __KERNEL_RCSID(0, "$NetBSD: kern_lwp.c,v 1.171 2012/07/22 22:40:19 rmind Exp $");
215
216 #include "opt_ddb.h"
217 #include "opt_lockdebug.h"
218 #include "opt_dtrace.h"
219
220 #define _LWP_API_PRIVATE
221
222 #include <sys/param.h>
223 #include <sys/systm.h>
224 #include <sys/cpu.h>
225 #include <sys/pool.h>
226 #include <sys/proc.h>
227 #include <sys/syscallargs.h>
228 #include <sys/syscall_stats.h>
229 #include <sys/kauth.h>
230 #include <sys/pserialize.h>
231 #include <sys/sleepq.h>
232 #include <sys/lockdebug.h>
233 #include <sys/kmem.h>
234 #include <sys/pset.h>
235 #include <sys/intr.h>
236 #include <sys/lwpctl.h>
237 #include <sys/atomic.h>
238 #include <sys/filedesc.h>
239 #include <sys/dtrace_bsd.h>
240 #include <sys/sdt.h>
241 #include <sys/xcall.h>
242 #include <sys/uidinfo.h>
243 #include <sys/sysctl.h>
244
245 #include <uvm/uvm_extern.h>
246 #include <uvm/uvm_object.h>
247
248 static pool_cache_t lwp_cache __read_mostly;
249 struct lwplist alllwp __cacheline_aligned;
250
251 static void lwp_dtor(void *, void *);
252
253 /* DTrace proc provider probes */
254 SDT_PROBE_DEFINE(proc,,,lwp_create,
255 "struct lwp *", NULL,
256 NULL, NULL, NULL, NULL,
257 NULL, NULL, NULL, NULL);
258 SDT_PROBE_DEFINE(proc,,,lwp_start,
259 "struct lwp *", NULL,
260 NULL, NULL, NULL, NULL,
261 NULL, NULL, NULL, NULL);
262 SDT_PROBE_DEFINE(proc,,,lwp_exit,
263 "struct lwp *", NULL,
264 NULL, NULL, NULL, NULL,
265 NULL, NULL, NULL, NULL);
266
267 struct turnstile turnstile0;
268 struct lwp lwp0 __aligned(MIN_LWP_ALIGNMENT) = {
269 #ifdef LWP0_CPU_INFO
270 .l_cpu = LWP0_CPU_INFO,
271 #endif
272 #ifdef LWP0_MD_INITIALIZER
273 .l_md = LWP0_MD_INITIALIZER,
274 #endif
275 .l_proc = &proc0,
276 .l_lid = 1,
277 .l_flag = LW_SYSTEM,
278 .l_stat = LSONPROC,
279 .l_ts = &turnstile0,
280 .l_syncobj = &sched_syncobj,
281 .l_refcnt = 1,
282 .l_priority = PRI_USER + NPRI_USER - 1,
283 .l_inheritedprio = -1,
284 .l_class = SCHED_OTHER,
285 .l_psid = PS_NONE,
286 .l_pi_lenders = SLIST_HEAD_INITIALIZER(&lwp0.l_pi_lenders),
287 .l_name = __UNCONST("swapper"),
288 .l_fd = &filedesc0,
289 };
290
291 static int sysctl_kern_maxlwp(SYSCTLFN_PROTO);
292
293 /*
294 * sysctl helper routine for kern.maxlwp. Ensures that the new
295 * values are not too low or too high.
296 */
297 static int
298 sysctl_kern_maxlwp(SYSCTLFN_ARGS)
299 {
300 int error, nmaxlwp;
301 struct sysctlnode node;
302
303 nmaxlwp = maxlwp;
304 node = *rnode;
305 node.sysctl_data = &nmaxlwp;
306 error = sysctl_lookup(SYSCTLFN_CALL(&node));
307 if (error || newp == NULL)
308 return error;
309
310 if (nmaxlwp < 0 || nmaxlwp >= 65536)
311 return EINVAL;
312 if (nmaxlwp > cpu_maxlwp())
313 return EINVAL;
314 maxlwp = nmaxlwp;
315
316 return 0;
317 }
318
319 static void
320 sysctl_kern_lwp_setup(void)
321 {
322 struct sysctllog *clog = NULL;
323
324 sysctl_createv(&clog, 0, NULL, NULL,
325 CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
326 CTLTYPE_INT, "maxlwp",
327 SYSCTL_DESCR("Maximum number of simultaneous threads"),
328 sysctl_kern_maxlwp, 0, NULL, 0,
329 CTL_KERN, CTL_CREATE, CTL_EOL);
330 }
331
332 void
333 lwpinit(void)
334 {
335
336 LIST_INIT(&alllwp);
337 lwpinit_specificdata();
338 lwp_sys_init();
339 lwp_cache = pool_cache_init(sizeof(lwp_t), MIN_LWP_ALIGNMENT, 0, 0,
340 "lwppl", NULL, IPL_NONE, NULL, lwp_dtor, NULL);
341
342 maxlwp = cpu_maxlwp();
343 sysctl_kern_lwp_setup();
344 }
345
346 void
347 lwp0_init(void)
348 {
349 struct lwp *l = &lwp0;
350
351 KASSERT((void *)uvm_lwp_getuarea(l) != NULL);
352 KASSERT(l->l_lid == proc0.p_nlwpid);
353
354 LIST_INSERT_HEAD(&alllwp, l, l_list);
355
356 callout_init(&l->l_timeout_ch, CALLOUT_MPSAFE);
357 callout_setfunc(&l->l_timeout_ch, sleepq_timeout, l);
358 cv_init(&l->l_sigcv, "sigwait");
359 cv_init(&l->l_waitcv, "vfork");
360
361 kauth_cred_hold(proc0.p_cred);
362 l->l_cred = proc0.p_cred;
363
364 kdtrace_thread_ctor(NULL, l);
365 lwp_initspecific(l);
366
367 SYSCALL_TIME_LWP_INIT(l);
368 }
369
370 static void
371 lwp_dtor(void *arg, void *obj)
372 {
373 lwp_t *l = obj;
374 uint64_t where;
375 (void)l;
376
377 /*
378 * Provide a barrier to ensure that all mutex_oncpu() and rw_oncpu()
379 * calls will exit before memory of LWP is returned to the pool, where
380 * KVA of LWP structure might be freed and re-used for other purposes.
381 * Kernel preemption is disabled around mutex_oncpu() and rw_oncpu()
382 * callers, therefore cross-call to all CPUs will do the job. Also,
383 * the value of l->l_cpu must be still valid at this point.
384 */
385 KASSERT(l->l_cpu != NULL);
386 where = xc_broadcast(0, (xcfunc_t)nullop, NULL, NULL);
387 xc_wait(where);
388 }
389
390 /*
391 * Set an suspended.
392 *
393 * Must be called with p_lock held, and the LWP locked. Will unlock the
394 * LWP before return.
395 */
396 int
397 lwp_suspend(struct lwp *curl, struct lwp *t)
398 {
399 int error;
400
401 KASSERT(mutex_owned(t->l_proc->p_lock));
402 KASSERT(lwp_locked(t, NULL));
403
404 KASSERT(curl != t || curl->l_stat == LSONPROC);
405
406 /*
407 * If the current LWP has been told to exit, we must not suspend anyone
408 * else or deadlock could occur. We won't return to userspace.
409 */
410 if ((curl->l_flag & (LW_WEXIT | LW_WCORE)) != 0) {
411 lwp_unlock(t);
412 return (EDEADLK);
413 }
414
415 error = 0;
416
417 switch (t->l_stat) {
418 case LSRUN:
419 case LSONPROC:
420 t->l_flag |= LW_WSUSPEND;
421 lwp_need_userret(t);
422 lwp_unlock(t);
423 break;
424
425 case LSSLEEP:
426 t->l_flag |= LW_WSUSPEND;
427
428 /*
429 * Kick the LWP and try to get it to the kernel boundary
430 * so that it will release any locks that it holds.
431 * setrunnable() will release the lock.
432 */
433 if ((t->l_flag & LW_SINTR) != 0)
434 setrunnable(t);
435 else
436 lwp_unlock(t);
437 break;
438
439 case LSSUSPENDED:
440 lwp_unlock(t);
441 break;
442
443 case LSSTOP:
444 t->l_flag |= LW_WSUSPEND;
445 setrunnable(t);
446 break;
447
448 case LSIDL:
449 case LSZOMB:
450 error = EINTR; /* It's what Solaris does..... */
451 lwp_unlock(t);
452 break;
453 }
454
455 return (error);
456 }
457
458 /*
459 * Restart a suspended LWP.
460 *
461 * Must be called with p_lock held, and the LWP locked. Will unlock the
462 * LWP before return.
463 */
464 void
465 lwp_continue(struct lwp *l)
466 {
467
468 KASSERT(mutex_owned(l->l_proc->p_lock));
469 KASSERT(lwp_locked(l, NULL));
470
471 /* If rebooting or not suspended, then just bail out. */
472 if ((l->l_flag & LW_WREBOOT) != 0) {
473 lwp_unlock(l);
474 return;
475 }
476
477 l->l_flag &= ~LW_WSUSPEND;
478
479 if (l->l_stat != LSSUSPENDED) {
480 lwp_unlock(l);
481 return;
482 }
483
484 /* setrunnable() will release the lock. */
485 setrunnable(l);
486 }
487
488 /*
489 * Restart a stopped LWP.
490 *
491 * Must be called with p_lock held, and the LWP NOT locked. Will unlock the
492 * LWP before return.
493 */
494 void
495 lwp_unstop(struct lwp *l)
496 {
497 struct proc *p = l->l_proc;
498
499 KASSERT(mutex_owned(proc_lock));
500 KASSERT(mutex_owned(p->p_lock));
501
502 lwp_lock(l);
503
504 /* If not stopped, then just bail out. */
505 if (l->l_stat != LSSTOP) {
506 lwp_unlock(l);
507 return;
508 }
509
510 p->p_stat = SACTIVE;
511 p->p_sflag &= ~PS_STOPPING;
512
513 if (!p->p_waited)
514 p->p_pptr->p_nstopchild--;
515
516 if (l->l_wchan == NULL) {
517 /* setrunnable() will release the lock. */
518 setrunnable(l);
519 } else if (p->p_xstat && (l->l_flag & LW_SINTR) != 0) {
520 /* setrunnable() so we can receive the signal */
521 setrunnable(l);
522 } else {
523 l->l_stat = LSSLEEP;
524 p->p_nrlwps++;
525 lwp_unlock(l);
526 }
527 }
528
529 /*
530 * Wait for an LWP within the current process to exit. If 'lid' is
531 * non-zero, we are waiting for a specific LWP.
532 *
533 * Must be called with p->p_lock held.
534 */
535 int
536 lwp_wait1(struct lwp *l, lwpid_t lid, lwpid_t *departed, int flags)
537 {
538 struct proc *p = l->l_proc;
539 struct lwp *l2;
540 int nfound, error;
541 lwpid_t curlid;
542 bool exiting;
543
544 KASSERT(mutex_owned(p->p_lock));
545
546 p->p_nlwpwait++;
547 l->l_waitingfor = lid;
548 curlid = l->l_lid;
549 exiting = ((flags & LWPWAIT_EXITCONTROL) != 0);
550
551 for (;;) {
552 /*
553 * Avoid a race between exit1() and sigexit(): if the
554 * process is dumping core, then we need to bail out: call
555 * into lwp_userret() where we will be suspended until the
556 * deed is done.
557 */
558 if ((p->p_sflag & PS_WCORE) != 0) {
559 mutex_exit(p->p_lock);
560 lwp_userret(l);
561 #ifdef DIAGNOSTIC
562 panic("lwp_wait1");
563 #endif
564 /* NOTREACHED */
565 }
566
567 /*
568 * First off, drain any detached LWP that is waiting to be
569 * reaped.
570 */
571 while ((l2 = p->p_zomblwp) != NULL) {
572 p->p_zomblwp = NULL;
573 lwp_free(l2, false, false);/* releases proc mutex */
574 mutex_enter(p->p_lock);
575 }
576
577 /*
578 * Now look for an LWP to collect. If the whole process is
579 * exiting, count detached LWPs as eligible to be collected,
580 * but don't drain them here.
581 */
582 nfound = 0;
583 error = 0;
584 LIST_FOREACH(l2, &p->p_lwps, l_sibling) {
585 /*
586 * If a specific wait and the target is waiting on
587 * us, then avoid deadlock. This also traps LWPs
588 * that try to wait on themselves.
589 *
590 * Note that this does not handle more complicated
591 * cycles, like: t1 -> t2 -> t3 -> t1. The process
592 * can still be killed so it is not a major problem.
593 */
594 if (l2->l_lid == lid && l2->l_waitingfor == curlid) {
595 error = EDEADLK;
596 break;
597 }
598 if (l2 == l)
599 continue;
600 if ((l2->l_prflag & LPR_DETACHED) != 0) {
601 nfound += exiting;
602 continue;
603 }
604 if (lid != 0) {
605 if (l2->l_lid != lid)
606 continue;
607 /*
608 * Mark this LWP as the first waiter, if there
609 * is no other.
610 */
611 if (l2->l_waiter == 0)
612 l2->l_waiter = curlid;
613 } else if (l2->l_waiter != 0) {
614 /*
615 * It already has a waiter - so don't
616 * collect it. If the waiter doesn't
617 * grab it we'll get another chance
618 * later.
619 */
620 nfound++;
621 continue;
622 }
623 nfound++;
624
625 /* No need to lock the LWP in order to see LSZOMB. */
626 if (l2->l_stat != LSZOMB)
627 continue;
628
629 /*
630 * We're no longer waiting. Reset the "first waiter"
631 * pointer on the target, in case it was us.
632 */
633 l->l_waitingfor = 0;
634 l2->l_waiter = 0;
635 p->p_nlwpwait--;
636 if (departed)
637 *departed = l2->l_lid;
638 sched_lwp_collect(l2);
639
640 /* lwp_free() releases the proc lock. */
641 lwp_free(l2, false, false);
642 mutex_enter(p->p_lock);
643 return 0;
644 }
645
646 if (error != 0)
647 break;
648 if (nfound == 0) {
649 error = ESRCH;
650 break;
651 }
652
653 /*
654 * The kernel is careful to ensure that it can not deadlock
655 * when exiting - just keep waiting.
656 */
657 if (exiting) {
658 KASSERT(p->p_nlwps > 1);
659 cv_wait(&p->p_lwpcv, p->p_lock);
660 continue;
661 }
662
663 /*
664 * If all other LWPs are waiting for exits or suspends
665 * and the supply of zombies and potential zombies is
666 * exhausted, then we are about to deadlock.
667 *
668 * If the process is exiting (and this LWP is not the one
669 * that is coordinating the exit) then bail out now.
670 */
671 if ((p->p_sflag & PS_WEXIT) != 0 ||
672 p->p_nrlwps + p->p_nzlwps - p->p_ndlwps <= p->p_nlwpwait) {
673 error = EDEADLK;
674 break;
675 }
676
677 /*
678 * Sit around and wait for something to happen. We'll be
679 * awoken if any of the conditions examined change: if an
680 * LWP exits, is collected, or is detached.
681 */
682 if ((error = cv_wait_sig(&p->p_lwpcv, p->p_lock)) != 0)
683 break;
684 }
685
686 /*
687 * We didn't find any LWPs to collect, we may have received a
688 * signal, or some other condition has caused us to bail out.
689 *
690 * If waiting on a specific LWP, clear the waiters marker: some
691 * other LWP may want it. Then, kick all the remaining waiters
692 * so that they can re-check for zombies and for deadlock.
693 */
694 if (lid != 0) {
695 LIST_FOREACH(l2, &p->p_lwps, l_sibling) {
696 if (l2->l_lid == lid) {
697 if (l2->l_waiter == curlid)
698 l2->l_waiter = 0;
699 break;
700 }
701 }
702 }
703 p->p_nlwpwait--;
704 l->l_waitingfor = 0;
705 cv_broadcast(&p->p_lwpcv);
706
707 return error;
708 }
709
710 /*
711 * Create a new LWP within process 'p2', using LWP 'l1' as a template.
712 * The new LWP is created in state LSIDL and must be set running,
713 * suspended, or stopped by the caller.
714 */
715 int
716 lwp_create(lwp_t *l1, proc_t *p2, vaddr_t uaddr, int flags,
717 void *stack, size_t stacksize, void (*func)(void *), void *arg,
718 lwp_t **rnewlwpp, int sclass)
719 {
720 struct lwp *l2, *isfree;
721 turnstile_t *ts;
722 lwpid_t lid;
723
724 KASSERT(l1 == curlwp || l1->l_proc == &proc0);
725
726 /*
727 * Enforce limits, excluding the first lwp and kthreads.
728 */
729 if (p2->p_nlwps != 0 && p2 != &proc0) {
730 uid_t uid = kauth_cred_getuid(l1->l_cred);
731 int count = chglwpcnt(uid, 1);
732 if (__predict_false(count >
733 p2->p_rlimit[RLIMIT_NTHR].rlim_cur)) {
734 if (kauth_authorize_process(l1->l_cred,
735 KAUTH_PROCESS_RLIMIT, p2,
736 KAUTH_ARG(KAUTH_REQ_PROCESS_RLIMIT_BYPASS),
737 &p2->p_rlimit[RLIMIT_NTHR], KAUTH_ARG(RLIMIT_NTHR))
738 != 0) {
739 (void)chglwpcnt(uid, -1);
740 return EAGAIN;
741 }
742 }
743 }
744
745 /*
746 * First off, reap any detached LWP waiting to be collected.
747 * We can re-use its LWP structure and turnstile.
748 */
749 isfree = NULL;
750 if (p2->p_zomblwp != NULL) {
751 mutex_enter(p2->p_lock);
752 if ((isfree = p2->p_zomblwp) != NULL) {
753 p2->p_zomblwp = NULL;
754 lwp_free(isfree, true, false);/* releases proc mutex */
755 } else
756 mutex_exit(p2->p_lock);
757 }
758 if (isfree == NULL) {
759 l2 = pool_cache_get(lwp_cache, PR_WAITOK);
760 memset(l2, 0, sizeof(*l2));
761 l2->l_ts = pool_cache_get(turnstile_cache, PR_WAITOK);
762 SLIST_INIT(&l2->l_pi_lenders);
763 } else {
764 l2 = isfree;
765 ts = l2->l_ts;
766 KASSERT(l2->l_inheritedprio == -1);
767 KASSERT(SLIST_EMPTY(&l2->l_pi_lenders));
768 memset(l2, 0, sizeof(*l2));
769 l2->l_ts = ts;
770 }
771
772 l2->l_stat = LSIDL;
773 l2->l_proc = p2;
774 l2->l_refcnt = 1;
775 l2->l_class = sclass;
776
777 /*
778 * If vfork(), we want the LWP to run fast and on the same CPU
779 * as its parent, so that it can reuse the VM context and cache
780 * footprint on the local CPU.
781 */
782 l2->l_kpriority = ((flags & LWP_VFORK) ? true : false);
783 l2->l_kpribase = PRI_KERNEL;
784 l2->l_priority = l1->l_priority;
785 l2->l_inheritedprio = -1;
786 l2->l_flag = 0;
787 l2->l_pflag = LP_MPSAFE;
788 TAILQ_INIT(&l2->l_ld_locks);
789
790 /*
791 * For vfork, borrow parent's lwpctl context if it exists.
792 * This also causes us to return via lwp_userret.
793 */
794 if (flags & LWP_VFORK && l1->l_lwpctl) {
795 l2->l_lwpctl = l1->l_lwpctl;
796 l2->l_flag |= LW_LWPCTL;
797 }
798
799 /*
800 * If not the first LWP in the process, grab a reference to the
801 * descriptor table.
802 */
803 l2->l_fd = p2->p_fd;
804 if (p2->p_nlwps != 0) {
805 KASSERT(l1->l_proc == p2);
806 fd_hold(l2);
807 } else {
808 KASSERT(l1->l_proc != p2);
809 }
810
811 if (p2->p_flag & PK_SYSTEM) {
812 /* Mark it as a system LWP. */
813 l2->l_flag |= LW_SYSTEM;
814 }
815
816 kpreempt_disable();
817 l2->l_mutex = l1->l_cpu->ci_schedstate.spc_mutex;
818 l2->l_cpu = l1->l_cpu;
819 kpreempt_enable();
820
821 kdtrace_thread_ctor(NULL, l2);
822 lwp_initspecific(l2);
823 sched_lwp_fork(l1, l2);
824 lwp_update_creds(l2);
825 callout_init(&l2->l_timeout_ch, CALLOUT_MPSAFE);
826 callout_setfunc(&l2->l_timeout_ch, sleepq_timeout, l2);
827 cv_init(&l2->l_sigcv, "sigwait");
828 cv_init(&l2->l_waitcv, "vfork");
829 l2->l_syncobj = &sched_syncobj;
830
831 if (rnewlwpp != NULL)
832 *rnewlwpp = l2;
833
834 /*
835 * PCU state needs to be saved before calling uvm_lwp_fork() so that
836 * the MD cpu_lwp_fork() can copy the saved state to the new LWP.
837 */
838 pcu_save_all(l1);
839
840 uvm_lwp_setuarea(l2, uaddr);
841 uvm_lwp_fork(l1, l2, stack, stacksize, func,
842 (arg != NULL) ? arg : l2);
843
844 if ((flags & LWP_PIDLID) != 0) {
845 lid = proc_alloc_pid(p2);
846 l2->l_pflag |= LP_PIDLID;
847 } else {
848 lid = 0;
849 }
850
851 mutex_enter(p2->p_lock);
852
853 if ((flags & LWP_DETACHED) != 0) {
854 l2->l_prflag = LPR_DETACHED;
855 p2->p_ndlwps++;
856 } else
857 l2->l_prflag = 0;
858
859 l2->l_sigstk = l1->l_sigstk;
860 l2->l_sigmask = l1->l_sigmask;
861 CIRCLEQ_INIT(&l2->l_sigpend.sp_info);
862 sigemptyset(&l2->l_sigpend.sp_set);
863
864 if (lid == 0) {
865 p2->p_nlwpid++;
866 if (p2->p_nlwpid == 0)
867 p2->p_nlwpid++;
868 lid = p2->p_nlwpid;
869 }
870 l2->l_lid = lid;
871 LIST_INSERT_HEAD(&p2->p_lwps, l2, l_sibling);
872 p2->p_nlwps++;
873 p2->p_nrlwps++;
874
875 KASSERT(l2->l_affinity == NULL);
876
877 if ((p2->p_flag & PK_SYSTEM) == 0) {
878 /* Inherit the affinity mask. */
879 if (l1->l_affinity) {
880 /*
881 * Note that we hold the state lock while inheriting
882 * the affinity to avoid race with sched_setaffinity().
883 */
884 lwp_lock(l1);
885 if (l1->l_affinity) {
886 kcpuset_use(l1->l_affinity);
887 l2->l_affinity = l1->l_affinity;
888 }
889 lwp_unlock(l1);
890 }
891 lwp_lock(l2);
892 /* Inherit a processor-set */
893 l2->l_psid = l1->l_psid;
894 /* Look for a CPU to start */
895 l2->l_cpu = sched_takecpu(l2);
896 lwp_unlock_to(l2, l2->l_cpu->ci_schedstate.spc_mutex);
897 }
898 mutex_exit(p2->p_lock);
899
900 SDT_PROBE(proc,,,lwp_create, l2, 0,0,0,0);
901
902 mutex_enter(proc_lock);
903 LIST_INSERT_HEAD(&alllwp, l2, l_list);
904 mutex_exit(proc_lock);
905
906 SYSCALL_TIME_LWP_INIT(l2);
907
908 if (p2->p_emul->e_lwp_fork)
909 (*p2->p_emul->e_lwp_fork)(l1, l2);
910
911 return (0);
912 }
913
914 /*
915 * Called by MD code when a new LWP begins execution. Must be called
916 * with the previous LWP locked (so at splsched), or if there is no
917 * previous LWP, at splsched.
918 */
919 void
920 lwp_startup(struct lwp *prev, struct lwp *new)
921 {
922
923 SDT_PROBE(proc,,,lwp_start, new, 0,0,0,0);
924
925 KASSERT(kpreempt_disabled());
926 if (prev != NULL) {
927 /*
928 * Normalize the count of the spin-mutexes, it was
929 * increased in mi_switch(). Unmark the state of
930 * context switch - it is finished for previous LWP.
931 */
932 curcpu()->ci_mtx_count++;
933 membar_exit();
934 prev->l_ctxswtch = 0;
935 }
936 KPREEMPT_DISABLE(new);
937 spl0();
938 if (__predict_true(new->l_proc->p_vmspace))
939 pmap_activate(new);
940
941 /* Note trip through cpu_switchto(). */
942 pserialize_switchpoint();
943
944 LOCKDEBUG_BARRIER(NULL, 0);
945 KPREEMPT_ENABLE(new);
946 if ((new->l_pflag & LP_MPSAFE) == 0) {
947 KERNEL_LOCK(1, new);
948 }
949 }
950
951 /*
952 * Exit an LWP.
953 */
954 void
955 lwp_exit(struct lwp *l)
956 {
957 struct proc *p = l->l_proc;
958 struct lwp *l2;
959 bool current;
960
961 current = (l == curlwp);
962
963 KASSERT(current || (l->l_stat == LSIDL && l->l_target_cpu == NULL));
964 KASSERT(p == curproc);
965
966 SDT_PROBE(proc,,,lwp_exit, l, 0,0,0,0);
967
968 /*
969 * Verify that we hold no locks other than the kernel lock.
970 */
971 LOCKDEBUG_BARRIER(&kernel_lock, 0);
972
973 /*
974 * If we are the last live LWP in a process, we need to exit the
975 * entire process. We do so with an exit status of zero, because
976 * it's a "controlled" exit, and because that's what Solaris does.
977 *
978 * We are not quite a zombie yet, but for accounting purposes we
979 * must increment the count of zombies here.
980 *
981 * Note: the last LWP's specificdata will be deleted here.
982 */
983 mutex_enter(p->p_lock);
984 if (p->p_nlwps - p->p_nzlwps == 1) {
985 KASSERT(current == true);
986 /* XXXSMP kernel_lock not held */
987 exit1(l, 0);
988 /* NOTREACHED */
989 }
990 p->p_nzlwps++;
991 mutex_exit(p->p_lock);
992
993 if (p->p_emul->e_lwp_exit)
994 (*p->p_emul->e_lwp_exit)(l);
995
996 /* Drop filedesc reference. */
997 fd_free();
998
999 /* Delete the specificdata while it's still safe to sleep. */
1000 lwp_finispecific(l);
1001
1002 /*
1003 * Release our cached credentials.
1004 */
1005 kauth_cred_free(l->l_cred);
1006 callout_destroy(&l->l_timeout_ch);
1007
1008 /*
1009 * Remove the LWP from the global list.
1010 * Free its LID from the PID namespace if needed.
1011 */
1012 mutex_enter(proc_lock);
1013 LIST_REMOVE(l, l_list);
1014 if ((l->l_pflag & LP_PIDLID) != 0 && l->l_lid != p->p_pid) {
1015 proc_free_pid(l->l_lid);
1016 }
1017 mutex_exit(proc_lock);
1018
1019 /*
1020 * Get rid of all references to the LWP that others (e.g. procfs)
1021 * may have, and mark the LWP as a zombie. If the LWP is detached,
1022 * mark it waiting for collection in the proc structure. Note that
1023 * before we can do that, we need to free any other dead, deatched
1024 * LWP waiting to meet its maker.
1025 */
1026 mutex_enter(p->p_lock);
1027 lwp_drainrefs(l);
1028
1029 if ((l->l_prflag & LPR_DETACHED) != 0) {
1030 while ((l2 = p->p_zomblwp) != NULL) {
1031 p->p_zomblwp = NULL;
1032 lwp_free(l2, false, false);/* releases proc mutex */
1033 mutex_enter(p->p_lock);
1034 l->l_refcnt++;
1035 lwp_drainrefs(l);
1036 }
1037 p->p_zomblwp = l;
1038 }
1039
1040 /*
1041 * If we find a pending signal for the process and we have been
1042 * asked to check for signals, then we lose: arrange to have
1043 * all other LWPs in the process check for signals.
1044 */
1045 if ((l->l_flag & LW_PENDSIG) != 0 &&
1046 firstsig(&p->p_sigpend.sp_set) != 0) {
1047 LIST_FOREACH(l2, &p->p_lwps, l_sibling) {
1048 lwp_lock(l2);
1049 l2->l_flag |= LW_PENDSIG;
1050 lwp_unlock(l2);
1051 }
1052 }
1053
1054 /*
1055 * Release any PCU resources before becoming a zombie.
1056 */
1057 pcu_discard_all(l);
1058
1059 lwp_lock(l);
1060 l->l_stat = LSZOMB;
1061 if (l->l_name != NULL) {
1062 strcpy(l->l_name, "(zombie)");
1063 }
1064 lwp_unlock(l);
1065 p->p_nrlwps--;
1066 cv_broadcast(&p->p_lwpcv);
1067 if (l->l_lwpctl != NULL)
1068 l->l_lwpctl->lc_curcpu = LWPCTL_CPU_EXITED;
1069 mutex_exit(p->p_lock);
1070
1071 /*
1072 * We can no longer block. At this point, lwp_free() may already
1073 * be gunning for us. On a multi-CPU system, we may be off p_lwps.
1074 *
1075 * Free MD LWP resources.
1076 */
1077 cpu_lwp_free(l, 0);
1078
1079 if (current) {
1080 pmap_deactivate(l);
1081
1082 /*
1083 * Release the kernel lock, and switch away into
1084 * oblivion.
1085 */
1086 #ifdef notyet
1087 /* XXXSMP hold in lwp_userret() */
1088 KERNEL_UNLOCK_LAST(l);
1089 #else
1090 KERNEL_UNLOCK_ALL(l, NULL);
1091 #endif
1092 lwp_exit_switchaway(l);
1093 }
1094 }
1095
1096 /*
1097 * Free a dead LWP's remaining resources.
1098 *
1099 * XXXLWP limits.
1100 */
1101 void
1102 lwp_free(struct lwp *l, bool recycle, bool last)
1103 {
1104 struct proc *p = l->l_proc;
1105 struct rusage *ru;
1106 ksiginfoq_t kq;
1107
1108 KASSERT(l != curlwp);
1109 KASSERT(last || mutex_owned(p->p_lock));
1110
1111 if (p != &proc0 && p->p_nlwps != 1)
1112 (void)chglwpcnt(kauth_cred_getuid(l->l_cred), -1);
1113 /*
1114 * If this was not the last LWP in the process, then adjust
1115 * counters and unlock.
1116 */
1117 if (!last) {
1118 /*
1119 * Add the LWP's run time to the process' base value.
1120 * This needs to co-incide with coming off p_lwps.
1121 */
1122 bintime_add(&p->p_rtime, &l->l_rtime);
1123 p->p_pctcpu += l->l_pctcpu;
1124 ru = &p->p_stats->p_ru;
1125 ruadd(ru, &l->l_ru);
1126 ru->ru_nvcsw += (l->l_ncsw - l->l_nivcsw);
1127 ru->ru_nivcsw += l->l_nivcsw;
1128 LIST_REMOVE(l, l_sibling);
1129 p->p_nlwps--;
1130 p->p_nzlwps--;
1131 if ((l->l_prflag & LPR_DETACHED) != 0)
1132 p->p_ndlwps--;
1133
1134 /*
1135 * Have any LWPs sleeping in lwp_wait() recheck for
1136 * deadlock.
1137 */
1138 cv_broadcast(&p->p_lwpcv);
1139 mutex_exit(p->p_lock);
1140 }
1141
1142 #ifdef MULTIPROCESSOR
1143 /*
1144 * In the unlikely event that the LWP is still on the CPU,
1145 * then spin until it has switched away. We need to release
1146 * all locks to avoid deadlock against interrupt handlers on
1147 * the target CPU.
1148 */
1149 if ((l->l_pflag & LP_RUNNING) != 0 || l->l_cpu->ci_curlwp == l) {
1150 int count;
1151 (void)count; /* XXXgcc */
1152 KERNEL_UNLOCK_ALL(curlwp, &count);
1153 while ((l->l_pflag & LP_RUNNING) != 0 ||
1154 l->l_cpu->ci_curlwp == l)
1155 SPINLOCK_BACKOFF_HOOK;
1156 KERNEL_LOCK(count, curlwp);
1157 }
1158 #endif
1159
1160 /*
1161 * Destroy the LWP's remaining signal information.
1162 */
1163 ksiginfo_queue_init(&kq);
1164 sigclear(&l->l_sigpend, NULL, &kq);
1165 ksiginfo_queue_drain(&kq);
1166 cv_destroy(&l->l_sigcv);
1167 cv_destroy(&l->l_waitcv);
1168
1169 /*
1170 * Free lwpctl structure and affinity.
1171 */
1172 if (l->l_lwpctl) {
1173 lwp_ctl_free(l);
1174 }
1175 if (l->l_affinity) {
1176 kcpuset_unuse(l->l_affinity, NULL);
1177 l->l_affinity = NULL;
1178 }
1179
1180 /*
1181 * Free the LWP's turnstile and the LWP structure itself unless the
1182 * caller wants to recycle them. Also, free the scheduler specific
1183 * data.
1184 *
1185 * We can't return turnstile0 to the pool (it didn't come from it),
1186 * so if it comes up just drop it quietly and move on.
1187 *
1188 * We don't recycle the VM resources at this time.
1189 */
1190
1191 if (!recycle && l->l_ts != &turnstile0)
1192 pool_cache_put(turnstile_cache, l->l_ts);
1193 if (l->l_name != NULL)
1194 kmem_free(l->l_name, MAXCOMLEN);
1195
1196 cpu_lwp_free2(l);
1197 uvm_lwp_exit(l);
1198
1199 KASSERT(SLIST_EMPTY(&l->l_pi_lenders));
1200 KASSERT(l->l_inheritedprio == -1);
1201 KASSERT(l->l_blcnt == 0);
1202 kdtrace_thread_dtor(NULL, l);
1203 if (!recycle)
1204 pool_cache_put(lwp_cache, l);
1205 }
1206
1207 /*
1208 * Migrate the LWP to the another CPU. Unlocks the LWP.
1209 */
1210 void
1211 lwp_migrate(lwp_t *l, struct cpu_info *tci)
1212 {
1213 struct schedstate_percpu *tspc;
1214 int lstat = l->l_stat;
1215
1216 KASSERT(lwp_locked(l, NULL));
1217 KASSERT(tci != NULL);
1218
1219 /* If LWP is still on the CPU, it must be handled like LSONPROC */
1220 if ((l->l_pflag & LP_RUNNING) != 0) {
1221 lstat = LSONPROC;
1222 }
1223
1224 /*
1225 * The destination CPU could be changed while previous migration
1226 * was not finished.
1227 */
1228 if (l->l_target_cpu != NULL) {
1229 l->l_target_cpu = tci;
1230 lwp_unlock(l);
1231 return;
1232 }
1233
1234 /* Nothing to do if trying to migrate to the same CPU */
1235 if (l->l_cpu == tci) {
1236 lwp_unlock(l);
1237 return;
1238 }
1239
1240 KASSERT(l->l_target_cpu == NULL);
1241 tspc = &tci->ci_schedstate;
1242 switch (lstat) {
1243 case LSRUN:
1244 l->l_target_cpu = tci;
1245 break;
1246 case LSIDL:
1247 l->l_cpu = tci;
1248 lwp_unlock_to(l, tspc->spc_mutex);
1249 return;
1250 case LSSLEEP:
1251 l->l_cpu = tci;
1252 break;
1253 case LSSTOP:
1254 case LSSUSPENDED:
1255 l->l_cpu = tci;
1256 if (l->l_wchan == NULL) {
1257 lwp_unlock_to(l, tspc->spc_lwplock);
1258 return;
1259 }
1260 break;
1261 case LSONPROC:
1262 l->l_target_cpu = tci;
1263 spc_lock(l->l_cpu);
1264 cpu_need_resched(l->l_cpu, RESCHED_KPREEMPT);
1265 spc_unlock(l->l_cpu);
1266 break;
1267 }
1268 lwp_unlock(l);
1269 }
1270
1271 /*
1272 * Find the LWP in the process. Arguments may be zero, in such case,
1273 * the calling process and first LWP in the list will be used.
1274 * On success - returns proc locked.
1275 */
1276 struct lwp *
1277 lwp_find2(pid_t pid, lwpid_t lid)
1278 {
1279 proc_t *p;
1280 lwp_t *l;
1281
1282 /* Find the process. */
1283 if (pid != 0) {
1284 mutex_enter(proc_lock);
1285 p = proc_find(pid);
1286 if (p == NULL) {
1287 mutex_exit(proc_lock);
1288 return NULL;
1289 }
1290 mutex_enter(p->p_lock);
1291 mutex_exit(proc_lock);
1292 } else {
1293 p = curlwp->l_proc;
1294 mutex_enter(p->p_lock);
1295 }
1296 /* Find the thread. */
1297 if (lid != 0) {
1298 l = lwp_find(p, lid);
1299 } else {
1300 l = LIST_FIRST(&p->p_lwps);
1301 }
1302 if (l == NULL) {
1303 mutex_exit(p->p_lock);
1304 }
1305 return l;
1306 }
1307
1308 /*
1309 * Look up a live LWP within the specified process.
1310 *
1311 * Must be called with p->p_lock held.
1312 */
1313 struct lwp *
1314 lwp_find(struct proc *p, lwpid_t id)
1315 {
1316 struct lwp *l;
1317
1318 KASSERT(mutex_owned(p->p_lock));
1319
1320 LIST_FOREACH(l, &p->p_lwps, l_sibling) {
1321 if (l->l_lid == id)
1322 break;
1323 }
1324
1325 /*
1326 * No need to lock - all of these conditions will
1327 * be visible with the process level mutex held.
1328 */
1329 if (l != NULL && (l->l_stat == LSIDL || l->l_stat == LSZOMB))
1330 l = NULL;
1331
1332 return l;
1333 }
1334
1335 /*
1336 * Update an LWP's cached credentials to mirror the process' master copy.
1337 *
1338 * This happens early in the syscall path, on user trap, and on LWP
1339 * creation. A long-running LWP can also voluntarily choose to update
1340 * it's credentials by calling this routine. This may be called from
1341 * LWP_CACHE_CREDS(), which checks l->l_cred != p->p_cred beforehand.
1342 */
1343 void
1344 lwp_update_creds(struct lwp *l)
1345 {
1346 kauth_cred_t oc;
1347 struct proc *p;
1348
1349 p = l->l_proc;
1350 oc = l->l_cred;
1351
1352 mutex_enter(p->p_lock);
1353 kauth_cred_hold(p->p_cred);
1354 l->l_cred = p->p_cred;
1355 l->l_prflag &= ~LPR_CRMOD;
1356 mutex_exit(p->p_lock);
1357 if (oc != NULL)
1358 kauth_cred_free(oc);
1359 }
1360
1361 /*
1362 * Verify that an LWP is locked, and optionally verify that the lock matches
1363 * one we specify.
1364 */
1365 int
1366 lwp_locked(struct lwp *l, kmutex_t *mtx)
1367 {
1368 kmutex_t *cur = l->l_mutex;
1369
1370 return mutex_owned(cur) && (mtx == cur || mtx == NULL);
1371 }
1372
1373 /*
1374 * Lend a new mutex to an LWP. The old mutex must be held.
1375 */
1376 void
1377 lwp_setlock(struct lwp *l, kmutex_t *new)
1378 {
1379
1380 KASSERT(mutex_owned(l->l_mutex));
1381
1382 membar_exit();
1383 l->l_mutex = new;
1384 }
1385
1386 /*
1387 * Lend a new mutex to an LWP, and release the old mutex. The old mutex
1388 * must be held.
1389 */
1390 void
1391 lwp_unlock_to(struct lwp *l, kmutex_t *new)
1392 {
1393 kmutex_t *old;
1394
1395 KASSERT(lwp_locked(l, NULL));
1396
1397 old = l->l_mutex;
1398 membar_exit();
1399 l->l_mutex = new;
1400 mutex_spin_exit(old);
1401 }
1402
1403 int
1404 lwp_trylock(struct lwp *l)
1405 {
1406 kmutex_t *old;
1407
1408 for (;;) {
1409 if (!mutex_tryenter(old = l->l_mutex))
1410 return 0;
1411 if (__predict_true(l->l_mutex == old))
1412 return 1;
1413 mutex_spin_exit(old);
1414 }
1415 }
1416
1417 void
1418 lwp_unsleep(lwp_t *l, bool cleanup)
1419 {
1420
1421 KASSERT(mutex_owned(l->l_mutex));
1422 (*l->l_syncobj->sobj_unsleep)(l, cleanup);
1423 }
1424
1425 /*
1426 * Handle exceptions for mi_userret(). Called if a member of LW_USERRET is
1427 * set.
1428 */
1429 void
1430 lwp_userret(struct lwp *l)
1431 {
1432 struct proc *p;
1433 int sig;
1434
1435 KASSERT(l == curlwp);
1436 KASSERT(l->l_stat == LSONPROC);
1437 p = l->l_proc;
1438
1439 #ifndef __HAVE_FAST_SOFTINTS
1440 /* Run pending soft interrupts. */
1441 if (l->l_cpu->ci_data.cpu_softints != 0)
1442 softint_overlay();
1443 #endif
1444
1445 /*
1446 * It is safe to do this read unlocked on a MP system..
1447 */
1448 while ((l->l_flag & LW_USERRET) != 0) {
1449 /*
1450 * Process pending signals first, unless the process
1451 * is dumping core or exiting, where we will instead
1452 * enter the LW_WSUSPEND case below.
1453 */
1454 if ((l->l_flag & (LW_PENDSIG | LW_WCORE | LW_WEXIT)) ==
1455 LW_PENDSIG) {
1456 mutex_enter(p->p_lock);
1457 while ((sig = issignal(l)) != 0)
1458 postsig(sig);
1459 mutex_exit(p->p_lock);
1460 }
1461
1462 /*
1463 * Core-dump or suspend pending.
1464 *
1465 * In case of core dump, suspend ourselves, so that the kernel
1466 * stack and therefore the userland registers saved in the
1467 * trapframe are around for coredump() to write them out.
1468 * We also need to save any PCU resources that we have so that
1469 * they accessible for coredump(). We issue a wakeup on
1470 * p->p_lwpcv so that sigexit() will write the core file out
1471 * once all other LWPs are suspended.
1472 */
1473 if ((l->l_flag & LW_WSUSPEND) != 0) {
1474 pcu_save_all(l);
1475 mutex_enter(p->p_lock);
1476 p->p_nrlwps--;
1477 cv_broadcast(&p->p_lwpcv);
1478 lwp_lock(l);
1479 l->l_stat = LSSUSPENDED;
1480 lwp_unlock(l);
1481 mutex_exit(p->p_lock);
1482 lwp_lock(l);
1483 mi_switch(l);
1484 }
1485
1486 /* Process is exiting. */
1487 if ((l->l_flag & LW_WEXIT) != 0) {
1488 lwp_exit(l);
1489 KASSERT(0);
1490 /* NOTREACHED */
1491 }
1492
1493 /* update lwpctl processor (for vfork child_return) */
1494 if (l->l_flag & LW_LWPCTL) {
1495 lwp_lock(l);
1496 KASSERT(kpreempt_disabled());
1497 l->l_lwpctl->lc_curcpu = (int)cpu_index(l->l_cpu);
1498 l->l_lwpctl->lc_pctr++;
1499 l->l_flag &= ~LW_LWPCTL;
1500 lwp_unlock(l);
1501 }
1502 }
1503 }
1504
1505 /*
1506 * Force an LWP to enter the kernel, to take a trip through lwp_userret().
1507 */
1508 void
1509 lwp_need_userret(struct lwp *l)
1510 {
1511 KASSERT(lwp_locked(l, NULL));
1512
1513 /*
1514 * Since the tests in lwp_userret() are done unlocked, make sure
1515 * that the condition will be seen before forcing the LWP to enter
1516 * kernel mode.
1517 */
1518 membar_producer();
1519 cpu_signotify(l);
1520 }
1521
1522 /*
1523 * Add one reference to an LWP. This will prevent the LWP from
1524 * exiting, thus keep the lwp structure and PCB around to inspect.
1525 */
1526 void
1527 lwp_addref(struct lwp *l)
1528 {
1529
1530 KASSERT(mutex_owned(l->l_proc->p_lock));
1531 KASSERT(l->l_stat != LSZOMB);
1532 KASSERT(l->l_refcnt != 0);
1533
1534 l->l_refcnt++;
1535 }
1536
1537 /*
1538 * Remove one reference to an LWP. If this is the last reference,
1539 * then we must finalize the LWP's death.
1540 */
1541 void
1542 lwp_delref(struct lwp *l)
1543 {
1544 struct proc *p = l->l_proc;
1545
1546 mutex_enter(p->p_lock);
1547 lwp_delref2(l);
1548 mutex_exit(p->p_lock);
1549 }
1550
1551 /*
1552 * Remove one reference to an LWP. If this is the last reference,
1553 * then we must finalize the LWP's death. The proc mutex is held
1554 * on entry.
1555 */
1556 void
1557 lwp_delref2(struct lwp *l)
1558 {
1559 struct proc *p = l->l_proc;
1560
1561 KASSERT(mutex_owned(p->p_lock));
1562 KASSERT(l->l_stat != LSZOMB);
1563 KASSERT(l->l_refcnt > 0);
1564 if (--l->l_refcnt == 0)
1565 cv_broadcast(&p->p_lwpcv);
1566 }
1567
1568 /*
1569 * Drain all references to the current LWP.
1570 */
1571 void
1572 lwp_drainrefs(struct lwp *l)
1573 {
1574 struct proc *p = l->l_proc;
1575
1576 KASSERT(mutex_owned(p->p_lock));
1577 KASSERT(l->l_refcnt != 0);
1578
1579 l->l_refcnt--;
1580 while (l->l_refcnt != 0)
1581 cv_wait(&p->p_lwpcv, p->p_lock);
1582 }
1583
1584 /*
1585 * Return true if the specified LWP is 'alive'. Only p->p_lock need
1586 * be held.
1587 */
1588 bool
1589 lwp_alive(lwp_t *l)
1590 {
1591
1592 KASSERT(mutex_owned(l->l_proc->p_lock));
1593
1594 switch (l->l_stat) {
1595 case LSSLEEP:
1596 case LSRUN:
1597 case LSONPROC:
1598 case LSSTOP:
1599 case LSSUSPENDED:
1600 return true;
1601 default:
1602 return false;
1603 }
1604 }
1605
1606 /*
1607 * Return first live LWP in the process.
1608 */
1609 lwp_t *
1610 lwp_find_first(proc_t *p)
1611 {
1612 lwp_t *l;
1613
1614 KASSERT(mutex_owned(p->p_lock));
1615
1616 LIST_FOREACH(l, &p->p_lwps, l_sibling) {
1617 if (lwp_alive(l)) {
1618 return l;
1619 }
1620 }
1621
1622 return NULL;
1623 }
1624
1625 /*
1626 * Allocate a new lwpctl structure for a user LWP.
1627 */
1628 int
1629 lwp_ctl_alloc(vaddr_t *uaddr)
1630 {
1631 lcproc_t *lp;
1632 u_int bit, i, offset;
1633 struct uvm_object *uao;
1634 int error;
1635 lcpage_t *lcp;
1636 proc_t *p;
1637 lwp_t *l;
1638
1639 l = curlwp;
1640 p = l->l_proc;
1641
1642 /* don't allow a vforked process to create lwp ctls */
1643 if (p->p_lflag & PL_PPWAIT)
1644 return EBUSY;
1645
1646 if (l->l_lcpage != NULL) {
1647 lcp = l->l_lcpage;
1648 *uaddr = lcp->lcp_uaddr + (vaddr_t)l->l_lwpctl - lcp->lcp_kaddr;
1649 return 0;
1650 }
1651
1652 /* First time around, allocate header structure for the process. */
1653 if ((lp = p->p_lwpctl) == NULL) {
1654 lp = kmem_alloc(sizeof(*lp), KM_SLEEP);
1655 mutex_init(&lp->lp_lock, MUTEX_DEFAULT, IPL_NONE);
1656 lp->lp_uao = NULL;
1657 TAILQ_INIT(&lp->lp_pages);
1658 mutex_enter(p->p_lock);
1659 if (p->p_lwpctl == NULL) {
1660 p->p_lwpctl = lp;
1661 mutex_exit(p->p_lock);
1662 } else {
1663 mutex_exit(p->p_lock);
1664 mutex_destroy(&lp->lp_lock);
1665 kmem_free(lp, sizeof(*lp));
1666 lp = p->p_lwpctl;
1667 }
1668 }
1669
1670 /*
1671 * Set up an anonymous memory region to hold the shared pages.
1672 * Map them into the process' address space. The user vmspace
1673 * gets the first reference on the UAO.
1674 */
1675 mutex_enter(&lp->lp_lock);
1676 if (lp->lp_uao == NULL) {
1677 lp->lp_uao = uao_create(LWPCTL_UAREA_SZ, 0);
1678 lp->lp_cur = 0;
1679 lp->lp_max = LWPCTL_UAREA_SZ;
1680 lp->lp_uva = p->p_emul->e_vm_default_addr(p,
1681 (vaddr_t)p->p_vmspace->vm_daddr, LWPCTL_UAREA_SZ);
1682 error = uvm_map(&p->p_vmspace->vm_map, &lp->lp_uva,
1683 LWPCTL_UAREA_SZ, lp->lp_uao, 0, 0, UVM_MAPFLAG(UVM_PROT_RW,
1684 UVM_PROT_RW, UVM_INH_NONE, UVM_ADV_NORMAL, 0));
1685 if (error != 0) {
1686 uao_detach(lp->lp_uao);
1687 lp->lp_uao = NULL;
1688 mutex_exit(&lp->lp_lock);
1689 return error;
1690 }
1691 }
1692
1693 /* Get a free block and allocate for this LWP. */
1694 TAILQ_FOREACH(lcp, &lp->lp_pages, lcp_chain) {
1695 if (lcp->lcp_nfree != 0)
1696 break;
1697 }
1698 if (lcp == NULL) {
1699 /* Nothing available - try to set up a free page. */
1700 if (lp->lp_cur == lp->lp_max) {
1701 mutex_exit(&lp->lp_lock);
1702 return ENOMEM;
1703 }
1704 lcp = kmem_alloc(LWPCTL_LCPAGE_SZ, KM_SLEEP);
1705 if (lcp == NULL) {
1706 mutex_exit(&lp->lp_lock);
1707 return ENOMEM;
1708 }
1709 /*
1710 * Wire the next page down in kernel space. Since this
1711 * is a new mapping, we must add a reference.
1712 */
1713 uao = lp->lp_uao;
1714 (*uao->pgops->pgo_reference)(uao);
1715 lcp->lcp_kaddr = vm_map_min(kernel_map);
1716 error = uvm_map(kernel_map, &lcp->lcp_kaddr, PAGE_SIZE,
1717 uao, lp->lp_cur, PAGE_SIZE,
1718 UVM_MAPFLAG(UVM_PROT_RW, UVM_PROT_RW,
1719 UVM_INH_NONE, UVM_ADV_RANDOM, 0));
1720 if (error != 0) {
1721 mutex_exit(&lp->lp_lock);
1722 kmem_free(lcp, LWPCTL_LCPAGE_SZ);
1723 (*uao->pgops->pgo_detach)(uao);
1724 return error;
1725 }
1726 error = uvm_map_pageable(kernel_map, lcp->lcp_kaddr,
1727 lcp->lcp_kaddr + PAGE_SIZE, FALSE, 0);
1728 if (error != 0) {
1729 mutex_exit(&lp->lp_lock);
1730 uvm_unmap(kernel_map, lcp->lcp_kaddr,
1731 lcp->lcp_kaddr + PAGE_SIZE);
1732 kmem_free(lcp, LWPCTL_LCPAGE_SZ);
1733 return error;
1734 }
1735 /* Prepare the page descriptor and link into the list. */
1736 lcp->lcp_uaddr = lp->lp_uva + lp->lp_cur;
1737 lp->lp_cur += PAGE_SIZE;
1738 lcp->lcp_nfree = LWPCTL_PER_PAGE;
1739 lcp->lcp_rotor = 0;
1740 memset(lcp->lcp_bitmap, 0xff, LWPCTL_BITMAP_SZ);
1741 TAILQ_INSERT_HEAD(&lp->lp_pages, lcp, lcp_chain);
1742 }
1743 for (i = lcp->lcp_rotor; lcp->lcp_bitmap[i] == 0;) {
1744 if (++i >= LWPCTL_BITMAP_ENTRIES)
1745 i = 0;
1746 }
1747 bit = ffs(lcp->lcp_bitmap[i]) - 1;
1748 lcp->lcp_bitmap[i] ^= (1 << bit);
1749 lcp->lcp_rotor = i;
1750 lcp->lcp_nfree--;
1751 l->l_lcpage = lcp;
1752 offset = (i << 5) + bit;
1753 l->l_lwpctl = (lwpctl_t *)lcp->lcp_kaddr + offset;
1754 *uaddr = lcp->lcp_uaddr + offset * sizeof(lwpctl_t);
1755 mutex_exit(&lp->lp_lock);
1756
1757 KPREEMPT_DISABLE(l);
1758 l->l_lwpctl->lc_curcpu = (int)curcpu()->ci_data.cpu_index;
1759 KPREEMPT_ENABLE(l);
1760
1761 return 0;
1762 }
1763
1764 /*
1765 * Free an lwpctl structure back to the per-process list.
1766 */
1767 void
1768 lwp_ctl_free(lwp_t *l)
1769 {
1770 struct proc *p = l->l_proc;
1771 lcproc_t *lp;
1772 lcpage_t *lcp;
1773 u_int map, offset;
1774
1775 /* don't free a lwp context we borrowed for vfork */
1776 if (p->p_lflag & PL_PPWAIT) {
1777 l->l_lwpctl = NULL;
1778 return;
1779 }
1780
1781 lp = p->p_lwpctl;
1782 KASSERT(lp != NULL);
1783
1784 lcp = l->l_lcpage;
1785 offset = (u_int)((lwpctl_t *)l->l_lwpctl - (lwpctl_t *)lcp->lcp_kaddr);
1786 KASSERT(offset < LWPCTL_PER_PAGE);
1787
1788 mutex_enter(&lp->lp_lock);
1789 lcp->lcp_nfree++;
1790 map = offset >> 5;
1791 lcp->lcp_bitmap[map] |= (1 << (offset & 31));
1792 if (lcp->lcp_bitmap[lcp->lcp_rotor] == 0)
1793 lcp->lcp_rotor = map;
1794 if (TAILQ_FIRST(&lp->lp_pages)->lcp_nfree == 0) {
1795 TAILQ_REMOVE(&lp->lp_pages, lcp, lcp_chain);
1796 TAILQ_INSERT_HEAD(&lp->lp_pages, lcp, lcp_chain);
1797 }
1798 mutex_exit(&lp->lp_lock);
1799 }
1800
1801 /*
1802 * Process is exiting; tear down lwpctl state. This can only be safely
1803 * called by the last LWP in the process.
1804 */
1805 void
1806 lwp_ctl_exit(void)
1807 {
1808 lcpage_t *lcp, *next;
1809 lcproc_t *lp;
1810 proc_t *p;
1811 lwp_t *l;
1812
1813 l = curlwp;
1814 l->l_lwpctl = NULL;
1815 l->l_lcpage = NULL;
1816 p = l->l_proc;
1817 lp = p->p_lwpctl;
1818
1819 KASSERT(lp != NULL);
1820 KASSERT(p->p_nlwps == 1);
1821
1822 for (lcp = TAILQ_FIRST(&lp->lp_pages); lcp != NULL; lcp = next) {
1823 next = TAILQ_NEXT(lcp, lcp_chain);
1824 uvm_unmap(kernel_map, lcp->lcp_kaddr,
1825 lcp->lcp_kaddr + PAGE_SIZE);
1826 kmem_free(lcp, LWPCTL_LCPAGE_SZ);
1827 }
1828
1829 if (lp->lp_uao != NULL) {
1830 uvm_unmap(&p->p_vmspace->vm_map, lp->lp_uva,
1831 lp->lp_uva + LWPCTL_UAREA_SZ);
1832 }
1833
1834 mutex_destroy(&lp->lp_lock);
1835 kmem_free(lp, sizeof(*lp));
1836 p->p_lwpctl = NULL;
1837 }
1838
1839 /*
1840 * Return the current LWP's "preemption counter". Used to detect
1841 * preemption across operations that can tolerate preemption without
1842 * crashing, but which may generate incorrect results if preempted.
1843 */
1844 uint64_t
1845 lwp_pctr(void)
1846 {
1847
1848 return curlwp->l_ncsw;
1849 }
1850
1851 /*
1852 * Set an LWP's private data pointer.
1853 */
1854 int
1855 lwp_setprivate(struct lwp *l, void *ptr)
1856 {
1857 int error = 0;
1858
1859 l->l_private = ptr;
1860 #ifdef __HAVE_CPU_LWP_SETPRIVATE
1861 error = cpu_lwp_setprivate(l, ptr);
1862 #endif
1863 return error;
1864 }
1865
1866 #if defined(DDB)
1867 #include <machine/pcb.h>
1868
1869 void
1870 lwp_whatis(uintptr_t addr, void (*pr)(const char *, ...))
1871 {
1872 lwp_t *l;
1873
1874 LIST_FOREACH(l, &alllwp, l_list) {
1875 uintptr_t stack = (uintptr_t)KSTACK_LOWEST_ADDR(l);
1876
1877 if (addr < stack || stack + KSTACK_SIZE <= addr) {
1878 continue;
1879 }
1880 (*pr)("%p is %p+%zu, LWP %p's stack\n",
1881 (void *)addr, (void *)stack,
1882 (size_t)(addr - stack), l);
1883 }
1884 }
1885 #endif /* defined(DDB) */
1886