kern_lwp.c revision 1.55.2.11 1 /* $NetBSD: kern_lwp.c,v 1.55.2.11 2007/04/16 23:31:20 ad Exp $ */
2
3 /*-
4 * Copyright (c) 2001, 2006, 2007 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Nathan J. Williams, and Andrew Doran.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 * 3. All advertising materials mentioning features or use of this software
19 * must display the following acknowledgement:
20 * This product includes software developed by the NetBSD
21 * Foundation, Inc. and its contributors.
22 * 4. Neither the name of The NetBSD Foundation nor the names of its
23 * contributors may be used to endorse or promote products derived
24 * from this software without specific prior written permission.
25 *
26 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
27 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
28 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
29 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
30 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
31 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
32 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
33 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
34 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
35 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
36 * POSSIBILITY OF SUCH DAMAGE.
37 */
38
39 /*
40 * Overview
41 *
42 * Lightweight processes (LWPs) are the basic unit (or thread) of
43 * execution within the kernel. The core state of an LWP is described
44 * by "struct lwp".
45 *
46 * Each LWP is contained within a process (described by "struct proc"),
47 * Every process contains at least one LWP, but may contain more. The
48 * process describes attributes shared among all of its LWPs such as a
49 * private address space, global execution state (stopped, active,
50 * zombie, ...), signal disposition and so on. On a multiprocessor
51 * machine, multiple LWPs be executing in kernel simultaneously.
52 *
53 * Note that LWPs differ from kernel threads (kthreads) in that kernel
54 * threads are distinct processes (system processes) with no user space
55 * component, which themselves may contain one or more LWPs.
56 *
57 * Execution states
58 *
59 * At any given time, an LWP has overall state that is described by
60 * lwp::l_stat. The states are broken into two sets below. The first
61 * set is guaranteed to represent the absolute, current state of the
62 * LWP:
63 *
64 * LSONPROC
65 *
66 * On processor: the LWP is executing on a CPU, either in the
67 * kernel or in user space.
68 *
69 * LSRUN
70 *
71 * Runnable: the LWP is parked on a run queue, and may soon be
72 * chosen to run by a idle processor, or by a processor that
73 * has been asked to preempt a currently runnning but lower
74 * priority LWP. If the LWP is not swapped in (L_INMEM == 0)
75 * then the LWP is not on a run queue, but may be soon.
76 *
77 * LSIDL
78 *
79 * Idle: the LWP has been created but has not yet executed.
80 * Whoever created the new LWP can be expected to set it to
81 * another state shortly.
82 *
83 * LSSUSPENDED:
84 *
85 * Suspended: the LWP has had its execution suspended by
86 * another LWP in the same process using the _lwp_suspend()
87 * system call. User-level LWPs also enter the suspended
88 * state when the system is shutting down.
89 *
90 * The second set represent a "statement of intent" on behalf of the
91 * LWP. The LWP may in fact be executing on a processor, may be
92 * sleeping, idle, or on a run queue. It is expected to take the
93 * necessary action to stop executing or become "running" again within
94 * a short timeframe.
95 *
96 * LSZOMB:
97 *
98 * Dead: the LWP has released most of its resources and is
99 * about to switch away into oblivion. When it switches away,
100 * its few remaining resources will be collected.
101 *
102 * LSSLEEP:
103 *
104 * Sleeping: the LWP has entered itself onto a sleep queue, and
105 * will switch away shortly to allow other LWPs to run on the
106 * CPU.
107 *
108 * LSSTOP:
109 *
110 * Stopped: the LWP has been stopped as a result of a job
111 * control signal, or as a result of the ptrace() interface.
112 * Stopped LWPs may run briefly within the kernel to handle
113 * signals that they receive, but will not return to user space
114 * until their process' state is changed away from stopped.
115 * Single LWPs within a process can not be set stopped
116 * selectively: all actions that can stop or continue LWPs
117 * occur at the process level.
118 *
119 * State transitions
120 *
121 * Note that the LSSTOP and LSSUSPENDED states may only be set
122 * when returning to user space in userret(), or when sleeping
123 * interruptably. Before setting those states, we try to ensure
124 * that the LWPs will release all kernel locks that they hold,
125 * and at a minimum try to ensure that the LWP can be set runnable
126 * again by a signal.
127 *
128 * LWPs may transition states in the following ways:
129 *
130 * RUN -------> ONPROC ONPROC -----> RUN
131 * > STOPPED > SLEEP
132 * > SUSPENDED > STOPPED
133 * > SUSPENDED
134 * > ZOMB
135 *
136 * STOPPED ---> RUN SUSPENDED --> RUN
137 * > SLEEP > SLEEP
138 *
139 * SLEEP -----> ONPROC IDL --------> RUN
140 * > RUN > SUSPENDED
141 * > STOPPED > STOPPED
142 * > SUSPENDED
143 *
144 * Locking
145 *
146 * The majority of fields in 'struct lwp' are covered by a single,
147 * general spin mutex pointed to by lwp::l_mutex. The locks covering
148 * each field are documented in sys/lwp.h.
149 *
150 * State transitions must be made with the LWP's general lock held. In
151 * a multiprocessor kernel, state transitions may cause the LWP's lock
152 * pointer to change. On uniprocessor kernels, most scheduler and
153 * synchronisation objects such as sleep queues and LWPs are protected
154 * by only one mutex (spc_mutex on single CPU). In this case, LWPs' lock
155 * pointers will never change and will always reference spc_mutex.
156 * Please note that in a multiprocessor kernel each CPU has own spc_mutex.
157 *
158 * Where spc_mutex is noted, it refers to l->l_cpu->ci_schedstate.spc_mutex
159
160 *
161 * Manipulation of the general lock is not performed directly, but
162 * through calls to lwp_lock(), lwp_relock() and similar.
163 *
164 * States and their associated locks:
165 *
166 * LSIDL, LSZOMB
167 *
168 * Always covered by spc_mutex.
169 *
170 * LSONPROC, LSRUN:
171 *
172 * Always covered by spc_mutex, which protects the run queues
173 * and other miscellaneous items. If the scheduler is changed
174 * to use per-CPU run queues, this may become a per-CPU mutex.
175 *
176 * LSSLEEP:
177 *
178 * Covered by a mutex associated with the sleep queue that the
179 * LWP resides on, indirectly referenced by l_sleepq->sq_mutex.
180 *
181 * LSSTOP, LSSUSPENDED:
182 *
183 * If the LWP was previously sleeping (l_wchan != NULL), then
184 * l_mutex references the sleep queue mutex. If the LWP was
185 * runnable or on the CPU when halted, or has been removed from
186 * the sleep queue since halted, then the mutex is spc_mutex.
187 *
188 * The lock order is as follows:
189 *
190 * sleepq_t::sq_mutex |---> spc_mutex
191 * tschain_t::tc_mutex |
192 *
193 * Each process has an scheduler state mutex (proc::p_smutex), and a
194 * number of counters on LWPs and their states: p_nzlwps, p_nrlwps, and
195 * so on. When an LWP is to be entered into or removed from one of the
196 * following states, p_mutex must be held and the process wide counters
197 * adjusted:
198 *
199 * LSIDL, LSZOMB, LSSTOP, LSSUSPENDED
200 *
201 * Note that an LWP is considered running or likely to run soon if in
202 * one of the following states. This affects the value of p_nrlwps:
203 *
204 * LSRUN, LSONPROC, LSSLEEP
205 *
206 * p_smutex does not need to be held when transitioning among these
207 * three states.
208 */
209
210 #include <sys/cdefs.h>
211 __KERNEL_RCSID(0, "$NetBSD: kern_lwp.c,v 1.55.2.11 2007/04/16 23:31:20 ad Exp $");
212
213 #include "opt_multiprocessor.h"
214 #include "opt_lockdebug.h"
215
216 #define _LWP_API_PRIVATE
217
218 #include <sys/param.h>
219 #include <sys/systm.h>
220 #include <sys/cpu.h>
221 #include <sys/pool.h>
222 #include <sys/proc.h>
223 #include <sys/syscallargs.h>
224 #include <sys/syscall_stats.h>
225 #include <sys/kauth.h>
226 #include <sys/sleepq.h>
227 #include <sys/lockdebug.h>
228 #include <sys/kmem.h>
229
230 #include <uvm/uvm_extern.h>
231
232 struct lwplist alllwp;
233
234 POOL_INIT(lwp_pool, sizeof(struct lwp), MIN_LWP_ALIGNMENT, 0, 0, "lwppl",
235 &pool_allocator_nointr, IPL_NONE);
236 POOL_INIT(lwp_uc_pool, sizeof(ucontext_t), 0, 0, 0, "lwpucpl",
237 &pool_allocator_nointr, IPL_NONE);
238
239 static specificdata_domain_t lwp_specificdata_domain;
240
241 #define LWP_DEBUG
242
243 #ifdef LWP_DEBUG
244 int lwp_debug = 0;
245 #define DPRINTF(x) if (lwp_debug) printf x
246 #else
247 #define DPRINTF(x)
248 #endif
249
250 void
251 lwpinit(void)
252 {
253
254 lwp_specificdata_domain = specificdata_domain_create();
255 KASSERT(lwp_specificdata_domain != NULL);
256 lwp_sys_init();
257 }
258
259 /*
260 * Set an suspended.
261 *
262 * Must be called with p_smutex held, and the LWP locked. Will unlock the
263 * LWP before return.
264 */
265 int
266 lwp_suspend(struct lwp *curl, struct lwp *t)
267 {
268 int error;
269
270 KASSERT(mutex_owned(&t->l_proc->p_smutex));
271 KASSERT(lwp_locked(t, NULL));
272
273 KASSERT(curl != t || curl->l_stat == LSONPROC);
274
275 /*
276 * If the current LWP has been told to exit, we must not suspend anyone
277 * else or deadlock could occur. We won't return to userspace.
278 */
279 if ((curl->l_stat & (LW_WEXIT | LW_WCORE)) != 0) {
280 lwp_unlock(t);
281 return (EDEADLK);
282 }
283
284 error = 0;
285
286 switch (t->l_stat) {
287 case LSRUN:
288 case LSONPROC:
289 t->l_flag |= LW_WSUSPEND;
290 lwp_need_userret(t);
291 lwp_unlock(t);
292 break;
293
294 case LSSLEEP:
295 t->l_flag |= LW_WSUSPEND;
296
297 /*
298 * Kick the LWP and try to get it to the kernel boundary
299 * so that it will release any locks that it holds.
300 * setrunnable() will release the lock.
301 */
302 if ((t->l_flag & LW_SINTR) != 0)
303 setrunnable(t);
304 else
305 lwp_unlock(t);
306 break;
307
308 case LSSUSPENDED:
309 lwp_unlock(t);
310 break;
311
312 case LSSTOP:
313 t->l_flag |= LW_WSUSPEND;
314 setrunnable(t);
315 break;
316
317 case LSIDL:
318 case LSZOMB:
319 error = EINTR; /* It's what Solaris does..... */
320 lwp_unlock(t);
321 break;
322 }
323
324 /*
325 * XXXLWP Wait for:
326 *
327 * o process exiting
328 * o target LWP suspended
329 * o target LWP not suspended and L_WSUSPEND clear
330 * o target LWP exited
331 */
332
333 return (error);
334 }
335
336 /*
337 * Restart a suspended LWP.
338 *
339 * Must be called with p_smutex held, and the LWP locked. Will unlock the
340 * LWP before return.
341 */
342 void
343 lwp_continue(struct lwp *l)
344 {
345
346 KASSERT(mutex_owned(&l->l_proc->p_smutex));
347 KASSERT(lwp_locked(l, NULL));
348
349 DPRINTF(("lwp_continue of %d.%d (%s), state %d, wchan %p\n",
350 l->l_proc->p_pid, l->l_lid, l->l_proc->p_comm, l->l_stat,
351 l->l_wchan));
352
353 /* If rebooting or not suspended, then just bail out. */
354 if ((l->l_flag & LW_WREBOOT) != 0) {
355 lwp_unlock(l);
356 return;
357 }
358
359 l->l_flag &= ~LW_WSUSPEND;
360
361 if (l->l_stat != LSSUSPENDED) {
362 lwp_unlock(l);
363 return;
364 }
365
366 /* setrunnable() will release the lock. */
367 setrunnable(l);
368 }
369
370 /*
371 * Wait for an LWP within the current process to exit. If 'lid' is
372 * non-zero, we are waiting for a specific LWP.
373 *
374 * Must be called with p->p_smutex held.
375 */
376 int
377 lwp_wait1(struct lwp *l, lwpid_t lid, lwpid_t *departed, int flags)
378 {
379 struct proc *p = l->l_proc;
380 struct lwp *l2;
381 int nfound, error;
382 lwpid_t curlid;
383 bool exiting;
384
385 DPRINTF(("lwp_wait1: %d.%d waiting for %d.\n",
386 p->p_pid, l->l_lid, lid));
387
388 KASSERT(mutex_owned(&p->p_smutex));
389
390 p->p_nlwpwait++;
391 l->l_waitingfor = lid;
392 curlid = l->l_lid;
393 exiting = ((flags & LWPWAIT_EXITCONTROL) != 0);
394
395 for (;;) {
396 /*
397 * Avoid a race between exit1() and sigexit(): if the
398 * process is dumping core, then we need to bail out: call
399 * into lwp_userret() where we will be suspended until the
400 * deed is done.
401 */
402 if ((p->p_sflag & PS_WCORE) != 0) {
403 mutex_exit(&p->p_smutex);
404 lwp_userret(l);
405 #ifdef DIAGNOSTIC
406 panic("lwp_wait1");
407 #endif
408 /* NOTREACHED */
409 }
410
411 /*
412 * First off, drain any detached LWP that is waiting to be
413 * reaped.
414 */
415 while ((l2 = p->p_zomblwp) != NULL) {
416 p->p_zomblwp = NULL;
417 lwp_free(l2, false, false);/* releases proc mutex */
418 mutex_enter(&p->p_smutex);
419 }
420
421 /*
422 * Now look for an LWP to collect. If the whole process is
423 * exiting, count detached LWPs as eligible to be collected,
424 * but don't drain them here.
425 */
426 nfound = 0;
427 error = 0;
428 LIST_FOREACH(l2, &p->p_lwps, l_sibling) {
429 /*
430 * If a specific wait and the target is waiting on
431 * us, then avoid deadlock. This also traps LWPs
432 * that try to wait on themselves.
433 *
434 * Note that this does not handle more complicated
435 * cycles, like: t1 -> t2 -> t3 -> t1. The process
436 * can still be killed so it is not a major problem.
437 */
438 if (l2->l_lid == lid && l2->l_waitingfor == curlid) {
439 error = EDEADLK;
440 break;
441 }
442 if (l2 == l)
443 continue;
444 if ((l2->l_prflag & LPR_DETACHED) != 0) {
445 nfound += exiting;
446 continue;
447 }
448 if (lid != 0) {
449 if (l2->l_lid != lid)
450 continue;
451 /*
452 * Mark this LWP as the first waiter, if there
453 * is no other.
454 */
455 if (l2->l_waiter == 0)
456 l2->l_waiter = curlid;
457 } else if (l2->l_waiter != 0) {
458 /*
459 * It already has a waiter - so don't
460 * collect it. If the waiter doesn't
461 * grab it we'll get another chance
462 * later.
463 */
464 nfound++;
465 continue;
466 }
467 nfound++;
468
469 /* No need to lock the LWP in order to see LSZOMB. */
470 if (l2->l_stat != LSZOMB)
471 continue;
472
473 /*
474 * We're no longer waiting. Reset the "first waiter"
475 * pointer on the target, in case it was us.
476 */
477 l->l_waitingfor = 0;
478 l2->l_waiter = 0;
479 p->p_nlwpwait--;
480 if (departed)
481 *departed = l2->l_lid;
482
483 /* lwp_free() releases the proc lock. */
484 lwp_free(l2, false, false);
485 mutex_enter(&p->p_smutex);
486 return 0;
487 }
488
489 if (error != 0)
490 break;
491 if (nfound == 0) {
492 error = ESRCH;
493 break;
494 }
495
496 /*
497 * The kernel is careful to ensure that it can not deadlock
498 * when exiting - just keep waiting.
499 */
500 if (exiting) {
501 KASSERT(p->p_nlwps > 1);
502 cv_wait(&p->p_lwpcv, &p->p_smutex);
503 continue;
504 }
505
506 /*
507 * If all other LWPs are waiting for exits or suspends
508 * and the supply of zombies and potential zombies is
509 * exhausted, then we are about to deadlock.
510 *
511 * If the process is exiting (and this LWP is not the one
512 * that is coordinating the exit) then bail out now.
513 */
514 if ((p->p_sflag & PS_WEXIT) != 0 ||
515 p->p_nrlwps + p->p_nzlwps - p->p_ndlwps <= p->p_nlwpwait) {
516 error = EDEADLK;
517 break;
518 }
519
520 /*
521 * Sit around and wait for something to happen. We'll be
522 * awoken if any of the conditions examined change: if an
523 * LWP exits, is collected, or is detached.
524 */
525 if ((error = cv_wait_sig(&p->p_lwpcv, &p->p_smutex)) != 0)
526 break;
527 }
528
529 /*
530 * We didn't find any LWPs to collect, we may have received a
531 * signal, or some other condition has caused us to bail out.
532 *
533 * If waiting on a specific LWP, clear the waiters marker: some
534 * other LWP may want it. Then, kick all the remaining waiters
535 * so that they can re-check for zombies and for deadlock.
536 */
537 if (lid != 0) {
538 LIST_FOREACH(l2, &p->p_lwps, l_sibling) {
539 if (l2->l_lid == lid) {
540 if (l2->l_waiter == curlid)
541 l2->l_waiter = 0;
542 break;
543 }
544 }
545 }
546 p->p_nlwpwait--;
547 l->l_waitingfor = 0;
548 cv_broadcast(&p->p_lwpcv);
549
550 return error;
551 }
552
553 /*
554 * Create a new LWP within process 'p2', using LWP 'l1' as a template.
555 * The new LWP is created in state LSIDL and must be set running,
556 * suspended, or stopped by the caller.
557 */
558 int
559 newlwp(struct lwp *l1, struct proc *p2, vaddr_t uaddr, bool inmem,
560 int flags, void *stack, size_t stacksize,
561 void (*func)(void *), void *arg, struct lwp **rnewlwpp)
562 {
563 struct lwp *l2, *isfree;
564 turnstile_t *ts;
565
566 /*
567 * First off, reap any detached LWP waiting to be collected.
568 * We can re-use its LWP structure and turnstile.
569 */
570 isfree = NULL;
571 if (p2->p_zomblwp != NULL) {
572 mutex_enter(&p2->p_smutex);
573 if ((isfree = p2->p_zomblwp) != NULL) {
574 p2->p_zomblwp = NULL;
575 lwp_free(isfree, true, false);/* releases proc mutex */
576 } else
577 mutex_exit(&p2->p_smutex);
578 }
579 if (isfree == NULL) {
580 l2 = pool_get(&lwp_pool, PR_WAITOK);
581 memset(l2, 0, sizeof(*l2));
582 l2->l_ts = pool_cache_get(&turnstile_cache, PR_WAITOK);
583 SLIST_INIT(&l2->l_pi_lenders);
584 } else {
585 l2 = isfree;
586 ts = l2->l_ts;
587 KASSERT(l2->l_inheritedprio == MAXPRI);
588 KASSERT(SLIST_EMPTY(&l2->l_pi_lenders));
589 memset(l2, 0, sizeof(*l2));
590 l2->l_ts = ts;
591 }
592
593 l2->l_stat = LSIDL;
594 l2->l_proc = p2;
595 l2->l_refcnt = 1;
596 l2->l_priority = l1->l_priority;
597 l2->l_usrpri = l1->l_usrpri;
598 l2->l_inheritedprio = MAXPRI;
599 l2->l_mutex = l1->l_cpu->ci_schedstate.spc_mutex;
600 l2->l_cpu = l1->l_cpu;
601 l2->l_flag = inmem ? LW_INMEM : 0;
602 lwp_initspecific(l2);
603 sched_lwp_fork(l2);
604
605 if (p2->p_flag & PK_SYSTEM) {
606 /*
607 * Mark it as a system process and not a candidate for
608 * swapping.
609 */
610 l2->l_flag |= LW_SYSTEM;
611 }
612
613 lwp_update_creds(l2);
614 callout_init(&l2->l_tsleep_ch);
615 cv_init(&l2->l_sigcv, "sigwait");
616 l2->l_syncobj = &sched_syncobj;
617
618 if (rnewlwpp != NULL)
619 *rnewlwpp = l2;
620
621 l2->l_addr = UAREA_TO_USER(uaddr);
622 uvm_lwp_fork(l1, l2, stack, stacksize, func,
623 (arg != NULL) ? arg : l2);
624
625 mutex_enter(&p2->p_smutex);
626
627 if ((flags & LWP_DETACHED) != 0) {
628 l2->l_prflag = LPR_DETACHED;
629 p2->p_ndlwps++;
630 } else
631 l2->l_prflag = 0;
632
633 l2->l_sigmask = l1->l_sigmask;
634 CIRCLEQ_INIT(&l2->l_sigpend.sp_info);
635 sigemptyset(&l2->l_sigpend.sp_set);
636
637 p2->p_nlwpid++;
638 if (p2->p_nlwpid == 0)
639 p2->p_nlwpid++;
640 l2->l_lid = p2->p_nlwpid;
641 LIST_INSERT_HEAD(&p2->p_lwps, l2, l_sibling);
642 p2->p_nlwps++;
643
644 mutex_exit(&p2->p_smutex);
645
646 mutex_enter(&proclist_mutex);
647 LIST_INSERT_HEAD(&alllwp, l2, l_list);
648 mutex_exit(&proclist_mutex);
649
650 SYSCALL_TIME_LWP_INIT(l2);
651
652 if (p2->p_emul->e_lwp_fork)
653 (*p2->p_emul->e_lwp_fork)(l1, l2);
654
655 return (0);
656 }
657
658 /*
659 * Called by MD code when a new LWP begins execution. Must be called
660 * with the previous LWP locked (so at splsched), or if there is no
661 * previous LWP, at splsched.
662 */
663 void
664 lwp_startup(struct lwp *prev, struct lwp *new)
665 {
666
667 curlwp = new;
668 if (prev != NULL) {
669 lwp_unlock(prev);
670 }
671 spl0();
672 pmap_activate(new);
673 LOCKDEBUG_BARRIER(NULL, 0);
674 KERNEL_LOCK(1, new);
675 }
676
677 /*
678 * Quit the process.
679 * this can only be used meaningfully if you're willing to switch away.
680 * Calling with l != curlwp would be weird.
681 */
682 void
683 lwp_exit(struct lwp *l)
684 {
685 struct proc *p = l->l_proc;
686 struct lwp *l2;
687
688 DPRINTF(("lwp_exit: %d.%d exiting.\n", p->p_pid, l->l_lid));
689 DPRINTF((" nlwps: %d nzlwps: %d\n", p->p_nlwps, p->p_nzlwps));
690
691 /*
692 * Verify that we hold no locks other than the kernel lock.
693 */
694 #ifdef MULTIPROCESSOR
695 LOCKDEBUG_BARRIER(&kernel_lock, 0);
696 #else
697 LOCKDEBUG_BARRIER(NULL, 0);
698 #endif
699
700 /*
701 * If we are the last live LWP in a process, we need to exit the
702 * entire process. We do so with an exit status of zero, because
703 * it's a "controlled" exit, and because that's what Solaris does.
704 *
705 * We are not quite a zombie yet, but for accounting purposes we
706 * must increment the count of zombies here.
707 *
708 * Note: the last LWP's specificdata will be deleted here.
709 */
710 mutex_enter(&p->p_smutex);
711 if (p->p_nlwps - p->p_nzlwps == 1) {
712 DPRINTF(("lwp_exit: %d.%d calling exit1()\n",
713 p->p_pid, l->l_lid));
714 exit1(l, 0);
715 /* NOTREACHED */
716 }
717 p->p_nzlwps++;
718 mutex_exit(&p->p_smutex);
719
720 if (p->p_emul->e_lwp_exit)
721 (*p->p_emul->e_lwp_exit)(l);
722
723 /* Delete the specificdata while it's still safe to sleep. */
724 specificdata_fini(lwp_specificdata_domain, &l->l_specdataref);
725
726 /*
727 * Release our cached credentials.
728 */
729 kauth_cred_free(l->l_cred);
730
731 /*
732 * Remove the LWP from the global list.
733 */
734 mutex_enter(&proclist_mutex);
735 LIST_REMOVE(l, l_list);
736 mutex_exit(&proclist_mutex);
737
738 /*
739 * Get rid of all references to the LWP that others (e.g. procfs)
740 * may have, and mark the LWP as a zombie. If the LWP is detached,
741 * mark it waiting for collection in the proc structure. Note that
742 * before we can do that, we need to free any other dead, deatched
743 * LWP waiting to meet its maker.
744 *
745 * XXXSMP disable preemption.
746 */
747 mutex_enter(&p->p_smutex);
748 lwp_drainrefs(l);
749
750 if ((l->l_prflag & LPR_DETACHED) != 0) {
751 while ((l2 = p->p_zomblwp) != NULL) {
752 p->p_zomblwp = NULL;
753 lwp_free(l2, false, false);/* releases proc mutex */
754 mutex_enter(&p->p_smutex);
755 }
756 p->p_zomblwp = l;
757 }
758
759 /*
760 * If we find a pending signal for the process and we have been
761 * asked to check for signals, then we loose: arrange to have
762 * all other LWPs in the process check for signals.
763 */
764 if ((l->l_flag & LW_PENDSIG) != 0 &&
765 firstsig(&p->p_sigpend.sp_set) != 0) {
766 LIST_FOREACH(l2, &p->p_lwps, l_sibling) {
767 lwp_lock(l2);
768 l2->l_flag |= LW_PENDSIG;
769 lwp_unlock(l2);
770 }
771 }
772
773 lwp_lock(l);
774 l->l_stat = LSZOMB;
775 lwp_unlock(l);
776 p->p_nrlwps--;
777 cv_broadcast(&p->p_lwpcv);
778 mutex_exit(&p->p_smutex);
779
780 /*
781 * We can no longer block. At this point, lwp_free() may already
782 * be gunning for us. On a multi-CPU system, we may be off p_lwps.
783 *
784 * Free MD LWP resources.
785 */
786 #ifndef __NO_CPU_LWP_FREE
787 cpu_lwp_free(l, 0);
788 #endif
789 pmap_deactivate(l);
790
791 /*
792 * Release the kernel lock, signal another LWP to collect us,
793 * and switch away into oblivion.
794 */
795 #ifdef notyet
796 /* XXXSMP hold in lwp_userret() */
797 KERNEL_UNLOCK_LAST(l);
798 #else
799 KERNEL_UNLOCK_ALL(l, NULL);
800 #endif
801
802 lwp_exit_switchaway(l);
803 }
804
805 void
806 lwp_exit_switchaway(struct lwp *l)
807 {
808 struct cpu_info *ci;
809 struct lwp *idlelwp;
810
811 /* Unlocked, but is for statistics only. */
812 uvmexp.swtch++;
813
814 (void)splsched();
815 ci = curcpu();
816 idlelwp = ci->ci_data.cpu_idlelwp;
817 idlelwp->l_stat = LSONPROC;
818 cpu_switchto(NULL, idlelwp);
819 }
820
821 /*
822 * Free a dead LWP's remaining resources.
823 *
824 * XXXLWP limits.
825 */
826 void
827 lwp_free(struct lwp *l, bool recycle, bool last)
828 {
829 struct proc *p = l->l_proc;
830 ksiginfoq_t kq;
831
832 /*
833 * If this was not the last LWP in the process, then adjust
834 * counters and unlock.
835 */
836 if (!last) {
837 /*
838 * Add the LWP's run time to the process' base value.
839 * This needs to co-incide with coming off p_lwps.
840 */
841 timeradd(&l->l_rtime, &p->p_rtime, &p->p_rtime);
842 LIST_REMOVE(l, l_sibling);
843 p->p_nlwps--;
844 p->p_nzlwps--;
845 if ((l->l_prflag & LPR_DETACHED) != 0)
846 p->p_ndlwps--;
847
848 /*
849 * Have any LWPs sleeping in lwp_wait() recheck for
850 * deadlock.
851 */
852 cv_broadcast(&p->p_lwpcv);
853 mutex_exit(&p->p_smutex);
854 }
855
856 #ifdef MULTIPROCESSOR
857 /*
858 * In the unlikely event that the LWP is still on the CPU,
859 * then spin until it has switched away. We need to release
860 * all locks to avoid deadlock against interrupt handlers on
861 * the target CPU.
862 */
863 if (l->l_cpu->ci_curlwp == l) {
864 int count;
865 (void)count; /* XXXgcc */
866 KERNEL_UNLOCK_ALL(curlwp, &count);
867 while (l->l_cpu->ci_curlwp == l)
868 SPINLOCK_BACKOFF_HOOK;
869 KERNEL_LOCK(count, curlwp);
870 }
871 #endif
872
873 /*
874 * Destroy the LWP's remaining signal information.
875 */
876 ksiginfo_queue_init(&kq);
877 sigclear(&l->l_sigpend, NULL, &kq);
878 ksiginfo_queue_drain(&kq);
879 cv_destroy(&l->l_sigcv);
880
881 /*
882 * Free the LWP's turnstile and the LWP structure itself unless the
883 * caller wants to recycle them. Also, free the scheduler specific data.
884 *
885 * We can't return turnstile0 to the pool (it didn't come from it),
886 * so if it comes up just drop it quietly and move on.
887 *
888 * We don't recycle the VM resources at this time.
889 */
890 KERNEL_LOCK(1, curlwp); /* XXXSMP */
891
892 sched_lwp_exit(l);
893
894 if (!recycle && l->l_ts != &turnstile0)
895 pool_cache_put(&turnstile_cache, l->l_ts);
896 #ifndef __NO_CPU_LWP_FREE
897 cpu_lwp_free2(l);
898 #endif
899 uvm_lwp_exit(l);
900 KASSERT(SLIST_EMPTY(&l->l_pi_lenders));
901 KASSERT(l->l_inheritedprio == MAXPRI);
902 if (!recycle)
903 pool_put(&lwp_pool, l);
904 KERNEL_UNLOCK_ONE(curlwp); /* XXXSMP */
905 }
906
907 /*
908 * Pick a LWP to represent the process for those operations which
909 * want information about a "process" that is actually associated
910 * with a LWP.
911 *
912 * If 'locking' is false, no locking or lock checks are performed.
913 * This is intended for use by DDB.
914 *
915 * We don't bother locking the LWP here, since code that uses this
916 * interface is broken by design and an exact match is not required.
917 */
918 struct lwp *
919 proc_representative_lwp(struct proc *p, int *nrlwps, int locking)
920 {
921 struct lwp *l, *onproc, *running, *sleeping, *stopped, *suspended;
922 struct lwp *signalled;
923 int cnt;
924
925 if (locking) {
926 KASSERT(mutex_owned(&p->p_smutex));
927 }
928
929 /* Trivial case: only one LWP */
930 if (p->p_nlwps == 1) {
931 l = LIST_FIRST(&p->p_lwps);
932 if (nrlwps)
933 *nrlwps = (l->l_stat == LSONPROC || LSRUN);
934 return l;
935 }
936
937 cnt = 0;
938 switch (p->p_stat) {
939 case SSTOP:
940 case SACTIVE:
941 /* Pick the most live LWP */
942 onproc = running = sleeping = stopped = suspended = NULL;
943 signalled = NULL;
944 LIST_FOREACH(l, &p->p_lwps, l_sibling) {
945 if ((l->l_flag & LW_IDLE) != 0) {
946 continue;
947 }
948 if (l->l_lid == p->p_sigctx.ps_lwp)
949 signalled = l;
950 switch (l->l_stat) {
951 case LSONPROC:
952 onproc = l;
953 cnt++;
954 break;
955 case LSRUN:
956 running = l;
957 cnt++;
958 break;
959 case LSSLEEP:
960 sleeping = l;
961 break;
962 case LSSTOP:
963 stopped = l;
964 break;
965 case LSSUSPENDED:
966 suspended = l;
967 break;
968 }
969 }
970 if (nrlwps)
971 *nrlwps = cnt;
972 if (signalled)
973 l = signalled;
974 else if (onproc)
975 l = onproc;
976 else if (running)
977 l = running;
978 else if (sleeping)
979 l = sleeping;
980 else if (stopped)
981 l = stopped;
982 else if (suspended)
983 l = suspended;
984 else
985 break;
986 return l;
987 if (nrlwps)
988 *nrlwps = 0;
989 l = LIST_FIRST(&p->p_lwps);
990 return l;
991 #ifdef DIAGNOSTIC
992 case SIDL:
993 case SZOMB:
994 case SDYING:
995 case SDEAD:
996 if (locking)
997 mutex_exit(&p->p_smutex);
998 /* We have more than one LWP and we're in SIDL?
999 * How'd that happen?
1000 */
1001 panic("Too many LWPs in idle/dying process %d (%s) stat = %d",
1002 p->p_pid, p->p_comm, p->p_stat);
1003 break;
1004 default:
1005 if (locking)
1006 mutex_exit(&p->p_smutex);
1007 panic("Process %d (%s) in unknown state %d",
1008 p->p_pid, p->p_comm, p->p_stat);
1009 #endif
1010 }
1011
1012 if (locking)
1013 mutex_exit(&p->p_smutex);
1014 panic("proc_representative_lwp: couldn't find a lwp for process"
1015 " %d (%s)", p->p_pid, p->p_comm);
1016 /* NOTREACHED */
1017 return NULL;
1018 }
1019
1020 /*
1021 * Look up a live LWP within the speicifed process, and return it locked.
1022 *
1023 * Must be called with p->p_smutex held.
1024 */
1025 struct lwp *
1026 lwp_find(struct proc *p, int id)
1027 {
1028 struct lwp *l;
1029
1030 KASSERT(mutex_owned(&p->p_smutex));
1031
1032 LIST_FOREACH(l, &p->p_lwps, l_sibling) {
1033 if (l->l_lid == id)
1034 break;
1035 }
1036
1037 /*
1038 * No need to lock - all of these conditions will
1039 * be visible with the process level mutex held.
1040 */
1041 if (l != NULL && (l->l_stat == LSIDL || l->l_stat == LSZOMB))
1042 l = NULL;
1043
1044 return l;
1045 }
1046
1047 /*
1048 * Update an LWP's cached credentials to mirror the process' master copy.
1049 *
1050 * This happens early in the syscall path, on user trap, and on LWP
1051 * creation. A long-running LWP can also voluntarily choose to update
1052 * it's credentials by calling this routine. This may be called from
1053 * LWP_CACHE_CREDS(), which checks l->l_cred != p->p_cred beforehand.
1054 */
1055 void
1056 lwp_update_creds(struct lwp *l)
1057 {
1058 kauth_cred_t oc;
1059 struct proc *p;
1060
1061 p = l->l_proc;
1062 oc = l->l_cred;
1063
1064 mutex_enter(&p->p_mutex);
1065 kauth_cred_hold(p->p_cred);
1066 l->l_cred = p->p_cred;
1067 mutex_exit(&p->p_mutex);
1068 if (oc != NULL) {
1069 KERNEL_LOCK(1, l); /* XXXSMP */
1070 kauth_cred_free(oc);
1071 KERNEL_UNLOCK_ONE(l); /* XXXSMP */
1072 }
1073 }
1074
1075 /*
1076 * Verify that an LWP is locked, and optionally verify that the lock matches
1077 * one we specify.
1078 */
1079 int
1080 lwp_locked(struct lwp *l, kmutex_t *mtx)
1081 {
1082 kmutex_t *cur = l->l_mutex;
1083
1084 return mutex_owned(cur) && (mtx == cur || mtx == NULL);
1085 }
1086
1087 /*
1088 * Lock an LWP.
1089 */
1090 void
1091 lwp_lock_retry(struct lwp *l, kmutex_t *old)
1092 {
1093
1094 /*
1095 * XXXgcc ignoring kmutex_t * volatile on i386
1096 *
1097 * gcc version 4.1.2 20061021 prerelease (NetBSD nb1 20061021)
1098 */
1099 #if 1
1100 while (l->l_mutex != old) {
1101 #else
1102 for (;;) {
1103 #endif
1104 mutex_spin_exit(old);
1105 old = l->l_mutex;
1106 mutex_spin_enter(old);
1107
1108 /*
1109 * mutex_enter() will have posted a read barrier. Re-test
1110 * l->l_mutex. If it has changed, we need to try again.
1111 */
1112 #if 1
1113 }
1114 #else
1115 } while (__predict_false(l->l_mutex != old));
1116 #endif
1117 }
1118
1119 /*
1120 * Lend a new mutex to an LWP. The old mutex must be held.
1121 */
1122 void
1123 lwp_setlock(struct lwp *l, kmutex_t *new)
1124 {
1125
1126 KASSERT(mutex_owned(l->l_mutex));
1127
1128 mb_write();
1129 l->l_mutex = new;
1130 }
1131
1132 /*
1133 * Lend a new mutex to an LWP, and release the old mutex. The old mutex
1134 * must be held.
1135 */
1136 void
1137 lwp_unlock_to(struct lwp *l, kmutex_t *new)
1138 {
1139 kmutex_t *old;
1140
1141 KASSERT(mutex_owned(l->l_mutex));
1142
1143 old = l->l_mutex;
1144 mb_write();
1145 l->l_mutex = new;
1146 mutex_spin_exit(old);
1147 }
1148
1149 /*
1150 * Acquire a new mutex, and donate it to an LWP. The LWP must already be
1151 * locked.
1152 */
1153 void
1154 lwp_relock(struct lwp *l, kmutex_t *new)
1155 {
1156 kmutex_t *old;
1157
1158 KASSERT(mutex_owned(l->l_mutex));
1159
1160 old = l->l_mutex;
1161 if (old != new) {
1162 mutex_spin_enter(new);
1163 l->l_mutex = new;
1164 mutex_spin_exit(old);
1165 }
1166 }
1167
1168 int
1169 lwp_trylock(struct lwp *l)
1170 {
1171 kmutex_t *old;
1172
1173 for (;;) {
1174 if (!mutex_tryenter(old = l->l_mutex))
1175 return 0;
1176 if (__predict_true(l->l_mutex == old))
1177 return 1;
1178 mutex_spin_exit(old);
1179 }
1180 }
1181
1182 /*
1183 * Handle exceptions for mi_userret(). Called if a member of LW_USERRET is
1184 * set.
1185 */
1186 void
1187 lwp_userret(struct lwp *l)
1188 {
1189 struct proc *p;
1190 void (*hook)(void);
1191 int sig;
1192
1193 p = l->l_proc;
1194
1195 /*
1196 * It should be safe to do this read unlocked on a multiprocessor
1197 * system..
1198 */
1199 while ((l->l_flag & LW_USERRET) != 0) {
1200 /*
1201 * Process pending signals first, unless the process
1202 * is dumping core or exiting, where we will instead
1203 * enter the L_WSUSPEND case below.
1204 */
1205 if ((l->l_flag & (LW_PENDSIG | LW_WCORE | LW_WEXIT)) ==
1206 LW_PENDSIG) {
1207 KERNEL_LOCK(1, l); /* XXXSMP pool_put() below */
1208 mutex_enter(&p->p_smutex);
1209 while ((sig = issignal(l)) != 0)
1210 postsig(sig);
1211 mutex_exit(&p->p_smutex);
1212 KERNEL_UNLOCK_LAST(l); /* XXXSMP */
1213 }
1214
1215 /*
1216 * Core-dump or suspend pending.
1217 *
1218 * In case of core dump, suspend ourselves, so that the
1219 * kernel stack and therefore the userland registers saved
1220 * in the trapframe are around for coredump() to write them
1221 * out. We issue a wakeup on p->p_lwpcv so that sigexit()
1222 * will write the core file out once all other LWPs are
1223 * suspended.
1224 */
1225 if ((l->l_flag & LW_WSUSPEND) != 0) {
1226 mutex_enter(&p->p_smutex);
1227 p->p_nrlwps--;
1228 cv_broadcast(&p->p_lwpcv);
1229 lwp_lock(l);
1230 l->l_stat = LSSUSPENDED;
1231 mutex_exit(&p->p_smutex);
1232 mi_switch(l);
1233 }
1234
1235 /* Process is exiting. */
1236 if ((l->l_flag & LW_WEXIT) != 0) {
1237 KERNEL_LOCK(1, l);
1238 lwp_exit(l);
1239 KASSERT(0);
1240 /* NOTREACHED */
1241 }
1242
1243 /* Call userret hook; used by Linux emulation. */
1244 if ((l->l_flag & LW_WUSERRET) != 0) {
1245 lwp_lock(l);
1246 l->l_flag &= ~LW_WUSERRET;
1247 lwp_unlock(l);
1248 hook = p->p_userret;
1249 p->p_userret = NULL;
1250 (*hook)();
1251 }
1252 }
1253 }
1254
1255 /*
1256 * Force an LWP to enter the kernel, to take a trip through lwp_userret().
1257 */
1258 void
1259 lwp_need_userret(struct lwp *l)
1260 {
1261 KASSERT(lwp_locked(l, NULL));
1262
1263 /*
1264 * Since the tests in lwp_userret() are done unlocked, make sure
1265 * that the condition will be seen before forcing the LWP to enter
1266 * kernel mode.
1267 */
1268 mb_write();
1269 cpu_signotify(l);
1270 }
1271
1272 /*
1273 * Add one reference to an LWP. This will prevent the LWP from
1274 * exiting, thus keep the lwp structure and PCB around to inspect.
1275 */
1276 void
1277 lwp_addref(struct lwp *l)
1278 {
1279
1280 KASSERT(mutex_owned(&l->l_proc->p_smutex));
1281 KASSERT(l->l_stat != LSZOMB);
1282 KASSERT(l->l_refcnt != 0);
1283
1284 l->l_refcnt++;
1285 }
1286
1287 /*
1288 * Remove one reference to an LWP. If this is the last reference,
1289 * then we must finalize the LWP's death.
1290 */
1291 void
1292 lwp_delref(struct lwp *l)
1293 {
1294 struct proc *p = l->l_proc;
1295
1296 mutex_enter(&p->p_smutex);
1297 if (--l->l_refcnt == 0)
1298 cv_broadcast(&p->p_refcv);
1299 mutex_exit(&p->p_smutex);
1300 }
1301
1302 /*
1303 * Drain all references to the current LWP.
1304 */
1305 void
1306 lwp_drainrefs(struct lwp *l)
1307 {
1308 struct proc *p = l->l_proc;
1309
1310 KASSERT(mutex_owned(&p->p_smutex));
1311 KASSERT(l->l_refcnt != 0);
1312
1313 l->l_refcnt--;
1314 while (l->l_refcnt != 0)
1315 cv_wait(&p->p_refcv, &p->p_smutex);
1316 }
1317
1318 /*
1319 * lwp_specific_key_create --
1320 * Create a key for subsystem lwp-specific data.
1321 */
1322 int
1323 lwp_specific_key_create(specificdata_key_t *keyp, specificdata_dtor_t dtor)
1324 {
1325
1326 return (specificdata_key_create(lwp_specificdata_domain, keyp, dtor));
1327 }
1328
1329 /*
1330 * lwp_specific_key_delete --
1331 * Delete a key for subsystem lwp-specific data.
1332 */
1333 void
1334 lwp_specific_key_delete(specificdata_key_t key)
1335 {
1336
1337 specificdata_key_delete(lwp_specificdata_domain, key);
1338 }
1339
1340 /*
1341 * lwp_initspecific --
1342 * Initialize an LWP's specificdata container.
1343 */
1344 void
1345 lwp_initspecific(struct lwp *l)
1346 {
1347 int error;
1348
1349 error = specificdata_init(lwp_specificdata_domain, &l->l_specdataref);
1350 KASSERT(error == 0);
1351 }
1352
1353 /*
1354 * lwp_finispecific --
1355 * Finalize an LWP's specificdata container.
1356 */
1357 void
1358 lwp_finispecific(struct lwp *l)
1359 {
1360
1361 specificdata_fini(lwp_specificdata_domain, &l->l_specdataref);
1362 }
1363
1364 /*
1365 * lwp_getspecific --
1366 * Return lwp-specific data corresponding to the specified key.
1367 *
1368 * Note: LWP specific data is NOT INTERLOCKED. An LWP should access
1369 * only its OWN SPECIFIC DATA. If it is necessary to access another
1370 * LWP's specifc data, care must be taken to ensure that doing so
1371 * would not cause internal data structure inconsistency (i.e. caller
1372 * can guarantee that the target LWP is not inside an lwp_getspecific()
1373 * or lwp_setspecific() call).
1374 */
1375 void *
1376 lwp_getspecific(specificdata_key_t key)
1377 {
1378
1379 return (specificdata_getspecific_unlocked(lwp_specificdata_domain,
1380 &curlwp->l_specdataref, key));
1381 }
1382
1383 void *
1384 _lwp_getspecific_by_lwp(struct lwp *l, specificdata_key_t key)
1385 {
1386
1387 return (specificdata_getspecific_unlocked(lwp_specificdata_domain,
1388 &l->l_specdataref, key));
1389 }
1390
1391 /*
1392 * lwp_setspecific --
1393 * Set lwp-specific data corresponding to the specified key.
1394 */
1395 void
1396 lwp_setspecific(specificdata_key_t key, void *data)
1397 {
1398
1399 specificdata_setspecific(lwp_specificdata_domain,
1400 &curlwp->l_specdataref, key, data);
1401 }
1402