kern_lwp.c revision 1.89 1 /* $NetBSD: kern_lwp.c,v 1.89 2008/01/07 11:41:29 yamt Exp $ */
2
3 /*-
4 * Copyright (c) 2001, 2006, 2007 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Nathan J. Williams, and Andrew Doran.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 * 3. All advertising materials mentioning features or use of this software
19 * must display the following acknowledgement:
20 * This product includes software developed by the NetBSD
21 * Foundation, Inc. and its contributors.
22 * 4. Neither the name of The NetBSD Foundation nor the names of its
23 * contributors may be used to endorse or promote products derived
24 * from this software without specific prior written permission.
25 *
26 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
27 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
28 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
29 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
30 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
31 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
32 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
33 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
34 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
35 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
36 * POSSIBILITY OF SUCH DAMAGE.
37 */
38
39 /*
40 * Overview
41 *
42 * Lightweight processes (LWPs) are the basic unit or thread of
43 * execution within the kernel. The core state of an LWP is described
44 * by "struct lwp", also known as lwp_t.
45 *
46 * Each LWP is contained within a process (described by "struct proc"),
47 * Every process contains at least one LWP, but may contain more. The
48 * process describes attributes shared among all of its LWPs such as a
49 * private address space, global execution state (stopped, active,
50 * zombie, ...), signal disposition and so on. On a multiprocessor
51 * machine, multiple LWPs be executing concurrently in the kernel.
52 *
53 * Execution states
54 *
55 * At any given time, an LWP has overall state that is described by
56 * lwp::l_stat. The states are broken into two sets below. The first
57 * set is guaranteed to represent the absolute, current state of the
58 * LWP:
59 *
60 * LSONPROC
61 *
62 * On processor: the LWP is executing on a CPU, either in the
63 * kernel or in user space.
64 *
65 * LSRUN
66 *
67 * Runnable: the LWP is parked on a run queue, and may soon be
68 * chosen to run by a idle processor, or by a processor that
69 * has been asked to preempt a currently runnning but lower
70 * priority LWP. If the LWP is not swapped in (L_INMEM == 0)
71 * then the LWP is not on a run queue, but may be soon.
72 *
73 * LSIDL
74 *
75 * Idle: the LWP has been created but has not yet executed,
76 * or it has ceased executing a unit of work and is waiting
77 * to be started again.
78 *
79 * LSSUSPENDED:
80 *
81 * Suspended: the LWP has had its execution suspended by
82 * another LWP in the same process using the _lwp_suspend()
83 * system call. User-level LWPs also enter the suspended
84 * state when the system is shutting down.
85 *
86 * The second set represent a "statement of intent" on behalf of the
87 * LWP. The LWP may in fact be executing on a processor, may be
88 * sleeping or idle. It is expected to take the necessary action to
89 * stop executing or become "running" again within a short timeframe.
90 * The LW_RUNNING flag in lwp::l_flag indicates that an LWP is running.
91 * Importantly, in indicates that its state is tied to a CPU.
92 *
93 * LSZOMB:
94 *
95 * Dead or dying: the LWP has released most of its resources
96 * and is a) about to switch away into oblivion b) has already
97 * switched away. When it switches away, its few remaining
98 * resources can be collected.
99 *
100 * LSSLEEP:
101 *
102 * Sleeping: the LWP has entered itself onto a sleep queue, and
103 * has switched away or will switch away shortly to allow other
104 * LWPs to run on the CPU.
105 *
106 * LSSTOP:
107 *
108 * Stopped: the LWP has been stopped as a result of a job
109 * control signal, or as a result of the ptrace() interface.
110 *
111 * Stopped LWPs may run briefly within the kernel to handle
112 * signals that they receive, but will not return to user space
113 * until their process' state is changed away from stopped.
114 *
115 * Single LWPs within a process can not be set stopped
116 * selectively: all actions that can stop or continue LWPs
117 * occur at the process level.
118 *
119 * State transitions
120 *
121 * Note that the LSSTOP state may only be set when returning to
122 * user space in userret(), or when sleeping interruptably. The
123 * LSSUSPENDED state may only be set in userret(). Before setting
124 * those states, we try to ensure that the LWPs will release all
125 * locks that they hold, and at a minimum try to ensure that the
126 * LWP can be set runnable again by a signal.
127 *
128 * LWPs may transition states in the following ways:
129 *
130 * RUN -------> ONPROC ONPROC -----> RUN
131 * > STOPPED > SLEEP
132 * > SUSPENDED > STOPPED
133 * > SUSPENDED
134 * > ZOMB
135 *
136 * STOPPED ---> RUN SUSPENDED --> RUN
137 * > SLEEP > SLEEP
138 *
139 * SLEEP -----> ONPROC IDL --------> RUN
140 * > RUN > SUSPENDED
141 * > STOPPED > STOPPED
142 * > SUSPENDED
143 *
144 * Other state transitions are possible with kernel threads (eg
145 * ONPROC -> IDL), but only happen under tightly controlled
146 * circumstances the side effects are understood.
147 *
148 * Locking
149 *
150 * The majority of fields in 'struct lwp' are covered by a single,
151 * general spin lock pointed to by lwp::l_mutex. The locks covering
152 * each field are documented in sys/lwp.h.
153 *
154 * State transitions must be made with the LWP's general lock held,
155 * and may cause the LWP's lock pointer to change. Manipulation of
156 * the general lock is not performed directly, but through calls to
157 * lwp_lock(), lwp_relock() and similar.
158 *
159 * States and their associated locks:
160 *
161 * LSONPROC, LSZOMB:
162 *
163 * Always covered by spc_lwplock, which protects running LWPs.
164 * This is a per-CPU lock.
165 *
166 * LSIDL, LSRUN:
167 *
168 * Always covered by spc_mutex, which protects the run queues.
169 * This may be a per-CPU lock, depending on the scheduler.
170 *
171 * LSSLEEP:
172 *
173 * Covered by a lock associated with the sleep queue that the
174 * LWP resides on, indirectly referenced by l_sleepq->sq_mutex.
175 *
176 * LSSTOP, LSSUSPENDED:
177 *
178 * If the LWP was previously sleeping (l_wchan != NULL), then
179 * l_mutex references the sleep queue lock. If the LWP was
180 * runnable or on the CPU when halted, or has been removed from
181 * the sleep queue since halted, then the lock is spc_lwplock.
182 *
183 * The lock order is as follows:
184 *
185 * spc::spc_lwplock ->
186 * sleepq_t::sq_mutex ->
187 * tschain_t::tc_mutex ->
188 * spc::spc_mutex
189 *
190 * Each process has an scheduler state lock (proc::p_smutex), and a
191 * number of counters on LWPs and their states: p_nzlwps, p_nrlwps, and
192 * so on. When an LWP is to be entered into or removed from one of the
193 * following states, p_mutex must be held and the process wide counters
194 * adjusted:
195 *
196 * LSIDL, LSZOMB, LSSTOP, LSSUSPENDED
197 *
198 * Note that an LWP is considered running or likely to run soon if in
199 * one of the following states. This affects the value of p_nrlwps:
200 *
201 * LSRUN, LSONPROC, LSSLEEP
202 *
203 * p_smutex does not need to be held when transitioning among these
204 * three states.
205 */
206
207 #include <sys/cdefs.h>
208 __KERNEL_RCSID(0, "$NetBSD: kern_lwp.c,v 1.89 2008/01/07 11:41:29 yamt Exp $");
209
210 #include "opt_ddb.h"
211 #include "opt_multiprocessor.h"
212 #include "opt_lockdebug.h"
213
214 #define _LWP_API_PRIVATE
215
216 #include <sys/param.h>
217 #include <sys/systm.h>
218 #include <sys/cpu.h>
219 #include <sys/pool.h>
220 #include <sys/proc.h>
221 #include <sys/syscallargs.h>
222 #include <sys/syscall_stats.h>
223 #include <sys/kauth.h>
224 #include <sys/sleepq.h>
225 #include <sys/user.h>
226 #include <sys/lockdebug.h>
227 #include <sys/kmem.h>
228 #include <sys/intr.h>
229 #include <sys/lwpctl.h>
230 #include <sys/atomic.h>
231
232 #include <uvm/uvm_extern.h>
233 #include <uvm/uvm_object.h>
234
235 struct lwplist alllwp = LIST_HEAD_INITIALIZER(alllwp);
236
237 POOL_INIT(lwp_uc_pool, sizeof(ucontext_t), 0, 0, 0, "lwpucpl",
238 &pool_allocator_nointr, IPL_NONE);
239
240 static pool_cache_t lwp_cache;
241 static specificdata_domain_t lwp_specificdata_domain;
242
243 void
244 lwpinit(void)
245 {
246
247 lwp_specificdata_domain = specificdata_domain_create();
248 KASSERT(lwp_specificdata_domain != NULL);
249 lwp_sys_init();
250 lwp_cache = pool_cache_init(sizeof(lwp_t), MIN_LWP_ALIGNMENT, 0, 0,
251 "lwppl", NULL, IPL_NONE, NULL, NULL, NULL);
252 }
253
254 /*
255 * Set an suspended.
256 *
257 * Must be called with p_smutex held, and the LWP locked. Will unlock the
258 * LWP before return.
259 */
260 int
261 lwp_suspend(struct lwp *curl, struct lwp *t)
262 {
263 int error;
264
265 KASSERT(mutex_owned(&t->l_proc->p_smutex));
266 KASSERT(lwp_locked(t, NULL));
267
268 KASSERT(curl != t || curl->l_stat == LSONPROC);
269
270 /*
271 * If the current LWP has been told to exit, we must not suspend anyone
272 * else or deadlock could occur. We won't return to userspace.
273 */
274 if ((curl->l_stat & (LW_WEXIT | LW_WCORE)) != 0) {
275 lwp_unlock(t);
276 return (EDEADLK);
277 }
278
279 error = 0;
280
281 switch (t->l_stat) {
282 case LSRUN:
283 case LSONPROC:
284 t->l_flag |= LW_WSUSPEND;
285 lwp_need_userret(t);
286 lwp_unlock(t);
287 break;
288
289 case LSSLEEP:
290 t->l_flag |= LW_WSUSPEND;
291
292 /*
293 * Kick the LWP and try to get it to the kernel boundary
294 * so that it will release any locks that it holds.
295 * setrunnable() will release the lock.
296 */
297 if ((t->l_flag & LW_SINTR) != 0)
298 setrunnable(t);
299 else
300 lwp_unlock(t);
301 break;
302
303 case LSSUSPENDED:
304 lwp_unlock(t);
305 break;
306
307 case LSSTOP:
308 t->l_flag |= LW_WSUSPEND;
309 setrunnable(t);
310 break;
311
312 case LSIDL:
313 case LSZOMB:
314 error = EINTR; /* It's what Solaris does..... */
315 lwp_unlock(t);
316 break;
317 }
318
319 return (error);
320 }
321
322 /*
323 * Restart a suspended LWP.
324 *
325 * Must be called with p_smutex held, and the LWP locked. Will unlock the
326 * LWP before return.
327 */
328 void
329 lwp_continue(struct lwp *l)
330 {
331
332 KASSERT(mutex_owned(&l->l_proc->p_smutex));
333 KASSERT(lwp_locked(l, NULL));
334
335 /* If rebooting or not suspended, then just bail out. */
336 if ((l->l_flag & LW_WREBOOT) != 0) {
337 lwp_unlock(l);
338 return;
339 }
340
341 l->l_flag &= ~LW_WSUSPEND;
342
343 if (l->l_stat != LSSUSPENDED) {
344 lwp_unlock(l);
345 return;
346 }
347
348 /* setrunnable() will release the lock. */
349 setrunnable(l);
350 }
351
352 /*
353 * Wait for an LWP within the current process to exit. If 'lid' is
354 * non-zero, we are waiting for a specific LWP.
355 *
356 * Must be called with p->p_smutex held.
357 */
358 int
359 lwp_wait1(struct lwp *l, lwpid_t lid, lwpid_t *departed, int flags)
360 {
361 struct proc *p = l->l_proc;
362 struct lwp *l2;
363 int nfound, error;
364 lwpid_t curlid;
365 bool exiting;
366
367 KASSERT(mutex_owned(&p->p_smutex));
368
369 p->p_nlwpwait++;
370 l->l_waitingfor = lid;
371 curlid = l->l_lid;
372 exiting = ((flags & LWPWAIT_EXITCONTROL) != 0);
373
374 for (;;) {
375 /*
376 * Avoid a race between exit1() and sigexit(): if the
377 * process is dumping core, then we need to bail out: call
378 * into lwp_userret() where we will be suspended until the
379 * deed is done.
380 */
381 if ((p->p_sflag & PS_WCORE) != 0) {
382 mutex_exit(&p->p_smutex);
383 lwp_userret(l);
384 #ifdef DIAGNOSTIC
385 panic("lwp_wait1");
386 #endif
387 /* NOTREACHED */
388 }
389
390 /*
391 * First off, drain any detached LWP that is waiting to be
392 * reaped.
393 */
394 while ((l2 = p->p_zomblwp) != NULL) {
395 p->p_zomblwp = NULL;
396 lwp_free(l2, false, false);/* releases proc mutex */
397 mutex_enter(&p->p_smutex);
398 }
399
400 /*
401 * Now look for an LWP to collect. If the whole process is
402 * exiting, count detached LWPs as eligible to be collected,
403 * but don't drain them here.
404 */
405 nfound = 0;
406 error = 0;
407 LIST_FOREACH(l2, &p->p_lwps, l_sibling) {
408 /*
409 * If a specific wait and the target is waiting on
410 * us, then avoid deadlock. This also traps LWPs
411 * that try to wait on themselves.
412 *
413 * Note that this does not handle more complicated
414 * cycles, like: t1 -> t2 -> t3 -> t1. The process
415 * can still be killed so it is not a major problem.
416 */
417 if (l2->l_lid == lid && l2->l_waitingfor == curlid) {
418 error = EDEADLK;
419 break;
420 }
421 if (l2 == l)
422 continue;
423 if ((l2->l_prflag & LPR_DETACHED) != 0) {
424 nfound += exiting;
425 continue;
426 }
427 if (lid != 0) {
428 if (l2->l_lid != lid)
429 continue;
430 /*
431 * Mark this LWP as the first waiter, if there
432 * is no other.
433 */
434 if (l2->l_waiter == 0)
435 l2->l_waiter = curlid;
436 } else if (l2->l_waiter != 0) {
437 /*
438 * It already has a waiter - so don't
439 * collect it. If the waiter doesn't
440 * grab it we'll get another chance
441 * later.
442 */
443 nfound++;
444 continue;
445 }
446 nfound++;
447
448 /* No need to lock the LWP in order to see LSZOMB. */
449 if (l2->l_stat != LSZOMB)
450 continue;
451
452 /*
453 * We're no longer waiting. Reset the "first waiter"
454 * pointer on the target, in case it was us.
455 */
456 l->l_waitingfor = 0;
457 l2->l_waiter = 0;
458 p->p_nlwpwait--;
459 if (departed)
460 *departed = l2->l_lid;
461 sched_lwp_collect(l2);
462
463 /* lwp_free() releases the proc lock. */
464 lwp_free(l2, false, false);
465 mutex_enter(&p->p_smutex);
466 return 0;
467 }
468
469 if (error != 0)
470 break;
471 if (nfound == 0) {
472 error = ESRCH;
473 break;
474 }
475
476 /*
477 * The kernel is careful to ensure that it can not deadlock
478 * when exiting - just keep waiting.
479 */
480 if (exiting) {
481 KASSERT(p->p_nlwps > 1);
482 cv_wait(&p->p_lwpcv, &p->p_smutex);
483 continue;
484 }
485
486 /*
487 * If all other LWPs are waiting for exits or suspends
488 * and the supply of zombies and potential zombies is
489 * exhausted, then we are about to deadlock.
490 *
491 * If the process is exiting (and this LWP is not the one
492 * that is coordinating the exit) then bail out now.
493 */
494 if ((p->p_sflag & PS_WEXIT) != 0 ||
495 p->p_nrlwps + p->p_nzlwps - p->p_ndlwps <= p->p_nlwpwait) {
496 error = EDEADLK;
497 break;
498 }
499
500 /*
501 * Sit around and wait for something to happen. We'll be
502 * awoken if any of the conditions examined change: if an
503 * LWP exits, is collected, or is detached.
504 */
505 if ((error = cv_wait_sig(&p->p_lwpcv, &p->p_smutex)) != 0)
506 break;
507 }
508
509 /*
510 * We didn't find any LWPs to collect, we may have received a
511 * signal, or some other condition has caused us to bail out.
512 *
513 * If waiting on a specific LWP, clear the waiters marker: some
514 * other LWP may want it. Then, kick all the remaining waiters
515 * so that they can re-check for zombies and for deadlock.
516 */
517 if (lid != 0) {
518 LIST_FOREACH(l2, &p->p_lwps, l_sibling) {
519 if (l2->l_lid == lid) {
520 if (l2->l_waiter == curlid)
521 l2->l_waiter = 0;
522 break;
523 }
524 }
525 }
526 p->p_nlwpwait--;
527 l->l_waitingfor = 0;
528 cv_broadcast(&p->p_lwpcv);
529
530 return error;
531 }
532
533 /*
534 * Create a new LWP within process 'p2', using LWP 'l1' as a template.
535 * The new LWP is created in state LSIDL and must be set running,
536 * suspended, or stopped by the caller.
537 */
538 int
539 lwp_create(lwp_t *l1, proc_t *p2, vaddr_t uaddr, bool inmem, int flags,
540 void *stack, size_t stacksize, void (*func)(void *), void *arg,
541 lwp_t **rnewlwpp, int sclass)
542 {
543 struct lwp *l2, *isfree;
544 turnstile_t *ts;
545
546 /*
547 * First off, reap any detached LWP waiting to be collected.
548 * We can re-use its LWP structure and turnstile.
549 */
550 isfree = NULL;
551 if (p2->p_zomblwp != NULL) {
552 mutex_enter(&p2->p_smutex);
553 if ((isfree = p2->p_zomblwp) != NULL) {
554 p2->p_zomblwp = NULL;
555 lwp_free(isfree, true, false);/* releases proc mutex */
556 } else
557 mutex_exit(&p2->p_smutex);
558 }
559 if (isfree == NULL) {
560 l2 = pool_cache_get(lwp_cache, PR_WAITOK);
561 memset(l2, 0, sizeof(*l2));
562 l2->l_ts = pool_cache_get(turnstile_cache, PR_WAITOK);
563 SLIST_INIT(&l2->l_pi_lenders);
564 } else {
565 l2 = isfree;
566 ts = l2->l_ts;
567 KASSERT(l2->l_inheritedprio == -1);
568 KASSERT(SLIST_EMPTY(&l2->l_pi_lenders));
569 memset(l2, 0, sizeof(*l2));
570 l2->l_ts = ts;
571 }
572
573 l2->l_stat = LSIDL;
574 l2->l_proc = p2;
575 l2->l_refcnt = 1;
576 l2->l_class = sclass;
577 l2->l_kpriority = l1->l_kpriority;
578 l2->l_kpribase = PRI_KERNEL;
579 l2->l_priority = l1->l_priority;
580 l2->l_inheritedprio = -1;
581 l2->l_mutex = l1->l_cpu->ci_schedstate.spc_mutex;
582 l2->l_cpu = l1->l_cpu;
583 l2->l_flag = inmem ? LW_INMEM : 0;
584 l2->l_pflag = LP_MPSAFE;
585
586 if (p2->p_flag & PK_SYSTEM) {
587 /*
588 * Mark it as a system process and not a candidate for
589 * swapping.
590 */
591 l2->l_flag |= LW_SYSTEM;
592 } else {
593 /* Look for a CPU to start */
594 l2->l_cpu = sched_takecpu(l2);
595 l2->l_mutex = l2->l_cpu->ci_schedstate.spc_mutex;
596 }
597
598 lwp_initspecific(l2);
599 sched_lwp_fork(l1, l2);
600 lwp_update_creds(l2);
601 callout_init(&l2->l_timeout_ch, CALLOUT_MPSAFE);
602 callout_setfunc(&l2->l_timeout_ch, sleepq_timeout, l2);
603 mutex_init(&l2->l_swaplock, MUTEX_DEFAULT, IPL_NONE);
604 cv_init(&l2->l_sigcv, "sigwait");
605 l2->l_syncobj = &sched_syncobj;
606
607 if (rnewlwpp != NULL)
608 *rnewlwpp = l2;
609
610 l2->l_addr = UAREA_TO_USER(uaddr);
611 uvm_lwp_fork(l1, l2, stack, stacksize, func,
612 (arg != NULL) ? arg : l2);
613
614 mutex_enter(&p2->p_smutex);
615
616 if ((flags & LWP_DETACHED) != 0) {
617 l2->l_prflag = LPR_DETACHED;
618 p2->p_ndlwps++;
619 } else
620 l2->l_prflag = 0;
621
622 l2->l_sigmask = l1->l_sigmask;
623 CIRCLEQ_INIT(&l2->l_sigpend.sp_info);
624 sigemptyset(&l2->l_sigpend.sp_set);
625
626 p2->p_nlwpid++;
627 if (p2->p_nlwpid == 0)
628 p2->p_nlwpid++;
629 l2->l_lid = p2->p_nlwpid;
630 LIST_INSERT_HEAD(&p2->p_lwps, l2, l_sibling);
631 p2->p_nlwps++;
632
633 mutex_exit(&p2->p_smutex);
634
635 mutex_enter(&proclist_lock);
636 LIST_INSERT_HEAD(&alllwp, l2, l_list);
637 mutex_exit(&proclist_lock);
638
639 SYSCALL_TIME_LWP_INIT(l2);
640
641 if (p2->p_emul->e_lwp_fork)
642 (*p2->p_emul->e_lwp_fork)(l1, l2);
643
644 return (0);
645 }
646
647 /*
648 * Called by MD code when a new LWP begins execution. Must be called
649 * with the previous LWP locked (so at splsched), or if there is no
650 * previous LWP, at splsched.
651 */
652 void
653 lwp_startup(struct lwp *prev, struct lwp *new)
654 {
655
656 if (prev != NULL) {
657 /*
658 * Normalize the count of the spin-mutexes, it was
659 * increased in mi_switch(). Unmark the state of
660 * context switch - it is finished for previous LWP.
661 */
662 curcpu()->ci_mtx_count++;
663 membar_exit();
664 prev->l_ctxswtch = 0;
665 }
666 spl0();
667 pmap_activate(new);
668 LOCKDEBUG_BARRIER(NULL, 0);
669 if ((new->l_pflag & LP_MPSAFE) == 0) {
670 KERNEL_LOCK(1, new);
671 }
672 }
673
674 /*
675 * Exit an LWP.
676 */
677 void
678 lwp_exit(struct lwp *l)
679 {
680 struct proc *p = l->l_proc;
681 struct lwp *l2;
682 bool current;
683
684 current = (l == curlwp);
685
686 KASSERT(current || l->l_stat == LSIDL);
687
688 /*
689 * Verify that we hold no locks other than the kernel lock.
690 */
691 #ifdef MULTIPROCESSOR
692 LOCKDEBUG_BARRIER(&kernel_lock, 0);
693 #else
694 LOCKDEBUG_BARRIER(NULL, 0);
695 #endif
696
697 /*
698 * If we are the last live LWP in a process, we need to exit the
699 * entire process. We do so with an exit status of zero, because
700 * it's a "controlled" exit, and because that's what Solaris does.
701 *
702 * We are not quite a zombie yet, but for accounting purposes we
703 * must increment the count of zombies here.
704 *
705 * Note: the last LWP's specificdata will be deleted here.
706 */
707 mutex_enter(&p->p_smutex);
708 if (p->p_nlwps - p->p_nzlwps == 1) {
709 KASSERT(current == true);
710 /* XXXSMP kernel_lock not held */
711 exit1(l, 0);
712 /* NOTREACHED */
713 }
714 p->p_nzlwps++;
715 mutex_exit(&p->p_smutex);
716
717 if (p->p_emul->e_lwp_exit)
718 (*p->p_emul->e_lwp_exit)(l);
719
720 /* Delete the specificdata while it's still safe to sleep. */
721 specificdata_fini(lwp_specificdata_domain, &l->l_specdataref);
722
723 /*
724 * Release our cached credentials.
725 */
726 kauth_cred_free(l->l_cred);
727 callout_destroy(&l->l_timeout_ch);
728
729 /*
730 * While we can still block, mark the LWP as unswappable to
731 * prevent conflicts with the with the swapper.
732 */
733 if (current)
734 uvm_lwp_hold(l);
735
736 /*
737 * Remove the LWP from the global list.
738 */
739 mutex_enter(&proclist_lock);
740 mutex_enter(&proclist_mutex);
741 LIST_REMOVE(l, l_list);
742 mutex_exit(&proclist_mutex);
743 mutex_exit(&proclist_lock);
744
745 /*
746 * Get rid of all references to the LWP that others (e.g. procfs)
747 * may have, and mark the LWP as a zombie. If the LWP is detached,
748 * mark it waiting for collection in the proc structure. Note that
749 * before we can do that, we need to free any other dead, deatched
750 * LWP waiting to meet its maker.
751 *
752 * XXXSMP disable preemption.
753 */
754 mutex_enter(&p->p_smutex);
755 lwp_drainrefs(l);
756
757 if ((l->l_prflag & LPR_DETACHED) != 0) {
758 while ((l2 = p->p_zomblwp) != NULL) {
759 p->p_zomblwp = NULL;
760 lwp_free(l2, false, false);/* releases proc mutex */
761 mutex_enter(&p->p_smutex);
762 l->l_refcnt++;
763 lwp_drainrefs(l);
764 }
765 p->p_zomblwp = l;
766 }
767
768 /*
769 * If we find a pending signal for the process and we have been
770 * asked to check for signals, then we loose: arrange to have
771 * all other LWPs in the process check for signals.
772 */
773 if ((l->l_flag & LW_PENDSIG) != 0 &&
774 firstsig(&p->p_sigpend.sp_set) != 0) {
775 LIST_FOREACH(l2, &p->p_lwps, l_sibling) {
776 lwp_lock(l2);
777 l2->l_flag |= LW_PENDSIG;
778 lwp_unlock(l2);
779 }
780 }
781
782 lwp_lock(l);
783 l->l_stat = LSZOMB;
784 lwp_unlock(l);
785 p->p_nrlwps--;
786 cv_broadcast(&p->p_lwpcv);
787 if (l->l_lwpctl != NULL)
788 l->l_lwpctl->lc_curcpu = LWPCTL_CPU_EXITED;
789 mutex_exit(&p->p_smutex);
790
791 /*
792 * We can no longer block. At this point, lwp_free() may already
793 * be gunning for us. On a multi-CPU system, we may be off p_lwps.
794 *
795 * Free MD LWP resources.
796 */
797 #ifndef __NO_CPU_LWP_FREE
798 cpu_lwp_free(l, 0);
799 #endif
800
801 if (current) {
802 pmap_deactivate(l);
803
804 /*
805 * Release the kernel lock, and switch away into
806 * oblivion.
807 */
808 #ifdef notyet
809 /* XXXSMP hold in lwp_userret() */
810 KERNEL_UNLOCK_LAST(l);
811 #else
812 KERNEL_UNLOCK_ALL(l, NULL);
813 #endif
814 lwp_exit_switchaway(l);
815 }
816 }
817
818 void
819 lwp_exit_switchaway(struct lwp *l)
820 {
821 struct cpu_info *ci;
822 struct lwp *idlelwp;
823
824 /* Unlocked, but is for statistics only. */
825 uvmexp.swtch++;
826
827 (void)splsched();
828 l->l_flag &= ~LW_RUNNING;
829 ci = curcpu();
830 idlelwp = ci->ci_data.cpu_idlelwp;
831 idlelwp->l_stat = LSONPROC;
832
833 /*
834 * cpu_onproc must be updated with the CPU locked, as
835 * aston() may try to set a AST pending on the LWP (and
836 * it does so with the CPU locked). Otherwise, the LWP
837 * may be destroyed before the AST can be set, leading
838 * to a user-after-free.
839 */
840 spc_lock(ci);
841 ci->ci_data.cpu_onproc = idlelwp;
842 spc_unlock(ci);
843 cpu_switchto(NULL, idlelwp, false);
844 }
845
846 /*
847 * Free a dead LWP's remaining resources.
848 *
849 * XXXLWP limits.
850 */
851 void
852 lwp_free(struct lwp *l, bool recycle, bool last)
853 {
854 struct proc *p = l->l_proc;
855 ksiginfoq_t kq;
856
857 /*
858 * If this was not the last LWP in the process, then adjust
859 * counters and unlock.
860 */
861 if (!last) {
862 /*
863 * Add the LWP's run time to the process' base value.
864 * This needs to co-incide with coming off p_lwps.
865 */
866 bintime_add(&p->p_rtime, &l->l_rtime);
867 p->p_pctcpu += l->l_pctcpu;
868 LIST_REMOVE(l, l_sibling);
869 p->p_nlwps--;
870 p->p_nzlwps--;
871 if ((l->l_prflag & LPR_DETACHED) != 0)
872 p->p_ndlwps--;
873
874 /*
875 * Have any LWPs sleeping in lwp_wait() recheck for
876 * deadlock.
877 */
878 cv_broadcast(&p->p_lwpcv);
879 mutex_exit(&p->p_smutex);
880 }
881
882 #ifdef MULTIPROCESSOR
883 /*
884 * In the unlikely event that the LWP is still on the CPU,
885 * then spin until it has switched away. We need to release
886 * all locks to avoid deadlock against interrupt handlers on
887 * the target CPU.
888 */
889 if ((l->l_flag & LW_RUNNING) != 0 || l->l_cpu->ci_curlwp == l) {
890 int count;
891 (void)count; /* XXXgcc */
892 KERNEL_UNLOCK_ALL(curlwp, &count);
893 while ((l->l_flag & LW_RUNNING) != 0 ||
894 l->l_cpu->ci_curlwp == l)
895 SPINLOCK_BACKOFF_HOOK;
896 KERNEL_LOCK(count, curlwp);
897 }
898 #endif
899
900 /*
901 * Destroy the LWP's remaining signal information.
902 */
903 ksiginfo_queue_init(&kq);
904 sigclear(&l->l_sigpend, NULL, &kq);
905 ksiginfo_queue_drain(&kq);
906 cv_destroy(&l->l_sigcv);
907 mutex_destroy(&l->l_swaplock);
908
909 /*
910 * Free the LWP's turnstile and the LWP structure itself unless the
911 * caller wants to recycle them. Also, free the scheduler specific data.
912 *
913 * We can't return turnstile0 to the pool (it didn't come from it),
914 * so if it comes up just drop it quietly and move on.
915 *
916 * We don't recycle the VM resources at this time.
917 */
918 if (l->l_lwpctl != NULL)
919 lwp_ctl_free(l);
920 sched_lwp_exit(l);
921
922 if (!recycle && l->l_ts != &turnstile0)
923 pool_cache_put(turnstile_cache, l->l_ts);
924 #ifndef __NO_CPU_LWP_FREE
925 cpu_lwp_free2(l);
926 #endif
927 uvm_lwp_exit(l);
928 KASSERT(SLIST_EMPTY(&l->l_pi_lenders));
929 KASSERT(l->l_inheritedprio == -1);
930 if (!recycle)
931 pool_cache_put(lwp_cache, l);
932 }
933
934 /*
935 * Pick a LWP to represent the process for those operations which
936 * want information about a "process" that is actually associated
937 * with a LWP.
938 *
939 * If 'locking' is false, no locking or lock checks are performed.
940 * This is intended for use by DDB.
941 *
942 * We don't bother locking the LWP here, since code that uses this
943 * interface is broken by design and an exact match is not required.
944 */
945 struct lwp *
946 proc_representative_lwp(struct proc *p, int *nrlwps, int locking)
947 {
948 struct lwp *l, *onproc, *running, *sleeping, *stopped, *suspended;
949 struct lwp *signalled;
950 int cnt;
951
952 if (locking) {
953 KASSERT(mutex_owned(&p->p_smutex));
954 }
955
956 /* Trivial case: only one LWP */
957 if (p->p_nlwps == 1) {
958 l = LIST_FIRST(&p->p_lwps);
959 if (nrlwps)
960 *nrlwps = (l->l_stat == LSONPROC || l->l_stat == LSRUN);
961 return l;
962 }
963
964 cnt = 0;
965 switch (p->p_stat) {
966 case SSTOP:
967 case SACTIVE:
968 /* Pick the most live LWP */
969 onproc = running = sleeping = stopped = suspended = NULL;
970 signalled = NULL;
971 LIST_FOREACH(l, &p->p_lwps, l_sibling) {
972 if ((l->l_flag & LW_IDLE) != 0) {
973 continue;
974 }
975 if (l->l_lid == p->p_sigctx.ps_lwp)
976 signalled = l;
977 switch (l->l_stat) {
978 case LSONPROC:
979 onproc = l;
980 cnt++;
981 break;
982 case LSRUN:
983 running = l;
984 cnt++;
985 break;
986 case LSSLEEP:
987 sleeping = l;
988 break;
989 case LSSTOP:
990 stopped = l;
991 break;
992 case LSSUSPENDED:
993 suspended = l;
994 break;
995 }
996 }
997 if (nrlwps)
998 *nrlwps = cnt;
999 if (signalled)
1000 l = signalled;
1001 else if (onproc)
1002 l = onproc;
1003 else if (running)
1004 l = running;
1005 else if (sleeping)
1006 l = sleeping;
1007 else if (stopped)
1008 l = stopped;
1009 else if (suspended)
1010 l = suspended;
1011 else
1012 break;
1013 return l;
1014 #ifdef DIAGNOSTIC
1015 case SIDL:
1016 case SZOMB:
1017 case SDYING:
1018 case SDEAD:
1019 if (locking)
1020 mutex_exit(&p->p_smutex);
1021 /* We have more than one LWP and we're in SIDL?
1022 * How'd that happen?
1023 */
1024 panic("Too many LWPs in idle/dying process %d (%s) stat = %d",
1025 p->p_pid, p->p_comm, p->p_stat);
1026 break;
1027 default:
1028 if (locking)
1029 mutex_exit(&p->p_smutex);
1030 panic("Process %d (%s) in unknown state %d",
1031 p->p_pid, p->p_comm, p->p_stat);
1032 #endif
1033 }
1034
1035 if (locking)
1036 mutex_exit(&p->p_smutex);
1037 panic("proc_representative_lwp: couldn't find a lwp for process"
1038 " %d (%s)", p->p_pid, p->p_comm);
1039 /* NOTREACHED */
1040 return NULL;
1041 }
1042
1043 /*
1044 * Look up a live LWP within the speicifed process, and return it locked.
1045 *
1046 * Must be called with p->p_smutex held.
1047 */
1048 struct lwp *
1049 lwp_find(struct proc *p, int id)
1050 {
1051 struct lwp *l;
1052
1053 KASSERT(mutex_owned(&p->p_smutex));
1054
1055 LIST_FOREACH(l, &p->p_lwps, l_sibling) {
1056 if (l->l_lid == id)
1057 break;
1058 }
1059
1060 /*
1061 * No need to lock - all of these conditions will
1062 * be visible with the process level mutex held.
1063 */
1064 if (l != NULL && (l->l_stat == LSIDL || l->l_stat == LSZOMB))
1065 l = NULL;
1066
1067 return l;
1068 }
1069
1070 /*
1071 * Update an LWP's cached credentials to mirror the process' master copy.
1072 *
1073 * This happens early in the syscall path, on user trap, and on LWP
1074 * creation. A long-running LWP can also voluntarily choose to update
1075 * it's credentials by calling this routine. This may be called from
1076 * LWP_CACHE_CREDS(), which checks l->l_cred != p->p_cred beforehand.
1077 */
1078 void
1079 lwp_update_creds(struct lwp *l)
1080 {
1081 kauth_cred_t oc;
1082 struct proc *p;
1083
1084 p = l->l_proc;
1085 oc = l->l_cred;
1086
1087 mutex_enter(&p->p_mutex);
1088 kauth_cred_hold(p->p_cred);
1089 l->l_cred = p->p_cred;
1090 mutex_exit(&p->p_mutex);
1091 if (oc != NULL)
1092 kauth_cred_free(oc);
1093 }
1094
1095 /*
1096 * Verify that an LWP is locked, and optionally verify that the lock matches
1097 * one we specify.
1098 */
1099 int
1100 lwp_locked(struct lwp *l, kmutex_t *mtx)
1101 {
1102 kmutex_t *cur = l->l_mutex;
1103
1104 return mutex_owned(cur) && (mtx == cur || mtx == NULL);
1105 }
1106
1107 /*
1108 * Lock an LWP.
1109 */
1110 void
1111 lwp_lock_retry(struct lwp *l, kmutex_t *old)
1112 {
1113
1114 /*
1115 * XXXgcc ignoring kmutex_t * volatile on i386
1116 *
1117 * gcc version 4.1.2 20061021 prerelease (NetBSD nb1 20061021)
1118 */
1119 #if 1
1120 while (l->l_mutex != old) {
1121 #else
1122 for (;;) {
1123 #endif
1124 mutex_spin_exit(old);
1125 old = l->l_mutex;
1126 mutex_spin_enter(old);
1127
1128 /*
1129 * mutex_enter() will have posted a read barrier. Re-test
1130 * l->l_mutex. If it has changed, we need to try again.
1131 */
1132 #if 1
1133 }
1134 #else
1135 } while (__predict_false(l->l_mutex != old));
1136 #endif
1137 }
1138
1139 /*
1140 * Lend a new mutex to an LWP. The old mutex must be held.
1141 */
1142 void
1143 lwp_setlock(struct lwp *l, kmutex_t *new)
1144 {
1145
1146 KASSERT(mutex_owned(l->l_mutex));
1147
1148 membar_producer();
1149 l->l_mutex = new;
1150 }
1151
1152 /*
1153 * Lend a new mutex to an LWP, and release the old mutex. The old mutex
1154 * must be held.
1155 */
1156 void
1157 lwp_unlock_to(struct lwp *l, kmutex_t *new)
1158 {
1159 kmutex_t *old;
1160
1161 KASSERT(mutex_owned(l->l_mutex));
1162
1163 old = l->l_mutex;
1164 membar_producer();
1165 l->l_mutex = new;
1166 mutex_spin_exit(old);
1167 }
1168
1169 /*
1170 * Acquire a new mutex, and donate it to an LWP. The LWP must already be
1171 * locked.
1172 */
1173 void
1174 lwp_relock(struct lwp *l, kmutex_t *new)
1175 {
1176 kmutex_t *old;
1177
1178 KASSERT(mutex_owned(l->l_mutex));
1179
1180 old = l->l_mutex;
1181 if (old != new) {
1182 mutex_spin_enter(new);
1183 l->l_mutex = new;
1184 mutex_spin_exit(old);
1185 }
1186 }
1187
1188 int
1189 lwp_trylock(struct lwp *l)
1190 {
1191 kmutex_t *old;
1192
1193 for (;;) {
1194 if (!mutex_tryenter(old = l->l_mutex))
1195 return 0;
1196 if (__predict_true(l->l_mutex == old))
1197 return 1;
1198 mutex_spin_exit(old);
1199 }
1200 }
1201
1202 /*
1203 * Handle exceptions for mi_userret(). Called if a member of LW_USERRET is
1204 * set.
1205 */
1206 void
1207 lwp_userret(struct lwp *l)
1208 {
1209 struct proc *p;
1210 void (*hook)(void);
1211 int sig;
1212
1213 p = l->l_proc;
1214
1215 #ifndef __HAVE_FAST_SOFTINTS
1216 /* Run pending soft interrupts. */
1217 if (l->l_cpu->ci_data.cpu_softints != 0)
1218 softint_overlay();
1219 #endif
1220
1221 /*
1222 * It should be safe to do this read unlocked on a multiprocessor
1223 * system..
1224 */
1225 while ((l->l_flag & LW_USERRET) != 0) {
1226 /*
1227 * Process pending signals first, unless the process
1228 * is dumping core or exiting, where we will instead
1229 * enter the L_WSUSPEND case below.
1230 */
1231 if ((l->l_flag & (LW_PENDSIG | LW_WCORE | LW_WEXIT)) ==
1232 LW_PENDSIG) {
1233 mutex_enter(&p->p_smutex);
1234 while ((sig = issignal(l)) != 0)
1235 postsig(sig);
1236 mutex_exit(&p->p_smutex);
1237 }
1238
1239 /*
1240 * Core-dump or suspend pending.
1241 *
1242 * In case of core dump, suspend ourselves, so that the
1243 * kernel stack and therefore the userland registers saved
1244 * in the trapframe are around for coredump() to write them
1245 * out. We issue a wakeup on p->p_lwpcv so that sigexit()
1246 * will write the core file out once all other LWPs are
1247 * suspended.
1248 */
1249 if ((l->l_flag & LW_WSUSPEND) != 0) {
1250 mutex_enter(&p->p_smutex);
1251 p->p_nrlwps--;
1252 cv_broadcast(&p->p_lwpcv);
1253 lwp_lock(l);
1254 l->l_stat = LSSUSPENDED;
1255 mutex_exit(&p->p_smutex);
1256 mi_switch(l);
1257 }
1258
1259 /* Process is exiting. */
1260 if ((l->l_flag & LW_WEXIT) != 0) {
1261 lwp_exit(l);
1262 KASSERT(0);
1263 /* NOTREACHED */
1264 }
1265
1266 /* Call userret hook; used by Linux emulation. */
1267 if ((l->l_flag & LW_WUSERRET) != 0) {
1268 lwp_lock(l);
1269 l->l_flag &= ~LW_WUSERRET;
1270 lwp_unlock(l);
1271 hook = p->p_userret;
1272 p->p_userret = NULL;
1273 (*hook)();
1274 }
1275 }
1276 }
1277
1278 /*
1279 * Force an LWP to enter the kernel, to take a trip through lwp_userret().
1280 */
1281 void
1282 lwp_need_userret(struct lwp *l)
1283 {
1284 KASSERT(lwp_locked(l, NULL));
1285
1286 /*
1287 * Since the tests in lwp_userret() are done unlocked, make sure
1288 * that the condition will be seen before forcing the LWP to enter
1289 * kernel mode.
1290 */
1291 membar_producer();
1292 cpu_signotify(l);
1293 }
1294
1295 /*
1296 * Add one reference to an LWP. This will prevent the LWP from
1297 * exiting, thus keep the lwp structure and PCB around to inspect.
1298 */
1299 void
1300 lwp_addref(struct lwp *l)
1301 {
1302
1303 KASSERT(mutex_owned(&l->l_proc->p_smutex));
1304 KASSERT(l->l_stat != LSZOMB);
1305 KASSERT(l->l_refcnt != 0);
1306
1307 l->l_refcnt++;
1308 }
1309
1310 /*
1311 * Remove one reference to an LWP. If this is the last reference,
1312 * then we must finalize the LWP's death.
1313 */
1314 void
1315 lwp_delref(struct lwp *l)
1316 {
1317 struct proc *p = l->l_proc;
1318
1319 mutex_enter(&p->p_smutex);
1320 KASSERT(l->l_stat != LSZOMB);
1321 KASSERT(l->l_refcnt > 0);
1322 if (--l->l_refcnt == 0)
1323 cv_broadcast(&p->p_lwpcv);
1324 mutex_exit(&p->p_smutex);
1325 }
1326
1327 /*
1328 * Drain all references to the current LWP.
1329 */
1330 void
1331 lwp_drainrefs(struct lwp *l)
1332 {
1333 struct proc *p = l->l_proc;
1334
1335 KASSERT(mutex_owned(&p->p_smutex));
1336 KASSERT(l->l_refcnt != 0);
1337
1338 l->l_refcnt--;
1339 while (l->l_refcnt != 0)
1340 cv_wait(&p->p_lwpcv, &p->p_smutex);
1341 }
1342
1343 /*
1344 * lwp_specific_key_create --
1345 * Create a key for subsystem lwp-specific data.
1346 */
1347 int
1348 lwp_specific_key_create(specificdata_key_t *keyp, specificdata_dtor_t dtor)
1349 {
1350
1351 return (specificdata_key_create(lwp_specificdata_domain, keyp, dtor));
1352 }
1353
1354 /*
1355 * lwp_specific_key_delete --
1356 * Delete a key for subsystem lwp-specific data.
1357 */
1358 void
1359 lwp_specific_key_delete(specificdata_key_t key)
1360 {
1361
1362 specificdata_key_delete(lwp_specificdata_domain, key);
1363 }
1364
1365 /*
1366 * lwp_initspecific --
1367 * Initialize an LWP's specificdata container.
1368 */
1369 void
1370 lwp_initspecific(struct lwp *l)
1371 {
1372 int error;
1373
1374 error = specificdata_init(lwp_specificdata_domain, &l->l_specdataref);
1375 KASSERT(error == 0);
1376 }
1377
1378 /*
1379 * lwp_finispecific --
1380 * Finalize an LWP's specificdata container.
1381 */
1382 void
1383 lwp_finispecific(struct lwp *l)
1384 {
1385
1386 specificdata_fini(lwp_specificdata_domain, &l->l_specdataref);
1387 }
1388
1389 /*
1390 * lwp_getspecific --
1391 * Return lwp-specific data corresponding to the specified key.
1392 *
1393 * Note: LWP specific data is NOT INTERLOCKED. An LWP should access
1394 * only its OWN SPECIFIC DATA. If it is necessary to access another
1395 * LWP's specifc data, care must be taken to ensure that doing so
1396 * would not cause internal data structure inconsistency (i.e. caller
1397 * can guarantee that the target LWP is not inside an lwp_getspecific()
1398 * or lwp_setspecific() call).
1399 */
1400 void *
1401 lwp_getspecific(specificdata_key_t key)
1402 {
1403
1404 return (specificdata_getspecific_unlocked(lwp_specificdata_domain,
1405 &curlwp->l_specdataref, key));
1406 }
1407
1408 void *
1409 _lwp_getspecific_by_lwp(struct lwp *l, specificdata_key_t key)
1410 {
1411
1412 return (specificdata_getspecific_unlocked(lwp_specificdata_domain,
1413 &l->l_specdataref, key));
1414 }
1415
1416 /*
1417 * lwp_setspecific --
1418 * Set lwp-specific data corresponding to the specified key.
1419 */
1420 void
1421 lwp_setspecific(specificdata_key_t key, void *data)
1422 {
1423
1424 specificdata_setspecific(lwp_specificdata_domain,
1425 &curlwp->l_specdataref, key, data);
1426 }
1427
1428 /*
1429 * Allocate a new lwpctl structure for a user LWP.
1430 */
1431 int
1432 lwp_ctl_alloc(vaddr_t *uaddr)
1433 {
1434 lcproc_t *lp;
1435 u_int bit, i, offset;
1436 struct uvm_object *uao;
1437 int error;
1438 lcpage_t *lcp;
1439 proc_t *p;
1440 lwp_t *l;
1441
1442 l = curlwp;
1443 p = l->l_proc;
1444
1445 if (l->l_lcpage != NULL) {
1446 lcp = l->l_lcpage;
1447 *uaddr = lcp->lcp_uaddr + (vaddr_t)l->l_lwpctl - lcp->lcp_kaddr;
1448 return (EINVAL);
1449 }
1450
1451 /* First time around, allocate header structure for the process. */
1452 if ((lp = p->p_lwpctl) == NULL) {
1453 lp = kmem_alloc(sizeof(*lp), KM_SLEEP);
1454 mutex_init(&lp->lp_lock, MUTEX_DEFAULT, IPL_NONE);
1455 lp->lp_uao = NULL;
1456 TAILQ_INIT(&lp->lp_pages);
1457 mutex_enter(&p->p_mutex);
1458 if (p->p_lwpctl == NULL) {
1459 p->p_lwpctl = lp;
1460 mutex_exit(&p->p_mutex);
1461 } else {
1462 mutex_exit(&p->p_mutex);
1463 mutex_destroy(&lp->lp_lock);
1464 kmem_free(lp, sizeof(*lp));
1465 lp = p->p_lwpctl;
1466 }
1467 }
1468
1469 /*
1470 * Set up an anonymous memory region to hold the shared pages.
1471 * Map them into the process' address space. The user vmspace
1472 * gets the first reference on the UAO.
1473 */
1474 mutex_enter(&lp->lp_lock);
1475 if (lp->lp_uao == NULL) {
1476 lp->lp_uao = uao_create(LWPCTL_UAREA_SZ, 0);
1477 lp->lp_cur = 0;
1478 lp->lp_max = LWPCTL_UAREA_SZ;
1479 lp->lp_uva = p->p_emul->e_vm_default_addr(p,
1480 (vaddr_t)p->p_vmspace->vm_daddr, LWPCTL_UAREA_SZ);
1481 error = uvm_map(&p->p_vmspace->vm_map, &lp->lp_uva,
1482 LWPCTL_UAREA_SZ, lp->lp_uao, 0, 0, UVM_MAPFLAG(UVM_PROT_RW,
1483 UVM_PROT_RW, UVM_INH_NONE, UVM_ADV_NORMAL, 0));
1484 if (error != 0) {
1485 uao_detach(lp->lp_uao);
1486 lp->lp_uao = NULL;
1487 mutex_exit(&lp->lp_lock);
1488 return error;
1489 }
1490 }
1491
1492 /* Get a free block and allocate for this LWP. */
1493 TAILQ_FOREACH(lcp, &lp->lp_pages, lcp_chain) {
1494 if (lcp->lcp_nfree != 0)
1495 break;
1496 }
1497 if (lcp == NULL) {
1498 /* Nothing available - try to set up a free page. */
1499 if (lp->lp_cur == lp->lp_max) {
1500 mutex_exit(&lp->lp_lock);
1501 return ENOMEM;
1502 }
1503 lcp = kmem_alloc(LWPCTL_LCPAGE_SZ, KM_SLEEP);
1504 if (lcp == NULL) {
1505 mutex_exit(&lp->lp_lock);
1506 return ENOMEM;
1507 }
1508 /*
1509 * Wire the next page down in kernel space. Since this
1510 * is a new mapping, we must add a reference.
1511 */
1512 uao = lp->lp_uao;
1513 (*uao->pgops->pgo_reference)(uao);
1514 error = uvm_map(kernel_map, &lcp->lcp_kaddr, PAGE_SIZE,
1515 uao, lp->lp_cur, PAGE_SIZE,
1516 UVM_MAPFLAG(UVM_PROT_RW, UVM_PROT_RW,
1517 UVM_INH_NONE, UVM_ADV_RANDOM, 0));
1518 if (error != 0) {
1519 mutex_exit(&lp->lp_lock);
1520 kmem_free(lcp, LWPCTL_LCPAGE_SZ);
1521 (*uao->pgops->pgo_detach)(uao);
1522 return error;
1523 }
1524 error = uvm_map_pageable(kernel_map, lcp->lcp_kaddr,
1525 lcp->lcp_kaddr + PAGE_SIZE, FALSE, 0);
1526 if (error != 0) {
1527 mutex_exit(&lp->lp_lock);
1528 uvm_unmap(kernel_map, lcp->lcp_kaddr,
1529 lcp->lcp_kaddr + PAGE_SIZE);
1530 kmem_free(lcp, LWPCTL_LCPAGE_SZ);
1531 return error;
1532 }
1533 /* Prepare the page descriptor and link into the list. */
1534 lcp->lcp_uaddr = lp->lp_uva + lp->lp_cur;
1535 lp->lp_cur += PAGE_SIZE;
1536 lcp->lcp_nfree = LWPCTL_PER_PAGE;
1537 lcp->lcp_rotor = 0;
1538 memset(lcp->lcp_bitmap, 0xff, LWPCTL_BITMAP_SZ);
1539 TAILQ_INSERT_HEAD(&lp->lp_pages, lcp, lcp_chain);
1540 }
1541 for (i = lcp->lcp_rotor; lcp->lcp_bitmap[i] == 0;) {
1542 if (++i >= LWPCTL_BITMAP_ENTRIES)
1543 i = 0;
1544 }
1545 bit = ffs(lcp->lcp_bitmap[i]) - 1;
1546 lcp->lcp_bitmap[i] ^= (1 << bit);
1547 lcp->lcp_rotor = i;
1548 lcp->lcp_nfree--;
1549 l->l_lcpage = lcp;
1550 offset = (i << 5) + bit;
1551 l->l_lwpctl = (lwpctl_t *)lcp->lcp_kaddr + offset;
1552 *uaddr = lcp->lcp_uaddr + offset * sizeof(lwpctl_t);
1553 mutex_exit(&lp->lp_lock);
1554
1555 l->l_lwpctl->lc_curcpu = (short)curcpu()->ci_data.cpu_index;
1556
1557 return 0;
1558 }
1559
1560 /*
1561 * Free an lwpctl structure back to the per-process list.
1562 */
1563 void
1564 lwp_ctl_free(lwp_t *l)
1565 {
1566 lcproc_t *lp;
1567 lcpage_t *lcp;
1568 u_int map, offset;
1569
1570 lp = l->l_proc->p_lwpctl;
1571 KASSERT(lp != NULL);
1572
1573 lcp = l->l_lcpage;
1574 offset = (u_int)((lwpctl_t *)l->l_lwpctl - (lwpctl_t *)lcp->lcp_kaddr);
1575 KASSERT(offset < LWPCTL_PER_PAGE);
1576
1577 mutex_enter(&lp->lp_lock);
1578 lcp->lcp_nfree++;
1579 map = offset >> 5;
1580 lcp->lcp_bitmap[map] |= (1 << (offset & 31));
1581 if (lcp->lcp_bitmap[lcp->lcp_rotor] == 0)
1582 lcp->lcp_rotor = map;
1583 if (TAILQ_FIRST(&lp->lp_pages)->lcp_nfree == 0) {
1584 TAILQ_REMOVE(&lp->lp_pages, lcp, lcp_chain);
1585 TAILQ_INSERT_HEAD(&lp->lp_pages, lcp, lcp_chain);
1586 }
1587 mutex_exit(&lp->lp_lock);
1588 }
1589
1590 /*
1591 * Process is exiting; tear down lwpctl state. This can only be safely
1592 * called by the last LWP in the process.
1593 */
1594 void
1595 lwp_ctl_exit(void)
1596 {
1597 lcpage_t *lcp, *next;
1598 lcproc_t *lp;
1599 proc_t *p;
1600 lwp_t *l;
1601
1602 l = curlwp;
1603 l->l_lwpctl = NULL;
1604 p = l->l_proc;
1605 lp = p->p_lwpctl;
1606
1607 KASSERT(lp != NULL);
1608 KASSERT(p->p_nlwps == 1);
1609
1610 for (lcp = TAILQ_FIRST(&lp->lp_pages); lcp != NULL; lcp = next) {
1611 next = TAILQ_NEXT(lcp, lcp_chain);
1612 uvm_unmap(kernel_map, lcp->lcp_kaddr,
1613 lcp->lcp_kaddr + PAGE_SIZE);
1614 kmem_free(lcp, LWPCTL_LCPAGE_SZ);
1615 }
1616
1617 if (lp->lp_uao != NULL) {
1618 uvm_unmap(&p->p_vmspace->vm_map, lp->lp_uva,
1619 lp->lp_uva + LWPCTL_UAREA_SZ);
1620 }
1621
1622 mutex_destroy(&lp->lp_lock);
1623 kmem_free(lp, sizeof(*lp));
1624 p->p_lwpctl = NULL;
1625 }
1626
1627 #if defined(DDB)
1628 void
1629 lwp_whatis(uintptr_t addr, void (*pr)(const char *, ...))
1630 {
1631 lwp_t *l;
1632
1633 LIST_FOREACH(l, &alllwp, l_list) {
1634 uintptr_t stack = (uintptr_t)KSTACK_LOWEST_ADDR(l);
1635
1636 if (addr < stack || stack + KSTACK_SIZE <= addr) {
1637 continue;
1638 }
1639 (*pr)("%p is %p+%zu, LWP %p's stack\n",
1640 (void *)addr, (void *)stack,
1641 (size_t)(addr - stack), l);
1642 }
1643 }
1644 #endif /* defined(DDB) */
1645