kern_lwp.c revision 1.221 1 1.221 ad /* $NetBSD: kern_lwp.c,v 1.221 2020/01/26 19:06:24 ad Exp $ */
2 1.2 thorpej
3 1.2 thorpej /*-
4 1.220 ad * Copyright (c) 2001, 2006, 2007, 2008, 2009, 2019, 2020
5 1.220 ad * The NetBSD Foundation, Inc.
6 1.2 thorpej * All rights reserved.
7 1.2 thorpej *
8 1.2 thorpej * This code is derived from software contributed to The NetBSD Foundation
9 1.52 ad * by Nathan J. Williams, and Andrew Doran.
10 1.2 thorpej *
11 1.2 thorpej * Redistribution and use in source and binary forms, with or without
12 1.2 thorpej * modification, are permitted provided that the following conditions
13 1.2 thorpej * are met:
14 1.2 thorpej * 1. Redistributions of source code must retain the above copyright
15 1.2 thorpej * notice, this list of conditions and the following disclaimer.
16 1.2 thorpej * 2. Redistributions in binary form must reproduce the above copyright
17 1.2 thorpej * notice, this list of conditions and the following disclaimer in the
18 1.2 thorpej * documentation and/or other materials provided with the distribution.
19 1.2 thorpej *
20 1.2 thorpej * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
21 1.2 thorpej * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
22 1.2 thorpej * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
23 1.2 thorpej * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
24 1.2 thorpej * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 1.2 thorpej * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 1.2 thorpej * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 1.2 thorpej * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 1.2 thorpej * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 1.2 thorpej * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 1.2 thorpej * POSSIBILITY OF SUCH DAMAGE.
31 1.2 thorpej */
32 1.9 lukem
33 1.52 ad /*
34 1.52 ad * Overview
35 1.52 ad *
36 1.66 ad * Lightweight processes (LWPs) are the basic unit or thread of
37 1.52 ad * execution within the kernel. The core state of an LWP is described
38 1.66 ad * by "struct lwp", also known as lwp_t.
39 1.52 ad *
40 1.52 ad * Each LWP is contained within a process (described by "struct proc"),
41 1.52 ad * Every process contains at least one LWP, but may contain more. The
42 1.52 ad * process describes attributes shared among all of its LWPs such as a
43 1.52 ad * private address space, global execution state (stopped, active,
44 1.52 ad * zombie, ...), signal disposition and so on. On a multiprocessor
45 1.66 ad * machine, multiple LWPs be executing concurrently in the kernel.
46 1.52 ad *
47 1.52 ad * Execution states
48 1.52 ad *
49 1.52 ad * At any given time, an LWP has overall state that is described by
50 1.52 ad * lwp::l_stat. The states are broken into two sets below. The first
51 1.52 ad * set is guaranteed to represent the absolute, current state of the
52 1.52 ad * LWP:
53 1.101 rmind *
54 1.101 rmind * LSONPROC
55 1.101 rmind *
56 1.101 rmind * On processor: the LWP is executing on a CPU, either in the
57 1.101 rmind * kernel or in user space.
58 1.101 rmind *
59 1.101 rmind * LSRUN
60 1.101 rmind *
61 1.101 rmind * Runnable: the LWP is parked on a run queue, and may soon be
62 1.101 rmind * chosen to run by an idle processor, or by a processor that
63 1.101 rmind * has been asked to preempt a currently runnning but lower
64 1.134 rmind * priority LWP.
65 1.101 rmind *
66 1.101 rmind * LSIDL
67 1.101 rmind *
68 1.101 rmind * Idle: the LWP has been created but has not yet executed,
69 1.66 ad * or it has ceased executing a unit of work and is waiting
70 1.66 ad * to be started again.
71 1.101 rmind *
72 1.101 rmind * LSSUSPENDED:
73 1.101 rmind *
74 1.101 rmind * Suspended: the LWP has had its execution suspended by
75 1.52 ad * another LWP in the same process using the _lwp_suspend()
76 1.52 ad * system call. User-level LWPs also enter the suspended
77 1.52 ad * state when the system is shutting down.
78 1.52 ad *
79 1.52 ad * The second set represent a "statement of intent" on behalf of the
80 1.52 ad * LWP. The LWP may in fact be executing on a processor, may be
81 1.66 ad * sleeping or idle. It is expected to take the necessary action to
82 1.101 rmind * stop executing or become "running" again within a short timeframe.
83 1.218 ad * The LW_RUNNING flag in lwp::l_flag indicates that an LWP is running.
84 1.101 rmind * Importantly, it indicates that its state is tied to a CPU.
85 1.101 rmind *
86 1.101 rmind * LSZOMB:
87 1.101 rmind *
88 1.101 rmind * Dead or dying: the LWP has released most of its resources
89 1.129 ad * and is about to switch away into oblivion, or has already
90 1.66 ad * switched away. When it switches away, its few remaining
91 1.66 ad * resources can be collected.
92 1.101 rmind *
93 1.101 rmind * LSSLEEP:
94 1.101 rmind *
95 1.101 rmind * Sleeping: the LWP has entered itself onto a sleep queue, and
96 1.101 rmind * has switched away or will switch away shortly to allow other
97 1.66 ad * LWPs to run on the CPU.
98 1.101 rmind *
99 1.101 rmind * LSSTOP:
100 1.101 rmind *
101 1.101 rmind * Stopped: the LWP has been stopped as a result of a job
102 1.101 rmind * control signal, or as a result of the ptrace() interface.
103 1.101 rmind *
104 1.101 rmind * Stopped LWPs may run briefly within the kernel to handle
105 1.101 rmind * signals that they receive, but will not return to user space
106 1.101 rmind * until their process' state is changed away from stopped.
107 1.101 rmind *
108 1.101 rmind * Single LWPs within a process can not be set stopped
109 1.101 rmind * selectively: all actions that can stop or continue LWPs
110 1.101 rmind * occur at the process level.
111 1.101 rmind *
112 1.52 ad * State transitions
113 1.52 ad *
114 1.66 ad * Note that the LSSTOP state may only be set when returning to
115 1.66 ad * user space in userret(), or when sleeping interruptably. The
116 1.66 ad * LSSUSPENDED state may only be set in userret(). Before setting
117 1.66 ad * those states, we try to ensure that the LWPs will release all
118 1.66 ad * locks that they hold, and at a minimum try to ensure that the
119 1.66 ad * LWP can be set runnable again by a signal.
120 1.52 ad *
121 1.52 ad * LWPs may transition states in the following ways:
122 1.52 ad *
123 1.52 ad * RUN -------> ONPROC ONPROC -----> RUN
124 1.129 ad * > SLEEP
125 1.129 ad * > STOPPED
126 1.52 ad * > SUSPENDED
127 1.52 ad * > ZOMB
128 1.129 ad * > IDL (special cases)
129 1.52 ad *
130 1.52 ad * STOPPED ---> RUN SUSPENDED --> RUN
131 1.129 ad * > SLEEP
132 1.52 ad *
133 1.52 ad * SLEEP -----> ONPROC IDL --------> RUN
134 1.101 rmind * > RUN > SUSPENDED
135 1.101 rmind * > STOPPED > STOPPED
136 1.129 ad * > ONPROC (special cases)
137 1.52 ad *
138 1.129 ad * Some state transitions are only possible with kernel threads (eg
139 1.129 ad * ONPROC -> IDL) and happen under tightly controlled circumstances
140 1.129 ad * free of unwanted side effects.
141 1.66 ad *
142 1.114 rmind * Migration
143 1.114 rmind *
144 1.114 rmind * Migration of threads from one CPU to another could be performed
145 1.114 rmind * internally by the scheduler via sched_takecpu() or sched_catchlwp()
146 1.114 rmind * functions. The universal lwp_migrate() function should be used for
147 1.114 rmind * any other cases. Subsystems in the kernel must be aware that CPU
148 1.114 rmind * of LWP may change, while it is not locked.
149 1.114 rmind *
150 1.52 ad * Locking
151 1.52 ad *
152 1.52 ad * The majority of fields in 'struct lwp' are covered by a single,
153 1.66 ad * general spin lock pointed to by lwp::l_mutex. The locks covering
154 1.52 ad * each field are documented in sys/lwp.h.
155 1.52 ad *
156 1.66 ad * State transitions must be made with the LWP's general lock held,
157 1.152 rmind * and may cause the LWP's lock pointer to change. Manipulation of
158 1.66 ad * the general lock is not performed directly, but through calls to
159 1.152 rmind * lwp_lock(), lwp_unlock() and others. It should be noted that the
160 1.152 rmind * adaptive locks are not allowed to be released while the LWP's lock
161 1.152 rmind * is being held (unlike for other spin-locks).
162 1.52 ad *
163 1.52 ad * States and their associated locks:
164 1.52 ad *
165 1.212 ad * LSIDL, LSONPROC, LSZOMB, LSSUPENDED:
166 1.52 ad *
167 1.212 ad * Always covered by spc_lwplock, which protects LWPs not
168 1.212 ad * associated with any other sync object. This is a per-CPU
169 1.212 ad * lock and matches lwp::l_cpu.
170 1.52 ad *
171 1.212 ad * LSRUN:
172 1.52 ad *
173 1.64 yamt * Always covered by spc_mutex, which protects the run queues.
174 1.129 ad * This is a per-CPU lock and matches lwp::l_cpu.
175 1.52 ad *
176 1.52 ad * LSSLEEP:
177 1.52 ad *
178 1.212 ad * Covered by a lock associated with the sleep queue (sometimes
179 1.221 ad * a turnstile sleep queue) that the LWP resides on. This can
180 1.221 ad * be spc_lwplock for SOBJ_SLEEPQ_NULL (an "untracked" sleep).
181 1.52 ad *
182 1.212 ad * LSSTOP:
183 1.101 rmind *
184 1.52 ad * If the LWP was previously sleeping (l_wchan != NULL), then
185 1.66 ad * l_mutex references the sleep queue lock. If the LWP was
186 1.52 ad * runnable or on the CPU when halted, or has been removed from
187 1.66 ad * the sleep queue since halted, then the lock is spc_lwplock.
188 1.52 ad *
189 1.52 ad * The lock order is as follows:
190 1.52 ad *
191 1.212 ad * sleepq -> turnstile -> spc_lwplock -> spc_mutex
192 1.52 ad *
193 1.103 ad * Each process has an scheduler state lock (proc::p_lock), and a
194 1.52 ad * number of counters on LWPs and their states: p_nzlwps, p_nrlwps, and
195 1.52 ad * so on. When an LWP is to be entered into or removed from one of the
196 1.103 ad * following states, p_lock must be held and the process wide counters
197 1.52 ad * adjusted:
198 1.52 ad *
199 1.52 ad * LSIDL, LSZOMB, LSSTOP, LSSUSPENDED
200 1.52 ad *
201 1.129 ad * (But not always for kernel threads. There are some special cases
202 1.212 ad * as mentioned above: soft interrupts, and the idle loops.)
203 1.129 ad *
204 1.52 ad * Note that an LWP is considered running or likely to run soon if in
205 1.52 ad * one of the following states. This affects the value of p_nrlwps:
206 1.52 ad *
207 1.52 ad * LSRUN, LSONPROC, LSSLEEP
208 1.52 ad *
209 1.103 ad * p_lock does not need to be held when transitioning among these
210 1.129 ad * three states, hence p_lock is rarely taken for state transitions.
211 1.52 ad */
212 1.52 ad
213 1.9 lukem #include <sys/cdefs.h>
214 1.221 ad __KERNEL_RCSID(0, "$NetBSD: kern_lwp.c,v 1.221 2020/01/26 19:06:24 ad Exp $");
215 1.8 martin
216 1.84 yamt #include "opt_ddb.h"
217 1.52 ad #include "opt_lockdebug.h"
218 1.139 darran #include "opt_dtrace.h"
219 1.2 thorpej
220 1.47 hannken #define _LWP_API_PRIVATE
221 1.47 hannken
222 1.2 thorpej #include <sys/param.h>
223 1.2 thorpej #include <sys/systm.h>
224 1.64 yamt #include <sys/cpu.h>
225 1.2 thorpej #include <sys/pool.h>
226 1.2 thorpej #include <sys/proc.h>
227 1.2 thorpej #include <sys/syscallargs.h>
228 1.57 dsl #include <sys/syscall_stats.h>
229 1.37 ad #include <sys/kauth.h>
230 1.52 ad #include <sys/sleepq.h>
231 1.52 ad #include <sys/lockdebug.h>
232 1.52 ad #include <sys/kmem.h>
233 1.91 rmind #include <sys/pset.h>
234 1.75 ad #include <sys/intr.h>
235 1.78 ad #include <sys/lwpctl.h>
236 1.81 ad #include <sys/atomic.h>
237 1.131 ad #include <sys/filedesc.h>
238 1.196 hannken #include <sys/fstrans.h>
239 1.138 darran #include <sys/dtrace_bsd.h>
240 1.141 darran #include <sys/sdt.h>
241 1.203 kamil #include <sys/ptrace.h>
242 1.157 rmind #include <sys/xcall.h>
243 1.169 christos #include <sys/uidinfo.h>
244 1.169 christos #include <sys/sysctl.h>
245 1.201 ozaki #include <sys/psref.h>
246 1.208 maxv #include <sys/msan.h>
247 1.138 darran
248 1.2 thorpej #include <uvm/uvm_extern.h>
249 1.80 skrll #include <uvm/uvm_object.h>
250 1.2 thorpej
251 1.152 rmind static pool_cache_t lwp_cache __read_mostly;
252 1.152 rmind struct lwplist alllwp __cacheline_aligned;
253 1.41 thorpej
254 1.157 rmind static void lwp_dtor(void *, void *);
255 1.157 rmind
256 1.141 darran /* DTrace proc provider probes */
257 1.180 christos SDT_PROVIDER_DEFINE(proc);
258 1.180 christos
259 1.180 christos SDT_PROBE_DEFINE1(proc, kernel, , lwp__create, "struct lwp *");
260 1.180 christos SDT_PROBE_DEFINE1(proc, kernel, , lwp__start, "struct lwp *");
261 1.180 christos SDT_PROBE_DEFINE1(proc, kernel, , lwp__exit, "struct lwp *");
262 1.141 darran
263 1.213 ad struct turnstile turnstile0 __cacheline_aligned;
264 1.147 pooka struct lwp lwp0 __aligned(MIN_LWP_ALIGNMENT) = {
265 1.147 pooka #ifdef LWP0_CPU_INFO
266 1.147 pooka .l_cpu = LWP0_CPU_INFO,
267 1.147 pooka #endif
268 1.154 matt #ifdef LWP0_MD_INITIALIZER
269 1.154 matt .l_md = LWP0_MD_INITIALIZER,
270 1.154 matt #endif
271 1.147 pooka .l_proc = &proc0,
272 1.147 pooka .l_lid = 1,
273 1.147 pooka .l_flag = LW_SYSTEM,
274 1.147 pooka .l_stat = LSONPROC,
275 1.147 pooka .l_ts = &turnstile0,
276 1.147 pooka .l_syncobj = &sched_syncobj,
277 1.147 pooka .l_refcnt = 1,
278 1.147 pooka .l_priority = PRI_USER + NPRI_USER - 1,
279 1.147 pooka .l_inheritedprio = -1,
280 1.147 pooka .l_class = SCHED_OTHER,
281 1.147 pooka .l_psid = PS_NONE,
282 1.147 pooka .l_pi_lenders = SLIST_HEAD_INITIALIZER(&lwp0.l_pi_lenders),
283 1.147 pooka .l_name = __UNCONST("swapper"),
284 1.147 pooka .l_fd = &filedesc0,
285 1.147 pooka };
286 1.147 pooka
287 1.169 christos static int sysctl_kern_maxlwp(SYSCTLFN_PROTO);
288 1.169 christos
289 1.169 christos /*
290 1.169 christos * sysctl helper routine for kern.maxlwp. Ensures that the new
291 1.169 christos * values are not too low or too high.
292 1.169 christos */
293 1.169 christos static int
294 1.169 christos sysctl_kern_maxlwp(SYSCTLFN_ARGS)
295 1.169 christos {
296 1.169 christos int error, nmaxlwp;
297 1.169 christos struct sysctlnode node;
298 1.169 christos
299 1.169 christos nmaxlwp = maxlwp;
300 1.169 christos node = *rnode;
301 1.169 christos node.sysctl_data = &nmaxlwp;
302 1.169 christos error = sysctl_lookup(SYSCTLFN_CALL(&node));
303 1.169 christos if (error || newp == NULL)
304 1.169 christos return error;
305 1.169 christos
306 1.169 christos if (nmaxlwp < 0 || nmaxlwp >= 65536)
307 1.169 christos return EINVAL;
308 1.169 christos if (nmaxlwp > cpu_maxlwp())
309 1.169 christos return EINVAL;
310 1.169 christos maxlwp = nmaxlwp;
311 1.169 christos
312 1.169 christos return 0;
313 1.169 christos }
314 1.169 christos
315 1.169 christos static void
316 1.169 christos sysctl_kern_lwp_setup(void)
317 1.169 christos {
318 1.169 christos struct sysctllog *clog = NULL;
319 1.169 christos
320 1.169 christos sysctl_createv(&clog, 0, NULL, NULL,
321 1.169 christos CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
322 1.169 christos CTLTYPE_INT, "maxlwp",
323 1.169 christos SYSCTL_DESCR("Maximum number of simultaneous threads"),
324 1.169 christos sysctl_kern_maxlwp, 0, NULL, 0,
325 1.169 christos CTL_KERN, CTL_CREATE, CTL_EOL);
326 1.169 christos }
327 1.169 christos
328 1.41 thorpej void
329 1.41 thorpej lwpinit(void)
330 1.41 thorpej {
331 1.41 thorpej
332 1.152 rmind LIST_INIT(&alllwp);
333 1.144 pooka lwpinit_specificdata();
334 1.52 ad lwp_sys_init();
335 1.87 ad lwp_cache = pool_cache_init(sizeof(lwp_t), MIN_LWP_ALIGNMENT, 0, 0,
336 1.157 rmind "lwppl", NULL, IPL_NONE, NULL, lwp_dtor, NULL);
337 1.169 christos
338 1.169 christos maxlwp = cpu_maxlwp();
339 1.169 christos sysctl_kern_lwp_setup();
340 1.41 thorpej }
341 1.41 thorpej
342 1.147 pooka void
343 1.147 pooka lwp0_init(void)
344 1.147 pooka {
345 1.147 pooka struct lwp *l = &lwp0;
346 1.147 pooka
347 1.147 pooka KASSERT((void *)uvm_lwp_getuarea(l) != NULL);
348 1.148 pooka KASSERT(l->l_lid == proc0.p_nlwpid);
349 1.147 pooka
350 1.147 pooka LIST_INSERT_HEAD(&alllwp, l, l_list);
351 1.147 pooka
352 1.147 pooka callout_init(&l->l_timeout_ch, CALLOUT_MPSAFE);
353 1.147 pooka callout_setfunc(&l->l_timeout_ch, sleepq_timeout, l);
354 1.147 pooka cv_init(&l->l_sigcv, "sigwait");
355 1.171 rmind cv_init(&l->l_waitcv, "vfork");
356 1.147 pooka
357 1.147 pooka kauth_cred_hold(proc0.p_cred);
358 1.147 pooka l->l_cred = proc0.p_cred;
359 1.147 pooka
360 1.164 yamt kdtrace_thread_ctor(NULL, l);
361 1.147 pooka lwp_initspecific(l);
362 1.147 pooka
363 1.147 pooka SYSCALL_TIME_LWP_INIT(l);
364 1.147 pooka }
365 1.147 pooka
366 1.157 rmind static void
367 1.157 rmind lwp_dtor(void *arg, void *obj)
368 1.157 rmind {
369 1.157 rmind lwp_t *l = obj;
370 1.157 rmind (void)l;
371 1.157 rmind
372 1.157 rmind /*
373 1.157 rmind * Provide a barrier to ensure that all mutex_oncpu() and rw_oncpu()
374 1.157 rmind * calls will exit before memory of LWP is returned to the pool, where
375 1.157 rmind * KVA of LWP structure might be freed and re-used for other purposes.
376 1.157 rmind * Kernel preemption is disabled around mutex_oncpu() and rw_oncpu()
377 1.157 rmind * callers, therefore cross-call to all CPUs will do the job. Also,
378 1.157 rmind * the value of l->l_cpu must be still valid at this point.
379 1.157 rmind */
380 1.157 rmind KASSERT(l->l_cpu != NULL);
381 1.205 uwe xc_barrier(0);
382 1.157 rmind }
383 1.157 rmind
384 1.52 ad /*
385 1.52 ad * Set an suspended.
386 1.52 ad *
387 1.103 ad * Must be called with p_lock held, and the LWP locked. Will unlock the
388 1.52 ad * LWP before return.
389 1.52 ad */
390 1.2 thorpej int
391 1.52 ad lwp_suspend(struct lwp *curl, struct lwp *t)
392 1.2 thorpej {
393 1.52 ad int error;
394 1.2 thorpej
395 1.103 ad KASSERT(mutex_owned(t->l_proc->p_lock));
396 1.63 ad KASSERT(lwp_locked(t, NULL));
397 1.33 chs
398 1.52 ad KASSERT(curl != t || curl->l_stat == LSONPROC);
399 1.2 thorpej
400 1.52 ad /*
401 1.52 ad * If the current LWP has been told to exit, we must not suspend anyone
402 1.52 ad * else or deadlock could occur. We won't return to userspace.
403 1.2 thorpej */
404 1.109 rmind if ((curl->l_flag & (LW_WEXIT | LW_WCORE)) != 0) {
405 1.52 ad lwp_unlock(t);
406 1.52 ad return (EDEADLK);
407 1.2 thorpej }
408 1.2 thorpej
409 1.204 kamil if ((t->l_flag & LW_DBGSUSPEND) != 0) {
410 1.204 kamil lwp_unlock(t);
411 1.204 kamil return 0;
412 1.204 kamil }
413 1.204 kamil
414 1.52 ad error = 0;
415 1.2 thorpej
416 1.52 ad switch (t->l_stat) {
417 1.52 ad case LSRUN:
418 1.52 ad case LSONPROC:
419 1.56 pavel t->l_flag |= LW_WSUSPEND;
420 1.52 ad lwp_need_userret(t);
421 1.52 ad lwp_unlock(t);
422 1.52 ad break;
423 1.2 thorpej
424 1.52 ad case LSSLEEP:
425 1.56 pavel t->l_flag |= LW_WSUSPEND;
426 1.2 thorpej
427 1.2 thorpej /*
428 1.52 ad * Kick the LWP and try to get it to the kernel boundary
429 1.52 ad * so that it will release any locks that it holds.
430 1.52 ad * setrunnable() will release the lock.
431 1.2 thorpej */
432 1.56 pavel if ((t->l_flag & LW_SINTR) != 0)
433 1.52 ad setrunnable(t);
434 1.52 ad else
435 1.52 ad lwp_unlock(t);
436 1.52 ad break;
437 1.2 thorpej
438 1.52 ad case LSSUSPENDED:
439 1.52 ad lwp_unlock(t);
440 1.52 ad break;
441 1.17 manu
442 1.52 ad case LSSTOP:
443 1.56 pavel t->l_flag |= LW_WSUSPEND;
444 1.52 ad setrunnable(t);
445 1.52 ad break;
446 1.2 thorpej
447 1.52 ad case LSIDL:
448 1.52 ad case LSZOMB:
449 1.52 ad error = EINTR; /* It's what Solaris does..... */
450 1.52 ad lwp_unlock(t);
451 1.52 ad break;
452 1.2 thorpej }
453 1.2 thorpej
454 1.69 rmind return (error);
455 1.2 thorpej }
456 1.2 thorpej
457 1.52 ad /*
458 1.52 ad * Restart a suspended LWP.
459 1.52 ad *
460 1.103 ad * Must be called with p_lock held, and the LWP locked. Will unlock the
461 1.52 ad * LWP before return.
462 1.52 ad */
463 1.2 thorpej void
464 1.2 thorpej lwp_continue(struct lwp *l)
465 1.2 thorpej {
466 1.2 thorpej
467 1.103 ad KASSERT(mutex_owned(l->l_proc->p_lock));
468 1.63 ad KASSERT(lwp_locked(l, NULL));
469 1.52 ad
470 1.52 ad /* If rebooting or not suspended, then just bail out. */
471 1.56 pavel if ((l->l_flag & LW_WREBOOT) != 0) {
472 1.52 ad lwp_unlock(l);
473 1.2 thorpej return;
474 1.10 fvdl }
475 1.2 thorpej
476 1.56 pavel l->l_flag &= ~LW_WSUSPEND;
477 1.2 thorpej
478 1.204 kamil if (l->l_stat != LSSUSPENDED || (l->l_flag & LW_DBGSUSPEND) != 0) {
479 1.52 ad lwp_unlock(l);
480 1.52 ad return;
481 1.2 thorpej }
482 1.2 thorpej
483 1.52 ad /* setrunnable() will release the lock. */
484 1.52 ad setrunnable(l);
485 1.2 thorpej }
486 1.2 thorpej
487 1.52 ad /*
488 1.142 christos * Restart a stopped LWP.
489 1.142 christos *
490 1.142 christos * Must be called with p_lock held, and the LWP NOT locked. Will unlock the
491 1.142 christos * LWP before return.
492 1.142 christos */
493 1.142 christos void
494 1.142 christos lwp_unstop(struct lwp *l)
495 1.142 christos {
496 1.142 christos struct proc *p = l->l_proc;
497 1.167 rmind
498 1.142 christos KASSERT(mutex_owned(proc_lock));
499 1.142 christos KASSERT(mutex_owned(p->p_lock));
500 1.142 christos
501 1.142 christos lwp_lock(l);
502 1.142 christos
503 1.204 kamil KASSERT((l->l_flag & LW_DBGSUSPEND) == 0);
504 1.204 kamil
505 1.142 christos /* If not stopped, then just bail out. */
506 1.142 christos if (l->l_stat != LSSTOP) {
507 1.142 christos lwp_unlock(l);
508 1.142 christos return;
509 1.142 christos }
510 1.142 christos
511 1.142 christos p->p_stat = SACTIVE;
512 1.142 christos p->p_sflag &= ~PS_STOPPING;
513 1.142 christos
514 1.142 christos if (!p->p_waited)
515 1.142 christos p->p_pptr->p_nstopchild--;
516 1.142 christos
517 1.142 christos if (l->l_wchan == NULL) {
518 1.142 christos /* setrunnable() will release the lock. */
519 1.142 christos setrunnable(l);
520 1.183 christos } else if (p->p_xsig && (l->l_flag & LW_SINTR) != 0) {
521 1.163 christos /* setrunnable() so we can receive the signal */
522 1.163 christos setrunnable(l);
523 1.142 christos } else {
524 1.142 christos l->l_stat = LSSLEEP;
525 1.142 christos p->p_nrlwps++;
526 1.142 christos lwp_unlock(l);
527 1.142 christos }
528 1.142 christos }
529 1.142 christos
530 1.142 christos /*
531 1.52 ad * Wait for an LWP within the current process to exit. If 'lid' is
532 1.52 ad * non-zero, we are waiting for a specific LWP.
533 1.52 ad *
534 1.103 ad * Must be called with p->p_lock held.
535 1.52 ad */
536 1.2 thorpej int
537 1.173 rmind lwp_wait(struct lwp *l, lwpid_t lid, lwpid_t *departed, bool exiting)
538 1.2 thorpej {
539 1.173 rmind const lwpid_t curlid = l->l_lid;
540 1.173 rmind proc_t *p = l->l_proc;
541 1.173 rmind lwp_t *l2;
542 1.173 rmind int error;
543 1.2 thorpej
544 1.103 ad KASSERT(mutex_owned(p->p_lock));
545 1.52 ad
546 1.52 ad p->p_nlwpwait++;
547 1.63 ad l->l_waitingfor = lid;
548 1.52 ad
549 1.52 ad for (;;) {
550 1.173 rmind int nfound;
551 1.173 rmind
552 1.52 ad /*
553 1.52 ad * Avoid a race between exit1() and sigexit(): if the
554 1.52 ad * process is dumping core, then we need to bail out: call
555 1.52 ad * into lwp_userret() where we will be suspended until the
556 1.52 ad * deed is done.
557 1.52 ad */
558 1.52 ad if ((p->p_sflag & PS_WCORE) != 0) {
559 1.103 ad mutex_exit(p->p_lock);
560 1.52 ad lwp_userret(l);
561 1.173 rmind KASSERT(false);
562 1.52 ad }
563 1.52 ad
564 1.52 ad /*
565 1.52 ad * First off, drain any detached LWP that is waiting to be
566 1.52 ad * reaped.
567 1.52 ad */
568 1.52 ad while ((l2 = p->p_zomblwp) != NULL) {
569 1.52 ad p->p_zomblwp = NULL;
570 1.63 ad lwp_free(l2, false, false);/* releases proc mutex */
571 1.103 ad mutex_enter(p->p_lock);
572 1.52 ad }
573 1.52 ad
574 1.52 ad /*
575 1.52 ad * Now look for an LWP to collect. If the whole process is
576 1.52 ad * exiting, count detached LWPs as eligible to be collected,
577 1.52 ad * but don't drain them here.
578 1.52 ad */
579 1.52 ad nfound = 0;
580 1.63 ad error = 0;
581 1.52 ad LIST_FOREACH(l2, &p->p_lwps, l_sibling) {
582 1.63 ad /*
583 1.63 ad * If a specific wait and the target is waiting on
584 1.63 ad * us, then avoid deadlock. This also traps LWPs
585 1.63 ad * that try to wait on themselves.
586 1.63 ad *
587 1.63 ad * Note that this does not handle more complicated
588 1.63 ad * cycles, like: t1 -> t2 -> t3 -> t1. The process
589 1.63 ad * can still be killed so it is not a major problem.
590 1.63 ad */
591 1.63 ad if (l2->l_lid == lid && l2->l_waitingfor == curlid) {
592 1.63 ad error = EDEADLK;
593 1.63 ad break;
594 1.63 ad }
595 1.63 ad if (l2 == l)
596 1.52 ad continue;
597 1.52 ad if ((l2->l_prflag & LPR_DETACHED) != 0) {
598 1.63 ad nfound += exiting;
599 1.63 ad continue;
600 1.63 ad }
601 1.63 ad if (lid != 0) {
602 1.63 ad if (l2->l_lid != lid)
603 1.63 ad continue;
604 1.63 ad /*
605 1.63 ad * Mark this LWP as the first waiter, if there
606 1.63 ad * is no other.
607 1.63 ad */
608 1.63 ad if (l2->l_waiter == 0)
609 1.63 ad l2->l_waiter = curlid;
610 1.63 ad } else if (l2->l_waiter != 0) {
611 1.63 ad /*
612 1.63 ad * It already has a waiter - so don't
613 1.63 ad * collect it. If the waiter doesn't
614 1.63 ad * grab it we'll get another chance
615 1.63 ad * later.
616 1.63 ad */
617 1.63 ad nfound++;
618 1.52 ad continue;
619 1.52 ad }
620 1.52 ad nfound++;
621 1.2 thorpej
622 1.52 ad /* No need to lock the LWP in order to see LSZOMB. */
623 1.52 ad if (l2->l_stat != LSZOMB)
624 1.52 ad continue;
625 1.2 thorpej
626 1.63 ad /*
627 1.63 ad * We're no longer waiting. Reset the "first waiter"
628 1.63 ad * pointer on the target, in case it was us.
629 1.63 ad */
630 1.63 ad l->l_waitingfor = 0;
631 1.63 ad l2->l_waiter = 0;
632 1.63 ad p->p_nlwpwait--;
633 1.2 thorpej if (departed)
634 1.2 thorpej *departed = l2->l_lid;
635 1.75 ad sched_lwp_collect(l2);
636 1.63 ad
637 1.63 ad /* lwp_free() releases the proc lock. */
638 1.63 ad lwp_free(l2, false, false);
639 1.103 ad mutex_enter(p->p_lock);
640 1.52 ad return 0;
641 1.52 ad }
642 1.2 thorpej
643 1.63 ad if (error != 0)
644 1.63 ad break;
645 1.52 ad if (nfound == 0) {
646 1.52 ad error = ESRCH;
647 1.52 ad break;
648 1.52 ad }
649 1.63 ad
650 1.63 ad /*
651 1.173 rmind * Note: since the lock will be dropped, need to restart on
652 1.173 rmind * wakeup to run all LWPs again, e.g. there may be new LWPs.
653 1.63 ad */
654 1.63 ad if (exiting) {
655 1.52 ad KASSERT(p->p_nlwps > 1);
656 1.192 christos cv_wait(&p->p_lwpcv, p->p_lock);
657 1.192 christos error = EAGAIN;
658 1.173 rmind break;
659 1.52 ad }
660 1.63 ad
661 1.63 ad /*
662 1.63 ad * If all other LWPs are waiting for exits or suspends
663 1.63 ad * and the supply of zombies and potential zombies is
664 1.63 ad * exhausted, then we are about to deadlock.
665 1.63 ad *
666 1.63 ad * If the process is exiting (and this LWP is not the one
667 1.63 ad * that is coordinating the exit) then bail out now.
668 1.63 ad */
669 1.52 ad if ((p->p_sflag & PS_WEXIT) != 0 ||
670 1.63 ad p->p_nrlwps + p->p_nzlwps - p->p_ndlwps <= p->p_nlwpwait) {
671 1.52 ad error = EDEADLK;
672 1.52 ad break;
673 1.2 thorpej }
674 1.63 ad
675 1.63 ad /*
676 1.63 ad * Sit around and wait for something to happen. We'll be
677 1.63 ad * awoken if any of the conditions examined change: if an
678 1.63 ad * LWP exits, is collected, or is detached.
679 1.63 ad */
680 1.103 ad if ((error = cv_wait_sig(&p->p_lwpcv, p->p_lock)) != 0)
681 1.52 ad break;
682 1.2 thorpej }
683 1.2 thorpej
684 1.63 ad /*
685 1.63 ad * We didn't find any LWPs to collect, we may have received a
686 1.63 ad * signal, or some other condition has caused us to bail out.
687 1.63 ad *
688 1.63 ad * If waiting on a specific LWP, clear the waiters marker: some
689 1.63 ad * other LWP may want it. Then, kick all the remaining waiters
690 1.63 ad * so that they can re-check for zombies and for deadlock.
691 1.63 ad */
692 1.63 ad if (lid != 0) {
693 1.63 ad LIST_FOREACH(l2, &p->p_lwps, l_sibling) {
694 1.63 ad if (l2->l_lid == lid) {
695 1.63 ad if (l2->l_waiter == curlid)
696 1.63 ad l2->l_waiter = 0;
697 1.63 ad break;
698 1.63 ad }
699 1.63 ad }
700 1.63 ad }
701 1.52 ad p->p_nlwpwait--;
702 1.63 ad l->l_waitingfor = 0;
703 1.63 ad cv_broadcast(&p->p_lwpcv);
704 1.63 ad
705 1.52 ad return error;
706 1.2 thorpej }
707 1.2 thorpej
708 1.174 dsl static lwpid_t
709 1.174 dsl lwp_find_free_lid(lwpid_t try_lid, lwp_t * new_lwp, proc_t *p)
710 1.174 dsl {
711 1.174 dsl #define LID_SCAN (1u << 31)
712 1.174 dsl lwp_t *scan, *free_before;
713 1.174 dsl lwpid_t nxt_lid;
714 1.174 dsl
715 1.174 dsl /*
716 1.174 dsl * We want the first unused lid greater than or equal to
717 1.174 dsl * try_lid (modulo 2^31).
718 1.174 dsl * (If nothing else ld.elf_so doesn't want lwpid with the top bit set.)
719 1.174 dsl * We must not return 0, and avoiding 'LID_SCAN - 1' makes
720 1.174 dsl * the outer test easier.
721 1.174 dsl * This would be much easier if the list were sorted in
722 1.174 dsl * increasing order.
723 1.174 dsl * The list is kept sorted in decreasing order.
724 1.174 dsl * This code is only used after a process has generated 2^31 lwp.
725 1.174 dsl *
726 1.174 dsl * Code assumes it can always find an id.
727 1.174 dsl */
728 1.174 dsl
729 1.174 dsl try_lid &= LID_SCAN - 1;
730 1.174 dsl if (try_lid <= 1)
731 1.174 dsl try_lid = 2;
732 1.174 dsl
733 1.174 dsl free_before = NULL;
734 1.174 dsl nxt_lid = LID_SCAN - 1;
735 1.174 dsl LIST_FOREACH(scan, &p->p_lwps, l_sibling) {
736 1.174 dsl if (scan->l_lid != nxt_lid) {
737 1.174 dsl /* There are available lid before this entry */
738 1.174 dsl free_before = scan;
739 1.174 dsl if (try_lid > scan->l_lid)
740 1.174 dsl break;
741 1.174 dsl }
742 1.174 dsl if (try_lid == scan->l_lid) {
743 1.174 dsl /* The ideal lid is busy, take a higher one */
744 1.174 dsl if (free_before != NULL) {
745 1.174 dsl try_lid = free_before->l_lid + 1;
746 1.174 dsl break;
747 1.174 dsl }
748 1.174 dsl /* No higher ones, reuse low numbers */
749 1.174 dsl try_lid = 2;
750 1.174 dsl }
751 1.174 dsl
752 1.174 dsl nxt_lid = scan->l_lid - 1;
753 1.174 dsl if (LIST_NEXT(scan, l_sibling) == NULL) {
754 1.174 dsl /* The value we have is lower than any existing lwp */
755 1.174 dsl LIST_INSERT_AFTER(scan, new_lwp, l_sibling);
756 1.174 dsl return try_lid;
757 1.174 dsl }
758 1.174 dsl }
759 1.174 dsl
760 1.174 dsl LIST_INSERT_BEFORE(free_before, new_lwp, l_sibling);
761 1.174 dsl return try_lid;
762 1.174 dsl }
763 1.174 dsl
764 1.52 ad /*
765 1.52 ad * Create a new LWP within process 'p2', using LWP 'l1' as a template.
766 1.52 ad * The new LWP is created in state LSIDL and must be set running,
767 1.52 ad * suspended, or stopped by the caller.
768 1.52 ad */
769 1.2 thorpej int
770 1.134 rmind lwp_create(lwp_t *l1, proc_t *p2, vaddr_t uaddr, int flags,
771 1.188 christos void *stack, size_t stacksize, void (*func)(void *), void *arg,
772 1.188 christos lwp_t **rnewlwpp, int sclass, const sigset_t *sigmask,
773 1.188 christos const stack_t *sigstk)
774 1.2 thorpej {
775 1.215 ad struct lwp *l2;
776 1.52 ad turnstile_t *ts;
777 1.151 chs lwpid_t lid;
778 1.2 thorpej
779 1.107 ad KASSERT(l1 == curlwp || l1->l_proc == &proc0);
780 1.107 ad
781 1.52 ad /*
782 1.215 ad * Enforce limits, excluding the first lwp and kthreads. We must
783 1.215 ad * use the process credentials here when adjusting the limit, as
784 1.215 ad * they are what's tied to the accounting entity. However for
785 1.215 ad * authorizing the action, we'll use the LWP's credentials.
786 1.169 christos */
787 1.215 ad mutex_enter(p2->p_lock);
788 1.169 christos if (p2->p_nlwps != 0 && p2 != &proc0) {
789 1.215 ad uid_t uid = kauth_cred_getuid(p2->p_cred);
790 1.169 christos int count = chglwpcnt(uid, 1);
791 1.169 christos if (__predict_false(count >
792 1.169 christos p2->p_rlimit[RLIMIT_NTHR].rlim_cur)) {
793 1.169 christos if (kauth_authorize_process(l1->l_cred,
794 1.169 christos KAUTH_PROCESS_RLIMIT, p2,
795 1.169 christos KAUTH_ARG(KAUTH_REQ_PROCESS_RLIMIT_BYPASS),
796 1.169 christos &p2->p_rlimit[RLIMIT_NTHR], KAUTH_ARG(RLIMIT_NTHR))
797 1.169 christos != 0) {
798 1.170 christos (void)chglwpcnt(uid, -1);
799 1.215 ad mutex_exit(p2->p_lock);
800 1.170 christos return EAGAIN;
801 1.169 christos }
802 1.169 christos }
803 1.169 christos }
804 1.169 christos
805 1.169 christos /*
806 1.52 ad * First off, reap any detached LWP waiting to be collected.
807 1.52 ad * We can re-use its LWP structure and turnstile.
808 1.52 ad */
809 1.215 ad if ((l2 = p2->p_zomblwp) != NULL) {
810 1.215 ad p2->p_zomblwp = NULL;
811 1.215 ad lwp_free(l2, true, false);
812 1.215 ad /* p2 now unlocked by lwp_free() */
813 1.52 ad ts = l2->l_ts;
814 1.75 ad KASSERT(l2->l_inheritedprio == -1);
815 1.60 yamt KASSERT(SLIST_EMPTY(&l2->l_pi_lenders));
816 1.52 ad memset(l2, 0, sizeof(*l2));
817 1.52 ad l2->l_ts = ts;
818 1.215 ad } else {
819 1.215 ad mutex_exit(p2->p_lock);
820 1.215 ad l2 = pool_cache_get(lwp_cache, PR_WAITOK);
821 1.215 ad memset(l2, 0, sizeof(*l2));
822 1.215 ad l2->l_ts = pool_cache_get(turnstile_cache, PR_WAITOK);
823 1.215 ad SLIST_INIT(&l2->l_pi_lenders);
824 1.52 ad }
825 1.2 thorpej
826 1.2 thorpej l2->l_stat = LSIDL;
827 1.2 thorpej l2->l_proc = p2;
828 1.52 ad l2->l_refcnt = 1;
829 1.75 ad l2->l_class = sclass;
830 1.116 ad
831 1.116 ad /*
832 1.116 ad * If vfork(), we want the LWP to run fast and on the same CPU
833 1.116 ad * as its parent, so that it can reuse the VM context and cache
834 1.116 ad * footprint on the local CPU.
835 1.116 ad */
836 1.116 ad l2->l_kpriority = ((flags & LWP_VFORK) ? true : false);
837 1.82 ad l2->l_kpribase = PRI_KERNEL;
838 1.52 ad l2->l_priority = l1->l_priority;
839 1.75 ad l2->l_inheritedprio = -1;
840 1.185 christos l2->l_protectprio = -1;
841 1.185 christos l2->l_auxprio = -1;
842 1.212 ad l2->l_flag = (l1->l_flag & (LW_WEXIT | LW_WREBOOT | LW_WCORE));
843 1.88 ad l2->l_pflag = LP_MPSAFE;
844 1.131 ad TAILQ_INIT(&l2->l_ld_locks);
845 1.197 ozaki l2->l_psrefs = 0;
846 1.208 maxv kmsan_lwp_alloc(l2);
847 1.131 ad
848 1.131 ad /*
849 1.156 pooka * For vfork, borrow parent's lwpctl context if it exists.
850 1.156 pooka * This also causes us to return via lwp_userret.
851 1.156 pooka */
852 1.156 pooka if (flags & LWP_VFORK && l1->l_lwpctl) {
853 1.156 pooka l2->l_lwpctl = l1->l_lwpctl;
854 1.156 pooka l2->l_flag |= LW_LWPCTL;
855 1.156 pooka }
856 1.156 pooka
857 1.156 pooka /*
858 1.131 ad * If not the first LWP in the process, grab a reference to the
859 1.131 ad * descriptor table.
860 1.131 ad */
861 1.97 ad l2->l_fd = p2->p_fd;
862 1.131 ad if (p2->p_nlwps != 0) {
863 1.131 ad KASSERT(l1->l_proc == p2);
864 1.136 rmind fd_hold(l2);
865 1.131 ad } else {
866 1.131 ad KASSERT(l1->l_proc != p2);
867 1.131 ad }
868 1.41 thorpej
869 1.56 pavel if (p2->p_flag & PK_SYSTEM) {
870 1.134 rmind /* Mark it as a system LWP. */
871 1.56 pavel l2->l_flag |= LW_SYSTEM;
872 1.52 ad }
873 1.2 thorpej
874 1.107 ad kpreempt_disable();
875 1.212 ad l2->l_mutex = l1->l_cpu->ci_schedstate.spc_lwplock;
876 1.107 ad l2->l_cpu = l1->l_cpu;
877 1.107 ad kpreempt_enable();
878 1.107 ad
879 1.138 darran kdtrace_thread_ctor(NULL, l2);
880 1.73 rmind lwp_initspecific(l2);
881 1.75 ad sched_lwp_fork(l1, l2);
882 1.37 ad lwp_update_creds(l2);
883 1.70 ad callout_init(&l2->l_timeout_ch, CALLOUT_MPSAFE);
884 1.70 ad callout_setfunc(&l2->l_timeout_ch, sleepq_timeout, l2);
885 1.52 ad cv_init(&l2->l_sigcv, "sigwait");
886 1.171 rmind cv_init(&l2->l_waitcv, "vfork");
887 1.52 ad l2->l_syncobj = &sched_syncobj;
888 1.201 ozaki PSREF_DEBUG_INIT_LWP(l2);
889 1.2 thorpej
890 1.2 thorpej if (rnewlwpp != NULL)
891 1.2 thorpej *rnewlwpp = l2;
892 1.2 thorpej
893 1.158 matt /*
894 1.158 matt * PCU state needs to be saved before calling uvm_lwp_fork() so that
895 1.158 matt * the MD cpu_lwp_fork() can copy the saved state to the new LWP.
896 1.158 matt */
897 1.158 matt pcu_save_all(l1);
898 1.158 matt
899 1.137 rmind uvm_lwp_setuarea(l2, uaddr);
900 1.190 skrll uvm_lwp_fork(l1, l2, stack, stacksize, func, (arg != NULL) ? arg : l2);
901 1.2 thorpej
902 1.151 chs if ((flags & LWP_PIDLID) != 0) {
903 1.151 chs lid = proc_alloc_pid(p2);
904 1.151 chs l2->l_pflag |= LP_PIDLID;
905 1.206 joerg } else if (p2->p_nlwps == 0) {
906 1.206 joerg lid = l1->l_lid;
907 1.207 joerg /*
908 1.207 joerg * Update next LWP ID, too. If this overflows to LID_SCAN,
909 1.207 joerg * the slow path of scanning will be used for the next LWP.
910 1.207 joerg */
911 1.207 joerg p2->p_nlwpid = lid + 1;
912 1.151 chs } else {
913 1.151 chs lid = 0;
914 1.151 chs }
915 1.151 chs
916 1.103 ad mutex_enter(p2->p_lock);
917 1.52 ad
918 1.52 ad if ((flags & LWP_DETACHED) != 0) {
919 1.52 ad l2->l_prflag = LPR_DETACHED;
920 1.52 ad p2->p_ndlwps++;
921 1.52 ad } else
922 1.52 ad l2->l_prflag = 0;
923 1.52 ad
924 1.188 christos l2->l_sigstk = *sigstk;
925 1.188 christos l2->l_sigmask = *sigmask;
926 1.176 christos TAILQ_INIT(&l2->l_sigpend.sp_info);
927 1.52 ad sigemptyset(&l2->l_sigpend.sp_set);
928 1.52 ad
929 1.174 dsl if (__predict_true(lid == 0)) {
930 1.174 dsl /*
931 1.174 dsl * XXX: l_lid are expected to be unique (for a process)
932 1.174 dsl * if LWP_PIDLID is sometimes set this won't be true.
933 1.174 dsl * Once 2^31 threads have been allocated we have to
934 1.174 dsl * scan to ensure we allocate a unique value.
935 1.174 dsl */
936 1.174 dsl lid = ++p2->p_nlwpid;
937 1.174 dsl if (__predict_false(lid & LID_SCAN)) {
938 1.174 dsl lid = lwp_find_free_lid(lid, l2, p2);
939 1.174 dsl p2->p_nlwpid = lid | LID_SCAN;
940 1.174 dsl /* l2 as been inserted into p_lwps in order */
941 1.174 dsl goto skip_insert;
942 1.174 dsl }
943 1.174 dsl p2->p_nlwpid = lid;
944 1.151 chs }
945 1.174 dsl LIST_INSERT_HEAD(&p2->p_lwps, l2, l_sibling);
946 1.174 dsl skip_insert:
947 1.151 chs l2->l_lid = lid;
948 1.2 thorpej p2->p_nlwps++;
949 1.149 yamt p2->p_nrlwps++;
950 1.2 thorpej
951 1.162 rmind KASSERT(l2->l_affinity == NULL);
952 1.162 rmind
953 1.210 ad /* Inherit the affinity mask. */
954 1.210 ad if (l1->l_affinity) {
955 1.210 ad /*
956 1.210 ad * Note that we hold the state lock while inheriting
957 1.210 ad * the affinity to avoid race with sched_setaffinity().
958 1.210 ad */
959 1.210 ad lwp_lock(l1);
960 1.162 rmind if (l1->l_affinity) {
961 1.210 ad kcpuset_use(l1->l_affinity);
962 1.210 ad l2->l_affinity = l1->l_affinity;
963 1.117 christos }
964 1.210 ad lwp_unlock(l1);
965 1.91 rmind }
966 1.128 rmind mutex_exit(p2->p_lock);
967 1.128 rmind
968 1.180 christos SDT_PROBE(proc, kernel, , lwp__create, l2, 0, 0, 0, 0);
969 1.141 darran
970 1.128 rmind mutex_enter(proc_lock);
971 1.128 rmind LIST_INSERT_HEAD(&alllwp, l2, l_list);
972 1.210 ad /* Inherit a processor-set */
973 1.210 ad l2->l_psid = l1->l_psid;
974 1.128 rmind mutex_exit(proc_lock);
975 1.91 rmind
976 1.57 dsl SYSCALL_TIME_LWP_INIT(l2);
977 1.57 dsl
978 1.16 manu if (p2->p_emul->e_lwp_fork)
979 1.16 manu (*p2->p_emul->e_lwp_fork)(l1, l2);
980 1.16 manu
981 1.2 thorpej return (0);
982 1.2 thorpej }
983 1.2 thorpej
984 1.2 thorpej /*
985 1.212 ad * Set a new LWP running. If the process is stopping, then the LWP is
986 1.212 ad * created stopped.
987 1.212 ad */
988 1.212 ad void
989 1.212 ad lwp_start(lwp_t *l, int flags)
990 1.212 ad {
991 1.212 ad proc_t *p = l->l_proc;
992 1.212 ad
993 1.212 ad mutex_enter(p->p_lock);
994 1.212 ad lwp_lock(l);
995 1.212 ad KASSERT(l->l_stat == LSIDL);
996 1.212 ad if ((flags & LWP_SUSPENDED) != 0) {
997 1.212 ad /* It'll suspend itself in lwp_userret(). */
998 1.212 ad l->l_flag |= LW_WSUSPEND;
999 1.212 ad }
1000 1.212 ad if (p->p_stat == SSTOP || (p->p_sflag & PS_STOPPING) != 0) {
1001 1.212 ad KASSERT(l->l_wchan == NULL);
1002 1.212 ad l->l_stat = LSSTOP;
1003 1.212 ad p->p_nrlwps--;
1004 1.212 ad lwp_unlock(l);
1005 1.212 ad } else {
1006 1.212 ad setrunnable(l);
1007 1.212 ad /* LWP now unlocked */
1008 1.212 ad }
1009 1.212 ad mutex_exit(p->p_lock);
1010 1.212 ad }
1011 1.212 ad
1012 1.212 ad /*
1013 1.64 yamt * Called by MD code when a new LWP begins execution. Must be called
1014 1.64 yamt * with the previous LWP locked (so at splsched), or if there is no
1015 1.64 yamt * previous LWP, at splsched.
1016 1.64 yamt */
1017 1.64 yamt void
1018 1.178 matt lwp_startup(struct lwp *prev, struct lwp *new_lwp)
1019 1.64 yamt {
1020 1.218 ad
1021 1.178 matt KASSERTMSG(new_lwp == curlwp, "l %p curlwp %p prevlwp %p", new_lwp, curlwp, prev);
1022 1.218 ad KASSERT(kpreempt_disabled());
1023 1.218 ad KASSERT(prev != NULL);
1024 1.218 ad KASSERT((prev->l_flag & LW_RUNNING) != 0);
1025 1.218 ad KASSERT(curcpu()->ci_mtx_count == -2);
1026 1.218 ad
1027 1.218 ad /* Immediately mark previous LWP as no longer running, and unlock. */
1028 1.218 ad prev->l_flag &= ~LW_RUNNING;
1029 1.218 ad lwp_unlock(prev);
1030 1.64 yamt
1031 1.218 ad /* Correct spin mutex count after mi_switch(). */
1032 1.218 ad curcpu()->ci_mtx_count = 0;
1033 1.141 darran
1034 1.218 ad /* Install new VM context. */
1035 1.218 ad if (__predict_true(new_lwp->l_proc->p_vmspace)) {
1036 1.218 ad pmap_activate(new_lwp);
1037 1.64 yamt }
1038 1.218 ad
1039 1.218 ad /* We remain at IPL_SCHED from mi_switch() - reset it. */
1040 1.181 skrll spl0();
1041 1.161 christos
1042 1.64 yamt LOCKDEBUG_BARRIER(NULL, 0);
1043 1.218 ad SDT_PROBE(proc, kernel, , lwp__start, new_lwp, 0, 0, 0, 0);
1044 1.218 ad
1045 1.218 ad /* For kthreads, acquire kernel lock if not MPSAFE. */
1046 1.218 ad if (__predict_false((new_lwp->l_pflag & LP_MPSAFE) == 0)) {
1047 1.178 matt KERNEL_LOCK(1, new_lwp);
1048 1.65 ad }
1049 1.64 yamt }
1050 1.64 yamt
1051 1.64 yamt /*
1052 1.65 ad * Exit an LWP.
1053 1.2 thorpej */
1054 1.2 thorpej void
1055 1.2 thorpej lwp_exit(struct lwp *l)
1056 1.2 thorpej {
1057 1.2 thorpej struct proc *p = l->l_proc;
1058 1.52 ad struct lwp *l2;
1059 1.65 ad bool current;
1060 1.65 ad
1061 1.65 ad current = (l == curlwp);
1062 1.2 thorpej
1063 1.114 rmind KASSERT(current || (l->l_stat == LSIDL && l->l_target_cpu == NULL));
1064 1.131 ad KASSERT(p == curproc);
1065 1.2 thorpej
1066 1.180 christos SDT_PROBE(proc, kernel, , lwp__exit, l, 0, 0, 0, 0);
1067 1.141 darran
1068 1.220 ad /* Verify that we hold no locks; for DIAGNOSTIC check kernel_lock. */
1069 1.218 ad LOCKDEBUG_BARRIER(NULL, 0);
1070 1.220 ad KASSERTMSG(curcpu()->ci_biglock_count == 0, "kernel_lock leaked");
1071 1.16 manu
1072 1.2 thorpej /*
1073 1.52 ad * If we are the last live LWP in a process, we need to exit the
1074 1.52 ad * entire process. We do so with an exit status of zero, because
1075 1.52 ad * it's a "controlled" exit, and because that's what Solaris does.
1076 1.52 ad *
1077 1.52 ad * We are not quite a zombie yet, but for accounting purposes we
1078 1.52 ad * must increment the count of zombies here.
1079 1.45 thorpej *
1080 1.45 thorpej * Note: the last LWP's specificdata will be deleted here.
1081 1.2 thorpej */
1082 1.103 ad mutex_enter(p->p_lock);
1083 1.52 ad if (p->p_nlwps - p->p_nzlwps == 1) {
1084 1.65 ad KASSERT(current == true);
1085 1.172 matt KASSERT(p != &proc0);
1086 1.184 christos exit1(l, 0, 0);
1087 1.19 jdolecek /* NOTREACHED */
1088 1.2 thorpej }
1089 1.52 ad p->p_nzlwps++;
1090 1.103 ad mutex_exit(p->p_lock);
1091 1.52 ad
1092 1.52 ad if (p->p_emul->e_lwp_exit)
1093 1.52 ad (*p->p_emul->e_lwp_exit)(l);
1094 1.2 thorpej
1095 1.131 ad /* Drop filedesc reference. */
1096 1.131 ad fd_free();
1097 1.131 ad
1098 1.196 hannken /* Release fstrans private data. */
1099 1.196 hannken fstrans_lwp_dtor(l);
1100 1.196 hannken
1101 1.45 thorpej /* Delete the specificdata while it's still safe to sleep. */
1102 1.145 pooka lwp_finispecific(l);
1103 1.45 thorpej
1104 1.52 ad /*
1105 1.52 ad * Release our cached credentials.
1106 1.52 ad */
1107 1.37 ad kauth_cred_free(l->l_cred);
1108 1.70 ad callout_destroy(&l->l_timeout_ch);
1109 1.65 ad
1110 1.65 ad /*
1111 1.198 kamil * If traced, report LWP exit event to the debugger.
1112 1.198 kamil *
1113 1.52 ad * Remove the LWP from the global list.
1114 1.151 chs * Free its LID from the PID namespace if needed.
1115 1.52 ad */
1116 1.102 ad mutex_enter(proc_lock);
1117 1.198 kamil
1118 1.199 kamil if ((p->p_slflag & (PSL_TRACED|PSL_TRACELWP_EXIT)) ==
1119 1.198 kamil (PSL_TRACED|PSL_TRACELWP_EXIT)) {
1120 1.198 kamil mutex_enter(p->p_lock);
1121 1.202 kamil if (ISSET(p->p_sflag, PS_WEXIT)) {
1122 1.202 kamil mutex_exit(p->p_lock);
1123 1.202 kamil /*
1124 1.202 kamil * We are exiting, bail out without informing parent
1125 1.202 kamil * about a terminating LWP as it would deadlock.
1126 1.202 kamil */
1127 1.202 kamil } else {
1128 1.203 kamil eventswitch(TRAP_LWP, PTRACE_LWP_EXIT, l->l_lid);
1129 1.202 kamil mutex_enter(proc_lock);
1130 1.202 kamil }
1131 1.198 kamil }
1132 1.198 kamil
1133 1.52 ad LIST_REMOVE(l, l_list);
1134 1.151 chs if ((l->l_pflag & LP_PIDLID) != 0 && l->l_lid != p->p_pid) {
1135 1.151 chs proc_free_pid(l->l_lid);
1136 1.151 chs }
1137 1.102 ad mutex_exit(proc_lock);
1138 1.19 jdolecek
1139 1.52 ad /*
1140 1.52 ad * Get rid of all references to the LWP that others (e.g. procfs)
1141 1.52 ad * may have, and mark the LWP as a zombie. If the LWP is detached,
1142 1.52 ad * mark it waiting for collection in the proc structure. Note that
1143 1.52 ad * before we can do that, we need to free any other dead, deatched
1144 1.52 ad * LWP waiting to meet its maker.
1145 1.52 ad */
1146 1.103 ad mutex_enter(p->p_lock);
1147 1.52 ad lwp_drainrefs(l);
1148 1.31 yamt
1149 1.52 ad if ((l->l_prflag & LPR_DETACHED) != 0) {
1150 1.52 ad while ((l2 = p->p_zomblwp) != NULL) {
1151 1.52 ad p->p_zomblwp = NULL;
1152 1.63 ad lwp_free(l2, false, false);/* releases proc mutex */
1153 1.103 ad mutex_enter(p->p_lock);
1154 1.72 ad l->l_refcnt++;
1155 1.72 ad lwp_drainrefs(l);
1156 1.52 ad }
1157 1.52 ad p->p_zomblwp = l;
1158 1.52 ad }
1159 1.31 yamt
1160 1.52 ad /*
1161 1.52 ad * If we find a pending signal for the process and we have been
1162 1.151 chs * asked to check for signals, then we lose: arrange to have
1163 1.52 ad * all other LWPs in the process check for signals.
1164 1.52 ad */
1165 1.56 pavel if ((l->l_flag & LW_PENDSIG) != 0 &&
1166 1.52 ad firstsig(&p->p_sigpend.sp_set) != 0) {
1167 1.52 ad LIST_FOREACH(l2, &p->p_lwps, l_sibling) {
1168 1.52 ad lwp_lock(l2);
1169 1.209 ad signotify(l2);
1170 1.52 ad lwp_unlock(l2);
1171 1.52 ad }
1172 1.31 yamt }
1173 1.31 yamt
1174 1.158 matt /*
1175 1.158 matt * Release any PCU resources before becoming a zombie.
1176 1.158 matt */
1177 1.158 matt pcu_discard_all(l);
1178 1.158 matt
1179 1.52 ad lwp_lock(l);
1180 1.52 ad l->l_stat = LSZOMB;
1181 1.162 rmind if (l->l_name != NULL) {
1182 1.90 ad strcpy(l->l_name, "(zombie)");
1183 1.128 rmind }
1184 1.52 ad lwp_unlock(l);
1185 1.2 thorpej p->p_nrlwps--;
1186 1.52 ad cv_broadcast(&p->p_lwpcv);
1187 1.78 ad if (l->l_lwpctl != NULL)
1188 1.78 ad l->l_lwpctl->lc_curcpu = LWPCTL_CPU_EXITED;
1189 1.103 ad mutex_exit(p->p_lock);
1190 1.52 ad
1191 1.52 ad /*
1192 1.52 ad * We can no longer block. At this point, lwp_free() may already
1193 1.52 ad * be gunning for us. On a multi-CPU system, we may be off p_lwps.
1194 1.52 ad *
1195 1.52 ad * Free MD LWP resources.
1196 1.52 ad */
1197 1.52 ad cpu_lwp_free(l, 0);
1198 1.2 thorpej
1199 1.65 ad if (current) {
1200 1.218 ad /* For the LW_RUNNING check in lwp_free(). */
1201 1.218 ad membar_exit();
1202 1.218 ad /* Switch away into oblivion. */
1203 1.218 ad lwp_lock(l);
1204 1.218 ad spc_lock(l->l_cpu);
1205 1.218 ad mi_switch(l);
1206 1.218 ad panic("lwp_exit");
1207 1.65 ad }
1208 1.2 thorpej }
1209 1.2 thorpej
1210 1.52 ad /*
1211 1.52 ad * Free a dead LWP's remaining resources.
1212 1.52 ad *
1213 1.52 ad * XXXLWP limits.
1214 1.52 ad */
1215 1.52 ad void
1216 1.63 ad lwp_free(struct lwp *l, bool recycle, bool last)
1217 1.52 ad {
1218 1.52 ad struct proc *p = l->l_proc;
1219 1.100 ad struct rusage *ru;
1220 1.52 ad ksiginfoq_t kq;
1221 1.52 ad
1222 1.92 yamt KASSERT(l != curlwp);
1223 1.160 yamt KASSERT(last || mutex_owned(p->p_lock));
1224 1.92 yamt
1225 1.177 christos /*
1226 1.177 christos * We use the process credentials instead of the lwp credentials here
1227 1.177 christos * because the lwp credentials maybe cached (just after a setuid call)
1228 1.177 christos * and we don't want pay for syncing, since the lwp is going away
1229 1.177 christos * anyway
1230 1.177 christos */
1231 1.169 christos if (p != &proc0 && p->p_nlwps != 1)
1232 1.177 christos (void)chglwpcnt(kauth_cred_getuid(p->p_cred), -1);
1233 1.218 ad
1234 1.52 ad /*
1235 1.52 ad * If this was not the last LWP in the process, then adjust
1236 1.52 ad * counters and unlock.
1237 1.52 ad */
1238 1.52 ad if (!last) {
1239 1.52 ad /*
1240 1.52 ad * Add the LWP's run time to the process' base value.
1241 1.52 ad * This needs to co-incide with coming off p_lwps.
1242 1.52 ad */
1243 1.86 yamt bintime_add(&p->p_rtime, &l->l_rtime);
1244 1.64 yamt p->p_pctcpu += l->l_pctcpu;
1245 1.100 ad ru = &p->p_stats->p_ru;
1246 1.100 ad ruadd(ru, &l->l_ru);
1247 1.100 ad ru->ru_nvcsw += (l->l_ncsw - l->l_nivcsw);
1248 1.100 ad ru->ru_nivcsw += l->l_nivcsw;
1249 1.52 ad LIST_REMOVE(l, l_sibling);
1250 1.52 ad p->p_nlwps--;
1251 1.52 ad p->p_nzlwps--;
1252 1.52 ad if ((l->l_prflag & LPR_DETACHED) != 0)
1253 1.52 ad p->p_ndlwps--;
1254 1.63 ad
1255 1.63 ad /*
1256 1.63 ad * Have any LWPs sleeping in lwp_wait() recheck for
1257 1.63 ad * deadlock.
1258 1.63 ad */
1259 1.63 ad cv_broadcast(&p->p_lwpcv);
1260 1.103 ad mutex_exit(p->p_lock);
1261 1.63 ad }
1262 1.52 ad
1263 1.52 ad #ifdef MULTIPROCESSOR
1264 1.63 ad /*
1265 1.63 ad * In the unlikely event that the LWP is still on the CPU,
1266 1.63 ad * then spin until it has switched away. We need to release
1267 1.63 ad * all locks to avoid deadlock against interrupt handlers on
1268 1.63 ad * the target CPU.
1269 1.63 ad */
1270 1.218 ad membar_enter();
1271 1.219 ad while (__predict_false((l->l_flag & LW_RUNNING) != 0)) {
1272 1.219 ad SPINLOCK_BACKOFF_HOOK;
1273 1.63 ad }
1274 1.52 ad #endif
1275 1.52 ad
1276 1.52 ad /*
1277 1.52 ad * Destroy the LWP's remaining signal information.
1278 1.52 ad */
1279 1.52 ad ksiginfo_queue_init(&kq);
1280 1.52 ad sigclear(&l->l_sigpend, NULL, &kq);
1281 1.52 ad ksiginfo_queue_drain(&kq);
1282 1.52 ad cv_destroy(&l->l_sigcv);
1283 1.171 rmind cv_destroy(&l->l_waitcv);
1284 1.2 thorpej
1285 1.19 jdolecek /*
1286 1.162 rmind * Free lwpctl structure and affinity.
1287 1.162 rmind */
1288 1.162 rmind if (l->l_lwpctl) {
1289 1.162 rmind lwp_ctl_free(l);
1290 1.162 rmind }
1291 1.162 rmind if (l->l_affinity) {
1292 1.162 rmind kcpuset_unuse(l->l_affinity, NULL);
1293 1.162 rmind l->l_affinity = NULL;
1294 1.162 rmind }
1295 1.162 rmind
1296 1.162 rmind /*
1297 1.52 ad * Free the LWP's turnstile and the LWP structure itself unless the
1298 1.93 yamt * caller wants to recycle them. Also, free the scheduler specific
1299 1.93 yamt * data.
1300 1.52 ad *
1301 1.52 ad * We can't return turnstile0 to the pool (it didn't come from it),
1302 1.52 ad * so if it comes up just drop it quietly and move on.
1303 1.52 ad *
1304 1.52 ad * We don't recycle the VM resources at this time.
1305 1.19 jdolecek */
1306 1.64 yamt
1307 1.52 ad if (!recycle && l->l_ts != &turnstile0)
1308 1.76 ad pool_cache_put(turnstile_cache, l->l_ts);
1309 1.90 ad if (l->l_name != NULL)
1310 1.90 ad kmem_free(l->l_name, MAXCOMLEN);
1311 1.135 rmind
1312 1.208 maxv kmsan_lwp_free(l);
1313 1.52 ad cpu_lwp_free2(l);
1314 1.19 jdolecek uvm_lwp_exit(l);
1315 1.134 rmind
1316 1.60 yamt KASSERT(SLIST_EMPTY(&l->l_pi_lenders));
1317 1.75 ad KASSERT(l->l_inheritedprio == -1);
1318 1.155 matt KASSERT(l->l_blcnt == 0);
1319 1.138 darran kdtrace_thread_dtor(NULL, l);
1320 1.52 ad if (!recycle)
1321 1.87 ad pool_cache_put(lwp_cache, l);
1322 1.2 thorpej }
1323 1.2 thorpej
1324 1.2 thorpej /*
1325 1.91 rmind * Migrate the LWP to the another CPU. Unlocks the LWP.
1326 1.91 rmind */
1327 1.91 rmind void
1328 1.114 rmind lwp_migrate(lwp_t *l, struct cpu_info *tci)
1329 1.91 rmind {
1330 1.114 rmind struct schedstate_percpu *tspc;
1331 1.121 rmind int lstat = l->l_stat;
1332 1.121 rmind
1333 1.91 rmind KASSERT(lwp_locked(l, NULL));
1334 1.114 rmind KASSERT(tci != NULL);
1335 1.114 rmind
1336 1.121 rmind /* If LWP is still on the CPU, it must be handled like LSONPROC */
1337 1.218 ad if ((l->l_flag & LW_RUNNING) != 0) {
1338 1.121 rmind lstat = LSONPROC;
1339 1.121 rmind }
1340 1.121 rmind
1341 1.114 rmind /*
1342 1.114 rmind * The destination CPU could be changed while previous migration
1343 1.114 rmind * was not finished.
1344 1.114 rmind */
1345 1.121 rmind if (l->l_target_cpu != NULL) {
1346 1.114 rmind l->l_target_cpu = tci;
1347 1.114 rmind lwp_unlock(l);
1348 1.114 rmind return;
1349 1.114 rmind }
1350 1.91 rmind
1351 1.114 rmind /* Nothing to do if trying to migrate to the same CPU */
1352 1.114 rmind if (l->l_cpu == tci) {
1353 1.91 rmind lwp_unlock(l);
1354 1.91 rmind return;
1355 1.91 rmind }
1356 1.91 rmind
1357 1.114 rmind KASSERT(l->l_target_cpu == NULL);
1358 1.114 rmind tspc = &tci->ci_schedstate;
1359 1.121 rmind switch (lstat) {
1360 1.91 rmind case LSRUN:
1361 1.134 rmind l->l_target_cpu = tci;
1362 1.134 rmind break;
1363 1.91 rmind case LSSLEEP:
1364 1.114 rmind l->l_cpu = tci;
1365 1.91 rmind break;
1366 1.212 ad case LSIDL:
1367 1.91 rmind case LSSTOP:
1368 1.91 rmind case LSSUSPENDED:
1369 1.114 rmind l->l_cpu = tci;
1370 1.114 rmind if (l->l_wchan == NULL) {
1371 1.114 rmind lwp_unlock_to(l, tspc->spc_lwplock);
1372 1.114 rmind return;
1373 1.91 rmind }
1374 1.114 rmind break;
1375 1.91 rmind case LSONPROC:
1376 1.114 rmind l->l_target_cpu = tci;
1377 1.114 rmind spc_lock(l->l_cpu);
1378 1.212 ad sched_resched_cpu(l->l_cpu, PRI_USER_RT, true);
1379 1.212 ad /* spc now unlocked */
1380 1.91 rmind break;
1381 1.91 rmind }
1382 1.91 rmind lwp_unlock(l);
1383 1.91 rmind }
1384 1.91 rmind
1385 1.91 rmind /*
1386 1.94 rmind * Find the LWP in the process. Arguments may be zero, in such case,
1387 1.94 rmind * the calling process and first LWP in the list will be used.
1388 1.103 ad * On success - returns proc locked.
1389 1.91 rmind */
1390 1.91 rmind struct lwp *
1391 1.91 rmind lwp_find2(pid_t pid, lwpid_t lid)
1392 1.91 rmind {
1393 1.91 rmind proc_t *p;
1394 1.91 rmind lwp_t *l;
1395 1.91 rmind
1396 1.150 rmind /* Find the process. */
1397 1.94 rmind if (pid != 0) {
1398 1.150 rmind mutex_enter(proc_lock);
1399 1.150 rmind p = proc_find(pid);
1400 1.150 rmind if (p == NULL) {
1401 1.150 rmind mutex_exit(proc_lock);
1402 1.150 rmind return NULL;
1403 1.150 rmind }
1404 1.150 rmind mutex_enter(p->p_lock);
1405 1.102 ad mutex_exit(proc_lock);
1406 1.150 rmind } else {
1407 1.150 rmind p = curlwp->l_proc;
1408 1.150 rmind mutex_enter(p->p_lock);
1409 1.150 rmind }
1410 1.150 rmind /* Find the thread. */
1411 1.150 rmind if (lid != 0) {
1412 1.150 rmind l = lwp_find(p, lid);
1413 1.150 rmind } else {
1414 1.150 rmind l = LIST_FIRST(&p->p_lwps);
1415 1.94 rmind }
1416 1.103 ad if (l == NULL) {
1417 1.103 ad mutex_exit(p->p_lock);
1418 1.103 ad }
1419 1.91 rmind return l;
1420 1.91 rmind }
1421 1.91 rmind
1422 1.91 rmind /*
1423 1.168 yamt * Look up a live LWP within the specified process.
1424 1.52 ad *
1425 1.103 ad * Must be called with p->p_lock held.
1426 1.52 ad */
1427 1.52 ad struct lwp *
1428 1.151 chs lwp_find(struct proc *p, lwpid_t id)
1429 1.52 ad {
1430 1.52 ad struct lwp *l;
1431 1.52 ad
1432 1.103 ad KASSERT(mutex_owned(p->p_lock));
1433 1.52 ad
1434 1.52 ad LIST_FOREACH(l, &p->p_lwps, l_sibling) {
1435 1.52 ad if (l->l_lid == id)
1436 1.52 ad break;
1437 1.52 ad }
1438 1.52 ad
1439 1.52 ad /*
1440 1.52 ad * No need to lock - all of these conditions will
1441 1.52 ad * be visible with the process level mutex held.
1442 1.52 ad */
1443 1.52 ad if (l != NULL && (l->l_stat == LSIDL || l->l_stat == LSZOMB))
1444 1.52 ad l = NULL;
1445 1.52 ad
1446 1.52 ad return l;
1447 1.52 ad }
1448 1.52 ad
1449 1.52 ad /*
1450 1.37 ad * Update an LWP's cached credentials to mirror the process' master copy.
1451 1.37 ad *
1452 1.37 ad * This happens early in the syscall path, on user trap, and on LWP
1453 1.37 ad * creation. A long-running LWP can also voluntarily choose to update
1454 1.179 snj * its credentials by calling this routine. This may be called from
1455 1.37 ad * LWP_CACHE_CREDS(), which checks l->l_cred != p->p_cred beforehand.
1456 1.37 ad */
1457 1.37 ad void
1458 1.37 ad lwp_update_creds(struct lwp *l)
1459 1.37 ad {
1460 1.37 ad kauth_cred_t oc;
1461 1.37 ad struct proc *p;
1462 1.37 ad
1463 1.37 ad p = l->l_proc;
1464 1.37 ad oc = l->l_cred;
1465 1.37 ad
1466 1.103 ad mutex_enter(p->p_lock);
1467 1.37 ad kauth_cred_hold(p->p_cred);
1468 1.37 ad l->l_cred = p->p_cred;
1469 1.98 ad l->l_prflag &= ~LPR_CRMOD;
1470 1.103 ad mutex_exit(p->p_lock);
1471 1.88 ad if (oc != NULL)
1472 1.37 ad kauth_cred_free(oc);
1473 1.52 ad }
1474 1.52 ad
1475 1.52 ad /*
1476 1.52 ad * Verify that an LWP is locked, and optionally verify that the lock matches
1477 1.52 ad * one we specify.
1478 1.52 ad */
1479 1.52 ad int
1480 1.52 ad lwp_locked(struct lwp *l, kmutex_t *mtx)
1481 1.52 ad {
1482 1.52 ad kmutex_t *cur = l->l_mutex;
1483 1.52 ad
1484 1.52 ad return mutex_owned(cur) && (mtx == cur || mtx == NULL);
1485 1.52 ad }
1486 1.52 ad
1487 1.52 ad /*
1488 1.52 ad * Lend a new mutex to an LWP. The old mutex must be held.
1489 1.52 ad */
1490 1.211 ad kmutex_t *
1491 1.178 matt lwp_setlock(struct lwp *l, kmutex_t *mtx)
1492 1.52 ad {
1493 1.211 ad kmutex_t *oldmtx = l->l_mutex;
1494 1.52 ad
1495 1.211 ad KASSERT(mutex_owned(oldmtx));
1496 1.52 ad
1497 1.107 ad membar_exit();
1498 1.178 matt l->l_mutex = mtx;
1499 1.211 ad return oldmtx;
1500 1.52 ad }
1501 1.52 ad
1502 1.52 ad /*
1503 1.52 ad * Lend a new mutex to an LWP, and release the old mutex. The old mutex
1504 1.52 ad * must be held.
1505 1.52 ad */
1506 1.52 ad void
1507 1.178 matt lwp_unlock_to(struct lwp *l, kmutex_t *mtx)
1508 1.52 ad {
1509 1.52 ad kmutex_t *old;
1510 1.52 ad
1511 1.152 rmind KASSERT(lwp_locked(l, NULL));
1512 1.52 ad
1513 1.52 ad old = l->l_mutex;
1514 1.107 ad membar_exit();
1515 1.178 matt l->l_mutex = mtx;
1516 1.52 ad mutex_spin_exit(old);
1517 1.52 ad }
1518 1.52 ad
1519 1.60 yamt int
1520 1.60 yamt lwp_trylock(struct lwp *l)
1521 1.60 yamt {
1522 1.60 yamt kmutex_t *old;
1523 1.60 yamt
1524 1.60 yamt for (;;) {
1525 1.60 yamt if (!mutex_tryenter(old = l->l_mutex))
1526 1.60 yamt return 0;
1527 1.60 yamt if (__predict_true(l->l_mutex == old))
1528 1.60 yamt return 1;
1529 1.60 yamt mutex_spin_exit(old);
1530 1.60 yamt }
1531 1.60 yamt }
1532 1.60 yamt
1533 1.134 rmind void
1534 1.211 ad lwp_unsleep(lwp_t *l, bool unlock)
1535 1.96 ad {
1536 1.96 ad
1537 1.96 ad KASSERT(mutex_owned(l->l_mutex));
1538 1.211 ad (*l->l_syncobj->sobj_unsleep)(l, unlock);
1539 1.96 ad }
1540 1.96 ad
1541 1.52 ad /*
1542 1.56 pavel * Handle exceptions for mi_userret(). Called if a member of LW_USERRET is
1543 1.52 ad * set.
1544 1.52 ad */
1545 1.52 ad void
1546 1.52 ad lwp_userret(struct lwp *l)
1547 1.52 ad {
1548 1.52 ad struct proc *p;
1549 1.52 ad int sig;
1550 1.52 ad
1551 1.114 rmind KASSERT(l == curlwp);
1552 1.114 rmind KASSERT(l->l_stat == LSONPROC);
1553 1.52 ad p = l->l_proc;
1554 1.52 ad
1555 1.75 ad #ifndef __HAVE_FAST_SOFTINTS
1556 1.75 ad /* Run pending soft interrupts. */
1557 1.75 ad if (l->l_cpu->ci_data.cpu_softints != 0)
1558 1.75 ad softint_overlay();
1559 1.75 ad #endif
1560 1.75 ad
1561 1.52 ad /*
1562 1.167 rmind * It is safe to do this read unlocked on a MP system..
1563 1.52 ad */
1564 1.167 rmind while ((l->l_flag & LW_USERRET) != 0) {
1565 1.52 ad /*
1566 1.52 ad * Process pending signals first, unless the process
1567 1.61 ad * is dumping core or exiting, where we will instead
1568 1.101 rmind * enter the LW_WSUSPEND case below.
1569 1.52 ad */
1570 1.61 ad if ((l->l_flag & (LW_PENDSIG | LW_WCORE | LW_WEXIT)) ==
1571 1.61 ad LW_PENDSIG) {
1572 1.103 ad mutex_enter(p->p_lock);
1573 1.52 ad while ((sig = issignal(l)) != 0)
1574 1.52 ad postsig(sig);
1575 1.103 ad mutex_exit(p->p_lock);
1576 1.52 ad }
1577 1.52 ad
1578 1.52 ad /*
1579 1.52 ad * Core-dump or suspend pending.
1580 1.52 ad *
1581 1.159 matt * In case of core dump, suspend ourselves, so that the kernel
1582 1.159 matt * stack and therefore the userland registers saved in the
1583 1.159 matt * trapframe are around for coredump() to write them out.
1584 1.159 matt * We also need to save any PCU resources that we have so that
1585 1.159 matt * they accessible for coredump(). We issue a wakeup on
1586 1.159 matt * p->p_lwpcv so that sigexit() will write the core file out
1587 1.159 matt * once all other LWPs are suspended.
1588 1.52 ad */
1589 1.56 pavel if ((l->l_flag & LW_WSUSPEND) != 0) {
1590 1.159 matt pcu_save_all(l);
1591 1.103 ad mutex_enter(p->p_lock);
1592 1.52 ad p->p_nrlwps--;
1593 1.52 ad cv_broadcast(&p->p_lwpcv);
1594 1.52 ad lwp_lock(l);
1595 1.52 ad l->l_stat = LSSUSPENDED;
1596 1.104 ad lwp_unlock(l);
1597 1.103 ad mutex_exit(p->p_lock);
1598 1.104 ad lwp_lock(l);
1599 1.217 ad spc_lock(l->l_cpu);
1600 1.64 yamt mi_switch(l);
1601 1.52 ad }
1602 1.52 ad
1603 1.52 ad /* Process is exiting. */
1604 1.56 pavel if ((l->l_flag & LW_WEXIT) != 0) {
1605 1.52 ad lwp_exit(l);
1606 1.52 ad KASSERT(0);
1607 1.52 ad /* NOTREACHED */
1608 1.52 ad }
1609 1.156 pooka
1610 1.156 pooka /* update lwpctl processor (for vfork child_return) */
1611 1.156 pooka if (l->l_flag & LW_LWPCTL) {
1612 1.156 pooka lwp_lock(l);
1613 1.156 pooka KASSERT(kpreempt_disabled());
1614 1.156 pooka l->l_lwpctl->lc_curcpu = (int)cpu_index(l->l_cpu);
1615 1.156 pooka l->l_lwpctl->lc_pctr++;
1616 1.156 pooka l->l_flag &= ~LW_LWPCTL;
1617 1.156 pooka lwp_unlock(l);
1618 1.156 pooka }
1619 1.52 ad }
1620 1.52 ad }
1621 1.52 ad
1622 1.52 ad /*
1623 1.52 ad * Force an LWP to enter the kernel, to take a trip through lwp_userret().
1624 1.52 ad */
1625 1.52 ad void
1626 1.52 ad lwp_need_userret(struct lwp *l)
1627 1.52 ad {
1628 1.209 ad
1629 1.209 ad KASSERT(!cpu_intr_p());
1630 1.63 ad KASSERT(lwp_locked(l, NULL));
1631 1.52 ad
1632 1.52 ad /*
1633 1.209 ad * If the LWP is in any state other than LSONPROC, we know that it
1634 1.209 ad * is executing in-kernel and will hit userret() on the way out.
1635 1.209 ad *
1636 1.209 ad * If the LWP is curlwp, then we know we'll be back out to userspace
1637 1.209 ad * soon (can't be called from a hardware interrupt here).
1638 1.209 ad *
1639 1.209 ad * Otherwise, we can't be sure what the LWP is doing, so first make
1640 1.209 ad * sure the update to l_flag will be globally visible, and then
1641 1.209 ad * force the LWP to take a trip through trap() where it will do
1642 1.209 ad * userret().
1643 1.209 ad */
1644 1.209 ad if (l->l_stat == LSONPROC && l != curlwp) {
1645 1.209 ad membar_producer();
1646 1.209 ad cpu_signotify(l);
1647 1.209 ad }
1648 1.52 ad }
1649 1.52 ad
1650 1.52 ad /*
1651 1.52 ad * Add one reference to an LWP. This will prevent the LWP from
1652 1.52 ad * exiting, thus keep the lwp structure and PCB around to inspect.
1653 1.52 ad */
1654 1.52 ad void
1655 1.52 ad lwp_addref(struct lwp *l)
1656 1.52 ad {
1657 1.52 ad
1658 1.103 ad KASSERT(mutex_owned(l->l_proc->p_lock));
1659 1.52 ad KASSERT(l->l_stat != LSZOMB);
1660 1.52 ad KASSERT(l->l_refcnt != 0);
1661 1.52 ad
1662 1.52 ad l->l_refcnt++;
1663 1.52 ad }
1664 1.52 ad
1665 1.52 ad /*
1666 1.52 ad * Remove one reference to an LWP. If this is the last reference,
1667 1.52 ad * then we must finalize the LWP's death.
1668 1.52 ad */
1669 1.52 ad void
1670 1.52 ad lwp_delref(struct lwp *l)
1671 1.52 ad {
1672 1.52 ad struct proc *p = l->l_proc;
1673 1.52 ad
1674 1.103 ad mutex_enter(p->p_lock);
1675 1.142 christos lwp_delref2(l);
1676 1.142 christos mutex_exit(p->p_lock);
1677 1.142 christos }
1678 1.142 christos
1679 1.142 christos /*
1680 1.142 christos * Remove one reference to an LWP. If this is the last reference,
1681 1.142 christos * then we must finalize the LWP's death. The proc mutex is held
1682 1.142 christos * on entry.
1683 1.142 christos */
1684 1.142 christos void
1685 1.142 christos lwp_delref2(struct lwp *l)
1686 1.142 christos {
1687 1.142 christos struct proc *p = l->l_proc;
1688 1.142 christos
1689 1.142 christos KASSERT(mutex_owned(p->p_lock));
1690 1.72 ad KASSERT(l->l_stat != LSZOMB);
1691 1.72 ad KASSERT(l->l_refcnt > 0);
1692 1.52 ad if (--l->l_refcnt == 0)
1693 1.76 ad cv_broadcast(&p->p_lwpcv);
1694 1.52 ad }
1695 1.52 ad
1696 1.52 ad /*
1697 1.52 ad * Drain all references to the current LWP.
1698 1.52 ad */
1699 1.52 ad void
1700 1.52 ad lwp_drainrefs(struct lwp *l)
1701 1.52 ad {
1702 1.52 ad struct proc *p = l->l_proc;
1703 1.52 ad
1704 1.103 ad KASSERT(mutex_owned(p->p_lock));
1705 1.52 ad KASSERT(l->l_refcnt != 0);
1706 1.52 ad
1707 1.52 ad l->l_refcnt--;
1708 1.52 ad while (l->l_refcnt != 0)
1709 1.103 ad cv_wait(&p->p_lwpcv, p->p_lock);
1710 1.37 ad }
1711 1.41 thorpej
1712 1.41 thorpej /*
1713 1.127 ad * Return true if the specified LWP is 'alive'. Only p->p_lock need
1714 1.127 ad * be held.
1715 1.127 ad */
1716 1.127 ad bool
1717 1.127 ad lwp_alive(lwp_t *l)
1718 1.127 ad {
1719 1.127 ad
1720 1.127 ad KASSERT(mutex_owned(l->l_proc->p_lock));
1721 1.127 ad
1722 1.127 ad switch (l->l_stat) {
1723 1.127 ad case LSSLEEP:
1724 1.127 ad case LSRUN:
1725 1.127 ad case LSONPROC:
1726 1.127 ad case LSSTOP:
1727 1.127 ad case LSSUSPENDED:
1728 1.127 ad return true;
1729 1.127 ad default:
1730 1.127 ad return false;
1731 1.127 ad }
1732 1.127 ad }
1733 1.127 ad
1734 1.127 ad /*
1735 1.127 ad * Return first live LWP in the process.
1736 1.127 ad */
1737 1.127 ad lwp_t *
1738 1.127 ad lwp_find_first(proc_t *p)
1739 1.127 ad {
1740 1.127 ad lwp_t *l;
1741 1.127 ad
1742 1.127 ad KASSERT(mutex_owned(p->p_lock));
1743 1.127 ad
1744 1.127 ad LIST_FOREACH(l, &p->p_lwps, l_sibling) {
1745 1.127 ad if (lwp_alive(l)) {
1746 1.127 ad return l;
1747 1.127 ad }
1748 1.127 ad }
1749 1.127 ad
1750 1.127 ad return NULL;
1751 1.127 ad }
1752 1.127 ad
1753 1.127 ad /*
1754 1.78 ad * Allocate a new lwpctl structure for a user LWP.
1755 1.78 ad */
1756 1.78 ad int
1757 1.78 ad lwp_ctl_alloc(vaddr_t *uaddr)
1758 1.78 ad {
1759 1.78 ad lcproc_t *lp;
1760 1.78 ad u_int bit, i, offset;
1761 1.78 ad struct uvm_object *uao;
1762 1.78 ad int error;
1763 1.78 ad lcpage_t *lcp;
1764 1.78 ad proc_t *p;
1765 1.78 ad lwp_t *l;
1766 1.78 ad
1767 1.78 ad l = curlwp;
1768 1.78 ad p = l->l_proc;
1769 1.78 ad
1770 1.156 pooka /* don't allow a vforked process to create lwp ctls */
1771 1.156 pooka if (p->p_lflag & PL_PPWAIT)
1772 1.156 pooka return EBUSY;
1773 1.156 pooka
1774 1.81 ad if (l->l_lcpage != NULL) {
1775 1.81 ad lcp = l->l_lcpage;
1776 1.81 ad *uaddr = lcp->lcp_uaddr + (vaddr_t)l->l_lwpctl - lcp->lcp_kaddr;
1777 1.143 njoly return 0;
1778 1.81 ad }
1779 1.78 ad
1780 1.78 ad /* First time around, allocate header structure for the process. */
1781 1.78 ad if ((lp = p->p_lwpctl) == NULL) {
1782 1.78 ad lp = kmem_alloc(sizeof(*lp), KM_SLEEP);
1783 1.78 ad mutex_init(&lp->lp_lock, MUTEX_DEFAULT, IPL_NONE);
1784 1.78 ad lp->lp_uao = NULL;
1785 1.78 ad TAILQ_INIT(&lp->lp_pages);
1786 1.103 ad mutex_enter(p->p_lock);
1787 1.78 ad if (p->p_lwpctl == NULL) {
1788 1.78 ad p->p_lwpctl = lp;
1789 1.103 ad mutex_exit(p->p_lock);
1790 1.78 ad } else {
1791 1.103 ad mutex_exit(p->p_lock);
1792 1.78 ad mutex_destroy(&lp->lp_lock);
1793 1.78 ad kmem_free(lp, sizeof(*lp));
1794 1.78 ad lp = p->p_lwpctl;
1795 1.78 ad }
1796 1.78 ad }
1797 1.78 ad
1798 1.78 ad /*
1799 1.78 ad * Set up an anonymous memory region to hold the shared pages.
1800 1.78 ad * Map them into the process' address space. The user vmspace
1801 1.78 ad * gets the first reference on the UAO.
1802 1.78 ad */
1803 1.78 ad mutex_enter(&lp->lp_lock);
1804 1.78 ad if (lp->lp_uao == NULL) {
1805 1.78 ad lp->lp_uao = uao_create(LWPCTL_UAREA_SZ, 0);
1806 1.78 ad lp->lp_cur = 0;
1807 1.78 ad lp->lp_max = LWPCTL_UAREA_SZ;
1808 1.78 ad lp->lp_uva = p->p_emul->e_vm_default_addr(p,
1809 1.182 martin (vaddr_t)p->p_vmspace->vm_daddr, LWPCTL_UAREA_SZ,
1810 1.182 martin p->p_vmspace->vm_map.flags & VM_MAP_TOPDOWN);
1811 1.78 ad error = uvm_map(&p->p_vmspace->vm_map, &lp->lp_uva,
1812 1.78 ad LWPCTL_UAREA_SZ, lp->lp_uao, 0, 0, UVM_MAPFLAG(UVM_PROT_RW,
1813 1.78 ad UVM_PROT_RW, UVM_INH_NONE, UVM_ADV_NORMAL, 0));
1814 1.78 ad if (error != 0) {
1815 1.78 ad uao_detach(lp->lp_uao);
1816 1.78 ad lp->lp_uao = NULL;
1817 1.78 ad mutex_exit(&lp->lp_lock);
1818 1.78 ad return error;
1819 1.78 ad }
1820 1.78 ad }
1821 1.78 ad
1822 1.78 ad /* Get a free block and allocate for this LWP. */
1823 1.78 ad TAILQ_FOREACH(lcp, &lp->lp_pages, lcp_chain) {
1824 1.78 ad if (lcp->lcp_nfree != 0)
1825 1.78 ad break;
1826 1.78 ad }
1827 1.78 ad if (lcp == NULL) {
1828 1.78 ad /* Nothing available - try to set up a free page. */
1829 1.78 ad if (lp->lp_cur == lp->lp_max) {
1830 1.78 ad mutex_exit(&lp->lp_lock);
1831 1.78 ad return ENOMEM;
1832 1.78 ad }
1833 1.78 ad lcp = kmem_alloc(LWPCTL_LCPAGE_SZ, KM_SLEEP);
1834 1.189 chs
1835 1.78 ad /*
1836 1.78 ad * Wire the next page down in kernel space. Since this
1837 1.78 ad * is a new mapping, we must add a reference.
1838 1.78 ad */
1839 1.78 ad uao = lp->lp_uao;
1840 1.78 ad (*uao->pgops->pgo_reference)(uao);
1841 1.99 ad lcp->lcp_kaddr = vm_map_min(kernel_map);
1842 1.78 ad error = uvm_map(kernel_map, &lcp->lcp_kaddr, PAGE_SIZE,
1843 1.78 ad uao, lp->lp_cur, PAGE_SIZE,
1844 1.78 ad UVM_MAPFLAG(UVM_PROT_RW, UVM_PROT_RW,
1845 1.78 ad UVM_INH_NONE, UVM_ADV_RANDOM, 0));
1846 1.78 ad if (error != 0) {
1847 1.78 ad mutex_exit(&lp->lp_lock);
1848 1.78 ad kmem_free(lcp, LWPCTL_LCPAGE_SZ);
1849 1.78 ad (*uao->pgops->pgo_detach)(uao);
1850 1.78 ad return error;
1851 1.78 ad }
1852 1.89 yamt error = uvm_map_pageable(kernel_map, lcp->lcp_kaddr,
1853 1.89 yamt lcp->lcp_kaddr + PAGE_SIZE, FALSE, 0);
1854 1.89 yamt if (error != 0) {
1855 1.89 yamt mutex_exit(&lp->lp_lock);
1856 1.89 yamt uvm_unmap(kernel_map, lcp->lcp_kaddr,
1857 1.89 yamt lcp->lcp_kaddr + PAGE_SIZE);
1858 1.89 yamt kmem_free(lcp, LWPCTL_LCPAGE_SZ);
1859 1.89 yamt return error;
1860 1.89 yamt }
1861 1.78 ad /* Prepare the page descriptor and link into the list. */
1862 1.78 ad lcp->lcp_uaddr = lp->lp_uva + lp->lp_cur;
1863 1.78 ad lp->lp_cur += PAGE_SIZE;
1864 1.78 ad lcp->lcp_nfree = LWPCTL_PER_PAGE;
1865 1.78 ad lcp->lcp_rotor = 0;
1866 1.78 ad memset(lcp->lcp_bitmap, 0xff, LWPCTL_BITMAP_SZ);
1867 1.78 ad TAILQ_INSERT_HEAD(&lp->lp_pages, lcp, lcp_chain);
1868 1.78 ad }
1869 1.78 ad for (i = lcp->lcp_rotor; lcp->lcp_bitmap[i] == 0;) {
1870 1.78 ad if (++i >= LWPCTL_BITMAP_ENTRIES)
1871 1.78 ad i = 0;
1872 1.78 ad }
1873 1.78 ad bit = ffs(lcp->lcp_bitmap[i]) - 1;
1874 1.193 kamil lcp->lcp_bitmap[i] ^= (1U << bit);
1875 1.78 ad lcp->lcp_rotor = i;
1876 1.78 ad lcp->lcp_nfree--;
1877 1.78 ad l->l_lcpage = lcp;
1878 1.78 ad offset = (i << 5) + bit;
1879 1.78 ad l->l_lwpctl = (lwpctl_t *)lcp->lcp_kaddr + offset;
1880 1.78 ad *uaddr = lcp->lcp_uaddr + offset * sizeof(lwpctl_t);
1881 1.78 ad mutex_exit(&lp->lp_lock);
1882 1.78 ad
1883 1.107 ad KPREEMPT_DISABLE(l);
1884 1.195 skrll l->l_lwpctl->lc_curcpu = (int)cpu_index(curcpu());
1885 1.107 ad KPREEMPT_ENABLE(l);
1886 1.78 ad
1887 1.78 ad return 0;
1888 1.78 ad }
1889 1.78 ad
1890 1.78 ad /*
1891 1.78 ad * Free an lwpctl structure back to the per-process list.
1892 1.78 ad */
1893 1.78 ad void
1894 1.78 ad lwp_ctl_free(lwp_t *l)
1895 1.78 ad {
1896 1.156 pooka struct proc *p = l->l_proc;
1897 1.78 ad lcproc_t *lp;
1898 1.78 ad lcpage_t *lcp;
1899 1.78 ad u_int map, offset;
1900 1.78 ad
1901 1.156 pooka /* don't free a lwp context we borrowed for vfork */
1902 1.156 pooka if (p->p_lflag & PL_PPWAIT) {
1903 1.156 pooka l->l_lwpctl = NULL;
1904 1.156 pooka return;
1905 1.156 pooka }
1906 1.156 pooka
1907 1.156 pooka lp = p->p_lwpctl;
1908 1.78 ad KASSERT(lp != NULL);
1909 1.78 ad
1910 1.78 ad lcp = l->l_lcpage;
1911 1.78 ad offset = (u_int)((lwpctl_t *)l->l_lwpctl - (lwpctl_t *)lcp->lcp_kaddr);
1912 1.78 ad KASSERT(offset < LWPCTL_PER_PAGE);
1913 1.78 ad
1914 1.78 ad mutex_enter(&lp->lp_lock);
1915 1.78 ad lcp->lcp_nfree++;
1916 1.78 ad map = offset >> 5;
1917 1.194 kamil lcp->lcp_bitmap[map] |= (1U << (offset & 31));
1918 1.78 ad if (lcp->lcp_bitmap[lcp->lcp_rotor] == 0)
1919 1.78 ad lcp->lcp_rotor = map;
1920 1.78 ad if (TAILQ_FIRST(&lp->lp_pages)->lcp_nfree == 0) {
1921 1.78 ad TAILQ_REMOVE(&lp->lp_pages, lcp, lcp_chain);
1922 1.78 ad TAILQ_INSERT_HEAD(&lp->lp_pages, lcp, lcp_chain);
1923 1.78 ad }
1924 1.78 ad mutex_exit(&lp->lp_lock);
1925 1.78 ad }
1926 1.78 ad
1927 1.78 ad /*
1928 1.78 ad * Process is exiting; tear down lwpctl state. This can only be safely
1929 1.78 ad * called by the last LWP in the process.
1930 1.78 ad */
1931 1.78 ad void
1932 1.78 ad lwp_ctl_exit(void)
1933 1.78 ad {
1934 1.78 ad lcpage_t *lcp, *next;
1935 1.78 ad lcproc_t *lp;
1936 1.78 ad proc_t *p;
1937 1.78 ad lwp_t *l;
1938 1.78 ad
1939 1.78 ad l = curlwp;
1940 1.78 ad l->l_lwpctl = NULL;
1941 1.95 ad l->l_lcpage = NULL;
1942 1.78 ad p = l->l_proc;
1943 1.78 ad lp = p->p_lwpctl;
1944 1.78 ad
1945 1.78 ad KASSERT(lp != NULL);
1946 1.78 ad KASSERT(p->p_nlwps == 1);
1947 1.78 ad
1948 1.78 ad for (lcp = TAILQ_FIRST(&lp->lp_pages); lcp != NULL; lcp = next) {
1949 1.78 ad next = TAILQ_NEXT(lcp, lcp_chain);
1950 1.78 ad uvm_unmap(kernel_map, lcp->lcp_kaddr,
1951 1.78 ad lcp->lcp_kaddr + PAGE_SIZE);
1952 1.78 ad kmem_free(lcp, LWPCTL_LCPAGE_SZ);
1953 1.78 ad }
1954 1.78 ad
1955 1.78 ad if (lp->lp_uao != NULL) {
1956 1.78 ad uvm_unmap(&p->p_vmspace->vm_map, lp->lp_uva,
1957 1.78 ad lp->lp_uva + LWPCTL_UAREA_SZ);
1958 1.78 ad }
1959 1.78 ad
1960 1.78 ad mutex_destroy(&lp->lp_lock);
1961 1.78 ad kmem_free(lp, sizeof(*lp));
1962 1.78 ad p->p_lwpctl = NULL;
1963 1.78 ad }
1964 1.84 yamt
1965 1.130 ad /*
1966 1.130 ad * Return the current LWP's "preemption counter". Used to detect
1967 1.130 ad * preemption across operations that can tolerate preemption without
1968 1.130 ad * crashing, but which may generate incorrect results if preempted.
1969 1.130 ad */
1970 1.130 ad uint64_t
1971 1.130 ad lwp_pctr(void)
1972 1.130 ad {
1973 1.130 ad
1974 1.130 ad return curlwp->l_ncsw;
1975 1.130 ad }
1976 1.130 ad
1977 1.151 chs /*
1978 1.151 chs * Set an LWP's private data pointer.
1979 1.151 chs */
1980 1.151 chs int
1981 1.151 chs lwp_setprivate(struct lwp *l, void *ptr)
1982 1.151 chs {
1983 1.151 chs int error = 0;
1984 1.151 chs
1985 1.151 chs l->l_private = ptr;
1986 1.151 chs #ifdef __HAVE_CPU_LWP_SETPRIVATE
1987 1.151 chs error = cpu_lwp_setprivate(l, ptr);
1988 1.151 chs #endif
1989 1.151 chs return error;
1990 1.151 chs }
1991 1.151 chs
1992 1.84 yamt #if defined(DDB)
1993 1.153 rmind #include <machine/pcb.h>
1994 1.153 rmind
1995 1.84 yamt void
1996 1.84 yamt lwp_whatis(uintptr_t addr, void (*pr)(const char *, ...))
1997 1.84 yamt {
1998 1.84 yamt lwp_t *l;
1999 1.84 yamt
2000 1.84 yamt LIST_FOREACH(l, &alllwp, l_list) {
2001 1.84 yamt uintptr_t stack = (uintptr_t)KSTACK_LOWEST_ADDR(l);
2002 1.84 yamt
2003 1.84 yamt if (addr < stack || stack + KSTACK_SIZE <= addr) {
2004 1.84 yamt continue;
2005 1.84 yamt }
2006 1.84 yamt (*pr)("%p is %p+%zu, LWP %p's stack\n",
2007 1.84 yamt (void *)addr, (void *)stack,
2008 1.84 yamt (size_t)(addr - stack), l);
2009 1.84 yamt }
2010 1.84 yamt }
2011 1.84 yamt #endif /* defined(DDB) */
2012