kern_synch.c revision 1.269 1 1.269 elad /* $NetBSD: kern_synch.c,v 1.269 2009/10/03 21:21:56 elad Exp $ */
2 1.63 thorpej
3 1.63 thorpej /*-
4 1.260 ad * Copyright (c) 1999, 2000, 2004, 2006, 2007, 2008, 2009
5 1.260 ad * The NetBSD Foundation, Inc.
6 1.63 thorpej * All rights reserved.
7 1.63 thorpej *
8 1.63 thorpej * This code is derived from software contributed to The NetBSD Foundation
9 1.63 thorpej * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
10 1.188 yamt * NASA Ames Research Center, by Charles M. Hannum, Andrew Doran and
11 1.188 yamt * Daniel Sieger.
12 1.63 thorpej *
13 1.63 thorpej * Redistribution and use in source and binary forms, with or without
14 1.63 thorpej * modification, are permitted provided that the following conditions
15 1.63 thorpej * are met:
16 1.63 thorpej * 1. Redistributions of source code must retain the above copyright
17 1.63 thorpej * notice, this list of conditions and the following disclaimer.
18 1.63 thorpej * 2. Redistributions in binary form must reproduce the above copyright
19 1.63 thorpej * notice, this list of conditions and the following disclaimer in the
20 1.63 thorpej * documentation and/or other materials provided with the distribution.
21 1.63 thorpej *
22 1.63 thorpej * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
23 1.63 thorpej * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
24 1.63 thorpej * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
25 1.63 thorpej * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
26 1.63 thorpej * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
27 1.63 thorpej * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
28 1.63 thorpej * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
29 1.63 thorpej * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
30 1.63 thorpej * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
31 1.63 thorpej * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
32 1.63 thorpej * POSSIBILITY OF SUCH DAMAGE.
33 1.63 thorpej */
34 1.26 cgd
35 1.26 cgd /*-
36 1.26 cgd * Copyright (c) 1982, 1986, 1990, 1991, 1993
37 1.26 cgd * The Regents of the University of California. All rights reserved.
38 1.26 cgd * (c) UNIX System Laboratories, Inc.
39 1.26 cgd * All or some portions of this file are derived from material licensed
40 1.26 cgd * to the University of California by American Telephone and Telegraph
41 1.26 cgd * Co. or Unix System Laboratories, Inc. and are reproduced herein with
42 1.26 cgd * the permission of UNIX System Laboratories, Inc.
43 1.26 cgd *
44 1.26 cgd * Redistribution and use in source and binary forms, with or without
45 1.26 cgd * modification, are permitted provided that the following conditions
46 1.26 cgd * are met:
47 1.26 cgd * 1. Redistributions of source code must retain the above copyright
48 1.26 cgd * notice, this list of conditions and the following disclaimer.
49 1.26 cgd * 2. Redistributions in binary form must reproduce the above copyright
50 1.26 cgd * notice, this list of conditions and the following disclaimer in the
51 1.26 cgd * documentation and/or other materials provided with the distribution.
52 1.136 agc * 3. Neither the name of the University nor the names of its contributors
53 1.26 cgd * may be used to endorse or promote products derived from this software
54 1.26 cgd * without specific prior written permission.
55 1.26 cgd *
56 1.26 cgd * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
57 1.26 cgd * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
58 1.26 cgd * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
59 1.26 cgd * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
60 1.26 cgd * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
61 1.26 cgd * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
62 1.26 cgd * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
63 1.26 cgd * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
64 1.26 cgd * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
65 1.26 cgd * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
66 1.26 cgd * SUCH DAMAGE.
67 1.26 cgd *
68 1.50 fvdl * @(#)kern_synch.c 8.9 (Berkeley) 5/19/95
69 1.26 cgd */
70 1.106 lukem
71 1.106 lukem #include <sys/cdefs.h>
72 1.269 elad __KERNEL_RCSID(0, "$NetBSD: kern_synch.c,v 1.269 2009/10/03 21:21:56 elad Exp $");
73 1.48 mrg
74 1.109 yamt #include "opt_kstack.h"
75 1.110 briggs #include "opt_perfctrs.h"
76 1.252 wrstuden #include "opt_sa.h"
77 1.26 cgd
78 1.174 ad #define __MUTEX_PRIVATE
79 1.174 ad
80 1.26 cgd #include <sys/param.h>
81 1.26 cgd #include <sys/systm.h>
82 1.26 cgd #include <sys/proc.h>
83 1.26 cgd #include <sys/kernel.h>
84 1.111 briggs #if defined(PERFCTRS)
85 1.110 briggs #include <sys/pmc.h>
86 1.111 briggs #endif
87 1.188 yamt #include <sys/cpu.h>
88 1.26 cgd #include <sys/resourcevar.h>
89 1.55 ross #include <sys/sched.h>
90 1.252 wrstuden #include <sys/sa.h>
91 1.252 wrstuden #include <sys/savar.h>
92 1.179 dsl #include <sys/syscall_stats.h>
93 1.174 ad #include <sys/sleepq.h>
94 1.174 ad #include <sys/lockdebug.h>
95 1.190 ad #include <sys/evcnt.h>
96 1.199 ad #include <sys/intr.h>
97 1.207 ad #include <sys/lwpctl.h>
98 1.209 ad #include <sys/atomic.h>
99 1.215 ad #include <sys/simplelock.h>
100 1.268 elad #include <sys/kauth.h>
101 1.47 mrg
102 1.47 mrg #include <uvm/uvm_extern.h>
103 1.47 mrg
104 1.231 ad #include <dev/lockstat.h>
105 1.231 ad
106 1.221 ad static u_int sched_unsleep(struct lwp *, bool);
107 1.188 yamt static void sched_changepri(struct lwp *, pri_t);
108 1.188 yamt static void sched_lendpri(struct lwp *, pri_t);
109 1.250 rmind static void resched_cpu(struct lwp *);
110 1.122 thorpej
111 1.174 ad syncobj_t sleep_syncobj = {
112 1.174 ad SOBJ_SLEEPQ_SORTED,
113 1.174 ad sleepq_unsleep,
114 1.184 yamt sleepq_changepri,
115 1.184 yamt sleepq_lendpri,
116 1.184 yamt syncobj_noowner,
117 1.174 ad };
118 1.174 ad
119 1.174 ad syncobj_t sched_syncobj = {
120 1.174 ad SOBJ_SLEEPQ_SORTED,
121 1.174 ad sched_unsleep,
122 1.184 yamt sched_changepri,
123 1.184 yamt sched_lendpri,
124 1.184 yamt syncobj_noowner,
125 1.174 ad };
126 1.122 thorpej
127 1.223 ad callout_t sched_pstats_ch;
128 1.223 ad unsigned sched_pstats_ticks;
129 1.223 ad kcondvar_t lbolt; /* once a second sleep address */
130 1.223 ad
131 1.269 elad static kauth_listener_t sched_listener;
132 1.268 elad
133 1.237 rmind /* Preemption event counters */
134 1.231 ad static struct evcnt kpreempt_ev_crit;
135 1.231 ad static struct evcnt kpreempt_ev_klock;
136 1.231 ad static struct evcnt kpreempt_ev_immed;
137 1.231 ad
138 1.231 ad /*
139 1.174 ad * During autoconfiguration or after a panic, a sleep will simply lower the
140 1.174 ad * priority briefly to allow interrupts, then return. The priority to be
141 1.174 ad * used (safepri) is machine-dependent, thus this value is initialized and
142 1.174 ad * maintained in the machine-dependent layers. This priority will typically
143 1.174 ad * be 0, or the lowest priority that is safe for use on the interrupt stack;
144 1.174 ad * it can be made higher to block network software interrupts after panics.
145 1.26 cgd */
146 1.174 ad int safepri;
147 1.26 cgd
148 1.268 elad static int
149 1.268 elad sched_listener_cb(kauth_cred_t cred, kauth_action_t action, void *cookie,
150 1.268 elad void *arg0, void *arg1, void *arg2, void *arg3)
151 1.268 elad {
152 1.268 elad struct proc *p;
153 1.268 elad int result;
154 1.268 elad
155 1.268 elad result = KAUTH_RESULT_DEFER;
156 1.268 elad p = arg0;
157 1.268 elad
158 1.268 elad switch (action) {
159 1.268 elad case KAUTH_PROCESS_SCHEDULER_GETPARAM:
160 1.268 elad if (kauth_cred_uidmatch(cred, p->p_cred))
161 1.268 elad result = KAUTH_RESULT_ALLOW;
162 1.268 elad break;
163 1.268 elad
164 1.268 elad case KAUTH_PROCESS_SCHEDULER_SETPARAM:
165 1.268 elad if (kauth_cred_uidmatch(cred, p->p_cred)) {
166 1.268 elad struct lwp *l;
167 1.268 elad int policy;
168 1.268 elad pri_t priority;
169 1.268 elad
170 1.268 elad l = arg1;
171 1.268 elad policy = (int)(unsigned long)arg2;
172 1.268 elad priority = (pri_t)(unsigned long)arg3;
173 1.268 elad
174 1.268 elad if ((policy == l->l_class ||
175 1.268 elad (policy != SCHED_FIFO && policy != SCHED_RR)) &&
176 1.268 elad priority <= l->l_priority)
177 1.268 elad result = KAUTH_RESULT_ALLOW;
178 1.268 elad }
179 1.268 elad
180 1.268 elad break;
181 1.268 elad
182 1.268 elad case KAUTH_PROCESS_SCHEDULER_GETAFFINITY:
183 1.268 elad result = KAUTH_RESULT_ALLOW;
184 1.268 elad break;
185 1.268 elad
186 1.268 elad case KAUTH_PROCESS_SCHEDULER_SETAFFINITY:
187 1.268 elad /* Privileged; we let the secmodel handle this. */
188 1.268 elad break;
189 1.268 elad
190 1.268 elad default:
191 1.268 elad break;
192 1.268 elad }
193 1.268 elad
194 1.268 elad return result;
195 1.268 elad }
196 1.268 elad
197 1.237 rmind void
198 1.237 rmind sched_init(void)
199 1.237 rmind {
200 1.237 rmind
201 1.237 rmind cv_init(&lbolt, "lbolt");
202 1.237 rmind callout_init(&sched_pstats_ch, CALLOUT_MPSAFE);
203 1.237 rmind callout_setfunc(&sched_pstats_ch, sched_pstats, NULL);
204 1.237 rmind
205 1.239 ad evcnt_attach_dynamic(&kpreempt_ev_crit, EVCNT_TYPE_MISC, NULL,
206 1.237 rmind "kpreempt", "defer: critical section");
207 1.239 ad evcnt_attach_dynamic(&kpreempt_ev_klock, EVCNT_TYPE_MISC, NULL,
208 1.237 rmind "kpreempt", "defer: kernel_lock");
209 1.239 ad evcnt_attach_dynamic(&kpreempt_ev_immed, EVCNT_TYPE_MISC, NULL,
210 1.237 rmind "kpreempt", "immediate");
211 1.237 rmind
212 1.237 rmind sched_pstats(NULL);
213 1.268 elad
214 1.268 elad sched_listener = kauth_listen_scope(KAUTH_SCOPE_PROCESS,
215 1.268 elad sched_listener_cb, NULL);
216 1.237 rmind }
217 1.237 rmind
218 1.26 cgd /*
219 1.174 ad * OBSOLETE INTERFACE
220 1.174 ad *
221 1.255 skrll * General sleep call. Suspends the current LWP until a wakeup is
222 1.255 skrll * performed on the specified identifier. The LWP will then be made
223 1.174 ad * runnable with the specified priority. Sleeps at most timo/hz seconds (0
224 1.174 ad * means no timeout). If pri includes PCATCH flag, signals are checked
225 1.26 cgd * before and after sleeping, else signals are not checked. Returns 0 if
226 1.26 cgd * awakened, EWOULDBLOCK if the timeout expires. If PCATCH is set and a
227 1.26 cgd * signal needs to be delivered, ERESTART is returned if the current system
228 1.26 cgd * call should be restarted if possible, and EINTR is returned if the system
229 1.26 cgd * call should be interrupted by the signal (return EINTR).
230 1.77 thorpej *
231 1.174 ad * The interlock is held until we are on a sleep queue. The interlock will
232 1.174 ad * be locked before returning back to the caller unless the PNORELOCK flag
233 1.174 ad * is specified, in which case the interlock will always be unlocked upon
234 1.174 ad * return.
235 1.26 cgd */
236 1.26 cgd int
237 1.185 yamt ltsleep(wchan_t ident, pri_t priority, const char *wmesg, int timo,
238 1.174 ad volatile struct simplelock *interlock)
239 1.26 cgd {
240 1.122 thorpej struct lwp *l = curlwp;
241 1.174 ad sleepq_t *sq;
242 1.244 ad kmutex_t *mp;
243 1.188 yamt int error;
244 1.26 cgd
245 1.204 ad KASSERT((l->l_pflag & LP_INTR) == 0);
246 1.204 ad
247 1.174 ad if (sleepq_dontsleep(l)) {
248 1.174 ad (void)sleepq_abort(NULL, 0);
249 1.174 ad if ((priority & PNORELOCK) != 0)
250 1.77 thorpej simple_unlock(interlock);
251 1.174 ad return 0;
252 1.26 cgd }
253 1.78 sommerfe
254 1.204 ad l->l_kpriority = true;
255 1.244 ad sq = sleeptab_lookup(&sleeptab, ident, &mp);
256 1.244 ad sleepq_enter(sq, l, mp);
257 1.204 ad sleepq_enqueue(sq, ident, wmesg, &sleep_syncobj);
258 1.42 cgd
259 1.174 ad if (interlock != NULL) {
260 1.204 ad KASSERT(simple_lock_held(interlock));
261 1.174 ad simple_unlock(interlock);
262 1.150 chs }
263 1.150 chs
264 1.188 yamt error = sleepq_block(timo, priority & PCATCH);
265 1.126 pk
266 1.174 ad if (interlock != NULL && (priority & PNORELOCK) == 0)
267 1.126 pk simple_lock(interlock);
268 1.174 ad
269 1.174 ad return error;
270 1.26 cgd }
271 1.26 cgd
272 1.187 ad int
273 1.187 ad mtsleep(wchan_t ident, pri_t priority, const char *wmesg, int timo,
274 1.187 ad kmutex_t *mtx)
275 1.187 ad {
276 1.187 ad struct lwp *l = curlwp;
277 1.187 ad sleepq_t *sq;
278 1.244 ad kmutex_t *mp;
279 1.188 yamt int error;
280 1.187 ad
281 1.204 ad KASSERT((l->l_pflag & LP_INTR) == 0);
282 1.204 ad
283 1.187 ad if (sleepq_dontsleep(l)) {
284 1.187 ad (void)sleepq_abort(mtx, (priority & PNORELOCK) != 0);
285 1.187 ad return 0;
286 1.187 ad }
287 1.187 ad
288 1.204 ad l->l_kpriority = true;
289 1.244 ad sq = sleeptab_lookup(&sleeptab, ident, &mp);
290 1.244 ad sleepq_enter(sq, l, mp);
291 1.204 ad sleepq_enqueue(sq, ident, wmesg, &sleep_syncobj);
292 1.187 ad mutex_exit(mtx);
293 1.188 yamt error = sleepq_block(timo, priority & PCATCH);
294 1.187 ad
295 1.187 ad if ((priority & PNORELOCK) == 0)
296 1.187 ad mutex_enter(mtx);
297 1.187 ad
298 1.187 ad return error;
299 1.187 ad }
300 1.187 ad
301 1.26 cgd /*
302 1.174 ad * General sleep call for situations where a wake-up is not expected.
303 1.26 cgd */
304 1.174 ad int
305 1.182 thorpej kpause(const char *wmesg, bool intr, int timo, kmutex_t *mtx)
306 1.26 cgd {
307 1.174 ad struct lwp *l = curlwp;
308 1.244 ad kmutex_t *mp;
309 1.174 ad sleepq_t *sq;
310 1.174 ad int error;
311 1.26 cgd
312 1.174 ad if (sleepq_dontsleep(l))
313 1.174 ad return sleepq_abort(NULL, 0);
314 1.26 cgd
315 1.174 ad if (mtx != NULL)
316 1.174 ad mutex_exit(mtx);
317 1.204 ad l->l_kpriority = true;
318 1.244 ad sq = sleeptab_lookup(&sleeptab, l, &mp);
319 1.244 ad sleepq_enter(sq, l, mp);
320 1.204 ad sleepq_enqueue(sq, l, wmesg, &sleep_syncobj);
321 1.188 yamt error = sleepq_block(timo, intr);
322 1.174 ad if (mtx != NULL)
323 1.174 ad mutex_enter(mtx);
324 1.83 thorpej
325 1.174 ad return error;
326 1.139 cl }
327 1.139 cl
328 1.252 wrstuden #ifdef KERN_SA
329 1.252 wrstuden /*
330 1.252 wrstuden * sa_awaken:
331 1.252 wrstuden *
332 1.252 wrstuden * We believe this lwp is an SA lwp. If it's yielding,
333 1.252 wrstuden * let it know it needs to wake up.
334 1.252 wrstuden *
335 1.252 wrstuden * We are called and exit with the lwp locked. We are
336 1.252 wrstuden * called in the middle of wakeup operations, so we need
337 1.252 wrstuden * to not touch the locks at all.
338 1.252 wrstuden */
339 1.252 wrstuden void
340 1.252 wrstuden sa_awaken(struct lwp *l)
341 1.252 wrstuden {
342 1.252 wrstuden /* LOCK_ASSERT(lwp_locked(l, NULL)); */
343 1.252 wrstuden
344 1.252 wrstuden if (l == l->l_savp->savp_lwp && l->l_flag & LW_SA_YIELD)
345 1.252 wrstuden l->l_flag &= ~LW_SA_IDLE;
346 1.252 wrstuden }
347 1.252 wrstuden #endif /* KERN_SA */
348 1.252 wrstuden
349 1.26 cgd /*
350 1.174 ad * OBSOLETE INTERFACE
351 1.174 ad *
352 1.255 skrll * Make all LWPs sleeping on the specified identifier runnable.
353 1.26 cgd */
354 1.26 cgd void
355 1.174 ad wakeup(wchan_t ident)
356 1.26 cgd {
357 1.174 ad sleepq_t *sq;
358 1.244 ad kmutex_t *mp;
359 1.83 thorpej
360 1.261 rmind if (__predict_false(cold))
361 1.174 ad return;
362 1.83 thorpej
363 1.244 ad sq = sleeptab_lookup(&sleeptab, ident, &mp);
364 1.244 ad sleepq_wake(sq, ident, (u_int)-1, mp);
365 1.63 thorpej }
366 1.63 thorpej
367 1.63 thorpej /*
368 1.174 ad * OBSOLETE INTERFACE
369 1.174 ad *
370 1.255 skrll * Make the highest priority LWP first in line on the specified
371 1.63 thorpej * identifier runnable.
372 1.63 thorpej */
373 1.174 ad void
374 1.174 ad wakeup_one(wchan_t ident)
375 1.63 thorpej {
376 1.174 ad sleepq_t *sq;
377 1.244 ad kmutex_t *mp;
378 1.63 thorpej
379 1.261 rmind if (__predict_false(cold))
380 1.174 ad return;
381 1.188 yamt
382 1.244 ad sq = sleeptab_lookup(&sleeptab, ident, &mp);
383 1.244 ad sleepq_wake(sq, ident, 1, mp);
384 1.174 ad }
385 1.63 thorpej
386 1.117 gmcgarry
387 1.117 gmcgarry /*
388 1.255 skrll * General yield call. Puts the current LWP back on its run queue and
389 1.117 gmcgarry * performs a voluntary context switch. Should only be called when the
390 1.255 skrll * current LWP explicitly requests it (eg sched_yield(2)).
391 1.117 gmcgarry */
392 1.117 gmcgarry void
393 1.117 gmcgarry yield(void)
394 1.117 gmcgarry {
395 1.122 thorpej struct lwp *l = curlwp;
396 1.117 gmcgarry
397 1.174 ad KERNEL_UNLOCK_ALL(l, &l->l_biglocks);
398 1.174 ad lwp_lock(l);
399 1.217 ad KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_lwplock));
400 1.188 yamt KASSERT(l->l_stat == LSONPROC);
401 1.204 ad l->l_kpriority = false;
402 1.188 yamt (void)mi_switch(l);
403 1.174 ad KERNEL_LOCK(l->l_biglocks, l);
404 1.69 thorpej }
405 1.69 thorpej
406 1.69 thorpej /*
407 1.255 skrll * General preemption call. Puts the current LWP back on its run queue
408 1.156 rpaulo * and performs an involuntary context switch.
409 1.69 thorpej */
410 1.69 thorpej void
411 1.174 ad preempt(void)
412 1.69 thorpej {
413 1.122 thorpej struct lwp *l = curlwp;
414 1.69 thorpej
415 1.174 ad KERNEL_UNLOCK_ALL(l, &l->l_biglocks);
416 1.174 ad lwp_lock(l);
417 1.217 ad KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_lwplock));
418 1.188 yamt KASSERT(l->l_stat == LSONPROC);
419 1.204 ad l->l_kpriority = false;
420 1.174 ad l->l_nivcsw++;
421 1.188 yamt (void)mi_switch(l);
422 1.174 ad KERNEL_LOCK(l->l_biglocks, l);
423 1.69 thorpej }
424 1.69 thorpej
425 1.234 ad /*
426 1.234 ad * Handle a request made by another agent to preempt the current LWP
427 1.234 ad * in-kernel. Usually called when l_dopreempt may be non-zero.
428 1.234 ad *
429 1.234 ad * Character addresses for lockstat only.
430 1.234 ad */
431 1.231 ad static char in_critical_section;
432 1.231 ad static char kernel_lock_held;
433 1.231 ad static char is_softint;
434 1.262 yamt static char cpu_kpreempt_enter_fail;
435 1.231 ad
436 1.231 ad bool
437 1.231 ad kpreempt(uintptr_t where)
438 1.231 ad {
439 1.231 ad uintptr_t failed;
440 1.231 ad lwp_t *l;
441 1.264 ad int s, dop, lsflag;
442 1.231 ad
443 1.231 ad l = curlwp;
444 1.231 ad failed = 0;
445 1.231 ad while ((dop = l->l_dopreempt) != 0) {
446 1.231 ad if (l->l_stat != LSONPROC) {
447 1.231 ad /*
448 1.231 ad * About to block (or die), let it happen.
449 1.231 ad * Doesn't really count as "preemption has
450 1.231 ad * been blocked", since we're going to
451 1.231 ad * context switch.
452 1.231 ad */
453 1.231 ad l->l_dopreempt = 0;
454 1.231 ad return true;
455 1.231 ad }
456 1.231 ad if (__predict_false((l->l_flag & LW_IDLE) != 0)) {
457 1.231 ad /* Can't preempt idle loop, don't count as failure. */
458 1.261 rmind l->l_dopreempt = 0;
459 1.261 rmind return true;
460 1.231 ad }
461 1.231 ad if (__predict_false(l->l_nopreempt != 0)) {
462 1.231 ad /* LWP holds preemption disabled, explicitly. */
463 1.231 ad if ((dop & DOPREEMPT_COUNTED) == 0) {
464 1.234 ad kpreempt_ev_crit.ev_count++;
465 1.231 ad }
466 1.231 ad failed = (uintptr_t)&in_critical_section;
467 1.231 ad break;
468 1.231 ad }
469 1.231 ad if (__predict_false((l->l_pflag & LP_INTR) != 0)) {
470 1.261 rmind /* Can't preempt soft interrupts yet. */
471 1.261 rmind l->l_dopreempt = 0;
472 1.261 rmind failed = (uintptr_t)&is_softint;
473 1.261 rmind break;
474 1.231 ad }
475 1.231 ad s = splsched();
476 1.231 ad if (__predict_false(l->l_blcnt != 0 ||
477 1.231 ad curcpu()->ci_biglock_wanted != NULL)) {
478 1.231 ad /* Hold or want kernel_lock, code is not MT safe. */
479 1.231 ad splx(s);
480 1.231 ad if ((dop & DOPREEMPT_COUNTED) == 0) {
481 1.234 ad kpreempt_ev_klock.ev_count++;
482 1.231 ad }
483 1.231 ad failed = (uintptr_t)&kernel_lock_held;
484 1.231 ad break;
485 1.231 ad }
486 1.231 ad if (__predict_false(!cpu_kpreempt_enter(where, s))) {
487 1.231 ad /*
488 1.231 ad * It may be that the IPL is too high.
489 1.231 ad * kpreempt_enter() can schedule an
490 1.231 ad * interrupt to retry later.
491 1.231 ad */
492 1.231 ad splx(s);
493 1.262 yamt failed = (uintptr_t)&cpu_kpreempt_enter_fail;
494 1.231 ad break;
495 1.231 ad }
496 1.231 ad /* Do it! */
497 1.231 ad if (__predict_true((dop & DOPREEMPT_COUNTED) == 0)) {
498 1.234 ad kpreempt_ev_immed.ev_count++;
499 1.231 ad }
500 1.231 ad lwp_lock(l);
501 1.231 ad mi_switch(l);
502 1.231 ad l->l_nopreempt++;
503 1.231 ad splx(s);
504 1.231 ad
505 1.231 ad /* Take care of any MD cleanup. */
506 1.231 ad cpu_kpreempt_exit(where);
507 1.231 ad l->l_nopreempt--;
508 1.231 ad }
509 1.231 ad
510 1.264 ad if (__predict_true(!failed)) {
511 1.264 ad return false;
512 1.264 ad }
513 1.264 ad
514 1.231 ad /* Record preemption failure for reporting via lockstat. */
515 1.264 ad atomic_or_uint(&l->l_dopreempt, DOPREEMPT_COUNTED);
516 1.264 ad lsflag = 0;
517 1.264 ad LOCKSTAT_ENTER(lsflag);
518 1.264 ad if (__predict_false(lsflag)) {
519 1.264 ad if (where == 0) {
520 1.264 ad where = (uintptr_t)__builtin_return_address(0);
521 1.264 ad }
522 1.264 ad /* Preemption is on, might recurse, so make it atomic. */
523 1.264 ad if (atomic_cas_ptr_ni((void *)&l->l_pfailaddr, NULL,
524 1.264 ad (void *)where) == NULL) {
525 1.264 ad LOCKSTAT_START_TIMER(lsflag, l->l_pfailtime);
526 1.264 ad l->l_pfaillock = failed;
527 1.231 ad }
528 1.231 ad }
529 1.264 ad LOCKSTAT_EXIT(lsflag);
530 1.264 ad return true;
531 1.231 ad }
532 1.231 ad
533 1.69 thorpej /*
534 1.231 ad * Return true if preemption is explicitly disabled.
535 1.230 ad */
536 1.231 ad bool
537 1.231 ad kpreempt_disabled(void)
538 1.231 ad {
539 1.261 rmind const lwp_t *l = curlwp;
540 1.231 ad
541 1.231 ad return l->l_nopreempt != 0 || l->l_stat == LSZOMB ||
542 1.231 ad (l->l_flag & LW_IDLE) != 0 || cpu_kpreempt_disabled();
543 1.231 ad }
544 1.230 ad
545 1.230 ad /*
546 1.231 ad * Disable kernel preemption.
547 1.230 ad */
548 1.230 ad void
549 1.231 ad kpreempt_disable(void)
550 1.230 ad {
551 1.230 ad
552 1.231 ad KPREEMPT_DISABLE(curlwp);
553 1.230 ad }
554 1.230 ad
555 1.230 ad /*
556 1.231 ad * Reenable kernel preemption.
557 1.230 ad */
558 1.231 ad void
559 1.231 ad kpreempt_enable(void)
560 1.230 ad {
561 1.230 ad
562 1.231 ad KPREEMPT_ENABLE(curlwp);
563 1.230 ad }
564 1.230 ad
565 1.230 ad /*
566 1.188 yamt * Compute the amount of time during which the current lwp was running.
567 1.130 nathanw *
568 1.188 yamt * - update l_rtime unless it's an idle lwp.
569 1.188 yamt */
570 1.188 yamt
571 1.199 ad void
572 1.212 yamt updatertime(lwp_t *l, const struct bintime *now)
573 1.188 yamt {
574 1.188 yamt
575 1.261 rmind if (__predict_false(l->l_flag & LW_IDLE))
576 1.188 yamt return;
577 1.188 yamt
578 1.212 yamt /* rtime += now - stime */
579 1.212 yamt bintime_add(&l->l_rtime, now);
580 1.212 yamt bintime_sub(&l->l_rtime, &l->l_stime);
581 1.188 yamt }
582 1.188 yamt
583 1.188 yamt /*
584 1.245 ad * Select next LWP from the current CPU to run..
585 1.245 ad */
586 1.245 ad static inline lwp_t *
587 1.245 ad nextlwp(struct cpu_info *ci, struct schedstate_percpu *spc)
588 1.245 ad {
589 1.245 ad lwp_t *newl;
590 1.245 ad
591 1.245 ad /*
592 1.245 ad * Let sched_nextlwp() select the LWP to run the CPU next.
593 1.245 ad * If no LWP is runnable, select the idle LWP.
594 1.245 ad *
595 1.245 ad * Note that spc_lwplock might not necessary be held, and
596 1.245 ad * new thread would be unlocked after setting the LWP-lock.
597 1.245 ad */
598 1.245 ad newl = sched_nextlwp();
599 1.245 ad if (newl != NULL) {
600 1.245 ad sched_dequeue(newl);
601 1.245 ad KASSERT(lwp_locked(newl, spc->spc_mutex));
602 1.245 ad newl->l_stat = LSONPROC;
603 1.245 ad newl->l_cpu = ci;
604 1.248 ad newl->l_pflag |= LP_RUNNING;
605 1.245 ad lwp_setlock(newl, spc->spc_lwplock);
606 1.245 ad } else {
607 1.245 ad newl = ci->ci_data.cpu_idlelwp;
608 1.245 ad newl->l_stat = LSONPROC;
609 1.248 ad newl->l_pflag |= LP_RUNNING;
610 1.245 ad }
611 1.261 rmind
612 1.245 ad /*
613 1.245 ad * Only clear want_resched if there are no pending (slow)
614 1.245 ad * software interrupts.
615 1.245 ad */
616 1.245 ad ci->ci_want_resched = ci->ci_data.cpu_softints;
617 1.245 ad spc->spc_flags &= ~SPCF_SWITCHCLEAR;
618 1.245 ad spc->spc_curpriority = lwp_eprio(newl);
619 1.245 ad
620 1.245 ad return newl;
621 1.245 ad }
622 1.245 ad
623 1.245 ad /*
624 1.188 yamt * The machine independent parts of context switch.
625 1.188 yamt *
626 1.188 yamt * Returns 1 if another LWP was actually run.
627 1.26 cgd */
628 1.122 thorpej int
629 1.199 ad mi_switch(lwp_t *l)
630 1.26 cgd {
631 1.246 rmind struct cpu_info *ci;
632 1.76 thorpej struct schedstate_percpu *spc;
633 1.188 yamt struct lwp *newl;
634 1.174 ad int retval, oldspl;
635 1.212 yamt struct bintime bt;
636 1.199 ad bool returning;
637 1.26 cgd
638 1.188 yamt KASSERT(lwp_locked(l, NULL));
639 1.231 ad KASSERT(kpreempt_disabled());
640 1.188 yamt LOCKDEBUG_BARRIER(l->l_mutex, 1);
641 1.174 ad
642 1.174 ad kstack_check_magic(l);
643 1.83 thorpej
644 1.212 yamt binuptime(&bt);
645 1.199 ad
646 1.267 yamt KASSERT((l->l_pflag & LP_RUNNING) != 0);
647 1.231 ad KASSERT(l->l_cpu == curcpu());
648 1.196 ad ci = l->l_cpu;
649 1.196 ad spc = &ci->ci_schedstate;
650 1.199 ad returning = false;
651 1.190 ad newl = NULL;
652 1.190 ad
653 1.199 ad /*
654 1.199 ad * If we have been asked to switch to a specific LWP, then there
655 1.199 ad * is no need to inspect the run queues. If a soft interrupt is
656 1.199 ad * blocking, then return to the interrupted thread without adjusting
657 1.199 ad * VM context or its start time: neither have been changed in order
658 1.199 ad * to take the interrupt.
659 1.199 ad */
660 1.190 ad if (l->l_switchto != NULL) {
661 1.204 ad if ((l->l_pflag & LP_INTR) != 0) {
662 1.199 ad returning = true;
663 1.199 ad softint_block(l);
664 1.248 ad if ((l->l_pflag & LP_TIMEINTR) != 0)
665 1.212 yamt updatertime(l, &bt);
666 1.199 ad }
667 1.190 ad newl = l->l_switchto;
668 1.190 ad l->l_switchto = NULL;
669 1.190 ad }
670 1.204 ad #ifndef __HAVE_FAST_SOFTINTS
671 1.204 ad else if (ci->ci_data.cpu_softints != 0) {
672 1.204 ad /* There are pending soft interrupts, so pick one. */
673 1.204 ad newl = softint_picklwp();
674 1.204 ad newl->l_stat = LSONPROC;
675 1.248 ad newl->l_pflag |= LP_RUNNING;
676 1.204 ad }
677 1.204 ad #endif /* !__HAVE_FAST_SOFTINTS */
678 1.190 ad
679 1.180 dsl /* Count time spent in current system call */
680 1.199 ad if (!returning) {
681 1.199 ad SYSCALL_TIME_SLEEP(l);
682 1.180 dsl
683 1.199 ad /*
684 1.199 ad * XXXSMP If we are using h/w performance counters,
685 1.199 ad * save context.
686 1.199 ad */
687 1.174 ad #if PERFCTRS
688 1.199 ad if (PMC_ENABLED(l->l_proc)) {
689 1.199 ad pmc_save_context(l->l_proc);
690 1.199 ad }
691 1.199 ad #endif
692 1.212 yamt updatertime(l, &bt);
693 1.174 ad }
694 1.113 gmcgarry
695 1.246 rmind /* Lock the runqueue */
696 1.246 rmind KASSERT(l->l_stat != LSRUN);
697 1.246 rmind mutex_spin_enter(spc->spc_mutex);
698 1.246 rmind
699 1.113 gmcgarry /*
700 1.174 ad * If on the CPU and we have gotten this far, then we must yield.
701 1.113 gmcgarry */
702 1.246 rmind if (l->l_stat == LSONPROC && l != newl) {
703 1.217 ad KASSERT(lwp_locked(l, spc->spc_lwplock));
704 1.188 yamt if ((l->l_flag & LW_IDLE) == 0) {
705 1.188 yamt l->l_stat = LSRUN;
706 1.246 rmind lwp_setlock(l, spc->spc_mutex);
707 1.246 rmind sched_enqueue(l, true);
708 1.246 rmind /* Handle migration case */
709 1.246 rmind KASSERT(spc->spc_migrating == NULL);
710 1.246 rmind if (l->l_target_cpu != NULL) {
711 1.246 rmind spc->spc_migrating = l;
712 1.216 rmind }
713 1.246 rmind } else
714 1.188 yamt l->l_stat = LSIDL;
715 1.174 ad }
716 1.174 ad
717 1.245 ad /* Pick new LWP to run. */
718 1.190 ad if (newl == NULL) {
719 1.245 ad newl = nextlwp(ci, spc);
720 1.199 ad }
721 1.199 ad
722 1.204 ad /* Items that must be updated with the CPU locked. */
723 1.199 ad if (!returning) {
724 1.204 ad /* Update the new LWP's start time. */
725 1.212 yamt newl->l_stime = bt;
726 1.204 ad
727 1.199 ad /*
728 1.204 ad * ci_curlwp changes when a fast soft interrupt occurs.
729 1.204 ad * We use cpu_onproc to keep track of which kernel or
730 1.204 ad * user thread is running 'underneath' the software
731 1.204 ad * interrupt. This is important for time accounting,
732 1.204 ad * itimers and forcing user threads to preempt (aston).
733 1.199 ad */
734 1.204 ad ci->ci_data.cpu_onproc = newl;
735 1.188 yamt }
736 1.188 yamt
737 1.241 ad /*
738 1.241 ad * Preemption related tasks. Must be done with the current
739 1.241 ad * CPU locked.
740 1.241 ad */
741 1.241 ad cpu_did_resched(l);
742 1.231 ad l->l_dopreempt = 0;
743 1.231 ad if (__predict_false(l->l_pfailaddr != 0)) {
744 1.231 ad LOCKSTAT_FLAG(lsflag);
745 1.231 ad LOCKSTAT_ENTER(lsflag);
746 1.231 ad LOCKSTAT_STOP_TIMER(lsflag, l->l_pfailtime);
747 1.231 ad LOCKSTAT_EVENT_RA(lsflag, l->l_pfaillock, LB_NOPREEMPT|LB_SPIN,
748 1.231 ad 1, l->l_pfailtime, l->l_pfailaddr);
749 1.231 ad LOCKSTAT_EXIT(lsflag);
750 1.231 ad l->l_pfailtime = 0;
751 1.231 ad l->l_pfaillock = 0;
752 1.231 ad l->l_pfailaddr = 0;
753 1.231 ad }
754 1.231 ad
755 1.188 yamt if (l != newl) {
756 1.188 yamt struct lwp *prevlwp;
757 1.174 ad
758 1.209 ad /* Release all locks, but leave the current LWP locked */
759 1.246 rmind if (l->l_mutex == spc->spc_mutex) {
760 1.209 ad /*
761 1.209 ad * Drop spc_lwplock, if the current LWP has been moved
762 1.209 ad * to the run queue (it is now locked by spc_mutex).
763 1.209 ad */
764 1.217 ad mutex_spin_exit(spc->spc_lwplock);
765 1.188 yamt } else {
766 1.209 ad /*
767 1.209 ad * Otherwise, drop the spc_mutex, we are done with the
768 1.209 ad * run queues.
769 1.209 ad */
770 1.188 yamt mutex_spin_exit(spc->spc_mutex);
771 1.188 yamt }
772 1.188 yamt
773 1.209 ad /*
774 1.253 skrll * Mark that context switch is going to be performed
775 1.209 ad * for this LWP, to protect it from being switched
776 1.209 ad * to on another CPU.
777 1.209 ad */
778 1.209 ad KASSERT(l->l_ctxswtch == 0);
779 1.209 ad l->l_ctxswtch = 1;
780 1.209 ad l->l_ncsw++;
781 1.267 yamt KASSERT((l->l_pflag & LP_RUNNING) != 0);
782 1.248 ad l->l_pflag &= ~LP_RUNNING;
783 1.209 ad
784 1.209 ad /*
785 1.209 ad * Increase the count of spin-mutexes before the release
786 1.209 ad * of the last lock - we must remain at IPL_SCHED during
787 1.209 ad * the context switch.
788 1.209 ad */
789 1.209 ad oldspl = MUTEX_SPIN_OLDSPL(ci);
790 1.209 ad ci->ci_mtx_count--;
791 1.209 ad lwp_unlock(l);
792 1.209 ad
793 1.218 ad /* Count the context switch on this CPU. */
794 1.218 ad ci->ci_data.cpu_nswtch++;
795 1.188 yamt
796 1.209 ad /* Update status for lwpctl, if present. */
797 1.209 ad if (l->l_lwpctl != NULL)
798 1.209 ad l->l_lwpctl->lc_curcpu = LWPCTL_CPU_NONE;
799 1.209 ad
800 1.199 ad /*
801 1.199 ad * Save old VM context, unless a soft interrupt
802 1.199 ad * handler is blocking.
803 1.199 ad */
804 1.199 ad if (!returning)
805 1.199 ad pmap_deactivate(l);
806 1.188 yamt
807 1.209 ad /*
808 1.209 ad * We may need to spin-wait for if 'newl' is still
809 1.209 ad * context switching on another CPU.
810 1.209 ad */
811 1.261 rmind if (__predict_false(newl->l_ctxswtch != 0)) {
812 1.209 ad u_int count;
813 1.209 ad count = SPINLOCK_BACKOFF_MIN;
814 1.209 ad while (newl->l_ctxswtch)
815 1.209 ad SPINLOCK_BACKOFF(count);
816 1.209 ad }
817 1.207 ad
818 1.188 yamt /* Switch to the new LWP.. */
819 1.204 ad prevlwp = cpu_switchto(l, newl, returning);
820 1.207 ad ci = curcpu();
821 1.207 ad
822 1.188 yamt /*
823 1.209 ad * Switched away - we have new curlwp.
824 1.209 ad * Restore VM context and IPL.
825 1.188 yamt */
826 1.209 ad pmap_activate(l);
827 1.265 rmind uvm_emap_switch(l);
828 1.265 rmind
829 1.188 yamt if (prevlwp != NULL) {
830 1.209 ad /* Normalize the count of the spin-mutexes */
831 1.209 ad ci->ci_mtx_count++;
832 1.209 ad /* Unmark the state of context switch */
833 1.209 ad membar_exit();
834 1.209 ad prevlwp->l_ctxswtch = 0;
835 1.188 yamt }
836 1.209 ad
837 1.209 ad /* Update status for lwpctl, if present. */
838 1.219 ad if (l->l_lwpctl != NULL) {
839 1.209 ad l->l_lwpctl->lc_curcpu = (int)cpu_index(ci);
840 1.219 ad l->l_lwpctl->lc_pctr++;
841 1.219 ad }
842 1.174 ad
843 1.231 ad KASSERT(l->l_cpu == ci);
844 1.231 ad splx(oldspl);
845 1.188 yamt retval = 1;
846 1.188 yamt } else {
847 1.188 yamt /* Nothing to do - just unlock and return. */
848 1.246 rmind mutex_spin_exit(spc->spc_mutex);
849 1.188 yamt lwp_unlock(l);
850 1.122 thorpej retval = 0;
851 1.122 thorpej }
852 1.110 briggs
853 1.188 yamt KASSERT(l == curlwp);
854 1.188 yamt KASSERT(l->l_stat == LSONPROC);
855 1.188 yamt
856 1.110 briggs /*
857 1.174 ad * XXXSMP If we are using h/w performance counters, restore context.
858 1.231 ad * XXXSMP preemption problem.
859 1.26 cgd */
860 1.114 gmcgarry #if PERFCTRS
861 1.175 christos if (PMC_ENABLED(l->l_proc)) {
862 1.175 christos pmc_restore_context(l->l_proc);
863 1.166 christos }
864 1.114 gmcgarry #endif
865 1.180 dsl SYSCALL_TIME_WAKEUP(l);
866 1.188 yamt LOCKDEBUG_BARRIER(NULL, 1);
867 1.169 yamt
868 1.122 thorpej return retval;
869 1.26 cgd }
870 1.26 cgd
871 1.26 cgd /*
872 1.245 ad * The machine independent parts of context switch to oblivion.
873 1.245 ad * Does not return. Call with the LWP unlocked.
874 1.245 ad */
875 1.245 ad void
876 1.245 ad lwp_exit_switchaway(lwp_t *l)
877 1.245 ad {
878 1.245 ad struct cpu_info *ci;
879 1.245 ad struct lwp *newl;
880 1.245 ad struct bintime bt;
881 1.245 ad
882 1.245 ad ci = l->l_cpu;
883 1.245 ad
884 1.245 ad KASSERT(kpreempt_disabled());
885 1.245 ad KASSERT(l->l_stat == LSZOMB || l->l_stat == LSIDL);
886 1.245 ad KASSERT(ci == curcpu());
887 1.245 ad LOCKDEBUG_BARRIER(NULL, 0);
888 1.245 ad
889 1.245 ad kstack_check_magic(l);
890 1.245 ad
891 1.245 ad /* Count time spent in current system call */
892 1.245 ad SYSCALL_TIME_SLEEP(l);
893 1.245 ad binuptime(&bt);
894 1.245 ad updatertime(l, &bt);
895 1.245 ad
896 1.245 ad /* Must stay at IPL_SCHED even after releasing run queue lock. */
897 1.245 ad (void)splsched();
898 1.245 ad
899 1.245 ad /*
900 1.245 ad * Let sched_nextlwp() select the LWP to run the CPU next.
901 1.245 ad * If no LWP is runnable, select the idle LWP.
902 1.245 ad *
903 1.245 ad * Note that spc_lwplock might not necessary be held, and
904 1.245 ad * new thread would be unlocked after setting the LWP-lock.
905 1.245 ad */
906 1.245 ad spc_lock(ci);
907 1.245 ad #ifndef __HAVE_FAST_SOFTINTS
908 1.245 ad if (ci->ci_data.cpu_softints != 0) {
909 1.245 ad /* There are pending soft interrupts, so pick one. */
910 1.245 ad newl = softint_picklwp();
911 1.245 ad newl->l_stat = LSONPROC;
912 1.248 ad newl->l_pflag |= LP_RUNNING;
913 1.245 ad } else
914 1.245 ad #endif /* !__HAVE_FAST_SOFTINTS */
915 1.245 ad {
916 1.245 ad newl = nextlwp(ci, &ci->ci_schedstate);
917 1.245 ad }
918 1.245 ad
919 1.245 ad /* Update the new LWP's start time. */
920 1.245 ad newl->l_stime = bt;
921 1.248 ad l->l_pflag &= ~LP_RUNNING;
922 1.245 ad
923 1.245 ad /*
924 1.245 ad * ci_curlwp changes when a fast soft interrupt occurs.
925 1.245 ad * We use cpu_onproc to keep track of which kernel or
926 1.245 ad * user thread is running 'underneath' the software
927 1.245 ad * interrupt. This is important for time accounting,
928 1.245 ad * itimers and forcing user threads to preempt (aston).
929 1.245 ad */
930 1.245 ad ci->ci_data.cpu_onproc = newl;
931 1.245 ad
932 1.245 ad /*
933 1.245 ad * Preemption related tasks. Must be done with the current
934 1.245 ad * CPU locked.
935 1.245 ad */
936 1.245 ad cpu_did_resched(l);
937 1.245 ad
938 1.245 ad /* Unlock the run queue. */
939 1.245 ad spc_unlock(ci);
940 1.245 ad
941 1.245 ad /* Count the context switch on this CPU. */
942 1.245 ad ci->ci_data.cpu_nswtch++;
943 1.245 ad
944 1.245 ad /* Update status for lwpctl, if present. */
945 1.245 ad if (l->l_lwpctl != NULL)
946 1.247 ad l->l_lwpctl->lc_curcpu = LWPCTL_CPU_EXITED;
947 1.245 ad
948 1.245 ad /*
949 1.245 ad * We may need to spin-wait for if 'newl' is still
950 1.245 ad * context switching on another CPU.
951 1.245 ad */
952 1.261 rmind if (__predict_false(newl->l_ctxswtch != 0)) {
953 1.245 ad u_int count;
954 1.245 ad count = SPINLOCK_BACKOFF_MIN;
955 1.245 ad while (newl->l_ctxswtch)
956 1.245 ad SPINLOCK_BACKOFF(count);
957 1.245 ad }
958 1.245 ad
959 1.245 ad /* Switch to the new LWP.. */
960 1.245 ad (void)cpu_switchto(NULL, newl, false);
961 1.245 ad
962 1.251 uwe for (;;) continue; /* XXX: convince gcc about "noreturn" */
963 1.245 ad /* NOTREACHED */
964 1.245 ad }
965 1.245 ad
966 1.245 ad /*
967 1.255 skrll * Change LWP state to be runnable, placing it on the run queue if it is
968 1.174 ad * in memory, and awakening the swapper if it isn't in memory.
969 1.174 ad *
970 1.174 ad * Call with the process and LWP locked. Will return with the LWP unlocked.
971 1.26 cgd */
972 1.26 cgd void
973 1.122 thorpej setrunnable(struct lwp *l)
974 1.26 cgd {
975 1.122 thorpej struct proc *p = l->l_proc;
976 1.205 ad struct cpu_info *ci;
977 1.26 cgd
978 1.188 yamt KASSERT((l->l_flag & LW_IDLE) == 0);
979 1.229 ad KASSERT(mutex_owned(p->p_lock));
980 1.183 ad KASSERT(lwp_locked(l, NULL));
981 1.205 ad KASSERT(l->l_mutex != l->l_cpu->ci_schedstate.spc_mutex);
982 1.83 thorpej
983 1.122 thorpej switch (l->l_stat) {
984 1.122 thorpej case LSSTOP:
985 1.33 mycroft /*
986 1.33 mycroft * If we're being traced (possibly because someone attached us
987 1.33 mycroft * while we were stopped), check for a signal from the debugger.
988 1.33 mycroft */
989 1.256 ad if ((p->p_slflag & PSL_TRACED) != 0 && p->p_xstat != 0)
990 1.174 ad signotify(l);
991 1.174 ad p->p_nrlwps++;
992 1.26 cgd break;
993 1.174 ad case LSSUSPENDED:
994 1.178 pavel l->l_flag &= ~LW_WSUSPEND;
995 1.174 ad p->p_nrlwps++;
996 1.192 rmind cv_broadcast(&p->p_lwpcv);
997 1.122 thorpej break;
998 1.174 ad case LSSLEEP:
999 1.174 ad KASSERT(l->l_wchan != NULL);
1000 1.26 cgd break;
1001 1.174 ad default:
1002 1.174 ad panic("setrunnable: lwp %p state was %d", l, l->l_stat);
1003 1.26 cgd }
1004 1.139 cl
1005 1.252 wrstuden #ifdef KERN_SA
1006 1.252 wrstuden if (l->l_proc->p_sa)
1007 1.252 wrstuden sa_awaken(l);
1008 1.252 wrstuden #endif /* KERN_SA */
1009 1.252 wrstuden
1010 1.174 ad /*
1011 1.174 ad * If the LWP was sleeping interruptably, then it's OK to start it
1012 1.174 ad * again. If not, mark it as still sleeping.
1013 1.174 ad */
1014 1.174 ad if (l->l_wchan != NULL) {
1015 1.174 ad l->l_stat = LSSLEEP;
1016 1.183 ad /* lwp_unsleep() will release the lock. */
1017 1.221 ad lwp_unsleep(l, true);
1018 1.174 ad return;
1019 1.174 ad }
1020 1.139 cl
1021 1.174 ad /*
1022 1.174 ad * If the LWP is still on the CPU, mark it as LSONPROC. It may be
1023 1.174 ad * about to call mi_switch(), in which case it will yield.
1024 1.174 ad */
1025 1.248 ad if ((l->l_pflag & LP_RUNNING) != 0) {
1026 1.174 ad l->l_stat = LSONPROC;
1027 1.174 ad l->l_slptime = 0;
1028 1.174 ad lwp_unlock(l);
1029 1.174 ad return;
1030 1.174 ad }
1031 1.122 thorpej
1032 1.174 ad /*
1033 1.205 ad * Look for a CPU to run.
1034 1.205 ad * Set the LWP runnable.
1035 1.174 ad */
1036 1.205 ad ci = sched_takecpu(l);
1037 1.205 ad l->l_cpu = ci;
1038 1.236 ad spc_lock(ci);
1039 1.236 ad lwp_unlock_to(l, ci->ci_schedstate.spc_mutex);
1040 1.188 yamt sched_setrunnable(l);
1041 1.174 ad l->l_stat = LSRUN;
1042 1.122 thorpej l->l_slptime = 0;
1043 1.174 ad
1044 1.205 ad /*
1045 1.205 ad * If thread is swapped out - wake the swapper to bring it back in.
1046 1.205 ad * Otherwise, enter it into a run queue.
1047 1.205 ad */
1048 1.178 pavel if (l->l_flag & LW_INMEM) {
1049 1.188 yamt sched_enqueue(l, false);
1050 1.188 yamt resched_cpu(l);
1051 1.174 ad lwp_unlock(l);
1052 1.174 ad } else {
1053 1.174 ad lwp_unlock(l);
1054 1.177 ad uvm_kick_scheduler();
1055 1.174 ad }
1056 1.26 cgd }
1057 1.26 cgd
1058 1.26 cgd /*
1059 1.174 ad * suspendsched:
1060 1.174 ad *
1061 1.266 yamt * Convert all non-LW_SYSTEM LSSLEEP or LSRUN LWPs to LSSUSPENDED.
1062 1.174 ad */
1063 1.94 bouyer void
1064 1.174 ad suspendsched(void)
1065 1.94 bouyer {
1066 1.174 ad CPU_INFO_ITERATOR cii;
1067 1.174 ad struct cpu_info *ci;
1068 1.122 thorpej struct lwp *l;
1069 1.174 ad struct proc *p;
1070 1.94 bouyer
1071 1.94 bouyer /*
1072 1.174 ad * We do this by process in order not to violate the locking rules.
1073 1.94 bouyer */
1074 1.228 ad mutex_enter(proc_lock);
1075 1.174 ad PROCLIST_FOREACH(p, &allproc) {
1076 1.238 ad if ((p->p_flag & PK_MARKER) != 0)
1077 1.238 ad continue;
1078 1.238 ad
1079 1.229 ad mutex_enter(p->p_lock);
1080 1.178 pavel if ((p->p_flag & PK_SYSTEM) != 0) {
1081 1.229 ad mutex_exit(p->p_lock);
1082 1.94 bouyer continue;
1083 1.174 ad }
1084 1.174 ad
1085 1.174 ad p->p_stat = SSTOP;
1086 1.174 ad
1087 1.174 ad LIST_FOREACH(l, &p->p_lwps, l_sibling) {
1088 1.174 ad if (l == curlwp)
1089 1.174 ad continue;
1090 1.174 ad
1091 1.174 ad lwp_lock(l);
1092 1.122 thorpej
1093 1.97 enami /*
1094 1.174 ad * Set L_WREBOOT so that the LWP will suspend itself
1095 1.174 ad * when it tries to return to user mode. We want to
1096 1.174 ad * try and get to get as many LWPs as possible to
1097 1.174 ad * the user / kernel boundary, so that they will
1098 1.174 ad * release any locks that they hold.
1099 1.97 enami */
1100 1.178 pavel l->l_flag |= (LW_WREBOOT | LW_WSUSPEND);
1101 1.174 ad
1102 1.174 ad if (l->l_stat == LSSLEEP &&
1103 1.178 pavel (l->l_flag & LW_SINTR) != 0) {
1104 1.174 ad /* setrunnable() will release the lock. */
1105 1.174 ad setrunnable(l);
1106 1.174 ad continue;
1107 1.174 ad }
1108 1.174 ad
1109 1.174 ad lwp_unlock(l);
1110 1.94 bouyer }
1111 1.174 ad
1112 1.229 ad mutex_exit(p->p_lock);
1113 1.94 bouyer }
1114 1.228 ad mutex_exit(proc_lock);
1115 1.174 ad
1116 1.174 ad /*
1117 1.174 ad * Kick all CPUs to make them preempt any LWPs running in user mode.
1118 1.174 ad * They'll trap into the kernel and suspend themselves in userret().
1119 1.174 ad */
1120 1.204 ad for (CPU_INFO_FOREACH(cii, ci)) {
1121 1.204 ad spc_lock(ci);
1122 1.204 ad cpu_need_resched(ci, RESCHED_IMMED);
1123 1.204 ad spc_unlock(ci);
1124 1.204 ad }
1125 1.174 ad }
1126 1.174 ad
1127 1.174 ad /*
1128 1.174 ad * sched_unsleep:
1129 1.174 ad *
1130 1.174 ad * The is called when the LWP has not been awoken normally but instead
1131 1.174 ad * interrupted: for example, if the sleep timed out. Because of this,
1132 1.174 ad * it's not a valid action for running or idle LWPs.
1133 1.174 ad */
1134 1.221 ad static u_int
1135 1.221 ad sched_unsleep(struct lwp *l, bool cleanup)
1136 1.174 ad {
1137 1.174 ad
1138 1.174 ad lwp_unlock(l);
1139 1.174 ad panic("sched_unsleep");
1140 1.174 ad }
1141 1.174 ad
1142 1.250 rmind static void
1143 1.188 yamt resched_cpu(struct lwp *l)
1144 1.188 yamt {
1145 1.250 rmind struct cpu_info *ci = ci = l->l_cpu;
1146 1.188 yamt
1147 1.250 rmind KASSERT(lwp_locked(l, NULL));
1148 1.204 ad if (lwp_eprio(l) > ci->ci_schedstate.spc_curpriority)
1149 1.188 yamt cpu_need_resched(ci, 0);
1150 1.188 yamt }
1151 1.188 yamt
1152 1.188 yamt static void
1153 1.185 yamt sched_changepri(struct lwp *l, pri_t pri)
1154 1.174 ad {
1155 1.174 ad
1156 1.188 yamt KASSERT(lwp_locked(l, NULL));
1157 1.174 ad
1158 1.204 ad if (l->l_stat == LSRUN && (l->l_flag & LW_INMEM) != 0) {
1159 1.204 ad KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_mutex));
1160 1.204 ad sched_dequeue(l);
1161 1.204 ad l->l_priority = pri;
1162 1.204 ad sched_enqueue(l, false);
1163 1.204 ad } else {
1164 1.174 ad l->l_priority = pri;
1165 1.157 yamt }
1166 1.188 yamt resched_cpu(l);
1167 1.184 yamt }
1168 1.184 yamt
1169 1.188 yamt static void
1170 1.185 yamt sched_lendpri(struct lwp *l, pri_t pri)
1171 1.184 yamt {
1172 1.184 yamt
1173 1.188 yamt KASSERT(lwp_locked(l, NULL));
1174 1.184 yamt
1175 1.204 ad if (l->l_stat == LSRUN && (l->l_flag & LW_INMEM) != 0) {
1176 1.204 ad KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_mutex));
1177 1.204 ad sched_dequeue(l);
1178 1.204 ad l->l_inheritedprio = pri;
1179 1.204 ad sched_enqueue(l, false);
1180 1.204 ad } else {
1181 1.184 yamt l->l_inheritedprio = pri;
1182 1.184 yamt }
1183 1.188 yamt resched_cpu(l);
1184 1.184 yamt }
1185 1.184 yamt
1186 1.184 yamt struct lwp *
1187 1.184 yamt syncobj_noowner(wchan_t wchan)
1188 1.184 yamt {
1189 1.184 yamt
1190 1.184 yamt return NULL;
1191 1.151 yamt }
1192 1.151 yamt
1193 1.250 rmind /* Decay 95% of proc::p_pctcpu in 60 seconds, ccpu = exp(-1/20) */
1194 1.250 rmind const fixpt_t ccpu = 0.95122942450071400909 * FSCALE;
1195 1.134 matt
1196 1.134 matt /*
1197 1.188 yamt * sched_pstats:
1198 1.188 yamt *
1199 1.188 yamt * Update process statistics and check CPU resource allocation.
1200 1.188 yamt * Call scheduler-specific hook to eventually adjust process/LWP
1201 1.188 yamt * priorities.
1202 1.130 nathanw */
1203 1.188 yamt /* ARGSUSED */
1204 1.113 gmcgarry void
1205 1.188 yamt sched_pstats(void *arg)
1206 1.113 gmcgarry {
1207 1.249 rmind const int clkhz = (stathz != 0 ? stathz : hz);
1208 1.260 ad static bool backwards;
1209 1.188 yamt struct rlimit *rlim;
1210 1.188 yamt struct lwp *l;
1211 1.188 yamt struct proc *p;
1212 1.188 yamt long runtm;
1213 1.249 rmind fixpt_t lpctcpu;
1214 1.249 rmind u_int lcpticks;
1215 1.249 rmind int sig;
1216 1.113 gmcgarry
1217 1.188 yamt sched_pstats_ticks++;
1218 1.174 ad
1219 1.228 ad mutex_enter(proc_lock);
1220 1.188 yamt PROCLIST_FOREACH(p, &allproc) {
1221 1.249 rmind if (__predict_false((p->p_flag & PK_MARKER) != 0))
1222 1.238 ad continue;
1223 1.238 ad
1224 1.188 yamt /*
1225 1.250 rmind * Increment time in/out of memory and sleep
1226 1.250 rmind * time (if sleeping), ignore overflow.
1227 1.188 yamt */
1228 1.229 ad mutex_enter(p->p_lock);
1229 1.212 yamt runtm = p->p_rtime.sec;
1230 1.188 yamt LIST_FOREACH(l, &p->p_lwps, l_sibling) {
1231 1.249 rmind if (__predict_false((l->l_flag & LW_IDLE) != 0))
1232 1.188 yamt continue;
1233 1.188 yamt lwp_lock(l);
1234 1.212 yamt runtm += l->l_rtime.sec;
1235 1.188 yamt l->l_swtime++;
1236 1.242 rmind sched_lwp_stats(l);
1237 1.188 yamt lwp_unlock(l);
1238 1.113 gmcgarry
1239 1.188 yamt l->l_pctcpu = (l->l_pctcpu * ccpu) >> FSHIFT;
1240 1.249 rmind if (l->l_slptime != 0)
1241 1.249 rmind continue;
1242 1.249 rmind
1243 1.249 rmind lpctcpu = l->l_pctcpu;
1244 1.249 rmind lcpticks = atomic_swap_uint(&l->l_cpticks, 0);
1245 1.249 rmind lpctcpu += ((FSCALE - ccpu) *
1246 1.249 rmind (lcpticks * FSCALE / clkhz)) >> FSHIFT;
1247 1.249 rmind l->l_pctcpu = lpctcpu;
1248 1.188 yamt }
1249 1.249 rmind /* Calculating p_pctcpu only for ps(1) */
1250 1.188 yamt p->p_pctcpu = (p->p_pctcpu * ccpu) >> FSHIFT;
1251 1.174 ad
1252 1.188 yamt /*
1253 1.188 yamt * Check if the process exceeds its CPU resource allocation.
1254 1.188 yamt * If over max, kill it.
1255 1.188 yamt */
1256 1.188 yamt rlim = &p->p_rlimit[RLIMIT_CPU];
1257 1.188 yamt sig = 0;
1258 1.249 rmind if (__predict_false(runtm >= rlim->rlim_cur)) {
1259 1.188 yamt if (runtm >= rlim->rlim_max)
1260 1.188 yamt sig = SIGKILL;
1261 1.188 yamt else {
1262 1.188 yamt sig = SIGXCPU;
1263 1.188 yamt if (rlim->rlim_cur < rlim->rlim_max)
1264 1.188 yamt rlim->rlim_cur += 5;
1265 1.188 yamt }
1266 1.188 yamt }
1267 1.229 ad mutex_exit(p->p_lock);
1268 1.259 rmind if (__predict_false(runtm < 0)) {
1269 1.260 ad if (!backwards) {
1270 1.260 ad backwards = true;
1271 1.260 ad printf("WARNING: negative runtime; "
1272 1.260 ad "monotonic clock has gone backwards\n");
1273 1.260 ad }
1274 1.259 rmind } else if (__predict_false(sig)) {
1275 1.259 rmind KASSERT((p->p_flag & PK_SYSTEM) == 0);
1276 1.188 yamt psignal(p, sig);
1277 1.259 rmind }
1278 1.174 ad }
1279 1.228 ad mutex_exit(proc_lock);
1280 1.188 yamt uvm_meter();
1281 1.191 ad cv_wakeup(&lbolt);
1282 1.188 yamt callout_schedule(&sched_pstats_ch, hz);
1283 1.113 gmcgarry }
1284