kern_sleepq.c revision 1.28.2.1
1 /* $NetBSD: kern_sleepq.c,v 1.28.2.1 2008/05/23 05:24:16 wrstuden Exp $ */ 2 3 /*- 4 * Copyright (c) 2006, 2007, 2008 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Sleep queue implementation, used by turnstiles and general sleep/wakeup 34 * interfaces. 35 */ 36 37 #include <sys/cdefs.h> 38 __KERNEL_RCSID(0, "$NetBSD: kern_sleepq.c,v 1.28.2.1 2008/05/23 05:24:16 wrstuden Exp $"); 39 40 #include <sys/param.h> 41 #include <sys/kernel.h> 42 #include <sys/cpu.h> 43 #include <sys/pool.h> 44 #include <sys/proc.h> 45 #include <sys/resourcevar.h> 46 #include <sys/sa.h> 47 #include <sys/savar.h> 48 #include <sys/sched.h> 49 #include <sys/systm.h> 50 #include <sys/sleepq.h> 51 #include <sys/ktrace.h> 52 53 #include <uvm/uvm_extern.h> 54 55 int sleepq_sigtoerror(lwp_t *, int); 56 57 /* General purpose sleep table, used by ltsleep() and condition variables. */ 58 sleeptab_t sleeptab; 59 60 /* 61 * sleeptab_init: 62 * 63 * Initialize a sleep table. 64 */ 65 void 66 sleeptab_init(sleeptab_t *st) 67 { 68 sleepq_t *sq; 69 int i; 70 71 for (i = 0; i < SLEEPTAB_HASH_SIZE; i++) { 72 sq = &st->st_queues[i].st_queue; 73 mutex_init(&st->st_queues[i].st_mutex, MUTEX_DEFAULT, 74 IPL_SCHED); 75 sleepq_init(sq, &st->st_queues[i].st_mutex); 76 } 77 } 78 79 /* 80 * sleepq_init: 81 * 82 * Prepare a sleep queue for use. 83 */ 84 void 85 sleepq_init(sleepq_t *sq, kmutex_t *mtx) 86 { 87 88 sq->sq_waiters = 0; 89 sq->sq_mutex = mtx; 90 TAILQ_INIT(&sq->sq_queue); 91 } 92 93 /* 94 * sleepq_remove: 95 * 96 * Remove an LWP from a sleep queue and wake it up. Return non-zero if 97 * the LWP is swapped out; if so the caller needs to awaken the swapper 98 * to bring the LWP into memory. 99 */ 100 int 101 sleepq_remove(sleepq_t *sq, lwp_t *l) 102 { 103 struct schedstate_percpu *spc; 104 struct cpu_info *ci; 105 106 KASSERT(lwp_locked(l, sq->sq_mutex)); 107 KASSERT(sq->sq_waiters > 0); 108 109 sq->sq_waiters--; 110 TAILQ_REMOVE(&sq->sq_queue, l, l_sleepchain); 111 112 #ifdef DIAGNOSTIC 113 if (sq->sq_waiters == 0) 114 KASSERT(TAILQ_FIRST(&sq->sq_queue) == NULL); 115 else 116 KASSERT(TAILQ_FIRST(&sq->sq_queue) != NULL); 117 #endif 118 119 l->l_syncobj = &sched_syncobj; 120 l->l_wchan = NULL; 121 l->l_sleepq = NULL; 122 l->l_flag &= ~LW_SINTR; 123 124 ci = l->l_cpu; 125 spc = &ci->ci_schedstate; 126 127 /* 128 * If not sleeping, the LWP must have been suspended. Let whoever 129 * holds it stopped set it running again. 130 */ 131 if (l->l_stat != LSSLEEP) { 132 KASSERT(l->l_stat == LSSTOP || l->l_stat == LSSUSPENDED); 133 lwp_setlock(l, spc->spc_lwplock); 134 return 0; 135 } 136 137 /* 138 * If the LWP is still on the CPU, mark it as LSONPROC. It may be 139 * about to call mi_switch(), in which case it will yield. 140 */ 141 if ((l->l_flag & LW_RUNNING) != 0) { 142 l->l_stat = LSONPROC; 143 l->l_slptime = 0; 144 lwp_setlock(l, spc->spc_lwplock); 145 return 0; 146 } 147 148 /* 149 * Call the wake-up handler of scheduler. 150 * It might change the CPU for this thread. 151 */ 152 sched_wakeup(l); 153 ci = l->l_cpu; 154 spc = &ci->ci_schedstate; 155 156 /* 157 * Set it running. 158 */ 159 spc_lock(ci); 160 lwp_setlock(l, spc->spc_mutex); 161 if (l->l_proc->p_sa != NULL) 162 sa_awaken(l); 163 sched_setrunnable(l); 164 l->l_stat = LSRUN; 165 l->l_slptime = 0; 166 if ((l->l_flag & LW_INMEM) != 0) { 167 sched_enqueue(l, false); 168 spc_unlock(ci); 169 return 0; 170 } 171 spc_unlock(ci); 172 return 1; 173 } 174 175 /* 176 * sleepq_insert: 177 * 178 * Insert an LWP into the sleep queue, optionally sorting by priority. 179 */ 180 inline void 181 sleepq_insert(sleepq_t *sq, lwp_t *l, syncobj_t *sobj) 182 { 183 lwp_t *l2; 184 const int pri = lwp_eprio(l); 185 186 if ((sobj->sobj_flag & SOBJ_SLEEPQ_SORTED) != 0) { 187 TAILQ_FOREACH(l2, &sq->sq_queue, l_sleepchain) { 188 if (lwp_eprio(l2) < pri) { 189 TAILQ_INSERT_BEFORE(l2, l, l_sleepchain); 190 return; 191 } 192 } 193 } 194 195 if ((sobj->sobj_flag & SOBJ_SLEEPQ_LIFO) != 0) 196 TAILQ_INSERT_HEAD(&sq->sq_queue, l, l_sleepchain); 197 else 198 TAILQ_INSERT_TAIL(&sq->sq_queue, l, l_sleepchain); 199 } 200 201 /* 202 * sleepq_enqueue: 203 * 204 * Enter an LWP into the sleep queue and prepare for sleep. The sleep 205 * queue must already be locked, and any interlock (such as the kernel 206 * lock) must have be released (see sleeptab_lookup(), sleepq_enter()). 207 */ 208 void 209 sleepq_enqueue(sleepq_t *sq, wchan_t wchan, const char *wmesg, syncobj_t *sobj) 210 { 211 lwp_t *l = curlwp; 212 213 KASSERT(lwp_locked(l, sq->sq_mutex)); 214 KASSERT(l->l_stat == LSONPROC); 215 KASSERT(l->l_wchan == NULL && l->l_sleepq == NULL); 216 217 l->l_syncobj = sobj; 218 l->l_wchan = wchan; 219 l->l_sleepq = sq; 220 l->l_wmesg = wmesg; 221 l->l_slptime = 0; 222 l->l_stat = LSSLEEP; 223 l->l_sleeperr = 0; 224 225 sq->sq_waiters++; 226 sleepq_insert(sq, l, sobj); 227 sched_slept(l); 228 } 229 230 /* 231 * sleepq_block: 232 * 233 * After any intermediate step such as releasing an interlock, switch. 234 * sleepq_block() may return early under exceptional conditions, for 235 * example if the LWP's containing process is exiting. 236 */ 237 int 238 sleepq_block(int timo, bool catch) 239 { 240 int error = 0, sig; 241 struct proc *p; 242 lwp_t *l = curlwp; 243 bool early = false; 244 245 ktrcsw(1, 0); 246 247 /* 248 * If sleeping interruptably, check for pending signals, exits or 249 * core dump events. 250 */ 251 if (catch) { 252 l->l_flag |= LW_SINTR; 253 if ((l->l_flag & (LW_CANCELLED|LW_WEXIT|LW_WCORE)) != 0) { 254 l->l_flag &= ~LW_CANCELLED; 255 error = EINTR; 256 early = true; 257 } else if ((l->l_flag & LW_PENDSIG) != 0 && sigispending(l, 0)) 258 early = true; 259 } 260 261 if (early) { 262 /* lwp_unsleep() will release the lock */ 263 lwp_unsleep(l, true); 264 } else { 265 if (timo) 266 callout_schedule(&l->l_timeout_ch, timo); 267 268 if ((l->l_flag & LW_SA) != 0) 269 sa_switch(l); 270 else 271 mi_switch(l); 272 273 /* The LWP and sleep queue are now unlocked. */ 274 if (timo) { 275 /* 276 * Even if the callout appears to have fired, we need to 277 * stop it in order to synchronise with other CPUs. 278 */ 279 if (callout_halt(&l->l_timeout_ch, NULL)) 280 error = EWOULDBLOCK; 281 } 282 } 283 284 if (catch && error == 0) { 285 p = l->l_proc; 286 if ((l->l_flag & (LW_CANCELLED | LW_WEXIT | LW_WCORE)) != 0) 287 error = EINTR; 288 else if ((l->l_flag & LW_PENDSIG) != 0) { 289 mutex_enter(p->p_lock); 290 if ((sig = issignal(l)) != 0) 291 error = sleepq_sigtoerror(l, sig); 292 mutex_exit(p->p_lock); 293 } 294 } 295 296 ktrcsw(0, 0); 297 298 KERNEL_LOCK(l->l_biglocks, l); 299 return error; 300 } 301 302 /* 303 * sleepq_wake: 304 * 305 * Wake zero or more LWPs blocked on a single wait channel. 306 */ 307 lwp_t * 308 sleepq_wake(sleepq_t *sq, wchan_t wchan, u_int expected) 309 { 310 lwp_t *l, *next; 311 int swapin = 0; 312 313 KASSERT(mutex_owned(sq->sq_mutex)); 314 315 for (l = TAILQ_FIRST(&sq->sq_queue); l != NULL; l = next) { 316 KASSERT(l->l_sleepq == sq); 317 KASSERT(l->l_mutex == sq->sq_mutex); 318 next = TAILQ_NEXT(l, l_sleepchain); 319 if (l->l_wchan != wchan) 320 continue; 321 swapin |= sleepq_remove(sq, l); 322 if (--expected == 0) 323 break; 324 } 325 326 sleepq_unlock(sq); 327 328 /* 329 * If there are newly awakend threads that need to be swapped in, 330 * then kick the swapper into action. 331 */ 332 if (swapin) 333 uvm_kick_scheduler(); 334 335 return l; 336 } 337 338 /* 339 * sleepq_unsleep: 340 * 341 * Remove an LWP from its sleep queue and set it runnable again. 342 * sleepq_unsleep() is called with the LWP's mutex held, and will 343 * always release it. 344 */ 345 u_int 346 sleepq_unsleep(lwp_t *l, bool cleanup) 347 { 348 sleepq_t *sq = l->l_sleepq; 349 int swapin; 350 351 KASSERT(lwp_locked(l, sq->sq_mutex)); 352 KASSERT(l->l_wchan != NULL); 353 354 swapin = sleepq_remove(sq, l); 355 356 if (cleanup) { 357 sleepq_unlock(sq); 358 if (swapin) 359 uvm_kick_scheduler(); 360 } 361 362 return swapin; 363 } 364 365 /* 366 * sleepq_timeout: 367 * 368 * Entered via the callout(9) subsystem to time out an LWP that is on a 369 * sleep queue. 370 */ 371 void 372 sleepq_timeout(void *arg) 373 { 374 lwp_t *l = arg; 375 376 /* 377 * Lock the LWP. Assuming it's still on the sleep queue, its 378 * current mutex will also be the sleep queue mutex. 379 */ 380 lwp_lock(l); 381 382 if (l->l_wchan == NULL) { 383 /* Somebody beat us to it. */ 384 lwp_unlock(l); 385 return; 386 } 387 388 lwp_unsleep(l, true); 389 } 390 391 /* 392 * sleepq_sigtoerror: 393 * 394 * Given a signal number, interpret and return an error code. 395 */ 396 int 397 sleepq_sigtoerror(lwp_t *l, int sig) 398 { 399 struct proc *p = l->l_proc; 400 int error; 401 402 KASSERT(mutex_owned(p->p_lock)); 403 404 /* 405 * If this sleep was canceled, don't let the syscall restart. 406 */ 407 if ((SIGACTION(p, sig).sa_flags & SA_RESTART) == 0) 408 error = EINTR; 409 else 410 error = ERESTART; 411 412 return error; 413 } 414 415 /* 416 * sleepq_abort: 417 * 418 * After a panic or during autoconfiguration, lower the interrupt 419 * priority level to give pending interrupts a chance to run, and 420 * then return. Called if sleepq_dontsleep() returns non-zero, and 421 * always returns zero. 422 */ 423 int 424 sleepq_abort(kmutex_t *mtx, int unlock) 425 { 426 extern int safepri; 427 int s; 428 429 s = splhigh(); 430 splx(safepri); 431 splx(s); 432 if (mtx != NULL && unlock != 0) 433 mutex_exit(mtx); 434 435 return 0; 436 } 437 438 /* 439 * sleepq_changepri: 440 * 441 * Adjust the priority of an LWP residing on a sleepq. This method 442 * will only alter the user priority; the effective priority is 443 * assumed to have been fixed at the time of insertion into the queue. 444 */ 445 void 446 sleepq_changepri(lwp_t *l, pri_t pri) 447 { 448 sleepq_t *sq = l->l_sleepq; 449 pri_t opri; 450 451 KASSERT(lwp_locked(l, sq->sq_mutex)); 452 453 opri = lwp_eprio(l); 454 l->l_priority = pri; 455 if (lwp_eprio(l) != opri) { 456 TAILQ_REMOVE(&sq->sq_queue, l, l_sleepchain); 457 sleepq_insert(sq, l, l->l_syncobj); 458 } 459 } 460 461 void 462 sleepq_lendpri(lwp_t *l, pri_t pri) 463 { 464 sleepq_t *sq = l->l_sleepq; 465 pri_t opri; 466 467 KASSERT(lwp_locked(l, sq->sq_mutex)); 468 469 opri = lwp_eprio(l); 470 l->l_inheritedprio = pri; 471 472 if (lwp_eprio(l) != opri && 473 (l->l_syncobj->sobj_flag & SOBJ_SLEEPQ_SORTED) != 0) { 474 TAILQ_REMOVE(&sq->sq_queue, l, l_sleepchain); 475 sleepq_insert(sq, l, l->l_syncobj); 476 } 477 } 478
Indexes created Sat Sep 20 22:09:52 GMT 2025