kern_lock.c revision 1.99
1 /* $NetBSD: kern_lock.c,v 1.99 2006/09/07 02:06:47 ad Exp $ */ 2 3 /*- 4 * Copyright (c) 1999, 2000 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center. 10 * 11 * This code is derived from software contributed to The NetBSD Foundation 12 * by Ross Harvey. 13 * 14 * Redistribution and use in source and binary forms, with or without 15 * modification, are permitted provided that the following conditions 16 * are met: 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in the 21 * documentation and/or other materials provided with the distribution. 22 * 3. All advertising materials mentioning features or use of this software 23 * must display the following acknowledgement: 24 * This product includes software developed by the NetBSD 25 * Foundation, Inc. and its contributors. 26 * 4. Neither the name of The NetBSD Foundation nor the names of its 27 * contributors may be used to endorse or promote products derived 28 * from this software without specific prior written permission. 29 * 30 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 31 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 32 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 33 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 34 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 35 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 36 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 37 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 38 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 39 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 40 * POSSIBILITY OF SUCH DAMAGE. 41 */ 42 43 /* 44 * Copyright (c) 1995 45 * The Regents of the University of California. All rights reserved. 46 * 47 * This code contains ideas from software contributed to Berkeley by 48 * Avadis Tevanian, Jr., Michael Wayne Young, and the Mach Operating 49 * System project at Carnegie-Mellon University. 50 * 51 * Redistribution and use in source and binary forms, with or without 52 * modification, are permitted provided that the following conditions 53 * are met: 54 * 1. Redistributions of source code must retain the above copyright 55 * notice, this list of conditions and the following disclaimer. 56 * 2. Redistributions in binary form must reproduce the above copyright 57 * notice, this list of conditions and the following disclaimer in the 58 * documentation and/or other materials provided with the distribution. 59 * 3. Neither the name of the University nor the names of its contributors 60 * may be used to endorse or promote products derived from this software 61 * without specific prior written permission. 62 * 63 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 64 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 65 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 66 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 67 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 68 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 69 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 70 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 71 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 72 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 73 * SUCH DAMAGE. 74 * 75 * @(#)kern_lock.c 8.18 (Berkeley) 5/21/95 76 */ 77 78 #include <sys/cdefs.h> 79 __KERNEL_RCSID(0, "$NetBSD: kern_lock.c,v 1.99 2006/09/07 02:06:47 ad Exp $"); 80 81 #include "opt_multiprocessor.h" 82 #include "opt_lockdebug.h" 83 #include "opt_ddb.h" 84 85 #include <sys/param.h> 86 #include <sys/proc.h> 87 #include <sys/lock.h> 88 #include <sys/systm.h> 89 #include <machine/cpu.h> 90 91 #include <dev/lockstat.h> 92 93 #if defined(LOCKDEBUG) 94 #include <sys/syslog.h> 95 /* 96 * note that stdarg.h and the ansi style va_start macro is used for both 97 * ansi and traditional c compiles. 98 * XXX: this requires that stdarg.h define: va_alist and va_dcl 99 */ 100 #include <machine/stdarg.h> 101 102 void lock_printf(const char *fmt, ...) 103 __attribute__((__format__(__printf__,1,2))); 104 105 static int acquire(volatile struct lock **, int *, int, int, int, uintptr_t ra); 106 107 int lock_debug_syslog = 0; /* defaults to printf, but can be patched */ 108 109 #ifdef DDB 110 #include <ddb/ddbvar.h> 111 #include <machine/db_machdep.h> 112 #include <ddb/db_command.h> 113 #include <ddb/db_interface.h> 114 #endif 115 #endif /* defined(LOCKDEBUG) */ 116 117 #if defined(MULTIPROCESSOR) 118 struct simplelock kernel_lock; 119 #endif 120 121 /* 122 * Locking primitives implementation. 123 * Locks provide shared/exclusive synchronization. 124 */ 125 126 #if defined(LOCKDEBUG) || defined(DIAGNOSTIC) /* { */ 127 #if defined(MULTIPROCESSOR) /* { */ 128 #define COUNT_CPU(cpu_id, x) \ 129 curcpu()->ci_spin_locks += (x) 130 #else 131 u_long spin_locks; 132 #define COUNT_CPU(cpu_id, x) spin_locks += (x) 133 #endif /* MULTIPROCESSOR */ /* } */ 134 135 #define COUNT(lkp, l, cpu_id, x) \ 136 do { \ 137 if ((lkp)->lk_flags & LK_SPIN) \ 138 COUNT_CPU((cpu_id), (x)); \ 139 else \ 140 (l)->l_locks += (x); \ 141 } while (/*CONSTCOND*/0) 142 #else 143 #define COUNT(lkp, p, cpu_id, x) 144 #define COUNT_CPU(cpu_id, x) 145 #endif /* LOCKDEBUG || DIAGNOSTIC */ /* } */ 146 147 #define INTERLOCK_ACQUIRE(lkp, flags, s) \ 148 do { \ 149 if ((flags) & LK_SPIN) \ 150 s = spllock(); \ 151 simple_lock(&(lkp)->lk_interlock); \ 152 } while (/*CONSTCOND*/ 0) 153 154 #define INTERLOCK_RELEASE(lkp, flags, s) \ 155 do { \ 156 simple_unlock(&(lkp)->lk_interlock); \ 157 if ((flags) & LK_SPIN) \ 158 splx(s); \ 159 } while (/*CONSTCOND*/ 0) 160 161 #ifdef DDB /* { */ 162 #if defined(MULTIPROCESSOR) || defined(LOCKDEBUG) 163 int simple_lock_debugger = 1; /* more serious on MP */ 164 #else 165 int simple_lock_debugger = 0; 166 #endif 167 #define SLOCK_DEBUGGER() if (simple_lock_debugger && db_onpanic) Debugger() 168 #define SLOCK_TRACE() \ 169 db_stack_trace_print((db_expr_t)__builtin_frame_address(0), \ 170 TRUE, 65535, "", lock_printf); 171 #else 172 #define SLOCK_DEBUGGER() /* nothing */ 173 #define SLOCK_TRACE() /* nothing */ 174 #endif /* } */ 175 176 #if defined(LOCKDEBUG) 177 #if defined(DDB) 178 #define SPINLOCK_SPINCHECK_DEBUGGER if (db_onpanic) Debugger() 179 #else 180 #define SPINLOCK_SPINCHECK_DEBUGGER /* nothing */ 181 #endif 182 183 #define SPINLOCK_SPINCHECK_DECL \ 184 /* 32-bits of count -- wrap constitutes a "spinout" */ \ 185 uint32_t __spinc = 0 186 187 #define SPINLOCK_SPINCHECK \ 188 do { \ 189 if (++__spinc == 0) { \ 190 lock_printf("LK_SPIN spinout, excl %d, share %d\n", \ 191 lkp->lk_exclusivecount, lkp->lk_sharecount); \ 192 if (lkp->lk_exclusivecount) \ 193 lock_printf("held by CPU %lu\n", \ 194 (u_long) lkp->lk_cpu); \ 195 if (lkp->lk_lock_file) \ 196 lock_printf("last locked at %s:%d\n", \ 197 lkp->lk_lock_file, lkp->lk_lock_line); \ 198 if (lkp->lk_unlock_file) \ 199 lock_printf("last unlocked at %s:%d\n", \ 200 lkp->lk_unlock_file, lkp->lk_unlock_line); \ 201 SLOCK_TRACE(); \ 202 SPINLOCK_SPINCHECK_DEBUGGER; \ 203 } \ 204 } while (/*CONSTCOND*/ 0) 205 #else 206 #define SPINLOCK_SPINCHECK_DECL /* nothing */ 207 #define SPINLOCK_SPINCHECK /* nothing */ 208 #endif /* LOCKDEBUG && DDB */ 209 210 #define RETURN_ADDRESS ((uintptr_t)__builtin_return_address(0)) 211 212 /* 213 * Acquire a resource. 214 */ 215 static int 216 acquire(volatile struct lock **lkpp, int *s, int extflags, 217 int drain, int wanted, uintptr_t ra) 218 { 219 int error; 220 volatile struct lock *lkp = *lkpp; 221 LOCKSTAT_TIMER(slptime); 222 223 KASSERT(drain || (wanted & LK_WAIT_NONZERO) == 0); 224 225 if (extflags & LK_SPIN) { 226 int interlocked; 227 228 SPINLOCK_SPINCHECK_DECL; 229 230 if (!drain) { 231 lkp->lk_waitcount++; 232 lkp->lk_flags |= LK_WAIT_NONZERO; 233 } 234 for (interlocked = 1;;) { 235 SPINLOCK_SPINCHECK; 236 if ((lkp->lk_flags & wanted) != 0) { 237 if (interlocked) { 238 INTERLOCK_RELEASE(lkp, LK_SPIN, *s); 239 interlocked = 0; 240 } 241 SPINLOCK_SPIN_HOOK; 242 } else if (interlocked) { 243 break; 244 } else { 245 INTERLOCK_ACQUIRE(lkp, LK_SPIN, *s); 246 interlocked = 1; 247 } 248 } 249 if (!drain) { 250 lkp->lk_waitcount--; 251 if (lkp->lk_waitcount == 0) 252 lkp->lk_flags &= ~LK_WAIT_NONZERO; 253 } 254 KASSERT((lkp->lk_flags & wanted) == 0); 255 error = 0; /* sanity */ 256 } else { 257 for (error = 0; (lkp->lk_flags & wanted) != 0; ) { 258 if (drain) 259 lkp->lk_flags |= LK_WAITDRAIN; 260 else { 261 lkp->lk_waitcount++; 262 lkp->lk_flags |= LK_WAIT_NONZERO; 263 } 264 /* XXX Cast away volatile. */ 265 LOCKSTAT_START_TIMER(slptime); 266 error = ltsleep(drain ? 267 (volatile const void *)&lkp->lk_flags : 268 (volatile const void *)lkp, lkp->lk_prio, 269 lkp->lk_wmesg, lkp->lk_timo, &lkp->lk_interlock); 270 LOCKSTAT_STOP_TIMER(slptime); 271 LOCKSTAT_EVENT_RA((void *)(uintptr_t)lkp, 272 LB_LOCKMGR | LB_SLEEP, 1, slptime, ra); 273 if (!drain) { 274 lkp->lk_waitcount--; 275 if (lkp->lk_waitcount == 0) 276 lkp->lk_flags &= ~LK_WAIT_NONZERO; 277 } 278 if (error) 279 break; 280 if (extflags & LK_SLEEPFAIL) { 281 error = ENOLCK; 282 break; 283 } 284 if (lkp->lk_newlock != NULL) { 285 simple_lock(&lkp->lk_newlock->lk_interlock); 286 simple_unlock(&lkp->lk_interlock); 287 if (lkp->lk_waitcount == 0) 288 wakeup(&lkp->lk_newlock); 289 *lkpp = lkp = lkp->lk_newlock; 290 } 291 } 292 } 293 294 return error; 295 } 296 297 #define SETHOLDER(lkp, pid, lid, cpu_id) \ 298 do { \ 299 if ((lkp)->lk_flags & LK_SPIN) \ 300 (lkp)->lk_cpu = cpu_id; \ 301 else { \ 302 (lkp)->lk_lockholder = pid; \ 303 (lkp)->lk_locklwp = lid; \ 304 } \ 305 } while (/*CONSTCOND*/0) 306 307 #define WEHOLDIT(lkp, pid, lid, cpu_id) \ 308 (((lkp)->lk_flags & LK_SPIN) != 0 ? \ 309 ((lkp)->lk_cpu == (cpu_id)) : \ 310 ((lkp)->lk_lockholder == (pid) && (lkp)->lk_locklwp == (lid))) 311 312 #define WAKEUP_WAITER(lkp) \ 313 do { \ 314 if (((lkp)->lk_flags & (LK_SPIN | LK_WAIT_NONZERO)) == \ 315 LK_WAIT_NONZERO) { \ 316 wakeup((lkp)); \ 317 } \ 318 } while (/*CONSTCOND*/0) 319 320 #if defined(LOCKDEBUG) /* { */ 321 #if defined(MULTIPROCESSOR) /* { */ 322 struct simplelock spinlock_list_slock = SIMPLELOCK_INITIALIZER; 323 324 #define SPINLOCK_LIST_LOCK() \ 325 __cpu_simple_lock(&spinlock_list_slock.lock_data) 326 327 #define SPINLOCK_LIST_UNLOCK() \ 328 __cpu_simple_unlock(&spinlock_list_slock.lock_data) 329 #else 330 #define SPINLOCK_LIST_LOCK() /* nothing */ 331 332 #define SPINLOCK_LIST_UNLOCK() /* nothing */ 333 #endif /* MULTIPROCESSOR */ /* } */ 334 335 _TAILQ_HEAD(, struct lock, volatile) spinlock_list = 336 TAILQ_HEAD_INITIALIZER(spinlock_list); 337 338 #define HAVEIT(lkp) \ 339 do { \ 340 if ((lkp)->lk_flags & LK_SPIN) { \ 341 int sp = spllock(); \ 342 SPINLOCK_LIST_LOCK(); \ 343 TAILQ_INSERT_TAIL(&spinlock_list, (lkp), lk_list); \ 344 SPINLOCK_LIST_UNLOCK(); \ 345 splx(sp); \ 346 } \ 347 } while (/*CONSTCOND*/0) 348 349 #define DONTHAVEIT(lkp) \ 350 do { \ 351 if ((lkp)->lk_flags & LK_SPIN) { \ 352 int sp = spllock(); \ 353 SPINLOCK_LIST_LOCK(); \ 354 TAILQ_REMOVE(&spinlock_list, (lkp), lk_list); \ 355 SPINLOCK_LIST_UNLOCK(); \ 356 splx(sp); \ 357 } \ 358 } while (/*CONSTCOND*/0) 359 #else 360 #define HAVEIT(lkp) /* nothing */ 361 362 #define DONTHAVEIT(lkp) /* nothing */ 363 #endif /* LOCKDEBUG */ /* } */ 364 365 #if defined(LOCKDEBUG) 366 /* 367 * Lock debug printing routine; can be configured to print to console 368 * or log to syslog. 369 */ 370 void 371 lock_printf(const char *fmt, ...) 372 { 373 char b[150]; 374 va_list ap; 375 376 va_start(ap, fmt); 377 if (lock_debug_syslog) 378 vlog(LOG_DEBUG, fmt, ap); 379 else { 380 vsnprintf(b, sizeof(b), fmt, ap); 381 printf_nolog("%s", b); 382 } 383 va_end(ap); 384 } 385 #endif /* LOCKDEBUG */ 386 387 /* 388 * Transfer any waiting processes from one lock to another. 389 */ 390 void 391 transferlockers(struct lock *from, struct lock *to) 392 { 393 394 KASSERT(from != to); 395 KASSERT((from->lk_flags & LK_WAITDRAIN) == 0); 396 if (from->lk_waitcount == 0) 397 return; 398 from->lk_newlock = to; 399 wakeup((void *)from); 400 tsleep((void *)&from->lk_newlock, from->lk_prio, "lkxfer", 0); 401 from->lk_newlock = NULL; 402 from->lk_flags &= ~(LK_WANT_EXCL | LK_WANT_UPGRADE); 403 KASSERT(from->lk_waitcount == 0); 404 } 405 406 407 /* 408 * Initialize a lock; required before use. 409 */ 410 void 411 lockinit(struct lock *lkp, int prio, const char *wmesg, int timo, int flags) 412 { 413 414 memset(lkp, 0, sizeof(struct lock)); 415 simple_lock_init(&lkp->lk_interlock); 416 lkp->lk_flags = flags & LK_EXTFLG_MASK; 417 if (flags & LK_SPIN) 418 lkp->lk_cpu = LK_NOCPU; 419 else { 420 lkp->lk_lockholder = LK_NOPROC; 421 lkp->lk_newlock = NULL; 422 lkp->lk_prio = prio; 423 lkp->lk_timo = timo; 424 } 425 lkp->lk_wmesg = wmesg; /* just a name for spin locks */ 426 #if defined(LOCKDEBUG) 427 lkp->lk_lock_file = NULL; 428 lkp->lk_unlock_file = NULL; 429 #endif 430 } 431 432 /* 433 * Determine the status of a lock. 434 */ 435 int 436 lockstatus(struct lock *lkp) 437 { 438 int s = 0; /* XXX: gcc */ 439 int lock_type = 0; 440 struct lwp *l = curlwp; /* XXX */ 441 pid_t pid; 442 lwpid_t lid; 443 cpuid_t cpu_num; 444 445 if ((lkp->lk_flags & LK_SPIN) || l == NULL) { 446 cpu_num = cpu_number(); 447 pid = LK_KERNPROC; 448 lid = 0; 449 } else { 450 cpu_num = LK_NOCPU; 451 pid = l->l_proc->p_pid; 452 lid = l->l_lid; 453 } 454 455 INTERLOCK_ACQUIRE(lkp, lkp->lk_flags, s); 456 if (lkp->lk_exclusivecount != 0) { 457 if (WEHOLDIT(lkp, pid, lid, cpu_num)) 458 lock_type = LK_EXCLUSIVE; 459 else 460 lock_type = LK_EXCLOTHER; 461 } else if (lkp->lk_sharecount != 0) 462 lock_type = LK_SHARED; 463 INTERLOCK_RELEASE(lkp, lkp->lk_flags, s); 464 return (lock_type); 465 } 466 467 #if defined(LOCKDEBUG) 468 /* 469 * Make sure no spin locks are held by a CPU that is about 470 * to context switch. 471 */ 472 void 473 spinlock_switchcheck(void) 474 { 475 u_long cnt; 476 int s; 477 478 s = spllock(); 479 #if defined(MULTIPROCESSOR) 480 cnt = curcpu()->ci_spin_locks; 481 #else 482 cnt = spin_locks; 483 #endif 484 splx(s); 485 486 if (cnt != 0) 487 panic("spinlock_switchcheck: CPU %lu has %lu spin locks", 488 (u_long) cpu_number(), cnt); 489 } 490 #endif /* LOCKDEBUG */ 491 492 /* 493 * Locks and IPLs (interrupt priority levels): 494 * 495 * Locks which may be taken from interrupt context must be handled 496 * very carefully; you must spl to the highest IPL where the lock 497 * is needed before acquiring the lock. 498 * 499 * It is also important to avoid deadlock, since certain (very high 500 * priority) interrupts are often needed to keep the system as a whole 501 * from deadlocking, and must not be blocked while you are spinning 502 * waiting for a lower-priority lock. 503 * 504 * In addition, the lock-debugging hooks themselves need to use locks! 505 * 506 * A raw __cpu_simple_lock may be used from interrupts are long as it 507 * is acquired and held at a single IPL. 508 * 509 * A simple_lock (which is a __cpu_simple_lock wrapped with some 510 * debugging hooks) may be used at or below spllock(), which is 511 * typically at or just below splhigh() (i.e. blocks everything 512 * but certain machine-dependent extremely high priority interrupts). 513 * 514 * spinlockmgr spinlocks should be used at or below splsched(). 515 * 516 * Some platforms may have interrupts of higher priority than splsched(), 517 * including hard serial interrupts, inter-processor interrupts, and 518 * kernel debugger traps. 519 */ 520 521 /* 522 * XXX XXX kludge around another kludge.. 523 * 524 * vfs_shutdown() may be called from interrupt context, either as a result 525 * of a panic, or from the debugger. It proceeds to call 526 * sys_sync(&proc0, ...), pretending its running on behalf of proc0 527 * 528 * We would like to make an attempt to sync the filesystems in this case, so 529 * if this happens, we treat attempts to acquire locks specially. 530 * All locks are acquired on behalf of proc0. 531 * 532 * If we've already paniced, we don't block waiting for locks, but 533 * just barge right ahead since we're already going down in flames. 534 */ 535 536 /* 537 * Set, change, or release a lock. 538 * 539 * Shared requests increment the shared count. Exclusive requests set the 540 * LK_WANT_EXCL flag (preventing further shared locks), and wait for already 541 * accepted shared locks and shared-to-exclusive upgrades to go away. 542 */ 543 int 544 #if defined(LOCKDEBUG) 545 _lockmgr(volatile struct lock *lkp, u_int flags, 546 struct simplelock *interlkp, const char *file, int line) 547 #else 548 lockmgr(volatile struct lock *lkp, u_int flags, 549 struct simplelock *interlkp) 550 #endif 551 { 552 int error; 553 pid_t pid; 554 lwpid_t lid; 555 int extflags; 556 cpuid_t cpu_num; 557 struct lwp *l = curlwp; 558 int lock_shutdown_noblock = 0; 559 int s = 0; 560 561 error = 0; 562 563 /* LK_RETRY is for vn_lock, not for lockmgr. */ 564 KASSERT((flags & LK_RETRY) == 0); 565 566 INTERLOCK_ACQUIRE(lkp, lkp->lk_flags, s); 567 if (flags & LK_INTERLOCK) 568 simple_unlock(interlkp); 569 extflags = (flags | lkp->lk_flags) & LK_EXTFLG_MASK; 570 571 #ifdef DIAGNOSTIC /* { */ 572 /* 573 * Don't allow spins on sleep locks and don't allow sleeps 574 * on spin locks. 575 */ 576 if ((flags ^ lkp->lk_flags) & LK_SPIN) 577 panic("lockmgr: sleep/spin mismatch"); 578 #endif /* } */ 579 580 if (extflags & LK_SPIN) { 581 pid = LK_KERNPROC; 582 lid = 0; 583 } else { 584 if (l == NULL) { 585 if (!doing_shutdown) { 586 panic("lockmgr: no context"); 587 } else { 588 l = &lwp0; 589 if (panicstr && (!(flags & LK_NOWAIT))) { 590 flags |= LK_NOWAIT; 591 lock_shutdown_noblock = 1; 592 } 593 } 594 } 595 lid = l->l_lid; 596 pid = l->l_proc->p_pid; 597 } 598 cpu_num = cpu_number(); 599 600 /* 601 * Once a lock has drained, the LK_DRAINING flag is set and an 602 * exclusive lock is returned. The only valid operation thereafter 603 * is a single release of that exclusive lock. This final release 604 * clears the LK_DRAINING flag and sets the LK_DRAINED flag. Any 605 * further requests of any sort will result in a panic. The bits 606 * selected for these two flags are chosen so that they will be set 607 * in memory that is freed (freed memory is filled with 0xdeadbeef). 608 * The final release is permitted to give a new lease on life to 609 * the lock by specifying LK_REENABLE. 610 */ 611 if (lkp->lk_flags & (LK_DRAINING|LK_DRAINED)) { 612 #ifdef DIAGNOSTIC /* { */ 613 if (lkp->lk_flags & LK_DRAINED) 614 panic("lockmgr: using decommissioned lock"); 615 if ((flags & LK_TYPE_MASK) != LK_RELEASE || 616 WEHOLDIT(lkp, pid, lid, cpu_num) == 0) 617 panic("lockmgr: non-release on draining lock: %d", 618 flags & LK_TYPE_MASK); 619 #endif /* DIAGNOSTIC */ /* } */ 620 lkp->lk_flags &= ~LK_DRAINING; 621 if ((flags & LK_REENABLE) == 0) 622 lkp->lk_flags |= LK_DRAINED; 623 } 624 625 switch (flags & LK_TYPE_MASK) { 626 627 case LK_SHARED: 628 if (WEHOLDIT(lkp, pid, lid, cpu_num) == 0) { 629 /* 630 * If just polling, check to see if we will block. 631 */ 632 if ((extflags & LK_NOWAIT) && (lkp->lk_flags & 633 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE))) { 634 error = EBUSY; 635 break; 636 } 637 /* 638 * Wait for exclusive locks and upgrades to clear. 639 */ 640 error = acquire(&lkp, &s, extflags, 0, 641 LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE, 642 RETURN_ADDRESS); 643 if (error) 644 break; 645 lkp->lk_sharecount++; 646 lkp->lk_flags |= LK_SHARE_NONZERO; 647 COUNT(lkp, l, cpu_num, 1); 648 break; 649 } 650 /* 651 * We hold an exclusive lock, so downgrade it to shared. 652 * An alternative would be to fail with EDEADLK. 653 */ 654 lkp->lk_sharecount++; 655 lkp->lk_flags |= LK_SHARE_NONZERO; 656 COUNT(lkp, l, cpu_num, 1); 657 /* fall into downgrade */ 658 659 case LK_DOWNGRADE: 660 if (WEHOLDIT(lkp, pid, lid, cpu_num) == 0 || 661 lkp->lk_exclusivecount == 0) 662 panic("lockmgr: not holding exclusive lock"); 663 lkp->lk_sharecount += lkp->lk_exclusivecount; 664 lkp->lk_flags |= LK_SHARE_NONZERO; 665 lkp->lk_exclusivecount = 0; 666 lkp->lk_recurselevel = 0; 667 lkp->lk_flags &= ~LK_HAVE_EXCL; 668 SETHOLDER(lkp, LK_NOPROC, 0, LK_NOCPU); 669 #if defined(LOCKDEBUG) 670 lkp->lk_unlock_file = file; 671 lkp->lk_unlock_line = line; 672 #endif 673 DONTHAVEIT(lkp); 674 WAKEUP_WAITER(lkp); 675 break; 676 677 case LK_EXCLUPGRADE: 678 /* 679 * If another process is ahead of us to get an upgrade, 680 * then we want to fail rather than have an intervening 681 * exclusive access. 682 */ 683 if (lkp->lk_flags & LK_WANT_UPGRADE) { 684 lkp->lk_sharecount--; 685 if (lkp->lk_sharecount == 0) 686 lkp->lk_flags &= ~LK_SHARE_NONZERO; 687 COUNT(lkp, l, cpu_num, -1); 688 error = EBUSY; 689 break; 690 } 691 /* fall into normal upgrade */ 692 693 case LK_UPGRADE: 694 /* 695 * Upgrade a shared lock to an exclusive one. If another 696 * shared lock has already requested an upgrade to an 697 * exclusive lock, our shared lock is released and an 698 * exclusive lock is requested (which will be granted 699 * after the upgrade). If we return an error, the file 700 * will always be unlocked. 701 */ 702 if (WEHOLDIT(lkp, pid, lid, cpu_num) || lkp->lk_sharecount <= 0) 703 panic("lockmgr: upgrade exclusive lock"); 704 lkp->lk_sharecount--; 705 if (lkp->lk_sharecount == 0) 706 lkp->lk_flags &= ~LK_SHARE_NONZERO; 707 COUNT(lkp, l, cpu_num, -1); 708 /* 709 * If we are just polling, check to see if we will block. 710 */ 711 if ((extflags & LK_NOWAIT) && 712 ((lkp->lk_flags & LK_WANT_UPGRADE) || 713 lkp->lk_sharecount > 1)) { 714 error = EBUSY; 715 break; 716 } 717 if ((lkp->lk_flags & LK_WANT_UPGRADE) == 0) { 718 /* 719 * We are first shared lock to request an upgrade, so 720 * request upgrade and wait for the shared count to 721 * drop to zero, then take exclusive lock. 722 */ 723 lkp->lk_flags |= LK_WANT_UPGRADE; 724 error = acquire(&lkp, &s, extflags, 0, LK_SHARE_NONZERO, 725 RETURN_ADDRESS); 726 lkp->lk_flags &= ~LK_WANT_UPGRADE; 727 if (error) { 728 WAKEUP_WAITER(lkp); 729 break; 730 } 731 lkp->lk_flags |= LK_HAVE_EXCL; 732 SETHOLDER(lkp, pid, lid, cpu_num); 733 #if defined(LOCKDEBUG) 734 lkp->lk_lock_file = file; 735 lkp->lk_lock_line = line; 736 #endif 737 HAVEIT(lkp); 738 if (lkp->lk_exclusivecount != 0) 739 panic("lockmgr: non-zero exclusive count"); 740 lkp->lk_exclusivecount = 1; 741 if (extflags & LK_SETRECURSE) 742 lkp->lk_recurselevel = 1; 743 COUNT(lkp, l, cpu_num, 1); 744 break; 745 } 746 /* 747 * Someone else has requested upgrade. Release our shared 748 * lock, awaken upgrade requestor if we are the last shared 749 * lock, then request an exclusive lock. 750 */ 751 if (lkp->lk_sharecount == 0) 752 WAKEUP_WAITER(lkp); 753 /* fall into exclusive request */ 754 755 case LK_EXCLUSIVE: 756 if (WEHOLDIT(lkp, pid, lid, cpu_num)) { 757 /* 758 * Recursive lock. 759 */ 760 if ((extflags & LK_CANRECURSE) == 0 && 761 lkp->lk_recurselevel == 0) { 762 if (extflags & LK_RECURSEFAIL) { 763 error = EDEADLK; 764 break; 765 } else 766 panic("lockmgr: locking against myself"); 767 } 768 lkp->lk_exclusivecount++; 769 if (extflags & LK_SETRECURSE && 770 lkp->lk_recurselevel == 0) 771 lkp->lk_recurselevel = lkp->lk_exclusivecount; 772 COUNT(lkp, l, cpu_num, 1); 773 break; 774 } 775 /* 776 * If we are just polling, check to see if we will sleep. 777 */ 778 if ((extflags & LK_NOWAIT) && (lkp->lk_flags & 779 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE | 780 LK_SHARE_NONZERO))) { 781 error = EBUSY; 782 break; 783 } 784 /* 785 * Try to acquire the want_exclusive flag. 786 */ 787 error = acquire(&lkp, &s, extflags, 0, 788 LK_HAVE_EXCL | LK_WANT_EXCL, RETURN_ADDRESS); 789 if (error) 790 break; 791 lkp->lk_flags |= LK_WANT_EXCL; 792 /* 793 * Wait for shared locks and upgrades to finish. 794 */ 795 error = acquire(&lkp, &s, extflags, 0, 796 LK_HAVE_EXCL | LK_WANT_UPGRADE | LK_SHARE_NONZERO, 797 RETURN_ADDRESS); 798 lkp->lk_flags &= ~LK_WANT_EXCL; 799 if (error) { 800 WAKEUP_WAITER(lkp); 801 break; 802 } 803 lkp->lk_flags |= LK_HAVE_EXCL; 804 SETHOLDER(lkp, pid, lid, cpu_num); 805 #if defined(LOCKDEBUG) 806 lkp->lk_lock_file = file; 807 lkp->lk_lock_line = line; 808 #endif 809 HAVEIT(lkp); 810 if (lkp->lk_exclusivecount != 0) 811 panic("lockmgr: non-zero exclusive count"); 812 lkp->lk_exclusivecount = 1; 813 if (extflags & LK_SETRECURSE) 814 lkp->lk_recurselevel = 1; 815 COUNT(lkp, l, cpu_num, 1); 816 break; 817 818 case LK_RELEASE: 819 if (lkp->lk_exclusivecount != 0) { 820 if (WEHOLDIT(lkp, pid, lid, cpu_num) == 0) { 821 if (lkp->lk_flags & LK_SPIN) { 822 panic("lockmgr: processor %lu, not " 823 "exclusive lock holder %lu " 824 "unlocking", cpu_num, lkp->lk_cpu); 825 } else { 826 panic("lockmgr: pid %d, not " 827 "exclusive lock holder %d " 828 "unlocking", pid, 829 lkp->lk_lockholder); 830 } 831 } 832 if (lkp->lk_exclusivecount == lkp->lk_recurselevel) 833 lkp->lk_recurselevel = 0; 834 lkp->lk_exclusivecount--; 835 COUNT(lkp, l, cpu_num, -1); 836 if (lkp->lk_exclusivecount == 0) { 837 lkp->lk_flags &= ~LK_HAVE_EXCL; 838 SETHOLDER(lkp, LK_NOPROC, 0, LK_NOCPU); 839 #if defined(LOCKDEBUG) 840 lkp->lk_unlock_file = file; 841 lkp->lk_unlock_line = line; 842 #endif 843 DONTHAVEIT(lkp); 844 } 845 } else if (lkp->lk_sharecount != 0) { 846 lkp->lk_sharecount--; 847 if (lkp->lk_sharecount == 0) 848 lkp->lk_flags &= ~LK_SHARE_NONZERO; 849 COUNT(lkp, l, cpu_num, -1); 850 } 851 #ifdef DIAGNOSTIC 852 else 853 panic("lockmgr: release of unlocked lock!"); 854 #endif 855 WAKEUP_WAITER(lkp); 856 break; 857 858 case LK_DRAIN: 859 /* 860 * Check that we do not already hold the lock, as it can 861 * never drain if we do. Unfortunately, we have no way to 862 * check for holding a shared lock, but at least we can 863 * check for an exclusive one. 864 */ 865 if (WEHOLDIT(lkp, pid, lid, cpu_num)) 866 panic("lockmgr: draining against myself"); 867 /* 868 * If we are just polling, check to see if we will sleep. 869 */ 870 if ((extflags & LK_NOWAIT) && (lkp->lk_flags & 871 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE | 872 LK_SHARE_NONZERO | LK_WAIT_NONZERO))) { 873 error = EBUSY; 874 break; 875 } 876 error = acquire(&lkp, &s, extflags, 1, 877 LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE | 878 LK_SHARE_NONZERO | LK_WAIT_NONZERO, 879 RETURN_ADDRESS); 880 if (error) 881 break; 882 lkp->lk_flags |= LK_DRAINING | LK_HAVE_EXCL; 883 SETHOLDER(lkp, pid, lid, cpu_num); 884 #if defined(LOCKDEBUG) 885 lkp->lk_lock_file = file; 886 lkp->lk_lock_line = line; 887 #endif 888 HAVEIT(lkp); 889 lkp->lk_exclusivecount = 1; 890 /* XXX unlikely that we'd want this */ 891 if (extflags & LK_SETRECURSE) 892 lkp->lk_recurselevel = 1; 893 COUNT(lkp, l, cpu_num, 1); 894 break; 895 896 default: 897 INTERLOCK_RELEASE(lkp, lkp->lk_flags, s); 898 panic("lockmgr: unknown locktype request %d", 899 flags & LK_TYPE_MASK); 900 /* NOTREACHED */ 901 } 902 if ((lkp->lk_flags & (LK_WAITDRAIN|LK_SPIN)) == LK_WAITDRAIN && 903 ((lkp->lk_flags & 904 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE | 905 LK_SHARE_NONZERO | LK_WAIT_NONZERO)) == 0)) { 906 lkp->lk_flags &= ~LK_WAITDRAIN; 907 wakeup(&lkp->lk_flags); 908 } 909 /* 910 * Note that this panic will be a recursive panic, since 911 * we only set lock_shutdown_noblock above if panicstr != NULL. 912 */ 913 if (error && lock_shutdown_noblock) 914 panic("lockmgr: deadlock (see previous panic)"); 915 916 INTERLOCK_RELEASE(lkp, lkp->lk_flags, s); 917 return (error); 918 } 919 920 /* 921 * For a recursive spinlock held one or more times by the current CPU, 922 * release all N locks, and return N. 923 * Intended for use in mi_switch() shortly before context switching. 924 */ 925 926 int 927 #if defined(LOCKDEBUG) 928 _spinlock_release_all(volatile struct lock *lkp, const char *file, int line) 929 #else 930 spinlock_release_all(volatile struct lock *lkp) 931 #endif 932 { 933 int s, count; 934 cpuid_t cpu_num; 935 936 KASSERT(lkp->lk_flags & LK_SPIN); 937 938 INTERLOCK_ACQUIRE(lkp, LK_SPIN, s); 939 940 cpu_num = cpu_number(); 941 count = lkp->lk_exclusivecount; 942 943 if (count != 0) { 944 #ifdef DIAGNOSTIC 945 if (WEHOLDIT(lkp, 0, 0, cpu_num) == 0) { 946 panic("spinlock_release_all: processor %lu, not " 947 "exclusive lock holder %lu " 948 "unlocking", (long)cpu_num, lkp->lk_cpu); 949 } 950 #endif 951 lkp->lk_recurselevel = 0; 952 lkp->lk_exclusivecount = 0; 953 COUNT_CPU(cpu_num, -count); 954 lkp->lk_flags &= ~LK_HAVE_EXCL; 955 SETHOLDER(lkp, LK_NOPROC, 0, LK_NOCPU); 956 #if defined(LOCKDEBUG) 957 lkp->lk_unlock_file = file; 958 lkp->lk_unlock_line = line; 959 #endif 960 DONTHAVEIT(lkp); 961 } 962 #ifdef DIAGNOSTIC 963 else if (lkp->lk_sharecount != 0) 964 panic("spinlock_release_all: release of shared lock!"); 965 else 966 panic("spinlock_release_all: release of unlocked lock!"); 967 #endif 968 INTERLOCK_RELEASE(lkp, LK_SPIN, s); 969 970 return (count); 971 } 972 973 /* 974 * For a recursive spinlock held one or more times by the current CPU, 975 * release all N locks, and return N. 976 * Intended for use in mi_switch() right after resuming execution. 977 */ 978 979 void 980 #if defined(LOCKDEBUG) 981 _spinlock_acquire_count(volatile struct lock *lkp, int count, 982 const char *file, int line) 983 #else 984 spinlock_acquire_count(volatile struct lock *lkp, int count) 985 #endif 986 { 987 int s, error; 988 cpuid_t cpu_num; 989 990 KASSERT(lkp->lk_flags & LK_SPIN); 991 992 INTERLOCK_ACQUIRE(lkp, LK_SPIN, s); 993 994 cpu_num = cpu_number(); 995 996 #ifdef DIAGNOSTIC 997 if (WEHOLDIT(lkp, LK_NOPROC, 0, cpu_num)) 998 panic("spinlock_acquire_count: processor %lu already holds lock", (long)cpu_num); 999 #endif 1000 /* 1001 * Try to acquire the want_exclusive flag. 1002 */ 1003 error = acquire(&lkp, &s, LK_SPIN, 0, LK_HAVE_EXCL | LK_WANT_EXCL, 1004 RETURN_ADDRESS); 1005 lkp->lk_flags |= LK_WANT_EXCL; 1006 /* 1007 * Wait for shared locks and upgrades to finish. 1008 */ 1009 error = acquire(&lkp, &s, LK_SPIN, 0, 1010 LK_HAVE_EXCL | LK_SHARE_NONZERO | LK_WANT_UPGRADE, 1011 RETURN_ADDRESS); 1012 lkp->lk_flags &= ~LK_WANT_EXCL; 1013 lkp->lk_flags |= LK_HAVE_EXCL; 1014 SETHOLDER(lkp, LK_NOPROC, 0, cpu_num); 1015 #if defined(LOCKDEBUG) 1016 lkp->lk_lock_file = file; 1017 lkp->lk_lock_line = line; 1018 #endif 1019 HAVEIT(lkp); 1020 if (lkp->lk_exclusivecount != 0) 1021 panic("lockmgr: non-zero exclusive count"); 1022 lkp->lk_exclusivecount = count; 1023 lkp->lk_recurselevel = 1; 1024 COUNT_CPU(cpu_num, count); 1025 1026 INTERLOCK_RELEASE(lkp, lkp->lk_flags, s); 1027 } 1028 1029 1030 1031 /* 1032 * Print out information about state of a lock. Used by VOP_PRINT 1033 * routines to display ststus about contained locks. 1034 */ 1035 void 1036 lockmgr_printinfo(volatile struct lock *lkp) 1037 { 1038 1039 if (lkp->lk_sharecount) 1040 printf(" lock type %s: SHARED (count %d)", lkp->lk_wmesg, 1041 lkp->lk_sharecount); 1042 else if (lkp->lk_flags & LK_HAVE_EXCL) { 1043 printf(" lock type %s: EXCL (count %d) by ", 1044 lkp->lk_wmesg, lkp->lk_exclusivecount); 1045 if (lkp->lk_flags & LK_SPIN) 1046 printf("processor %lu", lkp->lk_cpu); 1047 else 1048 printf("pid %d.%d", lkp->lk_lockholder, 1049 lkp->lk_locklwp); 1050 } else 1051 printf(" not locked"); 1052 if ((lkp->lk_flags & LK_SPIN) == 0 && lkp->lk_waitcount > 0) 1053 printf(" with %d pending", lkp->lk_waitcount); 1054 } 1055 1056 #if defined(LOCKDEBUG) /* { */ 1057 _TAILQ_HEAD(, struct simplelock, volatile) simplelock_list = 1058 TAILQ_HEAD_INITIALIZER(simplelock_list); 1059 1060 #if defined(MULTIPROCESSOR) /* { */ 1061 struct simplelock simplelock_list_slock = SIMPLELOCK_INITIALIZER; 1062 1063 #define SLOCK_LIST_LOCK() \ 1064 __cpu_simple_lock(&simplelock_list_slock.lock_data) 1065 1066 #define SLOCK_LIST_UNLOCK() \ 1067 __cpu_simple_unlock(&simplelock_list_slock.lock_data) 1068 1069 #define SLOCK_COUNT(x) \ 1070 curcpu()->ci_simple_locks += (x) 1071 #else 1072 u_long simple_locks; 1073 1074 #define SLOCK_LIST_LOCK() /* nothing */ 1075 1076 #define SLOCK_LIST_UNLOCK() /* nothing */ 1077 1078 #define SLOCK_COUNT(x) simple_locks += (x) 1079 #endif /* MULTIPROCESSOR */ /* } */ 1080 1081 #ifdef MULTIPROCESSOR 1082 #define SLOCK_MP() lock_printf("on CPU %ld\n", \ 1083 (u_long) cpu_number()) 1084 #else 1085 #define SLOCK_MP() /* nothing */ 1086 #endif 1087 1088 #define SLOCK_WHERE(str, alp, id, l) \ 1089 do { \ 1090 lock_printf("\n"); \ 1091 lock_printf(str); \ 1092 lock_printf("lock: %p, currently at: %s:%d\n", (alp), (id), (l)); \ 1093 SLOCK_MP(); \ 1094 if ((alp)->lock_file != NULL) \ 1095 lock_printf("last locked: %s:%d\n", (alp)->lock_file, \ 1096 (alp)->lock_line); \ 1097 if ((alp)->unlock_file != NULL) \ 1098 lock_printf("last unlocked: %s:%d\n", (alp)->unlock_file, \ 1099 (alp)->unlock_line); \ 1100 SLOCK_TRACE() \ 1101 SLOCK_DEBUGGER(); \ 1102 } while (/*CONSTCOND*/0) 1103 1104 /* 1105 * Simple lock functions so that the debugger can see from whence 1106 * they are being called. 1107 */ 1108 void 1109 simple_lock_init(volatile struct simplelock *alp) 1110 { 1111 1112 #if defined(MULTIPROCESSOR) /* { */ 1113 __cpu_simple_lock_init(&alp->lock_data); 1114 #else 1115 alp->lock_data = __SIMPLELOCK_UNLOCKED; 1116 #endif /* } */ 1117 alp->lock_file = NULL; 1118 alp->lock_line = 0; 1119 alp->unlock_file = NULL; 1120 alp->unlock_line = 0; 1121 alp->lock_holder = LK_NOCPU; 1122 } 1123 1124 void 1125 _simple_lock(volatile struct simplelock *alp, const char *id, int l) 1126 { 1127 cpuid_t cpu_num = cpu_number(); 1128 int s; 1129 1130 s = spllock(); 1131 1132 /* 1133 * MULTIPROCESSOR case: This is `safe' since if it's not us, we 1134 * don't take any action, and just fall into the normal spin case. 1135 */ 1136 if (alp->lock_data == __SIMPLELOCK_LOCKED) { 1137 #if defined(MULTIPROCESSOR) /* { */ 1138 if (alp->lock_holder == cpu_num) { 1139 SLOCK_WHERE("simple_lock: locking against myself\n", 1140 alp, id, l); 1141 goto out; 1142 } 1143 #else 1144 SLOCK_WHERE("simple_lock: lock held\n", alp, id, l); 1145 goto out; 1146 #endif /* MULTIPROCESSOR */ /* } */ 1147 } 1148 1149 #if defined(MULTIPROCESSOR) /* { */ 1150 /* Acquire the lock before modifying any fields. */ 1151 splx(s); 1152 __cpu_simple_lock(&alp->lock_data); 1153 s = spllock(); 1154 #else 1155 alp->lock_data = __SIMPLELOCK_LOCKED; 1156 #endif /* } */ 1157 1158 if (alp->lock_holder != LK_NOCPU) { 1159 SLOCK_WHERE("simple_lock: uninitialized lock\n", 1160 alp, id, l); 1161 } 1162 alp->lock_file = id; 1163 alp->lock_line = l; 1164 alp->lock_holder = cpu_num; 1165 1166 SLOCK_LIST_LOCK(); 1167 TAILQ_INSERT_TAIL(&simplelock_list, alp, list); 1168 SLOCK_LIST_UNLOCK(); 1169 1170 SLOCK_COUNT(1); 1171 1172 out: 1173 splx(s); 1174 } 1175 1176 int 1177 _simple_lock_held(volatile struct simplelock *alp) 1178 { 1179 #if defined(MULTIPROCESSOR) || defined(DIAGNOSTIC) 1180 cpuid_t cpu_num = cpu_number(); 1181 #endif 1182 int s, locked = 0; 1183 1184 s = spllock(); 1185 1186 #if defined(MULTIPROCESSOR) 1187 if (__cpu_simple_lock_try(&alp->lock_data) == 0) 1188 locked = (alp->lock_holder == cpu_num); 1189 else 1190 __cpu_simple_unlock(&alp->lock_data); 1191 #else 1192 if (alp->lock_data == __SIMPLELOCK_LOCKED) { 1193 locked = 1; 1194 KASSERT(alp->lock_holder == cpu_num); 1195 } 1196 #endif 1197 1198 splx(s); 1199 1200 return (locked); 1201 } 1202 1203 int 1204 _simple_lock_try(volatile struct simplelock *alp, const char *id, int l) 1205 { 1206 cpuid_t cpu_num = cpu_number(); 1207 int s, rv = 0; 1208 1209 s = spllock(); 1210 1211 /* 1212 * MULTIPROCESSOR case: This is `safe' since if it's not us, we 1213 * don't take any action. 1214 */ 1215 #if defined(MULTIPROCESSOR) /* { */ 1216 if ((rv = __cpu_simple_lock_try(&alp->lock_data)) == 0) { 1217 if (alp->lock_holder == cpu_num) 1218 SLOCK_WHERE("simple_lock_try: locking against myself\n", 1219 alp, id, l); 1220 goto out; 1221 } 1222 #else 1223 if (alp->lock_data == __SIMPLELOCK_LOCKED) { 1224 SLOCK_WHERE("simple_lock_try: lock held\n", alp, id, l); 1225 goto out; 1226 } 1227 alp->lock_data = __SIMPLELOCK_LOCKED; 1228 #endif /* MULTIPROCESSOR */ /* } */ 1229 1230 /* 1231 * At this point, we have acquired the lock. 1232 */ 1233 1234 rv = 1; 1235 1236 alp->lock_file = id; 1237 alp->lock_line = l; 1238 alp->lock_holder = cpu_num; 1239 1240 SLOCK_LIST_LOCK(); 1241 TAILQ_INSERT_TAIL(&simplelock_list, alp, list); 1242 SLOCK_LIST_UNLOCK(); 1243 1244 SLOCK_COUNT(1); 1245 1246 out: 1247 splx(s); 1248 return (rv); 1249 } 1250 1251 void 1252 _simple_unlock(volatile struct simplelock *alp, const char *id, int l) 1253 { 1254 int s; 1255 1256 s = spllock(); 1257 1258 /* 1259 * MULTIPROCESSOR case: This is `safe' because we think we hold 1260 * the lock, and if we don't, we don't take any action. 1261 */ 1262 if (alp->lock_data == __SIMPLELOCK_UNLOCKED) { 1263 SLOCK_WHERE("simple_unlock: lock not held\n", 1264 alp, id, l); 1265 goto out; 1266 } 1267 1268 SLOCK_LIST_LOCK(); 1269 TAILQ_REMOVE(&simplelock_list, alp, list); 1270 SLOCK_LIST_UNLOCK(); 1271 1272 SLOCK_COUNT(-1); 1273 1274 alp->list.tqe_next = NULL; /* sanity */ 1275 alp->list.tqe_prev = NULL; /* sanity */ 1276 1277 alp->unlock_file = id; 1278 alp->unlock_line = l; 1279 1280 #if defined(MULTIPROCESSOR) /* { */ 1281 alp->lock_holder = LK_NOCPU; 1282 /* Now that we've modified all fields, release the lock. */ 1283 __cpu_simple_unlock(&alp->lock_data); 1284 #else 1285 alp->lock_data = __SIMPLELOCK_UNLOCKED; 1286 KASSERT(alp->lock_holder == cpu_number()); 1287 alp->lock_holder = LK_NOCPU; 1288 #endif /* } */ 1289 1290 out: 1291 splx(s); 1292 } 1293 1294 void 1295 simple_lock_dump(void) 1296 { 1297 volatile struct simplelock *alp; 1298 int s; 1299 1300 s = spllock(); 1301 SLOCK_LIST_LOCK(); 1302 lock_printf("all simple locks:\n"); 1303 TAILQ_FOREACH(alp, &simplelock_list, list) { 1304 lock_printf("%p CPU %lu %s:%d\n", alp, alp->lock_holder, 1305 alp->lock_file, alp->lock_line); 1306 } 1307 SLOCK_LIST_UNLOCK(); 1308 splx(s); 1309 } 1310 1311 void 1312 simple_lock_freecheck(void *start, void *end) 1313 { 1314 volatile struct simplelock *alp; 1315 int s; 1316 1317 s = spllock(); 1318 SLOCK_LIST_LOCK(); 1319 TAILQ_FOREACH(alp, &simplelock_list, list) { 1320 if ((volatile void *)alp >= start && 1321 (volatile void *)alp < end) { 1322 lock_printf("freeing simple_lock %p CPU %lu %s:%d\n", 1323 alp, alp->lock_holder, alp->lock_file, 1324 alp->lock_line); 1325 SLOCK_DEBUGGER(); 1326 } 1327 } 1328 SLOCK_LIST_UNLOCK(); 1329 splx(s); 1330 } 1331 1332 /* 1333 * We must be holding exactly one lock: the sched_lock. 1334 */ 1335 1336 void 1337 simple_lock_switchcheck(void) 1338 { 1339 1340 simple_lock_only_held(&sched_lock, "switching"); 1341 } 1342 1343 /* 1344 * Drop into the debugger if lp isn't the only lock held. 1345 * lp may be NULL. 1346 */ 1347 void 1348 simple_lock_only_held(volatile struct simplelock *lp, const char *where) 1349 { 1350 volatile struct simplelock *alp; 1351 cpuid_t cpu_num = cpu_number(); 1352 int s; 1353 1354 if (lp) { 1355 LOCK_ASSERT(simple_lock_held(lp)); 1356 } 1357 s = spllock(); 1358 SLOCK_LIST_LOCK(); 1359 TAILQ_FOREACH(alp, &simplelock_list, list) { 1360 if (alp == lp) 1361 continue; 1362 #if defined(MULTIPROCESSOR) 1363 if (alp == &kernel_lock) 1364 continue; 1365 #endif /* defined(MULTIPROCESSOR) */ 1366 if (alp->lock_holder == cpu_num) 1367 break; 1368 } 1369 SLOCK_LIST_UNLOCK(); 1370 splx(s); 1371 1372 if (alp != NULL) { 1373 lock_printf("\n%s with held simple_lock %p " 1374 "CPU %lu %s:%d\n", 1375 where, alp, alp->lock_holder, alp->lock_file, 1376 alp->lock_line); 1377 SLOCK_TRACE(); 1378 SLOCK_DEBUGGER(); 1379 } 1380 } 1381 1382 /* 1383 * Set to 1 by simple_lock_assert_*(). 1384 * Can be cleared from ddb to avoid a panic. 1385 */ 1386 int slock_assert_will_panic; 1387 1388 /* 1389 * If the lock isn't held, print a traceback, optionally drop into the 1390 * debugger, then panic. 1391 * The panic can be avoided by clearing slock_assert_with_panic from the 1392 * debugger. 1393 */ 1394 void 1395 _simple_lock_assert_locked(volatile struct simplelock *alp, 1396 const char *lockname, const char *id, int l) 1397 { 1398 if (simple_lock_held(alp) == 0) { 1399 slock_assert_will_panic = 1; 1400 lock_printf("%s lock not held\n", lockname); 1401 SLOCK_WHERE("lock not held", alp, id, l); 1402 if (slock_assert_will_panic) 1403 panic("%s: not locked", lockname); 1404 } 1405 } 1406 1407 void 1408 _simple_lock_assert_unlocked(volatile struct simplelock *alp, 1409 const char *lockname, const char *id, int l) 1410 { 1411 if (simple_lock_held(alp)) { 1412 slock_assert_will_panic = 1; 1413 lock_printf("%s lock held\n", lockname); 1414 SLOCK_WHERE("lock held", alp, id, l); 1415 if (slock_assert_will_panic) 1416 panic("%s: locked", lockname); 1417 } 1418 } 1419 1420 void 1421 assert_sleepable(struct simplelock *interlock, const char *msg) 1422 { 1423 1424 if (curlwp == NULL) { 1425 panic("assert_sleepable: NULL curlwp"); 1426 } 1427 spinlock_switchcheck(); 1428 simple_lock_only_held(interlock, msg); 1429 } 1430 1431 #endif /* LOCKDEBUG */ /* } */ 1432 1433 #if defined(MULTIPROCESSOR) 1434 /* 1435 * Functions for manipulating the kernel_lock. We put them here 1436 * so that they show up in profiles. 1437 */ 1438 1439 /* 1440 * splbiglock: block IPLs which need to grab kernel_lock. 1441 * XXX splvm or splaudio should be enough. 1442 */ 1443 #if !defined(__HAVE_SPLBIGLOCK) 1444 #define splbiglock() splclock() 1445 #endif 1446 1447 void 1448 _kernel_lock_init(void) 1449 { 1450 1451 simple_lock_init(&kernel_lock); 1452 } 1453 1454 /* 1455 * Acquire/release the kernel lock. Intended for use in the scheduler 1456 * and the lower half of the kernel. 1457 */ 1458 void 1459 _kernel_lock(int flag) 1460 { 1461 struct cpu_info *ci = curcpu(); 1462 1463 SCHED_ASSERT_UNLOCKED(); 1464 1465 if (ci->ci_data.cpu_biglock_count > 0) { 1466 LOCK_ASSERT(simple_lock_held(&kernel_lock)); 1467 ci->ci_data.cpu_biglock_count++; 1468 } else { 1469 int s; 1470 1471 s = splbiglock(); 1472 while (!simple_lock_try(&kernel_lock)) { 1473 splx(s); 1474 SPINLOCK_SPIN_HOOK; 1475 s = splbiglock(); 1476 } 1477 ci->ci_data.cpu_biglock_count++; 1478 splx(s); 1479 } 1480 } 1481 1482 void 1483 _kernel_unlock(void) 1484 { 1485 struct cpu_info *ci = curcpu(); 1486 int s; 1487 1488 KASSERT(ci->ci_data.cpu_biglock_count > 0); 1489 1490 s = splbiglock(); 1491 if ((--ci->ci_data.cpu_biglock_count) == 0) 1492 simple_unlock(&kernel_lock); 1493 splx(s); 1494 } 1495 1496 /* 1497 * Acquire/release the kernel_lock on behalf of a process. Intended for 1498 * use in the top half of the kernel. 1499 */ 1500 void 1501 _kernel_proc_lock(struct lwp *l) 1502 { 1503 1504 SCHED_ASSERT_UNLOCKED(); 1505 _kernel_lock(0); 1506 } 1507 1508 void 1509 _kernel_proc_unlock(struct lwp *l) 1510 { 1511 1512 _kernel_unlock(); 1513 } 1514 1515 int 1516 _kernel_lock_release_all() 1517 { 1518 struct cpu_info *ci = curcpu(); 1519 int hold_count; 1520 1521 hold_count = ci->ci_data.cpu_biglock_count; 1522 1523 if (hold_count) { 1524 int s; 1525 1526 s = splbiglock(); 1527 ci->ci_data.cpu_biglock_count = 0; 1528 simple_unlock(&kernel_lock); 1529 splx(s); 1530 } 1531 1532 return hold_count; 1533 } 1534 1535 void 1536 _kernel_lock_acquire_count(int hold_count) 1537 { 1538 1539 KASSERT(curcpu()->ci_data.cpu_biglock_count == 0); 1540 1541 if (hold_count != 0) { 1542 struct cpu_info *ci = curcpu(); 1543 int s; 1544 1545 s = splbiglock(); 1546 while (!simple_lock_try(&kernel_lock)) { 1547 splx(s); 1548 SPINLOCK_SPIN_HOOK; 1549 s = splbiglock(); 1550 } 1551 ci->ci_data.cpu_biglock_count = hold_count; 1552 splx(s); 1553 } 1554 } 1555 #if defined(DEBUG) 1556 void 1557 _kernel_lock_assert_locked() 1558 { 1559 simple_lock_assert_locked(&kernel_lock, "kernel_lock"); 1560 } 1561 #endif 1562 1563 int 1564 lock_owner_onproc(uintptr_t owner) 1565 { 1566 CPU_INFO_ITERATOR cii; 1567 struct cpu_info *ci; 1568 1569 for (CPU_INFO_FOREACH(cii, ci)) 1570 if (owner == (uintptr_t)ci || owner == (uintptr_t)ci->ci_curlwp) 1571 return (1); 1572 1573 return (0); 1574 } 1575 1576 #else /* MULTIPROCESSOR */ 1577 1578 int 1579 lock_owner_onproc(uintptr_t owner) 1580 { 1581 1582 return 0; 1583 } 1584 1585 #endif /* MULTIPROCESSOR */ 1586
Indexes created Tue Sep 23 07:09:52 GMT 2025