kern_lock.c revision 1.99.2.4
1 /* $NetBSD: kern_lock.c,v 1.99.2.4 2006/11/17 16:34:36 ad Exp $ */ 2 3 /*- 4 * Copyright (c) 1999, 2000, 2006 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center, and by Andrew Doran. 10 * 11 * This code is derived from software contributed to The NetBSD Foundation 12 * by Ross Harvey. 13 * 14 * Redistribution and use in source and binary forms, with or without 15 * modification, are permitted provided that the following conditions 16 * are met: 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in the 21 * documentation and/or other materials provided with the distribution. 22 * 3. All advertising materials mentioning features or use of this software 23 * must display the following acknowledgement: 24 * This product includes software developed by the NetBSD 25 * Foundation, Inc. and its contributors. 26 * 4. Neither the name of The NetBSD Foundation nor the names of its 27 * contributors may be used to endorse or promote products derived 28 * from this software without specific prior written permission. 29 * 30 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 31 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 32 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 33 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 34 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 35 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 36 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 37 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 38 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 39 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 40 * POSSIBILITY OF SUCH DAMAGE. 41 */ 42 43 /* 44 * Copyright (c) 1995 45 * The Regents of the University of California. All rights reserved. 46 * 47 * This code contains ideas from software contributed to Berkeley by 48 * Avadis Tevanian, Jr., Michael Wayne Young, and the Mach Operating 49 * System project at Carnegie-Mellon University. 50 * 51 * Redistribution and use in source and binary forms, with or without 52 * modification, are permitted provided that the following conditions 53 * are met: 54 * 1. Redistributions of source code must retain the above copyright 55 * notice, this list of conditions and the following disclaimer. 56 * 2. Redistributions in binary form must reproduce the above copyright 57 * notice, this list of conditions and the following disclaimer in the 58 * documentation and/or other materials provided with the distribution. 59 * 3. Neither the name of the University nor the names of its contributors 60 * may be used to endorse or promote products derived from this software 61 * without specific prior written permission. 62 * 63 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 64 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 65 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 66 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 67 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 68 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 69 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 70 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 71 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 72 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 73 * SUCH DAMAGE. 74 * 75 * @(#)kern_lock.c 8.18 (Berkeley) 5/21/95 76 */ 77 78 #include <sys/cdefs.h> 79 __KERNEL_RCSID(0, "$NetBSD: kern_lock.c,v 1.99.2.4 2006/11/17 16:34:36 ad Exp $"); 80 81 #include "opt_multiprocessor.h" 82 #include "opt_ddb.h" 83 84 #define __MUTEX_PRIVATE 85 86 #include <sys/param.h> 87 #include <sys/proc.h> 88 #include <sys/lock.h> 89 #include <sys/systm.h> 90 #include <sys/lockdebug.h> 91 92 #include <machine/cpu.h> 93 94 #include <dev/lockstat.h> 95 96 #if defined(LOCKDEBUG) 97 #include <sys/syslog.h> 98 /* 99 * note that stdarg.h and the ansi style va_start macro is used for both 100 * ansi and traditional c compiles. 101 * XXX: this requires that stdarg.h define: va_alist and va_dcl 102 */ 103 #include <machine/stdarg.h> 104 105 void lock_printf(const char *fmt, ...) 106 __attribute__((__format__(__printf__,1,2))); 107 108 static int acquire(volatile struct lock **, int *, int, int, int, uintptr_t); 109 110 int lock_debug_syslog = 0; /* defaults to printf, but can be patched */ 111 112 #ifdef DDB 113 #include <ddb/ddbvar.h> 114 #include <machine/db_machdep.h> 115 #include <ddb/db_command.h> 116 #include <ddb/db_interface.h> 117 #endif 118 #endif /* defined(LOCKDEBUG) */ 119 120 #if defined(MULTIPROCESSOR) 121 /* 122 * IPL_BIGLOCK: block IPLs which need to grab kernel_mutex. 123 * XXX IPL_VM or IPL_AUDIO should be enough. 124 */ 125 #if !defined(__HAVE_SPLBIGLOCK) 126 #define splbiglock splclock 127 #endif 128 __cpu_simple_lock_t kernel_lock; 129 int kernel_lock_id; 130 #endif 131 132 /* 133 * Locking primitives implementation. 134 * Locks provide shared/exclusive synchronization. 135 */ 136 137 #if defined(LOCKDEBUG) || defined(DIAGNOSTIC) /* { */ 138 #if defined(MULTIPROCESSOR) /* { */ 139 #define COUNT_CPU(cpu_id, x) \ 140 curcpu()->ci_spin_locks += (x) 141 #else 142 u_long spin_locks; 143 #define COUNT_CPU(cpu_id, x) spin_locks += (x) 144 #endif /* MULTIPROCESSOR */ /* } */ 145 146 #define COUNT(lkp, l, cpu_id, x) \ 147 do { \ 148 if ((lkp)->lk_flags & LK_SPIN) \ 149 COUNT_CPU((cpu_id), (x)); \ 150 else \ 151 (l)->l_locks += (x); \ 152 } while (/*CONSTCOND*/0) 153 #else 154 #define COUNT(lkp, p, cpu_id, x) 155 #define COUNT_CPU(cpu_id, x) 156 #endif /* LOCKDEBUG || DIAGNOSTIC */ /* } */ 157 158 #define INTERLOCK_ACQUIRE(lkp, flags, s) \ 159 do { \ 160 if ((flags) & LK_SPIN) \ 161 s = spllock(); \ 162 simple_lock(&(lkp)->lk_interlock); \ 163 } while (/*CONSTCOND*/ 0) 164 165 #define INTERLOCK_RELEASE(lkp, flags, s) \ 166 do { \ 167 simple_unlock(&(lkp)->lk_interlock); \ 168 if ((flags) & LK_SPIN) \ 169 splx(s); \ 170 } while (/*CONSTCOND*/ 0) 171 172 #ifdef DDB /* { */ 173 #if defined(MULTIPROCESSOR) || defined(LOCKDEBUG) 174 int simple_lock_debugger = 1; /* more serious on MP */ 175 #else 176 int simple_lock_debugger = 0; 177 #endif 178 #define SLOCK_DEBUGGER() if (simple_lock_debugger && db_onpanic) Debugger() 179 #define SLOCK_TRACE() \ 180 db_stack_trace_print((db_expr_t)__builtin_frame_address(0), \ 181 TRUE, 65535, "", lock_printf); 182 #else 183 #define SLOCK_DEBUGGER() /* nothing */ 184 #define SLOCK_TRACE() /* nothing */ 185 #endif /* } */ 186 187 #if defined(LOCKDEBUG) 188 #if defined(DDB) 189 #define SPINLOCK_SPINCHECK_DEBUGGER if (db_onpanic) Debugger() 190 #else 191 #define SPINLOCK_SPINCHECK_DEBUGGER /* nothing */ 192 #endif 193 194 #define SPINLOCK_SPINCHECK_DECL \ 195 /* 32-bits of count -- wrap constitutes a "spinout" */ \ 196 uint32_t __spinc = 0 197 198 #define SPINLOCK_SPINCHECK \ 199 do { \ 200 if (++__spinc == 0) { \ 201 lock_printf("LK_SPIN spinout, excl %d, share %d\n", \ 202 lkp->lk_exclusivecount, lkp->lk_sharecount); \ 203 if (lkp->lk_exclusivecount) \ 204 lock_printf("held by CPU %lu\n", \ 205 (u_long) lkp->lk_cpu); \ 206 if (lkp->lk_lock_file) \ 207 lock_printf("last locked at %s:%d\n", \ 208 lkp->lk_lock_file, lkp->lk_lock_line); \ 209 if (lkp->lk_unlock_file) \ 210 lock_printf("last unlocked at %s:%d\n", \ 211 lkp->lk_unlock_file, lkp->lk_unlock_line); \ 212 SLOCK_TRACE(); \ 213 SPINLOCK_SPINCHECK_DEBUGGER; \ 214 } \ 215 } while (/*CONSTCOND*/ 0) 216 #else 217 #define SPINLOCK_SPINCHECK_DECL /* nothing */ 218 #define SPINLOCK_SPINCHECK /* nothing */ 219 #endif /* LOCKDEBUG && DDB */ 220 221 #define RETURN_ADDRESS ((uintptr_t)__builtin_return_address(0)) 222 223 /* 224 * Acquire a resource. 225 */ 226 static int 227 acquire(volatile struct lock **lkpp, int *s, int extflags, 228 int drain, int wanted, uintptr_t ra) 229 { 230 int error; 231 volatile struct lock *lkp = *lkpp; 232 LOCKSTAT_TIMER(slptime); 233 234 KASSERT(drain || (wanted & LK_WAIT_NONZERO) == 0); 235 236 if (extflags & LK_SPIN) { 237 int interlocked; 238 239 SPINLOCK_SPINCHECK_DECL; 240 241 if (!drain) { 242 lkp->lk_waitcount++; 243 lkp->lk_flags |= LK_WAIT_NONZERO; 244 } 245 for (interlocked = 1;;) { 246 SPINLOCK_SPINCHECK; 247 if ((lkp->lk_flags & wanted) != 0) { 248 if (interlocked) { 249 INTERLOCK_RELEASE(lkp, LK_SPIN, *s); 250 interlocked = 0; 251 } 252 SPINLOCK_SPIN_HOOK; 253 } else if (interlocked) { 254 break; 255 } else { 256 INTERLOCK_ACQUIRE(lkp, LK_SPIN, *s); 257 interlocked = 1; 258 } 259 } 260 if (!drain) { 261 lkp->lk_waitcount--; 262 if (lkp->lk_waitcount == 0) 263 lkp->lk_flags &= ~LK_WAIT_NONZERO; 264 } 265 KASSERT((lkp->lk_flags & wanted) == 0); 266 error = 0; /* sanity */ 267 } else { 268 for (error = 0; (lkp->lk_flags & wanted) != 0; ) { 269 if (drain) 270 lkp->lk_flags |= LK_WAITDRAIN; 271 else { 272 lkp->lk_waitcount++; 273 lkp->lk_flags |= LK_WAIT_NONZERO; 274 } 275 /* XXX Cast away volatile. */ 276 LOCKSTAT_START_TIMER(slptime); 277 error = ltsleep(drain ? 278 (volatile const void *)&lkp->lk_flags : 279 (volatile const void *)lkp, lkp->lk_prio, 280 lkp->lk_wmesg, lkp->lk_timo, &lkp->lk_interlock); 281 LOCKSTAT_STOP_TIMER(slptime); 282 LOCKSTAT_EVENT_RA((void *)(uintptr_t)lkp, 283 LB_LOCKMGR | LB_SLEEP, 1, slptime, ra); 284 if (!drain) { 285 lkp->lk_waitcount--; 286 if (lkp->lk_waitcount == 0) 287 lkp->lk_flags &= ~LK_WAIT_NONZERO; 288 } 289 if (error) 290 break; 291 if (extflags & LK_SLEEPFAIL) { 292 error = ENOLCK; 293 break; 294 } 295 if (lkp->lk_newlock != NULL) { 296 simple_lock(&lkp->lk_newlock->lk_interlock); 297 simple_unlock(&lkp->lk_interlock); 298 if (lkp->lk_waitcount == 0) 299 wakeup(&lkp->lk_newlock); 300 *lkpp = lkp = lkp->lk_newlock; 301 } 302 } 303 } 304 305 return error; 306 } 307 308 #define SETHOLDER(lkp, pid, lid, cpu_id) \ 309 do { \ 310 if ((lkp)->lk_flags & LK_SPIN) \ 311 (lkp)->lk_cpu = cpu_id; \ 312 else { \ 313 (lkp)->lk_lockholder = pid; \ 314 (lkp)->lk_locklwp = lid; \ 315 } \ 316 } while (/*CONSTCOND*/0) 317 318 #define WEHOLDIT(lkp, pid, lid, cpu_id) \ 319 (((lkp)->lk_flags & LK_SPIN) != 0 ? \ 320 ((lkp)->lk_cpu == (cpu_id)) : \ 321 ((lkp)->lk_lockholder == (pid) && (lkp)->lk_locklwp == (lid))) 322 323 #define WAKEUP_WAITER(lkp) \ 324 do { \ 325 if (((lkp)->lk_flags & (LK_SPIN | LK_WAIT_NONZERO)) == \ 326 LK_WAIT_NONZERO) { \ 327 wakeup((lkp)); \ 328 } \ 329 } while (/*CONSTCOND*/0) 330 331 #if defined(LOCKDEBUG) /* { */ 332 #if defined(MULTIPROCESSOR) /* { */ 333 struct simplelock spinlock_list_slock = SIMPLELOCK_INITIALIZER; 334 335 #define SPINLOCK_LIST_LOCK() \ 336 __cpu_simple_lock(&spinlock_list_slock.lock_data) 337 338 #define SPINLOCK_LIST_UNLOCK() \ 339 __cpu_simple_unlock(&spinlock_list_slock.lock_data) 340 #else 341 #define SPINLOCK_LIST_LOCK() /* nothing */ 342 343 #define SPINLOCK_LIST_UNLOCK() /* nothing */ 344 #endif /* MULTIPROCESSOR */ /* } */ 345 346 _TAILQ_HEAD(, struct lock, volatile) spinlock_list = 347 TAILQ_HEAD_INITIALIZER(spinlock_list); 348 349 #define HAVEIT(lkp) \ 350 do { \ 351 if ((lkp)->lk_flags & LK_SPIN) { \ 352 int sp = spllock(); \ 353 SPINLOCK_LIST_LOCK(); \ 354 TAILQ_INSERT_TAIL(&spinlock_list, (lkp), lk_list); \ 355 SPINLOCK_LIST_UNLOCK(); \ 356 splx(sp); \ 357 } \ 358 } while (/*CONSTCOND*/0) 359 360 #define DONTHAVEIT(lkp) \ 361 do { \ 362 if ((lkp)->lk_flags & LK_SPIN) { \ 363 int sp = spllock(); \ 364 SPINLOCK_LIST_LOCK(); \ 365 TAILQ_REMOVE(&spinlock_list, (lkp), lk_list); \ 366 SPINLOCK_LIST_UNLOCK(); \ 367 splx(sp); \ 368 } \ 369 } while (/*CONSTCOND*/0) 370 #else 371 #define HAVEIT(lkp) /* nothing */ 372 373 #define DONTHAVEIT(lkp) /* nothing */ 374 #endif /* LOCKDEBUG */ /* } */ 375 376 #if defined(LOCKDEBUG) 377 /* 378 * Lock debug printing routine; can be configured to print to console 379 * or log to syslog. 380 */ 381 void 382 lock_printf(const char *fmt, ...) 383 { 384 char b[150]; 385 va_list ap; 386 387 va_start(ap, fmt); 388 if (lock_debug_syslog) 389 vlog(LOG_DEBUG, fmt, ap); 390 else { 391 vsnprintf(b, sizeof(b), fmt, ap); 392 printf_nolog("%s", b); 393 } 394 va_end(ap); 395 } 396 #endif /* LOCKDEBUG */ 397 398 /* 399 * Transfer any waiting processes from one lock to another. 400 */ 401 void 402 transferlockers(struct lock *from, struct lock *to) 403 { 404 405 KASSERT(from != to); 406 KASSERT((from->lk_flags & LK_WAITDRAIN) == 0); 407 if (from->lk_waitcount == 0) 408 return; 409 from->lk_newlock = to; 410 wakeup((void *)from); 411 tsleep((void *)&from->lk_newlock, from->lk_prio, "lkxfer", 0); 412 from->lk_newlock = NULL; 413 from->lk_flags &= ~(LK_WANT_EXCL | LK_WANT_UPGRADE); 414 KASSERT(from->lk_waitcount == 0); 415 } 416 417 418 /* 419 * Initialize a lock; required before use. 420 */ 421 void 422 lockinit(struct lock *lkp, int prio, const char *wmesg, int timo, int flags) 423 { 424 425 memset(lkp, 0, sizeof(struct lock)); 426 simple_lock_init(&lkp->lk_interlock); 427 lkp->lk_flags = flags & LK_EXTFLG_MASK; 428 if (flags & LK_SPIN) 429 lkp->lk_cpu = LK_NOCPU; 430 else { 431 lkp->lk_lockholder = LK_NOPROC; 432 lkp->lk_newlock = NULL; 433 lkp->lk_prio = prio; 434 lkp->lk_timo = timo; 435 } 436 lkp->lk_wmesg = wmesg; /* just a name for spin locks */ 437 #if defined(LOCKDEBUG) 438 lkp->lk_lock_file = NULL; 439 lkp->lk_unlock_file = NULL; 440 #endif 441 } 442 443 /* 444 * Determine the status of a lock. 445 */ 446 int 447 lockstatus(struct lock *lkp) 448 { 449 int s = 0; /* XXX: gcc */ 450 int lock_type = 0; 451 struct lwp *l = curlwp; /* XXX */ 452 pid_t pid; 453 lwpid_t lid; 454 cpuid_t cpu_num; 455 456 if ((lkp->lk_flags & LK_SPIN) || l == NULL) { 457 cpu_num = cpu_number(); 458 pid = LK_KERNPROC; 459 lid = 0; 460 } else { 461 cpu_num = LK_NOCPU; 462 pid = l->l_proc->p_pid; 463 lid = l->l_lid; 464 } 465 466 INTERLOCK_ACQUIRE(lkp, lkp->lk_flags, s); 467 if (lkp->lk_exclusivecount != 0) { 468 if (WEHOLDIT(lkp, pid, lid, cpu_num)) 469 lock_type = LK_EXCLUSIVE; 470 else 471 lock_type = LK_EXCLOTHER; 472 } else if (lkp->lk_sharecount != 0) 473 lock_type = LK_SHARED; 474 INTERLOCK_RELEASE(lkp, lkp->lk_flags, s); 475 return (lock_type); 476 } 477 478 #if defined(LOCKDEBUG) 479 /* 480 * Make sure no spin locks are held by a CPU that is about 481 * to context switch. 482 */ 483 void 484 spinlock_switchcheck(void) 485 { 486 u_long cnt; 487 int s; 488 489 s = spllock(); 490 #if defined(MULTIPROCESSOR) 491 cnt = curcpu()->ci_spin_locks; 492 #else 493 cnt = spin_locks; 494 #endif 495 splx(s); 496 497 if (cnt != 0) 498 panic("spinlock_switchcheck: CPU %lu has %lu spin locks", 499 (u_long) cpu_number(), cnt); 500 } 501 #endif /* LOCKDEBUG */ 502 503 /* 504 * Locks and IPLs (interrupt priority levels): 505 * 506 * Locks which may be taken from interrupt context must be handled 507 * very carefully; you must spl to the highest IPL where the lock 508 * is needed before acquiring the lock. 509 * 510 * It is also important to avoid deadlock, since certain (very high 511 * priority) interrupts are often needed to keep the system as a whole 512 * from deadlocking, and must not be blocked while you are spinning 513 * waiting for a lower-priority lock. 514 * 515 * In addition, the lock-debugging hooks themselves need to use locks! 516 * 517 * A raw __cpu_simple_lock may be used from interrupts are long as it 518 * is acquired and held at a single IPL. 519 * 520 * A simple_lock (which is a __cpu_simple_lock wrapped with some 521 * debugging hooks) may be used at or below spllock(), which is 522 * typically at or just below splhigh() (i.e. blocks everything 523 * but certain machine-dependent extremely high priority interrupts). 524 * 525 * spinlockmgr spinlocks should be used at or below splsched(). 526 * 527 * Some platforms may have interrupts of higher priority than splsched(), 528 * including hard serial interrupts, inter-processor interrupts, and 529 * kernel debugger traps. 530 */ 531 532 /* 533 * XXX XXX kludge around another kludge.. 534 * 535 * vfs_shutdown() may be called from interrupt context, either as a result 536 * of a panic, or from the debugger. It proceeds to call 537 * sys_sync(&proc0, ...), pretending its running on behalf of proc0 538 * 539 * We would like to make an attempt to sync the filesystems in this case, so 540 * if this happens, we treat attempts to acquire locks specially. 541 * All locks are acquired on behalf of proc0. 542 * 543 * If we've already paniced, we don't block waiting for locks, but 544 * just barge right ahead since we're already going down in flames. 545 */ 546 547 /* 548 * Set, change, or release a lock. 549 * 550 * Shared requests increment the shared count. Exclusive requests set the 551 * LK_WANT_EXCL flag (preventing further shared locks), and wait for already 552 * accepted shared locks and shared-to-exclusive upgrades to go away. 553 */ 554 int 555 #if defined(LOCKDEBUG) 556 _lockmgr(volatile struct lock *lkp, u_int flags, 557 struct simplelock *interlkp, const char *file, int line) 558 #else 559 lockmgr(volatile struct lock *lkp, u_int flags, 560 struct simplelock *interlkp) 561 #endif 562 { 563 int error; 564 pid_t pid; 565 lwpid_t lid; 566 int extflags; 567 cpuid_t cpu_num; 568 struct lwp *l = curlwp; 569 int lock_shutdown_noblock = 0; 570 int s = 0; 571 572 error = 0; 573 574 /* LK_RETRY is for vn_lock, not for lockmgr. */ 575 KASSERT((flags & LK_RETRY) == 0); 576 577 INTERLOCK_ACQUIRE(lkp, lkp->lk_flags, s); 578 if (flags & LK_INTERLOCK) 579 simple_unlock(interlkp); 580 extflags = (flags | lkp->lk_flags) & LK_EXTFLG_MASK; 581 582 #ifdef DIAGNOSTIC /* { */ 583 /* 584 * Don't allow spins on sleep locks and don't allow sleeps 585 * on spin locks. 586 */ 587 if ((flags ^ lkp->lk_flags) & LK_SPIN) 588 panic("lockmgr: sleep/spin mismatch"); 589 #endif /* } */ 590 591 if (extflags & LK_SPIN) { 592 pid = LK_KERNPROC; 593 lid = 0; 594 } else { 595 if (l == NULL) { 596 if (!doing_shutdown) { 597 panic("lockmgr: no context"); 598 } else { 599 l = &lwp0; 600 if (panicstr && (!(flags & LK_NOWAIT))) { 601 flags |= LK_NOWAIT; 602 lock_shutdown_noblock = 1; 603 } 604 } 605 } 606 lid = l->l_lid; 607 pid = l->l_proc->p_pid; 608 } 609 cpu_num = cpu_number(); 610 611 /* 612 * Once a lock has drained, the LK_DRAINING flag is set and an 613 * exclusive lock is returned. The only valid operation thereafter 614 * is a single release of that exclusive lock. This final release 615 * clears the LK_DRAINING flag and sets the LK_DRAINED flag. Any 616 * further requests of any sort will result in a panic. The bits 617 * selected for these two flags are chosen so that they will be set 618 * in memory that is freed (freed memory is filled with 0xdeadbeef). 619 * The final release is permitted to give a new lease on life to 620 * the lock by specifying LK_REENABLE. 621 */ 622 if (lkp->lk_flags & (LK_DRAINING|LK_DRAINED)) { 623 #ifdef DIAGNOSTIC /* { */ 624 if (lkp->lk_flags & LK_DRAINED) 625 panic("lockmgr: using decommissioned lock"); 626 if ((flags & LK_TYPE_MASK) != LK_RELEASE || 627 WEHOLDIT(lkp, pid, lid, cpu_num) == 0) 628 panic("lockmgr: non-release on draining lock: %d", 629 flags & LK_TYPE_MASK); 630 #endif /* DIAGNOSTIC */ /* } */ 631 lkp->lk_flags &= ~LK_DRAINING; 632 if ((flags & LK_REENABLE) == 0) 633 lkp->lk_flags |= LK_DRAINED; 634 } 635 636 switch (flags & LK_TYPE_MASK) { 637 638 case LK_SHARED: 639 if (WEHOLDIT(lkp, pid, lid, cpu_num) == 0) { 640 /* 641 * If just polling, check to see if we will block. 642 */ 643 if ((extflags & LK_NOWAIT) && (lkp->lk_flags & 644 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE))) { 645 error = EBUSY; 646 break; 647 } 648 /* 649 * Wait for exclusive locks and upgrades to clear. 650 */ 651 error = acquire(&lkp, &s, extflags, 0, 652 LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE, 653 RETURN_ADDRESS); 654 if (error) 655 break; 656 lkp->lk_sharecount++; 657 lkp->lk_flags |= LK_SHARE_NONZERO; 658 COUNT(lkp, l, cpu_num, 1); 659 break; 660 } 661 /* 662 * We hold an exclusive lock, so downgrade it to shared. 663 * An alternative would be to fail with EDEADLK. 664 */ 665 lkp->lk_sharecount++; 666 lkp->lk_flags |= LK_SHARE_NONZERO; 667 COUNT(lkp, l, cpu_num, 1); 668 /* fall into downgrade */ 669 670 case LK_DOWNGRADE: 671 if (WEHOLDIT(lkp, pid, lid, cpu_num) == 0 || 672 lkp->lk_exclusivecount == 0) 673 panic("lockmgr: not holding exclusive lock"); 674 lkp->lk_sharecount += lkp->lk_exclusivecount; 675 lkp->lk_flags |= LK_SHARE_NONZERO; 676 lkp->lk_exclusivecount = 0; 677 lkp->lk_recurselevel = 0; 678 lkp->lk_flags &= ~LK_HAVE_EXCL; 679 SETHOLDER(lkp, LK_NOPROC, 0, LK_NOCPU); 680 #if defined(LOCKDEBUG) 681 lkp->lk_unlock_file = file; 682 lkp->lk_unlock_line = line; 683 #endif 684 DONTHAVEIT(lkp); 685 WAKEUP_WAITER(lkp); 686 break; 687 688 case LK_EXCLUPGRADE: 689 /* 690 * If another process is ahead of us to get an upgrade, 691 * then we want to fail rather than have an intervening 692 * exclusive access. 693 */ 694 if (lkp->lk_flags & LK_WANT_UPGRADE) { 695 lkp->lk_sharecount--; 696 if (lkp->lk_sharecount == 0) 697 lkp->lk_flags &= ~LK_SHARE_NONZERO; 698 COUNT(lkp, l, cpu_num, -1); 699 error = EBUSY; 700 break; 701 } 702 /* fall into normal upgrade */ 703 704 case LK_UPGRADE: 705 /* 706 * Upgrade a shared lock to an exclusive one. If another 707 * shared lock has already requested an upgrade to an 708 * exclusive lock, our shared lock is released and an 709 * exclusive lock is requested (which will be granted 710 * after the upgrade). If we return an error, the file 711 * will always be unlocked. 712 */ 713 if (WEHOLDIT(lkp, pid, lid, cpu_num) || lkp->lk_sharecount <= 0) 714 panic("lockmgr: upgrade exclusive lock"); 715 lkp->lk_sharecount--; 716 if (lkp->lk_sharecount == 0) 717 lkp->lk_flags &= ~LK_SHARE_NONZERO; 718 COUNT(lkp, l, cpu_num, -1); 719 /* 720 * If we are just polling, check to see if we will block. 721 */ 722 if ((extflags & LK_NOWAIT) && 723 ((lkp->lk_flags & LK_WANT_UPGRADE) || 724 lkp->lk_sharecount > 1)) { 725 error = EBUSY; 726 break; 727 } 728 if ((lkp->lk_flags & LK_WANT_UPGRADE) == 0) { 729 /* 730 * We are first shared lock to request an upgrade, so 731 * request upgrade and wait for the shared count to 732 * drop to zero, then take exclusive lock. 733 */ 734 lkp->lk_flags |= LK_WANT_UPGRADE; 735 error = acquire(&lkp, &s, extflags, 0, LK_SHARE_NONZERO, 736 RETURN_ADDRESS); 737 lkp->lk_flags &= ~LK_WANT_UPGRADE; 738 if (error) { 739 WAKEUP_WAITER(lkp); 740 break; 741 } 742 lkp->lk_flags |= LK_HAVE_EXCL; 743 SETHOLDER(lkp, pid, lid, cpu_num); 744 #if defined(LOCKDEBUG) 745 lkp->lk_lock_file = file; 746 lkp->lk_lock_line = line; 747 #endif 748 HAVEIT(lkp); 749 if (lkp->lk_exclusivecount != 0) 750 panic("lockmgr: non-zero exclusive count"); 751 lkp->lk_exclusivecount = 1; 752 if (extflags & LK_SETRECURSE) 753 lkp->lk_recurselevel = 1; 754 COUNT(lkp, l, cpu_num, 1); 755 break; 756 } 757 /* 758 * Someone else has requested upgrade. Release our shared 759 * lock, awaken upgrade requestor if we are the last shared 760 * lock, then request an exclusive lock. 761 */ 762 if (lkp->lk_sharecount == 0) 763 WAKEUP_WAITER(lkp); 764 /* fall into exclusive request */ 765 766 case LK_EXCLUSIVE: 767 if (WEHOLDIT(lkp, pid, lid, cpu_num)) { 768 /* 769 * Recursive lock. 770 */ 771 if ((extflags & LK_CANRECURSE) == 0 && 772 lkp->lk_recurselevel == 0) { 773 if (extflags & LK_RECURSEFAIL) { 774 error = EDEADLK; 775 break; 776 } else 777 panic("lockmgr: locking against myself"); 778 } 779 lkp->lk_exclusivecount++; 780 if (extflags & LK_SETRECURSE && 781 lkp->lk_recurselevel == 0) 782 lkp->lk_recurselevel = lkp->lk_exclusivecount; 783 COUNT(lkp, l, cpu_num, 1); 784 break; 785 } 786 /* 787 * If we are just polling, check to see if we will sleep. 788 */ 789 if ((extflags & LK_NOWAIT) && (lkp->lk_flags & 790 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE | 791 LK_SHARE_NONZERO))) { 792 error = EBUSY; 793 break; 794 } 795 /* 796 * Try to acquire the want_exclusive flag. 797 */ 798 error = acquire(&lkp, &s, extflags, 0, 799 LK_HAVE_EXCL | LK_WANT_EXCL, RETURN_ADDRESS); 800 if (error) 801 break; 802 lkp->lk_flags |= LK_WANT_EXCL; 803 /* 804 * Wait for shared locks and upgrades to finish. 805 */ 806 error = acquire(&lkp, &s, extflags, 0, 807 LK_HAVE_EXCL | LK_WANT_UPGRADE | LK_SHARE_NONZERO, 808 RETURN_ADDRESS); 809 lkp->lk_flags &= ~LK_WANT_EXCL; 810 if (error) { 811 WAKEUP_WAITER(lkp); 812 break; 813 } 814 lkp->lk_flags |= LK_HAVE_EXCL; 815 SETHOLDER(lkp, pid, lid, cpu_num); 816 #if defined(LOCKDEBUG) 817 lkp->lk_lock_file = file; 818 lkp->lk_lock_line = line; 819 #endif 820 HAVEIT(lkp); 821 if (lkp->lk_exclusivecount != 0) 822 panic("lockmgr: non-zero exclusive count"); 823 lkp->lk_exclusivecount = 1; 824 if (extflags & LK_SETRECURSE) 825 lkp->lk_recurselevel = 1; 826 COUNT(lkp, l, cpu_num, 1); 827 break; 828 829 case LK_RELEASE: 830 if (lkp->lk_exclusivecount != 0) { 831 if (WEHOLDIT(lkp, pid, lid, cpu_num) == 0) { 832 if (lkp->lk_flags & LK_SPIN) { 833 panic("lockmgr: processor %lu, not " 834 "exclusive lock holder %lu " 835 "unlocking", cpu_num, lkp->lk_cpu); 836 } else { 837 panic("lockmgr: pid %d, not " 838 "exclusive lock holder %d " 839 "unlocking", pid, 840 lkp->lk_lockholder); 841 } 842 } 843 if (lkp->lk_exclusivecount == lkp->lk_recurselevel) 844 lkp->lk_recurselevel = 0; 845 lkp->lk_exclusivecount--; 846 COUNT(lkp, l, cpu_num, -1); 847 if (lkp->lk_exclusivecount == 0) { 848 lkp->lk_flags &= ~LK_HAVE_EXCL; 849 SETHOLDER(lkp, LK_NOPROC, 0, LK_NOCPU); 850 #if defined(LOCKDEBUG) 851 lkp->lk_unlock_file = file; 852 lkp->lk_unlock_line = line; 853 #endif 854 DONTHAVEIT(lkp); 855 } 856 } else if (lkp->lk_sharecount != 0) { 857 lkp->lk_sharecount--; 858 if (lkp->lk_sharecount == 0) 859 lkp->lk_flags &= ~LK_SHARE_NONZERO; 860 COUNT(lkp, l, cpu_num, -1); 861 } 862 #ifdef DIAGNOSTIC 863 else 864 panic("lockmgr: release of unlocked lock!"); 865 #endif 866 WAKEUP_WAITER(lkp); 867 break; 868 869 case LK_DRAIN: 870 /* 871 * Check that we do not already hold the lock, as it can 872 * never drain if we do. Unfortunately, we have no way to 873 * check for holding a shared lock, but at least we can 874 * check for an exclusive one. 875 */ 876 if (WEHOLDIT(lkp, pid, lid, cpu_num)) 877 panic("lockmgr: draining against myself"); 878 /* 879 * If we are just polling, check to see if we will sleep. 880 */ 881 if ((extflags & LK_NOWAIT) && (lkp->lk_flags & 882 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE | 883 LK_SHARE_NONZERO | LK_WAIT_NONZERO))) { 884 error = EBUSY; 885 break; 886 } 887 error = acquire(&lkp, &s, extflags, 1, 888 LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE | 889 LK_SHARE_NONZERO | LK_WAIT_NONZERO, 890 RETURN_ADDRESS); 891 if (error) 892 break; 893 lkp->lk_flags |= LK_DRAINING | LK_HAVE_EXCL; 894 SETHOLDER(lkp, pid, lid, cpu_num); 895 #if defined(LOCKDEBUG) 896 lkp->lk_lock_file = file; 897 lkp->lk_lock_line = line; 898 #endif 899 HAVEIT(lkp); 900 lkp->lk_exclusivecount = 1; 901 /* XXX unlikely that we'd want this */ 902 if (extflags & LK_SETRECURSE) 903 lkp->lk_recurselevel = 1; 904 COUNT(lkp, l, cpu_num, 1); 905 break; 906 907 default: 908 INTERLOCK_RELEASE(lkp, lkp->lk_flags, s); 909 panic("lockmgr: unknown locktype request %d", 910 flags & LK_TYPE_MASK); 911 /* NOTREACHED */ 912 } 913 if ((lkp->lk_flags & (LK_WAITDRAIN|LK_SPIN)) == LK_WAITDRAIN && 914 ((lkp->lk_flags & 915 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE | 916 LK_SHARE_NONZERO | LK_WAIT_NONZERO)) == 0)) { 917 lkp->lk_flags &= ~LK_WAITDRAIN; 918 wakeup(&lkp->lk_flags); 919 } 920 /* 921 * Note that this panic will be a recursive panic, since 922 * we only set lock_shutdown_noblock above if panicstr != NULL. 923 */ 924 if (error && lock_shutdown_noblock) 925 panic("lockmgr: deadlock (see previous panic)"); 926 927 INTERLOCK_RELEASE(lkp, lkp->lk_flags, s); 928 return (error); 929 } 930 931 /* 932 * For a recursive spinlock held one or more times by the current CPU, 933 * release all N locks, and return N. 934 * Intended for use in mi_switch() shortly before context switching. 935 */ 936 937 int 938 #if defined(LOCKDEBUG) 939 _spinlock_release_all(volatile struct lock *lkp, const char *file, int line) 940 #else 941 spinlock_release_all(volatile struct lock *lkp) 942 #endif 943 { 944 int s, count; 945 cpuid_t cpu_num; 946 947 KASSERT(lkp->lk_flags & LK_SPIN); 948 949 INTERLOCK_ACQUIRE(lkp, LK_SPIN, s); 950 951 cpu_num = cpu_number(); 952 count = lkp->lk_exclusivecount; 953 954 if (count != 0) { 955 #ifdef DIAGNOSTIC 956 if (WEHOLDIT(lkp, 0, 0, cpu_num) == 0) { 957 panic("spinlock_release_all: processor %lu, not " 958 "exclusive lock holder %lu " 959 "unlocking", (long)cpu_num, lkp->lk_cpu); 960 } 961 #endif 962 lkp->lk_recurselevel = 0; 963 lkp->lk_exclusivecount = 0; 964 COUNT_CPU(cpu_num, -count); 965 lkp->lk_flags &= ~LK_HAVE_EXCL; 966 SETHOLDER(lkp, LK_NOPROC, 0, LK_NOCPU); 967 #if defined(LOCKDEBUG) 968 lkp->lk_unlock_file = file; 969 lkp->lk_unlock_line = line; 970 #endif 971 DONTHAVEIT(lkp); 972 } 973 #ifdef DIAGNOSTIC 974 else if (lkp->lk_sharecount != 0) 975 panic("spinlock_release_all: release of shared lock!"); 976 else 977 panic("spinlock_release_all: release of unlocked lock!"); 978 #endif 979 INTERLOCK_RELEASE(lkp, LK_SPIN, s); 980 981 return (count); 982 } 983 984 /* 985 * For a recursive spinlock held one or more times by the current CPU, 986 * release all N locks, and return N. 987 * Intended for use in mi_switch() right after resuming execution. 988 */ 989 990 void 991 #if defined(LOCKDEBUG) 992 _spinlock_acquire_count(volatile struct lock *lkp, int count, 993 const char *file, int line) 994 #else 995 spinlock_acquire_count(volatile struct lock *lkp, int count) 996 #endif 997 { 998 int s, error; 999 cpuid_t cpu_num; 1000 1001 KASSERT(lkp->lk_flags & LK_SPIN); 1002 1003 INTERLOCK_ACQUIRE(lkp, LK_SPIN, s); 1004 1005 cpu_num = cpu_number(); 1006 1007 #ifdef DIAGNOSTIC 1008 if (WEHOLDIT(lkp, LK_NOPROC, 0, cpu_num)) 1009 panic("spinlock_acquire_count: processor %lu already holds lock", (long)cpu_num); 1010 #endif 1011 /* 1012 * Try to acquire the want_exclusive flag. 1013 */ 1014 error = acquire(&lkp, &s, LK_SPIN, 0, LK_HAVE_EXCL | LK_WANT_EXCL, 1015 RETURN_ADDRESS); 1016 lkp->lk_flags |= LK_WANT_EXCL; 1017 /* 1018 * Wait for shared locks and upgrades to finish. 1019 */ 1020 error = acquire(&lkp, &s, LK_SPIN, 0, 1021 LK_HAVE_EXCL | LK_SHARE_NONZERO | LK_WANT_UPGRADE, 1022 RETURN_ADDRESS); 1023 lkp->lk_flags &= ~LK_WANT_EXCL; 1024 lkp->lk_flags |= LK_HAVE_EXCL; 1025 SETHOLDER(lkp, LK_NOPROC, 0, cpu_num); 1026 #if defined(LOCKDEBUG) 1027 lkp->lk_lock_file = file; 1028 lkp->lk_lock_line = line; 1029 #endif 1030 HAVEIT(lkp); 1031 if (lkp->lk_exclusivecount != 0) 1032 panic("lockmgr: non-zero exclusive count"); 1033 lkp->lk_exclusivecount = count; 1034 lkp->lk_recurselevel = 1; 1035 COUNT_CPU(cpu_num, count); 1036 1037 INTERLOCK_RELEASE(lkp, lkp->lk_flags, s); 1038 } 1039 1040 1041 1042 /* 1043 * Print out information about state of a lock. Used by VOP_PRINT 1044 * routines to display ststus about contained locks. 1045 */ 1046 void 1047 lockmgr_printinfo(volatile struct lock *lkp) 1048 { 1049 1050 if (lkp->lk_sharecount) 1051 printf(" lock type %s: SHARED (count %d)", lkp->lk_wmesg, 1052 lkp->lk_sharecount); 1053 else if (lkp->lk_flags & LK_HAVE_EXCL) { 1054 printf(" lock type %s: EXCL (count %d) by ", 1055 lkp->lk_wmesg, lkp->lk_exclusivecount); 1056 if (lkp->lk_flags & LK_SPIN) 1057 printf("processor %lu", lkp->lk_cpu); 1058 else 1059 printf("pid %d.%d", lkp->lk_lockholder, 1060 lkp->lk_locklwp); 1061 } else 1062 printf(" not locked"); 1063 if ((lkp->lk_flags & LK_SPIN) == 0 && lkp->lk_waitcount > 0) 1064 printf(" with %d pending", lkp->lk_waitcount); 1065 } 1066 1067 #if defined(LOCKDEBUG) /* { */ 1068 _TAILQ_HEAD(, struct simplelock, volatile) simplelock_list = 1069 TAILQ_HEAD_INITIALIZER(simplelock_list); 1070 1071 #if defined(MULTIPROCESSOR) /* { */ 1072 struct simplelock simplelock_list_slock = SIMPLELOCK_INITIALIZER; 1073 1074 #define SLOCK_LIST_LOCK() \ 1075 __cpu_simple_lock(&simplelock_list_slock.lock_data) 1076 1077 #define SLOCK_LIST_UNLOCK() \ 1078 __cpu_simple_unlock(&simplelock_list_slock.lock_data) 1079 1080 #define SLOCK_COUNT(x) \ 1081 curcpu()->ci_simple_locks += (x) 1082 #else 1083 u_long simple_locks; 1084 1085 #define SLOCK_LIST_LOCK() /* nothing */ 1086 1087 #define SLOCK_LIST_UNLOCK() /* nothing */ 1088 1089 #define SLOCK_COUNT(x) simple_locks += (x) 1090 #endif /* MULTIPROCESSOR */ /* } */ 1091 1092 #ifdef MULTIPROCESSOR 1093 #define SLOCK_MP() lock_printf("on CPU %ld\n", \ 1094 (u_long) cpu_number()) 1095 #else 1096 #define SLOCK_MP() /* nothing */ 1097 #endif 1098 1099 #define SLOCK_WHERE(str, alp, id, l) \ 1100 do { \ 1101 lock_printf("\n"); \ 1102 lock_printf(str); \ 1103 lock_printf("lock: %p, currently at: %s:%d\n", (alp), (id), (l)); \ 1104 SLOCK_MP(); \ 1105 if ((alp)->lock_file != NULL) \ 1106 lock_printf("last locked: %s:%d\n", (alp)->lock_file, \ 1107 (alp)->lock_line); \ 1108 if ((alp)->unlock_file != NULL) \ 1109 lock_printf("last unlocked: %s:%d\n", (alp)->unlock_file, \ 1110 (alp)->unlock_line); \ 1111 SLOCK_TRACE() \ 1112 SLOCK_DEBUGGER(); \ 1113 } while (/*CONSTCOND*/0) 1114 1115 /* 1116 * Simple lock functions so that the debugger can see from whence 1117 * they are being called. 1118 */ 1119 void 1120 simple_lock_init(volatile struct simplelock *alp) 1121 { 1122 1123 #if defined(MULTIPROCESSOR) /* { */ 1124 __cpu_simple_lock_init(&alp->lock_data); 1125 #else 1126 alp->lock_data = __SIMPLELOCK_UNLOCKED; 1127 #endif /* } */ 1128 alp->lock_file = NULL; 1129 alp->lock_line = 0; 1130 alp->unlock_file = NULL; 1131 alp->unlock_line = 0; 1132 alp->lock_holder = LK_NOCPU; 1133 } 1134 1135 void 1136 _simple_lock(volatile struct simplelock *alp, const char *id, int l) 1137 { 1138 cpuid_t cpu_num = cpu_number(); 1139 int s; 1140 1141 s = spllock(); 1142 1143 /* 1144 * MULTIPROCESSOR case: This is `safe' since if it's not us, we 1145 * don't take any action, and just fall into the normal spin case. 1146 */ 1147 if (alp->lock_data == __SIMPLELOCK_LOCKED) { 1148 #if defined(MULTIPROCESSOR) /* { */ 1149 if (alp->lock_holder == cpu_num) { 1150 SLOCK_WHERE("simple_lock: locking against myself\n", 1151 alp, id, l); 1152 goto out; 1153 } 1154 #else 1155 SLOCK_WHERE("simple_lock: lock held\n", alp, id, l); 1156 goto out; 1157 #endif /* MULTIPROCESSOR */ /* } */ 1158 } 1159 1160 #if defined(MULTIPROCESSOR) /* { */ 1161 /* Acquire the lock before modifying any fields. */ 1162 splx(s); 1163 __cpu_simple_lock(&alp->lock_data); 1164 s = spllock(); 1165 #else 1166 alp->lock_data = __SIMPLELOCK_LOCKED; 1167 #endif /* } */ 1168 1169 if (alp->lock_holder != LK_NOCPU) { 1170 SLOCK_WHERE("simple_lock: uninitialized lock\n", 1171 alp, id, l); 1172 } 1173 alp->lock_file = id; 1174 alp->lock_line = l; 1175 alp->lock_holder = cpu_num; 1176 1177 SLOCK_LIST_LOCK(); 1178 TAILQ_INSERT_TAIL(&simplelock_list, alp, list); 1179 SLOCK_LIST_UNLOCK(); 1180 1181 SLOCK_COUNT(1); 1182 1183 out: 1184 splx(s); 1185 } 1186 1187 int 1188 _simple_lock_held(volatile struct simplelock *alp) 1189 { 1190 #if defined(MULTIPROCESSOR) || defined(DIAGNOSTIC) 1191 cpuid_t cpu_num = cpu_number(); 1192 #endif 1193 int s, locked = 0; 1194 1195 s = spllock(); 1196 1197 #if defined(MULTIPROCESSOR) 1198 if (__cpu_simple_lock_try(&alp->lock_data) == 0) 1199 locked = (alp->lock_holder == cpu_num); 1200 else 1201 __cpu_simple_unlock(&alp->lock_data); 1202 #else 1203 if (alp->lock_data == __SIMPLELOCK_LOCKED) { 1204 locked = 1; 1205 KASSERT(alp->lock_holder == cpu_num); 1206 } 1207 #endif 1208 1209 splx(s); 1210 1211 return (locked); 1212 } 1213 1214 int 1215 _simple_lock_try(volatile struct simplelock *alp, const char *id, int l) 1216 { 1217 cpuid_t cpu_num = cpu_number(); 1218 int s, rv = 0; 1219 1220 s = spllock(); 1221 1222 /* 1223 * MULTIPROCESSOR case: This is `safe' since if it's not us, we 1224 * don't take any action. 1225 */ 1226 #if defined(MULTIPROCESSOR) /* { */ 1227 if ((rv = __cpu_simple_lock_try(&alp->lock_data)) == 0) { 1228 if (alp->lock_holder == cpu_num) 1229 SLOCK_WHERE("simple_lock_try: locking against myself\n", 1230 alp, id, l); 1231 goto out; 1232 } 1233 #else 1234 if (alp->lock_data == __SIMPLELOCK_LOCKED) { 1235 SLOCK_WHERE("simple_lock_try: lock held\n", alp, id, l); 1236 goto out; 1237 } 1238 alp->lock_data = __SIMPLELOCK_LOCKED; 1239 #endif /* MULTIPROCESSOR */ /* } */ 1240 1241 /* 1242 * At this point, we have acquired the lock. 1243 */ 1244 1245 rv = 1; 1246 1247 alp->lock_file = id; 1248 alp->lock_line = l; 1249 alp->lock_holder = cpu_num; 1250 1251 SLOCK_LIST_LOCK(); 1252 TAILQ_INSERT_TAIL(&simplelock_list, alp, list); 1253 SLOCK_LIST_UNLOCK(); 1254 1255 SLOCK_COUNT(1); 1256 1257 out: 1258 splx(s); 1259 return (rv); 1260 } 1261 1262 void 1263 _simple_unlock(volatile struct simplelock *alp, const char *id, int l) 1264 { 1265 int s; 1266 1267 s = spllock(); 1268 1269 /* 1270 * MULTIPROCESSOR case: This is `safe' because we think we hold 1271 * the lock, and if we don't, we don't take any action. 1272 */ 1273 if (alp->lock_data == __SIMPLELOCK_UNLOCKED) { 1274 SLOCK_WHERE("simple_unlock: lock not held\n", 1275 alp, id, l); 1276 goto out; 1277 } 1278 1279 SLOCK_LIST_LOCK(); 1280 TAILQ_REMOVE(&simplelock_list, alp, list); 1281 SLOCK_LIST_UNLOCK(); 1282 1283 SLOCK_COUNT(-1); 1284 1285 alp->list.tqe_next = NULL; /* sanity */ 1286 alp->list.tqe_prev = NULL; /* sanity */ 1287 1288 alp->unlock_file = id; 1289 alp->unlock_line = l; 1290 1291 #if defined(MULTIPROCESSOR) /* { */ 1292 alp->lock_holder = LK_NOCPU; 1293 /* Now that we've modified all fields, release the lock. */ 1294 __cpu_simple_unlock(&alp->lock_data); 1295 #else 1296 alp->lock_data = __SIMPLELOCK_UNLOCKED; 1297 KASSERT(alp->lock_holder == cpu_number()); 1298 alp->lock_holder = LK_NOCPU; 1299 #endif /* } */ 1300 1301 out: 1302 splx(s); 1303 } 1304 1305 void 1306 simple_lock_dump(void) 1307 { 1308 volatile struct simplelock *alp; 1309 int s; 1310 1311 s = spllock(); 1312 SLOCK_LIST_LOCK(); 1313 lock_printf("all simple locks:\n"); 1314 TAILQ_FOREACH(alp, &simplelock_list, list) { 1315 lock_printf("%p CPU %lu %s:%d\n", alp, alp->lock_holder, 1316 alp->lock_file, alp->lock_line); 1317 } 1318 SLOCK_LIST_UNLOCK(); 1319 splx(s); 1320 } 1321 1322 void 1323 simple_lock_freecheck(void *start, void *end) 1324 { 1325 volatile struct simplelock *alp; 1326 int s; 1327 1328 s = spllock(); 1329 SLOCK_LIST_LOCK(); 1330 TAILQ_FOREACH(alp, &simplelock_list, list) { 1331 if ((volatile void *)alp >= start && 1332 (volatile void *)alp < end) { 1333 lock_printf("freeing simple_lock %p CPU %lu %s:%d\n", 1334 alp, alp->lock_holder, alp->lock_file, 1335 alp->lock_line); 1336 SLOCK_DEBUGGER(); 1337 } 1338 } 1339 SLOCK_LIST_UNLOCK(); 1340 splx(s); 1341 } 1342 1343 /* 1344 * We must be holding exactly one lock: the sched_lock. 1345 */ 1346 1347 void 1348 simple_lock_switchcheck(void) 1349 { 1350 1351 simple_lock_only_held(NULL, "switching"); 1352 } 1353 1354 /* 1355 * Drop into the debugger if lp isn't the only lock held. 1356 * lp may be NULL. 1357 */ 1358 void 1359 simple_lock_only_held(volatile struct simplelock *lp, const char *where) 1360 { 1361 volatile struct simplelock *alp; 1362 cpuid_t cpu_num = cpu_number(); 1363 int s; 1364 1365 if (lp) { 1366 LOCK_ASSERT(simple_lock_held(lp)); 1367 } 1368 s = spllock(); 1369 SLOCK_LIST_LOCK(); 1370 TAILQ_FOREACH(alp, &simplelock_list, list) { 1371 if (alp == lp) 1372 continue; 1373 if (alp->lock_holder == cpu_num) 1374 break; 1375 } 1376 SLOCK_LIST_UNLOCK(); 1377 splx(s); 1378 1379 if (alp != NULL) { 1380 lock_printf("\n%s with held simple_lock %p " 1381 "CPU %lu %s:%d\n", 1382 where, alp, alp->lock_holder, alp->lock_file, 1383 alp->lock_line); 1384 SLOCK_TRACE(); 1385 SLOCK_DEBUGGER(); 1386 } 1387 } 1388 1389 /* 1390 * Set to 1 by simple_lock_assert_*(). 1391 * Can be cleared from ddb to avoid a panic. 1392 */ 1393 int slock_assert_will_panic; 1394 1395 /* 1396 * If the lock isn't held, print a traceback, optionally drop into the 1397 * debugger, then panic. 1398 * The panic can be avoided by clearing slock_assert_with_panic from the 1399 * debugger. 1400 */ 1401 void 1402 _simple_lock_assert_locked(volatile struct simplelock *alp, 1403 const char *lockname, const char *id, int l) 1404 { 1405 if (simple_lock_held(alp) == 0) { 1406 slock_assert_will_panic = 1; 1407 lock_printf("%s lock not held\n", lockname); 1408 SLOCK_WHERE("lock not held", alp, id, l); 1409 if (slock_assert_will_panic) 1410 panic("%s: not locked", lockname); 1411 } 1412 } 1413 1414 void 1415 _simple_lock_assert_unlocked(volatile struct simplelock *alp, 1416 const char *lockname, const char *id, int l) 1417 { 1418 if (simple_lock_held(alp)) { 1419 slock_assert_will_panic = 1; 1420 lock_printf("%s lock held\n", lockname); 1421 SLOCK_WHERE("lock held", alp, id, l); 1422 if (slock_assert_will_panic) 1423 panic("%s: locked", lockname); 1424 } 1425 } 1426 1427 void 1428 assert_sleepable(struct simplelock *interlock, const char *msg) 1429 { 1430 1431 if (curlwp == NULL) { 1432 panic("assert_sleepable: NULL curlwp"); 1433 } 1434 simple_lock_only_held(interlock, msg); 1435 } 1436 1437 #endif /* LOCKDEBUG */ /* } */ 1438 1439 #if defined(MULTIPROCESSOR) 1440 1441 /* 1442 * Functions for manipulating the kernel_lock. We put them here 1443 * so that they show up in profiles. 1444 */ 1445 1446 #define _KERNEL_LOCK_ABORT(msg) \ 1447 LOCKDEBUG_ABORT(kernel_lock_id, &kernel_lock, &_kernel_lock_ops, \ 1448 __FUNCTION__, msg) 1449 1450 int _kernel_lock_dump(volatile void *, char *, size_t); 1451 1452 lockops_t _kernel_lock_ops = { 1453 "Kernel lock", 1454 0, 1455 _kernel_lock_dump 1456 }; 1457 1458 /* 1459 * Initialize the kernel lock. 1460 */ 1461 void 1462 _kernel_lock_init(void) 1463 { 1464 1465 __cpu_simple_lock_init(&kernel_lock); 1466 kernel_lock_id = LOCKDEBUG_ALLOC(&kernel_lock, &_kernel_lock_ops); 1467 } 1468 1469 /* 1470 * Print debugging information about the kernel lock. 1471 */ 1472 int 1473 _kernel_lock_dump(volatile void *junk, char *buf, size_t l) 1474 { 1475 struct cpu_info *ci = curcpu(); 1476 1477 (void)junk; 1478 1479 return snprintf(buf, l, "curcpu holds : %18d want cnt.: %18d\n", 1480 ci->ci_biglock_count, ci->ci_biglock_wanted); 1481 } 1482 1483 /* 1484 * Acquire 'nlocks' holds on the kernel lock. If 'l' is non-null, the 1485 * acquisition is from process context. 1486 */ 1487 void 1488 _kernel_lock(u_int nlocks, struct lwp *l) 1489 { 1490 struct cpu_info *ci = curcpu(); 1491 LOCKSTAT_TIMER(spintime); 1492 u_int count; 1493 int s; 1494 1495 (void)l; 1496 1497 KASSERT(nlocks > 0); 1498 1499 s = splbiglock(); 1500 1501 if (ci->ci_biglock_count != 0) { 1502 KDASSERT(__cpu_simple_lock_held(&kernel_lock)); 1503 ci->ci_biglock_count += nlocks; 1504 splx(s); 1505 return; 1506 } 1507 1508 if (__cpu_simple_lock_try(&kernel_lock)) { 1509 ci->ci_biglock_count = nlocks; 1510 LOCKDEBUG_LOCKED(kernel_lock_id, 1511 (uintptr_t)__builtin_return_address(0), 0); 1512 splx(s); 1513 return; 1514 } 1515 1516 LOCKSTAT_START_TIMER(spintime); 1517 count = 0; 1518 ci->ci_biglock_wanted++; 1519 1520 while (!__cpu_simple_lock_try(&kernel_lock)) { 1521 splx(s); 1522 #ifdef LOCKDEBUG 1523 SPINLOCK_BACKOFF(count); 1524 #else 1525 if (++count > 0x0fffffff) 1526 _KERNEL_LOCK_ABORT("spinout"); 1527 #endif 1528 s = splbiglock(); 1529 } 1530 1531 ci->ci_biglock_wanted--; 1532 ci->ci_biglock_count += nlocks; 1533 LOCKSTAT_STOP_TIMER(spintime); 1534 LOCKDEBUG_LOCKED(kernel_lock_id, 1535 (uintptr_t)__builtin_return_address(0), 0); 1536 splx(s); 1537 1538 if (count == 0) 1539 return; 1540 1541 LOCKSTAT_EVENT(&kernel_lock, LB_KERNEL_LOCK | LB_SPIN, 1, spintime); 1542 } 1543 1544 /* 1545 * Release 'nlocks' holds on the kernel lock. If 'nlocks' is zero, release 1546 * all holds. If 'l' is non-null, the release is from process context. 1547 */ 1548 u_int 1549 _kernel_unlock(u_int nlocks, struct lwp *l) 1550 { 1551 struct cpu_info *ci = curcpu(); 1552 u_int olocks; 1553 int s; 1554 1555 (void)l; 1556 1557 KASSERT(nlocks < 2); 1558 1559 olocks = ci->ci_biglock_count; 1560 1561 if (olocks == 0) { 1562 KASSERT(nlocks == 0); 1563 return 0; 1564 } 1565 1566 KDASSERT(__cpu_simple_lock_held(&kernel_lock)); 1567 1568 if (nlocks == 0) 1569 nlocks = olocks; 1570 1571 s = splbiglock(); 1572 if ((ci->ci_biglock_count -= nlocks) == 0) { 1573 LOCKDEBUG_UNLOCKED(kernel_lock_id, 1574 (uintptr_t)__builtin_return_address(0), 0); 1575 __cpu_simple_unlock(&kernel_lock); 1576 } 1577 splx(s); 1578 1579 return olocks; 1580 } 1581 1582 #if defined(DEBUG) 1583 /* 1584 * Assert that the kernel lock is held. 1585 */ 1586 void 1587 _kernel_lock_assert_locked(void) 1588 { 1589 1590 if (!__cpu_simple_lock_held(&kernel_lock) || 1591 ci->ci_biglock_count == 0) 1592 _KERNEL_LOCK_ABORT("not locked"); 1593 } 1594 #endif 1595 1596 int 1597 lock_owner_onproc(uintptr_t owner) 1598 { 1599 CPU_INFO_ITERATOR cii; 1600 struct cpu_info *ci; 1601 1602 for (CPU_INFO_FOREACH(cii, ci)) 1603 if (owner == (uintptr_t)ci->ci_curlwp) 1604 return ci->ci_biglock_wanted == 0; 1605 1606 return (0); 1607 } 1608 1609 #else /* MULTIPROCESSOR */ 1610 1611 int 1612 lock_owner_onproc(uintptr_t owner) 1613 { 1614 1615 return 0; 1616 } 1617 1618 #endif /* MULTIPROCESSOR */ 1619
Indexes created Sun Sep 28 07:09:56 GMT 2025