kern_lock.c revision 1.98
1 /* $NetBSD: kern_lock.c,v 1.98 2006/09/07 01:08:45 ad Exp $ */ 2 3 /*- 4 * Copyright (c) 1999, 2000 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center. 10 * 11 * This code is derived from software contributed to The NetBSD Foundation 12 * by Ross Harvey. 13 * 14 * Redistribution and use in source and binary forms, with or without 15 * modification, are permitted provided that the following conditions 16 * are met: 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in the 21 * documentation and/or other materials provided with the distribution. 22 * 3. All advertising materials mentioning features or use of this software 23 * must display the following acknowledgement: 24 * This product includes software developed by the NetBSD 25 * Foundation, Inc. and its contributors. 26 * 4. Neither the name of The NetBSD Foundation nor the names of its 27 * contributors may be used to endorse or promote products derived 28 * from this software without specific prior written permission. 29 * 30 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 31 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 32 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 33 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 34 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 35 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 36 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 37 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 38 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 39 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 40 * POSSIBILITY OF SUCH DAMAGE. 41 */ 42 43 /* 44 * Copyright (c) 1995 45 * The Regents of the University of California. All rights reserved. 46 * 47 * This code contains ideas from software contributed to Berkeley by 48 * Avadis Tevanian, Jr., Michael Wayne Young, and the Mach Operating 49 * System project at Carnegie-Mellon University. 50 * 51 * Redistribution and use in source and binary forms, with or without 52 * modification, are permitted provided that the following conditions 53 * are met: 54 * 1. Redistributions of source code must retain the above copyright 55 * notice, this list of conditions and the following disclaimer. 56 * 2. Redistributions in binary form must reproduce the above copyright 57 * notice, this list of conditions and the following disclaimer in the 58 * documentation and/or other materials provided with the distribution. 59 * 3. Neither the name of the University nor the names of its contributors 60 * may be used to endorse or promote products derived from this software 61 * without specific prior written permission. 62 * 63 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 64 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 65 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 66 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 67 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 68 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 69 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 70 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 71 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 72 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 73 * SUCH DAMAGE. 74 * 75 * @(#)kern_lock.c 8.18 (Berkeley) 5/21/95 76 */ 77 78 #include <sys/cdefs.h> 79 __KERNEL_RCSID(0, "$NetBSD: kern_lock.c,v 1.98 2006/09/07 01:08:45 ad Exp $"); 80 81 #include "opt_multiprocessor.h" 82 #include "opt_lockdebug.h" 83 #include "opt_ddb.h" 84 85 #include <sys/param.h> 86 #include <sys/proc.h> 87 #include <sys/lock.h> 88 #include <sys/systm.h> 89 #include <machine/cpu.h> 90 91 #include <dev/lockstat.h> 92 93 #if defined(LOCKDEBUG) 94 #include <sys/syslog.h> 95 /* 96 * note that stdarg.h and the ansi style va_start macro is used for both 97 * ansi and traditional c compiles. 98 * XXX: this requires that stdarg.h define: va_alist and va_dcl 99 */ 100 #include <machine/stdarg.h> 101 102 void lock_printf(const char *fmt, ...) 103 __attribute__((__format__(__printf__,1,2))); 104 105 static int acquire(volatile struct lock **, int *, int, int, int, uintptr_t ra); 106 107 int lock_debug_syslog = 0; /* defaults to printf, but can be patched */ 108 109 #ifdef DDB 110 #include <ddb/ddbvar.h> 111 #include <machine/db_machdep.h> 112 #include <ddb/db_command.h> 113 #include <ddb/db_interface.h> 114 #endif 115 #endif /* defined(LOCKDEBUG) */ 116 117 #if defined(MULTIPROCESSOR) 118 struct simplelock kernel_lock; 119 #endif 120 121 /* 122 * Locking primitives implementation. 123 * Locks provide shared/exclusive synchronization. 124 */ 125 126 #if defined(LOCKDEBUG) || defined(DIAGNOSTIC) /* { */ 127 #if defined(MULTIPROCESSOR) /* { */ 128 #define COUNT_CPU(cpu_id, x) \ 129 curcpu()->ci_spin_locks += (x) 130 #else 131 u_long spin_locks; 132 #define COUNT_CPU(cpu_id, x) spin_locks += (x) 133 #endif /* MULTIPROCESSOR */ /* } */ 134 135 #define COUNT(lkp, l, cpu_id, x) \ 136 do { \ 137 if ((lkp)->lk_flags & LK_SPIN) \ 138 COUNT_CPU((cpu_id), (x)); \ 139 else \ 140 (l)->l_locks += (x); \ 141 } while (/*CONSTCOND*/0) 142 #else 143 #define COUNT(lkp, p, cpu_id, x) 144 #define COUNT_CPU(cpu_id, x) 145 #endif /* LOCKDEBUG || DIAGNOSTIC */ /* } */ 146 147 #ifndef SPINLOCK_SPIN_HOOK /* from <machine/lock.h> */ 148 #define SPINLOCK_SPIN_HOOK /* nothing */ 149 #endif 150 151 #define INTERLOCK_ACQUIRE(lkp, flags, s) \ 152 do { \ 153 if ((flags) & LK_SPIN) \ 154 s = spllock(); \ 155 simple_lock(&(lkp)->lk_interlock); \ 156 } while (/*CONSTCOND*/ 0) 157 158 #define INTERLOCK_RELEASE(lkp, flags, s) \ 159 do { \ 160 simple_unlock(&(lkp)->lk_interlock); \ 161 if ((flags) & LK_SPIN) \ 162 splx(s); \ 163 } while (/*CONSTCOND*/ 0) 164 165 #ifdef DDB /* { */ 166 #if defined(MULTIPROCESSOR) || defined(LOCKDEBUG) 167 int simple_lock_debugger = 1; /* more serious on MP */ 168 #else 169 int simple_lock_debugger = 0; 170 #endif 171 #define SLOCK_DEBUGGER() if (simple_lock_debugger && db_onpanic) Debugger() 172 #define SLOCK_TRACE() \ 173 db_stack_trace_print((db_expr_t)__builtin_frame_address(0), \ 174 TRUE, 65535, "", lock_printf); 175 #else 176 #define SLOCK_DEBUGGER() /* nothing */ 177 #define SLOCK_TRACE() /* nothing */ 178 #endif /* } */ 179 180 #if defined(LOCKDEBUG) 181 #if defined(DDB) 182 #define SPINLOCK_SPINCHECK_DEBUGGER if (db_onpanic) Debugger() 183 #else 184 #define SPINLOCK_SPINCHECK_DEBUGGER /* nothing */ 185 #endif 186 187 #define SPINLOCK_SPINCHECK_DECL \ 188 /* 32-bits of count -- wrap constitutes a "spinout" */ \ 189 uint32_t __spinc = 0 190 191 #define SPINLOCK_SPINCHECK \ 192 do { \ 193 if (++__spinc == 0) { \ 194 lock_printf("LK_SPIN spinout, excl %d, share %d\n", \ 195 lkp->lk_exclusivecount, lkp->lk_sharecount); \ 196 if (lkp->lk_exclusivecount) \ 197 lock_printf("held by CPU %lu\n", \ 198 (u_long) lkp->lk_cpu); \ 199 if (lkp->lk_lock_file) \ 200 lock_printf("last locked at %s:%d\n", \ 201 lkp->lk_lock_file, lkp->lk_lock_line); \ 202 if (lkp->lk_unlock_file) \ 203 lock_printf("last unlocked at %s:%d\n", \ 204 lkp->lk_unlock_file, lkp->lk_unlock_line); \ 205 SLOCK_TRACE(); \ 206 SPINLOCK_SPINCHECK_DEBUGGER; \ 207 } \ 208 } while (/*CONSTCOND*/ 0) 209 #else 210 #define SPINLOCK_SPINCHECK_DECL /* nothing */ 211 #define SPINLOCK_SPINCHECK /* nothing */ 212 #endif /* LOCKDEBUG && DDB */ 213 214 #define RETURN_ADDRESS ((uintptr_t)__builtin_return_address(0)) 215 216 /* 217 * Acquire a resource. 218 */ 219 static int 220 acquire(volatile struct lock **lkpp, int *s, int extflags, 221 int drain, int wanted, uintptr_t ra) 222 { 223 int error; 224 volatile struct lock *lkp = *lkpp; 225 LOCKSTAT_TIMER(slptime); 226 227 KASSERT(drain || (wanted & LK_WAIT_NONZERO) == 0); 228 229 if (extflags & LK_SPIN) { 230 int interlocked; 231 232 SPINLOCK_SPINCHECK_DECL; 233 234 if (!drain) { 235 lkp->lk_waitcount++; 236 lkp->lk_flags |= LK_WAIT_NONZERO; 237 } 238 for (interlocked = 1;;) { 239 SPINLOCK_SPINCHECK; 240 if ((lkp->lk_flags & wanted) != 0) { 241 if (interlocked) { 242 INTERLOCK_RELEASE(lkp, LK_SPIN, *s); 243 interlocked = 0; 244 } 245 SPINLOCK_SPIN_HOOK; 246 } else if (interlocked) { 247 break; 248 } else { 249 INTERLOCK_ACQUIRE(lkp, LK_SPIN, *s); 250 interlocked = 1; 251 } 252 } 253 if (!drain) { 254 lkp->lk_waitcount--; 255 if (lkp->lk_waitcount == 0) 256 lkp->lk_flags &= ~LK_WAIT_NONZERO; 257 } 258 KASSERT((lkp->lk_flags & wanted) == 0); 259 error = 0; /* sanity */ 260 } else { 261 for (error = 0; (lkp->lk_flags & wanted) != 0; ) { 262 if (drain) 263 lkp->lk_flags |= LK_WAITDRAIN; 264 else { 265 lkp->lk_waitcount++; 266 lkp->lk_flags |= LK_WAIT_NONZERO; 267 } 268 /* XXX Cast away volatile. */ 269 LOCKSTAT_START_TIMER(slptime); 270 error = ltsleep(drain ? 271 (volatile const void *)&lkp->lk_flags : 272 (volatile const void *)lkp, lkp->lk_prio, 273 lkp->lk_wmesg, lkp->lk_timo, &lkp->lk_interlock); 274 LOCKSTAT_STOP_TIMER(slptime); 275 LOCKSTAT_EVENT_RA((void *)(uintptr_t)lkp, 276 LB_LOCKMGR | LB_SLEEP, 1, slptime, ra); 277 if (!drain) { 278 lkp->lk_waitcount--; 279 if (lkp->lk_waitcount == 0) 280 lkp->lk_flags &= ~LK_WAIT_NONZERO; 281 } 282 if (error) 283 break; 284 if (extflags & LK_SLEEPFAIL) { 285 error = ENOLCK; 286 break; 287 } 288 if (lkp->lk_newlock != NULL) { 289 simple_lock(&lkp->lk_newlock->lk_interlock); 290 simple_unlock(&lkp->lk_interlock); 291 if (lkp->lk_waitcount == 0) 292 wakeup(&lkp->lk_newlock); 293 *lkpp = lkp = lkp->lk_newlock; 294 } 295 } 296 } 297 298 return error; 299 } 300 301 #define SETHOLDER(lkp, pid, lid, cpu_id) \ 302 do { \ 303 if ((lkp)->lk_flags & LK_SPIN) \ 304 (lkp)->lk_cpu = cpu_id; \ 305 else { \ 306 (lkp)->lk_lockholder = pid; \ 307 (lkp)->lk_locklwp = lid; \ 308 } \ 309 } while (/*CONSTCOND*/0) 310 311 #define WEHOLDIT(lkp, pid, lid, cpu_id) \ 312 (((lkp)->lk_flags & LK_SPIN) != 0 ? \ 313 ((lkp)->lk_cpu == (cpu_id)) : \ 314 ((lkp)->lk_lockholder == (pid) && (lkp)->lk_locklwp == (lid))) 315 316 #define WAKEUP_WAITER(lkp) \ 317 do { \ 318 if (((lkp)->lk_flags & (LK_SPIN | LK_WAIT_NONZERO)) == \ 319 LK_WAIT_NONZERO) { \ 320 wakeup((lkp)); \ 321 } \ 322 } while (/*CONSTCOND*/0) 323 324 #if defined(LOCKDEBUG) /* { */ 325 #if defined(MULTIPROCESSOR) /* { */ 326 struct simplelock spinlock_list_slock = SIMPLELOCK_INITIALIZER; 327 328 #define SPINLOCK_LIST_LOCK() \ 329 __cpu_simple_lock(&spinlock_list_slock.lock_data) 330 331 #define SPINLOCK_LIST_UNLOCK() \ 332 __cpu_simple_unlock(&spinlock_list_slock.lock_data) 333 #else 334 #define SPINLOCK_LIST_LOCK() /* nothing */ 335 336 #define SPINLOCK_LIST_UNLOCK() /* nothing */ 337 #endif /* MULTIPROCESSOR */ /* } */ 338 339 _TAILQ_HEAD(, struct lock, volatile) spinlock_list = 340 TAILQ_HEAD_INITIALIZER(spinlock_list); 341 342 #define HAVEIT(lkp) \ 343 do { \ 344 if ((lkp)->lk_flags & LK_SPIN) { \ 345 int sp = spllock(); \ 346 SPINLOCK_LIST_LOCK(); \ 347 TAILQ_INSERT_TAIL(&spinlock_list, (lkp), lk_list); \ 348 SPINLOCK_LIST_UNLOCK(); \ 349 splx(sp); \ 350 } \ 351 } while (/*CONSTCOND*/0) 352 353 #define DONTHAVEIT(lkp) \ 354 do { \ 355 if ((lkp)->lk_flags & LK_SPIN) { \ 356 int sp = spllock(); \ 357 SPINLOCK_LIST_LOCK(); \ 358 TAILQ_REMOVE(&spinlock_list, (lkp), lk_list); \ 359 SPINLOCK_LIST_UNLOCK(); \ 360 splx(sp); \ 361 } \ 362 } while (/*CONSTCOND*/0) 363 #else 364 #define HAVEIT(lkp) /* nothing */ 365 366 #define DONTHAVEIT(lkp) /* nothing */ 367 #endif /* LOCKDEBUG */ /* } */ 368 369 #if defined(LOCKDEBUG) 370 /* 371 * Lock debug printing routine; can be configured to print to console 372 * or log to syslog. 373 */ 374 void 375 lock_printf(const char *fmt, ...) 376 { 377 char b[150]; 378 va_list ap; 379 380 va_start(ap, fmt); 381 if (lock_debug_syslog) 382 vlog(LOG_DEBUG, fmt, ap); 383 else { 384 vsnprintf(b, sizeof(b), fmt, ap); 385 printf_nolog("%s", b); 386 } 387 va_end(ap); 388 } 389 #endif /* LOCKDEBUG */ 390 391 /* 392 * Transfer any waiting processes from one lock to another. 393 */ 394 void 395 transferlockers(struct lock *from, struct lock *to) 396 { 397 398 KASSERT(from != to); 399 KASSERT((from->lk_flags & LK_WAITDRAIN) == 0); 400 if (from->lk_waitcount == 0) 401 return; 402 from->lk_newlock = to; 403 wakeup((void *)from); 404 tsleep((void *)&from->lk_newlock, from->lk_prio, "lkxfer", 0); 405 from->lk_newlock = NULL; 406 from->lk_flags &= ~(LK_WANT_EXCL | LK_WANT_UPGRADE); 407 KASSERT(from->lk_waitcount == 0); 408 } 409 410 411 /* 412 * Initialize a lock; required before use. 413 */ 414 void 415 lockinit(struct lock *lkp, int prio, const char *wmesg, int timo, int flags) 416 { 417 418 memset(lkp, 0, sizeof(struct lock)); 419 simple_lock_init(&lkp->lk_interlock); 420 lkp->lk_flags = flags & LK_EXTFLG_MASK; 421 if (flags & LK_SPIN) 422 lkp->lk_cpu = LK_NOCPU; 423 else { 424 lkp->lk_lockholder = LK_NOPROC; 425 lkp->lk_newlock = NULL; 426 lkp->lk_prio = prio; 427 lkp->lk_timo = timo; 428 } 429 lkp->lk_wmesg = wmesg; /* just a name for spin locks */ 430 #if defined(LOCKDEBUG) 431 lkp->lk_lock_file = NULL; 432 lkp->lk_unlock_file = NULL; 433 #endif 434 } 435 436 /* 437 * Determine the status of a lock. 438 */ 439 int 440 lockstatus(struct lock *lkp) 441 { 442 int s = 0; /* XXX: gcc */ 443 int lock_type = 0; 444 struct lwp *l = curlwp; /* XXX */ 445 pid_t pid; 446 lwpid_t lid; 447 cpuid_t cpu_num; 448 449 if ((lkp->lk_flags & LK_SPIN) || l == NULL) { 450 cpu_num = cpu_number(); 451 pid = LK_KERNPROC; 452 lid = 0; 453 } else { 454 cpu_num = LK_NOCPU; 455 pid = l->l_proc->p_pid; 456 lid = l->l_lid; 457 } 458 459 INTERLOCK_ACQUIRE(lkp, lkp->lk_flags, s); 460 if (lkp->lk_exclusivecount != 0) { 461 if (WEHOLDIT(lkp, pid, lid, cpu_num)) 462 lock_type = LK_EXCLUSIVE; 463 else 464 lock_type = LK_EXCLOTHER; 465 } else if (lkp->lk_sharecount != 0) 466 lock_type = LK_SHARED; 467 INTERLOCK_RELEASE(lkp, lkp->lk_flags, s); 468 return (lock_type); 469 } 470 471 #if defined(LOCKDEBUG) 472 /* 473 * Make sure no spin locks are held by a CPU that is about 474 * to context switch. 475 */ 476 void 477 spinlock_switchcheck(void) 478 { 479 u_long cnt; 480 int s; 481 482 s = spllock(); 483 #if defined(MULTIPROCESSOR) 484 cnt = curcpu()->ci_spin_locks; 485 #else 486 cnt = spin_locks; 487 #endif 488 splx(s); 489 490 if (cnt != 0) 491 panic("spinlock_switchcheck: CPU %lu has %lu spin locks", 492 (u_long) cpu_number(), cnt); 493 } 494 #endif /* LOCKDEBUG */ 495 496 /* 497 * Locks and IPLs (interrupt priority levels): 498 * 499 * Locks which may be taken from interrupt context must be handled 500 * very carefully; you must spl to the highest IPL where the lock 501 * is needed before acquiring the lock. 502 * 503 * It is also important to avoid deadlock, since certain (very high 504 * priority) interrupts are often needed to keep the system as a whole 505 * from deadlocking, and must not be blocked while you are spinning 506 * waiting for a lower-priority lock. 507 * 508 * In addition, the lock-debugging hooks themselves need to use locks! 509 * 510 * A raw __cpu_simple_lock may be used from interrupts are long as it 511 * is acquired and held at a single IPL. 512 * 513 * A simple_lock (which is a __cpu_simple_lock wrapped with some 514 * debugging hooks) may be used at or below spllock(), which is 515 * typically at or just below splhigh() (i.e. blocks everything 516 * but certain machine-dependent extremely high priority interrupts). 517 * 518 * spinlockmgr spinlocks should be used at or below splsched(). 519 * 520 * Some platforms may have interrupts of higher priority than splsched(), 521 * including hard serial interrupts, inter-processor interrupts, and 522 * kernel debugger traps. 523 */ 524 525 /* 526 * XXX XXX kludge around another kludge.. 527 * 528 * vfs_shutdown() may be called from interrupt context, either as a result 529 * of a panic, or from the debugger. It proceeds to call 530 * sys_sync(&proc0, ...), pretending its running on behalf of proc0 531 * 532 * We would like to make an attempt to sync the filesystems in this case, so 533 * if this happens, we treat attempts to acquire locks specially. 534 * All locks are acquired on behalf of proc0. 535 * 536 * If we've already paniced, we don't block waiting for locks, but 537 * just barge right ahead since we're already going down in flames. 538 */ 539 540 /* 541 * Set, change, or release a lock. 542 * 543 * Shared requests increment the shared count. Exclusive requests set the 544 * LK_WANT_EXCL flag (preventing further shared locks), and wait for already 545 * accepted shared locks and shared-to-exclusive upgrades to go away. 546 */ 547 int 548 #if defined(LOCKDEBUG) 549 _lockmgr(volatile struct lock *lkp, u_int flags, 550 struct simplelock *interlkp, const char *file, int line) 551 #else 552 lockmgr(volatile struct lock *lkp, u_int flags, 553 struct simplelock *interlkp) 554 #endif 555 { 556 int error; 557 pid_t pid; 558 lwpid_t lid; 559 int extflags; 560 cpuid_t cpu_num; 561 struct lwp *l = curlwp; 562 int lock_shutdown_noblock = 0; 563 int s = 0; 564 565 error = 0; 566 567 /* LK_RETRY is for vn_lock, not for lockmgr. */ 568 KASSERT((flags & LK_RETRY) == 0); 569 570 INTERLOCK_ACQUIRE(lkp, lkp->lk_flags, s); 571 if (flags & LK_INTERLOCK) 572 simple_unlock(interlkp); 573 extflags = (flags | lkp->lk_flags) & LK_EXTFLG_MASK; 574 575 #ifdef DIAGNOSTIC /* { */ 576 /* 577 * Don't allow spins on sleep locks and don't allow sleeps 578 * on spin locks. 579 */ 580 if ((flags ^ lkp->lk_flags) & LK_SPIN) 581 panic("lockmgr: sleep/spin mismatch"); 582 #endif /* } */ 583 584 if (extflags & LK_SPIN) { 585 pid = LK_KERNPROC; 586 lid = 0; 587 } else { 588 if (l == NULL) { 589 if (!doing_shutdown) { 590 panic("lockmgr: no context"); 591 } else { 592 l = &lwp0; 593 if (panicstr && (!(flags & LK_NOWAIT))) { 594 flags |= LK_NOWAIT; 595 lock_shutdown_noblock = 1; 596 } 597 } 598 } 599 lid = l->l_lid; 600 pid = l->l_proc->p_pid; 601 } 602 cpu_num = cpu_number(); 603 604 /* 605 * Once a lock has drained, the LK_DRAINING flag is set and an 606 * exclusive lock is returned. The only valid operation thereafter 607 * is a single release of that exclusive lock. This final release 608 * clears the LK_DRAINING flag and sets the LK_DRAINED flag. Any 609 * further requests of any sort will result in a panic. The bits 610 * selected for these two flags are chosen so that they will be set 611 * in memory that is freed (freed memory is filled with 0xdeadbeef). 612 * The final release is permitted to give a new lease on life to 613 * the lock by specifying LK_REENABLE. 614 */ 615 if (lkp->lk_flags & (LK_DRAINING|LK_DRAINED)) { 616 #ifdef DIAGNOSTIC /* { */ 617 if (lkp->lk_flags & LK_DRAINED) 618 panic("lockmgr: using decommissioned lock"); 619 if ((flags & LK_TYPE_MASK) != LK_RELEASE || 620 WEHOLDIT(lkp, pid, lid, cpu_num) == 0) 621 panic("lockmgr: non-release on draining lock: %d", 622 flags & LK_TYPE_MASK); 623 #endif /* DIAGNOSTIC */ /* } */ 624 lkp->lk_flags &= ~LK_DRAINING; 625 if ((flags & LK_REENABLE) == 0) 626 lkp->lk_flags |= LK_DRAINED; 627 } 628 629 switch (flags & LK_TYPE_MASK) { 630 631 case LK_SHARED: 632 if (WEHOLDIT(lkp, pid, lid, cpu_num) == 0) { 633 /* 634 * If just polling, check to see if we will block. 635 */ 636 if ((extflags & LK_NOWAIT) && (lkp->lk_flags & 637 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE))) { 638 error = EBUSY; 639 break; 640 } 641 /* 642 * Wait for exclusive locks and upgrades to clear. 643 */ 644 error = acquire(&lkp, &s, extflags, 0, 645 LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE, 646 RETURN_ADDRESS); 647 if (error) 648 break; 649 lkp->lk_sharecount++; 650 lkp->lk_flags |= LK_SHARE_NONZERO; 651 COUNT(lkp, l, cpu_num, 1); 652 break; 653 } 654 /* 655 * We hold an exclusive lock, so downgrade it to shared. 656 * An alternative would be to fail with EDEADLK. 657 */ 658 lkp->lk_sharecount++; 659 lkp->lk_flags |= LK_SHARE_NONZERO; 660 COUNT(lkp, l, cpu_num, 1); 661 /* fall into downgrade */ 662 663 case LK_DOWNGRADE: 664 if (WEHOLDIT(lkp, pid, lid, cpu_num) == 0 || 665 lkp->lk_exclusivecount == 0) 666 panic("lockmgr: not holding exclusive lock"); 667 lkp->lk_sharecount += lkp->lk_exclusivecount; 668 lkp->lk_flags |= LK_SHARE_NONZERO; 669 lkp->lk_exclusivecount = 0; 670 lkp->lk_recurselevel = 0; 671 lkp->lk_flags &= ~LK_HAVE_EXCL; 672 SETHOLDER(lkp, LK_NOPROC, 0, LK_NOCPU); 673 #if defined(LOCKDEBUG) 674 lkp->lk_unlock_file = file; 675 lkp->lk_unlock_line = line; 676 #endif 677 DONTHAVEIT(lkp); 678 WAKEUP_WAITER(lkp); 679 break; 680 681 case LK_EXCLUPGRADE: 682 /* 683 * If another process is ahead of us to get an upgrade, 684 * then we want to fail rather than have an intervening 685 * exclusive access. 686 */ 687 if (lkp->lk_flags & LK_WANT_UPGRADE) { 688 lkp->lk_sharecount--; 689 if (lkp->lk_sharecount == 0) 690 lkp->lk_flags &= ~LK_SHARE_NONZERO; 691 COUNT(lkp, l, cpu_num, -1); 692 error = EBUSY; 693 break; 694 } 695 /* fall into normal upgrade */ 696 697 case LK_UPGRADE: 698 /* 699 * Upgrade a shared lock to an exclusive one. If another 700 * shared lock has already requested an upgrade to an 701 * exclusive lock, our shared lock is released and an 702 * exclusive lock is requested (which will be granted 703 * after the upgrade). If we return an error, the file 704 * will always be unlocked. 705 */ 706 if (WEHOLDIT(lkp, pid, lid, cpu_num) || lkp->lk_sharecount <= 0) 707 panic("lockmgr: upgrade exclusive lock"); 708 lkp->lk_sharecount--; 709 if (lkp->lk_sharecount == 0) 710 lkp->lk_flags &= ~LK_SHARE_NONZERO; 711 COUNT(lkp, l, cpu_num, -1); 712 /* 713 * If we are just polling, check to see if we will block. 714 */ 715 if ((extflags & LK_NOWAIT) && 716 ((lkp->lk_flags & LK_WANT_UPGRADE) || 717 lkp->lk_sharecount > 1)) { 718 error = EBUSY; 719 break; 720 } 721 if ((lkp->lk_flags & LK_WANT_UPGRADE) == 0) { 722 /* 723 * We are first shared lock to request an upgrade, so 724 * request upgrade and wait for the shared count to 725 * drop to zero, then take exclusive lock. 726 */ 727 lkp->lk_flags |= LK_WANT_UPGRADE; 728 error = acquire(&lkp, &s, extflags, 0, LK_SHARE_NONZERO, 729 RETURN_ADDRESS); 730 lkp->lk_flags &= ~LK_WANT_UPGRADE; 731 if (error) { 732 WAKEUP_WAITER(lkp); 733 break; 734 } 735 lkp->lk_flags |= LK_HAVE_EXCL; 736 SETHOLDER(lkp, pid, lid, cpu_num); 737 #if defined(LOCKDEBUG) 738 lkp->lk_lock_file = file; 739 lkp->lk_lock_line = line; 740 #endif 741 HAVEIT(lkp); 742 if (lkp->lk_exclusivecount != 0) 743 panic("lockmgr: non-zero exclusive count"); 744 lkp->lk_exclusivecount = 1; 745 if (extflags & LK_SETRECURSE) 746 lkp->lk_recurselevel = 1; 747 COUNT(lkp, l, cpu_num, 1); 748 break; 749 } 750 /* 751 * Someone else has requested upgrade. Release our shared 752 * lock, awaken upgrade requestor if we are the last shared 753 * lock, then request an exclusive lock. 754 */ 755 if (lkp->lk_sharecount == 0) 756 WAKEUP_WAITER(lkp); 757 /* fall into exclusive request */ 758 759 case LK_EXCLUSIVE: 760 if (WEHOLDIT(lkp, pid, lid, cpu_num)) { 761 /* 762 * Recursive lock. 763 */ 764 if ((extflags & LK_CANRECURSE) == 0 && 765 lkp->lk_recurselevel == 0) { 766 if (extflags & LK_RECURSEFAIL) { 767 error = EDEADLK; 768 break; 769 } else 770 panic("lockmgr: locking against myself"); 771 } 772 lkp->lk_exclusivecount++; 773 if (extflags & LK_SETRECURSE && 774 lkp->lk_recurselevel == 0) 775 lkp->lk_recurselevel = lkp->lk_exclusivecount; 776 COUNT(lkp, l, cpu_num, 1); 777 break; 778 } 779 /* 780 * If we are just polling, check to see if we will sleep. 781 */ 782 if ((extflags & LK_NOWAIT) && (lkp->lk_flags & 783 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE | 784 LK_SHARE_NONZERO))) { 785 error = EBUSY; 786 break; 787 } 788 /* 789 * Try to acquire the want_exclusive flag. 790 */ 791 error = acquire(&lkp, &s, extflags, 0, 792 LK_HAVE_EXCL | LK_WANT_EXCL, RETURN_ADDRESS); 793 if (error) 794 break; 795 lkp->lk_flags |= LK_WANT_EXCL; 796 /* 797 * Wait for shared locks and upgrades to finish. 798 */ 799 error = acquire(&lkp, &s, extflags, 0, 800 LK_HAVE_EXCL | LK_WANT_UPGRADE | LK_SHARE_NONZERO, 801 RETURN_ADDRESS); 802 lkp->lk_flags &= ~LK_WANT_EXCL; 803 if (error) { 804 WAKEUP_WAITER(lkp); 805 break; 806 } 807 lkp->lk_flags |= LK_HAVE_EXCL; 808 SETHOLDER(lkp, pid, lid, cpu_num); 809 #if defined(LOCKDEBUG) 810 lkp->lk_lock_file = file; 811 lkp->lk_lock_line = line; 812 #endif 813 HAVEIT(lkp); 814 if (lkp->lk_exclusivecount != 0) 815 panic("lockmgr: non-zero exclusive count"); 816 lkp->lk_exclusivecount = 1; 817 if (extflags & LK_SETRECURSE) 818 lkp->lk_recurselevel = 1; 819 COUNT(lkp, l, cpu_num, 1); 820 break; 821 822 case LK_RELEASE: 823 if (lkp->lk_exclusivecount != 0) { 824 if (WEHOLDIT(lkp, pid, lid, cpu_num) == 0) { 825 if (lkp->lk_flags & LK_SPIN) { 826 panic("lockmgr: processor %lu, not " 827 "exclusive lock holder %lu " 828 "unlocking", cpu_num, lkp->lk_cpu); 829 } else { 830 panic("lockmgr: pid %d, not " 831 "exclusive lock holder %d " 832 "unlocking", pid, 833 lkp->lk_lockholder); 834 } 835 } 836 if (lkp->lk_exclusivecount == lkp->lk_recurselevel) 837 lkp->lk_recurselevel = 0; 838 lkp->lk_exclusivecount--; 839 COUNT(lkp, l, cpu_num, -1); 840 if (lkp->lk_exclusivecount == 0) { 841 lkp->lk_flags &= ~LK_HAVE_EXCL; 842 SETHOLDER(lkp, LK_NOPROC, 0, LK_NOCPU); 843 #if defined(LOCKDEBUG) 844 lkp->lk_unlock_file = file; 845 lkp->lk_unlock_line = line; 846 #endif 847 DONTHAVEIT(lkp); 848 } 849 } else if (lkp->lk_sharecount != 0) { 850 lkp->lk_sharecount--; 851 if (lkp->lk_sharecount == 0) 852 lkp->lk_flags &= ~LK_SHARE_NONZERO; 853 COUNT(lkp, l, cpu_num, -1); 854 } 855 #ifdef DIAGNOSTIC 856 else 857 panic("lockmgr: release of unlocked lock!"); 858 #endif 859 WAKEUP_WAITER(lkp); 860 break; 861 862 case LK_DRAIN: 863 /* 864 * Check that we do not already hold the lock, as it can 865 * never drain if we do. Unfortunately, we have no way to 866 * check for holding a shared lock, but at least we can 867 * check for an exclusive one. 868 */ 869 if (WEHOLDIT(lkp, pid, lid, cpu_num)) 870 panic("lockmgr: draining against myself"); 871 /* 872 * If we are just polling, check to see if we will sleep. 873 */ 874 if ((extflags & LK_NOWAIT) && (lkp->lk_flags & 875 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE | 876 LK_SHARE_NONZERO | LK_WAIT_NONZERO))) { 877 error = EBUSY; 878 break; 879 } 880 error = acquire(&lkp, &s, extflags, 1, 881 LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE | 882 LK_SHARE_NONZERO | LK_WAIT_NONZERO, 883 RETURN_ADDRESS); 884 if (error) 885 break; 886 lkp->lk_flags |= LK_DRAINING | LK_HAVE_EXCL; 887 SETHOLDER(lkp, pid, lid, cpu_num); 888 #if defined(LOCKDEBUG) 889 lkp->lk_lock_file = file; 890 lkp->lk_lock_line = line; 891 #endif 892 HAVEIT(lkp); 893 lkp->lk_exclusivecount = 1; 894 /* XXX unlikely that we'd want this */ 895 if (extflags & LK_SETRECURSE) 896 lkp->lk_recurselevel = 1; 897 COUNT(lkp, l, cpu_num, 1); 898 break; 899 900 default: 901 INTERLOCK_RELEASE(lkp, lkp->lk_flags, s); 902 panic("lockmgr: unknown locktype request %d", 903 flags & LK_TYPE_MASK); 904 /* NOTREACHED */ 905 } 906 if ((lkp->lk_flags & (LK_WAITDRAIN|LK_SPIN)) == LK_WAITDRAIN && 907 ((lkp->lk_flags & 908 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE | 909 LK_SHARE_NONZERO | LK_WAIT_NONZERO)) == 0)) { 910 lkp->lk_flags &= ~LK_WAITDRAIN; 911 wakeup(&lkp->lk_flags); 912 } 913 /* 914 * Note that this panic will be a recursive panic, since 915 * we only set lock_shutdown_noblock above if panicstr != NULL. 916 */ 917 if (error && lock_shutdown_noblock) 918 panic("lockmgr: deadlock (see previous panic)"); 919 920 INTERLOCK_RELEASE(lkp, lkp->lk_flags, s); 921 return (error); 922 } 923 924 /* 925 * For a recursive spinlock held one or more times by the current CPU, 926 * release all N locks, and return N. 927 * Intended for use in mi_switch() shortly before context switching. 928 */ 929 930 int 931 #if defined(LOCKDEBUG) 932 _spinlock_release_all(volatile struct lock *lkp, const char *file, int line) 933 #else 934 spinlock_release_all(volatile struct lock *lkp) 935 #endif 936 { 937 int s, count; 938 cpuid_t cpu_num; 939 940 KASSERT(lkp->lk_flags & LK_SPIN); 941 942 INTERLOCK_ACQUIRE(lkp, LK_SPIN, s); 943 944 cpu_num = cpu_number(); 945 count = lkp->lk_exclusivecount; 946 947 if (count != 0) { 948 #ifdef DIAGNOSTIC 949 if (WEHOLDIT(lkp, 0, 0, cpu_num) == 0) { 950 panic("spinlock_release_all: processor %lu, not " 951 "exclusive lock holder %lu " 952 "unlocking", (long)cpu_num, lkp->lk_cpu); 953 } 954 #endif 955 lkp->lk_recurselevel = 0; 956 lkp->lk_exclusivecount = 0; 957 COUNT_CPU(cpu_num, -count); 958 lkp->lk_flags &= ~LK_HAVE_EXCL; 959 SETHOLDER(lkp, LK_NOPROC, 0, LK_NOCPU); 960 #if defined(LOCKDEBUG) 961 lkp->lk_unlock_file = file; 962 lkp->lk_unlock_line = line; 963 #endif 964 DONTHAVEIT(lkp); 965 } 966 #ifdef DIAGNOSTIC 967 else if (lkp->lk_sharecount != 0) 968 panic("spinlock_release_all: release of shared lock!"); 969 else 970 panic("spinlock_release_all: release of unlocked lock!"); 971 #endif 972 INTERLOCK_RELEASE(lkp, LK_SPIN, s); 973 974 return (count); 975 } 976 977 /* 978 * For a recursive spinlock held one or more times by the current CPU, 979 * release all N locks, and return N. 980 * Intended for use in mi_switch() right after resuming execution. 981 */ 982 983 void 984 #if defined(LOCKDEBUG) 985 _spinlock_acquire_count(volatile struct lock *lkp, int count, 986 const char *file, int line) 987 #else 988 spinlock_acquire_count(volatile struct lock *lkp, int count) 989 #endif 990 { 991 int s, error; 992 cpuid_t cpu_num; 993 994 KASSERT(lkp->lk_flags & LK_SPIN); 995 996 INTERLOCK_ACQUIRE(lkp, LK_SPIN, s); 997 998 cpu_num = cpu_number(); 999 1000 #ifdef DIAGNOSTIC 1001 if (WEHOLDIT(lkp, LK_NOPROC, 0, cpu_num)) 1002 panic("spinlock_acquire_count: processor %lu already holds lock", (long)cpu_num); 1003 #endif 1004 /* 1005 * Try to acquire the want_exclusive flag. 1006 */ 1007 error = acquire(&lkp, &s, LK_SPIN, 0, LK_HAVE_EXCL | LK_WANT_EXCL, 1008 RETURN_ADDRESS); 1009 lkp->lk_flags |= LK_WANT_EXCL; 1010 /* 1011 * Wait for shared locks and upgrades to finish. 1012 */ 1013 error = acquire(&lkp, &s, LK_SPIN, 0, 1014 LK_HAVE_EXCL | LK_SHARE_NONZERO | LK_WANT_UPGRADE, 1015 RETURN_ADDRESS); 1016 lkp->lk_flags &= ~LK_WANT_EXCL; 1017 lkp->lk_flags |= LK_HAVE_EXCL; 1018 SETHOLDER(lkp, LK_NOPROC, 0, cpu_num); 1019 #if defined(LOCKDEBUG) 1020 lkp->lk_lock_file = file; 1021 lkp->lk_lock_line = line; 1022 #endif 1023 HAVEIT(lkp); 1024 if (lkp->lk_exclusivecount != 0) 1025 panic("lockmgr: non-zero exclusive count"); 1026 lkp->lk_exclusivecount = count; 1027 lkp->lk_recurselevel = 1; 1028 COUNT_CPU(cpu_num, count); 1029 1030 INTERLOCK_RELEASE(lkp, lkp->lk_flags, s); 1031 } 1032 1033 1034 1035 /* 1036 * Print out information about state of a lock. Used by VOP_PRINT 1037 * routines to display ststus about contained locks. 1038 */ 1039 void 1040 lockmgr_printinfo(volatile struct lock *lkp) 1041 { 1042 1043 if (lkp->lk_sharecount) 1044 printf(" lock type %s: SHARED (count %d)", lkp->lk_wmesg, 1045 lkp->lk_sharecount); 1046 else if (lkp->lk_flags & LK_HAVE_EXCL) { 1047 printf(" lock type %s: EXCL (count %d) by ", 1048 lkp->lk_wmesg, lkp->lk_exclusivecount); 1049 if (lkp->lk_flags & LK_SPIN) 1050 printf("processor %lu", lkp->lk_cpu); 1051 else 1052 printf("pid %d.%d", lkp->lk_lockholder, 1053 lkp->lk_locklwp); 1054 } else 1055 printf(" not locked"); 1056 if ((lkp->lk_flags & LK_SPIN) == 0 && lkp->lk_waitcount > 0) 1057 printf(" with %d pending", lkp->lk_waitcount); 1058 } 1059 1060 #if defined(LOCKDEBUG) /* { */ 1061 _TAILQ_HEAD(, struct simplelock, volatile) simplelock_list = 1062 TAILQ_HEAD_INITIALIZER(simplelock_list); 1063 1064 #if defined(MULTIPROCESSOR) /* { */ 1065 struct simplelock simplelock_list_slock = SIMPLELOCK_INITIALIZER; 1066 1067 #define SLOCK_LIST_LOCK() \ 1068 __cpu_simple_lock(&simplelock_list_slock.lock_data) 1069 1070 #define SLOCK_LIST_UNLOCK() \ 1071 __cpu_simple_unlock(&simplelock_list_slock.lock_data) 1072 1073 #define SLOCK_COUNT(x) \ 1074 curcpu()->ci_simple_locks += (x) 1075 #else 1076 u_long simple_locks; 1077 1078 #define SLOCK_LIST_LOCK() /* nothing */ 1079 1080 #define SLOCK_LIST_UNLOCK() /* nothing */ 1081 1082 #define SLOCK_COUNT(x) simple_locks += (x) 1083 #endif /* MULTIPROCESSOR */ /* } */ 1084 1085 #ifdef MULTIPROCESSOR 1086 #define SLOCK_MP() lock_printf("on CPU %ld\n", \ 1087 (u_long) cpu_number()) 1088 #else 1089 #define SLOCK_MP() /* nothing */ 1090 #endif 1091 1092 #define SLOCK_WHERE(str, alp, id, l) \ 1093 do { \ 1094 lock_printf("\n"); \ 1095 lock_printf(str); \ 1096 lock_printf("lock: %p, currently at: %s:%d\n", (alp), (id), (l)); \ 1097 SLOCK_MP(); \ 1098 if ((alp)->lock_file != NULL) \ 1099 lock_printf("last locked: %s:%d\n", (alp)->lock_file, \ 1100 (alp)->lock_line); \ 1101 if ((alp)->unlock_file != NULL) \ 1102 lock_printf("last unlocked: %s:%d\n", (alp)->unlock_file, \ 1103 (alp)->unlock_line); \ 1104 SLOCK_TRACE() \ 1105 SLOCK_DEBUGGER(); \ 1106 } while (/*CONSTCOND*/0) 1107 1108 /* 1109 * Simple lock functions so that the debugger can see from whence 1110 * they are being called. 1111 */ 1112 void 1113 simple_lock_init(volatile struct simplelock *alp) 1114 { 1115 1116 #if defined(MULTIPROCESSOR) /* { */ 1117 __cpu_simple_lock_init(&alp->lock_data); 1118 #else 1119 alp->lock_data = __SIMPLELOCK_UNLOCKED; 1120 #endif /* } */ 1121 alp->lock_file = NULL; 1122 alp->lock_line = 0; 1123 alp->unlock_file = NULL; 1124 alp->unlock_line = 0; 1125 alp->lock_holder = LK_NOCPU; 1126 } 1127 1128 void 1129 _simple_lock(volatile struct simplelock *alp, const char *id, int l) 1130 { 1131 cpuid_t cpu_num = cpu_number(); 1132 int s; 1133 1134 s = spllock(); 1135 1136 /* 1137 * MULTIPROCESSOR case: This is `safe' since if it's not us, we 1138 * don't take any action, and just fall into the normal spin case. 1139 */ 1140 if (alp->lock_data == __SIMPLELOCK_LOCKED) { 1141 #if defined(MULTIPROCESSOR) /* { */ 1142 if (alp->lock_holder == cpu_num) { 1143 SLOCK_WHERE("simple_lock: locking against myself\n", 1144 alp, id, l); 1145 goto out; 1146 } 1147 #else 1148 SLOCK_WHERE("simple_lock: lock held\n", alp, id, l); 1149 goto out; 1150 #endif /* MULTIPROCESSOR */ /* } */ 1151 } 1152 1153 #if defined(MULTIPROCESSOR) /* { */ 1154 /* Acquire the lock before modifying any fields. */ 1155 splx(s); 1156 __cpu_simple_lock(&alp->lock_data); 1157 s = spllock(); 1158 #else 1159 alp->lock_data = __SIMPLELOCK_LOCKED; 1160 #endif /* } */ 1161 1162 if (alp->lock_holder != LK_NOCPU) { 1163 SLOCK_WHERE("simple_lock: uninitialized lock\n", 1164 alp, id, l); 1165 } 1166 alp->lock_file = id; 1167 alp->lock_line = l; 1168 alp->lock_holder = cpu_num; 1169 1170 SLOCK_LIST_LOCK(); 1171 TAILQ_INSERT_TAIL(&simplelock_list, alp, list); 1172 SLOCK_LIST_UNLOCK(); 1173 1174 SLOCK_COUNT(1); 1175 1176 out: 1177 splx(s); 1178 } 1179 1180 int 1181 _simple_lock_held(volatile struct simplelock *alp) 1182 { 1183 #if defined(MULTIPROCESSOR) || defined(DIAGNOSTIC) 1184 cpuid_t cpu_num = cpu_number(); 1185 #endif 1186 int s, locked = 0; 1187 1188 s = spllock(); 1189 1190 #if defined(MULTIPROCESSOR) 1191 if (__cpu_simple_lock_try(&alp->lock_data) == 0) 1192 locked = (alp->lock_holder == cpu_num); 1193 else 1194 __cpu_simple_unlock(&alp->lock_data); 1195 #else 1196 if (alp->lock_data == __SIMPLELOCK_LOCKED) { 1197 locked = 1; 1198 KASSERT(alp->lock_holder == cpu_num); 1199 } 1200 #endif 1201 1202 splx(s); 1203 1204 return (locked); 1205 } 1206 1207 int 1208 _simple_lock_try(volatile struct simplelock *alp, const char *id, int l) 1209 { 1210 cpuid_t cpu_num = cpu_number(); 1211 int s, rv = 0; 1212 1213 s = spllock(); 1214 1215 /* 1216 * MULTIPROCESSOR case: This is `safe' since if it's not us, we 1217 * don't take any action. 1218 */ 1219 #if defined(MULTIPROCESSOR) /* { */ 1220 if ((rv = __cpu_simple_lock_try(&alp->lock_data)) == 0) { 1221 if (alp->lock_holder == cpu_num) 1222 SLOCK_WHERE("simple_lock_try: locking against myself\n", 1223 alp, id, l); 1224 goto out; 1225 } 1226 #else 1227 if (alp->lock_data == __SIMPLELOCK_LOCKED) { 1228 SLOCK_WHERE("simple_lock_try: lock held\n", alp, id, l); 1229 goto out; 1230 } 1231 alp->lock_data = __SIMPLELOCK_LOCKED; 1232 #endif /* MULTIPROCESSOR */ /* } */ 1233 1234 /* 1235 * At this point, we have acquired the lock. 1236 */ 1237 1238 rv = 1; 1239 1240 alp->lock_file = id; 1241 alp->lock_line = l; 1242 alp->lock_holder = cpu_num; 1243 1244 SLOCK_LIST_LOCK(); 1245 TAILQ_INSERT_TAIL(&simplelock_list, alp, list); 1246 SLOCK_LIST_UNLOCK(); 1247 1248 SLOCK_COUNT(1); 1249 1250 out: 1251 splx(s); 1252 return (rv); 1253 } 1254 1255 void 1256 _simple_unlock(volatile struct simplelock *alp, const char *id, int l) 1257 { 1258 int s; 1259 1260 s = spllock(); 1261 1262 /* 1263 * MULTIPROCESSOR case: This is `safe' because we think we hold 1264 * the lock, and if we don't, we don't take any action. 1265 */ 1266 if (alp->lock_data == __SIMPLELOCK_UNLOCKED) { 1267 SLOCK_WHERE("simple_unlock: lock not held\n", 1268 alp, id, l); 1269 goto out; 1270 } 1271 1272 SLOCK_LIST_LOCK(); 1273 TAILQ_REMOVE(&simplelock_list, alp, list); 1274 SLOCK_LIST_UNLOCK(); 1275 1276 SLOCK_COUNT(-1); 1277 1278 alp->list.tqe_next = NULL; /* sanity */ 1279 alp->list.tqe_prev = NULL; /* sanity */ 1280 1281 alp->unlock_file = id; 1282 alp->unlock_line = l; 1283 1284 #if defined(MULTIPROCESSOR) /* { */ 1285 alp->lock_holder = LK_NOCPU; 1286 /* Now that we've modified all fields, release the lock. */ 1287 __cpu_simple_unlock(&alp->lock_data); 1288 #else 1289 alp->lock_data = __SIMPLELOCK_UNLOCKED; 1290 KASSERT(alp->lock_holder == cpu_number()); 1291 alp->lock_holder = LK_NOCPU; 1292 #endif /* } */ 1293 1294 out: 1295 splx(s); 1296 } 1297 1298 void 1299 simple_lock_dump(void) 1300 { 1301 volatile struct simplelock *alp; 1302 int s; 1303 1304 s = spllock(); 1305 SLOCK_LIST_LOCK(); 1306 lock_printf("all simple locks:\n"); 1307 TAILQ_FOREACH(alp, &simplelock_list, list) { 1308 lock_printf("%p CPU %lu %s:%d\n", alp, alp->lock_holder, 1309 alp->lock_file, alp->lock_line); 1310 } 1311 SLOCK_LIST_UNLOCK(); 1312 splx(s); 1313 } 1314 1315 void 1316 simple_lock_freecheck(void *start, void *end) 1317 { 1318 volatile struct simplelock *alp; 1319 int s; 1320 1321 s = spllock(); 1322 SLOCK_LIST_LOCK(); 1323 TAILQ_FOREACH(alp, &simplelock_list, list) { 1324 if ((volatile void *)alp >= start && 1325 (volatile void *)alp < end) { 1326 lock_printf("freeing simple_lock %p CPU %lu %s:%d\n", 1327 alp, alp->lock_holder, alp->lock_file, 1328 alp->lock_line); 1329 SLOCK_DEBUGGER(); 1330 } 1331 } 1332 SLOCK_LIST_UNLOCK(); 1333 splx(s); 1334 } 1335 1336 /* 1337 * We must be holding exactly one lock: the sched_lock. 1338 */ 1339 1340 void 1341 simple_lock_switchcheck(void) 1342 { 1343 1344 simple_lock_only_held(&sched_lock, "switching"); 1345 } 1346 1347 /* 1348 * Drop into the debugger if lp isn't the only lock held. 1349 * lp may be NULL. 1350 */ 1351 void 1352 simple_lock_only_held(volatile struct simplelock *lp, const char *where) 1353 { 1354 volatile struct simplelock *alp; 1355 cpuid_t cpu_num = cpu_number(); 1356 int s; 1357 1358 if (lp) { 1359 LOCK_ASSERT(simple_lock_held(lp)); 1360 } 1361 s = spllock(); 1362 SLOCK_LIST_LOCK(); 1363 TAILQ_FOREACH(alp, &simplelock_list, list) { 1364 if (alp == lp) 1365 continue; 1366 #if defined(MULTIPROCESSOR) 1367 if (alp == &kernel_lock) 1368 continue; 1369 #endif /* defined(MULTIPROCESSOR) */ 1370 if (alp->lock_holder == cpu_num) 1371 break; 1372 } 1373 SLOCK_LIST_UNLOCK(); 1374 splx(s); 1375 1376 if (alp != NULL) { 1377 lock_printf("\n%s with held simple_lock %p " 1378 "CPU %lu %s:%d\n", 1379 where, alp, alp->lock_holder, alp->lock_file, 1380 alp->lock_line); 1381 SLOCK_TRACE(); 1382 SLOCK_DEBUGGER(); 1383 } 1384 } 1385 1386 /* 1387 * Set to 1 by simple_lock_assert_*(). 1388 * Can be cleared from ddb to avoid a panic. 1389 */ 1390 int slock_assert_will_panic; 1391 1392 /* 1393 * If the lock isn't held, print a traceback, optionally drop into the 1394 * debugger, then panic. 1395 * The panic can be avoided by clearing slock_assert_with_panic from the 1396 * debugger. 1397 */ 1398 void 1399 _simple_lock_assert_locked(volatile struct simplelock *alp, 1400 const char *lockname, const char *id, int l) 1401 { 1402 if (simple_lock_held(alp) == 0) { 1403 slock_assert_will_panic = 1; 1404 lock_printf("%s lock not held\n", lockname); 1405 SLOCK_WHERE("lock not held", alp, id, l); 1406 if (slock_assert_will_panic) 1407 panic("%s: not locked", lockname); 1408 } 1409 } 1410 1411 void 1412 _simple_lock_assert_unlocked(volatile struct simplelock *alp, 1413 const char *lockname, const char *id, int l) 1414 { 1415 if (simple_lock_held(alp)) { 1416 slock_assert_will_panic = 1; 1417 lock_printf("%s lock held\n", lockname); 1418 SLOCK_WHERE("lock held", alp, id, l); 1419 if (slock_assert_will_panic) 1420 panic("%s: locked", lockname); 1421 } 1422 } 1423 1424 void 1425 assert_sleepable(struct simplelock *interlock, const char *msg) 1426 { 1427 1428 if (curlwp == NULL) { 1429 panic("assert_sleepable: NULL curlwp"); 1430 } 1431 spinlock_switchcheck(); 1432 simple_lock_only_held(interlock, msg); 1433 } 1434 1435 #endif /* LOCKDEBUG */ /* } */ 1436 1437 #if defined(MULTIPROCESSOR) 1438 /* 1439 * Functions for manipulating the kernel_lock. We put them here 1440 * so that they show up in profiles. 1441 */ 1442 1443 /* 1444 * splbiglock: block IPLs which need to grab kernel_lock. 1445 * XXX splvm or splaudio should be enough. 1446 */ 1447 #if !defined(__HAVE_SPLBIGLOCK) 1448 #define splbiglock() splclock() 1449 #endif 1450 1451 void 1452 _kernel_lock_init(void) 1453 { 1454 1455 simple_lock_init(&kernel_lock); 1456 } 1457 1458 /* 1459 * Acquire/release the kernel lock. Intended for use in the scheduler 1460 * and the lower half of the kernel. 1461 */ 1462 void 1463 _kernel_lock(int flag) 1464 { 1465 struct cpu_info *ci = curcpu(); 1466 1467 SCHED_ASSERT_UNLOCKED(); 1468 1469 if (ci->ci_data.cpu_biglock_count > 0) { 1470 LOCK_ASSERT(simple_lock_held(&kernel_lock)); 1471 ci->ci_data.cpu_biglock_count++; 1472 } else { 1473 int s; 1474 1475 s = splbiglock(); 1476 while (!simple_lock_try(&kernel_lock)) { 1477 splx(s); 1478 SPINLOCK_SPIN_HOOK; 1479 s = splbiglock(); 1480 } 1481 ci->ci_data.cpu_biglock_count++; 1482 splx(s); 1483 } 1484 } 1485 1486 void 1487 _kernel_unlock(void) 1488 { 1489 struct cpu_info *ci = curcpu(); 1490 int s; 1491 1492 KASSERT(ci->ci_data.cpu_biglock_count > 0); 1493 1494 s = splbiglock(); 1495 if ((--ci->ci_data.cpu_biglock_count) == 0) 1496 simple_unlock(&kernel_lock); 1497 splx(s); 1498 } 1499 1500 /* 1501 * Acquire/release the kernel_lock on behalf of a process. Intended for 1502 * use in the top half of the kernel. 1503 */ 1504 void 1505 _kernel_proc_lock(struct lwp *l) 1506 { 1507 1508 SCHED_ASSERT_UNLOCKED(); 1509 _kernel_lock(0); 1510 } 1511 1512 void 1513 _kernel_proc_unlock(struct lwp *l) 1514 { 1515 1516 _kernel_unlock(); 1517 } 1518 1519 int 1520 _kernel_lock_release_all() 1521 { 1522 struct cpu_info *ci = curcpu(); 1523 int hold_count; 1524 1525 hold_count = ci->ci_data.cpu_biglock_count; 1526 1527 if (hold_count) { 1528 int s; 1529 1530 s = splbiglock(); 1531 ci->ci_data.cpu_biglock_count = 0; 1532 simple_unlock(&kernel_lock); 1533 splx(s); 1534 } 1535 1536 return hold_count; 1537 } 1538 1539 void 1540 _kernel_lock_acquire_count(int hold_count) 1541 { 1542 1543 KASSERT(curcpu()->ci_data.cpu_biglock_count == 0); 1544 1545 if (hold_count != 0) { 1546 struct cpu_info *ci = curcpu(); 1547 int s; 1548 1549 s = splbiglock(); 1550 while (!simple_lock_try(&kernel_lock)) { 1551 splx(s); 1552 SPINLOCK_SPIN_HOOK; 1553 s = splbiglock(); 1554 } 1555 ci->ci_data.cpu_biglock_count = hold_count; 1556 splx(s); 1557 } 1558 } 1559 #if defined(DEBUG) 1560 void 1561 _kernel_lock_assert_locked() 1562 { 1563 simple_lock_assert_locked(&kernel_lock, "kernel_lock"); 1564 } 1565 #endif 1566 1567 #endif /* MULTIPROCESSOR */ 1568
Indexes created Thu Sep 25 16:09:42 GMT 2025