kern_lock.c revision 1.99.2.12
1 /* $NetBSD: kern_lock.c,v 1.99.2.12 2007/01/26 23:22:44 ad Exp $ */ 2 3 /*- 4 * Copyright (c) 1999, 2000, 2006 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center, and by Andrew Doran. 10 * 11 * This code is derived from software contributed to The NetBSD Foundation 12 * by Ross Harvey. 13 * 14 * Redistribution and use in source and binary forms, with or without 15 * modification, are permitted provided that the following conditions 16 * are met: 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in the 21 * documentation and/or other materials provided with the distribution. 22 * 3. All advertising materials mentioning features or use of this software 23 * must display the following acknowledgement: 24 * This product includes software developed by the NetBSD 25 * Foundation, Inc. and its contributors. 26 * 4. Neither the name of The NetBSD Foundation nor the names of its 27 * contributors may be used to endorse or promote products derived 28 * from this software without specific prior written permission. 29 * 30 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 31 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 32 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 33 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 34 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 35 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 36 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 37 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 38 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 39 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 40 * POSSIBILITY OF SUCH DAMAGE. 41 */ 42 43 /* 44 * Copyright (c) 1995 45 * The Regents of the University of California. All rights reserved. 46 * 47 * This code contains ideas from software contributed to Berkeley by 48 * Avadis Tevanian, Jr., Michael Wayne Young, and the Mach Operating 49 * System project at Carnegie-Mellon University. 50 * 51 * Redistribution and use in source and binary forms, with or without 52 * modification, are permitted provided that the following conditions 53 * are met: 54 * 1. Redistributions of source code must retain the above copyright 55 * notice, this list of conditions and the following disclaimer. 56 * 2. Redistributions in binary form must reproduce the above copyright 57 * notice, this list of conditions and the following disclaimer in the 58 * documentation and/or other materials provided with the distribution. 59 * 3. Neither the name of the University nor the names of its contributors 60 * may be used to endorse or promote products derived from this software 61 * without specific prior written permission. 62 * 63 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 64 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 65 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 66 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 67 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 68 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 69 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 70 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 71 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 72 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 73 * SUCH DAMAGE. 74 * 75 * @(#)kern_lock.c 8.18 (Berkeley) 5/21/95 76 */ 77 78 #include <sys/cdefs.h> 79 __KERNEL_RCSID(0, "$NetBSD: kern_lock.c,v 1.99.2.12 2007/01/26 23:22:44 ad Exp $"); 80 81 #include "opt_multiprocessor.h" 82 #include "opt_ddb.h" 83 84 #define __MUTEX_PRIVATE 85 86 #include <sys/param.h> 87 #include <sys/proc.h> 88 #include <sys/lock.h> 89 #include <sys/systm.h> 90 #include <sys/lockdebug.h> 91 92 #include <machine/cpu.h> 93 94 #include <dev/lockstat.h> 95 96 #if defined(LOCKDEBUG) 97 #include <sys/syslog.h> 98 /* 99 * note that stdarg.h and the ansi style va_start macro is used for both 100 * ansi and traditional c compiles. 101 * XXX: this requires that stdarg.h define: va_alist and va_dcl 102 */ 103 #include <machine/stdarg.h> 104 105 void lock_printf(const char *fmt, ...) 106 __attribute__((__format__(__printf__,1,2))); 107 108 static int acquire(volatile struct lock **, int *, int, int, int, uintptr_t); 109 110 int lock_debug_syslog = 0; /* defaults to printf, but can be patched */ 111 112 #ifdef DDB 113 #include <ddb/ddbvar.h> 114 #include <machine/db_machdep.h> 115 #include <ddb/db_command.h> 116 #include <ddb/db_interface.h> 117 #endif 118 #endif /* defined(LOCKDEBUG) */ 119 120 #if defined(MULTIPROCESSOR) 121 /* 122 * IPL_BIGLOCK: block IPLs which need to grab kernel_mutex. 123 * XXX IPL_VM or IPL_AUDIO should be enough. 124 */ 125 #if !defined(__HAVE_SPLBIGLOCK) 126 #define splbiglock splclock 127 #endif 128 __cpu_simple_lock_t kernel_lock; 129 int kernel_lock_id; 130 #endif 131 132 /* 133 * Locking primitives implementation. 134 * Locks provide shared/exclusive synchronization. 135 */ 136 137 #if defined(LOCKDEBUG) || defined(DIAGNOSTIC) /* { */ 138 #if defined(MULTIPROCESSOR) /* { */ 139 #define COUNT_CPU(cpu_id, x) \ 140 curcpu()->ci_spin_locks += (x) 141 #else 142 u_long spin_locks; 143 #define COUNT_CPU(cpu_id, x) spin_locks += (x) 144 #endif /* MULTIPROCESSOR */ /* } */ 145 146 #define COUNT(lkp, l, cpu_id, x) \ 147 do { \ 148 if ((lkp)->lk_flags & LK_SPIN) \ 149 COUNT_CPU((cpu_id), (x)); \ 150 else \ 151 (l)->l_locks += (x); \ 152 } while (/*CONSTCOND*/0) 153 #else 154 #define COUNT(lkp, p, cpu_id, x) 155 #define COUNT_CPU(cpu_id, x) 156 #endif /* LOCKDEBUG || DIAGNOSTIC */ /* } */ 157 158 #define INTERLOCK_ACQUIRE(lkp, flags, s) \ 159 do { \ 160 if ((flags) & LK_SPIN) \ 161 s = spllock(); \ 162 simple_lock(&(lkp)->lk_interlock); \ 163 } while (/*CONSTCOND*/ 0) 164 165 #define INTERLOCK_RELEASE(lkp, flags, s) \ 166 do { \ 167 simple_unlock(&(lkp)->lk_interlock); \ 168 if ((flags) & LK_SPIN) \ 169 splx(s); \ 170 } while (/*CONSTCOND*/ 0) 171 172 #ifdef DDB /* { */ 173 #if defined(MULTIPROCESSOR) || defined(LOCKDEBUG) 174 int simple_lock_debugger = 1; /* more serious on MP */ 175 #else 176 int simple_lock_debugger = 0; 177 #endif 178 #define SLOCK_DEBUGGER() if (simple_lock_debugger && db_onpanic) Debugger() 179 #define SLOCK_TRACE() \ 180 db_stack_trace_print((db_expr_t)__builtin_frame_address(0), \ 181 TRUE, 65535, "", lock_printf); 182 #else 183 #define SLOCK_DEBUGGER() /* nothing */ 184 #define SLOCK_TRACE() /* nothing */ 185 #endif /* } */ 186 187 #if defined(LOCKDEBUG) 188 #if defined(DDB) 189 #define SPINLOCK_SPINCHECK_DEBUGGER if (db_onpanic) Debugger() 190 #else 191 #define SPINLOCK_SPINCHECK_DEBUGGER /* nothing */ 192 #endif 193 194 #define SPINLOCK_SPINCHECK_DECL \ 195 /* 32-bits of count -- wrap constitutes a "spinout" */ \ 196 uint32_t __spinc = 0 197 198 #define SPINLOCK_SPINCHECK \ 199 do { \ 200 if (++__spinc == 0) { \ 201 lock_printf("LK_SPIN spinout, excl %d, share %d\n", \ 202 lkp->lk_exclusivecount, lkp->lk_sharecount); \ 203 if (lkp->lk_exclusivecount) \ 204 lock_printf("held by CPU %lu\n", \ 205 (u_long) lkp->lk_cpu); \ 206 if (lkp->lk_lock_file) \ 207 lock_printf("last locked at %s:%d\n", \ 208 lkp->lk_lock_file, lkp->lk_lock_line); \ 209 if (lkp->lk_unlock_file) \ 210 lock_printf("last unlocked at %s:%d\n", \ 211 lkp->lk_unlock_file, lkp->lk_unlock_line); \ 212 SLOCK_TRACE(); \ 213 SPINLOCK_SPINCHECK_DEBUGGER; \ 214 } \ 215 } while (/*CONSTCOND*/ 0) 216 #else 217 #define SPINLOCK_SPINCHECK_DECL /* nothing */ 218 #define SPINLOCK_SPINCHECK /* nothing */ 219 #endif /* LOCKDEBUG && DDB */ 220 221 #define RETURN_ADDRESS ((uintptr_t)__builtin_return_address(0)) 222 223 /* 224 * Acquire a resource. 225 */ 226 static int 227 acquire(volatile struct lock **lkpp, int *s, int extflags, 228 int drain, int wanted, uintptr_t ra) 229 { 230 int error; 231 volatile struct lock *lkp = *lkpp; 232 LOCKSTAT_TIMER(slptime); 233 234 KASSERT(drain || (wanted & LK_WAIT_NONZERO) == 0); 235 236 if (extflags & LK_SPIN) { 237 int interlocked; 238 239 SPINLOCK_SPINCHECK_DECL; 240 241 if (!drain) { 242 lkp->lk_waitcount++; 243 lkp->lk_flags |= LK_WAIT_NONZERO; 244 } 245 for (interlocked = 1;;) { 246 SPINLOCK_SPINCHECK; 247 if ((lkp->lk_flags & wanted) != 0) { 248 if (interlocked) { 249 INTERLOCK_RELEASE(lkp, LK_SPIN, *s); 250 interlocked = 0; 251 } 252 SPINLOCK_SPIN_HOOK; 253 } else if (interlocked) { 254 break; 255 } else { 256 INTERLOCK_ACQUIRE(lkp, LK_SPIN, *s); 257 interlocked = 1; 258 } 259 } 260 if (!drain) { 261 lkp->lk_waitcount--; 262 if (lkp->lk_waitcount == 0) 263 lkp->lk_flags &= ~LK_WAIT_NONZERO; 264 } 265 KASSERT((lkp->lk_flags & wanted) == 0); 266 error = 0; /* sanity */ 267 } else { 268 for (error = 0; (lkp->lk_flags & wanted) != 0; ) { 269 if (drain) 270 lkp->lk_flags |= LK_WAITDRAIN; 271 else { 272 lkp->lk_waitcount++; 273 lkp->lk_flags |= LK_WAIT_NONZERO; 274 } 275 /* XXX Cast away volatile. */ 276 LOCKSTAT_START_TIMER(slptime); 277 error = ltsleep(drain ? 278 (volatile const void *)&lkp->lk_flags : 279 (volatile const void *)lkp, lkp->lk_prio, 280 lkp->lk_wmesg, lkp->lk_timo, &lkp->lk_interlock); 281 LOCKSTAT_STOP_TIMER(slptime); 282 LOCKSTAT_EVENT_RA((void *)(uintptr_t)lkp, 283 LB_LOCKMGR | LB_SLEEP1, 1, slptime, ra); 284 if (!drain) { 285 lkp->lk_waitcount--; 286 if (lkp->lk_waitcount == 0) 287 lkp->lk_flags &= ~LK_WAIT_NONZERO; 288 } 289 if (error) 290 break; 291 if (extflags & LK_SLEEPFAIL) { 292 error = ENOLCK; 293 break; 294 } 295 if (lkp->lk_newlock != NULL) { 296 simple_lock(&lkp->lk_newlock->lk_interlock); 297 simple_unlock(&lkp->lk_interlock); 298 if (lkp->lk_waitcount == 0) 299 wakeup(&lkp->lk_newlock); 300 *lkpp = lkp = lkp->lk_newlock; 301 } 302 } 303 } 304 305 return error; 306 } 307 308 #define SETHOLDER(lkp, pid, lid, cpu_id) \ 309 do { \ 310 if ((lkp)->lk_flags & LK_SPIN) \ 311 (lkp)->lk_cpu = cpu_id; \ 312 else { \ 313 (lkp)->lk_lockholder = pid; \ 314 (lkp)->lk_locklwp = lid; \ 315 } \ 316 } while (/*CONSTCOND*/0) 317 318 #define WEHOLDIT(lkp, pid, lid, cpu_id) \ 319 (((lkp)->lk_flags & LK_SPIN) != 0 ? \ 320 ((lkp)->lk_cpu == (cpu_id)) : \ 321 ((lkp)->lk_lockholder == (pid) && (lkp)->lk_locklwp == (lid))) 322 323 #define WAKEUP_WAITER(lkp) \ 324 do { \ 325 if (((lkp)->lk_flags & (LK_SPIN | LK_WAIT_NONZERO)) == \ 326 LK_WAIT_NONZERO) { \ 327 wakeup((lkp)); \ 328 } \ 329 } while (/*CONSTCOND*/0) 330 331 #if defined(LOCKDEBUG) /* { */ 332 #if defined(MULTIPROCESSOR) /* { */ 333 struct simplelock spinlock_list_slock = SIMPLELOCK_INITIALIZER; 334 335 #define SPINLOCK_LIST_LOCK() \ 336 __cpu_simple_lock(&spinlock_list_slock.lock_data) 337 338 #define SPINLOCK_LIST_UNLOCK() \ 339 __cpu_simple_unlock(&spinlock_list_slock.lock_data) 340 #else 341 #define SPINLOCK_LIST_LOCK() /* nothing */ 342 343 #define SPINLOCK_LIST_UNLOCK() /* nothing */ 344 #endif /* MULTIPROCESSOR */ /* } */ 345 346 _TAILQ_HEAD(, struct lock, volatile) spinlock_list = 347 TAILQ_HEAD_INITIALIZER(spinlock_list); 348 349 #define HAVEIT(lkp) \ 350 do { \ 351 if ((lkp)->lk_flags & LK_SPIN) { \ 352 int sp = spllock(); \ 353 SPINLOCK_LIST_LOCK(); \ 354 TAILQ_INSERT_TAIL(&spinlock_list, (lkp), lk_list); \ 355 SPINLOCK_LIST_UNLOCK(); \ 356 splx(sp); \ 357 } \ 358 } while (/*CONSTCOND*/0) 359 360 #define DONTHAVEIT(lkp) \ 361 do { \ 362 if ((lkp)->lk_flags & LK_SPIN) { \ 363 int sp = spllock(); \ 364 SPINLOCK_LIST_LOCK(); \ 365 TAILQ_REMOVE(&spinlock_list, (lkp), lk_list); \ 366 SPINLOCK_LIST_UNLOCK(); \ 367 splx(sp); \ 368 } \ 369 } while (/*CONSTCOND*/0) 370 #else 371 #define HAVEIT(lkp) /* nothing */ 372 373 #define DONTHAVEIT(lkp) /* nothing */ 374 #endif /* LOCKDEBUG */ /* } */ 375 376 #if defined(LOCKDEBUG) 377 /* 378 * Lock debug printing routine; can be configured to print to console 379 * or log to syslog. 380 */ 381 void 382 lock_printf(const char *fmt, ...) 383 { 384 char b[150]; 385 va_list ap; 386 387 va_start(ap, fmt); 388 if (lock_debug_syslog) 389 vlog(LOG_DEBUG, fmt, ap); 390 else { 391 vsnprintf(b, sizeof(b), fmt, ap); 392 printf_nolog("%s", b); 393 } 394 va_end(ap); 395 } 396 #endif /* LOCKDEBUG */ 397 398 /* 399 * Transfer any waiting processes from one lock to another. 400 */ 401 void 402 transferlockers(struct lock *from, struct lock *to) 403 { 404 405 KASSERT(from != to); 406 KASSERT((from->lk_flags & LK_WAITDRAIN) == 0); 407 if (from->lk_waitcount == 0) 408 return; 409 from->lk_newlock = to; 410 wakeup((void *)from); 411 tsleep((void *)&from->lk_newlock, from->lk_prio, "lkxfer", 0); 412 from->lk_newlock = NULL; 413 from->lk_flags &= ~(LK_WANT_EXCL | LK_WANT_UPGRADE); 414 KASSERT(from->lk_waitcount == 0); 415 } 416 417 418 /* 419 * Initialize a lock; required before use. 420 */ 421 void 422 lockinit(struct lock *lkp, int prio, const char *wmesg, int timo, int flags) 423 { 424 425 memset(lkp, 0, sizeof(struct lock)); 426 simple_lock_init(&lkp->lk_interlock); 427 lkp->lk_flags = flags & LK_EXTFLG_MASK; 428 if (flags & LK_SPIN) 429 lkp->lk_cpu = LK_NOCPU; 430 else { 431 lkp->lk_lockholder = LK_NOPROC; 432 lkp->lk_newlock = NULL; 433 lkp->lk_prio = prio; 434 lkp->lk_timo = timo; 435 } 436 lkp->lk_wmesg = wmesg; /* just a name for spin locks */ 437 #if defined(LOCKDEBUG) 438 lkp->lk_lock_file = NULL; 439 lkp->lk_unlock_file = NULL; 440 #endif 441 } 442 443 /* 444 * Determine the status of a lock. 445 */ 446 int 447 lockstatus(struct lock *lkp) 448 { 449 int s = 0; /* XXX: gcc */ 450 int lock_type = 0; 451 struct lwp *l = curlwp; /* XXX */ 452 pid_t pid; 453 lwpid_t lid; 454 cpuid_t cpu_num; 455 456 if ((lkp->lk_flags & LK_SPIN) || l == NULL) { 457 cpu_num = cpu_number(); 458 pid = LK_KERNPROC; 459 lid = 0; 460 } else { 461 cpu_num = LK_NOCPU; 462 pid = l->l_proc->p_pid; 463 lid = l->l_lid; 464 } 465 466 INTERLOCK_ACQUIRE(lkp, lkp->lk_flags, s); 467 if (lkp->lk_exclusivecount != 0) { 468 if (WEHOLDIT(lkp, pid, lid, cpu_num)) 469 lock_type = LK_EXCLUSIVE; 470 else 471 lock_type = LK_EXCLOTHER; 472 } else if (lkp->lk_sharecount != 0) 473 lock_type = LK_SHARED; 474 else if (lkp->lk_flags & (LK_WANT_EXCL | LK_WANT_UPGRADE)) 475 lock_type = LK_EXCLOTHER; 476 INTERLOCK_RELEASE(lkp, lkp->lk_flags, s); 477 return (lock_type); 478 } 479 480 #if defined(LOCKDEBUG) 481 /* 482 * Make sure no spin locks are held by a CPU that is about 483 * to context switch. 484 */ 485 void 486 spinlock_switchcheck(void) 487 { 488 u_long cnt; 489 int s; 490 491 s = spllock(); 492 #if defined(MULTIPROCESSOR) 493 cnt = curcpu()->ci_spin_locks; 494 #else 495 cnt = spin_locks; 496 #endif 497 splx(s); 498 499 if (cnt != 0) 500 panic("spinlock_switchcheck: CPU %lu has %lu spin locks", 501 (u_long) cpu_number(), cnt); 502 } 503 #endif /* LOCKDEBUG */ 504 505 /* 506 * Locks and IPLs (interrupt priority levels): 507 * 508 * Locks which may be taken from interrupt context must be handled 509 * very carefully; you must spl to the highest IPL where the lock 510 * is needed before acquiring the lock. 511 * 512 * It is also important to avoid deadlock, since certain (very high 513 * priority) interrupts are often needed to keep the system as a whole 514 * from deadlocking, and must not be blocked while you are spinning 515 * waiting for a lower-priority lock. 516 * 517 * In addition, the lock-debugging hooks themselves need to use locks! 518 * 519 * A raw __cpu_simple_lock may be used from interrupts are long as it 520 * is acquired and held at a single IPL. 521 * 522 * A simple_lock (which is a __cpu_simple_lock wrapped with some 523 * debugging hooks) may be used at or below spllock(), which is 524 * typically at or just below splhigh() (i.e. blocks everything 525 * but certain machine-dependent extremely high priority interrupts). 526 * 527 * spinlockmgr spinlocks should be used at or below splsched(). 528 * 529 * Some platforms may have interrupts of higher priority than splsched(), 530 * including hard serial interrupts, inter-processor interrupts, and 531 * kernel debugger traps. 532 */ 533 534 /* 535 * XXX XXX kludge around another kludge.. 536 * 537 * vfs_shutdown() may be called from interrupt context, either as a result 538 * of a panic, or from the debugger. It proceeds to call 539 * sys_sync(&proc0, ...), pretending its running on behalf of proc0 540 * 541 * We would like to make an attempt to sync the filesystems in this case, so 542 * if this happens, we treat attempts to acquire locks specially. 543 * All locks are acquired on behalf of proc0. 544 * 545 * If we've already paniced, we don't block waiting for locks, but 546 * just barge right ahead since we're already going down in flames. 547 */ 548 549 /* 550 * Set, change, or release a lock. 551 * 552 * Shared requests increment the shared count. Exclusive requests set the 553 * LK_WANT_EXCL flag (preventing further shared locks), and wait for already 554 * accepted shared locks and shared-to-exclusive upgrades to go away. 555 */ 556 int 557 #if defined(LOCKDEBUG) 558 _lockmgr(volatile struct lock *lkp, u_int flags, 559 struct simplelock *interlkp, const char *file, int line) 560 #else 561 lockmgr(volatile struct lock *lkp, u_int flags, 562 struct simplelock *interlkp) 563 #endif 564 { 565 int error; 566 pid_t pid; 567 lwpid_t lid; 568 int extflags; 569 cpuid_t cpu_num; 570 struct lwp *l = curlwp; 571 int lock_shutdown_noblock = 0; 572 int s = 0; 573 574 error = 0; 575 576 /* LK_RETRY is for vn_lock, not for lockmgr. */ 577 KASSERT((flags & LK_RETRY) == 0); 578 579 INTERLOCK_ACQUIRE(lkp, lkp->lk_flags, s); 580 if (flags & LK_INTERLOCK) 581 simple_unlock(interlkp); 582 extflags = (flags | lkp->lk_flags) & LK_EXTFLG_MASK; 583 584 #ifdef DIAGNOSTIC /* { */ 585 /* 586 * Don't allow spins on sleep locks and don't allow sleeps 587 * on spin locks. 588 */ 589 if ((flags ^ lkp->lk_flags) & LK_SPIN) 590 panic("lockmgr: sleep/spin mismatch"); 591 #endif /* } */ 592 593 if (extflags & LK_SPIN) { 594 pid = LK_KERNPROC; 595 lid = 0; 596 } else { 597 if (l == NULL) { 598 if (!doing_shutdown) { 599 panic("lockmgr: no context"); 600 } else { 601 l = &lwp0; 602 if (panicstr && (!(flags & LK_NOWAIT))) { 603 flags |= LK_NOWAIT; 604 lock_shutdown_noblock = 1; 605 } 606 } 607 } 608 lid = l->l_lid; 609 pid = l->l_proc->p_pid; 610 } 611 cpu_num = cpu_number(); 612 613 /* 614 * Once a lock has drained, the LK_DRAINING flag is set and an 615 * exclusive lock is returned. The only valid operation thereafter 616 * is a single release of that exclusive lock. This final release 617 * clears the LK_DRAINING flag and sets the LK_DRAINED flag. Any 618 * further requests of any sort will result in a panic. The bits 619 * selected for these two flags are chosen so that they will be set 620 * in memory that is freed (freed memory is filled with 0xdeadbeef). 621 * The final release is permitted to give a new lease on life to 622 * the lock by specifying LK_REENABLE. 623 */ 624 if (lkp->lk_flags & (LK_DRAINING|LK_DRAINED)) { 625 #ifdef DIAGNOSTIC /* { */ 626 if (lkp->lk_flags & LK_DRAINED) 627 panic("lockmgr: using decommissioned lock"); 628 if ((flags & LK_TYPE_MASK) != LK_RELEASE || 629 WEHOLDIT(lkp, pid, lid, cpu_num) == 0) 630 panic("lockmgr: non-release on draining lock: %d", 631 flags & LK_TYPE_MASK); 632 #endif /* DIAGNOSTIC */ /* } */ 633 lkp->lk_flags &= ~LK_DRAINING; 634 if ((flags & LK_REENABLE) == 0) 635 lkp->lk_flags |= LK_DRAINED; 636 } 637 638 switch (flags & LK_TYPE_MASK) { 639 640 case LK_SHARED: 641 if (WEHOLDIT(lkp, pid, lid, cpu_num) == 0) { 642 /* 643 * If just polling, check to see if we will block. 644 */ 645 if ((extflags & LK_NOWAIT) && (lkp->lk_flags & 646 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE))) { 647 error = EBUSY; 648 break; 649 } 650 /* 651 * Wait for exclusive locks and upgrades to clear. 652 */ 653 error = acquire(&lkp, &s, extflags, 0, 654 LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE, 655 RETURN_ADDRESS); 656 if (error) 657 break; 658 lkp->lk_sharecount++; 659 lkp->lk_flags |= LK_SHARE_NONZERO; 660 COUNT(lkp, l, cpu_num, 1); 661 break; 662 } 663 /* 664 * We hold an exclusive lock, so downgrade it to shared. 665 * An alternative would be to fail with EDEADLK. 666 */ 667 lkp->lk_sharecount++; 668 lkp->lk_flags |= LK_SHARE_NONZERO; 669 COUNT(lkp, l, cpu_num, 1); 670 /* fall into downgrade */ 671 672 case LK_DOWNGRADE: 673 if (WEHOLDIT(lkp, pid, lid, cpu_num) == 0 || 674 lkp->lk_exclusivecount == 0) 675 panic("lockmgr: not holding exclusive lock"); 676 lkp->lk_sharecount += lkp->lk_exclusivecount; 677 lkp->lk_flags |= LK_SHARE_NONZERO; 678 lkp->lk_exclusivecount = 0; 679 lkp->lk_recurselevel = 0; 680 lkp->lk_flags &= ~LK_HAVE_EXCL; 681 SETHOLDER(lkp, LK_NOPROC, 0, LK_NOCPU); 682 #if defined(LOCKDEBUG) 683 lkp->lk_unlock_file = file; 684 lkp->lk_unlock_line = line; 685 #endif 686 DONTHAVEIT(lkp); 687 WAKEUP_WAITER(lkp); 688 break; 689 690 case LK_EXCLUPGRADE: 691 /* 692 * If another process is ahead of us to get an upgrade, 693 * then we want to fail rather than have an intervening 694 * exclusive access. 695 */ 696 if (lkp->lk_flags & LK_WANT_UPGRADE) { 697 lkp->lk_sharecount--; 698 if (lkp->lk_sharecount == 0) 699 lkp->lk_flags &= ~LK_SHARE_NONZERO; 700 COUNT(lkp, l, cpu_num, -1); 701 error = EBUSY; 702 break; 703 } 704 /* fall into normal upgrade */ 705 706 case LK_UPGRADE: 707 /* 708 * Upgrade a shared lock to an exclusive one. If another 709 * shared lock has already requested an upgrade to an 710 * exclusive lock, our shared lock is released and an 711 * exclusive lock is requested (which will be granted 712 * after the upgrade). If we return an error, the file 713 * will always be unlocked. 714 */ 715 if (WEHOLDIT(lkp, pid, lid, cpu_num) || lkp->lk_sharecount <= 0) 716 panic("lockmgr: upgrade exclusive lock"); 717 lkp->lk_sharecount--; 718 if (lkp->lk_sharecount == 0) 719 lkp->lk_flags &= ~LK_SHARE_NONZERO; 720 COUNT(lkp, l, cpu_num, -1); 721 /* 722 * If we are just polling, check to see if we will block. 723 */ 724 if ((extflags & LK_NOWAIT) && 725 ((lkp->lk_flags & LK_WANT_UPGRADE) || 726 lkp->lk_sharecount > 1)) { 727 error = EBUSY; 728 break; 729 } 730 if ((lkp->lk_flags & LK_WANT_UPGRADE) == 0) { 731 /* 732 * We are first shared lock to request an upgrade, so 733 * request upgrade and wait for the shared count to 734 * drop to zero, then take exclusive lock. 735 */ 736 lkp->lk_flags |= LK_WANT_UPGRADE; 737 error = acquire(&lkp, &s, extflags, 0, LK_SHARE_NONZERO, 738 RETURN_ADDRESS); 739 lkp->lk_flags &= ~LK_WANT_UPGRADE; 740 if (error) { 741 WAKEUP_WAITER(lkp); 742 break; 743 } 744 lkp->lk_flags |= LK_HAVE_EXCL; 745 SETHOLDER(lkp, pid, lid, cpu_num); 746 #if defined(LOCKDEBUG) 747 lkp->lk_lock_file = file; 748 lkp->lk_lock_line = line; 749 #endif 750 HAVEIT(lkp); 751 if (lkp->lk_exclusivecount != 0) 752 panic("lockmgr: non-zero exclusive count"); 753 lkp->lk_exclusivecount = 1; 754 if (extflags & LK_SETRECURSE) 755 lkp->lk_recurselevel = 1; 756 COUNT(lkp, l, cpu_num, 1); 757 break; 758 } 759 /* 760 * Someone else has requested upgrade. Release our shared 761 * lock, awaken upgrade requestor if we are the last shared 762 * lock, then request an exclusive lock. 763 */ 764 if (lkp->lk_sharecount == 0) 765 WAKEUP_WAITER(lkp); 766 /* fall into exclusive request */ 767 768 case LK_EXCLUSIVE: 769 if (WEHOLDIT(lkp, pid, lid, cpu_num)) { 770 /* 771 * Recursive lock. 772 */ 773 if ((extflags & LK_CANRECURSE) == 0 && 774 lkp->lk_recurselevel == 0) { 775 if (extflags & LK_RECURSEFAIL) { 776 error = EDEADLK; 777 break; 778 } else 779 panic("lockmgr: locking against myself"); 780 } 781 lkp->lk_exclusivecount++; 782 if (extflags & LK_SETRECURSE && 783 lkp->lk_recurselevel == 0) 784 lkp->lk_recurselevel = lkp->lk_exclusivecount; 785 COUNT(lkp, l, cpu_num, 1); 786 break; 787 } 788 /* 789 * If we are just polling, check to see if we will sleep. 790 */ 791 if ((extflags & LK_NOWAIT) && (lkp->lk_flags & 792 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE | 793 LK_SHARE_NONZERO))) { 794 error = EBUSY; 795 break; 796 } 797 /* 798 * Try to acquire the want_exclusive flag. 799 */ 800 error = acquire(&lkp, &s, extflags, 0, 801 LK_HAVE_EXCL | LK_WANT_EXCL, RETURN_ADDRESS); 802 if (error) 803 break; 804 lkp->lk_flags |= LK_WANT_EXCL; 805 /* 806 * Wait for shared locks and upgrades to finish. 807 */ 808 error = acquire(&lkp, &s, extflags, 0, 809 LK_HAVE_EXCL | LK_WANT_UPGRADE | LK_SHARE_NONZERO, 810 RETURN_ADDRESS); 811 lkp->lk_flags &= ~LK_WANT_EXCL; 812 if (error) { 813 WAKEUP_WAITER(lkp); 814 break; 815 } 816 lkp->lk_flags |= LK_HAVE_EXCL; 817 SETHOLDER(lkp, pid, lid, cpu_num); 818 #if defined(LOCKDEBUG) 819 lkp->lk_lock_file = file; 820 lkp->lk_lock_line = line; 821 #endif 822 HAVEIT(lkp); 823 if (lkp->lk_exclusivecount != 0) 824 panic("lockmgr: non-zero exclusive count"); 825 lkp->lk_exclusivecount = 1; 826 if (extflags & LK_SETRECURSE) 827 lkp->lk_recurselevel = 1; 828 COUNT(lkp, l, cpu_num, 1); 829 break; 830 831 case LK_RELEASE: 832 if (lkp->lk_exclusivecount != 0) { 833 if (WEHOLDIT(lkp, pid, lid, cpu_num) == 0) { 834 if (lkp->lk_flags & LK_SPIN) { 835 panic("lockmgr: processor %lu, not " 836 "exclusive lock holder %lu " 837 "unlocking", cpu_num, lkp->lk_cpu); 838 } else { 839 panic("lockmgr: pid %d, not " 840 "exclusive lock holder %d " 841 "unlocking", pid, 842 lkp->lk_lockholder); 843 } 844 } 845 if (lkp->lk_exclusivecount == lkp->lk_recurselevel) 846 lkp->lk_recurselevel = 0; 847 lkp->lk_exclusivecount--; 848 COUNT(lkp, l, cpu_num, -1); 849 if (lkp->lk_exclusivecount == 0) { 850 lkp->lk_flags &= ~LK_HAVE_EXCL; 851 SETHOLDER(lkp, LK_NOPROC, 0, LK_NOCPU); 852 #if defined(LOCKDEBUG) 853 lkp->lk_unlock_file = file; 854 lkp->lk_unlock_line = line; 855 #endif 856 DONTHAVEIT(lkp); 857 } 858 } else if (lkp->lk_sharecount != 0) { 859 lkp->lk_sharecount--; 860 if (lkp->lk_sharecount == 0) 861 lkp->lk_flags &= ~LK_SHARE_NONZERO; 862 COUNT(lkp, l, cpu_num, -1); 863 } 864 #ifdef DIAGNOSTIC 865 else 866 panic("lockmgr: release of unlocked lock!"); 867 #endif 868 WAKEUP_WAITER(lkp); 869 break; 870 871 case LK_DRAIN: 872 /* 873 * Check that we do not already hold the lock, as it can 874 * never drain if we do. Unfortunately, we have no way to 875 * check for holding a shared lock, but at least we can 876 * check for an exclusive one. 877 */ 878 if (WEHOLDIT(lkp, pid, lid, cpu_num)) 879 panic("lockmgr: draining against myself"); 880 /* 881 * If we are just polling, check to see if we will sleep. 882 */ 883 if ((extflags & LK_NOWAIT) && (lkp->lk_flags & 884 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE | 885 LK_SHARE_NONZERO | LK_WAIT_NONZERO))) { 886 error = EBUSY; 887 break; 888 } 889 error = acquire(&lkp, &s, extflags, 1, 890 LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE | 891 LK_SHARE_NONZERO | LK_WAIT_NONZERO, 892 RETURN_ADDRESS); 893 if (error) 894 break; 895 lkp->lk_flags |= LK_DRAINING | LK_HAVE_EXCL; 896 SETHOLDER(lkp, pid, lid, cpu_num); 897 #if defined(LOCKDEBUG) 898 lkp->lk_lock_file = file; 899 lkp->lk_lock_line = line; 900 #endif 901 HAVEIT(lkp); 902 lkp->lk_exclusivecount = 1; 903 /* XXX unlikely that we'd want this */ 904 if (extflags & LK_SETRECURSE) 905 lkp->lk_recurselevel = 1; 906 COUNT(lkp, l, cpu_num, 1); 907 break; 908 909 default: 910 INTERLOCK_RELEASE(lkp, lkp->lk_flags, s); 911 panic("lockmgr: unknown locktype request %d", 912 flags & LK_TYPE_MASK); 913 /* NOTREACHED */ 914 } 915 if ((lkp->lk_flags & (LK_WAITDRAIN|LK_SPIN)) == LK_WAITDRAIN && 916 ((lkp->lk_flags & 917 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE | 918 LK_SHARE_NONZERO | LK_WAIT_NONZERO)) == 0)) { 919 lkp->lk_flags &= ~LK_WAITDRAIN; 920 wakeup(&lkp->lk_flags); 921 } 922 /* 923 * Note that this panic will be a recursive panic, since 924 * we only set lock_shutdown_noblock above if panicstr != NULL. 925 */ 926 if (error && lock_shutdown_noblock) 927 panic("lockmgr: deadlock (see previous panic)"); 928 929 INTERLOCK_RELEASE(lkp, lkp->lk_flags, s); 930 return (error); 931 } 932 933 /* 934 * For a recursive spinlock held one or more times by the current CPU, 935 * release all N locks, and return N. 936 * Intended for use in mi_switch() shortly before context switching. 937 */ 938 939 int 940 #if defined(LOCKDEBUG) 941 _spinlock_release_all(volatile struct lock *lkp, const char *file, int line) 942 #else 943 spinlock_release_all(volatile struct lock *lkp) 944 #endif 945 { 946 int s, count; 947 cpuid_t cpu_num; 948 949 KASSERT(lkp->lk_flags & LK_SPIN); 950 951 INTERLOCK_ACQUIRE(lkp, LK_SPIN, s); 952 953 cpu_num = cpu_number(); 954 count = lkp->lk_exclusivecount; 955 956 if (count != 0) { 957 #ifdef DIAGNOSTIC 958 if (WEHOLDIT(lkp, 0, 0, cpu_num) == 0) { 959 panic("spinlock_release_all: processor %lu, not " 960 "exclusive lock holder %lu " 961 "unlocking", (long)cpu_num, lkp->lk_cpu); 962 } 963 #endif 964 lkp->lk_recurselevel = 0; 965 lkp->lk_exclusivecount = 0; 966 COUNT_CPU(cpu_num, -count); 967 lkp->lk_flags &= ~LK_HAVE_EXCL; 968 SETHOLDER(lkp, LK_NOPROC, 0, LK_NOCPU); 969 #if defined(LOCKDEBUG) 970 lkp->lk_unlock_file = file; 971 lkp->lk_unlock_line = line; 972 #endif 973 DONTHAVEIT(lkp); 974 } 975 #ifdef DIAGNOSTIC 976 else if (lkp->lk_sharecount != 0) 977 panic("spinlock_release_all: release of shared lock!"); 978 else 979 panic("spinlock_release_all: release of unlocked lock!"); 980 #endif 981 INTERLOCK_RELEASE(lkp, LK_SPIN, s); 982 983 return (count); 984 } 985 986 /* 987 * For a recursive spinlock held one or more times by the current CPU, 988 * release all N locks, and return N. 989 * Intended for use in mi_switch() right after resuming execution. 990 */ 991 992 void 993 #if defined(LOCKDEBUG) 994 _spinlock_acquire_count(volatile struct lock *lkp, int count, 995 const char *file, int line) 996 #else 997 spinlock_acquire_count(volatile struct lock *lkp, int count) 998 #endif 999 { 1000 int s, error; 1001 cpuid_t cpu_num; 1002 1003 KASSERT(lkp->lk_flags & LK_SPIN); 1004 1005 INTERLOCK_ACQUIRE(lkp, LK_SPIN, s); 1006 1007 cpu_num = cpu_number(); 1008 1009 #ifdef DIAGNOSTIC 1010 if (WEHOLDIT(lkp, LK_NOPROC, 0, cpu_num)) 1011 panic("spinlock_acquire_count: processor %lu already holds lock", (long)cpu_num); 1012 #endif 1013 /* 1014 * Try to acquire the want_exclusive flag. 1015 */ 1016 error = acquire(&lkp, &s, LK_SPIN, 0, LK_HAVE_EXCL | LK_WANT_EXCL, 1017 RETURN_ADDRESS); 1018 lkp->lk_flags |= LK_WANT_EXCL; 1019 /* 1020 * Wait for shared locks and upgrades to finish. 1021 */ 1022 error = acquire(&lkp, &s, LK_SPIN, 0, 1023 LK_HAVE_EXCL | LK_SHARE_NONZERO | LK_WANT_UPGRADE, 1024 RETURN_ADDRESS); 1025 lkp->lk_flags &= ~LK_WANT_EXCL; 1026 lkp->lk_flags |= LK_HAVE_EXCL; 1027 SETHOLDER(lkp, LK_NOPROC, 0, cpu_num); 1028 #if defined(LOCKDEBUG) 1029 lkp->lk_lock_file = file; 1030 lkp->lk_lock_line = line; 1031 #endif 1032 HAVEIT(lkp); 1033 if (lkp->lk_exclusivecount != 0) 1034 panic("lockmgr: non-zero exclusive count"); 1035 lkp->lk_exclusivecount = count; 1036 lkp->lk_recurselevel = 1; 1037 COUNT_CPU(cpu_num, count); 1038 1039 INTERLOCK_RELEASE(lkp, lkp->lk_flags, s); 1040 } 1041 1042 1043 1044 /* 1045 * Print out information about state of a lock. Used by VOP_PRINT 1046 * routines to display ststus about contained locks. 1047 */ 1048 void 1049 lockmgr_printinfo(volatile struct lock *lkp) 1050 { 1051 1052 if (lkp->lk_sharecount) 1053 printf(" lock type %s: SHARED (count %d)", lkp->lk_wmesg, 1054 lkp->lk_sharecount); 1055 else if (lkp->lk_flags & LK_HAVE_EXCL) { 1056 printf(" lock type %s: EXCL (count %d) by ", 1057 lkp->lk_wmesg, lkp->lk_exclusivecount); 1058 if (lkp->lk_flags & LK_SPIN) 1059 printf("processor %lu", lkp->lk_cpu); 1060 else 1061 printf("pid %d.%d", lkp->lk_lockholder, 1062 lkp->lk_locklwp); 1063 } else 1064 printf(" not locked"); 1065 if ((lkp->lk_flags & LK_SPIN) == 0 && lkp->lk_waitcount > 0) 1066 printf(" with %d pending", lkp->lk_waitcount); 1067 } 1068 1069 #if defined(LOCKDEBUG) /* { */ 1070 _TAILQ_HEAD(, struct simplelock, volatile) simplelock_list = 1071 TAILQ_HEAD_INITIALIZER(simplelock_list); 1072 1073 #if defined(MULTIPROCESSOR) /* { */ 1074 struct simplelock simplelock_list_slock = SIMPLELOCK_INITIALIZER; 1075 1076 #define SLOCK_LIST_LOCK() \ 1077 __cpu_simple_lock(&simplelock_list_slock.lock_data) 1078 1079 #define SLOCK_LIST_UNLOCK() \ 1080 __cpu_simple_unlock(&simplelock_list_slock.lock_data) 1081 1082 #define SLOCK_COUNT(x) \ 1083 curcpu()->ci_simple_locks += (x) 1084 #else 1085 u_long simple_locks; 1086 1087 #define SLOCK_LIST_LOCK() /* nothing */ 1088 1089 #define SLOCK_LIST_UNLOCK() /* nothing */ 1090 1091 #define SLOCK_COUNT(x) simple_locks += (x) 1092 #endif /* MULTIPROCESSOR */ /* } */ 1093 1094 #ifdef MULTIPROCESSOR 1095 #define SLOCK_MP() lock_printf("on CPU %ld\n", \ 1096 (u_long) cpu_number()) 1097 #else 1098 #define SLOCK_MP() /* nothing */ 1099 #endif 1100 1101 #define SLOCK_WHERE(str, alp, id, l) \ 1102 do { \ 1103 lock_printf("\n"); \ 1104 lock_printf(str); \ 1105 lock_printf("lock: %p, currently at: %s:%d\n", (alp), (id), (l)); \ 1106 SLOCK_MP(); \ 1107 if ((alp)->lock_file != NULL) \ 1108 lock_printf("last locked: %s:%d\n", (alp)->lock_file, \ 1109 (alp)->lock_line); \ 1110 if ((alp)->unlock_file != NULL) \ 1111 lock_printf("last unlocked: %s:%d\n", (alp)->unlock_file, \ 1112 (alp)->unlock_line); \ 1113 SLOCK_TRACE() \ 1114 SLOCK_DEBUGGER(); \ 1115 } while (/*CONSTCOND*/0) 1116 1117 /* 1118 * Simple lock functions so that the debugger can see from whence 1119 * they are being called. 1120 */ 1121 void 1122 simple_lock_init(volatile struct simplelock *alp) 1123 { 1124 1125 #if defined(MULTIPROCESSOR) /* { */ 1126 __cpu_simple_lock_init(&alp->lock_data); 1127 #else 1128 alp->lock_data = __SIMPLELOCK_UNLOCKED; 1129 #endif /* } */ 1130 alp->lock_file = NULL; 1131 alp->lock_line = 0; 1132 alp->unlock_file = NULL; 1133 alp->unlock_line = 0; 1134 alp->lock_holder = LK_NOCPU; 1135 } 1136 1137 void 1138 _simple_lock(volatile struct simplelock *alp, const char *id, int l) 1139 { 1140 cpuid_t cpu_num = cpu_number(); 1141 int s; 1142 1143 s = spllock(); 1144 1145 /* 1146 * MULTIPROCESSOR case: This is `safe' since if it's not us, we 1147 * don't take any action, and just fall into the normal spin case. 1148 */ 1149 if (alp->lock_data == __SIMPLELOCK_LOCKED) { 1150 #if defined(MULTIPROCESSOR) /* { */ 1151 if (alp->lock_holder == cpu_num) { 1152 SLOCK_WHERE("simple_lock: locking against myself\n", 1153 alp, id, l); 1154 goto out; 1155 } 1156 #else 1157 SLOCK_WHERE("simple_lock: lock held\n", alp, id, l); 1158 goto out; 1159 #endif /* MULTIPROCESSOR */ /* } */ 1160 } 1161 1162 #if defined(MULTIPROCESSOR) /* { */ 1163 /* Acquire the lock before modifying any fields. */ 1164 splx(s); 1165 __cpu_simple_lock(&alp->lock_data); 1166 s = spllock(); 1167 #else 1168 alp->lock_data = __SIMPLELOCK_LOCKED; 1169 #endif /* } */ 1170 1171 if (alp->lock_holder != LK_NOCPU) { 1172 SLOCK_WHERE("simple_lock: uninitialized lock\n", 1173 alp, id, l); 1174 } 1175 alp->lock_file = id; 1176 alp->lock_line = l; 1177 alp->lock_holder = cpu_num; 1178 1179 SLOCK_LIST_LOCK(); 1180 TAILQ_INSERT_TAIL(&simplelock_list, alp, list); 1181 SLOCK_LIST_UNLOCK(); 1182 1183 SLOCK_COUNT(1); 1184 1185 out: 1186 splx(s); 1187 } 1188 1189 int 1190 _simple_lock_held(volatile struct simplelock *alp) 1191 { 1192 #if defined(MULTIPROCESSOR) || defined(DIAGNOSTIC) 1193 cpuid_t cpu_num = cpu_number(); 1194 #endif 1195 int s, locked = 0; 1196 1197 s = spllock(); 1198 1199 #if defined(MULTIPROCESSOR) 1200 if (__cpu_simple_lock_try(&alp->lock_data) == 0) 1201 locked = (alp->lock_holder == cpu_num); 1202 else 1203 __cpu_simple_unlock(&alp->lock_data); 1204 #else 1205 if (alp->lock_data == __SIMPLELOCK_LOCKED) { 1206 locked = 1; 1207 KASSERT(alp->lock_holder == cpu_num); 1208 } 1209 #endif 1210 1211 splx(s); 1212 1213 return (locked); 1214 } 1215 1216 int 1217 _simple_lock_try(volatile struct simplelock *alp, const char *id, int l) 1218 { 1219 cpuid_t cpu_num = cpu_number(); 1220 int s, rv = 0; 1221 1222 s = spllock(); 1223 1224 /* 1225 * MULTIPROCESSOR case: This is `safe' since if it's not us, we 1226 * don't take any action. 1227 */ 1228 #if defined(MULTIPROCESSOR) /* { */ 1229 if ((rv = __cpu_simple_lock_try(&alp->lock_data)) == 0) { 1230 if (alp->lock_holder == cpu_num) 1231 SLOCK_WHERE("simple_lock_try: locking against myself\n", 1232 alp, id, l); 1233 goto out; 1234 } 1235 #else 1236 if (alp->lock_data == __SIMPLELOCK_LOCKED) { 1237 SLOCK_WHERE("simple_lock_try: lock held\n", alp, id, l); 1238 goto out; 1239 } 1240 alp->lock_data = __SIMPLELOCK_LOCKED; 1241 #endif /* MULTIPROCESSOR */ /* } */ 1242 1243 /* 1244 * At this point, we have acquired the lock. 1245 */ 1246 1247 rv = 1; 1248 1249 alp->lock_file = id; 1250 alp->lock_line = l; 1251 alp->lock_holder = cpu_num; 1252 1253 SLOCK_LIST_LOCK(); 1254 TAILQ_INSERT_TAIL(&simplelock_list, alp, list); 1255 SLOCK_LIST_UNLOCK(); 1256 1257 SLOCK_COUNT(1); 1258 1259 out: 1260 splx(s); 1261 return (rv); 1262 } 1263 1264 void 1265 _simple_unlock(volatile struct simplelock *alp, const char *id, int l) 1266 { 1267 int s; 1268 1269 s = spllock(); 1270 1271 /* 1272 * MULTIPROCESSOR case: This is `safe' because we think we hold 1273 * the lock, and if we don't, we don't take any action. 1274 */ 1275 if (alp->lock_data == __SIMPLELOCK_UNLOCKED) { 1276 SLOCK_WHERE("simple_unlock: lock not held\n", 1277 alp, id, l); 1278 goto out; 1279 } 1280 1281 SLOCK_LIST_LOCK(); 1282 TAILQ_REMOVE(&simplelock_list, alp, list); 1283 SLOCK_LIST_UNLOCK(); 1284 1285 SLOCK_COUNT(-1); 1286 1287 alp->list.tqe_next = NULL; /* sanity */ 1288 alp->list.tqe_prev = NULL; /* sanity */ 1289 1290 alp->unlock_file = id; 1291 alp->unlock_line = l; 1292 1293 #if defined(MULTIPROCESSOR) /* { */ 1294 alp->lock_holder = LK_NOCPU; 1295 /* Now that we've modified all fields, release the lock. */ 1296 __cpu_simple_unlock(&alp->lock_data); 1297 #else 1298 alp->lock_data = __SIMPLELOCK_UNLOCKED; 1299 KASSERT(alp->lock_holder == cpu_number()); 1300 alp->lock_holder = LK_NOCPU; 1301 #endif /* } */ 1302 1303 out: 1304 splx(s); 1305 } 1306 1307 void 1308 simple_lock_dump(void) 1309 { 1310 volatile struct simplelock *alp; 1311 int s; 1312 1313 s = spllock(); 1314 SLOCK_LIST_LOCK(); 1315 lock_printf("all simple locks:\n"); 1316 TAILQ_FOREACH(alp, &simplelock_list, list) { 1317 lock_printf("%p CPU %lu %s:%d\n", alp, alp->lock_holder, 1318 alp->lock_file, alp->lock_line); 1319 } 1320 SLOCK_LIST_UNLOCK(); 1321 splx(s); 1322 } 1323 1324 void 1325 simple_lock_freecheck(void *start, void *end) 1326 { 1327 volatile struct simplelock *alp; 1328 int s; 1329 1330 s = spllock(); 1331 SLOCK_LIST_LOCK(); 1332 TAILQ_FOREACH(alp, &simplelock_list, list) { 1333 if ((volatile void *)alp >= start && 1334 (volatile void *)alp < end) { 1335 lock_printf("freeing simple_lock %p CPU %lu %s:%d\n", 1336 alp, alp->lock_holder, alp->lock_file, 1337 alp->lock_line); 1338 SLOCK_DEBUGGER(); 1339 } 1340 } 1341 SLOCK_LIST_UNLOCK(); 1342 splx(s); 1343 } 1344 1345 /* 1346 * We must be holding exactly one lock: the sched_lock. 1347 */ 1348 1349 void 1350 simple_lock_switchcheck(void) 1351 { 1352 1353 simple_lock_only_held(NULL, "switching"); 1354 } 1355 1356 /* 1357 * Drop into the debugger if lp isn't the only lock held. 1358 * lp may be NULL. 1359 */ 1360 void 1361 simple_lock_only_held(volatile struct simplelock *lp, const char *where) 1362 { 1363 volatile struct simplelock *alp; 1364 cpuid_t cpu_num = cpu_number(); 1365 int s; 1366 1367 if (lp) { 1368 LOCK_ASSERT(simple_lock_held(lp)); 1369 } 1370 s = spllock(); 1371 SLOCK_LIST_LOCK(); 1372 TAILQ_FOREACH(alp, &simplelock_list, list) { 1373 if (alp == lp) 1374 continue; 1375 if (alp->lock_holder == cpu_num) 1376 break; 1377 } 1378 SLOCK_LIST_UNLOCK(); 1379 splx(s); 1380 1381 if (alp != NULL) { 1382 lock_printf("\n%s with held simple_lock %p " 1383 "CPU %lu %s:%d\n", 1384 where, alp, alp->lock_holder, alp->lock_file, 1385 alp->lock_line); 1386 SLOCK_TRACE(); 1387 SLOCK_DEBUGGER(); 1388 } 1389 } 1390 1391 /* 1392 * Set to 1 by simple_lock_assert_*(). 1393 * Can be cleared from ddb to avoid a panic. 1394 */ 1395 int slock_assert_will_panic; 1396 1397 /* 1398 * If the lock isn't held, print a traceback, optionally drop into the 1399 * debugger, then panic. 1400 * The panic can be avoided by clearing slock_assert_with_panic from the 1401 * debugger. 1402 */ 1403 void 1404 _simple_lock_assert_locked(volatile struct simplelock *alp, 1405 const char *lockname, const char *id, int l) 1406 { 1407 if (simple_lock_held(alp) == 0) { 1408 slock_assert_will_panic = 1; 1409 lock_printf("%s lock not held\n", lockname); 1410 SLOCK_WHERE("lock not held", alp, id, l); 1411 if (slock_assert_will_panic) 1412 panic("%s: not locked", lockname); 1413 } 1414 } 1415 1416 void 1417 _simple_lock_assert_unlocked(volatile struct simplelock *alp, 1418 const char *lockname, const char *id, int l) 1419 { 1420 if (simple_lock_held(alp)) { 1421 slock_assert_will_panic = 1; 1422 lock_printf("%s lock held\n", lockname); 1423 SLOCK_WHERE("lock held", alp, id, l); 1424 if (slock_assert_will_panic) 1425 panic("%s: locked", lockname); 1426 } 1427 } 1428 1429 void 1430 assert_sleepable(struct simplelock *interlock, const char *msg) 1431 { 1432 1433 if (curlwp == NULL) { 1434 panic("assert_sleepable: NULL curlwp"); 1435 } 1436 simple_lock_only_held(interlock, msg); 1437 } 1438 1439 #endif /* LOCKDEBUG */ /* } */ 1440 1441 #if defined(MULTIPROCESSOR) 1442 1443 /* 1444 * Functions for manipulating the kernel_lock. We put them here 1445 * so that they show up in profiles. 1446 */ 1447 1448 #define _KERNEL_LOCK_ABORT(msg) \ 1449 LOCKDEBUG_ABORT(kernel_lock_id, &kernel_lock, &_kernel_lock_ops, \ 1450 __FUNCTION__, msg) 1451 1452 #ifdef LOCKDEBUG 1453 #define _KERNEL_LOCK_ASSERT(cond) \ 1454 do { \ 1455 if (!(cond)) \ 1456 _KERNEL_LOCK_ABORT("assertion failed: " #cond); \ 1457 } while (/* CONSTCOND */ 0) 1458 #else 1459 #define _KERNEL_LOCK_ASSERT(cond) /* nothing */ 1460 #endif 1461 1462 void _kernel_lock_dump(volatile void *); 1463 1464 lockops_t _kernel_lock_ops = { 1465 "Kernel lock", 1466 0, 1467 _kernel_lock_dump 1468 }; 1469 1470 /* 1471 * Initialize the kernel lock. 1472 */ 1473 void 1474 _kernel_lock_init(void) 1475 { 1476 1477 __cpu_simple_lock_init(&kernel_lock); 1478 kernel_lock_id = LOCKDEBUG_ALLOC(&kernel_lock, &_kernel_lock_ops); 1479 } 1480 1481 /* 1482 * Print debugging information about the kernel lock. 1483 */ 1484 void 1485 _kernel_lock_dump(volatile void *junk) 1486 { 1487 struct cpu_info *ci = curcpu(); 1488 1489 (void)junk; 1490 1491 printf_nolog("curcpu holds : %18d wanted by: %#018lx\n", 1492 ci->ci_biglock_count, (long)ci->ci_biglock_wanted); 1493 } 1494 1495 /* 1496 * Acquire 'nlocks' holds on the kernel lock. If 'l' is non-null, the 1497 * acquisition is from process context. 1498 */ 1499 void 1500 _kernel_lock(int nlocks, struct lwp *l) 1501 { 1502 struct cpu_info *ci = curcpu(); 1503 LOCKSTAT_TIMER(spintime); 1504 struct lwp *owant; 1505 #ifdef LOCKDEBUG 1506 u_int spins; 1507 #endif 1508 int s; 1509 1510 (void)l; 1511 1512 if (nlocks == 0) 1513 return; 1514 _KERNEL_LOCK_ASSERT(nlocks > 0); 1515 1516 s = splbiglock(); 1517 1518 if (ci->ci_biglock_count != 0) { 1519 _KERNEL_LOCK_ASSERT(kernel_lock == __SIMPLELOCK_LOCKED); 1520 ci->ci_biglock_count += nlocks; 1521 splx(s); 1522 return; 1523 } 1524 1525 if (__cpu_simple_lock_try(&kernel_lock)) { 1526 ci->ci_biglock_count = nlocks; 1527 LOCKDEBUG_LOCKED(kernel_lock_id, 1528 (uintptr_t)__builtin_return_address(0), 0); 1529 splx(s); 1530 return; 1531 } 1532 1533 LOCKSTAT_START_TIMER(spintime); 1534 1535 /* 1536 * Before setting ci_biglock_wanted we must post a store 1537 * fence (see kern_mutex.c). This is accomplished by the 1538 * __cpu_simple_lock_try() above. 1539 */ 1540 owant = ci->ci_biglock_wanted; 1541 ci->ci_biglock_wanted = curlwp; /* XXXAD */ 1542 1543 #ifdef LOCKDEBUG 1544 spins = 0; 1545 #endif 1546 1547 do { 1548 while (kernel_lock == __SIMPLELOCK_LOCKED) { 1549 #ifdef LOCKDEBUG 1550 if (SPINLOCK_SPINOUT(spins)) 1551 _KERNEL_LOCK_ABORT("spinout"); 1552 #endif 1553 splx(s); 1554 SPINLOCK_SPIN_HOOK; 1555 (void)splbiglock(); 1556 } 1557 } while (!__cpu_simple_lock_try(&kernel_lock)); 1558 1559 ci->ci_biglock_wanted = owant; 1560 ci->ci_biglock_count += nlocks; 1561 LOCKSTAT_STOP_TIMER(spintime); 1562 LOCKDEBUG_LOCKED(kernel_lock_id, 1563 (uintptr_t)__builtin_return_address(0), 0); 1564 splx(s); 1565 1566 /* 1567 * Again, another store fence is required (see kern_mutex.c). 1568 */ 1569 mb_write(); 1570 LOCKSTAT_EVENT(&kernel_lock, LB_KERNEL_LOCK | LB_SPIN, 1, spintime); 1571 } 1572 1573 /* 1574 * Release 'nlocks' holds on the kernel lock. If 'nlocks' is zero, release 1575 * all holds. If 'l' is non-null, the release is from process context. 1576 */ 1577 void 1578 _kernel_unlock(int nlocks, struct lwp *l, int *countp) 1579 { 1580 struct cpu_info *ci = curcpu(); 1581 u_int olocks; 1582 int s; 1583 1584 (void)l; 1585 1586 _KERNEL_LOCK_ASSERT(nlocks < 2); 1587 1588 olocks = ci->ci_biglock_count; 1589 1590 if (olocks == 0) { 1591 _KERNEL_LOCK_ASSERT(nlocks <= 0); 1592 if (countp != NULL) 1593 *countp = 0; 1594 return; 1595 } 1596 1597 _KERNEL_LOCK_ASSERT(kernel_lock == __SIMPLELOCK_LOCKED); 1598 1599 if (nlocks == 0) 1600 nlocks = olocks; 1601 else if (nlocks == -1) { 1602 nlocks = 1; 1603 _KERNEL_LOCK_ASSERT(olocks == 1); 1604 } 1605 1606 s = splbiglock(); 1607 if ((ci->ci_biglock_count -= nlocks) == 0) { 1608 LOCKDEBUG_UNLOCKED(kernel_lock_id, 1609 (uintptr_t)__builtin_return_address(0), 0); 1610 __cpu_simple_unlock(&kernel_lock); 1611 } 1612 splx(s); 1613 1614 if (countp != NULL) 1615 *countp = olocks; 1616 } 1617 1618 #if defined(DEBUG) 1619 /* 1620 * Assert that the kernel lock is held. 1621 */ 1622 void 1623 _kernel_lock_assert_locked(void) 1624 { 1625 1626 if (kernel_lock != __SIMPLELOCK_LOCKED || 1627 curcpu()->ci_biglock_count == 0) 1628 _KERNEL_LOCK_ABORT("not locked"); 1629 } 1630 1631 void 1632 _kernel_lock_assert_unlocked() 1633 { 1634 1635 if (curcpu()->ci_biglock_count != 0) 1636 _KERNEL_LOCK_ABORT("locked"); 1637 } 1638 #endif 1639 1640 #endif /* MULTIPROCESSOR || LOCKDEBUG */ 1641
Indexes created Sun Sep 28 12:09:53 GMT 2025