kern_lock.c revision 1.121
1 /* $NetBSD: kern_lock.c,v 1.121 2007/10/10 17:37:40 ad Exp $ */ 2 3 /*- 4 * Copyright (c) 1999, 2000, 2006, 2007 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center, and by Andrew Doran. 10 * 11 * This code is derived from software contributed to The NetBSD Foundation 12 * by Ross Harvey. 13 * 14 * Redistribution and use in source and binary forms, with or without 15 * modification, are permitted provided that the following conditions 16 * are met: 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in the 21 * documentation and/or other materials provided with the distribution. 22 * 3. All advertising materials mentioning features or use of this software 23 * must display the following acknowledgement: 24 * This product includes software developed by the NetBSD 25 * Foundation, Inc. and its contributors. 26 * 4. Neither the name of The NetBSD Foundation nor the names of its 27 * contributors may be used to endorse or promote products derived 28 * from this software without specific prior written permission. 29 * 30 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 31 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 32 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 33 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 34 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 35 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 36 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 37 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 38 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 39 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 40 * POSSIBILITY OF SUCH DAMAGE. 41 */ 42 43 /* 44 * Copyright (c) 1995 45 * The Regents of the University of California. All rights reserved. 46 * 47 * This code contains ideas from software contributed to Berkeley by 48 * Avadis Tevanian, Jr., Michael Wayne Young, and the Mach Operating 49 * System project at Carnegie-Mellon University. 50 * 51 * Redistribution and use in source and binary forms, with or without 52 * modification, are permitted provided that the following conditions 53 * are met: 54 * 1. Redistributions of source code must retain the above copyright 55 * notice, this list of conditions and the following disclaimer. 56 * 2. Redistributions in binary form must reproduce the above copyright 57 * notice, this list of conditions and the following disclaimer in the 58 * documentation and/or other materials provided with the distribution. 59 * 3. Neither the name of the University nor the names of its contributors 60 * may be used to endorse or promote products derived from this software 61 * without specific prior written permission. 62 * 63 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 64 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 65 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 66 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 67 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 68 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 69 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 70 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 71 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 72 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 73 * SUCH DAMAGE. 74 * 75 * @(#)kern_lock.c 8.18 (Berkeley) 5/21/95 76 */ 77 78 #include <sys/cdefs.h> 79 __KERNEL_RCSID(0, "$NetBSD: kern_lock.c,v 1.121 2007/10/10 17:37:40 ad Exp $"); 80 81 #include "opt_multiprocessor.h" 82 #include "opt_ddb.h" 83 84 #define __MUTEX_PRIVATE 85 86 #include <sys/param.h> 87 #include <sys/proc.h> 88 #include <sys/lock.h> 89 #include <sys/systm.h> 90 #include <sys/lockdebug.h> 91 92 #include <machine/cpu.h> 93 #include <machine/stdarg.h> 94 95 #include <dev/lockstat.h> 96 97 #if defined(LOCKDEBUG) 98 #include <sys/syslog.h> 99 /* 100 * note that stdarg.h and the ansi style va_start macro is used for both 101 * ansi and traditional c compiles. 102 * XXX: this requires that stdarg.h define: va_alist and va_dcl 103 */ 104 #include <machine/stdarg.h> 105 106 void lock_printf(const char *fmt, ...) 107 __attribute__((__format__(__printf__,1,2))); 108 109 static int acquire(volatile struct lock **, int *, int, int, int, uintptr_t); 110 111 int lock_debug_syslog = 0; /* defaults to printf, but can be patched */ 112 113 #ifdef DDB 114 #include <ddb/ddbvar.h> 115 #include <machine/db_machdep.h> 116 #include <ddb/db_command.h> 117 #include <ddb/db_interface.h> 118 #endif 119 #endif /* defined(LOCKDEBUG) */ 120 121 /* 122 * Locking primitives implementation. 123 * Locks provide shared/exclusive synchronization. 124 */ 125 126 #if defined(LOCKDEBUG) || defined(DIAGNOSTIC) /* { */ 127 #if defined(MULTIPROCESSOR) /* { */ 128 #define COUNT_CPU(cpu_id, x) \ 129 curcpu()->ci_spin_locks += (x) 130 #else 131 u_long spin_locks; 132 #define COUNT_CPU(cpu_id, x) spin_locks += (x) 133 #endif /* MULTIPROCESSOR */ /* } */ 134 135 #define COUNT(lkp, l, cpu_id, x) \ 136 do { \ 137 if ((lkp)->lk_flags & LK_SPIN) \ 138 COUNT_CPU((cpu_id), (x)); \ 139 else \ 140 (l)->l_locks += (x); \ 141 } while (/*CONSTCOND*/0) 142 #else 143 #define COUNT(lkp, p, cpu_id, x) 144 #define COUNT_CPU(cpu_id, x) 145 #endif /* LOCKDEBUG || DIAGNOSTIC */ /* } */ 146 147 #define INTERLOCK_ACQUIRE(lkp, flags, s) \ 148 do { \ 149 if ((flags) & LK_SPIN) \ 150 s = splhigh(); \ 151 simple_lock(&(lkp)->lk_interlock); \ 152 } while (/*CONSTCOND*/ 0) 153 154 #define INTERLOCK_RELEASE(lkp, flags, s) \ 155 do { \ 156 simple_unlock(&(lkp)->lk_interlock); \ 157 if ((flags) & LK_SPIN) \ 158 splx(s); \ 159 } while (/*CONSTCOND*/ 0) 160 161 #ifdef DDB /* { */ 162 #if defined(MULTIPROCESSOR) || defined(LOCKDEBUG) 163 int simple_lock_debugger = 1; /* more serious on MP */ 164 #else 165 int simple_lock_debugger = 0; 166 #endif 167 #define SLOCK_DEBUGGER() if (simple_lock_debugger && db_onpanic) Debugger() 168 #define SLOCK_TRACE() \ 169 db_stack_trace_print((db_expr_t)__builtin_frame_address(0), \ 170 true, 65535, "", lock_printf); 171 #else 172 #define SLOCK_DEBUGGER() /* nothing */ 173 #define SLOCK_TRACE() /* nothing */ 174 #endif /* } */ 175 176 #if defined(LOCKDEBUG) 177 #if defined(DDB) 178 #define SPINLOCK_SPINCHECK_DEBUGGER if (db_onpanic) Debugger() 179 #else 180 #define SPINLOCK_SPINCHECK_DEBUGGER /* nothing */ 181 #endif 182 183 #define SPINLOCK_SPINCHECK_DECL \ 184 /* 32-bits of count -- wrap constitutes a "spinout" */ \ 185 uint32_t __spinc = 0 186 187 #define SPINLOCK_SPINCHECK \ 188 do { \ 189 if (++__spinc == 0) { \ 190 lock_printf("LK_SPIN spinout, excl %d, share %d\n", \ 191 lkp->lk_exclusivecount, lkp->lk_sharecount); \ 192 if (lkp->lk_exclusivecount) \ 193 lock_printf("held by CPU %lu\n", \ 194 (u_long) lkp->lk_cpu); \ 195 if (lkp->lk_lock_file) \ 196 lock_printf("last locked at %s:%d\n", \ 197 lkp->lk_lock_file, lkp->lk_lock_line); \ 198 if (lkp->lk_unlock_file) \ 199 lock_printf("last unlocked at %s:%d\n", \ 200 lkp->lk_unlock_file, lkp->lk_unlock_line); \ 201 SLOCK_TRACE(); \ 202 SPINLOCK_SPINCHECK_DEBUGGER; \ 203 } \ 204 } while (/*CONSTCOND*/ 0) 205 #else 206 #define SPINLOCK_SPINCHECK_DECL /* nothing */ 207 #define SPINLOCK_SPINCHECK /* nothing */ 208 #endif /* LOCKDEBUG && DDB */ 209 210 #define RETURN_ADDRESS ((uintptr_t)__builtin_return_address(0)) 211 212 /* 213 * Acquire a resource. 214 */ 215 static int 216 acquire(volatile struct lock **lkpp, int *s, int extflags, 217 int drain, int wanted, uintptr_t ra) 218 { 219 int error; 220 volatile struct lock *lkp = *lkpp; 221 LOCKSTAT_TIMER(slptime); 222 LOCKSTAT_FLAG(lsflag); 223 224 KASSERT(drain || (wanted & LK_WAIT_NONZERO) == 0); 225 226 if (extflags & LK_SPIN) { 227 int interlocked; 228 229 SPINLOCK_SPINCHECK_DECL; 230 231 if (!drain) { 232 lkp->lk_waitcount++; 233 lkp->lk_flags |= LK_WAIT_NONZERO; 234 } 235 for (interlocked = 1;;) { 236 SPINLOCK_SPINCHECK; 237 if ((lkp->lk_flags & wanted) != 0) { 238 if (interlocked) { 239 INTERLOCK_RELEASE(lkp, LK_SPIN, *s); 240 interlocked = 0; 241 } 242 SPINLOCK_SPIN_HOOK; 243 } else if (interlocked) { 244 break; 245 } else { 246 INTERLOCK_ACQUIRE(lkp, LK_SPIN, *s); 247 interlocked = 1; 248 } 249 } 250 if (!drain) { 251 lkp->lk_waitcount--; 252 if (lkp->lk_waitcount == 0) 253 lkp->lk_flags &= ~LK_WAIT_NONZERO; 254 } 255 KASSERT((lkp->lk_flags & wanted) == 0); 256 error = 0; /* sanity */ 257 } else { 258 LOCKSTAT_ENTER(lsflag); 259 260 for (error = 0; (lkp->lk_flags & wanted) != 0; ) { 261 if (drain) 262 lkp->lk_flags |= LK_WAITDRAIN; 263 else { 264 lkp->lk_waitcount++; 265 lkp->lk_flags |= LK_WAIT_NONZERO; 266 } 267 /* XXX Cast away volatile. */ 268 LOCKSTAT_START_TIMER(lsflag, slptime); 269 error = ltsleep(drain ? 270 (volatile const void *)&lkp->lk_flags : 271 (volatile const void *)lkp, lkp->lk_prio, 272 lkp->lk_wmesg, lkp->lk_timo, &lkp->lk_interlock); 273 LOCKSTAT_STOP_TIMER(lsflag, slptime); 274 LOCKSTAT_EVENT_RA(lsflag, (void *)(uintptr_t)lkp, 275 LB_LOCKMGR | LB_SLEEP1, 1, slptime, ra); 276 if (!drain) { 277 lkp->lk_waitcount--; 278 if (lkp->lk_waitcount == 0) 279 lkp->lk_flags &= ~LK_WAIT_NONZERO; 280 } 281 if (error) 282 break; 283 if (extflags & LK_SLEEPFAIL) { 284 error = ENOLCK; 285 break; 286 } 287 if (lkp->lk_newlock != NULL) { 288 simple_lock(&lkp->lk_newlock->lk_interlock); 289 simple_unlock(&lkp->lk_interlock); 290 if (lkp->lk_waitcount == 0) 291 wakeup(&lkp->lk_newlock); 292 *lkpp = lkp = lkp->lk_newlock; 293 } 294 } 295 296 LOCKSTAT_EXIT(lsflag); 297 } 298 299 return error; 300 } 301 302 #define SETHOLDER(lkp, pid, lid, cpu_id) \ 303 do { \ 304 if ((lkp)->lk_flags & LK_SPIN) \ 305 (lkp)->lk_cpu = cpu_id; \ 306 else { \ 307 (lkp)->lk_lockholder = pid; \ 308 (lkp)->lk_locklwp = lid; \ 309 } \ 310 } while (/*CONSTCOND*/0) 311 312 #define WEHOLDIT(lkp, pid, lid, cpu_id) \ 313 (((lkp)->lk_flags & LK_SPIN) != 0 ? \ 314 ((lkp)->lk_cpu == (cpu_id)) : \ 315 ((lkp)->lk_lockholder == (pid) && (lkp)->lk_locklwp == (lid))) 316 317 #define WAKEUP_WAITER(lkp) \ 318 do { \ 319 if (((lkp)->lk_flags & (LK_SPIN | LK_WAIT_NONZERO)) == \ 320 LK_WAIT_NONZERO) { \ 321 wakeup((lkp)); \ 322 } \ 323 } while (/*CONSTCOND*/0) 324 325 #if defined(LOCKDEBUG) /* { */ 326 #if defined(MULTIPROCESSOR) /* { */ 327 struct simplelock spinlock_list_slock = SIMPLELOCK_INITIALIZER; 328 329 #define SPINLOCK_LIST_LOCK() \ 330 __cpu_simple_lock(&spinlock_list_slock.lock_data) 331 332 #define SPINLOCK_LIST_UNLOCK() \ 333 __cpu_simple_unlock(&spinlock_list_slock.lock_data) 334 #else 335 #define SPINLOCK_LIST_LOCK() /* nothing */ 336 337 #define SPINLOCK_LIST_UNLOCK() /* nothing */ 338 #endif /* MULTIPROCESSOR */ /* } */ 339 340 _TAILQ_HEAD(, struct lock, volatile) spinlock_list = 341 TAILQ_HEAD_INITIALIZER(spinlock_list); 342 343 #define HAVEIT(lkp) \ 344 do { \ 345 if ((lkp)->lk_flags & LK_SPIN) { \ 346 int sp = splhigh(); \ 347 SPINLOCK_LIST_LOCK(); \ 348 TAILQ_INSERT_TAIL(&spinlock_list, (lkp), lk_list); \ 349 SPINLOCK_LIST_UNLOCK(); \ 350 splx(sp); \ 351 } \ 352 } while (/*CONSTCOND*/0) 353 354 #define DONTHAVEIT(lkp) \ 355 do { \ 356 if ((lkp)->lk_flags & LK_SPIN) { \ 357 int sp = splhigh(); \ 358 SPINLOCK_LIST_LOCK(); \ 359 TAILQ_REMOVE(&spinlock_list, (lkp), lk_list); \ 360 SPINLOCK_LIST_UNLOCK(); \ 361 splx(sp); \ 362 } \ 363 } while (/*CONSTCOND*/0) 364 #else 365 #define HAVEIT(lkp) /* nothing */ 366 367 #define DONTHAVEIT(lkp) /* nothing */ 368 #endif /* LOCKDEBUG */ /* } */ 369 370 #if defined(LOCKDEBUG) 371 /* 372 * Lock debug printing routine; can be configured to print to console 373 * or log to syslog. 374 */ 375 void 376 lock_printf(const char *fmt, ...) 377 { 378 char b[150]; 379 va_list ap; 380 381 va_start(ap, fmt); 382 if (lock_debug_syslog) 383 vlog(LOG_DEBUG, fmt, ap); 384 else { 385 vsnprintf(b, sizeof(b), fmt, ap); 386 printf_nolog("%s", b); 387 } 388 va_end(ap); 389 } 390 #endif /* LOCKDEBUG */ 391 392 static void 393 lockpanic(volatile struct lock *lkp, const char *fmt, ...) 394 { 395 char s[150], b[150]; 396 #ifdef LOCKDEBUG 397 static const char *locktype[] = { 398 "*0*", "shared", "exclusive", "upgrade", "exclupgrade", 399 "downgrade", "release", "drain", "exclother", "*9*", 400 "*10*", "*11*", "*12*", "*13*", "*14*", "*15*" 401 }; 402 #endif 403 va_list ap; 404 va_start(ap, fmt); 405 vsnprintf(s, sizeof(s), fmt, ap); 406 va_end(ap); 407 bitmask_snprintf(lkp->lk_flags, __LK_FLAG_BITS, b, sizeof(b)); 408 panic("%s (" 409 #ifdef LOCKDEBUG 410 "type %s " 411 #endif 412 "flags %s, sharecount %d, exclusivecount %d, " 413 "recurselevel %d, waitcount %d, wmesg %s" 414 #ifdef LOCKDEBUG 415 ", lock_file %s, unlock_file %s, lock_line %d, unlock_line %d" 416 #endif 417 ")\n", 418 s, 419 #ifdef LOCKDEBUG 420 locktype[lkp->lk_flags & LK_TYPE_MASK], 421 #endif 422 b, lkp->lk_sharecount, lkp->lk_exclusivecount, 423 lkp->lk_recurselevel, lkp->lk_waitcount, lkp->lk_wmesg 424 #ifdef LOCKDEBUG 425 , lkp->lk_lock_file, lkp->lk_unlock_file, lkp->lk_lock_line, 426 lkp->lk_unlock_line 427 #endif 428 ); 429 } 430 431 /* 432 * Initialize a lock; required before use. 433 */ 434 void 435 lockinit(struct lock *lkp, pri_t prio, const char *wmesg, int timo, int flags) 436 { 437 438 memset(lkp, 0, sizeof(struct lock)); 439 simple_lock_init(&lkp->lk_interlock); 440 lkp->lk_flags = flags & LK_EXTFLG_MASK; 441 if (flags & LK_SPIN) 442 lkp->lk_cpu = LK_NOCPU; 443 else { 444 lkp->lk_lockholder = LK_NOPROC; 445 lkp->lk_newlock = NULL; 446 lkp->lk_prio = prio; 447 lkp->lk_timo = timo; 448 } 449 lkp->lk_wmesg = wmesg; /* just a name for spin locks */ 450 #if defined(LOCKDEBUG) 451 lkp->lk_lock_file = NULL; 452 lkp->lk_unlock_file = NULL; 453 #endif 454 } 455 456 /* 457 * Determine the status of a lock. 458 */ 459 int 460 lockstatus(struct lock *lkp) 461 { 462 int s = 0; /* XXX: gcc */ 463 int lock_type = 0; 464 struct lwp *l = curlwp; /* XXX */ 465 pid_t pid; 466 lwpid_t lid; 467 cpuid_t cpu_num; 468 469 if ((lkp->lk_flags & LK_SPIN) || l == NULL) { 470 cpu_num = cpu_number(); 471 pid = LK_KERNPROC; 472 lid = 0; 473 } else { 474 cpu_num = LK_NOCPU; 475 pid = l->l_proc->p_pid; 476 lid = l->l_lid; 477 } 478 479 INTERLOCK_ACQUIRE(lkp, lkp->lk_flags, s); 480 if (lkp->lk_exclusivecount != 0) { 481 if (WEHOLDIT(lkp, pid, lid, cpu_num)) 482 lock_type = LK_EXCLUSIVE; 483 else 484 lock_type = LK_EXCLOTHER; 485 } else if (lkp->lk_sharecount != 0) 486 lock_type = LK_SHARED; 487 else if (lkp->lk_flags & (LK_WANT_EXCL | LK_WANT_UPGRADE)) 488 lock_type = LK_EXCLOTHER; 489 INTERLOCK_RELEASE(lkp, lkp->lk_flags, s); 490 return (lock_type); 491 } 492 493 #if defined(LOCKDEBUG) 494 /* 495 * Make sure no spin locks are held by a CPU that is about 496 * to context switch. 497 */ 498 void 499 spinlock_switchcheck(void) 500 { 501 u_long cnt; 502 int s; 503 504 if (panicstr != NULL) 505 return; 506 507 s = splhigh(); 508 #if defined(MULTIPROCESSOR) 509 cnt = curcpu()->ci_spin_locks; 510 #else 511 cnt = spin_locks; 512 #endif 513 splx(s); 514 515 if (cnt != 0) 516 panic("spinlock_switchcheck: CPU %lu has %lu spin locks", 517 (u_long) cpu_number(), cnt); 518 } 519 #endif /* LOCKDEBUG */ 520 521 /* 522 * Locks and IPLs (interrupt priority levels): 523 * 524 * Locks which may be taken from interrupt context must be handled 525 * very carefully; you must spl to the highest IPL where the lock 526 * is needed before acquiring the lock. 527 * 528 * It is also important to avoid deadlock, since certain (very high 529 * priority) interrupts are often needed to keep the system as a whole 530 * from deadlocking, and must not be blocked while you are spinning 531 * waiting for a lower-priority lock. 532 * 533 * In addition, the lock-debugging hooks themselves need to use locks! 534 * 535 * A raw __cpu_simple_lock may be used from interrupts are long as it 536 * is acquired and held at a single IPL. 537 */ 538 539 /* 540 * XXX XXX kludge around another kludge.. 541 * 542 * vfs_shutdown() may be called from interrupt context, either as a result 543 * of a panic, or from the debugger. It proceeds to call 544 * sys_sync(&proc0, ...), pretending its running on behalf of proc0 545 * 546 * We would like to make an attempt to sync the filesystems in this case, so 547 * if this happens, we treat attempts to acquire locks specially. 548 * All locks are acquired on behalf of proc0. 549 * 550 * If we've already paniced, we don't block waiting for locks, but 551 * just barge right ahead since we're already going down in flames. 552 */ 553 554 /* 555 * Set, change, or release a lock. 556 * 557 * Shared requests increment the shared count. Exclusive requests set the 558 * LK_WANT_EXCL flag (preventing further shared locks), and wait for already 559 * accepted shared locks and shared-to-exclusive upgrades to go away. 560 */ 561 int 562 #if defined(LOCKDEBUG) 563 _lockmgr(volatile struct lock *lkp, u_int flags, 564 struct simplelock *interlkp, const char *file, int line) 565 #else 566 lockmgr(volatile struct lock *lkp, u_int flags, 567 struct simplelock *interlkp) 568 #endif 569 { 570 int error; 571 pid_t pid; 572 lwpid_t lid; 573 int extflags; 574 cpuid_t cpu_num; 575 struct lwp *l = curlwp; 576 int lock_shutdown_noblock = 0; 577 int s = 0; 578 579 error = 0; 580 581 /* LK_RETRY is for vn_lock, not for lockmgr. */ 582 KASSERT((flags & LK_RETRY) == 0); 583 584 INTERLOCK_ACQUIRE(lkp, lkp->lk_flags, s); 585 if (flags & LK_INTERLOCK) 586 simple_unlock(interlkp); 587 extflags = (flags | lkp->lk_flags) & LK_EXTFLG_MASK; 588 589 #ifdef DIAGNOSTIC /* { */ 590 /* 591 * Don't allow spins on sleep locks and don't allow sleeps 592 * on spin locks. 593 */ 594 if ((flags ^ lkp->lk_flags) & LK_SPIN) 595 lockpanic(lkp, "lockmgr: sleep/spin mismatch"); 596 #endif /* } */ 597 598 if (extflags & LK_SPIN) { 599 pid = LK_KERNPROC; 600 lid = 0; 601 } else { 602 if (l == NULL) { 603 if (!doing_shutdown) { 604 panic("lockmgr: no context"); 605 } else { 606 l = &lwp0; 607 if (panicstr && (!(flags & LK_NOWAIT))) { 608 flags |= LK_NOWAIT; 609 lock_shutdown_noblock = 1; 610 } 611 } 612 } 613 lid = l->l_lid; 614 pid = l->l_proc->p_pid; 615 } 616 cpu_num = cpu_number(); 617 618 /* 619 * Once a lock has drained, the LK_DRAINING flag is set and an 620 * exclusive lock is returned. The only valid operation thereafter 621 * is a single release of that exclusive lock. This final release 622 * clears the LK_DRAINING flag and sets the LK_DRAINED flag. Any 623 * further requests of any sort will result in a panic. The bits 624 * selected for these two flags are chosen so that they will be set 625 * in memory that is freed (freed memory is filled with 0xdeadbeef). 626 * The final release is permitted to give a new lease on life to 627 * the lock by specifying LK_REENABLE. 628 */ 629 if (lkp->lk_flags & (LK_DRAINING|LK_DRAINED)) { 630 #ifdef DIAGNOSTIC /* { */ 631 if (lkp->lk_flags & LK_DRAINED) 632 lockpanic(lkp, "lockmgr: using decommissioned lock"); 633 if ((flags & LK_TYPE_MASK) != LK_RELEASE || 634 WEHOLDIT(lkp, pid, lid, cpu_num) == 0) 635 lockpanic(lkp, "lockmgr: non-release on draining lock: %d", 636 flags & LK_TYPE_MASK); 637 #endif /* DIAGNOSTIC */ /* } */ 638 lkp->lk_flags &= ~LK_DRAINING; 639 if ((flags & LK_REENABLE) == 0) 640 lkp->lk_flags |= LK_DRAINED; 641 } 642 643 switch (flags & LK_TYPE_MASK) { 644 645 case LK_SHARED: 646 if (WEHOLDIT(lkp, pid, lid, cpu_num) == 0) { 647 /* 648 * If just polling, check to see if we will block. 649 */ 650 if ((extflags & LK_NOWAIT) && (lkp->lk_flags & 651 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE))) { 652 error = EBUSY; 653 break; 654 } 655 /* 656 * Wait for exclusive locks and upgrades to clear. 657 */ 658 error = acquire(&lkp, &s, extflags, 0, 659 LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE, 660 RETURN_ADDRESS); 661 if (error) 662 break; 663 lkp->lk_sharecount++; 664 lkp->lk_flags |= LK_SHARE_NONZERO; 665 COUNT(lkp, l, cpu_num, 1); 666 break; 667 } 668 /* 669 * We hold an exclusive lock, so downgrade it to shared. 670 * An alternative would be to fail with EDEADLK. 671 */ 672 lkp->lk_sharecount++; 673 lkp->lk_flags |= LK_SHARE_NONZERO; 674 COUNT(lkp, l, cpu_num, 1); 675 /* fall into downgrade */ 676 677 case LK_DOWNGRADE: 678 if (WEHOLDIT(lkp, pid, lid, cpu_num) == 0 || 679 lkp->lk_exclusivecount == 0) 680 lockpanic(lkp, "lockmgr: not holding exclusive lock"); 681 lkp->lk_sharecount += lkp->lk_exclusivecount; 682 lkp->lk_flags |= LK_SHARE_NONZERO; 683 lkp->lk_exclusivecount = 0; 684 lkp->lk_recurselevel = 0; 685 lkp->lk_flags &= ~LK_HAVE_EXCL; 686 SETHOLDER(lkp, LK_NOPROC, 0, LK_NOCPU); 687 #if defined(LOCKDEBUG) 688 lkp->lk_unlock_file = file; 689 lkp->lk_unlock_line = line; 690 #endif 691 DONTHAVEIT(lkp); 692 WAKEUP_WAITER(lkp); 693 break; 694 695 case LK_EXCLUPGRADE: 696 /* 697 * If another process is ahead of us to get an upgrade, 698 * then we want to fail rather than have an intervening 699 * exclusive access. 700 */ 701 if (lkp->lk_flags & LK_WANT_UPGRADE) { 702 lkp->lk_sharecount--; 703 if (lkp->lk_sharecount == 0) 704 lkp->lk_flags &= ~LK_SHARE_NONZERO; 705 COUNT(lkp, l, cpu_num, -1); 706 error = EBUSY; 707 break; 708 } 709 /* fall into normal upgrade */ 710 711 case LK_UPGRADE: 712 /* 713 * Upgrade a shared lock to an exclusive one. If another 714 * shared lock has already requested an upgrade to an 715 * exclusive lock, our shared lock is released and an 716 * exclusive lock is requested (which will be granted 717 * after the upgrade). If we return an error, the file 718 * will always be unlocked. 719 */ 720 if (WEHOLDIT(lkp, pid, lid, cpu_num) || lkp->lk_sharecount <= 0) 721 lockpanic(lkp, "lockmgr: upgrade exclusive lock"); 722 lkp->lk_sharecount--; 723 if (lkp->lk_sharecount == 0) 724 lkp->lk_flags &= ~LK_SHARE_NONZERO; 725 COUNT(lkp, l, cpu_num, -1); 726 /* 727 * If we are just polling, check to see if we will block. 728 */ 729 if ((extflags & LK_NOWAIT) && 730 ((lkp->lk_flags & LK_WANT_UPGRADE) || 731 lkp->lk_sharecount > 1)) { 732 error = EBUSY; 733 break; 734 } 735 if ((lkp->lk_flags & LK_WANT_UPGRADE) == 0) { 736 /* 737 * We are first shared lock to request an upgrade, so 738 * request upgrade and wait for the shared count to 739 * drop to zero, then take exclusive lock. 740 */ 741 lkp->lk_flags |= LK_WANT_UPGRADE; 742 error = acquire(&lkp, &s, extflags, 0, LK_SHARE_NONZERO, 743 RETURN_ADDRESS); 744 lkp->lk_flags &= ~LK_WANT_UPGRADE; 745 if (error) { 746 WAKEUP_WAITER(lkp); 747 break; 748 } 749 lkp->lk_flags |= LK_HAVE_EXCL; 750 SETHOLDER(lkp, pid, lid, cpu_num); 751 #if defined(LOCKDEBUG) 752 lkp->lk_lock_file = file; 753 lkp->lk_lock_line = line; 754 #endif 755 HAVEIT(lkp); 756 if (lkp->lk_exclusivecount != 0) 757 lockpanic(lkp, "lockmgr: non-zero exclusive count"); 758 lkp->lk_exclusivecount = 1; 759 if (extflags & LK_SETRECURSE) 760 lkp->lk_recurselevel = 1; 761 COUNT(lkp, l, cpu_num, 1); 762 break; 763 } 764 /* 765 * Someone else has requested upgrade. Release our shared 766 * lock, awaken upgrade requestor if we are the last shared 767 * lock, then request an exclusive lock. 768 */ 769 if (lkp->lk_sharecount == 0) 770 WAKEUP_WAITER(lkp); 771 /* fall into exclusive request */ 772 773 case LK_EXCLUSIVE: 774 if (WEHOLDIT(lkp, pid, lid, cpu_num)) { 775 /* 776 * Recursive lock. 777 */ 778 if ((extflags & LK_CANRECURSE) == 0 && 779 lkp->lk_recurselevel == 0) { 780 if (extflags & LK_RECURSEFAIL) { 781 error = EDEADLK; 782 break; 783 } else 784 lockpanic(lkp, "lockmgr: locking against myself"); 785 } 786 lkp->lk_exclusivecount++; 787 if (extflags & LK_SETRECURSE && 788 lkp->lk_recurselevel == 0) 789 lkp->lk_recurselevel = lkp->lk_exclusivecount; 790 COUNT(lkp, l, cpu_num, 1); 791 break; 792 } 793 /* 794 * If we are just polling, check to see if we will sleep. 795 */ 796 if ((extflags & LK_NOWAIT) && (lkp->lk_flags & 797 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE | 798 LK_SHARE_NONZERO))) { 799 error = EBUSY; 800 break; 801 } 802 /* 803 * Try to acquire the want_exclusive flag. 804 */ 805 error = acquire(&lkp, &s, extflags, 0, 806 LK_HAVE_EXCL | LK_WANT_EXCL, RETURN_ADDRESS); 807 if (error) 808 break; 809 lkp->lk_flags |= LK_WANT_EXCL; 810 /* 811 * Wait for shared locks and upgrades to finish. 812 */ 813 error = acquire(&lkp, &s, extflags, 0, 814 LK_HAVE_EXCL | LK_WANT_UPGRADE | LK_SHARE_NONZERO, 815 RETURN_ADDRESS); 816 lkp->lk_flags &= ~LK_WANT_EXCL; 817 if (error) { 818 WAKEUP_WAITER(lkp); 819 break; 820 } 821 lkp->lk_flags |= LK_HAVE_EXCL; 822 SETHOLDER(lkp, pid, lid, cpu_num); 823 #if defined(LOCKDEBUG) 824 lkp->lk_lock_file = file; 825 lkp->lk_lock_line = line; 826 #endif 827 HAVEIT(lkp); 828 if (lkp->lk_exclusivecount != 0) 829 lockpanic(lkp, "lockmgr: non-zero exclusive count"); 830 lkp->lk_exclusivecount = 1; 831 if (extflags & LK_SETRECURSE) 832 lkp->lk_recurselevel = 1; 833 COUNT(lkp, l, cpu_num, 1); 834 break; 835 836 case LK_RELEASE: 837 if (lkp->lk_exclusivecount != 0) { 838 if (WEHOLDIT(lkp, pid, lid, cpu_num) == 0) { 839 if (lkp->lk_flags & LK_SPIN) { 840 lockpanic(lkp, 841 "lockmgr: processor %lu, not " 842 "exclusive lock holder %lu " 843 "unlocking", cpu_num, lkp->lk_cpu); 844 } else { 845 lockpanic(lkp, "lockmgr: pid %d.%d, not " 846 "exclusive lock holder %d.%d " 847 "unlocking", pid, lid, 848 lkp->lk_lockholder, 849 lkp->lk_locklwp); 850 } 851 } 852 if (lkp->lk_exclusivecount == lkp->lk_recurselevel) 853 lkp->lk_recurselevel = 0; 854 lkp->lk_exclusivecount--; 855 COUNT(lkp, l, cpu_num, -1); 856 if (lkp->lk_exclusivecount == 0) { 857 lkp->lk_flags &= ~LK_HAVE_EXCL; 858 SETHOLDER(lkp, LK_NOPROC, 0, LK_NOCPU); 859 #if defined(LOCKDEBUG) 860 lkp->lk_unlock_file = file; 861 lkp->lk_unlock_line = line; 862 #endif 863 DONTHAVEIT(lkp); 864 } 865 } else if (lkp->lk_sharecount != 0) { 866 lkp->lk_sharecount--; 867 if (lkp->lk_sharecount == 0) 868 lkp->lk_flags &= ~LK_SHARE_NONZERO; 869 COUNT(lkp, l, cpu_num, -1); 870 } 871 #ifdef DIAGNOSTIC 872 else 873 lockpanic(lkp, "lockmgr: release of unlocked lock!"); 874 #endif 875 WAKEUP_WAITER(lkp); 876 break; 877 878 case LK_DRAIN: 879 /* 880 * Check that we do not already hold the lock, as it can 881 * never drain if we do. Unfortunately, we have no way to 882 * check for holding a shared lock, but at least we can 883 * check for an exclusive one. 884 */ 885 if (WEHOLDIT(lkp, pid, lid, cpu_num)) 886 lockpanic(lkp, "lockmgr: draining against myself"); 887 /* 888 * If we are just polling, check to see if we will sleep. 889 */ 890 if ((extflags & LK_NOWAIT) && (lkp->lk_flags & 891 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE | 892 LK_SHARE_NONZERO | LK_WAIT_NONZERO))) { 893 error = EBUSY; 894 break; 895 } 896 error = acquire(&lkp, &s, extflags, 1, 897 LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE | 898 LK_SHARE_NONZERO | LK_WAIT_NONZERO, 899 RETURN_ADDRESS); 900 if (error) 901 break; 902 lkp->lk_flags |= LK_HAVE_EXCL; 903 if ((extflags & LK_RESURRECT) == 0) 904 lkp->lk_flags |= LK_DRAINING; 905 SETHOLDER(lkp, pid, lid, cpu_num); 906 #if defined(LOCKDEBUG) 907 lkp->lk_lock_file = file; 908 lkp->lk_lock_line = line; 909 #endif 910 HAVEIT(lkp); 911 lkp->lk_exclusivecount = 1; 912 /* XXX unlikely that we'd want this */ 913 if (extflags & LK_SETRECURSE) 914 lkp->lk_recurselevel = 1; 915 COUNT(lkp, l, cpu_num, 1); 916 break; 917 918 default: 919 INTERLOCK_RELEASE(lkp, lkp->lk_flags, s); 920 lockpanic(lkp, "lockmgr: unknown locktype request %d", 921 flags & LK_TYPE_MASK); 922 /* NOTREACHED */ 923 } 924 if ((lkp->lk_flags & (LK_WAITDRAIN|LK_SPIN)) == LK_WAITDRAIN && 925 ((lkp->lk_flags & 926 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE | 927 LK_SHARE_NONZERO | LK_WAIT_NONZERO)) == 0)) { 928 lkp->lk_flags &= ~LK_WAITDRAIN; 929 wakeup(&lkp->lk_flags); 930 } 931 /* 932 * Note that this panic will be a recursive panic, since 933 * we only set lock_shutdown_noblock above if panicstr != NULL. 934 */ 935 if (error && lock_shutdown_noblock) 936 lockpanic(lkp, "lockmgr: deadlock (see previous panic)"); 937 938 INTERLOCK_RELEASE(lkp, lkp->lk_flags, s); 939 return (error); 940 } 941 942 /* 943 * For a recursive spinlock held one or more times by the current CPU, 944 * release all N locks, and return N. 945 * Intended for use in mi_switch() shortly before context switching. 946 */ 947 948 int 949 #if defined(LOCKDEBUG) 950 _spinlock_release_all(volatile struct lock *lkp, const char *file, int line) 951 #else 952 spinlock_release_all(volatile struct lock *lkp) 953 #endif 954 { 955 int s, count; 956 cpuid_t cpu_num; 957 958 KASSERT(lkp->lk_flags & LK_SPIN); 959 960 INTERLOCK_ACQUIRE(lkp, LK_SPIN, s); 961 962 cpu_num = cpu_number(); 963 count = lkp->lk_exclusivecount; 964 965 if (count != 0) { 966 #ifdef DIAGNOSTIC 967 if (WEHOLDIT(lkp, 0, 0, cpu_num) == 0) { 968 lockpanic(lkp, "spinlock_release_all: processor %lu, not " 969 "exclusive lock holder %lu " 970 "unlocking", (long)cpu_num, lkp->lk_cpu); 971 } 972 #endif 973 lkp->lk_recurselevel = 0; 974 lkp->lk_exclusivecount = 0; 975 COUNT_CPU(cpu_num, -count); 976 lkp->lk_flags &= ~LK_HAVE_EXCL; 977 SETHOLDER(lkp, LK_NOPROC, 0, LK_NOCPU); 978 #if defined(LOCKDEBUG) 979 lkp->lk_unlock_file = file; 980 lkp->lk_unlock_line = line; 981 #endif 982 DONTHAVEIT(lkp); 983 } 984 #ifdef DIAGNOSTIC 985 else if (lkp->lk_sharecount != 0) 986 lockpanic(lkp, "spinlock_release_all: release of shared lock!"); 987 else 988 lockpanic(lkp, "spinlock_release_all: release of unlocked lock!"); 989 #endif 990 INTERLOCK_RELEASE(lkp, LK_SPIN, s); 991 992 return (count); 993 } 994 995 /* 996 * For a recursive spinlock held one or more times by the current CPU, 997 * release all N locks, and return N. 998 * Intended for use in mi_switch() right after resuming execution. 999 */ 1000 1001 void 1002 #if defined(LOCKDEBUG) 1003 _spinlock_acquire_count(volatile struct lock *lkp, int count, 1004 const char *file, int line) 1005 #else 1006 spinlock_acquire_count(volatile struct lock *lkp, int count) 1007 #endif 1008 { 1009 int s, error; 1010 cpuid_t cpu_num; 1011 1012 KASSERT(lkp->lk_flags & LK_SPIN); 1013 1014 INTERLOCK_ACQUIRE(lkp, LK_SPIN, s); 1015 1016 cpu_num = cpu_number(); 1017 1018 #ifdef DIAGNOSTIC 1019 if (WEHOLDIT(lkp, LK_NOPROC, 0, cpu_num)) 1020 lockpanic(lkp, "spinlock_acquire_count: processor %lu already holds lock", (long)cpu_num); 1021 #endif 1022 /* 1023 * Try to acquire the want_exclusive flag. 1024 */ 1025 error = acquire(&lkp, &s, LK_SPIN, 0, LK_HAVE_EXCL | LK_WANT_EXCL, 1026 RETURN_ADDRESS); 1027 lkp->lk_flags |= LK_WANT_EXCL; 1028 /* 1029 * Wait for shared locks and upgrades to finish. 1030 */ 1031 error = acquire(&lkp, &s, LK_SPIN, 0, 1032 LK_HAVE_EXCL | LK_SHARE_NONZERO | LK_WANT_UPGRADE, 1033 RETURN_ADDRESS); 1034 lkp->lk_flags &= ~LK_WANT_EXCL; 1035 lkp->lk_flags |= LK_HAVE_EXCL; 1036 SETHOLDER(lkp, LK_NOPROC, 0, cpu_num); 1037 #if defined(LOCKDEBUG) 1038 lkp->lk_lock_file = file; 1039 lkp->lk_lock_line = line; 1040 #endif 1041 HAVEIT(lkp); 1042 if (lkp->lk_exclusivecount != 0) 1043 lockpanic(lkp, "lockmgr: non-zero exclusive count"); 1044 lkp->lk_exclusivecount = count; 1045 lkp->lk_recurselevel = 1; 1046 COUNT_CPU(cpu_num, count); 1047 1048 INTERLOCK_RELEASE(lkp, lkp->lk_flags, s); 1049 } 1050 1051 1052 1053 /* 1054 * Print out information about state of a lock. Used by VOP_PRINT 1055 * routines to display ststus about contained locks. 1056 */ 1057 void 1058 lockmgr_printinfo(volatile struct lock *lkp) 1059 { 1060 1061 if (lkp->lk_sharecount) 1062 printf(" lock type %s: SHARED (count %d)", lkp->lk_wmesg, 1063 lkp->lk_sharecount); 1064 else if (lkp->lk_flags & LK_HAVE_EXCL) { 1065 printf(" lock type %s: EXCL (count %d) by ", 1066 lkp->lk_wmesg, lkp->lk_exclusivecount); 1067 if (lkp->lk_flags & LK_SPIN) 1068 printf("processor %lu", lkp->lk_cpu); 1069 else 1070 printf("pid %d.%d", lkp->lk_lockholder, 1071 lkp->lk_locklwp); 1072 } else 1073 printf(" not locked"); 1074 if ((lkp->lk_flags & LK_SPIN) == 0 && lkp->lk_waitcount > 0) 1075 printf(" with %d pending", lkp->lk_waitcount); 1076 } 1077 1078 #if defined(LOCKDEBUG) /* { */ 1079 _TAILQ_HEAD(, struct simplelock, volatile) simplelock_list = 1080 TAILQ_HEAD_INITIALIZER(simplelock_list); 1081 1082 #if defined(MULTIPROCESSOR) /* { */ 1083 struct simplelock simplelock_list_slock = SIMPLELOCK_INITIALIZER; 1084 1085 #define SLOCK_LIST_LOCK() \ 1086 __cpu_simple_lock(&simplelock_list_slock.lock_data) 1087 1088 #define SLOCK_LIST_UNLOCK() \ 1089 __cpu_simple_unlock(&simplelock_list_slock.lock_data) 1090 1091 #define SLOCK_COUNT(x) \ 1092 curcpu()->ci_simple_locks += (x) 1093 #else 1094 u_long simple_locks; 1095 1096 #define SLOCK_LIST_LOCK() /* nothing */ 1097 1098 #define SLOCK_LIST_UNLOCK() /* nothing */ 1099 1100 #define SLOCK_COUNT(x) simple_locks += (x) 1101 #endif /* MULTIPROCESSOR */ /* } */ 1102 1103 #ifdef MULTIPROCESSOR 1104 #define SLOCK_MP() lock_printf("on CPU %ld\n", \ 1105 (u_long) cpu_number()) 1106 #else 1107 #define SLOCK_MP() /* nothing */ 1108 #endif 1109 1110 #define SLOCK_WHERE(str, alp, id, l) \ 1111 do { \ 1112 lock_printf("\n"); \ 1113 lock_printf(str); \ 1114 lock_printf("lock: %p, currently at: %s:%d\n", (alp), (id), (l)); \ 1115 SLOCK_MP(); \ 1116 if ((alp)->lock_file != NULL) \ 1117 lock_printf("last locked: %s:%d\n", (alp)->lock_file, \ 1118 (alp)->lock_line); \ 1119 if ((alp)->unlock_file != NULL) \ 1120 lock_printf("last unlocked: %s:%d\n", (alp)->unlock_file, \ 1121 (alp)->unlock_line); \ 1122 SLOCK_TRACE() \ 1123 SLOCK_DEBUGGER(); \ 1124 } while (/*CONSTCOND*/0) 1125 1126 /* 1127 * Simple lock functions so that the debugger can see from whence 1128 * they are being called. 1129 */ 1130 void 1131 simple_lock_init(volatile struct simplelock *alp) 1132 { 1133 1134 #if defined(MULTIPROCESSOR) /* { */ 1135 __cpu_simple_lock_init(&alp->lock_data); 1136 #else 1137 __cpu_simple_lock_clear(&alp->lock_data); 1138 #endif /* } */ 1139 alp->lock_file = NULL; 1140 alp->lock_line = 0; 1141 alp->unlock_file = NULL; 1142 alp->unlock_line = 0; 1143 alp->lock_holder = LK_NOCPU; 1144 } 1145 1146 void 1147 _simple_lock(volatile struct simplelock *alp, const char *id, int l) 1148 { 1149 cpuid_t cpu_num = cpu_number(); 1150 int s; 1151 1152 s = splhigh(); 1153 1154 /* 1155 * MULTIPROCESSOR case: This is `safe' since if it's not us, we 1156 * don't take any action, and just fall into the normal spin case. 1157 */ 1158 if (__SIMPLELOCK_LOCKED_P(&alp->lock_data)) { 1159 #if defined(MULTIPROCESSOR) /* { */ 1160 if (alp->lock_holder == cpu_num) { 1161 SLOCK_WHERE("simple_lock: locking against myself\n", 1162 alp, id, l); 1163 goto out; 1164 } 1165 #else 1166 SLOCK_WHERE("simple_lock: lock held\n", alp, id, l); 1167 goto out; 1168 #endif /* MULTIPROCESSOR */ /* } */ 1169 } 1170 1171 #if defined(MULTIPROCESSOR) /* { */ 1172 /* Acquire the lock before modifying any fields. */ 1173 splx(s); 1174 __cpu_simple_lock(&alp->lock_data); 1175 s = splhigh(); 1176 #else 1177 __cpu_simple_lock_set(&alp->lock_data); 1178 #endif /* } */ 1179 1180 if (alp->lock_holder != LK_NOCPU) { 1181 SLOCK_WHERE("simple_lock: uninitialized lock\n", 1182 alp, id, l); 1183 } 1184 alp->lock_file = id; 1185 alp->lock_line = l; 1186 alp->lock_holder = cpu_num; 1187 1188 SLOCK_LIST_LOCK(); 1189 TAILQ_INSERT_TAIL(&simplelock_list, alp, list); 1190 SLOCK_LIST_UNLOCK(); 1191 1192 SLOCK_COUNT(1); 1193 1194 out: 1195 splx(s); 1196 } 1197 1198 int 1199 _simple_lock_held(volatile struct simplelock *alp) 1200 { 1201 #if defined(MULTIPROCESSOR) || defined(DIAGNOSTIC) 1202 cpuid_t cpu_num = cpu_number(); 1203 #endif 1204 int s, locked = 0; 1205 1206 s = splhigh(); 1207 1208 #if defined(MULTIPROCESSOR) 1209 if (__cpu_simple_lock_try(&alp->lock_data) == 0) 1210 locked = (alp->lock_holder == cpu_num); 1211 else 1212 __cpu_simple_unlock(&alp->lock_data); 1213 #else 1214 if (__SIMPLELOCK_LOCKED_P(&alp->lock_data)) { 1215 locked = 1; 1216 KASSERT(alp->lock_holder == cpu_num); 1217 } 1218 #endif 1219 1220 splx(s); 1221 1222 return (locked); 1223 } 1224 1225 int 1226 _simple_lock_try(volatile struct simplelock *alp, const char *id, int l) 1227 { 1228 cpuid_t cpu_num = cpu_number(); 1229 int s, rv = 0; 1230 1231 s = splhigh(); 1232 1233 /* 1234 * MULTIPROCESSOR case: This is `safe' since if it's not us, we 1235 * don't take any action. 1236 */ 1237 #if defined(MULTIPROCESSOR) /* { */ 1238 if ((rv = __cpu_simple_lock_try(&alp->lock_data)) == 0) { 1239 if (alp->lock_holder == cpu_num) 1240 SLOCK_WHERE("simple_lock_try: locking against myself\n", 1241 alp, id, l); 1242 goto out; 1243 } 1244 #else 1245 if (__SIMPLELOCK_LOCKED_P(&alp->lock_data)) { 1246 SLOCK_WHERE("simple_lock_try: lock held\n", alp, id, l); 1247 goto out; 1248 } 1249 __cpu_simple_lock_set(&alp->lock_data); 1250 #endif /* MULTIPROCESSOR */ /* } */ 1251 1252 /* 1253 * At this point, we have acquired the lock. 1254 */ 1255 1256 rv = 1; 1257 1258 alp->lock_file = id; 1259 alp->lock_line = l; 1260 alp->lock_holder = cpu_num; 1261 1262 SLOCK_LIST_LOCK(); 1263 TAILQ_INSERT_TAIL(&simplelock_list, alp, list); 1264 SLOCK_LIST_UNLOCK(); 1265 1266 SLOCK_COUNT(1); 1267 1268 out: 1269 splx(s); 1270 return (rv); 1271 } 1272 1273 void 1274 _simple_unlock(volatile struct simplelock *alp, const char *id, int l) 1275 { 1276 int s; 1277 1278 s = splhigh(); 1279 1280 /* 1281 * MULTIPROCESSOR case: This is `safe' because we think we hold 1282 * the lock, and if we don't, we don't take any action. 1283 */ 1284 if (__SIMPLELOCK_UNLOCKED_P(&alp->lock_data)) { 1285 SLOCK_WHERE("simple_unlock: lock not held\n", 1286 alp, id, l); 1287 goto out; 1288 } 1289 1290 SLOCK_LIST_LOCK(); 1291 TAILQ_REMOVE(&simplelock_list, alp, list); 1292 SLOCK_LIST_UNLOCK(); 1293 1294 SLOCK_COUNT(-1); 1295 1296 alp->list.tqe_next = NULL; /* sanity */ 1297 alp->list.tqe_prev = NULL; /* sanity */ 1298 1299 alp->unlock_file = id; 1300 alp->unlock_line = l; 1301 1302 #if defined(MULTIPROCESSOR) /* { */ 1303 alp->lock_holder = LK_NOCPU; 1304 /* Now that we've modified all fields, release the lock. */ 1305 __cpu_simple_unlock(&alp->lock_data); 1306 #else 1307 __cpu_simple_lock_clear(&alp->lock_data); 1308 KASSERT(alp->lock_holder == cpu_number()); 1309 alp->lock_holder = LK_NOCPU; 1310 #endif /* } */ 1311 1312 out: 1313 splx(s); 1314 } 1315 1316 void 1317 simple_lock_dump(void) 1318 { 1319 volatile struct simplelock *alp; 1320 int s; 1321 1322 s = splhigh(); 1323 SLOCK_LIST_LOCK(); 1324 lock_printf("all simple locks:\n"); 1325 TAILQ_FOREACH(alp, &simplelock_list, list) { 1326 lock_printf("%p CPU %lu %s:%d\n", alp, alp->lock_holder, 1327 alp->lock_file, alp->lock_line); 1328 } 1329 SLOCK_LIST_UNLOCK(); 1330 splx(s); 1331 } 1332 1333 void 1334 simple_lock_freecheck(void *start, void *end) 1335 { 1336 volatile struct simplelock *alp; 1337 int s; 1338 1339 s = splhigh(); 1340 SLOCK_LIST_LOCK(); 1341 TAILQ_FOREACH(alp, &simplelock_list, list) { 1342 if ((volatile void *)alp >= start && 1343 (volatile void *)alp < end) { 1344 lock_printf("freeing simple_lock %p CPU %lu %s:%d\n", 1345 alp, alp->lock_holder, alp->lock_file, 1346 alp->lock_line); 1347 SLOCK_DEBUGGER(); 1348 } 1349 } 1350 SLOCK_LIST_UNLOCK(); 1351 splx(s); 1352 } 1353 1354 /* 1355 * We must be holding exactly one lock: the spc_lock. 1356 */ 1357 1358 void 1359 simple_lock_switchcheck(void) 1360 { 1361 1362 simple_lock_only_held(NULL, "switching"); 1363 } 1364 1365 /* 1366 * Drop into the debugger if lp isn't the only lock held. 1367 * lp may be NULL. 1368 */ 1369 void 1370 simple_lock_only_held(volatile struct simplelock *lp, const char *where) 1371 { 1372 volatile struct simplelock *alp; 1373 cpuid_t cpu_num = cpu_number(); 1374 int s; 1375 1376 if (lp) { 1377 LOCK_ASSERT(simple_lock_held(lp)); 1378 } 1379 s = splhigh(); 1380 SLOCK_LIST_LOCK(); 1381 TAILQ_FOREACH(alp, &simplelock_list, list) { 1382 if (alp == lp) 1383 continue; 1384 if (alp->lock_holder == cpu_num) 1385 break; 1386 } 1387 SLOCK_LIST_UNLOCK(); 1388 splx(s); 1389 1390 if (alp != NULL) { 1391 lock_printf("\n%s with held simple_lock %p " 1392 "CPU %lu %s:%d\n", 1393 where, alp, alp->lock_holder, alp->lock_file, 1394 alp->lock_line); 1395 SLOCK_TRACE(); 1396 SLOCK_DEBUGGER(); 1397 } 1398 } 1399 1400 /* 1401 * Set to 1 by simple_lock_assert_*(). 1402 * Can be cleared from ddb to avoid a panic. 1403 */ 1404 int slock_assert_will_panic; 1405 1406 /* 1407 * If the lock isn't held, print a traceback, optionally drop into the 1408 * debugger, then panic. 1409 * The panic can be avoided by clearing slock_assert_with_panic from the 1410 * debugger. 1411 */ 1412 void 1413 _simple_lock_assert_locked(volatile struct simplelock *alp, 1414 const char *lockname, const char *id, int l) 1415 { 1416 if (simple_lock_held(alp) == 0) { 1417 slock_assert_will_panic = 1; 1418 lock_printf("%s lock not held\n", lockname); 1419 SLOCK_WHERE("lock not held", alp, id, l); 1420 if (slock_assert_will_panic && panicstr == NULL) 1421 panic("%s: not locked", lockname); 1422 } 1423 } 1424 1425 void 1426 _simple_lock_assert_unlocked(volatile struct simplelock *alp, 1427 const char *lockname, const char *id, int l) 1428 { 1429 if (simple_lock_held(alp)) { 1430 slock_assert_will_panic = 1; 1431 lock_printf("%s lock held\n", lockname); 1432 SLOCK_WHERE("lock held", alp, id, l); 1433 if (slock_assert_will_panic && panicstr == NULL) 1434 panic("%s: locked", lockname); 1435 } 1436 } 1437 1438 void 1439 assert_sleepable(struct simplelock *interlock, const char *msg) 1440 { 1441 1442 if (panicstr != NULL) 1443 return; 1444 if (CURCPU_IDLE_P()) { 1445 panic("assert_sleepable: idle"); 1446 } 1447 simple_lock_only_held(interlock, msg); 1448 } 1449 1450 #endif /* LOCKDEBUG */ /* } */ 1451 1452 int kernel_lock_id; 1453 __cpu_simple_lock_t kernel_lock; 1454 1455 #if defined(MULTIPROCESSOR) 1456 1457 /* 1458 * Functions for manipulating the kernel_lock. We put them here 1459 * so that they show up in profiles. 1460 */ 1461 1462 #define _KERNEL_LOCK_ABORT(msg) \ 1463 LOCKDEBUG_ABORT(kernel_lock_id, &kernel_lock, &_kernel_lock_ops, \ 1464 __func__, msg) 1465 1466 #ifdef LOCKDEBUG 1467 #define _KERNEL_LOCK_ASSERT(cond) \ 1468 do { \ 1469 if (!(cond)) \ 1470 _KERNEL_LOCK_ABORT("assertion failed: " #cond); \ 1471 } while (/* CONSTCOND */ 0) 1472 #else 1473 #define _KERNEL_LOCK_ASSERT(cond) /* nothing */ 1474 #endif 1475 1476 void _kernel_lock_dump(volatile void *); 1477 1478 lockops_t _kernel_lock_ops = { 1479 "Kernel lock", 1480 0, 1481 _kernel_lock_dump 1482 }; 1483 1484 /* 1485 * Initialize the kernel lock. 1486 */ 1487 void 1488 _kernel_lock_init(void) 1489 { 1490 1491 __cpu_simple_lock_init(&kernel_lock); 1492 kernel_lock_id = LOCKDEBUG_ALLOC(&kernel_lock, &_kernel_lock_ops); 1493 } 1494 1495 /* 1496 * Print debugging information about the kernel lock. 1497 */ 1498 void 1499 _kernel_lock_dump(volatile void *junk) 1500 { 1501 struct cpu_info *ci = curcpu(); 1502 1503 (void)junk; 1504 1505 printf_nolog("curcpu holds : %18d wanted by: %#018lx\n", 1506 ci->ci_biglock_count, (long)ci->ci_biglock_wanted); 1507 } 1508 1509 /* 1510 * Acquire 'nlocks' holds on the kernel lock. If 'l' is non-null, the 1511 * acquisition is from process context. 1512 */ 1513 void 1514 _kernel_lock(int nlocks, struct lwp *l) 1515 { 1516 struct cpu_info *ci = curcpu(); 1517 LOCKSTAT_TIMER(spintime); 1518 LOCKSTAT_FLAG(lsflag); 1519 struct lwp *owant; 1520 #ifdef LOCKDEBUG 1521 u_int spins; 1522 #endif 1523 int s; 1524 1525 (void)l; 1526 1527 if (nlocks == 0) 1528 return; 1529 _KERNEL_LOCK_ASSERT(nlocks > 0); 1530 1531 s = splsched(); /* XXX splvm() */ 1532 1533 if (ci->ci_biglock_count != 0) { 1534 _KERNEL_LOCK_ASSERT(__SIMPLELOCK_LOCKED_P(&kernel_lock)); 1535 ci->ci_biglock_count += nlocks; 1536 splx(s); 1537 return; 1538 } 1539 1540 LOCKDEBUG_WANTLOCK(kernel_lock_id, 1541 (uintptr_t)__builtin_return_address(0), 0); 1542 1543 if (__cpu_simple_lock_try(&kernel_lock)) { 1544 ci->ci_biglock_count = nlocks; 1545 LOCKDEBUG_LOCKED(kernel_lock_id, 1546 (uintptr_t)__builtin_return_address(0), 0); 1547 splx(s); 1548 return; 1549 } 1550 1551 LOCKSTAT_ENTER(lsflag); 1552 LOCKSTAT_START_TIMER(lsflag, spintime); 1553 1554 /* 1555 * Before setting ci_biglock_wanted we must post a store 1556 * fence (see kern_mutex.c). This is accomplished by the 1557 * __cpu_simple_lock_try() above. 1558 */ 1559 owant = ci->ci_biglock_wanted; 1560 ci->ci_biglock_wanted = curlwp; /* XXXAD */ 1561 1562 #ifdef LOCKDEBUG 1563 spins = 0; 1564 #endif 1565 1566 do { 1567 while (__SIMPLELOCK_LOCKED_P(&kernel_lock)) { 1568 #ifdef LOCKDEBUG 1569 if (SPINLOCK_SPINOUT(spins)) 1570 _KERNEL_LOCK_ABORT("spinout"); 1571 #endif 1572 splx(s); 1573 SPINLOCK_SPIN_HOOK; 1574 (void)splsched(); /* XXX splvm() */ 1575 } 1576 } while (!__cpu_simple_lock_try(&kernel_lock)); 1577 1578 ci->ci_biglock_wanted = owant; 1579 ci->ci_biglock_count += nlocks; 1580 LOCKSTAT_STOP_TIMER(lsflag, spintime); 1581 LOCKDEBUG_LOCKED(kernel_lock_id, 1582 (uintptr_t)__builtin_return_address(0), 0); 1583 splx(s); 1584 1585 /* 1586 * Again, another store fence is required (see kern_mutex.c). 1587 */ 1588 mb_write(); 1589 if (owant == NULL) { 1590 LOCKSTAT_EVENT(lsflag, &kernel_lock, LB_KERNEL_LOCK | LB_SPIN, 1591 1, spintime); 1592 } 1593 LOCKSTAT_EXIT(lsflag); 1594 } 1595 1596 /* 1597 * Release 'nlocks' holds on the kernel lock. If 'nlocks' is zero, release 1598 * all holds. If 'l' is non-null, the release is from process context. 1599 */ 1600 void 1601 _kernel_unlock(int nlocks, struct lwp *l, int *countp) 1602 { 1603 struct cpu_info *ci = curcpu(); 1604 u_int olocks; 1605 int s; 1606 1607 (void)l; 1608 1609 _KERNEL_LOCK_ASSERT(nlocks < 2); 1610 1611 olocks = ci->ci_biglock_count; 1612 1613 if (olocks == 0) { 1614 _KERNEL_LOCK_ASSERT(nlocks <= 0); 1615 if (countp != NULL) 1616 *countp = 0; 1617 return; 1618 } 1619 1620 _KERNEL_LOCK_ASSERT(__SIMPLELOCK_LOCKED_P(&kernel_lock)); 1621 1622 if (nlocks == 0) 1623 nlocks = olocks; 1624 else if (nlocks == -1) { 1625 nlocks = 1; 1626 _KERNEL_LOCK_ASSERT(olocks == 1); 1627 } 1628 1629 s = splsched(); /* XXX splvm() */ 1630 if ((ci->ci_biglock_count -= nlocks) == 0) { 1631 LOCKDEBUG_UNLOCKED(kernel_lock_id, 1632 (uintptr_t)__builtin_return_address(0), 0); 1633 __cpu_simple_unlock(&kernel_lock); 1634 } 1635 splx(s); 1636 1637 if (countp != NULL) 1638 *countp = olocks; 1639 } 1640 1641 #if defined(DEBUG) 1642 /* 1643 * Assert that the kernel lock is held. 1644 */ 1645 void 1646 _kernel_lock_assert_locked(void) 1647 { 1648 1649 if (!__SIMPLELOCK_LOCKED_P(&kernel_lock) || 1650 curcpu()->ci_biglock_count == 0) 1651 _KERNEL_LOCK_ABORT("not locked"); 1652 } 1653 1654 void 1655 _kernel_lock_assert_unlocked() 1656 { 1657 1658 if (curcpu()->ci_biglock_count != 0) 1659 _KERNEL_LOCK_ABORT("locked"); 1660 } 1661 #endif 1662 1663 #endif /* MULTIPROCESSOR || LOCKDEBUG */ 1664
Indexes created Sun Sep 28 07:09:56 GMT 2025