kern_lock.c revision 1.110.6.1
1 /* $NetBSD: kern_lock.c,v 1.110.6.1 2007/12/09 16:04:00 reinoud Exp $ */ 2 3 /*- 4 * Copyright (c) 1999, 2000, 2006 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center, and by Andrew Doran. 10 * 11 * This code is derived from software contributed to The NetBSD Foundation 12 * by Ross Harvey. 13 * 14 * Redistribution and use in source and binary forms, with or without 15 * modification, are permitted provided that the following conditions 16 * are met: 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in the 21 * documentation and/or other materials provided with the distribution. 22 * 3. All advertising materials mentioning features or use of this software 23 * must display the following acknowledgement: 24 * This product includes software developed by the NetBSD 25 * Foundation, Inc. and its contributors. 26 * 4. Neither the name of The NetBSD Foundation nor the names of its 27 * contributors may be used to endorse or promote products derived 28 * from this software without specific prior written permission. 29 * 30 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 31 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 32 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 33 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 34 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 35 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 36 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 37 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 38 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 39 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 40 * POSSIBILITY OF SUCH DAMAGE. 41 */ 42 43 /* 44 * Copyright (c) 1995 45 * The Regents of the University of California. All rights reserved. 46 * 47 * This code contains ideas from software contributed to Berkeley by 48 * Avadis Tevanian, Jr., Michael Wayne Young, and the Mach Operating 49 * System project at Carnegie-Mellon University. 50 * 51 * Redistribution and use in source and binary forms, with or without 52 * modification, are permitted provided that the following conditions 53 * are met: 54 * 1. Redistributions of source code must retain the above copyright 55 * notice, this list of conditions and the following disclaimer. 56 * 2. Redistributions in binary form must reproduce the above copyright 57 * notice, this list of conditions and the following disclaimer in the 58 * documentation and/or other materials provided with the distribution. 59 * 3. Neither the name of the University nor the names of its contributors 60 * may be used to endorse or promote products derived from this software 61 * without specific prior written permission. 62 * 63 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 64 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 65 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 66 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 67 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 68 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 69 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 70 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 71 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 72 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 73 * SUCH DAMAGE. 74 * 75 * @(#)kern_lock.c 8.18 (Berkeley) 5/21/95 76 */ 77 78 #include <sys/cdefs.h> 79 __KERNEL_RCSID(0, "$NetBSD: kern_lock.c,v 1.110.6.1 2007/12/09 16:04:00 reinoud Exp $"); 80 81 #include "opt_multiprocessor.h" 82 #include "opt_ddb.h" 83 84 #define __MUTEX_PRIVATE 85 86 #include <sys/param.h> 87 #include <sys/proc.h> 88 #include <sys/lock.h> 89 #include <sys/systm.h> 90 #include <sys/lockdebug.h> 91 92 #include <machine/cpu.h> 93 #include <machine/stdarg.h> 94 95 #include <dev/lockstat.h> 96 97 #if defined(LOCKDEBUG) 98 #include <sys/syslog.h> 99 /* 100 * note that stdarg.h and the ansi style va_start macro is used for both 101 * ansi and traditional c compiles. 102 * XXX: this requires that stdarg.h define: va_alist and va_dcl 103 */ 104 #include <machine/stdarg.h> 105 106 void lock_printf(const char *fmt, ...) 107 __attribute__((__format__(__printf__,1,2))); 108 109 static int acquire(volatile struct lock **, int *, int, int, int, uintptr_t); 110 111 int lock_debug_syslog = 0; /* defaults to printf, but can be patched */ 112 113 #ifdef DDB 114 #include <ddb/ddbvar.h> 115 #include <machine/db_machdep.h> 116 #include <ddb/db_command.h> 117 #include <ddb/db_interface.h> 118 #endif 119 #endif /* defined(LOCKDEBUG) */ 120 121 #if defined(MULTIPROCESSOR) 122 /* 123 * IPL_BIGLOCK: block IPLs which need to grab kernel_mutex. 124 * XXX IPL_VM or IPL_AUDIO should be enough. 125 */ 126 #if !defined(__HAVE_SPLBIGLOCK) 127 #define splbiglock splclock 128 #endif 129 __cpu_simple_lock_t kernel_lock; 130 int kernel_lock_id; 131 #endif 132 133 /* 134 * Locking primitives implementation. 135 * Locks provide shared/exclusive synchronization. 136 */ 137 138 #if defined(LOCKDEBUG) || defined(DIAGNOSTIC) /* { */ 139 #if defined(MULTIPROCESSOR) /* { */ 140 #define COUNT_CPU(cpu_id, x) \ 141 curcpu()->ci_spin_locks += (x) 142 #else 143 u_long spin_locks; 144 #define COUNT_CPU(cpu_id, x) spin_locks += (x) 145 #endif /* MULTIPROCESSOR */ /* } */ 146 147 #define COUNT(lkp, l, cpu_id, x) \ 148 do { \ 149 if ((lkp)->lk_flags & LK_SPIN) \ 150 COUNT_CPU((cpu_id), (x)); \ 151 else \ 152 (l)->l_locks += (x); \ 153 } while (/*CONSTCOND*/0) 154 #else 155 #define COUNT(lkp, p, cpu_id, x) 156 #define COUNT_CPU(cpu_id, x) 157 #endif /* LOCKDEBUG || DIAGNOSTIC */ /* } */ 158 159 #define INTERLOCK_ACQUIRE(lkp, flags, s) \ 160 do { \ 161 if ((flags) & LK_SPIN) \ 162 s = spllock(); \ 163 simple_lock(&(lkp)->lk_interlock); \ 164 } while (/*CONSTCOND*/ 0) 165 166 #define INTERLOCK_RELEASE(lkp, flags, s) \ 167 do { \ 168 simple_unlock(&(lkp)->lk_interlock); \ 169 if ((flags) & LK_SPIN) \ 170 splx(s); \ 171 } while (/*CONSTCOND*/ 0) 172 173 #ifdef DDB /* { */ 174 #if defined(MULTIPROCESSOR) || defined(LOCKDEBUG) 175 int simple_lock_debugger = 1; /* more serious on MP */ 176 #else 177 int simple_lock_debugger = 0; 178 #endif 179 #define SLOCK_DEBUGGER() if (simple_lock_debugger && db_onpanic) Debugger() 180 #define SLOCK_TRACE() \ 181 db_stack_trace_print((db_expr_t)__builtin_frame_address(0), \ 182 true, 65535, "", lock_printf); 183 #else 184 #define SLOCK_DEBUGGER() /* nothing */ 185 #define SLOCK_TRACE() /* nothing */ 186 #endif /* } */ 187 188 #if defined(LOCKDEBUG) 189 #if defined(DDB) 190 #define SPINLOCK_SPINCHECK_DEBUGGER if (db_onpanic) Debugger() 191 #else 192 #define SPINLOCK_SPINCHECK_DEBUGGER /* nothing */ 193 #endif 194 195 #define SPINLOCK_SPINCHECK_DECL \ 196 /* 32-bits of count -- wrap constitutes a "spinout" */ \ 197 uint32_t __spinc = 0 198 199 #define SPINLOCK_SPINCHECK \ 200 do { \ 201 if (++__spinc == 0) { \ 202 lock_printf("LK_SPIN spinout, excl %d, share %d\n", \ 203 lkp->lk_exclusivecount, lkp->lk_sharecount); \ 204 if (lkp->lk_exclusivecount) \ 205 lock_printf("held by CPU %lu\n", \ 206 (u_long) lkp->lk_cpu); \ 207 if (lkp->lk_lock_file) \ 208 lock_printf("last locked at %s:%d\n", \ 209 lkp->lk_lock_file, lkp->lk_lock_line); \ 210 if (lkp->lk_unlock_file) \ 211 lock_printf("last unlocked at %s:%d\n", \ 212 lkp->lk_unlock_file, lkp->lk_unlock_line); \ 213 SLOCK_TRACE(); \ 214 SPINLOCK_SPINCHECK_DEBUGGER; \ 215 } \ 216 } while (/*CONSTCOND*/ 0) 217 #else 218 #define SPINLOCK_SPINCHECK_DECL /* nothing */ 219 #define SPINLOCK_SPINCHECK /* nothing */ 220 #endif /* LOCKDEBUG && DDB */ 221 222 #define RETURN_ADDRESS ((uintptr_t)__builtin_return_address(0)) 223 224 /* 225 * Acquire a resource. 226 */ 227 static int 228 acquire(volatile struct lock **lkpp, int *s, int extflags, 229 int drain, int wanted, uintptr_t ra) 230 { 231 int error; 232 volatile struct lock *lkp = *lkpp; 233 LOCKSTAT_TIMER(slptime); 234 LOCKSTAT_FLAG(lsflag); 235 236 KASSERT(drain || (wanted & LK_WAIT_NONZERO) == 0); 237 238 if (extflags & LK_SPIN) { 239 int interlocked; 240 241 SPINLOCK_SPINCHECK_DECL; 242 243 if (!drain) { 244 lkp->lk_waitcount++; 245 lkp->lk_flags |= LK_WAIT_NONZERO; 246 } 247 for (interlocked = 1;;) { 248 SPINLOCK_SPINCHECK; 249 if ((lkp->lk_flags & wanted) != 0) { 250 if (interlocked) { 251 INTERLOCK_RELEASE(lkp, LK_SPIN, *s); 252 interlocked = 0; 253 } 254 SPINLOCK_SPIN_HOOK; 255 } else if (interlocked) { 256 break; 257 } else { 258 INTERLOCK_ACQUIRE(lkp, LK_SPIN, *s); 259 interlocked = 1; 260 } 261 } 262 if (!drain) { 263 lkp->lk_waitcount--; 264 if (lkp->lk_waitcount == 0) 265 lkp->lk_flags &= ~LK_WAIT_NONZERO; 266 } 267 KASSERT((lkp->lk_flags & wanted) == 0); 268 error = 0; /* sanity */ 269 } else { 270 LOCKSTAT_ENTER(lsflag); 271 272 for (error = 0; (lkp->lk_flags & wanted) != 0; ) { 273 if (drain) 274 lkp->lk_flags |= LK_WAITDRAIN; 275 else { 276 lkp->lk_waitcount++; 277 lkp->lk_flags |= LK_WAIT_NONZERO; 278 } 279 /* XXX Cast away volatile. */ 280 LOCKSTAT_START_TIMER(lsflag, slptime); 281 error = ltsleep(drain ? 282 (volatile const void *)&lkp->lk_flags : 283 (volatile const void *)lkp, lkp->lk_prio, 284 lkp->lk_wmesg, lkp->lk_timo, &lkp->lk_interlock); 285 LOCKSTAT_STOP_TIMER(lsflag, slptime); 286 LOCKSTAT_EVENT_RA(lsflag, (void *)(uintptr_t)lkp, 287 LB_LOCKMGR | LB_SLEEP1, 1, slptime, ra); 288 if (!drain) { 289 lkp->lk_waitcount--; 290 if (lkp->lk_waitcount == 0) 291 lkp->lk_flags &= ~LK_WAIT_NONZERO; 292 } 293 if (error) 294 break; 295 if (extflags & LK_SLEEPFAIL) { 296 error = ENOLCK; 297 break; 298 } 299 if (lkp->lk_newlock != NULL) { 300 simple_lock(&lkp->lk_newlock->lk_interlock); 301 simple_unlock(&lkp->lk_interlock); 302 if (lkp->lk_waitcount == 0) 303 wakeup(&lkp->lk_newlock); 304 *lkpp = lkp = lkp->lk_newlock; 305 } 306 } 307 308 LOCKSTAT_EXIT(lsflag); 309 } 310 311 return error; 312 } 313 314 #define SETHOLDER(lkp, pid, lid, cpu_id) \ 315 do { \ 316 if ((lkp)->lk_flags & LK_SPIN) \ 317 (lkp)->lk_cpu = cpu_id; \ 318 else { \ 319 (lkp)->lk_lockholder = pid; \ 320 (lkp)->lk_locklwp = lid; \ 321 } \ 322 } while (/*CONSTCOND*/0) 323 324 #define WEHOLDIT(lkp, pid, lid, cpu_id) \ 325 (((lkp)->lk_flags & LK_SPIN) != 0 ? \ 326 ((lkp)->lk_cpu == (cpu_id)) : \ 327 ((lkp)->lk_lockholder == (pid) && (lkp)->lk_locklwp == (lid))) 328 329 #define WAKEUP_WAITER(lkp) \ 330 do { \ 331 if (((lkp)->lk_flags & (LK_SPIN | LK_WAIT_NONZERO)) == \ 332 LK_WAIT_NONZERO) { \ 333 wakeup((lkp)); \ 334 } \ 335 } while (/*CONSTCOND*/0) 336 337 #if defined(LOCKDEBUG) /* { */ 338 #if defined(MULTIPROCESSOR) /* { */ 339 struct simplelock spinlock_list_slock = SIMPLELOCK_INITIALIZER; 340 341 #define SPINLOCK_LIST_LOCK() \ 342 __cpu_simple_lock(&spinlock_list_slock.lock_data) 343 344 #define SPINLOCK_LIST_UNLOCK() \ 345 __cpu_simple_unlock(&spinlock_list_slock.lock_data) 346 #else 347 #define SPINLOCK_LIST_LOCK() /* nothing */ 348 349 #define SPINLOCK_LIST_UNLOCK() /* nothing */ 350 #endif /* MULTIPROCESSOR */ /* } */ 351 352 _TAILQ_HEAD(, struct lock, volatile) spinlock_list = 353 TAILQ_HEAD_INITIALIZER(spinlock_list); 354 355 #define HAVEIT(lkp) \ 356 do { \ 357 if ((lkp)->lk_flags & LK_SPIN) { \ 358 int sp = spllock(); \ 359 SPINLOCK_LIST_LOCK(); \ 360 TAILQ_INSERT_TAIL(&spinlock_list, (lkp), lk_list); \ 361 SPINLOCK_LIST_UNLOCK(); \ 362 splx(sp); \ 363 } \ 364 } while (/*CONSTCOND*/0) 365 366 #define DONTHAVEIT(lkp) \ 367 do { \ 368 if ((lkp)->lk_flags & LK_SPIN) { \ 369 int sp = spllock(); \ 370 SPINLOCK_LIST_LOCK(); \ 371 TAILQ_REMOVE(&spinlock_list, (lkp), lk_list); \ 372 SPINLOCK_LIST_UNLOCK(); \ 373 splx(sp); \ 374 } \ 375 } while (/*CONSTCOND*/0) 376 #else 377 #define HAVEIT(lkp) /* nothing */ 378 379 #define DONTHAVEIT(lkp) /* nothing */ 380 #endif /* LOCKDEBUG */ /* } */ 381 382 #if defined(LOCKDEBUG) 383 /* 384 * Lock debug printing routine; can be configured to print to console 385 * or log to syslog. 386 */ 387 void 388 lock_printf(const char *fmt, ...) 389 { 390 char b[150]; 391 va_list ap; 392 393 va_start(ap, fmt); 394 if (lock_debug_syslog) 395 vlog(LOG_DEBUG, fmt, ap); 396 else { 397 vsnprintf(b, sizeof(b), fmt, ap); 398 printf_nolog("%s", b); 399 } 400 va_end(ap); 401 } 402 #endif /* LOCKDEBUG */ 403 404 static void 405 lockpanic(volatile struct lock *lkp, const char *fmt, ...) 406 { 407 char s[150], b[150]; 408 #ifdef LOCKDEBUG 409 static const char *locktype[] = { 410 "*0*", "shared", "exclusive", "*3*", "*4*", "downgrade", 411 "*release*", "drain", "exclother", "*9*", "*10*", 412 "*11*", "*12*", "*13*", "*14*", "*15*" 413 }; 414 #endif 415 416 va_list ap; 417 va_start(ap, fmt); 418 vsnprintf(s, sizeof(s), fmt, ap); 419 va_end(ap); 420 bitmask_snprintf(lkp->lk_flags, __LK_FLAG_BITS, b, sizeof(b)); 421 panic("%s (" 422 #ifdef LOCKDEBUG 423 "type %s " 424 #endif 425 "flags %s, sharecount %d, exclusivecount %d, " 426 "recurselevel %d, waitcount %d, wmesg %s" 427 #ifdef LOCKDEBUG 428 ", lock_file %s, unlock_file %s, lock_line %d, unlock_line %d" 429 #endif 430 ")\n", 431 s, 432 #ifdef LOCKDEBUG 433 locktype[lkp->lk_flags & LK_TYPE_MASK], 434 #endif 435 b, lkp->lk_sharecount, lkp->lk_exclusivecount, 436 lkp->lk_recurselevel, lkp->lk_waitcount, lkp->lk_wmesg 437 #ifdef LOCKDEBUG 438 , lkp->lk_lock_file, lkp->lk_unlock_file, lkp->lk_lock_line, 439 lkp->lk_unlock_line 440 #endif 441 ); 442 } 443 444 /* 445 * Transfer any waiting processes from one lock to another. 446 */ 447 void 448 transferlockers(struct lock *from, struct lock *to) 449 { 450 451 KASSERT(from != to); 452 KASSERT((from->lk_flags & LK_WAITDRAIN) == 0); 453 if (from->lk_waitcount == 0) 454 return; 455 from->lk_newlock = to; 456 wakeup((void *)from); 457 tsleep((void *)&from->lk_newlock, from->lk_prio, "lkxfer", 0); 458 from->lk_newlock = NULL; 459 from->lk_flags &= ~(LK_WANT_EXCL | LK_WANT_UPGRADE); 460 KASSERT(from->lk_waitcount == 0); 461 } 462 463 464 /* 465 * Initialize a lock; required before use. 466 */ 467 void 468 lockinit(struct lock *lkp, pri_t prio, const char *wmesg, int timo, int flags) 469 { 470 471 memset(lkp, 0, sizeof(struct lock)); 472 simple_lock_init(&lkp->lk_interlock); 473 lkp->lk_flags = flags & LK_EXTFLG_MASK; 474 if (flags & LK_SPIN) 475 lkp->lk_cpu = LK_NOCPU; 476 else { 477 lkp->lk_lockholder = LK_NOPROC; 478 lkp->lk_newlock = NULL; 479 lkp->lk_prio = prio; 480 lkp->lk_timo = timo; 481 } 482 lkp->lk_wmesg = wmesg; /* just a name for spin locks */ 483 #if defined(LOCKDEBUG) 484 lkp->lk_lock_file = NULL; 485 lkp->lk_unlock_file = NULL; 486 #endif 487 } 488 489 /* 490 * Determine the status of a lock. 491 */ 492 int 493 lockstatus(struct lock *lkp) 494 { 495 int s = 0; /* XXX: gcc */ 496 int lock_type = 0; 497 struct lwp *l = curlwp; /* XXX */ 498 pid_t pid; 499 lwpid_t lid; 500 cpuid_t cpu_num; 501 502 if ((lkp->lk_flags & LK_SPIN) || l == NULL) { 503 cpu_num = cpu_number(); 504 pid = LK_KERNPROC; 505 lid = 0; 506 } else { 507 cpu_num = LK_NOCPU; 508 pid = l->l_proc->p_pid; 509 lid = l->l_lid; 510 } 511 512 INTERLOCK_ACQUIRE(lkp, lkp->lk_flags, s); 513 if (lkp->lk_exclusivecount != 0) { 514 if (WEHOLDIT(lkp, pid, lid, cpu_num)) 515 lock_type = LK_EXCLUSIVE; 516 else 517 lock_type = LK_EXCLOTHER; 518 } else if (lkp->lk_sharecount != 0) 519 lock_type = LK_SHARED; 520 else if (lkp->lk_flags & LK_WANT_EXCL) 521 lock_type = LK_EXCLOTHER; 522 INTERLOCK_RELEASE(lkp, lkp->lk_flags, s); 523 return (lock_type); 524 } 525 526 #if defined(LOCKDEBUG) 527 /* 528 * Make sure no spin locks are held by a CPU that is about 529 * to context switch. 530 */ 531 void 532 spinlock_switchcheck(void) 533 { 534 u_long cnt; 535 int s; 536 537 s = spllock(); 538 #if defined(MULTIPROCESSOR) 539 cnt = curcpu()->ci_spin_locks; 540 #else 541 cnt = spin_locks; 542 #endif 543 splx(s); 544 545 if (cnt != 0) 546 panic("spinlock_switchcheck: CPU %lu has %lu spin locks", 547 (u_long) cpu_number(), cnt); 548 } 549 #endif /* LOCKDEBUG */ 550 551 /* 552 * Locks and IPLs (interrupt priority levels): 553 * 554 * Locks which may be taken from interrupt context must be handled 555 * very carefully; you must spl to the highest IPL where the lock 556 * is needed before acquiring the lock. 557 * 558 * It is also important to avoid deadlock, since certain (very high 559 * priority) interrupts are often needed to keep the system as a whole 560 * from deadlocking, and must not be blocked while you are spinning 561 * waiting for a lower-priority lock. 562 * 563 * In addition, the lock-debugging hooks themselves need to use locks! 564 * 565 * A raw __cpu_simple_lock may be used from interrupts are long as it 566 * is acquired and held at a single IPL. 567 * 568 * A simple_lock (which is a __cpu_simple_lock wrapped with some 569 * debugging hooks) may be used at or below spllock(), which is 570 * typically at or just below splhigh() (i.e. blocks everything 571 * but certain machine-dependent extremely high priority interrupts). 572 * 573 * spinlockmgr spinlocks should be used at or below splsched(). 574 * 575 * Some platforms may have interrupts of higher priority than splsched(), 576 * including hard serial interrupts, inter-processor interrupts, and 577 * kernel debugger traps. 578 */ 579 580 /* 581 * XXX XXX kludge around another kludge.. 582 * 583 * vfs_shutdown() may be called from interrupt context, either as a result 584 * of a panic, or from the debugger. It proceeds to call 585 * sys_sync(&proc0, ...), pretending its running on behalf of proc0 586 * 587 * We would like to make an attempt to sync the filesystems in this case, so 588 * if this happens, we treat attempts to acquire locks specially. 589 * All locks are acquired on behalf of proc0. 590 * 591 * If we've already paniced, we don't block waiting for locks, but 592 * just barge right ahead since we're already going down in flames. 593 */ 594 595 /* 596 * Set, change, or release a lock. 597 * 598 * Shared requests increment the shared count. Exclusive requests set the 599 * LK_WANT_EXCL flag (preventing further shared locks), and wait for already 600 * accepted shared locks to go away. 601 */ 602 int 603 #if defined(LOCKDEBUG) 604 _lockmgr(volatile struct lock *lkp, u_int flags, 605 struct simplelock *interlkp, const char *file, int line) 606 #else 607 lockmgr(volatile struct lock *lkp, u_int flags, 608 struct simplelock *interlkp) 609 #endif 610 { 611 int error; 612 pid_t pid; 613 lwpid_t lid; 614 int extflags; 615 cpuid_t cpu_num; 616 struct lwp *l = curlwp; 617 int lock_shutdown_noblock = 0; 618 int s = 0; 619 620 error = 0; 621 622 /* LK_RETRY is for vn_lock, not for lockmgr. */ 623 KASSERT((flags & LK_RETRY) == 0); 624 625 INTERLOCK_ACQUIRE(lkp, lkp->lk_flags, s); 626 if (flags & LK_INTERLOCK) 627 simple_unlock(interlkp); 628 extflags = (flags | lkp->lk_flags) & LK_EXTFLG_MASK; 629 630 #ifdef DIAGNOSTIC /* { */ 631 /* 632 * Don't allow spins on sleep locks and don't allow sleeps 633 * on spin locks. 634 */ 635 if ((flags ^ lkp->lk_flags) & LK_SPIN) 636 lockpanic(lkp, "lockmgr: sleep/spin mismatch"); 637 #endif /* } */ 638 639 if (extflags & LK_SPIN) { 640 pid = LK_KERNPROC; 641 lid = 0; 642 } else { 643 if (l == NULL) { 644 if (!doing_shutdown) { 645 panic("lockmgr: no context"); 646 } else { 647 l = &lwp0; 648 if (panicstr && (!(flags & LK_NOWAIT))) { 649 flags |= LK_NOWAIT; 650 lock_shutdown_noblock = 1; 651 } 652 } 653 } 654 lid = l->l_lid; 655 pid = l->l_proc->p_pid; 656 } 657 cpu_num = cpu_number(); 658 659 /* 660 * Once a lock has drained, the LK_DRAINING flag is set and an 661 * exclusive lock is returned. The only valid operation thereafter 662 * is a single release of that exclusive lock. This final release 663 * clears the LK_DRAINING flag and sets the LK_DRAINED flag. Any 664 * further requests of any sort will result in a panic. The bits 665 * selected for these two flags are chosen so that they will be set 666 * in memory that is freed (freed memory is filled with 0xdeadbeef). 667 * The final release is permitted to give a new lease on life to 668 * the lock by specifying LK_REENABLE. 669 */ 670 if (lkp->lk_flags & (LK_DRAINING|LK_DRAINED)) { 671 #ifdef DIAGNOSTIC /* { */ 672 if (lkp->lk_flags & LK_DRAINED) 673 lockpanic(lkp, "lockmgr: using decommissioned lock"); 674 if ((flags & LK_TYPE_MASK) != LK_RELEASE || 675 WEHOLDIT(lkp, pid, lid, cpu_num) == 0) 676 lockpanic(lkp, "lockmgr: non-release on draining lock: %d", 677 flags & LK_TYPE_MASK); 678 #endif /* DIAGNOSTIC */ /* } */ 679 lkp->lk_flags &= ~LK_DRAINING; 680 if ((flags & LK_REENABLE) == 0) 681 lkp->lk_flags |= LK_DRAINED; 682 } 683 684 switch (flags & LK_TYPE_MASK) { 685 686 case LK_SHARED: 687 if (WEHOLDIT(lkp, pid, lid, cpu_num) == 0) { 688 /* 689 * If just polling, check to see if we will block. 690 */ 691 if ((extflags & LK_NOWAIT) && (lkp->lk_flags & 692 (LK_HAVE_EXCL | LK_WANT_EXCL))) { 693 error = EBUSY; 694 break; 695 } 696 /* 697 * Wait for exclusive locks to clear. 698 */ 699 error = acquire(&lkp, &s, extflags, 0, 700 LK_HAVE_EXCL | LK_WANT_EXCL, 701 RETURN_ADDRESS); 702 if (error) 703 break; 704 lkp->lk_sharecount++; 705 lkp->lk_flags |= LK_SHARE_NONZERO; 706 COUNT(lkp, l, cpu_num, 1); 707 break; 708 } 709 /* 710 * We hold an exclusive lock, so downgrade it to shared. 711 * An alternative would be to fail with EDEADLK. 712 */ 713 lkp->lk_sharecount++; 714 lkp->lk_flags |= LK_SHARE_NONZERO; 715 COUNT(lkp, l, cpu_num, 1); 716 /* fall into downgrade */ 717 718 case LK_DOWNGRADE: 719 if (WEHOLDIT(lkp, pid, lid, cpu_num) == 0 || 720 lkp->lk_exclusivecount == 0) 721 lockpanic(lkp, "lockmgr: not holding exclusive lock"); 722 lkp->lk_sharecount += lkp->lk_exclusivecount; 723 lkp->lk_flags |= LK_SHARE_NONZERO; 724 lkp->lk_exclusivecount = 0; 725 lkp->lk_recurselevel = 0; 726 lkp->lk_flags &= ~LK_HAVE_EXCL; 727 SETHOLDER(lkp, LK_NOPROC, 0, LK_NOCPU); 728 #if defined(LOCKDEBUG) 729 lkp->lk_unlock_file = file; 730 lkp->lk_unlock_line = line; 731 #endif 732 DONTHAVEIT(lkp); 733 WAKEUP_WAITER(lkp); 734 break; 735 736 case LK_EXCLUSIVE: 737 if (WEHOLDIT(lkp, pid, lid, cpu_num)) { 738 /* 739 * Recursive lock. 740 */ 741 if ((extflags & LK_CANRECURSE) == 0 && 742 lkp->lk_recurselevel == 0) { 743 if (extflags & LK_RECURSEFAIL) { 744 error = EDEADLK; 745 break; 746 } else 747 lockpanic(lkp, "lockmgr: locking against myself"); 748 } 749 lkp->lk_exclusivecount++; 750 if (extflags & LK_SETRECURSE && 751 lkp->lk_recurselevel == 0) 752 lkp->lk_recurselevel = lkp->lk_exclusivecount; 753 COUNT(lkp, l, cpu_num, 1); 754 break; 755 } 756 /* 757 * If we are just polling, check to see if we will sleep. 758 */ 759 if ((extflags & LK_NOWAIT) && (lkp->lk_flags & 760 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_SHARE_NONZERO))) { 761 error = EBUSY; 762 break; 763 } 764 /* 765 * Try to acquire the want_exclusive flag. 766 */ 767 error = acquire(&lkp, &s, extflags, 0, 768 LK_HAVE_EXCL | LK_WANT_EXCL, RETURN_ADDRESS); 769 if (error) 770 break; 771 lkp->lk_flags |= LK_WANT_EXCL; 772 /* 773 * Wait for shared locks to finish. 774 */ 775 error = acquire(&lkp, &s, extflags, 0, 776 LK_HAVE_EXCL | LK_SHARE_NONZERO, 777 RETURN_ADDRESS); 778 lkp->lk_flags &= ~LK_WANT_EXCL; 779 if (error) { 780 WAKEUP_WAITER(lkp); 781 break; 782 } 783 lkp->lk_flags |= LK_HAVE_EXCL; 784 SETHOLDER(lkp, pid, lid, cpu_num); 785 #if defined(LOCKDEBUG) 786 lkp->lk_lock_file = file; 787 lkp->lk_lock_line = line; 788 #endif 789 HAVEIT(lkp); 790 if (lkp->lk_exclusivecount != 0) 791 lockpanic(lkp, "lockmgr: non-zero exclusive count"); 792 lkp->lk_exclusivecount = 1; 793 if (extflags & LK_SETRECURSE) 794 lkp->lk_recurselevel = 1; 795 COUNT(lkp, l, cpu_num, 1); 796 break; 797 798 case LK_RELEASE: 799 if (lkp->lk_exclusivecount != 0) { 800 if (WEHOLDIT(lkp, pid, lid, cpu_num) == 0) { 801 if (lkp->lk_flags & LK_SPIN) { 802 lockpanic(lkp, 803 "lockmgr: processor %lu, not " 804 "exclusive lock holder %lu " 805 "unlocking", cpu_num, lkp->lk_cpu); 806 } else { 807 lockpanic(lkp, "lockmgr: pid %d, not " 808 "exclusive lock holder %d " 809 "unlocking", pid, 810 lkp->lk_lockholder); 811 } 812 } 813 if (lkp->lk_exclusivecount == lkp->lk_recurselevel) 814 lkp->lk_recurselevel = 0; 815 lkp->lk_exclusivecount--; 816 COUNT(lkp, l, cpu_num, -1); 817 if (lkp->lk_exclusivecount == 0) { 818 lkp->lk_flags &= ~LK_HAVE_EXCL; 819 SETHOLDER(lkp, LK_NOPROC, 0, LK_NOCPU); 820 #if defined(LOCKDEBUG) 821 lkp->lk_unlock_file = file; 822 lkp->lk_unlock_line = line; 823 #endif 824 DONTHAVEIT(lkp); 825 } 826 } else if (lkp->lk_sharecount != 0) { 827 lkp->lk_sharecount--; 828 if (lkp->lk_sharecount == 0) 829 lkp->lk_flags &= ~LK_SHARE_NONZERO; 830 COUNT(lkp, l, cpu_num, -1); 831 } 832 #ifdef DIAGNOSTIC 833 else 834 lockpanic(lkp, "lockmgr: release of unlocked lock!"); 835 #endif 836 WAKEUP_WAITER(lkp); 837 break; 838 839 case LK_DRAIN: 840 /* 841 * Check that we do not already hold the lock, as it can 842 * never drain if we do. Unfortunately, we have no way to 843 * check for holding a shared lock, but at least we can 844 * check for an exclusive one. 845 */ 846 if (WEHOLDIT(lkp, pid, lid, cpu_num)) 847 lockpanic(lkp, "lockmgr: draining against myself"); 848 /* 849 * If we are just polling, check to see if we will sleep. 850 */ 851 if ((extflags & LK_NOWAIT) && (lkp->lk_flags & 852 (LK_HAVE_EXCL | LK_WANT_EXCL | 853 LK_SHARE_NONZERO | LK_WAIT_NONZERO))) { 854 error = EBUSY; 855 break; 856 } 857 error = acquire(&lkp, &s, extflags, 1, 858 LK_HAVE_EXCL | LK_WANT_EXCL | 859 LK_SHARE_NONZERO | LK_WAIT_NONZERO, 860 RETURN_ADDRESS); 861 if (error) 862 break; 863 lkp->lk_flags |= LK_DRAINING | LK_HAVE_EXCL; 864 SETHOLDER(lkp, pid, lid, cpu_num); 865 #if defined(LOCKDEBUG) 866 lkp->lk_lock_file = file; 867 lkp->lk_lock_line = line; 868 #endif 869 HAVEIT(lkp); 870 lkp->lk_exclusivecount = 1; 871 /* XXX unlikely that we'd want this */ 872 if (extflags & LK_SETRECURSE) 873 lkp->lk_recurselevel = 1; 874 COUNT(lkp, l, cpu_num, 1); 875 break; 876 877 default: 878 INTERLOCK_RELEASE(lkp, lkp->lk_flags, s); 879 lockpanic(lkp, "lockmgr: unknown locktype request %d", 880 flags & LK_TYPE_MASK); 881 /* NOTREACHED */ 882 } 883 if ((lkp->lk_flags & (LK_WAITDRAIN|LK_SPIN)) == LK_WAITDRAIN && 884 ((lkp->lk_flags & 885 (LK_HAVE_EXCL | LK_WANT_EXCL | 886 LK_SHARE_NONZERO | LK_WAIT_NONZERO)) == 0)) { 887 lkp->lk_flags &= ~LK_WAITDRAIN; 888 wakeup(&lkp->lk_flags); 889 } 890 /* 891 * Note that this panic will be a recursive panic, since 892 * we only set lock_shutdown_noblock above if panicstr != NULL. 893 */ 894 if (error && lock_shutdown_noblock) 895 lockpanic(lkp, "lockmgr: deadlock (see previous panic)"); 896 897 INTERLOCK_RELEASE(lkp, lkp->lk_flags, s); 898 return (error); 899 } 900 901 /* 902 * For a recursive spinlock held one or more times by the current CPU, 903 * release all N locks, and return N. 904 * Intended for use in mi_switch() shortly before context switching. 905 */ 906 907 int 908 #if defined(LOCKDEBUG) 909 _spinlock_release_all(volatile struct lock *lkp, const char *file, int line) 910 #else 911 spinlock_release_all(volatile struct lock *lkp) 912 #endif 913 { 914 int s, count; 915 cpuid_t cpu_num; 916 917 KASSERT(lkp->lk_flags & LK_SPIN); 918 919 INTERLOCK_ACQUIRE(lkp, LK_SPIN, s); 920 921 cpu_num = cpu_number(); 922 count = lkp->lk_exclusivecount; 923 924 if (count != 0) { 925 #ifdef DIAGNOSTIC 926 if (WEHOLDIT(lkp, 0, 0, cpu_num) == 0) { 927 lockpanic(lkp, "spinlock_release_all: processor %lu, not " 928 "exclusive lock holder %lu " 929 "unlocking", (long)cpu_num, lkp->lk_cpu); 930 } 931 #endif 932 lkp->lk_recurselevel = 0; 933 lkp->lk_exclusivecount = 0; 934 COUNT_CPU(cpu_num, -count); 935 lkp->lk_flags &= ~LK_HAVE_EXCL; 936 SETHOLDER(lkp, LK_NOPROC, 0, LK_NOCPU); 937 #if defined(LOCKDEBUG) 938 lkp->lk_unlock_file = file; 939 lkp->lk_unlock_line = line; 940 #endif 941 DONTHAVEIT(lkp); 942 } 943 #ifdef DIAGNOSTIC 944 else if (lkp->lk_sharecount != 0) 945 lockpanic(lkp, "spinlock_release_all: release of shared lock!"); 946 else 947 lockpanic(lkp, "spinlock_release_all: release of unlocked lock!"); 948 #endif 949 INTERLOCK_RELEASE(lkp, LK_SPIN, s); 950 951 return (count); 952 } 953 954 /* 955 * For a recursive spinlock held one or more times by the current CPU, 956 * release all N locks, and return N. 957 * Intended for use in mi_switch() right after resuming execution. 958 */ 959 960 void 961 #if defined(LOCKDEBUG) 962 _spinlock_acquire_count(volatile struct lock *lkp, int count, 963 const char *file, int line) 964 #else 965 spinlock_acquire_count(volatile struct lock *lkp, int count) 966 #endif 967 { 968 int s, error; 969 cpuid_t cpu_num; 970 971 KASSERT(lkp->lk_flags & LK_SPIN); 972 973 INTERLOCK_ACQUIRE(lkp, LK_SPIN, s); 974 975 cpu_num = cpu_number(); 976 977 #ifdef DIAGNOSTIC 978 if (WEHOLDIT(lkp, LK_NOPROC, 0, cpu_num)) 979 lockpanic(lkp, "spinlock_acquire_count: processor %lu already holds lock", (long)cpu_num); 980 #endif 981 /* 982 * Try to acquire the want_exclusive flag. 983 */ 984 error = acquire(&lkp, &s, LK_SPIN, 0, LK_HAVE_EXCL | LK_WANT_EXCL, 985 RETURN_ADDRESS); 986 lkp->lk_flags |= LK_WANT_EXCL; 987 /* 988 * Wait for shared locks and upgrades to finish. 989 */ 990 error = acquire(&lkp, &s, LK_SPIN, 0, 991 LK_HAVE_EXCL | LK_SHARE_NONZERO | LK_WANT_UPGRADE, 992 RETURN_ADDRESS); 993 lkp->lk_flags &= ~LK_WANT_EXCL; 994 lkp->lk_flags |= LK_HAVE_EXCL; 995 SETHOLDER(lkp, LK_NOPROC, 0, cpu_num); 996 #if defined(LOCKDEBUG) 997 lkp->lk_lock_file = file; 998 lkp->lk_lock_line = line; 999 #endif 1000 HAVEIT(lkp); 1001 if (lkp->lk_exclusivecount != 0) 1002 lockpanic(lkp, "lockmgr: non-zero exclusive count"); 1003 lkp->lk_exclusivecount = count; 1004 lkp->lk_recurselevel = 1; 1005 COUNT_CPU(cpu_num, count); 1006 1007 INTERLOCK_RELEASE(lkp, lkp->lk_flags, s); 1008 } 1009 1010 1011 1012 /* 1013 * Print out information about state of a lock. Used by VOP_PRINT 1014 * routines to display ststus about contained locks. 1015 */ 1016 void 1017 lockmgr_printinfo(volatile struct lock *lkp) 1018 { 1019 1020 if (lkp->lk_sharecount) 1021 printf(" lock type %s: SHARED (count %d)", lkp->lk_wmesg, 1022 lkp->lk_sharecount); 1023 else if (lkp->lk_flags & LK_HAVE_EXCL) { 1024 printf(" lock type %s: EXCL (count %d) by ", 1025 lkp->lk_wmesg, lkp->lk_exclusivecount); 1026 if (lkp->lk_flags & LK_SPIN) 1027 printf("processor %lu", lkp->lk_cpu); 1028 else 1029 printf("pid %d.%d", lkp->lk_lockholder, 1030 lkp->lk_locklwp); 1031 } else 1032 printf(" not locked"); 1033 if ((lkp->lk_flags & LK_SPIN) == 0 && lkp->lk_waitcount > 0) 1034 printf(" with %d pending", lkp->lk_waitcount); 1035 } 1036 1037 #if defined(LOCKDEBUG) /* { */ 1038 _TAILQ_HEAD(, struct simplelock, volatile) simplelock_list = 1039 TAILQ_HEAD_INITIALIZER(simplelock_list); 1040 1041 #if defined(MULTIPROCESSOR) /* { */ 1042 struct simplelock simplelock_list_slock = SIMPLELOCK_INITIALIZER; 1043 1044 #define SLOCK_LIST_LOCK() \ 1045 __cpu_simple_lock(&simplelock_list_slock.lock_data) 1046 1047 #define SLOCK_LIST_UNLOCK() \ 1048 __cpu_simple_unlock(&simplelock_list_slock.lock_data) 1049 1050 #define SLOCK_COUNT(x) \ 1051 curcpu()->ci_simple_locks += (x) 1052 #else 1053 u_long simple_locks; 1054 1055 #define SLOCK_LIST_LOCK() /* nothing */ 1056 1057 #define SLOCK_LIST_UNLOCK() /* nothing */ 1058 1059 #define SLOCK_COUNT(x) simple_locks += (x) 1060 #endif /* MULTIPROCESSOR */ /* } */ 1061 1062 #ifdef MULTIPROCESSOR 1063 #define SLOCK_MP() lock_printf("on CPU %ld\n", \ 1064 (u_long) cpu_number()) 1065 #else 1066 #define SLOCK_MP() /* nothing */ 1067 #endif 1068 1069 #define SLOCK_WHERE(str, alp, id, l) \ 1070 do { \ 1071 lock_printf("\n"); \ 1072 lock_printf(str); \ 1073 lock_printf("lock: %p, currently at: %s:%d\n", (alp), (id), (l)); \ 1074 SLOCK_MP(); \ 1075 if ((alp)->lock_file != NULL) \ 1076 lock_printf("last locked: %s:%d\n", (alp)->lock_file, \ 1077 (alp)->lock_line); \ 1078 if ((alp)->unlock_file != NULL) \ 1079 lock_printf("last unlocked: %s:%d\n", (alp)->unlock_file, \ 1080 (alp)->unlock_line); \ 1081 SLOCK_TRACE() \ 1082 SLOCK_DEBUGGER(); \ 1083 } while (/*CONSTCOND*/0) 1084 1085 /* 1086 * Simple lock functions so that the debugger can see from whence 1087 * they are being called. 1088 */ 1089 void 1090 simple_lock_init(volatile struct simplelock *alp) 1091 { 1092 1093 #if defined(MULTIPROCESSOR) /* { */ 1094 __cpu_simple_lock_init(&alp->lock_data); 1095 #else 1096 alp->lock_data = __SIMPLELOCK_UNLOCKED; 1097 #endif /* } */ 1098 alp->lock_file = NULL; 1099 alp->lock_line = 0; 1100 alp->unlock_file = NULL; 1101 alp->unlock_line = 0; 1102 alp->lock_holder = LK_NOCPU; 1103 } 1104 1105 void 1106 _simple_lock(volatile struct simplelock *alp, const char *id, int l) 1107 { 1108 cpuid_t cpu_num = cpu_number(); 1109 int s; 1110 1111 s = spllock(); 1112 1113 /* 1114 * MULTIPROCESSOR case: This is `safe' since if it's not us, we 1115 * don't take any action, and just fall into the normal spin case. 1116 */ 1117 if (alp->lock_data == __SIMPLELOCK_LOCKED) { 1118 #if defined(MULTIPROCESSOR) /* { */ 1119 if (alp->lock_holder == cpu_num) { 1120 SLOCK_WHERE("simple_lock: locking against myself\n", 1121 alp, id, l); 1122 goto out; 1123 } 1124 #else 1125 SLOCK_WHERE("simple_lock: lock held\n", alp, id, l); 1126 goto out; 1127 #endif /* MULTIPROCESSOR */ /* } */ 1128 } 1129 1130 #if defined(MULTIPROCESSOR) /* { */ 1131 /* Acquire the lock before modifying any fields. */ 1132 splx(s); 1133 __cpu_simple_lock(&alp->lock_data); 1134 s = spllock(); 1135 #else 1136 alp->lock_data = __SIMPLELOCK_LOCKED; 1137 #endif /* } */ 1138 1139 if (alp->lock_holder != LK_NOCPU) { 1140 SLOCK_WHERE("simple_lock: uninitialized lock\n", 1141 alp, id, l); 1142 } 1143 alp->lock_file = id; 1144 alp->lock_line = l; 1145 alp->lock_holder = cpu_num; 1146 1147 SLOCK_LIST_LOCK(); 1148 TAILQ_INSERT_TAIL(&simplelock_list, alp, list); 1149 SLOCK_LIST_UNLOCK(); 1150 1151 SLOCK_COUNT(1); 1152 1153 out: 1154 splx(s); 1155 } 1156 1157 int 1158 _simple_lock_held(volatile struct simplelock *alp) 1159 { 1160 #if defined(MULTIPROCESSOR) || defined(DIAGNOSTIC) 1161 cpuid_t cpu_num = cpu_number(); 1162 #endif 1163 int s, locked = 0; 1164 1165 s = spllock(); 1166 1167 #if defined(MULTIPROCESSOR) 1168 if (__cpu_simple_lock_try(&alp->lock_data) == 0) 1169 locked = (alp->lock_holder == cpu_num); 1170 else 1171 __cpu_simple_unlock(&alp->lock_data); 1172 #else 1173 if (alp->lock_data == __SIMPLELOCK_LOCKED) { 1174 locked = 1; 1175 KASSERT(alp->lock_holder == cpu_num); 1176 } 1177 #endif 1178 1179 splx(s); 1180 1181 return (locked); 1182 } 1183 1184 int 1185 _simple_lock_try(volatile struct simplelock *alp, const char *id, int l) 1186 { 1187 cpuid_t cpu_num = cpu_number(); 1188 int s, rv = 0; 1189 1190 s = spllock(); 1191 1192 /* 1193 * MULTIPROCESSOR case: This is `safe' since if it's not us, we 1194 * don't take any action. 1195 */ 1196 #if defined(MULTIPROCESSOR) /* { */ 1197 if ((rv = __cpu_simple_lock_try(&alp->lock_data)) == 0) { 1198 if (alp->lock_holder == cpu_num) 1199 SLOCK_WHERE("simple_lock_try: locking against myself\n", 1200 alp, id, l); 1201 goto out; 1202 } 1203 #else 1204 if (alp->lock_data == __SIMPLELOCK_LOCKED) { 1205 SLOCK_WHERE("simple_lock_try: lock held\n", alp, id, l); 1206 goto out; 1207 } 1208 alp->lock_data = __SIMPLELOCK_LOCKED; 1209 #endif /* MULTIPROCESSOR */ /* } */ 1210 1211 /* 1212 * At this point, we have acquired the lock. 1213 */ 1214 1215 rv = 1; 1216 1217 alp->lock_file = id; 1218 alp->lock_line = l; 1219 alp->lock_holder = cpu_num; 1220 1221 SLOCK_LIST_LOCK(); 1222 TAILQ_INSERT_TAIL(&simplelock_list, alp, list); 1223 SLOCK_LIST_UNLOCK(); 1224 1225 SLOCK_COUNT(1); 1226 1227 out: 1228 splx(s); 1229 return (rv); 1230 } 1231 1232 void 1233 _simple_unlock(volatile struct simplelock *alp, const char *id, int l) 1234 { 1235 int s; 1236 1237 s = spllock(); 1238 1239 /* 1240 * MULTIPROCESSOR case: This is `safe' because we think we hold 1241 * the lock, and if we don't, we don't take any action. 1242 */ 1243 if (alp->lock_data == __SIMPLELOCK_UNLOCKED) { 1244 SLOCK_WHERE("simple_unlock: lock not held\n", 1245 alp, id, l); 1246 goto out; 1247 } 1248 1249 SLOCK_LIST_LOCK(); 1250 TAILQ_REMOVE(&simplelock_list, alp, list); 1251 SLOCK_LIST_UNLOCK(); 1252 1253 SLOCK_COUNT(-1); 1254 1255 alp->list.tqe_next = NULL; /* sanity */ 1256 alp->list.tqe_prev = NULL; /* sanity */ 1257 1258 alp->unlock_file = id; 1259 alp->unlock_line = l; 1260 1261 #if defined(MULTIPROCESSOR) /* { */ 1262 alp->lock_holder = LK_NOCPU; 1263 /* Now that we've modified all fields, release the lock. */ 1264 __cpu_simple_unlock(&alp->lock_data); 1265 #else 1266 alp->lock_data = __SIMPLELOCK_UNLOCKED; 1267 KASSERT(alp->lock_holder == cpu_number()); 1268 alp->lock_holder = LK_NOCPU; 1269 #endif /* } */ 1270 1271 out: 1272 splx(s); 1273 } 1274 1275 void 1276 simple_lock_dump(void) 1277 { 1278 volatile struct simplelock *alp; 1279 int s; 1280 1281 s = spllock(); 1282 SLOCK_LIST_LOCK(); 1283 lock_printf("all simple locks:\n"); 1284 TAILQ_FOREACH(alp, &simplelock_list, list) { 1285 lock_printf("%p CPU %lu %s:%d\n", alp, alp->lock_holder, 1286 alp->lock_file, alp->lock_line); 1287 } 1288 SLOCK_LIST_UNLOCK(); 1289 splx(s); 1290 } 1291 1292 void 1293 simple_lock_freecheck(void *start, void *end) 1294 { 1295 volatile struct simplelock *alp; 1296 int s; 1297 1298 s = spllock(); 1299 SLOCK_LIST_LOCK(); 1300 TAILQ_FOREACH(alp, &simplelock_list, list) { 1301 if ((volatile void *)alp >= start && 1302 (volatile void *)alp < end) { 1303 lock_printf("freeing simple_lock %p CPU %lu %s:%d\n", 1304 alp, alp->lock_holder, alp->lock_file, 1305 alp->lock_line); 1306 SLOCK_DEBUGGER(); 1307 } 1308 } 1309 SLOCK_LIST_UNLOCK(); 1310 splx(s); 1311 } 1312 1313 /* 1314 * We must be holding exactly one lock: the sched_lock. 1315 */ 1316 1317 void 1318 simple_lock_switchcheck(void) 1319 { 1320 1321 simple_lock_only_held(NULL, "switching"); 1322 } 1323 1324 /* 1325 * Drop into the debugger if lp isn't the only lock held. 1326 * lp may be NULL. 1327 */ 1328 void 1329 simple_lock_only_held(volatile struct simplelock *lp, const char *where) 1330 { 1331 volatile struct simplelock *alp; 1332 cpuid_t cpu_num = cpu_number(); 1333 int s; 1334 1335 if (lp) { 1336 LOCK_ASSERT(simple_lock_held(lp)); 1337 } 1338 s = spllock(); 1339 SLOCK_LIST_LOCK(); 1340 TAILQ_FOREACH(alp, &simplelock_list, list) { 1341 if (alp == lp) 1342 continue; 1343 if (alp->lock_holder == cpu_num) 1344 break; 1345 } 1346 SLOCK_LIST_UNLOCK(); 1347 splx(s); 1348 1349 if (alp != NULL) { 1350 lock_printf("\n%s with held simple_lock %p " 1351 "CPU %lu %s:%d\n", 1352 where, alp, alp->lock_holder, alp->lock_file, 1353 alp->lock_line); 1354 SLOCK_TRACE(); 1355 SLOCK_DEBUGGER(); 1356 } 1357 } 1358 1359 /* 1360 * Set to 1 by simple_lock_assert_*(). 1361 * Can be cleared from ddb to avoid a panic. 1362 */ 1363 int slock_assert_will_panic; 1364 1365 /* 1366 * If the lock isn't held, print a traceback, optionally drop into the 1367 * debugger, then panic. 1368 * The panic can be avoided by clearing slock_assert_with_panic from the 1369 * debugger. 1370 */ 1371 void 1372 _simple_lock_assert_locked(volatile struct simplelock *alp, 1373 const char *lockname, const char *id, int l) 1374 { 1375 if (simple_lock_held(alp) == 0) { 1376 slock_assert_will_panic = 1; 1377 lock_printf("%s lock not held\n", lockname); 1378 SLOCK_WHERE("lock not held", alp, id, l); 1379 if (slock_assert_will_panic) 1380 panic("%s: not locked", lockname); 1381 } 1382 } 1383 1384 void 1385 _simple_lock_assert_unlocked(volatile struct simplelock *alp, 1386 const char *lockname, const char *id, int l) 1387 { 1388 if (simple_lock_held(alp)) { 1389 slock_assert_will_panic = 1; 1390 lock_printf("%s lock held\n", lockname); 1391 SLOCK_WHERE("lock held", alp, id, l); 1392 if (slock_assert_will_panic) 1393 panic("%s: locked", lockname); 1394 } 1395 } 1396 1397 void 1398 assert_sleepable(struct simplelock *interlock, const char *msg) 1399 { 1400 1401 if (curlwp == NULL) { 1402 panic("assert_sleepable: NULL curlwp"); 1403 } 1404 simple_lock_only_held(interlock, msg); 1405 } 1406 1407 #endif /* LOCKDEBUG */ /* } */ 1408 1409 #if defined(MULTIPROCESSOR) 1410 1411 /* 1412 * Functions for manipulating the kernel_lock. We put them here 1413 * so that they show up in profiles. 1414 */ 1415 1416 #define _KERNEL_LOCK_ABORT(msg) \ 1417 LOCKDEBUG_ABORT(kernel_lock_id, &kernel_lock, &_kernel_lock_ops, \ 1418 __FUNCTION__, msg) 1419 1420 #ifdef LOCKDEBUG 1421 #define _KERNEL_LOCK_ASSERT(cond) \ 1422 do { \ 1423 if (!(cond)) \ 1424 _KERNEL_LOCK_ABORT("assertion failed: " #cond); \ 1425 } while (/* CONSTCOND */ 0) 1426 #else 1427 #define _KERNEL_LOCK_ASSERT(cond) /* nothing */ 1428 #endif 1429 1430 void _kernel_lock_dump(volatile void *); 1431 1432 lockops_t _kernel_lock_ops = { 1433 "Kernel lock", 1434 0, 1435 _kernel_lock_dump 1436 }; 1437 1438 /* 1439 * Initialize the kernel lock. 1440 */ 1441 void 1442 _kernel_lock_init(void) 1443 { 1444 1445 __cpu_simple_lock_init(&kernel_lock); 1446 kernel_lock_id = LOCKDEBUG_ALLOC(&kernel_lock, &_kernel_lock_ops); 1447 } 1448 1449 /* 1450 * Print debugging information about the kernel lock. 1451 */ 1452 void 1453 _kernel_lock_dump(volatile void *junk) 1454 { 1455 struct cpu_info *ci = curcpu(); 1456 1457 (void)junk; 1458 1459 printf_nolog("curcpu holds : %18d wanted by: %#018lx\n", 1460 ci->ci_biglock_count, (long)ci->ci_biglock_wanted); 1461 } 1462 1463 /* 1464 * Acquire 'nlocks' holds on the kernel lock. If 'l' is non-null, the 1465 * acquisition is from process context. 1466 */ 1467 void 1468 _kernel_lock(int nlocks, struct lwp *l) 1469 { 1470 struct cpu_info *ci = curcpu(); 1471 LOCKSTAT_TIMER(spintime); 1472 LOCKSTAT_FLAG(lsflag); 1473 struct lwp *owant; 1474 #ifdef LOCKDEBUG 1475 u_int spins; 1476 #endif 1477 int s; 1478 1479 (void)l; 1480 1481 if (nlocks == 0) 1482 return; 1483 _KERNEL_LOCK_ASSERT(nlocks > 0); 1484 1485 s = splbiglock(); 1486 1487 if (ci->ci_biglock_count != 0) { 1488 _KERNEL_LOCK_ASSERT(kernel_lock == __SIMPLELOCK_LOCKED); 1489 ci->ci_biglock_count += nlocks; 1490 splx(s); 1491 return; 1492 } 1493 1494 LOCKDEBUG_WANTLOCK(kernel_lock_id, 1495 (uintptr_t)__builtin_return_address(0), 0); 1496 1497 if (__cpu_simple_lock_try(&kernel_lock)) { 1498 ci->ci_biglock_count = nlocks; 1499 LOCKDEBUG_LOCKED(kernel_lock_id, 1500 (uintptr_t)__builtin_return_address(0), 0); 1501 splx(s); 1502 return; 1503 } 1504 1505 LOCKSTAT_ENTER(lsflag); 1506 LOCKSTAT_START_TIMER(lsflag, spintime); 1507 1508 /* 1509 * Before setting ci_biglock_wanted we must post a store 1510 * fence (see kern_mutex.c). This is accomplished by the 1511 * __cpu_simple_lock_try() above. 1512 */ 1513 owant = ci->ci_biglock_wanted; 1514 ci->ci_biglock_wanted = curlwp; /* XXXAD */ 1515 1516 #ifdef LOCKDEBUG 1517 spins = 0; 1518 #endif 1519 1520 do { 1521 while (kernel_lock == __SIMPLELOCK_LOCKED) { 1522 #ifdef LOCKDEBUG 1523 if (SPINLOCK_SPINOUT(spins)) 1524 _KERNEL_LOCK_ABORT("spinout"); 1525 #endif 1526 splx(s); 1527 SPINLOCK_SPIN_HOOK; 1528 (void)splbiglock(); 1529 } 1530 } while (!__cpu_simple_lock_try(&kernel_lock)); 1531 1532 ci->ci_biglock_wanted = owant; 1533 ci->ci_biglock_count += nlocks; 1534 LOCKSTAT_STOP_TIMER(lsflag, spintime); 1535 LOCKDEBUG_LOCKED(kernel_lock_id, 1536 (uintptr_t)__builtin_return_address(0), 0); 1537 splx(s); 1538 1539 /* 1540 * Again, another store fence is required (see kern_mutex.c). 1541 */ 1542 mb_write(); 1543 if (owant == NULL) { 1544 LOCKSTAT_EVENT(lsflag, &kernel_lock, LB_KERNEL_LOCK | LB_SPIN, 1545 1, spintime); 1546 } 1547 LOCKSTAT_EXIT(lsflag); 1548 } 1549 1550 /* 1551 * Release 'nlocks' holds on the kernel lock. If 'nlocks' is zero, release 1552 * all holds. If 'l' is non-null, the release is from process context. 1553 */ 1554 void 1555 _kernel_unlock(int nlocks, struct lwp *l, int *countp) 1556 { 1557 struct cpu_info *ci = curcpu(); 1558 u_int olocks; 1559 int s; 1560 1561 (void)l; 1562 1563 _KERNEL_LOCK_ASSERT(nlocks < 2); 1564 1565 olocks = ci->ci_biglock_count; 1566 1567 if (olocks == 0) { 1568 _KERNEL_LOCK_ASSERT(nlocks <= 0); 1569 if (countp != NULL) 1570 *countp = 0; 1571 return; 1572 } 1573 1574 _KERNEL_LOCK_ASSERT(kernel_lock == __SIMPLELOCK_LOCKED); 1575 1576 if (nlocks == 0) 1577 nlocks = olocks; 1578 else if (nlocks == -1) { 1579 nlocks = 1; 1580 _KERNEL_LOCK_ASSERT(olocks == 1); 1581 } 1582 1583 s = splbiglock(); 1584 if ((ci->ci_biglock_count -= nlocks) == 0) { 1585 LOCKDEBUG_UNLOCKED(kernel_lock_id, 1586 (uintptr_t)__builtin_return_address(0), 0); 1587 __cpu_simple_unlock(&kernel_lock); 1588 } 1589 splx(s); 1590 1591 if (countp != NULL) 1592 *countp = olocks; 1593 } 1594 1595 #if defined(DEBUG) 1596 /* 1597 * Assert that the kernel lock is held. 1598 */ 1599 void 1600 _kernel_lock_assert_locked(void) 1601 { 1602 1603 if (kernel_lock != __SIMPLELOCK_LOCKED || 1604 curcpu()->ci_biglock_count == 0) 1605 _KERNEL_LOCK_ABORT("not locked"); 1606 } 1607 1608 void 1609 _kernel_lock_assert_unlocked() 1610 { 1611 1612 if (curcpu()->ci_biglock_count != 0) 1613 _KERNEL_LOCK_ABORT("locked"); 1614 } 1615 #endif 1616 1617 #endif /* MULTIPROCESSOR || LOCKDEBUG */ 1618
Indexes created Sun Sep 28 07:09:56 GMT 2025