kern_lock.c revision 1.117
1 /* $NetBSD: kern_lock.c,v 1.117 2007/07/29 11:45:21 ad Exp $ */ 2 3 /*- 4 * Copyright (c) 1999, 2000, 2006, 2007 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center, and by Andrew Doran. 10 * 11 * This code is derived from software contributed to The NetBSD Foundation 12 * by Ross Harvey. 13 * 14 * Redistribution and use in source and binary forms, with or without 15 * modification, are permitted provided that the following conditions 16 * are met: 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in the 21 * documentation and/or other materials provided with the distribution. 22 * 3. All advertising materials mentioning features or use of this software 23 * must display the following acknowledgement: 24 * This product includes software developed by the NetBSD 25 * Foundation, Inc. and its contributors. 26 * 4. Neither the name of The NetBSD Foundation nor the names of its 27 * contributors may be used to endorse or promote products derived 28 * from this software without specific prior written permission. 29 * 30 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 31 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 32 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 33 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 34 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 35 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 36 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 37 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 38 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 39 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 40 * POSSIBILITY OF SUCH DAMAGE. 41 */ 42 43 /* 44 * Copyright (c) 1995 45 * The Regents of the University of California. All rights reserved. 46 * 47 * This code contains ideas from software contributed to Berkeley by 48 * Avadis Tevanian, Jr., Michael Wayne Young, and the Mach Operating 49 * System project at Carnegie-Mellon University. 50 * 51 * Redistribution and use in source and binary forms, with or without 52 * modification, are permitted provided that the following conditions 53 * are met: 54 * 1. Redistributions of source code must retain the above copyright 55 * notice, this list of conditions and the following disclaimer. 56 * 2. Redistributions in binary form must reproduce the above copyright 57 * notice, this list of conditions and the following disclaimer in the 58 * documentation and/or other materials provided with the distribution. 59 * 3. Neither the name of the University nor the names of its contributors 60 * may be used to endorse or promote products derived from this software 61 * without specific prior written permission. 62 * 63 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 64 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 65 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 66 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 67 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 68 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 69 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 70 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 71 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 72 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 73 * SUCH DAMAGE. 74 * 75 * @(#)kern_lock.c 8.18 (Berkeley) 5/21/95 76 */ 77 78 #include <sys/cdefs.h> 79 __KERNEL_RCSID(0, "$NetBSD: kern_lock.c,v 1.117 2007/07/29 11:45:21 ad Exp $"); 80 81 #include "opt_multiprocessor.h" 82 #include "opt_ddb.h" 83 84 #define __MUTEX_PRIVATE 85 86 #include <sys/param.h> 87 #include <sys/proc.h> 88 #include <sys/lock.h> 89 #include <sys/systm.h> 90 #include <sys/lockdebug.h> 91 92 #include <machine/cpu.h> 93 #include <machine/stdarg.h> 94 95 #include <dev/lockstat.h> 96 97 #if defined(LOCKDEBUG) 98 #include <sys/syslog.h> 99 /* 100 * note that stdarg.h and the ansi style va_start macro is used for both 101 * ansi and traditional c compiles. 102 * XXX: this requires that stdarg.h define: va_alist and va_dcl 103 */ 104 #include <machine/stdarg.h> 105 106 void lock_printf(const char *fmt, ...) 107 __attribute__((__format__(__printf__,1,2))); 108 109 static int acquire(volatile struct lock **, int *, int, int, int, uintptr_t); 110 111 int lock_debug_syslog = 0; /* defaults to printf, but can be patched */ 112 113 #ifdef DDB 114 #include <ddb/ddbvar.h> 115 #include <machine/db_machdep.h> 116 #include <ddb/db_command.h> 117 #include <ddb/db_interface.h> 118 #endif 119 #endif /* defined(LOCKDEBUG) */ 120 121 /* 122 * Locking primitives implementation. 123 * Locks provide shared/exclusive synchronization. 124 */ 125 126 #if defined(LOCKDEBUG) || defined(DIAGNOSTIC) /* { */ 127 #if defined(MULTIPROCESSOR) /* { */ 128 #define COUNT_CPU(cpu_id, x) \ 129 curcpu()->ci_spin_locks += (x) 130 #else 131 u_long spin_locks; 132 #define COUNT_CPU(cpu_id, x) spin_locks += (x) 133 #endif /* MULTIPROCESSOR */ /* } */ 134 135 #define COUNT(lkp, l, cpu_id, x) \ 136 do { \ 137 if ((lkp)->lk_flags & LK_SPIN) \ 138 COUNT_CPU((cpu_id), (x)); \ 139 else \ 140 (l)->l_locks += (x); \ 141 } while (/*CONSTCOND*/0) 142 #else 143 #define COUNT(lkp, p, cpu_id, x) 144 #define COUNT_CPU(cpu_id, x) 145 #endif /* LOCKDEBUG || DIAGNOSTIC */ /* } */ 146 147 #define INTERLOCK_ACQUIRE(lkp, flags, s) \ 148 do { \ 149 if ((flags) & LK_SPIN) \ 150 s = splhigh(); \ 151 simple_lock(&(lkp)->lk_interlock); \ 152 } while (/*CONSTCOND*/ 0) 153 154 #define INTERLOCK_RELEASE(lkp, flags, s) \ 155 do { \ 156 simple_unlock(&(lkp)->lk_interlock); \ 157 if ((flags) & LK_SPIN) \ 158 splx(s); \ 159 } while (/*CONSTCOND*/ 0) 160 161 #ifdef DDB /* { */ 162 #if defined(MULTIPROCESSOR) || defined(LOCKDEBUG) 163 int simple_lock_debugger = 1; /* more serious on MP */ 164 #else 165 int simple_lock_debugger = 0; 166 #endif 167 #define SLOCK_DEBUGGER() if (simple_lock_debugger && db_onpanic) Debugger() 168 #define SLOCK_TRACE() \ 169 db_stack_trace_print((db_expr_t)__builtin_frame_address(0), \ 170 true, 65535, "", lock_printf); 171 #else 172 #define SLOCK_DEBUGGER() /* nothing */ 173 #define SLOCK_TRACE() /* nothing */ 174 #endif /* } */ 175 176 #if defined(LOCKDEBUG) 177 #if defined(DDB) 178 #define SPINLOCK_SPINCHECK_DEBUGGER if (db_onpanic) Debugger() 179 #else 180 #define SPINLOCK_SPINCHECK_DEBUGGER /* nothing */ 181 #endif 182 183 #define SPINLOCK_SPINCHECK_DECL \ 184 /* 32-bits of count -- wrap constitutes a "spinout" */ \ 185 uint32_t __spinc = 0 186 187 #define SPINLOCK_SPINCHECK \ 188 do { \ 189 if (++__spinc == 0) { \ 190 lock_printf("LK_SPIN spinout, excl %d, share %d\n", \ 191 lkp->lk_exclusivecount, lkp->lk_sharecount); \ 192 if (lkp->lk_exclusivecount) \ 193 lock_printf("held by CPU %lu\n", \ 194 (u_long) lkp->lk_cpu); \ 195 if (lkp->lk_lock_file) \ 196 lock_printf("last locked at %s:%d\n", \ 197 lkp->lk_lock_file, lkp->lk_lock_line); \ 198 if (lkp->lk_unlock_file) \ 199 lock_printf("last unlocked at %s:%d\n", \ 200 lkp->lk_unlock_file, lkp->lk_unlock_line); \ 201 SLOCK_TRACE(); \ 202 SPINLOCK_SPINCHECK_DEBUGGER; \ 203 } \ 204 } while (/*CONSTCOND*/ 0) 205 #else 206 #define SPINLOCK_SPINCHECK_DECL /* nothing */ 207 #define SPINLOCK_SPINCHECK /* nothing */ 208 #endif /* LOCKDEBUG && DDB */ 209 210 #define RETURN_ADDRESS ((uintptr_t)__builtin_return_address(0)) 211 212 /* 213 * Acquire a resource. 214 */ 215 static int 216 acquire(volatile struct lock **lkpp, int *s, int extflags, 217 int drain, int wanted, uintptr_t ra) 218 { 219 int error; 220 volatile struct lock *lkp = *lkpp; 221 LOCKSTAT_TIMER(slptime); 222 LOCKSTAT_FLAG(lsflag); 223 224 KASSERT(drain || (wanted & LK_WAIT_NONZERO) == 0); 225 226 if (extflags & LK_SPIN) { 227 int interlocked; 228 229 SPINLOCK_SPINCHECK_DECL; 230 231 if (!drain) { 232 lkp->lk_waitcount++; 233 lkp->lk_flags |= LK_WAIT_NONZERO; 234 } 235 for (interlocked = 1;;) { 236 SPINLOCK_SPINCHECK; 237 if ((lkp->lk_flags & wanted) != 0) { 238 if (interlocked) { 239 INTERLOCK_RELEASE(lkp, LK_SPIN, *s); 240 interlocked = 0; 241 } 242 SPINLOCK_SPIN_HOOK; 243 } else if (interlocked) { 244 break; 245 } else { 246 INTERLOCK_ACQUIRE(lkp, LK_SPIN, *s); 247 interlocked = 1; 248 } 249 } 250 if (!drain) { 251 lkp->lk_waitcount--; 252 if (lkp->lk_waitcount == 0) 253 lkp->lk_flags &= ~LK_WAIT_NONZERO; 254 } 255 KASSERT((lkp->lk_flags & wanted) == 0); 256 error = 0; /* sanity */ 257 } else { 258 LOCKSTAT_ENTER(lsflag); 259 260 for (error = 0; (lkp->lk_flags & wanted) != 0; ) { 261 if (drain) 262 lkp->lk_flags |= LK_WAITDRAIN; 263 else { 264 lkp->lk_waitcount++; 265 lkp->lk_flags |= LK_WAIT_NONZERO; 266 } 267 /* XXX Cast away volatile. */ 268 LOCKSTAT_START_TIMER(lsflag, slptime); 269 error = ltsleep(drain ? 270 (volatile const void *)&lkp->lk_flags : 271 (volatile const void *)lkp, lkp->lk_prio, 272 lkp->lk_wmesg, lkp->lk_timo, &lkp->lk_interlock); 273 LOCKSTAT_STOP_TIMER(lsflag, slptime); 274 LOCKSTAT_EVENT_RA(lsflag, (void *)(uintptr_t)lkp, 275 LB_LOCKMGR | LB_SLEEP1, 1, slptime, ra); 276 if (!drain) { 277 lkp->lk_waitcount--; 278 if (lkp->lk_waitcount == 0) 279 lkp->lk_flags &= ~LK_WAIT_NONZERO; 280 } 281 if (error) 282 break; 283 if (extflags & LK_SLEEPFAIL) { 284 error = ENOLCK; 285 break; 286 } 287 if (lkp->lk_newlock != NULL) { 288 simple_lock(&lkp->lk_newlock->lk_interlock); 289 simple_unlock(&lkp->lk_interlock); 290 if (lkp->lk_waitcount == 0) 291 wakeup(&lkp->lk_newlock); 292 *lkpp = lkp = lkp->lk_newlock; 293 } 294 } 295 296 LOCKSTAT_EXIT(lsflag); 297 } 298 299 return error; 300 } 301 302 #define SETHOLDER(lkp, pid, lid, cpu_id) \ 303 do { \ 304 if ((lkp)->lk_flags & LK_SPIN) \ 305 (lkp)->lk_cpu = cpu_id; \ 306 else { \ 307 (lkp)->lk_lockholder = pid; \ 308 (lkp)->lk_locklwp = lid; \ 309 } \ 310 } while (/*CONSTCOND*/0) 311 312 #define WEHOLDIT(lkp, pid, lid, cpu_id) \ 313 (((lkp)->lk_flags & LK_SPIN) != 0 ? \ 314 ((lkp)->lk_cpu == (cpu_id)) : \ 315 ((lkp)->lk_lockholder == (pid) && (lkp)->lk_locklwp == (lid))) 316 317 #define WAKEUP_WAITER(lkp) \ 318 do { \ 319 if (((lkp)->lk_flags & (LK_SPIN | LK_WAIT_NONZERO)) == \ 320 LK_WAIT_NONZERO) { \ 321 wakeup((lkp)); \ 322 } \ 323 } while (/*CONSTCOND*/0) 324 325 #if defined(LOCKDEBUG) /* { */ 326 #if defined(MULTIPROCESSOR) /* { */ 327 struct simplelock spinlock_list_slock = SIMPLELOCK_INITIALIZER; 328 329 #define SPINLOCK_LIST_LOCK() \ 330 __cpu_simple_lock(&spinlock_list_slock.lock_data) 331 332 #define SPINLOCK_LIST_UNLOCK() \ 333 __cpu_simple_unlock(&spinlock_list_slock.lock_data) 334 #else 335 #define SPINLOCK_LIST_LOCK() /* nothing */ 336 337 #define SPINLOCK_LIST_UNLOCK() /* nothing */ 338 #endif /* MULTIPROCESSOR */ /* } */ 339 340 _TAILQ_HEAD(, struct lock, volatile) spinlock_list = 341 TAILQ_HEAD_INITIALIZER(spinlock_list); 342 343 #define HAVEIT(lkp) \ 344 do { \ 345 if ((lkp)->lk_flags & LK_SPIN) { \ 346 int sp = splhigh(); \ 347 SPINLOCK_LIST_LOCK(); \ 348 TAILQ_INSERT_TAIL(&spinlock_list, (lkp), lk_list); \ 349 SPINLOCK_LIST_UNLOCK(); \ 350 splx(sp); \ 351 } \ 352 } while (/*CONSTCOND*/0) 353 354 #define DONTHAVEIT(lkp) \ 355 do { \ 356 if ((lkp)->lk_flags & LK_SPIN) { \ 357 int sp = splhigh(); \ 358 SPINLOCK_LIST_LOCK(); \ 359 TAILQ_REMOVE(&spinlock_list, (lkp), lk_list); \ 360 SPINLOCK_LIST_UNLOCK(); \ 361 splx(sp); \ 362 } \ 363 } while (/*CONSTCOND*/0) 364 #else 365 #define HAVEIT(lkp) /* nothing */ 366 367 #define DONTHAVEIT(lkp) /* nothing */ 368 #endif /* LOCKDEBUG */ /* } */ 369 370 #if defined(LOCKDEBUG) 371 /* 372 * Lock debug printing routine; can be configured to print to console 373 * or log to syslog. 374 */ 375 void 376 lock_printf(const char *fmt, ...) 377 { 378 char b[150]; 379 va_list ap; 380 381 va_start(ap, fmt); 382 if (lock_debug_syslog) 383 vlog(LOG_DEBUG, fmt, ap); 384 else { 385 vsnprintf(b, sizeof(b), fmt, ap); 386 printf_nolog("%s", b); 387 } 388 va_end(ap); 389 } 390 #endif /* LOCKDEBUG */ 391 392 static void 393 lockpanic(volatile struct lock *lkp, const char *fmt, ...) 394 { 395 char s[150], b[150]; 396 #ifdef LOCKDEBUG 397 static const char *locktype[] = { 398 "*0*", "shared", "exclusive", "upgrade", "exclupgrade", 399 "downgrade", "release", "drain", "exclother", "*9*", 400 "*10*", "*11*", "*12*", "*13*", "*14*", "*15*" 401 }; 402 #endif 403 va_list ap; 404 va_start(ap, fmt); 405 vsnprintf(s, sizeof(s), fmt, ap); 406 va_end(ap); 407 bitmask_snprintf(lkp->lk_flags, __LK_FLAG_BITS, b, sizeof(b)); 408 panic("%s (" 409 #ifdef LOCKDEBUG 410 "type %s " 411 #endif 412 "flags %s, sharecount %d, exclusivecount %d, " 413 "recurselevel %d, waitcount %d, wmesg %s" 414 #ifdef LOCKDEBUG 415 ", lock_file %s, unlock_file %s, lock_line %d, unlock_line %d" 416 #endif 417 ")\n", 418 s, 419 #ifdef LOCKDEBUG 420 locktype[lkp->lk_flags & LK_TYPE_MASK], 421 #endif 422 b, lkp->lk_sharecount, lkp->lk_exclusivecount, 423 lkp->lk_recurselevel, lkp->lk_waitcount, lkp->lk_wmesg 424 #ifdef LOCKDEBUG 425 , lkp->lk_lock_file, lkp->lk_unlock_file, lkp->lk_lock_line, 426 lkp->lk_unlock_line 427 #endif 428 ); 429 } 430 431 /* 432 * Transfer any waiting processes from one lock to another. 433 */ 434 void 435 transferlockers(struct lock *from, struct lock *to) 436 { 437 438 KASSERT(from != to); 439 KASSERT((from->lk_flags & LK_WAITDRAIN) == 0); 440 if (from->lk_waitcount == 0) 441 return; 442 from->lk_newlock = to; 443 wakeup((void *)from); 444 tsleep((void *)&from->lk_newlock, from->lk_prio, "lkxfer", 0); 445 from->lk_newlock = NULL; 446 from->lk_flags &= ~(LK_WANT_EXCL | LK_WANT_UPGRADE); 447 KASSERT(from->lk_waitcount == 0); 448 } 449 450 451 /* 452 * Initialize a lock; required before use. 453 */ 454 void 455 lockinit(struct lock *lkp, pri_t prio, const char *wmesg, int timo, int flags) 456 { 457 458 memset(lkp, 0, sizeof(struct lock)); 459 simple_lock_init(&lkp->lk_interlock); 460 lkp->lk_flags = flags & LK_EXTFLG_MASK; 461 if (flags & LK_SPIN) 462 lkp->lk_cpu = LK_NOCPU; 463 else { 464 lkp->lk_lockholder = LK_NOPROC; 465 lkp->lk_newlock = NULL; 466 lkp->lk_prio = prio; 467 lkp->lk_timo = timo; 468 } 469 lkp->lk_wmesg = wmesg; /* just a name for spin locks */ 470 #if defined(LOCKDEBUG) 471 lkp->lk_lock_file = NULL; 472 lkp->lk_unlock_file = NULL; 473 #endif 474 } 475 476 /* 477 * Determine the status of a lock. 478 */ 479 int 480 lockstatus(struct lock *lkp) 481 { 482 int s = 0; /* XXX: gcc */ 483 int lock_type = 0; 484 struct lwp *l = curlwp; /* XXX */ 485 pid_t pid; 486 lwpid_t lid; 487 cpuid_t cpu_num; 488 489 if ((lkp->lk_flags & LK_SPIN) || l == NULL) { 490 cpu_num = cpu_number(); 491 pid = LK_KERNPROC; 492 lid = 0; 493 } else { 494 cpu_num = LK_NOCPU; 495 pid = l->l_proc->p_pid; 496 lid = l->l_lid; 497 } 498 499 INTERLOCK_ACQUIRE(lkp, lkp->lk_flags, s); 500 if (lkp->lk_exclusivecount != 0) { 501 if (WEHOLDIT(lkp, pid, lid, cpu_num)) 502 lock_type = LK_EXCLUSIVE; 503 else 504 lock_type = LK_EXCLOTHER; 505 } else if (lkp->lk_sharecount != 0) 506 lock_type = LK_SHARED; 507 else if (lkp->lk_flags & (LK_WANT_EXCL | LK_WANT_UPGRADE)) 508 lock_type = LK_EXCLOTHER; 509 INTERLOCK_RELEASE(lkp, lkp->lk_flags, s); 510 return (lock_type); 511 } 512 513 #if defined(LOCKDEBUG) 514 /* 515 * Make sure no spin locks are held by a CPU that is about 516 * to context switch. 517 */ 518 void 519 spinlock_switchcheck(void) 520 { 521 u_long cnt; 522 int s; 523 524 if (panicstr != NULL) 525 return; 526 527 s = splhigh(); 528 #if defined(MULTIPROCESSOR) 529 cnt = curcpu()->ci_spin_locks; 530 #else 531 cnt = spin_locks; 532 #endif 533 splx(s); 534 535 if (cnt != 0) 536 panic("spinlock_switchcheck: CPU %lu has %lu spin locks", 537 (u_long) cpu_number(), cnt); 538 } 539 #endif /* LOCKDEBUG */ 540 541 /* 542 * Locks and IPLs (interrupt priority levels): 543 * 544 * Locks which may be taken from interrupt context must be handled 545 * very carefully; you must spl to the highest IPL where the lock 546 * is needed before acquiring the lock. 547 * 548 * It is also important to avoid deadlock, since certain (very high 549 * priority) interrupts are often needed to keep the system as a whole 550 * from deadlocking, and must not be blocked while you are spinning 551 * waiting for a lower-priority lock. 552 * 553 * In addition, the lock-debugging hooks themselves need to use locks! 554 * 555 * A raw __cpu_simple_lock may be used from interrupts are long as it 556 * is acquired and held at a single IPL. 557 */ 558 559 /* 560 * XXX XXX kludge around another kludge.. 561 * 562 * vfs_shutdown() may be called from interrupt context, either as a result 563 * of a panic, or from the debugger. It proceeds to call 564 * sys_sync(&proc0, ...), pretending its running on behalf of proc0 565 * 566 * We would like to make an attempt to sync the filesystems in this case, so 567 * if this happens, we treat attempts to acquire locks specially. 568 * All locks are acquired on behalf of proc0. 569 * 570 * If we've already paniced, we don't block waiting for locks, but 571 * just barge right ahead since we're already going down in flames. 572 */ 573 574 /* 575 * Set, change, or release a lock. 576 * 577 * Shared requests increment the shared count. Exclusive requests set the 578 * LK_WANT_EXCL flag (preventing further shared locks), and wait for already 579 * accepted shared locks and shared-to-exclusive upgrades to go away. 580 */ 581 int 582 #if defined(LOCKDEBUG) 583 _lockmgr(volatile struct lock *lkp, u_int flags, 584 struct simplelock *interlkp, const char *file, int line) 585 #else 586 lockmgr(volatile struct lock *lkp, u_int flags, 587 struct simplelock *interlkp) 588 #endif 589 { 590 int error; 591 pid_t pid; 592 lwpid_t lid; 593 int extflags; 594 cpuid_t cpu_num; 595 struct lwp *l = curlwp; 596 int lock_shutdown_noblock = 0; 597 int s = 0; 598 599 error = 0; 600 601 /* LK_RETRY is for vn_lock, not for lockmgr. */ 602 KASSERT((flags & LK_RETRY) == 0); 603 604 INTERLOCK_ACQUIRE(lkp, lkp->lk_flags, s); 605 if (flags & LK_INTERLOCK) 606 simple_unlock(interlkp); 607 extflags = (flags | lkp->lk_flags) & LK_EXTFLG_MASK; 608 609 #ifdef DIAGNOSTIC /* { */ 610 /* 611 * Don't allow spins on sleep locks and don't allow sleeps 612 * on spin locks. 613 */ 614 if ((flags ^ lkp->lk_flags) & LK_SPIN) 615 lockpanic(lkp, "lockmgr: sleep/spin mismatch"); 616 #endif /* } */ 617 618 if (extflags & LK_SPIN) { 619 pid = LK_KERNPROC; 620 lid = 0; 621 } else { 622 if (l == NULL) { 623 if (!doing_shutdown) { 624 panic("lockmgr: no context"); 625 } else { 626 l = &lwp0; 627 if (panicstr && (!(flags & LK_NOWAIT))) { 628 flags |= LK_NOWAIT; 629 lock_shutdown_noblock = 1; 630 } 631 } 632 } 633 lid = l->l_lid; 634 pid = l->l_proc->p_pid; 635 } 636 cpu_num = cpu_number(); 637 638 /* 639 * Once a lock has drained, the LK_DRAINING flag is set and an 640 * exclusive lock is returned. The only valid operation thereafter 641 * is a single release of that exclusive lock. This final release 642 * clears the LK_DRAINING flag and sets the LK_DRAINED flag. Any 643 * further requests of any sort will result in a panic. The bits 644 * selected for these two flags are chosen so that they will be set 645 * in memory that is freed (freed memory is filled with 0xdeadbeef). 646 * The final release is permitted to give a new lease on life to 647 * the lock by specifying LK_REENABLE. 648 */ 649 if (lkp->lk_flags & (LK_DRAINING|LK_DRAINED)) { 650 #ifdef DIAGNOSTIC /* { */ 651 if (lkp->lk_flags & LK_DRAINED) 652 lockpanic(lkp, "lockmgr: using decommissioned lock"); 653 if ((flags & LK_TYPE_MASK) != LK_RELEASE || 654 WEHOLDIT(lkp, pid, lid, cpu_num) == 0) 655 lockpanic(lkp, "lockmgr: non-release on draining lock: %d", 656 flags & LK_TYPE_MASK); 657 #endif /* DIAGNOSTIC */ /* } */ 658 lkp->lk_flags &= ~LK_DRAINING; 659 if ((flags & LK_REENABLE) == 0) 660 lkp->lk_flags |= LK_DRAINED; 661 } 662 663 switch (flags & LK_TYPE_MASK) { 664 665 case LK_SHARED: 666 if (WEHOLDIT(lkp, pid, lid, cpu_num) == 0) { 667 /* 668 * If just polling, check to see if we will block. 669 */ 670 if ((extflags & LK_NOWAIT) && (lkp->lk_flags & 671 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE))) { 672 error = EBUSY; 673 break; 674 } 675 /* 676 * Wait for exclusive locks and upgrades to clear. 677 */ 678 error = acquire(&lkp, &s, extflags, 0, 679 LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE, 680 RETURN_ADDRESS); 681 if (error) 682 break; 683 lkp->lk_sharecount++; 684 lkp->lk_flags |= LK_SHARE_NONZERO; 685 COUNT(lkp, l, cpu_num, 1); 686 break; 687 } 688 /* 689 * We hold an exclusive lock, so downgrade it to shared. 690 * An alternative would be to fail with EDEADLK. 691 */ 692 lkp->lk_sharecount++; 693 lkp->lk_flags |= LK_SHARE_NONZERO; 694 COUNT(lkp, l, cpu_num, 1); 695 /* fall into downgrade */ 696 697 case LK_DOWNGRADE: 698 if (WEHOLDIT(lkp, pid, lid, cpu_num) == 0 || 699 lkp->lk_exclusivecount == 0) 700 lockpanic(lkp, "lockmgr: not holding exclusive lock"); 701 lkp->lk_sharecount += lkp->lk_exclusivecount; 702 lkp->lk_flags |= LK_SHARE_NONZERO; 703 lkp->lk_exclusivecount = 0; 704 lkp->lk_recurselevel = 0; 705 lkp->lk_flags &= ~LK_HAVE_EXCL; 706 SETHOLDER(lkp, LK_NOPROC, 0, LK_NOCPU); 707 #if defined(LOCKDEBUG) 708 lkp->lk_unlock_file = file; 709 lkp->lk_unlock_line = line; 710 #endif 711 DONTHAVEIT(lkp); 712 WAKEUP_WAITER(lkp); 713 break; 714 715 case LK_EXCLUPGRADE: 716 /* 717 * If another process is ahead of us to get an upgrade, 718 * then we want to fail rather than have an intervening 719 * exclusive access. 720 */ 721 if (lkp->lk_flags & LK_WANT_UPGRADE) { 722 lkp->lk_sharecount--; 723 if (lkp->lk_sharecount == 0) 724 lkp->lk_flags &= ~LK_SHARE_NONZERO; 725 COUNT(lkp, l, cpu_num, -1); 726 error = EBUSY; 727 break; 728 } 729 /* fall into normal upgrade */ 730 731 case LK_UPGRADE: 732 /* 733 * Upgrade a shared lock to an exclusive one. If another 734 * shared lock has already requested an upgrade to an 735 * exclusive lock, our shared lock is released and an 736 * exclusive lock is requested (which will be granted 737 * after the upgrade). If we return an error, the file 738 * will always be unlocked. 739 */ 740 if (WEHOLDIT(lkp, pid, lid, cpu_num) || lkp->lk_sharecount <= 0) 741 lockpanic(lkp, "lockmgr: upgrade exclusive lock"); 742 lkp->lk_sharecount--; 743 if (lkp->lk_sharecount == 0) 744 lkp->lk_flags &= ~LK_SHARE_NONZERO; 745 COUNT(lkp, l, cpu_num, -1); 746 /* 747 * If we are just polling, check to see if we will block. 748 */ 749 if ((extflags & LK_NOWAIT) && 750 ((lkp->lk_flags & LK_WANT_UPGRADE) || 751 lkp->lk_sharecount > 1)) { 752 error = EBUSY; 753 break; 754 } 755 if ((lkp->lk_flags & LK_WANT_UPGRADE) == 0) { 756 /* 757 * We are first shared lock to request an upgrade, so 758 * request upgrade and wait for the shared count to 759 * drop to zero, then take exclusive lock. 760 */ 761 lkp->lk_flags |= LK_WANT_UPGRADE; 762 error = acquire(&lkp, &s, extflags, 0, LK_SHARE_NONZERO, 763 RETURN_ADDRESS); 764 lkp->lk_flags &= ~LK_WANT_UPGRADE; 765 if (error) { 766 WAKEUP_WAITER(lkp); 767 break; 768 } 769 lkp->lk_flags |= LK_HAVE_EXCL; 770 SETHOLDER(lkp, pid, lid, cpu_num); 771 #if defined(LOCKDEBUG) 772 lkp->lk_lock_file = file; 773 lkp->lk_lock_line = line; 774 #endif 775 HAVEIT(lkp); 776 if (lkp->lk_exclusivecount != 0) 777 lockpanic(lkp, "lockmgr: non-zero exclusive count"); 778 lkp->lk_exclusivecount = 1; 779 if (extflags & LK_SETRECURSE) 780 lkp->lk_recurselevel = 1; 781 COUNT(lkp, l, cpu_num, 1); 782 break; 783 } 784 /* 785 * Someone else has requested upgrade. Release our shared 786 * lock, awaken upgrade requestor if we are the last shared 787 * lock, then request an exclusive lock. 788 */ 789 if (lkp->lk_sharecount == 0) 790 WAKEUP_WAITER(lkp); 791 /* fall into exclusive request */ 792 793 case LK_EXCLUSIVE: 794 if (WEHOLDIT(lkp, pid, lid, cpu_num)) { 795 /* 796 * Recursive lock. 797 */ 798 if ((extflags & LK_CANRECURSE) == 0 && 799 lkp->lk_recurselevel == 0) { 800 if (extflags & LK_RECURSEFAIL) { 801 error = EDEADLK; 802 break; 803 } else 804 lockpanic(lkp, "lockmgr: locking against myself"); 805 } 806 lkp->lk_exclusivecount++; 807 if (extflags & LK_SETRECURSE && 808 lkp->lk_recurselevel == 0) 809 lkp->lk_recurselevel = lkp->lk_exclusivecount; 810 COUNT(lkp, l, cpu_num, 1); 811 break; 812 } 813 /* 814 * If we are just polling, check to see if we will sleep. 815 */ 816 if ((extflags & LK_NOWAIT) && (lkp->lk_flags & 817 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE | 818 LK_SHARE_NONZERO))) { 819 error = EBUSY; 820 break; 821 } 822 /* 823 * Try to acquire the want_exclusive flag. 824 */ 825 error = acquire(&lkp, &s, extflags, 0, 826 LK_HAVE_EXCL | LK_WANT_EXCL, RETURN_ADDRESS); 827 if (error) 828 break; 829 lkp->lk_flags |= LK_WANT_EXCL; 830 /* 831 * Wait for shared locks and upgrades to finish. 832 */ 833 error = acquire(&lkp, &s, extflags, 0, 834 LK_HAVE_EXCL | LK_WANT_UPGRADE | LK_SHARE_NONZERO, 835 RETURN_ADDRESS); 836 lkp->lk_flags &= ~LK_WANT_EXCL; 837 if (error) { 838 WAKEUP_WAITER(lkp); 839 break; 840 } 841 lkp->lk_flags |= LK_HAVE_EXCL; 842 SETHOLDER(lkp, pid, lid, cpu_num); 843 #if defined(LOCKDEBUG) 844 lkp->lk_lock_file = file; 845 lkp->lk_lock_line = line; 846 #endif 847 HAVEIT(lkp); 848 if (lkp->lk_exclusivecount != 0) 849 lockpanic(lkp, "lockmgr: non-zero exclusive count"); 850 lkp->lk_exclusivecount = 1; 851 if (extflags & LK_SETRECURSE) 852 lkp->lk_recurselevel = 1; 853 COUNT(lkp, l, cpu_num, 1); 854 break; 855 856 case LK_RELEASE: 857 if (lkp->lk_exclusivecount != 0) { 858 if (WEHOLDIT(lkp, pid, lid, cpu_num) == 0) { 859 if (lkp->lk_flags & LK_SPIN) { 860 lockpanic(lkp, 861 "lockmgr: processor %lu, not " 862 "exclusive lock holder %lu " 863 "unlocking", cpu_num, lkp->lk_cpu); 864 } else { 865 lockpanic(lkp, "lockmgr: pid %d.%d, not " 866 "exclusive lock holder %d.%d " 867 "unlocking", pid, lid, 868 lkp->lk_lockholder, 869 lkp->lk_locklwp); 870 } 871 } 872 if (lkp->lk_exclusivecount == lkp->lk_recurselevel) 873 lkp->lk_recurselevel = 0; 874 lkp->lk_exclusivecount--; 875 COUNT(lkp, l, cpu_num, -1); 876 if (lkp->lk_exclusivecount == 0) { 877 lkp->lk_flags &= ~LK_HAVE_EXCL; 878 SETHOLDER(lkp, LK_NOPROC, 0, LK_NOCPU); 879 #if defined(LOCKDEBUG) 880 lkp->lk_unlock_file = file; 881 lkp->lk_unlock_line = line; 882 #endif 883 DONTHAVEIT(lkp); 884 } 885 } else if (lkp->lk_sharecount != 0) { 886 lkp->lk_sharecount--; 887 if (lkp->lk_sharecount == 0) 888 lkp->lk_flags &= ~LK_SHARE_NONZERO; 889 COUNT(lkp, l, cpu_num, -1); 890 } 891 #ifdef DIAGNOSTIC 892 else 893 lockpanic(lkp, "lockmgr: release of unlocked lock!"); 894 #endif 895 WAKEUP_WAITER(lkp); 896 break; 897 898 case LK_DRAIN: 899 /* 900 * Check that we do not already hold the lock, as it can 901 * never drain if we do. Unfortunately, we have no way to 902 * check for holding a shared lock, but at least we can 903 * check for an exclusive one. 904 */ 905 if (WEHOLDIT(lkp, pid, lid, cpu_num)) 906 lockpanic(lkp, "lockmgr: draining against myself"); 907 /* 908 * If we are just polling, check to see if we will sleep. 909 */ 910 if ((extflags & LK_NOWAIT) && (lkp->lk_flags & 911 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE | 912 LK_SHARE_NONZERO | LK_WAIT_NONZERO))) { 913 error = EBUSY; 914 break; 915 } 916 error = acquire(&lkp, &s, extflags, 1, 917 LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE | 918 LK_SHARE_NONZERO | LK_WAIT_NONZERO, 919 RETURN_ADDRESS); 920 if (error) 921 break; 922 lkp->lk_flags |= LK_DRAINING | LK_HAVE_EXCL; 923 SETHOLDER(lkp, pid, lid, cpu_num); 924 #if defined(LOCKDEBUG) 925 lkp->lk_lock_file = file; 926 lkp->lk_lock_line = line; 927 #endif 928 HAVEIT(lkp); 929 lkp->lk_exclusivecount = 1; 930 /* XXX unlikely that we'd want this */ 931 if (extflags & LK_SETRECURSE) 932 lkp->lk_recurselevel = 1; 933 COUNT(lkp, l, cpu_num, 1); 934 break; 935 936 default: 937 INTERLOCK_RELEASE(lkp, lkp->lk_flags, s); 938 lockpanic(lkp, "lockmgr: unknown locktype request %d", 939 flags & LK_TYPE_MASK); 940 /* NOTREACHED */ 941 } 942 if ((lkp->lk_flags & (LK_WAITDRAIN|LK_SPIN)) == LK_WAITDRAIN && 943 ((lkp->lk_flags & 944 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE | 945 LK_SHARE_NONZERO | LK_WAIT_NONZERO)) == 0)) { 946 lkp->lk_flags &= ~LK_WAITDRAIN; 947 wakeup(&lkp->lk_flags); 948 } 949 /* 950 * Note that this panic will be a recursive panic, since 951 * we only set lock_shutdown_noblock above if panicstr != NULL. 952 */ 953 if (error && lock_shutdown_noblock) 954 lockpanic(lkp, "lockmgr: deadlock (see previous panic)"); 955 956 INTERLOCK_RELEASE(lkp, lkp->lk_flags, s); 957 return (error); 958 } 959 960 /* 961 * For a recursive spinlock held one or more times by the current CPU, 962 * release all N locks, and return N. 963 * Intended for use in mi_switch() shortly before context switching. 964 */ 965 966 int 967 #if defined(LOCKDEBUG) 968 _spinlock_release_all(volatile struct lock *lkp, const char *file, int line) 969 #else 970 spinlock_release_all(volatile struct lock *lkp) 971 #endif 972 { 973 int s, count; 974 cpuid_t cpu_num; 975 976 KASSERT(lkp->lk_flags & LK_SPIN); 977 978 INTERLOCK_ACQUIRE(lkp, LK_SPIN, s); 979 980 cpu_num = cpu_number(); 981 count = lkp->lk_exclusivecount; 982 983 if (count != 0) { 984 #ifdef DIAGNOSTIC 985 if (WEHOLDIT(lkp, 0, 0, cpu_num) == 0) { 986 lockpanic(lkp, "spinlock_release_all: processor %lu, not " 987 "exclusive lock holder %lu " 988 "unlocking", (long)cpu_num, lkp->lk_cpu); 989 } 990 #endif 991 lkp->lk_recurselevel = 0; 992 lkp->lk_exclusivecount = 0; 993 COUNT_CPU(cpu_num, -count); 994 lkp->lk_flags &= ~LK_HAVE_EXCL; 995 SETHOLDER(lkp, LK_NOPROC, 0, LK_NOCPU); 996 #if defined(LOCKDEBUG) 997 lkp->lk_unlock_file = file; 998 lkp->lk_unlock_line = line; 999 #endif 1000 DONTHAVEIT(lkp); 1001 } 1002 #ifdef DIAGNOSTIC 1003 else if (lkp->lk_sharecount != 0) 1004 lockpanic(lkp, "spinlock_release_all: release of shared lock!"); 1005 else 1006 lockpanic(lkp, "spinlock_release_all: release of unlocked lock!"); 1007 #endif 1008 INTERLOCK_RELEASE(lkp, LK_SPIN, s); 1009 1010 return (count); 1011 } 1012 1013 /* 1014 * For a recursive spinlock held one or more times by the current CPU, 1015 * release all N locks, and return N. 1016 * Intended for use in mi_switch() right after resuming execution. 1017 */ 1018 1019 void 1020 #if defined(LOCKDEBUG) 1021 _spinlock_acquire_count(volatile struct lock *lkp, int count, 1022 const char *file, int line) 1023 #else 1024 spinlock_acquire_count(volatile struct lock *lkp, int count) 1025 #endif 1026 { 1027 int s, error; 1028 cpuid_t cpu_num; 1029 1030 KASSERT(lkp->lk_flags & LK_SPIN); 1031 1032 INTERLOCK_ACQUIRE(lkp, LK_SPIN, s); 1033 1034 cpu_num = cpu_number(); 1035 1036 #ifdef DIAGNOSTIC 1037 if (WEHOLDIT(lkp, LK_NOPROC, 0, cpu_num)) 1038 lockpanic(lkp, "spinlock_acquire_count: processor %lu already holds lock", (long)cpu_num); 1039 #endif 1040 /* 1041 * Try to acquire the want_exclusive flag. 1042 */ 1043 error = acquire(&lkp, &s, LK_SPIN, 0, LK_HAVE_EXCL | LK_WANT_EXCL, 1044 RETURN_ADDRESS); 1045 lkp->lk_flags |= LK_WANT_EXCL; 1046 /* 1047 * Wait for shared locks and upgrades to finish. 1048 */ 1049 error = acquire(&lkp, &s, LK_SPIN, 0, 1050 LK_HAVE_EXCL | LK_SHARE_NONZERO | LK_WANT_UPGRADE, 1051 RETURN_ADDRESS); 1052 lkp->lk_flags &= ~LK_WANT_EXCL; 1053 lkp->lk_flags |= LK_HAVE_EXCL; 1054 SETHOLDER(lkp, LK_NOPROC, 0, cpu_num); 1055 #if defined(LOCKDEBUG) 1056 lkp->lk_lock_file = file; 1057 lkp->lk_lock_line = line; 1058 #endif 1059 HAVEIT(lkp); 1060 if (lkp->lk_exclusivecount != 0) 1061 lockpanic(lkp, "lockmgr: non-zero exclusive count"); 1062 lkp->lk_exclusivecount = count; 1063 lkp->lk_recurselevel = 1; 1064 COUNT_CPU(cpu_num, count); 1065 1066 INTERLOCK_RELEASE(lkp, lkp->lk_flags, s); 1067 } 1068 1069 1070 1071 /* 1072 * Print out information about state of a lock. Used by VOP_PRINT 1073 * routines to display ststus about contained locks. 1074 */ 1075 void 1076 lockmgr_printinfo(volatile struct lock *lkp) 1077 { 1078 1079 if (lkp->lk_sharecount) 1080 printf(" lock type %s: SHARED (count %d)", lkp->lk_wmesg, 1081 lkp->lk_sharecount); 1082 else if (lkp->lk_flags & LK_HAVE_EXCL) { 1083 printf(" lock type %s: EXCL (count %d) by ", 1084 lkp->lk_wmesg, lkp->lk_exclusivecount); 1085 if (lkp->lk_flags & LK_SPIN) 1086 printf("processor %lu", lkp->lk_cpu); 1087 else 1088 printf("pid %d.%d", lkp->lk_lockholder, 1089 lkp->lk_locklwp); 1090 } else 1091 printf(" not locked"); 1092 if ((lkp->lk_flags & LK_SPIN) == 0 && lkp->lk_waitcount > 0) 1093 printf(" with %d pending", lkp->lk_waitcount); 1094 } 1095 1096 #if defined(LOCKDEBUG) /* { */ 1097 _TAILQ_HEAD(, struct simplelock, volatile) simplelock_list = 1098 TAILQ_HEAD_INITIALIZER(simplelock_list); 1099 1100 #if defined(MULTIPROCESSOR) /* { */ 1101 struct simplelock simplelock_list_slock = SIMPLELOCK_INITIALIZER; 1102 1103 #define SLOCK_LIST_LOCK() \ 1104 __cpu_simple_lock(&simplelock_list_slock.lock_data) 1105 1106 #define SLOCK_LIST_UNLOCK() \ 1107 __cpu_simple_unlock(&simplelock_list_slock.lock_data) 1108 1109 #define SLOCK_COUNT(x) \ 1110 curcpu()->ci_simple_locks += (x) 1111 #else 1112 u_long simple_locks; 1113 1114 #define SLOCK_LIST_LOCK() /* nothing */ 1115 1116 #define SLOCK_LIST_UNLOCK() /* nothing */ 1117 1118 #define SLOCK_COUNT(x) simple_locks += (x) 1119 #endif /* MULTIPROCESSOR */ /* } */ 1120 1121 #ifdef MULTIPROCESSOR 1122 #define SLOCK_MP() lock_printf("on CPU %ld\n", \ 1123 (u_long) cpu_number()) 1124 #else 1125 #define SLOCK_MP() /* nothing */ 1126 #endif 1127 1128 #define SLOCK_WHERE(str, alp, id, l) \ 1129 do { \ 1130 lock_printf("\n"); \ 1131 lock_printf(str); \ 1132 lock_printf("lock: %p, currently at: %s:%d\n", (alp), (id), (l)); \ 1133 SLOCK_MP(); \ 1134 if ((alp)->lock_file != NULL) \ 1135 lock_printf("last locked: %s:%d\n", (alp)->lock_file, \ 1136 (alp)->lock_line); \ 1137 if ((alp)->unlock_file != NULL) \ 1138 lock_printf("last unlocked: %s:%d\n", (alp)->unlock_file, \ 1139 (alp)->unlock_line); \ 1140 SLOCK_TRACE() \ 1141 SLOCK_DEBUGGER(); \ 1142 } while (/*CONSTCOND*/0) 1143 1144 /* 1145 * Simple lock functions so that the debugger can see from whence 1146 * they are being called. 1147 */ 1148 void 1149 simple_lock_init(volatile struct simplelock *alp) 1150 { 1151 1152 #if defined(MULTIPROCESSOR) /* { */ 1153 __cpu_simple_lock_init(&alp->lock_data); 1154 #else 1155 alp->lock_data = __SIMPLELOCK_UNLOCKED; 1156 #endif /* } */ 1157 alp->lock_file = NULL; 1158 alp->lock_line = 0; 1159 alp->unlock_file = NULL; 1160 alp->unlock_line = 0; 1161 alp->lock_holder = LK_NOCPU; 1162 } 1163 1164 void 1165 _simple_lock(volatile struct simplelock *alp, const char *id, int l) 1166 { 1167 cpuid_t cpu_num = cpu_number(); 1168 int s; 1169 1170 s = splhigh(); 1171 1172 /* 1173 * MULTIPROCESSOR case: This is `safe' since if it's not us, we 1174 * don't take any action, and just fall into the normal spin case. 1175 */ 1176 if (alp->lock_data == __SIMPLELOCK_LOCKED) { 1177 #if defined(MULTIPROCESSOR) /* { */ 1178 if (alp->lock_holder == cpu_num) { 1179 SLOCK_WHERE("simple_lock: locking against myself\n", 1180 alp, id, l); 1181 goto out; 1182 } 1183 #else 1184 SLOCK_WHERE("simple_lock: lock held\n", alp, id, l); 1185 goto out; 1186 #endif /* MULTIPROCESSOR */ /* } */ 1187 } 1188 1189 #if defined(MULTIPROCESSOR) /* { */ 1190 /* Acquire the lock before modifying any fields. */ 1191 splx(s); 1192 __cpu_simple_lock(&alp->lock_data); 1193 s = splhigh(); 1194 #else 1195 alp->lock_data = __SIMPLELOCK_LOCKED; 1196 #endif /* } */ 1197 1198 if (alp->lock_holder != LK_NOCPU) { 1199 SLOCK_WHERE("simple_lock: uninitialized lock\n", 1200 alp, id, l); 1201 } 1202 alp->lock_file = id; 1203 alp->lock_line = l; 1204 alp->lock_holder = cpu_num; 1205 1206 SLOCK_LIST_LOCK(); 1207 TAILQ_INSERT_TAIL(&simplelock_list, alp, list); 1208 SLOCK_LIST_UNLOCK(); 1209 1210 SLOCK_COUNT(1); 1211 1212 out: 1213 splx(s); 1214 } 1215 1216 int 1217 _simple_lock_held(volatile struct simplelock *alp) 1218 { 1219 #if defined(MULTIPROCESSOR) || defined(DIAGNOSTIC) 1220 cpuid_t cpu_num = cpu_number(); 1221 #endif 1222 int s, locked = 0; 1223 1224 s = splhigh(); 1225 1226 #if defined(MULTIPROCESSOR) 1227 if (__cpu_simple_lock_try(&alp->lock_data) == 0) 1228 locked = (alp->lock_holder == cpu_num); 1229 else 1230 __cpu_simple_unlock(&alp->lock_data); 1231 #else 1232 if (alp->lock_data == __SIMPLELOCK_LOCKED) { 1233 locked = 1; 1234 KASSERT(alp->lock_holder == cpu_num); 1235 } 1236 #endif 1237 1238 splx(s); 1239 1240 return (locked); 1241 } 1242 1243 int 1244 _simple_lock_try(volatile struct simplelock *alp, const char *id, int l) 1245 { 1246 cpuid_t cpu_num = cpu_number(); 1247 int s, rv = 0; 1248 1249 s = splhigh(); 1250 1251 /* 1252 * MULTIPROCESSOR case: This is `safe' since if it's not us, we 1253 * don't take any action. 1254 */ 1255 #if defined(MULTIPROCESSOR) /* { */ 1256 if ((rv = __cpu_simple_lock_try(&alp->lock_data)) == 0) { 1257 if (alp->lock_holder == cpu_num) 1258 SLOCK_WHERE("simple_lock_try: locking against myself\n", 1259 alp, id, l); 1260 goto out; 1261 } 1262 #else 1263 if (alp->lock_data == __SIMPLELOCK_LOCKED) { 1264 SLOCK_WHERE("simple_lock_try: lock held\n", alp, id, l); 1265 goto out; 1266 } 1267 alp->lock_data = __SIMPLELOCK_LOCKED; 1268 #endif /* MULTIPROCESSOR */ /* } */ 1269 1270 /* 1271 * At this point, we have acquired the lock. 1272 */ 1273 1274 rv = 1; 1275 1276 alp->lock_file = id; 1277 alp->lock_line = l; 1278 alp->lock_holder = cpu_num; 1279 1280 SLOCK_LIST_LOCK(); 1281 TAILQ_INSERT_TAIL(&simplelock_list, alp, list); 1282 SLOCK_LIST_UNLOCK(); 1283 1284 SLOCK_COUNT(1); 1285 1286 out: 1287 splx(s); 1288 return (rv); 1289 } 1290 1291 void 1292 _simple_unlock(volatile struct simplelock *alp, const char *id, int l) 1293 { 1294 int s; 1295 1296 s = splhigh(); 1297 1298 /* 1299 * MULTIPROCESSOR case: This is `safe' because we think we hold 1300 * the lock, and if we don't, we don't take any action. 1301 */ 1302 if (alp->lock_data == __SIMPLELOCK_UNLOCKED) { 1303 SLOCK_WHERE("simple_unlock: lock not held\n", 1304 alp, id, l); 1305 goto out; 1306 } 1307 1308 SLOCK_LIST_LOCK(); 1309 TAILQ_REMOVE(&simplelock_list, alp, list); 1310 SLOCK_LIST_UNLOCK(); 1311 1312 SLOCK_COUNT(-1); 1313 1314 alp->list.tqe_next = NULL; /* sanity */ 1315 alp->list.tqe_prev = NULL; /* sanity */ 1316 1317 alp->unlock_file = id; 1318 alp->unlock_line = l; 1319 1320 #if defined(MULTIPROCESSOR) /* { */ 1321 alp->lock_holder = LK_NOCPU; 1322 /* Now that we've modified all fields, release the lock. */ 1323 __cpu_simple_unlock(&alp->lock_data); 1324 #else 1325 alp->lock_data = __SIMPLELOCK_UNLOCKED; 1326 KASSERT(alp->lock_holder == cpu_number()); 1327 alp->lock_holder = LK_NOCPU; 1328 #endif /* } */ 1329 1330 out: 1331 splx(s); 1332 } 1333 1334 void 1335 simple_lock_dump(void) 1336 { 1337 volatile struct simplelock *alp; 1338 int s; 1339 1340 s = splhigh(); 1341 SLOCK_LIST_LOCK(); 1342 lock_printf("all simple locks:\n"); 1343 TAILQ_FOREACH(alp, &simplelock_list, list) { 1344 lock_printf("%p CPU %lu %s:%d\n", alp, alp->lock_holder, 1345 alp->lock_file, alp->lock_line); 1346 } 1347 SLOCK_LIST_UNLOCK(); 1348 splx(s); 1349 } 1350 1351 void 1352 simple_lock_freecheck(void *start, void *end) 1353 { 1354 volatile struct simplelock *alp; 1355 int s; 1356 1357 s = splhigh(); 1358 SLOCK_LIST_LOCK(); 1359 TAILQ_FOREACH(alp, &simplelock_list, list) { 1360 if ((volatile void *)alp >= start && 1361 (volatile void *)alp < end) { 1362 lock_printf("freeing simple_lock %p CPU %lu %s:%d\n", 1363 alp, alp->lock_holder, alp->lock_file, 1364 alp->lock_line); 1365 SLOCK_DEBUGGER(); 1366 } 1367 } 1368 SLOCK_LIST_UNLOCK(); 1369 splx(s); 1370 } 1371 1372 /* 1373 * We must be holding exactly one lock: the spc_lock. 1374 */ 1375 1376 void 1377 simple_lock_switchcheck(void) 1378 { 1379 1380 simple_lock_only_held(NULL, "switching"); 1381 } 1382 1383 /* 1384 * Drop into the debugger if lp isn't the only lock held. 1385 * lp may be NULL. 1386 */ 1387 void 1388 simple_lock_only_held(volatile struct simplelock *lp, const char *where) 1389 { 1390 volatile struct simplelock *alp; 1391 cpuid_t cpu_num = cpu_number(); 1392 int s; 1393 1394 if (lp) { 1395 LOCK_ASSERT(simple_lock_held(lp)); 1396 } 1397 s = splhigh(); 1398 SLOCK_LIST_LOCK(); 1399 TAILQ_FOREACH(alp, &simplelock_list, list) { 1400 if (alp == lp) 1401 continue; 1402 if (alp->lock_holder == cpu_num) 1403 break; 1404 } 1405 SLOCK_LIST_UNLOCK(); 1406 splx(s); 1407 1408 if (alp != NULL) { 1409 lock_printf("\n%s with held simple_lock %p " 1410 "CPU %lu %s:%d\n", 1411 where, alp, alp->lock_holder, alp->lock_file, 1412 alp->lock_line); 1413 SLOCK_TRACE(); 1414 SLOCK_DEBUGGER(); 1415 } 1416 } 1417 1418 /* 1419 * Set to 1 by simple_lock_assert_*(). 1420 * Can be cleared from ddb to avoid a panic. 1421 */ 1422 int slock_assert_will_panic; 1423 1424 /* 1425 * If the lock isn't held, print a traceback, optionally drop into the 1426 * debugger, then panic. 1427 * The panic can be avoided by clearing slock_assert_with_panic from the 1428 * debugger. 1429 */ 1430 void 1431 _simple_lock_assert_locked(volatile struct simplelock *alp, 1432 const char *lockname, const char *id, int l) 1433 { 1434 if (simple_lock_held(alp) == 0) { 1435 slock_assert_will_panic = 1; 1436 lock_printf("%s lock not held\n", lockname); 1437 SLOCK_WHERE("lock not held", alp, id, l); 1438 if (slock_assert_will_panic && panicstr == NULL) 1439 panic("%s: not locked", lockname); 1440 } 1441 } 1442 1443 void 1444 _simple_lock_assert_unlocked(volatile struct simplelock *alp, 1445 const char *lockname, const char *id, int l) 1446 { 1447 if (simple_lock_held(alp)) { 1448 slock_assert_will_panic = 1; 1449 lock_printf("%s lock held\n", lockname); 1450 SLOCK_WHERE("lock held", alp, id, l); 1451 if (slock_assert_will_panic && panicstr == NULL) 1452 panic("%s: locked", lockname); 1453 } 1454 } 1455 1456 void 1457 assert_sleepable(struct simplelock *interlock, const char *msg) 1458 { 1459 1460 if (panicstr != NULL) 1461 return; 1462 if (CURCPU_IDLE_P()) { 1463 panic("assert_sleepable: idle"); 1464 } 1465 simple_lock_only_held(interlock, msg); 1466 } 1467 1468 #endif /* LOCKDEBUG */ /* } */ 1469 1470 int kernel_lock_id; 1471 __cpu_simple_lock_t kernel_lock; 1472 1473 #if defined(MULTIPROCESSOR) 1474 1475 /* 1476 * Functions for manipulating the kernel_lock. We put them here 1477 * so that they show up in profiles. 1478 */ 1479 1480 #define _KERNEL_LOCK_ABORT(msg) \ 1481 LOCKDEBUG_ABORT(kernel_lock_id, &kernel_lock, &_kernel_lock_ops, \ 1482 __FUNCTION__, msg) 1483 1484 #ifdef LOCKDEBUG 1485 #define _KERNEL_LOCK_ASSERT(cond) \ 1486 do { \ 1487 if (!(cond)) \ 1488 _KERNEL_LOCK_ABORT("assertion failed: " #cond); \ 1489 } while (/* CONSTCOND */ 0) 1490 #else 1491 #define _KERNEL_LOCK_ASSERT(cond) /* nothing */ 1492 #endif 1493 1494 void _kernel_lock_dump(volatile void *); 1495 1496 lockops_t _kernel_lock_ops = { 1497 "Kernel lock", 1498 0, 1499 _kernel_lock_dump 1500 }; 1501 1502 /* 1503 * Initialize the kernel lock. 1504 */ 1505 void 1506 _kernel_lock_init(void) 1507 { 1508 1509 __cpu_simple_lock_init(&kernel_lock); 1510 kernel_lock_id = LOCKDEBUG_ALLOC(&kernel_lock, &_kernel_lock_ops); 1511 } 1512 1513 /* 1514 * Print debugging information about the kernel lock. 1515 */ 1516 void 1517 _kernel_lock_dump(volatile void *junk) 1518 { 1519 struct cpu_info *ci = curcpu(); 1520 1521 (void)junk; 1522 1523 printf_nolog("curcpu holds : %18d wanted by: %#018lx\n", 1524 ci->ci_biglock_count, (long)ci->ci_biglock_wanted); 1525 } 1526 1527 /* 1528 * Acquire 'nlocks' holds on the kernel lock. If 'l' is non-null, the 1529 * acquisition is from process context. 1530 */ 1531 void 1532 _kernel_lock(int nlocks, struct lwp *l) 1533 { 1534 struct cpu_info *ci = curcpu(); 1535 LOCKSTAT_TIMER(spintime); 1536 LOCKSTAT_FLAG(lsflag); 1537 struct lwp *owant; 1538 #ifdef LOCKDEBUG 1539 u_int spins; 1540 #endif 1541 int s; 1542 1543 (void)l; 1544 1545 if (nlocks == 0) 1546 return; 1547 _KERNEL_LOCK_ASSERT(nlocks > 0); 1548 1549 s = splsched(); /* XXX splvm() */ 1550 1551 if (ci->ci_biglock_count != 0) { 1552 _KERNEL_LOCK_ASSERT(kernel_lock == __SIMPLELOCK_LOCKED); 1553 ci->ci_biglock_count += nlocks; 1554 splx(s); 1555 return; 1556 } 1557 1558 LOCKDEBUG_WANTLOCK(kernel_lock_id, 1559 (uintptr_t)__builtin_return_address(0), 0); 1560 1561 if (__cpu_simple_lock_try(&kernel_lock)) { 1562 ci->ci_biglock_count = nlocks; 1563 LOCKDEBUG_LOCKED(kernel_lock_id, 1564 (uintptr_t)__builtin_return_address(0), 0); 1565 splx(s); 1566 return; 1567 } 1568 1569 LOCKSTAT_ENTER(lsflag); 1570 LOCKSTAT_START_TIMER(lsflag, spintime); 1571 1572 /* 1573 * Before setting ci_biglock_wanted we must post a store 1574 * fence (see kern_mutex.c). This is accomplished by the 1575 * __cpu_simple_lock_try() above. 1576 */ 1577 owant = ci->ci_biglock_wanted; 1578 ci->ci_biglock_wanted = curlwp; /* XXXAD */ 1579 1580 #ifdef LOCKDEBUG 1581 spins = 0; 1582 #endif 1583 1584 do { 1585 while (kernel_lock == __SIMPLELOCK_LOCKED) { 1586 #ifdef LOCKDEBUG 1587 if (SPINLOCK_SPINOUT(spins)) 1588 _KERNEL_LOCK_ABORT("spinout"); 1589 #endif 1590 splx(s); 1591 SPINLOCK_SPIN_HOOK; 1592 (void)splsched(); /* XXX splvm() */ 1593 } 1594 } while (!__cpu_simple_lock_try(&kernel_lock)); 1595 1596 ci->ci_biglock_wanted = owant; 1597 ci->ci_biglock_count += nlocks; 1598 LOCKSTAT_STOP_TIMER(lsflag, spintime); 1599 LOCKDEBUG_LOCKED(kernel_lock_id, 1600 (uintptr_t)__builtin_return_address(0), 0); 1601 splx(s); 1602 1603 /* 1604 * Again, another store fence is required (see kern_mutex.c). 1605 */ 1606 mb_write(); 1607 if (owant == NULL) { 1608 LOCKSTAT_EVENT(lsflag, &kernel_lock, LB_KERNEL_LOCK | LB_SPIN, 1609 1, spintime); 1610 } 1611 LOCKSTAT_EXIT(lsflag); 1612 } 1613 1614 /* 1615 * Release 'nlocks' holds on the kernel lock. If 'nlocks' is zero, release 1616 * all holds. If 'l' is non-null, the release is from process context. 1617 */ 1618 void 1619 _kernel_unlock(int nlocks, struct lwp *l, int *countp) 1620 { 1621 struct cpu_info *ci = curcpu(); 1622 u_int olocks; 1623 int s; 1624 1625 (void)l; 1626 1627 _KERNEL_LOCK_ASSERT(nlocks < 2); 1628 1629 olocks = ci->ci_biglock_count; 1630 1631 if (olocks == 0) { 1632 _KERNEL_LOCK_ASSERT(nlocks <= 0); 1633 if (countp != NULL) 1634 *countp = 0; 1635 return; 1636 } 1637 1638 _KERNEL_LOCK_ASSERT(kernel_lock == __SIMPLELOCK_LOCKED); 1639 1640 if (nlocks == 0) 1641 nlocks = olocks; 1642 else if (nlocks == -1) { 1643 nlocks = 1; 1644 _KERNEL_LOCK_ASSERT(olocks == 1); 1645 } 1646 1647 s = splsched(); /* XXX splvm() */ 1648 if ((ci->ci_biglock_count -= nlocks) == 0) { 1649 LOCKDEBUG_UNLOCKED(kernel_lock_id, 1650 (uintptr_t)__builtin_return_address(0), 0); 1651 __cpu_simple_unlock(&kernel_lock); 1652 } 1653 splx(s); 1654 1655 if (countp != NULL) 1656 *countp = olocks; 1657 } 1658 1659 #if defined(DEBUG) 1660 /* 1661 * Assert that the kernel lock is held. 1662 */ 1663 void 1664 _kernel_lock_assert_locked(void) 1665 { 1666 1667 if (kernel_lock != __SIMPLELOCK_LOCKED || 1668 curcpu()->ci_biglock_count == 0) 1669 _KERNEL_LOCK_ABORT("not locked"); 1670 } 1671 1672 void 1673 _kernel_lock_assert_unlocked() 1674 { 1675 1676 if (curcpu()->ci_biglock_count != 0) 1677 _KERNEL_LOCK_ABORT("locked"); 1678 } 1679 #endif 1680 1681 #endif /* MULTIPROCESSOR || LOCKDEBUG */ 1682
Indexes created Mon Sep 22 05:09:51 GMT 2025