kern_lock.c revision 1.131
1 /* $NetBSD: kern_lock.c,v 1.131 2008/01/04 21:18:08 ad Exp $ */ 2 3 /*- 4 * Copyright (c) 1999, 2000, 2006, 2007 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center, and by Andrew Doran. 10 * 11 * This code is derived from software contributed to The NetBSD Foundation 12 * by Ross Harvey. 13 * 14 * Redistribution and use in source and binary forms, with or without 15 * modification, are permitted provided that the following conditions 16 * are met: 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in the 21 * documentation and/or other materials provided with the distribution. 22 * 3. All advertising materials mentioning features or use of this software 23 * must display the following acknowledgement: 24 * This product includes software developed by the NetBSD 25 * Foundation, Inc. and its contributors. 26 * 4. Neither the name of The NetBSD Foundation nor the names of its 27 * contributors may be used to endorse or promote products derived 28 * from this software without specific prior written permission. 29 * 30 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 31 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 32 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 33 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 34 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 35 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 36 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 37 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 38 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 39 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 40 * POSSIBILITY OF SUCH DAMAGE. 41 */ 42 43 /* 44 * Copyright (c) 1995 45 * The Regents of the University of California. All rights reserved. 46 * 47 * This code contains ideas from software contributed to Berkeley by 48 * Avadis Tevanian, Jr., Michael Wayne Young, and the Mach Operating 49 * System project at Carnegie-Mellon University. 50 * 51 * Redistribution and use in source and binary forms, with or without 52 * modification, are permitted provided that the following conditions 53 * are met: 54 * 1. Redistributions of source code must retain the above copyright 55 * notice, this list of conditions and the following disclaimer. 56 * 2. Redistributions in binary form must reproduce the above copyright 57 * notice, this list of conditions and the following disclaimer in the 58 * documentation and/or other materials provided with the distribution. 59 * 3. Neither the name of the University nor the names of its contributors 60 * may be used to endorse or promote products derived from this software 61 * without specific prior written permission. 62 * 63 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 64 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 65 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 66 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 67 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 68 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 69 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 70 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 71 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 72 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 73 * SUCH DAMAGE. 74 * 75 * @(#)kern_lock.c 8.18 (Berkeley) 5/21/95 76 */ 77 78 #include <sys/cdefs.h> 79 __KERNEL_RCSID(0, "$NetBSD: kern_lock.c,v 1.131 2008/01/04 21:18:08 ad Exp $"); 80 81 #include "opt_multiprocessor.h" 82 83 #include <sys/param.h> 84 #include <sys/proc.h> 85 #include <sys/lock.h> 86 #include <sys/systm.h> 87 #include <sys/kernel.h> 88 #include <sys/lockdebug.h> 89 #include <sys/cpu.h> 90 #include <sys/syslog.h> 91 #include <sys/atomic.h> 92 93 #include <machine/stdarg.h> 94 #include <machine/lock.h> 95 96 #include <dev/lockstat.h> 97 98 /* 99 * note that stdarg.h and the ansi style va_start macro is used for both 100 * ansi and traditional c compiles. 101 * XXX: this requires that stdarg.h define: va_alist and va_dcl 102 */ 103 void lock_printf(const char *fmt, ...) 104 __attribute__((__format__(__printf__,1,2))); 105 106 static int acquire(struct lock **, int *, int, int, int, uintptr_t); 107 108 int lock_debug_syslog = 0; /* defaults to printf, but can be patched */ 109 bool kernel_lock_dodebug; 110 __cpu_simple_lock_t kernel_lock; 111 112 #ifdef LOCKDEBUG 113 static lockops_t lockmgr_lockops = { 114 "lockmgr", 115 1, 116 (void *)nullop 117 }; 118 #endif 119 120 #if defined(LOCKDEBUG) || defined(DIAGNOSTIC) /* { */ 121 #define COUNT(lkp, l, cpu_id, x) (l)->l_locks += (x) 122 #else 123 #define COUNT(lkp, p, cpu_id, x) 124 #endif /* LOCKDEBUG || DIAGNOSTIC */ /* } */ 125 126 #define RETURN_ADDRESS ((uintptr_t)__builtin_return_address(0)) 127 128 /* 129 * Acquire a resource. 130 */ 131 static int 132 acquire(struct lock **lkpp, int *s, int extflags, 133 int drain, int wanted, uintptr_t ra) 134 { 135 int error; 136 struct lock *lkp = *lkpp; 137 LOCKSTAT_TIMER(slptime); 138 LOCKSTAT_FLAG(lsflag); 139 140 KASSERT(drain || (wanted & LK_WAIT_NONZERO) == 0); 141 142 LOCKSTAT_ENTER(lsflag); 143 144 for (error = 0; (lkp->lk_flags & wanted) != 0; ) { 145 if (drain) 146 lkp->lk_flags |= LK_WAITDRAIN; 147 else { 148 lkp->lk_waitcount++; 149 lkp->lk_flags |= LK_WAIT_NONZERO; 150 } 151 LOCKSTAT_START_TIMER(lsflag, slptime); 152 error = mtsleep(drain ? (void *)&lkp->lk_flags : (void *)lkp, 153 lkp->lk_prio, lkp->lk_wmesg, lkp->lk_timo, 154 __UNVOLATILE(&lkp->lk_interlock)); 155 LOCKSTAT_STOP_TIMER(lsflag, slptime); 156 LOCKSTAT_EVENT_RA(lsflag, (void *)(uintptr_t)lkp, 157 LB_LOCKMGR | LB_SLEEP1, 1, slptime, ra); 158 if (!drain) { 159 lkp->lk_waitcount--; 160 if (lkp->lk_waitcount == 0) 161 lkp->lk_flags &= ~LK_WAIT_NONZERO; 162 } 163 if (error) 164 break; 165 if (extflags & LK_SLEEPFAIL) { 166 error = ENOLCK; 167 break; 168 } 169 } 170 171 LOCKSTAT_EXIT(lsflag); 172 173 return error; 174 } 175 176 #define SETHOLDER(lkp, pid, lid, cpu_id) \ 177 do { \ 178 (lkp)->lk_lockholder = pid; \ 179 (lkp)->lk_locklwp = lid; \ 180 } while (/*CONSTCOND*/0) 181 182 #define WEHOLDIT(lkp, pid, lid, cpu_id) \ 183 ((lkp)->lk_lockholder == (pid) && (lkp)->lk_locklwp == (lid)) 184 185 #define WAKEUP_WAITER(lkp) \ 186 do { \ 187 if (((lkp)->lk_flags & LK_WAIT_NONZERO) != 0) { \ 188 wakeup((lkp)); \ 189 } \ 190 } while (/*CONSTCOND*/0) 191 192 #if defined(LOCKDEBUG) 193 /* 194 * Lock debug printing routine; can be configured to print to console 195 * or log to syslog. 196 */ 197 void 198 lock_printf(const char *fmt, ...) 199 { 200 char b[150]; 201 va_list ap; 202 203 va_start(ap, fmt); 204 if (lock_debug_syslog) 205 vlog(LOG_DEBUG, fmt, ap); 206 else { 207 vsnprintf(b, sizeof(b), fmt, ap); 208 printf_nolog("%s", b); 209 } 210 va_end(ap); 211 } 212 #endif /* LOCKDEBUG */ 213 214 static void 215 lockpanic(struct lock *lkp, const char *fmt, ...) 216 { 217 char s[150], b[150]; 218 static const char *locktype[] = { 219 "*0*", "shared", "exclusive", "*3*", "*4*", "downgrade", 220 "*release*", "drain", "exclother", "*9*", "*10*", 221 "*11*", "*12*", "*13*", "*14*", "*15*" 222 }; 223 va_list ap; 224 va_start(ap, fmt); 225 vsnprintf(s, sizeof(s), fmt, ap); 226 va_end(ap); 227 bitmask_snprintf(lkp->lk_flags, __LK_FLAG_BITS, b, sizeof(b)); 228 panic("%s (" 229 "type %s flags %s, sharecount %d, exclusivecount %d, " 230 "recurselevel %d, waitcount %d, wmesg %s" 231 ", lock_addr %p, unlock_addr %p" 232 ")\n", 233 s, locktype[lkp->lk_flags & LK_TYPE_MASK], 234 b, lkp->lk_sharecount, lkp->lk_exclusivecount, 235 lkp->lk_recurselevel, lkp->lk_waitcount, lkp->lk_wmesg, 236 (void *)lkp->lk_lock_addr, (void *)lkp->lk_unlock_addr 237 ); 238 } 239 240 /* 241 * Initialize a lock; required before use. 242 */ 243 void 244 lockinit(struct lock *lkp, pri_t prio, const char *wmesg, int timo, int flags) 245 { 246 247 memset(lkp, 0, sizeof(struct lock)); 248 lkp->lk_flags = flags & LK_EXTFLG_MASK; 249 mutex_init(&lkp->lk_interlock, MUTEX_DEFAULT, IPL_NONE); 250 lkp->lk_lockholder = LK_NOPROC; 251 lkp->lk_prio = prio; 252 lkp->lk_timo = timo; 253 lkp->lk_wmesg = wmesg; 254 lkp->lk_lock_addr = 0; 255 lkp->lk_unlock_addr = 0; 256 257 if (LOCKDEBUG_ALLOC(lkp, &lockmgr_lockops, 258 (uintptr_t)__builtin_return_address(0))) { 259 lkp->lk_flags |= LK_DODEBUG; 260 } 261 } 262 263 void 264 lockdestroy(struct lock *lkp) 265 { 266 267 LOCKDEBUG_FREE(((lkp->lk_flags & LK_DODEBUG) != 0), lkp); 268 mutex_destroy(&lkp->lk_interlock); 269 } 270 271 /* 272 * Determine the status of a lock. 273 */ 274 int 275 lockstatus(struct lock *lkp) 276 { 277 int lock_type = 0; 278 struct lwp *l = curlwp; /* XXX */ 279 pid_t pid; 280 lwpid_t lid; 281 cpuid_t cpu_num; 282 283 if (l == NULL) { 284 cpu_num = cpu_number(); 285 pid = LK_KERNPROC; 286 lid = 0; 287 } else { 288 cpu_num = LK_NOCPU; 289 pid = l->l_proc->p_pid; 290 lid = l->l_lid; 291 } 292 293 mutex_enter(&lkp->lk_interlock); 294 if (lkp->lk_exclusivecount != 0) { 295 if (WEHOLDIT(lkp, pid, lid, cpu_num)) 296 lock_type = LK_EXCLUSIVE; 297 else 298 lock_type = LK_EXCLOTHER; 299 } else if (lkp->lk_sharecount != 0) 300 lock_type = LK_SHARED; 301 else if (lkp->lk_flags & LK_WANT_EXCL) 302 lock_type = LK_EXCLOTHER; 303 mutex_exit(&lkp->lk_interlock); 304 return (lock_type); 305 } 306 307 /* 308 * XXX XXX kludge around another kludge.. 309 * 310 * vfs_shutdown() may be called from interrupt context, either as a result 311 * of a panic, or from the debugger. It proceeds to call 312 * sys_sync(&proc0, ...), pretending its running on behalf of proc0 313 * 314 * We would like to make an attempt to sync the filesystems in this case, so 315 * if this happens, we treat attempts to acquire locks specially. 316 * All locks are acquired on behalf of proc0. 317 * 318 * If we've already paniced, we don't block waiting for locks, but 319 * just barge right ahead since we're already going down in flames. 320 */ 321 322 /* 323 * Set, change, or release a lock. 324 * 325 * Shared requests increment the shared count. Exclusive requests set the 326 * LK_WANT_EXCL flag (preventing further shared locks), and wait for already 327 * accepted shared locks to go away. 328 */ 329 int 330 lockmgr(struct lock *lkp, u_int flags, kmutex_t *interlkp) 331 { 332 int error; 333 pid_t pid; 334 lwpid_t lid; 335 int extflags; 336 cpuid_t cpu_num; 337 struct lwp *l = curlwp; 338 int lock_shutdown_noblock = 0; 339 int s = 0; 340 341 error = 0; 342 343 /* LK_RETRY is for vn_lock, not for lockmgr. */ 344 KASSERT((flags & LK_RETRY) == 0); 345 KASSERT((l->l_pflag & LP_INTR) == 0 || panicstr != NULL); 346 347 mutex_enter(&lkp->lk_interlock); 348 if (flags & LK_INTERLOCK) 349 mutex_exit(interlkp); 350 extflags = (flags | lkp->lk_flags) & LK_EXTFLG_MASK; 351 352 if (l == NULL) { 353 if (!doing_shutdown) { 354 panic("lockmgr: no context"); 355 } else { 356 l = &lwp0; 357 if (panicstr && (!(flags & LK_NOWAIT))) { 358 flags |= LK_NOWAIT; 359 lock_shutdown_noblock = 1; 360 } 361 } 362 } 363 lid = l->l_lid; 364 pid = l->l_proc->p_pid; 365 cpu_num = cpu_number(); 366 367 /* 368 * Once a lock has drained, the LK_DRAINING flag is set and an 369 * exclusive lock is returned. The only valid operation thereafter 370 * is a single release of that exclusive lock. This final release 371 * clears the LK_DRAINING flag and sets the LK_DRAINED flag. Any 372 * further requests of any sort will result in a panic. The bits 373 * selected for these two flags are chosen so that they will be set 374 * in memory that is freed (freed memory is filled with 0xdeadbeef). 375 * The final release is permitted to give a new lease on life to 376 * the lock by specifying LK_REENABLE. 377 */ 378 if (lkp->lk_flags & (LK_DRAINING|LK_DRAINED)) { 379 #ifdef DIAGNOSTIC /* { */ 380 if (lkp->lk_flags & LK_DRAINED) 381 lockpanic(lkp, "lockmgr: using decommissioned lock"); 382 if ((flags & LK_TYPE_MASK) != LK_RELEASE || 383 WEHOLDIT(lkp, pid, lid, cpu_num) == 0) 384 lockpanic(lkp, "lockmgr: non-release on draining lock: %d", 385 flags & LK_TYPE_MASK); 386 #endif /* DIAGNOSTIC */ /* } */ 387 lkp->lk_flags &= ~LK_DRAINING; 388 if ((flags & LK_REENABLE) == 0) 389 lkp->lk_flags |= LK_DRAINED; 390 } 391 392 switch (flags & LK_TYPE_MASK) { 393 394 case LK_SHARED: 395 if (WEHOLDIT(lkp, pid, lid, cpu_num) == 0) { 396 /* 397 * If just polling, check to see if we will block. 398 */ 399 if ((extflags & LK_NOWAIT) && (lkp->lk_flags & 400 (LK_HAVE_EXCL | LK_WANT_EXCL))) { 401 error = EBUSY; 402 break; 403 } 404 /* 405 * Wait for exclusive locks to clear. 406 */ 407 error = acquire(&lkp, &s, extflags, 0, 408 LK_HAVE_EXCL | LK_WANT_EXCL, 409 RETURN_ADDRESS); 410 if (error) 411 break; 412 lkp->lk_sharecount++; 413 lkp->lk_flags |= LK_SHARE_NONZERO; 414 COUNT(lkp, l, cpu_num, 1); 415 break; 416 } 417 /* 418 * We hold an exclusive lock, so downgrade it to shared. 419 * An alternative would be to fail with EDEADLK. 420 */ 421 lkp->lk_sharecount++; 422 lkp->lk_flags |= LK_SHARE_NONZERO; 423 COUNT(lkp, l, cpu_num, 1); 424 /* fall into downgrade */ 425 426 case LK_DOWNGRADE: 427 if (WEHOLDIT(lkp, pid, lid, cpu_num) == 0 || 428 lkp->lk_exclusivecount == 0) 429 lockpanic(lkp, "lockmgr: not holding exclusive lock"); 430 lkp->lk_sharecount += lkp->lk_exclusivecount; 431 lkp->lk_flags |= LK_SHARE_NONZERO; 432 lkp->lk_exclusivecount = 0; 433 lkp->lk_recurselevel = 0; 434 lkp->lk_flags &= ~LK_HAVE_EXCL; 435 SETHOLDER(lkp, LK_NOPROC, 0, LK_NOCPU); 436 #if defined(LOCKDEBUG) 437 lkp->lk_unlock_addr = RETURN_ADDRESS; 438 #endif 439 WAKEUP_WAITER(lkp); 440 break; 441 442 case LK_EXCLUSIVE: 443 if (WEHOLDIT(lkp, pid, lid, cpu_num)) { 444 /* 445 * Recursive lock. 446 */ 447 if ((extflags & LK_CANRECURSE) == 0 && 448 lkp->lk_recurselevel == 0) { 449 if (extflags & LK_RECURSEFAIL) { 450 error = EDEADLK; 451 break; 452 } else 453 lockpanic(lkp, "lockmgr: locking against myself"); 454 } 455 lkp->lk_exclusivecount++; 456 COUNT(lkp, l, cpu_num, 1); 457 break; 458 } 459 /* 460 * If we are just polling, check to see if we will sleep. 461 */ 462 if ((extflags & LK_NOWAIT) && (lkp->lk_flags & 463 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_SHARE_NONZERO))) { 464 error = EBUSY; 465 break; 466 } 467 /* 468 * Try to acquire the want_exclusive flag. 469 */ 470 error = acquire(&lkp, &s, extflags, 0, 471 LK_HAVE_EXCL | LK_WANT_EXCL, RETURN_ADDRESS); 472 if (error) 473 break; 474 lkp->lk_flags |= LK_WANT_EXCL; 475 /* 476 * Wait for shared locks to finish. 477 */ 478 error = acquire(&lkp, &s, extflags, 0, 479 LK_HAVE_EXCL | LK_SHARE_NONZERO, 480 RETURN_ADDRESS); 481 lkp->lk_flags &= ~LK_WANT_EXCL; 482 if (error) { 483 WAKEUP_WAITER(lkp); 484 break; 485 } 486 lkp->lk_flags |= LK_HAVE_EXCL; 487 SETHOLDER(lkp, pid, lid, cpu_num); 488 #if defined(LOCKDEBUG) 489 lkp->lk_lock_addr = RETURN_ADDRESS; 490 #endif 491 if (lkp->lk_exclusivecount != 0) 492 lockpanic(lkp, "lockmgr: non-zero exclusive count"); 493 lkp->lk_exclusivecount = 1; 494 COUNT(lkp, l, cpu_num, 1); 495 break; 496 497 case LK_RELEASE: 498 if (lkp->lk_exclusivecount != 0) { 499 if (WEHOLDIT(lkp, pid, lid, cpu_num) == 0) { 500 lockpanic(lkp, "lockmgr: pid %d.%d, not " 501 "exclusive lock holder %d.%d " 502 "unlocking", pid, lid, 503 lkp->lk_lockholder, 504 lkp->lk_locklwp); 505 } 506 if (lkp->lk_exclusivecount == lkp->lk_recurselevel) 507 lkp->lk_recurselevel = 0; 508 lkp->lk_exclusivecount--; 509 COUNT(lkp, l, cpu_num, -1); 510 if (lkp->lk_exclusivecount == 0) { 511 lkp->lk_flags &= ~LK_HAVE_EXCL; 512 SETHOLDER(lkp, LK_NOPROC, 0, LK_NOCPU); 513 #if defined(LOCKDEBUG) 514 lkp->lk_unlock_addr = RETURN_ADDRESS; 515 #endif 516 } 517 } else if (lkp->lk_sharecount != 0) { 518 lkp->lk_sharecount--; 519 if (lkp->lk_sharecount == 0) 520 lkp->lk_flags &= ~LK_SHARE_NONZERO; 521 COUNT(lkp, l, cpu_num, -1); 522 } 523 #ifdef DIAGNOSTIC 524 else 525 lockpanic(lkp, "lockmgr: release of unlocked lock!"); 526 #endif 527 WAKEUP_WAITER(lkp); 528 break; 529 530 case LK_DRAIN: 531 /* 532 * Check that we do not already hold the lock, as it can 533 * never drain if we do. Unfortunately, we have no way to 534 * check for holding a shared lock, but at least we can 535 * check for an exclusive one. 536 */ 537 if (WEHOLDIT(lkp, pid, lid, cpu_num)) 538 lockpanic(lkp, "lockmgr: draining against myself"); 539 /* 540 * If we are just polling, check to see if we will sleep. 541 */ 542 if ((extflags & LK_NOWAIT) && (lkp->lk_flags & 543 (LK_HAVE_EXCL | LK_WANT_EXCL | 544 LK_SHARE_NONZERO | LK_WAIT_NONZERO))) { 545 error = EBUSY; 546 break; 547 } 548 error = acquire(&lkp, &s, extflags, 1, 549 LK_HAVE_EXCL | LK_WANT_EXCL | 550 LK_SHARE_NONZERO | LK_WAIT_NONZERO, 551 RETURN_ADDRESS); 552 if (error) 553 break; 554 lkp->lk_flags |= LK_HAVE_EXCL; 555 if ((extflags & LK_RESURRECT) == 0) 556 lkp->lk_flags |= LK_DRAINING; 557 SETHOLDER(lkp, pid, lid, cpu_num); 558 #if defined(LOCKDEBUG) 559 lkp->lk_lock_addr = RETURN_ADDRESS; 560 #endif 561 lkp->lk_exclusivecount = 1; 562 COUNT(lkp, l, cpu_num, 1); 563 break; 564 565 default: 566 mutex_exit(&lkp->lk_interlock); 567 lockpanic(lkp, "lockmgr: unknown locktype request %d", 568 flags & LK_TYPE_MASK); 569 /* NOTREACHED */ 570 } 571 if ((lkp->lk_flags & LK_WAITDRAIN) != 0 && 572 ((lkp->lk_flags & 573 (LK_HAVE_EXCL | LK_WANT_EXCL | 574 LK_SHARE_NONZERO | LK_WAIT_NONZERO)) == 0)) { 575 lkp->lk_flags &= ~LK_WAITDRAIN; 576 wakeup(&lkp->lk_flags); 577 } 578 /* 579 * Note that this panic will be a recursive panic, since 580 * we only set lock_shutdown_noblock above if panicstr != NULL. 581 */ 582 if (error && lock_shutdown_noblock) 583 lockpanic(lkp, "lockmgr: deadlock (see previous panic)"); 584 585 mutex_exit(&lkp->lk_interlock); 586 return (error); 587 } 588 589 /* 590 * Print out information about state of a lock. Used by VOP_PRINT 591 * routines to display ststus about contained locks. 592 */ 593 void 594 lockmgr_printinfo(struct lock *lkp) 595 { 596 597 if (lkp->lk_sharecount) 598 printf(" lock type %s: SHARED (count %d)", lkp->lk_wmesg, 599 lkp->lk_sharecount); 600 else if (lkp->lk_flags & LK_HAVE_EXCL) { 601 printf(" lock type %s: EXCL (count %d) by ", 602 lkp->lk_wmesg, lkp->lk_exclusivecount); 603 printf("pid %d.%d", lkp->lk_lockholder, 604 lkp->lk_locklwp); 605 } else 606 printf(" not locked"); 607 if (lkp->lk_waitcount > 0) 608 printf(" with %d pending", lkp->lk_waitcount); 609 } 610 611 #if defined(LOCKDEBUG) 612 void 613 assert_sleepable(struct simplelock *interlock, const char *msg) 614 { 615 616 if (panicstr != NULL) 617 return; 618 LOCKDEBUG_BARRIER(&kernel_lock, 1); 619 if (CURCPU_IDLE_P() && !cold) { 620 panic("assert_sleepable: idle"); 621 } 622 } 623 #endif 624 625 /* 626 * rump doesn't need the kernel lock so force it out. We cannot 627 * currently easily include it for compilation because of 628 * a) SPINLOCK_* b) membar_producer(). They are defined in different 629 * places / way for each arch, so just simply do not bother to 630 * fight a lot for no gain (i.e. pain but still no gain). 631 */ 632 #ifndef _RUMPKERNEL 633 /* 634 * Functions for manipulating the kernel_lock. We put them here 635 * so that they show up in profiles. 636 */ 637 638 #define _KERNEL_LOCK_ABORT(msg) \ 639 LOCKDEBUG_ABORT(&kernel_lock, &_kernel_lock_ops, __func__, msg) 640 641 #ifdef LOCKDEBUG 642 #define _KERNEL_LOCK_ASSERT(cond) \ 643 do { \ 644 if (!(cond)) \ 645 _KERNEL_LOCK_ABORT("assertion failed: " #cond); \ 646 } while (/* CONSTCOND */ 0) 647 #else 648 #define _KERNEL_LOCK_ASSERT(cond) /* nothing */ 649 #endif 650 651 void _kernel_lock_dump(volatile void *); 652 653 lockops_t _kernel_lock_ops = { 654 "Kernel lock", 655 0, 656 _kernel_lock_dump 657 }; 658 659 /* 660 * Initialize the kernel lock. 661 */ 662 void 663 kernel_lock_init(void) 664 { 665 666 __cpu_simple_lock_init(&kernel_lock); 667 kernel_lock_dodebug = LOCKDEBUG_ALLOC(&kernel_lock, &_kernel_lock_ops, 668 RETURN_ADDRESS); 669 } 670 671 /* 672 * Print debugging information about the kernel lock. 673 */ 674 void 675 _kernel_lock_dump(volatile void *junk) 676 { 677 struct cpu_info *ci = curcpu(); 678 679 (void)junk; 680 681 printf_nolog("curcpu holds : %18d wanted by: %#018lx\n", 682 ci->ci_biglock_count, (long)ci->ci_biglock_wanted); 683 } 684 685 /* 686 * Acquire 'nlocks' holds on the kernel lock. If 'l' is non-null, the 687 * acquisition is from process context. 688 */ 689 void 690 _kernel_lock(int nlocks, struct lwp *l) 691 { 692 struct cpu_info *ci = curcpu(); 693 LOCKSTAT_TIMER(spintime); 694 LOCKSTAT_FLAG(lsflag); 695 struct lwp *owant; 696 #ifdef LOCKDEBUG 697 u_int spins; 698 #endif 699 int s; 700 701 if (nlocks == 0) 702 return; 703 _KERNEL_LOCK_ASSERT(nlocks > 0); 704 705 l = curlwp; 706 707 if (ci->ci_biglock_count != 0) { 708 _KERNEL_LOCK_ASSERT(__SIMPLELOCK_LOCKED_P(&kernel_lock)); 709 ci->ci_biglock_count += nlocks; 710 l->l_blcnt += nlocks; 711 return; 712 } 713 714 _KERNEL_LOCK_ASSERT(l->l_blcnt == 0); 715 LOCKDEBUG_WANTLOCK(kernel_lock_dodebug, &kernel_lock, RETURN_ADDRESS, 716 0); 717 718 s = splvm(); 719 if (__cpu_simple_lock_try(&kernel_lock)) { 720 ci->ci_biglock_count = nlocks; 721 l->l_blcnt = nlocks; 722 LOCKDEBUG_LOCKED(kernel_lock_dodebug, &kernel_lock, 723 RETURN_ADDRESS, 0); 724 splx(s); 725 return; 726 } 727 728 LOCKSTAT_ENTER(lsflag); 729 LOCKSTAT_START_TIMER(lsflag, spintime); 730 731 /* 732 * Before setting ci_biglock_wanted we must post a store 733 * fence (see kern_mutex.c). This is accomplished by the 734 * __cpu_simple_lock_try() above. 735 */ 736 owant = ci->ci_biglock_wanted; 737 ci->ci_biglock_wanted = curlwp; /* XXXAD */ 738 739 #ifdef LOCKDEBUG 740 spins = 0; 741 #endif 742 743 do { 744 splx(s); 745 while (__SIMPLELOCK_LOCKED_P(&kernel_lock)) { 746 #ifdef LOCKDEBUG 747 if (SPINLOCK_SPINOUT(spins)) 748 _KERNEL_LOCK_ABORT("spinout"); 749 #endif 750 SPINLOCK_BACKOFF_HOOK; 751 SPINLOCK_SPIN_HOOK; 752 } 753 (void)splvm(); 754 } while (!__cpu_simple_lock_try(&kernel_lock)); 755 756 ci->ci_biglock_wanted = owant; 757 ci->ci_biglock_count = nlocks; 758 l->l_blcnt = nlocks; 759 LOCKSTAT_STOP_TIMER(lsflag, spintime); 760 LOCKDEBUG_LOCKED(kernel_lock_dodebug, &kernel_lock, RETURN_ADDRESS, 0); 761 splx(s); 762 763 /* 764 * Again, another store fence is required (see kern_mutex.c). 765 */ 766 membar_producer(); 767 if (owant == NULL) { 768 LOCKSTAT_EVENT(lsflag, &kernel_lock, LB_KERNEL_LOCK | LB_SPIN, 769 1, spintime); 770 } 771 LOCKSTAT_EXIT(lsflag); 772 } 773 774 /* 775 * Release 'nlocks' holds on the kernel lock. If 'nlocks' is zero, release 776 * all holds. If 'l' is non-null, the release is from process context. 777 */ 778 void 779 _kernel_unlock(int nlocks, struct lwp *l, int *countp) 780 { 781 struct cpu_info *ci = curcpu(); 782 u_int olocks; 783 int s; 784 785 l = curlwp; 786 787 _KERNEL_LOCK_ASSERT(nlocks < 2); 788 789 olocks = l->l_blcnt; 790 791 if (olocks == 0) { 792 _KERNEL_LOCK_ASSERT(nlocks <= 0); 793 if (countp != NULL) 794 *countp = 0; 795 return; 796 } 797 798 _KERNEL_LOCK_ASSERT(__SIMPLELOCK_LOCKED_P(&kernel_lock)); 799 800 if (nlocks == 0) 801 nlocks = olocks; 802 else if (nlocks == -1) { 803 nlocks = 1; 804 _KERNEL_LOCK_ASSERT(olocks == 1); 805 } 806 807 _KERNEL_LOCK_ASSERT(ci->ci_biglock_count >= l->l_blcnt); 808 809 l->l_blcnt -= nlocks; 810 if (ci->ci_biglock_count == nlocks) { 811 s = splvm(); 812 LOCKDEBUG_UNLOCKED(kernel_lock_dodebug, &kernel_lock, 813 RETURN_ADDRESS, 0); 814 ci->ci_biglock_count = 0; 815 __cpu_simple_unlock(&kernel_lock); 816 splx(s); 817 } else 818 ci->ci_biglock_count -= nlocks; 819 820 if (countp != NULL) 821 *countp = olocks; 822 } 823 #endif /* !_RUMPKERNEL */ 824
Indexes created Wed Sep 24 05:09:52 GMT 2025