subr_lockdebug.c revision 1.27
1 /* $NetBSD: subr_lockdebug.c,v 1.27 2008/02/18 16:02:41 ad Exp $ */ 2 3 /*- 4 * Copyright (c) 2006, 2007 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. All advertising materials mentioning features or use of this software 19 * must display the following acknowledgement: 20 * This product includes software developed by the NetBSD 21 * Foundation, Inc. and its contributors. 22 * 4. Neither the name of The NetBSD Foundation nor the names of its 23 * contributors may be used to endorse or promote products derived 24 * from this software without specific prior written permission. 25 * 26 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 27 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 28 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 29 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 30 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 31 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 32 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 33 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 34 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 35 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 36 * POSSIBILITY OF SUCH DAMAGE. 37 */ 38 39 /* 40 * Basic lock debugging code shared among lock primitives. 41 */ 42 43 #include <sys/cdefs.h> 44 __KERNEL_RCSID(0, "$NetBSD: subr_lockdebug.c,v 1.27 2008/02/18 16:02:41 ad Exp $"); 45 46 #include "opt_ddb.h" 47 48 #include <sys/param.h> 49 #include <sys/proc.h> 50 #include <sys/systm.h> 51 #include <sys/kernel.h> 52 #include <sys/kmem.h> 53 #include <sys/lockdebug.h> 54 #include <sys/sleepq.h> 55 #include <sys/cpu.h> 56 #include <sys/atomic.h> 57 #include <sys/lock.h> 58 59 #include <lib/libkern/rb.h> 60 61 #include <machine/lock.h> 62 63 #ifdef LOCKDEBUG 64 65 #define LD_BATCH_SHIFT 9 66 #define LD_BATCH (1 << LD_BATCH_SHIFT) 67 #define LD_BATCH_MASK (LD_BATCH - 1) 68 #define LD_MAX_LOCKS 1048576 69 #define LD_SLOP 16 70 71 #define LD_LOCKED 0x01 72 #define LD_SLEEPER 0x02 73 74 #define LD_WRITE_LOCK 0x80000000 75 76 typedef union lockdebuglk { 77 struct { 78 u_int lku_lock; 79 int lku_oldspl; 80 } ul; 81 uint8_t lk_pad[CACHE_LINE_SIZE]; 82 } volatile __aligned(CACHE_LINE_SIZE) lockdebuglk_t; 83 84 #define lk_lock ul.lku_lock 85 #define lk_oldspl ul.lku_oldspl 86 87 typedef struct lockdebug { 88 struct rb_node ld_rb_node; /* must be the first member */ 89 _TAILQ_ENTRY(struct lockdebug, volatile) ld_chain; 90 _TAILQ_ENTRY(struct lockdebug, volatile) ld_achain; 91 volatile void *ld_lock; 92 lockops_t *ld_lockops; 93 struct lwp *ld_lwp; 94 uintptr_t ld_locked; 95 uintptr_t ld_unlocked; 96 uintptr_t ld_initaddr; 97 uint16_t ld_shares; 98 uint16_t ld_cpu; 99 uint8_t ld_flags; 100 uint8_t ld_shwant; /* advisory */ 101 uint8_t ld_exwant; /* advisory */ 102 uint8_t ld_unused; 103 } volatile lockdebug_t; 104 105 typedef _TAILQ_HEAD(lockdebuglist, struct lockdebug, volatile) lockdebuglist_t; 106 107 lockdebuglk_t ld_tree_lk; 108 lockdebuglk_t ld_sleeper_lk; 109 lockdebuglk_t ld_spinner_lk; 110 lockdebuglk_t ld_free_lk; 111 112 lockdebuglist_t ld_sleepers = TAILQ_HEAD_INITIALIZER(ld_sleepers); 113 lockdebuglist_t ld_spinners = TAILQ_HEAD_INITIALIZER(ld_spinners); 114 lockdebuglist_t ld_free = TAILQ_HEAD_INITIALIZER(ld_free); 115 lockdebuglist_t ld_all = TAILQ_HEAD_INITIALIZER(ld_all); 116 int ld_nfree; 117 int ld_freeptr; 118 int ld_recurse; 119 bool ld_nomore; 120 lockdebug_t *ld_table[LD_MAX_LOCKS / LD_BATCH]; 121 unsigned int ld_panic; 122 123 lockdebug_t ld_prime[LD_BATCH]; 124 125 static void lockdebug_abort1(lockdebug_t *, lockdebuglk_t *lk, 126 const char *, const char *, bool); 127 static void lockdebug_more(void); 128 static void lockdebug_init(void); 129 130 static signed int 131 ld_rb_compare_nodes(const struct rb_node *n1, const struct rb_node *n2) 132 { 133 const lockdebug_t *ld1 = (const void *)n1; 134 const lockdebug_t *ld2 = (const void *)n2; 135 const uintptr_t a = (uintptr_t)ld1->ld_lock; 136 const uintptr_t b = (uintptr_t)ld2->ld_lock; 137 138 if (a < b) 139 return 1; 140 if (a > b) 141 return -1; 142 return 0; 143 } 144 145 static signed int 146 ld_rb_compare_key(const struct rb_node *n, const void *key) 147 { 148 const lockdebug_t *ld = (const void *)n; 149 const uintptr_t a = (uintptr_t)ld->ld_lock; 150 const uintptr_t b = (uintptr_t)key; 151 152 if (a < b) 153 return 1; 154 if (a > b) 155 return -1; 156 return 0; 157 } 158 159 static struct rb_tree ld_rb_tree; 160 161 static const struct rb_tree_ops ld_rb_tree_ops = { 162 .rb_compare_nodes = ld_rb_compare_nodes, 163 .rb_compare_key = ld_rb_compare_key, 164 }; 165 166 static void 167 lockdebug_lock_init(lockdebuglk_t *lk) 168 { 169 170 lk->lk_lock = 0; 171 } 172 173 static void 174 lockdebug_lock(lockdebuglk_t *lk) 175 { 176 int s; 177 178 s = splhigh(); 179 do { 180 while (lk->lk_lock != 0) { 181 SPINLOCK_SPIN_HOOK; 182 } 183 } while (atomic_cas_uint(&lk->lk_lock, 0, LD_WRITE_LOCK) != 0); 184 lk->lk_oldspl = s; 185 membar_enter(); 186 } 187 188 static void 189 lockdebug_unlock(lockdebuglk_t *lk) 190 { 191 int s; 192 193 s = lk->lk_oldspl; 194 membar_exit(); 195 lk->lk_lock = 0; 196 splx(s); 197 } 198 199 static int 200 lockdebug_lock_rd(lockdebuglk_t *lk) 201 { 202 u_int val; 203 int s; 204 205 s = splhigh(); 206 do { 207 while ((val = lk->lk_lock) == LD_WRITE_LOCK){ 208 SPINLOCK_SPIN_HOOK; 209 } 210 } while (atomic_cas_uint(&lk->lk_lock, val, val + 1) != val); 211 membar_enter(); 212 return s; 213 } 214 215 static void 216 lockdebug_unlock_rd(lockdebuglk_t *lk, int s) 217 { 218 219 membar_exit(); 220 atomic_dec_uint(&lk->lk_lock); 221 splx(s); 222 } 223 224 static inline lockdebug_t * 225 lockdebug_lookup1(volatile void *lock, lockdebuglk_t **lk) 226 { 227 lockdebug_t *ld; 228 int s; 229 230 s = lockdebug_lock_rd(&ld_tree_lk); 231 ld = (lockdebug_t *)rb_tree_find_node(&ld_rb_tree, __UNVOLATILE(lock)); 232 lockdebug_unlock_rd(&ld_tree_lk, s); 233 if (ld == NULL) 234 return NULL; 235 236 if ((ld->ld_flags & LD_SLEEPER) != 0) 237 *lk = &ld_sleeper_lk; 238 else 239 *lk = &ld_spinner_lk; 240 241 lockdebug_lock(*lk); 242 return ld; 243 } 244 245 /* 246 * lockdebug_lookup: 247 * 248 * Find a lockdebug structure by a pointer to a lock and return it locked. 249 */ 250 static inline lockdebug_t * 251 lockdebug_lookup(volatile void *lock, lockdebuglk_t **lk) 252 { 253 lockdebug_t *ld; 254 255 ld = lockdebug_lookup1(lock, lk); 256 if (ld == NULL) 257 panic("lockdebug_lookup: uninitialized lock (lock=%p)", lock); 258 return ld; 259 } 260 261 /* 262 * lockdebug_init: 263 * 264 * Initialize the lockdebug system. Allocate an initial pool of 265 * lockdebug structures before the VM system is up and running. 266 */ 267 static void 268 lockdebug_init(void) 269 { 270 lockdebug_t *ld; 271 int i; 272 273 lockdebug_lock_init(&ld_tree_lk); 274 lockdebug_lock_init(&ld_sleeper_lk); 275 lockdebug_lock_init(&ld_spinner_lk); 276 lockdebug_lock_init(&ld_free_lk); 277 278 rb_tree_init(&ld_rb_tree, &ld_rb_tree_ops); 279 280 ld = ld_prime; 281 ld_table[0] = ld; 282 for (i = 1, ld++; i < LD_BATCH; i++, ld++) { 283 TAILQ_INSERT_TAIL(&ld_free, ld, ld_chain); 284 TAILQ_INSERT_TAIL(&ld_all, ld, ld_achain); 285 } 286 ld_freeptr = 1; 287 ld_nfree = LD_BATCH - 1; 288 } 289 290 /* 291 * lockdebug_alloc: 292 * 293 * A lock is being initialized, so allocate an associated debug 294 * structure. 295 */ 296 bool 297 lockdebug_alloc(volatile void *lock, lockops_t *lo, uintptr_t initaddr) 298 { 299 struct cpu_info *ci; 300 lockdebug_t *ld; 301 lockdebuglk_t *lk; 302 303 if (lo == NULL || panicstr != NULL || ld_panic) 304 return false; 305 if (ld_freeptr == 0) 306 lockdebug_init(); 307 308 if ((ld = lockdebug_lookup1(lock, &lk)) != NULL) { 309 lockdebug_abort1(ld, lk, __func__, "already initialized", true); 310 return false; 311 } 312 313 /* 314 * Pinch a new debug structure. We may recurse because we call 315 * kmem_alloc(), which may need to initialize new locks somewhere 316 * down the path. If not recursing, we try to maintain at least 317 * LD_SLOP structures free, which should hopefully be enough to 318 * satisfy kmem_alloc(). If we can't provide a structure, not to 319 * worry: we'll just mark the lock as not having an ID. 320 */ 321 lockdebug_lock(&ld_free_lk); 322 ci = curcpu(); 323 ci->ci_lkdebug_recurse++; 324 325 if (TAILQ_EMPTY(&ld_free)) { 326 if (ci->ci_lkdebug_recurse > 1 || ld_nomore) { 327 ci->ci_lkdebug_recurse--; 328 lockdebug_unlock(&ld_free_lk); 329 return false; 330 } 331 lockdebug_more(); 332 } else if (ci->ci_lkdebug_recurse == 1 && ld_nfree < LD_SLOP) 333 lockdebug_more(); 334 335 if ((ld = TAILQ_FIRST(&ld_free)) == NULL) { 336 lockdebug_unlock(&ld_free_lk); 337 return false; 338 } 339 340 TAILQ_REMOVE(&ld_free, ld, ld_chain); 341 ld_nfree--; 342 343 ci->ci_lkdebug_recurse--; 344 lockdebug_unlock(&ld_free_lk); 345 346 if (ld->ld_lock != NULL) 347 panic("lockdebug_alloc: corrupt table"); 348 349 if (lo->lo_sleeplock) 350 lockdebug_lock(&ld_sleeper_lk); 351 else 352 lockdebug_lock(&ld_spinner_lk); 353 354 /* Initialise the structure. */ 355 ld->ld_lock = lock; 356 ld->ld_lockops = lo; 357 ld->ld_locked = 0; 358 ld->ld_unlocked = 0; 359 ld->ld_lwp = NULL; 360 ld->ld_initaddr = initaddr; 361 362 lockdebug_lock(&ld_tree_lk); 363 rb_tree_insert_node(&ld_rb_tree, __UNVOLATILE(&ld->ld_rb_node)); 364 lockdebug_unlock(&ld_tree_lk); 365 366 if (lo->lo_sleeplock) { 367 ld->ld_flags = LD_SLEEPER; 368 lockdebug_unlock(&ld_sleeper_lk); 369 } else { 370 ld->ld_flags = 0; 371 lockdebug_unlock(&ld_spinner_lk); 372 } 373 374 return true; 375 } 376 377 /* 378 * lockdebug_free: 379 * 380 * A lock is being destroyed, so release debugging resources. 381 */ 382 void 383 lockdebug_free(volatile void *lock) 384 { 385 lockdebug_t *ld; 386 lockdebuglk_t *lk; 387 388 if (panicstr != NULL || ld_panic) 389 return; 390 391 ld = lockdebug_lookup(lock, &lk); 392 if (ld == NULL) { 393 panic("lockdebug_free: destroying uninitialized lock %p" 394 "(ld_lock=%p)", lock, ld->ld_lock); 395 lockdebug_abort1(ld, lk, __func__, "lock record follows", 396 true); 397 return; 398 } 399 if ((ld->ld_flags & LD_LOCKED) != 0 || ld->ld_shares != 0) { 400 lockdebug_abort1(ld, lk, __func__, "is locked", true); 401 return; 402 } 403 lockdebug_lock(&ld_tree_lk); 404 rb_tree_remove_node(&ld_rb_tree, __UNVOLATILE(&ld->ld_rb_node)); 405 lockdebug_unlock(&ld_tree_lk); 406 ld->ld_lock = NULL; 407 lockdebug_unlock(lk); 408 409 lockdebug_lock(&ld_free_lk); 410 TAILQ_INSERT_TAIL(&ld_free, ld, ld_chain); 411 ld_nfree++; 412 lockdebug_unlock(&ld_free_lk); 413 } 414 415 /* 416 * lockdebug_more: 417 * 418 * Allocate a batch of debug structures and add to the free list. 419 * Must be called with ld_free_lk held. 420 */ 421 static void 422 lockdebug_more(void) 423 { 424 lockdebug_t *ld; 425 void *block; 426 int i, base, m; 427 428 while (ld_nfree < LD_SLOP) { 429 lockdebug_unlock(&ld_free_lk); 430 block = kmem_zalloc(LD_BATCH * sizeof(lockdebug_t), KM_SLEEP); 431 lockdebug_lock(&ld_free_lk); 432 433 if (block == NULL) 434 return; 435 436 if (ld_nfree > LD_SLOP) { 437 /* Somebody beat us to it. */ 438 lockdebug_unlock(&ld_free_lk); 439 kmem_free(block, LD_BATCH * sizeof(lockdebug_t)); 440 lockdebug_lock(&ld_free_lk); 441 continue; 442 } 443 444 base = ld_freeptr; 445 ld_nfree += LD_BATCH; 446 ld = block; 447 base <<= LD_BATCH_SHIFT; 448 m = min(LD_MAX_LOCKS, base + LD_BATCH); 449 450 if (m == LD_MAX_LOCKS) 451 ld_nomore = true; 452 453 for (i = base; i < m; i++, ld++) { 454 TAILQ_INSERT_TAIL(&ld_free, ld, ld_chain); 455 TAILQ_INSERT_TAIL(&ld_all, ld, ld_achain); 456 } 457 458 membar_producer(); 459 ld_table[ld_freeptr++] = block; 460 } 461 } 462 463 /* 464 * lockdebug_wantlock: 465 * 466 * Process the preamble to a lock acquire. 467 */ 468 void 469 lockdebug_wantlock(volatile void *lock, uintptr_t where, int shared) 470 { 471 struct lwp *l = curlwp; 472 lockdebuglk_t *lk; 473 lockdebug_t *ld; 474 bool recurse; 475 476 (void)shared; 477 recurse = false; 478 479 if (panicstr != NULL || ld_panic) 480 return; 481 482 if ((ld = lockdebug_lookup(lock, &lk)) == NULL) 483 return; 484 485 if ((ld->ld_flags & LD_LOCKED) != 0) { 486 if ((ld->ld_flags & LD_SLEEPER) != 0) { 487 if (ld->ld_lwp == l) 488 recurse = true; 489 } else if (ld->ld_cpu == (uint16_t)cpu_number()) 490 recurse = true; 491 } 492 493 if (cpu_intr_p()) { 494 if ((ld->ld_flags & LD_SLEEPER) != 0) { 495 lockdebug_abort1(ld, lk, __func__, 496 "acquiring sleep lock from interrupt context", 497 true); 498 return; 499 } 500 } 501 502 if (shared) 503 ld->ld_shwant++; 504 else 505 ld->ld_exwant++; 506 507 if (recurse) { 508 lockdebug_abort1(ld, lk, __func__, "locking against myself", 509 true); 510 return; 511 } 512 513 lockdebug_unlock(lk); 514 } 515 516 /* 517 * lockdebug_locked: 518 * 519 * Process a lock acquire operation. 520 */ 521 void 522 lockdebug_locked(volatile void *lock, uintptr_t where, int shared) 523 { 524 struct lwp *l = curlwp; 525 lockdebuglk_t *lk; 526 lockdebug_t *ld; 527 528 if (panicstr != NULL || ld_panic) 529 return; 530 531 if ((ld = lockdebug_lookup(lock, &lk)) == NULL) 532 return; 533 534 if (shared) { 535 l->l_shlocks++; 536 ld->ld_shares++; 537 ld->ld_shwant--; 538 } else { 539 if ((ld->ld_flags & LD_LOCKED) != 0) { 540 lockdebug_abort1(ld, lk, __func__, 541 "already locked", true); 542 return; 543 } 544 545 ld->ld_flags |= LD_LOCKED; 546 ld->ld_locked = where; 547 ld->ld_cpu = (uint16_t)cpu_number(); 548 ld->ld_lwp = l; 549 ld->ld_exwant--; 550 551 if ((ld->ld_flags & LD_SLEEPER) != 0) { 552 l->l_exlocks++; 553 TAILQ_INSERT_TAIL(&ld_sleepers, ld, ld_chain); 554 } else { 555 curcpu()->ci_spin_locks2++; 556 TAILQ_INSERT_TAIL(&ld_spinners, ld, ld_chain); 557 } 558 } 559 560 lockdebug_unlock(lk); 561 } 562 563 /* 564 * lockdebug_unlocked: 565 * 566 * Process a lock release operation. 567 */ 568 void 569 lockdebug_unlocked(volatile void *lock, uintptr_t where, int shared) 570 { 571 struct lwp *l = curlwp; 572 lockdebuglk_t *lk; 573 lockdebug_t *ld; 574 575 if (panicstr != NULL || ld_panic) 576 return; 577 578 if ((ld = lockdebug_lookup(lock, &lk)) == NULL) 579 return; 580 581 if (shared) { 582 if (l->l_shlocks == 0) { 583 lockdebug_abort1(ld, lk, __func__, 584 "no shared locks held by LWP", true); 585 return; 586 } 587 if (ld->ld_shares == 0) { 588 lockdebug_abort1(ld, lk, __func__, 589 "no shared holds on this lock", true); 590 return; 591 } 592 l->l_shlocks--; 593 ld->ld_shares--; 594 } else { 595 if ((ld->ld_flags & LD_LOCKED) == 0) { 596 lockdebug_abort1(ld, lk, __func__, "not locked", 597 true); 598 return; 599 } 600 601 if ((ld->ld_flags & LD_SLEEPER) != 0) { 602 if (ld->ld_lwp != curlwp) { 603 lockdebug_abort1(ld, lk, __func__, 604 "not held by current LWP", true); 605 return; 606 } 607 ld->ld_flags &= ~LD_LOCKED; 608 ld->ld_unlocked = where; 609 ld->ld_lwp = NULL; 610 curlwp->l_exlocks--; 611 TAILQ_REMOVE(&ld_sleepers, ld, ld_chain); 612 } else { 613 if (ld->ld_cpu != (uint16_t)cpu_number()) { 614 lockdebug_abort1(ld, lk, __func__, 615 "not held by current CPU", true); 616 return; 617 } 618 ld->ld_flags &= ~LD_LOCKED; 619 ld->ld_unlocked = where; 620 ld->ld_lwp = NULL; 621 curcpu()->ci_spin_locks2--; 622 TAILQ_REMOVE(&ld_spinners, ld, ld_chain); 623 } 624 } 625 626 lockdebug_unlock(lk); 627 } 628 629 /* 630 * lockdebug_barrier: 631 * 632 * Panic if we hold more than one specified spin lock, and optionally, 633 * if we hold sleep locks. 634 */ 635 void 636 lockdebug_barrier(volatile void *spinlock, int slplocks) 637 { 638 struct lwp *l = curlwp; 639 lockdebug_t *ld; 640 uint16_t cpuno; 641 int s; 642 643 if (panicstr != NULL || ld_panic) 644 return; 645 646 crit_enter(); 647 648 if (curcpu()->ci_spin_locks2 != 0) { 649 cpuno = (uint16_t)cpu_number(); 650 651 s = lockdebug_lock_rd(&ld_spinner_lk); 652 TAILQ_FOREACH(ld, &ld_spinners, ld_chain) { 653 if (ld->ld_lock == spinlock) { 654 if (ld->ld_cpu != cpuno) { 655 lockdebug_abort1(ld, &ld_spinner_lk, 656 __func__, 657 "not held by current CPU", true); 658 return; 659 } 660 continue; 661 } 662 if (ld->ld_cpu == cpuno && (l->l_pflag & LP_INTR) == 0) { 663 lockdebug_abort1(ld, &ld_spinner_lk, 664 __func__, "spin lock held", true); 665 return; 666 } 667 } 668 lockdebug_unlock_rd(&ld_spinner_lk, s); 669 } 670 671 if (!slplocks) { 672 if (l->l_exlocks != 0) { 673 s = lockdebug_lock_rd(&ld_sleeper_lk); 674 TAILQ_FOREACH(ld, &ld_sleepers, ld_chain) { 675 if (ld->ld_lwp == l) { 676 lockdebug_abort1(ld, &ld_sleeper_lk, 677 __func__, "sleep lock held", true); 678 return; 679 } 680 } 681 lockdebug_unlock_rd(&ld_sleeper_lk, s); 682 } 683 if (l->l_shlocks != 0) 684 panic("lockdebug_barrier: holding %d shared locks", 685 l->l_shlocks); 686 } 687 688 crit_exit(); 689 } 690 691 /* 692 * lockdebug_mem_check: 693 * 694 * Check for in-use locks within a memory region that is 695 * being freed. 696 */ 697 void 698 lockdebug_mem_check(const char *func, void *base, size_t sz) 699 { 700 lockdebug_t *ld; 701 lockdebuglk_t *lk; 702 int s; 703 704 if (panicstr != NULL || ld_panic) 705 return; 706 707 s = lockdebug_lock_rd(&ld_tree_lk); 708 ld = (lockdebug_t *)rb_tree_find_node_geq(&ld_rb_tree, base); 709 if (ld != NULL) { 710 const uintptr_t lock = (uintptr_t)ld->ld_lock; 711 712 if ((uintptr_t)base > lock) 713 panic("%s: corrupt tree ld=%p, base=%p, sz=%zu", 714 __func__, ld, base, sz); 715 if (lock >= (uintptr_t)base + sz) 716 ld = NULL; 717 } 718 lockdebug_unlock_rd(&ld_tree_lk, s); 719 if (ld == NULL) 720 return; 721 722 if ((ld->ld_flags & LD_SLEEPER) != 0) 723 lk = &ld_sleeper_lk; 724 else 725 lk = &ld_spinner_lk; 726 727 lockdebug_lock(lk); 728 lockdebug_abort1(ld, lk, func, 729 "allocation contains active lock", !cold); 730 } 731 732 /* 733 * lockdebug_dump: 734 * 735 * Dump information about a lock on panic, or for DDB. 736 */ 737 static void 738 lockdebug_dump(lockdebug_t *ld, void (*pr)(const char *, ...)) 739 { 740 int sleeper = (ld->ld_flags & LD_SLEEPER); 741 742 (*pr)( 743 "lock address : %#018lx type : %18s\n" 744 "shared holds : %18u exclusive: %18u\n" 745 "shares wanted: %18u exclusive: %18u\n" 746 "current cpu : %18u last held: %18u\n" 747 "current lwp : %#018lx last held: %#018lx\n" 748 "last locked : %#018lx unlocked : %#018lx\n" 749 "initialized : %#018lx\n", 750 (long)ld->ld_lock, (sleeper ? "sleep/adaptive" : "spin"), 751 (unsigned)ld->ld_shares, ((ld->ld_flags & LD_LOCKED) != 0), 752 (unsigned)ld->ld_shwant, (unsigned)ld->ld_exwant, 753 (unsigned)cpu_number(), (unsigned)ld->ld_cpu, 754 (long)curlwp, (long)ld->ld_lwp, 755 (long)ld->ld_locked, (long)ld->ld_unlocked, 756 (long)ld->ld_initaddr); 757 758 if (ld->ld_lockops->lo_dump != NULL) 759 (*ld->ld_lockops->lo_dump)(ld->ld_lock); 760 761 if (sleeper) { 762 (*pr)("\n"); 763 turnstile_print(ld->ld_lock, pr); 764 } 765 } 766 767 /* 768 * lockdebug_abort1: 769 * 770 * An error has been trapped - dump lock info and panic. 771 */ 772 static void 773 lockdebug_abort1(lockdebug_t *ld, lockdebuglk_t *lk, const char *func, 774 const char *msg, bool dopanic) 775 { 776 777 /* 778 * Don't make the situation wose if the system is already going 779 * down in flames. Once a panic is triggered, lockdebug state 780 * becomes stale and cannot be trusted. 781 */ 782 if (atomic_inc_uint_nv(&ld_panic) != 1) { 783 lockdebug_unlock(lk); 784 return; 785 } 786 787 printf_nolog("%s error: %s: %s\n\n", ld->ld_lockops->lo_name, 788 func, msg); 789 lockdebug_dump(ld, printf_nolog); 790 lockdebug_unlock(lk); 791 printf_nolog("\n"); 792 if (dopanic) 793 panic("LOCKDEBUG"); 794 } 795 796 #endif /* LOCKDEBUG */ 797 798 /* 799 * lockdebug_lock_print: 800 * 801 * Handle the DDB 'show lock' command. 802 */ 803 #ifdef DDB 804 void 805 lockdebug_lock_print(void *addr, void (*pr)(const char *, ...)) 806 { 807 #ifdef LOCKDEBUG 808 lockdebug_t *ld; 809 810 TAILQ_FOREACH(ld, &ld_all, ld_achain) { 811 if (ld->ld_lock == addr) { 812 lockdebug_dump(ld, pr); 813 return; 814 } 815 } 816 (*pr)("Sorry, no record of a lock with address %p found.\n", addr); 817 #else 818 (*pr)("Sorry, kernel not built with the LOCKDEBUG option.\n"); 819 #endif /* LOCKDEBUG */ 820 } 821 #endif /* DDB */ 822 823 /* 824 * lockdebug_abort: 825 * 826 * An error has been trapped - dump lock info and call panic(). 827 */ 828 void 829 lockdebug_abort(volatile void *lock, lockops_t *ops, const char *func, 830 const char *msg) 831 { 832 #ifdef LOCKDEBUG 833 lockdebug_t *ld; 834 lockdebuglk_t *lk; 835 836 if ((ld = lockdebug_lookup(lock, &lk)) != NULL) { 837 lockdebug_abort1(ld, lk, func, msg, true); 838 /* NOTREACHED */ 839 } 840 #endif /* LOCKDEBUG */ 841 842 /* 843 * Complain first on the occurrance only. Otherwise proceeed to 844 * panic where we will `rendezvous' with other CPUs if the machine 845 * is going down in flames. 846 */ 847 if (atomic_inc_uint_nv(&ld_panic) == 1) { 848 printf_nolog("%s error: %s: %s\n\n" 849 "lock address : %#018lx\n" 850 "current cpu : %18d\n" 851 "current lwp : %#018lx\n", 852 ops->lo_name, func, msg, (long)lock, (int)cpu_number(), 853 (long)curlwp); 854 (*ops->lo_dump)(lock); 855 printf_nolog("\n"); 856 } 857 858 panic("lock error"); 859 } 860
Indexes created Sun Sep 21 20:09:37 GMT 2025