subr_lockdebug.c revision 1.72.4.1 1 /* $NetBSD: subr_lockdebug.c,v 1.72.4.1 2020/01/17 21:47:35 ad Exp $ */
2
3 /*-
4 * Copyright (c) 2006, 2007, 2008, 2020 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Andrew Doran.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32 /*
33 * Basic lock debugging code shared among lock primitives.
34 */
35
36 #include <sys/cdefs.h>
37 __KERNEL_RCSID(0, "$NetBSD: subr_lockdebug.c,v 1.72.4.1 2020/01/17 21:47:35 ad Exp $");
38
39 #ifdef _KERNEL_OPT
40 #include "opt_ddb.h"
41 #endif
42
43 #include <sys/param.h>
44 #include <sys/proc.h>
45 #include <sys/systm.h>
46 #include <sys/kernel.h>
47 #include <sys/kmem.h>
48 #include <sys/lockdebug.h>
49 #include <sys/sleepq.h>
50 #include <sys/cpu.h>
51 #include <sys/atomic.h>
52 #include <sys/lock.h>
53 #include <sys/rbtree.h>
54 #include <sys/ksyms.h>
55
56 #include <machine/lock.h>
57
58 unsigned int ld_panic;
59
60 #ifdef LOCKDEBUG
61
62 #ifdef __ia64__
63 #define LD_BATCH_SHIFT 16
64 #else
65 #define LD_BATCH_SHIFT 9
66 #endif
67 #define LD_BATCH (1 << LD_BATCH_SHIFT)
68 #define LD_BATCH_MASK (LD_BATCH - 1)
69 #define LD_MAX_LOCKS 1048576
70 #define LD_SLOP 16
71
72 #define LD_LOCKED 0x01
73 #define LD_SLEEPER 0x02
74
75 #define LD_WRITE_LOCK 0x80000000
76
77 typedef struct lockdebug {
78 struct rb_node ld_rb_node;
79 __cpu_simple_lock_t ld_spinlock;
80 _TAILQ_ENTRY(struct lockdebug, volatile) ld_chain;
81 _TAILQ_ENTRY(struct lockdebug, volatile) ld_achain;
82 volatile void *ld_lock;
83 lockops_t *ld_lockops;
84 struct lwp *ld_lwp;
85 uintptr_t ld_locked;
86 uintptr_t ld_unlocked;
87 uintptr_t ld_initaddr;
88 uint16_t ld_shares;
89 uint16_t ld_cpu;
90 uint8_t ld_flags;
91 uint8_t ld_shwant; /* advisory */
92 uint8_t ld_exwant; /* advisory */
93 uint8_t ld_unused;
94 } volatile lockdebug_t;
95
96 typedef _TAILQ_HEAD(lockdebuglist, struct lockdebug, volatile) lockdebuglist_t;
97
98 __cpu_simple_lock_t ld_mod_lk;
99 lockdebuglist_t ld_free = TAILQ_HEAD_INITIALIZER(ld_free);
100 lockdebuglist_t ld_all = TAILQ_HEAD_INITIALIZER(ld_all);
101 int ld_nfree;
102 int ld_freeptr;
103 int ld_recurse;
104 bool ld_nomore;
105 lockdebug_t ld_prime[LD_BATCH];
106
107 static void lockdebug_abort1(const char *, size_t, lockdebug_t *, int,
108 const char *, bool);
109 static int lockdebug_more(int);
110 static void lockdebug_init(void);
111 static void lockdebug_dump(lockdebug_t *, void (*)(const char *, ...)
112 __printflike(1, 2));
113
114 static signed int
115 ld_rbto_compare_nodes(void *ctx, const void *n1, const void *n2)
116 {
117 const lockdebug_t *ld1 = n1;
118 const lockdebug_t *ld2 = n2;
119 const uintptr_t a = (uintptr_t)ld1->ld_lock;
120 const uintptr_t b = (uintptr_t)ld2->ld_lock;
121
122 if (a < b)
123 return -1;
124 if (a > b)
125 return 1;
126 return 0;
127 }
128
129 static signed int
130 ld_rbto_compare_key(void *ctx, const void *n, const void *key)
131 {
132 const lockdebug_t *ld = n;
133 const uintptr_t a = (uintptr_t)ld->ld_lock;
134 const uintptr_t b = (uintptr_t)key;
135
136 if (a < b)
137 return -1;
138 if (a > b)
139 return 1;
140 return 0;
141 }
142
143 static rb_tree_t ld_rb_tree;
144
145 static const rb_tree_ops_t ld_rb_tree_ops = {
146 .rbto_compare_nodes = ld_rbto_compare_nodes,
147 .rbto_compare_key = ld_rbto_compare_key,
148 .rbto_node_offset = offsetof(lockdebug_t, ld_rb_node),
149 .rbto_context = NULL
150 };
151
152 static inline lockdebug_t *
153 lockdebug_lookup1(const volatile void *lock)
154 {
155 lockdebug_t *ld;
156 struct cpu_info *ci;
157
158 ci = curcpu();
159 __cpu_simple_lock(&ci->ci_data.cpu_ld_lock);
160 ld = rb_tree_find_node(&ld_rb_tree, (void *)(intptr_t)lock);
161 __cpu_simple_unlock(&ci->ci_data.cpu_ld_lock);
162 if (ld == NULL) {
163 return NULL;
164 }
165 __cpu_simple_lock(&ld->ld_spinlock);
166
167 return ld;
168 }
169
170 static void
171 lockdebug_lock_cpus(void)
172 {
173 CPU_INFO_ITERATOR cii;
174 struct cpu_info *ci;
175
176 for (CPU_INFO_FOREACH(cii, ci)) {
177 __cpu_simple_lock(&ci->ci_data.cpu_ld_lock);
178 }
179 }
180
181 static void
182 lockdebug_unlock_cpus(void)
183 {
184 CPU_INFO_ITERATOR cii;
185 struct cpu_info *ci;
186
187 for (CPU_INFO_FOREACH(cii, ci)) {
188 __cpu_simple_unlock(&ci->ci_data.cpu_ld_lock);
189 }
190 }
191
192 /*
193 * lockdebug_lookup:
194 *
195 * Find a lockdebug structure by a pointer to a lock and return it locked.
196 */
197 static inline lockdebug_t *
198 lockdebug_lookup(const char *func, size_t line, const volatile void *lock,
199 uintptr_t where)
200 {
201 lockdebug_t *ld;
202
203 ld = lockdebug_lookup1(lock);
204 if (__predict_false(ld == NULL)) {
205 panic("%s,%zu: uninitialized lock (lock=%p, from=%08"
206 PRIxPTR ")", func, line, lock, where);
207 }
208 return ld;
209 }
210
211 /*
212 * lockdebug_init:
213 *
214 * Initialize the lockdebug system. Allocate an initial pool of
215 * lockdebug structures before the VM system is up and running.
216 */
217 static void
218 lockdebug_init(void)
219 {
220 lockdebug_t *ld;
221 int i;
222
223 TAILQ_INIT(&curcpu()->ci_data.cpu_ld_locks);
224 TAILQ_INIT(&curlwp->l_ld_locks);
225 __cpu_simple_lock_init(&curcpu()->ci_data.cpu_ld_lock);
226 __cpu_simple_lock_init(&ld_mod_lk);
227
228 rb_tree_init(&ld_rb_tree, &ld_rb_tree_ops);
229
230 ld = ld_prime;
231 for (i = 1, ld++; i < LD_BATCH; i++, ld++) {
232 __cpu_simple_lock_init(&ld->ld_spinlock);
233 TAILQ_INSERT_TAIL(&ld_free, ld, ld_chain);
234 TAILQ_INSERT_TAIL(&ld_all, ld, ld_achain);
235 }
236 ld_freeptr = 1;
237 ld_nfree = LD_BATCH - 1;
238 }
239
240 /*
241 * lockdebug_alloc:
242 *
243 * A lock is being initialized, so allocate an associated debug
244 * structure.
245 */
246 bool
247 lockdebug_alloc(const char *func, size_t line, volatile void *lock,
248 lockops_t *lo, uintptr_t initaddr)
249 {
250 struct cpu_info *ci;
251 lockdebug_t *ld;
252 int s;
253
254 if (__predict_false(lo == NULL || panicstr != NULL || ld_panic))
255 return false;
256 if (__predict_false(ld_freeptr == 0))
257 lockdebug_init();
258
259 s = splhigh();
260 __cpu_simple_lock(&ld_mod_lk);
261 if (__predict_false((ld = lockdebug_lookup1(lock)) != NULL)) {
262 __cpu_simple_unlock(&ld_mod_lk);
263 lockdebug_abort1(func, line, ld, s, "already initialized",
264 true);
265 return false;
266 }
267
268 /*
269 * Pinch a new debug structure. We may recurse because we call
270 * kmem_alloc(), which may need to initialize new locks somewhere
271 * down the path. If not recursing, we try to maintain at least
272 * LD_SLOP structures free, which should hopefully be enough to
273 * satisfy kmem_alloc(). If we can't provide a structure, not to
274 * worry: we'll just mark the lock as not having an ID.
275 */
276 ci = curcpu();
277 ci->ci_lkdebug_recurse++;
278 if (TAILQ_EMPTY(&ld_free)) {
279 if (ci->ci_lkdebug_recurse > 1 || ld_nomore) {
280 ci->ci_lkdebug_recurse--;
281 __cpu_simple_unlock(&ld_mod_lk);
282 splx(s);
283 return false;
284 }
285 s = lockdebug_more(s);
286 } else if (ci->ci_lkdebug_recurse == 1 && ld_nfree < LD_SLOP) {
287 s = lockdebug_more(s);
288 }
289 if (__predict_false((ld = TAILQ_FIRST(&ld_free)) == NULL)) {
290 __cpu_simple_unlock(&ld_mod_lk);
291 splx(s);
292 return false;
293 }
294 TAILQ_REMOVE(&ld_free, ld, ld_chain);
295 ld_nfree--;
296 ci->ci_lkdebug_recurse--;
297
298 if (__predict_false(ld->ld_lock != NULL)) {
299 panic("%s,%zu: corrupt table ld %p", func, line, ld);
300 }
301
302 /* Initialise the structure. */
303 ld->ld_lock = lock;
304 ld->ld_lockops = lo;
305 ld->ld_locked = 0;
306 ld->ld_unlocked = 0;
307 ld->ld_lwp = NULL;
308 ld->ld_initaddr = initaddr;
309 ld->ld_flags = (lo->lo_type == LOCKOPS_SLEEP ? LD_SLEEPER : 0);
310 lockdebug_lock_cpus();
311 (void)rb_tree_insert_node(&ld_rb_tree, __UNVOLATILE(ld));
312 lockdebug_unlock_cpus();
313 __cpu_simple_unlock(&ld_mod_lk);
314
315 splx(s);
316 return true;
317 }
318
319 /*
320 * lockdebug_free:
321 *
322 * A lock is being destroyed, so release debugging resources.
323 */
324 void
325 lockdebug_free(const char *func, size_t line, volatile void *lock)
326 {
327 lockdebug_t *ld;
328 int s;
329
330 if (__predict_false(panicstr != NULL || ld_panic))
331 return;
332
333 s = splhigh();
334 __cpu_simple_lock(&ld_mod_lk);
335 ld = lockdebug_lookup(func, line, lock,
336 (uintptr_t) __builtin_return_address(0));
337 if (__predict_false(ld == NULL)) {
338 __cpu_simple_unlock(&ld_mod_lk);
339 panic("%s,%zu: destroying uninitialized object %p"
340 "(ld_lock=%p)", func, line, lock, ld->ld_lock);
341 return;
342 }
343 if (__predict_false((ld->ld_flags & LD_LOCKED) != 0 ||
344 ld->ld_shares != 0)) {
345 __cpu_simple_unlock(&ld_mod_lk);
346 lockdebug_abort1(func, line, ld, s, "is locked or in use",
347 true);
348 return;
349 }
350 lockdebug_lock_cpus();
351 rb_tree_remove_node(&ld_rb_tree, __UNVOLATILE(ld));
352 lockdebug_unlock_cpus();
353 ld->ld_lock = NULL;
354 TAILQ_INSERT_TAIL(&ld_free, ld, ld_chain);
355 ld_nfree++;
356 __cpu_simple_unlock(&ld->ld_spinlock);
357 __cpu_simple_unlock(&ld_mod_lk);
358 splx(s);
359 }
360
361 /*
362 * lockdebug_more:
363 *
364 * Allocate a batch of debug structures and add to the free list.
365 * Must be called with ld_mod_lk held.
366 */
367 static int
368 lockdebug_more(int s)
369 {
370 lockdebug_t *ld;
371 void *block;
372 int i, base, m;
373
374 /*
375 * Can't call kmem_alloc() if in interrupt context. XXX We could
376 * deadlock, because we don't know which locks the caller holds.
377 */
378 if (cpu_intr_p() || cpu_softintr_p()) {
379 return s;
380 }
381
382 while (ld_nfree < LD_SLOP) {
383 __cpu_simple_unlock(&ld_mod_lk);
384 splx(s);
385 block = kmem_zalloc(LD_BATCH * sizeof(lockdebug_t), KM_SLEEP);
386 s = splhigh();
387 __cpu_simple_lock(&ld_mod_lk);
388
389 if (ld_nfree > LD_SLOP) {
390 /* Somebody beat us to it. */
391 __cpu_simple_unlock(&ld_mod_lk);
392 splx(s);
393 kmem_free(block, LD_BATCH * sizeof(lockdebug_t));
394 s = splhigh();
395 __cpu_simple_lock(&ld_mod_lk);
396 continue;
397 }
398
399 base = ld_freeptr;
400 ld_nfree += LD_BATCH;
401 ld = block;
402 base <<= LD_BATCH_SHIFT;
403 m = uimin(LD_MAX_LOCKS, base + LD_BATCH);
404
405 if (m == LD_MAX_LOCKS)
406 ld_nomore = true;
407
408 for (i = base; i < m; i++, ld++) {
409 __cpu_simple_lock_init(&ld->ld_spinlock);
410 TAILQ_INSERT_TAIL(&ld_free, ld, ld_chain);
411 TAILQ_INSERT_TAIL(&ld_all, ld, ld_achain);
412 }
413
414 membar_producer();
415 }
416
417 return s;
418 }
419
420 /*
421 * lockdebug_wantlock:
422 *
423 * Process the preamble to a lock acquire. The "shared"
424 * parameter controls which ld_{ex,sh}want counter is
425 * updated; a negative value of shared updates neither.
426 */
427 void
428 lockdebug_wantlock(const char *func, size_t line,
429 const volatile void *lock, uintptr_t where, int shared)
430 {
431 struct lwp *l = curlwp;
432 lockdebug_t *ld;
433 bool recurse;
434 int s;
435
436 (void)shared;
437 recurse = false;
438
439 if (__predict_false(panicstr != NULL || ld_panic))
440 return;
441
442 s = splhigh();
443 if ((ld = lockdebug_lookup(func, line, lock, where)) == NULL) {
444 splx(s);
445 return;
446 }
447 if ((ld->ld_flags & LD_LOCKED) != 0 || ld->ld_shares != 0) {
448 if ((ld->ld_flags & LD_SLEEPER) != 0) {
449 if (ld->ld_lwp == l)
450 recurse = true;
451 } else if (ld->ld_cpu == (uint16_t)cpu_index(curcpu()))
452 recurse = true;
453 }
454 if (cpu_intr_p()) {
455 if (__predict_false((ld->ld_flags & LD_SLEEPER) != 0)) {
456 lockdebug_abort1(func, line, ld, s,
457 "acquiring sleep lock from interrupt context",
458 true);
459 return;
460 }
461 }
462 if (shared > 0)
463 ld->ld_shwant++;
464 else if (shared == 0)
465 ld->ld_exwant++;
466 if (__predict_false(recurse)) {
467 lockdebug_abort1(func, line, ld, s, "locking against myself",
468 true);
469 return;
470 }
471 __cpu_simple_unlock(&ld->ld_spinlock);
472 splx(s);
473 }
474
475 /*
476 * lockdebug_locked:
477 *
478 * Process a lock acquire operation.
479 */
480 void
481 lockdebug_locked(const char *func, size_t line,
482 volatile void *lock, void *cvlock, uintptr_t where, int shared)
483 {
484 struct lwp *l = curlwp;
485 lockdebug_t *ld;
486 int s;
487
488 if (__predict_false(panicstr != NULL || ld_panic))
489 return;
490
491 s = splhigh();
492 if ((ld = lockdebug_lookup(func, line, lock, where)) == NULL) {
493 splx(s);
494 return;
495 }
496 if (cvlock) {
497 KASSERT(ld->ld_lockops->lo_type == LOCKOPS_CV);
498 if (lock == (void *)&lbolt) {
499 /* nothing */
500 } else if (ld->ld_shares++ == 0) {
501 ld->ld_locked = (uintptr_t)cvlock;
502 } else if (__predict_false(cvlock != (void *)ld->ld_locked)) {
503 lockdebug_abort1(func, line, ld, s,
504 "multiple locks used with condition variable",
505 true);
506 return;
507 }
508 } else if (shared) {
509 l->l_shlocks++;
510 ld->ld_locked = where;
511 ld->ld_shares++;
512 ld->ld_shwant--;
513 } else {
514 if (__predict_false((ld->ld_flags & LD_LOCKED) != 0)) {
515 lockdebug_abort1(func, line, ld, s, "already locked",
516 true);
517 return;
518 }
519 ld->ld_flags |= LD_LOCKED;
520 ld->ld_locked = where;
521 ld->ld_exwant--;
522 if ((ld->ld_flags & LD_SLEEPER) != 0) {
523 TAILQ_INSERT_TAIL(&l->l_ld_locks, ld, ld_chain);
524 } else {
525 TAILQ_INSERT_TAIL(&curcpu()->ci_data.cpu_ld_locks,
526 ld, ld_chain);
527 }
528 }
529 ld->ld_cpu = (uint16_t)cpu_index(curcpu());
530 ld->ld_lwp = l;
531 __cpu_simple_unlock(&ld->ld_spinlock);
532 splx(s);
533 }
534
535 /*
536 * lockdebug_unlocked:
537 *
538 * Process a lock release operation.
539 */
540 void
541 lockdebug_unlocked(const char *func, size_t line,
542 volatile void *lock, uintptr_t where, int shared)
543 {
544 struct lwp *l = curlwp;
545 lockdebug_t *ld;
546 int s;
547
548 if (__predict_false(panicstr != NULL || ld_panic))
549 return;
550
551 s = splhigh();
552 if ((ld = lockdebug_lookup(func, line, lock, where)) == NULL) {
553 splx(s);
554 return;
555 }
556 if (ld->ld_lockops->lo_type == LOCKOPS_CV) {
557 if (lock == (void *)&lbolt) {
558 /* nothing */
559 } else {
560 ld->ld_shares--;
561 }
562 } else if (shared) {
563 if (__predict_false(l->l_shlocks == 0)) {
564 lockdebug_abort1(func, line, ld, s,
565 "no shared locks held by LWP", true);
566 return;
567 }
568 if (__predict_false(ld->ld_shares == 0)) {
569 lockdebug_abort1(func, line, ld, s,
570 "no shared holds on this lock", true);
571 return;
572 }
573 l->l_shlocks--;
574 ld->ld_shares--;
575 if (ld->ld_lwp == l) {
576 ld->ld_unlocked = where;
577 ld->ld_lwp = NULL;
578 }
579 if (ld->ld_cpu == (uint16_t)cpu_index(curcpu()))
580 ld->ld_cpu = (uint16_t)-1;
581 } else {
582 if (__predict_false((ld->ld_flags & LD_LOCKED) == 0)) {
583 lockdebug_abort1(func, line, ld, s, "not locked", true);
584 return;
585 }
586
587 if ((ld->ld_flags & LD_SLEEPER) != 0) {
588 if (__predict_false(ld->ld_lwp != curlwp)) {
589 lockdebug_abort1(func, line, ld, s,
590 "not held by current LWP", true);
591 return;
592 }
593 TAILQ_REMOVE(&l->l_ld_locks, ld, ld_chain);
594 } else {
595 uint16_t idx = (uint16_t)cpu_index(curcpu());
596 if (__predict_false(ld->ld_cpu != idx)) {
597 lockdebug_abort1(func, line, ld, s,
598 "not held by current CPU", true);
599 return;
600 }
601 TAILQ_REMOVE(&curcpu()->ci_data.cpu_ld_locks, ld,
602 ld_chain);
603 }
604 ld->ld_flags &= ~LD_LOCKED;
605 ld->ld_unlocked = where;
606 ld->ld_lwp = NULL;
607 }
608 __cpu_simple_unlock(&ld->ld_spinlock);
609 splx(s);
610 }
611
612 /*
613 * lockdebug_wakeup:
614 *
615 * Process a wakeup on a condition variable.
616 */
617 void
618 lockdebug_wakeup(const char *func, size_t line, volatile void *lock,
619 uintptr_t where)
620 {
621 lockdebug_t *ld;
622 int s;
623
624 if (__predict_false(panicstr != NULL || ld_panic || lock == (void *)&lbolt))
625 return;
626
627 s = splhigh();
628 /* Find the CV... */
629 if ((ld = lockdebug_lookup(func, line, lock, where)) == NULL) {
630 splx(s);
631 return;
632 }
633 /*
634 * If it has any waiters, ensure that they are using the
635 * same interlock.
636 */
637 if (__predict_false(ld->ld_shares != 0 &&
638 !mutex_owned((kmutex_t *)ld->ld_locked))) {
639 lockdebug_abort1(func, line, ld, s, "interlocking mutex not "
640 "held during wakeup", true);
641 return;
642 }
643 __cpu_simple_unlock(&ld->ld_spinlock);
644 splx(s);
645 }
646
647 /*
648 * lockdebug_barrier:
649 *
650 * Panic if we hold more than one specified lock, and optionally, if we
651 * hold any sleep locks.
652 */
653 void
654 lockdebug_barrier(const char *func, size_t line, volatile void *onelock,
655 int slplocks)
656 {
657 struct lwp *l = curlwp;
658 lockdebug_t *ld;
659 int s;
660
661 if (__predict_false(panicstr != NULL || ld_panic))
662 return;
663
664 s = splhigh();
665 if ((l->l_pflag & LP_INTR) == 0) {
666 TAILQ_FOREACH(ld, &curcpu()->ci_data.cpu_ld_locks, ld_chain) {
667 if (ld->ld_lock == onelock) {
668 continue;
669 }
670 __cpu_simple_lock(&ld->ld_spinlock);
671 lockdebug_abort1(func, line, ld, s,
672 "spin lock held", true);
673 return;
674 }
675 }
676 if (slplocks) {
677 splx(s);
678 return;
679 }
680 ld = TAILQ_FIRST(&l->l_ld_locks);
681 if (__predict_false(ld != NULL && ld->ld_lock != onelock)) {
682 __cpu_simple_lock(&ld->ld_spinlock);
683 lockdebug_abort1(func, line, ld, s, "sleep lock held", true);
684 return;
685 }
686 splx(s);
687 if (l->l_shlocks != 0) {
688 TAILQ_FOREACH(ld, &ld_all, ld_achain) {
689 if (ld->ld_lock == onelock) {
690 continue;
691 }
692 if (ld->ld_lockops->lo_type == LOCKOPS_CV)
693 continue;
694 if (ld->ld_lwp == l)
695 lockdebug_dump(ld, printf);
696 }
697 panic("%s,%zu: holding %d shared locks", func, line,
698 l->l_shlocks);
699 }
700 }
701
702 /*
703 * lockdebug_mem_check:
704 *
705 * Check for in-use locks within a memory region that is
706 * being freed.
707 */
708 void
709 lockdebug_mem_check(const char *func, size_t line, void *base, size_t sz)
710 {
711 lockdebug_t *ld;
712 struct cpu_info *ci;
713 int s;
714
715 if (__predict_false(panicstr != NULL || ld_panic))
716 return;
717
718 s = splhigh();
719 ci = curcpu();
720 __cpu_simple_lock(&ci->ci_data.cpu_ld_lock);
721 ld = (lockdebug_t *)rb_tree_find_node_geq(&ld_rb_tree, base);
722 if (ld != NULL) {
723 const uintptr_t lock = (uintptr_t)ld->ld_lock;
724
725 if (__predict_false((uintptr_t)base > lock))
726 panic("%s,%zu: corrupt tree ld=%p, base=%p, sz=%zu",
727 func, line, ld, base, sz);
728 if (lock >= (uintptr_t)base + sz)
729 ld = NULL;
730 }
731 __cpu_simple_unlock(&ci->ci_data.cpu_ld_lock);
732 if (__predict_false(ld != NULL)) {
733 __cpu_simple_lock(&ld->ld_spinlock);
734 lockdebug_abort1(func, line, ld, s,
735 "allocation contains active lock", !cold);
736 return;
737 }
738 splx(s);
739 }
740
741 /*
742 * lockdebug_dump:
743 *
744 * Dump information about a lock on panic, or for DDB.
745 */
746 static void
747 lockdebug_dump(lockdebug_t *ld, void (*pr)(const char *, ...)
748 __printflike(1, 2))
749 {
750 int sleeper = (ld->ld_flags & LD_SLEEPER);
751
752 (*pr)(
753 "lock address : %#018lx type : %18s\n"
754 "initialized : %#018lx",
755 (long)ld->ld_lock, (sleeper ? "sleep/adaptive" : "spin"),
756 (long)ld->ld_initaddr);
757
758 if (ld->ld_lockops->lo_type == LOCKOPS_CV) {
759 (*pr)(" interlock: %#018lx\n", (long)ld->ld_locked);
760 } else {
761 (*pr)("\n"
762 "shared holds : %18u exclusive: %18u\n"
763 "shares wanted: %18u exclusive: %18u\n"
764 "current cpu : %18u last held: %18u\n"
765 "current lwp : %#018lx last held: %#018lx\n"
766 "last locked%c : %#018lx unlocked%c: %#018lx\n",
767 (unsigned)ld->ld_shares, ((ld->ld_flags & LD_LOCKED) != 0),
768 (unsigned)ld->ld_shwant, (unsigned)ld->ld_exwant,
769 (unsigned)cpu_index(curcpu()), (unsigned)ld->ld_cpu,
770 (long)curlwp, (long)ld->ld_lwp,
771 ((ld->ld_flags & LD_LOCKED) ? '*' : ' '),
772 (long)ld->ld_locked,
773 ((ld->ld_flags & LD_LOCKED) ? ' ' : '*'),
774 (long)ld->ld_unlocked);
775 }
776
777 if (ld->ld_lockops->lo_dump != NULL)
778 (*ld->ld_lockops->lo_dump)(ld->ld_lock, pr);
779
780 if (sleeper) {
781 (*pr)("\n");
782 turnstile_print(ld->ld_lock, pr);
783 }
784 }
785
786 /*
787 * lockdebug_abort1:
788 *
789 * An error has been trapped - dump lock info and panic.
790 */
791 static void
792 lockdebug_abort1(const char *func, size_t line, lockdebug_t *ld, int s,
793 const char *msg, bool dopanic)
794 {
795
796 /*
797 * Don't make the situation worse if the system is already going
798 * down in flames. Once a panic is triggered, lockdebug state
799 * becomes stale and cannot be trusted.
800 */
801 if (atomic_inc_uint_nv(&ld_panic) != 1) {
802 __cpu_simple_unlock(&ld->ld_spinlock);
803 splx(s);
804 return;
805 }
806
807 printf_nolog("%s error: %s,%zu: %s\n\n", ld->ld_lockops->lo_name,
808 func, line, msg);
809 lockdebug_dump(ld, printf_nolog);
810 __cpu_simple_unlock(&ld->ld_spinlock);
811 splx(s);
812 printf_nolog("\n");
813 if (dopanic)
814 panic("LOCKDEBUG: %s error: %s,%zu: %s",
815 ld->ld_lockops->lo_name, func, line, msg);
816 }
817
818 #endif /* LOCKDEBUG */
819
820 /*
821 * lockdebug_lock_print:
822 *
823 * Handle the DDB 'show lock' command.
824 */
825 #ifdef DDB
826 #include <machine/db_machdep.h>
827 #include <ddb/db_interface.h>
828
829 void
830 lockdebug_lock_print(void *addr,
831 void (*pr)(const char *, ...) __printflike(1, 2))
832 {
833 #ifdef LOCKDEBUG
834 lockdebug_t *ld;
835
836 TAILQ_FOREACH(ld, &ld_all, ld_achain) {
837 if (ld->ld_lock == NULL)
838 continue;
839 if (addr == NULL || ld->ld_lock == addr) {
840 lockdebug_dump(ld, pr);
841 if (addr != NULL)
842 return;
843 }
844 }
845 if (addr != NULL) {
846 (*pr)("Sorry, no record of a lock with address %p found.\n",
847 addr);
848 }
849 #else
850 (*pr)("Sorry, kernel not built with the LOCKDEBUG option.\n");
851 #endif /* LOCKDEBUG */
852 }
853
854 #ifdef LOCKDEBUG
855 static void
856 lockdebug_show_one(lockdebug_t *ld, int i,
857 void (*pr)(const char *, ...) __printflike(1, 2))
858 {
859 const char *sym;
860
861 ksyms_getname(NULL, &sym, (vaddr_t)ld->ld_initaddr,
862 KSYMS_CLOSEST|KSYMS_PROC|KSYMS_ANY);
863 (*pr)("Lock %d (initialized at %s)\n", i++, sym);
864 lockdebug_dump(ld, pr);
865 }
866
867 static void
868 lockdebug_show_trace(const void *ptr,
869 void (*pr)(const char *, ...) __printflike(1, 2))
870 {
871 db_stack_trace_print((db_expr_t)(intptr_t)ptr, true, 32, "a", pr);
872 }
873
874 static void
875 lockdebug_show_all_locks_lwp(void (*pr)(const char *, ...) __printflike(1, 2),
876 bool show_trace)
877 {
878 struct proc *p;
879
880 LIST_FOREACH(p, &allproc, p_list) {
881 struct lwp *l;
882 LIST_FOREACH(l, &p->p_lwps, l_sibling) {
883 lockdebug_t *ld;
884 int i = 0;
885 if (TAILQ_EMPTY(&l->l_ld_locks))
886 continue;
887 (*pr)("Locks held by an LWP (%s):\n",
888 l->l_name ? l->l_name : p->p_comm);
889 TAILQ_FOREACH(ld, &l->l_ld_locks, ld_chain) {
890 lockdebug_show_one(ld, i++, pr);
891 }
892 if (show_trace)
893 lockdebug_show_trace(l, pr);
894 (*pr)("\n");
895 }
896 }
897 }
898
899 static void
900 lockdebug_show_all_locks_cpu(void (*pr)(const char *, ...) __printflike(1, 2),
901 bool show_trace)
902 {
903 lockdebug_t *ld;
904 CPU_INFO_ITERATOR cii;
905 struct cpu_info *ci;
906
907 for (CPU_INFO_FOREACH(cii, ci)) {
908 int i = 0;
909 if (TAILQ_EMPTY(&ci->ci_data.cpu_ld_locks))
910 continue;
911 (*pr)("Locks held on CPU %u:\n", ci->ci_index);
912 TAILQ_FOREACH(ld, &ci->ci_data.cpu_ld_locks, ld_chain) {
913 lockdebug_show_one(ld, i++, pr);
914 if (show_trace)
915 #ifdef MULTIPROCESSOR
916 lockdebug_show_trace(ci->ci_curlwp, pr);
917 #else
918 lockdebug_show_trace(curlwp, pr);
919 #endif
920 (*pr)("\n");
921 }
922 }
923 }
924 #endif /* LOCKDEBUG */
925
926 void
927 lockdebug_show_all_locks(void (*pr)(const char *, ...) __printflike(1, 2),
928 const char *modif)
929 {
930 #ifdef LOCKDEBUG
931 bool show_trace = false;
932 if (modif[0] == 't')
933 show_trace = true;
934
935 (*pr)("[Locks tracked through LWPs]\n");
936 lockdebug_show_all_locks_lwp(pr, show_trace);
937 (*pr)("\n");
938
939 (*pr)("[Locks tracked through CPUs]\n");
940 lockdebug_show_all_locks_cpu(pr, show_trace);
941 (*pr)("\n");
942 #else
943 (*pr)("Sorry, kernel not built with the LOCKDEBUG option.\n");
944 #endif /* LOCKDEBUG */
945 }
946
947 void
948 lockdebug_show_lockstats(void (*pr)(const char *, ...) __printflike(1, 2))
949 {
950 #ifdef LOCKDEBUG
951 lockdebug_t *ld;
952 void *_ld;
953 uint32_t n_null = 0;
954 uint32_t n_spin_mutex = 0;
955 uint32_t n_adaptive_mutex = 0;
956 uint32_t n_rwlock = 0;
957 uint32_t n_cv = 0;
958 uint32_t n_others = 0;
959
960 RB_TREE_FOREACH(_ld, &ld_rb_tree) {
961 ld = _ld;
962 if (ld->ld_lock == NULL) {
963 n_null++;
964 continue;
965 }
966 if (ld->ld_lockops->lo_type == LOCKOPS_CV) {
967 n_cv++;
968 continue;
969 }
970 if (ld->ld_lockops->lo_name[0] == 'M') {
971 if (ld->ld_lockops->lo_type == LOCKOPS_SLEEP)
972 n_adaptive_mutex++;
973 else
974 n_spin_mutex++;
975 continue;
976 }
977 if (ld->ld_lockops->lo_name[0] == 'R') {
978 n_rwlock++;
979 continue;
980 }
981 n_others++;
982 }
983 (*pr)(
984 "condvar: %u\n"
985 "spin mutex: %u\n"
986 "adaptive mutex: %u\n"
987 "rwlock: %u\n"
988 "null locks: %u\n"
989 "others: %u\n",
990 n_cv, n_spin_mutex, n_adaptive_mutex, n_rwlock,
991 n_null, n_others);
992 #else
993 (*pr)("Sorry, kernel not built with the LOCKDEBUG option.\n");
994 #endif /* LOCKDEBUG */
995 }
996 #endif /* DDB */
997
998 /*
999 * lockdebug_dismiss:
1000 *
1001 * The system is rebooting, and potentially from an unsafe
1002 * place so avoid any future aborts.
1003 */
1004 void
1005 lockdebug_dismiss(void)
1006 {
1007
1008 atomic_inc_uint_nv(&ld_panic);
1009 }
1010
1011 /*
1012 * lockdebug_abort:
1013 *
1014 * An error has been trapped - dump lock info and call panic().
1015 */
1016 void
1017 lockdebug_abort(const char *func, size_t line, const volatile void *lock,
1018 lockops_t *ops, const char *msg)
1019 {
1020 #ifdef LOCKDEBUG
1021 lockdebug_t *ld;
1022 int s;
1023
1024 s = splhigh();
1025 if ((ld = lockdebug_lookup(func, line, lock,
1026 (uintptr_t) __builtin_return_address(0))) != NULL) {
1027 lockdebug_abort1(func, line, ld, s, msg, true);
1028 return;
1029 }
1030 splx(s);
1031 #endif /* LOCKDEBUG */
1032
1033 /*
1034 * Don't make the situation worse if the system is already going
1035 * down in flames. Once a panic is triggered, lockdebug state
1036 * becomes stale and cannot be trusted.
1037 */
1038 if (atomic_inc_uint_nv(&ld_panic) > 1)
1039 return;
1040
1041 printf_nolog("%s error: %s,%zu: %s\n\n"
1042 "lock address : %#018lx\n"
1043 "current cpu : %18d\n"
1044 "current lwp : %#018lx\n",
1045 ops->lo_name, func, line, msg, (long)lock,
1046 (int)cpu_index(curcpu()), (long)curlwp);
1047 (*ops->lo_dump)(lock, printf_nolog);
1048 printf_nolog("\n");
1049
1050 panic("lock error: %s: %s,%zu: %s: lock %p cpu %d lwp %p",
1051 ops->lo_name, func, line, msg, lock, cpu_index(curcpu()), curlwp);
1052 }
1053