subr_lockdebug.c revision 1.17 1 1.16 yamt /* $NetBSD: subr_lockdebug.c,v 1.17 2007/11/21 10:21:40 yamt Exp $ */
2 1.2 ad
3 1.2 ad /*-
4 1.2 ad * Copyright (c) 2006, 2007 The NetBSD Foundation, Inc.
5 1.2 ad * All rights reserved.
6 1.2 ad *
7 1.2 ad * This code is derived from software contributed to The NetBSD Foundation
8 1.2 ad * by Andrew Doran.
9 1.2 ad *
10 1.2 ad * Redistribution and use in source and binary forms, with or without
11 1.2 ad * modification, are permitted provided that the following conditions
12 1.2 ad * are met:
13 1.2 ad * 1. Redistributions of source code must retain the above copyright
14 1.2 ad * notice, this list of conditions and the following disclaimer.
15 1.2 ad * 2. Redistributions in binary form must reproduce the above copyright
16 1.2 ad * notice, this list of conditions and the following disclaimer in the
17 1.2 ad * documentation and/or other materials provided with the distribution.
18 1.2 ad * 3. All advertising materials mentioning features or use of this software
19 1.2 ad * must display the following acknowledgement:
20 1.2 ad * This product includes software developed by the NetBSD
21 1.2 ad * Foundation, Inc. and its contributors.
22 1.2 ad * 4. Neither the name of The NetBSD Foundation nor the names of its
23 1.2 ad * contributors may be used to endorse or promote products derived
24 1.2 ad * from this software without specific prior written permission.
25 1.2 ad *
26 1.2 ad * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
27 1.2 ad * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
28 1.2 ad * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
29 1.2 ad * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
30 1.2 ad * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
31 1.2 ad * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
32 1.2 ad * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
33 1.2 ad * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
34 1.2 ad * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
35 1.2 ad * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
36 1.2 ad * POSSIBILITY OF SUCH DAMAGE.
37 1.2 ad */
38 1.2 ad
39 1.2 ad /*
40 1.11 ad * Basic lock debugging code shared among lock primitives.
41 1.2 ad */
42 1.2 ad
43 1.9 dsl #include <sys/cdefs.h>
44 1.16 yamt __KERNEL_RCSID(0, "$NetBSD: subr_lockdebug.c,v 1.17 2007/11/21 10:21:40 yamt Exp $");
45 1.9 dsl
46 1.2 ad #include "opt_multiprocessor.h"
47 1.2 ad #include "opt_ddb.h"
48 1.2 ad
49 1.2 ad #include <sys/param.h>
50 1.2 ad #include <sys/proc.h>
51 1.2 ad #include <sys/systm.h>
52 1.10 ad #include <sys/kernel.h>
53 1.2 ad #include <sys/kmem.h>
54 1.2 ad #include <sys/lock.h>
55 1.2 ad #include <sys/lockdebug.h>
56 1.2 ad #include <sys/sleepq.h>
57 1.10 ad #include <sys/cpu.h>
58 1.2 ad
59 1.16 yamt #include <lib/libkern/rb.h>
60 1.16 yamt
61 1.2 ad #ifdef LOCKDEBUG
62 1.2 ad
63 1.2 ad #define LD_BATCH_SHIFT 9
64 1.2 ad #define LD_BATCH (1 << LD_BATCH_SHIFT)
65 1.2 ad #define LD_BATCH_MASK (LD_BATCH - 1)
66 1.2 ad #define LD_MAX_LOCKS 1048576
67 1.2 ad #define LD_SLOP 16
68 1.2 ad
69 1.2 ad #define LD_LOCKED 0x01
70 1.2 ad #define LD_SLEEPER 0x02
71 1.2 ad
72 1.2 ad typedef union lockdebuglk {
73 1.2 ad struct {
74 1.2 ad __cpu_simple_lock_t lku_lock;
75 1.2 ad int lku_oldspl;
76 1.2 ad } ul;
77 1.2 ad uint8_t lk_pad[64];
78 1.2 ad } volatile __aligned(64) lockdebuglk_t;
79 1.2 ad
80 1.2 ad #define lk_lock ul.lku_lock
81 1.2 ad #define lk_oldspl ul.lku_oldspl
82 1.2 ad
83 1.2 ad typedef struct lockdebug {
84 1.16 yamt struct rb_node ld_rb_node; /* must be the first member */
85 1.2 ad _TAILQ_ENTRY(struct lockdebug, volatile) ld_chain;
86 1.2 ad _TAILQ_ENTRY(struct lockdebug, volatile) ld_achain;
87 1.2 ad volatile void *ld_lock;
88 1.2 ad lockops_t *ld_lockops;
89 1.2 ad struct lwp *ld_lwp;
90 1.2 ad uintptr_t ld_locked;
91 1.2 ad uintptr_t ld_unlocked;
92 1.10 ad uintptr_t ld_initaddr;
93 1.2 ad uint16_t ld_shares;
94 1.2 ad uint16_t ld_cpu;
95 1.2 ad uint8_t ld_flags;
96 1.2 ad uint8_t ld_shwant; /* advisory */
97 1.2 ad uint8_t ld_exwant; /* advisory */
98 1.2 ad uint8_t ld_unused;
99 1.2 ad } volatile lockdebug_t;
100 1.2 ad
101 1.2 ad typedef _TAILQ_HEAD(lockdebuglist, struct lockdebug, volatile) lockdebuglist_t;
102 1.2 ad
103 1.16 yamt lockdebuglk_t ld_tree_lk;
104 1.15 matt lockdebuglk_t ld_sleeper_lk;
105 1.15 matt lockdebuglk_t ld_spinner_lk;
106 1.15 matt lockdebuglk_t ld_free_lk;
107 1.2 ad
108 1.13 matt lockdebuglist_t ld_sleepers = TAILQ_HEAD_INITIALIZER(ld_sleepers);
109 1.13 matt lockdebuglist_t ld_spinners = TAILQ_HEAD_INITIALIZER(ld_spinners);
110 1.13 matt lockdebuglist_t ld_free = TAILQ_HEAD_INITIALIZER(ld_free);
111 1.13 matt lockdebuglist_t ld_all = TAILQ_HEAD_INITIALIZER(ld_all);
112 1.2 ad int ld_nfree;
113 1.2 ad int ld_freeptr;
114 1.2 ad int ld_recurse;
115 1.5 ad bool ld_nomore;
116 1.2 ad lockdebug_t *ld_table[LD_MAX_LOCKS / LD_BATCH];
117 1.2 ad
118 1.2 ad lockdebug_t ld_prime[LD_BATCH];
119 1.2 ad
120 1.5 ad static void lockdebug_abort1(lockdebug_t *, lockdebuglk_t *lk,
121 1.10 ad const char *, const char *, bool);
122 1.5 ad static void lockdebug_more(void);
123 1.5 ad static void lockdebug_init(void);
124 1.2 ad
125 1.16 yamt static signed int
126 1.16 yamt ld_rb_compare_nodes(const struct rb_node *n1, const struct rb_node *n2)
127 1.16 yamt {
128 1.16 yamt const lockdebug_t *ld1 = (const void *)n1;
129 1.16 yamt const lockdebug_t *ld2 = (const void *)n2;
130 1.16 yamt intptr_t diff = (intptr_t)ld1->ld_lock - (intptr_t)ld2->ld_lock;
131 1.16 yamt if (diff < 0)
132 1.16 yamt return -1;
133 1.16 yamt else if (diff > 0)
134 1.16 yamt return 1;
135 1.16 yamt return 0;
136 1.16 yamt }
137 1.16 yamt
138 1.16 yamt static signed int
139 1.16 yamt ld_rb_compare_key(const struct rb_node *n, const void *key)
140 1.16 yamt {
141 1.16 yamt const lockdebug_t *ld = (const void *)n;
142 1.16 yamt intptr_t diff = (intptr_t)ld->ld_lock - (intptr_t)key;
143 1.16 yamt if (diff < 0)
144 1.16 yamt return -1;
145 1.16 yamt else if (diff > 0)
146 1.16 yamt return 1;
147 1.16 yamt return 0;
148 1.16 yamt }
149 1.16 yamt
150 1.16 yamt static struct rb_tree ld_rb_tree;
151 1.16 yamt
152 1.16 yamt static const struct rb_tree_ops ld_rb_tree_ops = {
153 1.16 yamt .rb_compare_nodes = ld_rb_compare_nodes,
154 1.16 yamt .rb_compare_key = ld_rb_compare_key,
155 1.16 yamt };
156 1.16 yamt
157 1.2 ad static inline void
158 1.2 ad lockdebug_lock(lockdebuglk_t *lk)
159 1.2 ad {
160 1.2 ad int s;
161 1.2 ad
162 1.8 ad s = splhigh();
163 1.2 ad __cpu_simple_lock(&lk->lk_lock);
164 1.2 ad lk->lk_oldspl = s;
165 1.2 ad }
166 1.2 ad
167 1.2 ad static inline void
168 1.2 ad lockdebug_unlock(lockdebuglk_t *lk)
169 1.2 ad {
170 1.2 ad int s;
171 1.2 ad
172 1.2 ad s = lk->lk_oldspl;
173 1.2 ad __cpu_simple_unlock(&(lk->lk_lock));
174 1.2 ad splx(s);
175 1.2 ad }
176 1.2 ad
177 1.2 ad /*
178 1.2 ad * lockdebug_lookup:
179 1.2 ad *
180 1.16 yamt * Find a lockdebug structure by a pointer to a lock and return it locked.
181 1.2 ad */
182 1.2 ad static inline lockdebug_t *
183 1.16 yamt lockdebug_lookup(volatile void *lock, lockdebuglk_t **lk)
184 1.2 ad {
185 1.16 yamt lockdebug_t *ld;
186 1.2 ad
187 1.16 yamt lockdebug_lock(&ld_tree_lk);
188 1.16 yamt ld = (lockdebug_t *)rb_tree_find_node(&ld_rb_tree, __UNVOLATILE(lock));
189 1.16 yamt lockdebug_unlock(&ld_tree_lk);
190 1.16 yamt if (ld == NULL)
191 1.16 yamt panic("lockdebug_lookup: uninitialized lock (lock=%p)", lock);
192 1.2 ad
193 1.2 ad if ((ld->ld_flags & LD_SLEEPER) != 0)
194 1.2 ad *lk = &ld_sleeper_lk;
195 1.2 ad else
196 1.2 ad *lk = &ld_spinner_lk;
197 1.2 ad
198 1.2 ad lockdebug_lock(*lk);
199 1.2 ad return ld;
200 1.2 ad }
201 1.2 ad
202 1.2 ad /*
203 1.2 ad * lockdebug_init:
204 1.2 ad *
205 1.2 ad * Initialize the lockdebug system. Allocate an initial pool of
206 1.2 ad * lockdebug structures before the VM system is up and running.
207 1.2 ad */
208 1.5 ad static void
209 1.2 ad lockdebug_init(void)
210 1.2 ad {
211 1.2 ad lockdebug_t *ld;
212 1.2 ad int i;
213 1.2 ad
214 1.16 yamt __cpu_simple_lock_init(&ld_tree_lk.lk_lock);
215 1.15 matt __cpu_simple_lock_init(&ld_sleeper_lk.lk_lock);
216 1.15 matt __cpu_simple_lock_init(&ld_spinner_lk.lk_lock);
217 1.15 matt __cpu_simple_lock_init(&ld_free_lk.lk_lock);
218 1.15 matt
219 1.16 yamt rb_tree_init(&ld_rb_tree, &ld_rb_tree_ops);
220 1.16 yamt
221 1.2 ad ld = ld_prime;
222 1.2 ad ld_table[0] = ld;
223 1.2 ad for (i = 1, ld++; i < LD_BATCH; i++, ld++) {
224 1.2 ad TAILQ_INSERT_TAIL(&ld_free, ld, ld_chain);
225 1.2 ad TAILQ_INSERT_TAIL(&ld_all, ld, ld_achain);
226 1.2 ad }
227 1.2 ad ld_freeptr = 1;
228 1.2 ad ld_nfree = LD_BATCH - 1;
229 1.2 ad }
230 1.2 ad
231 1.2 ad /*
232 1.2 ad * lockdebug_alloc:
233 1.2 ad *
234 1.2 ad * A lock is being initialized, so allocate an associated debug
235 1.2 ad * structure.
236 1.2 ad */
237 1.16 yamt bool
238 1.10 ad lockdebug_alloc(volatile void *lock, lockops_t *lo, uintptr_t initaddr)
239 1.2 ad {
240 1.2 ad struct cpu_info *ci;
241 1.2 ad lockdebug_t *ld;
242 1.2 ad
243 1.5 ad if (lo == NULL || panicstr != NULL)
244 1.16 yamt return false;
245 1.5 ad if (ld_freeptr == 0)
246 1.5 ad lockdebug_init();
247 1.2 ad
248 1.2 ad ci = curcpu();
249 1.2 ad
250 1.2 ad /*
251 1.2 ad * Pinch a new debug structure. We may recurse because we call
252 1.2 ad * kmem_alloc(), which may need to initialize new locks somewhere
253 1.7 skrll * down the path. If not recursing, we try to maintain at least
254 1.2 ad * LD_SLOP structures free, which should hopefully be enough to
255 1.2 ad * satisfy kmem_alloc(). If we can't provide a structure, not to
256 1.2 ad * worry: we'll just mark the lock as not having an ID.
257 1.2 ad */
258 1.2 ad lockdebug_lock(&ld_free_lk);
259 1.2 ad ci->ci_lkdebug_recurse++;
260 1.2 ad
261 1.2 ad if (TAILQ_EMPTY(&ld_free)) {
262 1.5 ad if (ci->ci_lkdebug_recurse > 1 || ld_nomore) {
263 1.2 ad ci->ci_lkdebug_recurse--;
264 1.2 ad lockdebug_unlock(&ld_free_lk);
265 1.16 yamt return false;
266 1.2 ad }
267 1.2 ad lockdebug_more();
268 1.2 ad } else if (ci->ci_lkdebug_recurse == 1 && ld_nfree < LD_SLOP)
269 1.2 ad lockdebug_more();
270 1.2 ad
271 1.2 ad if ((ld = TAILQ_FIRST(&ld_free)) == NULL) {
272 1.2 ad lockdebug_unlock(&ld_free_lk);
273 1.16 yamt return false;
274 1.2 ad }
275 1.2 ad
276 1.2 ad TAILQ_REMOVE(&ld_free, ld, ld_chain);
277 1.2 ad ld_nfree--;
278 1.2 ad
279 1.2 ad ci->ci_lkdebug_recurse--;
280 1.2 ad lockdebug_unlock(&ld_free_lk);
281 1.2 ad
282 1.2 ad if (ld->ld_lock != NULL)
283 1.2 ad panic("lockdebug_alloc: corrupt table");
284 1.2 ad
285 1.2 ad if (lo->lo_sleeplock)
286 1.2 ad lockdebug_lock(&ld_sleeper_lk);
287 1.2 ad else
288 1.2 ad lockdebug_lock(&ld_spinner_lk);
289 1.2 ad
290 1.2 ad /* Initialise the structure. */
291 1.2 ad ld->ld_lock = lock;
292 1.2 ad ld->ld_lockops = lo;
293 1.2 ad ld->ld_locked = 0;
294 1.2 ad ld->ld_unlocked = 0;
295 1.2 ad ld->ld_lwp = NULL;
296 1.10 ad ld->ld_initaddr = initaddr;
297 1.2 ad
298 1.16 yamt lockdebug_lock(&ld_tree_lk);
299 1.16 yamt rb_tree_insert_node(&ld_rb_tree, __UNVOLATILE(&ld->ld_rb_node));
300 1.16 yamt lockdebug_unlock(&ld_tree_lk);
301 1.16 yamt
302 1.2 ad if (lo->lo_sleeplock) {
303 1.2 ad ld->ld_flags = LD_SLEEPER;
304 1.2 ad lockdebug_unlock(&ld_sleeper_lk);
305 1.2 ad } else {
306 1.2 ad ld->ld_flags = 0;
307 1.2 ad lockdebug_unlock(&ld_spinner_lk);
308 1.2 ad }
309 1.2 ad
310 1.16 yamt return true;
311 1.2 ad }
312 1.2 ad
313 1.2 ad /*
314 1.2 ad * lockdebug_free:
315 1.2 ad *
316 1.2 ad * A lock is being destroyed, so release debugging resources.
317 1.2 ad */
318 1.2 ad void
319 1.16 yamt lockdebug_free(volatile void *lock)
320 1.2 ad {
321 1.2 ad lockdebug_t *ld;
322 1.2 ad lockdebuglk_t *lk;
323 1.2 ad
324 1.2 ad if (panicstr != NULL)
325 1.2 ad return;
326 1.2 ad
327 1.16 yamt ld = lockdebug_lookup(lock, &lk);
328 1.16 yamt if (ld == NULL) {
329 1.2 ad panic("lockdebug_free: destroying uninitialized lock %p"
330 1.16 yamt "(ld_lock=%p)", lock, ld->ld_lock);
331 1.10 ad lockdebug_abort1(ld, lk, __func__, "lock record follows",
332 1.10 ad true);
333 1.2 ad }
334 1.2 ad if ((ld->ld_flags & LD_LOCKED) != 0 || ld->ld_shares != 0)
335 1.10 ad lockdebug_abort1(ld, lk, __func__, "is locked", true);
336 1.16 yamt lockdebug_lock(&ld_tree_lk);
337 1.16 yamt rb_tree_remove_node(&ld_rb_tree, __UNVOLATILE(&ld->ld_rb_node));
338 1.16 yamt lockdebug_unlock(&ld_tree_lk);
339 1.2 ad ld->ld_lock = NULL;
340 1.2 ad lockdebug_unlock(lk);
341 1.2 ad
342 1.2 ad lockdebug_lock(&ld_free_lk);
343 1.2 ad TAILQ_INSERT_TAIL(&ld_free, ld, ld_chain);
344 1.2 ad ld_nfree++;
345 1.2 ad lockdebug_unlock(&ld_free_lk);
346 1.2 ad }
347 1.2 ad
348 1.2 ad /*
349 1.2 ad * lockdebug_more:
350 1.2 ad *
351 1.2 ad * Allocate a batch of debug structures and add to the free list.
352 1.2 ad * Must be called with ld_free_lk held.
353 1.2 ad */
354 1.5 ad static void
355 1.2 ad lockdebug_more(void)
356 1.2 ad {
357 1.2 ad lockdebug_t *ld;
358 1.2 ad void *block;
359 1.5 ad int i, base, m;
360 1.2 ad
361 1.2 ad while (ld_nfree < LD_SLOP) {
362 1.2 ad lockdebug_unlock(&ld_free_lk);
363 1.2 ad block = kmem_zalloc(LD_BATCH * sizeof(lockdebug_t), KM_SLEEP);
364 1.2 ad lockdebug_lock(&ld_free_lk);
365 1.2 ad
366 1.2 ad if (block == NULL)
367 1.2 ad return;
368 1.2 ad
369 1.2 ad if (ld_nfree > LD_SLOP) {
370 1.2 ad /* Somebody beat us to it. */
371 1.2 ad lockdebug_unlock(&ld_free_lk);
372 1.2 ad kmem_free(block, LD_BATCH * sizeof(lockdebug_t));
373 1.2 ad lockdebug_lock(&ld_free_lk);
374 1.2 ad continue;
375 1.2 ad }
376 1.2 ad
377 1.2 ad base = ld_freeptr;
378 1.2 ad ld_nfree += LD_BATCH;
379 1.2 ad ld = block;
380 1.2 ad base <<= LD_BATCH_SHIFT;
381 1.5 ad m = min(LD_MAX_LOCKS, base + LD_BATCH);
382 1.5 ad
383 1.5 ad if (m == LD_MAX_LOCKS)
384 1.5 ad ld_nomore = true;
385 1.2 ad
386 1.5 ad for (i = base; i < m; i++, ld++) {
387 1.2 ad TAILQ_INSERT_TAIL(&ld_free, ld, ld_chain);
388 1.2 ad TAILQ_INSERT_TAIL(&ld_all, ld, ld_achain);
389 1.2 ad }
390 1.2 ad
391 1.2 ad mb_write();
392 1.2 ad ld_table[ld_freeptr++] = block;
393 1.2 ad }
394 1.2 ad }
395 1.2 ad
396 1.2 ad /*
397 1.2 ad * lockdebug_wantlock:
398 1.2 ad *
399 1.2 ad * Process the preamble to a lock acquire.
400 1.2 ad */
401 1.2 ad void
402 1.16 yamt lockdebug_wantlock(volatile void *lock, uintptr_t where, int shared)
403 1.2 ad {
404 1.2 ad struct lwp *l = curlwp;
405 1.2 ad lockdebuglk_t *lk;
406 1.2 ad lockdebug_t *ld;
407 1.3 thorpej bool recurse;
408 1.2 ad
409 1.2 ad (void)shared;
410 1.4 thorpej recurse = false;
411 1.2 ad
412 1.2 ad if (panicstr != NULL)
413 1.2 ad return;
414 1.2 ad
415 1.16 yamt if ((ld = lockdebug_lookup(lock, &lk)) == NULL)
416 1.2 ad return;
417 1.2 ad
418 1.2 ad if ((ld->ld_flags & LD_LOCKED) != 0) {
419 1.2 ad if ((ld->ld_flags & LD_SLEEPER) != 0) {
420 1.2 ad if (ld->ld_lwp == l)
421 1.4 thorpej recurse = true;
422 1.2 ad } else if (ld->ld_cpu == (uint16_t)cpu_number())
423 1.4 thorpej recurse = true;
424 1.2 ad }
425 1.2 ad
426 1.10 ad #ifdef notyet
427 1.10 ad if (cpu_intr_p()) {
428 1.10 ad if ((ld->ld_flags & LD_SLEEPER) != 0)
429 1.10 ad lockdebug_abort1(ld, lk, __func__,
430 1.10 ad "acquiring sleep lock from interrupt context",
431 1.10 ad true);
432 1.10 ad }
433 1.10 ad #endif
434 1.10 ad
435 1.2 ad if (shared)
436 1.2 ad ld->ld_shwant++;
437 1.2 ad else
438 1.2 ad ld->ld_exwant++;
439 1.2 ad
440 1.2 ad if (recurse)
441 1.10 ad lockdebug_abort1(ld, lk, __func__, "locking against myself",
442 1.10 ad true);
443 1.2 ad
444 1.2 ad lockdebug_unlock(lk);
445 1.2 ad }
446 1.2 ad
447 1.2 ad /*
448 1.2 ad * lockdebug_locked:
449 1.2 ad *
450 1.2 ad * Process a lock acquire operation.
451 1.2 ad */
452 1.2 ad void
453 1.16 yamt lockdebug_locked(volatile void *lock, uintptr_t where, int shared)
454 1.2 ad {
455 1.2 ad struct lwp *l = curlwp;
456 1.2 ad lockdebuglk_t *lk;
457 1.2 ad lockdebug_t *ld;
458 1.2 ad
459 1.2 ad if (panicstr != NULL)
460 1.2 ad return;
461 1.2 ad
462 1.16 yamt if ((ld = lockdebug_lookup(lock, &lk)) == NULL)
463 1.2 ad return;
464 1.2 ad
465 1.2 ad if (shared) {
466 1.2 ad l->l_shlocks++;
467 1.2 ad ld->ld_shares++;
468 1.2 ad ld->ld_shwant--;
469 1.2 ad } else {
470 1.2 ad if ((ld->ld_flags & LD_LOCKED) != 0)
471 1.2 ad lockdebug_abort1(ld, lk, __func__,
472 1.10 ad "already locked", true);
473 1.2 ad
474 1.2 ad ld->ld_flags |= LD_LOCKED;
475 1.2 ad ld->ld_locked = where;
476 1.2 ad ld->ld_cpu = (uint16_t)cpu_number();
477 1.2 ad ld->ld_lwp = l;
478 1.2 ad ld->ld_exwant--;
479 1.2 ad
480 1.2 ad if ((ld->ld_flags & LD_SLEEPER) != 0) {
481 1.2 ad l->l_exlocks++;
482 1.2 ad TAILQ_INSERT_TAIL(&ld_sleepers, ld, ld_chain);
483 1.2 ad } else {
484 1.2 ad curcpu()->ci_spin_locks2++;
485 1.2 ad TAILQ_INSERT_TAIL(&ld_spinners, ld, ld_chain);
486 1.2 ad }
487 1.2 ad }
488 1.2 ad
489 1.2 ad lockdebug_unlock(lk);
490 1.2 ad }
491 1.2 ad
492 1.2 ad /*
493 1.2 ad * lockdebug_unlocked:
494 1.2 ad *
495 1.2 ad * Process a lock release operation.
496 1.2 ad */
497 1.2 ad void
498 1.16 yamt lockdebug_unlocked(volatile void *lock, uintptr_t where, int shared)
499 1.2 ad {
500 1.2 ad struct lwp *l = curlwp;
501 1.2 ad lockdebuglk_t *lk;
502 1.2 ad lockdebug_t *ld;
503 1.2 ad
504 1.2 ad if (panicstr != NULL)
505 1.2 ad return;
506 1.2 ad
507 1.16 yamt if ((ld = lockdebug_lookup(lock, &lk)) == NULL)
508 1.2 ad return;
509 1.2 ad
510 1.2 ad if (shared) {
511 1.2 ad if (l->l_shlocks == 0)
512 1.2 ad lockdebug_abort1(ld, lk, __func__,
513 1.10 ad "no shared locks held by LWP", true);
514 1.2 ad if (ld->ld_shares == 0)
515 1.2 ad lockdebug_abort1(ld, lk, __func__,
516 1.10 ad "no shared holds on this lock", true);
517 1.2 ad l->l_shlocks--;
518 1.2 ad ld->ld_shares--;
519 1.2 ad } else {
520 1.2 ad if ((ld->ld_flags & LD_LOCKED) == 0)
521 1.10 ad lockdebug_abort1(ld, lk, __func__, "not locked",
522 1.10 ad true);
523 1.2 ad
524 1.2 ad if ((ld->ld_flags & LD_SLEEPER) != 0) {
525 1.2 ad if (ld->ld_lwp != curlwp)
526 1.2 ad lockdebug_abort1(ld, lk, __func__,
527 1.10 ad "not held by current LWP", true);
528 1.2 ad ld->ld_flags &= ~LD_LOCKED;
529 1.2 ad ld->ld_unlocked = where;
530 1.2 ad ld->ld_lwp = NULL;
531 1.2 ad curlwp->l_exlocks--;
532 1.2 ad TAILQ_REMOVE(&ld_sleepers, ld, ld_chain);
533 1.2 ad } else {
534 1.2 ad if (ld->ld_cpu != (uint16_t)cpu_number())
535 1.2 ad lockdebug_abort1(ld, lk, __func__,
536 1.10 ad "not held by current CPU", true);
537 1.2 ad ld->ld_flags &= ~LD_LOCKED;
538 1.2 ad ld->ld_unlocked = where;
539 1.2 ad ld->ld_lwp = NULL;
540 1.2 ad curcpu()->ci_spin_locks2--;
541 1.2 ad TAILQ_REMOVE(&ld_spinners, ld, ld_chain);
542 1.2 ad }
543 1.2 ad }
544 1.2 ad
545 1.2 ad lockdebug_unlock(lk);
546 1.2 ad }
547 1.2 ad
548 1.2 ad /*
549 1.2 ad * lockdebug_barrier:
550 1.2 ad *
551 1.2 ad * Panic if we hold more than one specified spin lock, and optionally,
552 1.2 ad * if we hold sleep locks.
553 1.2 ad */
554 1.2 ad void
555 1.2 ad lockdebug_barrier(volatile void *spinlock, int slplocks)
556 1.2 ad {
557 1.2 ad struct lwp *l = curlwp;
558 1.2 ad lockdebug_t *ld;
559 1.2 ad uint16_t cpuno;
560 1.2 ad
561 1.2 ad if (panicstr != NULL)
562 1.2 ad return;
563 1.2 ad
564 1.2 ad if (curcpu()->ci_spin_locks2 != 0) {
565 1.2 ad cpuno = (uint16_t)cpu_number();
566 1.2 ad
567 1.2 ad lockdebug_lock(&ld_spinner_lk);
568 1.2 ad TAILQ_FOREACH(ld, &ld_spinners, ld_chain) {
569 1.2 ad if (ld->ld_lock == spinlock) {
570 1.2 ad if (ld->ld_cpu != cpuno)
571 1.2 ad lockdebug_abort1(ld, &ld_spinner_lk,
572 1.2 ad __func__,
573 1.10 ad "not held by current CPU", true);
574 1.2 ad continue;
575 1.2 ad }
576 1.12 ad if (ld->ld_cpu == cpuno && (l->l_pflag & LP_INTR) == 0)
577 1.2 ad lockdebug_abort1(ld, &ld_spinner_lk,
578 1.10 ad __func__, "spin lock held", true);
579 1.2 ad }
580 1.2 ad lockdebug_unlock(&ld_spinner_lk);
581 1.2 ad }
582 1.2 ad
583 1.2 ad if (!slplocks) {
584 1.2 ad if (l->l_exlocks != 0) {
585 1.2 ad lockdebug_lock(&ld_sleeper_lk);
586 1.2 ad TAILQ_FOREACH(ld, &ld_sleepers, ld_chain) {
587 1.2 ad if (ld->ld_lwp == l)
588 1.2 ad lockdebug_abort1(ld, &ld_sleeper_lk,
589 1.10 ad __func__, "sleep lock held", true);
590 1.2 ad }
591 1.2 ad lockdebug_unlock(&ld_sleeper_lk);
592 1.2 ad }
593 1.2 ad if (l->l_shlocks != 0)
594 1.2 ad panic("lockdebug_barrier: holding %d shared locks",
595 1.2 ad l->l_shlocks);
596 1.2 ad }
597 1.2 ad }
598 1.2 ad
599 1.2 ad /*
600 1.10 ad * lockdebug_mem_check:
601 1.10 ad *
602 1.10 ad * Check for in-use locks within a memory region that is
603 1.16 yamt * being freed.
604 1.10 ad */
605 1.10 ad void
606 1.10 ad lockdebug_mem_check(const char *func, void *base, size_t sz)
607 1.10 ad {
608 1.10 ad #if 0
609 1.16 yamt lockdebug_t *ld;
610 1.10 ad lockdebuglk_t *lk;
611 1.16 yamt uintptr_t lock;
612 1.10 ad
613 1.16 yamt lockdebug_lock(&ld_tree_lk);
614 1.16 yamt ld = (lockdebug_t *)rb_tree_find_node_geq(&ld_rb_tree, base);
615 1.16 yamt lockdebug_unlock(&ld_tree_lk);
616 1.16 yamt if (ld == NULL)
617 1.16 yamt return;
618 1.16 yamt
619 1.16 yamt if ((ld->ld_flags & LD_SLEEPER) != 0)
620 1.16 yamt lk = &ld_sleeper_lk;
621 1.16 yamt else
622 1.16 yamt lk = &ld_spinner_lk;
623 1.10 ad
624 1.16 yamt lockdebug_lock(lk);
625 1.16 yamt lock = (uintptr_t)ld->ld_lock;
626 1.16 yamt if ((uintptr_t)base <= lock && lock < (uintptr_t)base + sz) {
627 1.16 yamt lockdebug_abort1(ld, lk, func,
628 1.16 yamt "allocation contains active lock", !cold);
629 1.16 yamt return;
630 1.10 ad }
631 1.10 ad lockdebug_unlock(lk);
632 1.10 ad #endif
633 1.10 ad }
634 1.10 ad
635 1.10 ad /*
636 1.2 ad * lockdebug_dump:
637 1.2 ad *
638 1.2 ad * Dump information about a lock on panic, or for DDB.
639 1.2 ad */
640 1.2 ad static void
641 1.2 ad lockdebug_dump(lockdebug_t *ld, void (*pr)(const char *, ...))
642 1.2 ad {
643 1.2 ad int sleeper = (ld->ld_flags & LD_SLEEPER);
644 1.2 ad
645 1.2 ad (*pr)(
646 1.2 ad "lock address : %#018lx type : %18s\n"
647 1.2 ad "shared holds : %18u exclusive: %18u\n"
648 1.2 ad "shares wanted: %18u exclusive: %18u\n"
649 1.2 ad "current cpu : %18u last held: %18u\n"
650 1.2 ad "current lwp : %#018lx last held: %#018lx\n"
651 1.10 ad "last locked : %#018lx unlocked : %#018lx\n"
652 1.10 ad "initialized : %#018lx\n",
653 1.2 ad (long)ld->ld_lock, (sleeper ? "sleep/adaptive" : "spin"),
654 1.2 ad (unsigned)ld->ld_shares, ((ld->ld_flags & LD_LOCKED) != 0),
655 1.2 ad (unsigned)ld->ld_shwant, (unsigned)ld->ld_exwant,
656 1.2 ad (unsigned)cpu_number(), (unsigned)ld->ld_cpu,
657 1.2 ad (long)curlwp, (long)ld->ld_lwp,
658 1.10 ad (long)ld->ld_locked, (long)ld->ld_unlocked,
659 1.10 ad (long)ld->ld_initaddr);
660 1.2 ad
661 1.2 ad if (ld->ld_lockops->lo_dump != NULL)
662 1.2 ad (*ld->ld_lockops->lo_dump)(ld->ld_lock);
663 1.2 ad
664 1.2 ad if (sleeper) {
665 1.2 ad (*pr)("\n");
666 1.2 ad turnstile_print(ld->ld_lock, pr);
667 1.2 ad }
668 1.2 ad }
669 1.2 ad
670 1.2 ad /*
671 1.2 ad * lockdebug_dump:
672 1.2 ad *
673 1.2 ad * Dump information about a known lock.
674 1.2 ad */
675 1.5 ad static void
676 1.2 ad lockdebug_abort1(lockdebug_t *ld, lockdebuglk_t *lk, const char *func,
677 1.10 ad const char *msg, bool dopanic)
678 1.2 ad {
679 1.2 ad
680 1.2 ad printf_nolog("%s error: %s: %s\n\n", ld->ld_lockops->lo_name,
681 1.2 ad func, msg);
682 1.2 ad lockdebug_dump(ld, printf_nolog);
683 1.2 ad lockdebug_unlock(lk);
684 1.2 ad printf_nolog("\n");
685 1.10 ad if (dopanic)
686 1.10 ad panic("LOCKDEBUG");
687 1.2 ad }
688 1.2 ad
689 1.2 ad #endif /* LOCKDEBUG */
690 1.2 ad
691 1.2 ad /*
692 1.2 ad * lockdebug_lock_print:
693 1.2 ad *
694 1.2 ad * Handle the DDB 'show lock' command.
695 1.2 ad */
696 1.2 ad #ifdef DDB
697 1.2 ad void
698 1.2 ad lockdebug_lock_print(void *addr, void (*pr)(const char *, ...))
699 1.2 ad {
700 1.2 ad #ifdef LOCKDEBUG
701 1.2 ad lockdebug_t *ld;
702 1.2 ad
703 1.2 ad TAILQ_FOREACH(ld, &ld_all, ld_achain) {
704 1.2 ad if (ld->ld_lock == addr) {
705 1.2 ad lockdebug_dump(ld, pr);
706 1.2 ad return;
707 1.2 ad }
708 1.2 ad }
709 1.2 ad (*pr)("Sorry, no record of a lock with address %p found.\n", addr);
710 1.2 ad #else
711 1.2 ad (*pr)("Sorry, kernel not built with the LOCKDEBUG option.\n");
712 1.2 ad #endif /* LOCKDEBUG */
713 1.2 ad }
714 1.2 ad #endif /* DDB */
715 1.2 ad
716 1.2 ad /*
717 1.2 ad * lockdebug_abort:
718 1.2 ad *
719 1.2 ad * An error has been trapped - dump lock info and call panic().
720 1.2 ad */
721 1.2 ad void
722 1.16 yamt lockdebug_abort(volatile void *lock, lockops_t *ops, const char *func,
723 1.16 yamt const char *msg)
724 1.2 ad {
725 1.2 ad #ifdef LOCKDEBUG
726 1.2 ad lockdebug_t *ld;
727 1.2 ad lockdebuglk_t *lk;
728 1.2 ad
729 1.16 yamt if ((ld = lockdebug_lookup(lock, &lk)) != NULL) {
730 1.10 ad lockdebug_abort1(ld, lk, func, msg, true);
731 1.2 ad /* NOTREACHED */
732 1.2 ad }
733 1.2 ad #endif /* LOCKDEBUG */
734 1.2 ad
735 1.2 ad printf_nolog("%s error: %s: %s\n\n"
736 1.2 ad "lock address : %#018lx\n"
737 1.2 ad "current cpu : %18d\n"
738 1.2 ad "current lwp : %#018lx\n",
739 1.2 ad ops->lo_name, func, msg, (long)lock, (int)cpu_number(),
740 1.2 ad (long)curlwp);
741 1.2 ad
742 1.2 ad (*ops->lo_dump)(lock);
743 1.2 ad
744 1.2 ad printf_nolog("\n");
745 1.2 ad panic("lock error");
746 1.2 ad }
747