subr_lockdebug.c revision 1.8.6.5 1 1.8.6.5 joerg /* $NetBSD: subr_lockdebug.c,v 1.8.6.5 2007/11/14 19:04:44 joerg Exp $ */
2 1.2 ad
3 1.2 ad /*-
4 1.2 ad * Copyright (c) 2006, 2007 The NetBSD Foundation, Inc.
5 1.2 ad * All rights reserved.
6 1.2 ad *
7 1.2 ad * This code is derived from software contributed to The NetBSD Foundation
8 1.2 ad * by Andrew Doran.
9 1.2 ad *
10 1.2 ad * Redistribution and use in source and binary forms, with or without
11 1.2 ad * modification, are permitted provided that the following conditions
12 1.2 ad * are met:
13 1.2 ad * 1. Redistributions of source code must retain the above copyright
14 1.2 ad * notice, this list of conditions and the following disclaimer.
15 1.2 ad * 2. Redistributions in binary form must reproduce the above copyright
16 1.2 ad * notice, this list of conditions and the following disclaimer in the
17 1.2 ad * documentation and/or other materials provided with the distribution.
18 1.2 ad * 3. All advertising materials mentioning features or use of this software
19 1.2 ad * must display the following acknowledgement:
20 1.2 ad * This product includes software developed by the NetBSD
21 1.2 ad * Foundation, Inc. and its contributors.
22 1.2 ad * 4. Neither the name of The NetBSD Foundation nor the names of its
23 1.2 ad * contributors may be used to endorse or promote products derived
24 1.2 ad * from this software without specific prior written permission.
25 1.2 ad *
26 1.2 ad * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
27 1.2 ad * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
28 1.2 ad * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
29 1.2 ad * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
30 1.2 ad * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
31 1.2 ad * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
32 1.2 ad * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
33 1.2 ad * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
34 1.2 ad * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
35 1.2 ad * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
36 1.2 ad * POSSIBILITY OF SUCH DAMAGE.
37 1.2 ad */
38 1.2 ad
39 1.2 ad /*
40 1.8.6.3 joerg * Basic lock debugging code shared among lock primitives.
41 1.2 ad */
42 1.2 ad
43 1.8.6.1 joerg #include <sys/cdefs.h>
44 1.8.6.5 joerg __KERNEL_RCSID(0, "$NetBSD: subr_lockdebug.c,v 1.8.6.5 2007/11/14 19:04:44 joerg Exp $");
45 1.8.6.1 joerg
46 1.2 ad #include "opt_multiprocessor.h"
47 1.2 ad #include "opt_ddb.h"
48 1.2 ad
49 1.2 ad #include <sys/param.h>
50 1.2 ad #include <sys/proc.h>
51 1.2 ad #include <sys/systm.h>
52 1.8.6.2 joerg #include <sys/kernel.h>
53 1.2 ad #include <sys/kmem.h>
54 1.2 ad #include <sys/lock.h>
55 1.2 ad #include <sys/lockdebug.h>
56 1.2 ad #include <sys/sleepq.h>
57 1.8.6.2 joerg #include <sys/cpu.h>
58 1.2 ad
59 1.2 ad #ifdef LOCKDEBUG
60 1.2 ad
61 1.2 ad #define LD_BATCH_SHIFT 9
62 1.2 ad #define LD_BATCH (1 << LD_BATCH_SHIFT)
63 1.2 ad #define LD_BATCH_MASK (LD_BATCH - 1)
64 1.2 ad #define LD_MAX_LOCKS 1048576
65 1.2 ad #define LD_SLOP 16
66 1.2 ad
67 1.2 ad #define LD_LOCKED 0x01
68 1.2 ad #define LD_SLEEPER 0x02
69 1.8.6.2 joerg #define LD_MLOCKS 8
70 1.8.6.2 joerg #define LD_MLISTS 8192
71 1.2 ad
72 1.5 ad #define LD_NOID (LD_MAX_LOCKS + 1)
73 1.2 ad
74 1.2 ad typedef union lockdebuglk {
75 1.2 ad struct {
76 1.2 ad __cpu_simple_lock_t lku_lock;
77 1.2 ad int lku_oldspl;
78 1.2 ad } ul;
79 1.2 ad uint8_t lk_pad[64];
80 1.2 ad } volatile __aligned(64) lockdebuglk_t;
81 1.2 ad
82 1.2 ad #define lk_lock ul.lku_lock
83 1.2 ad #define lk_oldspl ul.lku_oldspl
84 1.2 ad
85 1.2 ad typedef struct lockdebug {
86 1.2 ad _TAILQ_ENTRY(struct lockdebug, volatile) ld_chain;
87 1.2 ad _TAILQ_ENTRY(struct lockdebug, volatile) ld_achain;
88 1.8.6.2 joerg _TAILQ_ENTRY(struct lockdebug, volatile) ld_mchain;
89 1.2 ad volatile void *ld_lock;
90 1.2 ad lockops_t *ld_lockops;
91 1.2 ad struct lwp *ld_lwp;
92 1.2 ad uintptr_t ld_locked;
93 1.2 ad uintptr_t ld_unlocked;
94 1.8.6.2 joerg uintptr_t ld_initaddr;
95 1.2 ad u_int ld_id;
96 1.2 ad uint16_t ld_shares;
97 1.2 ad uint16_t ld_cpu;
98 1.2 ad uint8_t ld_flags;
99 1.2 ad uint8_t ld_shwant; /* advisory */
100 1.2 ad uint8_t ld_exwant; /* advisory */
101 1.2 ad uint8_t ld_unused;
102 1.2 ad } volatile lockdebug_t;
103 1.2 ad
104 1.2 ad typedef _TAILQ_HEAD(lockdebuglist, struct lockdebug, volatile) lockdebuglist_t;
105 1.2 ad
106 1.2 ad lockdebuglk_t ld_sleeper_lk;
107 1.2 ad lockdebuglk_t ld_spinner_lk;
108 1.2 ad lockdebuglk_t ld_free_lk;
109 1.8.6.2 joerg lockdebuglk_t ld_mem_lk[LD_MLOCKS];
110 1.2 ad
111 1.8.6.2 joerg lockdebuglist_t ld_mem_list[LD_MLISTS];
112 1.8.6.5 joerg lockdebuglist_t ld_sleepers = TAILQ_HEAD_INITIALIZER(ld_sleepers);
113 1.8.6.5 joerg lockdebuglist_t ld_spinners = TAILQ_HEAD_INITIALIZER(ld_spinners);
114 1.8.6.5 joerg lockdebuglist_t ld_free = TAILQ_HEAD_INITIALIZER(ld_free);
115 1.8.6.5 joerg lockdebuglist_t ld_all = TAILQ_HEAD_INITIALIZER(ld_all);
116 1.2 ad int ld_nfree;
117 1.2 ad int ld_freeptr;
118 1.2 ad int ld_recurse;
119 1.5 ad bool ld_nomore;
120 1.2 ad lockdebug_t *ld_table[LD_MAX_LOCKS / LD_BATCH];
121 1.2 ad
122 1.2 ad lockdebug_t ld_prime[LD_BATCH];
123 1.2 ad
124 1.5 ad static void lockdebug_abort1(lockdebug_t *, lockdebuglk_t *lk,
125 1.8.6.2 joerg const char *, const char *, bool);
126 1.5 ad static void lockdebug_more(void);
127 1.5 ad static void lockdebug_init(void);
128 1.2 ad
129 1.2 ad static inline void
130 1.2 ad lockdebug_lock(lockdebuglk_t *lk)
131 1.2 ad {
132 1.2 ad int s;
133 1.2 ad
134 1.8 ad s = splhigh();
135 1.2 ad __cpu_simple_lock(&lk->lk_lock);
136 1.2 ad lk->lk_oldspl = s;
137 1.2 ad }
138 1.2 ad
139 1.2 ad static inline void
140 1.2 ad lockdebug_unlock(lockdebuglk_t *lk)
141 1.2 ad {
142 1.2 ad int s;
143 1.2 ad
144 1.2 ad s = lk->lk_oldspl;
145 1.2 ad __cpu_simple_unlock(&(lk->lk_lock));
146 1.2 ad splx(s);
147 1.2 ad }
148 1.2 ad
149 1.8.6.2 joerg static inline void
150 1.8.6.2 joerg lockdebug_mhash(volatile void *addr, lockdebuglk_t **lk, lockdebuglist_t **head)
151 1.8.6.2 joerg {
152 1.8.6.2 joerg u_int hash;
153 1.8.6.2 joerg
154 1.8.6.2 joerg hash = (uintptr_t)addr >> PGSHIFT;
155 1.8.6.2 joerg *lk = &ld_mem_lk[hash & (LD_MLOCKS - 1)];
156 1.8.6.2 joerg *head = &ld_mem_list[hash & (LD_MLISTS - 1)];
157 1.8.6.2 joerg lockdebug_lock(*lk);
158 1.8.6.2 joerg }
159 1.8.6.2 joerg
160 1.2 ad /*
161 1.2 ad * lockdebug_lookup:
162 1.2 ad *
163 1.2 ad * Find a lockdebug structure by ID and return it locked.
164 1.2 ad */
165 1.2 ad static inline lockdebug_t *
166 1.2 ad lockdebug_lookup(u_int id, lockdebuglk_t **lk)
167 1.2 ad {
168 1.2 ad lockdebug_t *base, *ld;
169 1.2 ad
170 1.2 ad if (id == LD_NOID)
171 1.2 ad return NULL;
172 1.2 ad
173 1.2 ad if (id == 0 || id >= LD_MAX_LOCKS)
174 1.2 ad panic("lockdebug_lookup: uninitialized lock (1, id=%d)", id);
175 1.2 ad
176 1.2 ad base = ld_table[id >> LD_BATCH_SHIFT];
177 1.2 ad ld = base + (id & LD_BATCH_MASK);
178 1.2 ad
179 1.2 ad if (base == NULL || ld->ld_lock == NULL || ld->ld_id != id)
180 1.2 ad panic("lockdebug_lookup: uninitialized lock (2, id=%d)", id);
181 1.2 ad
182 1.2 ad if ((ld->ld_flags & LD_SLEEPER) != 0)
183 1.2 ad *lk = &ld_sleeper_lk;
184 1.2 ad else
185 1.2 ad *lk = &ld_spinner_lk;
186 1.2 ad
187 1.2 ad lockdebug_lock(*lk);
188 1.2 ad return ld;
189 1.2 ad }
190 1.2 ad
191 1.2 ad /*
192 1.2 ad * lockdebug_init:
193 1.2 ad *
194 1.2 ad * Initialize the lockdebug system. Allocate an initial pool of
195 1.2 ad * lockdebug structures before the VM system is up and running.
196 1.2 ad */
197 1.5 ad static void
198 1.2 ad lockdebug_init(void)
199 1.2 ad {
200 1.2 ad lockdebug_t *ld;
201 1.2 ad int i;
202 1.2 ad
203 1.2 ad __cpu_simple_lock_init(&ld_sleeper_lk.lk_lock);
204 1.2 ad __cpu_simple_lock_init(&ld_spinner_lk.lk_lock);
205 1.2 ad __cpu_simple_lock_init(&ld_free_lk.lk_lock);
206 1.2 ad
207 1.2 ad ld = ld_prime;
208 1.2 ad ld_table[0] = ld;
209 1.2 ad for (i = 1, ld++; i < LD_BATCH; i++, ld++) {
210 1.2 ad ld->ld_id = i;
211 1.2 ad TAILQ_INSERT_TAIL(&ld_free, ld, ld_chain);
212 1.2 ad TAILQ_INSERT_TAIL(&ld_all, ld, ld_achain);
213 1.2 ad }
214 1.2 ad ld_freeptr = 1;
215 1.2 ad ld_nfree = LD_BATCH - 1;
216 1.8.6.2 joerg
217 1.8.6.2 joerg for (i = 0; i < LD_MLOCKS; i++)
218 1.8.6.2 joerg __cpu_simple_lock_init(&ld_mem_lk[i].lk_lock);
219 1.8.6.2 joerg for (i = 0; i < LD_MLISTS; i++)
220 1.8.6.2 joerg TAILQ_INIT(&ld_mem_list[i]);
221 1.2 ad }
222 1.2 ad
223 1.2 ad /*
224 1.2 ad * lockdebug_alloc:
225 1.2 ad *
226 1.2 ad * A lock is being initialized, so allocate an associated debug
227 1.2 ad * structure.
228 1.2 ad */
229 1.2 ad u_int
230 1.8.6.2 joerg lockdebug_alloc(volatile void *lock, lockops_t *lo, uintptr_t initaddr)
231 1.2 ad {
232 1.8.6.2 joerg #if 0
233 1.8.6.2 joerg lockdebuglist_t *head;
234 1.8.6.2 joerg lockdebuglk_t *lk;
235 1.8.6.2 joerg #endif
236 1.2 ad struct cpu_info *ci;
237 1.2 ad lockdebug_t *ld;
238 1.2 ad
239 1.5 ad if (lo == NULL || panicstr != NULL)
240 1.2 ad return LD_NOID;
241 1.5 ad if (ld_freeptr == 0)
242 1.5 ad lockdebug_init();
243 1.2 ad
244 1.2 ad ci = curcpu();
245 1.2 ad
246 1.2 ad /*
247 1.2 ad * Pinch a new debug structure. We may recurse because we call
248 1.2 ad * kmem_alloc(), which may need to initialize new locks somewhere
249 1.7 skrll * down the path. If not recursing, we try to maintain at least
250 1.2 ad * LD_SLOP structures free, which should hopefully be enough to
251 1.2 ad * satisfy kmem_alloc(). If we can't provide a structure, not to
252 1.2 ad * worry: we'll just mark the lock as not having an ID.
253 1.2 ad */
254 1.2 ad lockdebug_lock(&ld_free_lk);
255 1.2 ad ci->ci_lkdebug_recurse++;
256 1.2 ad
257 1.2 ad if (TAILQ_EMPTY(&ld_free)) {
258 1.5 ad if (ci->ci_lkdebug_recurse > 1 || ld_nomore) {
259 1.2 ad ci->ci_lkdebug_recurse--;
260 1.2 ad lockdebug_unlock(&ld_free_lk);
261 1.2 ad return LD_NOID;
262 1.2 ad }
263 1.2 ad lockdebug_more();
264 1.2 ad } else if (ci->ci_lkdebug_recurse == 1 && ld_nfree < LD_SLOP)
265 1.2 ad lockdebug_more();
266 1.2 ad
267 1.2 ad if ((ld = TAILQ_FIRST(&ld_free)) == NULL) {
268 1.2 ad lockdebug_unlock(&ld_free_lk);
269 1.2 ad return LD_NOID;
270 1.2 ad }
271 1.2 ad
272 1.2 ad TAILQ_REMOVE(&ld_free, ld, ld_chain);
273 1.2 ad ld_nfree--;
274 1.2 ad
275 1.2 ad ci->ci_lkdebug_recurse--;
276 1.2 ad lockdebug_unlock(&ld_free_lk);
277 1.2 ad
278 1.2 ad if (ld->ld_lock != NULL)
279 1.2 ad panic("lockdebug_alloc: corrupt table");
280 1.2 ad
281 1.2 ad if (lo->lo_sleeplock)
282 1.2 ad lockdebug_lock(&ld_sleeper_lk);
283 1.2 ad else
284 1.2 ad lockdebug_lock(&ld_spinner_lk);
285 1.2 ad
286 1.2 ad /* Initialise the structure. */
287 1.2 ad ld->ld_lock = lock;
288 1.2 ad ld->ld_lockops = lo;
289 1.2 ad ld->ld_locked = 0;
290 1.2 ad ld->ld_unlocked = 0;
291 1.2 ad ld->ld_lwp = NULL;
292 1.8.6.2 joerg ld->ld_initaddr = initaddr;
293 1.2 ad
294 1.2 ad if (lo->lo_sleeplock) {
295 1.2 ad ld->ld_flags = LD_SLEEPER;
296 1.2 ad lockdebug_unlock(&ld_sleeper_lk);
297 1.2 ad } else {
298 1.2 ad ld->ld_flags = 0;
299 1.2 ad lockdebug_unlock(&ld_spinner_lk);
300 1.2 ad }
301 1.2 ad
302 1.8.6.2 joerg #if 0
303 1.8.6.2 joerg /* Insert into address hash. */
304 1.8.6.2 joerg lockdebug_mhash(lock, &lk, &head);
305 1.8.6.2 joerg TAILQ_INSERT_HEAD(head, ld, ld_mchain);
306 1.8.6.2 joerg lockdebug_unlock(lk);
307 1.8.6.2 joerg #endif
308 1.8.6.2 joerg
309 1.2 ad return ld->ld_id;
310 1.2 ad }
311 1.2 ad
312 1.2 ad /*
313 1.2 ad * lockdebug_free:
314 1.2 ad *
315 1.2 ad * A lock is being destroyed, so release debugging resources.
316 1.2 ad */
317 1.2 ad void
318 1.2 ad lockdebug_free(volatile void *lock, u_int id)
319 1.2 ad {
320 1.8.6.2 joerg #if 0
321 1.8.6.2 joerg lockdebuglist_t *head;
322 1.8.6.2 joerg #endif
323 1.2 ad lockdebug_t *ld;
324 1.2 ad lockdebuglk_t *lk;
325 1.2 ad
326 1.2 ad if (panicstr != NULL)
327 1.2 ad return;
328 1.2 ad
329 1.2 ad if ((ld = lockdebug_lookup(id, &lk)) == NULL)
330 1.2 ad return;
331 1.2 ad
332 1.2 ad if (ld->ld_lock != lock) {
333 1.2 ad panic("lockdebug_free: destroying uninitialized lock %p"
334 1.2 ad "(ld_id=%d ld_lock=%p)", lock, id, ld->ld_lock);
335 1.8.6.2 joerg lockdebug_abort1(ld, lk, __func__, "lock record follows",
336 1.8.6.2 joerg true);
337 1.2 ad }
338 1.2 ad if ((ld->ld_flags & LD_LOCKED) != 0 || ld->ld_shares != 0)
339 1.8.6.2 joerg lockdebug_abort1(ld, lk, __func__, "is locked", true);
340 1.2 ad
341 1.2 ad ld->ld_lock = NULL;
342 1.2 ad
343 1.2 ad lockdebug_unlock(lk);
344 1.2 ad
345 1.2 ad lockdebug_lock(&ld_free_lk);
346 1.2 ad TAILQ_INSERT_TAIL(&ld_free, ld, ld_chain);
347 1.2 ad ld_nfree++;
348 1.2 ad lockdebug_unlock(&ld_free_lk);
349 1.8.6.2 joerg
350 1.8.6.2 joerg #if 0
351 1.8.6.2 joerg /* Remove from address hash. */
352 1.8.6.2 joerg lockdebug_mhash(lock, &lk, &head);
353 1.8.6.2 joerg TAILQ_REMOVE(head, ld, ld_mchain);
354 1.8.6.2 joerg lockdebug_unlock(lk);
355 1.8.6.2 joerg #endif
356 1.2 ad }
357 1.2 ad
358 1.2 ad /*
359 1.2 ad * lockdebug_more:
360 1.2 ad *
361 1.2 ad * Allocate a batch of debug structures and add to the free list.
362 1.2 ad * Must be called with ld_free_lk held.
363 1.2 ad */
364 1.5 ad static void
365 1.2 ad lockdebug_more(void)
366 1.2 ad {
367 1.2 ad lockdebug_t *ld;
368 1.2 ad void *block;
369 1.5 ad int i, base, m;
370 1.2 ad
371 1.2 ad while (ld_nfree < LD_SLOP) {
372 1.2 ad lockdebug_unlock(&ld_free_lk);
373 1.2 ad block = kmem_zalloc(LD_BATCH * sizeof(lockdebug_t), KM_SLEEP);
374 1.2 ad lockdebug_lock(&ld_free_lk);
375 1.2 ad
376 1.2 ad if (block == NULL)
377 1.2 ad return;
378 1.2 ad
379 1.2 ad if (ld_nfree > LD_SLOP) {
380 1.2 ad /* Somebody beat us to it. */
381 1.2 ad lockdebug_unlock(&ld_free_lk);
382 1.2 ad kmem_free(block, LD_BATCH * sizeof(lockdebug_t));
383 1.2 ad lockdebug_lock(&ld_free_lk);
384 1.2 ad continue;
385 1.2 ad }
386 1.2 ad
387 1.2 ad base = ld_freeptr;
388 1.2 ad ld_nfree += LD_BATCH;
389 1.2 ad ld = block;
390 1.2 ad base <<= LD_BATCH_SHIFT;
391 1.5 ad m = min(LD_MAX_LOCKS, base + LD_BATCH);
392 1.5 ad
393 1.5 ad if (m == LD_MAX_LOCKS)
394 1.5 ad ld_nomore = true;
395 1.2 ad
396 1.5 ad for (i = base; i < m; i++, ld++) {
397 1.5 ad ld->ld_id = i;
398 1.2 ad TAILQ_INSERT_TAIL(&ld_free, ld, ld_chain);
399 1.2 ad TAILQ_INSERT_TAIL(&ld_all, ld, ld_achain);
400 1.2 ad }
401 1.2 ad
402 1.2 ad mb_write();
403 1.2 ad ld_table[ld_freeptr++] = block;
404 1.2 ad }
405 1.2 ad }
406 1.2 ad
407 1.2 ad /*
408 1.2 ad * lockdebug_wantlock:
409 1.2 ad *
410 1.2 ad * Process the preamble to a lock acquire.
411 1.2 ad */
412 1.2 ad void
413 1.2 ad lockdebug_wantlock(u_int id, uintptr_t where, int shared)
414 1.2 ad {
415 1.2 ad struct lwp *l = curlwp;
416 1.2 ad lockdebuglk_t *lk;
417 1.2 ad lockdebug_t *ld;
418 1.3 thorpej bool recurse;
419 1.2 ad
420 1.2 ad (void)shared;
421 1.4 thorpej recurse = false;
422 1.2 ad
423 1.2 ad if (panicstr != NULL)
424 1.2 ad return;
425 1.2 ad
426 1.2 ad if ((ld = lockdebug_lookup(id, &lk)) == NULL)
427 1.2 ad return;
428 1.2 ad
429 1.2 ad if ((ld->ld_flags & LD_LOCKED) != 0) {
430 1.2 ad if ((ld->ld_flags & LD_SLEEPER) != 0) {
431 1.2 ad if (ld->ld_lwp == l)
432 1.4 thorpej recurse = true;
433 1.2 ad } else if (ld->ld_cpu == (uint16_t)cpu_number())
434 1.4 thorpej recurse = true;
435 1.2 ad }
436 1.2 ad
437 1.8.6.2 joerg #ifdef notyet
438 1.8.6.2 joerg if (cpu_intr_p()) {
439 1.8.6.2 joerg if ((ld->ld_flags & LD_SLEEPER) != 0)
440 1.8.6.2 joerg lockdebug_abort1(ld, lk, __func__,
441 1.8.6.2 joerg "acquiring sleep lock from interrupt context",
442 1.8.6.2 joerg true);
443 1.8.6.2 joerg }
444 1.8.6.2 joerg #endif
445 1.8.6.2 joerg
446 1.2 ad if (shared)
447 1.2 ad ld->ld_shwant++;
448 1.2 ad else
449 1.2 ad ld->ld_exwant++;
450 1.2 ad
451 1.2 ad if (recurse)
452 1.8.6.2 joerg lockdebug_abort1(ld, lk, __func__, "locking against myself",
453 1.8.6.2 joerg true);
454 1.2 ad
455 1.2 ad lockdebug_unlock(lk);
456 1.2 ad }
457 1.2 ad
458 1.2 ad /*
459 1.2 ad * lockdebug_locked:
460 1.2 ad *
461 1.2 ad * Process a lock acquire operation.
462 1.2 ad */
463 1.2 ad void
464 1.2 ad lockdebug_locked(u_int id, uintptr_t where, int shared)
465 1.2 ad {
466 1.2 ad struct lwp *l = curlwp;
467 1.2 ad lockdebuglk_t *lk;
468 1.2 ad lockdebug_t *ld;
469 1.2 ad
470 1.2 ad if (panicstr != NULL)
471 1.2 ad return;
472 1.2 ad
473 1.2 ad if ((ld = lockdebug_lookup(id, &lk)) == NULL)
474 1.2 ad return;
475 1.2 ad
476 1.2 ad if (shared) {
477 1.2 ad l->l_shlocks++;
478 1.2 ad ld->ld_shares++;
479 1.2 ad ld->ld_shwant--;
480 1.2 ad } else {
481 1.2 ad if ((ld->ld_flags & LD_LOCKED) != 0)
482 1.2 ad lockdebug_abort1(ld, lk, __func__,
483 1.8.6.2 joerg "already locked", true);
484 1.2 ad
485 1.2 ad ld->ld_flags |= LD_LOCKED;
486 1.2 ad ld->ld_locked = where;
487 1.2 ad ld->ld_cpu = (uint16_t)cpu_number();
488 1.2 ad ld->ld_lwp = l;
489 1.2 ad ld->ld_exwant--;
490 1.2 ad
491 1.2 ad if ((ld->ld_flags & LD_SLEEPER) != 0) {
492 1.2 ad l->l_exlocks++;
493 1.2 ad TAILQ_INSERT_TAIL(&ld_sleepers, ld, ld_chain);
494 1.2 ad } else {
495 1.2 ad curcpu()->ci_spin_locks2++;
496 1.2 ad TAILQ_INSERT_TAIL(&ld_spinners, ld, ld_chain);
497 1.2 ad }
498 1.2 ad }
499 1.2 ad
500 1.2 ad lockdebug_unlock(lk);
501 1.2 ad }
502 1.2 ad
503 1.2 ad /*
504 1.2 ad * lockdebug_unlocked:
505 1.2 ad *
506 1.2 ad * Process a lock release operation.
507 1.2 ad */
508 1.2 ad void
509 1.2 ad lockdebug_unlocked(u_int id, uintptr_t where, int shared)
510 1.2 ad {
511 1.2 ad struct lwp *l = curlwp;
512 1.2 ad lockdebuglk_t *lk;
513 1.2 ad lockdebug_t *ld;
514 1.2 ad
515 1.2 ad if (panicstr != NULL)
516 1.2 ad return;
517 1.2 ad
518 1.2 ad if ((ld = lockdebug_lookup(id, &lk)) == NULL)
519 1.2 ad return;
520 1.2 ad
521 1.2 ad if (shared) {
522 1.2 ad if (l->l_shlocks == 0)
523 1.2 ad lockdebug_abort1(ld, lk, __func__,
524 1.8.6.2 joerg "no shared locks held by LWP", true);
525 1.2 ad if (ld->ld_shares == 0)
526 1.2 ad lockdebug_abort1(ld, lk, __func__,
527 1.8.6.2 joerg "no shared holds on this lock", true);
528 1.2 ad l->l_shlocks--;
529 1.2 ad ld->ld_shares--;
530 1.2 ad } else {
531 1.2 ad if ((ld->ld_flags & LD_LOCKED) == 0)
532 1.8.6.2 joerg lockdebug_abort1(ld, lk, __func__, "not locked",
533 1.8.6.2 joerg true);
534 1.2 ad
535 1.2 ad if ((ld->ld_flags & LD_SLEEPER) != 0) {
536 1.2 ad if (ld->ld_lwp != curlwp)
537 1.2 ad lockdebug_abort1(ld, lk, __func__,
538 1.8.6.2 joerg "not held by current LWP", true);
539 1.2 ad ld->ld_flags &= ~LD_LOCKED;
540 1.2 ad ld->ld_unlocked = where;
541 1.2 ad ld->ld_lwp = NULL;
542 1.2 ad curlwp->l_exlocks--;
543 1.2 ad TAILQ_REMOVE(&ld_sleepers, ld, ld_chain);
544 1.2 ad } else {
545 1.2 ad if (ld->ld_cpu != (uint16_t)cpu_number())
546 1.2 ad lockdebug_abort1(ld, lk, __func__,
547 1.8.6.2 joerg "not held by current CPU", true);
548 1.2 ad ld->ld_flags &= ~LD_LOCKED;
549 1.2 ad ld->ld_unlocked = where;
550 1.2 ad ld->ld_lwp = NULL;
551 1.2 ad curcpu()->ci_spin_locks2--;
552 1.2 ad TAILQ_REMOVE(&ld_spinners, ld, ld_chain);
553 1.2 ad }
554 1.2 ad }
555 1.2 ad
556 1.2 ad lockdebug_unlock(lk);
557 1.2 ad }
558 1.2 ad
559 1.2 ad /*
560 1.2 ad * lockdebug_barrier:
561 1.2 ad *
562 1.2 ad * Panic if we hold more than one specified spin lock, and optionally,
563 1.2 ad * if we hold sleep locks.
564 1.2 ad */
565 1.2 ad void
566 1.2 ad lockdebug_barrier(volatile void *spinlock, int slplocks)
567 1.2 ad {
568 1.2 ad struct lwp *l = curlwp;
569 1.2 ad lockdebug_t *ld;
570 1.2 ad uint16_t cpuno;
571 1.2 ad
572 1.2 ad if (panicstr != NULL)
573 1.2 ad return;
574 1.2 ad
575 1.2 ad if (curcpu()->ci_spin_locks2 != 0) {
576 1.2 ad cpuno = (uint16_t)cpu_number();
577 1.2 ad
578 1.2 ad lockdebug_lock(&ld_spinner_lk);
579 1.2 ad TAILQ_FOREACH(ld, &ld_spinners, ld_chain) {
580 1.2 ad if (ld->ld_lock == spinlock) {
581 1.2 ad if (ld->ld_cpu != cpuno)
582 1.2 ad lockdebug_abort1(ld, &ld_spinner_lk,
583 1.2 ad __func__,
584 1.8.6.2 joerg "not held by current CPU", true);
585 1.2 ad continue;
586 1.2 ad }
587 1.8.6.4 joerg if (ld->ld_cpu == cpuno && (l->l_pflag & LP_INTR) == 0)
588 1.2 ad lockdebug_abort1(ld, &ld_spinner_lk,
589 1.8.6.2 joerg __func__, "spin lock held", true);
590 1.2 ad }
591 1.2 ad lockdebug_unlock(&ld_spinner_lk);
592 1.2 ad }
593 1.2 ad
594 1.2 ad if (!slplocks) {
595 1.2 ad if (l->l_exlocks != 0) {
596 1.2 ad lockdebug_lock(&ld_sleeper_lk);
597 1.2 ad TAILQ_FOREACH(ld, &ld_sleepers, ld_chain) {
598 1.2 ad if (ld->ld_lwp == l)
599 1.2 ad lockdebug_abort1(ld, &ld_sleeper_lk,
600 1.8.6.2 joerg __func__, "sleep lock held", true);
601 1.2 ad }
602 1.2 ad lockdebug_unlock(&ld_sleeper_lk);
603 1.2 ad }
604 1.2 ad if (l->l_shlocks != 0)
605 1.2 ad panic("lockdebug_barrier: holding %d shared locks",
606 1.2 ad l->l_shlocks);
607 1.2 ad }
608 1.2 ad }
609 1.2 ad
610 1.2 ad /*
611 1.8.6.2 joerg * lockdebug_mem_check:
612 1.8.6.2 joerg *
613 1.8.6.2 joerg * Check for in-use locks within a memory region that is
614 1.8.6.2 joerg * being freed. We only check for active locks within the
615 1.8.6.2 joerg * first page of the allocation.
616 1.8.6.2 joerg */
617 1.8.6.2 joerg void
618 1.8.6.2 joerg lockdebug_mem_check(const char *func, void *base, size_t sz)
619 1.8.6.2 joerg {
620 1.8.6.2 joerg #if 0
621 1.8.6.2 joerg lockdebuglist_t *head;
622 1.8.6.2 joerg lockdebuglk_t *lk;
623 1.8.6.2 joerg lockdebug_t *ld;
624 1.8.6.2 joerg uintptr_t sa, ea, la;
625 1.8.6.2 joerg
626 1.8.6.2 joerg sa = (uintptr_t)base;
627 1.8.6.2 joerg ea = sa + sz;
628 1.8.6.2 joerg
629 1.8.6.2 joerg lockdebug_mhash(base, &lk, &head);
630 1.8.6.2 joerg TAILQ_FOREACH(ld, head, ld_mchain) {
631 1.8.6.2 joerg la = (uintptr_t)ld->ld_lock;
632 1.8.6.2 joerg if (la >= sa && la < ea) {
633 1.8.6.2 joerg lockdebug_abort1(ld, lk, func,
634 1.8.6.2 joerg "allocation contains active lock", !cold);
635 1.8.6.2 joerg return;
636 1.8.6.2 joerg }
637 1.8.6.2 joerg }
638 1.8.6.2 joerg lockdebug_unlock(lk);
639 1.8.6.2 joerg #endif
640 1.8.6.2 joerg }
641 1.8.6.2 joerg
642 1.8.6.2 joerg /*
643 1.2 ad * lockdebug_dump:
644 1.2 ad *
645 1.2 ad * Dump information about a lock on panic, or for DDB.
646 1.2 ad */
647 1.2 ad static void
648 1.2 ad lockdebug_dump(lockdebug_t *ld, void (*pr)(const char *, ...))
649 1.2 ad {
650 1.2 ad int sleeper = (ld->ld_flags & LD_SLEEPER);
651 1.2 ad
652 1.2 ad (*pr)(
653 1.2 ad "lock address : %#018lx type : %18s\n"
654 1.2 ad "shared holds : %18u exclusive: %18u\n"
655 1.2 ad "shares wanted: %18u exclusive: %18u\n"
656 1.2 ad "current cpu : %18u last held: %18u\n"
657 1.2 ad "current lwp : %#018lx last held: %#018lx\n"
658 1.8.6.2 joerg "last locked : %#018lx unlocked : %#018lx\n"
659 1.8.6.2 joerg "initialized : %#018lx\n",
660 1.2 ad (long)ld->ld_lock, (sleeper ? "sleep/adaptive" : "spin"),
661 1.2 ad (unsigned)ld->ld_shares, ((ld->ld_flags & LD_LOCKED) != 0),
662 1.2 ad (unsigned)ld->ld_shwant, (unsigned)ld->ld_exwant,
663 1.2 ad (unsigned)cpu_number(), (unsigned)ld->ld_cpu,
664 1.2 ad (long)curlwp, (long)ld->ld_lwp,
665 1.8.6.2 joerg (long)ld->ld_locked, (long)ld->ld_unlocked,
666 1.8.6.2 joerg (long)ld->ld_initaddr);
667 1.2 ad
668 1.2 ad if (ld->ld_lockops->lo_dump != NULL)
669 1.2 ad (*ld->ld_lockops->lo_dump)(ld->ld_lock);
670 1.2 ad
671 1.2 ad if (sleeper) {
672 1.2 ad (*pr)("\n");
673 1.2 ad turnstile_print(ld->ld_lock, pr);
674 1.2 ad }
675 1.2 ad }
676 1.2 ad
677 1.2 ad /*
678 1.2 ad * lockdebug_dump:
679 1.2 ad *
680 1.2 ad * Dump information about a known lock.
681 1.2 ad */
682 1.5 ad static void
683 1.2 ad lockdebug_abort1(lockdebug_t *ld, lockdebuglk_t *lk, const char *func,
684 1.8.6.2 joerg const char *msg, bool dopanic)
685 1.2 ad {
686 1.2 ad
687 1.2 ad printf_nolog("%s error: %s: %s\n\n", ld->ld_lockops->lo_name,
688 1.2 ad func, msg);
689 1.2 ad lockdebug_dump(ld, printf_nolog);
690 1.2 ad lockdebug_unlock(lk);
691 1.2 ad printf_nolog("\n");
692 1.8.6.2 joerg if (dopanic)
693 1.8.6.2 joerg panic("LOCKDEBUG");
694 1.2 ad }
695 1.2 ad
696 1.2 ad #endif /* LOCKDEBUG */
697 1.2 ad
698 1.2 ad /*
699 1.2 ad * lockdebug_lock_print:
700 1.2 ad *
701 1.2 ad * Handle the DDB 'show lock' command.
702 1.2 ad */
703 1.2 ad #ifdef DDB
704 1.2 ad void
705 1.2 ad lockdebug_lock_print(void *addr, void (*pr)(const char *, ...))
706 1.2 ad {
707 1.2 ad #ifdef LOCKDEBUG
708 1.2 ad lockdebug_t *ld;
709 1.2 ad
710 1.2 ad TAILQ_FOREACH(ld, &ld_all, ld_achain) {
711 1.2 ad if (ld->ld_lock == addr) {
712 1.2 ad lockdebug_dump(ld, pr);
713 1.2 ad return;
714 1.2 ad }
715 1.2 ad }
716 1.2 ad (*pr)("Sorry, no record of a lock with address %p found.\n", addr);
717 1.2 ad #else
718 1.2 ad (*pr)("Sorry, kernel not built with the LOCKDEBUG option.\n");
719 1.2 ad #endif /* LOCKDEBUG */
720 1.2 ad }
721 1.2 ad #endif /* DDB */
722 1.2 ad
723 1.2 ad /*
724 1.2 ad * lockdebug_abort:
725 1.2 ad *
726 1.2 ad * An error has been trapped - dump lock info and call panic().
727 1.2 ad */
728 1.2 ad void
729 1.6 yamt lockdebug_abort(u_int id, volatile void *lock, lockops_t *ops,
730 1.2 ad const char *func, const char *msg)
731 1.2 ad {
732 1.2 ad #ifdef LOCKDEBUG
733 1.2 ad lockdebug_t *ld;
734 1.2 ad lockdebuglk_t *lk;
735 1.2 ad
736 1.2 ad if ((ld = lockdebug_lookup(id, &lk)) != NULL) {
737 1.8.6.2 joerg lockdebug_abort1(ld, lk, func, msg, true);
738 1.2 ad /* NOTREACHED */
739 1.2 ad }
740 1.2 ad #endif /* LOCKDEBUG */
741 1.2 ad
742 1.2 ad printf_nolog("%s error: %s: %s\n\n"
743 1.2 ad "lock address : %#018lx\n"
744 1.2 ad "current cpu : %18d\n"
745 1.2 ad "current lwp : %#018lx\n",
746 1.2 ad ops->lo_name, func, msg, (long)lock, (int)cpu_number(),
747 1.2 ad (long)curlwp);
748 1.2 ad
749 1.2 ad (*ops->lo_dump)(lock);
750 1.2 ad
751 1.2 ad printf_nolog("\n");
752 1.2 ad panic("lock error");
753 1.2 ad }
754