Home | History | Annotate | Line # | Download | only in kern
subr_lockdebug.c revision 1.8.6.5
      1  1.8.6.5    joerg /*	$NetBSD: subr_lockdebug.c,v 1.8.6.5 2007/11/14 19:04:44 joerg Exp $	*/
      2      1.2       ad 
      3      1.2       ad /*-
      4      1.2       ad  * Copyright (c) 2006, 2007 The NetBSD Foundation, Inc.
      5      1.2       ad  * All rights reserved.
      6      1.2       ad  *
      7      1.2       ad  * This code is derived from software contributed to The NetBSD Foundation
      8      1.2       ad  * by Andrew Doran.
      9      1.2       ad  *
     10      1.2       ad  * Redistribution and use in source and binary forms, with or without
     11      1.2       ad  * modification, are permitted provided that the following conditions
     12      1.2       ad  * are met:
     13      1.2       ad  * 1. Redistributions of source code must retain the above copyright
     14      1.2       ad  *    notice, this list of conditions and the following disclaimer.
     15      1.2       ad  * 2. Redistributions in binary form must reproduce the above copyright
     16      1.2       ad  *    notice, this list of conditions and the following disclaimer in the
     17      1.2       ad  *    documentation and/or other materials provided with the distribution.
     18      1.2       ad  * 3. All advertising materials mentioning features or use of this software
     19      1.2       ad  *    must display the following acknowledgement:
     20      1.2       ad  *	This product includes software developed by the NetBSD
     21      1.2       ad  *	Foundation, Inc. and its contributors.
     22      1.2       ad  * 4. Neither the name of The NetBSD Foundation nor the names of its
     23      1.2       ad  *    contributors may be used to endorse or promote products derived
     24      1.2       ad  *    from this software without specific prior written permission.
     25      1.2       ad  *
     26      1.2       ad  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     27      1.2       ad  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     28      1.2       ad  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     29      1.2       ad  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     30      1.2       ad  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     31      1.2       ad  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     32      1.2       ad  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     33      1.2       ad  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     34      1.2       ad  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     35      1.2       ad  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     36      1.2       ad  * POSSIBILITY OF SUCH DAMAGE.
     37      1.2       ad  */
     38      1.2       ad 
     39      1.2       ad /*
     40  1.8.6.3    joerg  * Basic lock debugging code shared among lock primitives.
     41      1.2       ad  */
     42      1.2       ad 
     43  1.8.6.1    joerg #include <sys/cdefs.h>
     44  1.8.6.5    joerg __KERNEL_RCSID(0, "$NetBSD: subr_lockdebug.c,v 1.8.6.5 2007/11/14 19:04:44 joerg Exp $");
     45  1.8.6.1    joerg 
     46      1.2       ad #include "opt_multiprocessor.h"
     47      1.2       ad #include "opt_ddb.h"
     48      1.2       ad 
     49      1.2       ad #include <sys/param.h>
     50      1.2       ad #include <sys/proc.h>
     51      1.2       ad #include <sys/systm.h>
     52  1.8.6.2    joerg #include <sys/kernel.h>
     53      1.2       ad #include <sys/kmem.h>
     54      1.2       ad #include <sys/lock.h>
     55      1.2       ad #include <sys/lockdebug.h>
     56      1.2       ad #include <sys/sleepq.h>
     57  1.8.6.2    joerg #include <sys/cpu.h>
     58      1.2       ad 
     59      1.2       ad #ifdef LOCKDEBUG
     60      1.2       ad 
     61      1.2       ad #define	LD_BATCH_SHIFT	9
     62      1.2       ad #define	LD_BATCH	(1 << LD_BATCH_SHIFT)
     63      1.2       ad #define	LD_BATCH_MASK	(LD_BATCH - 1)
     64      1.2       ad #define	LD_MAX_LOCKS	1048576
     65      1.2       ad #define	LD_SLOP		16
     66      1.2       ad 
     67      1.2       ad #define	LD_LOCKED	0x01
     68      1.2       ad #define	LD_SLEEPER	0x02
     69  1.8.6.2    joerg #define	LD_MLOCKS	8
     70  1.8.6.2    joerg #define	LD_MLISTS	8192
     71      1.2       ad 
     72      1.5       ad #define	LD_NOID		(LD_MAX_LOCKS + 1)
     73      1.2       ad 
     74      1.2       ad typedef union lockdebuglk {
     75      1.2       ad 	struct {
     76      1.2       ad 		__cpu_simple_lock_t	lku_lock;
     77      1.2       ad 		int			lku_oldspl;
     78      1.2       ad 	} ul;
     79      1.2       ad 	uint8_t	lk_pad[64];
     80      1.2       ad } volatile __aligned(64) lockdebuglk_t;
     81      1.2       ad 
     82      1.2       ad #define	lk_lock		ul.lku_lock
     83      1.2       ad #define	lk_oldspl	ul.lku_oldspl
     84      1.2       ad 
     85      1.2       ad typedef struct lockdebug {
     86      1.2       ad 	_TAILQ_ENTRY(struct lockdebug, volatile) ld_chain;
     87      1.2       ad 	_TAILQ_ENTRY(struct lockdebug, volatile) ld_achain;
     88  1.8.6.2    joerg 	_TAILQ_ENTRY(struct lockdebug, volatile) ld_mchain;
     89      1.2       ad 	volatile void	*ld_lock;
     90      1.2       ad 	lockops_t	*ld_lockops;
     91      1.2       ad 	struct lwp	*ld_lwp;
     92      1.2       ad 	uintptr_t	ld_locked;
     93      1.2       ad 	uintptr_t	ld_unlocked;
     94  1.8.6.2    joerg 	uintptr_t	ld_initaddr;
     95      1.2       ad 	u_int		ld_id;
     96      1.2       ad 	uint16_t	ld_shares;
     97      1.2       ad 	uint16_t	ld_cpu;
     98      1.2       ad 	uint8_t		ld_flags;
     99      1.2       ad 	uint8_t		ld_shwant;	/* advisory */
    100      1.2       ad 	uint8_t		ld_exwant;	/* advisory */
    101      1.2       ad 	uint8_t		ld_unused;
    102      1.2       ad } volatile lockdebug_t;
    103      1.2       ad 
    104      1.2       ad typedef _TAILQ_HEAD(lockdebuglist, struct lockdebug, volatile) lockdebuglist_t;
    105      1.2       ad 
    106      1.2       ad lockdebuglk_t		ld_sleeper_lk;
    107      1.2       ad lockdebuglk_t		ld_spinner_lk;
    108      1.2       ad lockdebuglk_t		ld_free_lk;
    109  1.8.6.2    joerg lockdebuglk_t		ld_mem_lk[LD_MLOCKS];
    110      1.2       ad 
    111  1.8.6.2    joerg lockdebuglist_t		ld_mem_list[LD_MLISTS];
    112  1.8.6.5    joerg lockdebuglist_t		ld_sleepers = TAILQ_HEAD_INITIALIZER(ld_sleepers);
    113  1.8.6.5    joerg lockdebuglist_t		ld_spinners = TAILQ_HEAD_INITIALIZER(ld_spinners);
    114  1.8.6.5    joerg lockdebuglist_t		ld_free = TAILQ_HEAD_INITIALIZER(ld_free);
    115  1.8.6.5    joerg lockdebuglist_t		ld_all = TAILQ_HEAD_INITIALIZER(ld_all);
    116      1.2       ad int			ld_nfree;
    117      1.2       ad int			ld_freeptr;
    118      1.2       ad int			ld_recurse;
    119      1.5       ad bool			ld_nomore;
    120      1.2       ad lockdebug_t		*ld_table[LD_MAX_LOCKS / LD_BATCH];
    121      1.2       ad 
    122      1.2       ad lockdebug_t		ld_prime[LD_BATCH];
    123      1.2       ad 
    124      1.5       ad static void	lockdebug_abort1(lockdebug_t *, lockdebuglk_t *lk,
    125  1.8.6.2    joerg 				 const char *, const char *, bool);
    126      1.5       ad static void	lockdebug_more(void);
    127      1.5       ad static void	lockdebug_init(void);
    128      1.2       ad 
    129      1.2       ad static inline void
    130      1.2       ad lockdebug_lock(lockdebuglk_t *lk)
    131      1.2       ad {
    132      1.2       ad 	int s;
    133      1.2       ad 
    134      1.8       ad 	s = splhigh();
    135      1.2       ad 	__cpu_simple_lock(&lk->lk_lock);
    136      1.2       ad 	lk->lk_oldspl = s;
    137      1.2       ad }
    138      1.2       ad 
    139      1.2       ad static inline void
    140      1.2       ad lockdebug_unlock(lockdebuglk_t *lk)
    141      1.2       ad {
    142      1.2       ad 	int s;
    143      1.2       ad 
    144      1.2       ad 	s = lk->lk_oldspl;
    145      1.2       ad 	__cpu_simple_unlock(&(lk->lk_lock));
    146      1.2       ad 	splx(s);
    147      1.2       ad }
    148      1.2       ad 
    149  1.8.6.2    joerg static inline void
    150  1.8.6.2    joerg lockdebug_mhash(volatile void *addr, lockdebuglk_t **lk, lockdebuglist_t **head)
    151  1.8.6.2    joerg {
    152  1.8.6.2    joerg 	u_int hash;
    153  1.8.6.2    joerg 
    154  1.8.6.2    joerg 	hash = (uintptr_t)addr >> PGSHIFT;
    155  1.8.6.2    joerg 	*lk = &ld_mem_lk[hash & (LD_MLOCKS - 1)];
    156  1.8.6.2    joerg 	*head = &ld_mem_list[hash & (LD_MLISTS - 1)];
    157  1.8.6.2    joerg 	lockdebug_lock(*lk);
    158  1.8.6.2    joerg }
    159  1.8.6.2    joerg 
    160      1.2       ad /*
    161      1.2       ad  * lockdebug_lookup:
    162      1.2       ad  *
    163      1.2       ad  *	Find a lockdebug structure by ID and return it locked.
    164      1.2       ad  */
    165      1.2       ad static inline lockdebug_t *
    166      1.2       ad lockdebug_lookup(u_int id, lockdebuglk_t **lk)
    167      1.2       ad {
    168      1.2       ad 	lockdebug_t *base, *ld;
    169      1.2       ad 
    170      1.2       ad 	if (id == LD_NOID)
    171      1.2       ad 		return NULL;
    172      1.2       ad 
    173      1.2       ad 	if (id == 0 || id >= LD_MAX_LOCKS)
    174      1.2       ad 		panic("lockdebug_lookup: uninitialized lock (1, id=%d)", id);
    175      1.2       ad 
    176      1.2       ad 	base = ld_table[id >> LD_BATCH_SHIFT];
    177      1.2       ad 	ld = base + (id & LD_BATCH_MASK);
    178      1.2       ad 
    179      1.2       ad 	if (base == NULL || ld->ld_lock == NULL || ld->ld_id != id)
    180      1.2       ad 		panic("lockdebug_lookup: uninitialized lock (2, id=%d)", id);
    181      1.2       ad 
    182      1.2       ad 	if ((ld->ld_flags & LD_SLEEPER) != 0)
    183      1.2       ad 		*lk = &ld_sleeper_lk;
    184      1.2       ad 	else
    185      1.2       ad 		*lk = &ld_spinner_lk;
    186      1.2       ad 
    187      1.2       ad 	lockdebug_lock(*lk);
    188      1.2       ad 	return ld;
    189      1.2       ad }
    190      1.2       ad 
    191      1.2       ad /*
    192      1.2       ad  * lockdebug_init:
    193      1.2       ad  *
    194      1.2       ad  *	Initialize the lockdebug system.  Allocate an initial pool of
    195      1.2       ad  *	lockdebug structures before the VM system is up and running.
    196      1.2       ad  */
    197      1.5       ad static void
    198      1.2       ad lockdebug_init(void)
    199      1.2       ad {
    200      1.2       ad 	lockdebug_t *ld;
    201      1.2       ad 	int i;
    202      1.2       ad 
    203      1.2       ad 	__cpu_simple_lock_init(&ld_sleeper_lk.lk_lock);
    204      1.2       ad 	__cpu_simple_lock_init(&ld_spinner_lk.lk_lock);
    205      1.2       ad 	__cpu_simple_lock_init(&ld_free_lk.lk_lock);
    206      1.2       ad 
    207      1.2       ad 	ld = ld_prime;
    208      1.2       ad 	ld_table[0] = ld;
    209      1.2       ad 	for (i = 1, ld++; i < LD_BATCH; i++, ld++) {
    210      1.2       ad 		ld->ld_id = i;
    211      1.2       ad 		TAILQ_INSERT_TAIL(&ld_free, ld, ld_chain);
    212      1.2       ad 		TAILQ_INSERT_TAIL(&ld_all, ld, ld_achain);
    213      1.2       ad 	}
    214      1.2       ad 	ld_freeptr = 1;
    215      1.2       ad 	ld_nfree = LD_BATCH - 1;
    216  1.8.6.2    joerg 
    217  1.8.6.2    joerg 	for (i = 0; i < LD_MLOCKS; i++)
    218  1.8.6.2    joerg 		__cpu_simple_lock_init(&ld_mem_lk[i].lk_lock);
    219  1.8.6.2    joerg 	for (i = 0; i < LD_MLISTS; i++)
    220  1.8.6.2    joerg 		TAILQ_INIT(&ld_mem_list[i]);
    221      1.2       ad }
    222      1.2       ad 
    223      1.2       ad /*
    224      1.2       ad  * lockdebug_alloc:
    225      1.2       ad  *
    226      1.2       ad  *	A lock is being initialized, so allocate an associated debug
    227      1.2       ad  *	structure.
    228      1.2       ad  */
    229      1.2       ad u_int
    230  1.8.6.2    joerg lockdebug_alloc(volatile void *lock, lockops_t *lo, uintptr_t initaddr)
    231      1.2       ad {
    232  1.8.6.2    joerg #if 0
    233  1.8.6.2    joerg 	lockdebuglist_t *head;
    234  1.8.6.2    joerg 	lockdebuglk_t *lk;
    235  1.8.6.2    joerg #endif
    236      1.2       ad 	struct cpu_info *ci;
    237      1.2       ad 	lockdebug_t *ld;
    238      1.2       ad 
    239      1.5       ad 	if (lo == NULL || panicstr != NULL)
    240      1.2       ad 		return LD_NOID;
    241      1.5       ad 	if (ld_freeptr == 0)
    242      1.5       ad 		lockdebug_init();
    243      1.2       ad 
    244      1.2       ad 	ci = curcpu();
    245      1.2       ad 
    246      1.2       ad 	/*
    247      1.2       ad 	 * Pinch a new debug structure.  We may recurse because we call
    248      1.2       ad 	 * kmem_alloc(), which may need to initialize new locks somewhere
    249      1.7    skrll 	 * down the path.  If not recursing, we try to maintain at least
    250      1.2       ad 	 * LD_SLOP structures free, which should hopefully be enough to
    251      1.2       ad 	 * satisfy kmem_alloc().  If we can't provide a structure, not to
    252      1.2       ad 	 * worry: we'll just mark the lock as not having an ID.
    253      1.2       ad 	 */
    254      1.2       ad 	lockdebug_lock(&ld_free_lk);
    255      1.2       ad 	ci->ci_lkdebug_recurse++;
    256      1.2       ad 
    257      1.2       ad 	if (TAILQ_EMPTY(&ld_free)) {
    258      1.5       ad 		if (ci->ci_lkdebug_recurse > 1 || ld_nomore) {
    259      1.2       ad 			ci->ci_lkdebug_recurse--;
    260      1.2       ad 			lockdebug_unlock(&ld_free_lk);
    261      1.2       ad 			return LD_NOID;
    262      1.2       ad 		}
    263      1.2       ad 		lockdebug_more();
    264      1.2       ad 	} else if (ci->ci_lkdebug_recurse == 1 && ld_nfree < LD_SLOP)
    265      1.2       ad 		lockdebug_more();
    266      1.2       ad 
    267      1.2       ad 	if ((ld = TAILQ_FIRST(&ld_free)) == NULL) {
    268      1.2       ad 		lockdebug_unlock(&ld_free_lk);
    269      1.2       ad 		return LD_NOID;
    270      1.2       ad 	}
    271      1.2       ad 
    272      1.2       ad 	TAILQ_REMOVE(&ld_free, ld, ld_chain);
    273      1.2       ad 	ld_nfree--;
    274      1.2       ad 
    275      1.2       ad 	ci->ci_lkdebug_recurse--;
    276      1.2       ad 	lockdebug_unlock(&ld_free_lk);
    277      1.2       ad 
    278      1.2       ad 	if (ld->ld_lock != NULL)
    279      1.2       ad 		panic("lockdebug_alloc: corrupt table");
    280      1.2       ad 
    281      1.2       ad 	if (lo->lo_sleeplock)
    282      1.2       ad 		lockdebug_lock(&ld_sleeper_lk);
    283      1.2       ad 	else
    284      1.2       ad 		lockdebug_lock(&ld_spinner_lk);
    285      1.2       ad 
    286      1.2       ad 	/* Initialise the structure. */
    287      1.2       ad 	ld->ld_lock = lock;
    288      1.2       ad 	ld->ld_lockops = lo;
    289      1.2       ad 	ld->ld_locked = 0;
    290      1.2       ad 	ld->ld_unlocked = 0;
    291      1.2       ad 	ld->ld_lwp = NULL;
    292  1.8.6.2    joerg 	ld->ld_initaddr = initaddr;
    293      1.2       ad 
    294      1.2       ad 	if (lo->lo_sleeplock) {
    295      1.2       ad 		ld->ld_flags = LD_SLEEPER;
    296      1.2       ad 		lockdebug_unlock(&ld_sleeper_lk);
    297      1.2       ad 	} else {
    298      1.2       ad 		ld->ld_flags = 0;
    299      1.2       ad 		lockdebug_unlock(&ld_spinner_lk);
    300      1.2       ad 	}
    301      1.2       ad 
    302  1.8.6.2    joerg #if 0
    303  1.8.6.2    joerg 	/* Insert into address hash. */
    304  1.8.6.2    joerg 	lockdebug_mhash(lock, &lk, &head);
    305  1.8.6.2    joerg 	TAILQ_INSERT_HEAD(head, ld, ld_mchain);
    306  1.8.6.2    joerg 	lockdebug_unlock(lk);
    307  1.8.6.2    joerg #endif
    308  1.8.6.2    joerg 
    309      1.2       ad 	return ld->ld_id;
    310      1.2       ad }
    311      1.2       ad 
    312      1.2       ad /*
    313      1.2       ad  * lockdebug_free:
    314      1.2       ad  *
    315      1.2       ad  *	A lock is being destroyed, so release debugging resources.
    316      1.2       ad  */
    317      1.2       ad void
    318      1.2       ad lockdebug_free(volatile void *lock, u_int id)
    319      1.2       ad {
    320  1.8.6.2    joerg #if 0
    321  1.8.6.2    joerg 	lockdebuglist_t *head;
    322  1.8.6.2    joerg #endif
    323      1.2       ad 	lockdebug_t *ld;
    324      1.2       ad 	lockdebuglk_t *lk;
    325      1.2       ad 
    326      1.2       ad 	if (panicstr != NULL)
    327      1.2       ad 		return;
    328      1.2       ad 
    329      1.2       ad 	if ((ld = lockdebug_lookup(id, &lk)) == NULL)
    330      1.2       ad 		return;
    331      1.2       ad 
    332      1.2       ad 	if (ld->ld_lock != lock) {
    333      1.2       ad 		panic("lockdebug_free: destroying uninitialized lock %p"
    334      1.2       ad 		    "(ld_id=%d ld_lock=%p)", lock, id, ld->ld_lock);
    335  1.8.6.2    joerg 		lockdebug_abort1(ld, lk, __func__, "lock record follows",
    336  1.8.6.2    joerg 		    true);
    337      1.2       ad 	}
    338      1.2       ad 	if ((ld->ld_flags & LD_LOCKED) != 0 || ld->ld_shares != 0)
    339  1.8.6.2    joerg 		lockdebug_abort1(ld, lk, __func__, "is locked", true);
    340      1.2       ad 
    341      1.2       ad 	ld->ld_lock = NULL;
    342      1.2       ad 
    343      1.2       ad 	lockdebug_unlock(lk);
    344      1.2       ad 
    345      1.2       ad 	lockdebug_lock(&ld_free_lk);
    346      1.2       ad 	TAILQ_INSERT_TAIL(&ld_free, ld, ld_chain);
    347      1.2       ad 	ld_nfree++;
    348      1.2       ad 	lockdebug_unlock(&ld_free_lk);
    349  1.8.6.2    joerg 
    350  1.8.6.2    joerg #if 0
    351  1.8.6.2    joerg 	/* Remove from address hash. */
    352  1.8.6.2    joerg 	lockdebug_mhash(lock, &lk, &head);
    353  1.8.6.2    joerg 	TAILQ_REMOVE(head, ld, ld_mchain);
    354  1.8.6.2    joerg 	lockdebug_unlock(lk);
    355  1.8.6.2    joerg #endif
    356      1.2       ad }
    357      1.2       ad 
    358      1.2       ad /*
    359      1.2       ad  * lockdebug_more:
    360      1.2       ad  *
    361      1.2       ad  *	Allocate a batch of debug structures and add to the free list.
    362      1.2       ad  *	Must be called with ld_free_lk held.
    363      1.2       ad  */
    364      1.5       ad static void
    365      1.2       ad lockdebug_more(void)
    366      1.2       ad {
    367      1.2       ad 	lockdebug_t *ld;
    368      1.2       ad 	void *block;
    369      1.5       ad 	int i, base, m;
    370      1.2       ad 
    371      1.2       ad 	while (ld_nfree < LD_SLOP) {
    372      1.2       ad 		lockdebug_unlock(&ld_free_lk);
    373      1.2       ad 		block = kmem_zalloc(LD_BATCH * sizeof(lockdebug_t), KM_SLEEP);
    374      1.2       ad 		lockdebug_lock(&ld_free_lk);
    375      1.2       ad 
    376      1.2       ad 		if (block == NULL)
    377      1.2       ad 			return;
    378      1.2       ad 
    379      1.2       ad 		if (ld_nfree > LD_SLOP) {
    380      1.2       ad 			/* Somebody beat us to it. */
    381      1.2       ad 			lockdebug_unlock(&ld_free_lk);
    382      1.2       ad 			kmem_free(block, LD_BATCH * sizeof(lockdebug_t));
    383      1.2       ad 			lockdebug_lock(&ld_free_lk);
    384      1.2       ad 			continue;
    385      1.2       ad 		}
    386      1.2       ad 
    387      1.2       ad 		base = ld_freeptr;
    388      1.2       ad 		ld_nfree += LD_BATCH;
    389      1.2       ad 		ld = block;
    390      1.2       ad 		base <<= LD_BATCH_SHIFT;
    391      1.5       ad 		m = min(LD_MAX_LOCKS, base + LD_BATCH);
    392      1.5       ad 
    393      1.5       ad 		if (m == LD_MAX_LOCKS)
    394      1.5       ad 			ld_nomore = true;
    395      1.2       ad 
    396      1.5       ad 		for (i = base; i < m; i++, ld++) {
    397      1.5       ad 			ld->ld_id = i;
    398      1.2       ad 			TAILQ_INSERT_TAIL(&ld_free, ld, ld_chain);
    399      1.2       ad 			TAILQ_INSERT_TAIL(&ld_all, ld, ld_achain);
    400      1.2       ad 		}
    401      1.2       ad 
    402      1.2       ad 		mb_write();
    403      1.2       ad 		ld_table[ld_freeptr++] = block;
    404      1.2       ad 	}
    405      1.2       ad }
    406      1.2       ad 
    407      1.2       ad /*
    408      1.2       ad  * lockdebug_wantlock:
    409      1.2       ad  *
    410      1.2       ad  *	Process the preamble to a lock acquire.
    411      1.2       ad  */
    412      1.2       ad void
    413      1.2       ad lockdebug_wantlock(u_int id, uintptr_t where, int shared)
    414      1.2       ad {
    415      1.2       ad 	struct lwp *l = curlwp;
    416      1.2       ad 	lockdebuglk_t *lk;
    417      1.2       ad 	lockdebug_t *ld;
    418      1.3  thorpej 	bool recurse;
    419      1.2       ad 
    420      1.2       ad 	(void)shared;
    421      1.4  thorpej 	recurse = false;
    422      1.2       ad 
    423      1.2       ad 	if (panicstr != NULL)
    424      1.2       ad 		return;
    425      1.2       ad 
    426      1.2       ad 	if ((ld = lockdebug_lookup(id, &lk)) == NULL)
    427      1.2       ad 		return;
    428      1.2       ad 
    429      1.2       ad 	if ((ld->ld_flags & LD_LOCKED) != 0) {
    430      1.2       ad 		if ((ld->ld_flags & LD_SLEEPER) != 0) {
    431      1.2       ad 			if (ld->ld_lwp == l)
    432      1.4  thorpej 				recurse = true;
    433      1.2       ad 		} else if (ld->ld_cpu == (uint16_t)cpu_number())
    434      1.4  thorpej 			recurse = true;
    435      1.2       ad 	}
    436      1.2       ad 
    437  1.8.6.2    joerg #ifdef notyet
    438  1.8.6.2    joerg 	if (cpu_intr_p()) {
    439  1.8.6.2    joerg 		if ((ld->ld_flags & LD_SLEEPER) != 0)
    440  1.8.6.2    joerg 			lockdebug_abort1(ld, lk, __func__,
    441  1.8.6.2    joerg 			    "acquiring sleep lock from interrupt context",
    442  1.8.6.2    joerg 			    true);
    443  1.8.6.2    joerg 	}
    444  1.8.6.2    joerg #endif
    445  1.8.6.2    joerg 
    446      1.2       ad 	if (shared)
    447      1.2       ad 		ld->ld_shwant++;
    448      1.2       ad 	else
    449      1.2       ad 		ld->ld_exwant++;
    450      1.2       ad 
    451      1.2       ad 	if (recurse)
    452  1.8.6.2    joerg 		lockdebug_abort1(ld, lk, __func__, "locking against myself",
    453  1.8.6.2    joerg 		    true);
    454      1.2       ad 
    455      1.2       ad 	lockdebug_unlock(lk);
    456      1.2       ad }
    457      1.2       ad 
    458      1.2       ad /*
    459      1.2       ad  * lockdebug_locked:
    460      1.2       ad  *
    461      1.2       ad  *	Process a lock acquire operation.
    462      1.2       ad  */
    463      1.2       ad void
    464      1.2       ad lockdebug_locked(u_int id, uintptr_t where, int shared)
    465      1.2       ad {
    466      1.2       ad 	struct lwp *l = curlwp;
    467      1.2       ad 	lockdebuglk_t *lk;
    468      1.2       ad 	lockdebug_t *ld;
    469      1.2       ad 
    470      1.2       ad 	if (panicstr != NULL)
    471      1.2       ad 		return;
    472      1.2       ad 
    473      1.2       ad 	if ((ld = lockdebug_lookup(id, &lk)) == NULL)
    474      1.2       ad 		return;
    475      1.2       ad 
    476      1.2       ad 	if (shared) {
    477      1.2       ad 		l->l_shlocks++;
    478      1.2       ad 		ld->ld_shares++;
    479      1.2       ad 		ld->ld_shwant--;
    480      1.2       ad 	} else {
    481      1.2       ad 		if ((ld->ld_flags & LD_LOCKED) != 0)
    482      1.2       ad 			lockdebug_abort1(ld, lk, __func__,
    483  1.8.6.2    joerg 			    "already locked", true);
    484      1.2       ad 
    485      1.2       ad 		ld->ld_flags |= LD_LOCKED;
    486      1.2       ad 		ld->ld_locked = where;
    487      1.2       ad 		ld->ld_cpu = (uint16_t)cpu_number();
    488      1.2       ad 		ld->ld_lwp = l;
    489      1.2       ad 		ld->ld_exwant--;
    490      1.2       ad 
    491      1.2       ad 		if ((ld->ld_flags & LD_SLEEPER) != 0) {
    492      1.2       ad 			l->l_exlocks++;
    493      1.2       ad 			TAILQ_INSERT_TAIL(&ld_sleepers, ld, ld_chain);
    494      1.2       ad 		} else {
    495      1.2       ad 			curcpu()->ci_spin_locks2++;
    496      1.2       ad 			TAILQ_INSERT_TAIL(&ld_spinners, ld, ld_chain);
    497      1.2       ad 		}
    498      1.2       ad 	}
    499      1.2       ad 
    500      1.2       ad 	lockdebug_unlock(lk);
    501      1.2       ad }
    502      1.2       ad 
    503      1.2       ad /*
    504      1.2       ad  * lockdebug_unlocked:
    505      1.2       ad  *
    506      1.2       ad  *	Process a lock release operation.
    507      1.2       ad  */
    508      1.2       ad void
    509      1.2       ad lockdebug_unlocked(u_int id, uintptr_t where, int shared)
    510      1.2       ad {
    511      1.2       ad 	struct lwp *l = curlwp;
    512      1.2       ad 	lockdebuglk_t *lk;
    513      1.2       ad 	lockdebug_t *ld;
    514      1.2       ad 
    515      1.2       ad 	if (panicstr != NULL)
    516      1.2       ad 		return;
    517      1.2       ad 
    518      1.2       ad 	if ((ld = lockdebug_lookup(id, &lk)) == NULL)
    519      1.2       ad 		return;
    520      1.2       ad 
    521      1.2       ad 	if (shared) {
    522      1.2       ad 		if (l->l_shlocks == 0)
    523      1.2       ad 			lockdebug_abort1(ld, lk, __func__,
    524  1.8.6.2    joerg 			    "no shared locks held by LWP", true);
    525      1.2       ad 		if (ld->ld_shares == 0)
    526      1.2       ad 			lockdebug_abort1(ld, lk, __func__,
    527  1.8.6.2    joerg 			    "no shared holds on this lock", true);
    528      1.2       ad 		l->l_shlocks--;
    529      1.2       ad 		ld->ld_shares--;
    530      1.2       ad 	} else {
    531      1.2       ad 		if ((ld->ld_flags & LD_LOCKED) == 0)
    532  1.8.6.2    joerg 			lockdebug_abort1(ld, lk, __func__, "not locked",
    533  1.8.6.2    joerg 			    true);
    534      1.2       ad 
    535      1.2       ad 		if ((ld->ld_flags & LD_SLEEPER) != 0) {
    536      1.2       ad 			if (ld->ld_lwp != curlwp)
    537      1.2       ad 				lockdebug_abort1(ld, lk, __func__,
    538  1.8.6.2    joerg 				    "not held by current LWP", true);
    539      1.2       ad 			ld->ld_flags &= ~LD_LOCKED;
    540      1.2       ad 			ld->ld_unlocked = where;
    541      1.2       ad 			ld->ld_lwp = NULL;
    542      1.2       ad 			curlwp->l_exlocks--;
    543      1.2       ad 			TAILQ_REMOVE(&ld_sleepers, ld, ld_chain);
    544      1.2       ad 		} else {
    545      1.2       ad 			if (ld->ld_cpu != (uint16_t)cpu_number())
    546      1.2       ad 				lockdebug_abort1(ld, lk, __func__,
    547  1.8.6.2    joerg 				    "not held by current CPU", true);
    548      1.2       ad 			ld->ld_flags &= ~LD_LOCKED;
    549      1.2       ad 			ld->ld_unlocked = where;
    550      1.2       ad 			ld->ld_lwp = NULL;
    551      1.2       ad 			curcpu()->ci_spin_locks2--;
    552      1.2       ad 			TAILQ_REMOVE(&ld_spinners, ld, ld_chain);
    553      1.2       ad 		}
    554      1.2       ad 	}
    555      1.2       ad 
    556      1.2       ad 	lockdebug_unlock(lk);
    557      1.2       ad }
    558      1.2       ad 
    559      1.2       ad /*
    560      1.2       ad  * lockdebug_barrier:
    561      1.2       ad  *
    562      1.2       ad  *	Panic if we hold more than one specified spin lock, and optionally,
    563      1.2       ad  *	if we hold sleep locks.
    564      1.2       ad  */
    565      1.2       ad void
    566      1.2       ad lockdebug_barrier(volatile void *spinlock, int slplocks)
    567      1.2       ad {
    568      1.2       ad 	struct lwp *l = curlwp;
    569      1.2       ad 	lockdebug_t *ld;
    570      1.2       ad 	uint16_t cpuno;
    571      1.2       ad 
    572      1.2       ad 	if (panicstr != NULL)
    573      1.2       ad 		return;
    574      1.2       ad 
    575      1.2       ad 	if (curcpu()->ci_spin_locks2 != 0) {
    576      1.2       ad 		cpuno = (uint16_t)cpu_number();
    577      1.2       ad 
    578      1.2       ad 		lockdebug_lock(&ld_spinner_lk);
    579      1.2       ad 		TAILQ_FOREACH(ld, &ld_spinners, ld_chain) {
    580      1.2       ad 			if (ld->ld_lock == spinlock) {
    581      1.2       ad 				if (ld->ld_cpu != cpuno)
    582      1.2       ad 					lockdebug_abort1(ld, &ld_spinner_lk,
    583      1.2       ad 					    __func__,
    584  1.8.6.2    joerg 					    "not held by current CPU", true);
    585      1.2       ad 				continue;
    586      1.2       ad 			}
    587  1.8.6.4    joerg 			if (ld->ld_cpu == cpuno && (l->l_pflag & LP_INTR) == 0)
    588      1.2       ad 				lockdebug_abort1(ld, &ld_spinner_lk,
    589  1.8.6.2    joerg 				    __func__, "spin lock held", true);
    590      1.2       ad 		}
    591      1.2       ad 		lockdebug_unlock(&ld_spinner_lk);
    592      1.2       ad 	}
    593      1.2       ad 
    594      1.2       ad 	if (!slplocks) {
    595      1.2       ad 		if (l->l_exlocks != 0) {
    596      1.2       ad 			lockdebug_lock(&ld_sleeper_lk);
    597      1.2       ad 			TAILQ_FOREACH(ld, &ld_sleepers, ld_chain) {
    598      1.2       ad 				if (ld->ld_lwp == l)
    599      1.2       ad 					lockdebug_abort1(ld, &ld_sleeper_lk,
    600  1.8.6.2    joerg 					    __func__, "sleep lock held", true);
    601      1.2       ad 			}
    602      1.2       ad 			lockdebug_unlock(&ld_sleeper_lk);
    603      1.2       ad 		}
    604      1.2       ad 		if (l->l_shlocks != 0)
    605      1.2       ad 			panic("lockdebug_barrier: holding %d shared locks",
    606      1.2       ad 			    l->l_shlocks);
    607      1.2       ad 	}
    608      1.2       ad }
    609      1.2       ad 
    610      1.2       ad /*
    611  1.8.6.2    joerg  * lockdebug_mem_check:
    612  1.8.6.2    joerg  *
    613  1.8.6.2    joerg  *	Check for in-use locks within a memory region that is
    614  1.8.6.2    joerg  *	being freed.  We only check for active locks within the
    615  1.8.6.2    joerg  *	first page of the allocation.
    616  1.8.6.2    joerg  */
    617  1.8.6.2    joerg void
    618  1.8.6.2    joerg lockdebug_mem_check(const char *func, void *base, size_t sz)
    619  1.8.6.2    joerg {
    620  1.8.6.2    joerg #if 0
    621  1.8.6.2    joerg 	lockdebuglist_t *head;
    622  1.8.6.2    joerg 	lockdebuglk_t *lk;
    623  1.8.6.2    joerg 	lockdebug_t *ld;
    624  1.8.6.2    joerg 	uintptr_t sa, ea, la;
    625  1.8.6.2    joerg 
    626  1.8.6.2    joerg 	sa = (uintptr_t)base;
    627  1.8.6.2    joerg 	ea = sa + sz;
    628  1.8.6.2    joerg 
    629  1.8.6.2    joerg 	lockdebug_mhash(base, &lk, &head);
    630  1.8.6.2    joerg 	TAILQ_FOREACH(ld, head, ld_mchain) {
    631  1.8.6.2    joerg 		la = (uintptr_t)ld->ld_lock;
    632  1.8.6.2    joerg 		if (la >= sa && la < ea) {
    633  1.8.6.2    joerg 			lockdebug_abort1(ld, lk, func,
    634  1.8.6.2    joerg 			    "allocation contains active lock", !cold);
    635  1.8.6.2    joerg 			return;
    636  1.8.6.2    joerg 		}
    637  1.8.6.2    joerg 	}
    638  1.8.6.2    joerg 	lockdebug_unlock(lk);
    639  1.8.6.2    joerg #endif
    640  1.8.6.2    joerg }
    641  1.8.6.2    joerg 
    642  1.8.6.2    joerg /*
    643      1.2       ad  * lockdebug_dump:
    644      1.2       ad  *
    645      1.2       ad  *	Dump information about a lock on panic, or for DDB.
    646      1.2       ad  */
    647      1.2       ad static void
    648      1.2       ad lockdebug_dump(lockdebug_t *ld, void (*pr)(const char *, ...))
    649      1.2       ad {
    650      1.2       ad 	int sleeper = (ld->ld_flags & LD_SLEEPER);
    651      1.2       ad 
    652      1.2       ad 	(*pr)(
    653      1.2       ad 	    "lock address : %#018lx type     : %18s\n"
    654      1.2       ad 	    "shared holds : %18u exclusive: %18u\n"
    655      1.2       ad 	    "shares wanted: %18u exclusive: %18u\n"
    656      1.2       ad 	    "current cpu  : %18u last held: %18u\n"
    657      1.2       ad 	    "current lwp  : %#018lx last held: %#018lx\n"
    658  1.8.6.2    joerg 	    "last locked  : %#018lx unlocked : %#018lx\n"
    659  1.8.6.2    joerg 	    "initialized  : %#018lx\n",
    660      1.2       ad 	    (long)ld->ld_lock, (sleeper ? "sleep/adaptive" : "spin"),
    661      1.2       ad 	    (unsigned)ld->ld_shares, ((ld->ld_flags & LD_LOCKED) != 0),
    662      1.2       ad 	    (unsigned)ld->ld_shwant, (unsigned)ld->ld_exwant,
    663      1.2       ad 	    (unsigned)cpu_number(), (unsigned)ld->ld_cpu,
    664      1.2       ad 	    (long)curlwp, (long)ld->ld_lwp,
    665  1.8.6.2    joerg 	    (long)ld->ld_locked, (long)ld->ld_unlocked,
    666  1.8.6.2    joerg 	    (long)ld->ld_initaddr);
    667      1.2       ad 
    668      1.2       ad 	if (ld->ld_lockops->lo_dump != NULL)
    669      1.2       ad 		(*ld->ld_lockops->lo_dump)(ld->ld_lock);
    670      1.2       ad 
    671      1.2       ad 	if (sleeper) {
    672      1.2       ad 		(*pr)("\n");
    673      1.2       ad 		turnstile_print(ld->ld_lock, pr);
    674      1.2       ad 	}
    675      1.2       ad }
    676      1.2       ad 
    677      1.2       ad /*
    678      1.2       ad  * lockdebug_dump:
    679      1.2       ad  *
    680      1.2       ad  *	Dump information about a known lock.
    681      1.2       ad  */
    682      1.5       ad static void
    683      1.2       ad lockdebug_abort1(lockdebug_t *ld, lockdebuglk_t *lk, const char *func,
    684  1.8.6.2    joerg 		 const char *msg, bool dopanic)
    685      1.2       ad {
    686      1.2       ad 
    687      1.2       ad 	printf_nolog("%s error: %s: %s\n\n", ld->ld_lockops->lo_name,
    688      1.2       ad 	    func, msg);
    689      1.2       ad 	lockdebug_dump(ld, printf_nolog);
    690      1.2       ad 	lockdebug_unlock(lk);
    691      1.2       ad 	printf_nolog("\n");
    692  1.8.6.2    joerg 	if (dopanic)
    693  1.8.6.2    joerg 		panic("LOCKDEBUG");
    694      1.2       ad }
    695      1.2       ad 
    696      1.2       ad #endif	/* LOCKDEBUG */
    697      1.2       ad 
    698      1.2       ad /*
    699      1.2       ad  * lockdebug_lock_print:
    700      1.2       ad  *
    701      1.2       ad  *	Handle the DDB 'show lock' command.
    702      1.2       ad  */
    703      1.2       ad #ifdef DDB
    704      1.2       ad void
    705      1.2       ad lockdebug_lock_print(void *addr, void (*pr)(const char *, ...))
    706      1.2       ad {
    707      1.2       ad #ifdef LOCKDEBUG
    708      1.2       ad 	lockdebug_t *ld;
    709      1.2       ad 
    710      1.2       ad 	TAILQ_FOREACH(ld, &ld_all, ld_achain) {
    711      1.2       ad 		if (ld->ld_lock == addr) {
    712      1.2       ad 			lockdebug_dump(ld, pr);
    713      1.2       ad 			return;
    714      1.2       ad 		}
    715      1.2       ad 	}
    716      1.2       ad 	(*pr)("Sorry, no record of a lock with address %p found.\n", addr);
    717      1.2       ad #else
    718      1.2       ad 	(*pr)("Sorry, kernel not built with the LOCKDEBUG option.\n");
    719      1.2       ad #endif	/* LOCKDEBUG */
    720      1.2       ad }
    721      1.2       ad #endif	/* DDB */
    722      1.2       ad 
    723      1.2       ad /*
    724      1.2       ad  * lockdebug_abort:
    725      1.2       ad  *
    726      1.2       ad  *	An error has been trapped - dump lock info and call panic().
    727      1.2       ad  */
    728      1.2       ad void
    729      1.6     yamt lockdebug_abort(u_int id, volatile void *lock, lockops_t *ops,
    730      1.2       ad 		const char *func, const char *msg)
    731      1.2       ad {
    732      1.2       ad #ifdef LOCKDEBUG
    733      1.2       ad 	lockdebug_t *ld;
    734      1.2       ad 	lockdebuglk_t *lk;
    735      1.2       ad 
    736      1.2       ad 	if ((ld = lockdebug_lookup(id, &lk)) != NULL) {
    737  1.8.6.2    joerg 		lockdebug_abort1(ld, lk, func, msg, true);
    738      1.2       ad 		/* NOTREACHED */
    739      1.2       ad 	}
    740      1.2       ad #endif	/* LOCKDEBUG */
    741      1.2       ad 
    742      1.2       ad 	printf_nolog("%s error: %s: %s\n\n"
    743      1.2       ad 	    "lock address : %#018lx\n"
    744      1.2       ad 	    "current cpu  : %18d\n"
    745      1.2       ad 	    "current lwp  : %#018lx\n",
    746      1.2       ad 	    ops->lo_name, func, msg, (long)lock, (int)cpu_number(),
    747      1.2       ad 	    (long)curlwp);
    748      1.2       ad 
    749      1.2       ad 	(*ops->lo_dump)(lock);
    750      1.2       ad 
    751      1.2       ad 	printf_nolog("\n");
    752      1.2       ad 	panic("lock error");
    753      1.2       ad }
    754