Home | History | Annotate | Line # | Download | only in kern
kern_lock.c revision 1.52
      1  1.52   thorpej /*	$NetBSD: kern_lock.c,v 1.52 2001/04/20 22:58:39 thorpej Exp $	*/
      2  1.19   thorpej 
      3  1.19   thorpej /*-
      4  1.28   thorpej  * Copyright (c) 1999, 2000 The NetBSD Foundation, Inc.
      5  1.19   thorpej  * All rights reserved.
      6  1.19   thorpej  *
      7  1.19   thorpej  * This code is derived from software contributed to The NetBSD Foundation
      8  1.19   thorpej  * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
      9  1.19   thorpej  * NASA Ames Research Center.
     10  1.19   thorpej  *
     11  1.19   thorpej  * This code is derived from software contributed to The NetBSD Foundation
     12  1.19   thorpej  * by Ross Harvey.
     13  1.19   thorpej  *
     14  1.19   thorpej  * Redistribution and use in source and binary forms, with or without
     15  1.19   thorpej  * modification, are permitted provided that the following conditions
     16  1.19   thorpej  * are met:
     17  1.19   thorpej  * 1. Redistributions of source code must retain the above copyright
     18  1.19   thorpej  *    notice, this list of conditions and the following disclaimer.
     19  1.19   thorpej  * 2. Redistributions in binary form must reproduce the above copyright
     20  1.19   thorpej  *    notice, this list of conditions and the following disclaimer in the
     21  1.19   thorpej  *    documentation and/or other materials provided with the distribution.
     22  1.19   thorpej  * 3. All advertising materials mentioning features or use of this software
     23  1.19   thorpej  *    must display the following acknowledgement:
     24  1.19   thorpej  *	This product includes software developed by the NetBSD
     25  1.19   thorpej  *	Foundation, Inc. and its contributors.
     26  1.19   thorpej  * 4. Neither the name of The NetBSD Foundation nor the names of its
     27  1.19   thorpej  *    contributors may be used to endorse or promote products derived
     28  1.19   thorpej  *    from this software without specific prior written permission.
     29  1.19   thorpej  *
     30  1.19   thorpej  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     31  1.19   thorpej  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     32  1.19   thorpej  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     33  1.19   thorpej  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     34  1.19   thorpej  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     35  1.19   thorpej  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     36  1.19   thorpej  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     37  1.19   thorpej  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     38  1.19   thorpej  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     39  1.19   thorpej  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     40  1.19   thorpej  * POSSIBILITY OF SUCH DAMAGE.
     41  1.19   thorpej  */
     42   1.2      fvdl 
     43   1.1      fvdl /*
     44   1.1      fvdl  * Copyright (c) 1995
     45   1.1      fvdl  *	The Regents of the University of California.  All rights reserved.
     46   1.1      fvdl  *
     47   1.1      fvdl  * This code contains ideas from software contributed to Berkeley by
     48   1.1      fvdl  * Avadis Tevanian, Jr., Michael Wayne Young, and the Mach Operating
     49   1.1      fvdl  * System project at Carnegie-Mellon University.
     50   1.1      fvdl  *
     51   1.1      fvdl  * Redistribution and use in source and binary forms, with or without
     52   1.1      fvdl  * modification, are permitted provided that the following conditions
     53   1.1      fvdl  * are met:
     54   1.1      fvdl  * 1. Redistributions of source code must retain the above copyright
     55   1.1      fvdl  *    notice, this list of conditions and the following disclaimer.
     56   1.1      fvdl  * 2. Redistributions in binary form must reproduce the above copyright
     57   1.1      fvdl  *    notice, this list of conditions and the following disclaimer in the
     58   1.1      fvdl  *    documentation and/or other materials provided with the distribution.
     59   1.1      fvdl  * 3. All advertising materials mentioning features or use of this software
     60   1.1      fvdl  *    must display the following acknowledgement:
     61   1.1      fvdl  *	This product includes software developed by the University of
     62   1.1      fvdl  *	California, Berkeley and its contributors.
     63   1.1      fvdl  * 4. Neither the name of the University nor the names of its contributors
     64   1.1      fvdl  *    may be used to endorse or promote products derived from this software
     65   1.1      fvdl  *    without specific prior written permission.
     66   1.1      fvdl  *
     67   1.1      fvdl  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
     68   1.1      fvdl  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     69   1.1      fvdl  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     70   1.1      fvdl  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
     71   1.1      fvdl  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     72   1.1      fvdl  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
     73   1.1      fvdl  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     74   1.1      fvdl  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     75   1.1      fvdl  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     76   1.1      fvdl  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     77   1.1      fvdl  * SUCH DAMAGE.
     78   1.1      fvdl  *
     79   1.1      fvdl  *	@(#)kern_lock.c	8.18 (Berkeley) 5/21/95
     80   1.1      fvdl  */
     81   1.7   thorpej 
     82  1.21   thorpej #include "opt_multiprocessor.h"
     83   1.7   thorpej #include "opt_lockdebug.h"
     84  1.18       chs #include "opt_ddb.h"
     85   1.1      fvdl 
     86   1.1      fvdl #include <sys/param.h>
     87   1.1      fvdl #include <sys/proc.h>
     88   1.1      fvdl #include <sys/lock.h>
     89   1.2      fvdl #include <sys/systm.h>
     90   1.1      fvdl #include <machine/cpu.h>
     91   1.1      fvdl 
     92  1.25   thorpej #if defined(LOCKDEBUG)
     93  1.25   thorpej #include <sys/syslog.h>
     94  1.25   thorpej /*
     95  1.25   thorpej  * note that stdarg.h and the ansi style va_start macro is used for both
     96  1.25   thorpej  * ansi and traditional c compiles.
     97  1.25   thorpej  * XXX: this requires that stdarg.h define: va_alist and va_dcl
     98  1.25   thorpej  */
     99  1.25   thorpej #include <machine/stdarg.h>
    100  1.25   thorpej 
    101  1.36   thorpej void	lock_printf(const char *fmt, ...)
    102  1.37       eeh     __attribute__((__format__(__printf__,1,2)));
    103  1.25   thorpej 
    104  1.51       jmc int	lock_debug_syslog = 1;	/* defaults to syslog, but can be patched */
    105  1.25   thorpej #endif
    106  1.25   thorpej 
    107   1.1      fvdl /*
    108   1.1      fvdl  * Locking primitives implementation.
    109   1.1      fvdl  * Locks provide shared/exclusive sychronization.
    110   1.1      fvdl  */
    111   1.1      fvdl 
    112  1.21   thorpej #if defined(LOCKDEBUG) || defined(DIAGNOSTIC) /* { */
    113  1.21   thorpej #if defined(MULTIPROCESSOR) /* { */
    114  1.21   thorpej #define	COUNT_CPU(cpu_id, x)						\
    115  1.47  sommerfe 	curcpu()->ci_spin_locks += (x)
    116  1.21   thorpej #else
    117  1.21   thorpej u_long	spin_locks;
    118  1.21   thorpej #define	COUNT_CPU(cpu_id, x)	spin_locks += (x)
    119  1.21   thorpej #endif /* MULTIPROCESSOR */ /* } */
    120  1.21   thorpej 
    121  1.21   thorpej #define	COUNT(lkp, p, cpu_id, x)					\
    122  1.21   thorpej do {									\
    123  1.21   thorpej 	if ((lkp)->lk_flags & LK_SPIN)					\
    124  1.21   thorpej 		COUNT_CPU((cpu_id), (x));				\
    125  1.21   thorpej 	else								\
    126  1.21   thorpej 		(p)->p_locks += (x);					\
    127  1.30   thorpej } while (/*CONSTCOND*/0)
    128   1.1      fvdl #else
    129  1.22    mellon #define COUNT(lkp, p, cpu_id, x)
    130  1.48  sommerfe #define COUNT_CPU(cpu_id, x)
    131  1.21   thorpej #endif /* LOCKDEBUG || DIAGNOSTIC */ /* } */
    132   1.1      fvdl 
    133  1.52   thorpej #ifndef SPINLOCK_SPIN_HOOK		/* from <machine/lock.h> */
    134  1.52   thorpej #define	SPINLOCK_SPIN_HOOK		/* nothing */
    135  1.49   thorpej #endif
    136  1.49   thorpej 
    137  1.43   thorpej #define	INTERLOCK_ACQUIRE(lkp, flags, s)				\
    138  1.40   thorpej do {									\
    139  1.43   thorpej 	if ((flags) & LK_SPIN)						\
    140  1.42   thorpej 		s = splsched();						\
    141  1.40   thorpej 	simple_lock(&(lkp)->lk_interlock);				\
    142  1.40   thorpej } while (0)
    143  1.40   thorpej 
    144  1.43   thorpej #define	INTERLOCK_RELEASE(lkp, flags, s)				\
    145  1.40   thorpej do {									\
    146  1.40   thorpej 	simple_unlock(&(lkp)->lk_interlock);				\
    147  1.52   thorpej 	if ((flags) & LK_SPIN)						\
    148  1.40   thorpej 		splx(s);						\
    149  1.40   thorpej } while (0)
    150  1.40   thorpej 
    151  1.50   thorpej #if defined(LOCKDEBUG)
    152  1.50   thorpej #if defined(DDB)
    153  1.50   thorpej #define	SPINLOCK_SPINCHECK_DEBUGGER	Debugger()
    154  1.50   thorpej #else
    155  1.50   thorpej #define	SPINLOCK_SPINCHECK_DEBUGGER	/* nothing */
    156  1.50   thorpej #endif
    157  1.50   thorpej 
    158  1.50   thorpej #define	SPINLOCK_SPINCHECK_DECL						\
    159  1.50   thorpej 	/* 32-bits of count -- wrap constitutes a "spinout" */		\
    160  1.50   thorpej 	uint32_t __spinc = 0
    161  1.50   thorpej 
    162  1.50   thorpej #define	SPINLOCK_SPINCHECK						\
    163  1.50   thorpej do {									\
    164  1.50   thorpej 	if (++__spinc == 0) {						\
    165  1.50   thorpej 		printf("LK_SPIN spinout, excl %d, share %d\n",		\
    166  1.50   thorpej 		    lkp->lk_exclusivecount, lkp->lk_sharecount);	\
    167  1.50   thorpej 		if (lkp->lk_exclusivecount)				\
    168  1.50   thorpej 			printf("held by CPU %lu\n",			\
    169  1.50   thorpej 			    (u_long) lkp->lk_cpu);			\
    170  1.50   thorpej 		if (lkp->lk_lock_file)					\
    171  1.50   thorpej 			printf("last locked at %s:%d\n",		\
    172  1.50   thorpej 			    lkp->lk_lock_file, lkp->lk_lock_line);	\
    173  1.50   thorpej 		if (lkp->lk_unlock_file)				\
    174  1.50   thorpej 			printf("last unlocked at %s:%d\n",		\
    175  1.50   thorpej 			    lkp->lk_unlock_file, lkp->lk_unlock_line);	\
    176  1.50   thorpej 		SPINLOCK_SPINCHECK_DEBUGGER;				\
    177  1.50   thorpej 	}								\
    178  1.50   thorpej } while (0)
    179  1.50   thorpej #else
    180  1.50   thorpej #define	SPINLOCK_SPINCHECK_DECL			/* nothing */
    181  1.50   thorpej #define	SPINLOCK_SPINCHECK			/* nothing */
    182  1.50   thorpej #endif /* LOCKDEBUG && DDB */
    183  1.50   thorpej 
    184   1.1      fvdl /*
    185   1.1      fvdl  * Acquire a resource.
    186   1.1      fvdl  */
    187  1.23   thorpej #define ACQUIRE(lkp, error, extflags, drain, wanted)			\
    188  1.19   thorpej 	if ((extflags) & LK_SPIN) {					\
    189  1.19   thorpej 		int interlocked;					\
    190  1.50   thorpej 		SPINLOCK_SPINCHECK_DECL;				\
    191  1.19   thorpej 									\
    192  1.23   thorpej 		if ((drain) == 0)					\
    193  1.23   thorpej 			(lkp)->lk_waitcount++;				\
    194  1.19   thorpej 		for (interlocked = 1;;) {				\
    195  1.50   thorpej 			SPINLOCK_SPINCHECK;				\
    196  1.19   thorpej 			if (wanted) {					\
    197  1.19   thorpej 				if (interlocked) {			\
    198  1.43   thorpej 					INTERLOCK_RELEASE((lkp),	\
    199  1.43   thorpej 					    LK_SPIN, s);		\
    200  1.19   thorpej 					interlocked = 0;		\
    201  1.19   thorpej 				}					\
    202  1.52   thorpej 				SPINLOCK_SPIN_HOOK;			\
    203  1.19   thorpej 			} else if (interlocked) {			\
    204  1.19   thorpej 				break;					\
    205  1.19   thorpej 			} else {					\
    206  1.43   thorpej 				INTERLOCK_ACQUIRE((lkp), LK_SPIN, s);	\
    207  1.19   thorpej 				interlocked = 1;			\
    208  1.19   thorpej 			}						\
    209  1.19   thorpej 		}							\
    210  1.23   thorpej 		if ((drain) == 0)					\
    211  1.23   thorpej 			(lkp)->lk_waitcount--;				\
    212  1.19   thorpej 		KASSERT((wanted) == 0);					\
    213  1.19   thorpej 		error = 0;	/* sanity */				\
    214  1.19   thorpej 	} else {							\
    215  1.19   thorpej 		for (error = 0; wanted; ) {				\
    216  1.23   thorpej 			if ((drain))					\
    217  1.23   thorpej 				(lkp)->lk_flags |= LK_WAITDRAIN;	\
    218  1.23   thorpej 			else						\
    219  1.23   thorpej 				(lkp)->lk_waitcount++;			\
    220  1.23   thorpej 			/* XXX Cast away volatile. */			\
    221  1.31   thorpej 			error = ltsleep((drain) ? &(lkp)->lk_flags :	\
    222  1.23   thorpej 			    (void *)(lkp), (lkp)->lk_prio,		\
    223  1.31   thorpej 			    (lkp)->lk_wmesg, (lkp)->lk_timo,		\
    224  1.31   thorpej 			    &(lkp)->lk_interlock);			\
    225  1.23   thorpej 			if ((drain) == 0)				\
    226  1.23   thorpej 				(lkp)->lk_waitcount--;			\
    227  1.19   thorpej 			if (error)					\
    228  1.19   thorpej 				break;					\
    229  1.19   thorpej 			if ((extflags) & LK_SLEEPFAIL) {		\
    230  1.19   thorpej 				error = ENOLCK;				\
    231  1.19   thorpej 				break;					\
    232  1.19   thorpej 			}						\
    233   1.1      fvdl 		}							\
    234   1.1      fvdl 	}
    235   1.1      fvdl 
    236  1.19   thorpej #define	SETHOLDER(lkp, pid, cpu_id)					\
    237  1.19   thorpej do {									\
    238  1.19   thorpej 	if ((lkp)->lk_flags & LK_SPIN)					\
    239  1.19   thorpej 		(lkp)->lk_cpu = cpu_id;					\
    240  1.19   thorpej 	else								\
    241  1.19   thorpej 		(lkp)->lk_lockholder = pid;				\
    242  1.30   thorpej } while (/*CONSTCOND*/0)
    243  1.19   thorpej 
    244  1.19   thorpej #define	WEHOLDIT(lkp, pid, cpu_id)					\
    245  1.19   thorpej 	(((lkp)->lk_flags & LK_SPIN) != 0 ?				\
    246  1.19   thorpej 	 ((lkp)->lk_cpu == (cpu_id)) : ((lkp)->lk_lockholder == (pid)))
    247  1.19   thorpej 
    248  1.23   thorpej #define	WAKEUP_WAITER(lkp)						\
    249  1.23   thorpej do {									\
    250  1.23   thorpej 	if (((lkp)->lk_flags & LK_SPIN) == 0 && (lkp)->lk_waitcount) {	\
    251  1.23   thorpej 		/* XXX Cast away volatile. */				\
    252  1.23   thorpej 		wakeup_one((void *)(lkp));				\
    253  1.23   thorpej 	}								\
    254  1.30   thorpej } while (/*CONSTCOND*/0)
    255  1.23   thorpej 
    256  1.21   thorpej #if defined(LOCKDEBUG) /* { */
    257  1.21   thorpej #if defined(MULTIPROCESSOR) /* { */
    258  1.21   thorpej struct simplelock spinlock_list_slock = SIMPLELOCK_INITIALIZER;
    259  1.21   thorpej 
    260  1.27   thorpej #define	SPINLOCK_LIST_LOCK()						\
    261  1.29  sommerfe 	__cpu_simple_lock(&spinlock_list_slock.lock_data)
    262  1.21   thorpej 
    263  1.27   thorpej #define	SPINLOCK_LIST_UNLOCK()						\
    264  1.29  sommerfe 	__cpu_simple_unlock(&spinlock_list_slock.lock_data)
    265  1.21   thorpej #else
    266  1.21   thorpej #define	SPINLOCK_LIST_LOCK()	/* nothing */
    267  1.21   thorpej 
    268  1.21   thorpej #define	SPINLOCK_LIST_UNLOCK()	/* nothing */
    269  1.21   thorpej #endif /* MULTIPROCESSOR */ /* } */
    270  1.21   thorpej 
    271  1.21   thorpej TAILQ_HEAD(, lock) spinlock_list =
    272  1.21   thorpej     TAILQ_HEAD_INITIALIZER(spinlock_list);
    273  1.21   thorpej 
    274  1.21   thorpej #define	HAVEIT(lkp)							\
    275  1.21   thorpej do {									\
    276  1.21   thorpej 	if ((lkp)->lk_flags & LK_SPIN) {				\
    277  1.44   thorpej 		int s = spllock();					\
    278  1.21   thorpej 		SPINLOCK_LIST_LOCK();					\
    279  1.21   thorpej 		/* XXX Cast away volatile. */				\
    280  1.21   thorpej 		TAILQ_INSERT_TAIL(&spinlock_list, (struct lock *)(lkp),	\
    281  1.21   thorpej 		    lk_list);						\
    282  1.21   thorpej 		SPINLOCK_LIST_UNLOCK();					\
    283  1.21   thorpej 		splx(s);						\
    284  1.21   thorpej 	}								\
    285  1.30   thorpej } while (/*CONSTCOND*/0)
    286  1.21   thorpej 
    287  1.21   thorpej #define	DONTHAVEIT(lkp)							\
    288  1.21   thorpej do {									\
    289  1.21   thorpej 	if ((lkp)->lk_flags & LK_SPIN) {				\
    290  1.44   thorpej 		int s = spllock();					\
    291  1.21   thorpej 		SPINLOCK_LIST_LOCK();					\
    292  1.21   thorpej 		/* XXX Cast away volatile. */				\
    293  1.21   thorpej 		TAILQ_REMOVE(&spinlock_list, (struct lock *)(lkp),	\
    294  1.21   thorpej 		    lk_list);						\
    295  1.21   thorpej 		SPINLOCK_LIST_UNLOCK();					\
    296  1.21   thorpej 		splx(s);						\
    297  1.21   thorpej 	}								\
    298  1.30   thorpej } while (/*CONSTCOND*/0)
    299  1.21   thorpej #else
    300  1.21   thorpej #define	HAVEIT(lkp)		/* nothing */
    301  1.21   thorpej 
    302  1.21   thorpej #define	DONTHAVEIT(lkp)		/* nothing */
    303  1.21   thorpej #endif /* LOCKDEBUG */ /* } */
    304  1.21   thorpej 
    305  1.25   thorpej #if defined(LOCKDEBUG)
    306  1.25   thorpej /*
    307  1.25   thorpej  * Lock debug printing routine; can be configured to print to console
    308  1.25   thorpej  * or log to syslog.
    309  1.25   thorpej  */
    310  1.25   thorpej void
    311  1.25   thorpej lock_printf(const char *fmt, ...)
    312  1.25   thorpej {
    313  1.25   thorpej 	va_list ap;
    314  1.25   thorpej 
    315  1.25   thorpej 	va_start(ap, fmt);
    316  1.25   thorpej 	if (lock_debug_syslog)
    317  1.25   thorpej 		vlog(LOG_DEBUG, fmt, ap);
    318  1.25   thorpej 	else
    319  1.25   thorpej 		vprintf(fmt, ap);
    320  1.25   thorpej 	va_end(ap);
    321  1.25   thorpej }
    322  1.25   thorpej #endif /* LOCKDEBUG */
    323  1.25   thorpej 
    324   1.1      fvdl /*
    325   1.1      fvdl  * Initialize a lock; required before use.
    326   1.1      fvdl  */
    327   1.1      fvdl void
    328  1.33   thorpej lockinit(struct lock *lkp, int prio, const char *wmesg, int timo, int flags)
    329   1.1      fvdl {
    330   1.1      fvdl 
    331   1.8     perry 	memset(lkp, 0, sizeof(struct lock));
    332   1.1      fvdl 	simple_lock_init(&lkp->lk_interlock);
    333   1.1      fvdl 	lkp->lk_flags = flags & LK_EXTFLG_MASK;
    334  1.19   thorpej 	if (flags & LK_SPIN)
    335  1.19   thorpej 		lkp->lk_cpu = LK_NOCPU;
    336  1.19   thorpej 	else {
    337  1.19   thorpej 		lkp->lk_lockholder = LK_NOPROC;
    338  1.19   thorpej 		lkp->lk_prio = prio;
    339  1.19   thorpej 		lkp->lk_timo = timo;
    340  1.19   thorpej 	}
    341  1.19   thorpej 	lkp->lk_wmesg = wmesg;	/* just a name for spin locks */
    342  1.50   thorpej #if defined(LOCKDEBUG)
    343  1.50   thorpej 	lkp->lk_lock_file = NULL;
    344  1.50   thorpej 	lkp->lk_unlock_file = NULL;
    345  1.50   thorpej #endif
    346   1.1      fvdl }
    347   1.1      fvdl 
    348   1.1      fvdl /*
    349   1.1      fvdl  * Determine the status of a lock.
    350   1.1      fvdl  */
    351   1.1      fvdl int
    352  1.33   thorpej lockstatus(struct lock *lkp)
    353   1.1      fvdl {
    354  1.40   thorpej 	int s, lock_type = 0;
    355   1.1      fvdl 
    356  1.43   thorpej 	INTERLOCK_ACQUIRE(lkp, lkp->lk_flags, s);
    357   1.1      fvdl 	if (lkp->lk_exclusivecount != 0)
    358   1.1      fvdl 		lock_type = LK_EXCLUSIVE;
    359   1.1      fvdl 	else if (lkp->lk_sharecount != 0)
    360   1.1      fvdl 		lock_type = LK_SHARED;
    361  1.43   thorpej 	INTERLOCK_RELEASE(lkp, lkp->lk_flags, s);
    362   1.1      fvdl 	return (lock_type);
    363   1.1      fvdl }
    364  1.35   thorpej 
    365  1.35   thorpej #if defined(LOCKDEBUG) || defined(DIAGNOSTIC)
    366  1.35   thorpej /*
    367  1.35   thorpej  * Make sure no spin locks are held by a CPU that is about
    368  1.35   thorpej  * to context switch.
    369  1.35   thorpej  */
    370  1.35   thorpej void
    371  1.35   thorpej spinlock_switchcheck(void)
    372  1.35   thorpej {
    373  1.35   thorpej 	u_long cnt;
    374  1.35   thorpej 	int s;
    375  1.35   thorpej 
    376  1.44   thorpej 	s = spllock();
    377  1.35   thorpej #if defined(MULTIPROCESSOR)
    378  1.35   thorpej 	cnt = curcpu()->ci_spin_locks;
    379  1.35   thorpej #else
    380  1.35   thorpej 	cnt = spin_locks;
    381  1.35   thorpej #endif
    382  1.35   thorpej 	splx(s);
    383  1.35   thorpej 
    384  1.35   thorpej 	if (cnt != 0)
    385  1.35   thorpej 		panic("spinlock_switchcheck: CPU %lu has %lu spin locks",
    386  1.35   thorpej 		    (u_long) cpu_number(), cnt);
    387  1.35   thorpej }
    388  1.35   thorpej #endif /* LOCKDEBUG || DIAGNOSTIC */
    389   1.1      fvdl 
    390   1.1      fvdl /*
    391  1.44   thorpej  * Locks and IPLs (interrupt priority levels):
    392  1.44   thorpej  *
    393  1.44   thorpej  * Locks which may be taken from interrupt context must be handled
    394  1.44   thorpej  * very carefully; you must spl to the highest IPL where the lock
    395  1.44   thorpej  * is needed before acquiring the lock.
    396  1.44   thorpej  *
    397  1.44   thorpej  * It is also important to avoid deadlock, since certain (very high
    398  1.44   thorpej  * priority) interrupts are often needed to keep the system as a whole
    399  1.44   thorpej  * from deadlocking, and must not be blocked while you are spinning
    400  1.44   thorpej  * waiting for a lower-priority lock.
    401  1.44   thorpej  *
    402  1.44   thorpej  * In addition, the lock-debugging hooks themselves need to use locks!
    403  1.44   thorpej  *
    404  1.44   thorpej  * A raw __cpu_simple_lock may be used from interrupts are long as it
    405  1.44   thorpej  * is acquired and held at a single IPL.
    406  1.44   thorpej  *
    407  1.44   thorpej  * A simple_lock (which is a __cpu_simple_lock wrapped with some
    408  1.44   thorpej  * debugging hooks) may be used at or below spllock(), which is
    409  1.44   thorpej  * typically at or just below splhigh() (i.e. blocks everything
    410  1.44   thorpej  * but certain machine-dependent extremely high priority interrupts).
    411  1.44   thorpej  *
    412  1.44   thorpej  * spinlockmgr spinlocks should be used at or below splsched().
    413  1.44   thorpej  *
    414  1.44   thorpej  * Some platforms may have interrupts of higher priority than splsched(),
    415  1.44   thorpej  * including hard serial interrupts, inter-processor interrupts, and
    416  1.44   thorpej  * kernel debugger traps.
    417  1.44   thorpej  */
    418  1.44   thorpej 
    419  1.44   thorpej /*
    420  1.32  sommerfe  * XXX XXX kludge around another kludge..
    421  1.32  sommerfe  *
    422  1.32  sommerfe  * vfs_shutdown() may be called from interrupt context, either as a result
    423  1.32  sommerfe  * of a panic, or from the debugger.   It proceeds to call
    424  1.32  sommerfe  * sys_sync(&proc0, ...), pretending its running on behalf of proc0
    425  1.32  sommerfe  *
    426  1.32  sommerfe  * We would like to make an attempt to sync the filesystems in this case, so
    427  1.32  sommerfe  * if this happens, we treat attempts to acquire locks specially.
    428  1.32  sommerfe  * All locks are acquired on behalf of proc0.
    429  1.32  sommerfe  *
    430  1.32  sommerfe  * If we've already paniced, we don't block waiting for locks, but
    431  1.32  sommerfe  * just barge right ahead since we're already going down in flames.
    432  1.32  sommerfe  */
    433  1.32  sommerfe 
    434  1.32  sommerfe /*
    435   1.1      fvdl  * Set, change, or release a lock.
    436   1.1      fvdl  *
    437   1.1      fvdl  * Shared requests increment the shared count. Exclusive requests set the
    438   1.1      fvdl  * LK_WANT_EXCL flag (preventing further shared locks), and wait for already
    439   1.1      fvdl  * accepted shared locks and shared-to-exclusive upgrades to go away.
    440   1.1      fvdl  */
    441   1.1      fvdl int
    442  1.50   thorpej #if defined(LOCKDEBUG)
    443  1.50   thorpej _lockmgr(__volatile struct lock *lkp, u_int flags,
    444  1.50   thorpej     struct simplelock *interlkp, const char *file, int line)
    445  1.50   thorpej #else
    446  1.33   thorpej lockmgr(__volatile struct lock *lkp, u_int flags,
    447  1.33   thorpej     struct simplelock *interlkp)
    448  1.50   thorpej #endif
    449   1.1      fvdl {
    450   1.1      fvdl 	int error;
    451   1.1      fvdl 	pid_t pid;
    452   1.1      fvdl 	int extflags;
    453  1.24   thorpej 	cpuid_t cpu_id;
    454   1.6      fvdl 	struct proc *p = curproc;
    455  1.32  sommerfe 	int lock_shutdown_noblock = 0;
    456  1.40   thorpej 	int s;
    457   1.1      fvdl 
    458   1.1      fvdl 	error = 0;
    459  1.19   thorpej 
    460  1.43   thorpej 	INTERLOCK_ACQUIRE(lkp, lkp->lk_flags, s);
    461   1.1      fvdl 	if (flags & LK_INTERLOCK)
    462   1.1      fvdl 		simple_unlock(interlkp);
    463   1.1      fvdl 	extflags = (flags | lkp->lk_flags) & LK_EXTFLG_MASK;
    464  1.19   thorpej 
    465  1.21   thorpej #ifdef DIAGNOSTIC /* { */
    466  1.19   thorpej 	/*
    467  1.19   thorpej 	 * Don't allow spins on sleep locks and don't allow sleeps
    468  1.19   thorpej 	 * on spin locks.
    469  1.19   thorpej 	 */
    470  1.19   thorpej 	if ((flags ^ lkp->lk_flags) & LK_SPIN)
    471  1.19   thorpej 		panic("lockmgr: sleep/spin mismatch\n");
    472  1.21   thorpej #endif /* } */
    473  1.19   thorpej 
    474  1.19   thorpej 	if (extflags & LK_SPIN)
    475  1.19   thorpej 		pid = LK_KERNPROC;
    476  1.19   thorpej 	else {
    477  1.32  sommerfe 		if (p == NULL) {
    478  1.32  sommerfe 			if (!doing_shutdown) {
    479  1.32  sommerfe #ifdef DIAGNOSTIC
    480  1.32  sommerfe 				panic("lockmgr: no context");
    481  1.32  sommerfe #endif
    482  1.32  sommerfe 			} else {
    483  1.32  sommerfe 				p = &proc0;
    484  1.32  sommerfe 				if (panicstr && (!(flags & LK_NOWAIT))) {
    485  1.32  sommerfe 					flags |= LK_NOWAIT;
    486  1.32  sommerfe 					lock_shutdown_noblock = 1;
    487  1.32  sommerfe 				}
    488  1.32  sommerfe 			}
    489  1.32  sommerfe 		}
    490  1.19   thorpej 		pid = p->p_pid;
    491  1.19   thorpej 	}
    492  1.24   thorpej 	cpu_id = cpu_number();
    493  1.19   thorpej 
    494   1.1      fvdl 	/*
    495   1.1      fvdl 	 * Once a lock has drained, the LK_DRAINING flag is set and an
    496   1.1      fvdl 	 * exclusive lock is returned. The only valid operation thereafter
    497   1.1      fvdl 	 * is a single release of that exclusive lock. This final release
    498   1.1      fvdl 	 * clears the LK_DRAINING flag and sets the LK_DRAINED flag. Any
    499   1.1      fvdl 	 * further requests of any sort will result in a panic. The bits
    500   1.1      fvdl 	 * selected for these two flags are chosen so that they will be set
    501   1.1      fvdl 	 * in memory that is freed (freed memory is filled with 0xdeadbeef).
    502   1.1      fvdl 	 * The final release is permitted to give a new lease on life to
    503   1.1      fvdl 	 * the lock by specifying LK_REENABLE.
    504   1.1      fvdl 	 */
    505   1.1      fvdl 	if (lkp->lk_flags & (LK_DRAINING|LK_DRAINED)) {
    506  1.28   thorpej #ifdef DIAGNOSTIC /* { */
    507   1.1      fvdl 		if (lkp->lk_flags & LK_DRAINED)
    508   1.1      fvdl 			panic("lockmgr: using decommissioned lock");
    509   1.1      fvdl 		if ((flags & LK_TYPE_MASK) != LK_RELEASE ||
    510  1.19   thorpej 		    WEHOLDIT(lkp, pid, cpu_id) == 0)
    511   1.1      fvdl 			panic("lockmgr: non-release on draining lock: %d\n",
    512   1.1      fvdl 			    flags & LK_TYPE_MASK);
    513  1.28   thorpej #endif /* DIAGNOSTIC */ /* } */
    514   1.1      fvdl 		lkp->lk_flags &= ~LK_DRAINING;
    515   1.1      fvdl 		if ((flags & LK_REENABLE) == 0)
    516   1.1      fvdl 			lkp->lk_flags |= LK_DRAINED;
    517   1.1      fvdl 	}
    518   1.1      fvdl 
    519   1.1      fvdl 	switch (flags & LK_TYPE_MASK) {
    520   1.1      fvdl 
    521   1.1      fvdl 	case LK_SHARED:
    522  1.19   thorpej 		if (WEHOLDIT(lkp, pid, cpu_id) == 0) {
    523   1.1      fvdl 			/*
    524   1.1      fvdl 			 * If just polling, check to see if we will block.
    525   1.1      fvdl 			 */
    526   1.1      fvdl 			if ((extflags & LK_NOWAIT) && (lkp->lk_flags &
    527   1.1      fvdl 			    (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE))) {
    528   1.1      fvdl 				error = EBUSY;
    529   1.1      fvdl 				break;
    530   1.1      fvdl 			}
    531   1.1      fvdl 			/*
    532   1.1      fvdl 			 * Wait for exclusive locks and upgrades to clear.
    533   1.1      fvdl 			 */
    534  1.23   thorpej 			ACQUIRE(lkp, error, extflags, 0, lkp->lk_flags &
    535   1.1      fvdl 			    (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE));
    536   1.1      fvdl 			if (error)
    537   1.1      fvdl 				break;
    538   1.1      fvdl 			lkp->lk_sharecount++;
    539  1.21   thorpej 			COUNT(lkp, p, cpu_id, 1);
    540   1.1      fvdl 			break;
    541   1.1      fvdl 		}
    542   1.1      fvdl 		/*
    543   1.1      fvdl 		 * We hold an exclusive lock, so downgrade it to shared.
    544   1.1      fvdl 		 * An alternative would be to fail with EDEADLK.
    545   1.1      fvdl 		 */
    546   1.1      fvdl 		lkp->lk_sharecount++;
    547  1.21   thorpej 		COUNT(lkp, p, cpu_id, 1);
    548   1.1      fvdl 		/* fall into downgrade */
    549   1.1      fvdl 
    550   1.1      fvdl 	case LK_DOWNGRADE:
    551  1.19   thorpej 		if (WEHOLDIT(lkp, pid, cpu_id) == 0 ||
    552  1.19   thorpej 		    lkp->lk_exclusivecount == 0)
    553   1.1      fvdl 			panic("lockmgr: not holding exclusive lock");
    554   1.1      fvdl 		lkp->lk_sharecount += lkp->lk_exclusivecount;
    555   1.1      fvdl 		lkp->lk_exclusivecount = 0;
    556  1.15      fvdl 		lkp->lk_recurselevel = 0;
    557   1.1      fvdl 		lkp->lk_flags &= ~LK_HAVE_EXCL;
    558  1.19   thorpej 		SETHOLDER(lkp, LK_NOPROC, LK_NOCPU);
    559  1.50   thorpej #if defined(LOCKDEBUG)
    560  1.50   thorpej 		lkp->lk_unlock_file = file;
    561  1.50   thorpej 		lkp->lk_unlock_line = line;
    562  1.50   thorpej #endif
    563  1.21   thorpej 		DONTHAVEIT(lkp);
    564  1.23   thorpej 		WAKEUP_WAITER(lkp);
    565   1.1      fvdl 		break;
    566   1.1      fvdl 
    567   1.1      fvdl 	case LK_EXCLUPGRADE:
    568   1.1      fvdl 		/*
    569   1.1      fvdl 		 * If another process is ahead of us to get an upgrade,
    570   1.1      fvdl 		 * then we want to fail rather than have an intervening
    571   1.1      fvdl 		 * exclusive access.
    572   1.1      fvdl 		 */
    573   1.1      fvdl 		if (lkp->lk_flags & LK_WANT_UPGRADE) {
    574   1.1      fvdl 			lkp->lk_sharecount--;
    575  1.21   thorpej 			COUNT(lkp, p, cpu_id, -1);
    576   1.1      fvdl 			error = EBUSY;
    577   1.1      fvdl 			break;
    578   1.1      fvdl 		}
    579   1.1      fvdl 		/* fall into normal upgrade */
    580   1.1      fvdl 
    581   1.1      fvdl 	case LK_UPGRADE:
    582   1.1      fvdl 		/*
    583   1.1      fvdl 		 * Upgrade a shared lock to an exclusive one. If another
    584   1.1      fvdl 		 * shared lock has already requested an upgrade to an
    585   1.1      fvdl 		 * exclusive lock, our shared lock is released and an
    586   1.1      fvdl 		 * exclusive lock is requested (which will be granted
    587   1.1      fvdl 		 * after the upgrade). If we return an error, the file
    588   1.1      fvdl 		 * will always be unlocked.
    589   1.1      fvdl 		 */
    590  1.19   thorpej 		if (WEHOLDIT(lkp, pid, cpu_id) || lkp->lk_sharecount <= 0)
    591   1.1      fvdl 			panic("lockmgr: upgrade exclusive lock");
    592   1.1      fvdl 		lkp->lk_sharecount--;
    593  1.21   thorpej 		COUNT(lkp, p, cpu_id, -1);
    594   1.1      fvdl 		/*
    595   1.1      fvdl 		 * If we are just polling, check to see if we will block.
    596   1.1      fvdl 		 */
    597   1.1      fvdl 		if ((extflags & LK_NOWAIT) &&
    598   1.1      fvdl 		    ((lkp->lk_flags & LK_WANT_UPGRADE) ||
    599   1.1      fvdl 		     lkp->lk_sharecount > 1)) {
    600   1.1      fvdl 			error = EBUSY;
    601   1.1      fvdl 			break;
    602   1.1      fvdl 		}
    603   1.1      fvdl 		if ((lkp->lk_flags & LK_WANT_UPGRADE) == 0) {
    604   1.1      fvdl 			/*
    605   1.1      fvdl 			 * We are first shared lock to request an upgrade, so
    606   1.1      fvdl 			 * request upgrade and wait for the shared count to
    607   1.1      fvdl 			 * drop to zero, then take exclusive lock.
    608   1.1      fvdl 			 */
    609   1.1      fvdl 			lkp->lk_flags |= LK_WANT_UPGRADE;
    610  1.23   thorpej 			ACQUIRE(lkp, error, extflags, 0, lkp->lk_sharecount);
    611   1.1      fvdl 			lkp->lk_flags &= ~LK_WANT_UPGRADE;
    612   1.1      fvdl 			if (error)
    613   1.1      fvdl 				break;
    614   1.1      fvdl 			lkp->lk_flags |= LK_HAVE_EXCL;
    615  1.19   thorpej 			SETHOLDER(lkp, pid, cpu_id);
    616  1.50   thorpej #if defined(LOCKDEBUG)
    617  1.50   thorpej 			lkp->lk_lock_file = file;
    618  1.50   thorpej 			lkp->lk_lock_line = line;
    619  1.50   thorpej #endif
    620  1.21   thorpej 			HAVEIT(lkp);
    621   1.1      fvdl 			if (lkp->lk_exclusivecount != 0)
    622   1.1      fvdl 				panic("lockmgr: non-zero exclusive count");
    623   1.1      fvdl 			lkp->lk_exclusivecount = 1;
    624  1.15      fvdl 			if (extflags & LK_SETRECURSE)
    625  1.15      fvdl 				lkp->lk_recurselevel = 1;
    626  1.21   thorpej 			COUNT(lkp, p, cpu_id, 1);
    627   1.1      fvdl 			break;
    628   1.1      fvdl 		}
    629   1.1      fvdl 		/*
    630   1.1      fvdl 		 * Someone else has requested upgrade. Release our shared
    631   1.1      fvdl 		 * lock, awaken upgrade requestor if we are the last shared
    632   1.1      fvdl 		 * lock, then request an exclusive lock.
    633   1.1      fvdl 		 */
    634  1.23   thorpej 		if (lkp->lk_sharecount == 0)
    635  1.23   thorpej 			WAKEUP_WAITER(lkp);
    636   1.1      fvdl 		/* fall into exclusive request */
    637   1.1      fvdl 
    638   1.1      fvdl 	case LK_EXCLUSIVE:
    639  1.19   thorpej 		if (WEHOLDIT(lkp, pid, cpu_id)) {
    640   1.1      fvdl 			/*
    641  1.19   thorpej 			 * Recursive lock.
    642   1.1      fvdl 			 */
    643  1.15      fvdl 			if ((extflags & LK_CANRECURSE) == 0 &&
    644  1.16  sommerfe 			     lkp->lk_recurselevel == 0) {
    645  1.16  sommerfe 				if (extflags & LK_RECURSEFAIL) {
    646  1.16  sommerfe 					error = EDEADLK;
    647  1.16  sommerfe 					break;
    648  1.16  sommerfe 				} else
    649  1.16  sommerfe 					panic("lockmgr: locking against myself");
    650  1.16  sommerfe 			}
    651   1.1      fvdl 			lkp->lk_exclusivecount++;
    652  1.15      fvdl 			if (extflags & LK_SETRECURSE &&
    653  1.15      fvdl 			    lkp->lk_recurselevel == 0)
    654  1.15      fvdl 				lkp->lk_recurselevel = lkp->lk_exclusivecount;
    655  1.21   thorpej 			COUNT(lkp, p, cpu_id, 1);
    656   1.1      fvdl 			break;
    657   1.1      fvdl 		}
    658   1.1      fvdl 		/*
    659   1.1      fvdl 		 * If we are just polling, check to see if we will sleep.
    660   1.1      fvdl 		 */
    661   1.1      fvdl 		if ((extflags & LK_NOWAIT) && ((lkp->lk_flags &
    662   1.1      fvdl 		     (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE)) ||
    663   1.1      fvdl 		     lkp->lk_sharecount != 0)) {
    664   1.1      fvdl 			error = EBUSY;
    665   1.1      fvdl 			break;
    666   1.1      fvdl 		}
    667   1.1      fvdl 		/*
    668   1.1      fvdl 		 * Try to acquire the want_exclusive flag.
    669   1.1      fvdl 		 */
    670  1.23   thorpej 		ACQUIRE(lkp, error, extflags, 0, lkp->lk_flags &
    671   1.1      fvdl 		    (LK_HAVE_EXCL | LK_WANT_EXCL));
    672   1.1      fvdl 		if (error)
    673   1.1      fvdl 			break;
    674   1.1      fvdl 		lkp->lk_flags |= LK_WANT_EXCL;
    675   1.1      fvdl 		/*
    676   1.1      fvdl 		 * Wait for shared locks and upgrades to finish.
    677   1.1      fvdl 		 */
    678  1.23   thorpej 		ACQUIRE(lkp, error, extflags, 0, lkp->lk_sharecount != 0 ||
    679   1.1      fvdl 		       (lkp->lk_flags & LK_WANT_UPGRADE));
    680   1.1      fvdl 		lkp->lk_flags &= ~LK_WANT_EXCL;
    681   1.1      fvdl 		if (error)
    682   1.1      fvdl 			break;
    683   1.1      fvdl 		lkp->lk_flags |= LK_HAVE_EXCL;
    684  1.19   thorpej 		SETHOLDER(lkp, pid, cpu_id);
    685  1.50   thorpej #if defined(LOCKDEBUG)
    686  1.50   thorpej 		lkp->lk_lock_file = file;
    687  1.50   thorpej 		lkp->lk_lock_line = line;
    688  1.50   thorpej #endif
    689  1.21   thorpej 		HAVEIT(lkp);
    690   1.1      fvdl 		if (lkp->lk_exclusivecount != 0)
    691   1.1      fvdl 			panic("lockmgr: non-zero exclusive count");
    692   1.1      fvdl 		lkp->lk_exclusivecount = 1;
    693  1.15      fvdl 		if (extflags & LK_SETRECURSE)
    694  1.15      fvdl 			lkp->lk_recurselevel = 1;
    695  1.21   thorpej 		COUNT(lkp, p, cpu_id, 1);
    696   1.1      fvdl 		break;
    697   1.1      fvdl 
    698   1.1      fvdl 	case LK_RELEASE:
    699   1.1      fvdl 		if (lkp->lk_exclusivecount != 0) {
    700  1.19   thorpej 			if (WEHOLDIT(lkp, pid, cpu_id) == 0) {
    701  1.19   thorpej 				if (lkp->lk_flags & LK_SPIN) {
    702  1.19   thorpej 					panic("lockmgr: processor %lu, not "
    703  1.19   thorpej 					    "exclusive lock holder %lu "
    704  1.19   thorpej 					    "unlocking", cpu_id, lkp->lk_cpu);
    705  1.19   thorpej 				} else {
    706  1.19   thorpej 					panic("lockmgr: pid %d, not "
    707  1.19   thorpej 					    "exclusive lock holder %d "
    708  1.19   thorpej 					    "unlocking", pid,
    709  1.19   thorpej 					    lkp->lk_lockholder);
    710  1.19   thorpej 				}
    711  1.19   thorpej 			}
    712  1.15      fvdl 			if (lkp->lk_exclusivecount == lkp->lk_recurselevel)
    713  1.15      fvdl 				lkp->lk_recurselevel = 0;
    714   1.1      fvdl 			lkp->lk_exclusivecount--;
    715  1.21   thorpej 			COUNT(lkp, p, cpu_id, -1);
    716   1.1      fvdl 			if (lkp->lk_exclusivecount == 0) {
    717   1.1      fvdl 				lkp->lk_flags &= ~LK_HAVE_EXCL;
    718  1.19   thorpej 				SETHOLDER(lkp, LK_NOPROC, LK_NOCPU);
    719  1.50   thorpej #if defined(LOCKDEBUG)
    720  1.50   thorpej 				lkp->lk_unlock_file = file;
    721  1.50   thorpej 				lkp->lk_unlock_line = line;
    722  1.50   thorpej #endif
    723  1.21   thorpej 				DONTHAVEIT(lkp);
    724   1.1      fvdl 			}
    725   1.1      fvdl 		} else if (lkp->lk_sharecount != 0) {
    726   1.1      fvdl 			lkp->lk_sharecount--;
    727  1.21   thorpej 			COUNT(lkp, p, cpu_id, -1);
    728   1.1      fvdl 		}
    729  1.39   thorpej #ifdef DIAGNOSTIC
    730  1.39   thorpej 		else
    731  1.39   thorpej 			panic("lockmgr: release of unlocked lock!");
    732  1.39   thorpej #endif
    733  1.23   thorpej 		WAKEUP_WAITER(lkp);
    734   1.1      fvdl 		break;
    735   1.1      fvdl 
    736   1.1      fvdl 	case LK_DRAIN:
    737   1.1      fvdl 		/*
    738   1.1      fvdl 		 * Check that we do not already hold the lock, as it can
    739   1.1      fvdl 		 * never drain if we do. Unfortunately, we have no way to
    740   1.1      fvdl 		 * check for holding a shared lock, but at least we can
    741   1.1      fvdl 		 * check for an exclusive one.
    742   1.1      fvdl 		 */
    743  1.19   thorpej 		if (WEHOLDIT(lkp, pid, cpu_id))
    744   1.1      fvdl 			panic("lockmgr: draining against myself");
    745   1.1      fvdl 		/*
    746   1.1      fvdl 		 * If we are just polling, check to see if we will sleep.
    747   1.1      fvdl 		 */
    748   1.1      fvdl 		if ((extflags & LK_NOWAIT) && ((lkp->lk_flags &
    749   1.1      fvdl 		     (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE)) ||
    750   1.1      fvdl 		     lkp->lk_sharecount != 0 || lkp->lk_waitcount != 0)) {
    751   1.1      fvdl 			error = EBUSY;
    752   1.1      fvdl 			break;
    753   1.1      fvdl 		}
    754  1.23   thorpej 		ACQUIRE(lkp, error, extflags, 1,
    755  1.23   thorpej 		    ((lkp->lk_flags &
    756  1.23   thorpej 		     (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE)) ||
    757  1.23   thorpej 		     lkp->lk_sharecount != 0 ||
    758  1.23   thorpej 		     lkp->lk_waitcount != 0));
    759  1.23   thorpej 		if (error)
    760  1.23   thorpej 			break;
    761   1.1      fvdl 		lkp->lk_flags |= LK_DRAINING | LK_HAVE_EXCL;
    762  1.19   thorpej 		SETHOLDER(lkp, pid, cpu_id);
    763  1.50   thorpej #if defined(LOCKDEBUG)
    764  1.50   thorpej 		lkp->lk_lock_file = file;
    765  1.50   thorpej 		lkp->lk_lock_line = line;
    766  1.50   thorpej #endif
    767  1.21   thorpej 		HAVEIT(lkp);
    768   1.1      fvdl 		lkp->lk_exclusivecount = 1;
    769  1.15      fvdl 		/* XXX unlikely that we'd want this */
    770  1.15      fvdl 		if (extflags & LK_SETRECURSE)
    771  1.15      fvdl 			lkp->lk_recurselevel = 1;
    772  1.21   thorpej 		COUNT(lkp, p, cpu_id, 1);
    773   1.1      fvdl 		break;
    774   1.1      fvdl 
    775   1.1      fvdl 	default:
    776  1.43   thorpej 		INTERLOCK_RELEASE(lkp, lkp->lk_flags, s);
    777   1.1      fvdl 		panic("lockmgr: unknown locktype request %d",
    778   1.1      fvdl 		    flags & LK_TYPE_MASK);
    779   1.1      fvdl 		/* NOTREACHED */
    780   1.1      fvdl 	}
    781  1.23   thorpej 	if ((lkp->lk_flags & (LK_WAITDRAIN|LK_SPIN)) == LK_WAITDRAIN &&
    782  1.23   thorpej 	    ((lkp->lk_flags &
    783  1.23   thorpej 	      (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE)) == 0 &&
    784   1.1      fvdl 	     lkp->lk_sharecount == 0 && lkp->lk_waitcount == 0)) {
    785   1.1      fvdl 		lkp->lk_flags &= ~LK_WAITDRAIN;
    786  1.20   thorpej 		wakeup_one((void *)&lkp->lk_flags);
    787   1.1      fvdl 	}
    788  1.32  sommerfe 	/*
    789  1.32  sommerfe 	 * Note that this panic will be a recursive panic, since
    790  1.32  sommerfe 	 * we only set lock_shutdown_noblock above if panicstr != NULL.
    791  1.32  sommerfe 	 */
    792  1.32  sommerfe 	if (error && lock_shutdown_noblock)
    793  1.32  sommerfe 		panic("lockmgr: deadlock (see previous panic)");
    794  1.32  sommerfe 
    795  1.43   thorpej 	INTERLOCK_RELEASE(lkp, lkp->lk_flags, s);
    796   1.1      fvdl 	return (error);
    797   1.1      fvdl }
    798   1.1      fvdl 
    799   1.1      fvdl /*
    800  1.47  sommerfe  * For a recursive spinlock held one or more times by the current CPU,
    801  1.47  sommerfe  * release all N locks, and return N.
    802  1.47  sommerfe  * Intended for use in mi_switch() shortly before context switching.
    803  1.47  sommerfe  */
    804  1.47  sommerfe 
    805  1.47  sommerfe int
    806  1.50   thorpej #if defined(LOCKDEBUG)
    807  1.50   thorpej _spinlock_release_all(__volatile struct lock *lkp, const char *file, int line)
    808  1.50   thorpej #else
    809  1.47  sommerfe spinlock_release_all(__volatile struct lock *lkp)
    810  1.50   thorpej #endif
    811  1.47  sommerfe {
    812  1.47  sommerfe 	int s, count;
    813  1.47  sommerfe 	cpuid_t cpu_id;
    814  1.47  sommerfe 
    815  1.47  sommerfe 	KASSERT(lkp->lk_flags & LK_SPIN);
    816  1.47  sommerfe 
    817  1.47  sommerfe 	INTERLOCK_ACQUIRE(lkp, LK_SPIN, s);
    818  1.47  sommerfe 
    819  1.47  sommerfe 	cpu_id = cpu_number();
    820  1.47  sommerfe 	count = lkp->lk_exclusivecount;
    821  1.47  sommerfe 
    822  1.47  sommerfe 	if (count != 0) {
    823  1.47  sommerfe #ifdef DIAGNOSTIC
    824  1.47  sommerfe 		if (WEHOLDIT(lkp, 0, cpu_id) == 0) {
    825  1.47  sommerfe 			panic("spinlock_release_all: processor %lu, not "
    826  1.47  sommerfe 			    "exclusive lock holder %lu "
    827  1.47  sommerfe 			    "unlocking", (long)cpu_id, lkp->lk_cpu);
    828  1.47  sommerfe 		}
    829  1.47  sommerfe #endif
    830  1.47  sommerfe 		lkp->lk_recurselevel = 0;
    831  1.47  sommerfe 		lkp->lk_exclusivecount = 0;
    832  1.47  sommerfe 		COUNT_CPU(cpu_id, -count);
    833  1.47  sommerfe 		lkp->lk_flags &= ~LK_HAVE_EXCL;
    834  1.47  sommerfe 		SETHOLDER(lkp, LK_NOPROC, LK_NOCPU);
    835  1.50   thorpej #if defined(LOCKDEBUG)
    836  1.50   thorpej 		lkp->lk_unlock_file = file;
    837  1.50   thorpej 		lkp->lk_unlock_line = line;
    838  1.50   thorpej #endif
    839  1.47  sommerfe 		DONTHAVEIT(lkp);
    840  1.47  sommerfe 	}
    841  1.47  sommerfe #ifdef DIAGNOSTIC
    842  1.47  sommerfe 	else if (lkp->lk_sharecount != 0)
    843  1.47  sommerfe 		panic("spinlock_release_all: release of shared lock!");
    844  1.47  sommerfe 	else
    845  1.47  sommerfe 		panic("spinlock_release_all: release of unlocked lock!");
    846  1.47  sommerfe #endif
    847  1.47  sommerfe 	INTERLOCK_RELEASE(lkp, LK_SPIN, s);
    848  1.47  sommerfe 
    849  1.47  sommerfe 	return (count);
    850  1.47  sommerfe }
    851  1.47  sommerfe 
    852  1.47  sommerfe /*
    853  1.47  sommerfe  * For a recursive spinlock held one or more times by the current CPU,
    854  1.47  sommerfe  * release all N locks, and return N.
    855  1.47  sommerfe  * Intended for use in mi_switch() right after resuming execution.
    856  1.47  sommerfe  */
    857  1.47  sommerfe 
    858  1.47  sommerfe void
    859  1.50   thorpej #if defined(LOCKDEBUG)
    860  1.50   thorpej _spinlock_acquire_count(__volatile struct lock *lkp, int count,
    861  1.50   thorpej     const char *file, int line)
    862  1.50   thorpej #else
    863  1.47  sommerfe spinlock_acquire_count(__volatile struct lock *lkp, int count)
    864  1.50   thorpej #endif
    865  1.47  sommerfe {
    866  1.47  sommerfe 	int s, error;
    867  1.47  sommerfe 	cpuid_t cpu_id;
    868  1.47  sommerfe 
    869  1.47  sommerfe 	KASSERT(lkp->lk_flags & LK_SPIN);
    870  1.47  sommerfe 
    871  1.47  sommerfe 	INTERLOCK_ACQUIRE(lkp, LK_SPIN, s);
    872  1.47  sommerfe 
    873  1.47  sommerfe 	cpu_id = cpu_number();
    874  1.47  sommerfe 
    875  1.47  sommerfe #ifdef DIAGNOSTIC
    876  1.47  sommerfe 	if (WEHOLDIT(lkp, LK_NOPROC, cpu_id))
    877  1.47  sommerfe 		panic("spinlock_acquire_count: processor %lu already holds lock\n", (long)cpu_id);
    878  1.47  sommerfe #endif
    879  1.47  sommerfe 	/*
    880  1.47  sommerfe 	 * Try to acquire the want_exclusive flag.
    881  1.47  sommerfe 	 */
    882  1.47  sommerfe 	ACQUIRE(lkp, error, LK_SPIN, 0, lkp->lk_flags &
    883  1.47  sommerfe 	    (LK_HAVE_EXCL | LK_WANT_EXCL));
    884  1.47  sommerfe 	lkp->lk_flags |= LK_WANT_EXCL;
    885  1.47  sommerfe 	/*
    886  1.47  sommerfe 	 * Wait for shared locks and upgrades to finish.
    887  1.47  sommerfe 	 */
    888  1.47  sommerfe 	ACQUIRE(lkp, error, LK_SPIN, 0, lkp->lk_sharecount != 0 ||
    889  1.47  sommerfe 	    (lkp->lk_flags & LK_WANT_UPGRADE));
    890  1.47  sommerfe 	lkp->lk_flags &= ~LK_WANT_EXCL;
    891  1.47  sommerfe 	lkp->lk_flags |= LK_HAVE_EXCL;
    892  1.47  sommerfe 	SETHOLDER(lkp, LK_NOPROC, cpu_id);
    893  1.50   thorpej #if defined(LOCKDEBUG)
    894  1.50   thorpej 	lkp->lk_lock_file = file;
    895  1.50   thorpej 	lkp->lk_lock_line = line;
    896  1.50   thorpej #endif
    897  1.47  sommerfe 	HAVEIT(lkp);
    898  1.47  sommerfe 	if (lkp->lk_exclusivecount != 0)
    899  1.47  sommerfe 		panic("lockmgr: non-zero exclusive count");
    900  1.47  sommerfe 	lkp->lk_exclusivecount = count;
    901  1.47  sommerfe 	lkp->lk_recurselevel = 1;
    902  1.47  sommerfe 	COUNT_CPU(cpu_id, count);
    903  1.47  sommerfe 
    904  1.47  sommerfe 	INTERLOCK_RELEASE(lkp, lkp->lk_flags, s);
    905  1.47  sommerfe }
    906  1.47  sommerfe 
    907  1.47  sommerfe 
    908  1.47  sommerfe 
    909  1.47  sommerfe /*
    910   1.1      fvdl  * Print out information about state of a lock. Used by VOP_PRINT
    911   1.1      fvdl  * routines to display ststus about contained locks.
    912   1.1      fvdl  */
    913   1.2      fvdl void
    914  1.33   thorpej lockmgr_printinfo(__volatile struct lock *lkp)
    915   1.1      fvdl {
    916   1.1      fvdl 
    917   1.1      fvdl 	if (lkp->lk_sharecount)
    918   1.1      fvdl 		printf(" lock type %s: SHARED (count %d)", lkp->lk_wmesg,
    919   1.1      fvdl 		    lkp->lk_sharecount);
    920  1.19   thorpej 	else if (lkp->lk_flags & LK_HAVE_EXCL) {
    921  1.19   thorpej 		printf(" lock type %s: EXCL (count %d) by ",
    922  1.19   thorpej 		    lkp->lk_wmesg, lkp->lk_exclusivecount);
    923  1.19   thorpej 		if (lkp->lk_flags & LK_SPIN)
    924  1.19   thorpej 			printf("processor %lu", lkp->lk_cpu);
    925  1.19   thorpej 		else
    926  1.19   thorpej 			printf("pid %d", lkp->lk_lockholder);
    927  1.19   thorpej 	} else
    928  1.19   thorpej 		printf(" not locked");
    929  1.19   thorpej 	if ((lkp->lk_flags & LK_SPIN) == 0 && lkp->lk_waitcount > 0)
    930   1.1      fvdl 		printf(" with %d pending", lkp->lk_waitcount);
    931   1.1      fvdl }
    932   1.1      fvdl 
    933  1.21   thorpej #if defined(LOCKDEBUG) /* { */
    934  1.21   thorpej TAILQ_HEAD(, simplelock) simplelock_list =
    935  1.21   thorpej     TAILQ_HEAD_INITIALIZER(simplelock_list);
    936  1.21   thorpej 
    937  1.21   thorpej #if defined(MULTIPROCESSOR) /* { */
    938  1.21   thorpej struct simplelock simplelock_list_slock = SIMPLELOCK_INITIALIZER;
    939  1.21   thorpej 
    940  1.21   thorpej #define	SLOCK_LIST_LOCK()						\
    941  1.29  sommerfe 	__cpu_simple_lock(&simplelock_list_slock.lock_data)
    942  1.21   thorpej 
    943  1.21   thorpej #define	SLOCK_LIST_UNLOCK()						\
    944  1.29  sommerfe 	__cpu_simple_unlock(&simplelock_list_slock.lock_data)
    945  1.21   thorpej 
    946  1.21   thorpej #define	SLOCK_COUNT(x)							\
    947  1.47  sommerfe 	curcpu()->ci_simple_locks += (x)
    948  1.21   thorpej #else
    949  1.21   thorpej u_long simple_locks;
    950  1.21   thorpej 
    951  1.21   thorpej #define	SLOCK_LIST_LOCK()	/* nothing */
    952  1.21   thorpej 
    953  1.21   thorpej #define	SLOCK_LIST_UNLOCK()	/* nothing */
    954  1.21   thorpej 
    955  1.21   thorpej #define	SLOCK_COUNT(x)		simple_locks += (x)
    956  1.21   thorpej #endif /* MULTIPROCESSOR */ /* } */
    957  1.21   thorpej 
    958  1.21   thorpej #ifdef DDB /* { */
    959  1.45  sommerfe #ifdef MULTIPROCESSOR
    960  1.45  sommerfe int simple_lock_debugger = 1;	/* more serious on MP */
    961  1.45  sommerfe #else
    962  1.18       chs int simple_lock_debugger = 0;
    963  1.45  sommerfe #endif
    964  1.21   thorpej #define	SLOCK_DEBUGGER()	if (simple_lock_debugger) Debugger()
    965  1.21   thorpej #else
    966  1.21   thorpej #define	SLOCK_DEBUGGER()	/* nothing */
    967  1.21   thorpej #endif /* } */
    968  1.21   thorpej 
    969  1.26  sommerfe #ifdef MULTIPROCESSOR
    970  1.46   thorpej #define SLOCK_MP()		lock_printf("on cpu %ld\n", 		\
    971  1.46   thorpej 				    (u_long) cpu_number())
    972  1.26  sommerfe #else
    973  1.26  sommerfe #define SLOCK_MP()		/* nothing */
    974  1.26  sommerfe #endif
    975  1.26  sommerfe 
    976  1.21   thorpej #define	SLOCK_WHERE(str, alp, id, l)					\
    977  1.21   thorpej do {									\
    978  1.25   thorpej 	lock_printf(str);						\
    979  1.33   thorpej 	lock_printf("lock: %p, currently at: %s:%d\n", (alp), (id), (l)); \
    980  1.26  sommerfe 	SLOCK_MP();							\
    981  1.21   thorpej 	if ((alp)->lock_file != NULL)					\
    982  1.25   thorpej 		lock_printf("last locked: %s:%d\n", (alp)->lock_file,	\
    983  1.21   thorpej 		    (alp)->lock_line);					\
    984  1.21   thorpej 	if ((alp)->unlock_file != NULL)					\
    985  1.25   thorpej 		lock_printf("last unlocked: %s:%d\n", (alp)->unlock_file, \
    986  1.21   thorpej 		    (alp)->unlock_line);				\
    987  1.21   thorpej 	SLOCK_DEBUGGER();						\
    988  1.30   thorpej } while (/*CONSTCOND*/0)
    989  1.12       chs 
    990   1.1      fvdl /*
    991   1.1      fvdl  * Simple lock functions so that the debugger can see from whence
    992   1.1      fvdl  * they are being called.
    993   1.1      fvdl  */
    994   1.1      fvdl void
    995  1.33   thorpej simple_lock_init(struct simplelock *alp)
    996   1.1      fvdl {
    997  1.21   thorpej 
    998  1.21   thorpej #if defined(MULTIPROCESSOR) /* { */
    999  1.27   thorpej 	__cpu_simple_lock_init(&alp->lock_data);
   1000  1.21   thorpej #else
   1001  1.27   thorpej 	alp->lock_data = __SIMPLELOCK_UNLOCKED;
   1002  1.21   thorpej #endif /* } */
   1003   1.5       chs 	alp->lock_file = NULL;
   1004   1.5       chs 	alp->lock_line = 0;
   1005   1.5       chs 	alp->unlock_file = NULL;
   1006   1.5       chs 	alp->unlock_line = 0;
   1007  1.41   thorpej 	alp->lock_holder = LK_NOCPU;
   1008   1.1      fvdl }
   1009   1.1      fvdl 
   1010   1.1      fvdl void
   1011  1.33   thorpej _simple_lock(__volatile struct simplelock *alp, const char *id, int l)
   1012   1.1      fvdl {
   1013  1.24   thorpej 	cpuid_t cpu_id = cpu_number();
   1014  1.12       chs 	int s;
   1015  1.12       chs 
   1016  1.44   thorpej 	s = spllock();
   1017  1.21   thorpej 
   1018  1.21   thorpej 	/*
   1019  1.21   thorpej 	 * MULTIPROCESSOR case: This is `safe' since if it's not us, we
   1020  1.21   thorpej 	 * don't take any action, and just fall into the normal spin case.
   1021  1.21   thorpej 	 */
   1022  1.27   thorpej 	if (alp->lock_data == __SIMPLELOCK_LOCKED) {
   1023  1.21   thorpej #if defined(MULTIPROCESSOR) /* { */
   1024  1.21   thorpej 		if (alp->lock_holder == cpu_id) {
   1025  1.21   thorpej 			SLOCK_WHERE("simple_lock: locking against myself\n",
   1026  1.21   thorpej 			    alp, id, l);
   1027  1.21   thorpej 			goto out;
   1028   1.1      fvdl 		}
   1029  1.21   thorpej #else
   1030  1.21   thorpej 		SLOCK_WHERE("simple_lock: lock held\n", alp, id, l);
   1031  1.21   thorpej 		goto out;
   1032  1.21   thorpej #endif /* MULTIPROCESSOR */ /* } */
   1033   1.1      fvdl 	}
   1034  1.21   thorpej 
   1035  1.21   thorpej #if defined(MULTIPROCESSOR) /* { */
   1036  1.21   thorpej 	/* Acquire the lock before modifying any fields. */
   1037  1.27   thorpej 	__cpu_simple_lock(&alp->lock_data);
   1038  1.21   thorpej #else
   1039  1.27   thorpej 	alp->lock_data = __SIMPLELOCK_LOCKED;
   1040  1.21   thorpej #endif /* } */
   1041  1.21   thorpej 
   1042  1.45  sommerfe 	if (alp->lock_holder != LK_NOCPU) {
   1043  1.45  sommerfe 		SLOCK_WHERE("simple_lock: uninitialized lock\n",
   1044  1.45  sommerfe 		    alp, id, l);
   1045  1.45  sommerfe 	}
   1046   1.5       chs 	alp->lock_file = id;
   1047   1.5       chs 	alp->lock_line = l;
   1048  1.21   thorpej 	alp->lock_holder = cpu_id;
   1049  1.21   thorpej 
   1050  1.21   thorpej 	SLOCK_LIST_LOCK();
   1051  1.21   thorpej 	/* XXX Cast away volatile */
   1052  1.21   thorpej 	TAILQ_INSERT_TAIL(&simplelock_list, (struct simplelock *)alp, list);
   1053  1.21   thorpej 	SLOCK_LIST_UNLOCK();
   1054  1.21   thorpej 
   1055  1.21   thorpej 	SLOCK_COUNT(1);
   1056  1.21   thorpej 
   1057  1.21   thorpej  out:
   1058  1.18       chs 	splx(s);
   1059  1.38   thorpej }
   1060  1.38   thorpej 
   1061  1.38   thorpej int
   1062  1.38   thorpej _simple_lock_held(__volatile struct simplelock *alp)
   1063  1.38   thorpej {
   1064  1.38   thorpej 	cpuid_t cpu_id = cpu_number();
   1065  1.38   thorpej 	int s, locked = 0;
   1066  1.38   thorpej 
   1067  1.44   thorpej 	s = spllock();
   1068  1.42   thorpej 
   1069  1.42   thorpej #if defined(MULTIPROCESSOR)
   1070  1.38   thorpej 	if (__cpu_simple_lock_try(&alp->lock_data) == 0)
   1071  1.38   thorpej 		locked = (alp->lock_holder == cpu_id);
   1072  1.38   thorpej 	else
   1073  1.38   thorpej 		__cpu_simple_unlock(&alp->lock_data);
   1074  1.38   thorpej #else
   1075  1.42   thorpej 	if (alp->lock_data == __SIMPLELOCK_LOCKED) {
   1076  1.42   thorpej 		locked = 1;
   1077  1.42   thorpej 		KASSERT(alp->lock_holder == cpu_id);
   1078  1.42   thorpej 	}
   1079  1.42   thorpej #endif
   1080  1.38   thorpej 
   1081  1.38   thorpej 	splx(s);
   1082  1.42   thorpej 
   1083  1.38   thorpej 	return (locked);
   1084   1.1      fvdl }
   1085   1.1      fvdl 
   1086   1.1      fvdl int
   1087  1.33   thorpej _simple_lock_try(__volatile struct simplelock *alp, const char *id, int l)
   1088   1.1      fvdl {
   1089  1.24   thorpej 	cpuid_t cpu_id = cpu_number();
   1090  1.21   thorpej 	int s, rv = 0;
   1091   1.1      fvdl 
   1092  1.44   thorpej 	s = spllock();
   1093  1.21   thorpej 
   1094  1.21   thorpej 	/*
   1095  1.21   thorpej 	 * MULTIPROCESSOR case: This is `safe' since if it's not us, we
   1096  1.21   thorpej 	 * don't take any action.
   1097  1.21   thorpej 	 */
   1098  1.21   thorpej #if defined(MULTIPROCESSOR) /* { */
   1099  1.27   thorpej 	if ((rv = __cpu_simple_lock_try(&alp->lock_data)) == 0) {
   1100  1.21   thorpej 		if (alp->lock_holder == cpu_id)
   1101  1.21   thorpej 			SLOCK_WHERE("simple_lock_try: locking against myself\n",
   1102  1.26  sommerfe 			    alp, id, l);
   1103  1.21   thorpej 		goto out;
   1104  1.21   thorpej 	}
   1105  1.21   thorpej #else
   1106  1.27   thorpej 	if (alp->lock_data == __SIMPLELOCK_LOCKED) {
   1107  1.21   thorpej 		SLOCK_WHERE("simple_lock_try: lock held\n", alp, id, l);
   1108  1.21   thorpej 		goto out;
   1109  1.18       chs 	}
   1110  1.27   thorpej 	alp->lock_data = __SIMPLELOCK_LOCKED;
   1111  1.21   thorpej #endif /* MULTIPROCESSOR */ /* } */
   1112  1.21   thorpej 
   1113  1.21   thorpej 	/*
   1114  1.21   thorpej 	 * At this point, we have acquired the lock.
   1115  1.21   thorpej 	 */
   1116  1.21   thorpej 
   1117  1.21   thorpej 	rv = 1;
   1118  1.18       chs 
   1119   1.5       chs 	alp->lock_file = id;
   1120   1.5       chs 	alp->lock_line = l;
   1121  1.21   thorpej 	alp->lock_holder = cpu_id;
   1122  1.21   thorpej 
   1123  1.21   thorpej 	SLOCK_LIST_LOCK();
   1124  1.21   thorpej 	/* XXX Cast away volatile. */
   1125  1.21   thorpej 	TAILQ_INSERT_TAIL(&simplelock_list, (struct simplelock *)alp, list);
   1126  1.21   thorpej 	SLOCK_LIST_UNLOCK();
   1127  1.21   thorpej 
   1128  1.21   thorpej 	SLOCK_COUNT(1);
   1129  1.21   thorpej 
   1130  1.21   thorpej  out:
   1131  1.12       chs 	splx(s);
   1132  1.21   thorpej 	return (rv);
   1133   1.1      fvdl }
   1134   1.1      fvdl 
   1135   1.1      fvdl void
   1136  1.33   thorpej _simple_unlock(__volatile struct simplelock *alp, const char *id, int l)
   1137   1.1      fvdl {
   1138  1.12       chs 	int s;
   1139   1.1      fvdl 
   1140  1.44   thorpej 	s = spllock();
   1141  1.21   thorpej 
   1142  1.21   thorpej 	/*
   1143  1.21   thorpej 	 * MULTIPROCESSOR case: This is `safe' because we think we hold
   1144  1.21   thorpej 	 * the lock, and if we don't, we don't take any action.
   1145  1.21   thorpej 	 */
   1146  1.27   thorpej 	if (alp->lock_data == __SIMPLELOCK_UNLOCKED) {
   1147  1.21   thorpej 		SLOCK_WHERE("simple_unlock: lock not held\n",
   1148  1.21   thorpej 		    alp, id, l);
   1149  1.21   thorpej 		goto out;
   1150  1.21   thorpej 	}
   1151  1.21   thorpej 
   1152  1.21   thorpej 	SLOCK_LIST_LOCK();
   1153  1.21   thorpej 	TAILQ_REMOVE(&simplelock_list, alp, list);
   1154  1.21   thorpej 	SLOCK_LIST_UNLOCK();
   1155  1.21   thorpej 
   1156  1.21   thorpej 	SLOCK_COUNT(-1);
   1157  1.21   thorpej 
   1158  1.21   thorpej 	alp->list.tqe_next = NULL;	/* sanity */
   1159  1.21   thorpej 	alp->list.tqe_prev = NULL;	/* sanity */
   1160  1.21   thorpej 
   1161   1.5       chs 	alp->unlock_file = id;
   1162   1.5       chs 	alp->unlock_line = l;
   1163  1.21   thorpej 
   1164  1.21   thorpej #if defined(MULTIPROCESSOR) /* { */
   1165  1.26  sommerfe 	alp->lock_holder = LK_NOCPU;
   1166  1.21   thorpej 	/* Now that we've modified all fields, release the lock. */
   1167  1.27   thorpej 	__cpu_simple_unlock(&alp->lock_data);
   1168  1.21   thorpej #else
   1169  1.27   thorpej 	alp->lock_data = __SIMPLELOCK_UNLOCKED;
   1170  1.41   thorpej 	KASSERT(alp->lock_holder == cpu_number());
   1171  1.41   thorpej 	alp->lock_holder = LK_NOCPU;
   1172  1.21   thorpej #endif /* } */
   1173  1.21   thorpej 
   1174  1.21   thorpej  out:
   1175  1.18       chs 	splx(s);
   1176  1.12       chs }
   1177  1.12       chs 
   1178  1.12       chs void
   1179  1.33   thorpej simple_lock_dump(void)
   1180  1.12       chs {
   1181  1.12       chs 	struct simplelock *alp;
   1182  1.12       chs 	int s;
   1183  1.12       chs 
   1184  1.44   thorpej 	s = spllock();
   1185  1.21   thorpej 	SLOCK_LIST_LOCK();
   1186  1.25   thorpej 	lock_printf("all simple locks:\n");
   1187  1.21   thorpej 	for (alp = TAILQ_FIRST(&simplelock_list); alp != NULL;
   1188  1.21   thorpej 	     alp = TAILQ_NEXT(alp, list)) {
   1189  1.25   thorpej 		lock_printf("%p CPU %lu %s:%d\n", alp, alp->lock_holder,
   1190  1.21   thorpej 		    alp->lock_file, alp->lock_line);
   1191  1.12       chs 	}
   1192  1.21   thorpej 	SLOCK_LIST_UNLOCK();
   1193  1.12       chs 	splx(s);
   1194  1.12       chs }
   1195  1.12       chs 
   1196  1.12       chs void
   1197  1.33   thorpej simple_lock_freecheck(void *start, void *end)
   1198  1.12       chs {
   1199  1.12       chs 	struct simplelock *alp;
   1200  1.12       chs 	int s;
   1201  1.12       chs 
   1202  1.44   thorpej 	s = spllock();
   1203  1.21   thorpej 	SLOCK_LIST_LOCK();
   1204  1.21   thorpej 	for (alp = TAILQ_FIRST(&simplelock_list); alp != NULL;
   1205  1.21   thorpej 	     alp = TAILQ_NEXT(alp, list)) {
   1206  1.12       chs 		if ((void *)alp >= start && (void *)alp < end) {
   1207  1.25   thorpej 			lock_printf("freeing simple_lock %p CPU %lu %s:%d\n",
   1208  1.34   thorpej 			    alp, alp->lock_holder, alp->lock_file,
   1209  1.34   thorpej 			    alp->lock_line);
   1210  1.34   thorpej 			SLOCK_DEBUGGER();
   1211  1.34   thorpej 		}
   1212  1.34   thorpej 	}
   1213  1.34   thorpej 	SLOCK_LIST_UNLOCK();
   1214  1.34   thorpej 	splx(s);
   1215  1.34   thorpej }
   1216  1.34   thorpej 
   1217  1.34   thorpej void
   1218  1.34   thorpej simple_lock_switchcheck(void)
   1219  1.34   thorpej {
   1220  1.34   thorpej 	struct simplelock *alp;
   1221  1.34   thorpej 	cpuid_t cpu_id = cpu_number();
   1222  1.34   thorpej 	int s;
   1223  1.34   thorpej 
   1224  1.42   thorpej 	/*
   1225  1.42   thorpej 	 * We must be holding exactly one lock: the sched_lock.
   1226  1.42   thorpej 	 */
   1227  1.42   thorpej 
   1228  1.42   thorpej 	SCHED_ASSERT_LOCKED();
   1229  1.42   thorpej 
   1230  1.44   thorpej 	s = spllock();
   1231  1.34   thorpej 	SLOCK_LIST_LOCK();
   1232  1.34   thorpej 	for (alp = TAILQ_FIRST(&simplelock_list); alp != NULL;
   1233  1.34   thorpej 	     alp = TAILQ_NEXT(alp, list)) {
   1234  1.42   thorpej 		if (alp == &sched_lock)
   1235  1.42   thorpej 			continue;
   1236  1.34   thorpej 		if (alp->lock_holder == cpu_id) {
   1237  1.34   thorpej 			lock_printf("switching with held simple_lock %p "
   1238  1.36   thorpej 			    "CPU %lu %s:%d\n",
   1239  1.21   thorpej 			    alp, alp->lock_holder, alp->lock_file,
   1240  1.21   thorpej 			    alp->lock_line);
   1241  1.21   thorpej 			SLOCK_DEBUGGER();
   1242  1.12       chs 		}
   1243  1.12       chs 	}
   1244  1.21   thorpej 	SLOCK_LIST_UNLOCK();
   1245  1.12       chs 	splx(s);
   1246   1.1      fvdl }
   1247  1.21   thorpej #endif /* LOCKDEBUG */ /* } */
   1248