kern_lock.c revision 1.52 1 1.52 thorpej /* $NetBSD: kern_lock.c,v 1.52 2001/04/20 22:58:39 thorpej Exp $ */
2 1.19 thorpej
3 1.19 thorpej /*-
4 1.28 thorpej * Copyright (c) 1999, 2000 The NetBSD Foundation, Inc.
5 1.19 thorpej * All rights reserved.
6 1.19 thorpej *
7 1.19 thorpej * This code is derived from software contributed to The NetBSD Foundation
8 1.19 thorpej * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
9 1.19 thorpej * NASA Ames Research Center.
10 1.19 thorpej *
11 1.19 thorpej * This code is derived from software contributed to The NetBSD Foundation
12 1.19 thorpej * by Ross Harvey.
13 1.19 thorpej *
14 1.19 thorpej * Redistribution and use in source and binary forms, with or without
15 1.19 thorpej * modification, are permitted provided that the following conditions
16 1.19 thorpej * are met:
17 1.19 thorpej * 1. Redistributions of source code must retain the above copyright
18 1.19 thorpej * notice, this list of conditions and the following disclaimer.
19 1.19 thorpej * 2. Redistributions in binary form must reproduce the above copyright
20 1.19 thorpej * notice, this list of conditions and the following disclaimer in the
21 1.19 thorpej * documentation and/or other materials provided with the distribution.
22 1.19 thorpej * 3. All advertising materials mentioning features or use of this software
23 1.19 thorpej * must display the following acknowledgement:
24 1.19 thorpej * This product includes software developed by the NetBSD
25 1.19 thorpej * Foundation, Inc. and its contributors.
26 1.19 thorpej * 4. Neither the name of The NetBSD Foundation nor the names of its
27 1.19 thorpej * contributors may be used to endorse or promote products derived
28 1.19 thorpej * from this software without specific prior written permission.
29 1.19 thorpej *
30 1.19 thorpej * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
31 1.19 thorpej * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
32 1.19 thorpej * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
33 1.19 thorpej * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
34 1.19 thorpej * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
35 1.19 thorpej * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
36 1.19 thorpej * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
37 1.19 thorpej * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
38 1.19 thorpej * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
39 1.19 thorpej * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
40 1.19 thorpej * POSSIBILITY OF SUCH DAMAGE.
41 1.19 thorpej */
42 1.2 fvdl
43 1.1 fvdl /*
44 1.1 fvdl * Copyright (c) 1995
45 1.1 fvdl * The Regents of the University of California. All rights reserved.
46 1.1 fvdl *
47 1.1 fvdl * This code contains ideas from software contributed to Berkeley by
48 1.1 fvdl * Avadis Tevanian, Jr., Michael Wayne Young, and the Mach Operating
49 1.1 fvdl * System project at Carnegie-Mellon University.
50 1.1 fvdl *
51 1.1 fvdl * Redistribution and use in source and binary forms, with or without
52 1.1 fvdl * modification, are permitted provided that the following conditions
53 1.1 fvdl * are met:
54 1.1 fvdl * 1. Redistributions of source code must retain the above copyright
55 1.1 fvdl * notice, this list of conditions and the following disclaimer.
56 1.1 fvdl * 2. Redistributions in binary form must reproduce the above copyright
57 1.1 fvdl * notice, this list of conditions and the following disclaimer in the
58 1.1 fvdl * documentation and/or other materials provided with the distribution.
59 1.1 fvdl * 3. All advertising materials mentioning features or use of this software
60 1.1 fvdl * must display the following acknowledgement:
61 1.1 fvdl * This product includes software developed by the University of
62 1.1 fvdl * California, Berkeley and its contributors.
63 1.1 fvdl * 4. Neither the name of the University nor the names of its contributors
64 1.1 fvdl * may be used to endorse or promote products derived from this software
65 1.1 fvdl * without specific prior written permission.
66 1.1 fvdl *
67 1.1 fvdl * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
68 1.1 fvdl * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
69 1.1 fvdl * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
70 1.1 fvdl * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
71 1.1 fvdl * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
72 1.1 fvdl * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
73 1.1 fvdl * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
74 1.1 fvdl * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
75 1.1 fvdl * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
76 1.1 fvdl * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
77 1.1 fvdl * SUCH DAMAGE.
78 1.1 fvdl *
79 1.1 fvdl * @(#)kern_lock.c 8.18 (Berkeley) 5/21/95
80 1.1 fvdl */
81 1.7 thorpej
82 1.21 thorpej #include "opt_multiprocessor.h"
83 1.7 thorpej #include "opt_lockdebug.h"
84 1.18 chs #include "opt_ddb.h"
85 1.1 fvdl
86 1.1 fvdl #include <sys/param.h>
87 1.1 fvdl #include <sys/proc.h>
88 1.1 fvdl #include <sys/lock.h>
89 1.2 fvdl #include <sys/systm.h>
90 1.1 fvdl #include <machine/cpu.h>
91 1.1 fvdl
92 1.25 thorpej #if defined(LOCKDEBUG)
93 1.25 thorpej #include <sys/syslog.h>
94 1.25 thorpej /*
95 1.25 thorpej * note that stdarg.h and the ansi style va_start macro is used for both
96 1.25 thorpej * ansi and traditional c compiles.
97 1.25 thorpej * XXX: this requires that stdarg.h define: va_alist and va_dcl
98 1.25 thorpej */
99 1.25 thorpej #include <machine/stdarg.h>
100 1.25 thorpej
101 1.36 thorpej void lock_printf(const char *fmt, ...)
102 1.37 eeh __attribute__((__format__(__printf__,1,2)));
103 1.25 thorpej
104 1.51 jmc int lock_debug_syslog = 1; /* defaults to syslog, but can be patched */
105 1.25 thorpej #endif
106 1.25 thorpej
107 1.1 fvdl /*
108 1.1 fvdl * Locking primitives implementation.
109 1.1 fvdl * Locks provide shared/exclusive sychronization.
110 1.1 fvdl */
111 1.1 fvdl
112 1.21 thorpej #if defined(LOCKDEBUG) || defined(DIAGNOSTIC) /* { */
113 1.21 thorpej #if defined(MULTIPROCESSOR) /* { */
114 1.21 thorpej #define COUNT_CPU(cpu_id, x) \
115 1.47 sommerfe curcpu()->ci_spin_locks += (x)
116 1.21 thorpej #else
117 1.21 thorpej u_long spin_locks;
118 1.21 thorpej #define COUNT_CPU(cpu_id, x) spin_locks += (x)
119 1.21 thorpej #endif /* MULTIPROCESSOR */ /* } */
120 1.21 thorpej
121 1.21 thorpej #define COUNT(lkp, p, cpu_id, x) \
122 1.21 thorpej do { \
123 1.21 thorpej if ((lkp)->lk_flags & LK_SPIN) \
124 1.21 thorpej COUNT_CPU((cpu_id), (x)); \
125 1.21 thorpej else \
126 1.21 thorpej (p)->p_locks += (x); \
127 1.30 thorpej } while (/*CONSTCOND*/0)
128 1.1 fvdl #else
129 1.22 mellon #define COUNT(lkp, p, cpu_id, x)
130 1.48 sommerfe #define COUNT_CPU(cpu_id, x)
131 1.21 thorpej #endif /* LOCKDEBUG || DIAGNOSTIC */ /* } */
132 1.1 fvdl
133 1.52 thorpej #ifndef SPINLOCK_SPIN_HOOK /* from <machine/lock.h> */
134 1.52 thorpej #define SPINLOCK_SPIN_HOOK /* nothing */
135 1.49 thorpej #endif
136 1.49 thorpej
137 1.43 thorpej #define INTERLOCK_ACQUIRE(lkp, flags, s) \
138 1.40 thorpej do { \
139 1.43 thorpej if ((flags) & LK_SPIN) \
140 1.42 thorpej s = splsched(); \
141 1.40 thorpej simple_lock(&(lkp)->lk_interlock); \
142 1.40 thorpej } while (0)
143 1.40 thorpej
144 1.43 thorpej #define INTERLOCK_RELEASE(lkp, flags, s) \
145 1.40 thorpej do { \
146 1.40 thorpej simple_unlock(&(lkp)->lk_interlock); \
147 1.52 thorpej if ((flags) & LK_SPIN) \
148 1.40 thorpej splx(s); \
149 1.40 thorpej } while (0)
150 1.40 thorpej
151 1.50 thorpej #if defined(LOCKDEBUG)
152 1.50 thorpej #if defined(DDB)
153 1.50 thorpej #define SPINLOCK_SPINCHECK_DEBUGGER Debugger()
154 1.50 thorpej #else
155 1.50 thorpej #define SPINLOCK_SPINCHECK_DEBUGGER /* nothing */
156 1.50 thorpej #endif
157 1.50 thorpej
158 1.50 thorpej #define SPINLOCK_SPINCHECK_DECL \
159 1.50 thorpej /* 32-bits of count -- wrap constitutes a "spinout" */ \
160 1.50 thorpej uint32_t __spinc = 0
161 1.50 thorpej
162 1.50 thorpej #define SPINLOCK_SPINCHECK \
163 1.50 thorpej do { \
164 1.50 thorpej if (++__spinc == 0) { \
165 1.50 thorpej printf("LK_SPIN spinout, excl %d, share %d\n", \
166 1.50 thorpej lkp->lk_exclusivecount, lkp->lk_sharecount); \
167 1.50 thorpej if (lkp->lk_exclusivecount) \
168 1.50 thorpej printf("held by CPU %lu\n", \
169 1.50 thorpej (u_long) lkp->lk_cpu); \
170 1.50 thorpej if (lkp->lk_lock_file) \
171 1.50 thorpej printf("last locked at %s:%d\n", \
172 1.50 thorpej lkp->lk_lock_file, lkp->lk_lock_line); \
173 1.50 thorpej if (lkp->lk_unlock_file) \
174 1.50 thorpej printf("last unlocked at %s:%d\n", \
175 1.50 thorpej lkp->lk_unlock_file, lkp->lk_unlock_line); \
176 1.50 thorpej SPINLOCK_SPINCHECK_DEBUGGER; \
177 1.50 thorpej } \
178 1.50 thorpej } while (0)
179 1.50 thorpej #else
180 1.50 thorpej #define SPINLOCK_SPINCHECK_DECL /* nothing */
181 1.50 thorpej #define SPINLOCK_SPINCHECK /* nothing */
182 1.50 thorpej #endif /* LOCKDEBUG && DDB */
183 1.50 thorpej
184 1.1 fvdl /*
185 1.1 fvdl * Acquire a resource.
186 1.1 fvdl */
187 1.23 thorpej #define ACQUIRE(lkp, error, extflags, drain, wanted) \
188 1.19 thorpej if ((extflags) & LK_SPIN) { \
189 1.19 thorpej int interlocked; \
190 1.50 thorpej SPINLOCK_SPINCHECK_DECL; \
191 1.19 thorpej \
192 1.23 thorpej if ((drain) == 0) \
193 1.23 thorpej (lkp)->lk_waitcount++; \
194 1.19 thorpej for (interlocked = 1;;) { \
195 1.50 thorpej SPINLOCK_SPINCHECK; \
196 1.19 thorpej if (wanted) { \
197 1.19 thorpej if (interlocked) { \
198 1.43 thorpej INTERLOCK_RELEASE((lkp), \
199 1.43 thorpej LK_SPIN, s); \
200 1.19 thorpej interlocked = 0; \
201 1.19 thorpej } \
202 1.52 thorpej SPINLOCK_SPIN_HOOK; \
203 1.19 thorpej } else if (interlocked) { \
204 1.19 thorpej break; \
205 1.19 thorpej } else { \
206 1.43 thorpej INTERLOCK_ACQUIRE((lkp), LK_SPIN, s); \
207 1.19 thorpej interlocked = 1; \
208 1.19 thorpej } \
209 1.19 thorpej } \
210 1.23 thorpej if ((drain) == 0) \
211 1.23 thorpej (lkp)->lk_waitcount--; \
212 1.19 thorpej KASSERT((wanted) == 0); \
213 1.19 thorpej error = 0; /* sanity */ \
214 1.19 thorpej } else { \
215 1.19 thorpej for (error = 0; wanted; ) { \
216 1.23 thorpej if ((drain)) \
217 1.23 thorpej (lkp)->lk_flags |= LK_WAITDRAIN; \
218 1.23 thorpej else \
219 1.23 thorpej (lkp)->lk_waitcount++; \
220 1.23 thorpej /* XXX Cast away volatile. */ \
221 1.31 thorpej error = ltsleep((drain) ? &(lkp)->lk_flags : \
222 1.23 thorpej (void *)(lkp), (lkp)->lk_prio, \
223 1.31 thorpej (lkp)->lk_wmesg, (lkp)->lk_timo, \
224 1.31 thorpej &(lkp)->lk_interlock); \
225 1.23 thorpej if ((drain) == 0) \
226 1.23 thorpej (lkp)->lk_waitcount--; \
227 1.19 thorpej if (error) \
228 1.19 thorpej break; \
229 1.19 thorpej if ((extflags) & LK_SLEEPFAIL) { \
230 1.19 thorpej error = ENOLCK; \
231 1.19 thorpej break; \
232 1.19 thorpej } \
233 1.1 fvdl } \
234 1.1 fvdl }
235 1.1 fvdl
236 1.19 thorpej #define SETHOLDER(lkp, pid, cpu_id) \
237 1.19 thorpej do { \
238 1.19 thorpej if ((lkp)->lk_flags & LK_SPIN) \
239 1.19 thorpej (lkp)->lk_cpu = cpu_id; \
240 1.19 thorpej else \
241 1.19 thorpej (lkp)->lk_lockholder = pid; \
242 1.30 thorpej } while (/*CONSTCOND*/0)
243 1.19 thorpej
244 1.19 thorpej #define WEHOLDIT(lkp, pid, cpu_id) \
245 1.19 thorpej (((lkp)->lk_flags & LK_SPIN) != 0 ? \
246 1.19 thorpej ((lkp)->lk_cpu == (cpu_id)) : ((lkp)->lk_lockholder == (pid)))
247 1.19 thorpej
248 1.23 thorpej #define WAKEUP_WAITER(lkp) \
249 1.23 thorpej do { \
250 1.23 thorpej if (((lkp)->lk_flags & LK_SPIN) == 0 && (lkp)->lk_waitcount) { \
251 1.23 thorpej /* XXX Cast away volatile. */ \
252 1.23 thorpej wakeup_one((void *)(lkp)); \
253 1.23 thorpej } \
254 1.30 thorpej } while (/*CONSTCOND*/0)
255 1.23 thorpej
256 1.21 thorpej #if defined(LOCKDEBUG) /* { */
257 1.21 thorpej #if defined(MULTIPROCESSOR) /* { */
258 1.21 thorpej struct simplelock spinlock_list_slock = SIMPLELOCK_INITIALIZER;
259 1.21 thorpej
260 1.27 thorpej #define SPINLOCK_LIST_LOCK() \
261 1.29 sommerfe __cpu_simple_lock(&spinlock_list_slock.lock_data)
262 1.21 thorpej
263 1.27 thorpej #define SPINLOCK_LIST_UNLOCK() \
264 1.29 sommerfe __cpu_simple_unlock(&spinlock_list_slock.lock_data)
265 1.21 thorpej #else
266 1.21 thorpej #define SPINLOCK_LIST_LOCK() /* nothing */
267 1.21 thorpej
268 1.21 thorpej #define SPINLOCK_LIST_UNLOCK() /* nothing */
269 1.21 thorpej #endif /* MULTIPROCESSOR */ /* } */
270 1.21 thorpej
271 1.21 thorpej TAILQ_HEAD(, lock) spinlock_list =
272 1.21 thorpej TAILQ_HEAD_INITIALIZER(spinlock_list);
273 1.21 thorpej
274 1.21 thorpej #define HAVEIT(lkp) \
275 1.21 thorpej do { \
276 1.21 thorpej if ((lkp)->lk_flags & LK_SPIN) { \
277 1.44 thorpej int s = spllock(); \
278 1.21 thorpej SPINLOCK_LIST_LOCK(); \
279 1.21 thorpej /* XXX Cast away volatile. */ \
280 1.21 thorpej TAILQ_INSERT_TAIL(&spinlock_list, (struct lock *)(lkp), \
281 1.21 thorpej lk_list); \
282 1.21 thorpej SPINLOCK_LIST_UNLOCK(); \
283 1.21 thorpej splx(s); \
284 1.21 thorpej } \
285 1.30 thorpej } while (/*CONSTCOND*/0)
286 1.21 thorpej
287 1.21 thorpej #define DONTHAVEIT(lkp) \
288 1.21 thorpej do { \
289 1.21 thorpej if ((lkp)->lk_flags & LK_SPIN) { \
290 1.44 thorpej int s = spllock(); \
291 1.21 thorpej SPINLOCK_LIST_LOCK(); \
292 1.21 thorpej /* XXX Cast away volatile. */ \
293 1.21 thorpej TAILQ_REMOVE(&spinlock_list, (struct lock *)(lkp), \
294 1.21 thorpej lk_list); \
295 1.21 thorpej SPINLOCK_LIST_UNLOCK(); \
296 1.21 thorpej splx(s); \
297 1.21 thorpej } \
298 1.30 thorpej } while (/*CONSTCOND*/0)
299 1.21 thorpej #else
300 1.21 thorpej #define HAVEIT(lkp) /* nothing */
301 1.21 thorpej
302 1.21 thorpej #define DONTHAVEIT(lkp) /* nothing */
303 1.21 thorpej #endif /* LOCKDEBUG */ /* } */
304 1.21 thorpej
305 1.25 thorpej #if defined(LOCKDEBUG)
306 1.25 thorpej /*
307 1.25 thorpej * Lock debug printing routine; can be configured to print to console
308 1.25 thorpej * or log to syslog.
309 1.25 thorpej */
310 1.25 thorpej void
311 1.25 thorpej lock_printf(const char *fmt, ...)
312 1.25 thorpej {
313 1.25 thorpej va_list ap;
314 1.25 thorpej
315 1.25 thorpej va_start(ap, fmt);
316 1.25 thorpej if (lock_debug_syslog)
317 1.25 thorpej vlog(LOG_DEBUG, fmt, ap);
318 1.25 thorpej else
319 1.25 thorpej vprintf(fmt, ap);
320 1.25 thorpej va_end(ap);
321 1.25 thorpej }
322 1.25 thorpej #endif /* LOCKDEBUG */
323 1.25 thorpej
324 1.1 fvdl /*
325 1.1 fvdl * Initialize a lock; required before use.
326 1.1 fvdl */
327 1.1 fvdl void
328 1.33 thorpej lockinit(struct lock *lkp, int prio, const char *wmesg, int timo, int flags)
329 1.1 fvdl {
330 1.1 fvdl
331 1.8 perry memset(lkp, 0, sizeof(struct lock));
332 1.1 fvdl simple_lock_init(&lkp->lk_interlock);
333 1.1 fvdl lkp->lk_flags = flags & LK_EXTFLG_MASK;
334 1.19 thorpej if (flags & LK_SPIN)
335 1.19 thorpej lkp->lk_cpu = LK_NOCPU;
336 1.19 thorpej else {
337 1.19 thorpej lkp->lk_lockholder = LK_NOPROC;
338 1.19 thorpej lkp->lk_prio = prio;
339 1.19 thorpej lkp->lk_timo = timo;
340 1.19 thorpej }
341 1.19 thorpej lkp->lk_wmesg = wmesg; /* just a name for spin locks */
342 1.50 thorpej #if defined(LOCKDEBUG)
343 1.50 thorpej lkp->lk_lock_file = NULL;
344 1.50 thorpej lkp->lk_unlock_file = NULL;
345 1.50 thorpej #endif
346 1.1 fvdl }
347 1.1 fvdl
348 1.1 fvdl /*
349 1.1 fvdl * Determine the status of a lock.
350 1.1 fvdl */
351 1.1 fvdl int
352 1.33 thorpej lockstatus(struct lock *lkp)
353 1.1 fvdl {
354 1.40 thorpej int s, lock_type = 0;
355 1.1 fvdl
356 1.43 thorpej INTERLOCK_ACQUIRE(lkp, lkp->lk_flags, s);
357 1.1 fvdl if (lkp->lk_exclusivecount != 0)
358 1.1 fvdl lock_type = LK_EXCLUSIVE;
359 1.1 fvdl else if (lkp->lk_sharecount != 0)
360 1.1 fvdl lock_type = LK_SHARED;
361 1.43 thorpej INTERLOCK_RELEASE(lkp, lkp->lk_flags, s);
362 1.1 fvdl return (lock_type);
363 1.1 fvdl }
364 1.35 thorpej
365 1.35 thorpej #if defined(LOCKDEBUG) || defined(DIAGNOSTIC)
366 1.35 thorpej /*
367 1.35 thorpej * Make sure no spin locks are held by a CPU that is about
368 1.35 thorpej * to context switch.
369 1.35 thorpej */
370 1.35 thorpej void
371 1.35 thorpej spinlock_switchcheck(void)
372 1.35 thorpej {
373 1.35 thorpej u_long cnt;
374 1.35 thorpej int s;
375 1.35 thorpej
376 1.44 thorpej s = spllock();
377 1.35 thorpej #if defined(MULTIPROCESSOR)
378 1.35 thorpej cnt = curcpu()->ci_spin_locks;
379 1.35 thorpej #else
380 1.35 thorpej cnt = spin_locks;
381 1.35 thorpej #endif
382 1.35 thorpej splx(s);
383 1.35 thorpej
384 1.35 thorpej if (cnt != 0)
385 1.35 thorpej panic("spinlock_switchcheck: CPU %lu has %lu spin locks",
386 1.35 thorpej (u_long) cpu_number(), cnt);
387 1.35 thorpej }
388 1.35 thorpej #endif /* LOCKDEBUG || DIAGNOSTIC */
389 1.1 fvdl
390 1.1 fvdl /*
391 1.44 thorpej * Locks and IPLs (interrupt priority levels):
392 1.44 thorpej *
393 1.44 thorpej * Locks which may be taken from interrupt context must be handled
394 1.44 thorpej * very carefully; you must spl to the highest IPL where the lock
395 1.44 thorpej * is needed before acquiring the lock.
396 1.44 thorpej *
397 1.44 thorpej * It is also important to avoid deadlock, since certain (very high
398 1.44 thorpej * priority) interrupts are often needed to keep the system as a whole
399 1.44 thorpej * from deadlocking, and must not be blocked while you are spinning
400 1.44 thorpej * waiting for a lower-priority lock.
401 1.44 thorpej *
402 1.44 thorpej * In addition, the lock-debugging hooks themselves need to use locks!
403 1.44 thorpej *
404 1.44 thorpej * A raw __cpu_simple_lock may be used from interrupts are long as it
405 1.44 thorpej * is acquired and held at a single IPL.
406 1.44 thorpej *
407 1.44 thorpej * A simple_lock (which is a __cpu_simple_lock wrapped with some
408 1.44 thorpej * debugging hooks) may be used at or below spllock(), which is
409 1.44 thorpej * typically at or just below splhigh() (i.e. blocks everything
410 1.44 thorpej * but certain machine-dependent extremely high priority interrupts).
411 1.44 thorpej *
412 1.44 thorpej * spinlockmgr spinlocks should be used at or below splsched().
413 1.44 thorpej *
414 1.44 thorpej * Some platforms may have interrupts of higher priority than splsched(),
415 1.44 thorpej * including hard serial interrupts, inter-processor interrupts, and
416 1.44 thorpej * kernel debugger traps.
417 1.44 thorpej */
418 1.44 thorpej
419 1.44 thorpej /*
420 1.32 sommerfe * XXX XXX kludge around another kludge..
421 1.32 sommerfe *
422 1.32 sommerfe * vfs_shutdown() may be called from interrupt context, either as a result
423 1.32 sommerfe * of a panic, or from the debugger. It proceeds to call
424 1.32 sommerfe * sys_sync(&proc0, ...), pretending its running on behalf of proc0
425 1.32 sommerfe *
426 1.32 sommerfe * We would like to make an attempt to sync the filesystems in this case, so
427 1.32 sommerfe * if this happens, we treat attempts to acquire locks specially.
428 1.32 sommerfe * All locks are acquired on behalf of proc0.
429 1.32 sommerfe *
430 1.32 sommerfe * If we've already paniced, we don't block waiting for locks, but
431 1.32 sommerfe * just barge right ahead since we're already going down in flames.
432 1.32 sommerfe */
433 1.32 sommerfe
434 1.32 sommerfe /*
435 1.1 fvdl * Set, change, or release a lock.
436 1.1 fvdl *
437 1.1 fvdl * Shared requests increment the shared count. Exclusive requests set the
438 1.1 fvdl * LK_WANT_EXCL flag (preventing further shared locks), and wait for already
439 1.1 fvdl * accepted shared locks and shared-to-exclusive upgrades to go away.
440 1.1 fvdl */
441 1.1 fvdl int
442 1.50 thorpej #if defined(LOCKDEBUG)
443 1.50 thorpej _lockmgr(__volatile struct lock *lkp, u_int flags,
444 1.50 thorpej struct simplelock *interlkp, const char *file, int line)
445 1.50 thorpej #else
446 1.33 thorpej lockmgr(__volatile struct lock *lkp, u_int flags,
447 1.33 thorpej struct simplelock *interlkp)
448 1.50 thorpej #endif
449 1.1 fvdl {
450 1.1 fvdl int error;
451 1.1 fvdl pid_t pid;
452 1.1 fvdl int extflags;
453 1.24 thorpej cpuid_t cpu_id;
454 1.6 fvdl struct proc *p = curproc;
455 1.32 sommerfe int lock_shutdown_noblock = 0;
456 1.40 thorpej int s;
457 1.1 fvdl
458 1.1 fvdl error = 0;
459 1.19 thorpej
460 1.43 thorpej INTERLOCK_ACQUIRE(lkp, lkp->lk_flags, s);
461 1.1 fvdl if (flags & LK_INTERLOCK)
462 1.1 fvdl simple_unlock(interlkp);
463 1.1 fvdl extflags = (flags | lkp->lk_flags) & LK_EXTFLG_MASK;
464 1.19 thorpej
465 1.21 thorpej #ifdef DIAGNOSTIC /* { */
466 1.19 thorpej /*
467 1.19 thorpej * Don't allow spins on sleep locks and don't allow sleeps
468 1.19 thorpej * on spin locks.
469 1.19 thorpej */
470 1.19 thorpej if ((flags ^ lkp->lk_flags) & LK_SPIN)
471 1.19 thorpej panic("lockmgr: sleep/spin mismatch\n");
472 1.21 thorpej #endif /* } */
473 1.19 thorpej
474 1.19 thorpej if (extflags & LK_SPIN)
475 1.19 thorpej pid = LK_KERNPROC;
476 1.19 thorpej else {
477 1.32 sommerfe if (p == NULL) {
478 1.32 sommerfe if (!doing_shutdown) {
479 1.32 sommerfe #ifdef DIAGNOSTIC
480 1.32 sommerfe panic("lockmgr: no context");
481 1.32 sommerfe #endif
482 1.32 sommerfe } else {
483 1.32 sommerfe p = &proc0;
484 1.32 sommerfe if (panicstr && (!(flags & LK_NOWAIT))) {
485 1.32 sommerfe flags |= LK_NOWAIT;
486 1.32 sommerfe lock_shutdown_noblock = 1;
487 1.32 sommerfe }
488 1.32 sommerfe }
489 1.32 sommerfe }
490 1.19 thorpej pid = p->p_pid;
491 1.19 thorpej }
492 1.24 thorpej cpu_id = cpu_number();
493 1.19 thorpej
494 1.1 fvdl /*
495 1.1 fvdl * Once a lock has drained, the LK_DRAINING flag is set and an
496 1.1 fvdl * exclusive lock is returned. The only valid operation thereafter
497 1.1 fvdl * is a single release of that exclusive lock. This final release
498 1.1 fvdl * clears the LK_DRAINING flag and sets the LK_DRAINED flag. Any
499 1.1 fvdl * further requests of any sort will result in a panic. The bits
500 1.1 fvdl * selected for these two flags are chosen so that they will be set
501 1.1 fvdl * in memory that is freed (freed memory is filled with 0xdeadbeef).
502 1.1 fvdl * The final release is permitted to give a new lease on life to
503 1.1 fvdl * the lock by specifying LK_REENABLE.
504 1.1 fvdl */
505 1.1 fvdl if (lkp->lk_flags & (LK_DRAINING|LK_DRAINED)) {
506 1.28 thorpej #ifdef DIAGNOSTIC /* { */
507 1.1 fvdl if (lkp->lk_flags & LK_DRAINED)
508 1.1 fvdl panic("lockmgr: using decommissioned lock");
509 1.1 fvdl if ((flags & LK_TYPE_MASK) != LK_RELEASE ||
510 1.19 thorpej WEHOLDIT(lkp, pid, cpu_id) == 0)
511 1.1 fvdl panic("lockmgr: non-release on draining lock: %d\n",
512 1.1 fvdl flags & LK_TYPE_MASK);
513 1.28 thorpej #endif /* DIAGNOSTIC */ /* } */
514 1.1 fvdl lkp->lk_flags &= ~LK_DRAINING;
515 1.1 fvdl if ((flags & LK_REENABLE) == 0)
516 1.1 fvdl lkp->lk_flags |= LK_DRAINED;
517 1.1 fvdl }
518 1.1 fvdl
519 1.1 fvdl switch (flags & LK_TYPE_MASK) {
520 1.1 fvdl
521 1.1 fvdl case LK_SHARED:
522 1.19 thorpej if (WEHOLDIT(lkp, pid, cpu_id) == 0) {
523 1.1 fvdl /*
524 1.1 fvdl * If just polling, check to see if we will block.
525 1.1 fvdl */
526 1.1 fvdl if ((extflags & LK_NOWAIT) && (lkp->lk_flags &
527 1.1 fvdl (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE))) {
528 1.1 fvdl error = EBUSY;
529 1.1 fvdl break;
530 1.1 fvdl }
531 1.1 fvdl /*
532 1.1 fvdl * Wait for exclusive locks and upgrades to clear.
533 1.1 fvdl */
534 1.23 thorpej ACQUIRE(lkp, error, extflags, 0, lkp->lk_flags &
535 1.1 fvdl (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE));
536 1.1 fvdl if (error)
537 1.1 fvdl break;
538 1.1 fvdl lkp->lk_sharecount++;
539 1.21 thorpej COUNT(lkp, p, cpu_id, 1);
540 1.1 fvdl break;
541 1.1 fvdl }
542 1.1 fvdl /*
543 1.1 fvdl * We hold an exclusive lock, so downgrade it to shared.
544 1.1 fvdl * An alternative would be to fail with EDEADLK.
545 1.1 fvdl */
546 1.1 fvdl lkp->lk_sharecount++;
547 1.21 thorpej COUNT(lkp, p, cpu_id, 1);
548 1.1 fvdl /* fall into downgrade */
549 1.1 fvdl
550 1.1 fvdl case LK_DOWNGRADE:
551 1.19 thorpej if (WEHOLDIT(lkp, pid, cpu_id) == 0 ||
552 1.19 thorpej lkp->lk_exclusivecount == 0)
553 1.1 fvdl panic("lockmgr: not holding exclusive lock");
554 1.1 fvdl lkp->lk_sharecount += lkp->lk_exclusivecount;
555 1.1 fvdl lkp->lk_exclusivecount = 0;
556 1.15 fvdl lkp->lk_recurselevel = 0;
557 1.1 fvdl lkp->lk_flags &= ~LK_HAVE_EXCL;
558 1.19 thorpej SETHOLDER(lkp, LK_NOPROC, LK_NOCPU);
559 1.50 thorpej #if defined(LOCKDEBUG)
560 1.50 thorpej lkp->lk_unlock_file = file;
561 1.50 thorpej lkp->lk_unlock_line = line;
562 1.50 thorpej #endif
563 1.21 thorpej DONTHAVEIT(lkp);
564 1.23 thorpej WAKEUP_WAITER(lkp);
565 1.1 fvdl break;
566 1.1 fvdl
567 1.1 fvdl case LK_EXCLUPGRADE:
568 1.1 fvdl /*
569 1.1 fvdl * If another process is ahead of us to get an upgrade,
570 1.1 fvdl * then we want to fail rather than have an intervening
571 1.1 fvdl * exclusive access.
572 1.1 fvdl */
573 1.1 fvdl if (lkp->lk_flags & LK_WANT_UPGRADE) {
574 1.1 fvdl lkp->lk_sharecount--;
575 1.21 thorpej COUNT(lkp, p, cpu_id, -1);
576 1.1 fvdl error = EBUSY;
577 1.1 fvdl break;
578 1.1 fvdl }
579 1.1 fvdl /* fall into normal upgrade */
580 1.1 fvdl
581 1.1 fvdl case LK_UPGRADE:
582 1.1 fvdl /*
583 1.1 fvdl * Upgrade a shared lock to an exclusive one. If another
584 1.1 fvdl * shared lock has already requested an upgrade to an
585 1.1 fvdl * exclusive lock, our shared lock is released and an
586 1.1 fvdl * exclusive lock is requested (which will be granted
587 1.1 fvdl * after the upgrade). If we return an error, the file
588 1.1 fvdl * will always be unlocked.
589 1.1 fvdl */
590 1.19 thorpej if (WEHOLDIT(lkp, pid, cpu_id) || lkp->lk_sharecount <= 0)
591 1.1 fvdl panic("lockmgr: upgrade exclusive lock");
592 1.1 fvdl lkp->lk_sharecount--;
593 1.21 thorpej COUNT(lkp, p, cpu_id, -1);
594 1.1 fvdl /*
595 1.1 fvdl * If we are just polling, check to see if we will block.
596 1.1 fvdl */
597 1.1 fvdl if ((extflags & LK_NOWAIT) &&
598 1.1 fvdl ((lkp->lk_flags & LK_WANT_UPGRADE) ||
599 1.1 fvdl lkp->lk_sharecount > 1)) {
600 1.1 fvdl error = EBUSY;
601 1.1 fvdl break;
602 1.1 fvdl }
603 1.1 fvdl if ((lkp->lk_flags & LK_WANT_UPGRADE) == 0) {
604 1.1 fvdl /*
605 1.1 fvdl * We are first shared lock to request an upgrade, so
606 1.1 fvdl * request upgrade and wait for the shared count to
607 1.1 fvdl * drop to zero, then take exclusive lock.
608 1.1 fvdl */
609 1.1 fvdl lkp->lk_flags |= LK_WANT_UPGRADE;
610 1.23 thorpej ACQUIRE(lkp, error, extflags, 0, lkp->lk_sharecount);
611 1.1 fvdl lkp->lk_flags &= ~LK_WANT_UPGRADE;
612 1.1 fvdl if (error)
613 1.1 fvdl break;
614 1.1 fvdl lkp->lk_flags |= LK_HAVE_EXCL;
615 1.19 thorpej SETHOLDER(lkp, pid, cpu_id);
616 1.50 thorpej #if defined(LOCKDEBUG)
617 1.50 thorpej lkp->lk_lock_file = file;
618 1.50 thorpej lkp->lk_lock_line = line;
619 1.50 thorpej #endif
620 1.21 thorpej HAVEIT(lkp);
621 1.1 fvdl if (lkp->lk_exclusivecount != 0)
622 1.1 fvdl panic("lockmgr: non-zero exclusive count");
623 1.1 fvdl lkp->lk_exclusivecount = 1;
624 1.15 fvdl if (extflags & LK_SETRECURSE)
625 1.15 fvdl lkp->lk_recurselevel = 1;
626 1.21 thorpej COUNT(lkp, p, cpu_id, 1);
627 1.1 fvdl break;
628 1.1 fvdl }
629 1.1 fvdl /*
630 1.1 fvdl * Someone else has requested upgrade. Release our shared
631 1.1 fvdl * lock, awaken upgrade requestor if we are the last shared
632 1.1 fvdl * lock, then request an exclusive lock.
633 1.1 fvdl */
634 1.23 thorpej if (lkp->lk_sharecount == 0)
635 1.23 thorpej WAKEUP_WAITER(lkp);
636 1.1 fvdl /* fall into exclusive request */
637 1.1 fvdl
638 1.1 fvdl case LK_EXCLUSIVE:
639 1.19 thorpej if (WEHOLDIT(lkp, pid, cpu_id)) {
640 1.1 fvdl /*
641 1.19 thorpej * Recursive lock.
642 1.1 fvdl */
643 1.15 fvdl if ((extflags & LK_CANRECURSE) == 0 &&
644 1.16 sommerfe lkp->lk_recurselevel == 0) {
645 1.16 sommerfe if (extflags & LK_RECURSEFAIL) {
646 1.16 sommerfe error = EDEADLK;
647 1.16 sommerfe break;
648 1.16 sommerfe } else
649 1.16 sommerfe panic("lockmgr: locking against myself");
650 1.16 sommerfe }
651 1.1 fvdl lkp->lk_exclusivecount++;
652 1.15 fvdl if (extflags & LK_SETRECURSE &&
653 1.15 fvdl lkp->lk_recurselevel == 0)
654 1.15 fvdl lkp->lk_recurselevel = lkp->lk_exclusivecount;
655 1.21 thorpej COUNT(lkp, p, cpu_id, 1);
656 1.1 fvdl break;
657 1.1 fvdl }
658 1.1 fvdl /*
659 1.1 fvdl * If we are just polling, check to see if we will sleep.
660 1.1 fvdl */
661 1.1 fvdl if ((extflags & LK_NOWAIT) && ((lkp->lk_flags &
662 1.1 fvdl (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE)) ||
663 1.1 fvdl lkp->lk_sharecount != 0)) {
664 1.1 fvdl error = EBUSY;
665 1.1 fvdl break;
666 1.1 fvdl }
667 1.1 fvdl /*
668 1.1 fvdl * Try to acquire the want_exclusive flag.
669 1.1 fvdl */
670 1.23 thorpej ACQUIRE(lkp, error, extflags, 0, lkp->lk_flags &
671 1.1 fvdl (LK_HAVE_EXCL | LK_WANT_EXCL));
672 1.1 fvdl if (error)
673 1.1 fvdl break;
674 1.1 fvdl lkp->lk_flags |= LK_WANT_EXCL;
675 1.1 fvdl /*
676 1.1 fvdl * Wait for shared locks and upgrades to finish.
677 1.1 fvdl */
678 1.23 thorpej ACQUIRE(lkp, error, extflags, 0, lkp->lk_sharecount != 0 ||
679 1.1 fvdl (lkp->lk_flags & LK_WANT_UPGRADE));
680 1.1 fvdl lkp->lk_flags &= ~LK_WANT_EXCL;
681 1.1 fvdl if (error)
682 1.1 fvdl break;
683 1.1 fvdl lkp->lk_flags |= LK_HAVE_EXCL;
684 1.19 thorpej SETHOLDER(lkp, pid, cpu_id);
685 1.50 thorpej #if defined(LOCKDEBUG)
686 1.50 thorpej lkp->lk_lock_file = file;
687 1.50 thorpej lkp->lk_lock_line = line;
688 1.50 thorpej #endif
689 1.21 thorpej HAVEIT(lkp);
690 1.1 fvdl if (lkp->lk_exclusivecount != 0)
691 1.1 fvdl panic("lockmgr: non-zero exclusive count");
692 1.1 fvdl lkp->lk_exclusivecount = 1;
693 1.15 fvdl if (extflags & LK_SETRECURSE)
694 1.15 fvdl lkp->lk_recurselevel = 1;
695 1.21 thorpej COUNT(lkp, p, cpu_id, 1);
696 1.1 fvdl break;
697 1.1 fvdl
698 1.1 fvdl case LK_RELEASE:
699 1.1 fvdl if (lkp->lk_exclusivecount != 0) {
700 1.19 thorpej if (WEHOLDIT(lkp, pid, cpu_id) == 0) {
701 1.19 thorpej if (lkp->lk_flags & LK_SPIN) {
702 1.19 thorpej panic("lockmgr: processor %lu, not "
703 1.19 thorpej "exclusive lock holder %lu "
704 1.19 thorpej "unlocking", cpu_id, lkp->lk_cpu);
705 1.19 thorpej } else {
706 1.19 thorpej panic("lockmgr: pid %d, not "
707 1.19 thorpej "exclusive lock holder %d "
708 1.19 thorpej "unlocking", pid,
709 1.19 thorpej lkp->lk_lockholder);
710 1.19 thorpej }
711 1.19 thorpej }
712 1.15 fvdl if (lkp->lk_exclusivecount == lkp->lk_recurselevel)
713 1.15 fvdl lkp->lk_recurselevel = 0;
714 1.1 fvdl lkp->lk_exclusivecount--;
715 1.21 thorpej COUNT(lkp, p, cpu_id, -1);
716 1.1 fvdl if (lkp->lk_exclusivecount == 0) {
717 1.1 fvdl lkp->lk_flags &= ~LK_HAVE_EXCL;
718 1.19 thorpej SETHOLDER(lkp, LK_NOPROC, LK_NOCPU);
719 1.50 thorpej #if defined(LOCKDEBUG)
720 1.50 thorpej lkp->lk_unlock_file = file;
721 1.50 thorpej lkp->lk_unlock_line = line;
722 1.50 thorpej #endif
723 1.21 thorpej DONTHAVEIT(lkp);
724 1.1 fvdl }
725 1.1 fvdl } else if (lkp->lk_sharecount != 0) {
726 1.1 fvdl lkp->lk_sharecount--;
727 1.21 thorpej COUNT(lkp, p, cpu_id, -1);
728 1.1 fvdl }
729 1.39 thorpej #ifdef DIAGNOSTIC
730 1.39 thorpej else
731 1.39 thorpej panic("lockmgr: release of unlocked lock!");
732 1.39 thorpej #endif
733 1.23 thorpej WAKEUP_WAITER(lkp);
734 1.1 fvdl break;
735 1.1 fvdl
736 1.1 fvdl case LK_DRAIN:
737 1.1 fvdl /*
738 1.1 fvdl * Check that we do not already hold the lock, as it can
739 1.1 fvdl * never drain if we do. Unfortunately, we have no way to
740 1.1 fvdl * check for holding a shared lock, but at least we can
741 1.1 fvdl * check for an exclusive one.
742 1.1 fvdl */
743 1.19 thorpej if (WEHOLDIT(lkp, pid, cpu_id))
744 1.1 fvdl panic("lockmgr: draining against myself");
745 1.1 fvdl /*
746 1.1 fvdl * If we are just polling, check to see if we will sleep.
747 1.1 fvdl */
748 1.1 fvdl if ((extflags & LK_NOWAIT) && ((lkp->lk_flags &
749 1.1 fvdl (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE)) ||
750 1.1 fvdl lkp->lk_sharecount != 0 || lkp->lk_waitcount != 0)) {
751 1.1 fvdl error = EBUSY;
752 1.1 fvdl break;
753 1.1 fvdl }
754 1.23 thorpej ACQUIRE(lkp, error, extflags, 1,
755 1.23 thorpej ((lkp->lk_flags &
756 1.23 thorpej (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE)) ||
757 1.23 thorpej lkp->lk_sharecount != 0 ||
758 1.23 thorpej lkp->lk_waitcount != 0));
759 1.23 thorpej if (error)
760 1.23 thorpej break;
761 1.1 fvdl lkp->lk_flags |= LK_DRAINING | LK_HAVE_EXCL;
762 1.19 thorpej SETHOLDER(lkp, pid, cpu_id);
763 1.50 thorpej #if defined(LOCKDEBUG)
764 1.50 thorpej lkp->lk_lock_file = file;
765 1.50 thorpej lkp->lk_lock_line = line;
766 1.50 thorpej #endif
767 1.21 thorpej HAVEIT(lkp);
768 1.1 fvdl lkp->lk_exclusivecount = 1;
769 1.15 fvdl /* XXX unlikely that we'd want this */
770 1.15 fvdl if (extflags & LK_SETRECURSE)
771 1.15 fvdl lkp->lk_recurselevel = 1;
772 1.21 thorpej COUNT(lkp, p, cpu_id, 1);
773 1.1 fvdl break;
774 1.1 fvdl
775 1.1 fvdl default:
776 1.43 thorpej INTERLOCK_RELEASE(lkp, lkp->lk_flags, s);
777 1.1 fvdl panic("lockmgr: unknown locktype request %d",
778 1.1 fvdl flags & LK_TYPE_MASK);
779 1.1 fvdl /* NOTREACHED */
780 1.1 fvdl }
781 1.23 thorpej if ((lkp->lk_flags & (LK_WAITDRAIN|LK_SPIN)) == LK_WAITDRAIN &&
782 1.23 thorpej ((lkp->lk_flags &
783 1.23 thorpej (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE)) == 0 &&
784 1.1 fvdl lkp->lk_sharecount == 0 && lkp->lk_waitcount == 0)) {
785 1.1 fvdl lkp->lk_flags &= ~LK_WAITDRAIN;
786 1.20 thorpej wakeup_one((void *)&lkp->lk_flags);
787 1.1 fvdl }
788 1.32 sommerfe /*
789 1.32 sommerfe * Note that this panic will be a recursive panic, since
790 1.32 sommerfe * we only set lock_shutdown_noblock above if panicstr != NULL.
791 1.32 sommerfe */
792 1.32 sommerfe if (error && lock_shutdown_noblock)
793 1.32 sommerfe panic("lockmgr: deadlock (see previous panic)");
794 1.32 sommerfe
795 1.43 thorpej INTERLOCK_RELEASE(lkp, lkp->lk_flags, s);
796 1.1 fvdl return (error);
797 1.1 fvdl }
798 1.1 fvdl
799 1.1 fvdl /*
800 1.47 sommerfe * For a recursive spinlock held one or more times by the current CPU,
801 1.47 sommerfe * release all N locks, and return N.
802 1.47 sommerfe * Intended for use in mi_switch() shortly before context switching.
803 1.47 sommerfe */
804 1.47 sommerfe
805 1.47 sommerfe int
806 1.50 thorpej #if defined(LOCKDEBUG)
807 1.50 thorpej _spinlock_release_all(__volatile struct lock *lkp, const char *file, int line)
808 1.50 thorpej #else
809 1.47 sommerfe spinlock_release_all(__volatile struct lock *lkp)
810 1.50 thorpej #endif
811 1.47 sommerfe {
812 1.47 sommerfe int s, count;
813 1.47 sommerfe cpuid_t cpu_id;
814 1.47 sommerfe
815 1.47 sommerfe KASSERT(lkp->lk_flags & LK_SPIN);
816 1.47 sommerfe
817 1.47 sommerfe INTERLOCK_ACQUIRE(lkp, LK_SPIN, s);
818 1.47 sommerfe
819 1.47 sommerfe cpu_id = cpu_number();
820 1.47 sommerfe count = lkp->lk_exclusivecount;
821 1.47 sommerfe
822 1.47 sommerfe if (count != 0) {
823 1.47 sommerfe #ifdef DIAGNOSTIC
824 1.47 sommerfe if (WEHOLDIT(lkp, 0, cpu_id) == 0) {
825 1.47 sommerfe panic("spinlock_release_all: processor %lu, not "
826 1.47 sommerfe "exclusive lock holder %lu "
827 1.47 sommerfe "unlocking", (long)cpu_id, lkp->lk_cpu);
828 1.47 sommerfe }
829 1.47 sommerfe #endif
830 1.47 sommerfe lkp->lk_recurselevel = 0;
831 1.47 sommerfe lkp->lk_exclusivecount = 0;
832 1.47 sommerfe COUNT_CPU(cpu_id, -count);
833 1.47 sommerfe lkp->lk_flags &= ~LK_HAVE_EXCL;
834 1.47 sommerfe SETHOLDER(lkp, LK_NOPROC, LK_NOCPU);
835 1.50 thorpej #if defined(LOCKDEBUG)
836 1.50 thorpej lkp->lk_unlock_file = file;
837 1.50 thorpej lkp->lk_unlock_line = line;
838 1.50 thorpej #endif
839 1.47 sommerfe DONTHAVEIT(lkp);
840 1.47 sommerfe }
841 1.47 sommerfe #ifdef DIAGNOSTIC
842 1.47 sommerfe else if (lkp->lk_sharecount != 0)
843 1.47 sommerfe panic("spinlock_release_all: release of shared lock!");
844 1.47 sommerfe else
845 1.47 sommerfe panic("spinlock_release_all: release of unlocked lock!");
846 1.47 sommerfe #endif
847 1.47 sommerfe INTERLOCK_RELEASE(lkp, LK_SPIN, s);
848 1.47 sommerfe
849 1.47 sommerfe return (count);
850 1.47 sommerfe }
851 1.47 sommerfe
852 1.47 sommerfe /*
853 1.47 sommerfe * For a recursive spinlock held one or more times by the current CPU,
854 1.47 sommerfe * release all N locks, and return N.
855 1.47 sommerfe * Intended for use in mi_switch() right after resuming execution.
856 1.47 sommerfe */
857 1.47 sommerfe
858 1.47 sommerfe void
859 1.50 thorpej #if defined(LOCKDEBUG)
860 1.50 thorpej _spinlock_acquire_count(__volatile struct lock *lkp, int count,
861 1.50 thorpej const char *file, int line)
862 1.50 thorpej #else
863 1.47 sommerfe spinlock_acquire_count(__volatile struct lock *lkp, int count)
864 1.50 thorpej #endif
865 1.47 sommerfe {
866 1.47 sommerfe int s, error;
867 1.47 sommerfe cpuid_t cpu_id;
868 1.47 sommerfe
869 1.47 sommerfe KASSERT(lkp->lk_flags & LK_SPIN);
870 1.47 sommerfe
871 1.47 sommerfe INTERLOCK_ACQUIRE(lkp, LK_SPIN, s);
872 1.47 sommerfe
873 1.47 sommerfe cpu_id = cpu_number();
874 1.47 sommerfe
875 1.47 sommerfe #ifdef DIAGNOSTIC
876 1.47 sommerfe if (WEHOLDIT(lkp, LK_NOPROC, cpu_id))
877 1.47 sommerfe panic("spinlock_acquire_count: processor %lu already holds lock\n", (long)cpu_id);
878 1.47 sommerfe #endif
879 1.47 sommerfe /*
880 1.47 sommerfe * Try to acquire the want_exclusive flag.
881 1.47 sommerfe */
882 1.47 sommerfe ACQUIRE(lkp, error, LK_SPIN, 0, lkp->lk_flags &
883 1.47 sommerfe (LK_HAVE_EXCL | LK_WANT_EXCL));
884 1.47 sommerfe lkp->lk_flags |= LK_WANT_EXCL;
885 1.47 sommerfe /*
886 1.47 sommerfe * Wait for shared locks and upgrades to finish.
887 1.47 sommerfe */
888 1.47 sommerfe ACQUIRE(lkp, error, LK_SPIN, 0, lkp->lk_sharecount != 0 ||
889 1.47 sommerfe (lkp->lk_flags & LK_WANT_UPGRADE));
890 1.47 sommerfe lkp->lk_flags &= ~LK_WANT_EXCL;
891 1.47 sommerfe lkp->lk_flags |= LK_HAVE_EXCL;
892 1.47 sommerfe SETHOLDER(lkp, LK_NOPROC, cpu_id);
893 1.50 thorpej #if defined(LOCKDEBUG)
894 1.50 thorpej lkp->lk_lock_file = file;
895 1.50 thorpej lkp->lk_lock_line = line;
896 1.50 thorpej #endif
897 1.47 sommerfe HAVEIT(lkp);
898 1.47 sommerfe if (lkp->lk_exclusivecount != 0)
899 1.47 sommerfe panic("lockmgr: non-zero exclusive count");
900 1.47 sommerfe lkp->lk_exclusivecount = count;
901 1.47 sommerfe lkp->lk_recurselevel = 1;
902 1.47 sommerfe COUNT_CPU(cpu_id, count);
903 1.47 sommerfe
904 1.47 sommerfe INTERLOCK_RELEASE(lkp, lkp->lk_flags, s);
905 1.47 sommerfe }
906 1.47 sommerfe
907 1.47 sommerfe
908 1.47 sommerfe
909 1.47 sommerfe /*
910 1.1 fvdl * Print out information about state of a lock. Used by VOP_PRINT
911 1.1 fvdl * routines to display ststus about contained locks.
912 1.1 fvdl */
913 1.2 fvdl void
914 1.33 thorpej lockmgr_printinfo(__volatile struct lock *lkp)
915 1.1 fvdl {
916 1.1 fvdl
917 1.1 fvdl if (lkp->lk_sharecount)
918 1.1 fvdl printf(" lock type %s: SHARED (count %d)", lkp->lk_wmesg,
919 1.1 fvdl lkp->lk_sharecount);
920 1.19 thorpej else if (lkp->lk_flags & LK_HAVE_EXCL) {
921 1.19 thorpej printf(" lock type %s: EXCL (count %d) by ",
922 1.19 thorpej lkp->lk_wmesg, lkp->lk_exclusivecount);
923 1.19 thorpej if (lkp->lk_flags & LK_SPIN)
924 1.19 thorpej printf("processor %lu", lkp->lk_cpu);
925 1.19 thorpej else
926 1.19 thorpej printf("pid %d", lkp->lk_lockholder);
927 1.19 thorpej } else
928 1.19 thorpej printf(" not locked");
929 1.19 thorpej if ((lkp->lk_flags & LK_SPIN) == 0 && lkp->lk_waitcount > 0)
930 1.1 fvdl printf(" with %d pending", lkp->lk_waitcount);
931 1.1 fvdl }
932 1.1 fvdl
933 1.21 thorpej #if defined(LOCKDEBUG) /* { */
934 1.21 thorpej TAILQ_HEAD(, simplelock) simplelock_list =
935 1.21 thorpej TAILQ_HEAD_INITIALIZER(simplelock_list);
936 1.21 thorpej
937 1.21 thorpej #if defined(MULTIPROCESSOR) /* { */
938 1.21 thorpej struct simplelock simplelock_list_slock = SIMPLELOCK_INITIALIZER;
939 1.21 thorpej
940 1.21 thorpej #define SLOCK_LIST_LOCK() \
941 1.29 sommerfe __cpu_simple_lock(&simplelock_list_slock.lock_data)
942 1.21 thorpej
943 1.21 thorpej #define SLOCK_LIST_UNLOCK() \
944 1.29 sommerfe __cpu_simple_unlock(&simplelock_list_slock.lock_data)
945 1.21 thorpej
946 1.21 thorpej #define SLOCK_COUNT(x) \
947 1.47 sommerfe curcpu()->ci_simple_locks += (x)
948 1.21 thorpej #else
949 1.21 thorpej u_long simple_locks;
950 1.21 thorpej
951 1.21 thorpej #define SLOCK_LIST_LOCK() /* nothing */
952 1.21 thorpej
953 1.21 thorpej #define SLOCK_LIST_UNLOCK() /* nothing */
954 1.21 thorpej
955 1.21 thorpej #define SLOCK_COUNT(x) simple_locks += (x)
956 1.21 thorpej #endif /* MULTIPROCESSOR */ /* } */
957 1.21 thorpej
958 1.21 thorpej #ifdef DDB /* { */
959 1.45 sommerfe #ifdef MULTIPROCESSOR
960 1.45 sommerfe int simple_lock_debugger = 1; /* more serious on MP */
961 1.45 sommerfe #else
962 1.18 chs int simple_lock_debugger = 0;
963 1.45 sommerfe #endif
964 1.21 thorpej #define SLOCK_DEBUGGER() if (simple_lock_debugger) Debugger()
965 1.21 thorpej #else
966 1.21 thorpej #define SLOCK_DEBUGGER() /* nothing */
967 1.21 thorpej #endif /* } */
968 1.21 thorpej
969 1.26 sommerfe #ifdef MULTIPROCESSOR
970 1.46 thorpej #define SLOCK_MP() lock_printf("on cpu %ld\n", \
971 1.46 thorpej (u_long) cpu_number())
972 1.26 sommerfe #else
973 1.26 sommerfe #define SLOCK_MP() /* nothing */
974 1.26 sommerfe #endif
975 1.26 sommerfe
976 1.21 thorpej #define SLOCK_WHERE(str, alp, id, l) \
977 1.21 thorpej do { \
978 1.25 thorpej lock_printf(str); \
979 1.33 thorpej lock_printf("lock: %p, currently at: %s:%d\n", (alp), (id), (l)); \
980 1.26 sommerfe SLOCK_MP(); \
981 1.21 thorpej if ((alp)->lock_file != NULL) \
982 1.25 thorpej lock_printf("last locked: %s:%d\n", (alp)->lock_file, \
983 1.21 thorpej (alp)->lock_line); \
984 1.21 thorpej if ((alp)->unlock_file != NULL) \
985 1.25 thorpej lock_printf("last unlocked: %s:%d\n", (alp)->unlock_file, \
986 1.21 thorpej (alp)->unlock_line); \
987 1.21 thorpej SLOCK_DEBUGGER(); \
988 1.30 thorpej } while (/*CONSTCOND*/0)
989 1.12 chs
990 1.1 fvdl /*
991 1.1 fvdl * Simple lock functions so that the debugger can see from whence
992 1.1 fvdl * they are being called.
993 1.1 fvdl */
994 1.1 fvdl void
995 1.33 thorpej simple_lock_init(struct simplelock *alp)
996 1.1 fvdl {
997 1.21 thorpej
998 1.21 thorpej #if defined(MULTIPROCESSOR) /* { */
999 1.27 thorpej __cpu_simple_lock_init(&alp->lock_data);
1000 1.21 thorpej #else
1001 1.27 thorpej alp->lock_data = __SIMPLELOCK_UNLOCKED;
1002 1.21 thorpej #endif /* } */
1003 1.5 chs alp->lock_file = NULL;
1004 1.5 chs alp->lock_line = 0;
1005 1.5 chs alp->unlock_file = NULL;
1006 1.5 chs alp->unlock_line = 0;
1007 1.41 thorpej alp->lock_holder = LK_NOCPU;
1008 1.1 fvdl }
1009 1.1 fvdl
1010 1.1 fvdl void
1011 1.33 thorpej _simple_lock(__volatile struct simplelock *alp, const char *id, int l)
1012 1.1 fvdl {
1013 1.24 thorpej cpuid_t cpu_id = cpu_number();
1014 1.12 chs int s;
1015 1.12 chs
1016 1.44 thorpej s = spllock();
1017 1.21 thorpej
1018 1.21 thorpej /*
1019 1.21 thorpej * MULTIPROCESSOR case: This is `safe' since if it's not us, we
1020 1.21 thorpej * don't take any action, and just fall into the normal spin case.
1021 1.21 thorpej */
1022 1.27 thorpej if (alp->lock_data == __SIMPLELOCK_LOCKED) {
1023 1.21 thorpej #if defined(MULTIPROCESSOR) /* { */
1024 1.21 thorpej if (alp->lock_holder == cpu_id) {
1025 1.21 thorpej SLOCK_WHERE("simple_lock: locking against myself\n",
1026 1.21 thorpej alp, id, l);
1027 1.21 thorpej goto out;
1028 1.1 fvdl }
1029 1.21 thorpej #else
1030 1.21 thorpej SLOCK_WHERE("simple_lock: lock held\n", alp, id, l);
1031 1.21 thorpej goto out;
1032 1.21 thorpej #endif /* MULTIPROCESSOR */ /* } */
1033 1.1 fvdl }
1034 1.21 thorpej
1035 1.21 thorpej #if defined(MULTIPROCESSOR) /* { */
1036 1.21 thorpej /* Acquire the lock before modifying any fields. */
1037 1.27 thorpej __cpu_simple_lock(&alp->lock_data);
1038 1.21 thorpej #else
1039 1.27 thorpej alp->lock_data = __SIMPLELOCK_LOCKED;
1040 1.21 thorpej #endif /* } */
1041 1.21 thorpej
1042 1.45 sommerfe if (alp->lock_holder != LK_NOCPU) {
1043 1.45 sommerfe SLOCK_WHERE("simple_lock: uninitialized lock\n",
1044 1.45 sommerfe alp, id, l);
1045 1.45 sommerfe }
1046 1.5 chs alp->lock_file = id;
1047 1.5 chs alp->lock_line = l;
1048 1.21 thorpej alp->lock_holder = cpu_id;
1049 1.21 thorpej
1050 1.21 thorpej SLOCK_LIST_LOCK();
1051 1.21 thorpej /* XXX Cast away volatile */
1052 1.21 thorpej TAILQ_INSERT_TAIL(&simplelock_list, (struct simplelock *)alp, list);
1053 1.21 thorpej SLOCK_LIST_UNLOCK();
1054 1.21 thorpej
1055 1.21 thorpej SLOCK_COUNT(1);
1056 1.21 thorpej
1057 1.21 thorpej out:
1058 1.18 chs splx(s);
1059 1.38 thorpej }
1060 1.38 thorpej
1061 1.38 thorpej int
1062 1.38 thorpej _simple_lock_held(__volatile struct simplelock *alp)
1063 1.38 thorpej {
1064 1.38 thorpej cpuid_t cpu_id = cpu_number();
1065 1.38 thorpej int s, locked = 0;
1066 1.38 thorpej
1067 1.44 thorpej s = spllock();
1068 1.42 thorpej
1069 1.42 thorpej #if defined(MULTIPROCESSOR)
1070 1.38 thorpej if (__cpu_simple_lock_try(&alp->lock_data) == 0)
1071 1.38 thorpej locked = (alp->lock_holder == cpu_id);
1072 1.38 thorpej else
1073 1.38 thorpej __cpu_simple_unlock(&alp->lock_data);
1074 1.38 thorpej #else
1075 1.42 thorpej if (alp->lock_data == __SIMPLELOCK_LOCKED) {
1076 1.42 thorpej locked = 1;
1077 1.42 thorpej KASSERT(alp->lock_holder == cpu_id);
1078 1.42 thorpej }
1079 1.42 thorpej #endif
1080 1.38 thorpej
1081 1.38 thorpej splx(s);
1082 1.42 thorpej
1083 1.38 thorpej return (locked);
1084 1.1 fvdl }
1085 1.1 fvdl
1086 1.1 fvdl int
1087 1.33 thorpej _simple_lock_try(__volatile struct simplelock *alp, const char *id, int l)
1088 1.1 fvdl {
1089 1.24 thorpej cpuid_t cpu_id = cpu_number();
1090 1.21 thorpej int s, rv = 0;
1091 1.1 fvdl
1092 1.44 thorpej s = spllock();
1093 1.21 thorpej
1094 1.21 thorpej /*
1095 1.21 thorpej * MULTIPROCESSOR case: This is `safe' since if it's not us, we
1096 1.21 thorpej * don't take any action.
1097 1.21 thorpej */
1098 1.21 thorpej #if defined(MULTIPROCESSOR) /* { */
1099 1.27 thorpej if ((rv = __cpu_simple_lock_try(&alp->lock_data)) == 0) {
1100 1.21 thorpej if (alp->lock_holder == cpu_id)
1101 1.21 thorpej SLOCK_WHERE("simple_lock_try: locking against myself\n",
1102 1.26 sommerfe alp, id, l);
1103 1.21 thorpej goto out;
1104 1.21 thorpej }
1105 1.21 thorpej #else
1106 1.27 thorpej if (alp->lock_data == __SIMPLELOCK_LOCKED) {
1107 1.21 thorpej SLOCK_WHERE("simple_lock_try: lock held\n", alp, id, l);
1108 1.21 thorpej goto out;
1109 1.18 chs }
1110 1.27 thorpej alp->lock_data = __SIMPLELOCK_LOCKED;
1111 1.21 thorpej #endif /* MULTIPROCESSOR */ /* } */
1112 1.21 thorpej
1113 1.21 thorpej /*
1114 1.21 thorpej * At this point, we have acquired the lock.
1115 1.21 thorpej */
1116 1.21 thorpej
1117 1.21 thorpej rv = 1;
1118 1.18 chs
1119 1.5 chs alp->lock_file = id;
1120 1.5 chs alp->lock_line = l;
1121 1.21 thorpej alp->lock_holder = cpu_id;
1122 1.21 thorpej
1123 1.21 thorpej SLOCK_LIST_LOCK();
1124 1.21 thorpej /* XXX Cast away volatile. */
1125 1.21 thorpej TAILQ_INSERT_TAIL(&simplelock_list, (struct simplelock *)alp, list);
1126 1.21 thorpej SLOCK_LIST_UNLOCK();
1127 1.21 thorpej
1128 1.21 thorpej SLOCK_COUNT(1);
1129 1.21 thorpej
1130 1.21 thorpej out:
1131 1.12 chs splx(s);
1132 1.21 thorpej return (rv);
1133 1.1 fvdl }
1134 1.1 fvdl
1135 1.1 fvdl void
1136 1.33 thorpej _simple_unlock(__volatile struct simplelock *alp, const char *id, int l)
1137 1.1 fvdl {
1138 1.12 chs int s;
1139 1.1 fvdl
1140 1.44 thorpej s = spllock();
1141 1.21 thorpej
1142 1.21 thorpej /*
1143 1.21 thorpej * MULTIPROCESSOR case: This is `safe' because we think we hold
1144 1.21 thorpej * the lock, and if we don't, we don't take any action.
1145 1.21 thorpej */
1146 1.27 thorpej if (alp->lock_data == __SIMPLELOCK_UNLOCKED) {
1147 1.21 thorpej SLOCK_WHERE("simple_unlock: lock not held\n",
1148 1.21 thorpej alp, id, l);
1149 1.21 thorpej goto out;
1150 1.21 thorpej }
1151 1.21 thorpej
1152 1.21 thorpej SLOCK_LIST_LOCK();
1153 1.21 thorpej TAILQ_REMOVE(&simplelock_list, alp, list);
1154 1.21 thorpej SLOCK_LIST_UNLOCK();
1155 1.21 thorpej
1156 1.21 thorpej SLOCK_COUNT(-1);
1157 1.21 thorpej
1158 1.21 thorpej alp->list.tqe_next = NULL; /* sanity */
1159 1.21 thorpej alp->list.tqe_prev = NULL; /* sanity */
1160 1.21 thorpej
1161 1.5 chs alp->unlock_file = id;
1162 1.5 chs alp->unlock_line = l;
1163 1.21 thorpej
1164 1.21 thorpej #if defined(MULTIPROCESSOR) /* { */
1165 1.26 sommerfe alp->lock_holder = LK_NOCPU;
1166 1.21 thorpej /* Now that we've modified all fields, release the lock. */
1167 1.27 thorpej __cpu_simple_unlock(&alp->lock_data);
1168 1.21 thorpej #else
1169 1.27 thorpej alp->lock_data = __SIMPLELOCK_UNLOCKED;
1170 1.41 thorpej KASSERT(alp->lock_holder == cpu_number());
1171 1.41 thorpej alp->lock_holder = LK_NOCPU;
1172 1.21 thorpej #endif /* } */
1173 1.21 thorpej
1174 1.21 thorpej out:
1175 1.18 chs splx(s);
1176 1.12 chs }
1177 1.12 chs
1178 1.12 chs void
1179 1.33 thorpej simple_lock_dump(void)
1180 1.12 chs {
1181 1.12 chs struct simplelock *alp;
1182 1.12 chs int s;
1183 1.12 chs
1184 1.44 thorpej s = spllock();
1185 1.21 thorpej SLOCK_LIST_LOCK();
1186 1.25 thorpej lock_printf("all simple locks:\n");
1187 1.21 thorpej for (alp = TAILQ_FIRST(&simplelock_list); alp != NULL;
1188 1.21 thorpej alp = TAILQ_NEXT(alp, list)) {
1189 1.25 thorpej lock_printf("%p CPU %lu %s:%d\n", alp, alp->lock_holder,
1190 1.21 thorpej alp->lock_file, alp->lock_line);
1191 1.12 chs }
1192 1.21 thorpej SLOCK_LIST_UNLOCK();
1193 1.12 chs splx(s);
1194 1.12 chs }
1195 1.12 chs
1196 1.12 chs void
1197 1.33 thorpej simple_lock_freecheck(void *start, void *end)
1198 1.12 chs {
1199 1.12 chs struct simplelock *alp;
1200 1.12 chs int s;
1201 1.12 chs
1202 1.44 thorpej s = spllock();
1203 1.21 thorpej SLOCK_LIST_LOCK();
1204 1.21 thorpej for (alp = TAILQ_FIRST(&simplelock_list); alp != NULL;
1205 1.21 thorpej alp = TAILQ_NEXT(alp, list)) {
1206 1.12 chs if ((void *)alp >= start && (void *)alp < end) {
1207 1.25 thorpej lock_printf("freeing simple_lock %p CPU %lu %s:%d\n",
1208 1.34 thorpej alp, alp->lock_holder, alp->lock_file,
1209 1.34 thorpej alp->lock_line);
1210 1.34 thorpej SLOCK_DEBUGGER();
1211 1.34 thorpej }
1212 1.34 thorpej }
1213 1.34 thorpej SLOCK_LIST_UNLOCK();
1214 1.34 thorpej splx(s);
1215 1.34 thorpej }
1216 1.34 thorpej
1217 1.34 thorpej void
1218 1.34 thorpej simple_lock_switchcheck(void)
1219 1.34 thorpej {
1220 1.34 thorpej struct simplelock *alp;
1221 1.34 thorpej cpuid_t cpu_id = cpu_number();
1222 1.34 thorpej int s;
1223 1.34 thorpej
1224 1.42 thorpej /*
1225 1.42 thorpej * We must be holding exactly one lock: the sched_lock.
1226 1.42 thorpej */
1227 1.42 thorpej
1228 1.42 thorpej SCHED_ASSERT_LOCKED();
1229 1.42 thorpej
1230 1.44 thorpej s = spllock();
1231 1.34 thorpej SLOCK_LIST_LOCK();
1232 1.34 thorpej for (alp = TAILQ_FIRST(&simplelock_list); alp != NULL;
1233 1.34 thorpej alp = TAILQ_NEXT(alp, list)) {
1234 1.42 thorpej if (alp == &sched_lock)
1235 1.42 thorpej continue;
1236 1.34 thorpej if (alp->lock_holder == cpu_id) {
1237 1.34 thorpej lock_printf("switching with held simple_lock %p "
1238 1.36 thorpej "CPU %lu %s:%d\n",
1239 1.21 thorpej alp, alp->lock_holder, alp->lock_file,
1240 1.21 thorpej alp->lock_line);
1241 1.21 thorpej SLOCK_DEBUGGER();
1242 1.12 chs }
1243 1.12 chs }
1244 1.21 thorpej SLOCK_LIST_UNLOCK();
1245 1.12 chs splx(s);
1246 1.1 fvdl }
1247 1.21 thorpej #endif /* LOCKDEBUG */ /* } */
1248