kern_lock.c revision 1.125 1 1.125 ad /* $NetBSD: kern_lock.c,v 1.125 2007/11/06 00:42:41 ad Exp $ */
2 1.19 thorpej
3 1.19 thorpej /*-
4 1.114 ad * Copyright (c) 1999, 2000, 2006, 2007 The NetBSD Foundation, Inc.
5 1.19 thorpej * All rights reserved.
6 1.19 thorpej *
7 1.19 thorpej * This code is derived from software contributed to The NetBSD Foundation
8 1.19 thorpej * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
9 1.105 ad * NASA Ames Research Center, and by Andrew Doran.
10 1.19 thorpej *
11 1.19 thorpej * This code is derived from software contributed to The NetBSD Foundation
12 1.19 thorpej * by Ross Harvey.
13 1.19 thorpej *
14 1.19 thorpej * Redistribution and use in source and binary forms, with or without
15 1.19 thorpej * modification, are permitted provided that the following conditions
16 1.19 thorpej * are met:
17 1.19 thorpej * 1. Redistributions of source code must retain the above copyright
18 1.19 thorpej * notice, this list of conditions and the following disclaimer.
19 1.19 thorpej * 2. Redistributions in binary form must reproduce the above copyright
20 1.19 thorpej * notice, this list of conditions and the following disclaimer in the
21 1.19 thorpej * documentation and/or other materials provided with the distribution.
22 1.19 thorpej * 3. All advertising materials mentioning features or use of this software
23 1.19 thorpej * must display the following acknowledgement:
24 1.19 thorpej * This product includes software developed by the NetBSD
25 1.19 thorpej * Foundation, Inc. and its contributors.
26 1.19 thorpej * 4. Neither the name of The NetBSD Foundation nor the names of its
27 1.19 thorpej * contributors may be used to endorse or promote products derived
28 1.19 thorpej * from this software without specific prior written permission.
29 1.19 thorpej *
30 1.19 thorpej * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
31 1.19 thorpej * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
32 1.19 thorpej * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
33 1.19 thorpej * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
34 1.19 thorpej * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
35 1.19 thorpej * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
36 1.19 thorpej * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
37 1.19 thorpej * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
38 1.19 thorpej * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
39 1.19 thorpej * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
40 1.19 thorpej * POSSIBILITY OF SUCH DAMAGE.
41 1.19 thorpej */
42 1.2 fvdl
43 1.86 perry /*
44 1.1 fvdl * Copyright (c) 1995
45 1.1 fvdl * The Regents of the University of California. All rights reserved.
46 1.1 fvdl *
47 1.1 fvdl * This code contains ideas from software contributed to Berkeley by
48 1.1 fvdl * Avadis Tevanian, Jr., Michael Wayne Young, and the Mach Operating
49 1.1 fvdl * System project at Carnegie-Mellon University.
50 1.1 fvdl *
51 1.1 fvdl * Redistribution and use in source and binary forms, with or without
52 1.1 fvdl * modification, are permitted provided that the following conditions
53 1.1 fvdl * are met:
54 1.1 fvdl * 1. Redistributions of source code must retain the above copyright
55 1.1 fvdl * notice, this list of conditions and the following disclaimer.
56 1.1 fvdl * 2. Redistributions in binary form must reproduce the above copyright
57 1.1 fvdl * notice, this list of conditions and the following disclaimer in the
58 1.1 fvdl * documentation and/or other materials provided with the distribution.
59 1.72 agc * 3. Neither the name of the University nor the names of its contributors
60 1.1 fvdl * may be used to endorse or promote products derived from this software
61 1.1 fvdl * without specific prior written permission.
62 1.1 fvdl *
63 1.1 fvdl * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
64 1.1 fvdl * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
65 1.1 fvdl * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
66 1.1 fvdl * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
67 1.1 fvdl * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
68 1.1 fvdl * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
69 1.1 fvdl * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
70 1.1 fvdl * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
71 1.1 fvdl * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
72 1.1 fvdl * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
73 1.1 fvdl * SUCH DAMAGE.
74 1.1 fvdl *
75 1.1 fvdl * @(#)kern_lock.c 8.18 (Berkeley) 5/21/95
76 1.1 fvdl */
77 1.60 lukem
78 1.60 lukem #include <sys/cdefs.h>
79 1.125 ad __KERNEL_RCSID(0, "$NetBSD: kern_lock.c,v 1.125 2007/11/06 00:42:41 ad Exp $");
80 1.7 thorpej
81 1.21 thorpej #include "opt_multiprocessor.h"
82 1.105 ad
83 1.1 fvdl #include <sys/param.h>
84 1.1 fvdl #include <sys/proc.h>
85 1.1 fvdl #include <sys/lock.h>
86 1.2 fvdl #include <sys/systm.h>
87 1.125 ad #include <sys/kernel.h>
88 1.105 ad #include <sys/lockdebug.h>
89 1.122 ad #include <sys/cpu.h>
90 1.122 ad #include <sys/syslog.h>
91 1.105 ad
92 1.110 christos #include <machine/stdarg.h>
93 1.1 fvdl
94 1.98 ad #include <dev/lockstat.h>
95 1.98 ad
96 1.25 thorpej /*
97 1.25 thorpej * note that stdarg.h and the ansi style va_start macro is used for both
98 1.25 thorpej * ansi and traditional c compiles.
99 1.25 thorpej * XXX: this requires that stdarg.h define: va_alist and va_dcl
100 1.25 thorpej */
101 1.36 thorpej void lock_printf(const char *fmt, ...)
102 1.37 eeh __attribute__((__format__(__printf__,1,2)));
103 1.25 thorpej
104 1.122 ad static int acquire(struct lock **, int *, int, int, int, uintptr_t);
105 1.73 yamt
106 1.57 sommerfe int lock_debug_syslog = 0; /* defaults to printf, but can be patched */
107 1.122 ad int kernel_lock_id;
108 1.122 ad __cpu_simple_lock_t kernel_lock;
109 1.1 fvdl
110 1.21 thorpej #if defined(LOCKDEBUG) || defined(DIAGNOSTIC) /* { */
111 1.122 ad #define COUNT(lkp, l, cpu_id, x) (l)->l_locks += (x)
112 1.1 fvdl #else
113 1.22 mellon #define COUNT(lkp, p, cpu_id, x)
114 1.21 thorpej #endif /* LOCKDEBUG || DIAGNOSTIC */ /* } */
115 1.1 fvdl
116 1.98 ad #define RETURN_ADDRESS ((uintptr_t)__builtin_return_address(0))
117 1.98 ad
118 1.1 fvdl /*
119 1.1 fvdl * Acquire a resource.
120 1.1 fvdl */
121 1.73 yamt static int
122 1.122 ad acquire(struct lock **lkpp, int *s, int extflags,
123 1.122 ad int drain, int wanted, uintptr_t ra)
124 1.73 yamt {
125 1.73 yamt int error;
126 1.122 ad struct lock *lkp = *lkpp;
127 1.98 ad LOCKSTAT_TIMER(slptime);
128 1.105 ad LOCKSTAT_FLAG(lsflag);
129 1.73 yamt
130 1.73 yamt KASSERT(drain || (wanted & LK_WAIT_NONZERO) == 0);
131 1.73 yamt
132 1.122 ad LOCKSTAT_ENTER(lsflag);
133 1.73 yamt
134 1.122 ad for (error = 0; (lkp->lk_flags & wanted) != 0; ) {
135 1.122 ad if (drain)
136 1.122 ad lkp->lk_flags |= LK_WAITDRAIN;
137 1.122 ad else {
138 1.73 yamt lkp->lk_waitcount++;
139 1.73 yamt lkp->lk_flags |= LK_WAIT_NONZERO;
140 1.73 yamt }
141 1.122 ad LOCKSTAT_START_TIMER(lsflag, slptime);
142 1.122 ad error = ltsleep(drain ? (void *)&lkp->lk_flags : (void *)lkp,
143 1.122 ad lkp->lk_prio, lkp->lk_wmesg, lkp->lk_timo,
144 1.122 ad &lkp->lk_interlock);
145 1.122 ad LOCKSTAT_STOP_TIMER(lsflag, slptime);
146 1.122 ad LOCKSTAT_EVENT_RA(lsflag, (void *)(uintptr_t)lkp,
147 1.122 ad LB_LOCKMGR | LB_SLEEP1, 1, slptime, ra);
148 1.73 yamt if (!drain) {
149 1.73 yamt lkp->lk_waitcount--;
150 1.73 yamt if (lkp->lk_waitcount == 0)
151 1.73 yamt lkp->lk_flags &= ~LK_WAIT_NONZERO;
152 1.73 yamt }
153 1.122 ad if (error)
154 1.122 ad break;
155 1.122 ad if (extflags & LK_SLEEPFAIL) {
156 1.122 ad error = ENOLCK;
157 1.122 ad break;
158 1.73 yamt }
159 1.122 ad }
160 1.105 ad
161 1.122 ad LOCKSTAT_EXIT(lsflag);
162 1.1 fvdl
163 1.73 yamt return error;
164 1.73 yamt }
165 1.73 yamt
166 1.69 thorpej #define SETHOLDER(lkp, pid, lid, cpu_id) \
167 1.19 thorpej do { \
168 1.122 ad (lkp)->lk_lockholder = pid; \
169 1.122 ad (lkp)->lk_locklwp = lid; \
170 1.30 thorpej } while (/*CONSTCOND*/0)
171 1.19 thorpej
172 1.69 thorpej #define WEHOLDIT(lkp, pid, lid, cpu_id) \
173 1.122 ad ((lkp)->lk_lockholder == (pid) && (lkp)->lk_locklwp == (lid))
174 1.19 thorpej
175 1.23 thorpej #define WAKEUP_WAITER(lkp) \
176 1.23 thorpej do { \
177 1.122 ad if (((lkp)->lk_flags & LK_WAIT_NONZERO) != 0) { \
178 1.87 christos wakeup((lkp)); \
179 1.23 thorpej } \
180 1.30 thorpej } while (/*CONSTCOND*/0)
181 1.23 thorpej
182 1.25 thorpej #if defined(LOCKDEBUG)
183 1.25 thorpej /*
184 1.25 thorpej * Lock debug printing routine; can be configured to print to console
185 1.25 thorpej * or log to syslog.
186 1.25 thorpej */
187 1.25 thorpej void
188 1.25 thorpej lock_printf(const char *fmt, ...)
189 1.25 thorpej {
190 1.68 pk char b[150];
191 1.25 thorpej va_list ap;
192 1.25 thorpej
193 1.25 thorpej va_start(ap, fmt);
194 1.25 thorpej if (lock_debug_syslog)
195 1.25 thorpej vlog(LOG_DEBUG, fmt, ap);
196 1.68 pk else {
197 1.68 pk vsnprintf(b, sizeof(b), fmt, ap);
198 1.68 pk printf_nolog("%s", b);
199 1.68 pk }
200 1.25 thorpej va_end(ap);
201 1.25 thorpej }
202 1.25 thorpej #endif /* LOCKDEBUG */
203 1.25 thorpej
204 1.110 christos static void
205 1.122 ad lockpanic(struct lock *lkp, const char *fmt, ...)
206 1.110 christos {
207 1.110 christos char s[150], b[150];
208 1.110 christos static const char *locktype[] = {
209 1.110 christos "*0*", "shared", "exclusive", "upgrade", "exclupgrade",
210 1.110 christos "downgrade", "release", "drain", "exclother", "*9*",
211 1.110 christos "*10*", "*11*", "*12*", "*13*", "*14*", "*15*"
212 1.110 christos };
213 1.110 christos va_list ap;
214 1.110 christos va_start(ap, fmt);
215 1.110 christos vsnprintf(s, sizeof(s), fmt, ap);
216 1.110 christos va_end(ap);
217 1.110 christos bitmask_snprintf(lkp->lk_flags, __LK_FLAG_BITS, b, sizeof(b));
218 1.110 christos panic("%s ("
219 1.122 ad "type %s flags %s, sharecount %d, exclusivecount %d, "
220 1.110 christos "recurselevel %d, waitcount %d, wmesg %s"
221 1.122 ad ", lock_addr %p, unlock_addr %p"
222 1.110 christos ")\n",
223 1.122 ad s, locktype[lkp->lk_flags & LK_TYPE_MASK],
224 1.110 christos b, lkp->lk_sharecount, lkp->lk_exclusivecount,
225 1.122 ad lkp->lk_recurselevel, lkp->lk_waitcount, lkp->lk_wmesg,
226 1.122 ad (void *)lkp->lk_lock_addr, (void *)lkp->lk_unlock_addr
227 1.110 christos );
228 1.110 christos }
229 1.110 christos
230 1.1 fvdl /*
231 1.1 fvdl * Initialize a lock; required before use.
232 1.1 fvdl */
233 1.1 fvdl void
234 1.109 yamt lockinit(struct lock *lkp, pri_t prio, const char *wmesg, int timo, int flags)
235 1.1 fvdl {
236 1.1 fvdl
237 1.8 perry memset(lkp, 0, sizeof(struct lock));
238 1.122 ad lkp->lk_flags = flags & LK_EXTFLG_MASK;
239 1.1 fvdl simple_lock_init(&lkp->lk_interlock);
240 1.122 ad lkp->lk_lockholder = LK_NOPROC;
241 1.122 ad lkp->lk_prio = prio;
242 1.122 ad lkp->lk_timo = timo;
243 1.122 ad lkp->lk_wmesg = wmesg;
244 1.122 ad lkp->lk_lock_addr = 0;
245 1.122 ad lkp->lk_unlock_addr = 0;
246 1.122 ad }
247 1.122 ad
248 1.122 ad void
249 1.122 ad lockdestroy(struct lock *lkp)
250 1.122 ad {
251 1.122 ad
252 1.122 ad /* nothing yet */
253 1.1 fvdl }
254 1.1 fvdl
255 1.1 fvdl /*
256 1.1 fvdl * Determine the status of a lock.
257 1.1 fvdl */
258 1.1 fvdl int
259 1.33 thorpej lockstatus(struct lock *lkp)
260 1.1 fvdl {
261 1.76 yamt int lock_type = 0;
262 1.76 yamt struct lwp *l = curlwp; /* XXX */
263 1.76 yamt pid_t pid;
264 1.76 yamt lwpid_t lid;
265 1.88 blymn cpuid_t cpu_num;
266 1.76 yamt
267 1.122 ad if (l == NULL) {
268 1.88 blymn cpu_num = cpu_number();
269 1.76 yamt pid = LK_KERNPROC;
270 1.76 yamt lid = 0;
271 1.76 yamt } else {
272 1.88 blymn cpu_num = LK_NOCPU;
273 1.76 yamt pid = l->l_proc->p_pid;
274 1.76 yamt lid = l->l_lid;
275 1.76 yamt }
276 1.1 fvdl
277 1.122 ad simple_lock(&lkp->lk_interlock);
278 1.76 yamt if (lkp->lk_exclusivecount != 0) {
279 1.88 blymn if (WEHOLDIT(lkp, pid, lid, cpu_num))
280 1.76 yamt lock_type = LK_EXCLUSIVE;
281 1.76 yamt else
282 1.76 yamt lock_type = LK_EXCLOTHER;
283 1.76 yamt } else if (lkp->lk_sharecount != 0)
284 1.1 fvdl lock_type = LK_SHARED;
285 1.103 chs else if (lkp->lk_flags & (LK_WANT_EXCL | LK_WANT_UPGRADE))
286 1.103 chs lock_type = LK_EXCLOTHER;
287 1.122 ad simple_unlock(&lkp->lk_interlock);
288 1.1 fvdl return (lock_type);
289 1.1 fvdl }
290 1.35 thorpej
291 1.44 thorpej /*
292 1.32 sommerfe * XXX XXX kludge around another kludge..
293 1.32 sommerfe *
294 1.32 sommerfe * vfs_shutdown() may be called from interrupt context, either as a result
295 1.32 sommerfe * of a panic, or from the debugger. It proceeds to call
296 1.32 sommerfe * sys_sync(&proc0, ...), pretending its running on behalf of proc0
297 1.32 sommerfe *
298 1.32 sommerfe * We would like to make an attempt to sync the filesystems in this case, so
299 1.32 sommerfe * if this happens, we treat attempts to acquire locks specially.
300 1.32 sommerfe * All locks are acquired on behalf of proc0.
301 1.32 sommerfe *
302 1.32 sommerfe * If we've already paniced, we don't block waiting for locks, but
303 1.32 sommerfe * just barge right ahead since we're already going down in flames.
304 1.32 sommerfe */
305 1.32 sommerfe
306 1.32 sommerfe /*
307 1.1 fvdl * Set, change, or release a lock.
308 1.1 fvdl *
309 1.1 fvdl * Shared requests increment the shared count. Exclusive requests set the
310 1.1 fvdl * LK_WANT_EXCL flag (preventing further shared locks), and wait for already
311 1.1 fvdl * accepted shared locks and shared-to-exclusive upgrades to go away.
312 1.1 fvdl */
313 1.1 fvdl int
314 1.122 ad lockmgr(struct lock *lkp, u_int flags, struct simplelock *interlkp)
315 1.1 fvdl {
316 1.1 fvdl int error;
317 1.1 fvdl pid_t pid;
318 1.69 thorpej lwpid_t lid;
319 1.1 fvdl int extflags;
320 1.88 blymn cpuid_t cpu_num;
321 1.69 thorpej struct lwp *l = curlwp;
322 1.32 sommerfe int lock_shutdown_noblock = 0;
323 1.67 scw int s = 0;
324 1.1 fvdl
325 1.1 fvdl error = 0;
326 1.19 thorpej
327 1.80 yamt /* LK_RETRY is for vn_lock, not for lockmgr. */
328 1.79 yamt KASSERT((flags & LK_RETRY) == 0);
329 1.125 ad KASSERT((l->l_pflag & LP_INTR) == 0 || panicstr != NULL);
330 1.79 yamt
331 1.122 ad simple_lock(&lkp->lk_interlock);
332 1.1 fvdl if (flags & LK_INTERLOCK)
333 1.1 fvdl simple_unlock(interlkp);
334 1.1 fvdl extflags = (flags | lkp->lk_flags) & LK_EXTFLG_MASK;
335 1.19 thorpej
336 1.122 ad if (l == NULL) {
337 1.122 ad if (!doing_shutdown) {
338 1.122 ad panic("lockmgr: no context");
339 1.122 ad } else {
340 1.122 ad l = &lwp0;
341 1.122 ad if (panicstr && (!(flags & LK_NOWAIT))) {
342 1.122 ad flags |= LK_NOWAIT;
343 1.122 ad lock_shutdown_noblock = 1;
344 1.32 sommerfe }
345 1.32 sommerfe }
346 1.19 thorpej }
347 1.122 ad lid = l->l_lid;
348 1.122 ad pid = l->l_proc->p_pid;
349 1.88 blymn cpu_num = cpu_number();
350 1.19 thorpej
351 1.1 fvdl /*
352 1.1 fvdl * Once a lock has drained, the LK_DRAINING flag is set and an
353 1.1 fvdl * exclusive lock is returned. The only valid operation thereafter
354 1.1 fvdl * is a single release of that exclusive lock. This final release
355 1.1 fvdl * clears the LK_DRAINING flag and sets the LK_DRAINED flag. Any
356 1.1 fvdl * further requests of any sort will result in a panic. The bits
357 1.1 fvdl * selected for these two flags are chosen so that they will be set
358 1.1 fvdl * in memory that is freed (freed memory is filled with 0xdeadbeef).
359 1.1 fvdl * The final release is permitted to give a new lease on life to
360 1.1 fvdl * the lock by specifying LK_REENABLE.
361 1.1 fvdl */
362 1.1 fvdl if (lkp->lk_flags & (LK_DRAINING|LK_DRAINED)) {
363 1.28 thorpej #ifdef DIAGNOSTIC /* { */
364 1.1 fvdl if (lkp->lk_flags & LK_DRAINED)
365 1.110 christos lockpanic(lkp, "lockmgr: using decommissioned lock");
366 1.1 fvdl if ((flags & LK_TYPE_MASK) != LK_RELEASE ||
367 1.88 blymn WEHOLDIT(lkp, pid, lid, cpu_num) == 0)
368 1.110 christos lockpanic(lkp, "lockmgr: non-release on draining lock: %d",
369 1.1 fvdl flags & LK_TYPE_MASK);
370 1.28 thorpej #endif /* DIAGNOSTIC */ /* } */
371 1.1 fvdl lkp->lk_flags &= ~LK_DRAINING;
372 1.1 fvdl if ((flags & LK_REENABLE) == 0)
373 1.1 fvdl lkp->lk_flags |= LK_DRAINED;
374 1.1 fvdl }
375 1.1 fvdl
376 1.1 fvdl switch (flags & LK_TYPE_MASK) {
377 1.1 fvdl
378 1.1 fvdl case LK_SHARED:
379 1.88 blymn if (WEHOLDIT(lkp, pid, lid, cpu_num) == 0) {
380 1.1 fvdl /*
381 1.1 fvdl * If just polling, check to see if we will block.
382 1.1 fvdl */
383 1.1 fvdl if ((extflags & LK_NOWAIT) && (lkp->lk_flags &
384 1.1 fvdl (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE))) {
385 1.1 fvdl error = EBUSY;
386 1.1 fvdl break;
387 1.1 fvdl }
388 1.1 fvdl /*
389 1.1 fvdl * Wait for exclusive locks and upgrades to clear.
390 1.1 fvdl */
391 1.78 hannken error = acquire(&lkp, &s, extflags, 0,
392 1.98 ad LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE,
393 1.98 ad RETURN_ADDRESS);
394 1.1 fvdl if (error)
395 1.1 fvdl break;
396 1.1 fvdl lkp->lk_sharecount++;
397 1.73 yamt lkp->lk_flags |= LK_SHARE_NONZERO;
398 1.88 blymn COUNT(lkp, l, cpu_num, 1);
399 1.1 fvdl break;
400 1.1 fvdl }
401 1.1 fvdl /*
402 1.1 fvdl * We hold an exclusive lock, so downgrade it to shared.
403 1.1 fvdl * An alternative would be to fail with EDEADLK.
404 1.1 fvdl */
405 1.1 fvdl lkp->lk_sharecount++;
406 1.73 yamt lkp->lk_flags |= LK_SHARE_NONZERO;
407 1.88 blymn COUNT(lkp, l, cpu_num, 1);
408 1.1 fvdl /* fall into downgrade */
409 1.1 fvdl
410 1.1 fvdl case LK_DOWNGRADE:
411 1.88 blymn if (WEHOLDIT(lkp, pid, lid, cpu_num) == 0 ||
412 1.19 thorpej lkp->lk_exclusivecount == 0)
413 1.110 christos lockpanic(lkp, "lockmgr: not holding exclusive lock");
414 1.1 fvdl lkp->lk_sharecount += lkp->lk_exclusivecount;
415 1.73 yamt lkp->lk_flags |= LK_SHARE_NONZERO;
416 1.1 fvdl lkp->lk_exclusivecount = 0;
417 1.15 fvdl lkp->lk_recurselevel = 0;
418 1.1 fvdl lkp->lk_flags &= ~LK_HAVE_EXCL;
419 1.69 thorpej SETHOLDER(lkp, LK_NOPROC, 0, LK_NOCPU);
420 1.50 thorpej #if defined(LOCKDEBUG)
421 1.122 ad lkp->lk_unlock_addr = RETURN_ADDRESS;
422 1.50 thorpej #endif
423 1.23 thorpej WAKEUP_WAITER(lkp);
424 1.1 fvdl break;
425 1.1 fvdl
426 1.1 fvdl case LK_EXCLUPGRADE:
427 1.1 fvdl /*
428 1.1 fvdl * If another process is ahead of us to get an upgrade,
429 1.1 fvdl * then we want to fail rather than have an intervening
430 1.1 fvdl * exclusive access.
431 1.1 fvdl */
432 1.1 fvdl if (lkp->lk_flags & LK_WANT_UPGRADE) {
433 1.1 fvdl lkp->lk_sharecount--;
434 1.73 yamt if (lkp->lk_sharecount == 0)
435 1.73 yamt lkp->lk_flags &= ~LK_SHARE_NONZERO;
436 1.88 blymn COUNT(lkp, l, cpu_num, -1);
437 1.1 fvdl error = EBUSY;
438 1.1 fvdl break;
439 1.1 fvdl }
440 1.1 fvdl /* fall into normal upgrade */
441 1.1 fvdl
442 1.1 fvdl case LK_UPGRADE:
443 1.1 fvdl /*
444 1.1 fvdl * Upgrade a shared lock to an exclusive one. If another
445 1.1 fvdl * shared lock has already requested an upgrade to an
446 1.1 fvdl * exclusive lock, our shared lock is released and an
447 1.1 fvdl * exclusive lock is requested (which will be granted
448 1.1 fvdl * after the upgrade). If we return an error, the file
449 1.1 fvdl * will always be unlocked.
450 1.1 fvdl */
451 1.88 blymn if (WEHOLDIT(lkp, pid, lid, cpu_num) || lkp->lk_sharecount <= 0)
452 1.110 christos lockpanic(lkp, "lockmgr: upgrade exclusive lock");
453 1.1 fvdl lkp->lk_sharecount--;
454 1.73 yamt if (lkp->lk_sharecount == 0)
455 1.73 yamt lkp->lk_flags &= ~LK_SHARE_NONZERO;
456 1.88 blymn COUNT(lkp, l, cpu_num, -1);
457 1.1 fvdl /*
458 1.1 fvdl * If we are just polling, check to see if we will block.
459 1.1 fvdl */
460 1.1 fvdl if ((extflags & LK_NOWAIT) &&
461 1.1 fvdl ((lkp->lk_flags & LK_WANT_UPGRADE) ||
462 1.1 fvdl lkp->lk_sharecount > 1)) {
463 1.1 fvdl error = EBUSY;
464 1.1 fvdl break;
465 1.1 fvdl }
466 1.1 fvdl if ((lkp->lk_flags & LK_WANT_UPGRADE) == 0) {
467 1.1 fvdl /*
468 1.1 fvdl * We are first shared lock to request an upgrade, so
469 1.1 fvdl * request upgrade and wait for the shared count to
470 1.1 fvdl * drop to zero, then take exclusive lock.
471 1.1 fvdl */
472 1.1 fvdl lkp->lk_flags |= LK_WANT_UPGRADE;
473 1.98 ad error = acquire(&lkp, &s, extflags, 0, LK_SHARE_NONZERO,
474 1.98 ad RETURN_ADDRESS);
475 1.1 fvdl lkp->lk_flags &= ~LK_WANT_UPGRADE;
476 1.83 yamt if (error) {
477 1.83 yamt WAKEUP_WAITER(lkp);
478 1.1 fvdl break;
479 1.83 yamt }
480 1.1 fvdl lkp->lk_flags |= LK_HAVE_EXCL;
481 1.88 blymn SETHOLDER(lkp, pid, lid, cpu_num);
482 1.50 thorpej #if defined(LOCKDEBUG)
483 1.122 ad lkp->lk_lock_addr = RETURN_ADDRESS;
484 1.50 thorpej #endif
485 1.1 fvdl if (lkp->lk_exclusivecount != 0)
486 1.110 christos lockpanic(lkp, "lockmgr: non-zero exclusive count");
487 1.1 fvdl lkp->lk_exclusivecount = 1;
488 1.15 fvdl if (extflags & LK_SETRECURSE)
489 1.15 fvdl lkp->lk_recurselevel = 1;
490 1.88 blymn COUNT(lkp, l, cpu_num, 1);
491 1.1 fvdl break;
492 1.1 fvdl }
493 1.1 fvdl /*
494 1.1 fvdl * Someone else has requested upgrade. Release our shared
495 1.1 fvdl * lock, awaken upgrade requestor if we are the last shared
496 1.1 fvdl * lock, then request an exclusive lock.
497 1.1 fvdl */
498 1.23 thorpej if (lkp->lk_sharecount == 0)
499 1.23 thorpej WAKEUP_WAITER(lkp);
500 1.1 fvdl /* fall into exclusive request */
501 1.1 fvdl
502 1.1 fvdl case LK_EXCLUSIVE:
503 1.88 blymn if (WEHOLDIT(lkp, pid, lid, cpu_num)) {
504 1.1 fvdl /*
505 1.19 thorpej * Recursive lock.
506 1.1 fvdl */
507 1.15 fvdl if ((extflags & LK_CANRECURSE) == 0 &&
508 1.16 sommerfe lkp->lk_recurselevel == 0) {
509 1.16 sommerfe if (extflags & LK_RECURSEFAIL) {
510 1.16 sommerfe error = EDEADLK;
511 1.16 sommerfe break;
512 1.16 sommerfe } else
513 1.110 christos lockpanic(lkp, "lockmgr: locking against myself");
514 1.16 sommerfe }
515 1.1 fvdl lkp->lk_exclusivecount++;
516 1.15 fvdl if (extflags & LK_SETRECURSE &&
517 1.15 fvdl lkp->lk_recurselevel == 0)
518 1.15 fvdl lkp->lk_recurselevel = lkp->lk_exclusivecount;
519 1.88 blymn COUNT(lkp, l, cpu_num, 1);
520 1.1 fvdl break;
521 1.1 fvdl }
522 1.1 fvdl /*
523 1.1 fvdl * If we are just polling, check to see if we will sleep.
524 1.1 fvdl */
525 1.73 yamt if ((extflags & LK_NOWAIT) && (lkp->lk_flags &
526 1.73 yamt (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE |
527 1.73 yamt LK_SHARE_NONZERO))) {
528 1.1 fvdl error = EBUSY;
529 1.1 fvdl break;
530 1.1 fvdl }
531 1.1 fvdl /*
532 1.1 fvdl * Try to acquire the want_exclusive flag.
533 1.1 fvdl */
534 1.82 yamt error = acquire(&lkp, &s, extflags, 0,
535 1.98 ad LK_HAVE_EXCL | LK_WANT_EXCL, RETURN_ADDRESS);
536 1.1 fvdl if (error)
537 1.1 fvdl break;
538 1.1 fvdl lkp->lk_flags |= LK_WANT_EXCL;
539 1.1 fvdl /*
540 1.1 fvdl * Wait for shared locks and upgrades to finish.
541 1.1 fvdl */
542 1.78 hannken error = acquire(&lkp, &s, extflags, 0,
543 1.98 ad LK_HAVE_EXCL | LK_WANT_UPGRADE | LK_SHARE_NONZERO,
544 1.98 ad RETURN_ADDRESS);
545 1.1 fvdl lkp->lk_flags &= ~LK_WANT_EXCL;
546 1.83 yamt if (error) {
547 1.83 yamt WAKEUP_WAITER(lkp);
548 1.1 fvdl break;
549 1.83 yamt }
550 1.1 fvdl lkp->lk_flags |= LK_HAVE_EXCL;
551 1.88 blymn SETHOLDER(lkp, pid, lid, cpu_num);
552 1.50 thorpej #if defined(LOCKDEBUG)
553 1.122 ad lkp->lk_lock_addr = RETURN_ADDRESS;
554 1.50 thorpej #endif
555 1.1 fvdl if (lkp->lk_exclusivecount != 0)
556 1.110 christos lockpanic(lkp, "lockmgr: non-zero exclusive count");
557 1.1 fvdl lkp->lk_exclusivecount = 1;
558 1.15 fvdl if (extflags & LK_SETRECURSE)
559 1.15 fvdl lkp->lk_recurselevel = 1;
560 1.88 blymn COUNT(lkp, l, cpu_num, 1);
561 1.1 fvdl break;
562 1.1 fvdl
563 1.1 fvdl case LK_RELEASE:
564 1.1 fvdl if (lkp->lk_exclusivecount != 0) {
565 1.88 blymn if (WEHOLDIT(lkp, pid, lid, cpu_num) == 0) {
566 1.122 ad lockpanic(lkp, "lockmgr: pid %d.%d, not "
567 1.122 ad "exclusive lock holder %d.%d "
568 1.122 ad "unlocking", pid, lid,
569 1.122 ad lkp->lk_lockholder,
570 1.122 ad lkp->lk_locklwp);
571 1.19 thorpej }
572 1.15 fvdl if (lkp->lk_exclusivecount == lkp->lk_recurselevel)
573 1.15 fvdl lkp->lk_recurselevel = 0;
574 1.1 fvdl lkp->lk_exclusivecount--;
575 1.88 blymn COUNT(lkp, l, cpu_num, -1);
576 1.1 fvdl if (lkp->lk_exclusivecount == 0) {
577 1.1 fvdl lkp->lk_flags &= ~LK_HAVE_EXCL;
578 1.69 thorpej SETHOLDER(lkp, LK_NOPROC, 0, LK_NOCPU);
579 1.50 thorpej #if defined(LOCKDEBUG)
580 1.122 ad lkp->lk_unlock_addr = RETURN_ADDRESS;
581 1.50 thorpej #endif
582 1.1 fvdl }
583 1.1 fvdl } else if (lkp->lk_sharecount != 0) {
584 1.1 fvdl lkp->lk_sharecount--;
585 1.73 yamt if (lkp->lk_sharecount == 0)
586 1.73 yamt lkp->lk_flags &= ~LK_SHARE_NONZERO;
587 1.88 blymn COUNT(lkp, l, cpu_num, -1);
588 1.1 fvdl }
589 1.39 thorpej #ifdef DIAGNOSTIC
590 1.39 thorpej else
591 1.110 christos lockpanic(lkp, "lockmgr: release of unlocked lock!");
592 1.39 thorpej #endif
593 1.23 thorpej WAKEUP_WAITER(lkp);
594 1.1 fvdl break;
595 1.1 fvdl
596 1.1 fvdl case LK_DRAIN:
597 1.1 fvdl /*
598 1.86 perry * Check that we do not already hold the lock, as it can
599 1.1 fvdl * never drain if we do. Unfortunately, we have no way to
600 1.1 fvdl * check for holding a shared lock, but at least we can
601 1.1 fvdl * check for an exclusive one.
602 1.1 fvdl */
603 1.88 blymn if (WEHOLDIT(lkp, pid, lid, cpu_num))
604 1.110 christos lockpanic(lkp, "lockmgr: draining against myself");
605 1.1 fvdl /*
606 1.1 fvdl * If we are just polling, check to see if we will sleep.
607 1.1 fvdl */
608 1.73 yamt if ((extflags & LK_NOWAIT) && (lkp->lk_flags &
609 1.73 yamt (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE |
610 1.73 yamt LK_SHARE_NONZERO | LK_WAIT_NONZERO))) {
611 1.1 fvdl error = EBUSY;
612 1.1 fvdl break;
613 1.1 fvdl }
614 1.78 hannken error = acquire(&lkp, &s, extflags, 1,
615 1.73 yamt LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE |
616 1.98 ad LK_SHARE_NONZERO | LK_WAIT_NONZERO,
617 1.98 ad RETURN_ADDRESS);
618 1.23 thorpej if (error)
619 1.23 thorpej break;
620 1.118 pooka lkp->lk_flags |= LK_HAVE_EXCL;
621 1.118 pooka if ((extflags & LK_RESURRECT) == 0)
622 1.118 pooka lkp->lk_flags |= LK_DRAINING;
623 1.88 blymn SETHOLDER(lkp, pid, lid, cpu_num);
624 1.50 thorpej #if defined(LOCKDEBUG)
625 1.122 ad lkp->lk_lock_addr = RETURN_ADDRESS;
626 1.50 thorpej #endif
627 1.1 fvdl lkp->lk_exclusivecount = 1;
628 1.15 fvdl /* XXX unlikely that we'd want this */
629 1.15 fvdl if (extflags & LK_SETRECURSE)
630 1.15 fvdl lkp->lk_recurselevel = 1;
631 1.88 blymn COUNT(lkp, l, cpu_num, 1);
632 1.1 fvdl break;
633 1.1 fvdl
634 1.1 fvdl default:
635 1.122 ad simple_unlock(&lkp->lk_interlock);
636 1.110 christos lockpanic(lkp, "lockmgr: unknown locktype request %d",
637 1.1 fvdl flags & LK_TYPE_MASK);
638 1.1 fvdl /* NOTREACHED */
639 1.1 fvdl }
640 1.122 ad if ((lkp->lk_flags & LK_WAITDRAIN) != 0 &&
641 1.23 thorpej ((lkp->lk_flags &
642 1.73 yamt (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE |
643 1.73 yamt LK_SHARE_NONZERO | LK_WAIT_NONZERO)) == 0)) {
644 1.1 fvdl lkp->lk_flags &= ~LK_WAITDRAIN;
645 1.87 christos wakeup(&lkp->lk_flags);
646 1.1 fvdl }
647 1.32 sommerfe /*
648 1.32 sommerfe * Note that this panic will be a recursive panic, since
649 1.32 sommerfe * we only set lock_shutdown_noblock above if panicstr != NULL.
650 1.32 sommerfe */
651 1.32 sommerfe if (error && lock_shutdown_noblock)
652 1.110 christos lockpanic(lkp, "lockmgr: deadlock (see previous panic)");
653 1.86 perry
654 1.122 ad simple_unlock(&lkp->lk_interlock);
655 1.1 fvdl return (error);
656 1.1 fvdl }
657 1.1 fvdl
658 1.1 fvdl /*
659 1.1 fvdl * Print out information about state of a lock. Used by VOP_PRINT
660 1.1 fvdl * routines to display ststus about contained locks.
661 1.1 fvdl */
662 1.2 fvdl void
663 1.122 ad lockmgr_printinfo(struct lock *lkp)
664 1.1 fvdl {
665 1.1 fvdl
666 1.1 fvdl if (lkp->lk_sharecount)
667 1.1 fvdl printf(" lock type %s: SHARED (count %d)", lkp->lk_wmesg,
668 1.1 fvdl lkp->lk_sharecount);
669 1.19 thorpej else if (lkp->lk_flags & LK_HAVE_EXCL) {
670 1.19 thorpej printf(" lock type %s: EXCL (count %d) by ",
671 1.19 thorpej lkp->lk_wmesg, lkp->lk_exclusivecount);
672 1.122 ad printf("pid %d.%d", lkp->lk_lockholder,
673 1.122 ad lkp->lk_locklwp);
674 1.19 thorpej } else
675 1.19 thorpej printf(" not locked");
676 1.122 ad if (lkp->lk_waitcount > 0)
677 1.1 fvdl printf(" with %d pending", lkp->lk_waitcount);
678 1.1 fvdl }
679 1.1 fvdl
680 1.122 ad #if defined(LOCKDEBUG)
681 1.96 yamt void
682 1.96 yamt assert_sleepable(struct simplelock *interlock, const char *msg)
683 1.96 yamt {
684 1.96 yamt
685 1.117 ad if (panicstr != NULL)
686 1.117 ad return;
687 1.122 ad LOCKDEBUG_BARRIER(&kernel_lock, 1);
688 1.125 ad if (CURCPU_IDLE_P() && !cold) {
689 1.113 yamt panic("assert_sleepable: idle");
690 1.97 yamt }
691 1.96 yamt }
692 1.122 ad #endif
693 1.105 ad
694 1.62 thorpej /*
695 1.124 pooka * rump doesn't need the kernel lock so force it out. We cannot
696 1.124 pooka * currently easily include it for compilation because of
697 1.124 pooka * a) SPINLOCK_* b) mb_write(). They are defined in different
698 1.124 pooka * places / way for each arch, so just simply do not bother to
699 1.124 pooka * fight a lot for no gain (i.e. pain but still no gain).
700 1.124 pooka */
701 1.124 pooka #ifndef _RUMPKERNEL
702 1.124 pooka /*
703 1.62 thorpej * Functions for manipulating the kernel_lock. We put them here
704 1.62 thorpej * so that they show up in profiles.
705 1.62 thorpej */
706 1.62 thorpej
707 1.105 ad #define _KERNEL_LOCK_ABORT(msg) \
708 1.105 ad LOCKDEBUG_ABORT(kernel_lock_id, &kernel_lock, &_kernel_lock_ops, \
709 1.120 ad __func__, msg)
710 1.105 ad
711 1.105 ad #ifdef LOCKDEBUG
712 1.105 ad #define _KERNEL_LOCK_ASSERT(cond) \
713 1.105 ad do { \
714 1.105 ad if (!(cond)) \
715 1.105 ad _KERNEL_LOCK_ABORT("assertion failed: " #cond); \
716 1.105 ad } while (/* CONSTCOND */ 0)
717 1.105 ad #else
718 1.105 ad #define _KERNEL_LOCK_ASSERT(cond) /* nothing */
719 1.105 ad #endif
720 1.105 ad
721 1.105 ad void _kernel_lock_dump(volatile void *);
722 1.105 ad
723 1.105 ad lockops_t _kernel_lock_ops = {
724 1.105 ad "Kernel lock",
725 1.105 ad 0,
726 1.105 ad _kernel_lock_dump
727 1.105 ad };
728 1.105 ad
729 1.85 yamt /*
730 1.105 ad * Initialize the kernel lock.
731 1.85 yamt */
732 1.62 thorpej void
733 1.122 ad kernel_lock_init(void)
734 1.62 thorpej {
735 1.62 thorpej
736 1.105 ad __cpu_simple_lock_init(&kernel_lock);
737 1.122 ad kernel_lock_id = LOCKDEBUG_ALLOC(&kernel_lock, &_kernel_lock_ops,
738 1.122 ad RETURN_ADDRESS);
739 1.62 thorpej }
740 1.62 thorpej
741 1.62 thorpej /*
742 1.105 ad * Print debugging information about the kernel lock.
743 1.62 thorpej */
744 1.62 thorpej void
745 1.105 ad _kernel_lock_dump(volatile void *junk)
746 1.62 thorpej {
747 1.85 yamt struct cpu_info *ci = curcpu();
748 1.62 thorpej
749 1.105 ad (void)junk;
750 1.85 yamt
751 1.105 ad printf_nolog("curcpu holds : %18d wanted by: %#018lx\n",
752 1.105 ad ci->ci_biglock_count, (long)ci->ci_biglock_wanted);
753 1.62 thorpej }
754 1.62 thorpej
755 1.105 ad /*
756 1.105 ad * Acquire 'nlocks' holds on the kernel lock. If 'l' is non-null, the
757 1.105 ad * acquisition is from process context.
758 1.105 ad */
759 1.62 thorpej void
760 1.105 ad _kernel_lock(int nlocks, struct lwp *l)
761 1.62 thorpej {
762 1.85 yamt struct cpu_info *ci = curcpu();
763 1.105 ad LOCKSTAT_TIMER(spintime);
764 1.105 ad LOCKSTAT_FLAG(lsflag);
765 1.105 ad struct lwp *owant;
766 1.105 ad #ifdef LOCKDEBUG
767 1.105 ad u_int spins;
768 1.105 ad #endif
769 1.85 yamt int s;
770 1.85 yamt
771 1.105 ad if (nlocks == 0)
772 1.105 ad return;
773 1.105 ad _KERNEL_LOCK_ASSERT(nlocks > 0);
774 1.62 thorpej
775 1.122 ad l = curlwp;
776 1.105 ad
777 1.105 ad if (ci->ci_biglock_count != 0) {
778 1.119 skrll _KERNEL_LOCK_ASSERT(__SIMPLELOCK_LOCKED_P(&kernel_lock));
779 1.105 ad ci->ci_biglock_count += nlocks;
780 1.122 ad l->l_blcnt += nlocks;
781 1.105 ad return;
782 1.105 ad }
783 1.105 ad
784 1.122 ad _KERNEL_LOCK_ASSERT(l->l_blcnt == 0);
785 1.122 ad LOCKDEBUG_WANTLOCK(kernel_lock_id, RETURN_ADDRESS, 0);
786 1.107 ad
787 1.122 ad s = splvm();
788 1.105 ad if (__cpu_simple_lock_try(&kernel_lock)) {
789 1.105 ad ci->ci_biglock_count = nlocks;
790 1.122 ad l->l_blcnt = nlocks;
791 1.122 ad LOCKDEBUG_LOCKED(kernel_lock_id, RETURN_ADDRESS, 0);
792 1.105 ad splx(s);
793 1.105 ad return;
794 1.105 ad }
795 1.105 ad
796 1.105 ad LOCKSTAT_ENTER(lsflag);
797 1.105 ad LOCKSTAT_START_TIMER(lsflag, spintime);
798 1.105 ad
799 1.105 ad /*
800 1.105 ad * Before setting ci_biglock_wanted we must post a store
801 1.105 ad * fence (see kern_mutex.c). This is accomplished by the
802 1.105 ad * __cpu_simple_lock_try() above.
803 1.105 ad */
804 1.105 ad owant = ci->ci_biglock_wanted;
805 1.105 ad ci->ci_biglock_wanted = curlwp; /* XXXAD */
806 1.105 ad
807 1.105 ad #ifdef LOCKDEBUG
808 1.105 ad spins = 0;
809 1.105 ad #endif
810 1.105 ad
811 1.105 ad do {
812 1.122 ad splx(s);
813 1.123 ad while (__SIMPLELOCK_LOCKED_P(&kernel_lock)) {
814 1.105 ad #ifdef LOCKDEBUG
815 1.105 ad if (SPINLOCK_SPINOUT(spins))
816 1.105 ad _KERNEL_LOCK_ABORT("spinout");
817 1.105 ad #endif
818 1.122 ad SPINLOCK_BACKOFF_HOOK;
819 1.105 ad SPINLOCK_SPIN_HOOK;
820 1.105 ad }
821 1.122 ad (void)splvm();
822 1.105 ad } while (!__cpu_simple_lock_try(&kernel_lock));
823 1.105 ad
824 1.105 ad ci->ci_biglock_wanted = owant;
825 1.122 ad ci->ci_biglock_count = nlocks;
826 1.122 ad l->l_blcnt = nlocks;
827 1.107 ad LOCKSTAT_STOP_TIMER(lsflag, spintime);
828 1.122 ad LOCKDEBUG_LOCKED(kernel_lock_id, RETURN_ADDRESS, 0);
829 1.85 yamt splx(s);
830 1.105 ad
831 1.105 ad /*
832 1.105 ad * Again, another store fence is required (see kern_mutex.c).
833 1.105 ad */
834 1.105 ad mb_write();
835 1.107 ad if (owant == NULL) {
836 1.107 ad LOCKSTAT_EVENT(lsflag, &kernel_lock, LB_KERNEL_LOCK | LB_SPIN,
837 1.107 ad 1, spintime);
838 1.107 ad }
839 1.105 ad LOCKSTAT_EXIT(lsflag);
840 1.62 thorpej }
841 1.62 thorpej
842 1.62 thorpej /*
843 1.105 ad * Release 'nlocks' holds on the kernel lock. If 'nlocks' is zero, release
844 1.105 ad * all holds. If 'l' is non-null, the release is from process context.
845 1.62 thorpej */
846 1.62 thorpej void
847 1.105 ad _kernel_unlock(int nlocks, struct lwp *l, int *countp)
848 1.62 thorpej {
849 1.105 ad struct cpu_info *ci = curcpu();
850 1.105 ad u_int olocks;
851 1.105 ad int s;
852 1.62 thorpej
853 1.122 ad l = curlwp;
854 1.62 thorpej
855 1.105 ad _KERNEL_LOCK_ASSERT(nlocks < 2);
856 1.62 thorpej
857 1.122 ad olocks = l->l_blcnt;
858 1.77 yamt
859 1.105 ad if (olocks == 0) {
860 1.105 ad _KERNEL_LOCK_ASSERT(nlocks <= 0);
861 1.105 ad if (countp != NULL)
862 1.105 ad *countp = 0;
863 1.105 ad return;
864 1.105 ad }
865 1.77 yamt
866 1.119 skrll _KERNEL_LOCK_ASSERT(__SIMPLELOCK_LOCKED_P(&kernel_lock));
867 1.85 yamt
868 1.105 ad if (nlocks == 0)
869 1.105 ad nlocks = olocks;
870 1.105 ad else if (nlocks == -1) {
871 1.105 ad nlocks = 1;
872 1.105 ad _KERNEL_LOCK_ASSERT(olocks == 1);
873 1.105 ad }
874 1.85 yamt
875 1.122 ad _KERNEL_LOCK_ASSERT(ci->ci_biglock_count >= l->l_blcnt);
876 1.122 ad
877 1.122 ad l->l_blcnt -= nlocks;
878 1.122 ad if (ci->ci_biglock_count == nlocks) {
879 1.122 ad s = splvm();
880 1.122 ad LOCKDEBUG_UNLOCKED(kernel_lock_id, RETURN_ADDRESS, 0);
881 1.122 ad ci->ci_biglock_count = 0;
882 1.105 ad __cpu_simple_unlock(&kernel_lock);
883 1.122 ad splx(s);
884 1.122 ad } else
885 1.122 ad ci->ci_biglock_count -= nlocks;
886 1.77 yamt
887 1.105 ad if (countp != NULL)
888 1.105 ad *countp = olocks;
889 1.77 yamt }
890 1.77 yamt
891 1.84 yamt #if defined(DEBUG)
892 1.105 ad /*
893 1.105 ad * Assert that the kernel lock is held.
894 1.105 ad */
895 1.84 yamt void
896 1.105 ad _kernel_lock_assert_locked(void)
897 1.84 yamt {
898 1.100 yamt
899 1.119 skrll if (!__SIMPLELOCK_LOCKED_P(&kernel_lock) ||
900 1.105 ad curcpu()->ci_biglock_count == 0)
901 1.105 ad _KERNEL_LOCK_ABORT("not locked");
902 1.84 yamt }
903 1.100 yamt
904 1.100 yamt void
905 1.100 yamt _kernel_lock_assert_unlocked()
906 1.100 yamt {
907 1.100 yamt
908 1.105 ad if (curcpu()->ci_biglock_count != 0)
909 1.105 ad _KERNEL_LOCK_ABORT("locked");
910 1.100 yamt }
911 1.84 yamt #endif
912 1.124 pooka #endif /* !_RUMPKERNEL */
913