kern_lock.c revision 1.32.2.2
1 /* $NetBSD: kern_lock.c,v 1.32.2.2 2000/08/11 23:10:15 thorpej Exp $ */ 2 3 /*- 4 * Copyright (c) 1999, 2000 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center. 10 * 11 * This code is derived from software contributed to The NetBSD Foundation 12 * by Ross Harvey. 13 * 14 * Redistribution and use in source and binary forms, with or without 15 * modification, are permitted provided that the following conditions 16 * are met: 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in the 21 * documentation and/or other materials provided with the distribution. 22 * 3. All advertising materials mentioning features or use of this software 23 * must display the following acknowledgement: 24 * This product includes software developed by the NetBSD 25 * Foundation, Inc. and its contributors. 26 * 4. Neither the name of The NetBSD Foundation nor the names of its 27 * contributors may be used to endorse or promote products derived 28 * from this software without specific prior written permission. 29 * 30 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 31 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 32 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 33 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 34 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 35 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 36 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 37 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 38 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 39 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 40 * POSSIBILITY OF SUCH DAMAGE. 41 */ 42 43 /* 44 * Copyright (c) 1995 45 * The Regents of the University of California. All rights reserved. 46 * 47 * This code contains ideas from software contributed to Berkeley by 48 * Avadis Tevanian, Jr., Michael Wayne Young, and the Mach Operating 49 * System project at Carnegie-Mellon University. 50 * 51 * Redistribution and use in source and binary forms, with or without 52 * modification, are permitted provided that the following conditions 53 * are met: 54 * 1. Redistributions of source code must retain the above copyright 55 * notice, this list of conditions and the following disclaimer. 56 * 2. Redistributions in binary form must reproduce the above copyright 57 * notice, this list of conditions and the following disclaimer in the 58 * documentation and/or other materials provided with the distribution. 59 * 3. All advertising materials mentioning features or use of this software 60 * must display the following acknowledgement: 61 * This product includes software developed by the University of 62 * California, Berkeley and its contributors. 63 * 4. Neither the name of the University nor the names of its contributors 64 * may be used to endorse or promote products derived from this software 65 * without specific prior written permission. 66 * 67 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 68 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 69 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 70 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 71 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 72 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 73 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 74 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 75 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 76 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 77 * SUCH DAMAGE. 78 * 79 * @(#)kern_lock.c 8.18 (Berkeley) 5/21/95 80 */ 81 82 #include "opt_multiprocessor.h" 83 #include "opt_lockdebug.h" 84 #include "opt_ddb.h" 85 86 #include <sys/param.h> 87 #include <sys/proc.h> 88 #include <sys/lock.h> 89 #include <sys/systm.h> 90 #include <machine/cpu.h> 91 92 #if defined(__HAVE_ATOMIC_OPERATIONS) 93 #include <machine/atomic.h> 94 #endif 95 96 #if defined(LOCKDEBUG) 97 #include <sys/syslog.h> 98 /* 99 * note that stdarg.h and the ansi style va_start macro is used for both 100 * ansi and traditional c compiles. 101 * XXX: this requires that stdarg.h define: va_alist and va_dcl 102 */ 103 #include <machine/stdarg.h> 104 105 void lock_printf __P((const char *fmt, ...)); 106 107 int lock_debug_syslog = 0; /* defaults to printf, but can be patched */ 108 #endif 109 110 /* 111 * Locking primitives implementation. 112 * Locks provide shared/exclusive sychronization. 113 */ 114 115 #if defined(LOCKDEBUG) || defined(DIAGNOSTIC) /* { */ 116 #if defined(MULTIPROCESSOR) /* { */ 117 #if defined(__HAVE_ATOMIC_OPERATIONS) /* { */ 118 #define COUNT_CPU(cpu_id, x) \ 119 atomic_add_ulong(&curcpu()->ci_spin_locks, (x)) 120 #else 121 #define COUNT_CPU(cpu_id, x) /* not safe */ 122 #endif /* __HAVE_ATOMIC_OPERATIONS */ /* } */ 123 #else 124 u_long spin_locks; 125 #define COUNT_CPU(cpu_id, x) spin_locks += (x) 126 #endif /* MULTIPROCESSOR */ /* } */ 127 128 #define COUNT(lkp, p, cpu_id, x) \ 129 do { \ 130 if ((lkp)->lk_flags & LK_SPIN) \ 131 COUNT_CPU((cpu_id), (x)); \ 132 else \ 133 (p)->p_locks += (x); \ 134 } while (/*CONSTCOND*/0) 135 #else 136 #define COUNT(lkp, p, cpu_id, x) 137 #endif /* LOCKDEBUG || DIAGNOSTIC */ /* } */ 138 139 /* 140 * Acquire a resource. 141 */ 142 #define ACQUIRE(lkp, error, extflags, drain, wanted) \ 143 if ((extflags) & LK_SPIN) { \ 144 int interlocked; \ 145 \ 146 if ((drain) == 0) \ 147 (lkp)->lk_waitcount++; \ 148 for (interlocked = 1;;) { \ 149 if (wanted) { \ 150 if (interlocked) { \ 151 simple_unlock(&(lkp)->lk_interlock); \ 152 interlocked = 0; \ 153 } \ 154 } else if (interlocked) { \ 155 break; \ 156 } else { \ 157 simple_lock(&(lkp)->lk_interlock); \ 158 interlocked = 1; \ 159 } \ 160 } \ 161 if ((drain) == 0) \ 162 (lkp)->lk_waitcount--; \ 163 KASSERT((wanted) == 0); \ 164 error = 0; /* sanity */ \ 165 } else { \ 166 for (error = 0; wanted; ) { \ 167 if ((drain)) \ 168 (lkp)->lk_flags |= LK_WAITDRAIN; \ 169 else \ 170 (lkp)->lk_waitcount++; \ 171 /* XXX Cast away volatile. */ \ 172 error = ltsleep((drain) ? &(lkp)->lk_flags : \ 173 (void *)(lkp), (lkp)->lk_prio, \ 174 (lkp)->lk_wmesg, (lkp)->lk_timo, \ 175 &(lkp)->lk_interlock); \ 176 if ((drain) == 0) \ 177 (lkp)->lk_waitcount--; \ 178 if (error) \ 179 break; \ 180 if ((extflags) & LK_SLEEPFAIL) { \ 181 error = ENOLCK; \ 182 break; \ 183 } \ 184 } \ 185 } 186 187 #define SETHOLDER(lkp, pid, cpu_id) \ 188 do { \ 189 if ((lkp)->lk_flags & LK_SPIN) \ 190 (lkp)->lk_cpu = cpu_id; \ 191 else \ 192 (lkp)->lk_lockholder = pid; \ 193 } while (/*CONSTCOND*/0) 194 195 #define WEHOLDIT(lkp, pid, cpu_id) \ 196 (((lkp)->lk_flags & LK_SPIN) != 0 ? \ 197 ((lkp)->lk_cpu == (cpu_id)) : ((lkp)->lk_lockholder == (pid))) 198 199 #define WAKEUP_WAITER(lkp) \ 200 do { \ 201 if (((lkp)->lk_flags & LK_SPIN) == 0 && (lkp)->lk_waitcount) { \ 202 /* XXX Cast away volatile. */ \ 203 wakeup_one((void *)(lkp)); \ 204 } \ 205 } while (/*CONSTCOND*/0) 206 207 #if defined(LOCKDEBUG) /* { */ 208 #if defined(MULTIPROCESSOR) /* { */ 209 struct simplelock spinlock_list_slock = SIMPLELOCK_INITIALIZER; 210 211 #define SPINLOCK_LIST_LOCK() \ 212 __cpu_simple_lock(&spinlock_list_slock.lock_data) 213 214 #define SPINLOCK_LIST_UNLOCK() \ 215 __cpu_simple_unlock(&spinlock_list_slock.lock_data) 216 #else 217 #define SPINLOCK_LIST_LOCK() /* nothing */ 218 219 #define SPINLOCK_LIST_UNLOCK() /* nothing */ 220 #endif /* MULTIPROCESSOR */ /* } */ 221 222 TAILQ_HEAD(, lock) spinlock_list = 223 TAILQ_HEAD_INITIALIZER(spinlock_list); 224 225 #define HAVEIT(lkp) \ 226 do { \ 227 if ((lkp)->lk_flags & LK_SPIN) { \ 228 int s = splhigh(); \ 229 SPINLOCK_LIST_LOCK(); \ 230 /* XXX Cast away volatile. */ \ 231 TAILQ_INSERT_TAIL(&spinlock_list, (struct lock *)(lkp), \ 232 lk_list); \ 233 SPINLOCK_LIST_UNLOCK(); \ 234 splx(s); \ 235 } \ 236 } while (/*CONSTCOND*/0) 237 238 #define DONTHAVEIT(lkp) \ 239 do { \ 240 if ((lkp)->lk_flags & LK_SPIN) { \ 241 int s = splhigh(); \ 242 SPINLOCK_LIST_LOCK(); \ 243 /* XXX Cast away volatile. */ \ 244 TAILQ_REMOVE(&spinlock_list, (struct lock *)(lkp), \ 245 lk_list); \ 246 SPINLOCK_LIST_UNLOCK(); \ 247 splx(s); \ 248 } \ 249 } while (/*CONSTCOND*/0) 250 #else 251 #define HAVEIT(lkp) /* nothing */ 252 253 #define DONTHAVEIT(lkp) /* nothing */ 254 #endif /* LOCKDEBUG */ /* } */ 255 256 #if defined(LOCKDEBUG) 257 /* 258 * Lock debug printing routine; can be configured to print to console 259 * or log to syslog. 260 */ 261 void 262 #ifdef __STDC__ 263 lock_printf(const char *fmt, ...) 264 #else 265 lock_printf(fmt, va_alist) 266 char *fmt; 267 va_dcl 268 #endif 269 { 270 va_list ap; 271 272 va_start(ap, fmt); 273 if (lock_debug_syslog) 274 vlog(LOG_DEBUG, fmt, ap); 275 else 276 vprintf(fmt, ap); 277 va_end(ap); 278 } 279 #endif /* LOCKDEBUG */ 280 281 /* 282 * Initialize a lock; required before use. 283 */ 284 void 285 lockinit(lkp, prio, wmesg, timo, flags) 286 struct lock *lkp; 287 int prio; 288 const char *wmesg; 289 int timo; 290 int flags; 291 { 292 293 memset(lkp, 0, sizeof(struct lock)); 294 simple_lock_init(&lkp->lk_interlock); 295 lkp->lk_flags = flags & LK_EXTFLG_MASK; 296 if (flags & LK_SPIN) 297 lkp->lk_cpu = LK_NOCPU; 298 else { 299 lkp->lk_lockholder = LK_NOPROC; 300 lkp->lk_prio = prio; 301 lkp->lk_timo = timo; 302 } 303 lkp->lk_wmesg = wmesg; /* just a name for spin locks */ 304 } 305 306 /* 307 * Determine the status of a lock. 308 */ 309 int 310 lockstatus(lkp) 311 struct lock *lkp; 312 { 313 int lock_type = 0; 314 315 simple_lock(&lkp->lk_interlock); 316 if (lkp->lk_exclusivecount != 0) 317 lock_type = LK_EXCLUSIVE; 318 else if (lkp->lk_sharecount != 0) 319 lock_type = LK_SHARED; 320 simple_unlock(&lkp->lk_interlock); 321 return (lock_type); 322 } 323 324 #if defined(LOCKDEBUG) || defined(DIAGNOSTIC) 325 /* 326 * Make sure no spin locks are held by a CPU that is about 327 * to context switch. 328 */ 329 void 330 spinlock_switchcheck(void) 331 { 332 u_long cnt; 333 int s; 334 335 s = splhigh(); 336 #if defined(MULTIPROCESSOR) 337 cnt = curcpu()->ci_spin_locks; 338 #else 339 cnt = spin_locks; 340 #endif 341 splx(s); 342 343 if (cnt != 0) 344 panic("spinlock_switchcheck: CPU %lu has %lu spin locks", 345 (u_long) cpu_number(), cnt); 346 } 347 #endif /* LOCKDEBUG || DIAGNOSTIC */ 348 349 /* 350 * XXX XXX kludge around another kludge.. 351 * 352 * vfs_shutdown() may be called from interrupt context, either as a result 353 * of a panic, or from the debugger. It proceeds to call 354 * sys_sync(&proc0, ...), pretending its running on behalf of proc0 355 * 356 * We would like to make an attempt to sync the filesystems in this case, so 357 * if this happens, we treat attempts to acquire locks specially. 358 * All locks are acquired on behalf of proc0. 359 * 360 * If we've already paniced, we don't block waiting for locks, but 361 * just barge right ahead since we're already going down in flames. 362 */ 363 364 /* 365 * Set, change, or release a lock. 366 * 367 * Shared requests increment the shared count. Exclusive requests set the 368 * LK_WANT_EXCL flag (preventing further shared locks), and wait for already 369 * accepted shared locks and shared-to-exclusive upgrades to go away. 370 */ 371 int 372 lockmgr(lkp, flags, interlkp) 373 __volatile struct lock *lkp; 374 u_int flags; 375 struct simplelock *interlkp; 376 { 377 int error; 378 pid_t pid; 379 int extflags; 380 cpuid_t cpu_id; 381 struct proc *p = curproc; 382 int lock_shutdown_noblock = 0; 383 384 error = 0; 385 386 simple_lock(&lkp->lk_interlock); 387 if (flags & LK_INTERLOCK) 388 simple_unlock(interlkp); 389 extflags = (flags | lkp->lk_flags) & LK_EXTFLG_MASK; 390 391 #ifdef DIAGNOSTIC /* { */ 392 /* 393 * Don't allow spins on sleep locks and don't allow sleeps 394 * on spin locks. 395 */ 396 if ((flags ^ lkp->lk_flags) & LK_SPIN) 397 panic("lockmgr: sleep/spin mismatch\n"); 398 #endif /* } */ 399 400 if (extflags & LK_SPIN) 401 pid = LK_KERNPROC; 402 else { 403 if (p == NULL) { 404 if (!doing_shutdown) { 405 #ifdef DIAGNOSTIC 406 panic("lockmgr: no context"); 407 #endif 408 } else { 409 p = &proc0; 410 if (panicstr && (!(flags & LK_NOWAIT))) { 411 flags |= LK_NOWAIT; 412 lock_shutdown_noblock = 1; 413 } 414 } 415 } 416 pid = p->p_pid; 417 } 418 cpu_id = cpu_number(); 419 420 /* 421 * Once a lock has drained, the LK_DRAINING flag is set and an 422 * exclusive lock is returned. The only valid operation thereafter 423 * is a single release of that exclusive lock. This final release 424 * clears the LK_DRAINING flag and sets the LK_DRAINED flag. Any 425 * further requests of any sort will result in a panic. The bits 426 * selected for these two flags are chosen so that they will be set 427 * in memory that is freed (freed memory is filled with 0xdeadbeef). 428 * The final release is permitted to give a new lease on life to 429 * the lock by specifying LK_REENABLE. 430 */ 431 if (lkp->lk_flags & (LK_DRAINING|LK_DRAINED)) { 432 #ifdef DIAGNOSTIC /* { */ 433 if (lkp->lk_flags & LK_DRAINED) 434 panic("lockmgr: using decommissioned lock"); 435 if ((flags & LK_TYPE_MASK) != LK_RELEASE || 436 WEHOLDIT(lkp, pid, cpu_id) == 0) 437 panic("lockmgr: non-release on draining lock: %d\n", 438 flags & LK_TYPE_MASK); 439 #endif /* DIAGNOSTIC */ /* } */ 440 lkp->lk_flags &= ~LK_DRAINING; 441 if ((flags & LK_REENABLE) == 0) 442 lkp->lk_flags |= LK_DRAINED; 443 } 444 445 switch (flags & LK_TYPE_MASK) { 446 447 case LK_SHARED: 448 if (WEHOLDIT(lkp, pid, cpu_id) == 0) { 449 /* 450 * If just polling, check to see if we will block. 451 */ 452 if ((extflags & LK_NOWAIT) && (lkp->lk_flags & 453 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE))) { 454 error = EBUSY; 455 break; 456 } 457 /* 458 * Wait for exclusive locks and upgrades to clear. 459 */ 460 ACQUIRE(lkp, error, extflags, 0, lkp->lk_flags & 461 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE)); 462 if (error) 463 break; 464 lkp->lk_sharecount++; 465 COUNT(lkp, p, cpu_id, 1); 466 break; 467 } 468 /* 469 * We hold an exclusive lock, so downgrade it to shared. 470 * An alternative would be to fail with EDEADLK. 471 */ 472 lkp->lk_sharecount++; 473 COUNT(lkp, p, cpu_id, 1); 474 /* fall into downgrade */ 475 476 case LK_DOWNGRADE: 477 if (WEHOLDIT(lkp, pid, cpu_id) == 0 || 478 lkp->lk_exclusivecount == 0) 479 panic("lockmgr: not holding exclusive lock"); 480 lkp->lk_sharecount += lkp->lk_exclusivecount; 481 lkp->lk_exclusivecount = 0; 482 lkp->lk_recurselevel = 0; 483 lkp->lk_flags &= ~LK_HAVE_EXCL; 484 SETHOLDER(lkp, LK_NOPROC, LK_NOCPU); 485 DONTHAVEIT(lkp); 486 WAKEUP_WAITER(lkp); 487 break; 488 489 case LK_EXCLUPGRADE: 490 /* 491 * If another process is ahead of us to get an upgrade, 492 * then we want to fail rather than have an intervening 493 * exclusive access. 494 */ 495 if (lkp->lk_flags & LK_WANT_UPGRADE) { 496 lkp->lk_sharecount--; 497 COUNT(lkp, p, cpu_id, -1); 498 error = EBUSY; 499 break; 500 } 501 /* fall into normal upgrade */ 502 503 case LK_UPGRADE: 504 /* 505 * Upgrade a shared lock to an exclusive one. If another 506 * shared lock has already requested an upgrade to an 507 * exclusive lock, our shared lock is released and an 508 * exclusive lock is requested (which will be granted 509 * after the upgrade). If we return an error, the file 510 * will always be unlocked. 511 */ 512 if (WEHOLDIT(lkp, pid, cpu_id) || lkp->lk_sharecount <= 0) 513 panic("lockmgr: upgrade exclusive lock"); 514 lkp->lk_sharecount--; 515 COUNT(lkp, p, cpu_id, -1); 516 /* 517 * If we are just polling, check to see if we will block. 518 */ 519 if ((extflags & LK_NOWAIT) && 520 ((lkp->lk_flags & LK_WANT_UPGRADE) || 521 lkp->lk_sharecount > 1)) { 522 error = EBUSY; 523 break; 524 } 525 if ((lkp->lk_flags & LK_WANT_UPGRADE) == 0) { 526 /* 527 * We are first shared lock to request an upgrade, so 528 * request upgrade and wait for the shared count to 529 * drop to zero, then take exclusive lock. 530 */ 531 lkp->lk_flags |= LK_WANT_UPGRADE; 532 ACQUIRE(lkp, error, extflags, 0, lkp->lk_sharecount); 533 lkp->lk_flags &= ~LK_WANT_UPGRADE; 534 if (error) 535 break; 536 lkp->lk_flags |= LK_HAVE_EXCL; 537 SETHOLDER(lkp, pid, cpu_id); 538 HAVEIT(lkp); 539 if (lkp->lk_exclusivecount != 0) 540 panic("lockmgr: non-zero exclusive count"); 541 lkp->lk_exclusivecount = 1; 542 if (extflags & LK_SETRECURSE) 543 lkp->lk_recurselevel = 1; 544 COUNT(lkp, p, cpu_id, 1); 545 break; 546 } 547 /* 548 * Someone else has requested upgrade. Release our shared 549 * lock, awaken upgrade requestor if we are the last shared 550 * lock, then request an exclusive lock. 551 */ 552 if (lkp->lk_sharecount == 0) 553 WAKEUP_WAITER(lkp); 554 /* fall into exclusive request */ 555 556 case LK_EXCLUSIVE: 557 if (WEHOLDIT(lkp, pid, cpu_id)) { 558 /* 559 * Recursive lock. 560 */ 561 if ((extflags & LK_CANRECURSE) == 0 && 562 lkp->lk_recurselevel == 0) { 563 if (extflags & LK_RECURSEFAIL) { 564 error = EDEADLK; 565 break; 566 } else 567 panic("lockmgr: locking against myself"); 568 } 569 lkp->lk_exclusivecount++; 570 if (extflags & LK_SETRECURSE && 571 lkp->lk_recurselevel == 0) 572 lkp->lk_recurselevel = lkp->lk_exclusivecount; 573 COUNT(lkp, p, cpu_id, 1); 574 break; 575 } 576 /* 577 * If we are just polling, check to see if we will sleep. 578 */ 579 if ((extflags & LK_NOWAIT) && ((lkp->lk_flags & 580 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE)) || 581 lkp->lk_sharecount != 0)) { 582 error = EBUSY; 583 break; 584 } 585 /* 586 * Try to acquire the want_exclusive flag. 587 */ 588 ACQUIRE(lkp, error, extflags, 0, lkp->lk_flags & 589 (LK_HAVE_EXCL | LK_WANT_EXCL)); 590 if (error) 591 break; 592 lkp->lk_flags |= LK_WANT_EXCL; 593 /* 594 * Wait for shared locks and upgrades to finish. 595 */ 596 ACQUIRE(lkp, error, extflags, 0, lkp->lk_sharecount != 0 || 597 (lkp->lk_flags & LK_WANT_UPGRADE)); 598 lkp->lk_flags &= ~LK_WANT_EXCL; 599 if (error) 600 break; 601 lkp->lk_flags |= LK_HAVE_EXCL; 602 SETHOLDER(lkp, pid, cpu_id); 603 HAVEIT(lkp); 604 if (lkp->lk_exclusivecount != 0) 605 panic("lockmgr: non-zero exclusive count"); 606 lkp->lk_exclusivecount = 1; 607 if (extflags & LK_SETRECURSE) 608 lkp->lk_recurselevel = 1; 609 COUNT(lkp, p, cpu_id, 1); 610 break; 611 612 case LK_RELEASE: 613 if (lkp->lk_exclusivecount != 0) { 614 if (WEHOLDIT(lkp, pid, cpu_id) == 0) { 615 if (lkp->lk_flags & LK_SPIN) { 616 panic("lockmgr: processor %lu, not " 617 "exclusive lock holder %lu " 618 "unlocking", cpu_id, lkp->lk_cpu); 619 } else { 620 panic("lockmgr: pid %d, not " 621 "exclusive lock holder %d " 622 "unlocking", pid, 623 lkp->lk_lockholder); 624 } 625 } 626 if (lkp->lk_exclusivecount == lkp->lk_recurselevel) 627 lkp->lk_recurselevel = 0; 628 lkp->lk_exclusivecount--; 629 COUNT(lkp, p, cpu_id, -1); 630 if (lkp->lk_exclusivecount == 0) { 631 lkp->lk_flags &= ~LK_HAVE_EXCL; 632 SETHOLDER(lkp, LK_NOPROC, LK_NOCPU); 633 DONTHAVEIT(lkp); 634 } 635 } else if (lkp->lk_sharecount != 0) { 636 lkp->lk_sharecount--; 637 COUNT(lkp, p, cpu_id, -1); 638 } 639 WAKEUP_WAITER(lkp); 640 break; 641 642 case LK_DRAIN: 643 /* 644 * Check that we do not already hold the lock, as it can 645 * never drain if we do. Unfortunately, we have no way to 646 * check for holding a shared lock, but at least we can 647 * check for an exclusive one. 648 */ 649 if (WEHOLDIT(lkp, pid, cpu_id)) 650 panic("lockmgr: draining against myself"); 651 /* 652 * If we are just polling, check to see if we will sleep. 653 */ 654 if ((extflags & LK_NOWAIT) && ((lkp->lk_flags & 655 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE)) || 656 lkp->lk_sharecount != 0 || lkp->lk_waitcount != 0)) { 657 error = EBUSY; 658 break; 659 } 660 ACQUIRE(lkp, error, extflags, 1, 661 ((lkp->lk_flags & 662 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE)) || 663 lkp->lk_sharecount != 0 || 664 lkp->lk_waitcount != 0)); 665 if (error) 666 break; 667 lkp->lk_flags |= LK_DRAINING | LK_HAVE_EXCL; 668 SETHOLDER(lkp, pid, cpu_id); 669 HAVEIT(lkp); 670 lkp->lk_exclusivecount = 1; 671 /* XXX unlikely that we'd want this */ 672 if (extflags & LK_SETRECURSE) 673 lkp->lk_recurselevel = 1; 674 COUNT(lkp, p, cpu_id, 1); 675 break; 676 677 default: 678 simple_unlock(&lkp->lk_interlock); 679 panic("lockmgr: unknown locktype request %d", 680 flags & LK_TYPE_MASK); 681 /* NOTREACHED */ 682 } 683 if ((lkp->lk_flags & (LK_WAITDRAIN|LK_SPIN)) == LK_WAITDRAIN && 684 ((lkp->lk_flags & 685 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE)) == 0 && 686 lkp->lk_sharecount == 0 && lkp->lk_waitcount == 0)) { 687 lkp->lk_flags &= ~LK_WAITDRAIN; 688 wakeup_one((void *)&lkp->lk_flags); 689 } 690 /* 691 * Note that this panic will be a recursive panic, since 692 * we only set lock_shutdown_noblock above if panicstr != NULL. 693 */ 694 if (error && lock_shutdown_noblock) 695 panic("lockmgr: deadlock (see previous panic)"); 696 697 simple_unlock(&lkp->lk_interlock); 698 return (error); 699 } 700 701 /* 702 * Print out information about state of a lock. Used by VOP_PRINT 703 * routines to display ststus about contained locks. 704 */ 705 void 706 lockmgr_printinfo(lkp) 707 __volatile struct lock *lkp; 708 { 709 710 if (lkp->lk_sharecount) 711 printf(" lock type %s: SHARED (count %d)", lkp->lk_wmesg, 712 lkp->lk_sharecount); 713 else if (lkp->lk_flags & LK_HAVE_EXCL) { 714 printf(" lock type %s: EXCL (count %d) by ", 715 lkp->lk_wmesg, lkp->lk_exclusivecount); 716 if (lkp->lk_flags & LK_SPIN) 717 printf("processor %lu", lkp->lk_cpu); 718 else 719 printf("pid %d", lkp->lk_lockholder); 720 } else 721 printf(" not locked"); 722 if ((lkp->lk_flags & LK_SPIN) == 0 && lkp->lk_waitcount > 0) 723 printf(" with %d pending", lkp->lk_waitcount); 724 } 725 726 #if defined(LOCKDEBUG) /* { */ 727 TAILQ_HEAD(, simplelock) simplelock_list = 728 TAILQ_HEAD_INITIALIZER(simplelock_list); 729 730 #if defined(MULTIPROCESSOR) /* { */ 731 struct simplelock simplelock_list_slock = SIMPLELOCK_INITIALIZER; 732 733 #define SLOCK_LIST_LOCK() \ 734 __cpu_simple_lock(&simplelock_list_slock.lock_data) 735 736 #define SLOCK_LIST_UNLOCK() \ 737 __cpu_simple_unlock(&simplelock_list_slock.lock_data) 738 739 #if defined(__HAVE_ATOMIC_OPERATIONS) /* { */ 740 #define SLOCK_COUNT(x) \ 741 atomic_add_ulong(&curcpu()->ci_simple_locks, (x)) 742 #else 743 #define SLOCK_COUNT(x) /* not safe */ 744 #endif /* __HAVE_ATOMIC_OPERATIONS */ /* } */ 745 #else 746 u_long simple_locks; 747 748 #define SLOCK_LIST_LOCK() /* nothing */ 749 750 #define SLOCK_LIST_UNLOCK() /* nothing */ 751 752 #define SLOCK_COUNT(x) simple_locks += (x) 753 #endif /* MULTIPROCESSOR */ /* } */ 754 755 #ifdef DDB /* { */ 756 int simple_lock_debugger = 0; 757 #define SLOCK_DEBUGGER() if (simple_lock_debugger) Debugger() 758 #else 759 #define SLOCK_DEBUGGER() /* nothing */ 760 #endif /* } */ 761 762 #ifdef MULTIPROCESSOR 763 #define SLOCK_MP() lock_printf("on cpu %d\n", cpu_number()) 764 #else 765 #define SLOCK_MP() /* nothing */ 766 #endif 767 768 #define SLOCK_WHERE(str, alp, id, l) \ 769 do { \ 770 lock_printf(str); \ 771 lock_printf("lock: %p, currently at: %s:%d\n", (alp), (id), (l)); \ 772 SLOCK_MP(); \ 773 if ((alp)->lock_file != NULL) \ 774 lock_printf("last locked: %s:%d\n", (alp)->lock_file, \ 775 (alp)->lock_line); \ 776 if ((alp)->unlock_file != NULL) \ 777 lock_printf("last unlocked: %s:%d\n", (alp)->unlock_file, \ 778 (alp)->unlock_line); \ 779 SLOCK_DEBUGGER(); \ 780 } while (/*CONSTCOND*/0) 781 782 /* 783 * Simple lock functions so that the debugger can see from whence 784 * they are being called. 785 */ 786 void 787 simple_lock_init(alp) 788 struct simplelock *alp; 789 { 790 791 #if defined(MULTIPROCESSOR) /* { */ 792 __cpu_simple_lock_init(&alp->lock_data); 793 #else 794 alp->lock_data = __SIMPLELOCK_UNLOCKED; 795 #endif /* } */ 796 alp->lock_file = NULL; 797 alp->lock_line = 0; 798 alp->unlock_file = NULL; 799 alp->unlock_line = 0; 800 alp->lock_holder = 0; 801 } 802 803 void 804 _simple_lock(alp, id, l) 805 __volatile struct simplelock *alp; 806 const char *id; 807 int l; 808 { 809 cpuid_t cpu_id = cpu_number(); 810 int s; 811 812 s = splhigh(); 813 814 /* 815 * MULTIPROCESSOR case: This is `safe' since if it's not us, we 816 * don't take any action, and just fall into the normal spin case. 817 */ 818 if (alp->lock_data == __SIMPLELOCK_LOCKED) { 819 #if defined(MULTIPROCESSOR) /* { */ 820 if (alp->lock_holder == cpu_id) { 821 SLOCK_WHERE("simple_lock: locking against myself\n", 822 alp, id, l); 823 goto out; 824 } 825 #else 826 SLOCK_WHERE("simple_lock: lock held\n", alp, id, l); 827 goto out; 828 #endif /* MULTIPROCESSOR */ /* } */ 829 } 830 831 #if defined(MULTIPROCESSOR) /* { */ 832 /* Acquire the lock before modifying any fields. */ 833 __cpu_simple_lock(&alp->lock_data); 834 #else 835 alp->lock_data = __SIMPLELOCK_LOCKED; 836 #endif /* } */ 837 838 alp->lock_file = id; 839 alp->lock_line = l; 840 alp->lock_holder = cpu_id; 841 842 SLOCK_LIST_LOCK(); 843 /* XXX Cast away volatile */ 844 TAILQ_INSERT_TAIL(&simplelock_list, (struct simplelock *)alp, list); 845 SLOCK_LIST_UNLOCK(); 846 847 SLOCK_COUNT(1); 848 849 out: 850 splx(s); 851 } 852 853 int 854 _simple_lock_try(alp, id, l) 855 __volatile struct simplelock *alp; 856 const char *id; 857 int l; 858 { 859 cpuid_t cpu_id = cpu_number(); 860 int s, rv = 0; 861 862 s = splhigh(); 863 864 /* 865 * MULTIPROCESSOR case: This is `safe' since if it's not us, we 866 * don't take any action. 867 */ 868 #if defined(MULTIPROCESSOR) /* { */ 869 if ((rv = __cpu_simple_lock_try(&alp->lock_data)) == 0) { 870 if (alp->lock_holder == cpu_id) 871 SLOCK_WHERE("simple_lock_try: locking against myself\n", 872 alp, id, l); 873 goto out; 874 } 875 #else 876 if (alp->lock_data == __SIMPLELOCK_LOCKED) { 877 SLOCK_WHERE("simple_lock_try: lock held\n", alp, id, l); 878 goto out; 879 } 880 alp->lock_data = __SIMPLELOCK_LOCKED; 881 #endif /* MULTIPROCESSOR */ /* } */ 882 883 /* 884 * At this point, we have acquired the lock. 885 */ 886 887 rv = 1; 888 889 alp->lock_file = id; 890 alp->lock_line = l; 891 alp->lock_holder = cpu_id; 892 893 SLOCK_LIST_LOCK(); 894 /* XXX Cast away volatile. */ 895 TAILQ_INSERT_TAIL(&simplelock_list, (struct simplelock *)alp, list); 896 SLOCK_LIST_UNLOCK(); 897 898 SLOCK_COUNT(1); 899 900 out: 901 splx(s); 902 return (rv); 903 } 904 905 void 906 _simple_unlock(alp, id, l) 907 __volatile struct simplelock *alp; 908 const char *id; 909 int l; 910 { 911 int s; 912 913 s = splhigh(); 914 915 /* 916 * MULTIPROCESSOR case: This is `safe' because we think we hold 917 * the lock, and if we don't, we don't take any action. 918 */ 919 if (alp->lock_data == __SIMPLELOCK_UNLOCKED) { 920 SLOCK_WHERE("simple_unlock: lock not held\n", 921 alp, id, l); 922 goto out; 923 } 924 925 SLOCK_LIST_LOCK(); 926 TAILQ_REMOVE(&simplelock_list, alp, list); 927 SLOCK_LIST_UNLOCK(); 928 929 SLOCK_COUNT(-1); 930 931 alp->list.tqe_next = NULL; /* sanity */ 932 alp->list.tqe_prev = NULL; /* sanity */ 933 934 alp->unlock_file = id; 935 alp->unlock_line = l; 936 937 #if defined(MULTIPROCESSOR) /* { */ 938 alp->lock_holder = LK_NOCPU; 939 /* Now that we've modified all fields, release the lock. */ 940 __cpu_simple_unlock(&alp->lock_data); 941 #else 942 alp->lock_data = __SIMPLELOCK_UNLOCKED; 943 #endif /* } */ 944 945 out: 946 splx(s); 947 } 948 949 void 950 simple_lock_dump() 951 { 952 struct simplelock *alp; 953 int s; 954 955 s = splhigh(); 956 SLOCK_LIST_LOCK(); 957 lock_printf("all simple locks:\n"); 958 for (alp = TAILQ_FIRST(&simplelock_list); alp != NULL; 959 alp = TAILQ_NEXT(alp, list)) { 960 lock_printf("%p CPU %lu %s:%d\n", alp, alp->lock_holder, 961 alp->lock_file, alp->lock_line); 962 } 963 SLOCK_LIST_UNLOCK(); 964 splx(s); 965 } 966 967 void 968 simple_lock_freecheck(start, end) 969 void *start, *end; 970 { 971 struct simplelock *alp; 972 int s; 973 974 s = splhigh(); 975 SLOCK_LIST_LOCK(); 976 for (alp = TAILQ_FIRST(&simplelock_list); alp != NULL; 977 alp = TAILQ_NEXT(alp, list)) { 978 if ((void *)alp >= start && (void *)alp < end) { 979 lock_printf("freeing simple_lock %p CPU %lu %s:%d\n", 980 alp, alp->lock_holder, alp->lock_file, 981 alp->lock_line); 982 SLOCK_DEBUGGER(); 983 } 984 } 985 SLOCK_LIST_UNLOCK(); 986 splx(s); 987 } 988 989 void 990 simple_lock_switchcheck(void) 991 { 992 struct simplelock *alp; 993 cpuid_t cpu_id = cpu_number(); 994 int s; 995 996 s = splhigh(); 997 SLOCK_LIST_LOCK(); 998 for (alp = TAILQ_FIRST(&simplelock_list); alp != NULL; 999 alp = TAILQ_NEXT(alp, list)) { 1000 if (alp->lock_holder == cpu_id) { 1001 lock_printf("switching with held simple_lock %p " 1002 "CPU %lu %s:%s\n", 1003 alp, alp->lock_holder, alp->lock_file, 1004 alp->lock_line); 1005 SLOCK_DEBUGGER(); 1006 } 1007 } 1008 SLOCK_LIST_UNLOCK(); 1009 splx(s); 1010 } 1011 #endif /* LOCKDEBUG */ /* } */ 1012
Indexes created Sun Sep 28 12:09:53 GMT 2025