kern_lock.c revision 1.32.2.3
1 /* $NetBSD: kern_lock.c,v 1.32.2.3 2000/09/05 23:17:01 gmcgarry Exp $ */ 2 3 /*- 4 * Copyright (c) 1999, 2000 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center. 10 * 11 * This code is derived from software contributed to The NetBSD Foundation 12 * by Ross Harvey. 13 * 14 * Redistribution and use in source and binary forms, with or without 15 * modification, are permitted provided that the following conditions 16 * are met: 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in the 21 * documentation and/or other materials provided with the distribution. 22 * 3. All advertising materials mentioning features or use of this software 23 * must display the following acknowledgement: 24 * This product includes software developed by the NetBSD 25 * Foundation, Inc. and its contributors. 26 * 4. Neither the name of The NetBSD Foundation nor the names of its 27 * contributors may be used to endorse or promote products derived 28 * from this software without specific prior written permission. 29 * 30 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 31 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 32 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 33 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 34 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 35 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 36 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 37 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 38 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 39 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 40 * POSSIBILITY OF SUCH DAMAGE. 41 */ 42 43 /* 44 * Copyright (c) 1995 45 * The Regents of the University of California. All rights reserved. 46 * 47 * This code contains ideas from software contributed to Berkeley by 48 * Avadis Tevanian, Jr., Michael Wayne Young, and the Mach Operating 49 * System project at Carnegie-Mellon University. 50 * 51 * Redistribution and use in source and binary forms, with or without 52 * modification, are permitted provided that the following conditions 53 * are met: 54 * 1. Redistributions of source code must retain the above copyright 55 * notice, this list of conditions and the following disclaimer. 56 * 2. Redistributions in binary form must reproduce the above copyright 57 * notice, this list of conditions and the following disclaimer in the 58 * documentation and/or other materials provided with the distribution. 59 * 3. All advertising materials mentioning features or use of this software 60 * must display the following acknowledgement: 61 * This product includes software developed by the University of 62 * California, Berkeley and its contributors. 63 * 4. Neither the name of the University nor the names of its contributors 64 * may be used to endorse or promote products derived from this software 65 * without specific prior written permission. 66 * 67 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 68 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 69 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 70 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 71 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 72 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 73 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 74 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 75 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 76 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 77 * SUCH DAMAGE. 78 * 79 * @(#)kern_lock.c 8.18 (Berkeley) 5/21/95 80 */ 81 82 #include "opt_multiprocessor.h" 83 #include "opt_lockdebug.h" 84 #include "opt_ddb.h" 85 86 #include <sys/param.h> 87 #include <sys/proc.h> 88 #include <sys/lock.h> 89 #include <sys/systm.h> 90 #include <machine/cpu.h> 91 92 #if defined(__HAVE_ATOMIC_OPERATIONS) 93 #include <machine/atomic.h> 94 #endif 95 96 #if defined(LOCKDEBUG) 97 #include <sys/syslog.h> 98 /* 99 * note that stdarg.h and the ansi style va_start macro is used for both 100 * ansi and traditional c compiles. 101 * XXX: this requires that stdarg.h define: va_alist and va_dcl 102 */ 103 #include <machine/stdarg.h> 104 105 void lock_printf(const char *fmt, ...) 106 __kprintf_attribute__((__format__(__kprintf__,1,2))); 107 108 int lock_debug_syslog = 0; /* defaults to printf, but can be patched */ 109 #endif 110 111 /* 112 * Locking primitives implementation. 113 * Locks provide shared/exclusive sychronization. 114 */ 115 116 #if defined(LOCKDEBUG) || defined(DIAGNOSTIC) /* { */ 117 #if defined(MULTIPROCESSOR) /* { */ 118 #if defined(__HAVE_ATOMIC_OPERATIONS) /* { */ 119 #define COUNT_CPU(cpu_id, x) \ 120 atomic_add_ulong(&curcpu()->ci_spin_locks, (x)) 121 #else 122 #define COUNT_CPU(cpu_id, x) /* not safe */ 123 #endif /* __HAVE_ATOMIC_OPERATIONS */ /* } */ 124 #else 125 u_long spin_locks; 126 #define COUNT_CPU(cpu_id, x) spin_locks += (x) 127 #endif /* MULTIPROCESSOR */ /* } */ 128 129 #define COUNT(lkp, p, cpu_id, x) \ 130 do { \ 131 if ((lkp)->lk_flags & LK_SPIN) \ 132 COUNT_CPU((cpu_id), (x)); \ 133 else \ 134 (p)->p_locks += (x); \ 135 } while (/*CONSTCOND*/0) 136 #else 137 #define COUNT(lkp, p, cpu_id, x) 138 #endif /* LOCKDEBUG || DIAGNOSTIC */ /* } */ 139 140 /* 141 * Acquire a resource. 142 */ 143 #define ACQUIRE(lkp, error, extflags, drain, wanted) \ 144 if ((extflags) & LK_SPIN) { \ 145 int interlocked; \ 146 \ 147 if ((drain) == 0) \ 148 (lkp)->lk_waitcount++; \ 149 for (interlocked = 1;;) { \ 150 if (wanted) { \ 151 if (interlocked) { \ 152 simple_unlock(&(lkp)->lk_interlock); \ 153 interlocked = 0; \ 154 } \ 155 } else if (interlocked) { \ 156 break; \ 157 } else { \ 158 simple_lock(&(lkp)->lk_interlock); \ 159 interlocked = 1; \ 160 } \ 161 } \ 162 if ((drain) == 0) \ 163 (lkp)->lk_waitcount--; \ 164 KASSERT((wanted) == 0); \ 165 error = 0; /* sanity */ \ 166 } else { \ 167 for (error = 0; wanted; ) { \ 168 if ((drain)) \ 169 (lkp)->lk_flags |= LK_WAITDRAIN; \ 170 else \ 171 (lkp)->lk_waitcount++; \ 172 /* XXX Cast away volatile. */ \ 173 error = ltsleep((drain) ? &(lkp)->lk_flags : \ 174 (void *)(lkp), (lkp)->lk_prio, \ 175 (lkp)->lk_wmesg, (lkp)->lk_timo, \ 176 &(lkp)->lk_interlock); \ 177 if ((drain) == 0) \ 178 (lkp)->lk_waitcount--; \ 179 if (error) \ 180 break; \ 181 if ((extflags) & LK_SLEEPFAIL) { \ 182 error = ENOLCK; \ 183 break; \ 184 } \ 185 } \ 186 } 187 188 #define SETHOLDER(lkp, pid, cpu_id) \ 189 do { \ 190 if ((lkp)->lk_flags & LK_SPIN) \ 191 (lkp)->lk_cpu = cpu_id; \ 192 else \ 193 (lkp)->lk_lockholder = pid; \ 194 } while (/*CONSTCOND*/0) 195 196 #define WEHOLDIT(lkp, pid, cpu_id) \ 197 (((lkp)->lk_flags & LK_SPIN) != 0 ? \ 198 ((lkp)->lk_cpu == (cpu_id)) : ((lkp)->lk_lockholder == (pid))) 199 200 #define WAKEUP_WAITER(lkp) \ 201 do { \ 202 if (((lkp)->lk_flags & LK_SPIN) == 0 && (lkp)->lk_waitcount) { \ 203 /* XXX Cast away volatile. */ \ 204 wakeup_one((void *)(lkp)); \ 205 } \ 206 } while (/*CONSTCOND*/0) 207 208 #if defined(LOCKDEBUG) /* { */ 209 #if defined(MULTIPROCESSOR) /* { */ 210 struct simplelock spinlock_list_slock = SIMPLELOCK_INITIALIZER; 211 212 #define SPINLOCK_LIST_LOCK() \ 213 __cpu_simple_lock(&spinlock_list_slock.lock_data) 214 215 #define SPINLOCK_LIST_UNLOCK() \ 216 __cpu_simple_unlock(&spinlock_list_slock.lock_data) 217 #else 218 #define SPINLOCK_LIST_LOCK() /* nothing */ 219 220 #define SPINLOCK_LIST_UNLOCK() /* nothing */ 221 #endif /* MULTIPROCESSOR */ /* } */ 222 223 TAILQ_HEAD(, lock) spinlock_list = 224 TAILQ_HEAD_INITIALIZER(spinlock_list); 225 226 #define HAVEIT(lkp) \ 227 do { \ 228 if ((lkp)->lk_flags & LK_SPIN) { \ 229 int s = splhigh(); \ 230 SPINLOCK_LIST_LOCK(); \ 231 /* XXX Cast away volatile. */ \ 232 TAILQ_INSERT_TAIL(&spinlock_list, (struct lock *)(lkp), \ 233 lk_list); \ 234 SPINLOCK_LIST_UNLOCK(); \ 235 splx(s); \ 236 } \ 237 } while (/*CONSTCOND*/0) 238 239 #define DONTHAVEIT(lkp) \ 240 do { \ 241 if ((lkp)->lk_flags & LK_SPIN) { \ 242 int s = splhigh(); \ 243 SPINLOCK_LIST_LOCK(); \ 244 /* XXX Cast away volatile. */ \ 245 TAILQ_REMOVE(&spinlock_list, (struct lock *)(lkp), \ 246 lk_list); \ 247 SPINLOCK_LIST_UNLOCK(); \ 248 splx(s); \ 249 } \ 250 } while (/*CONSTCOND*/0) 251 #else 252 #define HAVEIT(lkp) /* nothing */ 253 254 #define DONTHAVEIT(lkp) /* nothing */ 255 #endif /* LOCKDEBUG */ /* } */ 256 257 #if defined(LOCKDEBUG) 258 /* 259 * Lock debug printing routine; can be configured to print to console 260 * or log to syslog. 261 */ 262 void 263 #ifdef __STDC__ 264 lock_printf(const char *fmt, ...) 265 #else 266 lock_printf(fmt, va_alist) 267 char *fmt; 268 va_dcl 269 #endif 270 { 271 va_list ap; 272 273 va_start(ap, fmt); 274 if (lock_debug_syslog) 275 vlog(LOG_DEBUG, fmt, ap); 276 else 277 vprintf(fmt, ap); 278 va_end(ap); 279 } 280 #endif /* LOCKDEBUG */ 281 282 /* 283 * Initialize a lock; required before use. 284 */ 285 void 286 lockinit(lkp, prio, wmesg, timo, flags) 287 struct lock *lkp; 288 int prio; 289 const char *wmesg; 290 int timo; 291 int flags; 292 { 293 294 memset(lkp, 0, sizeof(struct lock)); 295 simple_lock_init(&lkp->lk_interlock); 296 lkp->lk_flags = flags & LK_EXTFLG_MASK; 297 if (flags & LK_SPIN) 298 lkp->lk_cpu = LK_NOCPU; 299 else { 300 lkp->lk_lockholder = LK_NOPROC; 301 lkp->lk_prio = prio; 302 lkp->lk_timo = timo; 303 } 304 lkp->lk_wmesg = wmesg; /* just a name for spin locks */ 305 } 306 307 /* 308 * Determine the status of a lock. 309 */ 310 int 311 lockstatus(lkp) 312 struct lock *lkp; 313 { 314 int lock_type = 0; 315 316 simple_lock(&lkp->lk_interlock); 317 if (lkp->lk_exclusivecount != 0) 318 lock_type = LK_EXCLUSIVE; 319 else if (lkp->lk_sharecount != 0) 320 lock_type = LK_SHARED; 321 simple_unlock(&lkp->lk_interlock); 322 return (lock_type); 323 } 324 325 #if defined(LOCKDEBUG) || defined(DIAGNOSTIC) 326 /* 327 * Make sure no spin locks are held by a CPU that is about 328 * to context switch. 329 */ 330 void 331 spinlock_switchcheck(void) 332 { 333 u_long cnt; 334 int s; 335 336 s = splhigh(); 337 #if defined(MULTIPROCESSOR) 338 cnt = curcpu()->ci_spin_locks; 339 #else 340 cnt = spin_locks; 341 #endif 342 splx(s); 343 344 if (cnt != 0) 345 panic("spinlock_switchcheck: CPU %lu has %lu spin locks", 346 (u_long) cpu_number(), cnt); 347 } 348 #endif /* LOCKDEBUG || DIAGNOSTIC */ 349 350 /* 351 * XXX XXX kludge around another kludge.. 352 * 353 * vfs_shutdown() may be called from interrupt context, either as a result 354 * of a panic, or from the debugger. It proceeds to call 355 * sys_sync(&proc0, ...), pretending its running on behalf of proc0 356 * 357 * We would like to make an attempt to sync the filesystems in this case, so 358 * if this happens, we treat attempts to acquire locks specially. 359 * All locks are acquired on behalf of proc0. 360 * 361 * If we've already paniced, we don't block waiting for locks, but 362 * just barge right ahead since we're already going down in flames. 363 */ 364 365 /* 366 * Set, change, or release a lock. 367 * 368 * Shared requests increment the shared count. Exclusive requests set the 369 * LK_WANT_EXCL flag (preventing further shared locks), and wait for already 370 * accepted shared locks and shared-to-exclusive upgrades to go away. 371 */ 372 int 373 lockmgr(lkp, flags, interlkp) 374 __volatile struct lock *lkp; 375 u_int flags; 376 struct simplelock *interlkp; 377 { 378 int error; 379 pid_t pid; 380 int extflags; 381 cpuid_t cpu_id; 382 struct proc *p = curproc; 383 int lock_shutdown_noblock = 0; 384 385 error = 0; 386 387 simple_lock(&lkp->lk_interlock); 388 if (flags & LK_INTERLOCK) 389 simple_unlock(interlkp); 390 extflags = (flags | lkp->lk_flags) & LK_EXTFLG_MASK; 391 392 #ifdef DIAGNOSTIC /* { */ 393 /* 394 * Don't allow spins on sleep locks and don't allow sleeps 395 * on spin locks. 396 */ 397 if ((flags ^ lkp->lk_flags) & LK_SPIN) 398 panic("lockmgr: sleep/spin mismatch\n"); 399 #endif /* } */ 400 401 if (extflags & LK_SPIN) 402 pid = LK_KERNPROC; 403 else { 404 if (p == NULL) { 405 if (!doing_shutdown) { 406 #ifdef DIAGNOSTIC 407 panic("lockmgr: no context"); 408 #endif 409 } else { 410 p = &proc0; 411 if (panicstr && (!(flags & LK_NOWAIT))) { 412 flags |= LK_NOWAIT; 413 lock_shutdown_noblock = 1; 414 } 415 } 416 } 417 pid = p->p_pid; 418 } 419 cpu_id = cpu_number(); 420 421 /* 422 * Once a lock has drained, the LK_DRAINING flag is set and an 423 * exclusive lock is returned. The only valid operation thereafter 424 * is a single release of that exclusive lock. This final release 425 * clears the LK_DRAINING flag and sets the LK_DRAINED flag. Any 426 * further requests of any sort will result in a panic. The bits 427 * selected for these two flags are chosen so that they will be set 428 * in memory that is freed (freed memory is filled with 0xdeadbeef). 429 * The final release is permitted to give a new lease on life to 430 * the lock by specifying LK_REENABLE. 431 */ 432 if (lkp->lk_flags & (LK_DRAINING|LK_DRAINED)) { 433 #ifdef DIAGNOSTIC /* { */ 434 if (lkp->lk_flags & LK_DRAINED) 435 panic("lockmgr: using decommissioned lock"); 436 if ((flags & LK_TYPE_MASK) != LK_RELEASE || 437 WEHOLDIT(lkp, pid, cpu_id) == 0) 438 panic("lockmgr: non-release on draining lock: %d\n", 439 flags & LK_TYPE_MASK); 440 #endif /* DIAGNOSTIC */ /* } */ 441 lkp->lk_flags &= ~LK_DRAINING; 442 if ((flags & LK_REENABLE) == 0) 443 lkp->lk_flags |= LK_DRAINED; 444 } 445 446 switch (flags & LK_TYPE_MASK) { 447 448 case LK_SHARED: 449 if (WEHOLDIT(lkp, pid, cpu_id) == 0) { 450 /* 451 * If just polling, check to see if we will block. 452 */ 453 if ((extflags & LK_NOWAIT) && (lkp->lk_flags & 454 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE))) { 455 error = EBUSY; 456 break; 457 } 458 /* 459 * Wait for exclusive locks and upgrades to clear. 460 */ 461 ACQUIRE(lkp, error, extflags, 0, lkp->lk_flags & 462 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE)); 463 if (error) 464 break; 465 lkp->lk_sharecount++; 466 COUNT(lkp, p, cpu_id, 1); 467 break; 468 } 469 /* 470 * We hold an exclusive lock, so downgrade it to shared. 471 * An alternative would be to fail with EDEADLK. 472 */ 473 lkp->lk_sharecount++; 474 COUNT(lkp, p, cpu_id, 1); 475 /* fall into downgrade */ 476 477 case LK_DOWNGRADE: 478 if (WEHOLDIT(lkp, pid, cpu_id) == 0 || 479 lkp->lk_exclusivecount == 0) 480 panic("lockmgr: not holding exclusive lock"); 481 lkp->lk_sharecount += lkp->lk_exclusivecount; 482 lkp->lk_exclusivecount = 0; 483 lkp->lk_recurselevel = 0; 484 lkp->lk_flags &= ~LK_HAVE_EXCL; 485 SETHOLDER(lkp, LK_NOPROC, LK_NOCPU); 486 DONTHAVEIT(lkp); 487 WAKEUP_WAITER(lkp); 488 break; 489 490 case LK_EXCLUPGRADE: 491 /* 492 * If another process is ahead of us to get an upgrade, 493 * then we want to fail rather than have an intervening 494 * exclusive access. 495 */ 496 if (lkp->lk_flags & LK_WANT_UPGRADE) { 497 lkp->lk_sharecount--; 498 COUNT(lkp, p, cpu_id, -1); 499 error = EBUSY; 500 break; 501 } 502 /* fall into normal upgrade */ 503 504 case LK_UPGRADE: 505 /* 506 * Upgrade a shared lock to an exclusive one. If another 507 * shared lock has already requested an upgrade to an 508 * exclusive lock, our shared lock is released and an 509 * exclusive lock is requested (which will be granted 510 * after the upgrade). If we return an error, the file 511 * will always be unlocked. 512 */ 513 if (WEHOLDIT(lkp, pid, cpu_id) || lkp->lk_sharecount <= 0) 514 panic("lockmgr: upgrade exclusive lock"); 515 lkp->lk_sharecount--; 516 COUNT(lkp, p, cpu_id, -1); 517 /* 518 * If we are just polling, check to see if we will block. 519 */ 520 if ((extflags & LK_NOWAIT) && 521 ((lkp->lk_flags & LK_WANT_UPGRADE) || 522 lkp->lk_sharecount > 1)) { 523 error = EBUSY; 524 break; 525 } 526 if ((lkp->lk_flags & LK_WANT_UPGRADE) == 0) { 527 /* 528 * We are first shared lock to request an upgrade, so 529 * request upgrade and wait for the shared count to 530 * drop to zero, then take exclusive lock. 531 */ 532 lkp->lk_flags |= LK_WANT_UPGRADE; 533 ACQUIRE(lkp, error, extflags, 0, lkp->lk_sharecount); 534 lkp->lk_flags &= ~LK_WANT_UPGRADE; 535 if (error) 536 break; 537 lkp->lk_flags |= LK_HAVE_EXCL; 538 SETHOLDER(lkp, pid, cpu_id); 539 HAVEIT(lkp); 540 if (lkp->lk_exclusivecount != 0) 541 panic("lockmgr: non-zero exclusive count"); 542 lkp->lk_exclusivecount = 1; 543 if (extflags & LK_SETRECURSE) 544 lkp->lk_recurselevel = 1; 545 COUNT(lkp, p, cpu_id, 1); 546 break; 547 } 548 /* 549 * Someone else has requested upgrade. Release our shared 550 * lock, awaken upgrade requestor if we are the last shared 551 * lock, then request an exclusive lock. 552 */ 553 if (lkp->lk_sharecount == 0) 554 WAKEUP_WAITER(lkp); 555 /* fall into exclusive request */ 556 557 case LK_EXCLUSIVE: 558 if (WEHOLDIT(lkp, pid, cpu_id)) { 559 /* 560 * Recursive lock. 561 */ 562 if ((extflags & LK_CANRECURSE) == 0 && 563 lkp->lk_recurselevel == 0) { 564 if (extflags & LK_RECURSEFAIL) { 565 error = EDEADLK; 566 break; 567 } else 568 panic("lockmgr: locking against myself"); 569 } 570 lkp->lk_exclusivecount++; 571 if (extflags & LK_SETRECURSE && 572 lkp->lk_recurselevel == 0) 573 lkp->lk_recurselevel = lkp->lk_exclusivecount; 574 COUNT(lkp, p, cpu_id, 1); 575 break; 576 } 577 /* 578 * If we are just polling, check to see if we will sleep. 579 */ 580 if ((extflags & LK_NOWAIT) && ((lkp->lk_flags & 581 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE)) || 582 lkp->lk_sharecount != 0)) { 583 error = EBUSY; 584 break; 585 } 586 /* 587 * Try to acquire the want_exclusive flag. 588 */ 589 ACQUIRE(lkp, error, extflags, 0, lkp->lk_flags & 590 (LK_HAVE_EXCL | LK_WANT_EXCL)); 591 if (error) 592 break; 593 lkp->lk_flags |= LK_WANT_EXCL; 594 /* 595 * Wait for shared locks and upgrades to finish. 596 */ 597 ACQUIRE(lkp, error, extflags, 0, lkp->lk_sharecount != 0 || 598 (lkp->lk_flags & LK_WANT_UPGRADE)); 599 lkp->lk_flags &= ~LK_WANT_EXCL; 600 if (error) 601 break; 602 lkp->lk_flags |= LK_HAVE_EXCL; 603 SETHOLDER(lkp, pid, cpu_id); 604 HAVEIT(lkp); 605 if (lkp->lk_exclusivecount != 0) 606 panic("lockmgr: non-zero exclusive count"); 607 lkp->lk_exclusivecount = 1; 608 if (extflags & LK_SETRECURSE) 609 lkp->lk_recurselevel = 1; 610 COUNT(lkp, p, cpu_id, 1); 611 break; 612 613 case LK_RELEASE: 614 if (lkp->lk_exclusivecount != 0) { 615 if (WEHOLDIT(lkp, pid, cpu_id) == 0) { 616 if (lkp->lk_flags & LK_SPIN) { 617 panic("lockmgr: processor %lu, not " 618 "exclusive lock holder %lu " 619 "unlocking", cpu_id, lkp->lk_cpu); 620 } else { 621 panic("lockmgr: pid %d, not " 622 "exclusive lock holder %d " 623 "unlocking", pid, 624 lkp->lk_lockholder); 625 } 626 } 627 if (lkp->lk_exclusivecount == lkp->lk_recurselevel) 628 lkp->lk_recurselevel = 0; 629 lkp->lk_exclusivecount--; 630 COUNT(lkp, p, cpu_id, -1); 631 if (lkp->lk_exclusivecount == 0) { 632 lkp->lk_flags &= ~LK_HAVE_EXCL; 633 SETHOLDER(lkp, LK_NOPROC, LK_NOCPU); 634 DONTHAVEIT(lkp); 635 } 636 } else if (lkp->lk_sharecount != 0) { 637 lkp->lk_sharecount--; 638 COUNT(lkp, p, cpu_id, -1); 639 } 640 WAKEUP_WAITER(lkp); 641 break; 642 643 case LK_DRAIN: 644 /* 645 * Check that we do not already hold the lock, as it can 646 * never drain if we do. Unfortunately, we have no way to 647 * check for holding a shared lock, but at least we can 648 * check for an exclusive one. 649 */ 650 if (WEHOLDIT(lkp, pid, cpu_id)) 651 panic("lockmgr: draining against myself"); 652 /* 653 * If we are just polling, check to see if we will sleep. 654 */ 655 if ((extflags & LK_NOWAIT) && ((lkp->lk_flags & 656 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE)) || 657 lkp->lk_sharecount != 0 || lkp->lk_waitcount != 0)) { 658 error = EBUSY; 659 break; 660 } 661 ACQUIRE(lkp, error, extflags, 1, 662 ((lkp->lk_flags & 663 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE)) || 664 lkp->lk_sharecount != 0 || 665 lkp->lk_waitcount != 0)); 666 if (error) 667 break; 668 lkp->lk_flags |= LK_DRAINING | LK_HAVE_EXCL; 669 SETHOLDER(lkp, pid, cpu_id); 670 HAVEIT(lkp); 671 lkp->lk_exclusivecount = 1; 672 /* XXX unlikely that we'd want this */ 673 if (extflags & LK_SETRECURSE) 674 lkp->lk_recurselevel = 1; 675 COUNT(lkp, p, cpu_id, 1); 676 break; 677 678 default: 679 simple_unlock(&lkp->lk_interlock); 680 panic("lockmgr: unknown locktype request %d", 681 flags & LK_TYPE_MASK); 682 /* NOTREACHED */ 683 } 684 if ((lkp->lk_flags & (LK_WAITDRAIN|LK_SPIN)) == LK_WAITDRAIN && 685 ((lkp->lk_flags & 686 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE)) == 0 && 687 lkp->lk_sharecount == 0 && lkp->lk_waitcount == 0)) { 688 lkp->lk_flags &= ~LK_WAITDRAIN; 689 wakeup_one((void *)&lkp->lk_flags); 690 } 691 /* 692 * Note that this panic will be a recursive panic, since 693 * we only set lock_shutdown_noblock above if panicstr != NULL. 694 */ 695 if (error && lock_shutdown_noblock) 696 panic("lockmgr: deadlock (see previous panic)"); 697 698 simple_unlock(&lkp->lk_interlock); 699 return (error); 700 } 701 702 /* 703 * Print out information about state of a lock. Used by VOP_PRINT 704 * routines to display ststus about contained locks. 705 */ 706 void 707 lockmgr_printinfo(lkp) 708 __volatile struct lock *lkp; 709 { 710 711 if (lkp->lk_sharecount) 712 printf(" lock type %s: SHARED (count %d)", lkp->lk_wmesg, 713 lkp->lk_sharecount); 714 else if (lkp->lk_flags & LK_HAVE_EXCL) { 715 printf(" lock type %s: EXCL (count %d) by ", 716 lkp->lk_wmesg, lkp->lk_exclusivecount); 717 if (lkp->lk_flags & LK_SPIN) 718 printf("processor %lu", lkp->lk_cpu); 719 else 720 printf("pid %d", lkp->lk_lockholder); 721 } else 722 printf(" not locked"); 723 if ((lkp->lk_flags & LK_SPIN) == 0 && lkp->lk_waitcount > 0) 724 printf(" with %d pending", lkp->lk_waitcount); 725 } 726 727 #if defined(LOCKDEBUG) /* { */ 728 TAILQ_HEAD(, simplelock) simplelock_list = 729 TAILQ_HEAD_INITIALIZER(simplelock_list); 730 731 #if defined(MULTIPROCESSOR) /* { */ 732 struct simplelock simplelock_list_slock = SIMPLELOCK_INITIALIZER; 733 734 #define SLOCK_LIST_LOCK() \ 735 __cpu_simple_lock(&simplelock_list_slock.lock_data) 736 737 #define SLOCK_LIST_UNLOCK() \ 738 __cpu_simple_unlock(&simplelock_list_slock.lock_data) 739 740 #if defined(__HAVE_ATOMIC_OPERATIONS) /* { */ 741 #define SLOCK_COUNT(x) \ 742 atomic_add_ulong(&curcpu()->ci_simple_locks, (x)) 743 #else 744 #define SLOCK_COUNT(x) /* not safe */ 745 #endif /* __HAVE_ATOMIC_OPERATIONS */ /* } */ 746 #else 747 u_long simple_locks; 748 749 #define SLOCK_LIST_LOCK() /* nothing */ 750 751 #define SLOCK_LIST_UNLOCK() /* nothing */ 752 753 #define SLOCK_COUNT(x) simple_locks += (x) 754 #endif /* MULTIPROCESSOR */ /* } */ 755 756 #ifdef DDB /* { */ 757 int simple_lock_debugger = 0; 758 #define SLOCK_DEBUGGER() if (simple_lock_debugger) Debugger() 759 #else 760 #define SLOCK_DEBUGGER() /* nothing */ 761 #endif /* } */ 762 763 #ifdef MULTIPROCESSOR 764 #define SLOCK_MP() lock_printf("on cpu %d\n", cpu_number()) 765 #else 766 #define SLOCK_MP() /* nothing */ 767 #endif 768 769 #define SLOCK_WHERE(str, alp, id, l) \ 770 do { \ 771 lock_printf(str); \ 772 lock_printf("lock: %p, currently at: %s:%d\n", (alp), (id), (l)); \ 773 SLOCK_MP(); \ 774 if ((alp)->lock_file != NULL) \ 775 lock_printf("last locked: %s:%d\n", (alp)->lock_file, \ 776 (alp)->lock_line); \ 777 if ((alp)->unlock_file != NULL) \ 778 lock_printf("last unlocked: %s:%d\n", (alp)->unlock_file, \ 779 (alp)->unlock_line); \ 780 SLOCK_DEBUGGER(); \ 781 } while (/*CONSTCOND*/0) 782 783 /* 784 * Simple lock functions so that the debugger can see from whence 785 * they are being called. 786 */ 787 void 788 simple_lock_init(alp) 789 struct simplelock *alp; 790 { 791 792 #if defined(MULTIPROCESSOR) /* { */ 793 __cpu_simple_lock_init(&alp->lock_data); 794 #else 795 alp->lock_data = __SIMPLELOCK_UNLOCKED; 796 #endif /* } */ 797 alp->lock_file = NULL; 798 alp->lock_line = 0; 799 alp->unlock_file = NULL; 800 alp->unlock_line = 0; 801 alp->lock_holder = 0; 802 } 803 804 void 805 _simple_lock(alp, id, l) 806 __volatile struct simplelock *alp; 807 const char *id; 808 int l; 809 { 810 cpuid_t cpu_id = cpu_number(); 811 int s; 812 813 s = splhigh(); 814 815 /* 816 * MULTIPROCESSOR case: This is `safe' since if it's not us, we 817 * don't take any action, and just fall into the normal spin case. 818 */ 819 if (alp->lock_data == __SIMPLELOCK_LOCKED) { 820 #if defined(MULTIPROCESSOR) /* { */ 821 if (alp->lock_holder == cpu_id) { 822 SLOCK_WHERE("simple_lock: locking against myself\n", 823 alp, id, l); 824 goto out; 825 } 826 #else 827 SLOCK_WHERE("simple_lock: lock held\n", alp, id, l); 828 goto out; 829 #endif /* MULTIPROCESSOR */ /* } */ 830 } 831 832 #if defined(MULTIPROCESSOR) /* { */ 833 /* Acquire the lock before modifying any fields. */ 834 __cpu_simple_lock(&alp->lock_data); 835 #else 836 alp->lock_data = __SIMPLELOCK_LOCKED; 837 #endif /* } */ 838 839 alp->lock_file = id; 840 alp->lock_line = l; 841 alp->lock_holder = cpu_id; 842 843 SLOCK_LIST_LOCK(); 844 /* XXX Cast away volatile */ 845 TAILQ_INSERT_TAIL(&simplelock_list, (struct simplelock *)alp, list); 846 SLOCK_LIST_UNLOCK(); 847 848 SLOCK_COUNT(1); 849 850 out: 851 splx(s); 852 } 853 854 int 855 _simple_lock_try(alp, id, l) 856 __volatile struct simplelock *alp; 857 const char *id; 858 int l; 859 { 860 cpuid_t cpu_id = cpu_number(); 861 int s, rv = 0; 862 863 s = splhigh(); 864 865 /* 866 * MULTIPROCESSOR case: This is `safe' since if it's not us, we 867 * don't take any action. 868 */ 869 #if defined(MULTIPROCESSOR) /* { */ 870 if ((rv = __cpu_simple_lock_try(&alp->lock_data)) == 0) { 871 if (alp->lock_holder == cpu_id) 872 SLOCK_WHERE("simple_lock_try: locking against myself\n", 873 alp, id, l); 874 goto out; 875 } 876 #else 877 if (alp->lock_data == __SIMPLELOCK_LOCKED) { 878 SLOCK_WHERE("simple_lock_try: lock held\n", alp, id, l); 879 goto out; 880 } 881 alp->lock_data = __SIMPLELOCK_LOCKED; 882 #endif /* MULTIPROCESSOR */ /* } */ 883 884 /* 885 * At this point, we have acquired the lock. 886 */ 887 888 rv = 1; 889 890 alp->lock_file = id; 891 alp->lock_line = l; 892 alp->lock_holder = cpu_id; 893 894 SLOCK_LIST_LOCK(); 895 /* XXX Cast away volatile. */ 896 TAILQ_INSERT_TAIL(&simplelock_list, (struct simplelock *)alp, list); 897 SLOCK_LIST_UNLOCK(); 898 899 SLOCK_COUNT(1); 900 901 out: 902 splx(s); 903 return (rv); 904 } 905 906 void 907 _simple_unlock(alp, id, l) 908 __volatile struct simplelock *alp; 909 const char *id; 910 int l; 911 { 912 int s; 913 914 s = splhigh(); 915 916 /* 917 * MULTIPROCESSOR case: This is `safe' because we think we hold 918 * the lock, and if we don't, we don't take any action. 919 */ 920 if (alp->lock_data == __SIMPLELOCK_UNLOCKED) { 921 SLOCK_WHERE("simple_unlock: lock not held\n", 922 alp, id, l); 923 goto out; 924 } 925 926 SLOCK_LIST_LOCK(); 927 TAILQ_REMOVE(&simplelock_list, alp, list); 928 SLOCK_LIST_UNLOCK(); 929 930 SLOCK_COUNT(-1); 931 932 alp->list.tqe_next = NULL; /* sanity */ 933 alp->list.tqe_prev = NULL; /* sanity */ 934 935 alp->unlock_file = id; 936 alp->unlock_line = l; 937 938 #if defined(MULTIPROCESSOR) /* { */ 939 alp->lock_holder = LK_NOCPU; 940 /* Now that we've modified all fields, release the lock. */ 941 __cpu_simple_unlock(&alp->lock_data); 942 #else 943 alp->lock_data = __SIMPLELOCK_UNLOCKED; 944 #endif /* } */ 945 946 out: 947 splx(s); 948 } 949 950 void 951 simple_lock_dump() 952 { 953 struct simplelock *alp; 954 int s; 955 956 s = splhigh(); 957 SLOCK_LIST_LOCK(); 958 lock_printf("all simple locks:\n"); 959 for (alp = TAILQ_FIRST(&simplelock_list); alp != NULL; 960 alp = TAILQ_NEXT(alp, list)) { 961 lock_printf("%p CPU %lu %s:%d\n", alp, alp->lock_holder, 962 alp->lock_file, alp->lock_line); 963 } 964 SLOCK_LIST_UNLOCK(); 965 splx(s); 966 } 967 968 void 969 simple_lock_freecheck(start, end) 970 void *start, *end; 971 { 972 struct simplelock *alp; 973 int s; 974 975 s = splhigh(); 976 SLOCK_LIST_LOCK(); 977 for (alp = TAILQ_FIRST(&simplelock_list); alp != NULL; 978 alp = TAILQ_NEXT(alp, list)) { 979 if ((void *)alp >= start && (void *)alp < end) { 980 lock_printf("freeing simple_lock %p CPU %lu %s:%d\n", 981 alp, alp->lock_holder, alp->lock_file, 982 alp->lock_line); 983 SLOCK_DEBUGGER(); 984 } 985 } 986 SLOCK_LIST_UNLOCK(); 987 splx(s); 988 } 989 990 void 991 simple_lock_switchcheck(void) 992 { 993 struct simplelock *alp; 994 cpuid_t cpu_id = cpu_number(); 995 int s; 996 997 s = splhigh(); 998 SLOCK_LIST_LOCK(); 999 for (alp = TAILQ_FIRST(&simplelock_list); alp != NULL; 1000 alp = TAILQ_NEXT(alp, list)) { 1001 if (alp->lock_holder == cpu_id) { 1002 lock_printf("switching with held simple_lock %p " 1003 "CPU %lu %s:%d\n", 1004 alp, alp->lock_holder, alp->lock_file, 1005 alp->lock_line); 1006 SLOCK_DEBUGGER(); 1007 } 1008 } 1009 SLOCK_LIST_UNLOCK(); 1010 splx(s); 1011 } 1012 #endif /* LOCKDEBUG */ /* } */ 1013
Indexes created Sun Sep 28 07:09:56 GMT 2025