kern_lock.c revision 1.41
1 /* $NetBSD: kern_lock.c,v 1.41 2000/08/19 19:36:18 thorpej Exp $ */ 2 3 /*- 4 * Copyright (c) 1999, 2000 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center. 10 * 11 * This code is derived from software contributed to The NetBSD Foundation 12 * by Ross Harvey. 13 * 14 * Redistribution and use in source and binary forms, with or without 15 * modification, are permitted provided that the following conditions 16 * are met: 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in the 21 * documentation and/or other materials provided with the distribution. 22 * 3. All advertising materials mentioning features or use of this software 23 * must display the following acknowledgement: 24 * This product includes software developed by the NetBSD 25 * Foundation, Inc. and its contributors. 26 * 4. Neither the name of The NetBSD Foundation nor the names of its 27 * contributors may be used to endorse or promote products derived 28 * from this software without specific prior written permission. 29 * 30 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 31 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 32 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 33 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 34 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 35 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 36 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 37 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 38 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 39 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 40 * POSSIBILITY OF SUCH DAMAGE. 41 */ 42 43 /* 44 * Copyright (c) 1995 45 * The Regents of the University of California. All rights reserved. 46 * 47 * This code contains ideas from software contributed to Berkeley by 48 * Avadis Tevanian, Jr., Michael Wayne Young, and the Mach Operating 49 * System project at Carnegie-Mellon University. 50 * 51 * Redistribution and use in source and binary forms, with or without 52 * modification, are permitted provided that the following conditions 53 * are met: 54 * 1. Redistributions of source code must retain the above copyright 55 * notice, this list of conditions and the following disclaimer. 56 * 2. Redistributions in binary form must reproduce the above copyright 57 * notice, this list of conditions and the following disclaimer in the 58 * documentation and/or other materials provided with the distribution. 59 * 3. All advertising materials mentioning features or use of this software 60 * must display the following acknowledgement: 61 * This product includes software developed by the University of 62 * California, Berkeley and its contributors. 63 * 4. Neither the name of the University nor the names of its contributors 64 * may be used to endorse or promote products derived from this software 65 * without specific prior written permission. 66 * 67 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 68 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 69 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 70 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 71 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 72 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 73 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 74 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 75 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 76 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 77 * SUCH DAMAGE. 78 * 79 * @(#)kern_lock.c 8.18 (Berkeley) 5/21/95 80 */ 81 82 #include "opt_multiprocessor.h" 83 #include "opt_lockdebug.h" 84 #include "opt_ddb.h" 85 86 #include <sys/param.h> 87 #include <sys/proc.h> 88 #include <sys/lock.h> 89 #include <sys/systm.h> 90 #include <machine/cpu.h> 91 92 #if defined(__HAVE_ATOMIC_OPERATIONS) 93 #include <machine/atomic.h> 94 #endif 95 96 #if defined(LOCKDEBUG) 97 #include <sys/syslog.h> 98 /* 99 * note that stdarg.h and the ansi style va_start macro is used for both 100 * ansi and traditional c compiles. 101 * XXX: this requires that stdarg.h define: va_alist and va_dcl 102 */ 103 #include <machine/stdarg.h> 104 105 void lock_printf(const char *fmt, ...) 106 __attribute__((__format__(__printf__,1,2))); 107 108 int lock_debug_syslog = 0; /* defaults to printf, but can be patched */ 109 #endif 110 111 /* 112 * Locking primitives implementation. 113 * Locks provide shared/exclusive sychronization. 114 */ 115 116 #if defined(LOCKDEBUG) || defined(DIAGNOSTIC) /* { */ 117 #if defined(MULTIPROCESSOR) /* { */ 118 #if defined(__HAVE_ATOMIC_OPERATIONS) /* { */ 119 #define COUNT_CPU(cpu_id, x) \ 120 atomic_add_ulong(&curcpu()->ci_spin_locks, (x)) 121 #else 122 #define COUNT_CPU(cpu_id, x) /* not safe */ 123 #endif /* __HAVE_ATOMIC_OPERATIONS */ /* } */ 124 #else 125 u_long spin_locks; 126 #define COUNT_CPU(cpu_id, x) spin_locks += (x) 127 #endif /* MULTIPROCESSOR */ /* } */ 128 129 #define COUNT(lkp, p, cpu_id, x) \ 130 do { \ 131 if ((lkp)->lk_flags & LK_SPIN) \ 132 COUNT_CPU((cpu_id), (x)); \ 133 else \ 134 (p)->p_locks += (x); \ 135 } while (/*CONSTCOND*/0) 136 #else 137 #define COUNT(lkp, p, cpu_id, x) 138 #endif /* LOCKDEBUG || DIAGNOSTIC */ /* } */ 139 140 #define INTERLOCK_ACQUIRE(lkp, s) \ 141 do { \ 142 if ((lkp)->lk_flags & LK_SPIN) \ 143 s = splhigh(); \ 144 simple_lock(&(lkp)->lk_interlock); \ 145 } while (0) 146 147 #define INTERLOCK_RELEASE(lkp, s) \ 148 do { \ 149 simple_unlock(&(lkp)->lk_interlock); \ 150 if ((lkp)->lk_flags & LK_SPIN) \ 151 splx(s); \ 152 } while (0) 153 154 /* 155 * Acquire a resource. 156 */ 157 #define ACQUIRE(lkp, error, extflags, drain, wanted) \ 158 if ((extflags) & LK_SPIN) { \ 159 int interlocked; \ 160 \ 161 if ((drain) == 0) \ 162 (lkp)->lk_waitcount++; \ 163 for (interlocked = 1;;) { \ 164 if (wanted) { \ 165 if (interlocked) { \ 166 INTERLOCK_RELEASE((lkp), s); \ 167 interlocked = 0; \ 168 } \ 169 } else if (interlocked) { \ 170 break; \ 171 } else { \ 172 INTERLOCK_ACQUIRE((lkp), s); \ 173 interlocked = 1; \ 174 } \ 175 } \ 176 if ((drain) == 0) \ 177 (lkp)->lk_waitcount--; \ 178 KASSERT((wanted) == 0); \ 179 error = 0; /* sanity */ \ 180 } else { \ 181 for (error = 0; wanted; ) { \ 182 if ((drain)) \ 183 (lkp)->lk_flags |= LK_WAITDRAIN; \ 184 else \ 185 (lkp)->lk_waitcount++; \ 186 /* XXX Cast away volatile. */ \ 187 error = ltsleep((drain) ? &(lkp)->lk_flags : \ 188 (void *)(lkp), (lkp)->lk_prio, \ 189 (lkp)->lk_wmesg, (lkp)->lk_timo, \ 190 &(lkp)->lk_interlock); \ 191 if ((drain) == 0) \ 192 (lkp)->lk_waitcount--; \ 193 if (error) \ 194 break; \ 195 if ((extflags) & LK_SLEEPFAIL) { \ 196 error = ENOLCK; \ 197 break; \ 198 } \ 199 } \ 200 } 201 202 #define SETHOLDER(lkp, pid, cpu_id) \ 203 do { \ 204 if ((lkp)->lk_flags & LK_SPIN) \ 205 (lkp)->lk_cpu = cpu_id; \ 206 else \ 207 (lkp)->lk_lockholder = pid; \ 208 } while (/*CONSTCOND*/0) 209 210 #define WEHOLDIT(lkp, pid, cpu_id) \ 211 (((lkp)->lk_flags & LK_SPIN) != 0 ? \ 212 ((lkp)->lk_cpu == (cpu_id)) : ((lkp)->lk_lockholder == (pid))) 213 214 #define WAKEUP_WAITER(lkp) \ 215 do { \ 216 if (((lkp)->lk_flags & LK_SPIN) == 0 && (lkp)->lk_waitcount) { \ 217 /* XXX Cast away volatile. */ \ 218 wakeup_one((void *)(lkp)); \ 219 } \ 220 } while (/*CONSTCOND*/0) 221 222 #if defined(LOCKDEBUG) /* { */ 223 #if defined(MULTIPROCESSOR) /* { */ 224 struct simplelock spinlock_list_slock = SIMPLELOCK_INITIALIZER; 225 226 #define SPINLOCK_LIST_LOCK() \ 227 __cpu_simple_lock(&spinlock_list_slock.lock_data) 228 229 #define SPINLOCK_LIST_UNLOCK() \ 230 __cpu_simple_unlock(&spinlock_list_slock.lock_data) 231 #else 232 #define SPINLOCK_LIST_LOCK() /* nothing */ 233 234 #define SPINLOCK_LIST_UNLOCK() /* nothing */ 235 #endif /* MULTIPROCESSOR */ /* } */ 236 237 TAILQ_HEAD(, lock) spinlock_list = 238 TAILQ_HEAD_INITIALIZER(spinlock_list); 239 240 #define HAVEIT(lkp) \ 241 do { \ 242 if ((lkp)->lk_flags & LK_SPIN) { \ 243 int s = splhigh(); \ 244 SPINLOCK_LIST_LOCK(); \ 245 /* XXX Cast away volatile. */ \ 246 TAILQ_INSERT_TAIL(&spinlock_list, (struct lock *)(lkp), \ 247 lk_list); \ 248 SPINLOCK_LIST_UNLOCK(); \ 249 splx(s); \ 250 } \ 251 } while (/*CONSTCOND*/0) 252 253 #define DONTHAVEIT(lkp) \ 254 do { \ 255 if ((lkp)->lk_flags & LK_SPIN) { \ 256 int s = splhigh(); \ 257 SPINLOCK_LIST_LOCK(); \ 258 /* XXX Cast away volatile. */ \ 259 TAILQ_REMOVE(&spinlock_list, (struct lock *)(lkp), \ 260 lk_list); \ 261 SPINLOCK_LIST_UNLOCK(); \ 262 splx(s); \ 263 } \ 264 } while (/*CONSTCOND*/0) 265 #else 266 #define HAVEIT(lkp) /* nothing */ 267 268 #define DONTHAVEIT(lkp) /* nothing */ 269 #endif /* LOCKDEBUG */ /* } */ 270 271 #if defined(LOCKDEBUG) 272 /* 273 * Lock debug printing routine; can be configured to print to console 274 * or log to syslog. 275 */ 276 void 277 lock_printf(const char *fmt, ...) 278 { 279 va_list ap; 280 281 va_start(ap, fmt); 282 if (lock_debug_syslog) 283 vlog(LOG_DEBUG, fmt, ap); 284 else 285 vprintf(fmt, ap); 286 va_end(ap); 287 } 288 #endif /* LOCKDEBUG */ 289 290 /* 291 * Initialize a lock; required before use. 292 */ 293 void 294 lockinit(struct lock *lkp, int prio, const char *wmesg, int timo, int flags) 295 { 296 297 memset(lkp, 0, sizeof(struct lock)); 298 simple_lock_init(&lkp->lk_interlock); 299 lkp->lk_flags = flags & LK_EXTFLG_MASK; 300 if (flags & LK_SPIN) 301 lkp->lk_cpu = LK_NOCPU; 302 else { 303 lkp->lk_lockholder = LK_NOPROC; 304 lkp->lk_prio = prio; 305 lkp->lk_timo = timo; 306 } 307 lkp->lk_wmesg = wmesg; /* just a name for spin locks */ 308 } 309 310 /* 311 * Determine the status of a lock. 312 */ 313 int 314 lockstatus(struct lock *lkp) 315 { 316 int s, lock_type = 0; 317 318 INTERLOCK_ACQUIRE(lkp, s); 319 if (lkp->lk_exclusivecount != 0) 320 lock_type = LK_EXCLUSIVE; 321 else if (lkp->lk_sharecount != 0) 322 lock_type = LK_SHARED; 323 INTERLOCK_RELEASE(lkp, s); 324 return (lock_type); 325 } 326 327 #if defined(LOCKDEBUG) || defined(DIAGNOSTIC) 328 /* 329 * Make sure no spin locks are held by a CPU that is about 330 * to context switch. 331 */ 332 void 333 spinlock_switchcheck(void) 334 { 335 u_long cnt; 336 int s; 337 338 s = splhigh(); 339 #if defined(MULTIPROCESSOR) 340 cnt = curcpu()->ci_spin_locks; 341 #else 342 cnt = spin_locks; 343 #endif 344 splx(s); 345 346 if (cnt != 0) 347 panic("spinlock_switchcheck: CPU %lu has %lu spin locks", 348 (u_long) cpu_number(), cnt); 349 } 350 #endif /* LOCKDEBUG || DIAGNOSTIC */ 351 352 /* 353 * XXX XXX kludge around another kludge.. 354 * 355 * vfs_shutdown() may be called from interrupt context, either as a result 356 * of a panic, or from the debugger. It proceeds to call 357 * sys_sync(&proc0, ...), pretending its running on behalf of proc0 358 * 359 * We would like to make an attempt to sync the filesystems in this case, so 360 * if this happens, we treat attempts to acquire locks specially. 361 * All locks are acquired on behalf of proc0. 362 * 363 * If we've already paniced, we don't block waiting for locks, but 364 * just barge right ahead since we're already going down in flames. 365 */ 366 367 /* 368 * Set, change, or release a lock. 369 * 370 * Shared requests increment the shared count. Exclusive requests set the 371 * LK_WANT_EXCL flag (preventing further shared locks), and wait for already 372 * accepted shared locks and shared-to-exclusive upgrades to go away. 373 */ 374 int 375 lockmgr(__volatile struct lock *lkp, u_int flags, 376 struct simplelock *interlkp) 377 { 378 int error; 379 pid_t pid; 380 int extflags; 381 cpuid_t cpu_id; 382 struct proc *p = curproc; 383 int lock_shutdown_noblock = 0; 384 int s; 385 386 error = 0; 387 388 INTERLOCK_ACQUIRE(lkp, s); 389 if (flags & LK_INTERLOCK) 390 simple_unlock(interlkp); 391 extflags = (flags | lkp->lk_flags) & LK_EXTFLG_MASK; 392 393 #ifdef DIAGNOSTIC /* { */ 394 /* 395 * Don't allow spins on sleep locks and don't allow sleeps 396 * on spin locks. 397 */ 398 if ((flags ^ lkp->lk_flags) & LK_SPIN) 399 panic("lockmgr: sleep/spin mismatch\n"); 400 #endif /* } */ 401 402 if (extflags & LK_SPIN) 403 pid = LK_KERNPROC; 404 else { 405 if (p == NULL) { 406 if (!doing_shutdown) { 407 #ifdef DIAGNOSTIC 408 panic("lockmgr: no context"); 409 #endif 410 } else { 411 p = &proc0; 412 if (panicstr && (!(flags & LK_NOWAIT))) { 413 flags |= LK_NOWAIT; 414 lock_shutdown_noblock = 1; 415 } 416 } 417 } 418 pid = p->p_pid; 419 } 420 cpu_id = cpu_number(); 421 422 /* 423 * Once a lock has drained, the LK_DRAINING flag is set and an 424 * exclusive lock is returned. The only valid operation thereafter 425 * is a single release of that exclusive lock. This final release 426 * clears the LK_DRAINING flag and sets the LK_DRAINED flag. Any 427 * further requests of any sort will result in a panic. The bits 428 * selected for these two flags are chosen so that they will be set 429 * in memory that is freed (freed memory is filled with 0xdeadbeef). 430 * The final release is permitted to give a new lease on life to 431 * the lock by specifying LK_REENABLE. 432 */ 433 if (lkp->lk_flags & (LK_DRAINING|LK_DRAINED)) { 434 #ifdef DIAGNOSTIC /* { */ 435 if (lkp->lk_flags & LK_DRAINED) 436 panic("lockmgr: using decommissioned lock"); 437 if ((flags & LK_TYPE_MASK) != LK_RELEASE || 438 WEHOLDIT(lkp, pid, cpu_id) == 0) 439 panic("lockmgr: non-release on draining lock: %d\n", 440 flags & LK_TYPE_MASK); 441 #endif /* DIAGNOSTIC */ /* } */ 442 lkp->lk_flags &= ~LK_DRAINING; 443 if ((flags & LK_REENABLE) == 0) 444 lkp->lk_flags |= LK_DRAINED; 445 } 446 447 switch (flags & LK_TYPE_MASK) { 448 449 case LK_SHARED: 450 if (WEHOLDIT(lkp, pid, cpu_id) == 0) { 451 /* 452 * If just polling, check to see if we will block. 453 */ 454 if ((extflags & LK_NOWAIT) && (lkp->lk_flags & 455 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE))) { 456 error = EBUSY; 457 break; 458 } 459 /* 460 * Wait for exclusive locks and upgrades to clear. 461 */ 462 ACQUIRE(lkp, error, extflags, 0, lkp->lk_flags & 463 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE)); 464 if (error) 465 break; 466 lkp->lk_sharecount++; 467 COUNT(lkp, p, cpu_id, 1); 468 break; 469 } 470 /* 471 * We hold an exclusive lock, so downgrade it to shared. 472 * An alternative would be to fail with EDEADLK. 473 */ 474 lkp->lk_sharecount++; 475 COUNT(lkp, p, cpu_id, 1); 476 /* fall into downgrade */ 477 478 case LK_DOWNGRADE: 479 if (WEHOLDIT(lkp, pid, cpu_id) == 0 || 480 lkp->lk_exclusivecount == 0) 481 panic("lockmgr: not holding exclusive lock"); 482 lkp->lk_sharecount += lkp->lk_exclusivecount; 483 lkp->lk_exclusivecount = 0; 484 lkp->lk_recurselevel = 0; 485 lkp->lk_flags &= ~LK_HAVE_EXCL; 486 SETHOLDER(lkp, LK_NOPROC, LK_NOCPU); 487 DONTHAVEIT(lkp); 488 WAKEUP_WAITER(lkp); 489 break; 490 491 case LK_EXCLUPGRADE: 492 /* 493 * If another process is ahead of us to get an upgrade, 494 * then we want to fail rather than have an intervening 495 * exclusive access. 496 */ 497 if (lkp->lk_flags & LK_WANT_UPGRADE) { 498 lkp->lk_sharecount--; 499 COUNT(lkp, p, cpu_id, -1); 500 error = EBUSY; 501 break; 502 } 503 /* fall into normal upgrade */ 504 505 case LK_UPGRADE: 506 /* 507 * Upgrade a shared lock to an exclusive one. If another 508 * shared lock has already requested an upgrade to an 509 * exclusive lock, our shared lock is released and an 510 * exclusive lock is requested (which will be granted 511 * after the upgrade). If we return an error, the file 512 * will always be unlocked. 513 */ 514 if (WEHOLDIT(lkp, pid, cpu_id) || lkp->lk_sharecount <= 0) 515 panic("lockmgr: upgrade exclusive lock"); 516 lkp->lk_sharecount--; 517 COUNT(lkp, p, cpu_id, -1); 518 /* 519 * If we are just polling, check to see if we will block. 520 */ 521 if ((extflags & LK_NOWAIT) && 522 ((lkp->lk_flags & LK_WANT_UPGRADE) || 523 lkp->lk_sharecount > 1)) { 524 error = EBUSY; 525 break; 526 } 527 if ((lkp->lk_flags & LK_WANT_UPGRADE) == 0) { 528 /* 529 * We are first shared lock to request an upgrade, so 530 * request upgrade and wait for the shared count to 531 * drop to zero, then take exclusive lock. 532 */ 533 lkp->lk_flags |= LK_WANT_UPGRADE; 534 ACQUIRE(lkp, error, extflags, 0, lkp->lk_sharecount); 535 lkp->lk_flags &= ~LK_WANT_UPGRADE; 536 if (error) 537 break; 538 lkp->lk_flags |= LK_HAVE_EXCL; 539 SETHOLDER(lkp, pid, cpu_id); 540 HAVEIT(lkp); 541 if (lkp->lk_exclusivecount != 0) 542 panic("lockmgr: non-zero exclusive count"); 543 lkp->lk_exclusivecount = 1; 544 if (extflags & LK_SETRECURSE) 545 lkp->lk_recurselevel = 1; 546 COUNT(lkp, p, cpu_id, 1); 547 break; 548 } 549 /* 550 * Someone else has requested upgrade. Release our shared 551 * lock, awaken upgrade requestor if we are the last shared 552 * lock, then request an exclusive lock. 553 */ 554 if (lkp->lk_sharecount == 0) 555 WAKEUP_WAITER(lkp); 556 /* fall into exclusive request */ 557 558 case LK_EXCLUSIVE: 559 if (WEHOLDIT(lkp, pid, cpu_id)) { 560 /* 561 * Recursive lock. 562 */ 563 if ((extflags & LK_CANRECURSE) == 0 && 564 lkp->lk_recurselevel == 0) { 565 if (extflags & LK_RECURSEFAIL) { 566 error = EDEADLK; 567 break; 568 } else 569 panic("lockmgr: locking against myself"); 570 } 571 lkp->lk_exclusivecount++; 572 if (extflags & LK_SETRECURSE && 573 lkp->lk_recurselevel == 0) 574 lkp->lk_recurselevel = lkp->lk_exclusivecount; 575 COUNT(lkp, p, cpu_id, 1); 576 break; 577 } 578 /* 579 * If we are just polling, check to see if we will sleep. 580 */ 581 if ((extflags & LK_NOWAIT) && ((lkp->lk_flags & 582 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE)) || 583 lkp->lk_sharecount != 0)) { 584 error = EBUSY; 585 break; 586 } 587 /* 588 * Try to acquire the want_exclusive flag. 589 */ 590 ACQUIRE(lkp, error, extflags, 0, lkp->lk_flags & 591 (LK_HAVE_EXCL | LK_WANT_EXCL)); 592 if (error) 593 break; 594 lkp->lk_flags |= LK_WANT_EXCL; 595 /* 596 * Wait for shared locks and upgrades to finish. 597 */ 598 ACQUIRE(lkp, error, extflags, 0, lkp->lk_sharecount != 0 || 599 (lkp->lk_flags & LK_WANT_UPGRADE)); 600 lkp->lk_flags &= ~LK_WANT_EXCL; 601 if (error) 602 break; 603 lkp->lk_flags |= LK_HAVE_EXCL; 604 SETHOLDER(lkp, pid, cpu_id); 605 HAVEIT(lkp); 606 if (lkp->lk_exclusivecount != 0) 607 panic("lockmgr: non-zero exclusive count"); 608 lkp->lk_exclusivecount = 1; 609 if (extflags & LK_SETRECURSE) 610 lkp->lk_recurselevel = 1; 611 COUNT(lkp, p, cpu_id, 1); 612 break; 613 614 case LK_RELEASE: 615 if (lkp->lk_exclusivecount != 0) { 616 if (WEHOLDIT(lkp, pid, cpu_id) == 0) { 617 if (lkp->lk_flags & LK_SPIN) { 618 panic("lockmgr: processor %lu, not " 619 "exclusive lock holder %lu " 620 "unlocking", cpu_id, lkp->lk_cpu); 621 } else { 622 panic("lockmgr: pid %d, not " 623 "exclusive lock holder %d " 624 "unlocking", pid, 625 lkp->lk_lockholder); 626 } 627 } 628 if (lkp->lk_exclusivecount == lkp->lk_recurselevel) 629 lkp->lk_recurselevel = 0; 630 lkp->lk_exclusivecount--; 631 COUNT(lkp, p, cpu_id, -1); 632 if (lkp->lk_exclusivecount == 0) { 633 lkp->lk_flags &= ~LK_HAVE_EXCL; 634 SETHOLDER(lkp, LK_NOPROC, LK_NOCPU); 635 DONTHAVEIT(lkp); 636 } 637 } else if (lkp->lk_sharecount != 0) { 638 lkp->lk_sharecount--; 639 COUNT(lkp, p, cpu_id, -1); 640 } 641 #ifdef DIAGNOSTIC 642 else 643 panic("lockmgr: release of unlocked lock!"); 644 #endif 645 WAKEUP_WAITER(lkp); 646 break; 647 648 case LK_DRAIN: 649 /* 650 * Check that we do not already hold the lock, as it can 651 * never drain if we do. Unfortunately, we have no way to 652 * check for holding a shared lock, but at least we can 653 * check for an exclusive one. 654 */ 655 if (WEHOLDIT(lkp, pid, cpu_id)) 656 panic("lockmgr: draining against myself"); 657 /* 658 * If we are just polling, check to see if we will sleep. 659 */ 660 if ((extflags & LK_NOWAIT) && ((lkp->lk_flags & 661 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE)) || 662 lkp->lk_sharecount != 0 || lkp->lk_waitcount != 0)) { 663 error = EBUSY; 664 break; 665 } 666 ACQUIRE(lkp, error, extflags, 1, 667 ((lkp->lk_flags & 668 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE)) || 669 lkp->lk_sharecount != 0 || 670 lkp->lk_waitcount != 0)); 671 if (error) 672 break; 673 lkp->lk_flags |= LK_DRAINING | LK_HAVE_EXCL; 674 SETHOLDER(lkp, pid, cpu_id); 675 HAVEIT(lkp); 676 lkp->lk_exclusivecount = 1; 677 /* XXX unlikely that we'd want this */ 678 if (extflags & LK_SETRECURSE) 679 lkp->lk_recurselevel = 1; 680 COUNT(lkp, p, cpu_id, 1); 681 break; 682 683 default: 684 INTERLOCK_RELEASE(lkp, s); 685 panic("lockmgr: unknown locktype request %d", 686 flags & LK_TYPE_MASK); 687 /* NOTREACHED */ 688 } 689 if ((lkp->lk_flags & (LK_WAITDRAIN|LK_SPIN)) == LK_WAITDRAIN && 690 ((lkp->lk_flags & 691 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE)) == 0 && 692 lkp->lk_sharecount == 0 && lkp->lk_waitcount == 0)) { 693 lkp->lk_flags &= ~LK_WAITDRAIN; 694 wakeup_one((void *)&lkp->lk_flags); 695 } 696 /* 697 * Note that this panic will be a recursive panic, since 698 * we only set lock_shutdown_noblock above if panicstr != NULL. 699 */ 700 if (error && lock_shutdown_noblock) 701 panic("lockmgr: deadlock (see previous panic)"); 702 703 INTERLOCK_RELEASE(lkp, s); 704 return (error); 705 } 706 707 /* 708 * Print out information about state of a lock. Used by VOP_PRINT 709 * routines to display ststus about contained locks. 710 */ 711 void 712 lockmgr_printinfo(__volatile struct lock *lkp) 713 { 714 715 if (lkp->lk_sharecount) 716 printf(" lock type %s: SHARED (count %d)", lkp->lk_wmesg, 717 lkp->lk_sharecount); 718 else if (lkp->lk_flags & LK_HAVE_EXCL) { 719 printf(" lock type %s: EXCL (count %d) by ", 720 lkp->lk_wmesg, lkp->lk_exclusivecount); 721 if (lkp->lk_flags & LK_SPIN) 722 printf("processor %lu", lkp->lk_cpu); 723 else 724 printf("pid %d", lkp->lk_lockholder); 725 } else 726 printf(" not locked"); 727 if ((lkp->lk_flags & LK_SPIN) == 0 && lkp->lk_waitcount > 0) 728 printf(" with %d pending", lkp->lk_waitcount); 729 } 730 731 #if defined(LOCKDEBUG) /* { */ 732 TAILQ_HEAD(, simplelock) simplelock_list = 733 TAILQ_HEAD_INITIALIZER(simplelock_list); 734 735 #if defined(MULTIPROCESSOR) /* { */ 736 struct simplelock simplelock_list_slock = SIMPLELOCK_INITIALIZER; 737 738 #define SLOCK_LIST_LOCK() \ 739 __cpu_simple_lock(&simplelock_list_slock.lock_data) 740 741 #define SLOCK_LIST_UNLOCK() \ 742 __cpu_simple_unlock(&simplelock_list_slock.lock_data) 743 744 #if defined(__HAVE_ATOMIC_OPERATIONS) /* { */ 745 #define SLOCK_COUNT(x) \ 746 atomic_add_ulong(&curcpu()->ci_simple_locks, (x)) 747 #else 748 #define SLOCK_COUNT(x) /* not safe */ 749 #endif /* __HAVE_ATOMIC_OPERATIONS */ /* } */ 750 #else 751 u_long simple_locks; 752 753 #define SLOCK_LIST_LOCK() /* nothing */ 754 755 #define SLOCK_LIST_UNLOCK() /* nothing */ 756 757 #define SLOCK_COUNT(x) simple_locks += (x) 758 #endif /* MULTIPROCESSOR */ /* } */ 759 760 #ifdef DDB /* { */ 761 int simple_lock_debugger = 0; 762 #define SLOCK_DEBUGGER() if (simple_lock_debugger) Debugger() 763 #else 764 #define SLOCK_DEBUGGER() /* nothing */ 765 #endif /* } */ 766 767 #ifdef MULTIPROCESSOR 768 #define SLOCK_MP() lock_printf("on cpu %d\n", cpu_number()) 769 #else 770 #define SLOCK_MP() /* nothing */ 771 #endif 772 773 #define SLOCK_WHERE(str, alp, id, l) \ 774 do { \ 775 lock_printf(str); \ 776 lock_printf("lock: %p, currently at: %s:%d\n", (alp), (id), (l)); \ 777 SLOCK_MP(); \ 778 if ((alp)->lock_file != NULL) \ 779 lock_printf("last locked: %s:%d\n", (alp)->lock_file, \ 780 (alp)->lock_line); \ 781 if ((alp)->unlock_file != NULL) \ 782 lock_printf("last unlocked: %s:%d\n", (alp)->unlock_file, \ 783 (alp)->unlock_line); \ 784 SLOCK_DEBUGGER(); \ 785 } while (/*CONSTCOND*/0) 786 787 /* 788 * Simple lock functions so that the debugger can see from whence 789 * they are being called. 790 */ 791 void 792 simple_lock_init(struct simplelock *alp) 793 { 794 795 #if defined(MULTIPROCESSOR) /* { */ 796 __cpu_simple_lock_init(&alp->lock_data); 797 #else 798 alp->lock_data = __SIMPLELOCK_UNLOCKED; 799 #endif /* } */ 800 alp->lock_file = NULL; 801 alp->lock_line = 0; 802 alp->unlock_file = NULL; 803 alp->unlock_line = 0; 804 alp->lock_holder = LK_NOCPU; 805 } 806 807 void 808 _simple_lock(__volatile struct simplelock *alp, const char *id, int l) 809 { 810 cpuid_t cpu_id = cpu_number(); 811 int s; 812 813 s = splhigh(); 814 815 /* 816 * MULTIPROCESSOR case: This is `safe' since if it's not us, we 817 * don't take any action, and just fall into the normal spin case. 818 */ 819 if (alp->lock_data == __SIMPLELOCK_LOCKED) { 820 #if defined(MULTIPROCESSOR) /* { */ 821 if (alp->lock_holder == cpu_id) { 822 SLOCK_WHERE("simple_lock: locking against myself\n", 823 alp, id, l); 824 goto out; 825 } 826 #else 827 SLOCK_WHERE("simple_lock: lock held\n", alp, id, l); 828 goto out; 829 #endif /* MULTIPROCESSOR */ /* } */ 830 } 831 832 #if defined(MULTIPROCESSOR) /* { */ 833 /* Acquire the lock before modifying any fields. */ 834 __cpu_simple_lock(&alp->lock_data); 835 #else 836 alp->lock_data = __SIMPLELOCK_LOCKED; 837 #endif /* } */ 838 839 KASSERT(alp->lock_holder == LK_NOCPU); 840 841 alp->lock_file = id; 842 alp->lock_line = l; 843 alp->lock_holder = cpu_id; 844 845 SLOCK_LIST_LOCK(); 846 /* XXX Cast away volatile */ 847 TAILQ_INSERT_TAIL(&simplelock_list, (struct simplelock *)alp, list); 848 SLOCK_LIST_UNLOCK(); 849 850 SLOCK_COUNT(1); 851 852 out: 853 splx(s); 854 } 855 856 int 857 _simple_lock_held(__volatile struct simplelock *alp) 858 { 859 #if defined(MULTIPROCESSOR) 860 cpuid_t cpu_id = cpu_number(); 861 int s, locked = 0; 862 863 s = splhigh(); 864 if (__cpu_simple_lock_try(&alp->lock_data) == 0) 865 locked = (alp->lock_holder == cpu_id); 866 else 867 __cpu_simple_unlock(&alp->lock_data); 868 splx(s); 869 #else 870 int s, locked; 871 872 s = splhigh(); 873 locked = (alp->lock_data == __SIMPLELOCK_LOCKED); 874 KASSERT(alp->lock_holder == cpu_number()); 875 splx(s); 876 #endif 877 return (locked); 878 } 879 880 int 881 _simple_lock_try(__volatile struct simplelock *alp, const char *id, int l) 882 { 883 cpuid_t cpu_id = cpu_number(); 884 int s, rv = 0; 885 886 s = splhigh(); 887 888 /* 889 * MULTIPROCESSOR case: This is `safe' since if it's not us, we 890 * don't take any action. 891 */ 892 #if defined(MULTIPROCESSOR) /* { */ 893 if ((rv = __cpu_simple_lock_try(&alp->lock_data)) == 0) { 894 if (alp->lock_holder == cpu_id) 895 SLOCK_WHERE("simple_lock_try: locking against myself\n", 896 alp, id, l); 897 goto out; 898 } 899 #else 900 if (alp->lock_data == __SIMPLELOCK_LOCKED) { 901 SLOCK_WHERE("simple_lock_try: lock held\n", alp, id, l); 902 goto out; 903 } 904 alp->lock_data = __SIMPLELOCK_LOCKED; 905 #endif /* MULTIPROCESSOR */ /* } */ 906 907 /* 908 * At this point, we have acquired the lock. 909 */ 910 911 rv = 1; 912 913 alp->lock_file = id; 914 alp->lock_line = l; 915 alp->lock_holder = cpu_id; 916 917 SLOCK_LIST_LOCK(); 918 /* XXX Cast away volatile. */ 919 TAILQ_INSERT_TAIL(&simplelock_list, (struct simplelock *)alp, list); 920 SLOCK_LIST_UNLOCK(); 921 922 SLOCK_COUNT(1); 923 924 out: 925 splx(s); 926 return (rv); 927 } 928 929 void 930 _simple_unlock(__volatile struct simplelock *alp, const char *id, int l) 931 { 932 int s; 933 934 s = splhigh(); 935 936 /* 937 * MULTIPROCESSOR case: This is `safe' because we think we hold 938 * the lock, and if we don't, we don't take any action. 939 */ 940 if (alp->lock_data == __SIMPLELOCK_UNLOCKED) { 941 SLOCK_WHERE("simple_unlock: lock not held\n", 942 alp, id, l); 943 goto out; 944 } 945 946 SLOCK_LIST_LOCK(); 947 TAILQ_REMOVE(&simplelock_list, alp, list); 948 SLOCK_LIST_UNLOCK(); 949 950 SLOCK_COUNT(-1); 951 952 alp->list.tqe_next = NULL; /* sanity */ 953 alp->list.tqe_prev = NULL; /* sanity */ 954 955 alp->unlock_file = id; 956 alp->unlock_line = l; 957 958 #if defined(MULTIPROCESSOR) /* { */ 959 alp->lock_holder = LK_NOCPU; 960 /* Now that we've modified all fields, release the lock. */ 961 __cpu_simple_unlock(&alp->lock_data); 962 #else 963 alp->lock_data = __SIMPLELOCK_UNLOCKED; 964 KASSERT(alp->lock_holder == cpu_number()); 965 alp->lock_holder = LK_NOCPU; 966 #endif /* } */ 967 968 out: 969 splx(s); 970 } 971 972 void 973 simple_lock_dump(void) 974 { 975 struct simplelock *alp; 976 int s; 977 978 s = splhigh(); 979 SLOCK_LIST_LOCK(); 980 lock_printf("all simple locks:\n"); 981 for (alp = TAILQ_FIRST(&simplelock_list); alp != NULL; 982 alp = TAILQ_NEXT(alp, list)) { 983 lock_printf("%p CPU %lu %s:%d\n", alp, alp->lock_holder, 984 alp->lock_file, alp->lock_line); 985 } 986 SLOCK_LIST_UNLOCK(); 987 splx(s); 988 } 989 990 void 991 simple_lock_freecheck(void *start, void *end) 992 { 993 struct simplelock *alp; 994 int s; 995 996 s = splhigh(); 997 SLOCK_LIST_LOCK(); 998 for (alp = TAILQ_FIRST(&simplelock_list); alp != NULL; 999 alp = TAILQ_NEXT(alp, list)) { 1000 if ((void *)alp >= start && (void *)alp < end) { 1001 lock_printf("freeing simple_lock %p CPU %lu %s:%d\n", 1002 alp, alp->lock_holder, alp->lock_file, 1003 alp->lock_line); 1004 SLOCK_DEBUGGER(); 1005 } 1006 } 1007 SLOCK_LIST_UNLOCK(); 1008 splx(s); 1009 } 1010 1011 void 1012 simple_lock_switchcheck(void) 1013 { 1014 struct simplelock *alp; 1015 cpuid_t cpu_id = cpu_number(); 1016 int s; 1017 1018 s = splhigh(); 1019 SLOCK_LIST_LOCK(); 1020 for (alp = TAILQ_FIRST(&simplelock_list); alp != NULL; 1021 alp = TAILQ_NEXT(alp, list)) { 1022 if (alp->lock_holder == cpu_id) { 1023 lock_printf("switching with held simple_lock %p " 1024 "CPU %lu %s:%d\n", 1025 alp, alp->lock_holder, alp->lock_file, 1026 alp->lock_line); 1027 SLOCK_DEBUGGER(); 1028 } 1029 } 1030 SLOCK_LIST_UNLOCK(); 1031 splx(s); 1032 } 1033 #endif /* LOCKDEBUG */ /* } */ 1034
Indexes created Mon Sep 22 21:09:42 GMT 2025