kern_lock.c revision 1.43
1 /* $NetBSD: kern_lock.c,v 1.43 2000/08/22 17:31:32 thorpej Exp $ */ 2 3 /*- 4 * Copyright (c) 1999, 2000 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center. 10 * 11 * This code is derived from software contributed to The NetBSD Foundation 12 * by Ross Harvey. 13 * 14 * Redistribution and use in source and binary forms, with or without 15 * modification, are permitted provided that the following conditions 16 * are met: 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in the 21 * documentation and/or other materials provided with the distribution. 22 * 3. All advertising materials mentioning features or use of this software 23 * must display the following acknowledgement: 24 * This product includes software developed by the NetBSD 25 * Foundation, Inc. and its contributors. 26 * 4. Neither the name of The NetBSD Foundation nor the names of its 27 * contributors may be used to endorse or promote products derived 28 * from this software without specific prior written permission. 29 * 30 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 31 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 32 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 33 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 34 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 35 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 36 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 37 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 38 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 39 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 40 * POSSIBILITY OF SUCH DAMAGE. 41 */ 42 43 /* 44 * Copyright (c) 1995 45 * The Regents of the University of California. All rights reserved. 46 * 47 * This code contains ideas from software contributed to Berkeley by 48 * Avadis Tevanian, Jr., Michael Wayne Young, and the Mach Operating 49 * System project at Carnegie-Mellon University. 50 * 51 * Redistribution and use in source and binary forms, with or without 52 * modification, are permitted provided that the following conditions 53 * are met: 54 * 1. Redistributions of source code must retain the above copyright 55 * notice, this list of conditions and the following disclaimer. 56 * 2. Redistributions in binary form must reproduce the above copyright 57 * notice, this list of conditions and the following disclaimer in the 58 * documentation and/or other materials provided with the distribution. 59 * 3. All advertising materials mentioning features or use of this software 60 * must display the following acknowledgement: 61 * This product includes software developed by the University of 62 * California, Berkeley and its contributors. 63 * 4. Neither the name of the University nor the names of its contributors 64 * may be used to endorse or promote products derived from this software 65 * without specific prior written permission. 66 * 67 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 68 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 69 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 70 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 71 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 72 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 73 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 74 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 75 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 76 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 77 * SUCH DAMAGE. 78 * 79 * @(#)kern_lock.c 8.18 (Berkeley) 5/21/95 80 */ 81 82 #include "opt_multiprocessor.h" 83 #include "opt_lockdebug.h" 84 #include "opt_ddb.h" 85 86 #include <sys/param.h> 87 #include <sys/proc.h> 88 #include <sys/lock.h> 89 #include <sys/systm.h> 90 #include <machine/cpu.h> 91 92 #if defined(__HAVE_ATOMIC_OPERATIONS) 93 #include <machine/atomic.h> 94 #endif 95 96 #if defined(LOCKDEBUG) 97 #include <sys/syslog.h> 98 /* 99 * note that stdarg.h and the ansi style va_start macro is used for both 100 * ansi and traditional c compiles. 101 * XXX: this requires that stdarg.h define: va_alist and va_dcl 102 */ 103 #include <machine/stdarg.h> 104 105 void lock_printf(const char *fmt, ...) 106 __attribute__((__format__(__printf__,1,2))); 107 108 int lock_debug_syslog = 0; /* defaults to printf, but can be patched */ 109 #endif 110 111 /* 112 * Locking primitives implementation. 113 * Locks provide shared/exclusive sychronization. 114 */ 115 116 #if defined(LOCKDEBUG) || defined(DIAGNOSTIC) /* { */ 117 #if defined(MULTIPROCESSOR) /* { */ 118 #if defined(__HAVE_ATOMIC_OPERATIONS) /* { */ 119 #define COUNT_CPU(cpu_id, x) \ 120 atomic_add_ulong(&curcpu()->ci_spin_locks, (x)) 121 #else 122 #define COUNT_CPU(cpu_id, x) /* not safe */ 123 #endif /* __HAVE_ATOMIC_OPERATIONS */ /* } */ 124 #else 125 u_long spin_locks; 126 #define COUNT_CPU(cpu_id, x) spin_locks += (x) 127 #endif /* MULTIPROCESSOR */ /* } */ 128 129 #define COUNT(lkp, p, cpu_id, x) \ 130 do { \ 131 if ((lkp)->lk_flags & LK_SPIN) \ 132 COUNT_CPU((cpu_id), (x)); \ 133 else \ 134 (p)->p_locks += (x); \ 135 } while (/*CONSTCOND*/0) 136 #else 137 #define COUNT(lkp, p, cpu_id, x) 138 #endif /* LOCKDEBUG || DIAGNOSTIC */ /* } */ 139 140 #define INTERLOCK_ACQUIRE(lkp, flags, s) \ 141 do { \ 142 if ((flags) & LK_SPIN) \ 143 s = splsched(); \ 144 simple_lock(&(lkp)->lk_interlock); \ 145 } while (0) 146 147 #define INTERLOCK_RELEASE(lkp, flags, s) \ 148 do { \ 149 simple_unlock(&(lkp)->lk_interlock); \ 150 if ((flags) & LK_SPIN) \ 151 splx(s); \ 152 } while (0) 153 154 /* 155 * Acquire a resource. 156 */ 157 #define ACQUIRE(lkp, error, extflags, drain, wanted) \ 158 if ((extflags) & LK_SPIN) { \ 159 int interlocked; \ 160 \ 161 if ((drain) == 0) \ 162 (lkp)->lk_waitcount++; \ 163 for (interlocked = 1;;) { \ 164 if (wanted) { \ 165 if (interlocked) { \ 166 INTERLOCK_RELEASE((lkp), \ 167 LK_SPIN, s); \ 168 interlocked = 0; \ 169 } \ 170 } else if (interlocked) { \ 171 break; \ 172 } else { \ 173 INTERLOCK_ACQUIRE((lkp), LK_SPIN, s); \ 174 interlocked = 1; \ 175 } \ 176 } \ 177 if ((drain) == 0) \ 178 (lkp)->lk_waitcount--; \ 179 KASSERT((wanted) == 0); \ 180 error = 0; /* sanity */ \ 181 } else { \ 182 for (error = 0; wanted; ) { \ 183 if ((drain)) \ 184 (lkp)->lk_flags |= LK_WAITDRAIN; \ 185 else \ 186 (lkp)->lk_waitcount++; \ 187 /* XXX Cast away volatile. */ \ 188 error = ltsleep((drain) ? &(lkp)->lk_flags : \ 189 (void *)(lkp), (lkp)->lk_prio, \ 190 (lkp)->lk_wmesg, (lkp)->lk_timo, \ 191 &(lkp)->lk_interlock); \ 192 if ((drain) == 0) \ 193 (lkp)->lk_waitcount--; \ 194 if (error) \ 195 break; \ 196 if ((extflags) & LK_SLEEPFAIL) { \ 197 error = ENOLCK; \ 198 break; \ 199 } \ 200 } \ 201 } 202 203 #define SETHOLDER(lkp, pid, cpu_id) \ 204 do { \ 205 if ((lkp)->lk_flags & LK_SPIN) \ 206 (lkp)->lk_cpu = cpu_id; \ 207 else \ 208 (lkp)->lk_lockholder = pid; \ 209 } while (/*CONSTCOND*/0) 210 211 #define WEHOLDIT(lkp, pid, cpu_id) \ 212 (((lkp)->lk_flags & LK_SPIN) != 0 ? \ 213 ((lkp)->lk_cpu == (cpu_id)) : ((lkp)->lk_lockholder == (pid))) 214 215 #define WAKEUP_WAITER(lkp) \ 216 do { \ 217 if (((lkp)->lk_flags & LK_SPIN) == 0 && (lkp)->lk_waitcount) { \ 218 /* XXX Cast away volatile. */ \ 219 wakeup_one((void *)(lkp)); \ 220 } \ 221 } while (/*CONSTCOND*/0) 222 223 #if defined(LOCKDEBUG) /* { */ 224 #if defined(MULTIPROCESSOR) /* { */ 225 struct simplelock spinlock_list_slock = SIMPLELOCK_INITIALIZER; 226 227 #define SPINLOCK_LIST_LOCK() \ 228 __cpu_simple_lock(&spinlock_list_slock.lock_data) 229 230 #define SPINLOCK_LIST_UNLOCK() \ 231 __cpu_simple_unlock(&spinlock_list_slock.lock_data) 232 #else 233 #define SPINLOCK_LIST_LOCK() /* nothing */ 234 235 #define SPINLOCK_LIST_UNLOCK() /* nothing */ 236 #endif /* MULTIPROCESSOR */ /* } */ 237 238 TAILQ_HEAD(, lock) spinlock_list = 239 TAILQ_HEAD_INITIALIZER(spinlock_list); 240 241 #define HAVEIT(lkp) \ 242 do { \ 243 if ((lkp)->lk_flags & LK_SPIN) { \ 244 int s = splhigh(); \ 245 SPINLOCK_LIST_LOCK(); \ 246 /* XXX Cast away volatile. */ \ 247 TAILQ_INSERT_TAIL(&spinlock_list, (struct lock *)(lkp), \ 248 lk_list); \ 249 SPINLOCK_LIST_UNLOCK(); \ 250 splx(s); \ 251 } \ 252 } while (/*CONSTCOND*/0) 253 254 #define DONTHAVEIT(lkp) \ 255 do { \ 256 if ((lkp)->lk_flags & LK_SPIN) { \ 257 int s = splhigh(); \ 258 SPINLOCK_LIST_LOCK(); \ 259 /* XXX Cast away volatile. */ \ 260 TAILQ_REMOVE(&spinlock_list, (struct lock *)(lkp), \ 261 lk_list); \ 262 SPINLOCK_LIST_UNLOCK(); \ 263 splx(s); \ 264 } \ 265 } while (/*CONSTCOND*/0) 266 #else 267 #define HAVEIT(lkp) /* nothing */ 268 269 #define DONTHAVEIT(lkp) /* nothing */ 270 #endif /* LOCKDEBUG */ /* } */ 271 272 #if defined(LOCKDEBUG) 273 /* 274 * Lock debug printing routine; can be configured to print to console 275 * or log to syslog. 276 */ 277 void 278 lock_printf(const char *fmt, ...) 279 { 280 va_list ap; 281 282 va_start(ap, fmt); 283 if (lock_debug_syslog) 284 vlog(LOG_DEBUG, fmt, ap); 285 else 286 vprintf(fmt, ap); 287 va_end(ap); 288 } 289 #endif /* LOCKDEBUG */ 290 291 /* 292 * Initialize a lock; required before use. 293 */ 294 void 295 lockinit(struct lock *lkp, int prio, const char *wmesg, int timo, int flags) 296 { 297 298 memset(lkp, 0, sizeof(struct lock)); 299 simple_lock_init(&lkp->lk_interlock); 300 lkp->lk_flags = flags & LK_EXTFLG_MASK; 301 if (flags & LK_SPIN) 302 lkp->lk_cpu = LK_NOCPU; 303 else { 304 lkp->lk_lockholder = LK_NOPROC; 305 lkp->lk_prio = prio; 306 lkp->lk_timo = timo; 307 } 308 lkp->lk_wmesg = wmesg; /* just a name for spin locks */ 309 } 310 311 /* 312 * Determine the status of a lock. 313 */ 314 int 315 lockstatus(struct lock *lkp) 316 { 317 int s, lock_type = 0; 318 319 INTERLOCK_ACQUIRE(lkp, lkp->lk_flags, s); 320 if (lkp->lk_exclusivecount != 0) 321 lock_type = LK_EXCLUSIVE; 322 else if (lkp->lk_sharecount != 0) 323 lock_type = LK_SHARED; 324 INTERLOCK_RELEASE(lkp, lkp->lk_flags, s); 325 return (lock_type); 326 } 327 328 #if defined(LOCKDEBUG) || defined(DIAGNOSTIC) 329 /* 330 * Make sure no spin locks are held by a CPU that is about 331 * to context switch. 332 */ 333 void 334 spinlock_switchcheck(void) 335 { 336 u_long cnt; 337 int s; 338 339 s = splhigh(); 340 #if defined(MULTIPROCESSOR) 341 cnt = curcpu()->ci_spin_locks; 342 #else 343 cnt = spin_locks; 344 #endif 345 splx(s); 346 347 if (cnt != 0) 348 panic("spinlock_switchcheck: CPU %lu has %lu spin locks", 349 (u_long) cpu_number(), cnt); 350 } 351 #endif /* LOCKDEBUG || DIAGNOSTIC */ 352 353 /* 354 * XXX XXX kludge around another kludge.. 355 * 356 * vfs_shutdown() may be called from interrupt context, either as a result 357 * of a panic, or from the debugger. It proceeds to call 358 * sys_sync(&proc0, ...), pretending its running on behalf of proc0 359 * 360 * We would like to make an attempt to sync the filesystems in this case, so 361 * if this happens, we treat attempts to acquire locks specially. 362 * All locks are acquired on behalf of proc0. 363 * 364 * If we've already paniced, we don't block waiting for locks, but 365 * just barge right ahead since we're already going down in flames. 366 */ 367 368 /* 369 * Set, change, or release a lock. 370 * 371 * Shared requests increment the shared count. Exclusive requests set the 372 * LK_WANT_EXCL flag (preventing further shared locks), and wait for already 373 * accepted shared locks and shared-to-exclusive upgrades to go away. 374 */ 375 int 376 lockmgr(__volatile struct lock *lkp, u_int flags, 377 struct simplelock *interlkp) 378 { 379 int error; 380 pid_t pid; 381 int extflags; 382 cpuid_t cpu_id; 383 struct proc *p = curproc; 384 int lock_shutdown_noblock = 0; 385 int s; 386 387 error = 0; 388 389 INTERLOCK_ACQUIRE(lkp, lkp->lk_flags, s); 390 if (flags & LK_INTERLOCK) 391 simple_unlock(interlkp); 392 extflags = (flags | lkp->lk_flags) & LK_EXTFLG_MASK; 393 394 #ifdef DIAGNOSTIC /* { */ 395 /* 396 * Don't allow spins on sleep locks and don't allow sleeps 397 * on spin locks. 398 */ 399 if ((flags ^ lkp->lk_flags) & LK_SPIN) 400 panic("lockmgr: sleep/spin mismatch\n"); 401 #endif /* } */ 402 403 if (extflags & LK_SPIN) 404 pid = LK_KERNPROC; 405 else { 406 if (p == NULL) { 407 if (!doing_shutdown) { 408 #ifdef DIAGNOSTIC 409 panic("lockmgr: no context"); 410 #endif 411 } else { 412 p = &proc0; 413 if (panicstr && (!(flags & LK_NOWAIT))) { 414 flags |= LK_NOWAIT; 415 lock_shutdown_noblock = 1; 416 } 417 } 418 } 419 pid = p->p_pid; 420 } 421 cpu_id = cpu_number(); 422 423 /* 424 * Once a lock has drained, the LK_DRAINING flag is set and an 425 * exclusive lock is returned. The only valid operation thereafter 426 * is a single release of that exclusive lock. This final release 427 * clears the LK_DRAINING flag and sets the LK_DRAINED flag. Any 428 * further requests of any sort will result in a panic. The bits 429 * selected for these two flags are chosen so that they will be set 430 * in memory that is freed (freed memory is filled with 0xdeadbeef). 431 * The final release is permitted to give a new lease on life to 432 * the lock by specifying LK_REENABLE. 433 */ 434 if (lkp->lk_flags & (LK_DRAINING|LK_DRAINED)) { 435 #ifdef DIAGNOSTIC /* { */ 436 if (lkp->lk_flags & LK_DRAINED) 437 panic("lockmgr: using decommissioned lock"); 438 if ((flags & LK_TYPE_MASK) != LK_RELEASE || 439 WEHOLDIT(lkp, pid, cpu_id) == 0) 440 panic("lockmgr: non-release on draining lock: %d\n", 441 flags & LK_TYPE_MASK); 442 #endif /* DIAGNOSTIC */ /* } */ 443 lkp->lk_flags &= ~LK_DRAINING; 444 if ((flags & LK_REENABLE) == 0) 445 lkp->lk_flags |= LK_DRAINED; 446 } 447 448 switch (flags & LK_TYPE_MASK) { 449 450 case LK_SHARED: 451 if (WEHOLDIT(lkp, pid, cpu_id) == 0) { 452 /* 453 * If just polling, check to see if we will block. 454 */ 455 if ((extflags & LK_NOWAIT) && (lkp->lk_flags & 456 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE))) { 457 error = EBUSY; 458 break; 459 } 460 /* 461 * Wait for exclusive locks and upgrades to clear. 462 */ 463 ACQUIRE(lkp, error, extflags, 0, lkp->lk_flags & 464 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE)); 465 if (error) 466 break; 467 lkp->lk_sharecount++; 468 COUNT(lkp, p, cpu_id, 1); 469 break; 470 } 471 /* 472 * We hold an exclusive lock, so downgrade it to shared. 473 * An alternative would be to fail with EDEADLK. 474 */ 475 lkp->lk_sharecount++; 476 COUNT(lkp, p, cpu_id, 1); 477 /* fall into downgrade */ 478 479 case LK_DOWNGRADE: 480 if (WEHOLDIT(lkp, pid, cpu_id) == 0 || 481 lkp->lk_exclusivecount == 0) 482 panic("lockmgr: not holding exclusive lock"); 483 lkp->lk_sharecount += lkp->lk_exclusivecount; 484 lkp->lk_exclusivecount = 0; 485 lkp->lk_recurselevel = 0; 486 lkp->lk_flags &= ~LK_HAVE_EXCL; 487 SETHOLDER(lkp, LK_NOPROC, LK_NOCPU); 488 DONTHAVEIT(lkp); 489 WAKEUP_WAITER(lkp); 490 break; 491 492 case LK_EXCLUPGRADE: 493 /* 494 * If another process is ahead of us to get an upgrade, 495 * then we want to fail rather than have an intervening 496 * exclusive access. 497 */ 498 if (lkp->lk_flags & LK_WANT_UPGRADE) { 499 lkp->lk_sharecount--; 500 COUNT(lkp, p, cpu_id, -1); 501 error = EBUSY; 502 break; 503 } 504 /* fall into normal upgrade */ 505 506 case LK_UPGRADE: 507 /* 508 * Upgrade a shared lock to an exclusive one. If another 509 * shared lock has already requested an upgrade to an 510 * exclusive lock, our shared lock is released and an 511 * exclusive lock is requested (which will be granted 512 * after the upgrade). If we return an error, the file 513 * will always be unlocked. 514 */ 515 if (WEHOLDIT(lkp, pid, cpu_id) || lkp->lk_sharecount <= 0) 516 panic("lockmgr: upgrade exclusive lock"); 517 lkp->lk_sharecount--; 518 COUNT(lkp, p, cpu_id, -1); 519 /* 520 * If we are just polling, check to see if we will block. 521 */ 522 if ((extflags & LK_NOWAIT) && 523 ((lkp->lk_flags & LK_WANT_UPGRADE) || 524 lkp->lk_sharecount > 1)) { 525 error = EBUSY; 526 break; 527 } 528 if ((lkp->lk_flags & LK_WANT_UPGRADE) == 0) { 529 /* 530 * We are first shared lock to request an upgrade, so 531 * request upgrade and wait for the shared count to 532 * drop to zero, then take exclusive lock. 533 */ 534 lkp->lk_flags |= LK_WANT_UPGRADE; 535 ACQUIRE(lkp, error, extflags, 0, lkp->lk_sharecount); 536 lkp->lk_flags &= ~LK_WANT_UPGRADE; 537 if (error) 538 break; 539 lkp->lk_flags |= LK_HAVE_EXCL; 540 SETHOLDER(lkp, pid, cpu_id); 541 HAVEIT(lkp); 542 if (lkp->lk_exclusivecount != 0) 543 panic("lockmgr: non-zero exclusive count"); 544 lkp->lk_exclusivecount = 1; 545 if (extflags & LK_SETRECURSE) 546 lkp->lk_recurselevel = 1; 547 COUNT(lkp, p, cpu_id, 1); 548 break; 549 } 550 /* 551 * Someone else has requested upgrade. Release our shared 552 * lock, awaken upgrade requestor if we are the last shared 553 * lock, then request an exclusive lock. 554 */ 555 if (lkp->lk_sharecount == 0) 556 WAKEUP_WAITER(lkp); 557 /* fall into exclusive request */ 558 559 case LK_EXCLUSIVE: 560 if (WEHOLDIT(lkp, pid, cpu_id)) { 561 /* 562 * Recursive lock. 563 */ 564 if ((extflags & LK_CANRECURSE) == 0 && 565 lkp->lk_recurselevel == 0) { 566 if (extflags & LK_RECURSEFAIL) { 567 error = EDEADLK; 568 break; 569 } else 570 panic("lockmgr: locking against myself"); 571 } 572 lkp->lk_exclusivecount++; 573 if (extflags & LK_SETRECURSE && 574 lkp->lk_recurselevel == 0) 575 lkp->lk_recurselevel = lkp->lk_exclusivecount; 576 COUNT(lkp, p, cpu_id, 1); 577 break; 578 } 579 /* 580 * If we are just polling, check to see if we will sleep. 581 */ 582 if ((extflags & LK_NOWAIT) && ((lkp->lk_flags & 583 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE)) || 584 lkp->lk_sharecount != 0)) { 585 error = EBUSY; 586 break; 587 } 588 /* 589 * Try to acquire the want_exclusive flag. 590 */ 591 ACQUIRE(lkp, error, extflags, 0, lkp->lk_flags & 592 (LK_HAVE_EXCL | LK_WANT_EXCL)); 593 if (error) 594 break; 595 lkp->lk_flags |= LK_WANT_EXCL; 596 /* 597 * Wait for shared locks and upgrades to finish. 598 */ 599 ACQUIRE(lkp, error, extflags, 0, lkp->lk_sharecount != 0 || 600 (lkp->lk_flags & LK_WANT_UPGRADE)); 601 lkp->lk_flags &= ~LK_WANT_EXCL; 602 if (error) 603 break; 604 lkp->lk_flags |= LK_HAVE_EXCL; 605 SETHOLDER(lkp, pid, cpu_id); 606 HAVEIT(lkp); 607 if (lkp->lk_exclusivecount != 0) 608 panic("lockmgr: non-zero exclusive count"); 609 lkp->lk_exclusivecount = 1; 610 if (extflags & LK_SETRECURSE) 611 lkp->lk_recurselevel = 1; 612 COUNT(lkp, p, cpu_id, 1); 613 break; 614 615 case LK_RELEASE: 616 if (lkp->lk_exclusivecount != 0) { 617 if (WEHOLDIT(lkp, pid, cpu_id) == 0) { 618 if (lkp->lk_flags & LK_SPIN) { 619 panic("lockmgr: processor %lu, not " 620 "exclusive lock holder %lu " 621 "unlocking", cpu_id, lkp->lk_cpu); 622 } else { 623 panic("lockmgr: pid %d, not " 624 "exclusive lock holder %d " 625 "unlocking", pid, 626 lkp->lk_lockholder); 627 } 628 } 629 if (lkp->lk_exclusivecount == lkp->lk_recurselevel) 630 lkp->lk_recurselevel = 0; 631 lkp->lk_exclusivecount--; 632 COUNT(lkp, p, cpu_id, -1); 633 if (lkp->lk_exclusivecount == 0) { 634 lkp->lk_flags &= ~LK_HAVE_EXCL; 635 SETHOLDER(lkp, LK_NOPROC, LK_NOCPU); 636 DONTHAVEIT(lkp); 637 } 638 } else if (lkp->lk_sharecount != 0) { 639 lkp->lk_sharecount--; 640 COUNT(lkp, p, cpu_id, -1); 641 } 642 #ifdef DIAGNOSTIC 643 else 644 panic("lockmgr: release of unlocked lock!"); 645 #endif 646 WAKEUP_WAITER(lkp); 647 break; 648 649 case LK_DRAIN: 650 /* 651 * Check that we do not already hold the lock, as it can 652 * never drain if we do. Unfortunately, we have no way to 653 * check for holding a shared lock, but at least we can 654 * check for an exclusive one. 655 */ 656 if (WEHOLDIT(lkp, pid, cpu_id)) 657 panic("lockmgr: draining against myself"); 658 /* 659 * If we are just polling, check to see if we will sleep. 660 */ 661 if ((extflags & LK_NOWAIT) && ((lkp->lk_flags & 662 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE)) || 663 lkp->lk_sharecount != 0 || lkp->lk_waitcount != 0)) { 664 error = EBUSY; 665 break; 666 } 667 ACQUIRE(lkp, error, extflags, 1, 668 ((lkp->lk_flags & 669 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE)) || 670 lkp->lk_sharecount != 0 || 671 lkp->lk_waitcount != 0)); 672 if (error) 673 break; 674 lkp->lk_flags |= LK_DRAINING | LK_HAVE_EXCL; 675 SETHOLDER(lkp, pid, cpu_id); 676 HAVEIT(lkp); 677 lkp->lk_exclusivecount = 1; 678 /* XXX unlikely that we'd want this */ 679 if (extflags & LK_SETRECURSE) 680 lkp->lk_recurselevel = 1; 681 COUNT(lkp, p, cpu_id, 1); 682 break; 683 684 default: 685 INTERLOCK_RELEASE(lkp, lkp->lk_flags, s); 686 panic("lockmgr: unknown locktype request %d", 687 flags & LK_TYPE_MASK); 688 /* NOTREACHED */ 689 } 690 if ((lkp->lk_flags & (LK_WAITDRAIN|LK_SPIN)) == LK_WAITDRAIN && 691 ((lkp->lk_flags & 692 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE)) == 0 && 693 lkp->lk_sharecount == 0 && lkp->lk_waitcount == 0)) { 694 lkp->lk_flags &= ~LK_WAITDRAIN; 695 wakeup_one((void *)&lkp->lk_flags); 696 } 697 /* 698 * Note that this panic will be a recursive panic, since 699 * we only set lock_shutdown_noblock above if panicstr != NULL. 700 */ 701 if (error && lock_shutdown_noblock) 702 panic("lockmgr: deadlock (see previous panic)"); 703 704 INTERLOCK_RELEASE(lkp, lkp->lk_flags, s); 705 return (error); 706 } 707 708 /* 709 * Print out information about state of a lock. Used by VOP_PRINT 710 * routines to display ststus about contained locks. 711 */ 712 void 713 lockmgr_printinfo(__volatile struct lock *lkp) 714 { 715 716 if (lkp->lk_sharecount) 717 printf(" lock type %s: SHARED (count %d)", lkp->lk_wmesg, 718 lkp->lk_sharecount); 719 else if (lkp->lk_flags & LK_HAVE_EXCL) { 720 printf(" lock type %s: EXCL (count %d) by ", 721 lkp->lk_wmesg, lkp->lk_exclusivecount); 722 if (lkp->lk_flags & LK_SPIN) 723 printf("processor %lu", lkp->lk_cpu); 724 else 725 printf("pid %d", lkp->lk_lockholder); 726 } else 727 printf(" not locked"); 728 if ((lkp->lk_flags & LK_SPIN) == 0 && lkp->lk_waitcount > 0) 729 printf(" with %d pending", lkp->lk_waitcount); 730 } 731 732 #if defined(LOCKDEBUG) /* { */ 733 TAILQ_HEAD(, simplelock) simplelock_list = 734 TAILQ_HEAD_INITIALIZER(simplelock_list); 735 736 #if defined(MULTIPROCESSOR) /* { */ 737 struct simplelock simplelock_list_slock = SIMPLELOCK_INITIALIZER; 738 739 #define SLOCK_LIST_LOCK() \ 740 __cpu_simple_lock(&simplelock_list_slock.lock_data) 741 742 #define SLOCK_LIST_UNLOCK() \ 743 __cpu_simple_unlock(&simplelock_list_slock.lock_data) 744 745 #if defined(__HAVE_ATOMIC_OPERATIONS) /* { */ 746 #define SLOCK_COUNT(x) \ 747 atomic_add_ulong(&curcpu()->ci_simple_locks, (x)) 748 #else 749 #define SLOCK_COUNT(x) /* not safe */ 750 #endif /* __HAVE_ATOMIC_OPERATIONS */ /* } */ 751 #else 752 u_long simple_locks; 753 754 #define SLOCK_LIST_LOCK() /* nothing */ 755 756 #define SLOCK_LIST_UNLOCK() /* nothing */ 757 758 #define SLOCK_COUNT(x) simple_locks += (x) 759 #endif /* MULTIPROCESSOR */ /* } */ 760 761 #ifdef DDB /* { */ 762 int simple_lock_debugger = 0; 763 #define SLOCK_DEBUGGER() if (simple_lock_debugger) Debugger() 764 #else 765 #define SLOCK_DEBUGGER() /* nothing */ 766 #endif /* } */ 767 768 #ifdef MULTIPROCESSOR 769 #define SLOCK_MP() lock_printf("on cpu %d\n", cpu_number()) 770 #else 771 #define SLOCK_MP() /* nothing */ 772 #endif 773 774 #define SLOCK_WHERE(str, alp, id, l) \ 775 do { \ 776 lock_printf(str); \ 777 lock_printf("lock: %p, currently at: %s:%d\n", (alp), (id), (l)); \ 778 SLOCK_MP(); \ 779 if ((alp)->lock_file != NULL) \ 780 lock_printf("last locked: %s:%d\n", (alp)->lock_file, \ 781 (alp)->lock_line); \ 782 if ((alp)->unlock_file != NULL) \ 783 lock_printf("last unlocked: %s:%d\n", (alp)->unlock_file, \ 784 (alp)->unlock_line); \ 785 SLOCK_DEBUGGER(); \ 786 } while (/*CONSTCOND*/0) 787 788 /* 789 * Simple lock functions so that the debugger can see from whence 790 * they are being called. 791 */ 792 void 793 simple_lock_init(struct simplelock *alp) 794 { 795 796 #if defined(MULTIPROCESSOR) /* { */ 797 __cpu_simple_lock_init(&alp->lock_data); 798 #else 799 alp->lock_data = __SIMPLELOCK_UNLOCKED; 800 #endif /* } */ 801 alp->lock_file = NULL; 802 alp->lock_line = 0; 803 alp->unlock_file = NULL; 804 alp->unlock_line = 0; 805 alp->lock_holder = LK_NOCPU; 806 } 807 808 void 809 _simple_lock(__volatile struct simplelock *alp, const char *id, int l) 810 { 811 cpuid_t cpu_id = cpu_number(); 812 int s; 813 814 s = splhigh(); 815 816 /* 817 * MULTIPROCESSOR case: This is `safe' since if it's not us, we 818 * don't take any action, and just fall into the normal spin case. 819 */ 820 if (alp->lock_data == __SIMPLELOCK_LOCKED) { 821 #if defined(MULTIPROCESSOR) /* { */ 822 if (alp->lock_holder == cpu_id) { 823 SLOCK_WHERE("simple_lock: locking against myself\n", 824 alp, id, l); 825 goto out; 826 } 827 #else 828 SLOCK_WHERE("simple_lock: lock held\n", alp, id, l); 829 goto out; 830 #endif /* MULTIPROCESSOR */ /* } */ 831 } 832 833 #if defined(MULTIPROCESSOR) /* { */ 834 /* Acquire the lock before modifying any fields. */ 835 __cpu_simple_lock(&alp->lock_data); 836 #else 837 alp->lock_data = __SIMPLELOCK_LOCKED; 838 #endif /* } */ 839 840 KASSERT(alp->lock_holder == LK_NOCPU); 841 842 alp->lock_file = id; 843 alp->lock_line = l; 844 alp->lock_holder = cpu_id; 845 846 SLOCK_LIST_LOCK(); 847 /* XXX Cast away volatile */ 848 TAILQ_INSERT_TAIL(&simplelock_list, (struct simplelock *)alp, list); 849 SLOCK_LIST_UNLOCK(); 850 851 SLOCK_COUNT(1); 852 853 out: 854 splx(s); 855 } 856 857 int 858 _simple_lock_held(__volatile struct simplelock *alp) 859 { 860 cpuid_t cpu_id = cpu_number(); 861 int s, locked = 0; 862 863 s = splhigh(); 864 865 #if defined(MULTIPROCESSOR) 866 if (__cpu_simple_lock_try(&alp->lock_data) == 0) 867 locked = (alp->lock_holder == cpu_id); 868 else 869 __cpu_simple_unlock(&alp->lock_data); 870 #else 871 if (alp->lock_data == __SIMPLELOCK_LOCKED) { 872 locked = 1; 873 KASSERT(alp->lock_holder == cpu_id); 874 } 875 #endif 876 877 splx(s); 878 879 return (locked); 880 } 881 882 int 883 _simple_lock_try(__volatile struct simplelock *alp, const char *id, int l) 884 { 885 cpuid_t cpu_id = cpu_number(); 886 int s, rv = 0; 887 888 s = splhigh(); 889 890 /* 891 * MULTIPROCESSOR case: This is `safe' since if it's not us, we 892 * don't take any action. 893 */ 894 #if defined(MULTIPROCESSOR) /* { */ 895 if ((rv = __cpu_simple_lock_try(&alp->lock_data)) == 0) { 896 if (alp->lock_holder == cpu_id) 897 SLOCK_WHERE("simple_lock_try: locking against myself\n", 898 alp, id, l); 899 goto out; 900 } 901 #else 902 if (alp->lock_data == __SIMPLELOCK_LOCKED) { 903 SLOCK_WHERE("simple_lock_try: lock held\n", alp, id, l); 904 goto out; 905 } 906 alp->lock_data = __SIMPLELOCK_LOCKED; 907 #endif /* MULTIPROCESSOR */ /* } */ 908 909 /* 910 * At this point, we have acquired the lock. 911 */ 912 913 rv = 1; 914 915 alp->lock_file = id; 916 alp->lock_line = l; 917 alp->lock_holder = cpu_id; 918 919 SLOCK_LIST_LOCK(); 920 /* XXX Cast away volatile. */ 921 TAILQ_INSERT_TAIL(&simplelock_list, (struct simplelock *)alp, list); 922 SLOCK_LIST_UNLOCK(); 923 924 SLOCK_COUNT(1); 925 926 out: 927 splx(s); 928 return (rv); 929 } 930 931 void 932 _simple_unlock(__volatile struct simplelock *alp, const char *id, int l) 933 { 934 int s; 935 936 s = splhigh(); 937 938 /* 939 * MULTIPROCESSOR case: This is `safe' because we think we hold 940 * the lock, and if we don't, we don't take any action. 941 */ 942 if (alp->lock_data == __SIMPLELOCK_UNLOCKED) { 943 SLOCK_WHERE("simple_unlock: lock not held\n", 944 alp, id, l); 945 goto out; 946 } 947 948 SLOCK_LIST_LOCK(); 949 TAILQ_REMOVE(&simplelock_list, alp, list); 950 SLOCK_LIST_UNLOCK(); 951 952 SLOCK_COUNT(-1); 953 954 alp->list.tqe_next = NULL; /* sanity */ 955 alp->list.tqe_prev = NULL; /* sanity */ 956 957 alp->unlock_file = id; 958 alp->unlock_line = l; 959 960 #if defined(MULTIPROCESSOR) /* { */ 961 alp->lock_holder = LK_NOCPU; 962 /* Now that we've modified all fields, release the lock. */ 963 __cpu_simple_unlock(&alp->lock_data); 964 #else 965 alp->lock_data = __SIMPLELOCK_UNLOCKED; 966 KASSERT(alp->lock_holder == cpu_number()); 967 alp->lock_holder = LK_NOCPU; 968 #endif /* } */ 969 970 out: 971 splx(s); 972 } 973 974 void 975 simple_lock_dump(void) 976 { 977 struct simplelock *alp; 978 int s; 979 980 s = splhigh(); 981 SLOCK_LIST_LOCK(); 982 lock_printf("all simple locks:\n"); 983 for (alp = TAILQ_FIRST(&simplelock_list); alp != NULL; 984 alp = TAILQ_NEXT(alp, list)) { 985 lock_printf("%p CPU %lu %s:%d\n", alp, alp->lock_holder, 986 alp->lock_file, alp->lock_line); 987 } 988 SLOCK_LIST_UNLOCK(); 989 splx(s); 990 } 991 992 void 993 simple_lock_freecheck(void *start, void *end) 994 { 995 struct simplelock *alp; 996 int s; 997 998 s = splhigh(); 999 SLOCK_LIST_LOCK(); 1000 for (alp = TAILQ_FIRST(&simplelock_list); alp != NULL; 1001 alp = TAILQ_NEXT(alp, list)) { 1002 if ((void *)alp >= start && (void *)alp < end) { 1003 lock_printf("freeing simple_lock %p CPU %lu %s:%d\n", 1004 alp, alp->lock_holder, alp->lock_file, 1005 alp->lock_line); 1006 SLOCK_DEBUGGER(); 1007 } 1008 } 1009 SLOCK_LIST_UNLOCK(); 1010 splx(s); 1011 } 1012 1013 void 1014 simple_lock_switchcheck(void) 1015 { 1016 struct simplelock *alp; 1017 cpuid_t cpu_id = cpu_number(); 1018 int s; 1019 1020 /* 1021 * We must be holding exactly one lock: the sched_lock. 1022 */ 1023 1024 SCHED_ASSERT_LOCKED(); 1025 1026 s = splhigh(); /* XXX spllock */ 1027 SLOCK_LIST_LOCK(); 1028 for (alp = TAILQ_FIRST(&simplelock_list); alp != NULL; 1029 alp = TAILQ_NEXT(alp, list)) { 1030 if (alp == &sched_lock) 1031 continue; 1032 if (alp->lock_holder == cpu_id) { 1033 lock_printf("switching with held simple_lock %p " 1034 "CPU %lu %s:%d\n", 1035 alp, alp->lock_holder, alp->lock_file, 1036 alp->lock_line); 1037 SLOCK_DEBUGGER(); 1038 } 1039 } 1040 SLOCK_LIST_UNLOCK(); 1041 splx(s); 1042 } 1043 #endif /* LOCKDEBUG */ /* } */ 1044
Indexes created Mon Sep 22 21:09:42 GMT 2025