kern_lock.c revision 1.36
1 /* $NetBSD: kern_lock.c,v 1.36 2000/08/08 19:55:26 thorpej Exp $ */ 2 3 /*- 4 * Copyright (c) 1999, 2000 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center. 10 * 11 * This code is derived from software contributed to The NetBSD Foundation 12 * by Ross Harvey. 13 * 14 * Redistribution and use in source and binary forms, with or without 15 * modification, are permitted provided that the following conditions 16 * are met: 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in the 21 * documentation and/or other materials provided with the distribution. 22 * 3. All advertising materials mentioning features or use of this software 23 * must display the following acknowledgement: 24 * This product includes software developed by the NetBSD 25 * Foundation, Inc. and its contributors. 26 * 4. Neither the name of The NetBSD Foundation nor the names of its 27 * contributors may be used to endorse or promote products derived 28 * from this software without specific prior written permission. 29 * 30 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 31 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 32 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 33 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 34 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 35 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 36 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 37 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 38 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 39 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 40 * POSSIBILITY OF SUCH DAMAGE. 41 */ 42 43 /* 44 * Copyright (c) 1995 45 * The Regents of the University of California. All rights reserved. 46 * 47 * This code contains ideas from software contributed to Berkeley by 48 * Avadis Tevanian, Jr., Michael Wayne Young, and the Mach Operating 49 * System project at Carnegie-Mellon University. 50 * 51 * Redistribution and use in source and binary forms, with or without 52 * modification, are permitted provided that the following conditions 53 * are met: 54 * 1. Redistributions of source code must retain the above copyright 55 * notice, this list of conditions and the following disclaimer. 56 * 2. Redistributions in binary form must reproduce the above copyright 57 * notice, this list of conditions and the following disclaimer in the 58 * documentation and/or other materials provided with the distribution. 59 * 3. All advertising materials mentioning features or use of this software 60 * must display the following acknowledgement: 61 * This product includes software developed by the University of 62 * California, Berkeley and its contributors. 63 * 4. Neither the name of the University nor the names of its contributors 64 * may be used to endorse or promote products derived from this software 65 * without specific prior written permission. 66 * 67 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 68 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 69 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 70 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 71 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 72 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 73 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 74 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 75 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 76 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 77 * SUCH DAMAGE. 78 * 79 * @(#)kern_lock.c 8.18 (Berkeley) 5/21/95 80 */ 81 82 #include "opt_multiprocessor.h" 83 #include "opt_lockdebug.h" 84 #include "opt_ddb.h" 85 86 #include <sys/param.h> 87 #include <sys/proc.h> 88 #include <sys/lock.h> 89 #include <sys/systm.h> 90 #include <machine/cpu.h> 91 92 #if defined(__HAVE_ATOMIC_OPERATIONS) 93 #include <machine/atomic.h> 94 #endif 95 96 #if defined(LOCKDEBUG) 97 #include <sys/syslog.h> 98 /* 99 * note that stdarg.h and the ansi style va_start macro is used for both 100 * ansi and traditional c compiles. 101 * XXX: this requires that stdarg.h define: va_alist and va_dcl 102 */ 103 #include <machine/stdarg.h> 104 105 void lock_printf(const char *fmt, ...) 106 __kprintf_attribute__((__format__(__kprintf__,1,2))); 107 108 int lock_debug_syslog = 0; /* defaults to printf, but can be patched */ 109 #endif 110 111 /* 112 * Locking primitives implementation. 113 * Locks provide shared/exclusive sychronization. 114 */ 115 116 #if defined(LOCKDEBUG) || defined(DIAGNOSTIC) /* { */ 117 #if defined(MULTIPROCESSOR) /* { */ 118 #if defined(__HAVE_ATOMIC_OPERATIONS) /* { */ 119 #define COUNT_CPU(cpu_id, x) \ 120 atomic_add_ulong(&curcpu()->ci_spin_locks, (x)) 121 #else 122 #define COUNT_CPU(cpu_id, x) /* not safe */ 123 #endif /* __HAVE_ATOMIC_OPERATIONS */ /* } */ 124 #else 125 u_long spin_locks; 126 #define COUNT_CPU(cpu_id, x) spin_locks += (x) 127 #endif /* MULTIPROCESSOR */ /* } */ 128 129 #define COUNT(lkp, p, cpu_id, x) \ 130 do { \ 131 if ((lkp)->lk_flags & LK_SPIN) \ 132 COUNT_CPU((cpu_id), (x)); \ 133 else \ 134 (p)->p_locks += (x); \ 135 } while (/*CONSTCOND*/0) 136 #else 137 #define COUNT(lkp, p, cpu_id, x) 138 #endif /* LOCKDEBUG || DIAGNOSTIC */ /* } */ 139 140 /* 141 * Acquire a resource. 142 */ 143 #define ACQUIRE(lkp, error, extflags, drain, wanted) \ 144 if ((extflags) & LK_SPIN) { \ 145 int interlocked; \ 146 \ 147 if ((drain) == 0) \ 148 (lkp)->lk_waitcount++; \ 149 for (interlocked = 1;;) { \ 150 if (wanted) { \ 151 if (interlocked) { \ 152 simple_unlock(&(lkp)->lk_interlock); \ 153 interlocked = 0; \ 154 } \ 155 } else if (interlocked) { \ 156 break; \ 157 } else { \ 158 simple_lock(&(lkp)->lk_interlock); \ 159 interlocked = 1; \ 160 } \ 161 } \ 162 if ((drain) == 0) \ 163 (lkp)->lk_waitcount--; \ 164 KASSERT((wanted) == 0); \ 165 error = 0; /* sanity */ \ 166 } else { \ 167 for (error = 0; wanted; ) { \ 168 if ((drain)) \ 169 (lkp)->lk_flags |= LK_WAITDRAIN; \ 170 else \ 171 (lkp)->lk_waitcount++; \ 172 /* XXX Cast away volatile. */ \ 173 error = ltsleep((drain) ? &(lkp)->lk_flags : \ 174 (void *)(lkp), (lkp)->lk_prio, \ 175 (lkp)->lk_wmesg, (lkp)->lk_timo, \ 176 &(lkp)->lk_interlock); \ 177 if ((drain) == 0) \ 178 (lkp)->lk_waitcount--; \ 179 if (error) \ 180 break; \ 181 if ((extflags) & LK_SLEEPFAIL) { \ 182 error = ENOLCK; \ 183 break; \ 184 } \ 185 } \ 186 } 187 188 #define SETHOLDER(lkp, pid, cpu_id) \ 189 do { \ 190 if ((lkp)->lk_flags & LK_SPIN) \ 191 (lkp)->lk_cpu = cpu_id; \ 192 else \ 193 (lkp)->lk_lockholder = pid; \ 194 } while (/*CONSTCOND*/0) 195 196 #define WEHOLDIT(lkp, pid, cpu_id) \ 197 (((lkp)->lk_flags & LK_SPIN) != 0 ? \ 198 ((lkp)->lk_cpu == (cpu_id)) : ((lkp)->lk_lockholder == (pid))) 199 200 #define WAKEUP_WAITER(lkp) \ 201 do { \ 202 if (((lkp)->lk_flags & LK_SPIN) == 0 && (lkp)->lk_waitcount) { \ 203 /* XXX Cast away volatile. */ \ 204 wakeup_one((void *)(lkp)); \ 205 } \ 206 } while (/*CONSTCOND*/0) 207 208 #if defined(LOCKDEBUG) /* { */ 209 #if defined(MULTIPROCESSOR) /* { */ 210 struct simplelock spinlock_list_slock = SIMPLELOCK_INITIALIZER; 211 212 #define SPINLOCK_LIST_LOCK() \ 213 __cpu_simple_lock(&spinlock_list_slock.lock_data) 214 215 #define SPINLOCK_LIST_UNLOCK() \ 216 __cpu_simple_unlock(&spinlock_list_slock.lock_data) 217 #else 218 #define SPINLOCK_LIST_LOCK() /* nothing */ 219 220 #define SPINLOCK_LIST_UNLOCK() /* nothing */ 221 #endif /* MULTIPROCESSOR */ /* } */ 222 223 TAILQ_HEAD(, lock) spinlock_list = 224 TAILQ_HEAD_INITIALIZER(spinlock_list); 225 226 #define HAVEIT(lkp) \ 227 do { \ 228 if ((lkp)->lk_flags & LK_SPIN) { \ 229 int s = splhigh(); \ 230 SPINLOCK_LIST_LOCK(); \ 231 /* XXX Cast away volatile. */ \ 232 TAILQ_INSERT_TAIL(&spinlock_list, (struct lock *)(lkp), \ 233 lk_list); \ 234 SPINLOCK_LIST_UNLOCK(); \ 235 splx(s); \ 236 } \ 237 } while (/*CONSTCOND*/0) 238 239 #define DONTHAVEIT(lkp) \ 240 do { \ 241 if ((lkp)->lk_flags & LK_SPIN) { \ 242 int s = splhigh(); \ 243 SPINLOCK_LIST_LOCK(); \ 244 /* XXX Cast away volatile. */ \ 245 TAILQ_REMOVE(&spinlock_list, (struct lock *)(lkp), \ 246 lk_list); \ 247 SPINLOCK_LIST_UNLOCK(); \ 248 splx(s); \ 249 } \ 250 } while (/*CONSTCOND*/0) 251 #else 252 #define HAVEIT(lkp) /* nothing */ 253 254 #define DONTHAVEIT(lkp) /* nothing */ 255 #endif /* LOCKDEBUG */ /* } */ 256 257 #if defined(LOCKDEBUG) 258 /* 259 * Lock debug printing routine; can be configured to print to console 260 * or log to syslog. 261 */ 262 void 263 lock_printf(const char *fmt, ...) 264 { 265 va_list ap; 266 267 va_start(ap, fmt); 268 if (lock_debug_syslog) 269 vlog(LOG_DEBUG, fmt, ap); 270 else 271 vprintf(fmt, ap); 272 va_end(ap); 273 } 274 #endif /* LOCKDEBUG */ 275 276 /* 277 * Initialize a lock; required before use. 278 */ 279 void 280 lockinit(struct lock *lkp, int prio, const char *wmesg, int timo, int flags) 281 { 282 283 memset(lkp, 0, sizeof(struct lock)); 284 simple_lock_init(&lkp->lk_interlock); 285 lkp->lk_flags = flags & LK_EXTFLG_MASK; 286 if (flags & LK_SPIN) 287 lkp->lk_cpu = LK_NOCPU; 288 else { 289 lkp->lk_lockholder = LK_NOPROC; 290 lkp->lk_prio = prio; 291 lkp->lk_timo = timo; 292 } 293 lkp->lk_wmesg = wmesg; /* just a name for spin locks */ 294 } 295 296 /* 297 * Determine the status of a lock. 298 */ 299 int 300 lockstatus(struct lock *lkp) 301 { 302 int lock_type = 0; 303 304 simple_lock(&lkp->lk_interlock); 305 if (lkp->lk_exclusivecount != 0) 306 lock_type = LK_EXCLUSIVE; 307 else if (lkp->lk_sharecount != 0) 308 lock_type = LK_SHARED; 309 simple_unlock(&lkp->lk_interlock); 310 return (lock_type); 311 } 312 313 #if defined(LOCKDEBUG) || defined(DIAGNOSTIC) 314 /* 315 * Make sure no spin locks are held by a CPU that is about 316 * to context switch. 317 */ 318 void 319 spinlock_switchcheck(void) 320 { 321 u_long cnt; 322 int s; 323 324 s = splhigh(); 325 #if defined(MULTIPROCESSOR) 326 cnt = curcpu()->ci_spin_locks; 327 #else 328 cnt = spin_locks; 329 #endif 330 splx(s); 331 332 if (cnt != 0) 333 panic("spinlock_switchcheck: CPU %lu has %lu spin locks", 334 (u_long) cpu_number(), cnt); 335 } 336 #endif /* LOCKDEBUG || DIAGNOSTIC */ 337 338 /* 339 * XXX XXX kludge around another kludge.. 340 * 341 * vfs_shutdown() may be called from interrupt context, either as a result 342 * of a panic, or from the debugger. It proceeds to call 343 * sys_sync(&proc0, ...), pretending its running on behalf of proc0 344 * 345 * We would like to make an attempt to sync the filesystems in this case, so 346 * if this happens, we treat attempts to acquire locks specially. 347 * All locks are acquired on behalf of proc0. 348 * 349 * If we've already paniced, we don't block waiting for locks, but 350 * just barge right ahead since we're already going down in flames. 351 */ 352 353 /* 354 * Set, change, or release a lock. 355 * 356 * Shared requests increment the shared count. Exclusive requests set the 357 * LK_WANT_EXCL flag (preventing further shared locks), and wait for already 358 * accepted shared locks and shared-to-exclusive upgrades to go away. 359 */ 360 int 361 lockmgr(__volatile struct lock *lkp, u_int flags, 362 struct simplelock *interlkp) 363 { 364 int error; 365 pid_t pid; 366 int extflags; 367 cpuid_t cpu_id; 368 struct proc *p = curproc; 369 int lock_shutdown_noblock = 0; 370 371 error = 0; 372 373 simple_lock(&lkp->lk_interlock); 374 if (flags & LK_INTERLOCK) 375 simple_unlock(interlkp); 376 extflags = (flags | lkp->lk_flags) & LK_EXTFLG_MASK; 377 378 #ifdef DIAGNOSTIC /* { */ 379 /* 380 * Don't allow spins on sleep locks and don't allow sleeps 381 * on spin locks. 382 */ 383 if ((flags ^ lkp->lk_flags) & LK_SPIN) 384 panic("lockmgr: sleep/spin mismatch\n"); 385 #endif /* } */ 386 387 if (extflags & LK_SPIN) 388 pid = LK_KERNPROC; 389 else { 390 if (p == NULL) { 391 if (!doing_shutdown) { 392 #ifdef DIAGNOSTIC 393 panic("lockmgr: no context"); 394 #endif 395 } else { 396 p = &proc0; 397 if (panicstr && (!(flags & LK_NOWAIT))) { 398 flags |= LK_NOWAIT; 399 lock_shutdown_noblock = 1; 400 } 401 } 402 } 403 pid = p->p_pid; 404 } 405 cpu_id = cpu_number(); 406 407 /* 408 * Once a lock has drained, the LK_DRAINING flag is set and an 409 * exclusive lock is returned. The only valid operation thereafter 410 * is a single release of that exclusive lock. This final release 411 * clears the LK_DRAINING flag and sets the LK_DRAINED flag. Any 412 * further requests of any sort will result in a panic. The bits 413 * selected for these two flags are chosen so that they will be set 414 * in memory that is freed (freed memory is filled with 0xdeadbeef). 415 * The final release is permitted to give a new lease on life to 416 * the lock by specifying LK_REENABLE. 417 */ 418 if (lkp->lk_flags & (LK_DRAINING|LK_DRAINED)) { 419 #ifdef DIAGNOSTIC /* { */ 420 if (lkp->lk_flags & LK_DRAINED) 421 panic("lockmgr: using decommissioned lock"); 422 if ((flags & LK_TYPE_MASK) != LK_RELEASE || 423 WEHOLDIT(lkp, pid, cpu_id) == 0) 424 panic("lockmgr: non-release on draining lock: %d\n", 425 flags & LK_TYPE_MASK); 426 #endif /* DIAGNOSTIC */ /* } */ 427 lkp->lk_flags &= ~LK_DRAINING; 428 if ((flags & LK_REENABLE) == 0) 429 lkp->lk_flags |= LK_DRAINED; 430 } 431 432 switch (flags & LK_TYPE_MASK) { 433 434 case LK_SHARED: 435 if (WEHOLDIT(lkp, pid, cpu_id) == 0) { 436 /* 437 * If just polling, check to see if we will block. 438 */ 439 if ((extflags & LK_NOWAIT) && (lkp->lk_flags & 440 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE))) { 441 error = EBUSY; 442 break; 443 } 444 /* 445 * Wait for exclusive locks and upgrades to clear. 446 */ 447 ACQUIRE(lkp, error, extflags, 0, lkp->lk_flags & 448 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE)); 449 if (error) 450 break; 451 lkp->lk_sharecount++; 452 COUNT(lkp, p, cpu_id, 1); 453 break; 454 } 455 /* 456 * We hold an exclusive lock, so downgrade it to shared. 457 * An alternative would be to fail with EDEADLK. 458 */ 459 lkp->lk_sharecount++; 460 COUNT(lkp, p, cpu_id, 1); 461 /* fall into downgrade */ 462 463 case LK_DOWNGRADE: 464 if (WEHOLDIT(lkp, pid, cpu_id) == 0 || 465 lkp->lk_exclusivecount == 0) 466 panic("lockmgr: not holding exclusive lock"); 467 lkp->lk_sharecount += lkp->lk_exclusivecount; 468 lkp->lk_exclusivecount = 0; 469 lkp->lk_recurselevel = 0; 470 lkp->lk_flags &= ~LK_HAVE_EXCL; 471 SETHOLDER(lkp, LK_NOPROC, LK_NOCPU); 472 DONTHAVEIT(lkp); 473 WAKEUP_WAITER(lkp); 474 break; 475 476 case LK_EXCLUPGRADE: 477 /* 478 * If another process is ahead of us to get an upgrade, 479 * then we want to fail rather than have an intervening 480 * exclusive access. 481 */ 482 if (lkp->lk_flags & LK_WANT_UPGRADE) { 483 lkp->lk_sharecount--; 484 COUNT(lkp, p, cpu_id, -1); 485 error = EBUSY; 486 break; 487 } 488 /* fall into normal upgrade */ 489 490 case LK_UPGRADE: 491 /* 492 * Upgrade a shared lock to an exclusive one. If another 493 * shared lock has already requested an upgrade to an 494 * exclusive lock, our shared lock is released and an 495 * exclusive lock is requested (which will be granted 496 * after the upgrade). If we return an error, the file 497 * will always be unlocked. 498 */ 499 if (WEHOLDIT(lkp, pid, cpu_id) || lkp->lk_sharecount <= 0) 500 panic("lockmgr: upgrade exclusive lock"); 501 lkp->lk_sharecount--; 502 COUNT(lkp, p, cpu_id, -1); 503 /* 504 * If we are just polling, check to see if we will block. 505 */ 506 if ((extflags & LK_NOWAIT) && 507 ((lkp->lk_flags & LK_WANT_UPGRADE) || 508 lkp->lk_sharecount > 1)) { 509 error = EBUSY; 510 break; 511 } 512 if ((lkp->lk_flags & LK_WANT_UPGRADE) == 0) { 513 /* 514 * We are first shared lock to request an upgrade, so 515 * request upgrade and wait for the shared count to 516 * drop to zero, then take exclusive lock. 517 */ 518 lkp->lk_flags |= LK_WANT_UPGRADE; 519 ACQUIRE(lkp, error, extflags, 0, lkp->lk_sharecount); 520 lkp->lk_flags &= ~LK_WANT_UPGRADE; 521 if (error) 522 break; 523 lkp->lk_flags |= LK_HAVE_EXCL; 524 SETHOLDER(lkp, pid, cpu_id); 525 HAVEIT(lkp); 526 if (lkp->lk_exclusivecount != 0) 527 panic("lockmgr: non-zero exclusive count"); 528 lkp->lk_exclusivecount = 1; 529 if (extflags & LK_SETRECURSE) 530 lkp->lk_recurselevel = 1; 531 COUNT(lkp, p, cpu_id, 1); 532 break; 533 } 534 /* 535 * Someone else has requested upgrade. Release our shared 536 * lock, awaken upgrade requestor if we are the last shared 537 * lock, then request an exclusive lock. 538 */ 539 if (lkp->lk_sharecount == 0) 540 WAKEUP_WAITER(lkp); 541 /* fall into exclusive request */ 542 543 case LK_EXCLUSIVE: 544 if (WEHOLDIT(lkp, pid, cpu_id)) { 545 /* 546 * Recursive lock. 547 */ 548 if ((extflags & LK_CANRECURSE) == 0 && 549 lkp->lk_recurselevel == 0) { 550 if (extflags & LK_RECURSEFAIL) { 551 error = EDEADLK; 552 break; 553 } else 554 panic("lockmgr: locking against myself"); 555 } 556 lkp->lk_exclusivecount++; 557 if (extflags & LK_SETRECURSE && 558 lkp->lk_recurselevel == 0) 559 lkp->lk_recurselevel = lkp->lk_exclusivecount; 560 COUNT(lkp, p, cpu_id, 1); 561 break; 562 } 563 /* 564 * If we are just polling, check to see if we will sleep. 565 */ 566 if ((extflags & LK_NOWAIT) && ((lkp->lk_flags & 567 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE)) || 568 lkp->lk_sharecount != 0)) { 569 error = EBUSY; 570 break; 571 } 572 /* 573 * Try to acquire the want_exclusive flag. 574 */ 575 ACQUIRE(lkp, error, extflags, 0, lkp->lk_flags & 576 (LK_HAVE_EXCL | LK_WANT_EXCL)); 577 if (error) 578 break; 579 lkp->lk_flags |= LK_WANT_EXCL; 580 /* 581 * Wait for shared locks and upgrades to finish. 582 */ 583 ACQUIRE(lkp, error, extflags, 0, lkp->lk_sharecount != 0 || 584 (lkp->lk_flags & LK_WANT_UPGRADE)); 585 lkp->lk_flags &= ~LK_WANT_EXCL; 586 if (error) 587 break; 588 lkp->lk_flags |= LK_HAVE_EXCL; 589 SETHOLDER(lkp, pid, cpu_id); 590 HAVEIT(lkp); 591 if (lkp->lk_exclusivecount != 0) 592 panic("lockmgr: non-zero exclusive count"); 593 lkp->lk_exclusivecount = 1; 594 if (extflags & LK_SETRECURSE) 595 lkp->lk_recurselevel = 1; 596 COUNT(lkp, p, cpu_id, 1); 597 break; 598 599 case LK_RELEASE: 600 if (lkp->lk_exclusivecount != 0) { 601 if (WEHOLDIT(lkp, pid, cpu_id) == 0) { 602 if (lkp->lk_flags & LK_SPIN) { 603 panic("lockmgr: processor %lu, not " 604 "exclusive lock holder %lu " 605 "unlocking", cpu_id, lkp->lk_cpu); 606 } else { 607 panic("lockmgr: pid %d, not " 608 "exclusive lock holder %d " 609 "unlocking", pid, 610 lkp->lk_lockholder); 611 } 612 } 613 if (lkp->lk_exclusivecount == lkp->lk_recurselevel) 614 lkp->lk_recurselevel = 0; 615 lkp->lk_exclusivecount--; 616 COUNT(lkp, p, cpu_id, -1); 617 if (lkp->lk_exclusivecount == 0) { 618 lkp->lk_flags &= ~LK_HAVE_EXCL; 619 SETHOLDER(lkp, LK_NOPROC, LK_NOCPU); 620 DONTHAVEIT(lkp); 621 } 622 } else if (lkp->lk_sharecount != 0) { 623 lkp->lk_sharecount--; 624 COUNT(lkp, p, cpu_id, -1); 625 } 626 WAKEUP_WAITER(lkp); 627 break; 628 629 case LK_DRAIN: 630 /* 631 * Check that we do not already hold the lock, as it can 632 * never drain if we do. Unfortunately, we have no way to 633 * check for holding a shared lock, but at least we can 634 * check for an exclusive one. 635 */ 636 if (WEHOLDIT(lkp, pid, cpu_id)) 637 panic("lockmgr: draining against myself"); 638 /* 639 * If we are just polling, check to see if we will sleep. 640 */ 641 if ((extflags & LK_NOWAIT) && ((lkp->lk_flags & 642 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE)) || 643 lkp->lk_sharecount != 0 || lkp->lk_waitcount != 0)) { 644 error = EBUSY; 645 break; 646 } 647 ACQUIRE(lkp, error, extflags, 1, 648 ((lkp->lk_flags & 649 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE)) || 650 lkp->lk_sharecount != 0 || 651 lkp->lk_waitcount != 0)); 652 if (error) 653 break; 654 lkp->lk_flags |= LK_DRAINING | LK_HAVE_EXCL; 655 SETHOLDER(lkp, pid, cpu_id); 656 HAVEIT(lkp); 657 lkp->lk_exclusivecount = 1; 658 /* XXX unlikely that we'd want this */ 659 if (extflags & LK_SETRECURSE) 660 lkp->lk_recurselevel = 1; 661 COUNT(lkp, p, cpu_id, 1); 662 break; 663 664 default: 665 simple_unlock(&lkp->lk_interlock); 666 panic("lockmgr: unknown locktype request %d", 667 flags & LK_TYPE_MASK); 668 /* NOTREACHED */ 669 } 670 if ((lkp->lk_flags & (LK_WAITDRAIN|LK_SPIN)) == LK_WAITDRAIN && 671 ((lkp->lk_flags & 672 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE)) == 0 && 673 lkp->lk_sharecount == 0 && lkp->lk_waitcount == 0)) { 674 lkp->lk_flags &= ~LK_WAITDRAIN; 675 wakeup_one((void *)&lkp->lk_flags); 676 } 677 /* 678 * Note that this panic will be a recursive panic, since 679 * we only set lock_shutdown_noblock above if panicstr != NULL. 680 */ 681 if (error && lock_shutdown_noblock) 682 panic("lockmgr: deadlock (see previous panic)"); 683 684 simple_unlock(&lkp->lk_interlock); 685 return (error); 686 } 687 688 /* 689 * Print out information about state of a lock. Used by VOP_PRINT 690 * routines to display ststus about contained locks. 691 */ 692 void 693 lockmgr_printinfo(__volatile struct lock *lkp) 694 { 695 696 if (lkp->lk_sharecount) 697 printf(" lock type %s: SHARED (count %d)", lkp->lk_wmesg, 698 lkp->lk_sharecount); 699 else if (lkp->lk_flags & LK_HAVE_EXCL) { 700 printf(" lock type %s: EXCL (count %d) by ", 701 lkp->lk_wmesg, lkp->lk_exclusivecount); 702 if (lkp->lk_flags & LK_SPIN) 703 printf("processor %lu", lkp->lk_cpu); 704 else 705 printf("pid %d", lkp->lk_lockholder); 706 } else 707 printf(" not locked"); 708 if ((lkp->lk_flags & LK_SPIN) == 0 && lkp->lk_waitcount > 0) 709 printf(" with %d pending", lkp->lk_waitcount); 710 } 711 712 #if defined(LOCKDEBUG) /* { */ 713 TAILQ_HEAD(, simplelock) simplelock_list = 714 TAILQ_HEAD_INITIALIZER(simplelock_list); 715 716 #if defined(MULTIPROCESSOR) /* { */ 717 struct simplelock simplelock_list_slock = SIMPLELOCK_INITIALIZER; 718 719 #define SLOCK_LIST_LOCK() \ 720 __cpu_simple_lock(&simplelock_list_slock.lock_data) 721 722 #define SLOCK_LIST_UNLOCK() \ 723 __cpu_simple_unlock(&simplelock_list_slock.lock_data) 724 725 #if defined(__HAVE_ATOMIC_OPERATIONS) /* { */ 726 #define SLOCK_COUNT(x) \ 727 atomic_add_ulong(&curcpu()->ci_simple_locks, (x)) 728 #else 729 #define SLOCK_COUNT(x) /* not safe */ 730 #endif /* __HAVE_ATOMIC_OPERATIONS */ /* } */ 731 #else 732 u_long simple_locks; 733 734 #define SLOCK_LIST_LOCK() /* nothing */ 735 736 #define SLOCK_LIST_UNLOCK() /* nothing */ 737 738 #define SLOCK_COUNT(x) simple_locks += (x) 739 #endif /* MULTIPROCESSOR */ /* } */ 740 741 #ifdef DDB /* { */ 742 int simple_lock_debugger = 0; 743 #define SLOCK_DEBUGGER() if (simple_lock_debugger) Debugger() 744 #else 745 #define SLOCK_DEBUGGER() /* nothing */ 746 #endif /* } */ 747 748 #ifdef MULTIPROCESSOR 749 #define SLOCK_MP() lock_printf("on cpu %d\n", cpu_number()) 750 #else 751 #define SLOCK_MP() /* nothing */ 752 #endif 753 754 #define SLOCK_WHERE(str, alp, id, l) \ 755 do { \ 756 lock_printf(str); \ 757 lock_printf("lock: %p, currently at: %s:%d\n", (alp), (id), (l)); \ 758 SLOCK_MP(); \ 759 if ((alp)->lock_file != NULL) \ 760 lock_printf("last locked: %s:%d\n", (alp)->lock_file, \ 761 (alp)->lock_line); \ 762 if ((alp)->unlock_file != NULL) \ 763 lock_printf("last unlocked: %s:%d\n", (alp)->unlock_file, \ 764 (alp)->unlock_line); \ 765 SLOCK_DEBUGGER(); \ 766 } while (/*CONSTCOND*/0) 767 768 /* 769 * Simple lock functions so that the debugger can see from whence 770 * they are being called. 771 */ 772 void 773 simple_lock_init(struct simplelock *alp) 774 { 775 776 #if defined(MULTIPROCESSOR) /* { */ 777 __cpu_simple_lock_init(&alp->lock_data); 778 #else 779 alp->lock_data = __SIMPLELOCK_UNLOCKED; 780 #endif /* } */ 781 alp->lock_file = NULL; 782 alp->lock_line = 0; 783 alp->unlock_file = NULL; 784 alp->unlock_line = 0; 785 alp->lock_holder = 0; 786 } 787 788 void 789 _simple_lock(__volatile struct simplelock *alp, const char *id, int l) 790 { 791 cpuid_t cpu_id = cpu_number(); 792 int s; 793 794 s = splhigh(); 795 796 /* 797 * MULTIPROCESSOR case: This is `safe' since if it's not us, we 798 * don't take any action, and just fall into the normal spin case. 799 */ 800 if (alp->lock_data == __SIMPLELOCK_LOCKED) { 801 #if defined(MULTIPROCESSOR) /* { */ 802 if (alp->lock_holder == cpu_id) { 803 SLOCK_WHERE("simple_lock: locking against myself\n", 804 alp, id, l); 805 goto out; 806 } 807 #else 808 SLOCK_WHERE("simple_lock: lock held\n", alp, id, l); 809 goto out; 810 #endif /* MULTIPROCESSOR */ /* } */ 811 } 812 813 #if defined(MULTIPROCESSOR) /* { */ 814 /* Acquire the lock before modifying any fields. */ 815 __cpu_simple_lock(&alp->lock_data); 816 #else 817 alp->lock_data = __SIMPLELOCK_LOCKED; 818 #endif /* } */ 819 820 alp->lock_file = id; 821 alp->lock_line = l; 822 alp->lock_holder = cpu_id; 823 824 SLOCK_LIST_LOCK(); 825 /* XXX Cast away volatile */ 826 TAILQ_INSERT_TAIL(&simplelock_list, (struct simplelock *)alp, list); 827 SLOCK_LIST_UNLOCK(); 828 829 SLOCK_COUNT(1); 830 831 out: 832 splx(s); 833 } 834 835 int 836 _simple_lock_try(__volatile struct simplelock *alp, const char *id, int l) 837 { 838 cpuid_t cpu_id = cpu_number(); 839 int s, rv = 0; 840 841 s = splhigh(); 842 843 /* 844 * MULTIPROCESSOR case: This is `safe' since if it's not us, we 845 * don't take any action. 846 */ 847 #if defined(MULTIPROCESSOR) /* { */ 848 if ((rv = __cpu_simple_lock_try(&alp->lock_data)) == 0) { 849 if (alp->lock_holder == cpu_id) 850 SLOCK_WHERE("simple_lock_try: locking against myself\n", 851 alp, id, l); 852 goto out; 853 } 854 #else 855 if (alp->lock_data == __SIMPLELOCK_LOCKED) { 856 SLOCK_WHERE("simple_lock_try: lock held\n", alp, id, l); 857 goto out; 858 } 859 alp->lock_data = __SIMPLELOCK_LOCKED; 860 #endif /* MULTIPROCESSOR */ /* } */ 861 862 /* 863 * At this point, we have acquired the lock. 864 */ 865 866 rv = 1; 867 868 alp->lock_file = id; 869 alp->lock_line = l; 870 alp->lock_holder = cpu_id; 871 872 SLOCK_LIST_LOCK(); 873 /* XXX Cast away volatile. */ 874 TAILQ_INSERT_TAIL(&simplelock_list, (struct simplelock *)alp, list); 875 SLOCK_LIST_UNLOCK(); 876 877 SLOCK_COUNT(1); 878 879 out: 880 splx(s); 881 return (rv); 882 } 883 884 void 885 _simple_unlock(__volatile struct simplelock *alp, const char *id, int l) 886 { 887 int s; 888 889 s = splhigh(); 890 891 /* 892 * MULTIPROCESSOR case: This is `safe' because we think we hold 893 * the lock, and if we don't, we don't take any action. 894 */ 895 if (alp->lock_data == __SIMPLELOCK_UNLOCKED) { 896 SLOCK_WHERE("simple_unlock: lock not held\n", 897 alp, id, l); 898 goto out; 899 } 900 901 SLOCK_LIST_LOCK(); 902 TAILQ_REMOVE(&simplelock_list, alp, list); 903 SLOCK_LIST_UNLOCK(); 904 905 SLOCK_COUNT(-1); 906 907 alp->list.tqe_next = NULL; /* sanity */ 908 alp->list.tqe_prev = NULL; /* sanity */ 909 910 alp->unlock_file = id; 911 alp->unlock_line = l; 912 913 #if defined(MULTIPROCESSOR) /* { */ 914 alp->lock_holder = LK_NOCPU; 915 /* Now that we've modified all fields, release the lock. */ 916 __cpu_simple_unlock(&alp->lock_data); 917 #else 918 alp->lock_data = __SIMPLELOCK_UNLOCKED; 919 #endif /* } */ 920 921 out: 922 splx(s); 923 } 924 925 void 926 simple_lock_dump(void) 927 { 928 struct simplelock *alp; 929 int s; 930 931 s = splhigh(); 932 SLOCK_LIST_LOCK(); 933 lock_printf("all simple locks:\n"); 934 for (alp = TAILQ_FIRST(&simplelock_list); alp != NULL; 935 alp = TAILQ_NEXT(alp, list)) { 936 lock_printf("%p CPU %lu %s:%d\n", alp, alp->lock_holder, 937 alp->lock_file, alp->lock_line); 938 } 939 SLOCK_LIST_UNLOCK(); 940 splx(s); 941 } 942 943 void 944 simple_lock_freecheck(void *start, void *end) 945 { 946 struct simplelock *alp; 947 int s; 948 949 s = splhigh(); 950 SLOCK_LIST_LOCK(); 951 for (alp = TAILQ_FIRST(&simplelock_list); alp != NULL; 952 alp = TAILQ_NEXT(alp, list)) { 953 if ((void *)alp >= start && (void *)alp < end) { 954 lock_printf("freeing simple_lock %p CPU %lu %s:%d\n", 955 alp, alp->lock_holder, alp->lock_file, 956 alp->lock_line); 957 SLOCK_DEBUGGER(); 958 } 959 } 960 SLOCK_LIST_UNLOCK(); 961 splx(s); 962 } 963 964 void 965 simple_lock_switchcheck(void) 966 { 967 struct simplelock *alp; 968 cpuid_t cpu_id = cpu_number(); 969 int s; 970 971 s = splhigh(); 972 SLOCK_LIST_LOCK(); 973 for (alp = TAILQ_FIRST(&simplelock_list); alp != NULL; 974 alp = TAILQ_NEXT(alp, list)) { 975 if (alp->lock_holder == cpu_id) { 976 lock_printf("switching with held simple_lock %p " 977 "CPU %lu %s:%d\n", 978 alp, alp->lock_holder, alp->lock_file, 979 alp->lock_line); 980 SLOCK_DEBUGGER(); 981 } 982 } 983 SLOCK_LIST_UNLOCK(); 984 splx(s); 985 } 986 #endif /* LOCKDEBUG */ /* } */ 987
Indexes created Mon Sep 22 13:09:51 GMT 2025