1 /* $NetBSD: kern_lock.c,v 1.198 2026/08/16 21:52:42 riastradh Exp $ */ 2 3 /*- 4 * Copyright (c) 2002, 2006, 2007, 2008, 2009, 2020, 2023 5 * The NetBSD Foundation, Inc. 6 * All rights reserved. 7 * 8 * This code is derived from software contributed to The NetBSD Foundation 9 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 10 * NASA Ames Research Center, and by Andrew Doran. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 23 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 24 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 25 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 * POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <sys/cdefs.h> 35 __KERNEL_RCSID(0, "$NetBSD: kern_lock.c,v 1.198 2026/08/16 21:52:42 riastradh Exp $"); 36 37 #ifdef _KERNEL_OPT 38 #include "opt_lockdebug.h" 39 #include "opt_multiprocessor.h" 40 #endif 41 42 #include <sys/param.h> 43 #include <sys/types.h> 44 45 #include <sys/atomic.h> 46 #include <sys/cpu.h> 47 #include <sys/kernel.h> 48 #include <sys/lock.h> 49 #include <sys/lockdebug.h> 50 #include <sys/lwp.h> 51 #include <sys/proc.h> 52 #include <sys/pserialize.h> 53 #include <sys/sdt.h> 54 #include <sys/syslog.h> 55 #include <sys/systm.h> 56 57 #if defined(DIAGNOSTIC) && !defined(LOCKDEBUG) 58 #include <sys/ksyms.h> 59 #endif 60 61 #include <machine/lock.h> 62 63 #include <dev/lockstat.h> 64 65 SDT_PROBE_DEFINE1(sdt, kernel, lock, entry, 66 "unsigned"/*nlocks*/); 67 SDT_PROBE_DEFINE1(sdt, kernel, lock, exit, 68 "unsigned"/*nlocks*/); 69 70 #define RETURN_ADDRESS (uintptr_t)__builtin_return_address(0) 71 72 bool kernel_lock_dodebug; 73 74 struct kernel_lock { 75 __cpu_simple_lock_t lock __aligned(CACHE_LINE_SIZE); 76 struct cpu_info *volatile holder; 77 } kernel_lock_cacheline[CACHE_LINE_SIZE / sizeof(struct kernel_lock)] 78 __cacheline_aligned; 79 __strong_alias(kernel_lock, kernel_lock_cacheline) 80 #define kernel_lock_holder (kernel_lock_cacheline[0].holder) 81 82 void 83 assert_sleepable(void) 84 { 85 const char *reason; 86 long pctr; 87 bool idle; 88 89 if (__predict_false(panicstr != NULL)) { 90 return; 91 } 92 93 LOCKDEBUG_BARRIER(kernel_lock, 1); 94 95 /* 96 * Avoid disabling/re-enabling preemption here since this 97 * routine may be called in delicate situations. 98 */ 99 do { 100 pctr = lwp_pctr(); 101 idle = CURCPU_IDLE_P(); 102 } while (__predict_false(pctr != lwp_pctr())); 103 104 reason = NULL; 105 if (__predict_false(idle) && !cold) { 106 reason = "idle"; 107 goto panic; 108 } 109 if (__predict_false(cpu_intr_p())) { 110 reason = "interrupt"; 111 goto panic; 112 } 113 if (__predict_false(cpu_softintr_p())) { 114 reason = "softint"; 115 goto panic; 116 } 117 if (__predict_false(!pserialize_not_in_read_section())) { 118 reason = "pserialize"; 119 goto panic; 120 } 121 return; 122 123 panic: panic("%s: %s caller=%p", __func__, reason, (void *)RETURN_ADDRESS); 124 } 125 126 /* 127 * Functions for manipulating the kernel_lock. We put them here 128 * so that they show up in profiles. 129 */ 130 131 #define _KERNEL_LOCK_ABORT(msg) \ 132 LOCKDEBUG_ABORT(__func__, __LINE__, kernel_lock, &_kernel_lock_ops, msg) 133 134 #ifdef LOCKDEBUG 135 #define _KERNEL_LOCK_ASSERT(cond) \ 136 do { \ 137 if (!(cond)) \ 138 _KERNEL_LOCK_ABORT("assertion failed: " #cond); \ 139 } while (/* CONSTCOND */ 0) 140 #else 141 #define _KERNEL_LOCK_ASSERT(cond) /* nothing */ 142 #endif 143 144 static void _kernel_lock_dump(const volatile void *, lockop_printer_t); 145 146 lockops_t _kernel_lock_ops = { 147 .lo_name = "Kernel lock", 148 .lo_type = LOCKOPS_SPIN, 149 .lo_dump = _kernel_lock_dump, 150 }; 151 152 #ifdef DDB 153 #include <ddb/ddb.h> 154 #endif 155 156 #ifdef MULTIPROCESSOR 157 static void 158 kernel_lock_trace_ipi(void *cookie) 159 { 160 161 printf("%s[%d %s]: hogging kernel lock\n", cpu_name(curcpu()), 162 curlwp->l_lid, 163 curlwp->l_name ? curlwp->l_name : curproc->p_comm); 164 #ifdef DDB 165 db_stacktrace(); 166 167 /* 168 * Make sure we leave a return address around for db_stacktrace 169 * to find. 170 */ 171 __insn_barrier(); 172 return; 173 #endif 174 } 175 176 static void 177 kernel_lock_spinout(void) 178 { 179 static volatile unsigned kernel_lock_last_report; 180 ipi_msg_t msg = { 181 .func = kernel_lock_trace_ipi, 182 }; 183 unsigned now, then; 184 struct cpu_info *holder; 185 186 /* 187 * Disable preemption so that curcpu() is stable in this function. 188 * Otherwise, it's possible for the curlwp to be migrated to the 189 * holder cpu in the meantime. 190 */ 191 kpreempt_disable(); 192 193 /* 194 * Find who holds the kernel lock. If nobody, we can't report 195 * anything, so pass -- but while this is possible in principle 196 * because we first take the lock and then set the holder, it 197 * is rather unlikely to actually happen in practice because we 198 * wait 10sec to take the lock before trying to report a 199 * problem anyway. 200 */ 201 if (!__SIMPLELOCK_LOCKED_P(kernel_lock)) 202 goto out; 203 204 /* 205 * Note: holder == NULL here basically means 206 * "no one has acquired kernel lock since the boot". 207 * 208 * Theoretically it's possible the first locker has acquired 209 * kernel_lock but has not updated kernel_lock_holder yet. 210 * But it's only theoretical, I suppose. 211 */ 212 holder = atomic_load_relaxed(&kernel_lock_holder); 213 if (holder == NULL) 214 goto out; 215 216 /* 217 * We know we don't have the kernel lock. 218 * 219 * However, the holder value is not reliable because we don't 220 * hold kernel lock. For example, an interrupt on this cpu may 221 * acquire/release the kernel lock and leave kernel_lock_holder 222 * pointing to us. 223 */ 224 if (holder == curcpu()) 225 goto out; 226 227 /* 228 * If we already reported kernel lock hogging in the last ten 229 * seconds, probably not worthwhile to fill the log buffer with 230 * repeated reports, so pass. 231 * 232 * XXX This can roll over, but only after decades of uptime. 233 */ 234 then = atomic_load_relaxed(&kernel_lock_last_report); 235 now = time_uptime; 236 if (now - then <= 10) 237 goto out; 238 if (atomic_cas_uint(&kernel_lock_last_report, then, now) != then) 239 goto out; 240 241 printf("%s[%d %s]: kernel lock spinout\n", cpu_name(curcpu()), 242 curlwp->l_lid, 243 curlwp->l_name ? curlwp->l_name : curproc->p_comm); 244 /* 245 * Send an IPI to whatever CPU holds the kernel lock. 246 */ 247 ipi_unicast(&msg, holder); 248 ipi_wait(&msg); 249 out: 250 kpreempt_enable(); 251 252 #ifdef LOCKDEBUG 253 _KERNEL_LOCK_ABORT("spinout"); 254 #endif 255 return; 256 } 257 #endif /* MULTIPROCESSOR */ 258 259 /* 260 * Initialize the kernel lock. 261 */ 262 void 263 kernel_lock_init(void) 264 { 265 266 __cpu_simple_lock_init(kernel_lock); 267 kernel_lock_dodebug = LOCKDEBUG_ALLOC(kernel_lock, &_kernel_lock_ops, 268 RETURN_ADDRESS); 269 } 270 CTASSERT(CACHE_LINE_SIZE >= sizeof(__cpu_simple_lock_t)); 271 272 /* 273 * Print debugging information about the kernel lock. 274 */ 275 static void 276 _kernel_lock_dump(const volatile void *junk, lockop_printer_t pr) 277 { 278 struct cpu_info *ci = curcpu(); 279 280 (void)junk; 281 282 pr("curcpu holds : %18d wanted by: %#018lx\n", 283 ci->ci_biglock_count, (long)ci->ci_biglock_wanted); 284 } 285 286 /* 287 * Acquire 'nlocks' holds on the kernel lock. 288 * 289 * Although it may not look it, this is one of the most central, intricate 290 * routines in the kernel, and tons of code elsewhere depends on its exact 291 * behaviour. If you change something in here, expect it to bite you in the 292 * rear. 293 */ 294 void 295 _kernel_lock(int nlocks) 296 { 297 struct cpu_info *ci; 298 #ifdef MULTIPROCESSOR 299 LOCKSTAT_TIMER(spintime); 300 LOCKSTAT_FLAG(lsflag); 301 #endif 302 int s; 303 struct lwp *l = curlwp; 304 volatile void *owanted; 305 306 _KERNEL_LOCK_ASSERT(nlocks > 0); 307 308 s = splvm(); 309 ci = curcpu(); 310 if (ci->ci_biglock_count != 0) { 311 _KERNEL_LOCK_ASSERT(__SIMPLELOCK_LOCKED_P(kernel_lock)); 312 SDT_PROBE1(sdt, kernel, lock, entry, nlocks); 313 ci->ci_biglock_count += nlocks; 314 l->l_blcnt += nlocks; 315 splx(s); 316 return; 317 } 318 319 _KERNEL_LOCK_ASSERT(l->l_blcnt == 0); 320 LOCKDEBUG_WANTLOCK(kernel_lock_dodebug, kernel_lock, RETURN_ADDRESS, 321 0, &owanted); 322 323 if (__predict_true(__cpu_simple_lock_try(kernel_lock))) { 324 atomic_store_relaxed(&kernel_lock_holder, curcpu()); 325 SDT_PROBE1(sdt, kernel, lock, entry, nlocks); 326 ci->ci_biglock_count = nlocks; 327 l->l_blcnt = nlocks; 328 LOCKDEBUG_LOCKED(kernel_lock_dodebug, kernel_lock, NULL, 329 RETURN_ADDRESS, 0, &owanted); 330 splx(s); 331 return; 332 } 333 334 #ifndef MULTIPROCESSOR 335 panic("%s: contended ???", __func__); 336 #else 337 struct lwp *owant; 338 u_int starttime; 339 340 /* 341 * To remove the ordering constraint between adaptive mutexes 342 * and kernel_lock we must make it appear as if this thread is 343 * blocking. For non-interlocked mutex release, a store fence 344 * is required to ensure that the result of any mutex_exit() 345 * by the current LWP becomes visible on the bus before the set 346 * of ci->ci_biglock_wanted becomes visible. 347 * 348 * This membar_producer matches the membar_consumer in 349 * mutex_vector_enter. 350 * 351 * That way, if l has just released a mutex, mutex_vector_enter 352 * can't see this store ci->ci_biglock_wanted := l until it 353 * will also see the mutex_exit store mtx->mtx_owner := 0 which 354 * clears the has-waiters bit. 355 */ 356 membar_producer(); 357 owant = ci->ci_biglock_wanted; 358 atomic_store_relaxed(&ci->ci_biglock_wanted, l); 359 360 /* 361 * Spin until we acquire the lock. Once we have it, record the 362 * time spent with lockstat. 363 */ 364 LOCKSTAT_ENTER(lsflag); 365 LOCKSTAT_START_TIMER(lsflag, spintime); 366 367 starttime = getticks(); 368 do { 369 splx(s); 370 while (__SIMPLELOCK_LOCKED_P(kernel_lock)) { 371 if (start_init_exec && 372 (getticks() - starttime) > 10*hz) { 373 kernel_lock_spinout(); 374 } 375 SPINLOCK_BACKOFF_HOOK; 376 SPINLOCK_SPIN_HOOK; 377 } 378 s = splvm(); 379 } while (!__cpu_simple_lock_try(kernel_lock)); 380 381 atomic_store_relaxed(&kernel_lock_holder, curcpu()); 382 383 SDT_PROBE1(sdt, kernel, lock, entry, nlocks); 384 ci->ci_biglock_count = nlocks; 385 l->l_blcnt = nlocks; 386 LOCKSTAT_STOP_TIMER(lsflag, spintime); 387 LOCKDEBUG_LOCKED(kernel_lock_dodebug, kernel_lock, NULL, 388 RETURN_ADDRESS, 0, &owanted); 389 if (owant == NULL) { 390 LOCKSTAT_EVENT_RA(lsflag, kernel_lock, 391 LB_KERNEL_LOCK | LB_SPIN, 1, spintime, RETURN_ADDRESS); 392 } 393 LOCKSTAT_EXIT(lsflag); 394 splx(s); 395 396 /* 397 * Now that we have kernel_lock, reset ci_biglock_wanted. This 398 * store must be visible on other CPUs before a mutex_exit() on 399 * this CPU can test the has-waiters bit. 400 * 401 * This membar_enter matches the membar_enter in 402 * mutex_vector_enter. (Yes, not membar_exit -- the legacy 403 * naming is confusing, but store-before-load usually pairs 404 * with store-before-load, in the extremely rare cases where it 405 * is used at all.) 406 * 407 * That way, mutex_vector_enter can't see this store 408 * ci->ci_biglock_wanted := owant until it has set the 409 * has-waiters bit. 410 */ 411 (void)atomic_swap_ptr(&ci->ci_biglock_wanted, owant); 412 #ifndef __HAVE_ATOMIC_AS_MEMBAR 413 membar_enter(); 414 #endif 415 #endif /* MULTIPROCESSOR */ 416 } 417 418 /* 419 * Release 'nlocks' holds on the kernel lock. If 'nlocks' is zero, release 420 * all holds. 421 */ 422 void 423 _kernel_unlock(int nlocks, int *countp) 424 { 425 struct cpu_info *ci; 426 u_int olocks; 427 int s; 428 struct lwp *l = curlwp; 429 430 _KERNEL_LOCK_ASSERT(nlocks < 2); 431 432 olocks = l->l_blcnt; 433 434 if (olocks == 0) { 435 _KERNEL_LOCK_ASSERT(nlocks <= 0); 436 if (countp != NULL) 437 *countp = 0; 438 return; 439 } 440 441 _KERNEL_LOCK_ASSERT(__SIMPLELOCK_LOCKED_P(kernel_lock)); 442 443 if (nlocks == 0) 444 nlocks = olocks; 445 else if (nlocks == -1) { 446 nlocks = 1; 447 _KERNEL_LOCK_ASSERT(olocks == 1); 448 } 449 s = splvm(); 450 ci = curcpu(); 451 _KERNEL_LOCK_ASSERT(ci->ci_biglock_count >= l->l_blcnt); 452 if (ci->ci_biglock_count == nlocks) { 453 LOCKDEBUG_UNLOCKED(kernel_lock_dodebug, kernel_lock, 454 RETURN_ADDRESS, 0); 455 ci->ci_biglock_count = 0; 456 __cpu_simple_unlock(kernel_lock); 457 l->l_blcnt -= nlocks; 458 splx(s); 459 if (l->l_dopreempt) 460 kpreempt(0); 461 } else { 462 ci->ci_biglock_count -= nlocks; 463 l->l_blcnt -= nlocks; 464 splx(s); 465 } 466 467 SDT_PROBE1(sdt, kernel, lock, exit, nlocks); 468 469 if (countp != NULL) 470 *countp = olocks; 471 } 472 473 bool 474 _kernel_locked_p(void) 475 { 476 return __SIMPLELOCK_LOCKED_P(kernel_lock); 477 } 478
Indexes created Thu Sep 24 00:25:51 UTC 2026