kern_lock.c revision 1.186
1 /* $NetBSD: kern_lock.c,v 1.186 2023/07/07 18:02:52 riastradh Exp $ */ 2 3 /*- 4 * Copyright (c) 2002, 2006, 2007, 2008, 2009, 2020 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center, and by Andrew Doran. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 #include <sys/cdefs.h> 34 __KERNEL_RCSID(0, "$NetBSD: kern_lock.c,v 1.186 2023/07/07 18:02:52 riastradh Exp $"); 35 36 #ifdef _KERNEL_OPT 37 #include "opt_lockdebug.h" 38 #endif 39 40 #include <sys/param.h> 41 #include <sys/proc.h> 42 #include <sys/lock.h> 43 #include <sys/systm.h> 44 #include <sys/kernel.h> 45 #include <sys/lockdebug.h> 46 #include <sys/cpu.h> 47 #include <sys/syslog.h> 48 #include <sys/atomic.h> 49 #include <sys/lwp.h> 50 #include <sys/pserialize.h> 51 52 #if defined(DIAGNOSTIC) && !defined(LOCKDEBUG) 53 #include <sys/ksyms.h> 54 #endif 55 56 #include <machine/lock.h> 57 58 #include <dev/lockstat.h> 59 60 #define RETURN_ADDRESS (uintptr_t)__builtin_return_address(0) 61 62 bool kernel_lock_dodebug; 63 64 __cpu_simple_lock_t kernel_lock[CACHE_LINE_SIZE / sizeof(__cpu_simple_lock_t)] 65 __cacheline_aligned; 66 67 void 68 assert_sleepable(void) 69 { 70 struct lwp *l = curlwp; 71 const char *reason; 72 uint64_t ncsw; 73 bool idle; 74 75 if (__predict_false(panicstr != NULL)) { 76 return; 77 } 78 79 LOCKDEBUG_BARRIER(kernel_lock, 1); 80 81 /* 82 * Avoid disabling/re-enabling preemption here since this 83 * routine may be called in delicate situations. 84 */ 85 do { 86 ncsw = l->l_ncsw; 87 __insn_barrier(); 88 idle = CURCPU_IDLE_P(); 89 __insn_barrier(); 90 } while (__predict_false(ncsw != l->l_ncsw)); 91 92 reason = NULL; 93 if (__predict_false(idle) && !cold) { 94 reason = "idle"; 95 goto panic; 96 } 97 if (__predict_false(cpu_intr_p())) { 98 reason = "interrupt"; 99 goto panic; 100 } 101 if (__predict_false(cpu_softintr_p())) { 102 reason = "softint"; 103 goto panic; 104 } 105 if (__predict_false(!pserialize_not_in_read_section())) { 106 reason = "pserialize"; 107 goto panic; 108 } 109 return; 110 111 panic: panic("%s: %s caller=%p", __func__, reason, (void *)RETURN_ADDRESS); 112 } 113 114 /* 115 * Functions for manipulating the kernel_lock. We put them here 116 * so that they show up in profiles. 117 */ 118 119 #define _KERNEL_LOCK_ABORT(msg) \ 120 LOCKDEBUG_ABORT(__func__, __LINE__, kernel_lock, &_kernel_lock_ops, msg) 121 122 #ifdef LOCKDEBUG 123 #define _KERNEL_LOCK_ASSERT(cond) \ 124 do { \ 125 if (!(cond)) \ 126 _KERNEL_LOCK_ABORT("assertion failed: " #cond); \ 127 } while (/* CONSTCOND */ 0) 128 #else 129 #define _KERNEL_LOCK_ASSERT(cond) /* nothing */ 130 #endif 131 132 static void _kernel_lock_dump(const volatile void *, lockop_printer_t); 133 134 lockops_t _kernel_lock_ops = { 135 .lo_name = "Kernel lock", 136 .lo_type = LOCKOPS_SPIN, 137 .lo_dump = _kernel_lock_dump, 138 }; 139 140 #ifdef LOCKDEBUG 141 142 #include <ddb/ddb.h> 143 144 static void 145 kernel_lock_trace_ipi(void *cookie) 146 { 147 148 printf("%s[%d %s]: hogging kernel lock\n", cpu_name(curcpu()), 149 curlwp->l_lid, 150 curlwp->l_name ? curlwp->l_name : curproc->p_comm); 151 db_stacktrace(); 152 } 153 154 #endif 155 156 /* 157 * Initialize the kernel lock. 158 */ 159 void 160 kernel_lock_init(void) 161 { 162 163 __cpu_simple_lock_init(kernel_lock); 164 kernel_lock_dodebug = LOCKDEBUG_ALLOC(kernel_lock, &_kernel_lock_ops, 165 RETURN_ADDRESS); 166 } 167 CTASSERT(CACHE_LINE_SIZE >= sizeof(__cpu_simple_lock_t)); 168 169 /* 170 * Print debugging information about the kernel lock. 171 */ 172 static void 173 _kernel_lock_dump(const volatile void *junk, lockop_printer_t pr) 174 { 175 struct cpu_info *ci = curcpu(); 176 177 (void)junk; 178 179 pr("curcpu holds : %18d wanted by: %#018lx\n", 180 ci->ci_biglock_count, (long)ci->ci_biglock_wanted); 181 } 182 183 /* 184 * Acquire 'nlocks' holds on the kernel lock. 185 * 186 * Although it may not look it, this is one of the most central, intricate 187 * routines in the kernel, and tons of code elsewhere depends on its exact 188 * behaviour. If you change something in here, expect it to bite you in the 189 * rear. 190 */ 191 void 192 _kernel_lock(int nlocks) 193 { 194 struct cpu_info *ci; 195 LOCKSTAT_TIMER(spintime); 196 LOCKSTAT_FLAG(lsflag); 197 struct lwp *owant; 198 #ifdef LOCKDEBUG 199 static struct cpu_info *kernel_lock_holder; 200 u_int spins = 0; 201 u_int starttime = getticks(); 202 #endif 203 int s; 204 struct lwp *l = curlwp; 205 206 _KERNEL_LOCK_ASSERT(nlocks > 0); 207 208 s = splvm(); 209 ci = curcpu(); 210 if (ci->ci_biglock_count != 0) { 211 _KERNEL_LOCK_ASSERT(__SIMPLELOCK_LOCKED_P(kernel_lock)); 212 ci->ci_biglock_count += nlocks; 213 l->l_blcnt += nlocks; 214 splx(s); 215 return; 216 } 217 218 _KERNEL_LOCK_ASSERT(l->l_blcnt == 0); 219 LOCKDEBUG_WANTLOCK(kernel_lock_dodebug, kernel_lock, RETURN_ADDRESS, 220 0); 221 222 if (__predict_true(__cpu_simple_lock_try(kernel_lock))) { 223 #ifdef LOCKDEBUG 224 kernel_lock_holder = curcpu(); 225 #endif 226 ci->ci_biglock_count = nlocks; 227 l->l_blcnt = nlocks; 228 LOCKDEBUG_LOCKED(kernel_lock_dodebug, kernel_lock, NULL, 229 RETURN_ADDRESS, 0); 230 splx(s); 231 return; 232 } 233 234 /* 235 * To remove the ordering constraint between adaptive mutexes 236 * and kernel_lock we must make it appear as if this thread is 237 * blocking. For non-interlocked mutex release, a store fence 238 * is required to ensure that the result of any mutex_exit() 239 * by the current LWP becomes visible on the bus before the set 240 * of ci->ci_biglock_wanted becomes visible. 241 * 242 * This membar_producer matches the membar_consumer in 243 * mutex_vector_enter. 244 * 245 * That way, if l has just released a mutex, mutex_vector_enter 246 * can't see this store ci->ci_biglock_wanted := l until it 247 * will also see the mutex_exit store mtx->mtx_owner := 0 which 248 * clears the has-waiters bit. 249 */ 250 membar_producer(); 251 owant = ci->ci_biglock_wanted; 252 atomic_store_relaxed(&ci->ci_biglock_wanted, l); 253 #if defined(DIAGNOSTIC) && !defined(LOCKDEBUG) 254 l->l_ld_wanted = __builtin_return_address(0); 255 #endif 256 257 /* 258 * Spin until we acquire the lock. Once we have it, record the 259 * time spent with lockstat. 260 */ 261 LOCKSTAT_ENTER(lsflag); 262 LOCKSTAT_START_TIMER(lsflag, spintime); 263 264 do { 265 splx(s); 266 while (__SIMPLELOCK_LOCKED_P(kernel_lock)) { 267 #ifdef LOCKDEBUG 268 if (SPINLOCK_SPINOUT(spins) && start_init_exec && 269 (getticks() - starttime) > 10*hz) { 270 ipi_msg_t msg = { 271 .func = kernel_lock_trace_ipi, 272 }; 273 kpreempt_disable(); 274 ipi_unicast(&msg, kernel_lock_holder); 275 ipi_wait(&msg); 276 kpreempt_enable(); 277 _KERNEL_LOCK_ABORT("spinout"); 278 } 279 #endif 280 SPINLOCK_BACKOFF_HOOK; 281 SPINLOCK_SPIN_HOOK; 282 } 283 s = splvm(); 284 } while (!__cpu_simple_lock_try(kernel_lock)); 285 286 ci->ci_biglock_count = nlocks; 287 l->l_blcnt = nlocks; 288 LOCKSTAT_STOP_TIMER(lsflag, spintime); 289 LOCKDEBUG_LOCKED(kernel_lock_dodebug, kernel_lock, NULL, 290 RETURN_ADDRESS, 0); 291 if (owant == NULL) { 292 LOCKSTAT_EVENT_RA(lsflag, kernel_lock, 293 LB_KERNEL_LOCK | LB_SPIN, 1, spintime, RETURN_ADDRESS); 294 } 295 LOCKSTAT_EXIT(lsflag); 296 splx(s); 297 298 /* 299 * Now that we have kernel_lock, reset ci_biglock_wanted. This 300 * store must be visible on other CPUs before a mutex_exit() on 301 * this CPU can test the has-waiters bit. 302 * 303 * This membar_enter matches the membar_enter in 304 * mutex_vector_enter. (Yes, not membar_exit -- the legacy 305 * naming is confusing, but store-before-load usually pairs 306 * with store-before-load, in the extremely rare cases where it 307 * is used at all.) 308 * 309 * That way, mutex_vector_enter can't see this store 310 * ci->ci_biglock_wanted := owant until it has set the 311 * has-waiters bit. 312 */ 313 (void)atomic_swap_ptr(&ci->ci_biglock_wanted, owant); 314 #ifndef __HAVE_ATOMIC_AS_MEMBAR 315 membar_enter(); 316 #endif 317 318 #ifdef LOCKDEBUG 319 kernel_lock_holder = curcpu(); 320 #endif 321 } 322 323 /* 324 * Release 'nlocks' holds on the kernel lock. If 'nlocks' is zero, release 325 * all holds. 326 */ 327 void 328 _kernel_unlock(int nlocks, int *countp) 329 { 330 struct cpu_info *ci; 331 u_int olocks; 332 int s; 333 struct lwp *l = curlwp; 334 335 _KERNEL_LOCK_ASSERT(nlocks < 2); 336 337 olocks = l->l_blcnt; 338 339 if (olocks == 0) { 340 _KERNEL_LOCK_ASSERT(nlocks <= 0); 341 if (countp != NULL) 342 *countp = 0; 343 return; 344 } 345 346 _KERNEL_LOCK_ASSERT(__SIMPLELOCK_LOCKED_P(kernel_lock)); 347 348 if (nlocks == 0) 349 nlocks = olocks; 350 else if (nlocks == -1) { 351 nlocks = 1; 352 _KERNEL_LOCK_ASSERT(olocks == 1); 353 } 354 s = splvm(); 355 ci = curcpu(); 356 _KERNEL_LOCK_ASSERT(ci->ci_biglock_count >= l->l_blcnt); 357 if (ci->ci_biglock_count == nlocks) { 358 LOCKDEBUG_UNLOCKED(kernel_lock_dodebug, kernel_lock, 359 RETURN_ADDRESS, 0); 360 ci->ci_biglock_count = 0; 361 __cpu_simple_unlock(kernel_lock); 362 l->l_blcnt -= nlocks; 363 splx(s); 364 if (l->l_dopreempt) 365 kpreempt(0); 366 } else { 367 ci->ci_biglock_count -= nlocks; 368 l->l_blcnt -= nlocks; 369 splx(s); 370 } 371 372 if (countp != NULL) 373 *countp = olocks; 374 } 375 376 bool 377 _kernel_locked_p(void) 378 { 379 return __SIMPLELOCK_LOCKED_P(kernel_lock); 380 } 381
Indexes created Sun Sep 28 07:09:56 GMT 2025