Home | History | Annotate | Line # | Download | only in kern
kern_lock.c revision 1.180
      1  1.180  riastrad /*	$NetBSD: kern_lock.c,v 1.180 2022/09/13 09:28:05 riastradh Exp $	*/
      2   1.19   thorpej 
      3   1.19   thorpej /*-
      4  1.165        ad  * Copyright (c) 2002, 2006, 2007, 2008, 2009, 2020 The NetBSD Foundation, Inc.
      5   1.19   thorpej  * All rights reserved.
      6   1.19   thorpej  *
      7   1.19   thorpej  * This code is derived from software contributed to The NetBSD Foundation
      8   1.19   thorpej  * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
      9  1.105        ad  * NASA Ames Research Center, and by Andrew Doran.
     10   1.19   thorpej  *
     11   1.19   thorpej  * Redistribution and use in source and binary forms, with or without
     12   1.19   thorpej  * modification, are permitted provided that the following conditions
     13   1.19   thorpej  * are met:
     14   1.19   thorpej  * 1. Redistributions of source code must retain the above copyright
     15   1.19   thorpej  *    notice, this list of conditions and the following disclaimer.
     16   1.19   thorpej  * 2. Redistributions in binary form must reproduce the above copyright
     17   1.19   thorpej  *    notice, this list of conditions and the following disclaimer in the
     18   1.19   thorpej  *    documentation and/or other materials provided with the distribution.
     19   1.19   thorpej  *
     20   1.19   thorpej  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     21   1.19   thorpej  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     22   1.19   thorpej  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     23   1.19   thorpej  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     24   1.19   thorpej  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     25   1.19   thorpej  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     26   1.19   thorpej  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     27   1.19   thorpej  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     28   1.19   thorpej  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     29   1.19   thorpej  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     30   1.19   thorpej  * POSSIBILITY OF SUCH DAMAGE.
     31   1.19   thorpej  */
     32    1.2      fvdl 
     33   1.60     lukem #include <sys/cdefs.h>
     34  1.180  riastrad __KERNEL_RCSID(0, "$NetBSD: kern_lock.c,v 1.180 2022/09/13 09:28:05 riastradh Exp $");
     35  1.168        ad 
     36  1.168        ad #ifdef _KERNEL_OPT
     37  1.168        ad #include "opt_lockdebug.h"
     38  1.168        ad #endif
     39  1.105        ad 
     40    1.1      fvdl #include <sys/param.h>
     41    1.1      fvdl #include <sys/proc.h>
     42    1.1      fvdl #include <sys/lock.h>
     43    1.2      fvdl #include <sys/systm.h>
     44  1.125        ad #include <sys/kernel.h>
     45  1.105        ad #include <sys/lockdebug.h>
     46  1.122        ad #include <sys/cpu.h>
     47  1.122        ad #include <sys/syslog.h>
     48  1.128        ad #include <sys/atomic.h>
     49  1.148        ad #include <sys/lwp.h>
     50  1.160     ozaki #include <sys/pserialize.h>
     51  1.105        ad 
     52  1.168        ad #if defined(DIAGNOSTIC) && !defined(LOCKDEBUG)
     53  1.168        ad #include <sys/ksyms.h>
     54  1.168        ad #endif
     55  1.168        ad 
     56  1.131        ad #include <machine/lock.h>
     57    1.1      fvdl 
     58   1.98        ad #include <dev/lockstat.h>
     59   1.98        ad 
     60  1.134        ad #define	RETURN_ADDRESS	(uintptr_t)__builtin_return_address(0)
     61   1.25   thorpej 
     62  1.127      yamt bool	kernel_lock_dodebug;
     63  1.132        ad 
     64  1.132        ad __cpu_simple_lock_t kernel_lock[CACHE_LINE_SIZE / sizeof(__cpu_simple_lock_t)]
     65  1.153      matt     __cacheline_aligned;
     66    1.1      fvdl 
     67   1.96      yamt void
     68  1.135      yamt assert_sleepable(void)
     69   1.96      yamt {
     70  1.135      yamt 	const char *reason;
     71  1.148        ad 	uint64_t pctr;
     72  1.148        ad 	bool idle;
     73   1.96      yamt 
     74  1.135      yamt 	if (panicstr != NULL) {
     75  1.117        ad 		return;
     76  1.135      yamt 	}
     77  1.135      yamt 
     78  1.132        ad 	LOCKDEBUG_BARRIER(kernel_lock, 1);
     79  1.135      yamt 
     80  1.148        ad 	/*
     81  1.148        ad 	 * Avoid disabling/re-enabling preemption here since this
     82  1.149    dyoung 	 * routine may be called in delicate situations.
     83  1.148        ad 	 */
     84  1.148        ad 	do {
     85  1.148        ad 		pctr = lwp_pctr();
     86  1.164  riastrad 		__insn_barrier();
     87  1.148        ad 		idle = CURCPU_IDLE_P();
     88  1.164  riastrad 		__insn_barrier();
     89  1.148        ad 	} while (pctr != lwp_pctr());
     90  1.148        ad 
     91  1.135      yamt 	reason = NULL;
     92  1.173     skrll 	if (idle && !cold) {
     93  1.135      yamt 		reason = "idle";
     94  1.135      yamt 	}
     95  1.135      yamt 	if (cpu_intr_p()) {
     96  1.135      yamt 		reason = "interrupt";
     97   1.97      yamt 	}
     98  1.148        ad 	if (cpu_softintr_p()) {
     99  1.135      yamt 		reason = "softint";
    100  1.135      yamt 	}
    101  1.160     ozaki 	if (!pserialize_not_in_read_section()) {
    102  1.160     ozaki 		reason = "pserialize";
    103  1.160     ozaki 	}
    104  1.135      yamt 
    105  1.135      yamt 	if (reason) {
    106  1.135      yamt 		panic("%s: %s caller=%p", __func__, reason,
    107  1.135      yamt 		    (void *)RETURN_ADDRESS);
    108  1.135      yamt 	}
    109   1.96      yamt }
    110  1.105        ad 
    111   1.62   thorpej /*
    112   1.62   thorpej  * Functions for manipulating the kernel_lock.  We put them here
    113   1.62   thorpej  * so that they show up in profiles.
    114   1.62   thorpej  */
    115   1.62   thorpej 
    116  1.105        ad #define	_KERNEL_LOCK_ABORT(msg)						\
    117  1.158  christos     LOCKDEBUG_ABORT(__func__, __LINE__, kernel_lock, &_kernel_lock_ops, msg)
    118  1.105        ad 
    119  1.105        ad #ifdef LOCKDEBUG
    120  1.105        ad #define	_KERNEL_LOCK_ASSERT(cond)					\
    121  1.105        ad do {									\
    122  1.105        ad 	if (!(cond))							\
    123  1.105        ad 		_KERNEL_LOCK_ABORT("assertion failed: " #cond);		\
    124  1.105        ad } while (/* CONSTCOND */ 0)
    125  1.105        ad #else
    126  1.105        ad #define	_KERNEL_LOCK_ASSERT(cond)	/* nothing */
    127  1.105        ad #endif
    128  1.105        ad 
    129  1.163     ozaki static void	_kernel_lock_dump(const volatile void *, lockop_printer_t);
    130  1.105        ad 
    131  1.105        ad lockops_t _kernel_lock_ops = {
    132  1.161     ozaki 	.lo_name = "Kernel lock",
    133  1.161     ozaki 	.lo_type = LOCKOPS_SPIN,
    134  1.161     ozaki 	.lo_dump = _kernel_lock_dump,
    135  1.105        ad };
    136  1.105        ad 
    137  1.174  riastrad #ifdef LOCKDEBUG
    138  1.174  riastrad 
    139  1.174  riastrad #include <ddb/ddb.h>
    140  1.174  riastrad 
    141  1.174  riastrad static void
    142  1.174  riastrad kernel_lock_trace_ipi(void *cookie)
    143  1.174  riastrad {
    144  1.174  riastrad 
    145  1.174  riastrad 	printf("%s[%d %s]: hogging kernel lock\n", cpu_name(curcpu()),
    146  1.174  riastrad 	    curlwp->l_lid,
    147  1.174  riastrad 	    curlwp->l_name ? curlwp->l_name : curproc->p_comm);
    148  1.174  riastrad 	db_stacktrace();
    149  1.174  riastrad }
    150  1.174  riastrad 
    151  1.174  riastrad #endif
    152  1.174  riastrad 
    153   1.85      yamt /*
    154  1.105        ad  * Initialize the kernel lock.
    155   1.85      yamt  */
    156   1.62   thorpej void
    157  1.122        ad kernel_lock_init(void)
    158   1.62   thorpej {
    159   1.62   thorpej 
    160  1.132        ad 	__cpu_simple_lock_init(kernel_lock);
    161  1.132        ad 	kernel_lock_dodebug = LOCKDEBUG_ALLOC(kernel_lock, &_kernel_lock_ops,
    162  1.122        ad 	    RETURN_ADDRESS);
    163   1.62   thorpej }
    164  1.155    martin CTASSERT(CACHE_LINE_SIZE >= sizeof(__cpu_simple_lock_t));
    165   1.62   thorpej 
    166   1.62   thorpej /*
    167  1.105        ad  * Print debugging information about the kernel lock.
    168   1.62   thorpej  */
    169  1.162     ozaki static void
    170  1.163     ozaki _kernel_lock_dump(const volatile void *junk, lockop_printer_t pr)
    171   1.62   thorpej {
    172   1.85      yamt 	struct cpu_info *ci = curcpu();
    173   1.62   thorpej 
    174  1.105        ad 	(void)junk;
    175   1.85      yamt 
    176  1.163     ozaki 	pr("curcpu holds : %18d wanted by: %#018lx\n",
    177  1.105        ad 	    ci->ci_biglock_count, (long)ci->ci_biglock_wanted);
    178   1.62   thorpej }
    179   1.62   thorpej 
    180  1.105        ad /*
    181  1.150       mrg  * Acquire 'nlocks' holds on the kernel lock.
    182  1.167        ad  *
    183  1.167        ad  * Although it may not look it, this is one of the most central, intricate
    184  1.167        ad  * routines in the kernel, and tons of code elsewhere depends on its exact
    185  1.167        ad  * behaviour.  If you change something in here, expect it to bite you in the
    186  1.167        ad  * rear.
    187  1.105        ad  */
    188   1.62   thorpej void
    189  1.137  drochner _kernel_lock(int nlocks)
    190   1.62   thorpej {
    191  1.138        ad 	struct cpu_info *ci;
    192  1.105        ad 	LOCKSTAT_TIMER(spintime);
    193  1.105        ad 	LOCKSTAT_FLAG(lsflag);
    194  1.105        ad 	struct lwp *owant;
    195  1.165        ad #ifdef LOCKDEBUG
    196  1.174  riastrad 	static struct cpu_info *kernel_lock_holder;
    197  1.165        ad 	u_int spins = 0;
    198  1.180  riastrad 	u_int starttime = getticks();
    199  1.165        ad #endif
    200   1.85      yamt 	int s;
    201  1.137  drochner 	struct lwp *l = curlwp;
    202   1.85      yamt 
    203  1.105        ad 	_KERNEL_LOCK_ASSERT(nlocks > 0);
    204   1.62   thorpej 
    205  1.138        ad 	s = splvm();
    206  1.138        ad 	ci = curcpu();
    207  1.105        ad 	if (ci->ci_biglock_count != 0) {
    208  1.132        ad 		_KERNEL_LOCK_ASSERT(__SIMPLELOCK_LOCKED_P(kernel_lock));
    209  1.105        ad 		ci->ci_biglock_count += nlocks;
    210  1.122        ad 		l->l_blcnt += nlocks;
    211  1.138        ad 		splx(s);
    212  1.105        ad 		return;
    213  1.105        ad 	}
    214  1.105        ad 
    215  1.122        ad 	_KERNEL_LOCK_ASSERT(l->l_blcnt == 0);
    216  1.132        ad 	LOCKDEBUG_WANTLOCK(kernel_lock_dodebug, kernel_lock, RETURN_ADDRESS,
    217  1.154   mlelstv 	    0);
    218  1.107        ad 
    219  1.165        ad 	if (__predict_true(__cpu_simple_lock_try(kernel_lock))) {
    220  1.177  riastrad #ifdef LOCKDEBUG
    221  1.176  riastrad 		kernel_lock_holder = curcpu();
    222  1.177  riastrad #endif
    223  1.105        ad 		ci->ci_biglock_count = nlocks;
    224  1.122        ad 		l->l_blcnt = nlocks;
    225  1.144        ad 		LOCKDEBUG_LOCKED(kernel_lock_dodebug, kernel_lock, NULL,
    226  1.127      yamt 		    RETURN_ADDRESS, 0);
    227  1.105        ad 		splx(s);
    228  1.105        ad 		return;
    229  1.105        ad 	}
    230  1.105        ad 
    231  1.132        ad 	/*
    232  1.132        ad 	 * To remove the ordering constraint between adaptive mutexes
    233  1.132        ad 	 * and kernel_lock we must make it appear as if this thread is
    234  1.132        ad 	 * blocking.  For non-interlocked mutex release, a store fence
    235  1.132        ad 	 * is required to ensure that the result of any mutex_exit()
    236  1.132        ad 	 * by the current LWP becomes visible on the bus before the set
    237  1.132        ad 	 * of ci->ci_biglock_wanted becomes visible.
    238  1.132        ad 	 */
    239  1.132        ad 	membar_producer();
    240  1.132        ad 	owant = ci->ci_biglock_wanted;
    241  1.167        ad 	ci->ci_biglock_wanted = l;
    242  1.168        ad #if defined(DIAGNOSTIC) && !defined(LOCKDEBUG)
    243  1.168        ad 	l->l_ld_wanted = __builtin_return_address(0);
    244  1.168        ad #endif
    245  1.105        ad 
    246  1.105        ad 	/*
    247  1.167        ad 	 * Spin until we acquire the lock.  Once we have it, record the
    248  1.167        ad 	 * time spent with lockstat.
    249  1.105        ad 	 */
    250  1.132        ad 	LOCKSTAT_ENTER(lsflag);
    251  1.132        ad 	LOCKSTAT_START_TIMER(lsflag, spintime);
    252  1.105        ad 
    253  1.105        ad 	do {
    254  1.122        ad 		splx(s);
    255  1.132        ad 		while (__SIMPLELOCK_LOCKED_P(kernel_lock)) {
    256  1.165        ad #ifdef LOCKDEBUG
    257  1.178  riastrad 			extern int start_init_exec;
    258  1.180  riastrad 			if (SPINLOCK_SPINOUT(spins) && start_init_exec &&
    259  1.180  riastrad 			    (getticks() - starttime) > 10*hz) {
    260  1.174  riastrad 				ipi_msg_t msg = {
    261  1.174  riastrad 					.func = kernel_lock_trace_ipi,
    262  1.174  riastrad 				};
    263  1.175  riastrad 				kpreempt_disable();
    264  1.174  riastrad 				ipi_unicast(&msg, kernel_lock_holder);
    265  1.174  riastrad 				ipi_wait(&msg);
    266  1.175  riastrad 				kpreempt_enable();
    267  1.178  riastrad 				_KERNEL_LOCK_ABORT("spinout");
    268  1.132        ad 			}
    269  1.179  riastrad #endif
    270  1.169  christos 			SPINLOCK_BACKOFF_HOOK;
    271  1.169  christos 			SPINLOCK_SPIN_HOOK;
    272  1.105        ad 		}
    273  1.132        ad 		s = splvm();
    274  1.132        ad 	} while (!__cpu_simple_lock_try(kernel_lock));
    275  1.105        ad 
    276  1.122        ad 	ci->ci_biglock_count = nlocks;
    277  1.122        ad 	l->l_blcnt = nlocks;
    278  1.107        ad 	LOCKSTAT_STOP_TIMER(lsflag, spintime);
    279  1.144        ad 	LOCKDEBUG_LOCKED(kernel_lock_dodebug, kernel_lock, NULL,
    280  1.144        ad 	    RETURN_ADDRESS, 0);
    281  1.132        ad 	if (owant == NULL) {
    282  1.132        ad 		LOCKSTAT_EVENT_RA(lsflag, kernel_lock,
    283  1.132        ad 		    LB_KERNEL_LOCK | LB_SPIN, 1, spintime, RETURN_ADDRESS);
    284  1.132        ad 	}
    285  1.132        ad 	LOCKSTAT_EXIT(lsflag);
    286  1.167        ad 	splx(s);
    287  1.105        ad 
    288  1.105        ad 	/*
    289  1.132        ad 	 * Now that we have kernel_lock, reset ci_biglock_wanted.  This
    290  1.132        ad 	 * store must be unbuffered (immediately visible on the bus) in
    291  1.157     skrll 	 * order for non-interlocked mutex release to work correctly.
    292  1.132        ad 	 * It must be visible before a mutex_exit() can execute on this
    293  1.132        ad 	 * processor.
    294  1.132        ad 	 *
    295  1.132        ad 	 * Note: only where CAS is available in hardware will this be
    296  1.132        ad 	 * an unbuffered write, but non-interlocked release cannot be
    297  1.132        ad 	 * done on CPUs without CAS in hardware.
    298  1.105        ad 	 */
    299  1.132        ad 	(void)atomic_swap_ptr(&ci->ci_biglock_wanted, owant);
    300  1.132        ad 
    301  1.132        ad 	/*
    302  1.132        ad 	 * Issue a memory barrier as we have acquired a lock.  This also
    303  1.132        ad 	 * prevents stores from a following mutex_exit() being reordered
    304  1.132        ad 	 * to occur before our store to ci_biglock_wanted above.
    305  1.132        ad 	 */
    306  1.165        ad #ifndef __HAVE_ATOMIC_AS_MEMBAR
    307  1.132        ad 	membar_enter();
    308  1.165        ad #endif
    309  1.174  riastrad 
    310  1.174  riastrad #ifdef LOCKDEBUG
    311  1.174  riastrad 	kernel_lock_holder = curcpu();
    312  1.174  riastrad #endif
    313   1.62   thorpej }
    314   1.62   thorpej 
    315   1.62   thorpej /*
    316  1.105        ad  * Release 'nlocks' holds on the kernel lock.  If 'nlocks' is zero, release
    317  1.150       mrg  * all holds.
    318   1.62   thorpej  */
    319   1.62   thorpej void
    320  1.137  drochner _kernel_unlock(int nlocks, int *countp)
    321   1.62   thorpej {
    322  1.138        ad 	struct cpu_info *ci;
    323  1.105        ad 	u_int olocks;
    324  1.105        ad 	int s;
    325  1.137  drochner 	struct lwp *l = curlwp;
    326   1.62   thorpej 
    327  1.105        ad 	_KERNEL_LOCK_ASSERT(nlocks < 2);
    328   1.62   thorpej 
    329  1.122        ad 	olocks = l->l_blcnt;
    330   1.77      yamt 
    331  1.105        ad 	if (olocks == 0) {
    332  1.105        ad 		_KERNEL_LOCK_ASSERT(nlocks <= 0);
    333  1.105        ad 		if (countp != NULL)
    334  1.105        ad 			*countp = 0;
    335  1.105        ad 		return;
    336  1.105        ad 	}
    337   1.77      yamt 
    338  1.132        ad 	_KERNEL_LOCK_ASSERT(__SIMPLELOCK_LOCKED_P(kernel_lock));
    339   1.85      yamt 
    340  1.105        ad 	if (nlocks == 0)
    341  1.105        ad 		nlocks = olocks;
    342  1.105        ad 	else if (nlocks == -1) {
    343  1.105        ad 		nlocks = 1;
    344  1.105        ad 		_KERNEL_LOCK_ASSERT(olocks == 1);
    345  1.105        ad 	}
    346  1.138        ad 	s = splvm();
    347  1.138        ad 	ci = curcpu();
    348  1.122        ad 	_KERNEL_LOCK_ASSERT(ci->ci_biglock_count >= l->l_blcnt);
    349  1.122        ad 	if (ci->ci_biglock_count == nlocks) {
    350  1.132        ad 		LOCKDEBUG_UNLOCKED(kernel_lock_dodebug, kernel_lock,
    351  1.127      yamt 		    RETURN_ADDRESS, 0);
    352  1.122        ad 		ci->ci_biglock_count = 0;
    353  1.132        ad 		__cpu_simple_unlock(kernel_lock);
    354  1.138        ad 		l->l_blcnt -= nlocks;
    355  1.122        ad 		splx(s);
    356  1.139        ad 		if (l->l_dopreempt)
    357  1.139        ad 			kpreempt(0);
    358  1.138        ad 	} else {
    359  1.122        ad 		ci->ci_biglock_count -= nlocks;
    360  1.138        ad 		l->l_blcnt -= nlocks;
    361  1.138        ad 		splx(s);
    362  1.138        ad 	}
    363   1.77      yamt 
    364  1.105        ad 	if (countp != NULL)
    365  1.105        ad 		*countp = olocks;
    366   1.77      yamt }
    367  1.152  jmcneill 
    368  1.152  jmcneill bool
    369  1.152  jmcneill _kernel_locked_p(void)
    370  1.152  jmcneill {
    371  1.152  jmcneill 	return __SIMPLELOCK_LOCKED_P(kernel_lock);
    372  1.152  jmcneill }
    373