Home | History | Annotate | Line # | Download | only in kern
subr_pserialize.c revision 1.11.2.2 
      1 /*	$NetBSD: subr_pserialize.c,v 1.11.2.2 2020/04/13 08:05:04 martin Exp $	*/
      2 
      3 /*-
      4  * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc.
      5  * All rights reserved.
      6  *
      7  * Redistribution and use in source and binary forms, with or without
      8  * modification, are permitted provided that the following conditions
      9  * are met:
     10  * 1. Redistributions of source code must retain the above copyright
     11  *    notice, this list of conditions and the following disclaimer.
     12  * 2. Redistributions in binary form must reproduce the above copyright
     13  *    notice, this list of conditions and the following disclaimer in the
     14  *    documentation and/or other materials provided with the distribution.
     15  *
     16  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     17  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     18  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     19  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     20  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     21  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     22  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     23  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     24  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     25  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     26  * POSSIBILITY OF SUCH DAMAGE.
     27  */
     28 
     29 /*
     30  * Passive serialization.
     31  */
     32 
     33 #include <sys/cdefs.h>
     34 __KERNEL_RCSID(0, "$NetBSD: subr_pserialize.c,v 1.11.2.2 2020/04/13 08:05:04 martin Exp $");
     35 
     36 #include <sys/param.h>
     37 #include <sys/atomic.h>
     38 #include <sys/cpu.h>
     39 #include <sys/evcnt.h>
     40 #include <sys/kmem.h>
     41 #include <sys/mutex.h>
     42 #include <sys/pserialize.h>
     43 #include <sys/xcall.h>
     44 
     45 struct pserialize {
     46 	lwp_t *			psz_owner;
     47 };
     48 
     49 static kmutex_t			psz_lock	__cacheline_aligned;
     50 static struct evcnt		psz_ev_excl	__cacheline_aligned =
     51     EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pserialize", "exclusive access");
     52 EVCNT_ATTACH_STATIC(psz_ev_excl);
     53 
     54 /*
     55  * pserialize_init:
     56  *
     57  *	Initialize passive serialization structures.
     58  */
     59 void
     60 pserialize_init(void)
     61 {
     62 
     63 	mutex_init(&psz_lock, MUTEX_DEFAULT, IPL_NONE);
     64 }
     65 
     66 /*
     67  * pserialize_create:
     68  *
     69  *	Create and initialize a passive serialization object.
     70  */
     71 pserialize_t
     72 pserialize_create(void)
     73 {
     74 	pserialize_t psz;
     75 
     76 	psz = kmem_zalloc(sizeof(*psz), KM_SLEEP);
     77 	return psz;
     78 }
     79 
     80 /*
     81  * pserialize_destroy:
     82  *
     83  *	Destroy a passive serialization object.
     84  */
     85 void
     86 pserialize_destroy(pserialize_t psz)
     87 {
     88 
     89 	KASSERT(psz->psz_owner == NULL);
     90 	kmem_free(psz, sizeof(*psz));
     91 }
     92 
     93 /*
     94  * pserialize_perform:
     95  *
     96  *	Perform the write side of passive serialization.  This operation
     97  *	MUST be serialized at a caller level (e.g. with a mutex or by a
     98  *	single-threaded use).
     99  */
    100 void
    101 pserialize_perform(pserialize_t psz)
    102 {
    103 
    104 	KASSERT(!cpu_intr_p());
    105 	KASSERT(!cpu_softintr_p());
    106 
    107 	if (__predict_false(panicstr != NULL)) {
    108 		return;
    109 	}
    110 	KASSERT(psz->psz_owner == NULL);
    111 
    112 	if (__predict_false(mp_online == false)) {
    113 		psz_ev_excl.ev_count++;
    114 		return;
    115 	}
    116 
    117 	psz->psz_owner = curlwp;
    118 
    119 	/*
    120 	 * Broadcast a NOP to all CPUs and wait until all of them complete.
    121 	 */
    122 	xc_barrier(XC_HIGHPRI);
    123 
    124 	KASSERT(psz->psz_owner == curlwp);
    125 	psz->psz_owner = NULL;
    126 	mutex_enter(&psz_lock);
    127 	psz_ev_excl.ev_count++;
    128 	mutex_exit(&psz_lock);
    129 }
    130 
    131 int
    132 pserialize_read_enter(void)
    133 {
    134 	int s;
    135 
    136 	s = splsoftserial();
    137 	curcpu()->ci_psz_read_depth++;
    138 	__insn_barrier();
    139 	return s;
    140 }
    141 
    142 void
    143 pserialize_read_exit(int s)
    144 {
    145 
    146 	KASSERT(kpreempt_disabled());
    147 
    148 	__insn_barrier();
    149 	if (__predict_false(curcpu()->ci_psz_read_depth-- == 0))
    150 		panic("mismatching pserialize_read_exit()");
    151 	splx(s);
    152 }
    153 
    154 /*
    155  * pserialize_in_read_section:
    156  *
    157  *	True if the caller is in a pserialize read section.  To be used
    158  *	only for diagnostic assertions where we want to guarantee the
    159  *	condition like:
    160  *
    161  *		KASSERT(pserialize_in_read_section());
    162  */
    163 bool
    164 pserialize_in_read_section(void)
    165 {
    166 
    167 	return kpreempt_disabled() && curcpu()->ci_psz_read_depth > 0;
    168 }
    169 
    170 /*
    171  * pserialize_not_in_read_section:
    172  *
    173  *	True if the caller is not in a pserialize read section.  To be
    174  *	used only for diagnostic assertions where we want to guarantee
    175  *	the condition like:
    176  *
    177  *		KASSERT(pserialize_not_in_read_section());
    178  */
    179 bool
    180 pserialize_not_in_read_section(void)
    181 {
    182 	bool notin;
    183 
    184 	kpreempt_disable();
    185 	notin = (curcpu()->ci_psz_read_depth == 0);
    186 	kpreempt_enable();
    187 
    188 	return notin;
    189 }
    190