Home | History | Annotate | Line # | Download | only in kern
subr_pserialize.c revision 1.15 
      1 /*	$NetBSD: subr_pserialize.c,v 1.15 2019/12/03 13:30:52 martin Exp $	*/
      2 
      3 /*-
      4  * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc.
      5  * All rights reserved.
      6  *
      7  * Redistribution and use in source and binary forms, with or without
      8  * modification, are permitted provided that the following conditions
      9  * are met:
     10  * 1. Redistributions of source code must retain the above copyright
     11  *    notice, this list of conditions and the following disclaimer.
     12  * 2. Redistributions in binary form must reproduce the above copyright
     13  *    notice, this list of conditions and the following disclaimer in the
     14  *    documentation and/or other materials provided with the distribution.
     15  *
     16  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     17  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     18  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     19  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     20  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     21  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     22  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     23  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     24  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     25  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     26  * POSSIBILITY OF SUCH DAMAGE.
     27  */
     28 
     29 /*
     30  * Passive serialization.
     31  */
     32 
     33 #include <sys/cdefs.h>
     34 __KERNEL_RCSID(0, "$NetBSD: subr_pserialize.c,v 1.15 2019/12/03 13:30:52 martin Exp $");
     35 
     36 #include <sys/param.h>
     37 #include <sys/atomic.h>
     38 #include <sys/cpu.h>
     39 #include <sys/evcnt.h>
     40 #include <sys/kmem.h>
     41 #include <sys/pserialize.h>
     42 #include <sys/xcall.h>
     43 
     44 struct pserialize {
     45 	lwp_t *			psz_owner;
     46 };
     47 
     48 static struct evcnt		psz_ev_excl	__cacheline_aligned;
     49 
     50 /*
     51  * pserialize_init:
     52  *
     53  *	Initialize passive serialization structures.
     54  */
     55 void
     56 pserialize_init(void)
     57 {
     58 
     59 	evcnt_attach_dynamic(&psz_ev_excl, EVCNT_TYPE_MISC, NULL,
     60 	    "pserialize", "exclusive access");
     61 }
     62 
     63 /*
     64  * pserialize_create:
     65  *
     66  *	Create and initialize a passive serialization object.
     67  */
     68 pserialize_t
     69 pserialize_create(void)
     70 {
     71 	pserialize_t psz;
     72 
     73 	psz = kmem_zalloc(sizeof(*psz), KM_SLEEP);
     74 	return psz;
     75 }
     76 
     77 /*
     78  * pserialize_destroy:
     79  *
     80  *	Destroy a passive serialization object.
     81  */
     82 void
     83 pserialize_destroy(pserialize_t psz)
     84 {
     85 
     86 	KASSERT(psz->psz_owner == NULL);
     87 	kmem_free(psz, sizeof(*psz));
     88 }
     89 
     90 /*
     91  * pserialize_perform:
     92  *
     93  *	Perform the write side of passive serialization.  This operation
     94  *	MUST be serialized at a caller level (e.g. with a mutex or by a
     95  *	single-threaded use).
     96  */
     97 void
     98 pserialize_perform(pserialize_t psz)
     99 {
    100 
    101 	KASSERT(!cpu_intr_p());
    102 	KASSERT(!cpu_softintr_p());
    103 
    104 	if (__predict_false(panicstr != NULL)) {
    105 		return;
    106 	}
    107 	KASSERT(psz->psz_owner == NULL);
    108 
    109 	if (__predict_false(mp_online == false)) {
    110 		psz_ev_excl.ev_count++;
    111 		return;
    112 	}
    113 
    114 	psz->psz_owner = curlwp;
    115 
    116 	/*
    117 	 * Broadcast a NOP to all CPUs and wait until all of them complete.
    118 	 */
    119 	xc_barrier(XC_HIGHPRI);
    120 
    121 	KASSERT(psz->psz_owner == curlwp);
    122 	psz->psz_owner = NULL;
    123 #ifdef __HAVE_ATOMIC64_LOADSTORE
    124 	atomic_store_relaxed(&psz_ev_excl.ev_count,
    125 	    1 + atomic_load_relaxed(&psz_ev_excl.ev_count));
    126 #endif
    127 }
    128 
    129 int
    130 pserialize_read_enter(void)
    131 {
    132 	int s;
    133 
    134 	s = splsoftserial();
    135 	curcpu()->ci_psz_read_depth++;
    136 	__insn_barrier();
    137 	return s;
    138 }
    139 
    140 void
    141 pserialize_read_exit(int s)
    142 {
    143 
    144 	KASSERT(kpreempt_disabled());
    145 
    146 	__insn_barrier();
    147 	if (__predict_false(curcpu()->ci_psz_read_depth-- == 0))
    148 		panic("mismatching pserialize_read_exit()");
    149 	splx(s);
    150 }
    151 
    152 /*
    153  * pserialize_in_read_section:
    154  *
    155  *	True if the caller is in a pserialize read section.  To be used
    156  *	only for diagnostic assertions where we want to guarantee the
    157  *	condition like:
    158  *
    159  *		KASSERT(pserialize_in_read_section());
    160  */
    161 bool
    162 pserialize_in_read_section(void)
    163 {
    164 
    165 	return kpreempt_disabled() && curcpu()->ci_psz_read_depth > 0;
    166 }
    167 
    168 /*
    169  * pserialize_not_in_read_section:
    170  *
    171  *	True if the caller is not in a pserialize read section.  To be
    172  *	used only for diagnostic assertions where we want to guarantee
    173  *	the condition like:
    174  *
    175  *		KASSERT(pserialize_not_in_read_section());
    176  */
    177 bool
    178 pserialize_not_in_read_section(void)
    179 {
    180 	bool notin;
    181 
    182 	kpreempt_disable();
    183 	notin = (curcpu()->ci_psz_read_depth == 0);
    184 	kpreempt_enable();
    185 
    186 	return notin;
    187 }
    188