Home | History | Annotate | Line # | Download | only in include 
      1 /*	$NetBSD: lock.h,v 1.29 2022/02/12 17:17:54 riastradh Exp $	*/
      2 
      3 /*-
      4  * Copyright (c) 2000, 2006 The NetBSD Foundation, Inc.
      5  * All rights reserved.
      6  *
      7  * This code is derived from software contributed to The NetBSD Foundation
      8  * by Jason R. Thorpe and Andrew Doran.
      9  *
     10  * Redistribution and use in source and binary forms, with or without
     11  * modification, are permitted provided that the following conditions
     12  * are met:
     13  * 1. Redistributions of source code must retain the above copyright
     14  *    notice, this list of conditions and the following disclaimer.
     15  * 2. Redistributions in binary form must reproduce the above copyright
     16  *    notice, this list of conditions and the following disclaimer in the
     17  *    documentation and/or other materials provided with the distribution.
     18  *
     19  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     20  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     21  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     22  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     23  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     24  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     25  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     26  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     27  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     28  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     29  * POSSIBILITY OF SUCH DAMAGE.
     30  */
     31 
     32 /*
     33  * Machine-dependent spin lock operations.
     34  */
     35 
     36 #ifndef _X86_LOCK_H_
     37 #define	_X86_LOCK_H_
     38 
     39 #include <sys/param.h>
     40 
     41 static __inline int
     42 __SIMPLELOCK_LOCKED_P(const __cpu_simple_lock_t *__ptr)
     43 {
     44 	return *__ptr == __SIMPLELOCK_LOCKED;
     45 }
     46 
     47 static __inline int
     48 __SIMPLELOCK_UNLOCKED_P(const __cpu_simple_lock_t *__ptr)
     49 {
     50 	return *__ptr == __SIMPLELOCK_UNLOCKED;
     51 }
     52 
     53 static __inline void
     54 __cpu_simple_lock_set(__cpu_simple_lock_t *__ptr)
     55 {
     56 
     57 	*__ptr = __SIMPLELOCK_LOCKED;
     58 }
     59 
     60 static __inline void
     61 __cpu_simple_lock_clear(__cpu_simple_lock_t *__ptr)
     62 {
     63 
     64 	*__ptr = __SIMPLELOCK_UNLOCKED;
     65 }
     66 
     67 #ifdef _HARDKERNEL
     68 # include <machine/cpufunc.h>
     69 # define SPINLOCK_SPIN_HOOK	/* nothing */
     70 # ifdef SPINLOCK_BACKOFF_HOOK
     71 #  undef SPINLOCK_BACKOFF_HOOK
     72 # endif
     73 # define SPINLOCK_BACKOFF_HOOK	x86_pause()
     74 # define SPINLOCK_INLINE
     75 #else /* !_HARDKERNEL */
     76 # define SPINLOCK_BODY
     77 # define SPINLOCK_INLINE static __inline __unused
     78 #endif /* _HARDKERNEL */
     79 
     80 SPINLOCK_INLINE void	__cpu_simple_lock_init(__cpu_simple_lock_t *);
     81 SPINLOCK_INLINE void	__cpu_simple_lock(__cpu_simple_lock_t *);
     82 SPINLOCK_INLINE int	__cpu_simple_lock_try(__cpu_simple_lock_t *);
     83 SPINLOCK_INLINE void	__cpu_simple_unlock(__cpu_simple_lock_t *);
     84 
     85 #ifdef SPINLOCK_BODY
     86 SPINLOCK_INLINE void
     87 __cpu_simple_lock_init(__cpu_simple_lock_t *lockp)
     88 {
     89 
     90 	*lockp = __SIMPLELOCK_UNLOCKED;
     91 }
     92 
     93 SPINLOCK_INLINE int
     94 __cpu_simple_lock_try(__cpu_simple_lock_t *lockp)
     95 {
     96 	uint8_t val;
     97 
     98 	val = __SIMPLELOCK_LOCKED;
     99 	__asm volatile ("xchgb %0,(%2)" :
    100 	    "=qQ" (val)
    101 	    :"0" (val), "r" (lockp));
    102 	__insn_barrier();
    103 	return val == __SIMPLELOCK_UNLOCKED;
    104 }
    105 
    106 SPINLOCK_INLINE void
    107 __cpu_simple_lock(__cpu_simple_lock_t *lockp)
    108 {
    109 
    110 	while (!__cpu_simple_lock_try(lockp))
    111 		/* nothing */;
    112 	__insn_barrier();
    113 }
    114 
    115 /*
    116  * Note on x86 memory ordering
    117  *
    118  * When releasing a lock we must ensure that no stores or loads from within
    119  * the critical section are re-ordered by the CPU to occur outside of it:
    120  * they must have completed and be visible to other processors once the lock
    121  * has been released.
    122  *
    123  * NetBSD usually runs with the kernel mapped (via MTRR) in a WB (write
    124  * back) memory region.  In that case, memory ordering on x86 platforms
    125  * looks like this:
    126  *
    127  * i386		All loads/stores occur in instruction sequence.
    128  *
    129  * i486		All loads/stores occur in instruction sequence.  In
    130  * Pentium	exceptional circumstances, loads can be re-ordered around
    131  *		stores, but for the purposes of releasing a lock it does
    132  *		not matter.  Stores may not be immediately visible to other
    133  *		processors as they can be buffered.  However, since the
    134  *		stores are buffered in order the lock release will always be
    135  *		the last operation in the critical section that becomes
    136  *		visible to other CPUs.
    137  *
    138  * Pentium Pro	The "Intel 64 and IA-32 Architectures Software Developer's
    139  * onwards	Manual" volume 3A (order number 248966) says that (1) "Reads
    140  *		can be carried out speculatively and in any order" and (2)
    141  *		"Reads can pass buffered stores, but the processor is
    142  *		self-consistent.".  This would be a problem for the below,
    143  *		and would mandate a locked instruction cycle or load fence
    144  *		before releasing the simple lock.
    145  *
    146  *		The "Intel Pentium 4 Processor Optimization" guide (order
    147  *		number 253668-022US) says: "Loads can be moved before stores
    148  *		that occurred earlier in the program if they are not
    149  *		predicted to load from the same linear address.".  This is
    150  *		not a problem since the only loads that can be re-ordered
    151  *		take place once the lock has been released via a store.
    152  *
    153  *		The above two documents seem to contradict each other,
    154  *		however with the exception of early steppings of the Pentium
    155  *		Pro, the second document is closer to the truth: a store
    156  *		will always act as a load fence for all loads that precede
    157  *		the store in instruction order.
    158  *
    159  *		Again, note that stores can be buffered and will not always
    160  *		become immediately visible to other CPUs: they are however
    161  *		buffered in order.
    162  *
    163  * AMD64	Stores occur in order and are buffered.  Loads can be
    164  *		reordered, however stores act as load fences, meaning that
    165  *		loads can not be reordered around stores.
    166  */
    167 SPINLOCK_INLINE void
    168 __cpu_simple_unlock(__cpu_simple_lock_t *lockp)
    169 {
    170 
    171 	__insn_barrier();
    172 	*lockp = __SIMPLELOCK_UNLOCKED;
    173 }
    174 
    175 #endif	/* SPINLOCK_BODY */
    176 
    177 #endif /* _X86_LOCK_H_ */
    178