Home | History | Annotate | Line # | Download | only in include
lock.h revision 1.14.18.2
      1  1.14.18.2     skrll /*	$NetBSD: lock.h,v 1.14.18.2 2007/08/15 08:24:07 skrll Exp $	*/
      2        1.1      fvdl 
      3        1.1      fvdl /*-
      4       1.12        ad  * Copyright (c) 2000, 2006 The NetBSD Foundation, Inc.
      5        1.1      fvdl  * All rights reserved.
      6        1.1      fvdl  *
      7        1.1      fvdl  * This code is derived from software contributed to The NetBSD Foundation
      8       1.12        ad  * by Jason R. Thorpe and Andrew Doran.
      9        1.1      fvdl  *
     10        1.1      fvdl  * Redistribution and use in source and binary forms, with or without
     11        1.1      fvdl  * modification, are permitted provided that the following conditions
     12        1.1      fvdl  * are met:
     13        1.1      fvdl  * 1. Redistributions of source code must retain the above copyright
     14        1.1      fvdl  *    notice, this list of conditions and the following disclaimer.
     15        1.1      fvdl  * 2. Redistributions in binary form must reproduce the above copyright
     16        1.1      fvdl  *    notice, this list of conditions and the following disclaimer in the
     17        1.1      fvdl  *    documentation and/or other materials provided with the distribution.
     18        1.1      fvdl  * 3. All advertising materials mentioning features or use of this software
     19        1.1      fvdl  *    must display the following acknowledgement:
     20        1.1      fvdl  *	This product includes software developed by the NetBSD
     21        1.1      fvdl  *	Foundation, Inc. and its contributors.
     22        1.1      fvdl  * 4. Neither the name of The NetBSD Foundation nor the names of its
     23        1.1      fvdl  *    contributors may be used to endorse or promote products derived
     24        1.1      fvdl  *    from this software without specific prior written permission.
     25        1.1      fvdl  *
     26        1.1      fvdl  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     27        1.1      fvdl  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     28        1.1      fvdl  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     29        1.1      fvdl  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     30        1.1      fvdl  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     31        1.1      fvdl  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     32        1.1      fvdl  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     33        1.1      fvdl  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     34        1.1      fvdl  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     35        1.1      fvdl  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     36        1.1      fvdl  * POSSIBILITY OF SUCH DAMAGE.
     37        1.1      fvdl  */
     38        1.1      fvdl 
     39        1.1      fvdl /*
     40        1.1      fvdl  * Machine-dependent spin lock operations.
     41        1.1      fvdl  */
     42        1.1      fvdl 
     43        1.9      yamt #ifndef _X86_LOCK_H_
     44        1.9      yamt #define	_X86_LOCK_H_
     45        1.1      fvdl 
     46        1.1      fvdl #if defined(_KERNEL_OPT)
     47        1.1      fvdl #include "opt_lockdebug.h"
     48        1.1      fvdl #endif
     49        1.1      fvdl 
     50        1.2      fvdl #include <machine/cpufunc.h>
     51        1.1      fvdl 
     52  1.14.18.2     skrll static __inline int
     53  1.14.18.2     skrll __SIMPLELOCK_LOCKED_P(__cpu_simple_lock_t *__ptr)
     54  1.14.18.2     skrll {
     55  1.14.18.2     skrll 	return *__ptr == __SIMPLELOCK_LOCKED;
     56  1.14.18.2     skrll }
     57  1.14.18.2     skrll 
     58  1.14.18.2     skrll static __inline int
     59  1.14.18.2     skrll __SIMPLELOCK_UNLOCKED_P(__cpu_simple_lock_t *__ptr)
     60  1.14.18.2     skrll {
     61  1.14.18.2     skrll 	return *__ptr == __SIMPLELOCK_UNLOCKED;
     62  1.14.18.2     skrll }
     63  1.14.18.2     skrll 
     64        1.1      fvdl #ifdef LOCKDEBUG
     65        1.1      fvdl 
     66        1.5  junyoung extern void __cpu_simple_lock_init(__cpu_simple_lock_t *);
     67        1.5  junyoung extern void __cpu_simple_lock(__cpu_simple_lock_t *);
     68        1.5  junyoung extern int __cpu_simple_lock_try(__cpu_simple_lock_t *);
     69        1.5  junyoung extern void __cpu_simple_unlock(__cpu_simple_lock_t *);
     70        1.1      fvdl 
     71        1.1      fvdl #else
     72        1.1      fvdl 
     73        1.1      fvdl #include <machine/atomic.h>
     74        1.1      fvdl 
     75       1.11     perry static __inline void __cpu_simple_lock_init(__cpu_simple_lock_t *)
     76        1.1      fvdl 	__attribute__((__unused__));
     77       1.11     perry static __inline void __cpu_simple_lock(__cpu_simple_lock_t *)
     78        1.1      fvdl 	__attribute__((__unused__));
     79       1.11     perry static __inline int __cpu_simple_lock_try(__cpu_simple_lock_t *)
     80        1.1      fvdl 	__attribute__((__unused__));
     81       1.11     perry static __inline void __cpu_simple_unlock(__cpu_simple_lock_t *)
     82        1.1      fvdl 	__attribute__((__unused__));
     83        1.1      fvdl 
     84  1.14.18.1     skrll static __inline void
     85  1.14.18.1     skrll __cpu_simple_lock_set(__cpu_simple_lock_t *__ptr)
     86  1.14.18.1     skrll {
     87  1.14.18.1     skrll 
     88  1.14.18.1     skrll 	*__ptr = __SIMPLELOCK_LOCKED;
     89  1.14.18.1     skrll }
     90  1.14.18.1     skrll 
     91  1.14.18.1     skrll static __inline void
     92  1.14.18.1     skrll __cpu_simple_lock_clear(__cpu_simple_lock_t *__ptr)
     93  1.14.18.1     skrll {
     94  1.14.18.1     skrll 
     95  1.14.18.1     skrll 	*__ptr = __SIMPLELOCK_UNLOCKED;
     96  1.14.18.1     skrll }
     97  1.14.18.1     skrll 
     98       1.11     perry static __inline void
     99        1.1      fvdl __cpu_simple_lock_init(__cpu_simple_lock_t *lockp)
    100        1.1      fvdl {
    101        1.1      fvdl 
    102        1.1      fvdl 	*lockp = __SIMPLELOCK_UNLOCKED;
    103        1.7      yamt 	__insn_barrier();
    104        1.1      fvdl }
    105        1.1      fvdl 
    106       1.11     perry static __inline void
    107        1.1      fvdl __cpu_simple_lock(__cpu_simple_lock_t *lockp)
    108        1.1      fvdl {
    109        1.1      fvdl 
    110       1.13        ad 	while (x86_atomic_testset_b(lockp, __SIMPLELOCK_LOCKED)
    111        1.6      yamt 	    != __SIMPLELOCK_UNLOCKED) {
    112        1.6      yamt 		do {
    113        1.6      yamt 			x86_pause();
    114        1.6      yamt 		} while (*lockp == __SIMPLELOCK_LOCKED);
    115        1.6      yamt 	}
    116        1.7      yamt 	__insn_barrier();
    117        1.1      fvdl }
    118        1.1      fvdl 
    119       1.11     perry static __inline int
    120        1.1      fvdl __cpu_simple_lock_try(__cpu_simple_lock_t *lockp)
    121        1.1      fvdl {
    122       1.13        ad 	int r = (x86_atomic_testset_b(lockp, __SIMPLELOCK_LOCKED)
    123        1.1      fvdl 	    == __SIMPLELOCK_UNLOCKED);
    124        1.1      fvdl 
    125        1.7      yamt 	__insn_barrier();
    126        1.1      fvdl 
    127        1.1      fvdl 	return (r);
    128        1.1      fvdl }
    129        1.1      fvdl 
    130       1.12        ad /*
    131       1.12        ad  * Note on x86 memory ordering
    132       1.12        ad  *
    133       1.12        ad  * When releasing a lock we must ensure that no stores or loads from within
    134       1.12        ad  * the critical section are re-ordered by the CPU to occur outside of it:
    135       1.12        ad  * they must have completed and be visible to other processors once the lock
    136       1.12        ad  * has been released.
    137       1.12        ad  *
    138       1.12        ad  * NetBSD usually runs with the kernel mapped (via MTRR) in a WB (write
    139       1.12        ad  * back) memory region.  In that case, memory ordering on x86 platforms
    140       1.12        ad  * looks like this:
    141       1.12        ad  *
    142       1.12        ad  * i386		All loads/stores occur in instruction sequence.
    143       1.12        ad  *
    144       1.12        ad  * i486		All loads/stores occur in instruction sequence.  In
    145       1.12        ad  * Pentium	exceptional circumstances, loads can be re-ordered around
    146       1.12        ad  *		stores, but for the purposes of releasing a lock it does
    147       1.12        ad  *		not matter.  Stores may not be immediately visible to other
    148       1.12        ad  *		processors as they can be buffered.  However, since the
    149       1.12        ad  *		stores are buffered in order the lock release will always be
    150       1.12        ad  *		the last operation in the critical section that becomes
    151       1.12        ad  *		visible to other CPUs.
    152       1.12        ad  *
    153       1.12        ad  * Pentium Pro	The "Intel 64 and IA-32 Architectures Software Developer's
    154       1.12        ad  * onwards	Manual" volume 3A (order number 248966) says that (1) "Reads
    155       1.12        ad  *		can be carried out speculatively and in any order" and (2)
    156       1.12        ad  *		"Reads can pass buffered stores, but the processor is
    157       1.12        ad  *		self-consistent.".  This would be a problem for the below,
    158       1.12        ad  *		and would mandate a locked instruction cycle or load fence
    159       1.12        ad  *		before releasing the simple lock.
    160       1.12        ad  *
    161       1.12        ad  *		The "Intel Pentium 4 Processor Optimization" guide (order
    162       1.12        ad  *		number 253668-022US) says: "Loads can be moved before stores
    163       1.12        ad  *		that occurred earlier in the program if they are not
    164       1.12        ad  *		predicted to load from the same linear address.".  This is
    165       1.12        ad  *		not a problem since the only loads that can be re-ordered
    166       1.12        ad  *		take place once the lock has been released via a store.
    167       1.12        ad  *
    168       1.12        ad  *		The above two documents seem to contradict each other,
    169       1.12        ad  *		however with the exception of early steppings of the Pentium
    170       1.12        ad  *		Pro, the second document is closer to the truth: a store
    171       1.12        ad  *		will always act as a load fence for all loads that precede
    172       1.12        ad  *		the store in instruction order.
    173       1.12        ad  *
    174       1.12        ad  *		Again, note that stores can be buffered and will not always
    175       1.12        ad  *		become immediately visible to other CPUs: they are however
    176       1.12        ad  *		buffered in order.
    177       1.12        ad  *
    178       1.12        ad  * AMD64	Stores occur in order and are buffered.  Loads can be
    179       1.12        ad  *		reordered, however stores act as load fences, meaning that
    180       1.12        ad  *		loads can not be reordered around stores.
    181       1.12        ad  */
    182       1.11     perry static __inline void
    183        1.1      fvdl __cpu_simple_unlock(__cpu_simple_lock_t *lockp)
    184        1.1      fvdl {
    185        1.1      fvdl 
    186        1.7      yamt 	__insn_barrier();
    187        1.1      fvdl 	*lockp = __SIMPLELOCK_UNLOCKED;
    188        1.1      fvdl }
    189        1.1      fvdl 
    190        1.1      fvdl #endif /* !LOCKDEBUG */
    191        1.4      yamt 
    192       1.13        ad #define	SPINLOCK_SPIN_HOOK	/* nothing */
    193       1.13        ad #define	SPINLOCK_BACKOFF_HOOK	x86_pause()
    194       1.13        ad 
    195        1.4      yamt #ifdef _KERNEL
    196       1.13        ad void	mb_read(void);
    197       1.13        ad void	mb_write(void);
    198       1.13        ad void	mb_memory(void);
    199       1.13        ad #else	/* _KERNEL */
    200       1.14        ad static __inline void
    201       1.13        ad mb_read(void)
    202       1.13        ad {
    203       1.13        ad 	x86_lfence();
    204       1.13        ad }
    205       1.13        ad 
    206       1.14        ad static __inline void
    207       1.13        ad mb_write(void)
    208       1.13        ad {
    209       1.13        ad 	__insn_barrier();
    210       1.13        ad }
    211       1.13        ad 
    212       1.14        ad static __inline void
    213       1.13        ad mb_memory(void)
    214       1.13        ad {
    215       1.13        ad 	x86_mfence();
    216       1.13        ad }
    217       1.13        ad #endif	/* _KERNEL */
    218        1.1      fvdl 
    219        1.9      yamt #endif /* _X86_LOCK_H_ */
    220