lock_stubs.S revision 1.3
1/* $NetBSD: lock_stubs.S,v 1.3 2007/05/17 14:51:13 yamt Exp $ */ 2 3/*- 4 * Copyright (c) 2006, 2007 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. All advertising materials mentioning features or use of this software 19 * must display the following acknowledgement: 20 * This product includes software developed by the NetBSD 21 * Foundation, Inc. and its contributors. 22 * 4. Neither the name of The NetBSD Foundation nor the names of its 23 * contributors may be used to endorse or promote products derived 24 * from this software without specific prior written permission. 25 * 26 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 27 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 28 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 29 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 30 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 31 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 32 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 33 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 34 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 35 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 36 * POSSIBILITY OF SUCH DAMAGE. 37 */ 38 39/* 40 * AMD64 lock stubs. Calling convention: 41 * 42 * %rdi arg 1 43 * %rsi arg 2 44 * %rdx arg 3 45 * %rax return value 46 */ 47 48#include "opt_multiprocessor.h" 49#include "opt_lockdebug.h" 50 51#include <machine/asm.h> 52#include <machine/intrdefs.h> 53 54#include "assym.h" 55 56#if defined(DIAGNOSTIC) || defined(MULTIPROCESSOR) || defined(LOCKDEBUG) 57#define FULL 58#endif 59 60#if defined(MULTIPROCESSOR) 61#define LOCK lock 62#else 63#define LOCK /* nothing */ 64#endif 65 66#define END(name,a) .align a; LABEL(name) 67 68#ifndef LOCKDEBUG 69 70/* 71 * void mutex_enter(kmutex_t *mtx); 72 * 73 * Acquire a mutex and post a load fence. 74 */ 75 .align 64 76 77NENTRY(mutex_enter) /* 0x0000, 25 bytes */ 78 movq CPUVAR(CURLWP), %rcx 79 xorq %rax, %rax 80 LOCK 81 cmpxchgq %rcx, MTX_OWNER(%rdi) 82 jnz,pn _C_LABEL(mutex_vector_enter) 83 ret 84 85/* 86 * void mutex_exit(kmutex_t *mtx); 87 * 88 * Release a mutex and post a load fence. 89 * 90 * See comments in mutex_vector_enter() about doing this operation unlocked 91 * on multiprocessor systems, and comments in arch/x86/include/lock.h about 92 * memory ordering on Intel x86 systems. 93 */ 94 .align 32 95 96NENTRY(mutex_exit) /* 0x0020, 24 bytes */ 97 movq CPUVAR(CURLWP), %rax 98 xorq %rdx, %rdx 99 cmpxchgq %rdx, MTX_OWNER(%rdi) 100 jnz,pn _C_LABEL(mutex_vector_exit) 101 ret 102 103/* 104 * void mutex_spin_enter(kmutex_t *mtx); 105 * 106 * Acquire a spin mutex and post a load fence. 107 */ 108 .align 64 109 110NENTRY(mutex_spin_enter) /* 0x0040, 58 bytes */ 111 movq CPUVAR(SELF200), %r8 112#if defined(FULL) 113 movl $0x0100, %eax /* new + expected value */ 114#endif 115 movl (CPU_INFO_ILEVEL-0x200)(%r8), %esi 116 subl $1, (CPU_INFO_MTX_COUNT-0x200)(%r8)/* decl doesnt set CF */ 117 movzbl MTX_IPL(%rdi), %ecx /* new SPL */ 118 cmovncl (CPU_INFO_MTX_OLDSPL-0x200)(%r8), %esi 119 cmpl %ecx, %esi /* higher? */ 120 movl %esi, (CPU_INFO_MTX_OLDSPL-0x200)(%r8) 121 cmovgl %esi, %ecx 122 movl %ecx, (CPU_INFO_ILEVEL-0x200)(%r8) /* splraiseipl() */ 123#if defined(FULL) 124 LOCK 125 cmpxchgb %ah, MTX_LOCK(%rdi) /* lock */ 126 jnz,pn _C_LABEL(mutex_spin_retry) /* failed; hard case */ 127#endif 128 ret 129 130/* 131 * void mutex_spin_exit(kmutex_t *mtx); 132 * 133 * Release a spin mutex and post a load fence. 134 */ 135 .align 64 136 137NENTRY(mutex_spin_exit) /* 0x0080, 60 bytes */ 138#ifdef DIAGNOSTIC 139 140 movl $0x0001, %eax /* new + expected value */ 141 movq CPUVAR(SELF200), %r8 142 cmpxchgb %ah, MTX_LOCK(%rdi) /* unlock */ 143 jnz,pn _C_LABEL(mutex_vector_exit) /* hard case if problems */ 144 movl (CPU_INFO_MTX_OLDSPL-0x200)(%r8), %edi 145 incl (CPU_INFO_MTX_COUNT-0x200)(%r8) 146 jnz 1f 147 cmpl (CPU_INFO_ILEVEL-0x200)(%r8), %edi 148 jae 1f 149 movl (CPU_INFO_IUNMASK-0x200)(%r8,%rdi,4), %esi 150 cli 151 testl (CPU_INFO_IPENDING-0x200)(%r8), %esi 152 jnz _C_LABEL(Xspllower) 153 movl %edi, (CPU_INFO_ILEVEL-0x200)(%r8) 154 sti 1551: rep /* double byte ret as branch */ 156 ret /* target: see AMD docs */ 157 158#else /* DIAGNOSTIC */ 159 160 movq CPUVAR(SELF200), %rsi 161#ifdef MULTIPROCESSOR 162 movb $0x00, MTX_LOCK(%rdi) 163#endif 164 movl (CPU_INFO_MTX_OLDSPL-0x200)(%rsi), %ecx 165 incl (CPU_INFO_MTX_COUNT-0x200)(%rsi) 166 movl (CPU_INFO_ILEVEL-0x200)(%rsi),%edx 167 cmovnzl %edx,%ecx 168 cmpl %edx,%ecx /* new level is lower? */ 169 pushq %rbx 170 jae,pn 2f 1711: 172 movl (CPU_INFO_IPENDING-0x200)(%rsi),%eax 173 testl %eax,(CPU_INFO_IUNMASK-0x200)(%rsi,%rcx,4)/* deferred interrupts? */ 174 movl %eax,%ebx 175 jnz,pn 3f 176 cmpxchg8b (CPU_INFO_ISTATE-0x200)(%rsi) /* swap in new ilevel */ 177 jnz,pn 1b 1782: 179 popq %rbx 180 ret 1813: 182 popq %rbx 183 movl %ecx, %edi 184 jmp _C_LABEL(Xspllower) 185 186#endif /* DIAGNOSTIC */ 187 188/* 189 * void rw_enter(krwlock_t *rwl, krw_t op); 190 * 191 * Acquire one hold on a RW lock. 192 */ 193 .align 64 194 195NENTRY(rw_enter) /* 0x00c0, 62 bytes */ 196 cmpl $RW_READER, %esi 197 jne 2f 198 199 /* 200 * Reader: this is the most common case. 201 */ 2021: movq RW_OWNER(%rdi), %rax 203 testb $(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al 204 leaq RW_READ_INCR(%rax), %rdx 205 jnz,pn _C_LABEL(rw_vector_enter) 206 LOCK 207 cmpxchgq %rdx, RW_OWNER(%rdi) 208 jnz,pn 1b 209 ret 210 211 /* 212 * Writer: if the compare-and-set fails, don't bother retrying. 213 */ 2142: movq CPUVAR(CURLWP), %rcx 215 xorq %rax, %rax 216 orq $RW_WRITE_LOCKED, %rcx 217 LOCK 218 cmpxchgq %rcx, RW_OWNER(%rdi) 219 jnz,pn _C_LABEL(rw_vector_enter) 220 ret 221 222/* 223 * void rw_exit(krwlock_t *rwl); 224 * 225 * Release one hold on a RW lock. 226 */ 227 .align 64 228 229NENTRY(rw_exit) /* 0x0100, 64 bytes */ 230 movq RW_OWNER(%rdi), %rax 231 testb $RW_WRITE_LOCKED, %al 232 jnz 2f 233 234 /* 235 * Reader 236 */ 2371: testb $RW_HAS_WAITERS, %al 238 jnz,pn 3f 239 cmpq $RW_READ_INCR, %rax 240 leaq -RW_READ_INCR(%rax), %rdx 241 jb,pn 3f 242 LOCK 243 cmpxchgq %rdx, RW_OWNER(%rdi) 244 jnz,pn 1b 245 ret 246 247 /* 248 * Writer 249 */ 2502: leaq -RW_WRITE_LOCKED(%rax), %rdx 251 subq CPUVAR(CURLWP), %rdx 252 jnz,pn 3f 253 LOCK 254 cmpxchgq %rdx, RW_OWNER(%rdi) 255 jnz 3f 256 ret 257 2583: jmp _C_LABEL(rw_vector_exit) 259 260#endif /* LOCKDEBUG */ 261 262/* 263 * int _lock_cas(uintptr_t *val, uintptr_t old, uintptr_t new); 264 * 265 * Perform an atomic compare-and-set operation. 266 */ 267 .align 32 268 269NENTRY(_lock_cas) /* 0x0140, 19 bytes */ 270 movq %rsi, %rax 271 LOCK 272 cmpxchgq %rdx, (%rdi) 273 movq $0, %rax 274 setz %al /* = 1 if success */ 275 ret 276 277/* 278 * Memory barrier operations. 279 */ 280 281 .align 8 282 283NENTRY(mb_read) 284 lfence 285 ret 286END(mb_read_end, 8) 287 288NENTRY(mb_write) 289 /* Nothing just yet */ 290 ret 291END(mb_write_end, 8) 292 293NENTRY(mb_memory) 294 mfence 295 ret 296END(mb_memory_end, 8) 297 298/* 299 * Make sure code after the ret is properly encoded with nopness 300 * by gas, or could stall newer processors. 301 */ 302 303NENTRY(x86_mb_nop) 304 ret 305END(x86_mb_nop_end, 8) 306