lock_stubs.S revision 1.13
1/* $NetBSD: lock_stubs.S,v 1.13 2008/01/25 19:02:32 ad Exp $ */ 2 3/*- 4 * Copyright (c) 2006, 2007, 2008 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. All advertising materials mentioning features or use of this software 19 * must display the following acknowledgement: 20 * This product includes software developed by the NetBSD 21 * Foundation, Inc. and its contributors. 22 * 4. Neither the name of The NetBSD Foundation nor the names of its 23 * contributors may be used to endorse or promote products derived 24 * from this software without specific prior written permission. 25 * 26 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 27 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 28 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 29 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 30 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 31 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 32 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 33 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 34 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 35 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 36 * POSSIBILITY OF SUCH DAMAGE. 37 */ 38 39/* 40 * AMD64 lock stubs. Calling convention: 41 * 42 * %rdi arg 1 43 * %rsi arg 2 44 * %rdx arg 3 45 * %rax return value 46 */ 47 48#include "opt_multiprocessor.h" 49#include "opt_lockdebug.h" 50 51#include <machine/asm.h> 52#include <machine/frameasm.h> 53 54#include "assym.h" 55 56#if defined(DIAGNOSTIC) || defined(MULTIPROCESSOR) || defined(LOCKDEBUG) 57#define FULL 58#endif 59 60#define END(name,a) .align a; LABEL(name) 61#define LOCK(num) .Lpatch/**/num: lock 62 63#ifndef LOCKDEBUG 64 65/* 66 * void mutex_enter(kmutex_t *mtx); 67 * 68 * Acquire a mutex and post a load fence. 69 */ 70 .align 64 71 72NENTRY(mutex_enter) 73 movq CPUVAR(CURLWP), %rcx 74 xorq %rax, %rax 75 LOCK(1) 76 cmpxchgq %rcx, MTX_OWNER(%rdi) 77 jnz,pn _C_LABEL(mutex_vector_enter) 78 ret 79 80/* 81 * void mutex_exit(kmutex_t *mtx); 82 * 83 * Release a mutex and post a load fence. 84 * 85 * See comments in mutex_vector_enter() about doing this operation unlocked 86 * on multiprocessor systems, and comments in arch/x86/include/lock.h about 87 * memory ordering on Intel x86 systems. 88 */ 89NENTRY(mutex_exit) 90 movq CPUVAR(CURLWP), %rax 91 xorq %rdx, %rdx 92 cmpxchgq %rdx, MTX_OWNER(%rdi) 93 jnz,pn _C_LABEL(mutex_vector_exit) 94 ret 95 96/* 97 * void mutex_spin_enter(kmutex_t *mtx); 98 * 99 * Acquire a spin mutex and post a load fence. 100 */ 101NENTRY(mutex_spin_enter) 102 movq CPUVAR(SELF), %r8 103#if defined(FULL) 104 movl $0x0100, %eax /* new + expected value */ 105#endif 106 movl CPU_INFO_ILEVEL(%r8), %esi 107 subl $1, CPU_INFO_MTX_COUNT(%r8) /* decl doesnt set CF */ 108 movzbl MTX_IPL(%rdi), %ecx /* new SPL */ 109 cmovncl CPU_INFO_MTX_OLDSPL(%r8), %esi 110 cmpl %ecx, %esi /* higher? */ 111 movl %esi, CPU_INFO_MTX_OLDSPL(%r8) 112 cmovgl %esi, %ecx 113 movl %ecx, CPU_INFO_ILEVEL(%r8) /* splraiseipl() */ 114#if defined(FULL) 115 LOCK(11) 116 cmpxchgb %ah, MTX_LOCK(%rdi) /* lock */ 117 jnz,pn _C_LABEL(mutex_spin_retry) /* failed; hard case */ 118#endif 119 ret 120 121/* 122 * void mutex_spin_exit(kmutex_t *mtx); 123 * 124 * Release a spin mutex and post a load fence. 125 */ 126NENTRY(mutex_spin_exit) 127#ifdef DIAGNOSTIC 128 129 movl $0x0001, %eax /* new + expected value */ 130 movq CPUVAR(SELF), %r8 131 cmpxchgb %ah, MTX_LOCK(%rdi) /* unlock */ 132 jnz,pn _C_LABEL(mutex_vector_exit) /* hard case if problems */ 133 movl CPU_INFO_MTX_OLDSPL(%r8), %edi 134 incl CPU_INFO_MTX_COUNT(%r8) 135 jnz 1f 136 cmpl CPU_INFO_ILEVEL(%r8), %edi 137 jae 1f 138 movl CPU_INFO_IUNMASK(%r8,%rdi,4), %esi 139 CLI(ax) 140 testl CPU_INFO_IPENDING(%r8), %esi 141 jnz _C_LABEL(Xspllower) 142 movl %edi, CPU_INFO_ILEVEL(%r8) 143 STI(ax) 1441: rep /* double byte ret as branch */ 145 ret /* target: see AMD docs */ 146 147#else /* DIAGNOSTIC */ 148 149 movq CPUVAR(SELF), %rsi 150#ifdef MULTIPROCESSOR 151 movb $0x00, MTX_LOCK(%rdi) 152#endif 153 movl CPU_INFO_MTX_OLDSPL(%rsi), %ecx 154 incl CPU_INFO_MTX_COUNT(%rsi) 155 movl CPU_INFO_ILEVEL(%rsi),%edx 156 cmovnzl %edx,%ecx 157 cmpl %edx,%ecx /* new level is lower? */ 158 pushq %rbx 159 jae,pn 2f 1601: 161 movl CPU_INFO_IPENDING(%rsi),%eax 162 testl %eax,CPU_INFO_IUNMASK(%rsi,%rcx,4)/* deferred interrupts? */ 163 movl %eax,%ebx 164 jnz,pn 3f 165 cmpxchg8b CPU_INFO_ISTATE(%rsi) /* swap in new ilevel */ 166 jnz,pn 1b 1672: 168 popq %rbx 169 ret 1703: 171 popq %rbx 172 movl %ecx, %edi 173 jmp _C_LABEL(Xspllower) 174 175#endif /* DIAGNOSTIC */ 176 177/* 178 * void rw_enter(krwlock_t *rwl, krw_t op); 179 * 180 * Acquire one hold on a RW lock. 181 */ 182NENTRY(rw_enter) 183 cmpl $RW_READER, %esi 184 jne 2f 185 186 /* 187 * Reader: this is the most common case. 188 */ 1891: movq RW_OWNER(%rdi), %rax 190 testb $(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al 191 leaq RW_READ_INCR(%rax), %rdx 192 jnz,pn _C_LABEL(rw_vector_enter) 193 LOCK(2) 194 cmpxchgq %rdx, RW_OWNER(%rdi) 195 jnz,pn 1b 196 ret 197 198 /* 199 * Writer: if the compare-and-set fails, don't bother retrying. 200 */ 2012: movq CPUVAR(CURLWP), %rcx 202 xorq %rax, %rax 203 orq $RW_WRITE_LOCKED, %rcx 204 LOCK(3) 205 cmpxchgq %rcx, RW_OWNER(%rdi) 206 jnz,pn _C_LABEL(rw_vector_enter) 207 ret 208 209/* 210 * void rw_exit(krwlock_t *rwl); 211 * 212 * Release one hold on a RW lock. 213 */ 214NENTRY(rw_exit) 215 movq RW_OWNER(%rdi), %rax 216 testb $RW_WRITE_LOCKED, %al 217 jnz 2f 218 219 /* 220 * Reader 221 */ 2221: testb $RW_HAS_WAITERS, %al 223 jnz,pn 3f 224 cmpq $RW_READ_INCR, %rax 225 leaq -RW_READ_INCR(%rax), %rdx 226 jb,pn 3f 227 LOCK(4) 228 cmpxchgq %rdx, RW_OWNER(%rdi) 229 jnz,pn 1b 230 ret 231 232 /* 233 * Writer 234 */ 2352: leaq -RW_WRITE_LOCKED(%rax), %rdx 236 subq CPUVAR(CURLWP), %rdx 237 jnz,pn 3f 238 LOCK(5) 239 cmpxchgq %rdx, RW_OWNER(%rdi) 240 jnz 3f 241 ret 242 2433: jmp _C_LABEL(rw_vector_exit) 244 245/* 246 * int rw_tryenter(krwlock_t *rwl, krw_t op); 247 * 248 * Try to acquire one hold on a RW lock. 249 */ 250NENTRY(rw_tryenter) 251 cmpl $RW_READER, %esi 252 jne 2f 253 254 /* 255 * Reader: this is the most common case. 256 */ 2571: movq RW_OWNER(%rdi), %rax 258 testb $(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al 259 leaq RW_READ_INCR(%rax), %rdx 260 jnz 3f 261 LOCK(8) 262 cmpxchgq %rdx, RW_OWNER(%rdi) 263 movl $0, %eax 264 setz %al 265 ret 266 267 /* 268 * Writer: if the compare-and-set fails, don't bother retrying. 269 */ 2702: movq CPUVAR(CURLWP), %rcx 271 xorq %rax, %rax 272 orq $RW_WRITE_LOCKED, %rcx 273 LOCK(9) 274 cmpxchgq %rcx, RW_OWNER(%rdi) 275 movl $0, %eax 276 setz %al 277 ret 278 2793: xorl %eax, %eax 280 ret 281 282#endif /* LOCKDEBUG */ 283 284/* 285 * Spinlocks. 286 */ 287NENTRY(__cpu_simple_lock_init) 288 movb $0, (%rdi) 289 ret 290 291NENTRY(__cpu_simple_lock) 292 movl $0x0100, %eax 2931: 294 LOCK(6) 295 cmpxchgb %ah, (%rdi) 296 jnz 2f 297 ret 2982: 299 movl $0x0100, %eax 300 pause 301 nop 302 nop 303 cmpb $0, (%rdi) 304 je 1b 305 jmp 2b 306 307NENTRY(__cpu_simple_unlock) 308 movb $0, (%rdi) 309 ret 310 311NENTRY(__cpu_simple_lock_try) 312 movl $0x0100, %eax 313 LOCK(7) 314 cmpxchgb %ah, (%rdi) 315 movl $0, %eax 316 setz %al 317 ret 318 319/* 320 * Patchpoints to replace with NOP when ncpu == 1. 321 */ 322#ifndef LOCKDEBUG 323LABEL(x86_lockpatch) 324 .quad .Lpatch1, .Lpatch2, .Lpatch3, .Lpatch4 325 .quad .Lpatch5, .Lpatch6, .Lpatch7, .Lpatch8 326 .quad .Lpatch9 327#ifdef FULL 328 .quad .Lpatch11 329#endif 330 .quad 0 331#endif 332