lock_stubs.S revision 1.18
1/* $NetBSD: lock_stubs.S,v 1.18 2008/05/06 23:03:03 ad Exp $ */ 2 3/*- 4 * Copyright (c) 2006, 2007, 2008 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32/* 33 * AMD64 lock stubs. Calling convention: 34 * 35 * %rdi arg 1 36 * %rsi arg 2 37 * %rdx arg 3 38 * %rax return value 39 */ 40 41#include "opt_multiprocessor.h" 42#include "opt_lockdebug.h" 43 44#include <machine/asm.h> 45#include <machine/frameasm.h> 46 47#include "assym.h" 48 49#if defined(DIAGNOSTIC) || defined(MULTIPROCESSOR) || defined(LOCKDEBUG) 50#define FULL 51#endif 52 53#define ENDLABEL(name,a) .align a; LABEL(name) 54#define LOCK(num) .Lpatch/**/num: lock 55 56#ifndef LOCKDEBUG 57 58/* 59 * void mutex_enter(kmutex_t *mtx); 60 * 61 * Acquire a mutex and post a load fence. 62 */ 63 .align 64 64 65NENTRY(mutex_enter) 66 movq CPUVAR(CURLWP), %rcx 67 xorq %rax, %rax 68 LOCK(1) 69 cmpxchgq %rcx, MTX_OWNER(%rdi) 70 jnz 1f 71 ret 721: 73 jmp _C_LABEL(mutex_vector_enter) 74 75/* 76 * void mutex_exit(kmutex_t *mtx); 77 * 78 * Release a mutex and post a load fence. 79 * 80 * See comments in mutex_vector_enter() about doing this operation unlocked 81 * on multiprocessor systems, and comments in arch/x86/include/lock.h about 82 * memory ordering on Intel x86 systems. 83 */ 84NENTRY(mutex_exit) 85 movq CPUVAR(CURLWP), %rax 86 xorq %rdx, %rdx 87 cmpxchgq %rdx, MTX_OWNER(%rdi) 88 jnz 1f 89 ret 901: 91 jmp _C_LABEL(mutex_vector_exit) 92 93/* 94 * void mutex_spin_enter(kmutex_t *mtx); 95 * 96 * Acquire a spin mutex and post a load fence. 97 */ 98NENTRY(mutex_spin_enter) 99#if defined(FULL) 100 movl $0x0100, %eax /* new + expected value */ 101#endif 102 movl CPUVAR(ILEVEL), %esi 103 movzbl MTX_IPL(%rdi), %ecx /* new SPL */ 104 cmpl %ecx, %esi /* higher? */ 105 cmovgl %esi, %ecx 106 movl %ecx, CPUVAR(ILEVEL) /* splraiseipl() */ 107 subl $1, CPUVAR(MTX_COUNT) /* decl doesnt set CF */ 108 cmovncl CPUVAR(MTX_OLDSPL), %esi 109 movl %esi, CPUVAR(MTX_OLDSPL) 110#if defined(FULL) 111 LOCK(11) 112 cmpxchgb %ah, MTX_LOCK(%rdi) /* lock */ 113 jnz 1f 114#endif 115 ret 1161: 117 jmp _C_LABEL(mutex_spin_retry) /* failed; hard case */ 118 119/* 120 * void mutex_spin_exit(kmutex_t *mtx); 121 * 122 * Release a spin mutex and post a load fence. 123 */ 124NENTRY(mutex_spin_exit) 125#ifdef DIAGNOSTIC 126 127 movl $0x0001, %eax /* new + expected value */ 128 movq CPUVAR(SELF), %r8 129 cmpxchgb %ah, MTX_LOCK(%rdi) /* unlock */ 130 jnz,pn _C_LABEL(mutex_vector_exit) /* hard case if problems */ 131 movl CPU_INFO_MTX_OLDSPL(%r8), %edi 132 incl CPU_INFO_MTX_COUNT(%r8) 133 jnz 1f 134 cmpl CPU_INFO_ILEVEL(%r8), %edi 135 jae 1f 136 movl CPU_INFO_IUNMASK(%r8,%rdi,4), %esi 137 CLI(ax) 138 testl CPU_INFO_IPENDING(%r8), %esi 139 jnz _C_LABEL(Xspllower) 140 movl %edi, CPU_INFO_ILEVEL(%r8) 141 STI(ax) 1421: rep /* double byte ret as branch */ 143 ret /* target: see AMD docs */ 144 145#else /* DIAGNOSTIC */ 146 147 movq CPUVAR(SELF), %rsi 148#ifdef MULTIPROCESSOR 149 movb $0x00, MTX_LOCK(%rdi) 150#endif 151 movl CPU_INFO_MTX_OLDSPL(%rsi), %ecx 152 incl CPU_INFO_MTX_COUNT(%rsi) 153 movl CPU_INFO_ILEVEL(%rsi),%edx 154 cmovnzl %edx,%ecx 155 cmpl %edx,%ecx /* new level is lower? */ 156 pushq %rbx 157 jae,pn 2f 1581: 159 movl CPU_INFO_IPENDING(%rsi),%eax 160 testl %eax,CPU_INFO_IUNMASK(%rsi,%rcx,4)/* deferred interrupts? */ 161 movl %eax,%ebx 162 jnz,pn 3f 163 cmpxchg8b CPU_INFO_ISTATE(%rsi) /* swap in new ilevel */ 164 jnz,pn 1b 1652: 166 popq %rbx 167 ret 1683: 169 popq %rbx 170 movl %ecx, %edi 171 jmp _C_LABEL(Xspllower) 172 173#endif /* DIAGNOSTIC */ 174 175/* 176 * void rw_enter(krwlock_t *rwl, krw_t op); 177 * 178 * Acquire one hold on a RW lock. 179 */ 180NENTRY(rw_enter) 181 cmpl $RW_READER, %esi 182 jne 2f 183 184 /* 185 * Reader: this is the most common case. 186 */ 1871: movq RW_OWNER(%rdi), %rax 188 testb $(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al 189 leaq RW_READ_INCR(%rax), %rdx 190 jnz 3f 191 LOCK(2) 192 cmpxchgq %rdx, RW_OWNER(%rdi) 193 jnz,pn 1b 194 ret 195 196 /* 197 * Writer: if the compare-and-set fails, don't bother retrying. 198 */ 1992: movq CPUVAR(CURLWP), %rcx 200 xorq %rax, %rax 201 orq $RW_WRITE_LOCKED, %rcx 202 LOCK(3) 203 cmpxchgq %rcx, RW_OWNER(%rdi) 204 jnz 3f 205 ret 2063: 207 jmp _C_LABEL(rw_vector_enter) 208 209/* 210 * void rw_exit(krwlock_t *rwl); 211 * 212 * Release one hold on a RW lock. 213 */ 214NENTRY(rw_exit) 215 movq RW_OWNER(%rdi), %rax 216 testb $RW_WRITE_LOCKED, %al 217 jnz 2f 218 219 /* 220 * Reader 221 */ 2221: testb $RW_HAS_WAITERS, %al 223 jnz 3f 224 cmpq $RW_READ_INCR, %rax 225 leaq -RW_READ_INCR(%rax), %rdx 226 jb 3f 227 LOCK(4) 228 cmpxchgq %rdx, RW_OWNER(%rdi) 229 jnz,pn 1b 230 ret 231 232 /* 233 * Writer 234 */ 2352: leaq -RW_WRITE_LOCKED(%rax), %rdx 236 subq CPUVAR(CURLWP), %rdx 237 jnz 3f 238 LOCK(5) 239 cmpxchgq %rdx, RW_OWNER(%rdi) 240 jnz 3f 241 ret 242 2433: jmp _C_LABEL(rw_vector_exit) 244 245/* 246 * int rw_tryenter(krwlock_t *rwl, krw_t op); 247 * 248 * Try to acquire one hold on a RW lock. 249 */ 250NENTRY(rw_tryenter) 251 cmpl $RW_READER, %esi 252 jne 2f 253 254 /* 255 * Reader: this is the most common case. 256 */ 257 movq RW_OWNER(%rdi), %rax 2581: 259 testb $(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al 260 leaq RW_READ_INCR(%rax), %rdx 261 jnz 3f 262 LOCK(8) 263 cmpxchgq %rdx, RW_OWNER(%rdi) 264 jnz 1b 265 movl %edi, %eax /* nonzero */ 266 ret 267 268 /* 269 * Writer: if the compare-and-set fails, don't bother retrying. 270 */ 2712: movq CPUVAR(CURLWP), %rcx 272 xorq %rax, %rax 273 orq $RW_WRITE_LOCKED, %rcx 274 LOCK(9) 275 cmpxchgq %rcx, RW_OWNER(%rdi) 276 movl $0, %eax 277 setz %al 278 ret 279 2803: xorl %eax, %eax 281 ret 282 283#endif /* LOCKDEBUG */ 284 285/* 286 * Spinlocks. 287 */ 288NENTRY(__cpu_simple_lock_init) 289 movb $0, (%rdi) 290 ret 291 292NENTRY(__cpu_simple_lock) 293 movl $0x0100, %eax 2941: 295 LOCK(6) 296 cmpxchgb %ah, (%rdi) 297 jnz 2f 298 ret 2992: 300 movl $0x0100, %eax 301 pause 302 nop 303 nop 304 cmpb $0, (%rdi) 305 je 1b 306 jmp 2b 307 308NENTRY(__cpu_simple_unlock) 309 movb $0, (%rdi) 310 ret 311 312NENTRY(__cpu_simple_lock_try) 313 movl $0x0100, %eax 314 LOCK(7) 315 cmpxchgb %ah, (%rdi) 316 movl $0, %eax 317 setz %al 318 ret 319 320/* 321 * Patchpoints to replace with NOP when ncpu == 1. 322 */ 323#ifndef LOCKDEBUG 324LABEL(x86_lockpatch) 325 .quad .Lpatch1, .Lpatch2, .Lpatch3, .Lpatch4 326 .quad .Lpatch5, .Lpatch6, .Lpatch7, .Lpatch8 327 .quad .Lpatch9 328#ifdef FULL 329 .quad .Lpatch11 330#endif 331 .quad 0 332#endif 333