lock_stubs.S revision 1.24
1/* $NetBSD: lock_stubs.S,v 1.24 2011/01/12 23:12:11 joerg Exp $ */ 2 3/*- 4 * Copyright (c) 2006, 2007, 2008, 2009 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32/* 33 * AMD64 lock stubs. Calling convention: 34 * 35 * %rdi arg 1 36 * %rsi arg 2 37 * %rdx arg 3 38 * %rax return value 39 */ 40 41#include "opt_multiprocessor.h" 42#include "opt_lockdebug.h" 43 44#include <machine/asm.h> 45#include <machine/frameasm.h> 46 47#include "assym.h" 48 49#define ENDLABEL(name,a) .align a; LABEL(name) 50#define LOCK(num) .Lpatch ## num: lock 51#define RET(num) .Lret ## num: ret; nop; nop; ret 52 53#ifndef LOCKDEBUG 54 55/* 56 * void mutex_enter(kmutex_t *mtx); 57 * 58 * Acquire a mutex and post a load fence. 59 */ 60 .align 64 61 62ENTRY(mutex_enter) 63 movq CPUVAR(CURLWP), %rcx 64 xorq %rax, %rax 65 LOCK(1) 66 cmpxchgq %rcx, (%rdi) 67 jnz 1f 68 RET(1) 691: 70 jmp _C_LABEL(mutex_vector_enter) 71 72/* 73 * void mutex_exit(kmutex_t *mtx); 74 * 75 * Release a mutex and post a load fence. 76 * 77 * See comments in mutex_vector_enter() about doing this operation unlocked 78 * on multiprocessor systems, and comments in arch/x86/include/lock.h about 79 * memory ordering on Intel x86 systems. 80 */ 81ENTRY(mutex_exit) 82 movq CPUVAR(CURLWP), %rax 83 xorq %rdx, %rdx 84 cmpxchgq %rdx, (%rdi) 85 jnz 1f 86 ret 871: 88 jmp _C_LABEL(mutex_vector_exit) 89 90/* 91 * void mutex_spin_enter(kmutex_t *mtx); 92 * 93 * Acquire a spin mutex and post a load fence. 94 */ 95ENTRY(mutex_spin_enter) 96 movl $1, %eax 97 movl CPUVAR(ILEVEL), %esi 98 movzbl MTX_IPL(%rdi), %ecx /* new SPL */ 99 cmpl %ecx, %esi /* higher? */ 100 cmovgl %esi, %ecx 101 movl %ecx, CPUVAR(ILEVEL) /* splraiseipl() */ 102 subl %eax, CPUVAR(MTX_COUNT) /* decl doesnt set CF */ 103 cmovncl CPUVAR(MTX_OLDSPL), %esi 104 movl %esi, CPUVAR(MTX_OLDSPL) 105 xchgb %al, MTX_LOCK(%rdi) /* lock */ 106#ifdef MULTIPROCESSOR /* XXX for xen */ 107 testb %al, %al 108 jnz 1f 109#endif 110 RET(2) 1111: 112 jmp _C_LABEL(mutex_spin_retry) /* failed; hard case */ 113 114/* 115 * void mutex_spin_exit(kmutex_t *mtx); 116 * 117 * Release a spin mutex and post a load fence. 118 */ 119ENTRY(mutex_spin_exit) 120#ifdef DIAGNOSTIC 121 122 movl $0x0001, %eax /* new + expected value */ 123 movq CPUVAR(SELF), %r8 124 cmpxchgb %ah, MTX_LOCK(%rdi) /* unlock */ 125 jnz _C_LABEL(mutex_vector_exit) /* hard case if problems */ 126 movl CPU_INFO_MTX_OLDSPL(%r8), %edi 127 incl CPU_INFO_MTX_COUNT(%r8) 128 jnz 1f 129 cmpl CPU_INFO_ILEVEL(%r8), %edi 130 jae 1f 131 movl CPU_INFO_IUNMASK(%r8,%rdi,4), %esi 132 CLI(ax) 133 testl CPU_INFO_IPENDING(%r8), %esi 134 jnz _C_LABEL(Xspllower) 135 movl %edi, CPU_INFO_ILEVEL(%r8) 136 STI(ax) 1371: rep /* double byte ret as branch */ 138 ret /* target: see AMD docs */ 139 140#else /* DIAGNOSTIC */ 141 142 movq CPUVAR(SELF), %rsi 143 movb $0x00, MTX_LOCK(%rdi) 144 movl CPU_INFO_MTX_OLDSPL(%rsi), %ecx 145 incl CPU_INFO_MTX_COUNT(%rsi) 146 movl CPU_INFO_ILEVEL(%rsi),%edx 147 cmovnzl %edx,%ecx 148 pushq %rbx 149 cmpl %edx,%ecx /* new level is lower? */ 150 jae 2f 1511: 152 movl CPU_INFO_IPENDING(%rsi),%eax 153 testl %eax,CPU_INFO_IUNMASK(%rsi,%rcx,4)/* deferred interrupts? */ 154 jnz 3f 155 movl %eax,%ebx 156 cmpxchg8b CPU_INFO_ISTATE(%rsi) /* swap in new ilevel */ 157 jnz 4f 1582: 159 popq %rbx 160 ret 1613: 162 popq %rbx 163 movl %ecx, %edi 164 jmp _C_LABEL(Xspllower) 1654: 166 jmp 1b 167 168#endif /* DIAGNOSTIC */ 169 170/* 171 * void rw_enter(krwlock_t *rwl, krw_t op); 172 * 173 * Acquire one hold on a RW lock. 174 */ 175ENTRY(rw_enter) 176 cmpl $RW_READER, %esi 177 jne 2f 178 179 /* 180 * Reader: this is the most common case. 181 */ 182 movq (%rdi), %rax 1830: 184 testb $(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al 185 jnz 3f 186 leaq RW_READ_INCR(%rax), %rdx 187 LOCK(2) 188 cmpxchgq %rdx, (%rdi) 189 jnz 1f 190 RET(3) 1911: 192 jmp 0b 193 194 /* 195 * Writer: if the compare-and-set fails, don't bother retrying. 196 */ 1972: movq CPUVAR(CURLWP), %rcx 198 xorq %rax, %rax 199 orq $RW_WRITE_LOCKED, %rcx 200 LOCK(3) 201 cmpxchgq %rcx, (%rdi) 202 jnz 3f 203 RET(4) 2043: 205 jmp _C_LABEL(rw_vector_enter) 206 207/* 208 * void rw_exit(krwlock_t *rwl); 209 * 210 * Release one hold on a RW lock. 211 */ 212ENTRY(rw_exit) 213 movq (%rdi), %rax 214 testb $RW_WRITE_LOCKED, %al 215 jnz 2f 216 217 /* 218 * Reader 219 */ 2200: testb $RW_HAS_WAITERS, %al 221 jnz 3f 222 cmpq $RW_READ_INCR, %rax 223 jb 3f 224 leaq -RW_READ_INCR(%rax), %rdx 225 LOCK(4) 226 cmpxchgq %rdx, (%rdi) 227 jnz 1f 228 ret 2291: 230 jmp 0b 231 232 /* 233 * Writer 234 */ 2352: leaq -RW_WRITE_LOCKED(%rax), %rdx 236 subq CPUVAR(CURLWP), %rdx 237 jnz 3f 238 LOCK(5) 239 cmpxchgq %rdx, (%rdi) 240 jnz 3f 241 ret 242 2433: jmp _C_LABEL(rw_vector_exit) 244 245/* 246 * int rw_tryenter(krwlock_t *rwl, krw_t op); 247 * 248 * Try to acquire one hold on a RW lock. 249 */ 250ENTRY(rw_tryenter) 251 cmpl $RW_READER, %esi 252 jne 2f 253 254 /* 255 * Reader: this is the most common case. 256 */ 257 movq (%rdi), %rax 2580: 259 testb $(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al 260 jnz 4f 261 leaq RW_READ_INCR(%rax), %rdx 262 LOCK(8) 263 cmpxchgq %rdx, (%rdi) 264 jnz 1f 265 movl %edx, %eax /* nonzero */ 266 RET(5) 2671: 268 jmp 0b 269 270 /* 271 * Writer: if the compare-and-set fails, don't bother retrying. 272 */ 2732: movq CPUVAR(CURLWP), %rcx 274 xorq %rax, %rax 275 orq $RW_WRITE_LOCKED, %rcx 276 LOCK(9) 277 cmpxchgq %rcx, (%rdi) 278 movl $0, %eax 279 setz %al 2803: 281 RET(6) 282 ret 2834: 284 xorl %eax, %eax 285 jmp 3b 286 287#endif /* LOCKDEBUG */ 288 289/* 290 * Spinlocks. 291 */ 292ENTRY(__cpu_simple_lock_init) 293 movb $0, (%rdi) 294 ret 295 296NENTRY(__cpu_simple_lock) 297 movl $0x0100, %eax 2981: 299 LOCK(6) 300 cmpxchgb %ah, (%rdi) 301 jnz 2f 302 RET(7) 3032: 304 movl $0x0100, %eax 305 pause 306 nop 307 nop 308 cmpb $0, (%rdi) 309 je 1b 310 jmp 2b 311 312ENTRY(__cpu_simple_unlock) 313 movb $0, (%rdi) 314 ret 315 316ENTRY(__cpu_simple_lock_try) 317 movl $0x0100, %eax 318 LOCK(7) 319 cmpxchgb %ah, (%rdi) 320 movl $0, %eax 321 setz %al 322 RET(8) 323 324/* 325 * Patchpoints to replace with NOP when ncpu == 1. 326 */ 327#ifndef LOCKDEBUG 328LABEL(x86_lockpatch) 329 .quad .Lpatch1, .Lpatch2, .Lpatch3, .Lpatch4 330 .quad .Lpatch5, .Lpatch6, .Lpatch7, .Lpatch8 331 .quad .Lpatch9 332 .quad 0 333#endif 334 335LABEL(x86_retpatch) 336#ifndef LOCKDEBUG 337 .quad .Lret1, .Lret2, .Lret3, .Lret4, .Lret5, .Lret6 338#endif 339 .quad .Lret7, .Lret8 340 .quad 0 341