lock_stubs.S revision 1.21
1/* $NetBSD: lock_stubs.S,v 1.21 2008/12/05 13:08:11 ad Exp $ */ 2 3/*- 4 * Copyright (c) 2006, 2007, 2008 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32/* 33 * AMD64 lock stubs. Calling convention: 34 * 35 * %rdi arg 1 36 * %rsi arg 2 37 * %rdx arg 3 38 * %rax return value 39 */ 40 41#include "opt_multiprocessor.h" 42#include "opt_lockdebug.h" 43 44#include <machine/asm.h> 45#include <machine/frameasm.h> 46 47#include "assym.h" 48 49#define ENDLABEL(name,a) .align a; LABEL(name) 50#define LOCK(num) .Lpatch/**/num: lock 51 52#ifndef LOCKDEBUG 53 54/* 55 * void mutex_enter(kmutex_t *mtx); 56 * 57 * Acquire a mutex and post a load fence. 58 */ 59 .align 64 60 61ENTRY(mutex_enter) 62 movq CPUVAR(CURLWP), %rcx 63 xorq %rax, %rax 64 LOCK(1) 65 cmpxchgq %rcx, (%rdi) 66 jnz 1f 67 ret 681: 69 jmp _C_LABEL(mutex_vector_enter) 70 71/* 72 * void mutex_exit(kmutex_t *mtx); 73 * 74 * Release a mutex and post a load fence. 75 * 76 * See comments in mutex_vector_enter() about doing this operation unlocked 77 * on multiprocessor systems, and comments in arch/x86/include/lock.h about 78 * memory ordering on Intel x86 systems. 79 */ 80ENTRY(mutex_exit) 81 movq CPUVAR(CURLWP), %rax 82 xorq %rdx, %rdx 83 cmpxchgq %rdx, (%rdi) 84 jnz 1f 85 ret 861: 87 jmp _C_LABEL(mutex_vector_exit) 88 89/* 90 * void mutex_spin_enter(kmutex_t *mtx); 91 * 92 * Acquire a spin mutex and post a load fence. 93 */ 94ENTRY(mutex_spin_enter) 95 movl $1, %eax 96 movl CPUVAR(ILEVEL), %esi 97 movzbl MTX_IPL(%rdi), %ecx /* new SPL */ 98 cmpl %ecx, %esi /* higher? */ 99 cmovgl %esi, %ecx 100 movl %ecx, CPUVAR(ILEVEL) /* splraiseipl() */ 101 subl %eax, CPUVAR(MTX_COUNT) /* decl doesnt set CF */ 102 cmovncl CPUVAR(MTX_OLDSPL), %esi 103 movl %esi, CPUVAR(MTX_OLDSPL) 104 xchgb %al, MTX_LOCK(%rdi) /* lock */ 105#ifdef MULTIPROCESSOR /* XXX for xen */ 106 testb %al, %al 107 jnz 1f 108#endif 109 ret 1101: 111 jmp _C_LABEL(mutex_spin_retry) /* failed; hard case */ 112 113/* 114 * void mutex_spin_exit(kmutex_t *mtx); 115 * 116 * Release a spin mutex and post a load fence. 117 */ 118ENTRY(mutex_spin_exit) 119#ifdef DIAGNOSTIC 120 121 movl $0x0001, %eax /* new + expected value */ 122 movq CPUVAR(SELF), %r8 123 cmpxchgb %ah, MTX_LOCK(%rdi) /* unlock */ 124 jnz _C_LABEL(mutex_vector_exit) /* hard case if problems */ 125 movl CPU_INFO_MTX_OLDSPL(%r8), %edi 126 incl CPU_INFO_MTX_COUNT(%r8) 127 jnz 1f 128 cmpl CPU_INFO_ILEVEL(%r8), %edi 129 jae 1f 130 movl CPU_INFO_IUNMASK(%r8,%rdi,4), %esi 131 CLI(ax) 132 testl CPU_INFO_IPENDING(%r8), %esi 133 jnz _C_LABEL(Xspllower) 134 movl %edi, CPU_INFO_ILEVEL(%r8) 135 STI(ax) 1361: rep /* double byte ret as branch */ 137 ret /* target: see AMD docs */ 138 139#else /* DIAGNOSTIC */ 140 141 movq CPUVAR(SELF), %rsi 142 movb $0x00, MTX_LOCK(%rdi) 143 movl CPU_INFO_MTX_OLDSPL(%rsi), %ecx 144 incl CPU_INFO_MTX_COUNT(%rsi) 145 movl CPU_INFO_ILEVEL(%rsi),%edx 146 cmovnzl %edx,%ecx 147 pushq %rbx 148 cmpl %edx,%ecx /* new level is lower? */ 149 jae 2f 1501: 151 movl CPU_INFO_IPENDING(%rsi),%eax 152 testl %eax,CPU_INFO_IUNMASK(%rsi,%rcx,4)/* deferred interrupts? */ 153 jnz 3f 154 movl %eax,%ebx 155 cmpxchg8b CPU_INFO_ISTATE(%rsi) /* swap in new ilevel */ 156 jnz 4f 1572: 158 popq %rbx 159 ret 1603: 161 popq %rbx 162 movl %ecx, %edi 163 jmp _C_LABEL(Xspllower) 1644: 165 jmp 1b 166 167#endif /* DIAGNOSTIC */ 168 169/* 170 * void rw_enter(krwlock_t *rwl, krw_t op); 171 * 172 * Acquire one hold on a RW lock. 173 */ 174ENTRY(rw_enter) 175 cmpl $RW_READER, %esi 176 jne 2f 177 178 /* 179 * Reader: this is the most common case. 180 */ 181 movq (%rdi), %rax 1820: 183 testb $(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al 184 jnz 3f 185 leaq RW_READ_INCR(%rax), %rdx 186 LOCK(2) 187 cmpxchgq %rdx, (%rdi) 188 jnz 1f 189 ret 1901: 191 jmp 0b 192 193 /* 194 * Writer: if the compare-and-set fails, don't bother retrying. 195 */ 1962: movq CPUVAR(CURLWP), %rcx 197 xorq %rax, %rax 198 orq $RW_WRITE_LOCKED, %rcx 199 LOCK(3) 200 cmpxchgq %rcx, (%rdi) 201 jnz 3f 202 ret 2033: 204 jmp _C_LABEL(rw_vector_enter) 205 206/* 207 * void rw_exit(krwlock_t *rwl); 208 * 209 * Release one hold on a RW lock. 210 */ 211ENTRY(rw_exit) 212 movq (%rdi), %rax 213 testb $RW_WRITE_LOCKED, %al 214 jnz 2f 215 216 /* 217 * Reader 218 */ 2190: testb $RW_HAS_WAITERS, %al 220 jnz 3f 221 cmpq $RW_READ_INCR, %rax 222 jb 3f 223 leaq -RW_READ_INCR(%rax), %rdx 224 LOCK(4) 225 cmpxchgq %rdx, (%rdi) 226 jnz 1f 227 ret 2281: 229 jmp 0b 230 231 /* 232 * Writer 233 */ 2342: leaq -RW_WRITE_LOCKED(%rax), %rdx 235 subq CPUVAR(CURLWP), %rdx 236 jnz 3f 237 LOCK(5) 238 cmpxchgq %rdx, (%rdi) 239 jnz 3f 240 ret 241 2423: jmp _C_LABEL(rw_vector_exit) 243 244/* 245 * int rw_tryenter(krwlock_t *rwl, krw_t op); 246 * 247 * Try to acquire one hold on a RW lock. 248 */ 249ENTRY(rw_tryenter) 250 cmpl $RW_READER, %esi 251 jne 2f 252 253 /* 254 * Reader: this is the most common case. 255 */ 256 movq (%rdi), %rax 2570: 258 testb $(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al 259 jnz 3f 260 leaq RW_READ_INCR(%rax), %rdx 261 LOCK(8) 262 cmpxchgq %rdx, (%rdi) 263 jnz 1f 264 movl %edx, %eax /* nonzero */ 265 ret 2661: 267 jmp 0b 268 269 /* 270 * Writer: if the compare-and-set fails, don't bother retrying. 271 */ 2722: movq CPUVAR(CURLWP), %rcx 273 xorq %rax, %rax 274 orq $RW_WRITE_LOCKED, %rcx 275 LOCK(9) 276 cmpxchgq %rcx, (%rdi) 277 movl $0, %eax 278 setz %al 279 ret 280 2813: xorl %eax, %eax 282 ret 283 284#endif /* LOCKDEBUG */ 285 286/* 287 * Spinlocks. 288 */ 289ENTRY(__cpu_simple_lock_init) 290 movb $0, (%rdi) 291 ret 292 293NENTRY(__cpu_simple_lock) 294 movl $0x0100, %eax 2951: 296 LOCK(6) 297 cmpxchgb %ah, (%rdi) 298 jnz 2f 299 ret 3002: 301 movl $0x0100, %eax 302 pause 303 nop 304 nop 305 cmpb $0, (%rdi) 306 je 1b 307 jmp 2b 308 309ENTRY(__cpu_simple_unlock) 310 movb $0, (%rdi) 311 ret 312 313ENTRY(__cpu_simple_lock_try) 314 movl $0x0100, %eax 315 LOCK(7) 316 cmpxchgb %ah, (%rdi) 317 movl $0, %eax 318 setz %al 319 ret 320 321/* 322 * Patchpoints to replace with NOP when ncpu == 1. 323 */ 324#ifndef LOCKDEBUG 325LABEL(x86_lockpatch) 326 .quad .Lpatch1, .Lpatch2, .Lpatch3, .Lpatch4 327 .quad .Lpatch5, .Lpatch6, .Lpatch7, .Lpatch8 328 .quad .Lpatch9 329 .quad 0 330#endif 331