lock_stubs.S revision 1.20
1/* $NetBSD: lock_stubs.S,v 1.20 2008/06/27 18:16:02 ad Exp $ */ 2 3/*- 4 * Copyright (c) 2006, 2007, 2008 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32/* 33 * AMD64 lock stubs. Calling convention: 34 * 35 * %rdi arg 1 36 * %rsi arg 2 37 * %rdx arg 3 38 * %rax return value 39 */ 40 41#include "opt_multiprocessor.h" 42#include "opt_lockdebug.h" 43 44#include <machine/asm.h> 45#include <machine/frameasm.h> 46 47#include "assym.h" 48 49#if defined(DIAGNOSTIC) || defined(MULTIPROCESSOR) || defined(LOCKDEBUG) 50#define FULL 51#endif 52 53#define ENDLABEL(name,a) .align a; LABEL(name) 54#define LOCK(num) .Lpatch/**/num: lock 55 56#ifndef LOCKDEBUG 57 58/* 59 * void mutex_enter(kmutex_t *mtx); 60 * 61 * Acquire a mutex and post a load fence. 62 */ 63 .align 64 64 65ENTRY(mutex_enter) 66 movq CPUVAR(CURLWP), %rcx 67 xorq %rax, %rax 68 LOCK(1) 69 cmpxchgq %rcx, MTX_OWNER(%rdi) 70 jnz 1f 71 ret 721: 73 jmp _C_LABEL(mutex_vector_enter) 74 75/* 76 * void mutex_exit(kmutex_t *mtx); 77 * 78 * Release a mutex and post a load fence. 79 * 80 * See comments in mutex_vector_enter() about doing this operation unlocked 81 * on multiprocessor systems, and comments in arch/x86/include/lock.h about 82 * memory ordering on Intel x86 systems. 83 */ 84ENTRY(mutex_exit) 85 movq CPUVAR(CURLWP), %rax 86 xorq %rdx, %rdx 87 cmpxchgq %rdx, MTX_OWNER(%rdi) 88 jnz 1f 89 ret 901: 91 jmp _C_LABEL(mutex_vector_exit) 92 93/* 94 * void mutex_spin_enter(kmutex_t *mtx); 95 * 96 * Acquire a spin mutex and post a load fence. 97 */ 98ENTRY(mutex_spin_enter) 99 movl $1, %eax 100 movl CPUVAR(ILEVEL), %esi 101 movzbl MTX_IPL(%rdi), %ecx /* new SPL */ 102 cmpl %ecx, %esi /* higher? */ 103 cmovgl %esi, %ecx 104 movl %ecx, CPUVAR(ILEVEL) /* splraiseipl() */ 105 subl %eax, CPUVAR(MTX_COUNT) /* decl doesnt set CF */ 106 cmovncl CPUVAR(MTX_OLDSPL), %esi 107 movl %esi, CPUVAR(MTX_OLDSPL) 108#if defined(FULL) 109 xchgb %al, MTX_LOCK(%rdi) /* lock */ 110 testb %al, %al 111 jnz 1f 112#endif 113 ret 1141: 115 jmp _C_LABEL(mutex_spin_retry) /* failed; hard case */ 116 117/* 118 * void mutex_spin_exit(kmutex_t *mtx); 119 * 120 * Release a spin mutex and post a load fence. 121 */ 122ENTRY(mutex_spin_exit) 123#ifdef DIAGNOSTIC 124 125 movl $0x0001, %eax /* new + expected value */ 126 movq CPUVAR(SELF), %r8 127 cmpxchgb %ah, MTX_LOCK(%rdi) /* unlock */ 128 jnz,pn _C_LABEL(mutex_vector_exit) /* hard case if problems */ 129 movl CPU_INFO_MTX_OLDSPL(%r8), %edi 130 incl CPU_INFO_MTX_COUNT(%r8) 131 jnz 1f 132 cmpl CPU_INFO_ILEVEL(%r8), %edi 133 jae 1f 134 movl CPU_INFO_IUNMASK(%r8,%rdi,4), %esi 135 CLI(ax) 136 testl CPU_INFO_IPENDING(%r8), %esi 137 jnz _C_LABEL(Xspllower) 138 movl %edi, CPU_INFO_ILEVEL(%r8) 139 STI(ax) 1401: rep /* double byte ret as branch */ 141 ret /* target: see AMD docs */ 142 143#else /* DIAGNOSTIC */ 144 145 movq CPUVAR(SELF), %rsi 146#ifdef MULTIPROCESSOR 147 movb $0x00, MTX_LOCK(%rdi) 148#endif 149 movl CPU_INFO_MTX_OLDSPL(%rsi), %ecx 150 incl CPU_INFO_MTX_COUNT(%rsi) 151 movl CPU_INFO_ILEVEL(%rsi),%edx 152 cmovnzl %edx,%ecx 153 cmpl %edx,%ecx /* new level is lower? */ 154 pushq %rbx 155 jae,pn 2f 1561: 157 movl CPU_INFO_IPENDING(%rsi),%eax 158 testl %eax,CPU_INFO_IUNMASK(%rsi,%rcx,4)/* deferred interrupts? */ 159 movl %eax,%ebx 160 jnz,pn 3f 161 cmpxchg8b CPU_INFO_ISTATE(%rsi) /* swap in new ilevel */ 162 jnz,pn 1b 1632: 164 popq %rbx 165 ret 1663: 167 popq %rbx 168 movl %ecx, %edi 169 jmp _C_LABEL(Xspllower) 170 171#endif /* DIAGNOSTIC */ 172 173/* 174 * void rw_enter(krwlock_t *rwl, krw_t op); 175 * 176 * Acquire one hold on a RW lock. 177 */ 178ENTRY(rw_enter) 179 cmpl $RW_READER, %esi 180 jne 2f 181 182 /* 183 * Reader: this is the most common case. 184 */ 1851: movq RW_OWNER(%rdi), %rax 186 testb $(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al 187 leaq RW_READ_INCR(%rax), %rdx 188 jnz 3f 189 LOCK(2) 190 cmpxchgq %rdx, RW_OWNER(%rdi) 191 jnz,pn 1b 192 ret 193 194 /* 195 * Writer: if the compare-and-set fails, don't bother retrying. 196 */ 1972: movq CPUVAR(CURLWP), %rcx 198 xorq %rax, %rax 199 orq $RW_WRITE_LOCKED, %rcx 200 LOCK(3) 201 cmpxchgq %rcx, RW_OWNER(%rdi) 202 jnz 3f 203 ret 2043: 205 jmp _C_LABEL(rw_vector_enter) 206 207/* 208 * void rw_exit(krwlock_t *rwl); 209 * 210 * Release one hold on a RW lock. 211 */ 212ENTRY(rw_exit) 213 movq RW_OWNER(%rdi), %rax 214 testb $RW_WRITE_LOCKED, %al 215 jnz 2f 216 217 /* 218 * Reader 219 */ 2201: testb $RW_HAS_WAITERS, %al 221 jnz 3f 222 cmpq $RW_READ_INCR, %rax 223 leaq -RW_READ_INCR(%rax), %rdx 224 jb 3f 225 LOCK(4) 226 cmpxchgq %rdx, RW_OWNER(%rdi) 227 jnz,pn 1b 228 ret 229 230 /* 231 * Writer 232 */ 2332: leaq -RW_WRITE_LOCKED(%rax), %rdx 234 subq CPUVAR(CURLWP), %rdx 235 jnz 3f 236 LOCK(5) 237 cmpxchgq %rdx, RW_OWNER(%rdi) 238 jnz 3f 239 ret 240 2413: jmp _C_LABEL(rw_vector_exit) 242 243/* 244 * int rw_tryenter(krwlock_t *rwl, krw_t op); 245 * 246 * Try to acquire one hold on a RW lock. 247 */ 248ENTRY(rw_tryenter) 249 cmpl $RW_READER, %esi 250 jne 2f 251 252 /* 253 * Reader: this is the most common case. 254 */ 255 movq RW_OWNER(%rdi), %rax 2561: 257 testb $(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al 258 leaq RW_READ_INCR(%rax), %rdx 259 jnz 3f 260 LOCK(8) 261 cmpxchgq %rdx, RW_OWNER(%rdi) 262 jnz 1b 263 movl %edi, %eax /* nonzero */ 264 ret 265 266 /* 267 * Writer: if the compare-and-set fails, don't bother retrying. 268 */ 2692: movq CPUVAR(CURLWP), %rcx 270 xorq %rax, %rax 271 orq $RW_WRITE_LOCKED, %rcx 272 LOCK(9) 273 cmpxchgq %rcx, RW_OWNER(%rdi) 274 movl $0, %eax 275 setz %al 276 ret 277 2783: xorl %eax, %eax 279 ret 280 281#endif /* LOCKDEBUG */ 282 283/* 284 * Spinlocks. 285 */ 286ENTRY(__cpu_simple_lock_init) 287 movb $0, (%rdi) 288 ret 289 290NENTRY(__cpu_simple_lock) 291 movl $0x0100, %eax 2921: 293 LOCK(6) 294 cmpxchgb %ah, (%rdi) 295 jnz 2f 296 ret 2972: 298 movl $0x0100, %eax 299 pause 300 nop 301 nop 302 cmpb $0, (%rdi) 303 je 1b 304 jmp 2b 305 306ENTRY(__cpu_simple_unlock) 307 movb $0, (%rdi) 308 ret 309 310ENTRY(__cpu_simple_lock_try) 311 movl $0x0100, %eax 312 LOCK(7) 313 cmpxchgb %ah, (%rdi) 314 movl $0, %eax 315 setz %al 316 ret 317 318/* 319 * Patchpoints to replace with NOP when ncpu == 1. 320 */ 321#ifndef LOCKDEBUG 322LABEL(x86_lockpatch) 323 .quad .Lpatch1, .Lpatch2, .Lpatch3, .Lpatch4 324 .quad .Lpatch5, .Lpatch6, .Lpatch7, .Lpatch8 325 .quad .Lpatch9 326 .quad 0 327#endif 328