lock_stubs.S revision 1.14
1/* $NetBSD: lock_stubs.S,v 1.14 2008/04/28 18:31:15 ad Exp $ */ 2 3/*- 4 * Copyright (c) 2006, 2007, 2008 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. All advertising materials mentioning features or use of this software 19 * must display the following acknowledgement: 20 * This product includes software developed by the NetBSD 21 * Foundation, Inc. and its contributors. 22 * 4. Neither the name of The NetBSD Foundation nor the names of its 23 * contributors may be used to endorse or promote products derived 24 * from this software without specific prior written permission. 25 * 26 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 27 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 28 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 29 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 30 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 31 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 32 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 33 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 34 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 35 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 36 * POSSIBILITY OF SUCH DAMAGE. 37 */ 38 39/* 40 * AMD64 lock stubs. Calling convention: 41 * 42 * %rdi arg 1 43 * %rsi arg 2 44 * %rdx arg 3 45 * %rax return value 46 */ 47 48#include "opt_multiprocessor.h" 49#include "opt_lockdebug.h" 50 51#include <machine/asm.h> 52#include <machine/frameasm.h> 53 54#include "assym.h" 55 56#if defined(DIAGNOSTIC) || defined(MULTIPROCESSOR) || defined(LOCKDEBUG) 57#define FULL 58#endif 59 60#define END(name,a) .align a; LABEL(name) 61#define LOCK(num) .Lpatch/**/num: lock 62 63#ifndef LOCKDEBUG 64 65/* 66 * void mutex_enter(kmutex_t *mtx); 67 * 68 * Acquire a mutex and post a load fence. 69 */ 70 .align 64 71 72NENTRY(mutex_enter) 73 movq CPUVAR(CURLWP), %rcx 74 xorq %rax, %rax 75 LOCK(1) 76 cmpxchgq %rcx, MTX_OWNER(%rdi) 77 jnz 1f 78 ret 791: 80 jmp _C_LABEL(mutex_vector_enter) 81 82/* 83 * void mutex_exit(kmutex_t *mtx); 84 * 85 * Release a mutex and post a load fence. 86 * 87 * See comments in mutex_vector_enter() about doing this operation unlocked 88 * on multiprocessor systems, and comments in arch/x86/include/lock.h about 89 * memory ordering on Intel x86 systems. 90 */ 91NENTRY(mutex_exit) 92 movq CPUVAR(CURLWP), %rax 93 xorq %rdx, %rdx 94 cmpxchgq %rdx, MTX_OWNER(%rdi) 95 jnz 1f 96 ret 971: 98 jmp _C_LABEL(mutex_vector_exit) 99 100/* 101 * void mutex_spin_enter(kmutex_t *mtx); 102 * 103 * Acquire a spin mutex and post a load fence. 104 */ 105NENTRY(mutex_spin_enter) 106#if defined(FULL) 107 movl $0x0100, %eax /* new + expected value */ 108#endif 109 movl CPUVAR(ILEVEL), %esi 110 movzbl MTX_IPL(%rdi), %ecx /* new SPL */ 111 cmpl %ecx, %esi /* higher? */ 112 cmovgl %esi, %ecx 113 movl %ecx, CPUVAR(ILEVEL) /* splraiseipl() */ 114 subl $1, CPUVAR(MTX_COUNT) /* decl doesnt set CF */ 115 cmovncl CPUVAR(MTX_OLDSPL), %esi 116 movl %esi, CPUVAR(MTX_OLDSPL) 117#if defined(FULL) 118 LOCK(11) 119 cmpxchgb %ah, MTX_LOCK(%rdi) /* lock */ 120 jnz 1f 121#endif 122 ret 1231: 124 jmp _C_LABEL(mutex_spin_retry) /* failed; hard case */ 125 126/* 127 * void mutex_spin_exit(kmutex_t *mtx); 128 * 129 * Release a spin mutex and post a load fence. 130 */ 131NENTRY(mutex_spin_exit) 132#ifdef DIAGNOSTIC 133 134 movl $0x0001, %eax /* new + expected value */ 135 movq CPUVAR(SELF), %r8 136 cmpxchgb %ah, MTX_LOCK(%rdi) /* unlock */ 137 jnz,pn _C_LABEL(mutex_vector_exit) /* hard case if problems */ 138 movl CPU_INFO_MTX_OLDSPL(%r8), %edi 139 incl CPU_INFO_MTX_COUNT(%r8) 140 jnz 1f 141 cmpl CPU_INFO_ILEVEL(%r8), %edi 142 jae 1f 143 movl CPU_INFO_IUNMASK(%r8,%rdi,4), %esi 144 CLI(ax) 145 testl CPU_INFO_IPENDING(%r8), %esi 146 jnz _C_LABEL(Xspllower) 147 movl %edi, CPU_INFO_ILEVEL(%r8) 148 STI(ax) 1491: rep /* double byte ret as branch */ 150 ret /* target: see AMD docs */ 151 152#else /* DIAGNOSTIC */ 153 154 movq CPUVAR(SELF), %rsi 155#ifdef MULTIPROCESSOR 156 movb $0x00, MTX_LOCK(%rdi) 157#endif 158 movl CPU_INFO_MTX_OLDSPL(%rsi), %ecx 159 incl CPU_INFO_MTX_COUNT(%rsi) 160 movl CPU_INFO_ILEVEL(%rsi),%edx 161 cmovnzl %edx,%ecx 162 cmpl %edx,%ecx /* new level is lower? */ 163 pushq %rbx 164 jae,pn 2f 1651: 166 movl CPU_INFO_IPENDING(%rsi),%eax 167 testl %eax,CPU_INFO_IUNMASK(%rsi,%rcx,4)/* deferred interrupts? */ 168 movl %eax,%ebx 169 jnz,pn 3f 170 cmpxchg8b CPU_INFO_ISTATE(%rsi) /* swap in new ilevel */ 171 jnz,pn 1b 1722: 173 popq %rbx 174 ret 1753: 176 popq %rbx 177 movl %ecx, %edi 178 jmp _C_LABEL(Xspllower) 179 180#endif /* DIAGNOSTIC */ 181 182/* 183 * void rw_enter(krwlock_t *rwl, krw_t op); 184 * 185 * Acquire one hold on a RW lock. 186 */ 187NENTRY(rw_enter) 188 cmpl $RW_READER, %esi 189 jne 2f 190 191 /* 192 * Reader: this is the most common case. 193 */ 1941: movq RW_OWNER(%rdi), %rax 195 testb $(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al 196 leaq RW_READ_INCR(%rax), %rdx 197 jnz 3f 198 LOCK(2) 199 cmpxchgq %rdx, RW_OWNER(%rdi) 200 jnz,pn 1b 201 ret 202 203 /* 204 * Writer: if the compare-and-set fails, don't bother retrying. 205 */ 2062: movq CPUVAR(CURLWP), %rcx 207 xorq %rax, %rax 208 orq $RW_WRITE_LOCKED, %rcx 209 LOCK(3) 210 cmpxchgq %rcx, RW_OWNER(%rdi) 211 jnz 3f 212 ret 2133: 214 jmp _C_LABEL(rw_vector_enter) 215 216/* 217 * void rw_exit(krwlock_t *rwl); 218 * 219 * Release one hold on a RW lock. 220 */ 221NENTRY(rw_exit) 222 movq RW_OWNER(%rdi), %rax 223 testb $RW_WRITE_LOCKED, %al 224 jnz 2f 225 226 /* 227 * Reader 228 */ 2291: testb $RW_HAS_WAITERS, %al 230 jnz 3f 231 cmpq $RW_READ_INCR, %rax 232 leaq -RW_READ_INCR(%rax), %rdx 233 jb 3f 234 LOCK(4) 235 cmpxchgq %rdx, RW_OWNER(%rdi) 236 jnz,pn 1b 237 ret 238 239 /* 240 * Writer 241 */ 2422: leaq -RW_WRITE_LOCKED(%rax), %rdx 243 subq CPUVAR(CURLWP), %rdx 244 jnz 3f 245 LOCK(5) 246 cmpxchgq %rdx, RW_OWNER(%rdi) 247 jnz 3f 248 ret 249 2503: jmp _C_LABEL(rw_vector_exit) 251 252/* 253 * int rw_tryenter(krwlock_t *rwl, krw_t op); 254 * 255 * Try to acquire one hold on a RW lock. 256 */ 257NENTRY(rw_tryenter) 258 cmpl $RW_READER, %esi 259 jne 2f 260 261 /* 262 * Reader: this is the most common case. 263 */ 2641: movq RW_OWNER(%rdi), %rax 265 testb $(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al 266 leaq RW_READ_INCR(%rax), %rdx 267 jnz 3f 268 LOCK(8) 269 cmpxchgq %rdx, RW_OWNER(%rdi) 270 movl $0, %eax 271 setz %al 272 ret 273 274 /* 275 * Writer: if the compare-and-set fails, don't bother retrying. 276 */ 2772: movq CPUVAR(CURLWP), %rcx 278 xorq %rax, %rax 279 orq $RW_WRITE_LOCKED, %rcx 280 LOCK(9) 281 cmpxchgq %rcx, RW_OWNER(%rdi) 282 movl $0, %eax 283 setz %al 284 ret 285 2863: xorl %eax, %eax 287 ret 288 289#endif /* LOCKDEBUG */ 290 291/* 292 * Spinlocks. 293 */ 294NENTRY(__cpu_simple_lock_init) 295 movb $0, (%rdi) 296 ret 297 298NENTRY(__cpu_simple_lock) 299 movl $0x0100, %eax 3001: 301 LOCK(6) 302 cmpxchgb %ah, (%rdi) 303 jnz 2f 304 ret 3052: 306 movl $0x0100, %eax 307 pause 308 nop 309 nop 310 cmpb $0, (%rdi) 311 je 1b 312 jmp 2b 313 314NENTRY(__cpu_simple_unlock) 315 movb $0, (%rdi) 316 ret 317 318NENTRY(__cpu_simple_lock_try) 319 movl $0x0100, %eax 320 LOCK(7) 321 cmpxchgb %ah, (%rdi) 322 movl $0, %eax 323 setz %al 324 ret 325 326/* 327 * Patchpoints to replace with NOP when ncpu == 1. 328 */ 329#ifndef LOCKDEBUG 330LABEL(x86_lockpatch) 331 .quad .Lpatch1, .Lpatch2, .Lpatch3, .Lpatch4 332 .quad .Lpatch5, .Lpatch6, .Lpatch7, .Lpatch8 333 .quad .Lpatch9 334#ifdef FULL 335 .quad .Lpatch11 336#endif 337 .quad 0 338#endif 339