lock_stubs.S revision 1.11
1/* $NetBSD: lock_stubs.S,v 1.11 2007/12/20 23:46:10 ad Exp $ */ 2 3/*- 4 * Copyright (c) 2006, 2007 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. All advertising materials mentioning features or use of this software 19 * must display the following acknowledgement: 20 * This product includes software developed by the NetBSD 21 * Foundation, Inc. and its contributors. 22 * 4. Neither the name of The NetBSD Foundation nor the names of its 23 * contributors may be used to endorse or promote products derived 24 * from this software without specific prior written permission. 25 * 26 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 27 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 28 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 29 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 30 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 31 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 32 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 33 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 34 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 35 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 36 * POSSIBILITY OF SUCH DAMAGE. 37 */ 38 39/* 40 * AMD64 lock stubs. Calling convention: 41 * 42 * %rdi arg 1 43 * %rsi arg 2 44 * %rdx arg 3 45 * %rax return value 46 */ 47 48#include "opt_multiprocessor.h" 49#include "opt_lockdebug.h" 50 51#include <machine/asm.h> 52#include <machine/frameasm.h> 53 54#include "assym.h" 55 56#if defined(DIAGNOSTIC) || defined(MULTIPROCESSOR) || defined(LOCKDEBUG) 57#define FULL 58#endif 59 60#define END(name,a) .align a; LABEL(name) 61#define LOCK(num) .Lpatch/**/num: lock 62 63#ifndef LOCKDEBUG 64 65/* 66 * void mutex_enter(kmutex_t *mtx); 67 * 68 * Acquire a mutex and post a load fence. 69 */ 70 .align 64 71 72NENTRY(mutex_enter) 73 movq CPUVAR(CURLWP), %rcx 74 xorq %rax, %rax 75 LOCK(1) 76 cmpxchgq %rcx, MTX_OWNER(%rdi) 77 jnz,pn _C_LABEL(mutex_vector_enter) 78 ret 79 80/* 81 * void mutex_exit(kmutex_t *mtx); 82 * 83 * Release a mutex and post a load fence. 84 * 85 * See comments in mutex_vector_enter() about doing this operation unlocked 86 * on multiprocessor systems, and comments in arch/x86/include/lock.h about 87 * memory ordering on Intel x86 systems. 88 */ 89NENTRY(mutex_exit) 90 movq CPUVAR(CURLWP), %rax 91 xorq %rdx, %rdx 92 cmpxchgq %rdx, MTX_OWNER(%rdi) 93 jnz,pn _C_LABEL(mutex_vector_exit) 94 ret 95 96/* 97 * void mutex_spin_enter(kmutex_t *mtx); 98 * 99 * Acquire a spin mutex and post a load fence. 100 */ 101NENTRY(mutex_spin_enter) 102 movq CPUVAR(SELF), %r8 103#if defined(FULL) 104 movl $0x0100, %eax /* new + expected value */ 105#endif 106 movl CPU_INFO_ILEVEL(%r8), %esi 107 subl $1, CPU_INFO_MTX_COUNT(%r8) /* decl doesnt set CF */ 108 movzbl MTX_IPL(%rdi), %ecx /* new SPL */ 109 cmovncl CPU_INFO_MTX_OLDSPL(%r8), %esi 110 cmpl %ecx, %esi /* higher? */ 111 movl %esi, CPU_INFO_MTX_OLDSPL(%r8) 112 cmovgl %esi, %ecx 113 movl %ecx, CPU_INFO_ILEVEL(%r8) /* splraiseipl() */ 114#if defined(FULL) 115 LOCK(11) 116 cmpxchgb %ah, MTX_LOCK(%rdi) /* lock */ 117 jnz,pn _C_LABEL(mutex_spin_retry) /* failed; hard case */ 118#endif 119 ret 120 121/* 122 * void mutex_spin_exit(kmutex_t *mtx); 123 * 124 * Release a spin mutex and post a load fence. 125 */ 126NENTRY(mutex_spin_exit) 127#ifdef DIAGNOSTIC 128 129 movl $0x0001, %eax /* new + expected value */ 130 movq CPUVAR(SELF), %r8 131 cmpxchgb %ah, MTX_LOCK(%rdi) /* unlock */ 132 jnz,pn _C_LABEL(mutex_vector_exit) /* hard case if problems */ 133 movl CPU_INFO_MTX_OLDSPL(%r8), %edi 134 incl CPU_INFO_MTX_COUNT(%r8) 135 jnz 1f 136 cmpl CPU_INFO_ILEVEL(%r8), %edi 137 jae 1f 138 movl CPU_INFO_IUNMASK(%r8,%rdi,4), %esi 139 CLI(ax,10) 140 testl CPU_INFO_IPENDING(%r8), %esi 141 jnz _C_LABEL(Xspllower) 142 movl %edi, CPU_INFO_ILEVEL(%r8) 143 STI(ax,10) 1441: rep /* double byte ret as branch */ 145 ret /* target: see AMD docs */ 146 147#else /* DIAGNOSTIC */ 148 149 movq CPUVAR(SELF), %rsi 150#ifdef MULTIPROCESSOR 151 movb $0x00, MTX_LOCK(%rdi) 152#endif 153 movl CPU_INFO_MTX_OLDSPL(%rsi), %ecx 154 incl CPU_INFO_MTX_COUNT(%rsi) 155 movl CPU_INFO_ILEVEL(%rsi),%edx 156 cmovnzl %edx,%ecx 157 cmpl %edx,%ecx /* new level is lower? */ 158 pushq %rbx 159 jae,pn 2f 1601: 161 movl CPU_INFO_IPENDING(%rsi),%eax 162 testl %eax,CPU_INFO_IUNMASK(%rsi,%rcx,4)/* deferred interrupts? */ 163 movl %eax,%ebx 164 jnz,pn 3f 165 cmpxchg8b CPU_INFO_ISTATE(%rsi) /* swap in new ilevel */ 166 jnz,pn 1b 1672: 168 popq %rbx 169 ret 1703: 171 popq %rbx 172 movl %ecx, %edi 173 jmp _C_LABEL(Xspllower) 174 175#endif /* DIAGNOSTIC */ 176 177/* 178 * void rw_enter(krwlock_t *rwl, krw_t op); 179 * 180 * Acquire one hold on a RW lock. 181 */ 182NENTRY(rw_enter) 183 cmpl $RW_READER, %esi 184 jne 2f 185 186 /* 187 * Reader: this is the most common case. 188 */ 1891: movq RW_OWNER(%rdi), %rax 190 testb $(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al 191 leaq RW_READ_INCR(%rax), %rdx 192 jnz,pn _C_LABEL(rw_vector_enter) 193 LOCK(2) 194 cmpxchgq %rdx, RW_OWNER(%rdi) 195 jnz,pn 1b 196 ret 197 198 /* 199 * Writer: if the compare-and-set fails, don't bother retrying. 200 */ 2012: movq CPUVAR(CURLWP), %rcx 202 xorq %rax, %rax 203 orq $RW_WRITE_LOCKED, %rcx 204 LOCK(3) 205 cmpxchgq %rcx, RW_OWNER(%rdi) 206 jnz,pn _C_LABEL(rw_vector_enter) 207 ret 208 209/* 210 * void rw_exit(krwlock_t *rwl); 211 * 212 * Release one hold on a RW lock. 213 */ 214NENTRY(rw_exit) 215 movq RW_OWNER(%rdi), %rax 216 testb $RW_WRITE_LOCKED, %al 217 jnz 2f 218 219 /* 220 * Reader 221 */ 2221: testb $RW_HAS_WAITERS, %al 223 jnz,pn 3f 224 cmpq $RW_READ_INCR, %rax 225 leaq -RW_READ_INCR(%rax), %rdx 226 jb,pn 3f 227 LOCK(4) 228 cmpxchgq %rdx, RW_OWNER(%rdi) 229 jnz,pn 1b 230 ret 231 232 /* 233 * Writer 234 */ 2352: leaq -RW_WRITE_LOCKED(%rax), %rdx 236 subq CPUVAR(CURLWP), %rdx 237 jnz,pn 3f 238 LOCK(5) 239 cmpxchgq %rdx, RW_OWNER(%rdi) 240 jnz 3f 241 ret 242 2433: jmp _C_LABEL(rw_vector_exit) 244 245#endif /* LOCKDEBUG */ 246 247/* 248 * Spinlocks. 249 */ 250NENTRY(__cpu_simple_lock_init) 251 movb $0, (%rdi) 252 ret 253 254NENTRY(__cpu_simple_lock) 255 movl $0x0100, %eax 2561: 257 LOCK(6) 258 cmpxchgb %ah, (%rdi) 259 jnz 2f 260 ret 2612: 262 movl $0x0100, %eax 263 pause 264 nop 265 nop 266 cmpb $0, (%rdi) 267 je 1b 268 jmp 2b 269 270NENTRY(__cpu_simple_unlock) 271 movb $0, (%rdi) 272 ret 273 274NENTRY(__cpu_simple_lock_try) 275 movl $0x0100, %eax 276 LOCK(7) 277 cmpxchgb %ah, (%rdi) 278 movl $0, %eax 279 setz %al 280 ret 281 282/* 283 * Patchpoints to replace with NOP when ncpu == 1. 284 */ 285#ifndef LOCKDEBUG 286LABEL(x86_lockpatch) 287 .quad .Lpatch1, .Lpatch2, .Lpatch3, .Lpatch4 288 .quad .Lpatch5, .Lpatch6, .Lpatch7 289#ifdef FULL 290 .quad .Lpatch11 291#endif 292 .quad 0 293#endif 294