1 /* $NetBSD: lock_stubs.S,v 1.38 2022/09/08 06:57:44 knakahara Exp $ */ 2 3 /*- 4 * Copyright (c) 2006, 2007, 2008, 2009 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Where possible we make each routine fit into an assumed 64-byte cache 34 * line. Please check alignment with 'objdump -d' after making changes. 35 */ 36 37 #include <machine/asm.h> 38 __KERNEL_RCSID(0, "$NetBSD: lock_stubs.S,v 1.38 2022/09/08 06:57:44 knakahara Exp $"); 39 40 #include "opt_lockdebug.h" 41 42 #include <machine/cputypes.h> 43 #include <machine/frameasm.h> 44 45 #include "assym.h" 46 47 #define ALIGN64 .align 64 48 #define ALIGN32 .align 32 49 #define LOCK(num) \ 50 HOTPATCH(HP_NAME_NOLOCK, 1) ; \ 51 lock 52 #define RET(num) \ 53 HOTPATCH(HP_NAME_RETFENCE, 3) ; \ 54 ret; nop; nop ; \ 55 ret 56 57 #define ENDLABEL(name,a) .align a; LABEL(name) 58 59 #if !defined(LOCKDEBUG) 60 61 /* 62 * void mutex_enter(kmutex_t *mtx); 63 * 64 * Acquire a mutex and post a load fence. 65 */ 66 ALIGN64 67 68 ENTRY(mutex_enter) 69 movl 4(%esp), %edx 70 xorl %eax, %eax 71 movl %fs:CPU_INFO_CURLWP(%eax), %ecx 72 LOCK(1) 73 cmpxchgl %ecx, (%edx) 74 jnz 1f 75 RET(1) 76 1: 77 jmp _C_LABEL(mutex_vector_enter) 78 END(mutex_enter) 79 80 /* 81 * void mutex_exit(kmutex_t *mtx); 82 * 83 * Release a mutex and post a load fence. 84 * 85 * See comments in mutex_vector_enter() about doing this operation unlocked 86 * on multiprocessor systems, and comments in arch/x86/include/lock.h about 87 * memory ordering on Intel x86 systems. 88 */ 89 ENTRY(mutex_exit) 90 movl 4(%esp), %edx 91 xorl %ecx, %ecx 92 movl %fs:CPU_INFO_CURLWP(%ecx), %eax 93 cmpxchgl %ecx, (%edx) 94 jnz 1f 95 ret 96 1: 97 jmp _C_LABEL(mutex_vector_exit) 98 END(mutex_exit) 99 100 /* 101 * void rw_enter(krwlock_t *rwl, krw_t op); 102 * 103 * Acquire one hold on a RW lock. 104 */ 105 ENTRY(rw_enter) 106 movl 4(%esp), %edx 107 cmpl $RW_READER, 8(%esp) 108 jne 2f 109 110 /* 111 * Reader 112 */ 113 movl (%edx), %eax 114 0: 115 testb $(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al 116 jnz 3f 117 leal RW_READ_INCR(%eax), %ecx 118 LOCK(2) 119 cmpxchgl %ecx, (%edx) 120 jnz 1f 121 RET(2) 122 1: 123 jmp 0b 124 125 /* 126 * Writer 127 */ 128 2: xorl %eax, %eax 129 movl %fs:CPU_INFO_CURLWP(%eax), %ecx 130 orl $RW_WRITE_LOCKED, %ecx 131 LOCK(3) 132 cmpxchgl %ecx, (%edx) 133 jnz 3f 134 RET(3) 135 3: 136 jmp _C_LABEL(rw_vector_enter) 137 END(rw_enter) 138 139 /* 140 * void rw_exit(krwlock_t *rwl); 141 * 142 * Release one hold on a RW lock. 143 */ 144 ENTRY(rw_exit) 145 movl 4(%esp), %edx 146 movl (%edx), %eax 147 testb $RW_WRITE_LOCKED, %al 148 jnz 2f 149 150 /* 151 * Reader 152 */ 153 0: testb $RW_HAS_WAITERS, %al 154 jnz 3f 155 cmpl $RW_READ_INCR, %eax 156 jb 3f 157 leal -RW_READ_INCR(%eax), %ecx 158 LOCK(4) 159 cmpxchgl %ecx, (%edx) 160 jnz 1f 161 ret 162 1: 163 jmp 0b 164 165 /* 166 * Writer 167 */ 168 2: leal -RW_WRITE_LOCKED(%eax), %ecx 169 subl CPUVAR(CURLWP), %ecx 170 jnz 3f 171 LOCK(5) 172 cmpxchgl %ecx, (%edx) 173 jnz 3f 174 ret 175 176 /* 177 * Slow path. 178 */ 179 3: jmp _C_LABEL(rw_vector_exit) 180 END(rw_exit) 181 182 /* 183 * int rw_tryenter(krwlock_t *rwl, krw_t op); 184 * 185 * Try to acquire one hold on a RW lock. 186 */ 187 ENTRY(rw_tryenter) 188 movl 4(%esp), %edx 189 cmpl $RW_READER, 8(%esp) 190 jne 2f 191 192 /* 193 * Reader 194 */ 195 movl (%edx), %eax 196 0: 197 testb $(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al 198 jnz 4f 199 leal RW_READ_INCR(%eax), %ecx 200 LOCK(12) 201 cmpxchgl %ecx, (%edx) 202 jnz 1f 203 movl %edx, %eax /* nonzero */ 204 RET(4) 205 1: 206 jmp 0b 207 208 /* 209 * Writer 210 */ 211 2: 212 xorl %eax, %eax 213 movl %fs:CPU_INFO_CURLWP(%eax), %ecx 214 orl $RW_WRITE_LOCKED, %ecx 215 LOCK(13) 216 cmpxchgl %ecx, (%edx) 217 movl $0, %eax 218 setz %al 219 3: 220 RET(5) 221 4: 222 xorl %eax, %eax 223 jmp 3b 224 END(rw_tryenter) 225 226 227 /* 228 * void mutex_spin_enter(kmutex_t *mtx); 229 * 230 * Acquire a spin mutex and post a load fence. 231 */ 232 ENTRY(mutex_spin_enter) 233 movl 4(%esp), %edx 234 movb CPUVAR(ILEVEL), %cl 235 movb MTX_IPL(%edx), %ch 236 movl $0x01, %eax 237 cmpb %ch, %cl 238 jg 1f 239 movb %ch, CPUVAR(ILEVEL) /* splraiseipl() */ 240 1: 241 subl %eax, CPUVAR(MTX_COUNT) /* decl does not set CF */ 242 jnc 2f 243 movb %cl, CPUVAR(MTX_OLDSPL) 244 2: 245 xchgb %al, MTX_LOCK(%edx) /* lock it */ 246 testb %al, %al 247 jnz 3f 248 RET(6) 249 3: 250 jmp _C_LABEL(mutex_spin_retry) 251 252 ALIGN64 253 LABEL(mutex_spin_enter_end) 254 END(mutex_spin_enter) 255 256 #ifndef XENPV 257 /* 258 * Release a spin mutex and post a store fence. Must occupy 128 bytes. 259 */ 260 ENTRY(mutex_spin_exit) 261 HOTPATCH(HP_NAME_MUTEX_EXIT, 128) 262 movl 4(%esp), %edx 263 movl CPUVAR(MTX_OLDSPL), %ecx 264 incl CPUVAR(MTX_COUNT) 265 movb $0, MTX_LOCK(%edx) /* zero */ 266 jnz 1f 267 movl CPUVAR(IUNMASK)(,%ecx,8), %edx 268 movl CPUVAR(IUNMASK)+4(,%ecx,8), %eax 269 cli 270 testl CPUVAR(IPENDING), %edx 271 movl %ecx, 4(%esp) 272 jnz _C_LABEL(Xspllower) /* does sti */ 273 testl CPUVAR(IPENDING)+4, %eax 274 jnz _C_LABEL(Xspllower) /* does sti */ 275 movb %cl, CPUVAR(ILEVEL) 276 sti 277 1: ret 278 .space 32, 0xCC 279 .align 32 280 END(mutex_spin_exit) 281 #else /* XENPV */ 282 STRONG_ALIAS(mutex_spin_exit, i686_mutex_spin_exit) 283 #endif /* !XENPV */ 284 285 /* 286 * Patch for i686 CPUs where cli/sti is prohibitively expensive. 287 * Must be the same size as mutex_spin_exit(), that is, 128 bytes. 288 */ 289 ENTRY(i686_mutex_spin_exit) 290 mov 4(%esp),%edx 291 movl CPUVAR(MTX_OLDSPL), %ecx 292 incl CPUVAR(MTX_COUNT) 293 movb $0, MTX_LOCK(%edx) /* zero */ 294 jnz 1f 295 pushl %ebx 296 pushl %esi 297 pushl %edi 298 movl %ecx, %esi 299 movl %ecx, %edi 300 shll $24, %edi 301 0: 302 movl CPUVAR(IPENDING), %eax 303 testl %eax, CPUVAR(IUNMASK)(,%esi,8) 304 jnz 2f 305 movl CPUVAR(IPENDING)+4, %edx 306 testl %edx, CPUVAR(IUNMASK)+4(,%esi,8) 307 jnz 2f 308 movl %eax, %ebx 309 movl %edx, %ecx 310 andl $0x00ffffff, %ecx 311 orl %edi, %ecx 312 cmpxchg8b CPUVAR(ISTATE) /* swap in new ilevel */ 313 jnz 0b 314 popl %edi 315 popl %esi 316 popl %ebx 317 1: 318 ret 319 2: 320 movl %esi,%ecx 321 popl %edi 322 popl %esi 323 popl %ebx 324 movl %ecx,4(%esp) 325 326 /* The reference must be absolute, hence the indirect jump. */ 327 movl $Xspllower,%eax 328 jmp *%eax 329 330 .space 16, 0xCC 331 .align 32 332 LABEL(i686_mutex_spin_exit_end) 333 END(i686_mutex_spin_exit) 334 335 #endif /* !LOCKDEBUG */ 336 337 /* 338 * Spinlocks. 339 */ 340 ENTRY(__cpu_simple_lock_init) 341 movl 4(%esp), %edx 342 movb $0, (%edx) 343 ret 344 END(__cpu_simple_lock_init) 345 346 ENTRY(__cpu_simple_lock) 347 movl 4(%esp), %edx 348 movl $0x0100, %eax 349 1: 350 LOCK(6) 351 cmpxchgb %ah, (%edx) 352 jnz 2f 353 RET(7) 354 2: 355 movl $0x0100, %eax 356 pause 357 nop 358 nop 359 cmpb $0, (%edx) 360 je 1b 361 jmp 2b 362 END(__cpu_simple_lock) 363 364 ENTRY(__cpu_simple_unlock) 365 movl 4(%esp), %edx 366 movb $0, (%edx) 367 ret 368 END(__cpu_simple_unlock) 369 370 ENTRY(__cpu_simple_lock_try) 371 movl 4(%esp), %edx 372 movl $0x0100, %eax 373 LOCK(7) 374 cmpxchgb %ah, (%edx) 375 movl $0, %eax 376 setz %al 377 RET(8) 378 END(__cpu_simple_lock_try) 379 380
Indexes created Sun Sep 21 20:09:37 GMT 2025