atomic.S revision 1.19.6.1
1 /* $NetBSD: atomic.S,v 1.19.6.1 2014/05/22 11:26:28 yamt Exp $ */ 2 3 /*- 4 * Copyright (c) 2007 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/param.h> 33 #include <machine/asm.h> 34 /* 35 * __HAVE_ constants should not be in <machine/types.h> 36 * because we can't use them from assembly. OTOH we 37 * only need __HAVE_ATOMIC64_OPS here, and we don't. 38 */ 39 #ifdef _KERNEL 40 #define ALIAS(f, t) STRONG_ALIAS(f,t) 41 #else 42 #define ALIAS(f, t) WEAK_ALIAS(f,t) 43 #endif 44 45 #ifdef _HARDKERNEL 46 #define LOCK(n) .Lpatch ## n: lock 47 #define ENDLABEL(a) _ALIGN_TEXT; LABEL(a) 48 #else 49 #define LOCK(n) lock 50 #define ENDLABEL(a) /* nothing */ 51 #endif 52 53 .text 54 55 ENTRY(_atomic_add_32) 56 movl 4(%esp), %edx 57 movl 8(%esp), %eax 58 LOCK(1) 59 addl %eax, (%edx) 60 ret 61 62 ENTRY(_atomic_add_32_nv) 63 movl 4(%esp), %edx 64 movl 8(%esp), %eax 65 movl %eax, %ecx 66 LOCK(2) 67 xaddl %eax, (%edx) 68 addl %ecx, %eax 69 ret 70 71 ENTRY(_atomic_and_32) 72 movl 4(%esp), %edx 73 movl 8(%esp), %eax 74 LOCK(3) 75 andl %eax, (%edx) 76 ret 77 78 ENTRY(_atomic_and_32_nv) 79 movl 4(%esp), %edx 80 movl (%edx), %eax 81 0: 82 movl %eax, %ecx 83 andl 8(%esp), %ecx 84 LOCK(4) 85 cmpxchgl %ecx, (%edx) 86 jnz 1f 87 movl %ecx, %eax 88 ret 89 1: 90 jmp 0b 91 92 ENTRY(_atomic_dec_32) 93 movl 4(%esp), %edx 94 LOCK(5) 95 decl (%edx) 96 ret 97 98 ENTRY(_atomic_dec_32_nv) 99 movl 4(%esp), %edx 100 movl $-1, %eax 101 LOCK(6) 102 xaddl %eax, (%edx) 103 decl %eax 104 ret 105 106 ENTRY(_atomic_inc_32) 107 movl 4(%esp), %edx 108 LOCK(7) 109 incl (%edx) 110 ret 111 112 ENTRY(_atomic_inc_32_nv) 113 movl 4(%esp), %edx 114 movl $1, %eax 115 LOCK(8) 116 xaddl %eax, (%edx) 117 incl %eax 118 ret 119 120 ENTRY(_atomic_or_32) 121 movl 4(%esp), %edx 122 movl 8(%esp), %eax 123 LOCK(9) 124 orl %eax, (%edx) 125 ret 126 127 ENTRY(_atomic_or_32_nv) 128 movl 4(%esp), %edx 129 movl (%edx), %eax 130 0: 131 movl %eax, %ecx 132 orl 8(%esp), %ecx 133 LOCK(10) 134 cmpxchgl %ecx, (%edx) 135 jnz 1f 136 movl %ecx, %eax 137 ret 138 1: 139 jmp 0b 140 141 ENTRY(_atomic_swap_32) 142 movl 4(%esp), %edx 143 movl 8(%esp), %eax 144 xchgl %eax, (%edx) 145 ret 146 147 ENTRY(_atomic_cas_32) 148 movl 4(%esp), %edx 149 movl 8(%esp), %eax 150 movl 12(%esp), %ecx 151 LOCK(12) 152 cmpxchgl %ecx, (%edx) 153 /* %eax now contains the old value */ 154 ret 155 156 ENTRY(_atomic_cas_32_ni) 157 movl 4(%esp), %edx 158 movl 8(%esp), %eax 159 movl 12(%esp), %ecx 160 cmpxchgl %ecx, (%edx) 161 /* %eax now contains the old value */ 162 ret 163 164 ENTRY(_membar_consumer) 165 LOCK(13) 166 addl $0, -4(%esp) 167 ret 168 ENDLABEL(membar_consumer_end) 169 170 ENTRY(_membar_producer) 171 /* A store is enough */ 172 movl $0, -4(%esp) 173 ret 174 ENDLABEL(membar_producer_end) 175 176 ENTRY(_membar_sync) 177 LOCK(14) 178 addl $0, -4(%esp) 179 ret 180 ENDLABEL(membar_sync_end) 181 182 #if defined(__HAVE_ATOMIC64_OPS) || defined(_KERNEL) 183 ENTRY(_atomic_cas_64) 184 #ifdef _HARDKERNEL 185 pushf 186 cli 187 #endif /* _HARDKERNEL */ 188 pushl %edi 189 pushl %ebx 190 movl 12(%esp), %edi 191 movl 16(%esp), %eax 192 movl 20(%esp), %edx 193 movl 24(%esp), %ebx 194 movl 28(%esp), %ecx 195 cmpl 0(%edi), %eax 196 jne 2f 197 cmpl 4(%edi), %edx 198 jne 2f 199 movl %ebx, 0(%edi) 200 movl %ecx, 4(%edi) 201 1: 202 popl %ebx 203 popl %edi 204 #ifdef _HARDKERNEL 205 popf 206 #endif /* _HARDKERNEL */ 207 ret 208 2: 209 movl 0(%edi), %eax 210 movl 4(%edi), %edx 211 jmp 1b 212 ENDLABEL(_atomic_cas_64_end) 213 214 ENTRY(_atomic_cas_cx8) 215 pushl %edi 216 pushl %ebx 217 movl 12(%esp), %edi 218 movl 16(%esp), %eax 219 movl 20(%esp), %edx 220 movl 24(%esp), %ebx 221 movl 28(%esp), %ecx 222 LOCK(15) 223 cmpxchg8b (%edi) 224 popl %ebx 225 popl %edi 226 ret 227 #ifdef _HARDKERNEL 228 #ifdef GPROF 229 .space 16, 0x90 230 #else 231 .space 32, 0x90 232 #endif 233 #endif /* _HARDKERNEL */ 234 ENDLABEL(_atomic_cas_cx8_end) 235 #endif /* __HAVE_ATOMIC64_OPS || _KERNEL */ 236 237 #ifdef _HARDKERNEL 238 ENTRY(sse2_lfence) 239 lfence 240 ret 241 ENDLABEL(sse2_lfence_end) 242 243 ENTRY(sse2_mfence) 244 mfence 245 ret 246 ENDLABEL(sse2_mfence_end) 247 248 atomic_lockpatch: 249 .globl atomic_lockpatch 250 .long .Lpatch1, .Lpatch2, .Lpatch3, .Lpatch4, .Lpatch5 251 .long .Lpatch6, .Lpatch7, .Lpatch8, .Lpatch9, .Lpatch10 252 .long .Lpatch12, .Lpatch13, .Lpatch14, .Lpatch15, 0 253 #endif /* _HARDKERNEL */ 254 255 ALIAS(atomic_add_32,_atomic_add_32) 256 ALIAS(atomic_add_int,_atomic_add_32) 257 ALIAS(atomic_add_long,_atomic_add_32) 258 ALIAS(atomic_add_ptr,_atomic_add_32) 259 260 ALIAS(atomic_add_32_nv,_atomic_add_32_nv) 261 ALIAS(atomic_add_int_nv,_atomic_add_32_nv) 262 ALIAS(atomic_add_long_nv,_atomic_add_32_nv) 263 ALIAS(atomic_add_ptr_nv,_atomic_add_32_nv) 264 265 ALIAS(atomic_and_32,_atomic_and_32) 266 ALIAS(atomic_and_uint,_atomic_and_32) 267 ALIAS(atomic_and_ulong,_atomic_and_32) 268 ALIAS(atomic_and_ptr,_atomic_and_32) 269 270 ALIAS(atomic_and_32_nv,_atomic_and_32_nv) 271 ALIAS(atomic_and_uint_nv,_atomic_and_32_nv) 272 ALIAS(atomic_and_ulong_nv,_atomic_and_32_nv) 273 ALIAS(atomic_and_ptr_nv,_atomic_and_32_nv) 274 275 ALIAS(atomic_dec_32,_atomic_dec_32) 276 ALIAS(atomic_dec_uint,_atomic_dec_32) 277 ALIAS(atomic_dec_ulong,_atomic_dec_32) 278 ALIAS(atomic_dec_ptr,_atomic_dec_32) 279 280 ALIAS(atomic_dec_32_nv,_atomic_dec_32_nv) 281 ALIAS(atomic_dec_uint_nv,_atomic_dec_32_nv) 282 ALIAS(atomic_dec_ulong_nv,_atomic_dec_32_nv) 283 ALIAS(atomic_dec_ptr_nv,_atomic_dec_32_nv) 284 285 ALIAS(atomic_inc_32,_atomic_inc_32) 286 ALIAS(atomic_inc_uint,_atomic_inc_32) 287 ALIAS(atomic_inc_ulong,_atomic_inc_32) 288 ALIAS(atomic_inc_ptr,_atomic_inc_32) 289 290 ALIAS(atomic_inc_32_nv,_atomic_inc_32_nv) 291 ALIAS(atomic_inc_uint_nv,_atomic_inc_32_nv) 292 ALIAS(atomic_inc_ulong_nv,_atomic_inc_32_nv) 293 ALIAS(atomic_inc_ptr_nv,_atomic_inc_32_nv) 294 295 ALIAS(atomic_or_32,_atomic_or_32) 296 ALIAS(atomic_or_uint,_atomic_or_32) 297 ALIAS(atomic_or_ulong,_atomic_or_32) 298 ALIAS(atomic_or_ptr,_atomic_or_32) 299 300 ALIAS(atomic_or_32_nv,_atomic_or_32_nv) 301 ALIAS(atomic_or_uint_nv,_atomic_or_32_nv) 302 ALIAS(atomic_or_ulong_nv,_atomic_or_32_nv) 303 ALIAS(atomic_or_ptr_nv,_atomic_or_32_nv) 304 305 ALIAS(atomic_swap_32,_atomic_swap_32) 306 ALIAS(atomic_swap_uint,_atomic_swap_32) 307 ALIAS(atomic_swap_ulong,_atomic_swap_32) 308 ALIAS(atomic_swap_ptr,_atomic_swap_32) 309 310 ALIAS(atomic_cas_32,_atomic_cas_32) 311 ALIAS(atomic_cas_uint,_atomic_cas_32) 312 ALIAS(atomic_cas_ulong,_atomic_cas_32) 313 ALIAS(atomic_cas_ptr,_atomic_cas_32) 314 315 ALIAS(atomic_cas_32_ni,_atomic_cas_32_ni) 316 ALIAS(atomic_cas_uint_ni,_atomic_cas_32_ni) 317 ALIAS(atomic_cas_ulong_ni,_atomic_cas_32_ni) 318 ALIAS(atomic_cas_ptr_ni,_atomic_cas_32_ni) 319 320 #if defined(__HAVE_ATOMIC64_OPS) || defined(_KERNEL) 321 ALIAS(atomic_cas_64,_atomic_cas_64) 322 ALIAS(atomic_cas_64_ni,_atomic_cas_64) 323 ALIAS(__sync_val_compare_and_swap_8,_atomic_cas_64) 324 #endif /* __HAVE_ATOMIC64_OPS || _KERNEL */ 325 326 ALIAS(membar_consumer,_membar_consumer) 327 ALIAS(membar_producer,_membar_producer) 328 ALIAS(membar_enter,_membar_consumer) 329 ALIAS(membar_exit,_membar_producer) 330 ALIAS(membar_sync,_membar_sync) 331 332 STRONG_ALIAS(_atomic_add_int,_atomic_add_32) 333 STRONG_ALIAS(_atomic_add_long,_atomic_add_32) 334 STRONG_ALIAS(_atomic_add_ptr,_atomic_add_32) 335 336 STRONG_ALIAS(_atomic_add_int_nv,_atomic_add_32_nv) 337 STRONG_ALIAS(_atomic_add_long_nv,_atomic_add_32_nv) 338 STRONG_ALIAS(_atomic_add_ptr_nv,_atomic_add_32_nv) 339 340 STRONG_ALIAS(_atomic_and_uint,_atomic_and_32) 341 STRONG_ALIAS(_atomic_and_ulong,_atomic_and_32) 342 STRONG_ALIAS(_atomic_and_ptr,_atomic_and_32) 343 344 STRONG_ALIAS(_atomic_and_uint_nv,_atomic_and_32_nv) 345 STRONG_ALIAS(_atomic_and_ulong_nv,_atomic_and_32_nv) 346 STRONG_ALIAS(_atomic_and_ptr_nv,_atomic_and_32_nv) 347 348 STRONG_ALIAS(_atomic_dec_uint,_atomic_dec_32) 349 STRONG_ALIAS(_atomic_dec_ulong,_atomic_dec_32) 350 STRONG_ALIAS(_atomic_dec_ptr,_atomic_dec_32) 351 352 STRONG_ALIAS(_atomic_dec_uint_nv,_atomic_dec_32_nv) 353 STRONG_ALIAS(_atomic_dec_ulong_nv,_atomic_dec_32_nv) 354 STRONG_ALIAS(_atomic_dec_ptr_nv,_atomic_dec_32_nv) 355 356 STRONG_ALIAS(_atomic_inc_uint,_atomic_inc_32) 357 STRONG_ALIAS(_atomic_inc_ulong,_atomic_inc_32) 358 STRONG_ALIAS(_atomic_inc_ptr,_atomic_inc_32) 359 360 STRONG_ALIAS(_atomic_inc_uint_nv,_atomic_inc_32_nv) 361 STRONG_ALIAS(_atomic_inc_ulong_nv,_atomic_inc_32_nv) 362 STRONG_ALIAS(_atomic_inc_ptr_nv,_atomic_inc_32_nv) 363 364 STRONG_ALIAS(_atomic_or_uint,_atomic_or_32) 365 STRONG_ALIAS(_atomic_or_ulong,_atomic_or_32) 366 STRONG_ALIAS(_atomic_or_ptr,_atomic_or_32) 367 368 STRONG_ALIAS(_atomic_or_uint_nv,_atomic_or_32_nv) 369 STRONG_ALIAS(_atomic_or_ulong_nv,_atomic_or_32_nv) 370 STRONG_ALIAS(_atomic_or_ptr_nv,_atomic_or_32_nv) 371 372 STRONG_ALIAS(_atomic_swap_uint,_atomic_swap_32) 373 STRONG_ALIAS(_atomic_swap_ulong,_atomic_swap_32) 374 STRONG_ALIAS(_atomic_swap_ptr,_atomic_swap_32) 375 376 STRONG_ALIAS(_atomic_cas_uint,_atomic_cas_32) 377 STRONG_ALIAS(_atomic_cas_ulong,_atomic_cas_32) 378 STRONG_ALIAS(_atomic_cas_ptr,_atomic_cas_32) 379 380 STRONG_ALIAS(_atomic_cas_uint_ni,_atomic_cas_32_ni) 381 STRONG_ALIAS(_atomic_cas_ulong_ni,_atomic_cas_32_ni) 382 STRONG_ALIAS(_atomic_cas_ptr_ni,_atomic_cas_32_ni) 383 384 STRONG_ALIAS(_membar_enter,_membar_consumer) 385 STRONG_ALIAS(_membar_exit,_membar_producer) 386
Indexes created Fri Oct 03 02:09:57 GMT 2025