atomic.S revision 1.26
1 /* $NetBSD: atomic.S,v 1.26 2020/04/26 13:54:02 maxv Exp $ */ 2 3 /*- 4 * Copyright (c) 2007 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/param.h> 33 #include <machine/asm.h> 34 /* 35 * __HAVE_ constants should not be in <machine/types.h> 36 * because we can't use them from assembly. OTOH we 37 * only need __HAVE_ATOMIC64_OPS here, and we don't. 38 */ 39 #ifdef _KERNEL 40 #define ALIAS(f, t) STRONG_ALIAS(f,t) 41 #else 42 #define ALIAS(f, t) WEAK_ALIAS(f,t) 43 #endif 44 45 #ifdef _HARDKERNEL 46 #include "opt_xen.h" 47 #include <machine/frameasm.h> 48 #define LOCK(n) HOTPATCH(HP_NAME_NOLOCK, 1); lock 49 #define ENDLABEL(a) _ALIGN_TEXT; LABEL(a) 50 #else 51 #define LOCK(n) lock 52 #define ENDLABEL(a) /* nothing */ 53 #endif 54 55 .text 56 57 ENTRY(_atomic_add_32) 58 movl 4(%esp), %edx 59 movl 8(%esp), %eax 60 LOCK(1) 61 addl %eax, (%edx) 62 ret 63 END(_atomic_add_32) 64 65 ENTRY(_atomic_add_32_nv) 66 movl 4(%esp), %edx 67 movl 8(%esp), %eax 68 movl %eax, %ecx 69 LOCK(2) 70 xaddl %eax, (%edx) 71 addl %ecx, %eax 72 ret 73 END(_atomic_add_32_nv) 74 75 ENTRY(_atomic_and_32) 76 movl 4(%esp), %edx 77 movl 8(%esp), %eax 78 LOCK(3) 79 andl %eax, (%edx) 80 ret 81 END(_atomic_and_32) 82 83 ENTRY(_atomic_and_32_nv) 84 movl 4(%esp), %edx 85 movl (%edx), %eax 86 0: 87 movl %eax, %ecx 88 andl 8(%esp), %ecx 89 LOCK(4) 90 cmpxchgl %ecx, (%edx) 91 jnz 1f 92 movl %ecx, %eax 93 ret 94 1: 95 jmp 0b 96 END(_atomic_and_32_nv) 97 98 ENTRY(_atomic_dec_32) 99 movl 4(%esp), %edx 100 LOCK(5) 101 decl (%edx) 102 ret 103 END(_atomic_dec_32) 104 105 ENTRY(_atomic_dec_32_nv) 106 movl 4(%esp), %edx 107 movl $-1, %eax 108 LOCK(6) 109 xaddl %eax, (%edx) 110 decl %eax 111 ret 112 END(_atomic_dec_32_nv) 113 114 ENTRY(_atomic_inc_32) 115 movl 4(%esp), %edx 116 LOCK(7) 117 incl (%edx) 118 ret 119 END(_atomic_inc_32) 120 121 ENTRY(_atomic_inc_32_nv) 122 movl 4(%esp), %edx 123 movl $1, %eax 124 LOCK(8) 125 xaddl %eax, (%edx) 126 incl %eax 127 ret 128 END(_atomic_inc_32_nv) 129 130 ENTRY(_atomic_or_32) 131 movl 4(%esp), %edx 132 movl 8(%esp), %eax 133 LOCK(9) 134 orl %eax, (%edx) 135 ret 136 END(_atomic_or_32) 137 138 ENTRY(_atomic_or_32_nv) 139 movl 4(%esp), %edx 140 movl (%edx), %eax 141 0: 142 movl %eax, %ecx 143 orl 8(%esp), %ecx 144 LOCK(10) 145 cmpxchgl %ecx, (%edx) 146 jnz 1f 147 movl %ecx, %eax 148 ret 149 1: 150 jmp 0b 151 END(_atomic_or_32_nv) 152 153 ENTRY(_atomic_swap_32) 154 movl 4(%esp), %edx 155 movl 8(%esp), %eax 156 xchgl %eax, (%edx) 157 ret 158 END(_atomic_swap_32) 159 160 ENTRY(_atomic_cas_32) 161 movl 4(%esp), %edx 162 movl 8(%esp), %eax 163 movl 12(%esp), %ecx 164 LOCK(12) 165 cmpxchgl %ecx, (%edx) 166 /* %eax now contains the old value */ 167 ret 168 END(_atomic_cas_32) 169 170 ENTRY(_atomic_cas_32_ni) 171 movl 4(%esp), %edx 172 movl 8(%esp), %eax 173 movl 12(%esp), %ecx 174 cmpxchgl %ecx, (%edx) 175 /* %eax now contains the old value */ 176 ret 177 END(_atomic_cas_32_ni) 178 179 ENTRY(_membar_consumer) 180 LOCK(13) 181 addl $0, -4(%esp) 182 ret 183 END(_membar_consumer) 184 ENDLABEL(membar_consumer_end) 185 186 ENTRY(_membar_producer) 187 /* A store is enough */ 188 movl $0, -4(%esp) 189 ret 190 END(_membar_producer) 191 192 ENTRY(_membar_sync) 193 LOCK(14) 194 addl $0, -4(%esp) 195 ret 196 END(_membar_sync) 197 ENDLABEL(membar_sync_end) 198 199 #if defined(__HAVE_ATOMIC64_OPS) || defined(_KERNEL) 200 #ifdef XENPV 201 STRONG_ALIAS(_atomic_cas_64,_atomic_cas_cx8) 202 #else 203 ENTRY(_atomic_cas_64) 204 #ifdef _HARDKERNEL 205 pushf 206 cli 207 #endif /* _HARDKERNEL */ 208 pushl %edi 209 pushl %ebx 210 movl 12(%esp), %edi 211 movl 16(%esp), %eax 212 movl 20(%esp), %edx 213 movl 24(%esp), %ebx 214 movl 28(%esp), %ecx 215 cmpl 0(%edi), %eax 216 jne 2f 217 cmpl 4(%edi), %edx 218 jne 2f 219 movl %ebx, 0(%edi) 220 movl %ecx, 4(%edi) 221 1: 222 popl %ebx 223 popl %edi 224 #ifdef _HARDKERNEL 225 popf 226 #endif /* _HARDKERNEL */ 227 ret 228 2: 229 movl 0(%edi), %eax 230 movl 4(%edi), %edx 231 jmp 1b 232 END(_atomic_cas_64) 233 ENDLABEL(_atomic_cas_64_end) 234 #endif /* !XEN */ 235 236 ENTRY(_atomic_cas_cx8) 237 pushl %edi 238 pushl %ebx 239 movl 12(%esp), %edi 240 movl 16(%esp), %eax 241 movl 20(%esp), %edx 242 movl 24(%esp), %ebx 243 movl 28(%esp), %ecx 244 LOCK(15) 245 cmpxchg8b (%edi) 246 popl %ebx 247 popl %edi 248 ret 249 #ifdef _HARDKERNEL 250 #ifdef GPROF 251 .space 16, 0x90 252 #else 253 .space 32, 0x90 254 #endif 255 #endif /* _HARDKERNEL */ 256 END(_atomic_cas_cx8) 257 ENDLABEL(_atomic_cas_cx8_end) 258 #endif /* __HAVE_ATOMIC64_OPS || _KERNEL */ 259 260 #ifdef _HARDKERNEL 261 ENTRY(sse2_lfence) 262 lfence 263 ret 264 END(sse2_lfence) 265 ENDLABEL(sse2_lfence_end) 266 267 ENTRY(sse2_mfence) 268 mfence 269 ret 270 END(sse2_mfence) 271 ENDLABEL(sse2_mfence_end) 272 #endif /* _HARDKERNEL */ 273 274 ALIAS(atomic_add_32,_atomic_add_32) 275 ALIAS(atomic_add_int,_atomic_add_32) 276 ALIAS(atomic_add_long,_atomic_add_32) 277 ALIAS(atomic_add_ptr,_atomic_add_32) 278 279 ALIAS(atomic_add_32_nv,_atomic_add_32_nv) 280 ALIAS(atomic_add_int_nv,_atomic_add_32_nv) 281 ALIAS(atomic_add_long_nv,_atomic_add_32_nv) 282 ALIAS(atomic_add_ptr_nv,_atomic_add_32_nv) 283 284 ALIAS(atomic_and_32,_atomic_and_32) 285 ALIAS(atomic_and_uint,_atomic_and_32) 286 ALIAS(atomic_and_ulong,_atomic_and_32) 287 ALIAS(atomic_and_ptr,_atomic_and_32) 288 289 ALIAS(atomic_and_32_nv,_atomic_and_32_nv) 290 ALIAS(atomic_and_uint_nv,_atomic_and_32_nv) 291 ALIAS(atomic_and_ulong_nv,_atomic_and_32_nv) 292 ALIAS(atomic_and_ptr_nv,_atomic_and_32_nv) 293 294 ALIAS(atomic_dec_32,_atomic_dec_32) 295 ALIAS(atomic_dec_uint,_atomic_dec_32) 296 ALIAS(atomic_dec_ulong,_atomic_dec_32) 297 ALIAS(atomic_dec_ptr,_atomic_dec_32) 298 299 ALIAS(atomic_dec_32_nv,_atomic_dec_32_nv) 300 ALIAS(atomic_dec_uint_nv,_atomic_dec_32_nv) 301 ALIAS(atomic_dec_ulong_nv,_atomic_dec_32_nv) 302 ALIAS(atomic_dec_ptr_nv,_atomic_dec_32_nv) 303 304 ALIAS(atomic_inc_32,_atomic_inc_32) 305 ALIAS(atomic_inc_uint,_atomic_inc_32) 306 ALIAS(atomic_inc_ulong,_atomic_inc_32) 307 ALIAS(atomic_inc_ptr,_atomic_inc_32) 308 309 ALIAS(atomic_inc_32_nv,_atomic_inc_32_nv) 310 ALIAS(atomic_inc_uint_nv,_atomic_inc_32_nv) 311 ALIAS(atomic_inc_ulong_nv,_atomic_inc_32_nv) 312 ALIAS(atomic_inc_ptr_nv,_atomic_inc_32_nv) 313 314 ALIAS(atomic_or_32,_atomic_or_32) 315 ALIAS(atomic_or_uint,_atomic_or_32) 316 ALIAS(atomic_or_ulong,_atomic_or_32) 317 ALIAS(atomic_or_ptr,_atomic_or_32) 318 319 ALIAS(atomic_or_32_nv,_atomic_or_32_nv) 320 ALIAS(atomic_or_uint_nv,_atomic_or_32_nv) 321 ALIAS(atomic_or_ulong_nv,_atomic_or_32_nv) 322 ALIAS(atomic_or_ptr_nv,_atomic_or_32_nv) 323 324 ALIAS(atomic_swap_32,_atomic_swap_32) 325 ALIAS(atomic_swap_uint,_atomic_swap_32) 326 ALIAS(atomic_swap_ulong,_atomic_swap_32) 327 ALIAS(atomic_swap_ptr,_atomic_swap_32) 328 329 ALIAS(atomic_cas_32,_atomic_cas_32) 330 ALIAS(atomic_cas_uint,_atomic_cas_32) 331 ALIAS(atomic_cas_ulong,_atomic_cas_32) 332 ALIAS(atomic_cas_ptr,_atomic_cas_32) 333 334 ALIAS(atomic_cas_32_ni,_atomic_cas_32_ni) 335 ALIAS(atomic_cas_uint_ni,_atomic_cas_32_ni) 336 ALIAS(atomic_cas_ulong_ni,_atomic_cas_32_ni) 337 ALIAS(atomic_cas_ptr_ni,_atomic_cas_32_ni) 338 339 #if defined(__HAVE_ATOMIC64_OPS) || defined(_KERNEL) 340 ALIAS(atomic_cas_64,_atomic_cas_64) 341 ALIAS(atomic_cas_64_ni,_atomic_cas_64) 342 ALIAS(__sync_val_compare_and_swap_8,_atomic_cas_64) 343 #endif /* __HAVE_ATOMIC64_OPS || _KERNEL */ 344 345 ALIAS(membar_consumer,_membar_consumer) 346 ALIAS(membar_producer,_membar_producer) 347 ALIAS(membar_enter,_membar_consumer) 348 ALIAS(membar_exit,_membar_producer) 349 ALIAS(membar_sync,_membar_sync) 350 351 STRONG_ALIAS(_atomic_add_int,_atomic_add_32) 352 STRONG_ALIAS(_atomic_add_long,_atomic_add_32) 353 STRONG_ALIAS(_atomic_add_ptr,_atomic_add_32) 354 355 STRONG_ALIAS(_atomic_add_int_nv,_atomic_add_32_nv) 356 STRONG_ALIAS(_atomic_add_long_nv,_atomic_add_32_nv) 357 STRONG_ALIAS(_atomic_add_ptr_nv,_atomic_add_32_nv) 358 359 STRONG_ALIAS(_atomic_and_uint,_atomic_and_32) 360 STRONG_ALIAS(_atomic_and_ulong,_atomic_and_32) 361 STRONG_ALIAS(_atomic_and_ptr,_atomic_and_32) 362 363 STRONG_ALIAS(_atomic_and_uint_nv,_atomic_and_32_nv) 364 STRONG_ALIAS(_atomic_and_ulong_nv,_atomic_and_32_nv) 365 STRONG_ALIAS(_atomic_and_ptr_nv,_atomic_and_32_nv) 366 367 STRONG_ALIAS(_atomic_dec_uint,_atomic_dec_32) 368 STRONG_ALIAS(_atomic_dec_ulong,_atomic_dec_32) 369 STRONG_ALIAS(_atomic_dec_ptr,_atomic_dec_32) 370 371 STRONG_ALIAS(_atomic_dec_uint_nv,_atomic_dec_32_nv) 372 STRONG_ALIAS(_atomic_dec_ulong_nv,_atomic_dec_32_nv) 373 STRONG_ALIAS(_atomic_dec_ptr_nv,_atomic_dec_32_nv) 374 375 STRONG_ALIAS(_atomic_inc_uint,_atomic_inc_32) 376 STRONG_ALIAS(_atomic_inc_ulong,_atomic_inc_32) 377 STRONG_ALIAS(_atomic_inc_ptr,_atomic_inc_32) 378 379 STRONG_ALIAS(_atomic_inc_uint_nv,_atomic_inc_32_nv) 380 STRONG_ALIAS(_atomic_inc_ulong_nv,_atomic_inc_32_nv) 381 STRONG_ALIAS(_atomic_inc_ptr_nv,_atomic_inc_32_nv) 382 383 STRONG_ALIAS(_atomic_or_uint,_atomic_or_32) 384 STRONG_ALIAS(_atomic_or_ulong,_atomic_or_32) 385 STRONG_ALIAS(_atomic_or_ptr,_atomic_or_32) 386 387 STRONG_ALIAS(_atomic_or_uint_nv,_atomic_or_32_nv) 388 STRONG_ALIAS(_atomic_or_ulong_nv,_atomic_or_32_nv) 389 STRONG_ALIAS(_atomic_or_ptr_nv,_atomic_or_32_nv) 390 391 STRONG_ALIAS(_atomic_swap_uint,_atomic_swap_32) 392 STRONG_ALIAS(_atomic_swap_ulong,_atomic_swap_32) 393 STRONG_ALIAS(_atomic_swap_ptr,_atomic_swap_32) 394 395 STRONG_ALIAS(_atomic_cas_uint,_atomic_cas_32) 396 STRONG_ALIAS(_atomic_cas_ulong,_atomic_cas_32) 397 STRONG_ALIAS(_atomic_cas_ptr,_atomic_cas_32) 398 399 STRONG_ALIAS(_atomic_cas_uint_ni,_atomic_cas_32_ni) 400 STRONG_ALIAS(_atomic_cas_ulong_ni,_atomic_cas_32_ni) 401 STRONG_ALIAS(_atomic_cas_ptr_ni,_atomic_cas_32_ni) 402 403 STRONG_ALIAS(_membar_enter,_membar_consumer) 404 STRONG_ALIAS(_membar_exit,_membar_producer) 405
Indexes created Fri Oct 03 02:09:57 GMT 2025