atomic.S revision 1.28
1 /* $NetBSD: atomic.S,v 1.28 2020/04/26 14:49:17 maxv Exp $ */ 2 3 /*- 4 * Copyright (c) 2007 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/param.h> 33 #include <machine/asm.h> 34 /* 35 * __HAVE_ constants should not be in <machine/types.h> 36 * because we can't use them from assembly. OTOH we 37 * only need __HAVE_ATOMIC64_OPS here, and we don't. 38 */ 39 #ifdef _KERNEL 40 #define ALIAS(f, t) STRONG_ALIAS(f,t) 41 #else 42 #define ALIAS(f, t) WEAK_ALIAS(f,t) 43 #endif 44 45 #ifdef _HARDKERNEL 46 #include "opt_xen.h" 47 #include <machine/frameasm.h> 48 #define LOCK HOTPATCH(HP_NAME_NOLOCK, 1); lock 49 #define HOTPATCH_SSE2_LFENCE HOTPATCH(HP_NAME_SSE2_LFENCE, 7); 50 #define HOTPATCH_SSE2_MFENCE HOTPATCH(HP_NAME_SSE2_MFENCE, 7); 51 #define ENDLABEL(a) _ALIGN_TEXT; LABEL(a) 52 #else 53 #define LOCK lock 54 #define HOTPATCH_SSE2_LFENCE /* nothing */ 55 #define HOTPATCH_SSE2_MFENCE /* nothing */ 56 #define ENDLABEL(a) /* nothing */ 57 #endif 58 59 .text 60 61 ENTRY(_atomic_add_32) 62 movl 4(%esp), %edx 63 movl 8(%esp), %eax 64 LOCK 65 addl %eax, (%edx) 66 ret 67 END(_atomic_add_32) 68 69 ENTRY(_atomic_add_32_nv) 70 movl 4(%esp), %edx 71 movl 8(%esp), %eax 72 movl %eax, %ecx 73 LOCK 74 xaddl %eax, (%edx) 75 addl %ecx, %eax 76 ret 77 END(_atomic_add_32_nv) 78 79 ENTRY(_atomic_and_32) 80 movl 4(%esp), %edx 81 movl 8(%esp), %eax 82 LOCK 83 andl %eax, (%edx) 84 ret 85 END(_atomic_and_32) 86 87 ENTRY(_atomic_and_32_nv) 88 movl 4(%esp), %edx 89 movl (%edx), %eax 90 0: 91 movl %eax, %ecx 92 andl 8(%esp), %ecx 93 LOCK 94 cmpxchgl %ecx, (%edx) 95 jnz 1f 96 movl %ecx, %eax 97 ret 98 1: 99 jmp 0b 100 END(_atomic_and_32_nv) 101 102 ENTRY(_atomic_dec_32) 103 movl 4(%esp), %edx 104 LOCK 105 decl (%edx) 106 ret 107 END(_atomic_dec_32) 108 109 ENTRY(_atomic_dec_32_nv) 110 movl 4(%esp), %edx 111 movl $-1, %eax 112 LOCK 113 xaddl %eax, (%edx) 114 decl %eax 115 ret 116 END(_atomic_dec_32_nv) 117 118 ENTRY(_atomic_inc_32) 119 movl 4(%esp), %edx 120 LOCK 121 incl (%edx) 122 ret 123 END(_atomic_inc_32) 124 125 ENTRY(_atomic_inc_32_nv) 126 movl 4(%esp), %edx 127 movl $1, %eax 128 LOCK 129 xaddl %eax, (%edx) 130 incl %eax 131 ret 132 END(_atomic_inc_32_nv) 133 134 ENTRY(_atomic_or_32) 135 movl 4(%esp), %edx 136 movl 8(%esp), %eax 137 LOCK 138 orl %eax, (%edx) 139 ret 140 END(_atomic_or_32) 141 142 ENTRY(_atomic_or_32_nv) 143 movl 4(%esp), %edx 144 movl (%edx), %eax 145 0: 146 movl %eax, %ecx 147 orl 8(%esp), %ecx 148 LOCK 149 cmpxchgl %ecx, (%edx) 150 jnz 1f 151 movl %ecx, %eax 152 ret 153 1: 154 jmp 0b 155 END(_atomic_or_32_nv) 156 157 ENTRY(_atomic_swap_32) 158 movl 4(%esp), %edx 159 movl 8(%esp), %eax 160 xchgl %eax, (%edx) 161 ret 162 END(_atomic_swap_32) 163 164 ENTRY(_atomic_cas_32) 165 movl 4(%esp), %edx 166 movl 8(%esp), %eax 167 movl 12(%esp), %ecx 168 LOCK 169 cmpxchgl %ecx, (%edx) 170 /* %eax now contains the old value */ 171 ret 172 END(_atomic_cas_32) 173 174 ENTRY(_atomic_cas_32_ni) 175 movl 4(%esp), %edx 176 movl 8(%esp), %eax 177 movl 12(%esp), %ecx 178 cmpxchgl %ecx, (%edx) 179 /* %eax now contains the old value */ 180 ret 181 END(_atomic_cas_32_ni) 182 183 ENTRY(_membar_consumer) 184 HOTPATCH_SSE2_LFENCE 185 /* 7 bytes of instructions */ 186 LOCK 187 addl $0, -4(%esp) 188 ret 189 END(_membar_consumer) 190 191 ENTRY(_membar_producer) 192 /* A store is enough */ 193 movl $0, -4(%esp) 194 ret 195 END(_membar_producer) 196 197 ENTRY(_membar_sync) 198 HOTPATCH_SSE2_MFENCE 199 /* 7 bytes of instructions */ 200 LOCK 201 addl $0, -4(%esp) 202 ret 203 END(_membar_sync) 204 205 #if defined(__HAVE_ATOMIC64_OPS) || defined(_KERNEL) 206 #ifdef XENPV 207 STRONG_ALIAS(_atomic_cas_64,_atomic_cas_cx8) 208 #else 209 ENTRY(_atomic_cas_64) 210 #ifdef _HARDKERNEL 211 pushf 212 cli 213 #endif /* _HARDKERNEL */ 214 pushl %edi 215 pushl %ebx 216 movl 12(%esp), %edi 217 movl 16(%esp), %eax 218 movl 20(%esp), %edx 219 movl 24(%esp), %ebx 220 movl 28(%esp), %ecx 221 cmpl 0(%edi), %eax 222 jne 2f 223 cmpl 4(%edi), %edx 224 jne 2f 225 movl %ebx, 0(%edi) 226 movl %ecx, 4(%edi) 227 1: 228 popl %ebx 229 popl %edi 230 #ifdef _HARDKERNEL 231 popf 232 #endif /* _HARDKERNEL */ 233 ret 234 2: 235 movl 0(%edi), %eax 236 movl 4(%edi), %edx 237 jmp 1b 238 END(_atomic_cas_64) 239 ENDLABEL(_atomic_cas_64_end) 240 #endif /* !XEN */ 241 242 ENTRY(_atomic_cas_cx8) 243 pushl %edi 244 pushl %ebx 245 movl 12(%esp), %edi 246 movl 16(%esp), %eax 247 movl 20(%esp), %edx 248 movl 24(%esp), %ebx 249 movl 28(%esp), %ecx 250 LOCK 251 cmpxchg8b (%edi) 252 popl %ebx 253 popl %edi 254 ret 255 #ifdef _HARDKERNEL 256 #ifdef GPROF 257 .space 16, 0x90 258 #else 259 .space 32, 0x90 260 #endif 261 #endif /* _HARDKERNEL */ 262 END(_atomic_cas_cx8) 263 ENDLABEL(_atomic_cas_cx8_end) 264 #endif /* __HAVE_ATOMIC64_OPS || _KERNEL */ 265 266 ALIAS(atomic_add_32,_atomic_add_32) 267 ALIAS(atomic_add_int,_atomic_add_32) 268 ALIAS(atomic_add_long,_atomic_add_32) 269 ALIAS(atomic_add_ptr,_atomic_add_32) 270 271 ALIAS(atomic_add_32_nv,_atomic_add_32_nv) 272 ALIAS(atomic_add_int_nv,_atomic_add_32_nv) 273 ALIAS(atomic_add_long_nv,_atomic_add_32_nv) 274 ALIAS(atomic_add_ptr_nv,_atomic_add_32_nv) 275 276 ALIAS(atomic_and_32,_atomic_and_32) 277 ALIAS(atomic_and_uint,_atomic_and_32) 278 ALIAS(atomic_and_ulong,_atomic_and_32) 279 ALIAS(atomic_and_ptr,_atomic_and_32) 280 281 ALIAS(atomic_and_32_nv,_atomic_and_32_nv) 282 ALIAS(atomic_and_uint_nv,_atomic_and_32_nv) 283 ALIAS(atomic_and_ulong_nv,_atomic_and_32_nv) 284 ALIAS(atomic_and_ptr_nv,_atomic_and_32_nv) 285 286 ALIAS(atomic_dec_32,_atomic_dec_32) 287 ALIAS(atomic_dec_uint,_atomic_dec_32) 288 ALIAS(atomic_dec_ulong,_atomic_dec_32) 289 ALIAS(atomic_dec_ptr,_atomic_dec_32) 290 291 ALIAS(atomic_dec_32_nv,_atomic_dec_32_nv) 292 ALIAS(atomic_dec_uint_nv,_atomic_dec_32_nv) 293 ALIAS(atomic_dec_ulong_nv,_atomic_dec_32_nv) 294 ALIAS(atomic_dec_ptr_nv,_atomic_dec_32_nv) 295 296 ALIAS(atomic_inc_32,_atomic_inc_32) 297 ALIAS(atomic_inc_uint,_atomic_inc_32) 298 ALIAS(atomic_inc_ulong,_atomic_inc_32) 299 ALIAS(atomic_inc_ptr,_atomic_inc_32) 300 301 ALIAS(atomic_inc_32_nv,_atomic_inc_32_nv) 302 ALIAS(atomic_inc_uint_nv,_atomic_inc_32_nv) 303 ALIAS(atomic_inc_ulong_nv,_atomic_inc_32_nv) 304 ALIAS(atomic_inc_ptr_nv,_atomic_inc_32_nv) 305 306 ALIAS(atomic_or_32,_atomic_or_32) 307 ALIAS(atomic_or_uint,_atomic_or_32) 308 ALIAS(atomic_or_ulong,_atomic_or_32) 309 ALIAS(atomic_or_ptr,_atomic_or_32) 310 311 ALIAS(atomic_or_32_nv,_atomic_or_32_nv) 312 ALIAS(atomic_or_uint_nv,_atomic_or_32_nv) 313 ALIAS(atomic_or_ulong_nv,_atomic_or_32_nv) 314 ALIAS(atomic_or_ptr_nv,_atomic_or_32_nv) 315 316 ALIAS(atomic_swap_32,_atomic_swap_32) 317 ALIAS(atomic_swap_uint,_atomic_swap_32) 318 ALIAS(atomic_swap_ulong,_atomic_swap_32) 319 ALIAS(atomic_swap_ptr,_atomic_swap_32) 320 321 ALIAS(atomic_cas_32,_atomic_cas_32) 322 ALIAS(atomic_cas_uint,_atomic_cas_32) 323 ALIAS(atomic_cas_ulong,_atomic_cas_32) 324 ALIAS(atomic_cas_ptr,_atomic_cas_32) 325 326 ALIAS(atomic_cas_32_ni,_atomic_cas_32_ni) 327 ALIAS(atomic_cas_uint_ni,_atomic_cas_32_ni) 328 ALIAS(atomic_cas_ulong_ni,_atomic_cas_32_ni) 329 ALIAS(atomic_cas_ptr_ni,_atomic_cas_32_ni) 330 331 #if defined(__HAVE_ATOMIC64_OPS) || defined(_KERNEL) 332 ALIAS(atomic_cas_64,_atomic_cas_64) 333 ALIAS(atomic_cas_64_ni,_atomic_cas_64) 334 ALIAS(__sync_val_compare_and_swap_8,_atomic_cas_64) 335 #endif /* __HAVE_ATOMIC64_OPS || _KERNEL */ 336 337 ALIAS(membar_consumer,_membar_consumer) 338 ALIAS(membar_producer,_membar_producer) 339 ALIAS(membar_enter,_membar_consumer) 340 ALIAS(membar_exit,_membar_producer) 341 ALIAS(membar_sync,_membar_sync) 342 343 STRONG_ALIAS(_atomic_add_int,_atomic_add_32) 344 STRONG_ALIAS(_atomic_add_long,_atomic_add_32) 345 STRONG_ALIAS(_atomic_add_ptr,_atomic_add_32) 346 347 STRONG_ALIAS(_atomic_add_int_nv,_atomic_add_32_nv) 348 STRONG_ALIAS(_atomic_add_long_nv,_atomic_add_32_nv) 349 STRONG_ALIAS(_atomic_add_ptr_nv,_atomic_add_32_nv) 350 351 STRONG_ALIAS(_atomic_and_uint,_atomic_and_32) 352 STRONG_ALIAS(_atomic_and_ulong,_atomic_and_32) 353 STRONG_ALIAS(_atomic_and_ptr,_atomic_and_32) 354 355 STRONG_ALIAS(_atomic_and_uint_nv,_atomic_and_32_nv) 356 STRONG_ALIAS(_atomic_and_ulong_nv,_atomic_and_32_nv) 357 STRONG_ALIAS(_atomic_and_ptr_nv,_atomic_and_32_nv) 358 359 STRONG_ALIAS(_atomic_dec_uint,_atomic_dec_32) 360 STRONG_ALIAS(_atomic_dec_ulong,_atomic_dec_32) 361 STRONG_ALIAS(_atomic_dec_ptr,_atomic_dec_32) 362 363 STRONG_ALIAS(_atomic_dec_uint_nv,_atomic_dec_32_nv) 364 STRONG_ALIAS(_atomic_dec_ulong_nv,_atomic_dec_32_nv) 365 STRONG_ALIAS(_atomic_dec_ptr_nv,_atomic_dec_32_nv) 366 367 STRONG_ALIAS(_atomic_inc_uint,_atomic_inc_32) 368 STRONG_ALIAS(_atomic_inc_ulong,_atomic_inc_32) 369 STRONG_ALIAS(_atomic_inc_ptr,_atomic_inc_32) 370 371 STRONG_ALIAS(_atomic_inc_uint_nv,_atomic_inc_32_nv) 372 STRONG_ALIAS(_atomic_inc_ulong_nv,_atomic_inc_32_nv) 373 STRONG_ALIAS(_atomic_inc_ptr_nv,_atomic_inc_32_nv) 374 375 STRONG_ALIAS(_atomic_or_uint,_atomic_or_32) 376 STRONG_ALIAS(_atomic_or_ulong,_atomic_or_32) 377 STRONG_ALIAS(_atomic_or_ptr,_atomic_or_32) 378 379 STRONG_ALIAS(_atomic_or_uint_nv,_atomic_or_32_nv) 380 STRONG_ALIAS(_atomic_or_ulong_nv,_atomic_or_32_nv) 381 STRONG_ALIAS(_atomic_or_ptr_nv,_atomic_or_32_nv) 382 383 STRONG_ALIAS(_atomic_swap_uint,_atomic_swap_32) 384 STRONG_ALIAS(_atomic_swap_ulong,_atomic_swap_32) 385 STRONG_ALIAS(_atomic_swap_ptr,_atomic_swap_32) 386 387 STRONG_ALIAS(_atomic_cas_uint,_atomic_cas_32) 388 STRONG_ALIAS(_atomic_cas_ulong,_atomic_cas_32) 389 STRONG_ALIAS(_atomic_cas_ptr,_atomic_cas_32) 390 391 STRONG_ALIAS(_atomic_cas_uint_ni,_atomic_cas_32_ni) 392 STRONG_ALIAS(_atomic_cas_ulong_ni,_atomic_cas_32_ni) 393 STRONG_ALIAS(_atomic_cas_ptr_ni,_atomic_cas_32_ni) 394 395 STRONG_ALIAS(_membar_enter,_membar_consumer) 396 STRONG_ALIAS(_membar_exit,_membar_producer) 397 398 #ifdef _HARDKERNEL 399 .section .rodata 400 401 LABEL(sse2_lfence) 402 lfence 403 ret 404 nop; nop; nop; 405 LABEL(sse2_lfence_end) 406 407 LABEL(sse2_mfence) 408 mfence 409 ret 410 nop; nop; nop; 411 LABEL(sse2_mfence_end) 412 #endif /* _HARDKERNEL */ 413
Indexes created Wed Oct 01 07:09:59 GMT 2025