atomic.S revision 1.21
1 /* $NetBSD: atomic.S,v 1.21 2020/04/26 13:59:44 maxv Exp $ */ 2 3 /*- 4 * Copyright (c) 2007 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/param.h> 33 #include <machine/asm.h> 34 35 #ifdef _KERNEL 36 #define ALIAS(f, t) STRONG_ALIAS(f,t) 37 #else 38 #define ALIAS(f, t) WEAK_ALIAS(f,t) 39 #endif 40 41 #ifdef _HARDKERNEL 42 #include <machine/frameasm.h> 43 #define LOCK HOTPATCH(HP_NAME_NOLOCK, 1); lock 44 #define ENDLABEL(a) _ALIGN_TEXT; LABEL(a) 45 #else 46 #define LOCK lock 47 #define ENDLABEL(a) /* nothing */ 48 #endif 49 50 .text 51 52 /* 32-bit */ 53 54 ENTRY(_atomic_add_32) 55 LOCK 56 addl %esi, (%rdi) 57 ret 58 END(_atomic_add_32) 59 60 ENTRY(_atomic_add_32_nv) 61 movl %esi, %eax 62 LOCK 63 xaddl %eax, (%rdi) 64 addl %esi, %eax 65 ret 66 END(_atomic_add_32_nv) 67 68 ENTRY(_atomic_and_32) 69 LOCK 70 andl %esi, (%rdi) 71 ret 72 END(_atomic_and_32) 73 74 ENTRY(_atomic_and_32_nv) 75 movl (%rdi), %eax 76 1: 77 movl %eax, %ecx 78 andl %esi, %ecx 79 LOCK 80 cmpxchgl %ecx, (%rdi) 81 jnz 1b 82 movl %ecx, %eax 83 ret 84 END(_atomic_and_32_nv) 85 86 ENTRY(_atomic_dec_32) 87 LOCK 88 decl (%rdi) 89 ret 90 END(_atomic_dec_32) 91 92 ENTRY(_atomic_dec_32_nv) 93 movl $-1, %eax 94 LOCK 95 xaddl %eax, (%rdi) 96 decl %eax 97 ret 98 END(_atomic_dec_32_nv) 99 100 ENTRY(_atomic_inc_32) 101 LOCK 102 incl (%rdi) 103 ret 104 END(_atomic_inc_32) 105 106 ENTRY(_atomic_inc_32_nv) 107 movl $1, %eax 108 LOCK 109 xaddl %eax, (%rdi) 110 incl %eax 111 ret 112 END(_atomic_inc_32_nv) 113 114 ENTRY(_atomic_or_32) 115 LOCK 116 orl %esi, (%rdi) 117 ret 118 END(_atomic_or_32) 119 120 ENTRY(_atomic_or_32_nv) 121 movl (%rdi), %eax 122 1: 123 movl %eax, %ecx 124 orl %esi, %ecx 125 LOCK 126 cmpxchgl %ecx, (%rdi) 127 jnz 1b 128 movl %ecx, %eax 129 ret 130 END(_atomic_or_32_nv) 131 132 ENTRY(_atomic_swap_32) 133 movl %esi, %eax 134 xchgl %eax, (%rdi) 135 ret 136 END(_atomic_swap_32) 137 138 ENTRY(_atomic_cas_32) 139 movl %esi, %eax 140 LOCK 141 cmpxchgl %edx, (%rdi) 142 /* %eax now contains the old value */ 143 ret 144 END(_atomic_cas_32) 145 146 ENTRY(_atomic_cas_32_ni) 147 movl %esi, %eax 148 cmpxchgl %edx, (%rdi) 149 /* %eax now contains the old value */ 150 ret 151 END(_atomic_cas_32_ni) 152 153 /* 64-bit */ 154 155 ENTRY(_atomic_add_64) 156 LOCK 157 addq %rsi, (%rdi) 158 ret 159 END(_atomic_add_64) 160 161 ENTRY(_atomic_add_64_nv) 162 movq %rsi, %rax 163 LOCK 164 xaddq %rax, (%rdi) 165 addq %rsi, %rax 166 ret 167 END(_atomic_add_64_nv) 168 169 ENTRY(_atomic_and_64) 170 LOCK 171 andq %rsi, (%rdi) 172 ret 173 END(_atomic_and_64) 174 175 ENTRY(_atomic_and_64_nv) 176 movq (%rdi), %rax 177 1: 178 movq %rax, %rcx 179 andq %rsi, %rcx 180 LOCK 181 cmpxchgq %rcx, (%rdi) 182 jnz 1b 183 movq %rcx, %rax 184 ret 185 END(_atomic_and_64_nv) 186 187 ENTRY(_atomic_dec_64) 188 LOCK 189 decq (%rdi) 190 ret 191 END(_atomic_dec_64) 192 193 ENTRY(_atomic_dec_64_nv) 194 movq $-1, %rax 195 LOCK 196 xaddq %rax, (%rdi) 197 decq %rax 198 ret 199 END(_atomic_dec_64_nv) 200 201 ENTRY(_atomic_inc_64) 202 LOCK 203 incq (%rdi) 204 ret 205 END(_atomic_inc_64) 206 207 ENTRY(_atomic_inc_64_nv) 208 movq $1, %rax 209 LOCK 210 xaddq %rax, (%rdi) 211 incq %rax 212 ret 213 END(_atomic_inc_64_nv) 214 215 ENTRY(_atomic_or_64) 216 LOCK 217 orq %rsi, (%rdi) 218 ret 219 END(_atomic_or_64) 220 221 ENTRY(_atomic_or_64_nv) 222 movq (%rdi), %rax 223 1: 224 movq %rax, %rcx 225 orq %rsi, %rcx 226 LOCK 227 cmpxchgq %rcx, (%rdi) 228 jnz 1b 229 movq %rcx, %rax 230 ret 231 END(_atomic_or_64_nv) 232 233 ENTRY(_atomic_swap_64) 234 movq %rsi, %rax 235 xchgq %rax, (%rdi) 236 ret 237 END(_atomic_swap_64) 238 239 ENTRY(_atomic_cas_64) 240 movq %rsi, %rax 241 LOCK 242 cmpxchgq %rdx, (%rdi) 243 /* %eax now contains the old value */ 244 ret 245 END(_atomic_cas_64) 246 247 ENTRY(_atomic_cas_64_ni) 248 movq %rsi, %rax 249 cmpxchgq %rdx, (%rdi) 250 /* %eax now contains the old value */ 251 ret 252 END(_atomic_cas_64_ni) 253 254 /* memory barriers */ 255 256 ENTRY(_membar_consumer) 257 LOCK 258 addq $0, -8(%rsp) 259 ret 260 END(_membar_consumer) 261 ENDLABEL(membar_consumer_end) 262 263 ENTRY(_membar_producer) 264 /* A store is enough */ 265 movq $0, -8(%rsp) 266 ret 267 END(_membar_producer) 268 269 ENTRY(_membar_sync) 270 LOCK 271 addq $0, -8(%rsp) 272 ret 273 END(_membar_sync) 274 ENDLABEL(membar_sync_end) 275 276 #ifdef _HARDKERNEL 277 ENTRY(sse2_lfence) 278 lfence 279 ret 280 END(sse2_lfence) 281 ENDLABEL(sse2_lfence_end) 282 283 ENTRY(sse2_mfence) 284 mfence 285 ret 286 END(sse2_mfence) 287 ENDLABEL(sse2_mfence_end) 288 #endif /* _HARDKERNEL */ 289 290 ALIAS(atomic_add_32,_atomic_add_32) 291 ALIAS(atomic_add_64,_atomic_add_64) 292 ALIAS(atomic_add_int,_atomic_add_32) 293 ALIAS(atomic_add_long,_atomic_add_64) 294 ALIAS(atomic_add_ptr,_atomic_add_64) 295 296 ALIAS(atomic_add_32_nv,_atomic_add_32_nv) 297 ALIAS(atomic_add_64_nv,_atomic_add_64_nv) 298 ALIAS(atomic_add_int_nv,_atomic_add_32_nv) 299 ALIAS(atomic_add_long_nv,_atomic_add_64_nv) 300 ALIAS(atomic_add_ptr_nv,_atomic_add_64_nv) 301 302 ALIAS(atomic_and_32,_atomic_and_32) 303 ALIAS(atomic_and_64,_atomic_and_64) 304 ALIAS(atomic_and_uint,_atomic_and_32) 305 ALIAS(atomic_and_ulong,_atomic_and_64) 306 ALIAS(atomic_and_ptr,_atomic_and_64) 307 308 ALIAS(atomic_and_32_nv,_atomic_and_32_nv) 309 ALIAS(atomic_and_64_nv,_atomic_and_64_nv) 310 ALIAS(atomic_and_uint_nv,_atomic_and_32_nv) 311 ALIAS(atomic_and_ulong_nv,_atomic_and_64_nv) 312 ALIAS(atomic_and_ptr_nv,_atomic_and_64_nv) 313 314 ALIAS(atomic_dec_32,_atomic_dec_32) 315 ALIAS(atomic_dec_64,_atomic_dec_64) 316 ALIAS(atomic_dec_uint,_atomic_dec_32) 317 ALIAS(atomic_dec_ulong,_atomic_dec_64) 318 ALIAS(atomic_dec_ptr,_atomic_dec_64) 319 320 ALIAS(atomic_dec_32_nv,_atomic_dec_32_nv) 321 ALIAS(atomic_dec_64_nv,_atomic_dec_64_nv) 322 ALIAS(atomic_dec_uint_nv,_atomic_dec_32_nv) 323 ALIAS(atomic_dec_ulong_nv,_atomic_dec_64_nv) 324 ALIAS(atomic_dec_ptr_nv,_atomic_dec_64_nv) 325 326 ALIAS(atomic_inc_32,_atomic_inc_32) 327 ALIAS(atomic_inc_64,_atomic_inc_64) 328 ALIAS(atomic_inc_uint,_atomic_inc_32) 329 ALIAS(atomic_inc_ulong,_atomic_inc_64) 330 ALIAS(atomic_inc_ptr,_atomic_inc_64) 331 332 ALIAS(atomic_inc_32_nv,_atomic_inc_32_nv) 333 ALIAS(atomic_inc_64_nv,_atomic_inc_64_nv) 334 ALIAS(atomic_inc_uint_nv,_atomic_inc_32_nv) 335 ALIAS(atomic_inc_ulong_nv,_atomic_inc_64_nv) 336 ALIAS(atomic_inc_ptr_nv,_atomic_inc_64_nv) 337 338 ALIAS(atomic_or_32,_atomic_or_32) 339 ALIAS(atomic_or_64,_atomic_or_64) 340 ALIAS(atomic_or_uint,_atomic_or_32) 341 ALIAS(atomic_or_ulong,_atomic_or_64) 342 ALIAS(atomic_or_ptr,_atomic_or_64) 343 344 ALIAS(atomic_or_32_nv,_atomic_or_32_nv) 345 ALIAS(atomic_or_64_nv,_atomic_or_64_nv) 346 ALIAS(atomic_or_uint_nv,_atomic_or_32_nv) 347 ALIAS(atomic_or_ulong_nv,_atomic_or_64_nv) 348 ALIAS(atomic_or_ptr_nv,_atomic_or_64_nv) 349 350 ALIAS(atomic_swap_32,_atomic_swap_32) 351 ALIAS(atomic_swap_64,_atomic_swap_64) 352 ALIAS(atomic_swap_uint,_atomic_swap_32) 353 ALIAS(atomic_swap_ulong,_atomic_swap_64) 354 ALIAS(atomic_swap_ptr,_atomic_swap_64) 355 356 ALIAS(atomic_cas_32,_atomic_cas_32) 357 ALIAS(atomic_cas_64,_atomic_cas_64) 358 ALIAS(atomic_cas_uint,_atomic_cas_32) 359 ALIAS(atomic_cas_ulong,_atomic_cas_64) 360 ALIAS(atomic_cas_ptr,_atomic_cas_64) 361 362 ALIAS(atomic_cas_32_ni,_atomic_cas_32_ni) 363 ALIAS(atomic_cas_64_ni,_atomic_cas_64_ni) 364 ALIAS(atomic_cas_uint_ni,_atomic_cas_32_ni) 365 ALIAS(atomic_cas_ulong_ni,_atomic_cas_64_ni) 366 ALIAS(atomic_cas_ptr_ni,_atomic_cas_64_ni) 367 368 ALIAS(membar_consumer,_membar_consumer) 369 ALIAS(membar_producer,_membar_producer) 370 ALIAS(membar_enter,_membar_consumer) 371 ALIAS(membar_exit,_membar_producer) 372 ALIAS(membar_sync,_membar_sync) 373 374 STRONG_ALIAS(_atomic_add_int,_atomic_add_32) 375 STRONG_ALIAS(_atomic_add_long,_atomic_add_64) 376 STRONG_ALIAS(_atomic_add_ptr,_atomic_add_64) 377 378 STRONG_ALIAS(_atomic_add_int_nv,_atomic_add_32_nv) 379 STRONG_ALIAS(_atomic_add_long_nv,_atomic_add_64_nv) 380 STRONG_ALIAS(_atomic_add_ptr_nv,_atomic_add_64_nv) 381 382 STRONG_ALIAS(_atomic_and_uint,_atomic_and_32) 383 STRONG_ALIAS(_atomic_and_ulong,_atomic_and_64) 384 STRONG_ALIAS(_atomic_and_ptr,_atomic_and_64) 385 386 STRONG_ALIAS(_atomic_and_uint_nv,_atomic_and_32_nv) 387 STRONG_ALIAS(_atomic_and_ulong_nv,_atomic_and_64_nv) 388 STRONG_ALIAS(_atomic_and_ptr_nv,_atomic_and_64_nv) 389 390 STRONG_ALIAS(_atomic_dec_uint,_atomic_dec_32) 391 STRONG_ALIAS(_atomic_dec_ulong,_atomic_dec_64) 392 STRONG_ALIAS(_atomic_dec_ptr,_atomic_dec_64) 393 394 STRONG_ALIAS(_atomic_dec_uint_nv,_atomic_dec_32_nv) 395 STRONG_ALIAS(_atomic_dec_ulong_nv,_atomic_dec_64_nv) 396 STRONG_ALIAS(_atomic_dec_ptr_nv,_atomic_dec_64_nv) 397 398 STRONG_ALIAS(_atomic_inc_uint,_atomic_inc_32) 399 STRONG_ALIAS(_atomic_inc_ulong,_atomic_inc_64) 400 STRONG_ALIAS(_atomic_inc_ptr,_atomic_inc_64) 401 402 STRONG_ALIAS(_atomic_inc_uint_nv,_atomic_inc_32_nv) 403 STRONG_ALIAS(_atomic_inc_ulong_nv,_atomic_inc_64_nv) 404 STRONG_ALIAS(_atomic_inc_ptr_nv,_atomic_inc_64_nv) 405 406 STRONG_ALIAS(_atomic_or_uint,_atomic_or_32) 407 STRONG_ALIAS(_atomic_or_ulong,_atomic_or_64) 408 STRONG_ALIAS(_atomic_or_ptr,_atomic_or_64) 409 410 STRONG_ALIAS(_atomic_or_uint_nv,_atomic_or_32_nv) 411 STRONG_ALIAS(_atomic_or_ulong_nv,_atomic_or_64_nv) 412 STRONG_ALIAS(_atomic_or_ptr_nv,_atomic_or_64_nv) 413 414 STRONG_ALIAS(_atomic_swap_uint,_atomic_swap_32) 415 STRONG_ALIAS(_atomic_swap_ulong,_atomic_swap_64) 416 STRONG_ALIAS(_atomic_swap_ptr,_atomic_swap_64) 417 418 STRONG_ALIAS(_atomic_cas_uint,_atomic_cas_32) 419 STRONG_ALIAS(_atomic_cas_ulong,_atomic_cas_64) 420 STRONG_ALIAS(_atomic_cas_ptr,_atomic_cas_64) 421 422 STRONG_ALIAS(_atomic_cas_uint_ni,_atomic_cas_32_ni) 423 STRONG_ALIAS(_atomic_cas_ulong_ni,_atomic_cas_64_ni) 424 STRONG_ALIAS(_atomic_cas_ptr_ni,_atomic_cas_64_ni) 425 426 STRONG_ALIAS(_membar_enter,_membar_consumer) 427 STRONG_ALIAS(_membar_exit,_membar_producer) 428
Indexes created Tue Sep 30 20:09:53 GMT 2025