atomic.S revision 1.19
1 /* $NetBSD: atomic.S,v 1.19 2020/04/26 13:37:14 maxv Exp $ */ 2 3 /*- 4 * Copyright (c) 2007 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/param.h> 33 #include <machine/asm.h> 34 35 #ifdef _KERNEL 36 #define ALIAS(f, t) STRONG_ALIAS(f,t) 37 #else 38 #define ALIAS(f, t) WEAK_ALIAS(f,t) 39 #endif 40 41 #ifdef _HARDKERNEL 42 #include <machine/frameasm.h> 43 #define LOCK(n) HOTPATCH(HP_NAME_NOLOCK, 1); lock 44 #define ENDLABEL(a) _ALIGN_TEXT; LABEL(a) 45 #else 46 #define LOCK(n) lock 47 #define ENDLABEL(a) /* nothing */ 48 #endif 49 50 .text 51 52 /* 32-bit */ 53 54 ENTRY(_atomic_add_32) 55 LOCK(1) 56 addl %esi, (%rdi) 57 ret 58 END(_atomic_add_32) 59 60 ENTRY(_atomic_add_32_nv) 61 movl %esi, %eax 62 LOCK(2) 63 xaddl %eax, (%rdi) 64 addl %esi, %eax 65 ret 66 END(_atomic_add_32_nv) 67 68 ENTRY(_atomic_and_32) 69 LOCK(3) 70 andl %esi, (%rdi) 71 ret 72 END(_atomic_and_32) 73 74 ENTRY(_atomic_and_32_nv) 75 movl (%rdi), %eax 76 1: 77 movl %eax, %ecx 78 andl %esi, %ecx 79 LOCK(4) 80 cmpxchgl %ecx, (%rdi) 81 jnz 1b 82 movl %ecx, %eax 83 ret 84 END(_atomic_and_32_nv) 85 86 ENTRY(_atomic_dec_32) 87 LOCK(5) 88 decl (%rdi) 89 ret 90 END(_atomic_dec_32) 91 92 ENTRY(_atomic_dec_32_nv) 93 movl $-1, %eax 94 LOCK(6) 95 xaddl %eax, (%rdi) 96 decl %eax 97 ret 98 END(_atomic_dec_32_nv) 99 100 ENTRY(_atomic_inc_32) 101 LOCK(7) 102 incl (%rdi) 103 ret 104 END(_atomic_inc_32) 105 106 ENTRY(_atomic_inc_32_nv) 107 movl $1, %eax 108 LOCK(8) 109 xaddl %eax, (%rdi) 110 incl %eax 111 ret 112 END(_atomic_inc_32_nv) 113 114 ENTRY(_atomic_or_32) 115 LOCK(9) 116 orl %esi, (%rdi) 117 ret 118 END(_atomic_or_32) 119 120 ENTRY(_atomic_or_32_nv) 121 movl (%rdi), %eax 122 1: 123 movl %eax, %ecx 124 orl %esi, %ecx 125 LOCK(10) 126 cmpxchgl %ecx, (%rdi) 127 jnz 1b 128 movl %ecx, %eax 129 ret 130 END(_atomic_or_32_nv) 131 132 ENTRY(_atomic_swap_32) 133 movl %esi, %eax 134 xchgl %eax, (%rdi) 135 ret 136 END(_atomic_swap_32) 137 138 ENTRY(_atomic_cas_32) 139 movl %esi, %eax 140 LOCK(12) 141 cmpxchgl %edx, (%rdi) 142 /* %eax now contains the old value */ 143 ret 144 END(_atomic_cas_32) 145 146 ENTRY(_atomic_cas_32_ni) 147 movl %esi, %eax 148 cmpxchgl %edx, (%rdi) 149 /* %eax now contains the old value */ 150 ret 151 END(_atomic_cas_32_ni) 152 153 /* 64-bit */ 154 155 ENTRY(_atomic_add_64) 156 LOCK(13) 157 addq %rsi, (%rdi) 158 ret 159 END(_atomic_add_64) 160 161 ENTRY(_atomic_add_64_nv) 162 movq %rsi, %rax 163 LOCK(14) 164 xaddq %rax, (%rdi) 165 addq %rsi, %rax 166 ret 167 END(_atomic_add_64_nv) 168 169 ENTRY(_atomic_and_64) 170 LOCK(15) 171 andq %rsi, (%rdi) 172 ret 173 END(_atomic_and_64) 174 175 ENTRY(_atomic_and_64_nv) 176 movq (%rdi), %rax 177 1: 178 movq %rax, %rcx 179 andq %rsi, %rcx 180 LOCK(16) 181 cmpxchgq %rcx, (%rdi) 182 jnz 1b 183 movq %rcx, %rax 184 ret 185 END(_atomic_and_64_nv) 186 187 ENTRY(_atomic_dec_64) 188 LOCK(17) 189 decq (%rdi) 190 ret 191 END(_atomic_dec_64) 192 193 ENTRY(_atomic_dec_64_nv) 194 movq $-1, %rax 195 LOCK(18) 196 xaddq %rax, (%rdi) 197 decq %rax 198 ret 199 END(_atomic_dec_64_nv) 200 201 ENTRY(_atomic_inc_64) 202 LOCK(19) 203 incq (%rdi) 204 ret 205 END(_atomic_inc_64) 206 207 ENTRY(_atomic_inc_64_nv) 208 movq $1, %rax 209 LOCK(20) 210 xaddq %rax, (%rdi) 211 incq %rax 212 ret 213 END(_atomic_inc_64_nv) 214 215 ENTRY(_atomic_or_64) 216 LOCK(21) 217 orq %rsi, (%rdi) 218 ret 219 END(_atomic_or_64) 220 221 ENTRY(_atomic_or_64_nv) 222 movq (%rdi), %rax 223 1: 224 movq %rax, %rcx 225 orq %rsi, %rcx 226 LOCK(22) 227 cmpxchgq %rcx, (%rdi) 228 jnz 1b 229 movq %rcx, %rax 230 ret 231 END(_atomic_or_64_nv) 232 233 ENTRY(_atomic_swap_64) 234 movq %rsi, %rax 235 xchgq %rax, (%rdi) 236 ret 237 END(_atomic_swap_64) 238 239 ENTRY(_atomic_cas_64) 240 movq %rsi, %rax 241 LOCK(24) 242 cmpxchgq %rdx, (%rdi) 243 /* %eax now contains the old value */ 244 ret 245 END(_atomic_cas_64) 246 247 ENTRY(_atomic_cas_64_ni) 248 movq %rsi, %rax 249 cmpxchgq %rdx, (%rdi) 250 /* %eax now contains the old value */ 251 ret 252 END(_atomic_cas_64_ni) 253 254 /* memory barriers */ 255 256 ENTRY(_membar_consumer) 257 LOCK(25) 258 addq $0, -8(%rsp) 259 ret 260 END(_membar_consumer) 261 ENDLABEL(membar_consumer_end) 262 263 ENTRY(_membar_producer) 264 /* A store is enough */ 265 movq $0, -8(%rsp) 266 ret 267 END(_membar_producer) 268 ENDLABEL(membar_producer_end) 269 270 ENTRY(_membar_sync) 271 LOCK(26) 272 addq $0, -8(%rsp) 273 ret 274 END(_membar_sync) 275 ENDLABEL(membar_sync_end) 276 277 #ifdef _HARDKERNEL 278 ENTRY(sse2_lfence) 279 lfence 280 ret 281 END(sse2_lfence) 282 ENDLABEL(sse2_lfence_end) 283 284 ENTRY(sse2_mfence) 285 mfence 286 ret 287 END(sse2_mfence) 288 ENDLABEL(sse2_mfence_end) 289 #endif /* _HARDKERNEL */ 290 291 ALIAS(atomic_add_32,_atomic_add_32) 292 ALIAS(atomic_add_64,_atomic_add_64) 293 ALIAS(atomic_add_int,_atomic_add_32) 294 ALIAS(atomic_add_long,_atomic_add_64) 295 ALIAS(atomic_add_ptr,_atomic_add_64) 296 297 ALIAS(atomic_add_32_nv,_atomic_add_32_nv) 298 ALIAS(atomic_add_64_nv,_atomic_add_64_nv) 299 ALIAS(atomic_add_int_nv,_atomic_add_32_nv) 300 ALIAS(atomic_add_long_nv,_atomic_add_64_nv) 301 ALIAS(atomic_add_ptr_nv,_atomic_add_64_nv) 302 303 ALIAS(atomic_and_32,_atomic_and_32) 304 ALIAS(atomic_and_64,_atomic_and_64) 305 ALIAS(atomic_and_uint,_atomic_and_32) 306 ALIAS(atomic_and_ulong,_atomic_and_64) 307 ALIAS(atomic_and_ptr,_atomic_and_64) 308 309 ALIAS(atomic_and_32_nv,_atomic_and_32_nv) 310 ALIAS(atomic_and_64_nv,_atomic_and_64_nv) 311 ALIAS(atomic_and_uint_nv,_atomic_and_32_nv) 312 ALIAS(atomic_and_ulong_nv,_atomic_and_64_nv) 313 ALIAS(atomic_and_ptr_nv,_atomic_and_64_nv) 314 315 ALIAS(atomic_dec_32,_atomic_dec_32) 316 ALIAS(atomic_dec_64,_atomic_dec_64) 317 ALIAS(atomic_dec_uint,_atomic_dec_32) 318 ALIAS(atomic_dec_ulong,_atomic_dec_64) 319 ALIAS(atomic_dec_ptr,_atomic_dec_64) 320 321 ALIAS(atomic_dec_32_nv,_atomic_dec_32_nv) 322 ALIAS(atomic_dec_64_nv,_atomic_dec_64_nv) 323 ALIAS(atomic_dec_uint_nv,_atomic_dec_32_nv) 324 ALIAS(atomic_dec_ulong_nv,_atomic_dec_64_nv) 325 ALIAS(atomic_dec_ptr_nv,_atomic_dec_64_nv) 326 327 ALIAS(atomic_inc_32,_atomic_inc_32) 328 ALIAS(atomic_inc_64,_atomic_inc_64) 329 ALIAS(atomic_inc_uint,_atomic_inc_32) 330 ALIAS(atomic_inc_ulong,_atomic_inc_64) 331 ALIAS(atomic_inc_ptr,_atomic_inc_64) 332 333 ALIAS(atomic_inc_32_nv,_atomic_inc_32_nv) 334 ALIAS(atomic_inc_64_nv,_atomic_inc_64_nv) 335 ALIAS(atomic_inc_uint_nv,_atomic_inc_32_nv) 336 ALIAS(atomic_inc_ulong_nv,_atomic_inc_64_nv) 337 ALIAS(atomic_inc_ptr_nv,_atomic_inc_64_nv) 338 339 ALIAS(atomic_or_32,_atomic_or_32) 340 ALIAS(atomic_or_64,_atomic_or_64) 341 ALIAS(atomic_or_uint,_atomic_or_32) 342 ALIAS(atomic_or_ulong,_atomic_or_64) 343 ALIAS(atomic_or_ptr,_atomic_or_64) 344 345 ALIAS(atomic_or_32_nv,_atomic_or_32_nv) 346 ALIAS(atomic_or_64_nv,_atomic_or_64_nv) 347 ALIAS(atomic_or_uint_nv,_atomic_or_32_nv) 348 ALIAS(atomic_or_ulong_nv,_atomic_or_64_nv) 349 ALIAS(atomic_or_ptr_nv,_atomic_or_64_nv) 350 351 ALIAS(atomic_swap_32,_atomic_swap_32) 352 ALIAS(atomic_swap_64,_atomic_swap_64) 353 ALIAS(atomic_swap_uint,_atomic_swap_32) 354 ALIAS(atomic_swap_ulong,_atomic_swap_64) 355 ALIAS(atomic_swap_ptr,_atomic_swap_64) 356 357 ALIAS(atomic_cas_32,_atomic_cas_32) 358 ALIAS(atomic_cas_64,_atomic_cas_64) 359 ALIAS(atomic_cas_uint,_atomic_cas_32) 360 ALIAS(atomic_cas_ulong,_atomic_cas_64) 361 ALIAS(atomic_cas_ptr,_atomic_cas_64) 362 363 ALIAS(atomic_cas_32_ni,_atomic_cas_32_ni) 364 ALIAS(atomic_cas_64_ni,_atomic_cas_64_ni) 365 ALIAS(atomic_cas_uint_ni,_atomic_cas_32_ni) 366 ALIAS(atomic_cas_ulong_ni,_atomic_cas_64_ni) 367 ALIAS(atomic_cas_ptr_ni,_atomic_cas_64_ni) 368 369 ALIAS(membar_consumer,_membar_consumer) 370 ALIAS(membar_producer,_membar_producer) 371 ALIAS(membar_enter,_membar_consumer) 372 ALIAS(membar_exit,_membar_producer) 373 ALIAS(membar_sync,_membar_sync) 374 375 STRONG_ALIAS(_atomic_add_int,_atomic_add_32) 376 STRONG_ALIAS(_atomic_add_long,_atomic_add_64) 377 STRONG_ALIAS(_atomic_add_ptr,_atomic_add_64) 378 379 STRONG_ALIAS(_atomic_add_int_nv,_atomic_add_32_nv) 380 STRONG_ALIAS(_atomic_add_long_nv,_atomic_add_64_nv) 381 STRONG_ALIAS(_atomic_add_ptr_nv,_atomic_add_64_nv) 382 383 STRONG_ALIAS(_atomic_and_uint,_atomic_and_32) 384 STRONG_ALIAS(_atomic_and_ulong,_atomic_and_64) 385 STRONG_ALIAS(_atomic_and_ptr,_atomic_and_64) 386 387 STRONG_ALIAS(_atomic_and_uint_nv,_atomic_and_32_nv) 388 STRONG_ALIAS(_atomic_and_ulong_nv,_atomic_and_64_nv) 389 STRONG_ALIAS(_atomic_and_ptr_nv,_atomic_and_64_nv) 390 391 STRONG_ALIAS(_atomic_dec_uint,_atomic_dec_32) 392 STRONG_ALIAS(_atomic_dec_ulong,_atomic_dec_64) 393 STRONG_ALIAS(_atomic_dec_ptr,_atomic_dec_64) 394 395 STRONG_ALIAS(_atomic_dec_uint_nv,_atomic_dec_32_nv) 396 STRONG_ALIAS(_atomic_dec_ulong_nv,_atomic_dec_64_nv) 397 STRONG_ALIAS(_atomic_dec_ptr_nv,_atomic_dec_64_nv) 398 399 STRONG_ALIAS(_atomic_inc_uint,_atomic_inc_32) 400 STRONG_ALIAS(_atomic_inc_ulong,_atomic_inc_64) 401 STRONG_ALIAS(_atomic_inc_ptr,_atomic_inc_64) 402 403 STRONG_ALIAS(_atomic_inc_uint_nv,_atomic_inc_32_nv) 404 STRONG_ALIAS(_atomic_inc_ulong_nv,_atomic_inc_64_nv) 405 STRONG_ALIAS(_atomic_inc_ptr_nv,_atomic_inc_64_nv) 406 407 STRONG_ALIAS(_atomic_or_uint,_atomic_or_32) 408 STRONG_ALIAS(_atomic_or_ulong,_atomic_or_64) 409 STRONG_ALIAS(_atomic_or_ptr,_atomic_or_64) 410 411 STRONG_ALIAS(_atomic_or_uint_nv,_atomic_or_32_nv) 412 STRONG_ALIAS(_atomic_or_ulong_nv,_atomic_or_64_nv) 413 STRONG_ALIAS(_atomic_or_ptr_nv,_atomic_or_64_nv) 414 415 STRONG_ALIAS(_atomic_swap_uint,_atomic_swap_32) 416 STRONG_ALIAS(_atomic_swap_ulong,_atomic_swap_64) 417 STRONG_ALIAS(_atomic_swap_ptr,_atomic_swap_64) 418 419 STRONG_ALIAS(_atomic_cas_uint,_atomic_cas_32) 420 STRONG_ALIAS(_atomic_cas_ulong,_atomic_cas_64) 421 STRONG_ALIAS(_atomic_cas_ptr,_atomic_cas_64) 422 423 STRONG_ALIAS(_atomic_cas_uint_ni,_atomic_cas_32_ni) 424 STRONG_ALIAS(_atomic_cas_ulong_ni,_atomic_cas_64_ni) 425 STRONG_ALIAS(_atomic_cas_ptr_ni,_atomic_cas_64_ni) 426 427 STRONG_ALIAS(_membar_enter,_membar_consumer) 428 STRONG_ALIAS(_membar_exit,_membar_producer) 429
Indexes created Tue Sep 30 17:09:57 GMT 2025