atomic.S revision 1.17
1 /* $NetBSD: atomic.S,v 1.17 2014/05/22 15:23:11 uebayasi Exp $ */ 2 3 /*- 4 * Copyright (c) 2007 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/param.h> 33 #include <machine/asm.h> 34 35 #ifdef _KERNEL 36 #define ALIAS(f, t) STRONG_ALIAS(f,t) 37 #else 38 #define ALIAS(f, t) WEAK_ALIAS(f,t) 39 #endif 40 41 #ifdef _HARDKERNEL 42 #define LOCK(n) .Lpatch ## n: lock 43 #define ENDLABEL(a) _ALIGN_TEXT; LABEL(a) 44 #else 45 #define LOCK(n) lock 46 #define ENDLABEL(a) /* nothing */ 47 #endif 48 49 .text 50 51 /* 32-bit */ 52 53 ENTRY(_atomic_add_32) 54 LOCK(1) 55 addl %esi, (%rdi) 56 ret 57 END(_atomic_add_32) 58 59 ENTRY(_atomic_add_32_nv) 60 movl %esi, %eax 61 LOCK(2) 62 xaddl %eax, (%rdi) 63 addl %esi, %eax 64 ret 65 END(_atomic_add_32_nv) 66 67 ENTRY(_atomic_and_32) 68 LOCK(3) 69 andl %esi, (%rdi) 70 ret 71 END(_atomic_and_32) 72 73 ENTRY(_atomic_and_32_nv) 74 movl (%rdi), %eax 75 1: 76 movl %eax, %ecx 77 andl %esi, %ecx 78 LOCK(4) 79 cmpxchgl %ecx, (%rdi) 80 jnz 1b 81 movl %ecx, %eax 82 ret 83 END(_atomic_and_32_nv) 84 85 ENTRY(_atomic_dec_32) 86 LOCK(5) 87 decl (%rdi) 88 ret 89 END(_atomic_dec_32) 90 91 ENTRY(_atomic_dec_32_nv) 92 movl $-1, %eax 93 LOCK(6) 94 xaddl %eax, (%rdi) 95 decl %eax 96 ret 97 END(_atomic_dec_32_nv) 98 99 ENTRY(_atomic_inc_32) 100 LOCK(7) 101 incl (%rdi) 102 ret 103 END(_atomic_inc_32) 104 105 ENTRY(_atomic_inc_32_nv) 106 movl $1, %eax 107 LOCK(8) 108 xaddl %eax, (%rdi) 109 incl %eax 110 ret 111 END(_atomic_inc_32_nv) 112 113 ENTRY(_atomic_or_32) 114 LOCK(9) 115 orl %esi, (%rdi) 116 ret 117 END(_atomic_or_32) 118 119 ENTRY(_atomic_or_32_nv) 120 movl (%rdi), %eax 121 1: 122 movl %eax, %ecx 123 orl %esi, %ecx 124 LOCK(10) 125 cmpxchgl %ecx, (%rdi) 126 jnz 1b 127 movl %ecx, %eax 128 ret 129 END(_atomic_or_32_nv) 130 131 ENTRY(_atomic_swap_32) 132 movl %esi, %eax 133 xchgl %eax, (%rdi) 134 ret 135 END(_atomic_swap_32) 136 137 ENTRY(_atomic_cas_32) 138 movl %esi, %eax 139 LOCK(12) 140 cmpxchgl %edx, (%rdi) 141 /* %eax now contains the old value */ 142 ret 143 END(_atomic_cas_32) 144 145 ENTRY(_atomic_cas_32_ni) 146 movl %esi, %eax 147 cmpxchgl %edx, (%rdi) 148 /* %eax now contains the old value */ 149 ret 150 END(_atomic_cas_32_ni) 151 152 /* 64-bit */ 153 154 ENTRY(_atomic_add_64) 155 LOCK(13) 156 addq %rsi, (%rdi) 157 ret 158 END(_atomic_add_64) 159 160 ENTRY(_atomic_add_64_nv) 161 movq %rsi, %rax 162 LOCK(14) 163 xaddq %rax, (%rdi) 164 addq %rsi, %rax 165 ret 166 END(_atomic_add_64_nv) 167 168 ENTRY(_atomic_and_64) 169 LOCK(15) 170 andq %rsi, (%rdi) 171 ret 172 END(_atomic_and_64) 173 174 ENTRY(_atomic_and_64_nv) 175 movq (%rdi), %rax 176 1: 177 movq %rax, %rcx 178 andq %rsi, %rcx 179 LOCK(16) 180 cmpxchgq %rcx, (%rdi) 181 jnz 1b 182 movq %rcx, %rax 183 ret 184 END(_atomic_and_64_nv) 185 186 ENTRY(_atomic_dec_64) 187 LOCK(17) 188 decq (%rdi) 189 ret 190 END(_atomic_dec_64) 191 192 ENTRY(_atomic_dec_64_nv) 193 movq $-1, %rax 194 LOCK(18) 195 xaddq %rax, (%rdi) 196 decq %rax 197 ret 198 END(_atomic_dec_64_nv) 199 200 ENTRY(_atomic_inc_64) 201 LOCK(19) 202 incq (%rdi) 203 ret 204 END(_atomic_inc_64) 205 206 ENTRY(_atomic_inc_64_nv) 207 movq $1, %rax 208 LOCK(20) 209 xaddq %rax, (%rdi) 210 incq %rax 211 ret 212 END(_atomic_inc_64_nv) 213 214 ENTRY(_atomic_or_64) 215 LOCK(21) 216 orq %rsi, (%rdi) 217 ret 218 END(_atomic_or_64) 219 220 ENTRY(_atomic_or_64_nv) 221 movq (%rdi), %rax 222 1: 223 movq %rax, %rcx 224 orq %rsi, %rcx 225 LOCK(22) 226 cmpxchgq %rcx, (%rdi) 227 jnz 1b 228 movq %rcx, %rax 229 ret 230 END(_atomic_or_64_nv) 231 232 ENTRY(_atomic_swap_64) 233 movq %rsi, %rax 234 xchgq %rax, (%rdi) 235 ret 236 END(_atomic_swap_64) 237 238 ENTRY(_atomic_cas_64) 239 movq %rsi, %rax 240 LOCK(24) 241 cmpxchgq %rdx, (%rdi) 242 /* %eax now contains the old value */ 243 ret 244 END(_atomic_cas_64) 245 246 ENTRY(_atomic_cas_64_ni) 247 movq %rsi, %rax 248 cmpxchgq %rdx, (%rdi) 249 /* %eax now contains the old value */ 250 ret 251 END(_atomic_cas_64_ni) 252 253 /* memory barriers */ 254 255 ENTRY(_membar_consumer) 256 LOCK(25) 257 addq $0, -8(%rsp) 258 ret 259 END(_membar_consumer) 260 ENDLABEL(membar_consumer_end) 261 262 ENTRY(_membar_producer) 263 /* A store is enough */ 264 movq $0, -8(%rsp) 265 ret 266 END(_membar_producer) 267 ENDLABEL(membar_producer_end) 268 269 ENTRY(_membar_sync) 270 LOCK(26) 271 addq $0, -8(%rsp) 272 ret 273 END(_membar_sync) 274 ENDLABEL(membar_sync_end) 275 276 #ifdef _HARDKERNEL 277 ENTRY(sse2_lfence) 278 lfence 279 ret 280 END(sse2_lfence) 281 ENDLABEL(sse2_lfence_end) 282 283 ENTRY(sse2_mfence) 284 mfence 285 ret 286 END(sse2_mfence) 287 ENDLABEL(sse2_mfence_end) 288 289 atomic_lockpatch: 290 .globl atomic_lockpatch 291 .quad .Lpatch1, .Lpatch2, .Lpatch3, .Lpatch4, .Lpatch5 292 .quad .Lpatch6, .Lpatch7, .Lpatch8, .Lpatch9, .Lpatch10 293 .quad .Lpatch12, .Lpatch13, .Lpatch14, .Lpatch15 294 .quad .Lpatch16, .Lpatch17, .Lpatch18, .Lpatch19, .Lpatch20 295 .quad .Lpatch21, .Lpatch22, .Lpatch24, .Lpatch25 296 .quad .Lpatch26, 0 297 #endif /* _HARDKERNEL */ 298 299 ALIAS(atomic_add_32,_atomic_add_32) 300 ALIAS(atomic_add_64,_atomic_add_64) 301 ALIAS(atomic_add_int,_atomic_add_32) 302 ALIAS(atomic_add_long,_atomic_add_64) 303 ALIAS(atomic_add_ptr,_atomic_add_64) 304 305 ALIAS(atomic_add_32_nv,_atomic_add_32_nv) 306 ALIAS(atomic_add_64_nv,_atomic_add_64_nv) 307 ALIAS(atomic_add_int_nv,_atomic_add_32_nv) 308 ALIAS(atomic_add_long_nv,_atomic_add_64_nv) 309 ALIAS(atomic_add_ptr_nv,_atomic_add_64_nv) 310 311 ALIAS(atomic_and_32,_atomic_and_32) 312 ALIAS(atomic_and_64,_atomic_and_64) 313 ALIAS(atomic_and_uint,_atomic_and_32) 314 ALIAS(atomic_and_ulong,_atomic_and_64) 315 ALIAS(atomic_and_ptr,_atomic_and_64) 316 317 ALIAS(atomic_and_32_nv,_atomic_and_32_nv) 318 ALIAS(atomic_and_64_nv,_atomic_and_64_nv) 319 ALIAS(atomic_and_uint_nv,_atomic_and_32_nv) 320 ALIAS(atomic_and_ulong_nv,_atomic_and_64_nv) 321 ALIAS(atomic_and_ptr_nv,_atomic_and_64_nv) 322 323 ALIAS(atomic_dec_32,_atomic_dec_32) 324 ALIAS(atomic_dec_64,_atomic_dec_64) 325 ALIAS(atomic_dec_uint,_atomic_dec_32) 326 ALIAS(atomic_dec_ulong,_atomic_dec_64) 327 ALIAS(atomic_dec_ptr,_atomic_dec_64) 328 329 ALIAS(atomic_dec_32_nv,_atomic_dec_32_nv) 330 ALIAS(atomic_dec_64_nv,_atomic_dec_64_nv) 331 ALIAS(atomic_dec_uint_nv,_atomic_dec_32_nv) 332 ALIAS(atomic_dec_ulong_nv,_atomic_dec_64_nv) 333 ALIAS(atomic_dec_ptr_nv,_atomic_dec_64_nv) 334 335 ALIAS(atomic_inc_32,_atomic_inc_32) 336 ALIAS(atomic_inc_64,_atomic_inc_64) 337 ALIAS(atomic_inc_uint,_atomic_inc_32) 338 ALIAS(atomic_inc_ulong,_atomic_inc_64) 339 ALIAS(atomic_inc_ptr,_atomic_inc_64) 340 341 ALIAS(atomic_inc_32_nv,_atomic_inc_32_nv) 342 ALIAS(atomic_inc_64_nv,_atomic_inc_64_nv) 343 ALIAS(atomic_inc_uint_nv,_atomic_inc_32_nv) 344 ALIAS(atomic_inc_ulong_nv,_atomic_inc_64_nv) 345 ALIAS(atomic_inc_ptr_nv,_atomic_inc_64_nv) 346 347 ALIAS(atomic_or_32,_atomic_or_32) 348 ALIAS(atomic_or_uint,_atomic_or_32) 349 ALIAS(atomic_or_ulong,_atomic_or_64) 350 ALIAS(atomic_or_ptr,_atomic_or_64) 351 352 ALIAS(atomic_or_32_nv,_atomic_or_32_nv) 353 ALIAS(atomic_or_64_nv,_atomic_or_64_nv) 354 ALIAS(atomic_or_uint_nv,_atomic_or_32_nv) 355 ALIAS(atomic_or_ulong_nv,_atomic_or_64_nv) 356 ALIAS(atomic_or_ptr_nv,_atomic_or_64_nv) 357 358 ALIAS(atomic_swap_32,_atomic_swap_32) 359 ALIAS(atomic_swap_64,_atomic_swap_64) 360 ALIAS(atomic_swap_uint,_atomic_swap_32) 361 ALIAS(atomic_swap_ulong,_atomic_swap_64) 362 ALIAS(atomic_swap_ptr,_atomic_swap_64) 363 364 ALIAS(atomic_cas_32,_atomic_cas_32) 365 ALIAS(atomic_cas_64,_atomic_cas_64) 366 ALIAS(atomic_cas_uint,_atomic_cas_32) 367 ALIAS(atomic_cas_ulong,_atomic_cas_64) 368 ALIAS(atomic_cas_ptr,_atomic_cas_64) 369 370 ALIAS(atomic_cas_32_ni,_atomic_cas_32_ni) 371 ALIAS(atomic_cas_64_ni,_atomic_cas_64_ni) 372 ALIAS(atomic_cas_uint_ni,_atomic_cas_32_ni) 373 ALIAS(atomic_cas_ulong_ni,_atomic_cas_64_ni) 374 ALIAS(atomic_cas_ptr_ni,_atomic_cas_64_ni) 375 376 ALIAS(membar_consumer,_membar_consumer) 377 ALIAS(membar_producer,_membar_producer) 378 ALIAS(membar_enter,_membar_consumer) 379 ALIAS(membar_exit,_membar_producer) 380 ALIAS(membar_sync,_membar_sync) 381 382 STRONG_ALIAS(_atomic_add_int,_atomic_add_32) 383 STRONG_ALIAS(_atomic_add_long,_atomic_add_64) 384 STRONG_ALIAS(_atomic_add_ptr,_atomic_add_64) 385 386 STRONG_ALIAS(_atomic_add_int_nv,_atomic_add_32_nv) 387 STRONG_ALIAS(_atomic_add_long_nv,_atomic_add_64_nv) 388 STRONG_ALIAS(_atomic_add_ptr_nv,_atomic_add_64_nv) 389 390 STRONG_ALIAS(_atomic_and_uint,_atomic_and_32) 391 STRONG_ALIAS(_atomic_and_ulong,_atomic_and_64) 392 STRONG_ALIAS(_atomic_and_ptr,_atomic_and_64) 393 394 STRONG_ALIAS(_atomic_and_uint_nv,_atomic_and_32_nv) 395 STRONG_ALIAS(_atomic_and_ulong_nv,_atomic_and_64_nv) 396 STRONG_ALIAS(_atomic_and_ptr_nv,_atomic_and_64_nv) 397 398 STRONG_ALIAS(_atomic_dec_uint,_atomic_dec_32) 399 STRONG_ALIAS(_atomic_dec_ulong,_atomic_dec_64) 400 STRONG_ALIAS(_atomic_dec_ptr,_atomic_dec_64) 401 402 STRONG_ALIAS(_atomic_dec_uint_nv,_atomic_dec_32_nv) 403 STRONG_ALIAS(_atomic_dec_ulong_nv,_atomic_dec_64_nv) 404 STRONG_ALIAS(_atomic_dec_ptr_nv,_atomic_dec_64_nv) 405 406 STRONG_ALIAS(_atomic_inc_uint,_atomic_inc_32) 407 STRONG_ALIAS(_atomic_inc_ulong,_atomic_inc_64) 408 STRONG_ALIAS(_atomic_inc_ptr,_atomic_inc_64) 409 410 STRONG_ALIAS(_atomic_inc_uint_nv,_atomic_inc_32_nv) 411 STRONG_ALIAS(_atomic_inc_ulong_nv,_atomic_inc_64_nv) 412 STRONG_ALIAS(_atomic_inc_ptr_nv,_atomic_inc_64_nv) 413 414 STRONG_ALIAS(_atomic_or_uint,_atomic_or_32) 415 STRONG_ALIAS(_atomic_or_ulong,_atomic_or_64) 416 STRONG_ALIAS(_atomic_or_ptr,_atomic_or_64) 417 418 STRONG_ALIAS(_atomic_or_uint_nv,_atomic_or_32_nv) 419 STRONG_ALIAS(_atomic_or_ulong_nv,_atomic_or_64_nv) 420 STRONG_ALIAS(_atomic_or_ptr_nv,_atomic_or_64_nv) 421 422 STRONG_ALIAS(_atomic_swap_uint,_atomic_swap_32) 423 STRONG_ALIAS(_atomic_swap_ulong,_atomic_swap_64) 424 STRONG_ALIAS(_atomic_swap_ptr,_atomic_swap_64) 425 426 STRONG_ALIAS(_atomic_cas_uint,_atomic_cas_32) 427 STRONG_ALIAS(_atomic_cas_ulong,_atomic_cas_64) 428 STRONG_ALIAS(_atomic_cas_ptr,_atomic_cas_64) 429 430 STRONG_ALIAS(_atomic_cas_uint_ni,_atomic_cas_32_ni) 431 STRONG_ALIAS(_atomic_cas_ulong_ni,_atomic_cas_64_ni) 432 STRONG_ALIAS(_atomic_cas_ptr_ni,_atomic_cas_64_ni) 433 434 STRONG_ALIAS(_membar_enter,_membar_consumer) 435 STRONG_ALIAS(_membar_exit,_membar_producer) 436
Indexes created Tue Sep 30 20:09:53 GMT 2025