atomic.S revision 1.9
1 /* $NetBSD: atomic.S,v 1.9 2008/02/10 13:25:46 ad Exp $ */ 2 3 /*- 4 * Copyright (c) 2007 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. All advertising materials mentioning features or use of this software 19 * must display the following acknowledgement: 20 * This product includes software developed by the NetBSD 21 * Foundation, Inc. and its contributors. 22 * 4. Neither the name of The NetBSD Foundation nor the names of its 23 * contributors may be used to endorse or promote products derived 24 * from this software without specific prior written permission. 25 * 26 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 27 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 28 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 29 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 30 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 31 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 32 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 33 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 34 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 35 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 36 * POSSIBILITY OF SUCH DAMAGE. 37 */ 38 39 #include <machine/asm.h> 40 41 #ifdef _KERNEL 42 #define LOCK(n) .Lpatch/**/n: lock 43 #define ALIAS(f, t) STRONG_ALIAS(f,t) 44 #define END(a) _ALIGN_TEXT; LABEL(a) 45 #else 46 #define LOCK(n) lock 47 #define ALIAS(f, t) WEAK_ALIAS(f,t) 48 #define END(a) /* nothing */ 49 #endif 50 51 .text 52 53 /* 32-bit */ 54 55 NENTRY(_atomic_add_32) 56 LOCK(1) 57 addl %esi, (%rdi) 58 ret 59 60 NENTRY(_atomic_add_32_nv) 61 movl %esi, %eax 62 LOCK(2) 63 xaddl %eax, (%rdi) 64 addl %esi, %eax 65 ret 66 67 NENTRY(_atomic_and_32) 68 LOCK(3) 69 andl %esi, (%rdi) 70 ret 71 72 NENTRY(_atomic_and_32_nv) 73 movl (%rdi), %eax 74 1: 75 movl %eax, %ecx 76 andl %esi, %ecx 77 LOCK(4) 78 cmpxchgl %ecx, (%rdi) 79 jnz 1b 80 movl %ecx, %eax 81 ret 82 83 NENTRY(_atomic_dec_32) 84 LOCK(5) 85 decl (%rdi) 86 ret 87 88 NENTRY(_atomic_dec_32_nv) 89 movl $-1, %eax 90 LOCK(6) 91 xaddl %eax, (%rdi) 92 decl %eax 93 ret 94 95 NENTRY(_atomic_inc_32) 96 LOCK(7) 97 incl (%rdi) 98 ret 99 100 NENTRY(_atomic_inc_32_nv) 101 movl $1, %eax 102 LOCK(8) 103 xaddl %eax, (%rdi) 104 incl %eax 105 ret 106 107 NENTRY(_atomic_or_32) 108 LOCK(9) 109 orl %esi, (%rdi) 110 ret 111 112 NENTRY(_atomic_or_32_nv) 113 movl (%rdi), %eax 114 1: 115 movl %eax, %ecx 116 orl %esi, %ecx 117 LOCK(10) 118 cmpxchgl %ecx, (%rdi) 119 jnz 1b 120 movl %ecx, %eax 121 ret 122 123 NENTRY(_atomic_swap_32) 124 movl %esi, %eax 125 xchgl %eax, (%rdi) 126 ret 127 128 NENTRY(_atomic_cas_32) 129 movl %esi, %eax 130 LOCK(12) 131 cmpxchgl %edx, (%rdi) 132 /* %eax now contains the old value */ 133 ret 134 135 NENTRY(_atomic_cas_32_ni) 136 movl %esi, %eax 137 cmpxchgl %edx, (%rdi) 138 /* %eax now contains the old value */ 139 ret 140 141 /* 64-bit */ 142 143 NENTRY(_atomic_add_64) 144 LOCK(13) 145 addq %rsi, (%rdi) 146 ret 147 148 NENTRY(_atomic_add_64_nv) 149 movq %rsi, %rax 150 LOCK(14) 151 xaddq %rax, (%rdi) 152 addq %rsi, %rax 153 ret 154 155 NENTRY(_atomic_and_64) 156 LOCK(15) 157 andq %rsi, (%rdi) 158 ret 159 160 NENTRY(_atomic_and_64_nv) 161 movq (%rdi), %rax 162 1: 163 movq %rax, %rcx 164 andq %rsi, %rcx 165 LOCK(16) 166 cmpxchgq %rcx, (%rdi) 167 jnz 1b 168 movq %rcx, %rax 169 ret 170 171 NENTRY(_atomic_dec_64) 172 LOCK(17) 173 decq (%rdi) 174 ret 175 176 NENTRY(_atomic_dec_64_nv) 177 movq $-1, %rax 178 LOCK(18) 179 xaddq %rax, (%rdi) 180 decq %rax 181 ret 182 183 NENTRY(_atomic_inc_64) 184 LOCK(19) 185 incq (%rdi) 186 ret 187 188 NENTRY(_atomic_inc_64_nv) 189 movq $1, %rax 190 LOCK(20) 191 xaddq %rax, (%rdi) 192 incq %rax 193 ret 194 195 NENTRY(_atomic_or_64) 196 LOCK(21) 197 orq %rsi, (%rdi) 198 ret 199 200 NENTRY(_atomic_or_64_nv) 201 movq (%rdi), %rax 202 1: 203 movq %rax, %rcx 204 orq %rsi, %rcx 205 LOCK(22) 206 cmpxchgq %rcx, (%rdi) 207 jnz 1b 208 movq %rcx, %rax 209 ret 210 211 NENTRY(_atomic_swap_64) 212 movq %rsi, %rax 213 xchgq %rax, (%rdi) 214 ret 215 216 NENTRY(_atomic_cas_64) 217 movq %rsi, %rax 218 LOCK(24) 219 cmpxchgq %rdx, (%rdi) 220 /* %eax now contains the old value */ 221 ret 222 223 NENTRY(_atomic_cas_64_ni) 224 movq %rsi, %rax 225 cmpxchgq %rdx, (%rdi) 226 /* %eax now contains the old value */ 227 ret 228 229 /* memory barriers */ 230 231 NENTRY(_membar_consumer) 232 LOCK(25) 233 addq $0, -8(%rsp) 234 ret 235 END(membar_consumer_end) 236 237 NENTRY(_membar_producer) 238 /* A store is enough */ 239 movq $0, -8(%rsp) 240 ret 241 END(membar_producer_end) 242 243 NENTRY(_membar_sync) 244 LOCK(26) 245 addq $0, -8(%rsp) 246 ret 247 END(membar_sync_end) 248 249 #ifdef _KERNEL 250 NENTRY(sse2_lfence) 251 lfence 252 ret 253 END(sse2_lfence_end) 254 255 NENTRY(sse2_mfence) 256 mfence 257 ret 258 END(sse2_mfence_end) 259 260 atomic_lockpatch: 261 .globl atomic_lockpatch 262 .quad .Lpatch1, .Lpatch2, .Lpatch3, .Lpatch4, .Lpatch5 263 .quad .Lpatch6, .Lpatch7, .Lpatch8, .Lpatch9, .Lpatch10 264 .quad .Lpatch12, .Lpatch13, .Lpatch14, .Lpatch15 265 .quad .Lpatch16, .Lpatch17, .Lpatch18, .Lpatch19, .Lpatch20 266 .quad .Lpatch21, .Lpatch22, .Lpatch24, .Lpatch25 267 .quad .Lpatch26, 0 268 #endif /* _KERNEL */ 269 270 ALIAS(atomic_add_32,_atomic_add_32) 271 ALIAS(atomic_add_64,_atomic_add_64) 272 ALIAS(atomic_add_int,_atomic_add_32) 273 ALIAS(atomic_add_long,_atomic_add_64) 274 ALIAS(atomic_add_ptr,_atomic_add_64) 275 276 ALIAS(atomic_add_32_nv,_atomic_add_32_nv) 277 ALIAS(atomic_add_64_nv,_atomic_add_64_nv) 278 ALIAS(atomic_add_int_nv,_atomic_add_32_nv) 279 ALIAS(atomic_add_long_nv,_atomic_add_64_nv) 280 ALIAS(atomic_add_ptr_nv,_atomic_add_64_nv) 281 282 ALIAS(atomic_and_32,_atomic_and_32) 283 ALIAS(atomic_and_64,_atomic_and_64) 284 ALIAS(atomic_and_uint,_atomic_and_32) 285 ALIAS(atomic_and_ulong,_atomic_and_64) 286 ALIAS(atomic_and_ptr,_atomic_and_64) 287 288 ALIAS(atomic_and_32_nv,_atomic_and_32_nv) 289 ALIAS(atomic_and_64_nv,_atomic_and_64_nv) 290 ALIAS(atomic_and_uint_nv,_atomic_and_32_nv) 291 ALIAS(atomic_and_ulong_nv,_atomic_and_64_nv) 292 ALIAS(atomic_and_ptr_nv,_atomic_and_64_nv) 293 294 ALIAS(atomic_dec_32,_atomic_dec_32) 295 ALIAS(atomic_dec_64,_atomic_dec_64) 296 ALIAS(atomic_dec_uint,_atomic_dec_32) 297 ALIAS(atomic_dec_ulong,_atomic_dec_64) 298 ALIAS(atomic_dec_ptr,_atomic_dec_64) 299 300 ALIAS(atomic_dec_32_nv,_atomic_dec_32_nv) 301 ALIAS(atomic_dec_64_nv,_atomic_dec_64_nv) 302 ALIAS(atomic_dec_uint_nv,_atomic_dec_32_nv) 303 ALIAS(atomic_dec_ulong_nv,_atomic_dec_64_nv) 304 ALIAS(atomic_dec_ptr_nv,_atomic_dec_64_nv) 305 306 ALIAS(atomic_inc_32,_atomic_inc_32) 307 ALIAS(atomic_inc_64,_atomic_inc_64) 308 ALIAS(atomic_inc_uint,_atomic_inc_32) 309 ALIAS(atomic_inc_ulong,_atomic_inc_64) 310 ALIAS(atomic_inc_ptr,_atomic_inc_64) 311 312 ALIAS(atomic_inc_32_nv,_atomic_inc_32_nv) 313 ALIAS(atomic_inc_64_nv,_atomic_inc_64_nv) 314 ALIAS(atomic_inc_uint_nv,_atomic_inc_32_nv) 315 ALIAS(atomic_inc_ulong_nv,_atomic_inc_64_nv) 316 ALIAS(atomic_inc_ptr_nv,_atomic_inc_64_nv) 317 318 ALIAS(atomic_or_32,_atomic_or_32) 319 ALIAS(atomic_or_uint,_atomic_or_32) 320 ALIAS(atomic_or_ulong,_atomic_or_64) 321 ALIAS(atomic_or_ptr,_atomic_or_64) 322 323 ALIAS(atomic_or_32_nv,_atomic_or_32_nv) 324 ALIAS(atomic_or_64_nv,_atomic_or_64_nv) 325 ALIAS(atomic_or_uint_nv,_atomic_or_32_nv) 326 ALIAS(atomic_or_ulong_nv,_atomic_or_64_nv) 327 ALIAS(atomic_or_ptr_nv,_atomic_or_64_nv) 328 329 ALIAS(atomic_swap_32,_atomic_swap_32) 330 ALIAS(atomic_swap_64,_atomic_swap_64) 331 ALIAS(atomic_swap_uint,_atomic_swap_32) 332 ALIAS(atomic_swap_ulong,_atomic_swap_64) 333 ALIAS(atomic_swap_ptr,_atomic_swap_64) 334 335 ALIAS(atomic_cas_32,_atomic_cas_32) 336 ALIAS(atomic_cas_64,_atomic_cas_64) 337 ALIAS(atomic_cas_uint,_atomic_cas_32) 338 ALIAS(atomic_cas_ulong,_atomic_cas_64) 339 ALIAS(atomic_cas_ptr,_atomic_cas_64) 340 341 ALIAS(atomic_cas_32_ni,_atomic_cas_32_ni) 342 ALIAS(atomic_cas_64_ni,_atomic_cas_64_ni) 343 ALIAS(atomic_cas_uint_ni,_atomic_cas_32_ni) 344 ALIAS(atomic_cas_ulong_ni,_atomic_cas_64_ni) 345 ALIAS(atomic_cas_ptr_ni,_atomic_cas_64_ni) 346 347 ALIAS(membar_consumer,_membar_consumer) 348 ALIAS(membar_producer,_membar_producer) 349 ALIAS(membar_enter,_membar_consumer) 350 ALIAS(membar_exit,_membar_producer) 351 ALIAS(membar_sync,_membar_sync) 352 353 STRONG_ALIAS(_atomic_add_int,_atomic_add_32) 354 STRONG_ALIAS(_atomic_add_long,_atomic_add_64) 355 STRONG_ALIAS(_atomic_add_ptr,_atomic_add_64) 356 357 STRONG_ALIAS(_atomic_add_int_nv,_atomic_add_32_nv) 358 STRONG_ALIAS(_atomic_add_long_nv,_atomic_add_64_nv) 359 STRONG_ALIAS(_atomic_add_ptr_nv,_atomic_add_64_nv) 360 361 STRONG_ALIAS(_atomic_and_uint,_atomic_and_32) 362 STRONG_ALIAS(_atomic_and_ulong,_atomic_and_64) 363 STRONG_ALIAS(_atomic_and_ptr,_atomic_and_64) 364 365 STRONG_ALIAS(_atomic_and_uint_nv,_atomic_and_32_nv) 366 STRONG_ALIAS(_atomic_and_ulong_nv,_atomic_and_64_nv) 367 STRONG_ALIAS(_atomic_and_ptr_nv,_atomic_and_64_nv) 368 369 STRONG_ALIAS(_atomic_dec_uint,_atomic_dec_32) 370 STRONG_ALIAS(_atomic_dec_ulong,_atomic_dec_64) 371 STRONG_ALIAS(_atomic_dec_ptr,_atomic_dec_64) 372 373 STRONG_ALIAS(_atomic_dec_uint_nv,_atomic_dec_32_nv) 374 STRONG_ALIAS(_atomic_dec_ulong_nv,_atomic_dec_64_nv) 375 STRONG_ALIAS(_atomic_dec_ptr_nv,_atomic_dec_64_nv) 376 377 STRONG_ALIAS(_atomic_inc_uint,_atomic_inc_32) 378 STRONG_ALIAS(_atomic_inc_ulong,_atomic_inc_64) 379 STRONG_ALIAS(_atomic_inc_ptr,_atomic_inc_64) 380 381 STRONG_ALIAS(_atomic_inc_uint_nv,_atomic_inc_32_nv) 382 STRONG_ALIAS(_atomic_inc_ulong_nv,_atomic_inc_64_nv) 383 STRONG_ALIAS(_atomic_inc_ptr_nv,_atomic_inc_64_nv) 384 385 STRONG_ALIAS(_atomic_or_uint,_atomic_or_32) 386 STRONG_ALIAS(_atomic_or_ulong,_atomic_or_64) 387 STRONG_ALIAS(_atomic_or_ptr,_atomic_or_64) 388 389 STRONG_ALIAS(_atomic_or_uint_nv,_atomic_or_32_nv) 390 STRONG_ALIAS(_atomic_or_ulong_nv,_atomic_or_64_nv) 391 STRONG_ALIAS(_atomic_or_ptr_nv,_atomic_or_64_nv) 392 393 STRONG_ALIAS(_atomic_swap_uint,_atomic_swap_32) 394 STRONG_ALIAS(_atomic_swap_ulong,_atomic_swap_64) 395 STRONG_ALIAS(_atomic_swap_ptr,_atomic_swap_64) 396 397 STRONG_ALIAS(_atomic_cas_uint,_atomic_cas_32) 398 STRONG_ALIAS(_atomic_cas_ulong,_atomic_cas_64) 399 STRONG_ALIAS(_atomic_cas_ptr,_atomic_cas_64) 400 401 STRONG_ALIAS(_atomic_cas_uint_ni,_atomic_cas_32_ni) 402 STRONG_ALIAS(_atomic_cas_ulong_ni,_atomic_cas_64_ni) 403 STRONG_ALIAS(_atomic_cas_ptr_ni,_atomic_cas_64_ni) 404 405 STRONG_ALIAS(_membar_enter,_membar_consumer) 406 STRONG_ALIAS(_membar_exit,_membar_producer) 407
Indexes created Tue Sep 30 07:10:03 GMT 2025