atomic.S revision 1.12
1 /* $NetBSD: atomic.S,v 1.12 2008/05/25 15:56:12 chs Exp $ */ 2 3 /*- 4 * Copyright (c) 2007 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <machine/asm.h> 33 34 #ifdef _KERNEL 35 #define LOCK(n) .Lpatch/**/n: lock 36 #define ALIAS(f, t) STRONG_ALIAS(f,t) 37 #define ENDLABEL(a) _ALIGN_TEXT; LABEL(a) 38 #else 39 #define LOCK(n) lock 40 #define ALIAS(f, t) WEAK_ALIAS(f,t) 41 #define ENDLABEL(a) /* nothing */ 42 #endif 43 44 .text 45 46 /* 32-bit */ 47 48 ENTRY(_atomic_add_32) 49 LOCK(1) 50 addl %esi, (%rdi) 51 ret 52 53 ENTRY(_atomic_add_32_nv) 54 movl %esi, %eax 55 LOCK(2) 56 xaddl %eax, (%rdi) 57 addl %esi, %eax 58 ret 59 60 ENTRY(_atomic_and_32) 61 LOCK(3) 62 andl %esi, (%rdi) 63 ret 64 65 ENTRY(_atomic_and_32_nv) 66 movl (%rdi), %eax 67 1: 68 movl %eax, %ecx 69 andl %esi, %ecx 70 LOCK(4) 71 cmpxchgl %ecx, (%rdi) 72 jnz 1b 73 movl %ecx, %eax 74 ret 75 76 ENTRY(_atomic_dec_32) 77 LOCK(5) 78 decl (%rdi) 79 ret 80 81 ENTRY(_atomic_dec_32_nv) 82 movl $-1, %eax 83 LOCK(6) 84 xaddl %eax, (%rdi) 85 decl %eax 86 ret 87 88 ENTRY(_atomic_inc_32) 89 LOCK(7) 90 incl (%rdi) 91 ret 92 93 ENTRY(_atomic_inc_32_nv) 94 movl $1, %eax 95 LOCK(8) 96 xaddl %eax, (%rdi) 97 incl %eax 98 ret 99 100 ENTRY(_atomic_or_32) 101 LOCK(9) 102 orl %esi, (%rdi) 103 ret 104 105 ENTRY(_atomic_or_32_nv) 106 movl (%rdi), %eax 107 1: 108 movl %eax, %ecx 109 orl %esi, %ecx 110 LOCK(10) 111 cmpxchgl %ecx, (%rdi) 112 jnz 1b 113 movl %ecx, %eax 114 ret 115 116 ENTRY(_atomic_swap_32) 117 movl %esi, %eax 118 xchgl %eax, (%rdi) 119 ret 120 121 ENTRY(_atomic_cas_32) 122 movl %esi, %eax 123 LOCK(12) 124 cmpxchgl %edx, (%rdi) 125 /* %eax now contains the old value */ 126 ret 127 128 ENTRY(_atomic_cas_32_ni) 129 movl %esi, %eax 130 cmpxchgl %edx, (%rdi) 131 /* %eax now contains the old value */ 132 ret 133 134 /* 64-bit */ 135 136 ENTRY(_atomic_add_64) 137 LOCK(13) 138 addq %rsi, (%rdi) 139 ret 140 141 ENTRY(_atomic_add_64_nv) 142 movq %rsi, %rax 143 LOCK(14) 144 xaddq %rax, (%rdi) 145 addq %rsi, %rax 146 ret 147 148 ENTRY(_atomic_and_64) 149 LOCK(15) 150 andq %rsi, (%rdi) 151 ret 152 153 ENTRY(_atomic_and_64_nv) 154 movq (%rdi), %rax 155 1: 156 movq %rax, %rcx 157 andq %rsi, %rcx 158 LOCK(16) 159 cmpxchgq %rcx, (%rdi) 160 jnz 1b 161 movq %rcx, %rax 162 ret 163 164 ENTRY(_atomic_dec_64) 165 LOCK(17) 166 decq (%rdi) 167 ret 168 169 ENTRY(_atomic_dec_64_nv) 170 movq $-1, %rax 171 LOCK(18) 172 xaddq %rax, (%rdi) 173 decq %rax 174 ret 175 176 ENTRY(_atomic_inc_64) 177 LOCK(19) 178 incq (%rdi) 179 ret 180 181 ENTRY(_atomic_inc_64_nv) 182 movq $1, %rax 183 LOCK(20) 184 xaddq %rax, (%rdi) 185 incq %rax 186 ret 187 188 ENTRY(_atomic_or_64) 189 LOCK(21) 190 orq %rsi, (%rdi) 191 ret 192 193 ENTRY(_atomic_or_64_nv) 194 movq (%rdi), %rax 195 1: 196 movq %rax, %rcx 197 orq %rsi, %rcx 198 LOCK(22) 199 cmpxchgq %rcx, (%rdi) 200 jnz 1b 201 movq %rcx, %rax 202 ret 203 204 ENTRY(_atomic_swap_64) 205 movq %rsi, %rax 206 xchgq %rax, (%rdi) 207 ret 208 209 ENTRY(_atomic_cas_64) 210 movq %rsi, %rax 211 LOCK(24) 212 cmpxchgq %rdx, (%rdi) 213 /* %eax now contains the old value */ 214 ret 215 216 ENTRY(_atomic_cas_64_ni) 217 movq %rsi, %rax 218 cmpxchgq %rdx, (%rdi) 219 /* %eax now contains the old value */ 220 ret 221 222 /* memory barriers */ 223 224 ENTRY(_membar_consumer) 225 LOCK(25) 226 addq $0, -8(%rsp) 227 ret 228 ENDLABEL(membar_consumer_end) 229 230 ENTRY(_membar_producer) 231 /* A store is enough */ 232 movq $0, -8(%rsp) 233 ret 234 ENDLABEL(membar_producer_end) 235 236 ENTRY(_membar_sync) 237 LOCK(26) 238 addq $0, -8(%rsp) 239 ret 240 ENDLABEL(membar_sync_end) 241 242 #ifdef _KERNEL 243 ENTRY(sse2_lfence) 244 lfence 245 ret 246 ENDLABEL(sse2_lfence_end) 247 248 ENTRY(sse2_mfence) 249 mfence 250 ret 251 ENDLABEL(sse2_mfence_end) 252 253 atomic_lockpatch: 254 .globl atomic_lockpatch 255 .quad .Lpatch1, .Lpatch2, .Lpatch3, .Lpatch4, .Lpatch5 256 .quad .Lpatch6, .Lpatch7, .Lpatch8, .Lpatch9, .Lpatch10 257 .quad .Lpatch12, .Lpatch13, .Lpatch14, .Lpatch15 258 .quad .Lpatch16, .Lpatch17, .Lpatch18, .Lpatch19, .Lpatch20 259 .quad .Lpatch21, .Lpatch22, .Lpatch24, .Lpatch25 260 .quad .Lpatch26, 0 261 #endif /* _KERNEL */ 262 263 ALIAS(atomic_add_32,_atomic_add_32) 264 ALIAS(atomic_add_64,_atomic_add_64) 265 ALIAS(atomic_add_int,_atomic_add_32) 266 ALIAS(atomic_add_long,_atomic_add_64) 267 ALIAS(atomic_add_ptr,_atomic_add_64) 268 269 ALIAS(atomic_add_32_nv,_atomic_add_32_nv) 270 ALIAS(atomic_add_64_nv,_atomic_add_64_nv) 271 ALIAS(atomic_add_int_nv,_atomic_add_32_nv) 272 ALIAS(atomic_add_long_nv,_atomic_add_64_nv) 273 ALIAS(atomic_add_ptr_nv,_atomic_add_64_nv) 274 275 ALIAS(atomic_and_32,_atomic_and_32) 276 ALIAS(atomic_and_64,_atomic_and_64) 277 ALIAS(atomic_and_uint,_atomic_and_32) 278 ALIAS(atomic_and_ulong,_atomic_and_64) 279 ALIAS(atomic_and_ptr,_atomic_and_64) 280 281 ALIAS(atomic_and_32_nv,_atomic_and_32_nv) 282 ALIAS(atomic_and_64_nv,_atomic_and_64_nv) 283 ALIAS(atomic_and_uint_nv,_atomic_and_32_nv) 284 ALIAS(atomic_and_ulong_nv,_atomic_and_64_nv) 285 ALIAS(atomic_and_ptr_nv,_atomic_and_64_nv) 286 287 ALIAS(atomic_dec_32,_atomic_dec_32) 288 ALIAS(atomic_dec_64,_atomic_dec_64) 289 ALIAS(atomic_dec_uint,_atomic_dec_32) 290 ALIAS(atomic_dec_ulong,_atomic_dec_64) 291 ALIAS(atomic_dec_ptr,_atomic_dec_64) 292 293 ALIAS(atomic_dec_32_nv,_atomic_dec_32_nv) 294 ALIAS(atomic_dec_64_nv,_atomic_dec_64_nv) 295 ALIAS(atomic_dec_uint_nv,_atomic_dec_32_nv) 296 ALIAS(atomic_dec_ulong_nv,_atomic_dec_64_nv) 297 ALIAS(atomic_dec_ptr_nv,_atomic_dec_64_nv) 298 299 ALIAS(atomic_inc_32,_atomic_inc_32) 300 ALIAS(atomic_inc_64,_atomic_inc_64) 301 ALIAS(atomic_inc_uint,_atomic_inc_32) 302 ALIAS(atomic_inc_ulong,_atomic_inc_64) 303 ALIAS(atomic_inc_ptr,_atomic_inc_64) 304 305 ALIAS(atomic_inc_32_nv,_atomic_inc_32_nv) 306 ALIAS(atomic_inc_64_nv,_atomic_inc_64_nv) 307 ALIAS(atomic_inc_uint_nv,_atomic_inc_32_nv) 308 ALIAS(atomic_inc_ulong_nv,_atomic_inc_64_nv) 309 ALIAS(atomic_inc_ptr_nv,_atomic_inc_64_nv) 310 311 ALIAS(atomic_or_32,_atomic_or_32) 312 ALIAS(atomic_or_uint,_atomic_or_32) 313 ALIAS(atomic_or_ulong,_atomic_or_64) 314 ALIAS(atomic_or_ptr,_atomic_or_64) 315 316 ALIAS(atomic_or_32_nv,_atomic_or_32_nv) 317 ALIAS(atomic_or_64_nv,_atomic_or_64_nv) 318 ALIAS(atomic_or_uint_nv,_atomic_or_32_nv) 319 ALIAS(atomic_or_ulong_nv,_atomic_or_64_nv) 320 ALIAS(atomic_or_ptr_nv,_atomic_or_64_nv) 321 322 ALIAS(atomic_swap_32,_atomic_swap_32) 323 ALIAS(atomic_swap_64,_atomic_swap_64) 324 ALIAS(atomic_swap_uint,_atomic_swap_32) 325 ALIAS(atomic_swap_ulong,_atomic_swap_64) 326 ALIAS(atomic_swap_ptr,_atomic_swap_64) 327 328 ALIAS(atomic_cas_32,_atomic_cas_32) 329 ALIAS(atomic_cas_64,_atomic_cas_64) 330 ALIAS(atomic_cas_uint,_atomic_cas_32) 331 ALIAS(atomic_cas_ulong,_atomic_cas_64) 332 ALIAS(atomic_cas_ptr,_atomic_cas_64) 333 334 ALIAS(atomic_cas_32_ni,_atomic_cas_32_ni) 335 ALIAS(atomic_cas_64_ni,_atomic_cas_64_ni) 336 ALIAS(atomic_cas_uint_ni,_atomic_cas_32_ni) 337 ALIAS(atomic_cas_ulong_ni,_atomic_cas_64_ni) 338 ALIAS(atomic_cas_ptr_ni,_atomic_cas_64_ni) 339 340 ALIAS(membar_consumer,_membar_consumer) 341 ALIAS(membar_producer,_membar_producer) 342 ALIAS(membar_enter,_membar_consumer) 343 ALIAS(membar_exit,_membar_producer) 344 ALIAS(membar_sync,_membar_sync) 345 346 STRONG_ALIAS(_atomic_add_int,_atomic_add_32) 347 STRONG_ALIAS(_atomic_add_long,_atomic_add_64) 348 STRONG_ALIAS(_atomic_add_ptr,_atomic_add_64) 349 350 STRONG_ALIAS(_atomic_add_int_nv,_atomic_add_32_nv) 351 STRONG_ALIAS(_atomic_add_long_nv,_atomic_add_64_nv) 352 STRONG_ALIAS(_atomic_add_ptr_nv,_atomic_add_64_nv) 353 354 STRONG_ALIAS(_atomic_and_uint,_atomic_and_32) 355 STRONG_ALIAS(_atomic_and_ulong,_atomic_and_64) 356 STRONG_ALIAS(_atomic_and_ptr,_atomic_and_64) 357 358 STRONG_ALIAS(_atomic_and_uint_nv,_atomic_and_32_nv) 359 STRONG_ALIAS(_atomic_and_ulong_nv,_atomic_and_64_nv) 360 STRONG_ALIAS(_atomic_and_ptr_nv,_atomic_and_64_nv) 361 362 STRONG_ALIAS(_atomic_dec_uint,_atomic_dec_32) 363 STRONG_ALIAS(_atomic_dec_ulong,_atomic_dec_64) 364 STRONG_ALIAS(_atomic_dec_ptr,_atomic_dec_64) 365 366 STRONG_ALIAS(_atomic_dec_uint_nv,_atomic_dec_32_nv) 367 STRONG_ALIAS(_atomic_dec_ulong_nv,_atomic_dec_64_nv) 368 STRONG_ALIAS(_atomic_dec_ptr_nv,_atomic_dec_64_nv) 369 370 STRONG_ALIAS(_atomic_inc_uint,_atomic_inc_32) 371 STRONG_ALIAS(_atomic_inc_ulong,_atomic_inc_64) 372 STRONG_ALIAS(_atomic_inc_ptr,_atomic_inc_64) 373 374 STRONG_ALIAS(_atomic_inc_uint_nv,_atomic_inc_32_nv) 375 STRONG_ALIAS(_atomic_inc_ulong_nv,_atomic_inc_64_nv) 376 STRONG_ALIAS(_atomic_inc_ptr_nv,_atomic_inc_64_nv) 377 378 STRONG_ALIAS(_atomic_or_uint,_atomic_or_32) 379 STRONG_ALIAS(_atomic_or_ulong,_atomic_or_64) 380 STRONG_ALIAS(_atomic_or_ptr,_atomic_or_64) 381 382 STRONG_ALIAS(_atomic_or_uint_nv,_atomic_or_32_nv) 383 STRONG_ALIAS(_atomic_or_ulong_nv,_atomic_or_64_nv) 384 STRONG_ALIAS(_atomic_or_ptr_nv,_atomic_or_64_nv) 385 386 STRONG_ALIAS(_atomic_swap_uint,_atomic_swap_32) 387 STRONG_ALIAS(_atomic_swap_ulong,_atomic_swap_64) 388 STRONG_ALIAS(_atomic_swap_ptr,_atomic_swap_64) 389 390 STRONG_ALIAS(_atomic_cas_uint,_atomic_cas_32) 391 STRONG_ALIAS(_atomic_cas_ulong,_atomic_cas_64) 392 STRONG_ALIAS(_atomic_cas_ptr,_atomic_cas_64) 393 394 STRONG_ALIAS(_atomic_cas_uint_ni,_atomic_cas_32_ni) 395 STRONG_ALIAS(_atomic_cas_ulong_ni,_atomic_cas_64_ni) 396 STRONG_ALIAS(_atomic_cas_ptr_ni,_atomic_cas_64_ni) 397 398 STRONG_ALIAS(_membar_enter,_membar_consumer) 399 STRONG_ALIAS(_membar_exit,_membar_producer) 400
Indexes created Tue Sep 30 20:09:53 GMT 2025