atomic.S revision 1.15
1 /* $NetBSD: atomic.S,v 1.15 2009/11/26 07:40:34 pooka Exp $ */ 2 3 /*- 4 * Copyright (c) 2007 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/param.h> 33 #include <machine/asm.h> 34 35 #ifdef _KERNEL 36 #define ALIAS(f, t) STRONG_ALIAS(f,t) 37 #else 38 #define ALIAS(f, t) WEAK_ALIAS(f,t) 39 #endif 40 41 #ifdef _HARDKERNEL 42 #define LOCK(n) .Lpatch/**/n: lock 43 #define ENDLABEL(a) _ALIGN_TEXT; LABEL(a) 44 #else 45 #define LOCK(n) lock 46 #define ENDLABEL(a) /* nothing */ 47 #endif 48 49 .text 50 51 /* 32-bit */ 52 53 ENTRY(_atomic_add_32) 54 LOCK(1) 55 addl %esi, (%rdi) 56 ret 57 58 ENTRY(_atomic_add_32_nv) 59 movl %esi, %eax 60 LOCK(2) 61 xaddl %eax, (%rdi) 62 addl %esi, %eax 63 ret 64 65 ENTRY(_atomic_and_32) 66 LOCK(3) 67 andl %esi, (%rdi) 68 ret 69 70 ENTRY(_atomic_and_32_nv) 71 movl (%rdi), %eax 72 1: 73 movl %eax, %ecx 74 andl %esi, %ecx 75 LOCK(4) 76 cmpxchgl %ecx, (%rdi) 77 jnz 1b 78 movl %ecx, %eax 79 ret 80 81 ENTRY(_atomic_dec_32) 82 LOCK(5) 83 decl (%rdi) 84 ret 85 86 ENTRY(_atomic_dec_32_nv) 87 movl $-1, %eax 88 LOCK(6) 89 xaddl %eax, (%rdi) 90 decl %eax 91 ret 92 93 ENTRY(_atomic_inc_32) 94 LOCK(7) 95 incl (%rdi) 96 ret 97 98 ENTRY(_atomic_inc_32_nv) 99 movl $1, %eax 100 LOCK(8) 101 xaddl %eax, (%rdi) 102 incl %eax 103 ret 104 105 ENTRY(_atomic_or_32) 106 LOCK(9) 107 orl %esi, (%rdi) 108 ret 109 110 ENTRY(_atomic_or_32_nv) 111 movl (%rdi), %eax 112 1: 113 movl %eax, %ecx 114 orl %esi, %ecx 115 LOCK(10) 116 cmpxchgl %ecx, (%rdi) 117 jnz 1b 118 movl %ecx, %eax 119 ret 120 121 ENTRY(_atomic_swap_32) 122 movl %esi, %eax 123 xchgl %eax, (%rdi) 124 ret 125 126 ENTRY(_atomic_cas_32) 127 movl %esi, %eax 128 LOCK(12) 129 cmpxchgl %edx, (%rdi) 130 /* %eax now contains the old value */ 131 ret 132 133 ENTRY(_atomic_cas_32_ni) 134 movl %esi, %eax 135 cmpxchgl %edx, (%rdi) 136 /* %eax now contains the old value */ 137 ret 138 139 /* 64-bit */ 140 141 ENTRY(_atomic_add_64) 142 LOCK(13) 143 addq %rsi, (%rdi) 144 ret 145 146 ENTRY(_atomic_add_64_nv) 147 movq %rsi, %rax 148 LOCK(14) 149 xaddq %rax, (%rdi) 150 addq %rsi, %rax 151 ret 152 153 ENTRY(_atomic_and_64) 154 LOCK(15) 155 andq %rsi, (%rdi) 156 ret 157 158 ENTRY(_atomic_and_64_nv) 159 movq (%rdi), %rax 160 1: 161 movq %rax, %rcx 162 andq %rsi, %rcx 163 LOCK(16) 164 cmpxchgq %rcx, (%rdi) 165 jnz 1b 166 movq %rcx, %rax 167 ret 168 169 ENTRY(_atomic_dec_64) 170 LOCK(17) 171 decq (%rdi) 172 ret 173 174 ENTRY(_atomic_dec_64_nv) 175 movq $-1, %rax 176 LOCK(18) 177 xaddq %rax, (%rdi) 178 decq %rax 179 ret 180 181 ENTRY(_atomic_inc_64) 182 LOCK(19) 183 incq (%rdi) 184 ret 185 186 ENTRY(_atomic_inc_64_nv) 187 movq $1, %rax 188 LOCK(20) 189 xaddq %rax, (%rdi) 190 incq %rax 191 ret 192 193 ENTRY(_atomic_or_64) 194 LOCK(21) 195 orq %rsi, (%rdi) 196 ret 197 198 ENTRY(_atomic_or_64_nv) 199 movq (%rdi), %rax 200 1: 201 movq %rax, %rcx 202 orq %rsi, %rcx 203 LOCK(22) 204 cmpxchgq %rcx, (%rdi) 205 jnz 1b 206 movq %rcx, %rax 207 ret 208 209 ENTRY(_atomic_swap_64) 210 movq %rsi, %rax 211 xchgq %rax, (%rdi) 212 ret 213 214 ENTRY(_atomic_cas_64) 215 movq %rsi, %rax 216 LOCK(24) 217 cmpxchgq %rdx, (%rdi) 218 /* %eax now contains the old value */ 219 ret 220 221 ENTRY(_atomic_cas_64_ni) 222 movq %rsi, %rax 223 cmpxchgq %rdx, (%rdi) 224 /* %eax now contains the old value */ 225 ret 226 227 /* memory barriers */ 228 229 ENTRY(_membar_consumer) 230 LOCK(25) 231 addq $0, -8(%rsp) 232 ret 233 ENDLABEL(membar_consumer_end) 234 235 ENTRY(_membar_producer) 236 /* A store is enough */ 237 movq $0, -8(%rsp) 238 ret 239 ENDLABEL(membar_producer_end) 240 241 ENTRY(_membar_sync) 242 LOCK(26) 243 addq $0, -8(%rsp) 244 ret 245 ENDLABEL(membar_sync_end) 246 247 #ifdef _HARDKERNEL 248 ENTRY(sse2_lfence) 249 lfence 250 ret 251 ENDLABEL(sse2_lfence_end) 252 253 ENTRY(sse2_mfence) 254 mfence 255 ret 256 ENDLABEL(sse2_mfence_end) 257 258 atomic_lockpatch: 259 .globl atomic_lockpatch 260 .quad .Lpatch1, .Lpatch2, .Lpatch3, .Lpatch4, .Lpatch5 261 .quad .Lpatch6, .Lpatch7, .Lpatch8, .Lpatch9, .Lpatch10 262 .quad .Lpatch12, .Lpatch13, .Lpatch14, .Lpatch15 263 .quad .Lpatch16, .Lpatch17, .Lpatch18, .Lpatch19, .Lpatch20 264 .quad .Lpatch21, .Lpatch22, .Lpatch24, .Lpatch25 265 .quad .Lpatch26, 0 266 #endif /* _HARDKERNEL */ 267 268 ALIAS(atomic_add_32,_atomic_add_32) 269 ALIAS(atomic_add_64,_atomic_add_64) 270 ALIAS(atomic_add_int,_atomic_add_32) 271 ALIAS(atomic_add_long,_atomic_add_64) 272 ALIAS(atomic_add_ptr,_atomic_add_64) 273 274 ALIAS(atomic_add_32_nv,_atomic_add_32_nv) 275 ALIAS(atomic_add_64_nv,_atomic_add_64_nv) 276 ALIAS(atomic_add_int_nv,_atomic_add_32_nv) 277 ALIAS(atomic_add_long_nv,_atomic_add_64_nv) 278 ALIAS(atomic_add_ptr_nv,_atomic_add_64_nv) 279 280 ALIAS(atomic_and_32,_atomic_and_32) 281 ALIAS(atomic_and_64,_atomic_and_64) 282 ALIAS(atomic_and_uint,_atomic_and_32) 283 ALIAS(atomic_and_ulong,_atomic_and_64) 284 ALIAS(atomic_and_ptr,_atomic_and_64) 285 286 ALIAS(atomic_and_32_nv,_atomic_and_32_nv) 287 ALIAS(atomic_and_64_nv,_atomic_and_64_nv) 288 ALIAS(atomic_and_uint_nv,_atomic_and_32_nv) 289 ALIAS(atomic_and_ulong_nv,_atomic_and_64_nv) 290 ALIAS(atomic_and_ptr_nv,_atomic_and_64_nv) 291 292 ALIAS(atomic_dec_32,_atomic_dec_32) 293 ALIAS(atomic_dec_64,_atomic_dec_64) 294 ALIAS(atomic_dec_uint,_atomic_dec_32) 295 ALIAS(atomic_dec_ulong,_atomic_dec_64) 296 ALIAS(atomic_dec_ptr,_atomic_dec_64) 297 298 ALIAS(atomic_dec_32_nv,_atomic_dec_32_nv) 299 ALIAS(atomic_dec_64_nv,_atomic_dec_64_nv) 300 ALIAS(atomic_dec_uint_nv,_atomic_dec_32_nv) 301 ALIAS(atomic_dec_ulong_nv,_atomic_dec_64_nv) 302 ALIAS(atomic_dec_ptr_nv,_atomic_dec_64_nv) 303 304 ALIAS(atomic_inc_32,_atomic_inc_32) 305 ALIAS(atomic_inc_64,_atomic_inc_64) 306 ALIAS(atomic_inc_uint,_atomic_inc_32) 307 ALIAS(atomic_inc_ulong,_atomic_inc_64) 308 ALIAS(atomic_inc_ptr,_atomic_inc_64) 309 310 ALIAS(atomic_inc_32_nv,_atomic_inc_32_nv) 311 ALIAS(atomic_inc_64_nv,_atomic_inc_64_nv) 312 ALIAS(atomic_inc_uint_nv,_atomic_inc_32_nv) 313 ALIAS(atomic_inc_ulong_nv,_atomic_inc_64_nv) 314 ALIAS(atomic_inc_ptr_nv,_atomic_inc_64_nv) 315 316 ALIAS(atomic_or_32,_atomic_or_32) 317 ALIAS(atomic_or_uint,_atomic_or_32) 318 ALIAS(atomic_or_ulong,_atomic_or_64) 319 ALIAS(atomic_or_ptr,_atomic_or_64) 320 321 ALIAS(atomic_or_32_nv,_atomic_or_32_nv) 322 ALIAS(atomic_or_64_nv,_atomic_or_64_nv) 323 ALIAS(atomic_or_uint_nv,_atomic_or_32_nv) 324 ALIAS(atomic_or_ulong_nv,_atomic_or_64_nv) 325 ALIAS(atomic_or_ptr_nv,_atomic_or_64_nv) 326 327 ALIAS(atomic_swap_32,_atomic_swap_32) 328 ALIAS(atomic_swap_64,_atomic_swap_64) 329 ALIAS(atomic_swap_uint,_atomic_swap_32) 330 ALIAS(atomic_swap_ulong,_atomic_swap_64) 331 ALIAS(atomic_swap_ptr,_atomic_swap_64) 332 333 ALIAS(atomic_cas_32,_atomic_cas_32) 334 ALIAS(atomic_cas_64,_atomic_cas_64) 335 ALIAS(atomic_cas_uint,_atomic_cas_32) 336 ALIAS(atomic_cas_ulong,_atomic_cas_64) 337 ALIAS(atomic_cas_ptr,_atomic_cas_64) 338 339 ALIAS(atomic_cas_32_ni,_atomic_cas_32_ni) 340 ALIAS(atomic_cas_64_ni,_atomic_cas_64_ni) 341 ALIAS(atomic_cas_uint_ni,_atomic_cas_32_ni) 342 ALIAS(atomic_cas_ulong_ni,_atomic_cas_64_ni) 343 ALIAS(atomic_cas_ptr_ni,_atomic_cas_64_ni) 344 345 ALIAS(membar_consumer,_membar_consumer) 346 ALIAS(membar_producer,_membar_producer) 347 ALIAS(membar_enter,_membar_consumer) 348 ALIAS(membar_exit,_membar_producer) 349 ALIAS(membar_sync,_membar_sync) 350 351 STRONG_ALIAS(_atomic_add_int,_atomic_add_32) 352 STRONG_ALIAS(_atomic_add_long,_atomic_add_64) 353 STRONG_ALIAS(_atomic_add_ptr,_atomic_add_64) 354 355 STRONG_ALIAS(_atomic_add_int_nv,_atomic_add_32_nv) 356 STRONG_ALIAS(_atomic_add_long_nv,_atomic_add_64_nv) 357 STRONG_ALIAS(_atomic_add_ptr_nv,_atomic_add_64_nv) 358 359 STRONG_ALIAS(_atomic_and_uint,_atomic_and_32) 360 STRONG_ALIAS(_atomic_and_ulong,_atomic_and_64) 361 STRONG_ALIAS(_atomic_and_ptr,_atomic_and_64) 362 363 STRONG_ALIAS(_atomic_and_uint_nv,_atomic_and_32_nv) 364 STRONG_ALIAS(_atomic_and_ulong_nv,_atomic_and_64_nv) 365 STRONG_ALIAS(_atomic_and_ptr_nv,_atomic_and_64_nv) 366 367 STRONG_ALIAS(_atomic_dec_uint,_atomic_dec_32) 368 STRONG_ALIAS(_atomic_dec_ulong,_atomic_dec_64) 369 STRONG_ALIAS(_atomic_dec_ptr,_atomic_dec_64) 370 371 STRONG_ALIAS(_atomic_dec_uint_nv,_atomic_dec_32_nv) 372 STRONG_ALIAS(_atomic_dec_ulong_nv,_atomic_dec_64_nv) 373 STRONG_ALIAS(_atomic_dec_ptr_nv,_atomic_dec_64_nv) 374 375 STRONG_ALIAS(_atomic_inc_uint,_atomic_inc_32) 376 STRONG_ALIAS(_atomic_inc_ulong,_atomic_inc_64) 377 STRONG_ALIAS(_atomic_inc_ptr,_atomic_inc_64) 378 379 STRONG_ALIAS(_atomic_inc_uint_nv,_atomic_inc_32_nv) 380 STRONG_ALIAS(_atomic_inc_ulong_nv,_atomic_inc_64_nv) 381 STRONG_ALIAS(_atomic_inc_ptr_nv,_atomic_inc_64_nv) 382 383 STRONG_ALIAS(_atomic_or_uint,_atomic_or_32) 384 STRONG_ALIAS(_atomic_or_ulong,_atomic_or_64) 385 STRONG_ALIAS(_atomic_or_ptr,_atomic_or_64) 386 387 STRONG_ALIAS(_atomic_or_uint_nv,_atomic_or_32_nv) 388 STRONG_ALIAS(_atomic_or_ulong_nv,_atomic_or_64_nv) 389 STRONG_ALIAS(_atomic_or_ptr_nv,_atomic_or_64_nv) 390 391 STRONG_ALIAS(_atomic_swap_uint,_atomic_swap_32) 392 STRONG_ALIAS(_atomic_swap_ulong,_atomic_swap_64) 393 STRONG_ALIAS(_atomic_swap_ptr,_atomic_swap_64) 394 395 STRONG_ALIAS(_atomic_cas_uint,_atomic_cas_32) 396 STRONG_ALIAS(_atomic_cas_ulong,_atomic_cas_64) 397 STRONG_ALIAS(_atomic_cas_ptr,_atomic_cas_64) 398 399 STRONG_ALIAS(_atomic_cas_uint_ni,_atomic_cas_32_ni) 400 STRONG_ALIAS(_atomic_cas_ulong_ni,_atomic_cas_64_ni) 401 STRONG_ALIAS(_atomic_cas_ptr_ni,_atomic_cas_64_ni) 402 403 STRONG_ALIAS(_membar_enter,_membar_consumer) 404 STRONG_ALIAS(_membar_exit,_membar_producer) 405
Indexes created Wed Oct 01 15:09:59 GMT 2025