atomic.S revision 1.18
1 /* $NetBSD: atomic.S,v 1.18 2009/11/26 07:40:34 pooka Exp $ */ 2 3 /*- 4 * Copyright (c) 2007 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/param.h> 33 #include <machine/asm.h> 34 35 #ifdef _KERNEL 36 #define ALIAS(f, t) STRONG_ALIAS(f,t) 37 #else 38 #define ALIAS(f, t) WEAK_ALIAS(f,t) 39 #endif 40 41 #ifdef _HARDKERNEL 42 #define LOCK(n) .Lpatch/**/n: lock 43 #define ENDLABEL(a) _ALIGN_TEXT; LABEL(a) 44 #else 45 #define LOCK(n) lock 46 #define ENDLABEL(a) /* nothing */ 47 #endif 48 49 .text 50 51 ENTRY(_atomic_add_32) 52 movl 4(%esp), %edx 53 movl 8(%esp), %eax 54 LOCK(1) 55 addl %eax, (%edx) 56 ret 57 58 ENTRY(_atomic_add_32_nv) 59 movl 4(%esp), %edx 60 movl 8(%esp), %eax 61 movl %eax, %ecx 62 LOCK(2) 63 xaddl %eax, (%edx) 64 addl %ecx, %eax 65 ret 66 67 ENTRY(_atomic_and_32) 68 movl 4(%esp), %edx 69 movl 8(%esp), %eax 70 LOCK(3) 71 andl %eax, (%edx) 72 ret 73 74 ENTRY(_atomic_and_32_nv) 75 movl 4(%esp), %edx 76 movl (%edx), %eax 77 0: 78 movl %eax, %ecx 79 andl 8(%esp), %ecx 80 LOCK(4) 81 cmpxchgl %ecx, (%edx) 82 jnz 1f 83 movl %ecx, %eax 84 ret 85 1: 86 jmp 0b 87 88 ENTRY(_atomic_dec_32) 89 movl 4(%esp), %edx 90 LOCK(5) 91 decl (%edx) 92 ret 93 94 ENTRY(_atomic_dec_32_nv) 95 movl 4(%esp), %edx 96 movl $-1, %eax 97 LOCK(6) 98 xaddl %eax, (%edx) 99 decl %eax 100 ret 101 102 ENTRY(_atomic_inc_32) 103 movl 4(%esp), %edx 104 LOCK(7) 105 incl (%edx) 106 ret 107 108 ENTRY(_atomic_inc_32_nv) 109 movl 4(%esp), %edx 110 movl $1, %eax 111 LOCK(8) 112 xaddl %eax, (%edx) 113 incl %eax 114 ret 115 116 ENTRY(_atomic_or_32) 117 movl 4(%esp), %edx 118 movl 8(%esp), %eax 119 LOCK(9) 120 orl %eax, (%edx) 121 ret 122 123 ENTRY(_atomic_or_32_nv) 124 movl 4(%esp), %edx 125 movl (%edx), %eax 126 0: 127 movl %eax, %ecx 128 orl 8(%esp), %ecx 129 LOCK(10) 130 cmpxchgl %ecx, (%edx) 131 jnz 1f 132 movl %ecx, %eax 133 ret 134 1: 135 jmp 0b 136 137 ENTRY(_atomic_swap_32) 138 movl 4(%esp), %edx 139 movl 8(%esp), %eax 140 xchgl %eax, (%edx) 141 ret 142 143 ENTRY(_atomic_cas_32) 144 movl 4(%esp), %edx 145 movl 8(%esp), %eax 146 movl 12(%esp), %ecx 147 LOCK(12) 148 cmpxchgl %ecx, (%edx) 149 /* %eax now contains the old value */ 150 ret 151 152 ENTRY(_atomic_cas_32_ni) 153 movl 4(%esp), %edx 154 movl 8(%esp), %eax 155 movl 12(%esp), %ecx 156 cmpxchgl %ecx, (%edx) 157 /* %eax now contains the old value */ 158 ret 159 160 ENTRY(_membar_consumer) 161 LOCK(13) 162 addl $0, -4(%esp) 163 ret 164 ENDLABEL(membar_consumer_end) 165 166 ENTRY(_membar_producer) 167 /* A store is enough */ 168 movl $0, -4(%esp) 169 ret 170 ENDLABEL(membar_producer_end) 171 172 ENTRY(_membar_sync) 173 LOCK(14) 174 addl $0, -4(%esp) 175 ret 176 ENDLABEL(membar_sync_end) 177 178 #ifdef _HARDKERNEL 179 ENTRY(_atomic_cas_64) 180 pushf 181 cli 182 pushl %edi 183 pushl %ebx 184 movl 12(%esp), %edi 185 movl 16(%esp), %eax 186 movl 20(%esp), %edx 187 movl 24(%esp), %ebx 188 movl 28(%esp), %ecx 189 cmpl 0(%edi), %eax 190 jne 2f 191 cmpl 4(%edi), %edx 192 jne 2f 193 movl %ebx, 0(%edi) 194 movl %ecx, 4(%edi) 195 1: 196 popl %ebx 197 popl %edi 198 popf 199 ret 200 2: 201 movl 0(%edi), %eax 202 movl 4(%edi), %edx 203 jmp 1b 204 ENDLABEL(_atomic_cas_64_end) 205 206 ENTRY(_atomic_cas_cx8) 207 pushl %edi 208 pushl %ebx 209 movl 12(%esp), %edi 210 movl 16(%esp), %eax 211 movl 20(%esp), %edx 212 movl 24(%esp), %ebx 213 movl 28(%esp), %ecx 214 LOCK(15) 215 cmpxchg8b (%edi) 216 popl %ebx 217 popl %edi 218 ret 219 #ifdef GPROF 220 .space 16, 0x90 221 #else 222 .space 32, 0x90 223 #endif 224 ENDLABEL(_atomic_cas_cx8_end) 225 226 ENTRY(sse2_lfence) 227 lfence 228 ret 229 ENDLABEL(sse2_lfence_end) 230 231 ENTRY(sse2_mfence) 232 mfence 233 ret 234 ENDLABEL(sse2_mfence_end) 235 236 atomic_lockpatch: 237 .globl atomic_lockpatch 238 .long .Lpatch1, .Lpatch2, .Lpatch3, .Lpatch4, .Lpatch5 239 .long .Lpatch6, .Lpatch7, .Lpatch8, .Lpatch9, .Lpatch10 240 .long .Lpatch12, .Lpatch13, .Lpatch14, .Lpatch15, 0 241 #else 242 ENTRY(_atomic_cas_64) 243 pushl %edi 244 pushl %ebx 245 movl 12(%esp), %edi 246 movl 16(%esp), %eax 247 movl 20(%esp), %edx 248 movl 24(%esp), %ebx 249 movl 28(%esp), %ecx 250 lock 251 cmpxchg8b (%edi) 252 popl %ebx 253 popl %edi 254 ret 255 #endif /* _HARDKERNEL */ 256 257 ALIAS(atomic_add_32,_atomic_add_32) 258 ALIAS(atomic_add_int,_atomic_add_32) 259 ALIAS(atomic_add_long,_atomic_add_32) 260 ALIAS(atomic_add_ptr,_atomic_add_32) 261 262 ALIAS(atomic_add_32_nv,_atomic_add_32_nv) 263 ALIAS(atomic_add_int_nv,_atomic_add_32_nv) 264 ALIAS(atomic_add_long_nv,_atomic_add_32_nv) 265 ALIAS(atomic_add_ptr_nv,_atomic_add_32_nv) 266 267 ALIAS(atomic_and_32,_atomic_and_32) 268 ALIAS(atomic_and_uint,_atomic_and_32) 269 ALIAS(atomic_and_ulong,_atomic_and_32) 270 ALIAS(atomic_and_ptr,_atomic_and_32) 271 272 ALIAS(atomic_and_32_nv,_atomic_and_32_nv) 273 ALIAS(atomic_and_uint_nv,_atomic_and_32_nv) 274 ALIAS(atomic_and_ulong_nv,_atomic_and_32_nv) 275 ALIAS(atomic_and_ptr_nv,_atomic_and_32_nv) 276 277 ALIAS(atomic_dec_32,_atomic_dec_32) 278 ALIAS(atomic_dec_uint,_atomic_dec_32) 279 ALIAS(atomic_dec_ulong,_atomic_dec_32) 280 ALIAS(atomic_dec_ptr,_atomic_dec_32) 281 282 ALIAS(atomic_dec_32_nv,_atomic_dec_32_nv) 283 ALIAS(atomic_dec_uint_nv,_atomic_dec_32_nv) 284 ALIAS(atomic_dec_ulong_nv,_atomic_dec_32_nv) 285 ALIAS(atomic_dec_ptr_nv,_atomic_dec_32_nv) 286 287 ALIAS(atomic_inc_32,_atomic_inc_32) 288 ALIAS(atomic_inc_uint,_atomic_inc_32) 289 ALIAS(atomic_inc_ulong,_atomic_inc_32) 290 ALIAS(atomic_inc_ptr,_atomic_inc_32) 291 292 ALIAS(atomic_inc_32_nv,_atomic_inc_32_nv) 293 ALIAS(atomic_inc_uint_nv,_atomic_inc_32_nv) 294 ALIAS(atomic_inc_ulong_nv,_atomic_inc_32_nv) 295 ALIAS(atomic_inc_ptr_nv,_atomic_inc_32_nv) 296 297 ALIAS(atomic_or_32,_atomic_or_32) 298 ALIAS(atomic_or_uint,_atomic_or_32) 299 ALIAS(atomic_or_ulong,_atomic_or_32) 300 ALIAS(atomic_or_ptr,_atomic_or_32) 301 302 ALIAS(atomic_or_32_nv,_atomic_or_32_nv) 303 ALIAS(atomic_or_uint_nv,_atomic_or_32_nv) 304 ALIAS(atomic_or_ulong_nv,_atomic_or_32_nv) 305 ALIAS(atomic_or_ptr_nv,_atomic_or_32_nv) 306 307 ALIAS(atomic_swap_32,_atomic_swap_32) 308 ALIAS(atomic_swap_uint,_atomic_swap_32) 309 ALIAS(atomic_swap_ulong,_atomic_swap_32) 310 ALIAS(atomic_swap_ptr,_atomic_swap_32) 311 312 ALIAS(atomic_cas_32,_atomic_cas_32) 313 ALIAS(atomic_cas_uint,_atomic_cas_32) 314 ALIAS(atomic_cas_ulong,_atomic_cas_32) 315 ALIAS(atomic_cas_ptr,_atomic_cas_32) 316 317 ALIAS(atomic_cas_32_ni,_atomic_cas_32_ni) 318 ALIAS(atomic_cas_uint_ni,_atomic_cas_32_ni) 319 ALIAS(atomic_cas_ulong_ni,_atomic_cas_32_ni) 320 ALIAS(atomic_cas_ptr_ni,_atomic_cas_32_ni) 321 322 ALIAS(atomic_cas_64,_atomic_cas_64) 323 ALIAS(atomic_cas_64_ni,_atomic_cas_64) 324 325 ALIAS(membar_consumer,_membar_consumer) 326 ALIAS(membar_producer,_membar_producer) 327 ALIAS(membar_enter,_membar_consumer) 328 ALIAS(membar_exit,_membar_producer) 329 ALIAS(membar_sync,_membar_sync) 330 331 STRONG_ALIAS(_atomic_add_int,_atomic_add_32) 332 STRONG_ALIAS(_atomic_add_long,_atomic_add_32) 333 STRONG_ALIAS(_atomic_add_ptr,_atomic_add_32) 334 335 STRONG_ALIAS(_atomic_add_int_nv,_atomic_add_32_nv) 336 STRONG_ALIAS(_atomic_add_long_nv,_atomic_add_32_nv) 337 STRONG_ALIAS(_atomic_add_ptr_nv,_atomic_add_32_nv) 338 339 STRONG_ALIAS(_atomic_and_uint,_atomic_and_32) 340 STRONG_ALIAS(_atomic_and_ulong,_atomic_and_32) 341 STRONG_ALIAS(_atomic_and_ptr,_atomic_and_32) 342 343 STRONG_ALIAS(_atomic_and_uint_nv,_atomic_and_32_nv) 344 STRONG_ALIAS(_atomic_and_ulong_nv,_atomic_and_32_nv) 345 STRONG_ALIAS(_atomic_and_ptr_nv,_atomic_and_32_nv) 346 347 STRONG_ALIAS(_atomic_dec_uint,_atomic_dec_32) 348 STRONG_ALIAS(_atomic_dec_ulong,_atomic_dec_32) 349 STRONG_ALIAS(_atomic_dec_ptr,_atomic_dec_32) 350 351 STRONG_ALIAS(_atomic_dec_uint_nv,_atomic_dec_32_nv) 352 STRONG_ALIAS(_atomic_dec_ulong_nv,_atomic_dec_32_nv) 353 STRONG_ALIAS(_atomic_dec_ptr_nv,_atomic_dec_32_nv) 354 355 STRONG_ALIAS(_atomic_inc_uint,_atomic_inc_32) 356 STRONG_ALIAS(_atomic_inc_ulong,_atomic_inc_32) 357 STRONG_ALIAS(_atomic_inc_ptr,_atomic_inc_32) 358 359 STRONG_ALIAS(_atomic_inc_uint_nv,_atomic_inc_32_nv) 360 STRONG_ALIAS(_atomic_inc_ulong_nv,_atomic_inc_32_nv) 361 STRONG_ALIAS(_atomic_inc_ptr_nv,_atomic_inc_32_nv) 362 363 STRONG_ALIAS(_atomic_or_uint,_atomic_or_32) 364 STRONG_ALIAS(_atomic_or_ulong,_atomic_or_32) 365 STRONG_ALIAS(_atomic_or_ptr,_atomic_or_32) 366 367 STRONG_ALIAS(_atomic_or_uint_nv,_atomic_or_32_nv) 368 STRONG_ALIAS(_atomic_or_ulong_nv,_atomic_or_32_nv) 369 STRONG_ALIAS(_atomic_or_ptr_nv,_atomic_or_32_nv) 370 371 STRONG_ALIAS(_atomic_swap_uint,_atomic_swap_32) 372 STRONG_ALIAS(_atomic_swap_ulong,_atomic_swap_32) 373 STRONG_ALIAS(_atomic_swap_ptr,_atomic_swap_32) 374 375 STRONG_ALIAS(_atomic_cas_uint,_atomic_cas_32) 376 STRONG_ALIAS(_atomic_cas_ulong,_atomic_cas_32) 377 STRONG_ALIAS(_atomic_cas_ptr,_atomic_cas_32) 378 379 STRONG_ALIAS(_atomic_cas_uint_ni,_atomic_cas_32_ni) 380 STRONG_ALIAS(_atomic_cas_ulong_ni,_atomic_cas_32_ni) 381 STRONG_ALIAS(_atomic_cas_ptr_ni,_atomic_cas_32_ni) 382 383 STRONG_ALIAS(_membar_enter,_membar_consumer) 384 STRONG_ALIAS(_membar_exit,_membar_producer) 385
Indexes created Wed Oct 01 15:09:59 GMT 2025