atomic.S revision 1.11
1 /* $NetBSD: atomic.S,v 1.11 2008/05/03 05:41:40 yamt Exp $ */ 2 3 /*- 4 * Copyright (c) 2007 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <machine/asm.h> 33 34 #ifdef _KERNEL 35 #define LOCK(n) .Lpatch/**/n: lock 36 #define ALIAS(f, t) STRONG_ALIAS(f,t) 37 #define ENDLABEL(a) _ALIGN_TEXT; LABEL(a) 38 #else 39 #define LOCK(n) lock 40 #define ALIAS(f, t) WEAK_ALIAS(f,t) 41 #define ENDLABEL(a) /* nothing */ 42 #endif 43 44 .text 45 46 NENTRY(_atomic_add_32) 47 movl 4(%esp), %edx 48 movl 8(%esp), %eax 49 LOCK(1) 50 addl %eax, (%edx) 51 ret 52 53 NENTRY(_atomic_add_32_nv) 54 movl 4(%esp), %edx 55 movl 8(%esp), %eax 56 movl %eax, %ecx 57 LOCK(2) 58 xaddl %eax, (%edx) 59 addl %ecx, %eax 60 ret 61 62 NENTRY(_atomic_and_32) 63 movl 4(%esp), %edx 64 movl 8(%esp), %eax 65 LOCK(3) 66 andl %eax, (%edx) 67 ret 68 69 NENTRY(_atomic_and_32_nv) 70 movl 4(%esp), %edx 71 movl (%edx), %eax 72 1: 73 movl %eax, %ecx 74 andl 8(%esp), %ecx 75 LOCK(4) 76 cmpxchgl %ecx, (%edx) 77 jnz 1b 78 movl %ecx, %eax 79 ret 80 81 NENTRY(_atomic_dec_32) 82 movl 4(%esp), %edx 83 LOCK(5) 84 decl (%edx) 85 ret 86 87 NENTRY(_atomic_dec_32_nv) 88 movl 4(%esp), %edx 89 movl $-1, %eax 90 LOCK(6) 91 xaddl %eax, (%edx) 92 decl %eax 93 ret 94 95 NENTRY(_atomic_inc_32) 96 movl 4(%esp), %edx 97 LOCK(7) 98 incl (%edx) 99 ret 100 101 NENTRY(_atomic_inc_32_nv) 102 movl 4(%esp), %edx 103 movl $1, %eax 104 LOCK(8) 105 xaddl %eax, (%edx) 106 incl %eax 107 ret 108 109 NENTRY(_atomic_or_32) 110 movl 4(%esp), %edx 111 movl 8(%esp), %eax 112 LOCK(9) 113 orl %eax, (%edx) 114 ret 115 116 NENTRY(_atomic_or_32_nv) 117 movl 4(%esp), %edx 118 movl (%edx), %eax 119 1: 120 movl %eax, %ecx 121 orl 8(%esp), %ecx 122 LOCK(10) 123 cmpxchgl %ecx, (%edx) 124 jnz 1b 125 movl %ecx, %eax 126 ret 127 128 NENTRY(_atomic_swap_32) 129 movl 4(%esp), %edx 130 movl 8(%esp), %eax 131 xchgl %eax, (%edx) 132 ret 133 134 NENTRY(_atomic_cas_32) 135 movl 4(%esp), %edx 136 movl 8(%esp), %eax 137 movl 12(%esp), %ecx 138 LOCK(12) 139 cmpxchgl %ecx, (%edx) 140 /* %eax now contains the old value */ 141 ret 142 143 NENTRY(_atomic_cas_32_ni) 144 movl 4(%esp), %edx 145 movl 8(%esp), %eax 146 movl 12(%esp), %ecx 147 cmpxchgl %ecx, (%edx) 148 /* %eax now contains the old value */ 149 ret 150 151 NENTRY(_membar_consumer) 152 LOCK(13) 153 addl $0, -4(%esp) 154 ret 155 ENDLABEL(membar_consumer_end) 156 157 NENTRY(_membar_producer) 158 /* A store is enough */ 159 movl $0, -4(%esp) 160 ret 161 ENDLABEL(membar_producer_end) 162 163 NENTRY(_membar_sync) 164 LOCK(14) 165 addl $0, -4(%esp) 166 ret 167 ENDLABEL(membar_sync_end) 168 169 #ifdef _KERNEL 170 NENTRY(_atomic_cas_64) 171 cli 172 pushl %edi 173 pushl %ebx 174 movl 12(%esp), %edi 175 movl 16(%esp), %eax 176 movl 20(%esp), %edx 177 movl 24(%esp), %ebx 178 movl 28(%esp), %ecx 179 cmpl 0(%edi), %eax 180 jne 2f 181 cmpl 4(%edi), %edx 182 jne 2f 183 movl %ebx, 0(%edi) 184 movl %ecx, 4(%edi) 185 1: 186 popl %ebx 187 popl %edi 188 sti 189 ret 190 2: 191 movl 0(%edi), %eax 192 movl 4(%edi), %edx 193 jmp 1b 194 ENDLABEL(_atomic_cas_64_end) 195 196 NENTRY(_atomic_cas_cx8) 197 pushl %edi 198 pushl %ebx 199 movl 12(%esp), %edi 200 movl 16(%esp), %eax 201 movl 20(%esp), %edx 202 movl 24(%esp), %ebx 203 movl 28(%esp), %ecx 204 LOCK(15) 205 cmpxchg8b (%edi) 206 popl %ebx 207 popl %edi 208 ret 209 .space 32, 0x90 210 ENDLABEL(_atomic_cas_cx8_end) 211 212 NENTRY(sse2_lfence) 213 lfence 214 ret 215 ENDLABEL(sse2_lfence_end) 216 217 NENTRY(sse2_mfence) 218 mfence 219 ret 220 ENDLABEL(sse2_mfence_end) 221 222 atomic_lockpatch: 223 .globl atomic_lockpatch 224 .long .Lpatch1, .Lpatch2, .Lpatch3, .Lpatch4, .Lpatch5 225 .long .Lpatch6, .Lpatch7, .Lpatch8, .Lpatch9, .Lpatch10 226 .long .Lpatch12, .Lpatch13, .Lpatch14, .Lpatch15, 0 227 #else 228 NENTRY(_atomic_cas_64) 229 pushl %edi 230 pushl %ebx 231 movl 12(%esp), %edi 232 movl 16(%esp), %eax 233 movl 20(%esp), %edx 234 movl 24(%esp), %ebx 235 movl 28(%esp), %ecx 236 lock 237 cmpxchg8b (%edi) 238 popl %ebx 239 popl %edi 240 ret 241 #endif /* _KERNEL */ 242 243 ALIAS(atomic_add_32,_atomic_add_32) 244 ALIAS(atomic_add_int,_atomic_add_32) 245 ALIAS(atomic_add_long,_atomic_add_32) 246 ALIAS(atomic_add_ptr,_atomic_add_32) 247 248 ALIAS(atomic_add_32_nv,_atomic_add_32_nv) 249 ALIAS(atomic_add_int_nv,_atomic_add_32_nv) 250 ALIAS(atomic_add_long_nv,_atomic_add_32_nv) 251 ALIAS(atomic_add_ptr_nv,_atomic_add_32_nv) 252 253 ALIAS(atomic_and_32,_atomic_and_32) 254 ALIAS(atomic_and_uint,_atomic_and_32) 255 ALIAS(atomic_and_ulong,_atomic_and_32) 256 ALIAS(atomic_and_ptr,_atomic_and_32) 257 258 ALIAS(atomic_and_32_nv,_atomic_and_32_nv) 259 ALIAS(atomic_and_uint_nv,_atomic_and_32_nv) 260 ALIAS(atomic_and_ulong_nv,_atomic_and_32_nv) 261 ALIAS(atomic_and_ptr_nv,_atomic_and_32_nv) 262 263 ALIAS(atomic_dec_32,_atomic_dec_32) 264 ALIAS(atomic_dec_uint,_atomic_dec_32) 265 ALIAS(atomic_dec_ulong,_atomic_dec_32) 266 ALIAS(atomic_dec_ptr,_atomic_dec_32) 267 268 ALIAS(atomic_dec_32_nv,_atomic_dec_32_nv) 269 ALIAS(atomic_dec_uint_nv,_atomic_dec_32_nv) 270 ALIAS(atomic_dec_ulong_nv,_atomic_dec_32_nv) 271 ALIAS(atomic_dec_ptr_nv,_atomic_dec_32_nv) 272 273 ALIAS(atomic_inc_32,_atomic_inc_32) 274 ALIAS(atomic_inc_uint,_atomic_inc_32) 275 ALIAS(atomic_inc_ulong,_atomic_inc_32) 276 ALIAS(atomic_inc_ptr,_atomic_inc_32) 277 278 ALIAS(atomic_inc_32_nv,_atomic_inc_32_nv) 279 ALIAS(atomic_inc_uint_nv,_atomic_inc_32_nv) 280 ALIAS(atomic_inc_ulong_nv,_atomic_inc_32_nv) 281 ALIAS(atomic_inc_ptr_nv,_atomic_inc_32_nv) 282 283 ALIAS(atomic_or_32,_atomic_or_32) 284 ALIAS(atomic_or_uint,_atomic_or_32) 285 ALIAS(atomic_or_ulong,_atomic_or_32) 286 ALIAS(atomic_or_ptr,_atomic_or_32) 287 288 ALIAS(atomic_or_32_nv,_atomic_or_32_nv) 289 ALIAS(atomic_or_uint_nv,_atomic_or_32_nv) 290 ALIAS(atomic_or_ulong_nv,_atomic_or_32_nv) 291 ALIAS(atomic_or_ptr_nv,_atomic_or_32_nv) 292 293 ALIAS(atomic_swap_32,_atomic_swap_32) 294 ALIAS(atomic_swap_uint,_atomic_swap_32) 295 ALIAS(atomic_swap_ulong,_atomic_swap_32) 296 ALIAS(atomic_swap_ptr,_atomic_swap_32) 297 298 ALIAS(atomic_cas_32,_atomic_cas_32) 299 ALIAS(atomic_cas_uint,_atomic_cas_32) 300 ALIAS(atomic_cas_ulong,_atomic_cas_32) 301 ALIAS(atomic_cas_ptr,_atomic_cas_32) 302 303 ALIAS(atomic_cas_32_ni,_atomic_cas_32_ni) 304 ALIAS(atomic_cas_uint_ni,_atomic_cas_32_ni) 305 ALIAS(atomic_cas_ulong_ni,_atomic_cas_32_ni) 306 ALIAS(atomic_cas_ptr_ni,_atomic_cas_32_ni) 307 308 ALIAS(atomic_cas_64,_atomic_cas_64) 309 ALIAS(atomic_cas_64_ni,_atomic_cas_64) 310 311 ALIAS(membar_consumer,_membar_consumer) 312 ALIAS(membar_producer,_membar_producer) 313 ALIAS(membar_enter,_membar_consumer) 314 ALIAS(membar_exit,_membar_producer) 315 ALIAS(membar_sync,_membar_sync) 316 317 STRONG_ALIAS(_atomic_add_int,_atomic_add_32) 318 STRONG_ALIAS(_atomic_add_long,_atomic_add_32) 319 STRONG_ALIAS(_atomic_add_ptr,_atomic_add_32) 320 321 STRONG_ALIAS(_atomic_add_int_nv,_atomic_add_32_nv) 322 STRONG_ALIAS(_atomic_add_long_nv,_atomic_add_32_nv) 323 STRONG_ALIAS(_atomic_add_ptr_nv,_atomic_add_32_nv) 324 325 STRONG_ALIAS(_atomic_and_uint,_atomic_and_32) 326 STRONG_ALIAS(_atomic_and_ulong,_atomic_and_32) 327 STRONG_ALIAS(_atomic_and_ptr,_atomic_and_32) 328 329 STRONG_ALIAS(_atomic_and_uint_nv,_atomic_and_32_nv) 330 STRONG_ALIAS(_atomic_and_ulong_nv,_atomic_and_32_nv) 331 STRONG_ALIAS(_atomic_and_ptr_nv,_atomic_and_32_nv) 332 333 STRONG_ALIAS(_atomic_dec_uint,_atomic_dec_32) 334 STRONG_ALIAS(_atomic_dec_ulong,_atomic_dec_32) 335 STRONG_ALIAS(_atomic_dec_ptr,_atomic_dec_32) 336 337 STRONG_ALIAS(_atomic_dec_uint_nv,_atomic_dec_32_nv) 338 STRONG_ALIAS(_atomic_dec_ulong_nv,_atomic_dec_32_nv) 339 STRONG_ALIAS(_atomic_dec_ptr_nv,_atomic_dec_32_nv) 340 341 STRONG_ALIAS(_atomic_inc_uint,_atomic_inc_32) 342 STRONG_ALIAS(_atomic_inc_ulong,_atomic_inc_32) 343 STRONG_ALIAS(_atomic_inc_ptr,_atomic_inc_32) 344 345 STRONG_ALIAS(_atomic_inc_uint_nv,_atomic_inc_32_nv) 346 STRONG_ALIAS(_atomic_inc_ulong_nv,_atomic_inc_32_nv) 347 STRONG_ALIAS(_atomic_inc_ptr_nv,_atomic_inc_32_nv) 348 349 STRONG_ALIAS(_atomic_or_uint,_atomic_or_32) 350 STRONG_ALIAS(_atomic_or_ulong,_atomic_or_32) 351 STRONG_ALIAS(_atomic_or_ptr,_atomic_or_32) 352 353 STRONG_ALIAS(_atomic_or_uint_nv,_atomic_or_32_nv) 354 STRONG_ALIAS(_atomic_or_ulong_nv,_atomic_or_32_nv) 355 STRONG_ALIAS(_atomic_or_ptr_nv,_atomic_or_32_nv) 356 357 STRONG_ALIAS(_atomic_swap_uint,_atomic_swap_32) 358 STRONG_ALIAS(_atomic_swap_ulong,_atomic_swap_32) 359 STRONG_ALIAS(_atomic_swap_ptr,_atomic_swap_32) 360 361 STRONG_ALIAS(_atomic_cas_uint,_atomic_cas_32) 362 STRONG_ALIAS(_atomic_cas_ulong,_atomic_cas_32) 363 STRONG_ALIAS(_atomic_cas_ptr,_atomic_cas_32) 364 365 STRONG_ALIAS(_atomic_cas_uint_ni,_atomic_cas_32_ni) 366 STRONG_ALIAS(_atomic_cas_ulong_ni,_atomic_cas_32_ni) 367 STRONG_ALIAS(_atomic_cas_ptr_ni,_atomic_cas_32_ni) 368 369 STRONG_ALIAS(_membar_enter,_membar_consumer) 370 STRONG_ALIAS(_membar_exit,_membar_producer) 371
Indexes created Wed Oct 01 07:09:59 GMT 2025