atomic.S revision 1.9
1 /* $NetBSD: atomic.S,v 1.9 2008/02/10 13:25:46 ad Exp $ */ 2 3 /*- 4 * Copyright (c) 2007 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. All advertising materials mentioning features or use of this software 19 * must display the following acknowledgement: 20 * This product includes software developed by the NetBSD 21 * Foundation, Inc. and its contributors. 22 * 4. Neither the name of The NetBSD Foundation nor the names of its 23 * contributors may be used to endorse or promote products derived 24 * from this software without specific prior written permission. 25 * 26 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 27 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 28 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 29 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 30 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 31 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 32 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 33 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 34 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 35 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 36 * POSSIBILITY OF SUCH DAMAGE. 37 */ 38 39 #include <machine/asm.h> 40 41 #ifdef _KERNEL 42 #define LOCK(n) .Lpatch/**/n: lock 43 #define ALIAS(f, t) STRONG_ALIAS(f,t) 44 #define END(a) _ALIGN_TEXT; LABEL(a) 45 #else 46 #define LOCK(n) lock 47 #define ALIAS(f, t) WEAK_ALIAS(f,t) 48 #define END(a) /* nothing */ 49 #endif 50 51 .text 52 53 NENTRY(_atomic_add_32) 54 movl 4(%esp), %edx 55 movl 8(%esp), %eax 56 LOCK(1) 57 addl %eax, (%edx) 58 ret 59 60 NENTRY(_atomic_add_32_nv) 61 movl 4(%esp), %edx 62 movl 8(%esp), %eax 63 movl %eax, %ecx 64 LOCK(2) 65 xaddl %eax, (%edx) 66 addl %ecx, %eax 67 ret 68 69 NENTRY(_atomic_and_32) 70 movl 4(%esp), %edx 71 movl 8(%esp), %eax 72 LOCK(3) 73 andl %eax, (%edx) 74 ret 75 76 NENTRY(_atomic_and_32_nv) 77 movl 4(%esp), %edx 78 movl (%edx), %eax 79 1: 80 movl %eax, %ecx 81 andl 8(%esp), %ecx 82 LOCK(4) 83 cmpxchgl %ecx, (%edx) 84 jnz 1b 85 movl %ecx, %eax 86 ret 87 88 NENTRY(_atomic_dec_32) 89 movl 4(%esp), %edx 90 LOCK(5) 91 decl (%edx) 92 ret 93 94 NENTRY(_atomic_dec_32_nv) 95 movl 4(%esp), %edx 96 movl $-1, %eax 97 LOCK(6) 98 xaddl %eax, (%edx) 99 decl %eax 100 ret 101 102 NENTRY(_atomic_inc_32) 103 movl 4(%esp), %edx 104 LOCK(7) 105 incl (%edx) 106 ret 107 108 NENTRY(_atomic_inc_32_nv) 109 movl 4(%esp), %edx 110 movl $1, %eax 111 LOCK(8) 112 xaddl %eax, (%edx) 113 incl %eax 114 ret 115 116 NENTRY(_atomic_or_32) 117 movl 4(%esp), %edx 118 movl 8(%esp), %eax 119 LOCK(9) 120 orl %eax, (%edx) 121 ret 122 123 NENTRY(_atomic_or_32_nv) 124 movl 4(%esp), %edx 125 movl (%edx), %eax 126 1: 127 movl %eax, %ecx 128 orl 8(%esp), %ecx 129 LOCK(10) 130 cmpxchgl %ecx, (%edx) 131 jnz 1b 132 movl %ecx, %eax 133 ret 134 135 NENTRY(_atomic_swap_32) 136 movl 4(%esp), %edx 137 movl 8(%esp), %eax 138 xchgl %eax, (%edx) 139 ret 140 141 NENTRY(_atomic_cas_32) 142 movl 4(%esp), %edx 143 movl 8(%esp), %eax 144 movl 12(%esp), %ecx 145 LOCK(12) 146 cmpxchgl %ecx, (%edx) 147 /* %eax now contains the old value */ 148 ret 149 150 NENTRY(_atomic_cas_32_ni) 151 movl 4(%esp), %edx 152 movl 8(%esp), %eax 153 movl 12(%esp), %ecx 154 cmpxchgl %ecx, (%edx) 155 /* %eax now contains the old value */ 156 ret 157 158 NENTRY(_membar_consumer) 159 LOCK(13) 160 addl $0, -4(%esp) 161 ret 162 END(membar_consumer_end) 163 164 NENTRY(_membar_producer) 165 /* A store is enough */ 166 movl $0, -4(%esp) 167 ret 168 END(membar_producer_end) 169 170 NENTRY(_membar_sync) 171 LOCK(14) 172 addl $0, -4(%esp) 173 ret 174 END(membar_sync_end) 175 176 #ifdef _KERNEL 177 NENTRY(_atomic_cas_64) 178 cli 179 pushl %edi 180 pushl %ebx 181 movl 12(%esp), %edi 182 movl 16(%esp), %eax 183 movl 20(%esp), %edx 184 movl 24(%esp), %ebx 185 movl 28(%esp), %ecx 186 cmpl 0(%edi), %eax 187 jne 2f 188 cmpl 4(%edi), %edx 189 jne 2f 190 movl %ebx, 0(%edi) 191 movl %ecx, 4(%edi) 192 1: 193 popl %ebx 194 popl %edi 195 sti 196 ret 197 2: 198 movl 0(%edi), %eax 199 movl 4(%edi), %edx 200 jmp 1b 201 END(_atomic_cas_64_end) 202 203 NENTRY(_atomic_cas_cx8) 204 pushl %edi 205 pushl %ebx 206 movl 12(%esp), %edi 207 movl 16(%esp), %eax 208 movl 20(%esp), %edx 209 movl 24(%esp), %ebx 210 movl 28(%esp), %ecx 211 LOCK(15) 212 cmpxchg8b (%edi) 213 popl %ebx 214 popl %edi 215 ret 216 .space 32, 0x90 217 END(_atomic_cas_cx8_end) 218 219 NENTRY(sse2_lfence) 220 lfence 221 ret 222 END(sse2_lfence_end) 223 224 NENTRY(sse2_mfence) 225 mfence 226 ret 227 END(sse2_mfence_end) 228 229 atomic_lockpatch: 230 .globl atomic_lockpatch 231 .long .Lpatch1, .Lpatch2, .Lpatch3, .Lpatch4, .Lpatch5 232 .long .Lpatch6, .Lpatch7, .Lpatch8, .Lpatch9, .Lpatch10 233 .long .Lpatch12, .Lpatch13, .Lpatch14, .Lpatch15, 0 234 #else 235 NENTRY(_atomic_cas_64) 236 pushl %edi 237 pushl %ebx 238 movl 12(%esp), %edi 239 movl 16(%esp), %eax 240 movl 20(%esp), %edx 241 movl 24(%esp), %ebx 242 movl 28(%esp), %ecx 243 lock 244 cmpxchg8b (%edi) 245 popl %ebx 246 popl %edi 247 ret 248 #endif /* _KERNEL */ 249 250 ALIAS(atomic_add_32,_atomic_add_32) 251 ALIAS(atomic_add_int,_atomic_add_32) 252 ALIAS(atomic_add_long,_atomic_add_32) 253 ALIAS(atomic_add_ptr,_atomic_add_32) 254 255 ALIAS(atomic_add_32_nv,_atomic_add_32_nv) 256 ALIAS(atomic_add_int_nv,_atomic_add_32_nv) 257 ALIAS(atomic_add_long_nv,_atomic_add_32_nv) 258 ALIAS(atomic_add_ptr_nv,_atomic_add_32_nv) 259 260 ALIAS(atomic_and_32,_atomic_and_32) 261 ALIAS(atomic_and_uint,_atomic_and_32) 262 ALIAS(atomic_and_ulong,_atomic_and_32) 263 ALIAS(atomic_and_ptr,_atomic_and_32) 264 265 ALIAS(atomic_and_32_nv,_atomic_and_32_nv) 266 ALIAS(atomic_and_uint_nv,_atomic_and_32_nv) 267 ALIAS(atomic_and_ulong_nv,_atomic_and_32_nv) 268 ALIAS(atomic_and_ptr_nv,_atomic_and_32_nv) 269 270 ALIAS(atomic_dec_32,_atomic_dec_32) 271 ALIAS(atomic_dec_uint,_atomic_dec_32) 272 ALIAS(atomic_dec_ulong,_atomic_dec_32) 273 ALIAS(atomic_dec_ptr,_atomic_dec_32) 274 275 ALIAS(atomic_dec_32_nv,_atomic_dec_32_nv) 276 ALIAS(atomic_dec_uint_nv,_atomic_dec_32_nv) 277 ALIAS(atomic_dec_ulong_nv,_atomic_dec_32_nv) 278 ALIAS(atomic_dec_ptr_nv,_atomic_dec_32_nv) 279 280 ALIAS(atomic_inc_32,_atomic_inc_32) 281 ALIAS(atomic_inc_uint,_atomic_inc_32) 282 ALIAS(atomic_inc_ulong,_atomic_inc_32) 283 ALIAS(atomic_inc_ptr,_atomic_inc_32) 284 285 ALIAS(atomic_inc_32_nv,_atomic_inc_32_nv) 286 ALIAS(atomic_inc_uint_nv,_atomic_inc_32_nv) 287 ALIAS(atomic_inc_ulong_nv,_atomic_inc_32_nv) 288 ALIAS(atomic_inc_ptr_nv,_atomic_inc_32_nv) 289 290 ALIAS(atomic_or_32,_atomic_or_32) 291 ALIAS(atomic_or_uint,_atomic_or_32) 292 ALIAS(atomic_or_ulong,_atomic_or_32) 293 ALIAS(atomic_or_ptr,_atomic_or_32) 294 295 ALIAS(atomic_or_32_nv,_atomic_or_32_nv) 296 ALIAS(atomic_or_uint_nv,_atomic_or_32_nv) 297 ALIAS(atomic_or_ulong_nv,_atomic_or_32_nv) 298 ALIAS(atomic_or_ptr_nv,_atomic_or_32_nv) 299 300 ALIAS(atomic_swap_32,_atomic_swap_32) 301 ALIAS(atomic_swap_uint,_atomic_swap_32) 302 ALIAS(atomic_swap_ulong,_atomic_swap_32) 303 ALIAS(atomic_swap_ptr,_atomic_swap_32) 304 305 ALIAS(atomic_cas_32,_atomic_cas_32) 306 ALIAS(atomic_cas_uint,_atomic_cas_32) 307 ALIAS(atomic_cas_ulong,_atomic_cas_32) 308 ALIAS(atomic_cas_ptr,_atomic_cas_32) 309 310 ALIAS(atomic_cas_32_ni,_atomic_cas_32_ni) 311 ALIAS(atomic_cas_uint_ni,_atomic_cas_32_ni) 312 ALIAS(atomic_cas_ulong_ni,_atomic_cas_32_ni) 313 ALIAS(atomic_cas_ptr_ni,_atomic_cas_32_ni) 314 315 ALIAS(atomic_cas_64,_atomic_cas_64) 316 ALIAS(atomic_cas_64_ni,_atomic_cas_64) 317 318 ALIAS(membar_consumer,_membar_consumer) 319 ALIAS(membar_producer,_membar_producer) 320 ALIAS(membar_enter,_membar_consumer) 321 ALIAS(membar_exit,_membar_producer) 322 ALIAS(membar_sync,_membar_sync) 323 324 STRONG_ALIAS(_atomic_add_int,_atomic_add_32) 325 STRONG_ALIAS(_atomic_add_long,_atomic_add_32) 326 STRONG_ALIAS(_atomic_add_ptr,_atomic_add_32) 327 328 STRONG_ALIAS(_atomic_add_int_nv,_atomic_add_32_nv) 329 STRONG_ALIAS(_atomic_add_long_nv,_atomic_add_32_nv) 330 STRONG_ALIAS(_atomic_add_ptr_nv,_atomic_add_32_nv) 331 332 STRONG_ALIAS(_atomic_and_uint,_atomic_and_32) 333 STRONG_ALIAS(_atomic_and_ulong,_atomic_and_32) 334 STRONG_ALIAS(_atomic_and_ptr,_atomic_and_32) 335 336 STRONG_ALIAS(_atomic_and_uint_nv,_atomic_and_32_nv) 337 STRONG_ALIAS(_atomic_and_ulong_nv,_atomic_and_32_nv) 338 STRONG_ALIAS(_atomic_and_ptr_nv,_atomic_and_32_nv) 339 340 STRONG_ALIAS(_atomic_dec_uint,_atomic_dec_32) 341 STRONG_ALIAS(_atomic_dec_ulong,_atomic_dec_32) 342 STRONG_ALIAS(_atomic_dec_ptr,_atomic_dec_32) 343 344 STRONG_ALIAS(_atomic_dec_uint_nv,_atomic_dec_32_nv) 345 STRONG_ALIAS(_atomic_dec_ulong_nv,_atomic_dec_32_nv) 346 STRONG_ALIAS(_atomic_dec_ptr_nv,_atomic_dec_32_nv) 347 348 STRONG_ALIAS(_atomic_inc_uint,_atomic_inc_32) 349 STRONG_ALIAS(_atomic_inc_ulong,_atomic_inc_32) 350 STRONG_ALIAS(_atomic_inc_ptr,_atomic_inc_32) 351 352 STRONG_ALIAS(_atomic_inc_uint_nv,_atomic_inc_32_nv) 353 STRONG_ALIAS(_atomic_inc_ulong_nv,_atomic_inc_32_nv) 354 STRONG_ALIAS(_atomic_inc_ptr_nv,_atomic_inc_32_nv) 355 356 STRONG_ALIAS(_atomic_or_uint,_atomic_or_32) 357 STRONG_ALIAS(_atomic_or_ulong,_atomic_or_32) 358 STRONG_ALIAS(_atomic_or_ptr,_atomic_or_32) 359 360 STRONG_ALIAS(_atomic_or_uint_nv,_atomic_or_32_nv) 361 STRONG_ALIAS(_atomic_or_ulong_nv,_atomic_or_32_nv) 362 STRONG_ALIAS(_atomic_or_ptr_nv,_atomic_or_32_nv) 363 364 STRONG_ALIAS(_atomic_swap_uint,_atomic_swap_32) 365 STRONG_ALIAS(_atomic_swap_ulong,_atomic_swap_32) 366 STRONG_ALIAS(_atomic_swap_ptr,_atomic_swap_32) 367 368 STRONG_ALIAS(_atomic_cas_uint,_atomic_cas_32) 369 STRONG_ALIAS(_atomic_cas_ulong,_atomic_cas_32) 370 STRONG_ALIAS(_atomic_cas_ptr,_atomic_cas_32) 371 372 STRONG_ALIAS(_atomic_cas_uint_ni,_atomic_cas_32_ni) 373 STRONG_ALIAS(_atomic_cas_ulong_ni,_atomic_cas_32_ni) 374 STRONG_ALIAS(_atomic_cas_ptr_ni,_atomic_cas_32_ni) 375 376 STRONG_ALIAS(_membar_enter,_membar_consumer) 377 STRONG_ALIAS(_membar_exit,_membar_producer) 378
Indexes created Wed Oct 01 07:09:59 GMT 2025