atomic.S revision 1.17
1 /* $NetBSD: atomic.S,v 1.17 2009/04/02 00:19:02 enami Exp $ */ 2 3 /*- 4 * Copyright (c) 2007 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/param.h> 33 #include <machine/asm.h> 34 35 #ifdef _HARDKERNEL 36 #define LOCK(n) .Lpatch/**/n: lock 37 #define ALIAS(f, t) STRONG_ALIAS(f,t) 38 #define ENDLABEL(a) _ALIGN_TEXT; LABEL(a) 39 #else 40 #define LOCK(n) lock 41 #define ALIAS(f, t) WEAK_ALIAS(f,t) 42 #define ENDLABEL(a) /* nothing */ 43 #endif 44 45 .text 46 47 ENTRY(_atomic_add_32) 48 movl 4(%esp), %edx 49 movl 8(%esp), %eax 50 LOCK(1) 51 addl %eax, (%edx) 52 ret 53 54 ENTRY(_atomic_add_32_nv) 55 movl 4(%esp), %edx 56 movl 8(%esp), %eax 57 movl %eax, %ecx 58 LOCK(2) 59 xaddl %eax, (%edx) 60 addl %ecx, %eax 61 ret 62 63 ENTRY(_atomic_and_32) 64 movl 4(%esp), %edx 65 movl 8(%esp), %eax 66 LOCK(3) 67 andl %eax, (%edx) 68 ret 69 70 ENTRY(_atomic_and_32_nv) 71 movl 4(%esp), %edx 72 movl (%edx), %eax 73 0: 74 movl %eax, %ecx 75 andl 8(%esp), %ecx 76 LOCK(4) 77 cmpxchgl %ecx, (%edx) 78 jnz 1f 79 movl %ecx, %eax 80 ret 81 1: 82 jmp 0b 83 84 ENTRY(_atomic_dec_32) 85 movl 4(%esp), %edx 86 LOCK(5) 87 decl (%edx) 88 ret 89 90 ENTRY(_atomic_dec_32_nv) 91 movl 4(%esp), %edx 92 movl $-1, %eax 93 LOCK(6) 94 xaddl %eax, (%edx) 95 decl %eax 96 ret 97 98 ENTRY(_atomic_inc_32) 99 movl 4(%esp), %edx 100 LOCK(7) 101 incl (%edx) 102 ret 103 104 ENTRY(_atomic_inc_32_nv) 105 movl 4(%esp), %edx 106 movl $1, %eax 107 LOCK(8) 108 xaddl %eax, (%edx) 109 incl %eax 110 ret 111 112 ENTRY(_atomic_or_32) 113 movl 4(%esp), %edx 114 movl 8(%esp), %eax 115 LOCK(9) 116 orl %eax, (%edx) 117 ret 118 119 ENTRY(_atomic_or_32_nv) 120 movl 4(%esp), %edx 121 movl (%edx), %eax 122 0: 123 movl %eax, %ecx 124 orl 8(%esp), %ecx 125 LOCK(10) 126 cmpxchgl %ecx, (%edx) 127 jnz 1f 128 movl %ecx, %eax 129 ret 130 1: 131 jmp 0b 132 133 ENTRY(_atomic_swap_32) 134 movl 4(%esp), %edx 135 movl 8(%esp), %eax 136 xchgl %eax, (%edx) 137 ret 138 139 ENTRY(_atomic_cas_32) 140 movl 4(%esp), %edx 141 movl 8(%esp), %eax 142 movl 12(%esp), %ecx 143 LOCK(12) 144 cmpxchgl %ecx, (%edx) 145 /* %eax now contains the old value */ 146 ret 147 148 ENTRY(_atomic_cas_32_ni) 149 movl 4(%esp), %edx 150 movl 8(%esp), %eax 151 movl 12(%esp), %ecx 152 cmpxchgl %ecx, (%edx) 153 /* %eax now contains the old value */ 154 ret 155 156 ENTRY(_membar_consumer) 157 LOCK(13) 158 addl $0, -4(%esp) 159 ret 160 ENDLABEL(membar_consumer_end) 161 162 ENTRY(_membar_producer) 163 /* A store is enough */ 164 movl $0, -4(%esp) 165 ret 166 ENDLABEL(membar_producer_end) 167 168 ENTRY(_membar_sync) 169 LOCK(14) 170 addl $0, -4(%esp) 171 ret 172 ENDLABEL(membar_sync_end) 173 174 #ifdef _HARDKERNEL 175 ENTRY(_atomic_cas_64) 176 pushf 177 cli 178 pushl %edi 179 pushl %ebx 180 movl 12(%esp), %edi 181 movl 16(%esp), %eax 182 movl 20(%esp), %edx 183 movl 24(%esp), %ebx 184 movl 28(%esp), %ecx 185 cmpl 0(%edi), %eax 186 jne 2f 187 cmpl 4(%edi), %edx 188 jne 2f 189 movl %ebx, 0(%edi) 190 movl %ecx, 4(%edi) 191 1: 192 popl %ebx 193 popl %edi 194 popf 195 ret 196 2: 197 movl 0(%edi), %eax 198 movl 4(%edi), %edx 199 jmp 1b 200 ENDLABEL(_atomic_cas_64_end) 201 202 ENTRY(_atomic_cas_cx8) 203 pushl %edi 204 pushl %ebx 205 movl 12(%esp), %edi 206 movl 16(%esp), %eax 207 movl 20(%esp), %edx 208 movl 24(%esp), %ebx 209 movl 28(%esp), %ecx 210 LOCK(15) 211 cmpxchg8b (%edi) 212 popl %ebx 213 popl %edi 214 ret 215 #ifdef GPROF 216 .space 16, 0x90 217 #else 218 .space 32, 0x90 219 #endif 220 ENDLABEL(_atomic_cas_cx8_end) 221 222 ENTRY(sse2_lfence) 223 lfence 224 ret 225 ENDLABEL(sse2_lfence_end) 226 227 ENTRY(sse2_mfence) 228 mfence 229 ret 230 ENDLABEL(sse2_mfence_end) 231 232 atomic_lockpatch: 233 .globl atomic_lockpatch 234 .long .Lpatch1, .Lpatch2, .Lpatch3, .Lpatch4, .Lpatch5 235 .long .Lpatch6, .Lpatch7, .Lpatch8, .Lpatch9, .Lpatch10 236 .long .Lpatch12, .Lpatch13, .Lpatch14, .Lpatch15, 0 237 #else 238 ENTRY(_atomic_cas_64) 239 pushl %edi 240 pushl %ebx 241 movl 12(%esp), %edi 242 movl 16(%esp), %eax 243 movl 20(%esp), %edx 244 movl 24(%esp), %ebx 245 movl 28(%esp), %ecx 246 lock 247 cmpxchg8b (%edi) 248 popl %ebx 249 popl %edi 250 ret 251 #endif /* _HARDKERNEL */ 252 253 ALIAS(atomic_add_32,_atomic_add_32) 254 ALIAS(atomic_add_int,_atomic_add_32) 255 ALIAS(atomic_add_long,_atomic_add_32) 256 ALIAS(atomic_add_ptr,_atomic_add_32) 257 258 ALIAS(atomic_add_32_nv,_atomic_add_32_nv) 259 ALIAS(atomic_add_int_nv,_atomic_add_32_nv) 260 ALIAS(atomic_add_long_nv,_atomic_add_32_nv) 261 ALIAS(atomic_add_ptr_nv,_atomic_add_32_nv) 262 263 ALIAS(atomic_and_32,_atomic_and_32) 264 ALIAS(atomic_and_uint,_atomic_and_32) 265 ALIAS(atomic_and_ulong,_atomic_and_32) 266 ALIAS(atomic_and_ptr,_atomic_and_32) 267 268 ALIAS(atomic_and_32_nv,_atomic_and_32_nv) 269 ALIAS(atomic_and_uint_nv,_atomic_and_32_nv) 270 ALIAS(atomic_and_ulong_nv,_atomic_and_32_nv) 271 ALIAS(atomic_and_ptr_nv,_atomic_and_32_nv) 272 273 ALIAS(atomic_dec_32,_atomic_dec_32) 274 ALIAS(atomic_dec_uint,_atomic_dec_32) 275 ALIAS(atomic_dec_ulong,_atomic_dec_32) 276 ALIAS(atomic_dec_ptr,_atomic_dec_32) 277 278 ALIAS(atomic_dec_32_nv,_atomic_dec_32_nv) 279 ALIAS(atomic_dec_uint_nv,_atomic_dec_32_nv) 280 ALIAS(atomic_dec_ulong_nv,_atomic_dec_32_nv) 281 ALIAS(atomic_dec_ptr_nv,_atomic_dec_32_nv) 282 283 ALIAS(atomic_inc_32,_atomic_inc_32) 284 ALIAS(atomic_inc_uint,_atomic_inc_32) 285 ALIAS(atomic_inc_ulong,_atomic_inc_32) 286 ALIAS(atomic_inc_ptr,_atomic_inc_32) 287 288 ALIAS(atomic_inc_32_nv,_atomic_inc_32_nv) 289 ALIAS(atomic_inc_uint_nv,_atomic_inc_32_nv) 290 ALIAS(atomic_inc_ulong_nv,_atomic_inc_32_nv) 291 ALIAS(atomic_inc_ptr_nv,_atomic_inc_32_nv) 292 293 ALIAS(atomic_or_32,_atomic_or_32) 294 ALIAS(atomic_or_uint,_atomic_or_32) 295 ALIAS(atomic_or_ulong,_atomic_or_32) 296 ALIAS(atomic_or_ptr,_atomic_or_32) 297 298 ALIAS(atomic_or_32_nv,_atomic_or_32_nv) 299 ALIAS(atomic_or_uint_nv,_atomic_or_32_nv) 300 ALIAS(atomic_or_ulong_nv,_atomic_or_32_nv) 301 ALIAS(atomic_or_ptr_nv,_atomic_or_32_nv) 302 303 ALIAS(atomic_swap_32,_atomic_swap_32) 304 ALIAS(atomic_swap_uint,_atomic_swap_32) 305 ALIAS(atomic_swap_ulong,_atomic_swap_32) 306 ALIAS(atomic_swap_ptr,_atomic_swap_32) 307 308 ALIAS(atomic_cas_32,_atomic_cas_32) 309 ALIAS(atomic_cas_uint,_atomic_cas_32) 310 ALIAS(atomic_cas_ulong,_atomic_cas_32) 311 ALIAS(atomic_cas_ptr,_atomic_cas_32) 312 313 ALIAS(atomic_cas_32_ni,_atomic_cas_32_ni) 314 ALIAS(atomic_cas_uint_ni,_atomic_cas_32_ni) 315 ALIAS(atomic_cas_ulong_ni,_atomic_cas_32_ni) 316 ALIAS(atomic_cas_ptr_ni,_atomic_cas_32_ni) 317 318 ALIAS(atomic_cas_64,_atomic_cas_64) 319 ALIAS(atomic_cas_64_ni,_atomic_cas_64) 320 321 ALIAS(membar_consumer,_membar_consumer) 322 ALIAS(membar_producer,_membar_producer) 323 ALIAS(membar_enter,_membar_consumer) 324 ALIAS(membar_exit,_membar_producer) 325 ALIAS(membar_sync,_membar_sync) 326 327 STRONG_ALIAS(_atomic_add_int,_atomic_add_32) 328 STRONG_ALIAS(_atomic_add_long,_atomic_add_32) 329 STRONG_ALIAS(_atomic_add_ptr,_atomic_add_32) 330 331 STRONG_ALIAS(_atomic_add_int_nv,_atomic_add_32_nv) 332 STRONG_ALIAS(_atomic_add_long_nv,_atomic_add_32_nv) 333 STRONG_ALIAS(_atomic_add_ptr_nv,_atomic_add_32_nv) 334 335 STRONG_ALIAS(_atomic_and_uint,_atomic_and_32) 336 STRONG_ALIAS(_atomic_and_ulong,_atomic_and_32) 337 STRONG_ALIAS(_atomic_and_ptr,_atomic_and_32) 338 339 STRONG_ALIAS(_atomic_and_uint_nv,_atomic_and_32_nv) 340 STRONG_ALIAS(_atomic_and_ulong_nv,_atomic_and_32_nv) 341 STRONG_ALIAS(_atomic_and_ptr_nv,_atomic_and_32_nv) 342 343 STRONG_ALIAS(_atomic_dec_uint,_atomic_dec_32) 344 STRONG_ALIAS(_atomic_dec_ulong,_atomic_dec_32) 345 STRONG_ALIAS(_atomic_dec_ptr,_atomic_dec_32) 346 347 STRONG_ALIAS(_atomic_dec_uint_nv,_atomic_dec_32_nv) 348 STRONG_ALIAS(_atomic_dec_ulong_nv,_atomic_dec_32_nv) 349 STRONG_ALIAS(_atomic_dec_ptr_nv,_atomic_dec_32_nv) 350 351 STRONG_ALIAS(_atomic_inc_uint,_atomic_inc_32) 352 STRONG_ALIAS(_atomic_inc_ulong,_atomic_inc_32) 353 STRONG_ALIAS(_atomic_inc_ptr,_atomic_inc_32) 354 355 STRONG_ALIAS(_atomic_inc_uint_nv,_atomic_inc_32_nv) 356 STRONG_ALIAS(_atomic_inc_ulong_nv,_atomic_inc_32_nv) 357 STRONG_ALIAS(_atomic_inc_ptr_nv,_atomic_inc_32_nv) 358 359 STRONG_ALIAS(_atomic_or_uint,_atomic_or_32) 360 STRONG_ALIAS(_atomic_or_ulong,_atomic_or_32) 361 STRONG_ALIAS(_atomic_or_ptr,_atomic_or_32) 362 363 STRONG_ALIAS(_atomic_or_uint_nv,_atomic_or_32_nv) 364 STRONG_ALIAS(_atomic_or_ulong_nv,_atomic_or_32_nv) 365 STRONG_ALIAS(_atomic_or_ptr_nv,_atomic_or_32_nv) 366 367 STRONG_ALIAS(_atomic_swap_uint,_atomic_swap_32) 368 STRONG_ALIAS(_atomic_swap_ulong,_atomic_swap_32) 369 STRONG_ALIAS(_atomic_swap_ptr,_atomic_swap_32) 370 371 STRONG_ALIAS(_atomic_cas_uint,_atomic_cas_32) 372 STRONG_ALIAS(_atomic_cas_ulong,_atomic_cas_32) 373 STRONG_ALIAS(_atomic_cas_ptr,_atomic_cas_32) 374 375 STRONG_ALIAS(_atomic_cas_uint_ni,_atomic_cas_32_ni) 376 STRONG_ALIAS(_atomic_cas_ulong_ni,_atomic_cas_32_ni) 377 STRONG_ALIAS(_atomic_cas_ptr_ni,_atomic_cas_32_ni) 378 379 STRONG_ALIAS(_membar_enter,_membar_consumer) 380 STRONG_ALIAS(_membar_exit,_membar_producer) 381
Indexes created Wed Oct 01 15:09:59 GMT 2025