atomic.S revision 1.14
1 /* $NetBSD: atomic.S,v 1.14 2008/12/19 11:21:24 ad Exp $ */ 2 3 /*- 4 * Copyright (c) 2007 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <machine/asm.h> 33 34 #ifdef _KERNEL 35 #define LOCK(n) .Lpatch/**/n: lock 36 #define ALIAS(f, t) STRONG_ALIAS(f,t) 37 #define ENDLABEL(a) _ALIGN_TEXT; LABEL(a) 38 #else 39 #define LOCK(n) lock 40 #define ALIAS(f, t) WEAK_ALIAS(f,t) 41 #define ENDLABEL(a) /* nothing */ 42 #endif 43 44 .text 45 46 ENTRY(_atomic_add_32) 47 movl 4(%esp), %edx 48 movl 8(%esp), %eax 49 LOCK(1) 50 addl %eax, (%edx) 51 ret 52 53 ENTRY(_atomic_add_32_nv) 54 movl 4(%esp), %edx 55 movl 8(%esp), %eax 56 movl %eax, %ecx 57 LOCK(2) 58 xaddl %eax, (%edx) 59 addl %ecx, %eax 60 ret 61 62 ENTRY(_atomic_and_32) 63 movl 4(%esp), %edx 64 movl 8(%esp), %eax 65 LOCK(3) 66 andl %eax, (%edx) 67 ret 68 69 ENTRY(_atomic_and_32_nv) 70 movl 4(%esp), %edx 71 movl (%edx), %eax 72 0: 73 movl %eax, %ecx 74 andl 8(%esp), %ecx 75 LOCK(4) 76 cmpxchgl %ecx, (%edx) 77 jnz 1f 78 movl %ecx, %eax 79 ret 80 1: 81 jmp 0b 82 83 ENTRY(_atomic_dec_32) 84 movl 4(%esp), %edx 85 LOCK(5) 86 decl (%edx) 87 ret 88 89 ENTRY(_atomic_dec_32_nv) 90 movl 4(%esp), %edx 91 movl $-1, %eax 92 LOCK(6) 93 xaddl %eax, (%edx) 94 decl %eax 95 ret 96 97 ENTRY(_atomic_inc_32) 98 movl 4(%esp), %edx 99 LOCK(7) 100 incl (%edx) 101 ret 102 103 ENTRY(_atomic_inc_32_nv) 104 movl 4(%esp), %edx 105 movl $1, %eax 106 LOCK(8) 107 xaddl %eax, (%edx) 108 incl %eax 109 ret 110 111 ENTRY(_atomic_or_32) 112 movl 4(%esp), %edx 113 movl 8(%esp), %eax 114 LOCK(9) 115 orl %eax, (%edx) 116 ret 117 118 ENTRY(_atomic_or_32_nv) 119 movl 4(%esp), %edx 120 movl (%edx), %eax 121 0: 122 movl %eax, %ecx 123 orl 8(%esp), %ecx 124 LOCK(10) 125 cmpxchgl %ecx, (%edx) 126 jnz 1f 127 movl %ecx, %eax 128 ret 129 1: 130 jmp 0b 131 132 ENTRY(_atomic_swap_32) 133 movl 4(%esp), %edx 134 movl 8(%esp), %eax 135 xchgl %eax, (%edx) 136 ret 137 138 ENTRY(_atomic_cas_32) 139 movl 4(%esp), %edx 140 movl 8(%esp), %eax 141 movl 12(%esp), %ecx 142 LOCK(12) 143 cmpxchgl %ecx, (%edx) 144 /* %eax now contains the old value */ 145 ret 146 147 ENTRY(_atomic_cas_32_ni) 148 movl 4(%esp), %edx 149 movl 8(%esp), %eax 150 movl 12(%esp), %ecx 151 cmpxchgl %ecx, (%edx) 152 /* %eax now contains the old value */ 153 ret 154 155 ENTRY(_membar_consumer) 156 LOCK(13) 157 addl $0, -4(%esp) 158 ret 159 ENDLABEL(membar_consumer_end) 160 161 ENTRY(_membar_producer) 162 /* A store is enough */ 163 movl $0, -4(%esp) 164 ret 165 ENDLABEL(membar_producer_end) 166 167 ENTRY(_membar_sync) 168 LOCK(14) 169 addl $0, -4(%esp) 170 ret 171 ENDLABEL(membar_sync_end) 172 173 #ifdef _KERNEL 174 ENTRY(_atomic_cas_64) 175 pushf 176 cli 177 pushl %edi 178 pushl %ebx 179 movl 12(%esp), %edi 180 movl 16(%esp), %eax 181 movl 20(%esp), %edx 182 movl 24(%esp), %ebx 183 movl 28(%esp), %ecx 184 cmpl 0(%edi), %eax 185 jne 2f 186 cmpl 4(%edi), %edx 187 jne 2f 188 movl %ebx, 0(%edi) 189 movl %ecx, 4(%edi) 190 1: 191 popl %ebx 192 popl %edi 193 popf 194 ret 195 2: 196 movl 0(%edi), %eax 197 movl 4(%edi), %edx 198 jmp 1b 199 ENDLABEL(_atomic_cas_64_end) 200 201 ENTRY(_atomic_cas_cx8) 202 pushl %edi 203 pushl %ebx 204 movl 12(%esp), %edi 205 movl 16(%esp), %eax 206 movl 20(%esp), %edx 207 movl 24(%esp), %ebx 208 movl 28(%esp), %ecx 209 LOCK(15) 210 cmpxchg8b (%edi) 211 popl %ebx 212 popl %edi 213 ret 214 .space 32, 0x90 215 ENDLABEL(_atomic_cas_cx8_end) 216 217 ENTRY(sse2_lfence) 218 lfence 219 ret 220 ENDLABEL(sse2_lfence_end) 221 222 ENTRY(sse2_mfence) 223 mfence 224 ret 225 ENDLABEL(sse2_mfence_end) 226 227 atomic_lockpatch: 228 .globl atomic_lockpatch 229 .long .Lpatch1, .Lpatch2, .Lpatch3, .Lpatch4, .Lpatch5 230 .long .Lpatch6, .Lpatch7, .Lpatch8, .Lpatch9, .Lpatch10 231 .long .Lpatch12, .Lpatch13, .Lpatch14, .Lpatch15, 0 232 #else 233 ENTRY(_atomic_cas_64) 234 pushl %edi 235 pushl %ebx 236 movl 12(%esp), %edi 237 movl 16(%esp), %eax 238 movl 20(%esp), %edx 239 movl 24(%esp), %ebx 240 movl 28(%esp), %ecx 241 lock 242 cmpxchg8b (%edi) 243 popl %ebx 244 popl %edi 245 ret 246 #endif /* _KERNEL */ 247 248 ALIAS(atomic_add_32,_atomic_add_32) 249 ALIAS(atomic_add_int,_atomic_add_32) 250 ALIAS(atomic_add_long,_atomic_add_32) 251 ALIAS(atomic_add_ptr,_atomic_add_32) 252 253 ALIAS(atomic_add_32_nv,_atomic_add_32_nv) 254 ALIAS(atomic_add_int_nv,_atomic_add_32_nv) 255 ALIAS(atomic_add_long_nv,_atomic_add_32_nv) 256 ALIAS(atomic_add_ptr_nv,_atomic_add_32_nv) 257 258 ALIAS(atomic_and_32,_atomic_and_32) 259 ALIAS(atomic_and_uint,_atomic_and_32) 260 ALIAS(atomic_and_ulong,_atomic_and_32) 261 ALIAS(atomic_and_ptr,_atomic_and_32) 262 263 ALIAS(atomic_and_32_nv,_atomic_and_32_nv) 264 ALIAS(atomic_and_uint_nv,_atomic_and_32_nv) 265 ALIAS(atomic_and_ulong_nv,_atomic_and_32_nv) 266 ALIAS(atomic_and_ptr_nv,_atomic_and_32_nv) 267 268 ALIAS(atomic_dec_32,_atomic_dec_32) 269 ALIAS(atomic_dec_uint,_atomic_dec_32) 270 ALIAS(atomic_dec_ulong,_atomic_dec_32) 271 ALIAS(atomic_dec_ptr,_atomic_dec_32) 272 273 ALIAS(atomic_dec_32_nv,_atomic_dec_32_nv) 274 ALIAS(atomic_dec_uint_nv,_atomic_dec_32_nv) 275 ALIAS(atomic_dec_ulong_nv,_atomic_dec_32_nv) 276 ALIAS(atomic_dec_ptr_nv,_atomic_dec_32_nv) 277 278 ALIAS(atomic_inc_32,_atomic_inc_32) 279 ALIAS(atomic_inc_uint,_atomic_inc_32) 280 ALIAS(atomic_inc_ulong,_atomic_inc_32) 281 ALIAS(atomic_inc_ptr,_atomic_inc_32) 282 283 ALIAS(atomic_inc_32_nv,_atomic_inc_32_nv) 284 ALIAS(atomic_inc_uint_nv,_atomic_inc_32_nv) 285 ALIAS(atomic_inc_ulong_nv,_atomic_inc_32_nv) 286 ALIAS(atomic_inc_ptr_nv,_atomic_inc_32_nv) 287 288 ALIAS(atomic_or_32,_atomic_or_32) 289 ALIAS(atomic_or_uint,_atomic_or_32) 290 ALIAS(atomic_or_ulong,_atomic_or_32) 291 ALIAS(atomic_or_ptr,_atomic_or_32) 292 293 ALIAS(atomic_or_32_nv,_atomic_or_32_nv) 294 ALIAS(atomic_or_uint_nv,_atomic_or_32_nv) 295 ALIAS(atomic_or_ulong_nv,_atomic_or_32_nv) 296 ALIAS(atomic_or_ptr_nv,_atomic_or_32_nv) 297 298 ALIAS(atomic_swap_32,_atomic_swap_32) 299 ALIAS(atomic_swap_uint,_atomic_swap_32) 300 ALIAS(atomic_swap_ulong,_atomic_swap_32) 301 ALIAS(atomic_swap_ptr,_atomic_swap_32) 302 303 ALIAS(atomic_cas_32,_atomic_cas_32) 304 ALIAS(atomic_cas_uint,_atomic_cas_32) 305 ALIAS(atomic_cas_ulong,_atomic_cas_32) 306 ALIAS(atomic_cas_ptr,_atomic_cas_32) 307 308 ALIAS(atomic_cas_32_ni,_atomic_cas_32_ni) 309 ALIAS(atomic_cas_uint_ni,_atomic_cas_32_ni) 310 ALIAS(atomic_cas_ulong_ni,_atomic_cas_32_ni) 311 ALIAS(atomic_cas_ptr_ni,_atomic_cas_32_ni) 312 313 ALIAS(atomic_cas_64,_atomic_cas_64) 314 ALIAS(atomic_cas_64_ni,_atomic_cas_64) 315 316 ALIAS(membar_consumer,_membar_consumer) 317 ALIAS(membar_producer,_membar_producer) 318 ALIAS(membar_enter,_membar_consumer) 319 ALIAS(membar_exit,_membar_producer) 320 ALIAS(membar_sync,_membar_sync) 321 322 STRONG_ALIAS(_atomic_add_int,_atomic_add_32) 323 STRONG_ALIAS(_atomic_add_long,_atomic_add_32) 324 STRONG_ALIAS(_atomic_add_ptr,_atomic_add_32) 325 326 STRONG_ALIAS(_atomic_add_int_nv,_atomic_add_32_nv) 327 STRONG_ALIAS(_atomic_add_long_nv,_atomic_add_32_nv) 328 STRONG_ALIAS(_atomic_add_ptr_nv,_atomic_add_32_nv) 329 330 STRONG_ALIAS(_atomic_and_uint,_atomic_and_32) 331 STRONG_ALIAS(_atomic_and_ulong,_atomic_and_32) 332 STRONG_ALIAS(_atomic_and_ptr,_atomic_and_32) 333 334 STRONG_ALIAS(_atomic_and_uint_nv,_atomic_and_32_nv) 335 STRONG_ALIAS(_atomic_and_ulong_nv,_atomic_and_32_nv) 336 STRONG_ALIAS(_atomic_and_ptr_nv,_atomic_and_32_nv) 337 338 STRONG_ALIAS(_atomic_dec_uint,_atomic_dec_32) 339 STRONG_ALIAS(_atomic_dec_ulong,_atomic_dec_32) 340 STRONG_ALIAS(_atomic_dec_ptr,_atomic_dec_32) 341 342 STRONG_ALIAS(_atomic_dec_uint_nv,_atomic_dec_32_nv) 343 STRONG_ALIAS(_atomic_dec_ulong_nv,_atomic_dec_32_nv) 344 STRONG_ALIAS(_atomic_dec_ptr_nv,_atomic_dec_32_nv) 345 346 STRONG_ALIAS(_atomic_inc_uint,_atomic_inc_32) 347 STRONG_ALIAS(_atomic_inc_ulong,_atomic_inc_32) 348 STRONG_ALIAS(_atomic_inc_ptr,_atomic_inc_32) 349 350 STRONG_ALIAS(_atomic_inc_uint_nv,_atomic_inc_32_nv) 351 STRONG_ALIAS(_atomic_inc_ulong_nv,_atomic_inc_32_nv) 352 STRONG_ALIAS(_atomic_inc_ptr_nv,_atomic_inc_32_nv) 353 354 STRONG_ALIAS(_atomic_or_uint,_atomic_or_32) 355 STRONG_ALIAS(_atomic_or_ulong,_atomic_or_32) 356 STRONG_ALIAS(_atomic_or_ptr,_atomic_or_32) 357 358 STRONG_ALIAS(_atomic_or_uint_nv,_atomic_or_32_nv) 359 STRONG_ALIAS(_atomic_or_ulong_nv,_atomic_or_32_nv) 360 STRONG_ALIAS(_atomic_or_ptr_nv,_atomic_or_32_nv) 361 362 STRONG_ALIAS(_atomic_swap_uint,_atomic_swap_32) 363 STRONG_ALIAS(_atomic_swap_ulong,_atomic_swap_32) 364 STRONG_ALIAS(_atomic_swap_ptr,_atomic_swap_32) 365 366 STRONG_ALIAS(_atomic_cas_uint,_atomic_cas_32) 367 STRONG_ALIAS(_atomic_cas_ulong,_atomic_cas_32) 368 STRONG_ALIAS(_atomic_cas_ptr,_atomic_cas_32) 369 370 STRONG_ALIAS(_atomic_cas_uint_ni,_atomic_cas_32_ni) 371 STRONG_ALIAS(_atomic_cas_ulong_ni,_atomic_cas_32_ni) 372 STRONG_ALIAS(_atomic_cas_ptr_ni,_atomic_cas_32_ni) 373 374 STRONG_ALIAS(_membar_enter,_membar_consumer) 375 STRONG_ALIAS(_membar_exit,_membar_producer) 376
Indexes created Wed Oct 01 07:09:59 GMT 2025