atomic.S revision 1.3
1 /* $NetBSD: atomic.S,v 1.3 2007/11/28 18:02:29 ad Exp $ */ 2 3 /*- 4 * Copyright (c) 2007 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. All advertising materials mentioning features or use of this software 19 * must display the following acknowledgement: 20 * This product includes software developed by the NetBSD 21 * Foundation, Inc. and its contributors. 22 * 4. Neither the name of The NetBSD Foundation nor the names of its 23 * contributors may be used to endorse or promote products derived 24 * from this software without specific prior written permission. 25 * 26 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 27 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 28 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 29 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 30 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 31 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 32 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 33 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 34 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 35 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 36 * POSSIBILITY OF SUCH DAMAGE. 37 */ 38 39 #include <machine/asm.h> 40 41 #ifdef _KERNEL 42 #define LOCK(n) .Lpatch/**/n: lock 43 #define ALIAS(f, t) STRONG_ALIAS(f,t) 44 #define END(a) _ALIGN_TEXT; LABEL(a) 45 #else 46 #define LOCK(n) lock 47 #define ALIAS(f, t) WEAK_ALIAS(f,t) 48 #define END(a) /* nothing */ 49 #endif 50 51 .text 52 53 /* 32-bit */ 54 55 NENTRY(_atomic_add_32) 56 LOCK(1) 57 addl %esi, (%rdi) 58 ret 59 60 NENTRY(_atomic_add_32_nv) 61 movl %esi, %eax 62 LOCK(2) 63 xaddl %eax, (%rdi) 64 addl %esi, %eax 65 ret 66 67 NENTRY(_atomic_and_32) 68 LOCK(3) 69 andl %esi, (%rdi) 70 ret 71 72 NENTRY(_atomic_and_32_nv) 73 movl (%rdi), %eax 74 1: 75 movl %eax, %ecx 76 andl %esi, %ecx 77 LOCK(4) 78 cmpxchgl %ecx, (%rdi) 79 jnz 1b 80 movl %ecx, %eax 81 ret 82 83 NENTRY(_atomic_dec_32) 84 LOCK(5) 85 decl (%rdi) 86 ret 87 88 NENTRY(_atomic_dec_32_nv) 89 movl $-1, %eax 90 LOCK(6) 91 xaddl %eax, (%rdi) 92 decl %eax 93 ret 94 95 NENTRY(_atomic_inc_32) 96 LOCK(7) 97 incl (%rdi) 98 ret 99 100 NENTRY(_atomic_inc_32_nv) 101 movl $1, %eax 102 LOCK(8) 103 xaddl %eax, (%rdi) 104 incl %eax 105 ret 106 107 NENTRY(_atomic_or_32) 108 LOCK(9) 109 orl %esi, (%rdi) 110 ret 111 112 NENTRY(_atomic_or_32_nv) 113 movl (%rdi), %eax 114 1: 115 movl %eax, %ecx 116 orl %esi, %ecx 117 LOCK(10) 118 cmpxchgl %ecx, (%rdi) 119 jnz 1b 120 movl %ecx, %eax 121 ret 122 123 NENTRY(_atomic_swap_32) 124 xchgl %esi, (%rdi) 125 ret 126 127 NENTRY(_atomic_cas_32) 128 movl %esi, %eax 129 LOCK(12) 130 cmpxchgl %edx, (%rdi) 131 /* %eax now contains the old value */ 132 ret 133 134 /* 64-bit */ 135 136 NENTRY(_atomic_add_64) 137 LOCK(13) 138 addq %rsi, (%rdi) 139 ret 140 141 NENTRY(_atomic_add_64_nv) 142 movq %rsi, %rax 143 LOCK(14) 144 xaddq %rax, (%rdi) 145 addq %rsi, %rax 146 ret 147 148 NENTRY(_atomic_and_64) 149 LOCK(15) 150 andq %rsi, (%rdi) 151 ret 152 153 NENTRY(_atomic_and_64_nv) 154 movq (%rdi), %rax 155 1: 156 movq %rax, %rcx 157 andq %rsi, %rcx 158 LOCK(16) 159 cmpxchgq %rcx, (%rdi) 160 jnz 1b 161 movq %rcx, %rax 162 ret 163 164 NENTRY(_atomic_dec_64) 165 LOCK(17) 166 decq (%rdi) 167 ret 168 169 NENTRY(_atomic_dec_64_nv) 170 movq $-1, %rax 171 LOCK(18) 172 xaddq %rax, (%rdi) 173 decq %rax 174 ret 175 176 NENTRY(_atomic_inc_64) 177 LOCK(19) 178 incq (%rdi) 179 ret 180 181 NENTRY(_atomic_inc_64_nv) 182 movq $1, %rax 183 LOCK(20) 184 xaddq %rax, (%rdi) 185 incq %rax 186 ret 187 188 NENTRY(_atomic_or_64) 189 LOCK(21) 190 orq %rsi, (%rdi) 191 ret 192 193 NENTRY(_atomic_or_64_nv) 194 movq (%rdi), %rax 195 1: 196 movq %rax, %rcx 197 orq %rsi, %rcx 198 LOCK(22) 199 cmpxchgq %rcx, (%rdi) 200 jnz 1b 201 movq %rcx, %rax 202 ret 203 204 NENTRY(_atomic_swap_64) 205 xchgq %rsi, (%rdi) 206 ret 207 208 NENTRY(_atomic_cas_64) 209 movq %rsi, %rax 210 LOCK(24) 211 cmpxchgl %edx, (%rdi) 212 /* %eax now contains the old value */ 213 ret 214 215 /* memory barriers */ 216 217 NENTRY(_membar_consumer) 218 LOCK(25) 219 addq $0, -8(%rsp) 220 ret 221 END(membar_consumer_end) 222 223 NENTRY(_membar_producer) 224 /* A store is enough */ 225 movq $0, -8(%rsp) 226 ret 227 END(membar_producer_end) 228 229 NENTRY(_membar_enter) 230 /* A store is enough */ 231 movq $0, -8(%rsp) 232 ret 233 END(membar_enter_end) 234 235 NENTRY(_membar_exit) 236 /* A store is enough */ 237 movq $0, -8(%rsp) 238 ret 239 END(membar_exit_end) 240 241 NENTRY(_membar_sync) 242 LOCK(26) 243 addq $0, -8(%rsp) 244 ret 245 END(membar_sync_end) 246 247 #ifdef _KERNEL 248 NENTRY(sse2_lfence) 249 lfence 250 ret 251 END(sse2_lfence_end) 252 253 NENTRY(sse2_mfence) 254 mfence 255 ret 256 END(sse2_mfence_end) 257 258 atomic_lockpatch: 259 .globl atomic_lockpatch 260 .quad .Lpatch1, .Lpatch2, .Lpatch3, .Lpatch4, .Lpatch5 261 .quad .Lpatch6, .Lpatch7, .Lpatch8, .Lpatch9, .Lpatch10 262 .quad .Lpatch12, .Lpatch13, .Lpatch14, .Lpatch15 263 .quad .Lpatch16, .Lpatch17, .Lpatch18, .Lpatch19, .Lpatch20 264 .quad .Lpatch21, .Lpatch22, .Lpatch24, .Lpatch25 265 .quad .Lpatch26, 0 266 #endif /* _KERNEL */ 267 268 ALIAS(atomic_add_32,_atomic_add_32) 269 ALIAS(atomic_add_64,_atomic_add_64) 270 ALIAS(atomic_add_uint,_atomic_add_32) 271 ALIAS(atomic_add_ulong,_atomic_add_64) 272 ALIAS(atomic_add_ptr,_atomic_add_64) 273 274 ALIAS(atomic_add_32_nv,_atomic_add_32_nv) 275 ALIAS(atomic_add_64_nv,_atomic_add_64_nv) 276 ALIAS(atomic_add_uint_nv,_atomic_add_32_nv) 277 ALIAS(atomic_add_ulong_nv,_atomic_add_64_nv) 278 ALIAS(atomic_add_ptr_nv,_atomic_add_64_nv) 279 280 ALIAS(atomic_and_32,_atomic_and_32) 281 ALIAS(atomic_and_64,_atomic_and_64) 282 ALIAS(atomic_and_uint,_atomic_and_32) 283 ALIAS(atomic_and_ulong,_atomic_and_64) 284 ALIAS(atomic_and_ptr,_atomic_and_64) 285 286 ALIAS(atomic_and_32_nv,_atomic_and_32_nv) 287 ALIAS(atomic_and_64_nv,_atomic_and_64_nv) 288 ALIAS(atomic_and_uint_nv,_atomic_and_32_nv) 289 ALIAS(atomic_and_ulong_nv,_atomic_and_64_nv) 290 ALIAS(atomic_and_ptr_nv,_atomic_and_64_nv) 291 292 ALIAS(atomic_dec_32,_atomic_dec_32) 293 ALIAS(atomic_dec_64,_atomic_dec_64) 294 ALIAS(atomic_dec_uint,_atomic_dec_32) 295 ALIAS(atomic_dec_ulong,_atomic_dec_64) 296 ALIAS(atomic_dec_ptr,_atomic_dec_64) 297 298 ALIAS(atomic_dec_32_nv,_atomic_dec_32_nv) 299 ALIAS(atomic_dec_64_nv,_atomic_dec_64_nv) 300 ALIAS(atomic_dec_uint_nv,_atomic_dec_32_nv) 301 ALIAS(atomic_dec_ulong_nv,_atomic_dec_64_nv) 302 ALIAS(atomic_dec_ptr_nv,_atomic_dec_64_nv) 303 304 ALIAS(atomic_inc_32,_atomic_inc_32) 305 ALIAS(atomic_inc_64,_atomic_inc_64) 306 ALIAS(atomic_inc_uint,_atomic_inc_32) 307 ALIAS(atomic_inc_ulong,_atomic_inc_64) 308 ALIAS(atomic_inc_ptr,_atomic_inc_64) 309 310 ALIAS(atomic_inc_32_nv,_atomic_inc_32_nv) 311 ALIAS(atomic_inc_64_nv,_atomic_inc_64_nv) 312 ALIAS(atomic_inc_uint_nv,_atomic_inc_32_nv) 313 ALIAS(atomic_inc_ulong_nv,_atomic_inc_64_nv) 314 ALIAS(atomic_inc_ptr_nv,_atomic_inc_64_nv) 315 316 ALIAS(atomic_or_32,_atomic_or_32) 317 ALIAS(atomic_or_uint,_atomic_or_32) 318 ALIAS(atomic_or_ulong,_atomic_or_64) 319 ALIAS(atomic_or_ptr,_atomic_or_64) 320 321 ALIAS(atomic_or_32_nv,_atomic_or_32_nv) 322 ALIAS(atomic_or_64_nv,_atomic_or_64_nv) 323 ALIAS(atomic_or_uint_nv,_atomic_or_32_nv) 324 ALIAS(atomic_or_ulong_nv,_atomic_or_64_nv) 325 ALIAS(atomic_or_ptr_nv,_atomic_or_64_nv) 326 327 ALIAS(atomic_swap_32,_atomic_swap_32) 328 ALIAS(atomic_swap_64,_atomic_swap_64) 329 ALIAS(atomic_swap_uint,_atomic_swap_32) 330 ALIAS(atomic_swap_ulong,_atomic_swap_64) 331 ALIAS(atomic_swap_ptr,_atomic_swap_64) 332 333 ALIAS(atomic_cas_32,_atomic_cas_32) 334 ALIAS(atomic_cas_64,_atomic_cas_64) 335 ALIAS(atomic_cas_uint,_atomic_cas_32) 336 ALIAS(atomic_cas_ulong,_atomic_cas_64) 337 ALIAS(atomic_cas_ptr,_atomic_cas_64) 338 339 ALIAS(membar_consumer,_membar_consumer) 340 ALIAS(membar_producer,_membar_producer) 341 ALIAS(membar_enter,_membar_enter) 342 ALIAS(membar_exit,_membar_exit) 343 ALIAS(membar_sync,_membar_sync) 344
Indexes created Tue Sep 30 11:09:46 GMT 2025