1 /* $NetBSD: atomic_init_testset.c,v 1.19 2024/01/21 03:42:08 thorpej Exp $ */ 2 3 /*- 4 * Copyright (c) 2008 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 17 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 18 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 * POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 /* 30 * libc glue for atomic operations where the hardware does not provide 31 * compare-and-swap. It's assumed that this will only be used on 32-bit 32 * platforms. 33 * 34 * This should be compiled with '-fno-reorder-blocks -fomit-frame-pointer' 35 * if using gcc. 36 */ 37 38 #include <sys/cdefs.h> 39 __RCSID("$NetBSD: atomic_init_testset.c,v 1.19 2024/01/21 03:42:08 thorpej Exp $"); 40 41 #include "extern.h" 42 #include "atomic_op_namespace.h" 43 44 #include <sys/types.h> 45 #include <sys/atomic.h> 46 #include <sys/lock.h> 47 #include <sys/ras.h> 48 #include <sys/sysctl.h> 49 50 #include <string.h> 51 52 #define I2 __SIMPLELOCK_UNLOCKED, __SIMPLELOCK_UNLOCKED, 53 #define I16 I2 I2 I2 I2 I2 I2 I2 I2 54 #define I128 I16 I16 I16 I16 I16 I16 I16 I16 55 56 static __cpu_simple_lock_t atomic_locks[128] = { I128 }; 57 /* 58 * Pick a lock out of above array depending on the object address 59 * passed. Most variables used atomically will not be in the same 60 * cacheline - and if they are, using the same lock is fine. 61 */ 62 #define HASH(PTR) (((uintptr_t)(PTR) >> 3) & 127) 63 64 #ifdef __HAVE_ASM_ATOMIC_CAS_UP 65 extern uint32_t _atomic_cas_up(volatile uint32_t *, uint32_t, uint32_t); 66 #else 67 static uint32_t _atomic_cas_up(volatile uint32_t *, uint32_t, uint32_t); 68 #endif 69 static uint32_t (*_atomic_cas_fn)(volatile uint32_t *, uint32_t, uint32_t) = 70 _atomic_cas_up; 71 RAS_DECL(_atomic_cas); 72 73 #ifdef __HAVE_ATOMIC_CAS_64_UP 74 #ifdef __HAVE_ASM_ATOMIC_CAS_64_UP 75 extern uint64_t _atomic_cas_64_up(volatile uint64_t *, uint64_t, uint64_t); 76 #else 77 static uint64_t _atomic_cas_64_up(volatile uint64_t *, uint64_t, uint64_t); 78 #endif 79 static uint64_t (*_atomic_cas_64_fn)(volatile uint64_t *, uint64_t, uint64_t) = 80 _atomic_cas_64_up; 81 RAS_DECL(_atomic_cas_64); 82 #endif 83 84 #ifdef __HAVE_ASM_ATOMIC_CAS_16_UP 85 extern uint16_t _atomic_cas_16_up(volatile uint16_t *, uint16_t, uint16_t); 86 #else 87 static uint16_t _atomic_cas_16_up(volatile uint16_t *, uint16_t, uint16_t); 88 #endif 89 static uint16_t (*_atomic_cas_16_fn)(volatile uint16_t *, uint16_t, uint16_t) = 90 _atomic_cas_16_up; 91 RAS_DECL(_atomic_cas_16); 92 93 #ifdef __HAVE_ASM_ATOMIC_CAS_8_UP 94 extern uint8_t _atomic_cas_8_up(volatile uint8_t *, uint8_t, uint8_t); 95 #else 96 static uint8_t _atomic_cas_8_up(volatile uint8_t *, uint8_t, uint8_t); 97 #endif 98 static uint8_t (*_atomic_cas_8_fn)(volatile uint8_t *, uint8_t, uint8_t) = 99 _atomic_cas_8_up; 100 RAS_DECL(_atomic_cas_8); 101 102 #ifndef __HAVE_ASM_ATOMIC_CAS_UP 103 static uint32_t 104 _atomic_cas_up(volatile uint32_t *ptr, uint32_t old, uint32_t new) 105 { 106 uint32_t ret; 107 108 RAS_START(_atomic_cas); 109 ret = *ptr; 110 if (__predict_false(ret != old)) { 111 return ret; 112 } 113 *ptr = new; 114 RAS_END(_atomic_cas); 115 116 return ret; 117 } 118 #endif 119 120 #if defined(__HAVE_ATOMIC_CAS_64_UP) && !defined(__HAVE_ASM_ATOMIC_CAS_64_UP) 121 static uint64_t 122 _atomic_cas_64_up(volatile uint64_t *ptr, uint64_t old, uint64_t new) 123 { 124 uint64_t ret; 125 126 RAS_START(_atomic_cas_64); 127 ret = *ptr; 128 if (__predict_false(ret != old)) { 129 return ret; 130 } 131 *ptr = new; 132 RAS_END(_atomic_cas_64); 133 134 return ret; 135 } 136 #endif 137 138 #ifndef __HAVE_ASM_ATOMIC_CAS_16_UP 139 static uint16_t 140 _atomic_cas_16_up(volatile uint16_t *ptr, uint16_t old, uint16_t new) 141 { 142 uint16_t ret; 143 144 RAS_START(_atomic_cas_16); 145 ret = *ptr; 146 if (__predict_false(ret != old)) { 147 return ret; 148 } 149 *ptr = new; 150 RAS_END(_atomic_cas_16); 151 152 return ret; 153 } 154 #endif 155 156 #ifndef __HAVE_ASM_ATOMIC_CAS_8_UP 157 static uint8_t 158 _atomic_cas_8_up(volatile uint8_t *ptr, uint8_t old, uint8_t new) 159 { 160 uint8_t ret; 161 162 RAS_START(_atomic_cas_8); 163 ret = *ptr; 164 if (__predict_false(ret != old)) { 165 return ret; 166 } 167 *ptr = new; 168 RAS_END(_atomic_cas_8); 169 170 return ret; 171 } 172 #endif 173 174 static uint32_t 175 _atomic_cas_mp(volatile uint32_t *ptr, uint32_t old, uint32_t new) 176 { 177 __cpu_simple_lock_t *lock; 178 uint32_t ret; 179 180 lock = &atomic_locks[HASH(ptr)]; 181 __cpu_simple_lock(lock); 182 ret = *ptr; 183 if (__predict_true(ret == old)) { 184 *ptr = new; 185 } 186 __cpu_simple_unlock(lock); 187 188 return ret; 189 } 190 191 #ifdef __HAVE_ATOMIC_CAS_64_UP 192 static uint64_t 193 _atomic_cas_64_mp(volatile uint64_t *ptr, uint64_t old, uint64_t new) 194 { 195 __cpu_simple_lock_t *lock; 196 uint64_t ret; 197 198 lock = &atomic_locks[HASH(ptr)]; 199 __cpu_simple_lock(lock); 200 ret = *ptr; 201 if (__predict_true(ret == old)) { 202 *ptr = new; 203 } 204 __cpu_simple_unlock(lock); 205 206 return ret; 207 } 208 #endif 209 210 static uint16_t 211 _atomic_cas_16_mp(volatile uint16_t *ptr, uint16_t old, uint16_t new) 212 { 213 __cpu_simple_lock_t *lock; 214 uint16_t ret; 215 216 lock = &atomic_locks[HASH(ptr)]; 217 __cpu_simple_lock(lock); 218 ret = *ptr; 219 if (__predict_true(ret == old)) { 220 *ptr = new; 221 } 222 __cpu_simple_unlock(lock); 223 224 return ret; 225 } 226 227 static uint8_t 228 _atomic_cas_8_mp(volatile uint8_t *ptr, uint8_t old, uint8_t new) 229 { 230 __cpu_simple_lock_t *lock; 231 uint8_t ret; 232 233 lock = &atomic_locks[HASH(ptr)]; 234 __cpu_simple_lock(lock); 235 ret = *ptr; 236 if (__predict_true(ret == old)) { 237 *ptr = new; 238 } 239 __cpu_simple_unlock(lock); 240 241 return ret; 242 } 243 244 uint32_t 245 _atomic_cas_32(volatile uint32_t *ptr, uint32_t old, uint32_t new) 246 { 247 248 return (*_atomic_cas_fn)(ptr, old, new); 249 } 250 251 #ifdef __HAVE_ATOMIC_CAS_64_UP 252 uint64_t _atomic_cas_64(volatile uint64_t *, uint64_t, uint64_t); 253 254 uint64_t 255 _atomic_cas_64(volatile uint64_t *ptr, uint64_t old, uint64_t new) 256 { 257 258 return (*_atomic_cas_64_fn)(ptr, old, new); 259 } 260 #endif 261 262 uint16_t 263 _atomic_cas_16(volatile uint16_t *ptr, uint16_t old, uint16_t new) 264 { 265 266 return (*_atomic_cas_16_fn)(ptr, old, new); 267 } 268 269 uint8_t _atomic_cas_8(volatile uint8_t *, uint8_t, uint8_t); 270 271 uint8_t 272 _atomic_cas_8(volatile uint8_t *ptr, uint8_t old, uint8_t new) 273 { 274 275 return (*_atomic_cas_8_fn)(ptr, old, new); 276 } 277 278 void __section(".text.startup") __attribute__ ((__visibility__("hidden"))) 279 __libc_atomic_init(void) 280 { 281 int ncpu, mib[2]; 282 size_t len; 283 284 _atomic_cas_fn = _atomic_cas_mp; 285 #ifdef __HAVE_ATOMIC_CAS_64_UP 286 _atomic_cas_64_fn = _atomic_cas_64_mp; 287 #endif 288 _atomic_cas_16_fn = _atomic_cas_16_mp; 289 _atomic_cas_8_fn = _atomic_cas_8_mp; 290 291 mib[0] = CTL_HW; 292 mib[1] = HW_NCPU; 293 len = sizeof(ncpu); 294 if (sysctl(mib, 2, &ncpu, &len, NULL, 0) == -1) 295 return; 296 if (ncpu > 1) 297 return; 298 299 if (rasctl(RAS_ADDR(_atomic_cas), RAS_SIZE(_atomic_cas), 300 RAS_INSTALL) == 0) { 301 _atomic_cas_fn = _atomic_cas_up; 302 } 303 304 305 #ifdef __HAVE_ATOMIC_CAS_64_UP 306 if (rasctl(RAS_ADDR(_atomic_cas_64), RAS_SIZE(_atomic_cas_64), 307 RAS_INSTALL) == 0) { 308 _atomic_cas_64_fn = _atomic_cas_64_up; 309 } 310 #endif 311 312 if (rasctl(RAS_ADDR(_atomic_cas_16), RAS_SIZE(_atomic_cas_16), 313 RAS_INSTALL) == 0) { 314 _atomic_cas_16_fn = _atomic_cas_16_up; 315 } 316 317 if (rasctl(RAS_ADDR(_atomic_cas_8), RAS_SIZE(_atomic_cas_8), 318 RAS_INSTALL) == 0) { 319 _atomic_cas_8_fn = _atomic_cas_8_up; 320 } 321 } 322 323 #undef atomic_cas_32 324 #undef atomic_cas_uint 325 #undef atomic_cas_ulong 326 #undef atomic_cas_ptr 327 #undef atomic_cas_32_ni 328 #undef atomic_cas_uint_ni 329 #undef atomic_cas_ulong_ni 330 #undef atomic_cas_ptr_ni 331 332 atomic_op_alias(atomic_cas_32,_atomic_cas_32) 333 atomic_op_alias(atomic_cas_uint,_atomic_cas_32) 334 __strong_alias(_atomic_cas_uint,_atomic_cas_32) 335 atomic_op_alias(atomic_cas_ulong,_atomic_cas_32) 336 __strong_alias(_atomic_cas_ulong,_atomic_cas_32) 337 atomic_op_alias(atomic_cas_ptr,_atomic_cas_32) 338 __strong_alias(_atomic_cas_ptr,_atomic_cas_32) 339 340 atomic_op_alias(atomic_cas_32_ni,_atomic_cas_32) 341 __strong_alias(_atomic_cas_32_ni,_atomic_cas_32) 342 atomic_op_alias(atomic_cas_uint_ni,_atomic_cas_32) 343 __strong_alias(_atomic_cas_uint_ni,_atomic_cas_32) 344 atomic_op_alias(atomic_cas_ulong_ni,_atomic_cas_32) 345 __strong_alias(_atomic_cas_ulong_ni,_atomic_cas_32) 346 atomic_op_alias(atomic_cas_ptr_ni,_atomic_cas_32) 347 __strong_alias(_atomic_cas_ptr_ni,_atomic_cas_32) 348 349 //atomic_op_alias(atomic_cas_16,_atomic_cas_16) 350 //atomic_op_alias(atomic_cas_16_ni,_atomic_cas_16) 351 //atomic_op_alias(atomic_cas_8,_atomic_cas_8) 352 //atomic_op_alias(atomic_cas_8_ni,_atomic_cas_8) 353 #ifdef __HAVE_ATOMIC_CAS_64_UP 354 atomic_op_alias(atomic_cas_64_ni,_atomic_cas_64) 355 __strong_alias(_atomic_cas_64_ni,_atomic_cas_64) 356 crt_alias(__sync_val_compare_and_swap_8,_atomic_cas_64) 357 #endif 358 crt_alias(__sync_val_compare_and_swap_4,_atomic_cas_32) 359 crt_alias(__sync_val_compare_and_swap_2,_atomic_cas_16) 360 crt_alias(__sync_val_compare_and_swap_1,_atomic_cas_8) 361
Indexes created Tue Sep 30 07:10:03 GMT 2025