1 /* $NetBSD: atomic.h,v 1.30 2026/07/02 14:38:19 riastradh Exp $ */ 2 3 /*- 4 * Copyright (c) 2007, 2008 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #ifndef _SYS_ATOMIC_H_ 33 #define _SYS_ATOMIC_H_ 34 35 #include <sys/types.h> 36 #include <sys/endian.h> 37 #if !defined(_KERNEL) && !defined(_STANDALONE) 38 #include <stdint.h> 39 #endif 40 41 #if defined(_KERNEL) && defined(_KERNEL_OPT) 42 #include "opt_kasan.h" 43 #include "opt_kcsan.h" 44 #include "opt_kmsan.h" 45 #endif 46 47 #if defined(KASAN) 48 #define ATOMIC_PROTO_ADD(name, tret, targ1, targ2) \ 49 void kasan_atomic_add_##name(volatile targ1 *, targ2); \ 50 tret kasan_atomic_add_##name##_nv(volatile targ1 *, targ2) 51 #define ATOMIC_PROTO_AND(name, tret, targ1, targ2) \ 52 void kasan_atomic_and_##name(volatile targ1 *, targ2); \ 53 tret kasan_atomic_and_##name##_nv(volatile targ1 *, targ2) 54 #define ATOMIC_PROTO_OR(name, tret, targ1, targ2) \ 55 void kasan_atomic_or_##name(volatile targ1 *, targ2); \ 56 tret kasan_atomic_or_##name##_nv(volatile targ1 *, targ2) 57 #define ATOMIC_PROTO_CAS(name, tret, targ1, targ2) \ 58 tret kasan_atomic_cas_##name(volatile targ1 *, targ2, targ2); \ 59 tret kasan_atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2) 60 #define ATOMIC_PROTO_SWAP(name, tret, targ1, targ2) \ 61 tret kasan_atomic_swap_##name(volatile targ1 *, targ2) 62 #define ATOMIC_PROTO_DEC(name, tret, targ1) \ 63 void kasan_atomic_dec_##name(volatile targ1 *); \ 64 tret kasan_atomic_dec_##name##_nv(volatile targ1 *) 65 #define ATOMIC_PROTO_INC(name, tret, targ1) \ 66 void kasan_atomic_inc_##name(volatile targ1 *); \ 67 tret kasan_atomic_inc_##name##_nv(volatile targ1 *) 68 #elif defined(KCSAN) 69 #define ATOMIC_PROTO_ADD(name, tret, targ1, targ2) \ 70 void kcsan_atomic_add_##name(volatile targ1 *, targ2); \ 71 tret kcsan_atomic_add_##name##_nv(volatile targ1 *, targ2) 72 #define ATOMIC_PROTO_AND(name, tret, targ1, targ2) \ 73 void kcsan_atomic_and_##name(volatile targ1 *, targ2); \ 74 tret kcsan_atomic_and_##name##_nv(volatile targ1 *, targ2) 75 #define ATOMIC_PROTO_OR(name, tret, targ1, targ2) \ 76 void kcsan_atomic_or_##name(volatile targ1 *, targ2); \ 77 tret kcsan_atomic_or_##name##_nv(volatile targ1 *, targ2) 78 #define ATOMIC_PROTO_CAS(name, tret, targ1, targ2) \ 79 tret kcsan_atomic_cas_##name(volatile targ1 *, targ2, targ2); \ 80 tret kcsan_atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2) 81 #define ATOMIC_PROTO_SWAP(name, tret, targ1, targ2) \ 82 tret kcsan_atomic_swap_##name(volatile targ1 *, targ2) 83 #define ATOMIC_PROTO_DEC(name, tret, targ1) \ 84 void kcsan_atomic_dec_##name(volatile targ1 *); \ 85 tret kcsan_atomic_dec_##name##_nv(volatile targ1 *) 86 #define ATOMIC_PROTO_INC(name, tret, targ1) \ 87 void kcsan_atomic_inc_##name(volatile targ1 *); \ 88 tret kcsan_atomic_inc_##name##_nv(volatile targ1 *) 89 #elif defined(KMSAN) 90 #define ATOMIC_PROTO_ADD(name, tret, targ1, targ2) \ 91 void kmsan_atomic_add_##name(volatile targ1 *, targ2); \ 92 tret kmsan_atomic_add_##name##_nv(volatile targ1 *, targ2) 93 #define ATOMIC_PROTO_AND(name, tret, targ1, targ2) \ 94 void kmsan_atomic_and_##name(volatile targ1 *, targ2); \ 95 tret kmsan_atomic_and_##name##_nv(volatile targ1 *, targ2) 96 #define ATOMIC_PROTO_OR(name, tret, targ1, targ2) \ 97 void kmsan_atomic_or_##name(volatile targ1 *, targ2); \ 98 tret kmsan_atomic_or_##name##_nv(volatile targ1 *, targ2) 99 #define ATOMIC_PROTO_CAS(name, tret, targ1, targ2) \ 100 tret kmsan_atomic_cas_##name(volatile targ1 *, targ2, targ2); \ 101 tret kmsan_atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2) 102 #define ATOMIC_PROTO_SWAP(name, tret, targ1, targ2) \ 103 tret kmsan_atomic_swap_##name(volatile targ1 *, targ2) 104 #define ATOMIC_PROTO_DEC(name, tret, targ1) \ 105 void kmsan_atomic_dec_##name(volatile targ1 *); \ 106 tret kmsan_atomic_dec_##name##_nv(volatile targ1 *) 107 #define ATOMIC_PROTO_INC(name, tret, targ1) \ 108 void kmsan_atomic_inc_##name(volatile targ1 *); \ 109 tret kmsan_atomic_inc_##name##_nv(volatile targ1 *) 110 #else 111 #define ATOMIC_PROTO_ADD(name, tret, targ1, targ2) \ 112 void atomic_add_##name(volatile targ1 *, targ2); \ 113 tret atomic_add_##name##_nv(volatile targ1 *, targ2) 114 #define ATOMIC_PROTO_AND(name, tret, targ1, targ2) \ 115 void atomic_and_##name(volatile targ1 *, targ2); \ 116 tret atomic_and_##name##_nv(volatile targ1 *, targ2) 117 #define ATOMIC_PROTO_OR(name, tret, targ1, targ2) \ 118 void atomic_or_##name(volatile targ1 *, targ2); \ 119 tret atomic_or_##name##_nv(volatile targ1 *, targ2) 120 #define ATOMIC_PROTO_CAS(name, tret, targ1, targ2) \ 121 tret atomic_cas_##name(volatile targ1 *, targ2, targ2); \ 122 tret atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2) 123 #define ATOMIC_PROTO_SWAP(name, tret, targ1, targ2) \ 124 tret atomic_swap_##name(volatile targ1 *, targ2) 125 #define ATOMIC_PROTO_DEC(name, tret, targ1) \ 126 void atomic_dec_##name(volatile targ1 *); \ 127 tret atomic_dec_##name##_nv(volatile targ1 *) 128 #define ATOMIC_PROTO_INC(name, tret, targ1) \ 129 void atomic_inc_##name(volatile targ1 *); \ 130 tret atomic_inc_##name##_nv(volatile targ1 *) 131 #endif 132 133 __BEGIN_DECLS 134 135 ATOMIC_PROTO_ADD(32, uint32_t, uint32_t, int32_t); 136 ATOMIC_PROTO_ADD(64, uint64_t, uint64_t, int64_t); 137 ATOMIC_PROTO_ADD(int, unsigned int, unsigned int, int); 138 ATOMIC_PROTO_ADD(long, unsigned long, unsigned long, long); 139 ATOMIC_PROTO_ADD(ptr, void *, void, ssize_t); 140 141 ATOMIC_PROTO_AND(32, uint32_t, uint32_t, uint32_t); 142 ATOMIC_PROTO_AND(64, uint64_t, uint64_t, uint64_t); 143 ATOMIC_PROTO_AND(uint, unsigned int, unsigned int, unsigned int); 144 ATOMIC_PROTO_AND(ulong, unsigned long, unsigned long, unsigned long); 145 146 ATOMIC_PROTO_OR(32, uint32_t, uint32_t, uint32_t); 147 ATOMIC_PROTO_OR(64, uint64_t, uint64_t, uint64_t); 148 ATOMIC_PROTO_OR(uint, unsigned int, unsigned int, unsigned int); 149 ATOMIC_PROTO_OR(ulong, unsigned long, unsigned long, unsigned long); 150 151 ATOMIC_PROTO_CAS(32, uint32_t, uint32_t, uint32_t); 152 ATOMIC_PROTO_CAS(64, uint64_t, uint64_t, uint64_t); 153 ATOMIC_PROTO_CAS(uint, unsigned int, unsigned int, unsigned int); 154 ATOMIC_PROTO_CAS(ulong, unsigned long, unsigned long, unsigned long); 155 ATOMIC_PROTO_CAS(ptr, void *, void, void *); 156 157 ATOMIC_PROTO_SWAP(32, uint32_t, uint32_t, uint32_t); 158 ATOMIC_PROTO_SWAP(64, uint64_t, uint64_t, uint64_t); 159 ATOMIC_PROTO_SWAP(uint, unsigned int, unsigned int, unsigned int); 160 ATOMIC_PROTO_SWAP(ulong, unsigned long, unsigned long, unsigned long); 161 ATOMIC_PROTO_SWAP(ptr, void *, void, void *); 162 163 ATOMIC_PROTO_DEC(32, uint32_t, uint32_t); 164 ATOMIC_PROTO_DEC(64, uint64_t, uint64_t); 165 ATOMIC_PROTO_DEC(uint, unsigned int, unsigned int); 166 ATOMIC_PROTO_DEC(ulong, unsigned long, unsigned long); 167 ATOMIC_PROTO_DEC(ptr, void *, void); 168 169 ATOMIC_PROTO_INC(32, uint32_t, uint32_t); 170 ATOMIC_PROTO_INC(64, uint64_t, uint64_t); 171 ATOMIC_PROTO_INC(uint, unsigned int, unsigned int); 172 ATOMIC_PROTO_INC(ulong, unsigned long, unsigned long); 173 ATOMIC_PROTO_INC(ptr, void *, void); 174 175 /* 176 * These operations will be provided for userland, but may not be 177 * implemented efficiently. 178 */ 179 uint16_t atomic_cas_16(volatile uint16_t *, uint16_t, uint16_t); 180 uint8_t atomic_cas_8(volatile uint8_t *, uint8_t, uint8_t); 181 182 /* 183 * Memory barrier operations 184 */ 185 void membar_acquire(void); 186 void membar_release(void); 187 void membar_producer(void); 188 void membar_consumer(void); 189 void membar_sync(void); 190 191 /* 192 * Deprecated memory barriers 193 */ 194 void membar_enter(void); 195 void membar_exit(void); 196 197 #ifdef __HAVE_MEMBAR_DATADEP_CONSUMER 198 void membar_datadep_consumer(void); 199 #else 200 #define membar_datadep_consumer() ((void)0) 201 #endif 202 203 __END_DECLS 204 205 #if defined(KASAN) 206 #define atomic_add_32 kasan_atomic_add_32 207 #define atomic_add_int kasan_atomic_add_int 208 #define atomic_add_long kasan_atomic_add_long 209 #define atomic_add_ptr kasan_atomic_add_ptr 210 #define atomic_add_64 kasan_atomic_add_64 211 #define atomic_add_32_nv kasan_atomic_add_32_nv 212 #define atomic_add_int_nv kasan_atomic_add_int_nv 213 #define atomic_add_long_nv kasan_atomic_add_long_nv 214 #define atomic_add_ptr_nv kasan_atomic_add_ptr_nv 215 #define atomic_add_64_nv kasan_atomic_add_64_nv 216 #define atomic_and_32 kasan_atomic_and_32 217 #define atomic_and_uint kasan_atomic_and_uint 218 #define atomic_and_ulong kasan_atomic_and_ulong 219 #define atomic_and_64 kasan_atomic_and_64 220 #define atomic_and_32_nv kasan_atomic_and_32_nv 221 #define atomic_and_uint_nv kasan_atomic_and_uint_nv 222 #define atomic_and_ulong_nv kasan_atomic_and_ulong_nv 223 #define atomic_and_64_nv kasan_atomic_and_64_nv 224 #define atomic_or_32 kasan_atomic_or_32 225 #define atomic_or_uint kasan_atomic_or_uint 226 #define atomic_or_ulong kasan_atomic_or_ulong 227 #define atomic_or_64 kasan_atomic_or_64 228 #define atomic_or_32_nv kasan_atomic_or_32_nv 229 #define atomic_or_uint_nv kasan_atomic_or_uint_nv 230 #define atomic_or_ulong_nv kasan_atomic_or_ulong_nv 231 #define atomic_or_64_nv kasan_atomic_or_64_nv 232 #define atomic_cas_32 kasan_atomic_cas_32 233 #define atomic_cas_uint kasan_atomic_cas_uint 234 #define atomic_cas_ulong kasan_atomic_cas_ulong 235 #define atomic_cas_ptr kasan_atomic_cas_ptr 236 #define atomic_cas_64 kasan_atomic_cas_64 237 #define atomic_cas_32_ni kasan_atomic_cas_32_ni 238 #define atomic_cas_uint_ni kasan_atomic_cas_uint_ni 239 #define atomic_cas_ulong_ni kasan_atomic_cas_ulong_ni 240 #define atomic_cas_ptr_ni kasan_atomic_cas_ptr_ni 241 #define atomic_cas_64_ni kasan_atomic_cas_64_ni 242 #define atomic_swap_32 kasan_atomic_swap_32 243 #define atomic_swap_uint kasan_atomic_swap_uint 244 #define atomic_swap_ulong kasan_atomic_swap_ulong 245 #define atomic_swap_ptr kasan_atomic_swap_ptr 246 #define atomic_swap_64 kasan_atomic_swap_64 247 #define atomic_dec_32 kasan_atomic_dec_32 248 #define atomic_dec_uint kasan_atomic_dec_uint 249 #define atomic_dec_ulong kasan_atomic_dec_ulong 250 #define atomic_dec_ptr kasan_atomic_dec_ptr 251 #define atomic_dec_64 kasan_atomic_dec_64 252 #define atomic_dec_32_nv kasan_atomic_dec_32_nv 253 #define atomic_dec_uint_nv kasan_atomic_dec_uint_nv 254 #define atomic_dec_ulong_nv kasan_atomic_dec_ulong_nv 255 #define atomic_dec_ptr_nv kasan_atomic_dec_ptr_nv 256 #define atomic_dec_64_nv kasan_atomic_dec_64_nv 257 #define atomic_inc_32 kasan_atomic_inc_32 258 #define atomic_inc_uint kasan_atomic_inc_uint 259 #define atomic_inc_ulong kasan_atomic_inc_ulong 260 #define atomic_inc_ptr kasan_atomic_inc_ptr 261 #define atomic_inc_64 kasan_atomic_inc_64 262 #define atomic_inc_32_nv kasan_atomic_inc_32_nv 263 #define atomic_inc_uint_nv kasan_atomic_inc_uint_nv 264 #define atomic_inc_ulong_nv kasan_atomic_inc_ulong_nv 265 #define atomic_inc_ptr_nv kasan_atomic_inc_ptr_nv 266 #define atomic_inc_64_nv kasan_atomic_inc_64_nv 267 #elif defined(KCSAN) 268 #define atomic_add_32 kcsan_atomic_add_32 269 #define atomic_add_int kcsan_atomic_add_int 270 #define atomic_add_long kcsan_atomic_add_long 271 #define atomic_add_ptr kcsan_atomic_add_ptr 272 #define atomic_add_64 kcsan_atomic_add_64 273 #define atomic_add_32_nv kcsan_atomic_add_32_nv 274 #define atomic_add_int_nv kcsan_atomic_add_int_nv 275 #define atomic_add_long_nv kcsan_atomic_add_long_nv 276 #define atomic_add_ptr_nv kcsan_atomic_add_ptr_nv 277 #define atomic_add_64_nv kcsan_atomic_add_64_nv 278 #define atomic_and_32 kcsan_atomic_and_32 279 #define atomic_and_uint kcsan_atomic_and_uint 280 #define atomic_and_ulong kcsan_atomic_and_ulong 281 #define atomic_and_64 kcsan_atomic_and_64 282 #define atomic_and_32_nv kcsan_atomic_and_32_nv 283 #define atomic_and_uint_nv kcsan_atomic_and_uint_nv 284 #define atomic_and_ulong_nv kcsan_atomic_and_ulong_nv 285 #define atomic_and_64_nv kcsan_atomic_and_64_nv 286 #define atomic_or_32 kcsan_atomic_or_32 287 #define atomic_or_uint kcsan_atomic_or_uint 288 #define atomic_or_ulong kcsan_atomic_or_ulong 289 #define atomic_or_64 kcsan_atomic_or_64 290 #define atomic_or_32_nv kcsan_atomic_or_32_nv 291 #define atomic_or_uint_nv kcsan_atomic_or_uint_nv 292 #define atomic_or_ulong_nv kcsan_atomic_or_ulong_nv 293 #define atomic_or_64_nv kcsan_atomic_or_64_nv 294 #define atomic_cas_32 kcsan_atomic_cas_32 295 #define atomic_cas_uint kcsan_atomic_cas_uint 296 #define atomic_cas_ulong kcsan_atomic_cas_ulong 297 #define atomic_cas_ptr kcsan_atomic_cas_ptr 298 #define atomic_cas_64 kcsan_atomic_cas_64 299 #define atomic_cas_32_ni kcsan_atomic_cas_32_ni 300 #define atomic_cas_uint_ni kcsan_atomic_cas_uint_ni 301 #define atomic_cas_ulong_ni kcsan_atomic_cas_ulong_ni 302 #define atomic_cas_ptr_ni kcsan_atomic_cas_ptr_ni 303 #define atomic_cas_64_ni kcsan_atomic_cas_64_ni 304 #define atomic_swap_32 kcsan_atomic_swap_32 305 #define atomic_swap_uint kcsan_atomic_swap_uint 306 #define atomic_swap_ulong kcsan_atomic_swap_ulong 307 #define atomic_swap_ptr kcsan_atomic_swap_ptr 308 #define atomic_swap_64 kcsan_atomic_swap_64 309 #define atomic_dec_32 kcsan_atomic_dec_32 310 #define atomic_dec_uint kcsan_atomic_dec_uint 311 #define atomic_dec_ulong kcsan_atomic_dec_ulong 312 #define atomic_dec_ptr kcsan_atomic_dec_ptr 313 #define atomic_dec_64 kcsan_atomic_dec_64 314 #define atomic_dec_32_nv kcsan_atomic_dec_32_nv 315 #define atomic_dec_uint_nv kcsan_atomic_dec_uint_nv 316 #define atomic_dec_ulong_nv kcsan_atomic_dec_ulong_nv 317 #define atomic_dec_ptr_nv kcsan_atomic_dec_ptr_nv 318 #define atomic_dec_64_nv kcsan_atomic_dec_64_nv 319 #define atomic_inc_32 kcsan_atomic_inc_32 320 #define atomic_inc_uint kcsan_atomic_inc_uint 321 #define atomic_inc_ulong kcsan_atomic_inc_ulong 322 #define atomic_inc_ptr kcsan_atomic_inc_ptr 323 #define atomic_inc_64 kcsan_atomic_inc_64 324 #define atomic_inc_32_nv kcsan_atomic_inc_32_nv 325 #define atomic_inc_uint_nv kcsan_atomic_inc_uint_nv 326 #define atomic_inc_ulong_nv kcsan_atomic_inc_ulong_nv 327 #define atomic_inc_ptr_nv kcsan_atomic_inc_ptr_nv 328 #define atomic_inc_64_nv kcsan_atomic_inc_64_nv 329 #elif defined(KMSAN) 330 #define atomic_add_32 kmsan_atomic_add_32 331 #define atomic_add_int kmsan_atomic_add_int 332 #define atomic_add_long kmsan_atomic_add_long 333 #define atomic_add_ptr kmsan_atomic_add_ptr 334 #define atomic_add_64 kmsan_atomic_add_64 335 #define atomic_add_32_nv kmsan_atomic_add_32_nv 336 #define atomic_add_int_nv kmsan_atomic_add_int_nv 337 #define atomic_add_long_nv kmsan_atomic_add_long_nv 338 #define atomic_add_ptr_nv kmsan_atomic_add_ptr_nv 339 #define atomic_add_64_nv kmsan_atomic_add_64_nv 340 #define atomic_and_32 kmsan_atomic_and_32 341 #define atomic_and_uint kmsan_atomic_and_uint 342 #define atomic_and_ulong kmsan_atomic_and_ulong 343 #define atomic_and_64 kmsan_atomic_and_64 344 #define atomic_and_32_nv kmsan_atomic_and_32_nv 345 #define atomic_and_uint_nv kmsan_atomic_and_uint_nv 346 #define atomic_and_ulong_nv kmsan_atomic_and_ulong_nv 347 #define atomic_and_64_nv kmsan_atomic_and_64_nv 348 #define atomic_or_32 kmsan_atomic_or_32 349 #define atomic_or_uint kmsan_atomic_or_uint 350 #define atomic_or_ulong kmsan_atomic_or_ulong 351 #define atomic_or_64 kmsan_atomic_or_64 352 #define atomic_or_32_nv kmsan_atomic_or_32_nv 353 #define atomic_or_uint_nv kmsan_atomic_or_uint_nv 354 #define atomic_or_ulong_nv kmsan_atomic_or_ulong_nv 355 #define atomic_or_64_nv kmsan_atomic_or_64_nv 356 #define atomic_cas_32 kmsan_atomic_cas_32 357 #define atomic_cas_uint kmsan_atomic_cas_uint 358 #define atomic_cas_ulong kmsan_atomic_cas_ulong 359 #define atomic_cas_ptr kmsan_atomic_cas_ptr 360 #define atomic_cas_64 kmsan_atomic_cas_64 361 #define atomic_cas_32_ni kmsan_atomic_cas_32_ni 362 #define atomic_cas_uint_ni kmsan_atomic_cas_uint_ni 363 #define atomic_cas_ulong_ni kmsan_atomic_cas_ulong_ni 364 #define atomic_cas_ptr_ni kmsan_atomic_cas_ptr_ni 365 #define atomic_cas_64_ni kmsan_atomic_cas_64_ni 366 #define atomic_swap_32 kmsan_atomic_swap_32 367 #define atomic_swap_uint kmsan_atomic_swap_uint 368 #define atomic_swap_ulong kmsan_atomic_swap_ulong 369 #define atomic_swap_ptr kmsan_atomic_swap_ptr 370 #define atomic_swap_64 kmsan_atomic_swap_64 371 #define atomic_dec_32 kmsan_atomic_dec_32 372 #define atomic_dec_uint kmsan_atomic_dec_uint 373 #define atomic_dec_ulong kmsan_atomic_dec_ulong 374 #define atomic_dec_ptr kmsan_atomic_dec_ptr 375 #define atomic_dec_64 kmsan_atomic_dec_64 376 #define atomic_dec_32_nv kmsan_atomic_dec_32_nv 377 #define atomic_dec_uint_nv kmsan_atomic_dec_uint_nv 378 #define atomic_dec_ulong_nv kmsan_atomic_dec_ulong_nv 379 #define atomic_dec_ptr_nv kmsan_atomic_dec_ptr_nv 380 #define atomic_dec_64_nv kmsan_atomic_dec_64_nv 381 #define atomic_inc_32 kmsan_atomic_inc_32 382 #define atomic_inc_uint kmsan_atomic_inc_uint 383 #define atomic_inc_ulong kmsan_atomic_inc_ulong 384 #define atomic_inc_ptr kmsan_atomic_inc_ptr 385 #define atomic_inc_64 kmsan_atomic_inc_64 386 #define atomic_inc_32_nv kmsan_atomic_inc_32_nv 387 #define atomic_inc_uint_nv kmsan_atomic_inc_uint_nv 388 #define atomic_inc_ulong_nv kmsan_atomic_inc_ulong_nv 389 #define atomic_inc_ptr_nv kmsan_atomic_inc_ptr_nv 390 #define atomic_inc_64_nv kmsan_atomic_inc_64_nv 391 #endif 392 393 #ifdef _KERNEL 394 395 #if 1 // XXX: __STDC_VERSION__ < 201112L 396 397 /* Pre-C11 definitions */ 398 399 #include <sys/cdefs.h> 400 401 #include <lib/libkern/libkern.h> 402 403 #ifdef _LP64 404 #define __HAVE_ATOMIC64_LOADSTORE 1 405 #define __ATOMIC_SIZE_MAX 8 406 #else 407 #define __ATOMIC_SIZE_MAX 4 408 #endif 409 410 #if __GNUC_PREREQ__(14, 0) && !defined(__clang__) 411 /* 412 * Can enable this for clang too once we update past clang>=19 and get: 413 * https://github.com/llvm/llvm-project/pull/87392 414 */ 415 #define __atomic_typeof_unqual __typeof_unqual__ 416 #else 417 #define __atomic_typeof_unqual __typeof__ 418 #endif 419 420 /* 421 * We assume that access to an aligned pointer to a volatile object of 422 * at most __ATOMIC_SIZE_MAX bytes is guaranteed to be atomic. This is 423 * an assumption that may be wrong, but we hope it won't be wrong 424 * before we just adopt the C11 atomic API. 425 */ 426 #define __ATOMIC_PTR_CHECK(p) do \ 427 { \ 428 CTASSERT(sizeof(*(p)) <= __ATOMIC_SIZE_MAX); \ 429 KASSERT(((uintptr_t)(p) & (sizeof(*(p)) - 1)) == 0); \ 430 } while (0) 431 432 #ifdef KCSAN 433 void kcsan_atomic_load(const volatile void *, void *, int); 434 void kcsan_atomic_store(volatile void *, const void *, int); 435 #define __BEGIN_ATOMIC_LOAD(p, v) \ 436 union { __atomic_typeof_unqual(*(p)) __al_val; char __al_buf[1]; } v; \ 437 kcsan_atomic_load(p, v.__al_buf, sizeof(v.__al_val)) 438 #define __END_ATOMIC_LOAD(v) \ 439 (v).__al_val 440 #define __DO_ATOMIC_STORE(p, v) \ 441 kcsan_atomic_store(p, __UNVOLATILE(&v), sizeof(v)) 442 #else 443 #define __BEGIN_ATOMIC_LOAD(p, v) \ 444 const __atomic_typeof_unqual(*(p)) v = *(p) 445 #define __END_ATOMIC_LOAD(v) \ 446 v 447 #ifdef __HAVE_HASHLOCKED_ATOMICS 448 #define __DO_ATOMIC_STORE(p, v) \ 449 __do_atomic_store(p, __UNVOLATILE(&v), sizeof(v)) 450 #else /* !__HAVE_HASHLOCKED_ATOMICS */ 451 #define __DO_ATOMIC_STORE(p, v) \ 452 *p = v 453 #endif 454 #endif 455 456 #define atomic_load_relaxed(p) \ 457 ({ \ 458 const volatile __typeof__(*(p)) *__al_ptr = (p); \ 459 __ATOMIC_PTR_CHECK(__al_ptr); \ 460 __BEGIN_ATOMIC_LOAD(__al_ptr, __al_val); \ 461 __END_ATOMIC_LOAD(__al_val); \ 462 }) 463 464 #define atomic_load_consume(p) \ 465 ({ \ 466 const volatile __typeof__(*(p)) *__al_ptr = (p); \ 467 __ATOMIC_PTR_CHECK(__al_ptr); \ 468 __BEGIN_ATOMIC_LOAD(__al_ptr, __al_val); \ 469 membar_datadep_consumer(); \ 470 __END_ATOMIC_LOAD(__al_val); \ 471 }) 472 473 #define atomic_load_acquire(p) \ 474 ({ \ 475 const volatile __typeof__(*(p)) *__al_ptr = (p); \ 476 __ATOMIC_PTR_CHECK(__al_ptr); \ 477 __BEGIN_ATOMIC_LOAD(__al_ptr, __al_val); \ 478 membar_acquire(); \ 479 __END_ATOMIC_LOAD(__al_val); \ 480 }) 481 482 #define atomic_store_relaxed(p,v) \ 483 ({ \ 484 volatile __typeof__(*(p)) *__as_ptr = (p); \ 485 __atomic_typeof_unqual(*(p)) __as_val = (v); \ 486 __ATOMIC_PTR_CHECK(__as_ptr); \ 487 __DO_ATOMIC_STORE(__as_ptr, __as_val); \ 488 }) 489 490 #define atomic_store_release(p,v) \ 491 ({ \ 492 volatile __typeof__(*(p)) *__as_ptr = (p); \ 493 __atomic_typeof_unqual(*(p)) __as_val = (v); \ 494 __ATOMIC_PTR_CHECK(__as_ptr); \ 495 membar_release(); \ 496 __DO_ATOMIC_STORE(__as_ptr, __as_val); \ 497 }) 498 499 #ifdef __HAVE_HASHLOCKED_ATOMICS 500 static __inline __always_inline void 501 __do_atomic_store(volatile void *p, const void *q, size_t size) 502 { 503 volatile uint32_t *p32 = (volatile uint32_t *)((uintptr_t)p & ~3); 504 505 switch (size) { 506 case 1: { 507 uint8_t v; 508 #if _BYTE_ORDER == _LITTLE_ENDIAN 509 unsigned s = 8 * ((uintptr_t)p & 3); 510 #elif _BYTE_ORDER == _BIG_ENDIAN 511 unsigned s = 8 * (3 - ((uintptr_t)p & 3)); 512 #else 513 # error atomic endianness kablooie 514 #endif 515 uint32_t o, n, m = ~(0xffU << s); 516 517 memcpy(&v, q, 1); 518 do { 519 o = atomic_load_relaxed(p32); 520 n = (o & m) | ((uint32_t)v << s); 521 } while (atomic_cas_32(p32, o, n) != o); 522 break; 523 } 524 case 2: { 525 uint16_t v; 526 #if _BYTE_ORDER == _LITTLE_ENDIAN 527 unsigned s = 8 * ((uintptr_t)p & 2); 528 #elif _BYTE_ORDER == _BIG_ENDIAN 529 unsigned s = 8 * (2 - ((uintptr_t)p & 2)); 530 #else 531 # error atomic endianness kablooie 532 #endif 533 uint32_t o, n, m = ~(0xffffU << s); 534 535 memcpy(&v, q, 2); 536 do { 537 o = atomic_load_relaxed(p32); 538 n = (o & m) | ((uint32_t)v << s); 539 } while (atomic_cas_32(p32, o, n) != o); 540 break; 541 } 542 case 4: { 543 uint32_t v; 544 545 memcpy(&v, q, 4); 546 (void)atomic_swap_32(p, v); 547 break; 548 } 549 #ifdef __HAVE_ATOMIC64_LOADSTORE 550 case 8: { 551 uint64_t v; 552 memcpy(&v, q, 8); 553 (void)atomic_swap_64(p, v); 554 break; 555 } 556 #endif 557 } 558 } 559 #endif /* __HAVE_HASHLOCKED_ATOMICS */ 560 561 #else /* __STDC_VERSION__ >= 201112L */ 562 563 /* C11 definitions, not yet available */ 564 565 #include <stdatomic.h> 566 567 #define atomic_load_relaxed(p) \ 568 atomic_load_explicit((p), memory_order_relaxed) 569 #if 0 /* memory_order_consume is not there yet */ 570 #define atomic_load_consume(p) \ 571 atomic_load_explicit((p), memory_order_consume) 572 #else 573 #define atomic_load_consume(p) \ 574 ({ \ 575 const __atomic_typeof_unqual(*(p)) __al_val = atomic_load_relaxed(p); \ 576 membar_datadep_consumer(); \ 577 __al_val; \ 578 }) 579 #endif 580 #define atomic_load_acquire(p) \ 581 atomic_load_explicit((p), memory_order_acquire) 582 #define atomic_store_relaxed(p, v) \ 583 atomic_store_explicit((p), (v), memory_order_relaxed) 584 #define atomic_store_release(p, v) \ 585 atomic_store_explicit((p), (v), memory_order_release) 586 587 #endif /* __STDC_VERSION__ */ 588 589 #endif /* _KERNEL */ 590 591 #endif /* ! _SYS_ATOMIC_H_ */ 592
Indexes created Fri Aug 14 00:25:39 UTC 2026