1 /* $NetBSD: atomic.h,v 1.27 2025/04/22 01:34:38 riastradh Exp $ */ 2 3 /*- 4 * Copyright (c) 2007, 2008 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #ifndef _SYS_ATOMIC_H_ 33 #define _SYS_ATOMIC_H_ 34 35 #include <sys/types.h> 36 #if !defined(_KERNEL) && !defined(_STANDALONE) 37 #include <stdint.h> 38 #endif 39 40 #if defined(_KERNEL) && defined(_KERNEL_OPT) 41 #include "opt_kasan.h" 42 #include "opt_kcsan.h" 43 #include "opt_kmsan.h" 44 #endif 45 46 #if defined(KASAN) 47 #define ATOMIC_PROTO_ADD(name, tret, targ1, targ2) \ 48 void kasan_atomic_add_##name(volatile targ1 *, targ2); \ 49 tret kasan_atomic_add_##name##_nv(volatile targ1 *, targ2) 50 #define ATOMIC_PROTO_AND(name, tret, targ1, targ2) \ 51 void kasan_atomic_and_##name(volatile targ1 *, targ2); \ 52 tret kasan_atomic_and_##name##_nv(volatile targ1 *, targ2) 53 #define ATOMIC_PROTO_OR(name, tret, targ1, targ2) \ 54 void kasan_atomic_or_##name(volatile targ1 *, targ2); \ 55 tret kasan_atomic_or_##name##_nv(volatile targ1 *, targ2) 56 #define ATOMIC_PROTO_CAS(name, tret, targ1, targ2) \ 57 tret kasan_atomic_cas_##name(volatile targ1 *, targ2, targ2); \ 58 tret kasan_atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2) 59 #define ATOMIC_PROTO_SWAP(name, tret, targ1, targ2) \ 60 tret kasan_atomic_swap_##name(volatile targ1 *, targ2) 61 #define ATOMIC_PROTO_DEC(name, tret, targ1) \ 62 void kasan_atomic_dec_##name(volatile targ1 *); \ 63 tret kasan_atomic_dec_##name##_nv(volatile targ1 *) 64 #define ATOMIC_PROTO_INC(name, tret, targ1) \ 65 void kasan_atomic_inc_##name(volatile targ1 *); \ 66 tret kasan_atomic_inc_##name##_nv(volatile targ1 *) 67 #elif defined(KCSAN) 68 #define ATOMIC_PROTO_ADD(name, tret, targ1, targ2) \ 69 void kcsan_atomic_add_##name(volatile targ1 *, targ2); \ 70 tret kcsan_atomic_add_##name##_nv(volatile targ1 *, targ2) 71 #define ATOMIC_PROTO_AND(name, tret, targ1, targ2) \ 72 void kcsan_atomic_and_##name(volatile targ1 *, targ2); \ 73 tret kcsan_atomic_and_##name##_nv(volatile targ1 *, targ2) 74 #define ATOMIC_PROTO_OR(name, tret, targ1, targ2) \ 75 void kcsan_atomic_or_##name(volatile targ1 *, targ2); \ 76 tret kcsan_atomic_or_##name##_nv(volatile targ1 *, targ2) 77 #define ATOMIC_PROTO_CAS(name, tret, targ1, targ2) \ 78 tret kcsan_atomic_cas_##name(volatile targ1 *, targ2, targ2); \ 79 tret kcsan_atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2) 80 #define ATOMIC_PROTO_SWAP(name, tret, targ1, targ2) \ 81 tret kcsan_atomic_swap_##name(volatile targ1 *, targ2) 82 #define ATOMIC_PROTO_DEC(name, tret, targ1) \ 83 void kcsan_atomic_dec_##name(volatile targ1 *); \ 84 tret kcsan_atomic_dec_##name##_nv(volatile targ1 *) 85 #define ATOMIC_PROTO_INC(name, tret, targ1) \ 86 void kcsan_atomic_inc_##name(volatile targ1 *); \ 87 tret kcsan_atomic_inc_##name##_nv(volatile targ1 *) 88 #elif defined(KMSAN) 89 #define ATOMIC_PROTO_ADD(name, tret, targ1, targ2) \ 90 void kmsan_atomic_add_##name(volatile targ1 *, targ2); \ 91 tret kmsan_atomic_add_##name##_nv(volatile targ1 *, targ2) 92 #define ATOMIC_PROTO_AND(name, tret, targ1, targ2) \ 93 void kmsan_atomic_and_##name(volatile targ1 *, targ2); \ 94 tret kmsan_atomic_and_##name##_nv(volatile targ1 *, targ2) 95 #define ATOMIC_PROTO_OR(name, tret, targ1, targ2) \ 96 void kmsan_atomic_or_##name(volatile targ1 *, targ2); \ 97 tret kmsan_atomic_or_##name##_nv(volatile targ1 *, targ2) 98 #define ATOMIC_PROTO_CAS(name, tret, targ1, targ2) \ 99 tret kmsan_atomic_cas_##name(volatile targ1 *, targ2, targ2); \ 100 tret kmsan_atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2) 101 #define ATOMIC_PROTO_SWAP(name, tret, targ1, targ2) \ 102 tret kmsan_atomic_swap_##name(volatile targ1 *, targ2) 103 #define ATOMIC_PROTO_DEC(name, tret, targ1) \ 104 void kmsan_atomic_dec_##name(volatile targ1 *); \ 105 tret kmsan_atomic_dec_##name##_nv(volatile targ1 *) 106 #define ATOMIC_PROTO_INC(name, tret, targ1) \ 107 void kmsan_atomic_inc_##name(volatile targ1 *); \ 108 tret kmsan_atomic_inc_##name##_nv(volatile targ1 *) 109 #else 110 #define ATOMIC_PROTO_ADD(name, tret, targ1, targ2) \ 111 void atomic_add_##name(volatile targ1 *, targ2); \ 112 tret atomic_add_##name##_nv(volatile targ1 *, targ2) 113 #define ATOMIC_PROTO_AND(name, tret, targ1, targ2) \ 114 void atomic_and_##name(volatile targ1 *, targ2); \ 115 tret atomic_and_##name##_nv(volatile targ1 *, targ2) 116 #define ATOMIC_PROTO_OR(name, tret, targ1, targ2) \ 117 void atomic_or_##name(volatile targ1 *, targ2); \ 118 tret atomic_or_##name##_nv(volatile targ1 *, targ2) 119 #define ATOMIC_PROTO_CAS(name, tret, targ1, targ2) \ 120 tret atomic_cas_##name(volatile targ1 *, targ2, targ2); \ 121 tret atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2) 122 #define ATOMIC_PROTO_SWAP(name, tret, targ1, targ2) \ 123 tret atomic_swap_##name(volatile targ1 *, targ2) 124 #define ATOMIC_PROTO_DEC(name, tret, targ1) \ 125 void atomic_dec_##name(volatile targ1 *); \ 126 tret atomic_dec_##name##_nv(volatile targ1 *) 127 #define ATOMIC_PROTO_INC(name, tret, targ1) \ 128 void atomic_inc_##name(volatile targ1 *); \ 129 tret atomic_inc_##name##_nv(volatile targ1 *) 130 #endif 131 132 __BEGIN_DECLS 133 134 ATOMIC_PROTO_ADD(32, uint32_t, uint32_t, int32_t); 135 ATOMIC_PROTO_ADD(64, uint64_t, uint64_t, int64_t); 136 ATOMIC_PROTO_ADD(int, unsigned int, unsigned int, int); 137 ATOMIC_PROTO_ADD(long, unsigned long, unsigned long, long); 138 ATOMIC_PROTO_ADD(ptr, void *, void, ssize_t); 139 140 ATOMIC_PROTO_AND(32, uint32_t, uint32_t, uint32_t); 141 ATOMIC_PROTO_AND(64, uint64_t, uint64_t, uint64_t); 142 ATOMIC_PROTO_AND(uint, unsigned int, unsigned int, unsigned int); 143 ATOMIC_PROTO_AND(ulong, unsigned long, unsigned long, unsigned long); 144 145 ATOMIC_PROTO_OR(32, uint32_t, uint32_t, uint32_t); 146 ATOMIC_PROTO_OR(64, uint64_t, uint64_t, uint64_t); 147 ATOMIC_PROTO_OR(uint, unsigned int, unsigned int, unsigned int); 148 ATOMIC_PROTO_OR(ulong, unsigned long, unsigned long, unsigned long); 149 150 ATOMIC_PROTO_CAS(32, uint32_t, uint32_t, uint32_t); 151 ATOMIC_PROTO_CAS(64, uint64_t, uint64_t, uint64_t); 152 ATOMIC_PROTO_CAS(uint, unsigned int, unsigned int, unsigned int); 153 ATOMIC_PROTO_CAS(ulong, unsigned long, unsigned long, unsigned long); 154 ATOMIC_PROTO_CAS(ptr, void *, void, void *); 155 156 ATOMIC_PROTO_SWAP(32, uint32_t, uint32_t, uint32_t); 157 ATOMIC_PROTO_SWAP(64, uint64_t, uint64_t, uint64_t); 158 ATOMIC_PROTO_SWAP(uint, unsigned int, unsigned int, unsigned int); 159 ATOMIC_PROTO_SWAP(ulong, unsigned long, unsigned long, unsigned long); 160 ATOMIC_PROTO_SWAP(ptr, void *, void, void *); 161 162 ATOMIC_PROTO_DEC(32, uint32_t, uint32_t); 163 ATOMIC_PROTO_DEC(64, uint64_t, uint64_t); 164 ATOMIC_PROTO_DEC(uint, unsigned int, unsigned int); 165 ATOMIC_PROTO_DEC(ulong, unsigned long, unsigned long); 166 ATOMIC_PROTO_DEC(ptr, void *, void); 167 168 ATOMIC_PROTO_INC(32, uint32_t, uint32_t); 169 ATOMIC_PROTO_INC(64, uint64_t, uint64_t); 170 ATOMIC_PROTO_INC(uint, unsigned int, unsigned int); 171 ATOMIC_PROTO_INC(ulong, unsigned long, unsigned long); 172 ATOMIC_PROTO_INC(ptr, void *, void); 173 174 /* 175 * These operations will be provided for userland, but may not be 176 * implemented efficiently. 177 */ 178 uint16_t atomic_cas_16(volatile uint16_t *, uint16_t, uint16_t); 179 uint8_t atomic_cas_8(volatile uint8_t *, uint8_t, uint8_t); 180 181 /* 182 * Memory barrier operations 183 */ 184 void membar_acquire(void); 185 void membar_release(void); 186 void membar_producer(void); 187 void membar_consumer(void); 188 void membar_sync(void); 189 190 /* 191 * Deprecated memory barriers 192 */ 193 void membar_enter(void); 194 void membar_exit(void); 195 196 #ifdef __HAVE_MEMBAR_DATADEP_CONSUMER 197 void membar_datadep_consumer(void); 198 #else 199 #define membar_datadep_consumer() ((void)0) 200 #endif 201 202 __END_DECLS 203 204 #if defined(KASAN) 205 #define atomic_add_32 kasan_atomic_add_32 206 #define atomic_add_int kasan_atomic_add_int 207 #define atomic_add_long kasan_atomic_add_long 208 #define atomic_add_ptr kasan_atomic_add_ptr 209 #define atomic_add_64 kasan_atomic_add_64 210 #define atomic_add_32_nv kasan_atomic_add_32_nv 211 #define atomic_add_int_nv kasan_atomic_add_int_nv 212 #define atomic_add_long_nv kasan_atomic_add_long_nv 213 #define atomic_add_ptr_nv kasan_atomic_add_ptr_nv 214 #define atomic_add_64_nv kasan_atomic_add_64_nv 215 #define atomic_and_32 kasan_atomic_and_32 216 #define atomic_and_uint kasan_atomic_and_uint 217 #define atomic_and_ulong kasan_atomic_and_ulong 218 #define atomic_and_64 kasan_atomic_and_64 219 #define atomic_and_32_nv kasan_atomic_and_32_nv 220 #define atomic_and_uint_nv kasan_atomic_and_uint_nv 221 #define atomic_and_ulong_nv kasan_atomic_and_ulong_nv 222 #define atomic_and_64_nv kasan_atomic_and_64_nv 223 #define atomic_or_32 kasan_atomic_or_32 224 #define atomic_or_uint kasan_atomic_or_uint 225 #define atomic_or_ulong kasan_atomic_or_ulong 226 #define atomic_or_64 kasan_atomic_or_64 227 #define atomic_or_32_nv kasan_atomic_or_32_nv 228 #define atomic_or_uint_nv kasan_atomic_or_uint_nv 229 #define atomic_or_ulong_nv kasan_atomic_or_ulong_nv 230 #define atomic_or_64_nv kasan_atomic_or_64_nv 231 #define atomic_cas_32 kasan_atomic_cas_32 232 #define atomic_cas_uint kasan_atomic_cas_uint 233 #define atomic_cas_ulong kasan_atomic_cas_ulong 234 #define atomic_cas_ptr kasan_atomic_cas_ptr 235 #define atomic_cas_64 kasan_atomic_cas_64 236 #define atomic_cas_32_ni kasan_atomic_cas_32_ni 237 #define atomic_cas_uint_ni kasan_atomic_cas_uint_ni 238 #define atomic_cas_ulong_ni kasan_atomic_cas_ulong_ni 239 #define atomic_cas_ptr_ni kasan_atomic_cas_ptr_ni 240 #define atomic_cas_64_ni kasan_atomic_cas_64_ni 241 #define atomic_swap_32 kasan_atomic_swap_32 242 #define atomic_swap_uint kasan_atomic_swap_uint 243 #define atomic_swap_ulong kasan_atomic_swap_ulong 244 #define atomic_swap_ptr kasan_atomic_swap_ptr 245 #define atomic_swap_64 kasan_atomic_swap_64 246 #define atomic_dec_32 kasan_atomic_dec_32 247 #define atomic_dec_uint kasan_atomic_dec_uint 248 #define atomic_dec_ulong kasan_atomic_dec_ulong 249 #define atomic_dec_ptr kasan_atomic_dec_ptr 250 #define atomic_dec_64 kasan_atomic_dec_64 251 #define atomic_dec_32_nv kasan_atomic_dec_32_nv 252 #define atomic_dec_uint_nv kasan_atomic_dec_uint_nv 253 #define atomic_dec_ulong_nv kasan_atomic_dec_ulong_nv 254 #define atomic_dec_ptr_nv kasan_atomic_dec_ptr_nv 255 #define atomic_dec_64_nv kasan_atomic_dec_64_nv 256 #define atomic_inc_32 kasan_atomic_inc_32 257 #define atomic_inc_uint kasan_atomic_inc_uint 258 #define atomic_inc_ulong kasan_atomic_inc_ulong 259 #define atomic_inc_ptr kasan_atomic_inc_ptr 260 #define atomic_inc_64 kasan_atomic_inc_64 261 #define atomic_inc_32_nv kasan_atomic_inc_32_nv 262 #define atomic_inc_uint_nv kasan_atomic_inc_uint_nv 263 #define atomic_inc_ulong_nv kasan_atomic_inc_ulong_nv 264 #define atomic_inc_ptr_nv kasan_atomic_inc_ptr_nv 265 #define atomic_inc_64_nv kasan_atomic_inc_64_nv 266 #elif defined(KCSAN) 267 #define atomic_add_32 kcsan_atomic_add_32 268 #define atomic_add_int kcsan_atomic_add_int 269 #define atomic_add_long kcsan_atomic_add_long 270 #define atomic_add_ptr kcsan_atomic_add_ptr 271 #define atomic_add_64 kcsan_atomic_add_64 272 #define atomic_add_32_nv kcsan_atomic_add_32_nv 273 #define atomic_add_int_nv kcsan_atomic_add_int_nv 274 #define atomic_add_long_nv kcsan_atomic_add_long_nv 275 #define atomic_add_ptr_nv kcsan_atomic_add_ptr_nv 276 #define atomic_add_64_nv kcsan_atomic_add_64_nv 277 #define atomic_and_32 kcsan_atomic_and_32 278 #define atomic_and_uint kcsan_atomic_and_uint 279 #define atomic_and_ulong kcsan_atomic_and_ulong 280 #define atomic_and_64 kcsan_atomic_and_64 281 #define atomic_and_32_nv kcsan_atomic_and_32_nv 282 #define atomic_and_uint_nv kcsan_atomic_and_uint_nv 283 #define atomic_and_ulong_nv kcsan_atomic_and_ulong_nv 284 #define atomic_and_64_nv kcsan_atomic_and_64_nv 285 #define atomic_or_32 kcsan_atomic_or_32 286 #define atomic_or_uint kcsan_atomic_or_uint 287 #define atomic_or_ulong kcsan_atomic_or_ulong 288 #define atomic_or_64 kcsan_atomic_or_64 289 #define atomic_or_32_nv kcsan_atomic_or_32_nv 290 #define atomic_or_uint_nv kcsan_atomic_or_uint_nv 291 #define atomic_or_ulong_nv kcsan_atomic_or_ulong_nv 292 #define atomic_or_64_nv kcsan_atomic_or_64_nv 293 #define atomic_cas_32 kcsan_atomic_cas_32 294 #define atomic_cas_uint kcsan_atomic_cas_uint 295 #define atomic_cas_ulong kcsan_atomic_cas_ulong 296 #define atomic_cas_ptr kcsan_atomic_cas_ptr 297 #define atomic_cas_64 kcsan_atomic_cas_64 298 #define atomic_cas_32_ni kcsan_atomic_cas_32_ni 299 #define atomic_cas_uint_ni kcsan_atomic_cas_uint_ni 300 #define atomic_cas_ulong_ni kcsan_atomic_cas_ulong_ni 301 #define atomic_cas_ptr_ni kcsan_atomic_cas_ptr_ni 302 #define atomic_cas_64_ni kcsan_atomic_cas_64_ni 303 #define atomic_swap_32 kcsan_atomic_swap_32 304 #define atomic_swap_uint kcsan_atomic_swap_uint 305 #define atomic_swap_ulong kcsan_atomic_swap_ulong 306 #define atomic_swap_ptr kcsan_atomic_swap_ptr 307 #define atomic_swap_64 kcsan_atomic_swap_64 308 #define atomic_dec_32 kcsan_atomic_dec_32 309 #define atomic_dec_uint kcsan_atomic_dec_uint 310 #define atomic_dec_ulong kcsan_atomic_dec_ulong 311 #define atomic_dec_ptr kcsan_atomic_dec_ptr 312 #define atomic_dec_64 kcsan_atomic_dec_64 313 #define atomic_dec_32_nv kcsan_atomic_dec_32_nv 314 #define atomic_dec_uint_nv kcsan_atomic_dec_uint_nv 315 #define atomic_dec_ulong_nv kcsan_atomic_dec_ulong_nv 316 #define atomic_dec_ptr_nv kcsan_atomic_dec_ptr_nv 317 #define atomic_dec_64_nv kcsan_atomic_dec_64_nv 318 #define atomic_inc_32 kcsan_atomic_inc_32 319 #define atomic_inc_uint kcsan_atomic_inc_uint 320 #define atomic_inc_ulong kcsan_atomic_inc_ulong 321 #define atomic_inc_ptr kcsan_atomic_inc_ptr 322 #define atomic_inc_64 kcsan_atomic_inc_64 323 #define atomic_inc_32_nv kcsan_atomic_inc_32_nv 324 #define atomic_inc_uint_nv kcsan_atomic_inc_uint_nv 325 #define atomic_inc_ulong_nv kcsan_atomic_inc_ulong_nv 326 #define atomic_inc_ptr_nv kcsan_atomic_inc_ptr_nv 327 #define atomic_inc_64_nv kcsan_atomic_inc_64_nv 328 #elif defined(KMSAN) 329 #define atomic_add_32 kmsan_atomic_add_32 330 #define atomic_add_int kmsan_atomic_add_int 331 #define atomic_add_long kmsan_atomic_add_long 332 #define atomic_add_ptr kmsan_atomic_add_ptr 333 #define atomic_add_64 kmsan_atomic_add_64 334 #define atomic_add_32_nv kmsan_atomic_add_32_nv 335 #define atomic_add_int_nv kmsan_atomic_add_int_nv 336 #define atomic_add_long_nv kmsan_atomic_add_long_nv 337 #define atomic_add_ptr_nv kmsan_atomic_add_ptr_nv 338 #define atomic_add_64_nv kmsan_atomic_add_64_nv 339 #define atomic_and_32 kmsan_atomic_and_32 340 #define atomic_and_uint kmsan_atomic_and_uint 341 #define atomic_and_ulong kmsan_atomic_and_ulong 342 #define atomic_and_64 kmsan_atomic_and_64 343 #define atomic_and_32_nv kmsan_atomic_and_32_nv 344 #define atomic_and_uint_nv kmsan_atomic_and_uint_nv 345 #define atomic_and_ulong_nv kmsan_atomic_and_ulong_nv 346 #define atomic_and_64_nv kmsan_atomic_and_64_nv 347 #define atomic_or_32 kmsan_atomic_or_32 348 #define atomic_or_uint kmsan_atomic_or_uint 349 #define atomic_or_ulong kmsan_atomic_or_ulong 350 #define atomic_or_64 kmsan_atomic_or_64 351 #define atomic_or_32_nv kmsan_atomic_or_32_nv 352 #define atomic_or_uint_nv kmsan_atomic_or_uint_nv 353 #define atomic_or_ulong_nv kmsan_atomic_or_ulong_nv 354 #define atomic_or_64_nv kmsan_atomic_or_64_nv 355 #define atomic_cas_32 kmsan_atomic_cas_32 356 #define atomic_cas_uint kmsan_atomic_cas_uint 357 #define atomic_cas_ulong kmsan_atomic_cas_ulong 358 #define atomic_cas_ptr kmsan_atomic_cas_ptr 359 #define atomic_cas_64 kmsan_atomic_cas_64 360 #define atomic_cas_32_ni kmsan_atomic_cas_32_ni 361 #define atomic_cas_uint_ni kmsan_atomic_cas_uint_ni 362 #define atomic_cas_ulong_ni kmsan_atomic_cas_ulong_ni 363 #define atomic_cas_ptr_ni kmsan_atomic_cas_ptr_ni 364 #define atomic_cas_64_ni kmsan_atomic_cas_64_ni 365 #define atomic_swap_32 kmsan_atomic_swap_32 366 #define atomic_swap_uint kmsan_atomic_swap_uint 367 #define atomic_swap_ulong kmsan_atomic_swap_ulong 368 #define atomic_swap_ptr kmsan_atomic_swap_ptr 369 #define atomic_swap_64 kmsan_atomic_swap_64 370 #define atomic_dec_32 kmsan_atomic_dec_32 371 #define atomic_dec_uint kmsan_atomic_dec_uint 372 #define atomic_dec_ulong kmsan_atomic_dec_ulong 373 #define atomic_dec_ptr kmsan_atomic_dec_ptr 374 #define atomic_dec_64 kmsan_atomic_dec_64 375 #define atomic_dec_32_nv kmsan_atomic_dec_32_nv 376 #define atomic_dec_uint_nv kmsan_atomic_dec_uint_nv 377 #define atomic_dec_ulong_nv kmsan_atomic_dec_ulong_nv 378 #define atomic_dec_ptr_nv kmsan_atomic_dec_ptr_nv 379 #define atomic_dec_64_nv kmsan_atomic_dec_64_nv 380 #define atomic_inc_32 kmsan_atomic_inc_32 381 #define atomic_inc_uint kmsan_atomic_inc_uint 382 #define atomic_inc_ulong kmsan_atomic_inc_ulong 383 #define atomic_inc_ptr kmsan_atomic_inc_ptr 384 #define atomic_inc_64 kmsan_atomic_inc_64 385 #define atomic_inc_32_nv kmsan_atomic_inc_32_nv 386 #define atomic_inc_uint_nv kmsan_atomic_inc_uint_nv 387 #define atomic_inc_ulong_nv kmsan_atomic_inc_ulong_nv 388 #define atomic_inc_ptr_nv kmsan_atomic_inc_ptr_nv 389 #define atomic_inc_64_nv kmsan_atomic_inc_64_nv 390 #endif 391 392 #ifdef _KERNEL 393 394 #if 1 // XXX: __STDC_VERSION__ < 201112L 395 396 /* Pre-C11 definitions */ 397 398 #include <sys/cdefs.h> 399 400 #include <lib/libkern/libkern.h> 401 402 #ifdef _LP64 403 #define __HAVE_ATOMIC64_LOADSTORE 1 404 #define __ATOMIC_SIZE_MAX 8 405 #else 406 #define __ATOMIC_SIZE_MAX 4 407 #endif 408 409 /* 410 * We assume that access to an aligned pointer to a volatile object of 411 * at most __ATOMIC_SIZE_MAX bytes is guaranteed to be atomic. This is 412 * an assumption that may be wrong, but we hope it won't be wrong 413 * before we just adopt the C11 atomic API. 414 */ 415 #define __ATOMIC_PTR_CHECK(p) do \ 416 { \ 417 CTASSERT(sizeof(*(p)) <= __ATOMIC_SIZE_MAX); \ 418 KASSERT(((uintptr_t)(p) & (sizeof(*(p)) - 1)) == 0); \ 419 } while (0) 420 421 #ifdef KCSAN 422 void kcsan_atomic_load(const volatile void *, void *, int); 423 void kcsan_atomic_store(volatile void *, const void *, int); 424 #define __BEGIN_ATOMIC_LOAD(p, v) \ 425 union { __typeof__(*(p)) __al_val; char __al_buf[1]; } v; \ 426 kcsan_atomic_load(p, v.__al_buf, sizeof(v.__al_val)) 427 #define __END_ATOMIC_LOAD(v) \ 428 (v).__al_val 429 #define __DO_ATOMIC_STORE(p, v) \ 430 kcsan_atomic_store(p, __UNVOLATILE(&v), sizeof(v)) 431 #else 432 #define __BEGIN_ATOMIC_LOAD(p, v) \ 433 __typeof__(*(p)) v = *(p) 434 #define __END_ATOMIC_LOAD(v) \ 435 v 436 #ifdef __HAVE_HASHLOCKED_ATOMICS 437 #define __DO_ATOMIC_STORE(p, v) \ 438 __do_atomic_store(p, __UNVOLATILE(&v), sizeof(v)) 439 #else /* !__HAVE_HASHLOCKED_ATOMICS */ 440 #define __DO_ATOMIC_STORE(p, v) \ 441 *p = v 442 #endif 443 #endif 444 445 #define atomic_load_relaxed(p) \ 446 ({ \ 447 const volatile __typeof__(*(p)) *__al_ptr = (p); \ 448 __ATOMIC_PTR_CHECK(__al_ptr); \ 449 __BEGIN_ATOMIC_LOAD(__al_ptr, __al_val); \ 450 __END_ATOMIC_LOAD(__al_val); \ 451 }) 452 453 #define atomic_load_consume(p) \ 454 ({ \ 455 const volatile __typeof__(*(p)) *__al_ptr = (p); \ 456 __ATOMIC_PTR_CHECK(__al_ptr); \ 457 __BEGIN_ATOMIC_LOAD(__al_ptr, __al_val); \ 458 membar_datadep_consumer(); \ 459 __END_ATOMIC_LOAD(__al_val); \ 460 }) 461 462 #define atomic_load_acquire(p) \ 463 ({ \ 464 const volatile __typeof__(*(p)) *__al_ptr = (p); \ 465 __ATOMIC_PTR_CHECK(__al_ptr); \ 466 __BEGIN_ATOMIC_LOAD(__al_ptr, __al_val); \ 467 membar_acquire(); \ 468 __END_ATOMIC_LOAD(__al_val); \ 469 }) 470 471 #define atomic_store_relaxed(p,v) \ 472 ({ \ 473 volatile __typeof__(*(p)) *__as_ptr = (p); \ 474 __typeof__(*(p)) __as_val = (v); \ 475 __ATOMIC_PTR_CHECK(__as_ptr); \ 476 __DO_ATOMIC_STORE(__as_ptr, __as_val); \ 477 }) 478 479 #define atomic_store_release(p,v) \ 480 ({ \ 481 volatile __typeof__(*(p)) *__as_ptr = (p); \ 482 __typeof__(*(p)) __as_val = (v); \ 483 __ATOMIC_PTR_CHECK(__as_ptr); \ 484 membar_release(); \ 485 __DO_ATOMIC_STORE(__as_ptr, __as_val); \ 486 }) 487 488 #ifdef __HAVE_HASHLOCKED_ATOMICS 489 static __inline __always_inline void 490 __do_atomic_store(volatile void *p, const void *q, size_t size) 491 { 492 volatile uint32_t *p32 = (volatile uint32_t *)((uintptr_t)p & ~3); 493 494 switch (size) { 495 case 1: { 496 uint8_t v; 497 unsigned s = 8 * ((uintptr_t)p & 3); 498 uint32_t o, n, m = ~(0xffU << s); 499 500 memcpy(&v, q, 1); 501 do { 502 o = atomic_load_relaxed(p32); 503 n = (o & m) | ((uint32_t)v << s); 504 } while (atomic_cas_32(p32, o, n) != o); 505 break; 506 } 507 case 2: { 508 uint16_t v; 509 unsigned s = 8 * (((uintptr_t)p & 2) >> 1); 510 uint32_t o, n, m = ~(0xffffU << s); 511 512 memcpy(&v, q, 2); 513 do { 514 o = atomic_load_relaxed(p32); 515 n = (o & m) | ((uint32_t)v << s); 516 } while (atomic_cas_32(p32, o, n) != o); 517 break; 518 } 519 case 4: { 520 uint32_t v; 521 522 memcpy(&v, q, 4); 523 (void)atomic_swap_32(p, v); 524 break; 525 } 526 #ifdef __HAVE_ATOMIC64_LOADSTORE 527 case 8: { 528 uint64_t v; 529 memcpy(&v, q, 8); 530 (void)atomic_swap_64(p, v); 531 break; 532 } 533 #endif 534 } 535 } 536 #endif /* __HAVE_HASHLOCKED_ATOMICS */ 537 538 #else /* __STDC_VERSION__ >= 201112L */ 539 540 /* C11 definitions, not yet available */ 541 542 #include <stdatomic.h> 543 544 #define atomic_load_relaxed(p) \ 545 atomic_load_explicit((p), memory_order_relaxed) 546 #if 0 /* memory_order_consume is not there yet */ 547 #define atomic_load_consume(p) \ 548 atomic_load_explicit((p), memory_order_consume) 549 #else 550 #define atomic_load_consume(p) \ 551 ({ \ 552 const __typeof__(*(p)) __al_val = atomic_load_relaxed(p); \ 553 membar_datadep_consumer(); \ 554 __al_val; \ 555 }) 556 #endif 557 #define atomic_load_acquire(p) \ 558 atomic_load_explicit((p), memory_order_acquire) 559 #define atomic_store_relaxed(p, v) \ 560 atomic_store_explicit((p), (v), memory_order_relaxed) 561 #define atomic_store_release(p, v) \ 562 atomic_store_explicit((p), (v), memory_order_release) 563 564 #endif /* __STDC_VERSION__ */ 565 566 #endif /* _KERNEL */ 567 568 #endif /* ! _SYS_ATOMIC_H_ */ 569
Indexes created Sat Sep 20 22:09:52 GMT 2025