1 // SPDX-FileCopyrightText: 1991-1994 by Xerox Corporation. All rights reserved. 2 // SPDX-FileCopyrightText: 1996-1999 by Silicon Graphics. All rights reserved. 3 // SPDX-FileCopyrightText: 1999-2004 Hewlett-Packard Development Company, L.P. 4 // SPDX-FileCopyrightText: 2009 Mathieu Desnoyers <mathieu.desnoyers (at) efficios.com> 5 // SPDX-FileCopyrightText: 2010 Paolo Bonzini 6 // 7 // SPDX-License-Identifier: LicenseRef-Boehm-GC 8 9 #ifndef _URCU_UATOMIC_GENERIC_H 10 #define _URCU_UATOMIC_GENERIC_H 11 12 /* 13 * Code inspired from libuatomic_ops-1.2, inherited in part from the 14 * Boehm-Demers-Weiser conservative garbage collector. 15 */ 16 17 #include <stdint.h> 18 #include <urcu/compiler.h> 19 #include <urcu/system.h> 20 21 #ifdef __cplusplus 22 extern "C" { 23 #endif 24 25 /* 26 * Can be defined for the architecture. 27 * 28 * What needs to be emitted _before_ the `operation' with memory ordering `mo'. 29 */ 30 #ifndef _cmm_compat_c11_smp_mb__before_mo 31 # define _cmm_compat_c11_smp_mb__before_mo(operation, mo) \ 32 do { \ 33 switch (mo) { \ 34 case CMM_SEQ_CST_FENCE: \ 35 case CMM_SEQ_CST: \ 36 case CMM_ACQ_REL: \ 37 case CMM_RELEASE: \ 38 cmm_smp_mb(); \ 39 break; \ 40 case CMM_ACQUIRE: \ 41 case CMM_CONSUME: \ 42 case CMM_RELAXED: \ 43 break; \ 44 default: \ 45 abort(); \ 46 break; \ 47 \ 48 } \ 49 } while(0) 50 51 #endif /* _cmm_compat_c11_smp_mb__before_mo */ 52 53 /* 54 * Can be defined for the architecture. 55 * 56 * What needs to be emitted _after_ the `operation' with memory ordering `mo'. 57 */ 58 #ifndef _cmm_compat_c11_smp_mb__after_mo 59 # define _cmm_compat_c11_smp_mb__after_mo(operation, mo) \ 60 do { \ 61 switch (mo) { \ 62 case CMM_SEQ_CST_FENCE: \ 63 case CMM_SEQ_CST: \ 64 case CMM_ACQUIRE: \ 65 case CMM_CONSUME: \ 66 case CMM_ACQ_REL: \ 67 cmm_smp_mb(); \ 68 break; \ 69 case CMM_RELEASE: \ 70 case CMM_RELAXED: \ 71 break; \ 72 default: \ 73 abort(); \ 74 break; \ 75 \ 76 } \ 77 } while(0) 78 #endif /* _cmm_compat_c11_smp_mb__after_mo */ 79 80 /* 81 * If the toolchain supports the C11 memory model, then it is safe to implement 82 * `uatomic_store_mo()' in term of __atomic builtins. This has the effect of 83 * reducing the number of emitted memory barriers except for the 84 * CMM_SEQ_CST_FENCE memory order. 85 */ 86 #ifndef uatomic_store_mo 87 # ifdef _CMM_TOOLCHAIN_SUPPORT_C11_MM 88 # define uatomic_store_mo(addr, v, mo) \ 89 do { \ 90 __atomic_store_n(cmm_cast_volatile(addr), v, \ 91 cmm_to_c11(mo)); \ 92 cmm_seq_cst_fence_after_atomic(mo); \ 93 } while (0) 94 # else 95 # define uatomic_store_mo(addr, v, mo) \ 96 do { \ 97 _cmm_compat_c11_smp_mb__before_mo(uatomic_store, mo); \ 98 (void) CMM_STORE_SHARED(*(addr), v); \ 99 _cmm_compat_c11_smp_mb__after_mo(uatomic_store, mo); \ 100 } while (0) 101 # endif /* _CMM_TOOLCHAIN_SUPPORT_C11_MM */ 102 #endif /* !uatomic_store */ 103 104 /* 105 * If the toolchain supports the C11 memory model, then it is safe to implement 106 * `uatomic_load_mo()' in term of __atomic builtins. This has the effect of 107 * reducing the number of emitted memory barriers except for the 108 * CMM_SEQ_CST_FENCE memory order. 109 */ 110 #ifndef uatomic_load_mo 111 # ifdef _CMM_TOOLCHAIN_SUPPORT_C11_MM 112 # define uatomic_load_mo(addr, mo) \ 113 __extension__ \ 114 ({ \ 115 __typeof__(*(addr)) _value = \ 116 __atomic_load_n(cmm_cast_volatile(addr), \ 117 cmm_to_c11(mo)); \ 118 cmm_seq_cst_fence_after_atomic(mo); \ 119 \ 120 _value; \ 121 }) 122 # else 123 # define uatomic_load_mo(addr, mo) \ 124 __extension__ \ 125 ({ \ 126 _cmm_compat_c11_smp_mb__before_mo(uatomic_load, mo); \ 127 __typeof__(*(addr)) _rcu_value = CMM_LOAD_SHARED(*(addr)); \ 128 _cmm_compat_c11_smp_mb__after_mo(uatomic_load, mo); \ 129 \ 130 _rcu_value; \ 131 }) 132 # endif /* _CMM_TOOLCHAIN_SUPPORT_C11_MM */ 133 #endif /* !uatomic_load */ 134 135 #if !defined __OPTIMIZE__ || defined UATOMIC_NO_LINK_ERROR 136 #ifdef ILLEGAL_INSTR 137 static inline __attribute__((always_inline)) 138 void _uatomic_link_error(void) 139 { 140 /* 141 * generate an illegal instruction. Cannot catch this with 142 * linker tricks when optimizations are disabled. 143 */ 144 __asm__ __volatile__(ILLEGAL_INSTR); 145 } 146 #else 147 static inline __attribute__((always_inline, __noreturn__)) 148 void _uatomic_link_error(void) 149 { 150 __builtin_trap(); 151 } 152 #endif 153 154 155 /* 156 * NOTE: All RMW operations are implemented using the `__sync' builtins. All 157 * builtins used are documented to be considered a "full barrier". Therefore, 158 * for RMW operations, nothing is emitted for any memory order. 159 */ 160 161 #else /* #if !defined __OPTIMIZE__ || defined UATOMIC_NO_LINK_ERROR */ 162 extern void _uatomic_link_error(void); 163 #endif /* #else #if !defined __OPTIMIZE__ || defined UATOMIC_NO_LINK_ERROR */ 164 165 /* uatomic_cmpxchg_mo */ 166 167 #ifndef uatomic_cmpxchg_mo 168 static inline __attribute__((always_inline)) 169 unsigned long _uatomic_cmpxchg(void *addr, unsigned long old, 170 unsigned long _new, int len) 171 { 172 switch (len) { 173 #ifdef UATOMIC_HAS_ATOMIC_BYTE 174 case 1: 175 return __sync_val_compare_and_swap_1((uint8_t *) addr, old, 176 _new); 177 #endif 178 #ifdef UATOMIC_HAS_ATOMIC_SHORT 179 case 2: 180 return __sync_val_compare_and_swap_2((uint16_t *) addr, old, 181 _new); 182 #endif 183 case 4: 184 return __sync_val_compare_and_swap_4((uint32_t *) addr, old, 185 _new); 186 #if (CAA_BITS_PER_LONG == 64) 187 case 8: 188 return __sync_val_compare_and_swap_8((uint64_t *) addr, old, 189 _new); 190 #endif 191 } 192 _uatomic_link_error(); 193 return 0; 194 } 195 196 #define uatomic_cmpxchg_mo(addr, old, _new, mos, mof) \ 197 ((__typeof__(*(addr))) _uatomic_cmpxchg((addr), \ 198 caa_cast_long_keep_sign(old), \ 199 caa_cast_long_keep_sign(_new), \ 200 sizeof(*(addr)))) 201 /* uatomic_and_mo */ 202 203 #ifndef uatomic_and_mo 204 static inline __attribute__((always_inline)) 205 void _uatomic_and(void *addr, unsigned long val, 206 int len) 207 { 208 switch (len) { 209 #ifdef UATOMIC_HAS_ATOMIC_BYTE 210 case 1: 211 __sync_and_and_fetch_1((uint8_t *) addr, val); 212 return; 213 #endif 214 #ifdef UATOMIC_HAS_ATOMIC_SHORT 215 case 2: 216 __sync_and_and_fetch_2((uint16_t *) addr, val); 217 return; 218 #endif 219 case 4: 220 __sync_and_and_fetch_4((uint32_t *) addr, val); 221 return; 222 #if (CAA_BITS_PER_LONG == 64) 223 case 8: 224 __sync_and_and_fetch_8((uint64_t *) addr, val); 225 return; 226 #endif 227 } 228 _uatomic_link_error(); 229 } 230 231 #define uatomic_and_mo(addr, v, mo) \ 232 (_uatomic_and((addr), \ 233 caa_cast_long_keep_sign(v), \ 234 sizeof(*(addr)))) 235 #define cmm_smp_mb__before_uatomic_and() cmm_barrier() 236 #define cmm_smp_mb__after_uatomic_and() cmm_barrier() 237 238 #endif 239 240 /* uatomic_or_mo */ 241 242 #ifndef uatomic_or_mo 243 static inline __attribute__((always_inline)) 244 void _uatomic_or(void *addr, unsigned long val, 245 int len) 246 { 247 switch (len) { 248 #ifdef UATOMIC_HAS_ATOMIC_BYTE 249 case 1: 250 __sync_or_and_fetch_1((uint8_t *) addr, val); 251 return; 252 #endif 253 #ifdef UATOMIC_HAS_ATOMIC_SHORT 254 case 2: 255 __sync_or_and_fetch_2((uint16_t *) addr, val); 256 return; 257 #endif 258 case 4: 259 __sync_or_and_fetch_4((uint32_t *) addr, val); 260 return; 261 #if (CAA_BITS_PER_LONG == 64) 262 case 8: 263 __sync_or_and_fetch_8((uint64_t *) addr, val); 264 return; 265 #endif 266 } 267 _uatomic_link_error(); 268 return; 269 } 270 271 #define uatomic_or_mo(addr, v, mo) \ 272 (_uatomic_or((addr), \ 273 caa_cast_long_keep_sign(v), \ 274 sizeof(*(addr)))) 275 #define cmm_smp_mb__before_uatomic_or() cmm_barrier() 276 #define cmm_smp_mb__after_uatomic_or() cmm_barrier() 277 278 #endif 279 280 281 /* uatomic_add_return_mo */ 282 283 #ifndef uatomic_add_return_mo 284 static inline __attribute__((always_inline)) 285 unsigned long _uatomic_add_return(void *addr, unsigned long val, 286 int len) 287 { 288 switch (len) { 289 #ifdef UATOMIC_HAS_ATOMIC_BYTE 290 case 1: 291 return __sync_add_and_fetch_1((uint8_t *) addr, val); 292 #endif 293 #ifdef UATOMIC_HAS_ATOMIC_SHORT 294 case 2: 295 return __sync_add_and_fetch_2((uint16_t *) addr, val); 296 #endif 297 case 4: 298 return __sync_add_and_fetch_4((uint32_t *) addr, val); 299 #if (CAA_BITS_PER_LONG == 64) 300 case 8: 301 return __sync_add_and_fetch_8((uint64_t *) addr, val); 302 #endif 303 } 304 _uatomic_link_error(); 305 return 0; 306 } 307 308 309 #define uatomic_add_return_mo(addr, v, mo) \ 310 ((__typeof__(*(addr))) _uatomic_add_return((addr), \ 311 caa_cast_long_keep_sign(v), \ 312 sizeof(*(addr)))) 313 #endif /* #ifndef uatomic_add_return */ 314 315 #ifndef uatomic_xchg_mo 316 /* xchg */ 317 318 static inline __attribute__((always_inline)) 319 unsigned long _uatomic_exchange(void *addr, unsigned long val, int len) 320 { 321 switch (len) { 322 #ifdef UATOMIC_HAS_ATOMIC_BYTE 323 case 1: 324 { 325 uint8_t old; 326 327 do { 328 old = uatomic_read((uint8_t *) addr); 329 } while (!__sync_bool_compare_and_swap_1((uint8_t *) addr, 330 old, val)); 331 332 return old; 333 } 334 #endif 335 #ifdef UATOMIC_HAS_ATOMIC_SHORT 336 case 2: 337 { 338 uint16_t old; 339 340 do { 341 old = uatomic_read((uint16_t *) addr); 342 } while (!__sync_bool_compare_and_swap_2((uint16_t *) addr, 343 old, val)); 344 345 return old; 346 } 347 #endif 348 case 4: 349 { 350 uint32_t old; 351 352 do { 353 old = uatomic_read((uint32_t *) addr); 354 } while (!__sync_bool_compare_and_swap_4((uint32_t *) addr, 355 old, val)); 356 357 return old; 358 } 359 #if (CAA_BITS_PER_LONG == 64) 360 case 8: 361 { 362 uint64_t old; 363 364 do { 365 old = uatomic_read((uint64_t *) addr); 366 } while (!__sync_bool_compare_and_swap_8((uint64_t *) addr, 367 old, val)); 368 369 return old; 370 } 371 #endif 372 } 373 _uatomic_link_error(); 374 return 0; 375 } 376 377 #define uatomic_xchg_mo(addr, v, mo) \ 378 ((__typeof__(*(addr))) _uatomic_exchange((addr), \ 379 caa_cast_long_keep_sign(v), \ 380 sizeof(*(addr)))) 381 #endif /* #ifndef uatomic_xchg_mo */ 382 383 #else /* #ifndef uatomic_cmpxchg_mo */ 384 385 #ifndef uatomic_and_mo 386 /* uatomic_and_mo */ 387 388 static inline __attribute__((always_inline)) 389 void _uatomic_and(void *addr, unsigned long val, int len) 390 { 391 switch (len) { 392 #ifdef UATOMIC_HAS_ATOMIC_BYTE 393 case 1: 394 { 395 uint8_t old, oldt; 396 397 oldt = uatomic_read((uint8_t *) addr); 398 do { 399 old = oldt; 400 oldt = _uatomic_cmpxchg(addr, old, old & val, 1); 401 } while (oldt != old); 402 403 return; 404 } 405 #endif 406 #ifdef UATOMIC_HAS_ATOMIC_SHORT 407 case 2: 408 { 409 uint16_t old, oldt; 410 411 oldt = uatomic_read((uint16_t *) addr); 412 do { 413 old = oldt; 414 oldt = _uatomic_cmpxchg(addr, old, old & val, 2); 415 } while (oldt != old); 416 } 417 #endif 418 case 4: 419 { 420 uint32_t old, oldt; 421 422 oldt = uatomic_read((uint32_t *) addr); 423 do { 424 old = oldt; 425 oldt = _uatomic_cmpxchg(addr, old, old & val, 4); 426 } while (oldt != old); 427 428 return; 429 } 430 #if (CAA_BITS_PER_LONG == 64) 431 case 8: 432 { 433 uint64_t old, oldt; 434 435 oldt = uatomic_read((uint64_t *) addr); 436 do { 437 old = oldt; 438 oldt = _uatomic_cmpxchg(addr, old, old & val, 8); 439 } while (oldt != old); 440 441 return; 442 } 443 #endif 444 } 445 _uatomic_link_error(); 446 } 447 448 #define uatomic_and_mo(addr, v, mo) \ 449 (_uatomic_and((addr), \ 450 caa_cast_long_keep_sign(v), \ 451 sizeof(*(addr)))) 452 #define cmm_smp_mb__before_uatomic_and() cmm_barrier() 453 #define cmm_smp_mb__after_uatomic_and() cmm_barrier() 454 455 #endif /* #ifndef uatomic_and_mo */ 456 457 #ifndef uatomic_or_mo 458 /* uatomic_or_mo */ 459 460 static inline __attribute__((always_inline)) 461 void _uatomic_or(void *addr, unsigned long val, int len) 462 { 463 switch (len) { 464 #ifdef UATOMIC_HAS_ATOMIC_BYTE 465 case 1: 466 { 467 uint8_t old, oldt; 468 469 oldt = uatomic_read((uint8_t *) addr); 470 do { 471 old = oldt; 472 oldt = _uatomic_cmpxchg(addr, old, old | val, 1); 473 } while (oldt != old); 474 475 return; 476 } 477 #endif 478 #ifdef UATOMIC_HAS_ATOMIC_SHORT 479 case 2: 480 { 481 uint16_t old, oldt; 482 483 oldt = uatomic_read((uint16_t *) addr); 484 do { 485 old = oldt; 486 oldt = _uatomic_cmpxchg(addr, old, old | val, 2); 487 } while (oldt != old); 488 489 return; 490 } 491 #endif 492 case 4: 493 { 494 uint32_t old, oldt; 495 496 oldt = uatomic_read((uint32_t *) addr); 497 do { 498 old = oldt; 499 oldt = _uatomic_cmpxchg(addr, old, old | val, 4); 500 } while (oldt != old); 501 502 return; 503 } 504 #if (CAA_BITS_PER_LONG == 64) 505 case 8: 506 { 507 uint64_t old, oldt; 508 509 oldt = uatomic_read((uint64_t *) addr); 510 do { 511 old = oldt; 512 oldt = _uatomic_cmpxchg(addr, old, old | val, 8); 513 } while (oldt != old); 514 515 return; 516 } 517 #endif 518 } 519 _uatomic_link_error(); 520 } 521 522 #define uatomic_or_mo(addr, v, mo) \ 523 (_uatomic_or((addr), \ 524 caa_cast_long_keep_sign(v), \ 525 sizeof(*(addr)))) 526 #define cmm_smp_mb__before_uatomic_or() cmm_barrier() 527 #define cmm_smp_mb__after_uatomic_or() cmm_barrier() 528 529 #endif /* #ifndef uatomic_or_mo */ 530 531 #ifndef uatomic_add_return_mo 532 /* uatomic_add_return_mo */ 533 534 static inline __attribute__((always_inline)) 535 unsigned long _uatomic_add_return(void *addr, unsigned long val, int len) 536 { 537 switch (len) { 538 #ifdef UATOMIC_HAS_ATOMIC_BYTE 539 case 1: 540 { 541 uint8_t old, oldt; 542 543 oldt = uatomic_read((uint8_t *) addr); 544 do { 545 old = oldt; 546 oldt = uatomic_cmpxchg((uint8_t *) addr, 547 old, old + val); 548 } while (oldt != old); 549 550 return old + val; 551 } 552 #endif 553 #ifdef UATOMIC_HAS_ATOMIC_SHORT 554 case 2: 555 { 556 uint16_t old, oldt; 557 558 oldt = uatomic_read((uint16_t *) addr); 559 do { 560 old = oldt; 561 oldt = uatomic_cmpxchg((uint16_t *) addr, 562 old, old + val); 563 } while (oldt != old); 564 565 return old + val; 566 } 567 #endif 568 case 4: 569 { 570 uint32_t old, oldt; 571 572 oldt = uatomic_read((uint32_t *) addr); 573 do { 574 old = oldt; 575 oldt = uatomic_cmpxchg((uint32_t *) addr, 576 old, old + val); 577 } while (oldt != old); 578 579 return old + val; 580 } 581 #if (CAA_BITS_PER_LONG == 64) 582 case 8: 583 { 584 uint64_t old, oldt; 585 586 oldt = uatomic_read((uint64_t *) addr); 587 do { 588 old = oldt; 589 oldt = uatomic_cmpxchg((uint64_t *) addr, 590 old, old + val); 591 } while (oldt != old); 592 593 return old + val; 594 } 595 #endif 596 } 597 _uatomic_link_error(); 598 return 0; 599 } 600 601 #define uatomic_add_return_mo(addr, v, mo) \ 602 ((__typeof__(*(addr))) _uatomic_add_return((addr), \ 603 caa_cast_long_keep_sign(v), \ 604 sizeof(*(addr)))) 605 #endif /* #ifndef uatomic_add_return_mo */ 606 607 #ifndef uatomic_xchg_mo 608 /* uatomic_xchg_mo */ 609 610 static inline __attribute__((always_inline)) 611 unsigned long _uatomic_exchange(void *addr, unsigned long val, int len) 612 { 613 switch (len) { 614 #ifdef UATOMIC_HAS_ATOMIC_BYTE 615 case 1: 616 { 617 uint8_t old, oldt; 618 619 oldt = uatomic_read((uint8_t *) addr); 620 do { 621 old = oldt; 622 oldt = uatomic_cmpxchg((uint8_t *) addr, 623 old, val); 624 } while (oldt != old); 625 626 return old; 627 } 628 #endif 629 #ifdef UATOMIC_HAS_ATOMIC_SHORT 630 case 2: 631 { 632 uint16_t old, oldt; 633 634 oldt = uatomic_read((uint16_t *) addr); 635 do { 636 old = oldt; 637 oldt = uatomic_cmpxchg((uint16_t *) addr, 638 old, val); 639 } while (oldt != old); 640 641 return old; 642 } 643 #endif 644 case 4: 645 { 646 uint32_t old, oldt; 647 648 oldt = uatomic_read((uint32_t *) addr); 649 do { 650 old = oldt; 651 oldt = uatomic_cmpxchg((uint32_t *) addr, 652 old, val); 653 } while (oldt != old); 654 655 return old; 656 } 657 #if (CAA_BITS_PER_LONG == 64) 658 case 8: 659 { 660 uint64_t old, oldt; 661 662 oldt = uatomic_read((uint64_t *) addr); 663 do { 664 old = oldt; 665 oldt = uatomic_cmpxchg((uint64_t *) addr, 666 old, val); 667 } while (oldt != old); 668 669 return old; 670 } 671 #endif 672 } 673 _uatomic_link_error(); 674 return 0; 675 } 676 677 #define uatomic_xchg_mo(addr, v, mo) \ 678 ((__typeof__(*(addr))) _uatomic_exchange((addr), \ 679 caa_cast_long_keep_sign(v), \ 680 sizeof(*(addr)))) 681 #endif /* #ifndef uatomic_xchg_mo */ 682 683 #endif /* #else #ifndef uatomic_cmpxchg_mo */ 684 685 /* uatomic_sub_return_mo, uatomic_add_mo, uatomic_sub_mo, uatomic_inc_mo, uatomic_dec_mo */ 686 687 #ifndef uatomic_add_mo 688 #define uatomic_add_mo(addr, v, mo) (void)uatomic_add_return_mo((addr), (v), mo) 689 #define cmm_smp_mb__before_uatomic_add() cmm_barrier() 690 #define cmm_smp_mb__after_uatomic_add() cmm_barrier() 691 #endif 692 693 #define uatomic_sub_return_mo(addr, v, mo) \ 694 uatomic_add_return_mo((addr), -(caa_cast_long_keep_sign(v)), mo) 695 #define uatomic_sub_mo(addr, v, mo) \ 696 uatomic_add_mo((addr), -(caa_cast_long_keep_sign(v)), mo) 697 #define cmm_smp_mb__before_uatomic_sub() cmm_smp_mb__before_uatomic_add() 698 #define cmm_smp_mb__after_uatomic_sub() cmm_smp_mb__after_uatomic_add() 699 700 #ifndef uatomic_inc_mo 701 #define uatomic_inc_mo(addr, mo) uatomic_add_mo((addr), 1, mo) 702 #define cmm_smp_mb__before_uatomic_inc() cmm_smp_mb__before_uatomic_add() 703 #define cmm_smp_mb__after_uatomic_inc() cmm_smp_mb__after_uatomic_add() 704 #endif 705 706 #ifndef uatomic_dec_mo 707 #define uatomic_dec_mo(addr, mo) uatomic_add((addr), -1, mo) 708 #define cmm_smp_mb__before_uatomic_dec() cmm_smp_mb__before_uatomic_add() 709 #define cmm_smp_mb__after_uatomic_dec() cmm_smp_mb__after_uatomic_add() 710 #endif 711 712 #ifdef __cplusplus 713 } 714 #endif 715 716 #endif /* _URCU_UATOMIC_GENERIC_H */ 717
Indexes created Sat Jun 27 00:24:51 UTC 2026