1 // SPDX-FileCopyrightText: 1991-1994 by Xerox Corporation. All rights reserved. 2 // SPDX-FileCopyrightText: 1996-1999 by Silicon Graphics. All rights reserved. 3 // SPDX-FileCopyrightText: 1999-2004 Hewlett-Packard Development Company, L.P. 4 // SPDX-FileCopyrightText: 2009 Mathieu Desnoyers <mathieu.desnoyers (at) efficios.com> 5 // 6 // SPDX-License-Identifier: LicenseRef-Boehm-GC 7 8 #ifndef _URCU_ARCH_UATOMIC_PPC_H 9 #define _URCU_ARCH_UATOMIC_PPC_H 10 11 /* 12 * Code inspired from libuatomic_ops-1.2, inherited in part from the 13 * Boehm-Demers-Weiser conservative garbage collector. 14 */ 15 16 #include <urcu/compiler.h> 17 #include <urcu/system.h> 18 19 #ifdef __cplusplus 20 extern "C" { 21 #endif 22 23 #define ILLEGAL_INSTR ".long 0xd00d00" 24 25 /* 26 * Providing sequential consistency semantic with respect to other 27 * instructions for cmpxchg and add_return family of atomic primitives. 28 * 29 * This is achieved with: 30 * lwsync (prior stores can be reordered after following loads) 31 * lwarx 32 * stwcx. 33 * test if success (retry) 34 * sync 35 * 36 * Explanation of the sequential consistency provided by this scheme 37 * from Paul E. McKenney: 38 * 39 * The reason we can get away with the lwsync before is that if a prior 40 * store reorders with the lwarx, then you have to store to the atomic 41 * variable from some other CPU to detect it. 42 * 43 * And if you do that, the lwarx will lose its reservation, so the stwcx 44 * will fail. The atomic operation will retry, so that the caller won't be 45 * able to see the misordering. 46 */ 47 48 /* xchg */ 49 50 static inline __attribute__((always_inline)) 51 unsigned long _uatomic_exchange(void *addr, unsigned long val, int len) 52 { 53 switch (len) { 54 case 4: 55 { 56 unsigned int result; 57 58 __asm__ __volatile__( 59 LWSYNC_OPCODE 60 "1:\t" "lwarx %0,0,%1\n" /* load and reserve */ 61 "stwcx. %2,0,%1\n" /* else store conditional */ 62 "bne- 1b\n" /* retry if lost reservation */ 63 "sync\n" 64 : "=&r"(result) 65 : "r"(addr), "r"(val) 66 : "memory", "cc"); 67 68 return result; 69 } 70 #if (CAA_BITS_PER_LONG == 64) 71 case 8: 72 { 73 unsigned long result; 74 75 __asm__ __volatile__( 76 LWSYNC_OPCODE 77 "1:\t" "ldarx %0,0,%1\n" /* load and reserve */ 78 "stdcx. %2,0,%1\n" /* else store conditional */ 79 "bne- 1b\n" /* retry if lost reservation */ 80 "sync\n" 81 : "=&r"(result) 82 : "r"(addr), "r"(val) 83 : "memory", "cc"); 84 85 return result; 86 } 87 #endif 88 } 89 /* 90 * generate an illegal instruction. Cannot catch this with 91 * linker tricks when optimizations are disabled. 92 */ 93 __asm__ __volatile__(ILLEGAL_INSTR); 94 return 0; 95 } 96 97 #define uatomic_xchg_mo(addr, v, mo) \ 98 ((__typeof__(*(addr))) _uatomic_exchange((addr), \ 99 caa_cast_long_keep_sign(v), \ 100 sizeof(*(addr)))) 101 /* cmpxchg */ 102 103 static inline __attribute__((always_inline)) 104 unsigned long _uatomic_cmpxchg(void *addr, unsigned long old, 105 unsigned long _new, int len) 106 { 107 switch (len) { 108 case 4: 109 { 110 unsigned int old_val; 111 112 __asm__ __volatile__( 113 LWSYNC_OPCODE 114 "1:\t" "lwarx %0,0,%1\n" /* load and reserve */ 115 "cmpw %0,%3\n" /* if load is not equal to */ 116 "bne 2f\n" /* old, fail */ 117 "stwcx. %2,0,%1\n" /* else store conditional */ 118 "bne- 1b\n" /* retry if lost reservation */ 119 "sync\n" 120 "2:\n" 121 : "=&r"(old_val) 122 : "r"(addr), "r"((unsigned int)_new), 123 "r"((unsigned int)old) 124 : "memory", "cc"); 125 126 return old_val; 127 } 128 #if (CAA_BITS_PER_LONG == 64) 129 case 8: 130 { 131 unsigned long old_val; 132 133 __asm__ __volatile__( 134 LWSYNC_OPCODE 135 "1:\t" "ldarx %0,0,%1\n" /* load and reserve */ 136 "cmpd %0,%3\n" /* if load is not equal to */ 137 "bne 2f\n" /* old, fail */ 138 "stdcx. %2,0,%1\n" /* else store conditional */ 139 "bne- 1b\n" /* retry if lost reservation */ 140 "sync\n" 141 "2:\n" 142 : "=&r"(old_val) 143 : "r"(addr), "r"((unsigned long)_new), 144 "r"((unsigned long)old) 145 : "memory", "cc"); 146 147 return old_val; 148 } 149 #endif 150 } 151 /* 152 * generate an illegal instruction. Cannot catch this with 153 * linker tricks when optimizations are disabled. 154 */ 155 __asm__ __volatile__(ILLEGAL_INSTR); 156 return 0; 157 } 158 159 160 #define uatomic_cmpxchg_mo(addr, old, _new, mos, mof) \ 161 ((__typeof__(*(addr))) _uatomic_cmpxchg((addr), \ 162 caa_cast_long_keep_sign(old), \ 163 caa_cast_long_keep_sign(_new),\ 164 sizeof(*(addr)))) 165 166 /* uatomic_add_return */ 167 168 static inline __attribute__((always_inline)) 169 unsigned long _uatomic_add_return(void *addr, unsigned long val, 170 int len) 171 { 172 switch (len) { 173 case 4: 174 { 175 unsigned int result; 176 177 __asm__ __volatile__( 178 LWSYNC_OPCODE 179 "1:\t" "lwarx %0,0,%1\n" /* load and reserve */ 180 "add %0,%2,%0\n" /* add val to value loaded */ 181 "stwcx. %0,0,%1\n" /* store conditional */ 182 "bne- 1b\n" /* retry if lost reservation */ 183 "sync\n" 184 : "=&r"(result) 185 : "r"(addr), "r"(val) 186 : "memory", "cc"); 187 188 return result; 189 } 190 #if (CAA_BITS_PER_LONG == 64) 191 case 8: 192 { 193 unsigned long result; 194 195 __asm__ __volatile__( 196 LWSYNC_OPCODE 197 "1:\t" "ldarx %0,0,%1\n" /* load and reserve */ 198 "add %0,%2,%0\n" /* add val to value loaded */ 199 "stdcx. %0,0,%1\n" /* store conditional */ 200 "bne- 1b\n" /* retry if lost reservation */ 201 "sync\n" 202 : "=&r"(result) 203 : "r"(addr), "r"(val) 204 : "memory", "cc"); 205 206 return result; 207 } 208 #endif 209 } 210 /* 211 * generate an illegal instruction. Cannot catch this with 212 * linker tricks when optimizations are disabled. 213 */ 214 __asm__ __volatile__(ILLEGAL_INSTR); 215 return 0; 216 } 217 218 219 #define uatomic_add_return_mo(addr, v, mo) \ 220 ((__typeof__(*(addr))) _uatomic_add_return((addr), \ 221 caa_cast_long_keep_sign(v), \ 222 sizeof(*(addr)))) 223 224 #ifdef __cplusplus 225 } 226 #endif 227 228 #include <urcu/uatomic/generic.h> 229 230 #endif /* _URCU_ARCH_UATOMIC_PPC_H */ 231
Indexes created Wed Mar 04 05:31:52 UTC 2026