1 /* Linux-specific atomic operations for PA Linux. 2 Copyright (C) 2008-2024 Free Software Foundation, Inc. 3 Based on code contributed by CodeSourcery for ARM EABI Linux. 4 Modifications for PA Linux by Helge Deller <deller (at) gmx.de> 5 6 This file is part of GCC. 7 8 GCC is free software; you can redistribute it and/or modify it under 9 the terms of the GNU General Public License as published by the Free 10 Software Foundation; either version 3, or (at your option) any later 11 version. 12 13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY 14 WARRANTY; without even the implied warranty of MERCHANTABILITY or 15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 16 for more details. 17 18 Under Section 7 of GPL version 3, you are granted additional 19 permissions described in the GCC Runtime Library Exception, version 20 3.1, as published by the Free Software Foundation. 21 22 You should have received a copy of the GNU General Public License and 23 a copy of the GCC Runtime Library Exception along with this program; 24 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see 25 <http://www.gnu.org/licenses/>. */ 26 27 #define EFAULT 14 28 #define EBUSY 16 29 #define ENOSYS 251 30 31 #define _ASM_EFAULT "-14" 32 33 typedef unsigned char u8; 34 typedef short unsigned int u16; 35 typedef unsigned int u32; 36 #ifdef __LP64__ 37 typedef long unsigned int u64; 38 #else 39 typedef long long unsigned int u64; 40 #endif 41 42 /* PA-RISC 2.0 supports out-of-order execution for loads and stores. 43 Thus, we need to synchonize memory accesses. For more info, see: 44 "Advanced Performance Features of the 64-bit PA-8000" by Doug Hunt. 45 46 We implement byte, short and int versions of each atomic operation 47 using the kernel helper defined below. There is no support for 48 64-bit operations yet. */ 49 50 /* Determine kernel LWS function call (0=32-bit, 1=64-bit userspace). */ 51 #define LWS_CAS (sizeof(long) == 4 ? 0 : 1) 52 53 /* Kernel helper for compare-and-exchange a 32-bit value. */ 54 static inline long 55 __kernel_cmpxchg (volatile void *mem, int oldval, int newval) 56 { 57 register unsigned long lws_mem asm("r26") = (unsigned long) (mem); 58 register int lws_old asm("r25") = oldval; 59 register int lws_new asm("r24") = newval; 60 register long lws_ret asm("r28"); 61 register long lws_errno asm("r21"); 62 asm volatile ( "ble 0xb0(%%sr2, %%r0) \n\t" 63 "ldi %2, %%r20 \n\t" 64 "cmpiclr,<> " _ASM_EFAULT ", %%r21, %%r0\n\t" 65 "iitlbp %%r0,(%%sr0, %%r0) \n\t" 66 : "=r" (lws_ret), "=r" (lws_errno) 67 : "i" (LWS_CAS), "r" (lws_mem), "r" (lws_old), "r" (lws_new) 68 : "r1", "r20", "r22", "r23", "r29", "r31", "memory" 69 ); 70 71 /* If the kernel LWS call succeeded (lws_errno == 0), lws_ret contains 72 the old value from memory. If this value is equal to OLDVAL, the 73 new value was written to memory. If not, return -EBUSY. */ 74 if (!lws_errno && lws_ret != oldval) 75 return -EBUSY; 76 77 return lws_errno; 78 } 79 80 static inline long 81 __kernel_cmpxchg2 (volatile void *mem, const void *oldval, const void *newval, 82 int val_size) 83 { 84 register unsigned long lws_mem asm("r26") = (unsigned long) (mem); 85 register unsigned long lws_old asm("r25") = (unsigned long) oldval; 86 register unsigned long lws_new asm("r24") = (unsigned long) newval; 87 register int lws_size asm("r23") = val_size; 88 register long lws_ret asm("r28"); 89 register long lws_errno asm("r21"); 90 asm volatile ( "ble 0xb0(%%sr2, %%r0) \n\t" 91 "ldi %6, %%r20 \n\t" 92 "cmpiclr,<> " _ASM_EFAULT ", %%r21, %%r0\n\t" 93 "iitlbp %%r0,(%%sr0, %%r0) \n\t" 94 : "=r" (lws_ret), "=r" (lws_errno), "+r" (lws_mem), 95 "+r" (lws_old), "+r" (lws_new), "+r" (lws_size) 96 : "i" (2) 97 : "r1", "r20", "r22", "r29", "r31", "fr4", "memory" 98 ); 99 100 /* If the kernel LWS call is successful, lws_ret contains 0. */ 101 if (__builtin_expect (lws_ret == 0, 1)) 102 return 0; 103 104 /* If the kernel LWS call fails with no error, return -EBUSY */ 105 if (__builtin_expect (!lws_errno, 0)) 106 return -EBUSY; 107 108 return lws_errno; 109 } 110 #define HIDDEN __attribute__ ((visibility ("hidden"))) 111 112 /* Big endian masks */ 113 #define INVERT_MASK_1 24 114 #define INVERT_MASK_2 16 115 116 #define MASK_1 0xffu 117 #define MASK_2 0xffffu 118 119 /* Load value with an atomic processor load if possible. */ 120 #define ATOMIC_LOAD(TYPE, WIDTH) \ 121 static inline TYPE \ 122 atomic_load_##WIDTH (volatile void *ptr) \ 123 { \ 124 return *(volatile TYPE *)ptr; \ 125 } 126 127 #if defined(__LP64__) || defined(__SOFTFP__) 128 ATOMIC_LOAD (u64, 8) 129 #else 130 static inline u64 131 atomic_load_8 (volatile void *ptr) 132 { 133 u64 result; 134 double tmp; 135 136 asm volatile ("{fldds|fldd} 0(%2),%1\n\t" 137 "{fstds|fstd} %1,-16(%%sp)\n\t" 138 "{ldws|ldw} -16(%%sp),%0\n\t" 139 "{ldws|ldw} -12(%%sp),%R0" 140 : "=r" (result), "=f" (tmp) : "r" (ptr): "memory"); 141 return result; 142 } 143 #endif 144 145 ATOMIC_LOAD (u32, 4) 146 ATOMIC_LOAD (u16, 2) 147 ATOMIC_LOAD (u8, 1) 148 149 #define FETCH_AND_OP_2(OP, PFX_OP, INF_OP, TYPE, WIDTH, INDEX) \ 150 TYPE HIDDEN \ 151 __sync_fetch_and_##OP##_##WIDTH (volatile void *ptr, TYPE val) \ 152 { \ 153 TYPE tmp, newval; \ 154 long failure; \ 155 \ 156 do { \ 157 tmp = atomic_load_##WIDTH ((volatile TYPE *)ptr); \ 158 newval = PFX_OP (tmp INF_OP val); \ 159 failure = __kernel_cmpxchg2 (ptr, &tmp, &newval, INDEX); \ 160 } while (failure != 0); \ 161 \ 162 return tmp; \ 163 } 164 165 FETCH_AND_OP_2 (add, , +, u64, 8, 3) 166 FETCH_AND_OP_2 (sub, , -, u64, 8, 3) 167 FETCH_AND_OP_2 (or, , |, u64, 8, 3) 168 FETCH_AND_OP_2 (and, , &, u64, 8, 3) 169 FETCH_AND_OP_2 (xor, , ^, u64, 8, 3) 170 FETCH_AND_OP_2 (nand, ~, &, u64, 8, 3) 171 172 FETCH_AND_OP_2 (add, , +, u16, 2, 1) 173 FETCH_AND_OP_2 (sub, , -, u16, 2, 1) 174 FETCH_AND_OP_2 (or, , |, u16, 2, 1) 175 FETCH_AND_OP_2 (and, , &, u16, 2, 1) 176 FETCH_AND_OP_2 (xor, , ^, u16, 2, 1) 177 FETCH_AND_OP_2 (nand, ~, &, u16, 2, 1) 178 179 FETCH_AND_OP_2 (add, , +, u8, 1, 0) 180 FETCH_AND_OP_2 (sub, , -, u8, 1, 0) 181 FETCH_AND_OP_2 (or, , |, u8, 1, 0) 182 FETCH_AND_OP_2 (and, , &, u8, 1, 0) 183 FETCH_AND_OP_2 (xor, , ^, u8, 1, 0) 184 FETCH_AND_OP_2 (nand, ~, &, u8, 1, 0) 185 186 #define OP_AND_FETCH_2(OP, PFX_OP, INF_OP, TYPE, WIDTH, INDEX) \ 187 TYPE HIDDEN \ 188 __sync_##OP##_and_fetch_##WIDTH (volatile void *ptr, TYPE val) \ 189 { \ 190 TYPE tmp, newval; \ 191 long failure; \ 192 \ 193 do { \ 194 tmp = atomic_load_##WIDTH ((volatile TYPE *)ptr); \ 195 newval = PFX_OP (tmp INF_OP val); \ 196 failure = __kernel_cmpxchg2 (ptr, &tmp, &newval, INDEX); \ 197 } while (failure != 0); \ 198 \ 199 return PFX_OP (tmp INF_OP val); \ 200 } 201 202 OP_AND_FETCH_2 (add, , +, u64, 8, 3) 203 OP_AND_FETCH_2 (sub, , -, u64, 8, 3) 204 OP_AND_FETCH_2 (or, , |, u64, 8, 3) 205 OP_AND_FETCH_2 (and, , &, u64, 8, 3) 206 OP_AND_FETCH_2 (xor, , ^, u64, 8, 3) 207 OP_AND_FETCH_2 (nand, ~, &, u64, 8, 3) 208 209 OP_AND_FETCH_2 (add, , +, u16, 2, 1) 210 OP_AND_FETCH_2 (sub, , -, u16, 2, 1) 211 OP_AND_FETCH_2 (or, , |, u16, 2, 1) 212 OP_AND_FETCH_2 (and, , &, u16, 2, 1) 213 OP_AND_FETCH_2 (xor, , ^, u16, 2, 1) 214 OP_AND_FETCH_2 (nand, ~, &, u16, 2, 1) 215 216 OP_AND_FETCH_2 (add, , +, u8, 1, 0) 217 OP_AND_FETCH_2 (sub, , -, u8, 1, 0) 218 OP_AND_FETCH_2 (or, , |, u8, 1, 0) 219 OP_AND_FETCH_2 (and, , &, u8, 1, 0) 220 OP_AND_FETCH_2 (xor, , ^, u8, 1, 0) 221 OP_AND_FETCH_2 (nand, ~, &, u8, 1, 0) 222 223 #define FETCH_AND_OP_WORD(OP, PFX_OP, INF_OP) \ 224 unsigned int HIDDEN \ 225 __sync_fetch_and_##OP##_4 (volatile void *ptr, unsigned int val) \ 226 { \ 227 unsigned int tmp; \ 228 long failure; \ 229 \ 230 do { \ 231 tmp = atomic_load_4 ((volatile unsigned int *)ptr); \ 232 failure = __kernel_cmpxchg (ptr, tmp, PFX_OP (tmp INF_OP val)); \ 233 } while (failure != 0); \ 234 \ 235 return tmp; \ 236 } 237 238 FETCH_AND_OP_WORD (add, , +) 239 FETCH_AND_OP_WORD (sub, , -) 240 FETCH_AND_OP_WORD (or, , |) 241 FETCH_AND_OP_WORD (and, , &) 242 FETCH_AND_OP_WORD (xor, , ^) 243 FETCH_AND_OP_WORD (nand, ~, &) 244 245 #define OP_AND_FETCH_WORD(OP, PFX_OP, INF_OP) \ 246 unsigned int HIDDEN \ 247 __sync_##OP##_and_fetch_4 (volatile void *ptr, unsigned int val) \ 248 { \ 249 unsigned int tmp; \ 250 long failure; \ 251 \ 252 do { \ 253 tmp = atomic_load_4 ((volatile unsigned int *)ptr); \ 254 failure = __kernel_cmpxchg (ptr, tmp, PFX_OP (tmp INF_OP val)); \ 255 } while (failure != 0); \ 256 \ 257 return PFX_OP (tmp INF_OP val); \ 258 } 259 260 OP_AND_FETCH_WORD (add, , +) 261 OP_AND_FETCH_WORD (sub, , -) 262 OP_AND_FETCH_WORD (or, , |) 263 OP_AND_FETCH_WORD (and, , &) 264 OP_AND_FETCH_WORD (xor, , ^) 265 OP_AND_FETCH_WORD (nand, ~, &) 266 267 typedef unsigned char bool; 268 269 #define COMPARE_AND_SWAP_2(TYPE, WIDTH, INDEX) \ 270 TYPE HIDDEN \ 271 __sync_val_compare_and_swap_##WIDTH (volatile void *ptr, TYPE oldval, \ 272 TYPE newval) \ 273 { \ 274 TYPE actual_oldval; \ 275 long fail; \ 276 \ 277 while (1) \ 278 { \ 279 actual_oldval = atomic_load_##WIDTH ((volatile TYPE *)ptr); \ 280 \ 281 if (__builtin_expect (oldval != actual_oldval, 0)) \ 282 return actual_oldval; \ 283 \ 284 fail = __kernel_cmpxchg2 (ptr, &actual_oldval, &newval, INDEX); \ 285 \ 286 if (__builtin_expect (!fail, 1)) \ 287 return actual_oldval; \ 288 } \ 289 } \ 290 \ 291 _Bool HIDDEN \ 292 __sync_bool_compare_and_swap_##WIDTH (volatile void *ptr, \ 293 TYPE oldval, TYPE newval) \ 294 { \ 295 long failure = __kernel_cmpxchg2 (ptr, &oldval, &newval, INDEX); \ 296 return (failure == 0); \ 297 } 298 299 COMPARE_AND_SWAP_2 (u64, 8, 3) 300 COMPARE_AND_SWAP_2 (u16, 2, 1) 301 COMPARE_AND_SWAP_2 (u8, 1, 0) 302 303 unsigned int HIDDEN 304 __sync_val_compare_and_swap_4 (volatile void *ptr, unsigned int oldval, 305 unsigned int newval) 306 { 307 long fail; 308 unsigned int actual_oldval; 309 310 while (1) 311 { 312 actual_oldval = atomic_load_4 ((volatile unsigned int *)ptr); 313 314 if (__builtin_expect (oldval != actual_oldval, 0)) 315 return actual_oldval; 316 317 fail = __kernel_cmpxchg (ptr, actual_oldval, newval); 318 319 if (__builtin_expect (!fail, 1)) 320 return actual_oldval; 321 } 322 } 323 324 _Bool HIDDEN 325 __sync_bool_compare_and_swap_4 (volatile void *ptr, unsigned int oldval, 326 unsigned int newval) 327 { 328 long failure = __kernel_cmpxchg (ptr, oldval, newval); 329 return (failure == 0); 330 } 331 332 #define SYNC_LOCK_TEST_AND_SET_2(TYPE, WIDTH, INDEX) \ 333 TYPE HIDDEN \ 334 __sync_lock_test_and_set_##WIDTH (volatile void *ptr, TYPE val) \ 335 { \ 336 TYPE oldval; \ 337 long failure; \ 338 \ 339 do { \ 340 oldval = atomic_load_##WIDTH ((volatile TYPE *)ptr); \ 341 failure = __kernel_cmpxchg2 (ptr, &oldval, &val, INDEX); \ 342 } while (failure != 0); \ 343 \ 344 return oldval; \ 345 } 346 347 SYNC_LOCK_TEST_AND_SET_2 (u64, 8, 3) 348 SYNC_LOCK_TEST_AND_SET_2 (u16, 2, 1) 349 SYNC_LOCK_TEST_AND_SET_2 (u8, 1, 0) 350 351 u32 HIDDEN 352 __sync_lock_test_and_set_4 (volatile void *ptr, unsigned int val) 353 { 354 long failure; 355 unsigned int oldval; 356 357 do { 358 oldval = atomic_load_4 ((volatile unsigned int *)ptr); 359 failure = __kernel_cmpxchg (ptr, oldval, val); 360 } while (failure != 0); 361 362 return oldval; 363 } 364 365 #define SYNC_LOCK_RELEASE_1(TYPE, WIDTH, INDEX) \ 366 void HIDDEN \ 367 __sync_lock_release_##WIDTH (volatile void *ptr) \ 368 { \ 369 TYPE oldval, val = 0; \ 370 long failure; \ 371 \ 372 do { \ 373 oldval = atomic_load_##WIDTH ((volatile TYPE *)ptr); \ 374 failure = __kernel_cmpxchg2 (ptr, &oldval, &val, INDEX); \ 375 } while (failure != 0); \ 376 } 377 378 SYNC_LOCK_RELEASE_1 (u64, 8, 3) 379 SYNC_LOCK_RELEASE_1 (u16, 2, 1) 380 SYNC_LOCK_RELEASE_1 (u8, 1, 0) 381 382 void HIDDEN 383 __sync_lock_release_4 (volatile void *ptr) 384 { 385 long failure; 386 unsigned int oldval; 387 388 do { 389 oldval = atomic_load_4 ((volatile unsigned int *)ptr); 390 failure = __kernel_cmpxchg (ptr, oldval, 0); 391 } while (failure != 0); 392 } 393 394 #ifndef __LP64__ 395 #define SYNC_LOCK_LOAD_2(TYPE, WIDTH, INDEX) \ 396 TYPE __sync_lock_load_##WIDTH (volatile void *) HIDDEN; \ 397 TYPE \ 398 __sync_lock_load_##WIDTH (volatile void *ptr) \ 399 { \ 400 TYPE oldval; \ 401 long failure; \ 402 \ 403 do { \ 404 oldval = atomic_load_##WIDTH ((volatile TYPE *)ptr); \ 405 failure = __kernel_cmpxchg2 (ptr, &oldval, &oldval, INDEX); \ 406 } while (failure != 0); \ 407 \ 408 return oldval; \ 409 } 410 411 SYNC_LOCK_LOAD_2 (u64, 8, 3) 412 #endif 413
Indexes created Mon Mar 02 05:31:46 UTC 2026