1 // Copyright (C) 2002-2022 Free Software Foundation, Inc. 2 // 3 // This file is part of GCC. 4 // 5 // GCC is free software; you can redistribute it and/or modify 6 // it under the terms of the GNU General Public License as published by 7 // the Free Software Foundation; either version 3, or (at your option) 8 // any later version. 9 10 // GCC is distributed in the hope that it will be useful, 11 // but WITHOUT ANY WARRANTY; without even the implied warranty of 12 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 // GNU General Public License for more details. 14 15 // Under Section 7 of GPL version 3, you are granted additional 16 // permissions described in the GCC Runtime Library Exception, version 17 // 3.1, as published by the Free Software Foundation. 18 19 // You should have received a copy of the GNU General Public License and 20 // a copy of the GCC Runtime Library Exception along with this program; 21 // see the files COPYING3 and COPYING.RUNTIME respectively. If not, see 22 // <http://www.gnu.org/licenses/>. 23 24 // Written by Mark Mitchell, CodeSourcery LLC, <mark (at) codesourcery.com> 25 // Thread support written by Jason Merrill, Red Hat Inc. <jason (at) redhat.com> 26 27 #include <bits/c++config.h> 28 #include <cxxabi.h> 29 #include <exception> 30 #include <new> 31 #include <ext/atomicity.h> 32 #include <ext/concurrence.h> 33 #include <bits/atomic_lockfree_defines.h> 34 #if defined(__GTHREADS) && defined(__GTHREAD_HAS_COND) \ 35 && (ATOMIC_INT_LOCK_FREE > 1) && defined(_GLIBCXX_HAVE_LINUX_FUTEX) 36 # include <climits> 37 # include <syscall.h> 38 # include <unistd.h> 39 # define _GLIBCXX_USE_FUTEX 40 # define _GLIBCXX_FUTEX_WAIT 0 41 # define _GLIBCXX_FUTEX_WAKE 1 42 #endif 43 44 // The IA64/generic ABI uses the first byte of the guard variable. 45 // The ARM EABI uses the least significant bit. 46 47 // Thread-safe static local initialization support. 48 #ifdef __GTHREADS 49 # ifndef _GLIBCXX_USE_FUTEX 50 namespace 51 { 52 // A single mutex controlling all static initializations. 53 static __gnu_cxx::__recursive_mutex* static_mutex; 54 55 typedef char fake_recursive_mutex[sizeof(__gnu_cxx::__recursive_mutex)] 56 __attribute__ ((aligned(__alignof__(__gnu_cxx::__recursive_mutex)))); 57 fake_recursive_mutex fake_mutex; 58 59 static void init() 60 { static_mutex = new (&fake_mutex) __gnu_cxx::__recursive_mutex(); } 61 62 __gnu_cxx::__recursive_mutex& 63 get_static_mutex() 64 { 65 static __gthread_once_t once = __GTHREAD_ONCE_INIT; 66 __gthread_once(&once, init); 67 return *static_mutex; 68 } 69 70 // Simple wrapper for exception safety. 71 struct mutex_wrapper 72 { 73 bool unlock; 74 mutex_wrapper() : unlock(true) 75 { get_static_mutex().lock(); } 76 77 ~mutex_wrapper() 78 { 79 if (unlock) 80 static_mutex->unlock(); 81 } 82 }; 83 } 84 # endif 85 86 # if defined(__GTHREAD_HAS_COND) && !defined(_GLIBCXX_USE_FUTEX) 87 namespace 88 { 89 // A single condition variable controlling all static initializations. 90 static __gnu_cxx::__cond* static_cond; 91 92 // using a fake type to avoid initializing a static class. 93 typedef char fake_cond_t[sizeof(__gnu_cxx::__cond)] 94 __attribute__ ((aligned(__alignof__(__gnu_cxx::__cond)))); 95 fake_cond_t fake_cond; 96 97 static void init_static_cond() 98 { static_cond = new (&fake_cond) __gnu_cxx::__cond(); } 99 100 __gnu_cxx::__cond& 101 get_static_cond() 102 { 103 static __gthread_once_t once = __GTHREAD_ONCE_INIT; 104 __gthread_once(&once, init_static_cond); 105 return *static_cond; 106 } 107 } 108 # endif 109 110 # ifndef _GLIBCXX_GUARD_TEST_AND_ACQUIRE 111 112 // Test the guard variable with a memory load with 113 // acquire semantics. 114 115 inline bool 116 __test_and_acquire (__cxxabiv1::__guard *g) 117 { 118 unsigned char __c; 119 unsigned char *__p = reinterpret_cast<unsigned char *>(g); 120 __atomic_load (__p, &__c, __ATOMIC_ACQUIRE); 121 (void) __p; 122 return _GLIBCXX_GUARD_TEST(&__c); 123 } 124 # define _GLIBCXX_GUARD_TEST_AND_ACQUIRE(G) __test_and_acquire (G) 125 # endif 126 127 # ifndef _GLIBCXX_GUARD_SET_AND_RELEASE 128 129 // Set the guard variable to 1 with memory order release semantics. 130 131 inline void 132 __set_and_release (__cxxabiv1::__guard *g) 133 { 134 unsigned char *__p = reinterpret_cast<unsigned char *>(g); 135 unsigned char val = 1; 136 __atomic_store (__p, &val, __ATOMIC_RELEASE); 137 (void) __p; 138 } 139 # define _GLIBCXX_GUARD_SET_AND_RELEASE(G) __set_and_release (G) 140 # endif 141 142 #else /* !__GTHREADS */ 143 144 # undef _GLIBCXX_GUARD_TEST_AND_ACQUIRE 145 # undef _GLIBCXX_GUARD_SET_AND_RELEASE 146 # define _GLIBCXX_GUARD_SET_AND_RELEASE(G) _GLIBCXX_GUARD_SET (G) 147 148 #endif /* __GTHREADS */ 149 150 // 151 // Here are C++ run-time routines for guarded initialization of static 152 // variables. There are 4 scenarios under which these routines are called: 153 // 154 // 1. Threads not supported (__GTHREADS not defined) 155 // 2. Threads are supported but not enabled at run-time. 156 // 3. Threads enabled at run-time but __gthreads_* are not fully POSIX. 157 // 4. Threads enabled at run-time and __gthreads_* support all POSIX threads 158 // primitives we need here. 159 // 160 // The old code supported scenarios 1-3 but was broken since it used a global 161 // mutex for all threads and had the mutex locked during the whole duration of 162 // initialization of a guarded static variable. The following created a 163 // dead-lock with the old code. 164 // 165 // Thread 1 acquires the global mutex. 166 // Thread 1 starts initializing static variable. 167 // Thread 1 creates thread 2 during initialization. 168 // Thread 2 attempts to acquire mutex to initialize another variable. 169 // Thread 2 blocks since thread 1 is locking the mutex. 170 // Thread 1 waits for result from thread 2 and also blocks. A deadlock. 171 // 172 // The new code here can handle this situation and thus is more robust. However, 173 // we need to use the POSIX thread condition variable, which is not supported 174 // in all platforms, notably older versions of Microsoft Windows. The gthr*.h 175 // headers define a symbol __GTHREAD_HAS_COND for platforms that support POSIX 176 // like condition variables. For platforms that do not support condition 177 // variables, we need to fall back to the old code. 178 179 // If _GLIBCXX_USE_FUTEX, no global mutex or condition variable is used, 180 // only atomic operations are used together with futex syscall. 181 // Valid values of the first integer in guard are: 182 // 0 No thread encountered the guarded init 183 // yet or it has been aborted. 184 // _GLIBCXX_GUARD_BIT The guarded static var has been successfully 185 // initialized. 186 // _GLIBCXX_GUARD_PENDING_BIT The guarded static var is being initialized 187 // and no other thread is waiting for its 188 // initialization. 189 // (_GLIBCXX_GUARD_PENDING_BIT The guarded static var is being initialized 190 // | _GLIBCXX_GUARD_WAITING_BIT) and some other threads are waiting until 191 // it is initialized. 192 193 namespace __cxxabiv1 194 { 195 #ifdef _GLIBCXX_USE_FUTEX 196 namespace 197 { 198 static inline int __guard_test_bit (const int __byte, const int __val) 199 { 200 union { int __i; char __c[sizeof (int)]; } __u = { 0 }; 201 __u.__c[__byte] = __val; 202 return __u.__i; 203 } 204 } 205 #endif 206 207 static inline int 208 init_in_progress_flag(__guard* g) 209 { return ((char *)g)[1]; } 210 211 static inline void 212 set_init_in_progress_flag(__guard* g, int v) 213 { ((char *)g)[1] = v; } 214 215 static inline void 216 throw_recursive_init_exception() 217 { 218 #if __cpp_exceptions 219 throw __gnu_cxx::recursive_init_error(); 220 #else 221 // Use __builtin_trap so we don't require abort(). 222 __builtin_trap(); 223 #endif 224 } 225 226 // acquire() is a helper function used to acquire guard if thread support is 227 // not compiled in or is compiled in but not enabled at run-time. 228 static int 229 acquire(__guard *g) 230 { 231 // Quit if the object is already initialized. 232 if (_GLIBCXX_GUARD_TEST(g)) 233 return 0; 234 235 if (init_in_progress_flag(g)) 236 throw_recursive_init_exception(); 237 238 set_init_in_progress_flag(g, 1); 239 return 1; 240 } 241 242 extern "C" 243 int __cxa_guard_acquire (__guard *g) 244 { 245 #ifdef __GTHREADS 246 // If the target can reorder loads, we need to insert a read memory 247 // barrier so that accesses to the guarded variable happen after the 248 // guard test. 249 if (_GLIBCXX_GUARD_TEST_AND_ACQUIRE (g)) 250 return 0; 251 252 # ifdef _GLIBCXX_USE_FUTEX 253 // If __atomic_* and futex syscall are supported, don't use any global 254 // mutex. 255 256 // Use the same bits in the guard variable whether single-threaded or not, 257 // so that __cxa_guard_release and __cxa_guard_abort match the logic here 258 // even if __libc_single_threaded becomes false between now and then. 259 260 if (__gnu_cxx::__is_single_threaded()) 261 { 262 // No need to use atomics, and no need to wait for other threads. 263 int *gi = (int *) (void *) g; 264 if (*gi == 0) 265 { 266 *gi = _GLIBCXX_GUARD_PENDING_BIT; 267 return 1; 268 } 269 else 270 throw_recursive_init_exception(); 271 } 272 else 273 { 274 int *gi = (int *) (void *) g; 275 const int guard_bit = _GLIBCXX_GUARD_BIT; 276 const int pending_bit = _GLIBCXX_GUARD_PENDING_BIT; 277 const int waiting_bit = _GLIBCXX_GUARD_WAITING_BIT; 278 279 while (1) 280 { 281 int expected(0); 282 if (__atomic_compare_exchange_n(gi, &expected, pending_bit, false, 283 __ATOMIC_ACQ_REL, 284 __ATOMIC_ACQUIRE)) 285 { 286 // This thread should do the initialization. 287 return 1; 288 } 289 290 if (expected == guard_bit) 291 { 292 // Already initialized. 293 return 0; 294 } 295 296 if (expected == pending_bit) 297 { 298 // Use acquire here. 299 int newv = expected | waiting_bit; 300 if (!__atomic_compare_exchange_n(gi, &expected, newv, false, 301 __ATOMIC_ACQ_REL, 302 __ATOMIC_ACQUIRE)) 303 { 304 if (expected == guard_bit) 305 { 306 // Make a thread that failed to set the 307 // waiting bit exit the function earlier, 308 // if it detects that another thread has 309 // successfully finished initialising. 310 return 0; 311 } 312 if (expected == 0) 313 continue; 314 } 315 316 expected = newv; 317 } 318 319 syscall (SYS_futex, gi, _GLIBCXX_FUTEX_WAIT, expected, 0); 320 } 321 } 322 # else // ! _GLIBCXX_USE_FUTEX 323 if (__gthread_active_p ()) 324 { 325 mutex_wrapper mw; 326 327 while (1) // When this loop is executing, mutex is locked. 328 { 329 # ifdef __GTHREAD_HAS_COND 330 // The static is already initialized. 331 if (_GLIBCXX_GUARD_TEST(g)) 332 return 0; // The mutex will be unlocked via wrapper 333 334 if (init_in_progress_flag(g)) 335 { 336 // The guarded static is currently being initialized by 337 // another thread, so we release mutex and wait for the 338 // condition variable. We will lock the mutex again after 339 // this. 340 get_static_cond().wait_recursive(&get_static_mutex()); 341 } 342 else 343 { 344 set_init_in_progress_flag(g, 1); 345 return 1; // The mutex will be unlocked via wrapper. 346 } 347 # else 348 // This provides compatibility with older systems not supporting 349 // POSIX like condition variables. 350 if (acquire(g)) 351 { 352 mw.unlock = false; 353 return 1; // The mutex still locked. 354 } 355 return 0; // The mutex will be unlocked via wrapper. 356 # endif 357 } 358 } 359 # endif 360 #endif // ! __GTHREADS 361 362 return acquire (g); 363 } 364 365 extern "C" 366 void __cxa_guard_abort (__guard *g) noexcept 367 { 368 #ifdef _GLIBCXX_USE_FUTEX 369 // If __atomic_* and futex syscall are supported, don't use any global 370 // mutex. 371 372 if (__gnu_cxx::__is_single_threaded()) 373 { 374 // No need to use atomics, and no other threads to wake. 375 int *gi = (int *) (void *) g; 376 *gi = 0; 377 return; 378 } 379 else 380 { 381 int *gi = (int *) (void *) g; 382 const int waiting_bit = _GLIBCXX_GUARD_WAITING_BIT; 383 int old = __atomic_exchange_n (gi, 0, __ATOMIC_ACQ_REL); 384 385 if ((old & waiting_bit) != 0) 386 syscall (SYS_futex, gi, _GLIBCXX_FUTEX_WAKE, INT_MAX); 387 return; 388 } 389 #elif defined(__GTHREAD_HAS_COND) 390 if (__gthread_active_p()) 391 { 392 mutex_wrapper mw; 393 394 set_init_in_progress_flag(g, 0); 395 396 // If we abort, we still need to wake up all other threads waiting for 397 // the condition variable. 398 get_static_cond().broadcast(); 399 return; 400 } 401 #endif 402 403 set_init_in_progress_flag(g, 0); 404 #if defined(__GTHREADS) && !defined(__GTHREAD_HAS_COND) 405 // This provides compatibility with older systems not supporting POSIX like 406 // condition variables. 407 if (__gthread_active_p ()) 408 static_mutex->unlock(); 409 #endif 410 } 411 412 extern "C" 413 void __cxa_guard_release (__guard *g) noexcept 414 { 415 #ifdef _GLIBCXX_USE_FUTEX 416 // If __atomic_* and futex syscall are supported, don't use any global 417 // mutex. 418 419 if (__gnu_cxx::__is_single_threaded()) 420 { 421 int *gi = (int *) (void *) g; 422 *gi = _GLIBCXX_GUARD_BIT; 423 return; 424 } 425 else 426 { 427 int *gi = (int *) (void *) g; 428 const int guard_bit = _GLIBCXX_GUARD_BIT; 429 const int waiting_bit = _GLIBCXX_GUARD_WAITING_BIT; 430 int old = __atomic_exchange_n (gi, guard_bit, __ATOMIC_ACQ_REL); 431 432 if ((old & waiting_bit) != 0) 433 syscall (SYS_futex, gi, _GLIBCXX_FUTEX_WAKE, INT_MAX); 434 return; 435 } 436 437 #elif defined(__GTHREAD_HAS_COND) 438 if (__gthread_active_p()) 439 { 440 mutex_wrapper mw; 441 442 set_init_in_progress_flag(g, 0); 443 _GLIBCXX_GUARD_SET_AND_RELEASE(g); 444 445 get_static_cond().broadcast(); 446 return; 447 } 448 #endif 449 450 set_init_in_progress_flag(g, 0); 451 _GLIBCXX_GUARD_SET_AND_RELEASE (g); 452 453 #if defined(__GTHREADS) && !defined(__GTHREAD_HAS_COND) 454 // This provides compatibility with older systems not supporting POSIX like 455 // condition variables. 456 if (__gthread_active_p()) 457 static_mutex->unlock(); 458 #endif 459 } 460 } 461
Indexes created Tue Feb 24 01:34:59 UTC 2026