1 /* $NetBSD: tls.c,v 1.25 2025/09/30 06:12:53 skrll Exp $ */ 2 /*- 3 * Copyright (c) 2011 The NetBSD Foundation, Inc. 4 * All rights reserved. 5 * 6 * This code is derived from software contributed to The NetBSD Foundation 7 * by Joerg Sonnenberger. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 19 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 20 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 21 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 22 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 * POSSIBILITY OF SUCH DAMAGE. 29 */ 30 31 #include <sys/cdefs.h> 32 __RCSID("$NetBSD: tls.c,v 1.25 2025/09/30 06:12:53 skrll Exp $"); 33 34 /* 35 * Thread-local storage 36 * 37 * Reference: 38 * 39 * [ELFTLS] Ulrich Drepper, `ELF Handling For Thread-Local 40 * Storage', Version 0.21, 2023-08-22. 41 * https://akkadia.org/drepper/tls.pdf 42 * https://web.archive.org/web/20240718081934/https://akkadia.org/drepper/tls.pdf 43 */ 44 45 #include <sys/param.h> 46 #include <sys/ucontext.h> 47 #include <lwp.h> 48 #include <stdalign.h> 49 #include <stddef.h> 50 #include <string.h> 51 #include "debug.h" 52 #include "rtld.h" 53 54 #include <machine/lwp_private.h> 55 56 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II) 57 58 static struct tls_tcb *_rtld_tls_allocate_locked(void); 59 static void *_rtld_tls_module_allocate(struct tls_tcb *, size_t); 60 61 /* A macro to test correct alignment of a pointer. */ 62 #define ALIGNED_P(ptr, algnmt) (((uintptr_t)(ptr) & ((algnmt) - 1)) == 0) 63 64 /* 65 * DTV offset 66 * 67 * On some architectures (m68k, mips, or1k, powerpc, and riscv), 68 * the DTV offsets passed to __tls_get_addr have a bias relative 69 * to the start of the DTV, in order to maximize the range of TLS 70 * offsets that can be used by instruction encodings with signed 71 * displacements. 72 */ 73 #ifndef TLS_DTV_OFFSET 74 #define TLS_DTV_OFFSET 0 75 #endif 76 77 static size_t _rtld_tls_static_space; /* Static TLS space allocated */ 78 static size_t _rtld_tls_static_offset; /* Next offset for static TLS to use */ 79 size_t _rtld_tls_dtv_generation = 1; /* Bumped on each load of obj w/ TLS */ 80 size_t _rtld_tls_max_index = 1; /* Max index into up-to-date DTV */ 81 82 /* 83 * DTV -- Dynamic Thread Vector 84 * 85 * The DTV is a per-thread array that maps each module with 86 * thread-local storage to a pointer into part of the thread's TCB 87 * (thread control block), or dynamically loaded TLS blocks, 88 * reserved for that module's storage. 89 * 90 * The TCB itself, struct tls_tcb, has a pointer to the DTV at 91 * tcb->tcb_dtv. 92 * 93 * The layout is: 94 * 95 * +---------------+ 96 * | max index | -1 max index i for which dtv[i] is alloced 97 * +---------------+ 98 * | generation | 0 void **dtv points here 99 * +---------------+ 100 * | obj 1 tls ptr | 1 TLS pointer for obj w/ obj->tlsindex 1 101 * +---------------+ 102 * | obj 2 tls ptr | 2 TLS pointer for obj w/ obj->tlsindex 2 103 * +---------------+ 104 * . 105 * . 106 * . 107 * 108 * The values of obj->tlsindex start at 1; this way, 109 * dtv[obj->tlsindex] works, when dtv[0] is the generation. The 110 * TLS pointers go either into the static thread-local storage, 111 * for the initial objects (i.e., those loaded at startup), or 112 * into TLS blocks dynamically allocated for objects that 113 * dynamically loaded by dlopen. 114 * 115 * The generation field is a cache of the global generation number 116 * _rtld_tls_dtv_generation, which is bumped every time an object 117 * with TLS is loaded in _rtld_map_object, and cached by 118 * __tls_get_addr (via _rtld_tls_get_addr) when a newly loaded 119 * module lies outside the bounds of the current DTV. 120 * 121 * XXX Why do we keep max index and generation separately? They 122 * appear to be initialized the same, always incremented together, 123 * and always stored together. 124 * 125 * XXX Why is this not a struct? 126 * 127 * struct dtv { 128 * size_t dtv_gen; 129 * void *dtv_module[]; 130 * }; 131 */ 132 #define DTV_GENERATION(dtv) ((size_t)((dtv)[0])) 133 #define DTV_MAX_INDEX(dtv) ((size_t)((dtv)[-1])) 134 #define SET_DTV_GENERATION(dtv, val) (dtv)[0] = (void *)(size_t)(val) 135 #define SET_DTV_MAX_INDEX(dtv, val) (dtv)[-1] = (void *)(size_t)(val) 136 137 /* 138 * _rtld_tls_get_addr(tcb, idx, offset) 139 * 140 * Slow path for __tls_get_addr (see below), called to allocate 141 * TLS space if needed for the object obj with obj->tlsindex idx, 142 * at offset, which must be below obj->tlssize. 143 * 144 * This may allocate a DTV if the current one is too old, and it 145 * may allocate a dynamically loaded TLS block if there isn't one 146 * already allocated for it. 147 * 148 * XXX Why is the first argument passed as `void *tls' instead of 149 * just `struct tls_tcb *tcb'? 150 */ 151 void * 152 _rtld_tls_get_addr(void *tls, size_t idx, size_t offset) 153 { 154 struct tls_tcb *tcb = tls; 155 void **dtv, **new_dtv; 156 sigset_t mask; 157 158 _rtld_exclusive_enter(&mask); 159 160 dtv = tcb->tcb_dtv; 161 162 /* 163 * If the generation number has changed, we have to allocate a 164 * new DTV. 165 * 166 * XXX Do we really? Isn't it enough to check whether idx <= 167 * DTV_MAX_INDEX(dtv)? 168 */ 169 if (__predict_false(DTV_GENERATION(dtv) != _rtld_tls_dtv_generation)) { 170 size_t to_copy = DTV_MAX_INDEX(dtv); 171 172 /* 173 * "2 +" because the first element is the generation and 174 * the second one is the maximum index. 175 */ 176 new_dtv = xcalloc((2 + _rtld_tls_max_index) * sizeof(*dtv)); 177 ++new_dtv; /* advance past DTV_MAX_INDEX */ 178 if (to_copy > _rtld_tls_max_index) /* XXX How? */ 179 to_copy = _rtld_tls_max_index; 180 memcpy(new_dtv + 1, dtv + 1, to_copy * sizeof(*dtv)); 181 xfree(dtv - 1); /* retreat back to DTV_MAX_INDEX */ 182 dtv = tcb->tcb_dtv = new_dtv; 183 SET_DTV_MAX_INDEX(dtv, _rtld_tls_max_index); 184 SET_DTV_GENERATION(dtv, _rtld_tls_dtv_generation); 185 } 186 187 if (__predict_false(dtv[idx] == NULL)) 188 dtv[idx] = _rtld_tls_module_allocate(tcb, idx); 189 190 _rtld_exclusive_exit(&mask); 191 192 return (uint8_t *)dtv[idx] + offset; 193 } 194 195 /* 196 * _rtld_tls_initial_allocation() 197 * 198 * Allocate the TCB (thread control block) for the initial thread, 199 * once the static TLS space usage has been determined (plus some 200 * slop to allow certain special cases like Mesa to be dlopened). 201 * 202 * This must be done _after_ all initial objects (i.e., those 203 * loaded at startup, as opposed to objects dynamically loaded by 204 * dlopen) have had TLS offsets allocated if need be by 205 * _rtld_tls_offset_allocate, and have had relocations processed. 206 */ 207 void 208 _rtld_tls_initial_allocation(void) 209 { 210 struct tls_tcb *tcb; 211 212 _rtld_tls_static_space = _rtld_tls_static_offset + 213 RTLD_STATIC_TLS_RESERVATION; 214 215 #ifndef __HAVE_TLS_VARIANT_I 216 _rtld_tls_static_space = roundup2(_rtld_tls_static_space, 217 alignof(max_align_t)); 218 #endif 219 dbg(("_rtld_tls_static_space %zu", _rtld_tls_static_space)); 220 221 tcb = _rtld_tls_allocate_locked(); 222 #ifdef __HAVE___LWP_SETTCB 223 __lwp_settcb(tcb); 224 #else 225 _lwp_setprivate(tcb); 226 #endif 227 } 228 229 /* 230 * _rtld_tls_allocate_locked() 231 * 232 * Internal subroutine to allocate a TCB (thread control block) 233 * for the current thread. 234 * 235 * This allocates a DTV and a TCB that points to it, including 236 * static space in the TCB for the TLS of the initial objects. 237 * TLS blocks for dynamically loaded objects are allocated lazily. 238 * 239 * Caller must either be single-threaded (at startup via 240 * _rtld_tls_initial_allocation) or hold the rtld exclusive lock 241 * (via _rtld_tls_allocate). 242 */ 243 static struct tls_tcb * 244 _rtld_tls_allocate_locked(void) 245 { 246 Obj_Entry *obj; 247 struct tls_tcb *tcb; 248 uint8_t *p, *q; 249 250 p = xcalloc(_rtld_tls_static_space + sizeof(struct tls_tcb)); 251 #ifdef __HAVE_TLS_VARIANT_I 252 tcb = (struct tls_tcb *)p; 253 p += sizeof(struct tls_tcb); 254 #else 255 p += _rtld_tls_static_space; 256 tcb = (struct tls_tcb *)p; 257 tcb->tcb_self = tcb; 258 #endif 259 dbg(("lwp %d tls tcb %p", _lwp_self(), tcb)); 260 /* 261 * "2 +" because the first element is the generation and the second 262 * one is the maximum index. 263 */ 264 tcb->tcb_dtv = xcalloc(sizeof(*tcb->tcb_dtv) * (2 + _rtld_tls_max_index)); 265 ++tcb->tcb_dtv; /* advance past DTV_MAX_INDEX */ 266 SET_DTV_MAX_INDEX(tcb->tcb_dtv, _rtld_tls_max_index); 267 SET_DTV_GENERATION(tcb->tcb_dtv, _rtld_tls_dtv_generation); 268 269 for (obj = _rtld_objlist; obj != NULL; obj = obj->next) { 270 if (obj->tls_static) { 271 #ifdef __HAVE_TLS_VARIANT_I 272 q = p + obj->tlsoffset; 273 #else 274 q = p - obj->tlsoffset; 275 #endif 276 dbg(("%s: [lwp %d] tls dtv %p-%p index %zu " 277 "offset %zx alignment %zx tlsinit %p%s", 278 obj->path, _lwp_self(), 279 q, q + obj->tlsinitsize, obj->tlsindex, 280 obj->tlsoffset, obj->tlsalign, obj->tlsinit, 281 ALIGNED_P(q, obj->tlsalign) ? "" : 282 " BAD ALIGNMENT")); 283 if (obj->tlsinitsize) 284 memcpy(q, obj->tlsinit, obj->tlsinitsize); 285 tcb->tcb_dtv[obj->tlsindex] = q; 286 } 287 } 288 289 return tcb; 290 } 291 292 /* 293 * _rtld_tls_allocate() 294 * 295 * Allocate a TCB (thread control block) for the current thread. 296 * 297 * Called by pthread_create for non-initial threads. (The initial 298 * thread's TCB is allocated by _rtld_tls_initial_allocation.) 299 */ 300 struct tls_tcb * 301 _rtld_tls_allocate(void) 302 { 303 struct tls_tcb *tcb; 304 sigset_t mask; 305 306 _rtld_exclusive_enter(&mask); 307 tcb = _rtld_tls_allocate_locked(); 308 _rtld_exclusive_exit(&mask); 309 310 return tcb; 311 } 312 313 /* 314 * _rtld_tls_free(tcb) 315 * 316 * Free a TCB allocated with _rtld_tls_allocate. 317 * 318 * Frees any TLS blocks for dynamically loaded objects that tcb's 319 * DTV points to, and frees tcb's DTV, and frees tcb. 320 */ 321 void 322 _rtld_tls_free(struct tls_tcb *tcb) 323 { 324 size_t i, max_index; 325 uint8_t *p, *p_end; 326 sigset_t mask; 327 328 _rtld_exclusive_enter(&mask); 329 330 #ifdef __HAVE_TLS_VARIANT_I 331 p = (uint8_t *)tcb; 332 #else 333 p = (uint8_t *)tcb - _rtld_tls_static_space; 334 #endif 335 p_end = p + _rtld_tls_static_space; 336 337 max_index = DTV_MAX_INDEX(tcb->tcb_dtv); 338 for (i = 1; i <= max_index; ++i) { 339 if ((uint8_t *)tcb->tcb_dtv[i] < p || 340 (uint8_t *)tcb->tcb_dtv[i] >= p_end) 341 xfree(tcb->tcb_dtv[i]); 342 } 343 xfree(tcb->tcb_dtv - 1); /* retreat back to DTV_MAX_INDEX */ 344 xfree(p); 345 346 _rtld_exclusive_exit(&mask); 347 } 348 349 /* 350 * _rtld_tls_module_allocate(tcb, idx) 351 * 352 * Allocate thread-local storage in the thread with the given TCB 353 * (thread control block) for the object obj whose obj->tlsindex 354 * is idx. 355 * 356 * If obj has had space in static TLS reserved (obj->tls_static), 357 * return a pointer into that. Otherwise, allocate a TLS block, 358 * mark obj as having a TLS block allocated (obj->tls_dynamic), 359 * and return it. 360 * 361 * Called by _rtld_tls_get_addr to get the thread-local storage 362 * for an object the first time around. 363 */ 364 static void * 365 _rtld_tls_module_allocate(struct tls_tcb *tcb, size_t idx) 366 { 367 Obj_Entry *obj; 368 uint8_t *p; 369 370 for (obj = _rtld_objlist; obj != NULL; obj = obj->next) { 371 if (obj->tlsindex == idx) 372 break; 373 } 374 if (obj == NULL) { 375 _rtld_error("Module for TLS index %zu missing", idx); 376 _rtld_die(); 377 } 378 if (obj->tls_static) { 379 #ifdef __HAVE_TLS_VARIANT_I 380 p = (uint8_t *)tcb + obj->tlsoffset + sizeof(struct tls_tcb); 381 #else 382 p = (uint8_t *)tcb - obj->tlsoffset; 383 #endif 384 return p; 385 } 386 387 p = xmalloc(obj->tlssize); 388 memcpy(p, obj->tlsinit, obj->tlsinitsize); 389 memset(p + obj->tlsinitsize, 0, obj->tlssize - obj->tlsinitsize); 390 391 obj->tls_dynamic = 1; 392 393 return p; 394 } 395 396 /* 397 * _rtld_tls_offset_allocate(obj) 398 * 399 * Allocate a static thread-local storage offset for obj. 400 * 401 * Called by _rtld at startup for all initial objects. Called 402 * also by MD relocation logic, which is allowed (for Mesa) to 403 * allocate an additional 64 bytes (RTLD_STATIC_TLS_RESERVATION) 404 * of static thread-local storage in dlopened objects. 405 */ 406 int 407 _rtld_tls_offset_allocate(Obj_Entry *obj) 408 { 409 size_t offset, next_offset; 410 411 if (obj->tls_dynamic) 412 return -1; 413 414 if (obj->tls_static) 415 return 0; 416 417 if (obj->tlssize == 0) { 418 obj->tlsoffset = 0; 419 obj->tls_static = 1; 420 return 0; 421 } 422 423 #ifdef __HAVE_TLS_VARIANT_I 424 offset = roundup2(_rtld_tls_static_offset, obj->tlsalign); 425 next_offset = offset + obj->tlssize; 426 #else 427 offset = roundup2(_rtld_tls_static_offset + obj->tlssize, 428 obj->tlsalign); 429 next_offset = offset; 430 #endif 431 432 /* 433 * Check if the static allocation was already done. 434 * This happens if dynamically loaded modules want to use 435 * static TLS space. 436 * 437 * XXX Keep an actual free list and callbacks for initialisation. 438 */ 439 if (_rtld_tls_static_space) { 440 if (obj->tlsinitsize) { 441 _rtld_error("%s: Use of initialized " 442 "Thread Local Storage with model initial-exec " 443 "and dlopen is not supported", 444 obj->path); 445 return -1; 446 } 447 if (next_offset > _rtld_tls_static_space) { 448 _rtld_error("%s: No space available " 449 "for static Thread Local Storage", 450 obj->path); 451 return -1; 452 } 453 } 454 obj->tlsoffset = offset; 455 dbg(("%s: static tls offset 0x%zx size %zu align %zu (%zx/%zx)\n", 456 obj->path, obj->tlsoffset, obj->tlssize, obj->tlsalign, 457 _rtld_tls_static_offset, next_offset)); 458 _rtld_tls_static_offset = next_offset; 459 obj->tls_static = 1; 460 461 return 0; 462 } 463 464 /* 465 * _rtld_tls_offset_free(obj) 466 * 467 * Free a static thread-local storage offset for obj. 468 * 469 * Called by dlclose (via _rtld_unload_object -> _rtld_obj_free). 470 * 471 * Since static thread-local storage is normally not used by 472 * dlopened objects (with the exception of Mesa), this doesn't do 473 * anything to recycle the space right now. 474 */ 475 void 476 _rtld_tls_offset_free(Obj_Entry *obj) 477 { 478 479 /* 480 * XXX See above. 481 */ 482 obj->tls_static = 0; 483 return; 484 } 485 486 #if defined(__HAVE_COMMON___TLS_GET_ADDR) && defined(RTLD_LOADER) 487 /* 488 * __tls_get_addr(tlsindex) 489 * 490 * Symbol directly called by code generated by the compiler for 491 * references thread-local storage in the general-dynamic or 492 * local-dynamic TLS models (but not initial-exec or local-exec). 493 * 494 * The argument is a pointer to 495 * 496 * struct { 497 * unsigned long int ti_module; 498 * unsigned long int ti_offset; 499 * }; 500 * 501 * as in, e.g., [ELFTLS] Sec. 3.4.3. This coincides with the 502 * type size_t[2] on all architectures that use this common 503 * __tls_get_addr definition (XXX but why do we write it as 504 * size_t[2]?). 505 * 506 * ti_module, i.e., arg[0], is the obj->tlsindex assigned at 507 * load-time by _rtld_map_object, and ti_offset, i.e., arg[1], is 508 * assigned at link-time by ld(1), possibly adjusted by 509 * TLS_DTV_OFFSET. 510 * 511 * Some architectures -- specifically IA-64 -- use a different 512 * calling convention. Some architectures -- specifically i386 513 * -- also use another entry point ___tls_get_addr (that's three 514 * leading underscores) with a different calling convention. 515 */ 516 void * 517 __tls_get_addr(void *arg_) 518 { 519 size_t *arg = (size_t *)arg_; 520 void **dtv; 521 #ifdef __HAVE___LWP_GETTCB_FAST 522 struct tls_tcb * const tcb = __lwp_gettcb_fast(); 523 #else 524 struct tls_tcb * const tcb = __lwp_getprivate_fast(); 525 #endif 526 size_t idx = arg[0], offset = arg[1] + TLS_DTV_OFFSET; 527 528 dtv = tcb->tcb_dtv; 529 530 /* 531 * Fast path: access to an already allocated DTV entry. This 532 * checks the current limit and the entry without needing any 533 * locking. Entries are only freed on dlclose() and it is an 534 * application bug if code of the module is still running at 535 * that point. 536 */ 537 if (__predict_true(idx <= DTV_MAX_INDEX(dtv) && dtv[idx] != NULL)) 538 return (uint8_t *)dtv[idx] + offset; 539 540 return _rtld_tls_get_addr(tcb, idx, offset); 541 } 542 #endif 543 544 #endif /* __HAVE_TLS_VARIANT_I || __HAVE_TLS_VARIANT_II */ 545
Indexes created Tue Sep 30 10:09:55 GMT 2025