1 #define JEMALLOC_PROF_SYS_C_ 2 #include "jemalloc/internal/jemalloc_preamble.h" 3 #include "jemalloc/internal/jemalloc_internal_includes.h" 4 5 #include "jemalloc/internal/buf_writer.h" 6 #include "jemalloc/internal/ctl.h" 7 #include "jemalloc/internal/prof_data.h" 8 #include "jemalloc/internal/prof_sys.h" 9 10 #ifdef JEMALLOC_PROF_LIBUNWIND 11 #define UNW_LOCAL_ONLY 12 #include <libunwind.h> 13 #endif 14 15 #ifdef JEMALLOC_PROF_LIBGCC 16 /* 17 * We have a circular dependency -- jemalloc_internal.h tells us if we should 18 * use libgcc's unwinding functionality, but after we've included that, we've 19 * already hooked _Unwind_Backtrace. We'll temporarily disable hooking. 20 */ 21 #undef _Unwind_Backtrace 22 #include <unwind.h> 23 #define _Unwind_Backtrace JEMALLOC_TEST_HOOK(_Unwind_Backtrace, test_hooks_libc_hook) 24 #endif 25 26 /******************************************************************************/ 27 28 malloc_mutex_t prof_dump_filename_mtx; 29 30 bool prof_do_mock = false; 31 32 static uint64_t prof_dump_seq; 33 static uint64_t prof_dump_iseq; 34 static uint64_t prof_dump_mseq; 35 static uint64_t prof_dump_useq; 36 37 static char *prof_prefix = NULL; 38 39 /* The fallback allocator profiling functionality will use. */ 40 base_t *prof_base; 41 42 void 43 bt_init(prof_bt_t *bt, void **vec) { 44 cassert(config_prof); 45 46 bt->vec = vec; 47 bt->len = 0; 48 } 49 50 #ifdef JEMALLOC_PROF_LIBUNWIND 51 static void 52 prof_backtrace_impl(void **vec, unsigned *len, unsigned max_len) { 53 int nframes; 54 55 cassert(config_prof); 56 assert(*len == 0); 57 assert(vec != NULL); 58 assert(max_len == PROF_BT_MAX); 59 60 nframes = unw_backtrace(vec, PROF_BT_MAX); 61 if (nframes <= 0) { 62 return; 63 } 64 *len = nframes; 65 } 66 #elif (defined(JEMALLOC_PROF_LIBGCC)) 67 static _Unwind_Reason_Code 68 prof_unwind_init_callback(struct _Unwind_Context *context, void *arg) { 69 cassert(config_prof); 70 71 return _URC_NO_REASON; 72 } 73 74 static _Unwind_Reason_Code 75 prof_unwind_callback(struct _Unwind_Context *context, void *arg) { 76 prof_unwind_data_t *data = (prof_unwind_data_t *)arg; 77 void *ip; 78 79 cassert(config_prof); 80 81 ip = (void *)_Unwind_GetIP(context); 82 if (ip == NULL) { 83 return _URC_END_OF_STACK; 84 } 85 data->vec[*data->len] = ip; 86 (*data->len)++; 87 if (*data->len == data->max) { 88 return _URC_END_OF_STACK; 89 } 90 91 return _URC_NO_REASON; 92 } 93 94 static void 95 prof_backtrace_impl(void **vec, unsigned *len, unsigned max_len) { 96 prof_unwind_data_t data = {vec, len, max_len}; 97 98 cassert(config_prof); 99 assert(vec != NULL); 100 assert(max_len == PROF_BT_MAX); 101 102 _Unwind_Backtrace(prof_unwind_callback, &data); 103 } 104 #elif (defined(JEMALLOC_PROF_GCC)) 105 static void 106 prof_backtrace_impl(void **vec, unsigned *len, unsigned max_len) { 107 #define BT_FRAME(i) \ 108 if ((i) < max_len) { \ 109 void *p; \ 110 if (__builtin_frame_address(i) == 0) { \ 111 return; \ 112 } \ 113 p = __builtin_return_address(i); \ 114 if (p == NULL) { \ 115 return; \ 116 } \ 117 vec[(i)] = p; \ 118 *len = (i) + 1; \ 119 } else { \ 120 return; \ 121 } 122 123 cassert(config_prof); 124 assert(vec != NULL); 125 assert(max_len == PROF_BT_MAX); 126 127 BT_FRAME(0) 128 BT_FRAME(1) 129 BT_FRAME(2) 130 BT_FRAME(3) 131 BT_FRAME(4) 132 BT_FRAME(5) 133 BT_FRAME(6) 134 BT_FRAME(7) 135 BT_FRAME(8) 136 BT_FRAME(9) 137 138 BT_FRAME(10) 139 BT_FRAME(11) 140 BT_FRAME(12) 141 BT_FRAME(13) 142 BT_FRAME(14) 143 BT_FRAME(15) 144 BT_FRAME(16) 145 BT_FRAME(17) 146 BT_FRAME(18) 147 BT_FRAME(19) 148 149 BT_FRAME(20) 150 BT_FRAME(21) 151 BT_FRAME(22) 152 BT_FRAME(23) 153 BT_FRAME(24) 154 BT_FRAME(25) 155 BT_FRAME(26) 156 BT_FRAME(27) 157 BT_FRAME(28) 158 BT_FRAME(29) 159 160 BT_FRAME(30) 161 BT_FRAME(31) 162 BT_FRAME(32) 163 BT_FRAME(33) 164 BT_FRAME(34) 165 BT_FRAME(35) 166 BT_FRAME(36) 167 BT_FRAME(37) 168 BT_FRAME(38) 169 BT_FRAME(39) 170 171 BT_FRAME(40) 172 BT_FRAME(41) 173 BT_FRAME(42) 174 BT_FRAME(43) 175 BT_FRAME(44) 176 BT_FRAME(45) 177 BT_FRAME(46) 178 BT_FRAME(47) 179 BT_FRAME(48) 180 BT_FRAME(49) 181 182 BT_FRAME(50) 183 BT_FRAME(51) 184 BT_FRAME(52) 185 BT_FRAME(53) 186 BT_FRAME(54) 187 BT_FRAME(55) 188 BT_FRAME(56) 189 BT_FRAME(57) 190 BT_FRAME(58) 191 BT_FRAME(59) 192 193 BT_FRAME(60) 194 BT_FRAME(61) 195 BT_FRAME(62) 196 BT_FRAME(63) 197 BT_FRAME(64) 198 BT_FRAME(65) 199 BT_FRAME(66) 200 BT_FRAME(67) 201 BT_FRAME(68) 202 BT_FRAME(69) 203 204 BT_FRAME(70) 205 BT_FRAME(71) 206 BT_FRAME(72) 207 BT_FRAME(73) 208 BT_FRAME(74) 209 BT_FRAME(75) 210 BT_FRAME(76) 211 BT_FRAME(77) 212 BT_FRAME(78) 213 BT_FRAME(79) 214 215 BT_FRAME(80) 216 BT_FRAME(81) 217 BT_FRAME(82) 218 BT_FRAME(83) 219 BT_FRAME(84) 220 BT_FRAME(85) 221 BT_FRAME(86) 222 BT_FRAME(87) 223 BT_FRAME(88) 224 BT_FRAME(89) 225 226 BT_FRAME(90) 227 BT_FRAME(91) 228 BT_FRAME(92) 229 BT_FRAME(93) 230 BT_FRAME(94) 231 BT_FRAME(95) 232 BT_FRAME(96) 233 BT_FRAME(97) 234 BT_FRAME(98) 235 BT_FRAME(99) 236 237 BT_FRAME(100) 238 BT_FRAME(101) 239 BT_FRAME(102) 240 BT_FRAME(103) 241 BT_FRAME(104) 242 BT_FRAME(105) 243 BT_FRAME(106) 244 BT_FRAME(107) 245 BT_FRAME(108) 246 BT_FRAME(109) 247 248 BT_FRAME(110) 249 BT_FRAME(111) 250 BT_FRAME(112) 251 BT_FRAME(113) 252 BT_FRAME(114) 253 BT_FRAME(115) 254 BT_FRAME(116) 255 BT_FRAME(117) 256 BT_FRAME(118) 257 BT_FRAME(119) 258 259 BT_FRAME(120) 260 BT_FRAME(121) 261 BT_FRAME(122) 262 BT_FRAME(123) 263 BT_FRAME(124) 264 BT_FRAME(125) 265 BT_FRAME(126) 266 BT_FRAME(127) 267 #undef BT_FRAME 268 } 269 #else 270 static void 271 prof_backtrace_impl(void **vec, unsigned *len, unsigned max_len) { 272 cassert(config_prof); 273 not_reached(); 274 } 275 #endif 276 277 void 278 prof_backtrace(tsd_t *tsd, prof_bt_t *bt) { 279 cassert(config_prof); 280 prof_backtrace_hook_t prof_backtrace_hook = prof_backtrace_hook_get(); 281 assert(prof_backtrace_hook != NULL); 282 283 pre_reentrancy(tsd, NULL); 284 prof_backtrace_hook(bt->vec, &bt->len, PROF_BT_MAX); 285 post_reentrancy(tsd); 286 } 287 288 void 289 prof_hooks_init(void) { 290 prof_backtrace_hook_set(&prof_backtrace_impl); 291 prof_dump_hook_set(NULL); 292 } 293 294 void 295 prof_unwind_init(void) { 296 #ifdef JEMALLOC_PROF_LIBGCC 297 /* 298 * Cause the backtracing machinery to allocate its internal 299 * state before enabling profiling. 300 */ 301 _Unwind_Backtrace(prof_unwind_init_callback, NULL); 302 #endif 303 } 304 305 static int 306 prof_sys_thread_name_read_impl(char *buf, size_t limit) { 307 #if defined(JEMALLOC_HAVE_PTHREAD_GETNAME_NP) 308 return pthread_getname_np(pthread_self(), buf, limit); 309 #elif defined(JEMALLOC_HAVE_PTHREAD_GET_NAME_NP) 310 pthread_get_name_np(pthread_self(), buf, limit); 311 return 0; 312 #else 313 return ENOSYS; 314 #endif 315 } 316 prof_sys_thread_name_read_t *JET_MUTABLE prof_sys_thread_name_read = 317 prof_sys_thread_name_read_impl; 318 319 void 320 prof_sys_thread_name_fetch(tsd_t *tsd) { 321 #define THREAD_NAME_MAX_LEN 16 322 char buf[THREAD_NAME_MAX_LEN]; 323 if (!prof_sys_thread_name_read(buf, THREAD_NAME_MAX_LEN)) { 324 prof_thread_name_set_impl(tsd, buf); 325 } 326 #undef THREAD_NAME_MAX_LEN 327 } 328 329 int 330 prof_getpid(void) { 331 #ifdef _WIN32 332 return GetCurrentProcessId(); 333 #else 334 return getpid(); 335 #endif 336 } 337 338 /* 339 * This buffer is rather large for stack allocation, so use a single buffer for 340 * all profile dumps; protected by prof_dump_mtx. 341 */ 342 static char prof_dump_buf[PROF_DUMP_BUFSIZE]; 343 344 typedef struct prof_dump_arg_s prof_dump_arg_t; 345 struct prof_dump_arg_s { 346 /* 347 * Whether error should be handled locally: if true, then we print out 348 * error message as well as abort (if opt_abort is true) when an error 349 * occurred, and we also report the error back to the caller in the end; 350 * if false, then we only report the error back to the caller in the 351 * end. 352 */ 353 const bool handle_error_locally; 354 /* 355 * Whether there has been an error in the dumping process, which could 356 * have happened either in file opening or in file writing. When an 357 * error has already occurred, we will stop further writing to the file. 358 */ 359 bool error; 360 /* File descriptor of the dump file. */ 361 int prof_dump_fd; 362 }; 363 364 static void 365 prof_dump_check_possible_error(prof_dump_arg_t *arg, bool err_cond, 366 const char *format, ...) { 367 assert(!arg->error); 368 if (!err_cond) { 369 return; 370 } 371 372 arg->error = true; 373 if (!arg->handle_error_locally) { 374 return; 375 } 376 377 va_list ap; 378 char buf[PROF_PRINTF_BUFSIZE]; 379 va_start(ap, format); 380 malloc_vsnprintf(buf, sizeof(buf), format, ap); 381 va_end(ap); 382 malloc_write(buf); 383 384 if (opt_abort) { 385 abort(); 386 } 387 } 388 389 static int 390 prof_dump_open_file_impl(const char *filename, int mode) { 391 return creat(filename, mode); 392 } 393 prof_dump_open_file_t *JET_MUTABLE prof_dump_open_file = 394 prof_dump_open_file_impl; 395 396 static void 397 prof_dump_open(prof_dump_arg_t *arg, const char *filename) { 398 arg->prof_dump_fd = prof_dump_open_file(filename, 0644); 399 prof_dump_check_possible_error(arg, arg->prof_dump_fd == -1, 400 "<jemalloc>: failed to open \"%s\"\n", filename); 401 } 402 403 prof_dump_write_file_t *JET_MUTABLE prof_dump_write_file = malloc_write_fd; 404 405 static void 406 prof_dump_flush(void *opaque, const char *s) { 407 cassert(config_prof); 408 prof_dump_arg_t *arg = (prof_dump_arg_t *)opaque; 409 if (!arg->error) { 410 ssize_t err = prof_dump_write_file(arg->prof_dump_fd, s, 411 strlen(s)); 412 prof_dump_check_possible_error(arg, err == -1, 413 "<jemalloc>: failed to write during heap profile flush\n"); 414 } 415 } 416 417 static void 418 prof_dump_close(prof_dump_arg_t *arg) { 419 if (arg->prof_dump_fd != -1) { 420 close(arg->prof_dump_fd); 421 } 422 } 423 424 #ifndef _WIN32 425 JEMALLOC_FORMAT_PRINTF(1, 2) 426 static int 427 prof_open_maps_internal(const char *format, ...) { 428 int mfd; 429 va_list ap; 430 char filename[PATH_MAX + 1]; 431 432 va_start(ap, format); 433 malloc_vsnprintf(filename, sizeof(filename), format, ap); 434 va_end(ap); 435 436 #if defined(O_CLOEXEC) 437 mfd = open(filename, O_RDONLY | O_CLOEXEC); 438 #else 439 mfd = open(filename, O_RDONLY); 440 if (mfd != -1) { 441 fcntl(mfd, F_SETFD, fcntl(mfd, F_GETFD) | FD_CLOEXEC); 442 } 443 #endif 444 445 return mfd; 446 } 447 #endif 448 449 static int 450 prof_dump_open_maps_impl(void) { 451 int mfd; 452 453 cassert(config_prof); 454 #if defined(__FreeBSD__) || defined(__DragonFly__) 455 mfd = prof_open_maps_internal("/proc/curproc/map"); 456 #elif defined(_WIN32) 457 mfd = -1; // Not implemented 458 #else 459 int pid = prof_getpid(); 460 461 mfd = prof_open_maps_internal("/proc/%d/task/%d/maps", pid, pid); 462 if (mfd == -1) { 463 mfd = prof_open_maps_internal("/proc/%d/maps", pid); 464 } 465 #endif 466 return mfd; 467 } 468 prof_dump_open_maps_t *JET_MUTABLE prof_dump_open_maps = 469 prof_dump_open_maps_impl; 470 471 static ssize_t 472 prof_dump_read_maps_cb(void *read_cbopaque, void *buf, size_t limit) { 473 int mfd = *(int *)read_cbopaque; 474 assert(mfd != -1); 475 return malloc_read_fd(mfd, buf, limit); 476 } 477 478 static void 479 prof_dump_maps(buf_writer_t *buf_writer) { 480 int mfd = prof_dump_open_maps(); 481 if (mfd == -1) { 482 return; 483 } 484 485 buf_writer_cb(buf_writer, "\nMAPPED_LIBRARIES:\n"); 486 buf_writer_pipe(buf_writer, prof_dump_read_maps_cb, &mfd); 487 close(mfd); 488 } 489 490 static bool 491 prof_dump(tsd_t *tsd, bool propagate_err, const char *filename, 492 bool leakcheck) { 493 cassert(config_prof); 494 assert(tsd_reentrancy_level_get(tsd) == 0); 495 496 prof_tdata_t * tdata = prof_tdata_get(tsd, true); 497 if (tdata == NULL) { 498 return true; 499 } 500 501 prof_dump_arg_t arg = {/* handle_error_locally */ !propagate_err, 502 /* error */ false, /* prof_dump_fd */ -1}; 503 504 pre_reentrancy(tsd, NULL); 505 malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_mtx); 506 507 prof_dump_open(&arg, filename); 508 buf_writer_t buf_writer; 509 bool err = buf_writer_init(tsd_tsdn(tsd), &buf_writer, prof_dump_flush, 510 &arg, prof_dump_buf, PROF_DUMP_BUFSIZE); 511 assert(!err); 512 prof_dump_impl(tsd, buf_writer_cb, &buf_writer, tdata, leakcheck); 513 prof_dump_maps(&buf_writer); 514 buf_writer_terminate(tsd_tsdn(tsd), &buf_writer); 515 prof_dump_close(&arg); 516 517 prof_dump_hook_t dump_hook = prof_dump_hook_get(); 518 if (dump_hook != NULL) { 519 dump_hook(filename); 520 } 521 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_mtx); 522 post_reentrancy(tsd); 523 524 return arg.error; 525 } 526 527 /* 528 * If profiling is off, then PROF_DUMP_FILENAME_LEN is 1, so we'll end up 529 * calling strncpy with a size of 0, which triggers a -Wstringop-truncation 530 * warning (strncpy can never actually be called in this case, since we bail out 531 * much earlier when config_prof is false). This function works around the 532 * warning to let us leave the warning on. 533 */ 534 static inline void 535 prof_strncpy(char *UNUSED dest, const char *UNUSED src, size_t UNUSED size) { 536 cassert(config_prof); 537 #ifdef JEMALLOC_PROF 538 strncpy(dest, src, size); 539 #endif 540 } 541 542 static const char * 543 prof_prefix_get(tsdn_t* tsdn) { 544 malloc_mutex_assert_owner(tsdn, &prof_dump_filename_mtx); 545 546 return prof_prefix == NULL ? opt_prof_prefix : prof_prefix; 547 } 548 549 static bool 550 prof_prefix_is_empty(tsdn_t *tsdn) { 551 malloc_mutex_lock(tsdn, &prof_dump_filename_mtx); 552 bool ret = (prof_prefix_get(tsdn)[0] == '\0'); 553 malloc_mutex_unlock(tsdn, &prof_dump_filename_mtx); 554 return ret; 555 } 556 557 #define DUMP_FILENAME_BUFSIZE (PATH_MAX + 1) 558 #define VSEQ_INVALID UINT64_C(0xffffffffffffffff) 559 static void 560 prof_dump_filename(tsd_t *tsd, char *filename, char v, uint64_t vseq) { 561 cassert(config_prof); 562 563 assert(tsd_reentrancy_level_get(tsd) == 0); 564 const char *prefix = prof_prefix_get(tsd_tsdn(tsd)); 565 566 if (vseq != VSEQ_INVALID) { 567 /* "<prefix>.<pid>.<seq>.v<vseq>.heap" */ 568 malloc_snprintf(filename, DUMP_FILENAME_BUFSIZE, 569 "%s.%d.%"FMTu64".%c%"FMTu64".heap", prefix, prof_getpid(), 570 prof_dump_seq, v, vseq); 571 } else { 572 /* "<prefix>.<pid>.<seq>.<v>.heap" */ 573 malloc_snprintf(filename, DUMP_FILENAME_BUFSIZE, 574 "%s.%d.%"FMTu64".%c.heap", prefix, prof_getpid(), 575 prof_dump_seq, v); 576 } 577 prof_dump_seq++; 578 } 579 580 void 581 prof_get_default_filename(tsdn_t *tsdn, char *filename, uint64_t ind) { 582 malloc_mutex_lock(tsdn, &prof_dump_filename_mtx); 583 malloc_snprintf(filename, PROF_DUMP_FILENAME_LEN, 584 "%s.%d.%"FMTu64".json", prof_prefix_get(tsdn), prof_getpid(), ind); 585 malloc_mutex_unlock(tsdn, &prof_dump_filename_mtx); 586 } 587 588 void 589 prof_fdump_impl(tsd_t *tsd) { 590 char filename[DUMP_FILENAME_BUFSIZE]; 591 592 assert(!prof_prefix_is_empty(tsd_tsdn(tsd))); 593 malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_filename_mtx); 594 prof_dump_filename(tsd, filename, 'f', VSEQ_INVALID); 595 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_filename_mtx); 596 prof_dump(tsd, false, filename, opt_prof_leak); 597 } 598 599 bool 600 prof_prefix_set(tsdn_t *tsdn, const char *prefix) { 601 cassert(config_prof); 602 ctl_mtx_assert_held(tsdn); 603 malloc_mutex_lock(tsdn, &prof_dump_filename_mtx); 604 if (prof_prefix == NULL) { 605 malloc_mutex_unlock(tsdn, &prof_dump_filename_mtx); 606 /* Everything is still guarded by ctl_mtx. */ 607 char *buffer = base_alloc(tsdn, prof_base, 608 PROF_DUMP_FILENAME_LEN, QUANTUM); 609 if (buffer == NULL) { 610 return true; 611 } 612 malloc_mutex_lock(tsdn, &prof_dump_filename_mtx); 613 prof_prefix = buffer; 614 } 615 assert(prof_prefix != NULL); 616 617 prof_strncpy(prof_prefix, prefix, PROF_DUMP_FILENAME_LEN - 1); 618 prof_prefix[PROF_DUMP_FILENAME_LEN - 1] = '\0'; 619 malloc_mutex_unlock(tsdn, &prof_dump_filename_mtx); 620 621 return false; 622 } 623 624 void 625 prof_idump_impl(tsd_t *tsd) { 626 malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_filename_mtx); 627 if (prof_prefix_get(tsd_tsdn(tsd))[0] == '\0') { 628 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_filename_mtx); 629 return; 630 } 631 char filename[PATH_MAX + 1]; 632 prof_dump_filename(tsd, filename, 'i', prof_dump_iseq); 633 prof_dump_iseq++; 634 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_filename_mtx); 635 prof_dump(tsd, false, filename, false); 636 } 637 638 bool 639 prof_mdump_impl(tsd_t *tsd, const char *filename) { 640 char filename_buf[DUMP_FILENAME_BUFSIZE]; 641 if (filename == NULL) { 642 /* No filename specified, so automatically generate one. */ 643 malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_filename_mtx); 644 if (prof_prefix_get(tsd_tsdn(tsd))[0] == '\0') { 645 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_filename_mtx); 646 return true; 647 } 648 prof_dump_filename(tsd, filename_buf, 'm', prof_dump_mseq); 649 prof_dump_mseq++; 650 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_filename_mtx); 651 filename = filename_buf; 652 } 653 return prof_dump(tsd, true, filename, false); 654 } 655 656 void 657 prof_gdump_impl(tsd_t *tsd) { 658 tsdn_t *tsdn = tsd_tsdn(tsd); 659 malloc_mutex_lock(tsdn, &prof_dump_filename_mtx); 660 if (prof_prefix_get(tsdn)[0] == '\0') { 661 malloc_mutex_unlock(tsdn, &prof_dump_filename_mtx); 662 return; 663 } 664 char filename[DUMP_FILENAME_BUFSIZE]; 665 prof_dump_filename(tsd, filename, 'u', prof_dump_useq); 666 prof_dump_useq++; 667 malloc_mutex_unlock(tsdn, &prof_dump_filename_mtx); 668 prof_dump(tsd, false, filename, false); 669 } 670
Indexes created Tue Feb 24 08:35:24 UTC 2026