1 /*===-- InstrProfData.inc - instr profiling runtime structures -*- C++ -*-=== *\ 2 |* 3 |* The LLVM Compiler Infrastructure 4 |* 5 |* This file is distributed under the University of Illinois Open Source 6 |* License. See LICENSE.TXT for details. 7 |* 8 \*===----------------------------------------------------------------------===*/ 9 /* 10 * This is the master file that defines all the data structure, signature, 11 * constant literals that are shared across profiling runtime library, 12 * compiler (instrumentation), and host tools (reader/writer). The entities 13 * defined in this file affect the profile runtime ABI, the raw profile format, 14 * or both. 15 * 16 * The file has two identical copies. The master copy lives in LLVM and 17 * the other one sits in compiler-rt/lib/profile directory. To make changes 18 * in this file, first modify the master copy and copy it over to compiler-rt. 19 * Testing of any change in this file can start only after the two copies are 20 * synced up. 21 * 22 * The first part of the file includes macros that defines types, names, and 23 * initializers for the member fields of the core data structures. The field 24 * declarations for one structure is enabled by defining the field activation 25 * macro associated with that structure. Only one field activation record 26 * can be defined at one time and the rest definitions will be filtered out by 27 * the preprocessor. 28 * 29 * Examples of how the template is used to instantiate structure definition: 30 * 1. To declare a structure: 31 * 32 * struct ProfData { 33 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ 34 * Type Name; 35 * #include "llvm/ProfileData/InstrProfData.inc" 36 * }; 37 * 38 * 2. To construct LLVM type arrays for the struct type: 39 * 40 * Type *DataTypes[] = { 41 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ 42 * LLVMType, 43 * #include "llvm/ProfileData/InstrProfData.inc" 44 * }; 45 * 46 * 4. To construct constant array for the initializers: 47 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ 48 * Initializer, 49 * Constant *ConstantVals[] = { 50 * #include "llvm/ProfileData/InstrProfData.inc" 51 * }; 52 * 53 * 54 * The second part of the file includes definitions all other entities that 55 * are related to runtime ABI and format. When no field activation macro is 56 * defined, this file can be included to introduce the definitions. 57 * 58 \*===----------------------------------------------------------------------===*/ 59 60 /* INSTR_PROF_DATA start. */ 61 /* Definition of member fields of the per-function control structure. */ 62 #ifndef INSTR_PROF_DATA 63 #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) 64 #else 65 #define INSTR_PROF_DATA_DEFINED 66 #endif 67 68 INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NameSize, \ 69 ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \ 70 NamePtr->getType()->getPointerElementType()->getArrayNumElements())) 71 INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \ 72 ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters)) 73 INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \ 74 ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ 75 Inc->getHash()->getZExtValue())) 76 INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), NamePtr, \ 77 ConstantExpr::getBitCast(NamePtr, llvm::Type::getInt8PtrTy(Ctx))) 78 INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt64PtrTy(Ctx), CounterPtr, \ 79 ConstantExpr::getBitCast(CounterPtr, \ 80 llvm::Type::getInt64PtrTy(Ctx))) 81 INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), FunctionPointer, \ 82 FunctionAddr) 83 INSTR_PROF_DATA(IntPtrT, llvm::Type::getInt8PtrTy(Ctx), Values, \ 84 ConstantPointerNull::get(Int8PtrTy)) 85 INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \ 86 ConstantArray::get(Int16ArrayTy, Int16ArrayVals)) 87 #undef INSTR_PROF_DATA 88 /* INSTR_PROF_DATA end. */ 89 90 /* INSTR_PROF_RAW_HEADER start */ 91 /* Definition of member fields of the raw profile header data structure. */ 92 #ifndef INSTR_PROF_RAW_HEADER 93 #define INSTR_PROF_RAW_HEADER(Type, Name, Initializer) 94 #else 95 #define INSTR_PROF_DATA_DEFINED 96 #endif 97 INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic()) 98 INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version()) 99 INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize) 100 INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize) 101 INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) 102 INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin) 103 INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) 104 INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) 105 INSTR_PROF_RAW_HEADER(uint64_t, ValueDataSize, ValueDataSize) 106 INSTR_PROF_RAW_HEADER(uint64_t, ValueDataDelta, (uintptr_t)ValueDataBegin) 107 #undef INSTR_PROF_RAW_HEADER 108 /* INSTR_PROF_RAW_HEADER end */ 109 110 /* VALUE_PROF_FUNC_PARAM start */ 111 /* Definition of parameter types of the runtime API used to do value profiling 112 * for a given value site. 113 */ 114 #ifndef VALUE_PROF_FUNC_PARAM 115 #define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType) 116 #define INSTR_PROF_COMMA 117 #else 118 #define INSTR_PROF_DATA_DEFINED 119 #define INSTR_PROF_COMMA , 120 #endif 121 VALUE_PROF_FUNC_PARAM(uint64_t, TargetValue, Type::getInt64Ty(Ctx)) \ 122 INSTR_PROF_COMMA 123 VALUE_PROF_FUNC_PARAM(void *, Data, Type::getInt8PtrTy(Ctx)) INSTR_PROF_COMMA 124 VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx)) 125 #undef VALUE_PROF_FUNC_PARAM 126 #undef INSTR_PROF_COMMA 127 /* VALUE_PROF_FUNC_PARAM end */ 128 129 /* VALUE_PROF_KIND start */ 130 #ifndef VALUE_PROF_KIND 131 #define VALUE_PROF_KIND(Enumerator, Value) 132 #else 133 #define INSTR_PROF_DATA_DEFINED 134 #endif 135 VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0) 136 /* These two kinds must be the last to be 137 * declared. This is to make sure the string 138 * array created with the template can be 139 * indexed with the kind value. 140 */ 141 VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget) 142 VALUE_PROF_KIND(IPVK_Last, IPVK_IndirectCallTarget) 143 144 #undef VALUE_PROF_KIND 145 /* VALUE_PROF_KIND end */ 146 147 /* COVMAP_FUNC_RECORD start */ 148 /* Definition of member fields of the function record structure in coverage 149 * map. 150 */ 151 #ifndef COVMAP_FUNC_RECORD 152 #define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Initializer) 153 #else 154 #define INSTR_PROF_DATA_DEFINED 155 #endif 156 COVMAP_FUNC_RECORD(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), \ 157 NamePtr, llvm::ConstantExpr::getBitCast(NamePtr, \ 158 llvm::Type::getInt8PtrTy(Ctx))) 159 COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), NameSize, \ 160 llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx),\ 161 NameValue.size())) 162 COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), DataSize, \ 163 llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx),\ 164 CoverageMapping.size())) 165 COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \ 166 llvm::ConstantInt::get(llvm::Type::getInt64Ty(Ctx), FuncHash)) 167 #undef COVMAP_FUNC_RECORD 168 /* COVMAP_FUNC_RECORD end. */ 169 170 /* COVMAP_HEADER start */ 171 /* Definition of member fields of coverage map header. 172 */ 173 #ifndef COVMAP_HEADER 174 #define COVMAP_HEADER(Type, LLVMType, Name, Initializer) 175 #else 176 #define INSTR_PROF_DATA_DEFINED 177 #endif 178 COVMAP_HEADER(uint32_t, Int32Ty, NRecords, \ 179 llvm::ConstantInt::get(Int32Ty, FunctionRecords.size())) 180 COVMAP_HEADER(uint32_t, Int32Ty, FilenamesSize, \ 181 llvm::ConstantInt::get(Int32Ty, FilenamesSize)) 182 COVMAP_HEADER(uint32_t, Int32Ty, CoverageSize, \ 183 llvm::ConstantInt::get(Int32Ty, CoverageMappingSize)) 184 COVMAP_HEADER(uint32_t, Int32Ty, Version, \ 185 llvm::ConstantInt::get(Int32Ty, CovMapVersion::CurrentVersion)) 186 #undef COVMAP_HEADER 187 /* COVMAP_HEADER end. */ 188 189 190 #ifdef INSTR_PROF_VALUE_PROF_DATA 191 #define INSTR_PROF_DATA_DEFINED 192 193 #define INSTR_PROF_MAX_NUM_VAL_PER_SITE 255 194 /*! 195 * This is the header of the data structure that defines the on-disk 196 * layout of the value profile data of a particular kind for one function. 197 */ 198 typedef struct ValueProfRecord { 199 /* The kind of the value profile record. */ 200 uint32_t Kind; 201 /* 202 * The number of value profile sites. It is guaranteed to be non-zero; 203 * otherwise the record for this kind won't be emitted. 204 */ 205 uint32_t NumValueSites; 206 /* 207 * The first element of the array that stores the number of profiled 208 * values for each value site. The size of the array is NumValueSites. 209 * Since NumValueSites is greater than zero, there is at least one 210 * element in the array. 211 */ 212 uint8_t SiteCountArray[1]; 213 214 /* 215 * The fake declaration is for documentation purpose only. 216 * Align the start of next field to be on 8 byte boundaries. 217 uint8_t Padding[X]; 218 */ 219 220 /* The array of value profile data. The size of the array is the sum 221 * of all elements in SiteCountArray[]. 222 InstrProfValueData ValueData[]; 223 */ 224 225 #ifdef __cplusplus 226 /*! 227 * \brief Return the number of value sites. 228 */ 229 uint32_t getNumValueSites() const { return NumValueSites; } 230 /*! 231 * \brief Read data from this record and save it to Record. 232 */ 233 void deserializeTo(InstrProfRecord &Record, 234 InstrProfRecord::ValueMapType *VMap); 235 /* 236 * In-place byte swap: 237 * Do byte swap for this instance. \c Old is the original order before 238 * the swap, and \c New is the New byte order. 239 */ 240 void swapBytes(support::endianness Old, support::endianness New); 241 #endif 242 } ValueProfRecord; 243 244 /*! 245 * Per-function header/control data structure for value profiling 246 * data in indexed format. 247 */ 248 typedef struct ValueProfData { 249 /* 250 * Total size in bytes including this field. It must be a multiple 251 * of sizeof(uint64_t). 252 */ 253 uint32_t TotalSize; 254 /* 255 *The number of value profile kinds that has value profile data. 256 * In this implementation, a value profile kind is considered to 257 * have profile data if the number of value profile sites for the 258 * kind is not zero. More aggressively, the implementation can 259 * choose to check the actual data value: if none of the value sites 260 * has any profiled values, the kind can be skipped. 261 */ 262 uint32_t NumValueKinds; 263 264 /* 265 * Following are a sequence of variable length records. The prefix/header 266 * of each record is defined by ValueProfRecord type. The number of 267 * records is NumValueKinds. 268 * ValueProfRecord Record_1; 269 * ValueProfRecord Record_N; 270 */ 271 272 #if __cplusplus 273 /*! 274 * Return the total size in bytes of the on-disk value profile data 275 * given the data stored in Record. 276 */ 277 static uint32_t getSize(const InstrProfRecord &Record); 278 /*! 279 * Return a pointer to \c ValueProfData instance ready to be streamed. 280 */ 281 static std::unique_ptr<ValueProfData> 282 serializeFrom(const InstrProfRecord &Record); 283 /*! 284 * Check the integrity of the record. Return the error code when 285 * an error is detected, otherwise return instrprof_error::success. 286 */ 287 instrprof_error checkIntegrity(); 288 /*! 289 * Return a pointer to \c ValueProfileData instance ready to be read. 290 * All data in the instance are properly byte swapped. The input 291 * data is assumed to be in little endian order. 292 */ 293 static ErrorOr<std::unique_ptr<ValueProfData>> 294 getValueProfData(const unsigned char *SrcBuffer, 295 const unsigned char *const SrcBufferEnd, 296 support::endianness SrcDataEndianness); 297 /*! 298 * Swap byte order from \c Endianness order to host byte order. 299 */ 300 void swapBytesToHost(support::endianness Endianness); 301 /*! 302 * Swap byte order from host byte order to \c Endianness order. 303 */ 304 void swapBytesFromHost(support::endianness Endianness); 305 /*! 306 * Return the total size of \c ValueProfileData. 307 */ 308 uint32_t getSize() const { return TotalSize; } 309 /*! 310 * Read data from this data and save it to \c Record. 311 */ 312 void deserializeTo(InstrProfRecord &Record, 313 InstrProfRecord::ValueMapType *VMap); 314 void operator delete(void *ptr) { ::operator delete(ptr); } 315 #endif 316 } ValueProfData; 317 318 /* 319 * The closure is designed to abstact away two types of value profile data: 320 * - InstrProfRecord which is the primary data structure used to 321 * represent profile data in host tools (reader, writer, and profile-use) 322 * - value profile runtime data structure suitable to be used by C 323 * runtime library. 324 * 325 * Both sources of data need to serialize to disk/memory-buffer in common 326 * format: ValueProfData. The abstraction allows compiler-rt's raw profiler 327 * writer to share the same format and code with indexed profile writer. 328 * 329 * For documentation of the member methods below, refer to corresponding methods 330 * in class InstrProfRecord. 331 */ 332 typedef struct ValueProfRecordClosure { 333 const void *Record; 334 uint32_t (*GetNumValueKinds)(const void *Record); 335 uint32_t (*GetNumValueSites)(const void *Record, uint32_t VKind); 336 uint32_t (*GetNumValueData)(const void *Record, uint32_t VKind); 337 uint32_t (*GetNumValueDataForSite)(const void *R, uint32_t VK, uint32_t S); 338 339 /* 340 * After extracting the value profile data from the value profile record, 341 * this method is used to map the in-memory value to on-disk value. If 342 * the method is null, value will be written out untranslated. 343 */ 344 uint64_t (*RemapValueData)(uint32_t, uint64_t Value); 345 void (*GetValueForSite)(const void *R, InstrProfValueData *Dst, uint32_t K, 346 uint32_t S, uint64_t (*Mapper)(uint32_t, uint64_t)); 347 ValueProfData *(*AllocValueProfData)(size_t TotalSizeInBytes); 348 } ValueProfRecordClosure; 349 350 /* 351 * A wrapper struct that represents value profile runtime data. 352 * Like InstrProfRecord class which is used by profiling host tools, 353 * ValueProfRuntimeRecord also implements the abstract intefaces defined in 354 * ValueProfRecordClosure so that the runtime data can be serialized using 355 * shared C implementation. In this structure, NumValueSites and Nodes 356 * members are the primary fields while other fields hold the derived 357 * information for fast implementation of closure interfaces. 358 */ 359 typedef struct ValueProfRuntimeRecord { 360 /* Number of sites for each value profile kind. */ 361 const uint16_t *NumValueSites; 362 /* An array of linked-list headers. The size of of the array is the 363 * total number of value profile sites : sum(NumValueSites[*])). Each 364 * linked-list stores the values profiled for a value profile site. */ 365 ValueProfNode **Nodes; 366 367 /* Total number of value profile kinds which have at least one 368 * value profile sites. */ 369 uint32_t NumValueKinds; 370 /* An array recording the number of values tracked at each site. 371 * The size of the array is TotalNumValueSites. */ 372 uint8_t *SiteCountArray[IPVK_Last + 1]; 373 ValueProfNode **NodesKind[IPVK_Last + 1]; 374 } ValueProfRuntimeRecord; 375 376 /* Forward declarations of C interfaces. */ 377 int initializeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord, 378 const uint16_t *NumValueSites, 379 ValueProfNode **Nodes); 380 void finalizeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord); 381 uint32_t getValueProfDataSizeRT(const ValueProfRuntimeRecord *Record); 382 ValueProfData * 383 serializeValueProfDataFromRT(const ValueProfRuntimeRecord *Record, 384 ValueProfData *Dst); 385 uint32_t getNumValueKindsRT(const void *R); 386 387 #undef INSTR_PROF_VALUE_PROF_DATA 388 #endif /* INSTR_PROF_VALUE_PROF_DATA */ 389 390 391 #ifdef INSTR_PROF_COMMON_API_IMPL 392 #define INSTR_PROF_DATA_DEFINED 393 #ifdef __cplusplus 394 #define INSTR_PROF_INLINE inline 395 #define INSTR_PROF_NULLPTR nullptr 396 #else 397 #define INSTR_PROF_INLINE 398 #define INSTR_PROF_NULLPTR NULL 399 #endif 400 401 #ifndef offsetof 402 #define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER) 403 #endif 404 405 /*! 406 * \brief Return the \c ValueProfRecord header size including the 407 * padding bytes. 408 */ 409 INSTR_PROF_INLINE 410 uint32_t getValueProfRecordHeaderSize(uint32_t NumValueSites) { 411 uint32_t Size = offsetof(ValueProfRecord, SiteCountArray) + 412 sizeof(uint8_t) * NumValueSites; 413 /* Round the size to multiple of 8 bytes. */ 414 Size = (Size + 7) & ~7; 415 return Size; 416 } 417 418 /*! 419 * \brief Return the total size of the value profile record including the 420 * header and the value data. 421 */ 422 INSTR_PROF_INLINE 423 uint32_t getValueProfRecordSize(uint32_t NumValueSites, 424 uint32_t NumValueData) { 425 return getValueProfRecordHeaderSize(NumValueSites) + 426 sizeof(InstrProfValueData) * NumValueData; 427 } 428 429 /*! 430 * \brief Return the pointer to the start of value data array. 431 */ 432 INSTR_PROF_INLINE 433 InstrProfValueData *getValueProfRecordValueData(ValueProfRecord *This) { 434 return (InstrProfValueData *)((char *)This + getValueProfRecordHeaderSize( 435 This->NumValueSites)); 436 } 437 438 /*! 439 * \brief Return the total number of value data for \c This record. 440 */ 441 INSTR_PROF_INLINE 442 uint32_t getValueProfRecordNumValueData(ValueProfRecord *This) { 443 uint32_t NumValueData = 0; 444 uint32_t I; 445 for (I = 0; I < This->NumValueSites; I++) 446 NumValueData += This->SiteCountArray[I]; 447 return NumValueData; 448 } 449 450 /*! 451 * \brief Use this method to advance to the next \c This \c ValueProfRecord. 452 */ 453 INSTR_PROF_INLINE 454 ValueProfRecord *getValueProfRecordNext(ValueProfRecord *This) { 455 uint32_t NumValueData = getValueProfRecordNumValueData(This); 456 return (ValueProfRecord *)((char *)This + 457 getValueProfRecordSize(This->NumValueSites, 458 NumValueData)); 459 } 460 461 /*! 462 * \brief Return the first \c ValueProfRecord instance. 463 */ 464 INSTR_PROF_INLINE 465 ValueProfRecord *getFirstValueProfRecord(ValueProfData *This) { 466 return (ValueProfRecord *)((char *)This + sizeof(ValueProfData)); 467 } 468 469 /* Closure based interfaces. */ 470 471 /*! 472 * Return the total size in bytes of the on-disk value profile data 473 * given the data stored in Record. 474 */ 475 uint32_t getValueProfDataSize(ValueProfRecordClosure *Closure) { 476 uint32_t Kind; 477 uint32_t TotalSize = sizeof(ValueProfData); 478 const void *Record = Closure->Record; 479 uint32_t NumValueKinds = Closure->GetNumValueKinds(Record); 480 if (NumValueKinds == 0) 481 return TotalSize; 482 483 for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) { 484 uint32_t NumValueSites = Closure->GetNumValueSites(Record, Kind); 485 if (!NumValueSites) 486 continue; 487 TotalSize += getValueProfRecordSize(NumValueSites, 488 Closure->GetNumValueData(Record, Kind)); 489 } 490 return TotalSize; 491 } 492 493 /*! 494 * Extract value profile data of a function for the profile kind \c ValueKind 495 * from the \c Closure and serialize the data into \c This record instance. 496 */ 497 void serializeValueProfRecordFrom(ValueProfRecord *This, 498 ValueProfRecordClosure *Closure, 499 uint32_t ValueKind, uint32_t NumValueSites) { 500 uint32_t S; 501 const void *Record = Closure->Record; 502 This->Kind = ValueKind; 503 This->NumValueSites = NumValueSites; 504 InstrProfValueData *DstVD = getValueProfRecordValueData(This); 505 506 for (S = 0; S < NumValueSites; S++) { 507 uint32_t ND = Closure->GetNumValueDataForSite(Record, ValueKind, S); 508 This->SiteCountArray[S] = ND; 509 Closure->GetValueForSite(Record, DstVD, ValueKind, S, 510 Closure->RemapValueData); 511 DstVD += ND; 512 } 513 } 514 515 /*! 516 * Extract value profile data of a function from the \c Closure 517 * and serialize the data into \c DstData if it is not NULL or heap 518 * memory allocated by the \c Closure's allocator method. 519 */ 520 ValueProfData *serializeValueProfDataFrom(ValueProfRecordClosure *Closure, 521 ValueProfData *DstData) { 522 uint32_t Kind; 523 uint32_t TotalSize = getValueProfDataSize(Closure); 524 525 ValueProfData *VPD = 526 DstData ? DstData : Closure->AllocValueProfData(TotalSize); 527 528 VPD->TotalSize = TotalSize; 529 VPD->NumValueKinds = Closure->GetNumValueKinds(Closure->Record); 530 ValueProfRecord *VR = getFirstValueProfRecord(VPD); 531 for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) { 532 uint32_t NumValueSites = Closure->GetNumValueSites(Closure->Record, Kind); 533 if (!NumValueSites) 534 continue; 535 serializeValueProfRecordFrom(VR, Closure, Kind, NumValueSites); 536 VR = getValueProfRecordNext(VR); 537 } 538 return VPD; 539 } 540 541 /* 542 * The value profiler runtime library stores the value profile data 543 * for a given function in \c NumValueSites and \c Nodes structures. 544 * \c ValueProfRuntimeRecord class is used to encapsulate the runtime 545 * profile data and provides fast interfaces to retrieve the profile 546 * information. This interface is used to initialize the runtime record 547 * and pre-compute the information needed for efficient implementation 548 * of callbacks required by ValueProfRecordClosure class. 549 */ 550 int initializeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord, 551 const uint16_t *NumValueSites, 552 ValueProfNode **Nodes) { 553 unsigned I, J, S = 0, NumValueKinds = 0; 554 RuntimeRecord->NumValueSites = NumValueSites; 555 RuntimeRecord->Nodes = Nodes; 556 for (I = 0; I <= IPVK_Last; I++) { 557 uint16_t N = NumValueSites[I]; 558 if (!N) { 559 RuntimeRecord->SiteCountArray[I] = INSTR_PROF_NULLPTR; 560 continue; 561 } 562 NumValueKinds++; 563 RuntimeRecord->SiteCountArray[I] = (uint8_t *)calloc(N, 1); 564 if (!RuntimeRecord->SiteCountArray[I]) 565 return 1; 566 RuntimeRecord->NodesKind[I] = Nodes ? &Nodes[S] : INSTR_PROF_NULLPTR; 567 for (J = 0; J < N; J++) { 568 /* Compute value count for each site. */ 569 uint32_t C = 0; 570 ValueProfNode *Site = 571 Nodes ? RuntimeRecord->NodesKind[I][J] : INSTR_PROF_NULLPTR; 572 while (Site) { 573 C++; 574 Site = Site->Next; 575 } 576 if (C > UCHAR_MAX) 577 C = UCHAR_MAX; 578 RuntimeRecord->SiteCountArray[I][J] = C; 579 } 580 S += N; 581 } 582 RuntimeRecord->NumValueKinds = NumValueKinds; 583 return 0; 584 } 585 586 void finalizeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord) { 587 unsigned I; 588 for (I = 0; I <= IPVK_Last; I++) { 589 if (RuntimeRecord->SiteCountArray[I]) 590 free(RuntimeRecord->SiteCountArray[I]); 591 } 592 } 593 594 /* ValueProfRecordClosure Interface implementation for 595 * ValueProfDataRuntimeRecord. */ 596 uint32_t getNumValueKindsRT(const void *R) { 597 return ((const ValueProfRuntimeRecord *)R)->NumValueKinds; 598 } 599 600 uint32_t getNumValueSitesRT(const void *R, uint32_t VK) { 601 return ((const ValueProfRuntimeRecord *)R)->NumValueSites[VK]; 602 } 603 604 uint32_t getNumValueDataForSiteRT(const void *R, uint32_t VK, uint32_t S) { 605 const ValueProfRuntimeRecord *Record = (const ValueProfRuntimeRecord *)R; 606 return Record->SiteCountArray[VK][S]; 607 } 608 609 uint32_t getNumValueDataRT(const void *R, uint32_t VK) { 610 unsigned I, S = 0; 611 const ValueProfRuntimeRecord *Record = (const ValueProfRuntimeRecord *)R; 612 if (Record->SiteCountArray[VK] == INSTR_PROF_NULLPTR) 613 return 0; 614 for (I = 0; I < Record->NumValueSites[VK]; I++) 615 S += Record->SiteCountArray[VK][I]; 616 return S; 617 } 618 619 void getValueForSiteRT(const void *R, InstrProfValueData *Dst, uint32_t VK, 620 uint32_t S, uint64_t (*Mapper)(uint32_t, uint64_t)) { 621 unsigned I, N = 0; 622 const ValueProfRuntimeRecord *Record = (const ValueProfRuntimeRecord *)R; 623 N = getNumValueDataForSiteRT(R, VK, S); 624 if (N == 0) 625 return; 626 ValueProfNode *VNode = Record->NodesKind[VK][S]; 627 for (I = 0; I < N; I++) { 628 Dst[I] = VNode->VData; 629 VNode = VNode->Next; 630 } 631 } 632 633 ValueProfData *allocValueProfDataRT(size_t TotalSizeInBytes) { 634 return (ValueProfData *)calloc(TotalSizeInBytes, 1); 635 } 636 637 static ValueProfRecordClosure RTRecordClosure = { 638 INSTR_PROF_NULLPTR, getNumValueKindsRT, getNumValueSitesRT, 639 getNumValueDataRT, getNumValueDataForSiteRT, INSTR_PROF_NULLPTR, 640 getValueForSiteRT, allocValueProfDataRT}; 641 642 /* 643 * Return the size of ValueProfData structure to store data 644 * recorded in the runtime record. 645 */ 646 uint32_t getValueProfDataSizeRT(const ValueProfRuntimeRecord *Record) { 647 RTRecordClosure.Record = Record; 648 return getValueProfDataSize(&RTRecordClosure); 649 } 650 651 /* 652 * Return a ValueProfData instance that stores the data collected 653 * from runtime. If \c DstData is provided by the caller, the value 654 * profile data will be store in *DstData and DstData is returned, 655 * otherwise the method will allocate space for the value data and 656 * return pointer to the newly allocated space. 657 */ 658 ValueProfData * 659 serializeValueProfDataFromRT(const ValueProfRuntimeRecord *Record, 660 ValueProfData *DstData) { 661 RTRecordClosure.Record = Record; 662 return serializeValueProfDataFrom(&RTRecordClosure, DstData); 663 } 664 665 #undef INSTR_PROF_COMMON_API_IMPL 666 #endif /* INSTR_PROF_COMMON_API_IMPL */ 667 668 /*============================================================================*/ 669 670 #ifndef INSTR_PROF_DATA_DEFINED 671 672 #ifndef INSTR_PROF_DATA_INC 673 #define INSTR_PROF_DATA_INC 674 675 /* Helper macros. */ 676 #define INSTR_PROF_SIMPLE_QUOTE(x) #x 677 #define INSTR_PROF_QUOTE(x) INSTR_PROF_SIMPLE_QUOTE(x) 678 #define INSTR_PROF_SIMPLE_CONCAT(x,y) x ## y 679 #define INSTR_PROF_CONCAT(x,y) INSTR_PROF_SIMPLE_CONCAT(x,y) 680 681 /* Magic number to detect file format and endianness. 682 * Use 255 at one end, since no UTF-8 file can use that character. Avoid 0, 683 * so that utilities, like strings, don't grab it as a string. 129 is also 684 * invalid UTF-8, and high enough to be interesting. 685 * Use "lprofr" in the centre to stand for "LLVM Profile Raw", or "lprofR" 686 * for 32-bit platforms. 687 */ 688 #define INSTR_PROF_RAW_MAGIC_64 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \ 689 (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \ 690 (uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129 691 #define INSTR_PROF_RAW_MAGIC_32 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \ 692 (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \ 693 (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129 694 695 /* Raw profile format version. */ 696 #define INSTR_PROF_RAW_VERSION 2 697 #define INSTR_PROF_INDEX_VERSION 3 698 #define INSTR_PROF_COVMAP_VERSION 0 699 700 /* Profile version is always of type uint64_t. Reserve the upper 8 bits in the 701 * version for other variants of profile. We set the lowest bit of the upper 8 702 * bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentaiton 703 * generated profile, and 0 if this is a Clang FE generated profile. 704 */ 705 #define VARIANT_MASKS_ALL 0xff00000000000000ULL 706 #define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL) 707 708 /* Runtime section names and name strings. */ 709 #define INSTR_PROF_DATA_SECT_NAME __llvm_prf_data 710 #define INSTR_PROF_NAME_SECT_NAME __llvm_prf_names 711 #define INSTR_PROF_CNTS_SECT_NAME __llvm_prf_cnts 712 #define INSTR_PROF_COVMAP_SECT_NAME __llvm_covmap 713 714 #define INSTR_PROF_DATA_SECT_NAME_STR \ 715 INSTR_PROF_QUOTE(INSTR_PROF_DATA_SECT_NAME) 716 #define INSTR_PROF_NAME_SECT_NAME_STR \ 717 INSTR_PROF_QUOTE(INSTR_PROF_NAME_SECT_NAME) 718 #define INSTR_PROF_CNTS_SECT_NAME_STR \ 719 INSTR_PROF_QUOTE(INSTR_PROF_CNTS_SECT_NAME) 720 #define INSTR_PROF_COVMAP_SECT_NAME_STR \ 721 INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_SECT_NAME) 722 723 /* Macros to define start/stop section symbol for a given 724 * section on Linux. For instance 725 * INSTR_PROF_SECT_START(INSTR_PROF_DATA_SECT_NAME) will 726 * expand to __start___llvm_prof_data 727 */ 728 #define INSTR_PROF_SECT_START(Sect) \ 729 INSTR_PROF_CONCAT(__start_,Sect) 730 #define INSTR_PROF_SECT_STOP(Sect) \ 731 INSTR_PROF_CONCAT(__stop_,Sect) 732 733 /* Value Profiling API linkage name. */ 734 #define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target 735 #define INSTR_PROF_VALUE_PROF_FUNC_STR \ 736 INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC) 737 738 /* InstrProfile per-function control data alignment. */ 739 #define INSTR_PROF_DATA_ALIGNMENT 8 740 741 /* The data structure that represents a tracked value by the 742 * value profiler. 743 */ 744 typedef struct InstrProfValueData { 745 /* Profiled value. */ 746 uint64_t Value; 747 /* Number of times the value appears in the training run. */ 748 uint64_t Count; 749 } InstrProfValueData; 750 751 /* This is an internal data structure used by value profiler. It 752 * is defined here to allow serialization code sharing by LLVM 753 * to be used in unit test. 754 */ 755 typedef struct ValueProfNode { 756 InstrProfValueData VData; 757 struct ValueProfNode *Next; 758 } ValueProfNode; 759 760 #endif /* INSTR_PROF_DATA_INC */ 761 762 #else 763 #undef INSTR_PROF_DATA_DEFINED 764 #endif 765
Indexes created Sat Sep 20 22:09:52 GMT 2025