Home | History | Annotate | Line # | Download | only in Instrumentation 
      1 //===- MemProfiler.cpp - memory allocation and access profiler ------------===//
      2 //
      3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
      4 // See https://llvm.org/LICENSE.txt for license information.
      5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
      6 //
      7 //===----------------------------------------------------------------------===//
      8 //
      9 // This file is a part of MemProfiler. Memory accesses are instrumented
     10 // to increment the access count held in a shadow memory location, or
     11 // alternatively to call into the runtime. Memory intrinsic calls (memmove,
     12 // memcpy, memset) are changed to call the memory profiling runtime version
     13 // instead.
     14 //
     15 //===----------------------------------------------------------------------===//
     16 
     17 #include "llvm/Transforms/Instrumentation/MemProfiler.h"
     18 #include "llvm/ADT/SmallVector.h"
     19 #include "llvm/ADT/Statistic.h"
     20 #include "llvm/ADT/StringRef.h"
     21 #include "llvm/ADT/Triple.h"
     22 #include "llvm/IR/Constant.h"
     23 #include "llvm/IR/DataLayout.h"
     24 #include "llvm/IR/Function.h"
     25 #include "llvm/IR/GlobalValue.h"
     26 #include "llvm/IR/IRBuilder.h"
     27 #include "llvm/IR/Instruction.h"
     28 #include "llvm/IR/LLVMContext.h"
     29 #include "llvm/IR/Module.h"
     30 #include "llvm/IR/Type.h"
     31 #include "llvm/IR/Value.h"
     32 #include "llvm/InitializePasses.h"
     33 #include "llvm/Pass.h"
     34 #include "llvm/Support/CommandLine.h"
     35 #include "llvm/Support/Debug.h"
     36 #include "llvm/Transforms/Instrumentation.h"
     37 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
     38 #include "llvm/Transforms/Utils/ModuleUtils.h"
     39 
     40 using namespace llvm;
     41 
     42 #define DEBUG_TYPE "memprof"
     43 
     44 constexpr int LLVM_MEM_PROFILER_VERSION = 1;
     45 
     46 // Size of memory mapped to a single shadow location.
     47 constexpr uint64_t DefaultShadowGranularity = 64;
     48 
     49 // Scale from granularity down to shadow size.
     50 constexpr uint64_t DefaultShadowScale = 3;
     51 
     52 constexpr char MemProfModuleCtorName[] = "memprof.module_ctor";
     53 constexpr uint64_t MemProfCtorAndDtorPriority = 1;
     54 // On Emscripten, the system needs more than one priorities for constructors.
     55 constexpr uint64_t MemProfEmscriptenCtorAndDtorPriority = 50;
     56 constexpr char MemProfInitName[] = "__memprof_init";
     57 constexpr char MemProfVersionCheckNamePrefix[] =
     58     "__memprof_version_mismatch_check_v";
     59 
     60 constexpr char MemProfShadowMemoryDynamicAddress[] =
     61     "__memprof_shadow_memory_dynamic_address";
     62 
     63 constexpr char MemProfFilenameVar[] = "__memprof_profile_filename";
     64 
     65 // Command-line flags.
     66 
     67 static cl::opt<bool> ClInsertVersionCheck(
     68     "memprof-guard-against-version-mismatch",
     69     cl::desc("Guard against compiler/runtime version mismatch."), cl::Hidden,
     70     cl::init(true));
     71 
     72 // This flag may need to be replaced with -f[no-]memprof-reads.
     73 static cl::opt<bool> ClInstrumentReads("memprof-instrument-reads",
     74                                        cl::desc("instrument read instructions"),
     75                                        cl::Hidden, cl::init(true));
     76 
     77 static cl::opt<bool>
     78     ClInstrumentWrites("memprof-instrument-writes",
     79                        cl::desc("instrument write instructions"), cl::Hidden,
     80                        cl::init(true));
     81 
     82 static cl::opt<bool> ClInstrumentAtomics(
     83     "memprof-instrument-atomics",
     84     cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden,
     85     cl::init(true));
     86 
     87 static cl::opt<bool> ClUseCalls(
     88     "memprof-use-callbacks",
     89     cl::desc("Use callbacks instead of inline instrumentation sequences."),
     90     cl::Hidden, cl::init(false));
     91 
     92 static cl::opt<std::string>
     93     ClMemoryAccessCallbackPrefix("memprof-memory-access-callback-prefix",
     94                                  cl::desc("Prefix for memory access callbacks"),
     95                                  cl::Hidden, cl::init("__memprof_"));
     96 
     97 // These flags allow to change the shadow mapping.
     98 // The shadow mapping looks like
     99 //    Shadow = ((Mem & mask) >> scale) + offset
    100 
    101 static cl::opt<int> ClMappingScale("memprof-mapping-scale",
    102                                    cl::desc("scale of memprof shadow mapping"),
    103                                    cl::Hidden, cl::init(DefaultShadowScale));
    104 
    105 static cl::opt<int>
    106     ClMappingGranularity("memprof-mapping-granularity",
    107                          cl::desc("granularity of memprof shadow mapping"),
    108                          cl::Hidden, cl::init(DefaultShadowGranularity));
    109 
    110 // Debug flags.
    111 
    112 static cl::opt<int> ClDebug("memprof-debug", cl::desc("debug"), cl::Hidden,
    113                             cl::init(0));
    114 
    115 static cl::opt<std::string> ClDebugFunc("memprof-debug-func", cl::Hidden,
    116                                         cl::desc("Debug func"));
    117 
    118 static cl::opt<int> ClDebugMin("memprof-debug-min", cl::desc("Debug min inst"),
    119                                cl::Hidden, cl::init(-1));
    120 
    121 static cl::opt<int> ClDebugMax("memprof-debug-max", cl::desc("Debug max inst"),
    122                                cl::Hidden, cl::init(-1));
    123 
    124 STATISTIC(NumInstrumentedReads, "Number of instrumented reads");
    125 STATISTIC(NumInstrumentedWrites, "Number of instrumented writes");
    126 
    127 namespace {
    128 
    129 /// This struct defines the shadow mapping using the rule:
    130 ///   shadow = ((mem & mask) >> Scale) ADD DynamicShadowOffset.
    131 struct ShadowMapping {
    132   ShadowMapping() {
    133     Scale = ClMappingScale;
    134     Granularity = ClMappingGranularity;
    135     Mask = ~(Granularity - 1);
    136   }
    137 
    138   int Scale;
    139   int Granularity;
    140   uint64_t Mask; // Computed as ~(Granularity-1)
    141 };
    142 
    143 static uint64_t getCtorAndDtorPriority(Triple &TargetTriple) {
    144   return TargetTriple.isOSEmscripten() ? MemProfEmscriptenCtorAndDtorPriority
    145                                        : MemProfCtorAndDtorPriority;
    146 }
    147 
    148 struct InterestingMemoryAccess {
    149   Value *Addr = nullptr;
    150   bool IsWrite;
    151   unsigned Alignment;
    152   uint64_t TypeSize;
    153   Value *MaybeMask = nullptr;
    154 };
    155 
    156 /// Instrument the code in module to profile memory accesses.
    157 class MemProfiler {
    158 public:
    159   MemProfiler(Module &M) {
    160     C = &(M.getContext());
    161     LongSize = M.getDataLayout().getPointerSizeInBits();
    162     IntptrTy = Type::getIntNTy(*C, LongSize);
    163   }
    164 
    165   /// If it is an interesting memory access, populate information
    166   /// about the access and return a InterestingMemoryAccess struct.
    167   /// Otherwise return None.
    168   Optional<InterestingMemoryAccess>
    169   isInterestingMemoryAccess(Instruction *I) const;
    170 
    171   void instrumentMop(Instruction *I, const DataLayout &DL,
    172                      InterestingMemoryAccess &Access);
    173   void instrumentAddress(Instruction *OrigIns, Instruction *InsertBefore,
    174                          Value *Addr, uint32_t TypeSize, bool IsWrite);
    175   void instrumentMaskedLoadOrStore(const DataLayout &DL, Value *Mask,
    176                                    Instruction *I, Value *Addr,
    177                                    unsigned Alignment, uint32_t TypeSize,
    178                                    bool IsWrite);
    179   void instrumentMemIntrinsic(MemIntrinsic *MI);
    180   Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);
    181   bool instrumentFunction(Function &F);
    182   bool maybeInsertMemProfInitAtFunctionEntry(Function &F);
    183   bool insertDynamicShadowAtFunctionEntry(Function &F);
    184 
    185 private:
    186   void initializeCallbacks(Module &M);
    187 
    188   LLVMContext *C;
    189   int LongSize;
    190   Type *IntptrTy;
    191   ShadowMapping Mapping;
    192 
    193   // These arrays is indexed by AccessIsWrite
    194   FunctionCallee MemProfMemoryAccessCallback[2];
    195   FunctionCallee MemProfMemoryAccessCallbackSized[2];
    196 
    197   FunctionCallee MemProfMemmove, MemProfMemcpy, MemProfMemset;
    198   Value *DynamicShadowOffset = nullptr;
    199 };
    200 
    201 class MemProfilerLegacyPass : public FunctionPass {
    202 public:
    203   static char ID;
    204 
    205   explicit MemProfilerLegacyPass() : FunctionPass(ID) {
    206     initializeMemProfilerLegacyPassPass(*PassRegistry::getPassRegistry());
    207   }
    208 
    209   StringRef getPassName() const override { return "MemProfilerFunctionPass"; }
    210 
    211   bool runOnFunction(Function &F) override {
    212     MemProfiler Profiler(*F.getParent());
    213     return Profiler.instrumentFunction(F);
    214   }
    215 };
    216 
    217 class ModuleMemProfiler {
    218 public:
    219   ModuleMemProfiler(Module &M) { TargetTriple = Triple(M.getTargetTriple()); }
    220 
    221   bool instrumentModule(Module &);
    222 
    223 private:
    224   Triple TargetTriple;
    225   ShadowMapping Mapping;
    226   Function *MemProfCtorFunction = nullptr;
    227 };
    228 
    229 class ModuleMemProfilerLegacyPass : public ModulePass {
    230 public:
    231   static char ID;
    232 
    233   explicit ModuleMemProfilerLegacyPass() : ModulePass(ID) {
    234     initializeModuleMemProfilerLegacyPassPass(*PassRegistry::getPassRegistry());
    235   }
    236 
    237   StringRef getPassName() const override { return "ModuleMemProfiler"; }
    238 
    239   void getAnalysisUsage(AnalysisUsage &AU) const override {}
    240 
    241   bool runOnModule(Module &M) override {
    242     ModuleMemProfiler MemProfiler(M);
    243     return MemProfiler.instrumentModule(M);
    244   }
    245 };
    246 
    247 } // end anonymous namespace
    248 
    249 MemProfilerPass::MemProfilerPass() {}
    250 
    251 PreservedAnalyses MemProfilerPass::run(Function &F,
    252                                        AnalysisManager<Function> &AM) {
    253   Module &M = *F.getParent();
    254   MemProfiler Profiler(M);
    255   if (Profiler.instrumentFunction(F))
    256     return PreservedAnalyses::none();
    257   return PreservedAnalyses::all();
    258 
    259   return PreservedAnalyses::all();
    260 }
    261 
    262 ModuleMemProfilerPass::ModuleMemProfilerPass() {}
    263 
    264 PreservedAnalyses ModuleMemProfilerPass::run(Module &M,
    265                                              AnalysisManager<Module> &AM) {
    266   ModuleMemProfiler Profiler(M);
    267   if (Profiler.instrumentModule(M))
    268     return PreservedAnalyses::none();
    269   return PreservedAnalyses::all();
    270 }
    271 
    272 char MemProfilerLegacyPass::ID = 0;
    273 
    274 INITIALIZE_PASS_BEGIN(MemProfilerLegacyPass, "memprof",
    275                       "MemProfiler: profile memory allocations and accesses.",
    276                       false, false)
    277 INITIALIZE_PASS_END(MemProfilerLegacyPass, "memprof",
    278                     "MemProfiler: profile memory allocations and accesses.",
    279                     false, false)
    280 
    281 FunctionPass *llvm::createMemProfilerFunctionPass() {
    282   return new MemProfilerLegacyPass();
    283 }
    284 
    285 char ModuleMemProfilerLegacyPass::ID = 0;
    286 
    287 INITIALIZE_PASS(ModuleMemProfilerLegacyPass, "memprof-module",
    288                 "MemProfiler: profile memory allocations and accesses."
    289                 "ModulePass",
    290                 false, false)
    291 
    292 ModulePass *llvm::createModuleMemProfilerLegacyPassPass() {
    293   return new ModuleMemProfilerLegacyPass();
    294 }
    295 
    296 Value *MemProfiler::memToShadow(Value *Shadow, IRBuilder<> &IRB) {
    297   // (Shadow & mask) >> scale
    298   Shadow = IRB.CreateAnd(Shadow, Mapping.Mask);
    299   Shadow = IRB.CreateLShr(Shadow, Mapping.Scale);
    300   // (Shadow >> scale) | offset
    301   assert(DynamicShadowOffset);
    302   return IRB.CreateAdd(Shadow, DynamicShadowOffset);
    303 }
    304 
    305 // Instrument memset/memmove/memcpy
    306 void MemProfiler::instrumentMemIntrinsic(MemIntrinsic *MI) {
    307   IRBuilder<> IRB(MI);
    308   if (isa<MemTransferInst>(MI)) {
    309     IRB.CreateCall(
    310         isa<MemMoveInst>(MI) ? MemProfMemmove : MemProfMemcpy,
    311         {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
    312          IRB.CreatePointerCast(MI->getOperand(1), IRB.getInt8PtrTy()),
    313          IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
    314   } else if (isa<MemSetInst>(MI)) {
    315     IRB.CreateCall(
    316         MemProfMemset,
    317         {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
    318          IRB.CreateIntCast(MI->getOperand(1), IRB.getInt32Ty(), false),
    319          IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
    320   }
    321   MI->eraseFromParent();
    322 }
    323 
    324 Optional<InterestingMemoryAccess>
    325 MemProfiler::isInterestingMemoryAccess(Instruction *I) const {
    326   // Do not instrument the load fetching the dynamic shadow address.
    327   if (DynamicShadowOffset == I)
    328     return None;
    329 
    330   InterestingMemoryAccess Access;
    331 
    332   const DataLayout &DL = I->getModule()->getDataLayout();
    333   if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
    334     if (!ClInstrumentReads)
    335       return None;
    336     Access.IsWrite = false;
    337     Access.TypeSize = DL.getTypeStoreSizeInBits(LI->getType());
    338     Access.Alignment = LI->getAlignment();
    339     Access.Addr = LI->getPointerOperand();
    340   } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
    341     if (!ClInstrumentWrites)
    342       return None;
    343     Access.IsWrite = true;
    344     Access.TypeSize =
    345         DL.getTypeStoreSizeInBits(SI->getValueOperand()->getType());
    346     Access.Alignment = SI->getAlignment();
    347     Access.Addr = SI->getPointerOperand();
    348   } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I)) {
    349     if (!ClInstrumentAtomics)
    350       return None;
    351     Access.IsWrite = true;
    352     Access.TypeSize =
    353         DL.getTypeStoreSizeInBits(RMW->getValOperand()->getType());
    354     Access.Alignment = 0;
    355     Access.Addr = RMW->getPointerOperand();
    356   } else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I)) {
    357     if (!ClInstrumentAtomics)
    358       return None;
    359     Access.IsWrite = true;
    360     Access.TypeSize =
    361         DL.getTypeStoreSizeInBits(XCHG->getCompareOperand()->getType());
    362     Access.Alignment = 0;
    363     Access.Addr = XCHG->getPointerOperand();
    364   } else if (auto *CI = dyn_cast<CallInst>(I)) {
    365     auto *F = CI->getCalledFunction();
    366     if (F && (F->getIntrinsicID() == Intrinsic::masked_load ||
    367               F->getIntrinsicID() == Intrinsic::masked_store)) {
    368       unsigned OpOffset = 0;
    369       if (F->getIntrinsicID() == Intrinsic::masked_store) {
    370         if (!ClInstrumentWrites)
    371           return None;
    372         // Masked store has an initial operand for the value.
    373         OpOffset = 1;
    374         Access.IsWrite = true;
    375       } else {
    376         if (!ClInstrumentReads)
    377           return None;
    378         Access.IsWrite = false;
    379       }
    380 
    381       auto *BasePtr = CI->getOperand(0 + OpOffset);
    382       auto *Ty = cast<PointerType>(BasePtr->getType())->getElementType();
    383       Access.TypeSize = DL.getTypeStoreSizeInBits(Ty);
    384       if (auto *AlignmentConstant =
    385               dyn_cast<ConstantInt>(CI->getOperand(1 + OpOffset)))
    386         Access.Alignment = (unsigned)AlignmentConstant->getZExtValue();
    387       else
    388         Access.Alignment = 1; // No alignment guarantees. We probably got Undef
    389       Access.MaybeMask = CI->getOperand(2 + OpOffset);
    390       Access.Addr = BasePtr;
    391     }
    392   }
    393 
    394   if (!Access.Addr)
    395     return None;
    396 
    397   // Do not instrument acesses from different address spaces; we cannot deal
    398   // with them.
    399   Type *PtrTy = cast<PointerType>(Access.Addr->getType()->getScalarType());
    400   if (PtrTy->getPointerAddressSpace() != 0)
    401     return None;
    402 
    403   // Ignore swifterror addresses.
    404   // swifterror memory addresses are mem2reg promoted by instruction
    405   // selection. As such they cannot have regular uses like an instrumentation
    406   // function and it makes no sense to track them as memory.
    407   if (Access.Addr->isSwiftError())
    408     return None;
    409 
    410   return Access;
    411 }
    412 
    413 void MemProfiler::instrumentMaskedLoadOrStore(const DataLayout &DL, Value *Mask,
    414                                               Instruction *I, Value *Addr,
    415                                               unsigned Alignment,
    416                                               uint32_t TypeSize, bool IsWrite) {
    417   auto *VTy = cast<FixedVectorType>(
    418       cast<PointerType>(Addr->getType())->getElementType());
    419   uint64_t ElemTypeSize = DL.getTypeStoreSizeInBits(VTy->getScalarType());
    420   unsigned Num = VTy->getNumElements();
    421   auto *Zero = ConstantInt::get(IntptrTy, 0);
    422   for (unsigned Idx = 0; Idx < Num; ++Idx) {
    423     Value *InstrumentedAddress = nullptr;
    424     Instruction *InsertBefore = I;
    425     if (auto *Vector = dyn_cast<ConstantVector>(Mask)) {
    426       // dyn_cast as we might get UndefValue
    427       if (auto *Masked = dyn_cast<ConstantInt>(Vector->getOperand(Idx))) {
    428         if (Masked->isZero())
    429           // Mask is constant false, so no instrumentation needed.
    430           continue;
    431         // If we have a true or undef value, fall through to instrumentAddress.
    432         // with InsertBefore == I
    433       }
    434     } else {
    435       IRBuilder<> IRB(I);
    436       Value *MaskElem = IRB.CreateExtractElement(Mask, Idx);
    437       Instruction *ThenTerm = SplitBlockAndInsertIfThen(MaskElem, I, false);
    438       InsertBefore = ThenTerm;
    439     }
    440 
    441     IRBuilder<> IRB(InsertBefore);
    442     InstrumentedAddress =
    443         IRB.CreateGEP(VTy, Addr, {Zero, ConstantInt::get(IntptrTy, Idx)});
    444     instrumentAddress(I, InsertBefore, InstrumentedAddress, ElemTypeSize,
    445                       IsWrite);
    446   }
    447 }
    448 
    449 void MemProfiler::instrumentMop(Instruction *I, const DataLayout &DL,
    450                                 InterestingMemoryAccess &Access) {
    451   if (Access.IsWrite)
    452     NumInstrumentedWrites++;
    453   else
    454     NumInstrumentedReads++;
    455 
    456   if (Access.MaybeMask) {
    457     instrumentMaskedLoadOrStore(DL, Access.MaybeMask, I, Access.Addr,
    458                                 Access.Alignment, Access.TypeSize,
    459                                 Access.IsWrite);
    460   } else {
    461     // Since the access counts will be accumulated across the entire allocation,
    462     // we only update the shadow access count for the first location and thus
    463     // don't need to worry about alignment and type size.
    464     instrumentAddress(I, I, Access.Addr, Access.TypeSize, Access.IsWrite);
    465   }
    466 }
    467 
    468 void MemProfiler::instrumentAddress(Instruction *OrigIns,
    469                                     Instruction *InsertBefore, Value *Addr,
    470                                     uint32_t TypeSize, bool IsWrite) {
    471   IRBuilder<> IRB(InsertBefore);
    472   Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy);
    473 
    474   if (ClUseCalls) {
    475     IRB.CreateCall(MemProfMemoryAccessCallback[IsWrite], AddrLong);
    476     return;
    477   }
    478 
    479   // Create an inline sequence to compute shadow location, and increment the
    480   // value by one.
    481   Type *ShadowTy = Type::getInt64Ty(*C);
    482   Type *ShadowPtrTy = PointerType::get(ShadowTy, 0);
    483   Value *ShadowPtr = memToShadow(AddrLong, IRB);
    484   Value *ShadowAddr = IRB.CreateIntToPtr(ShadowPtr, ShadowPtrTy);
    485   Value *ShadowValue = IRB.CreateLoad(ShadowTy, ShadowAddr);
    486   Value *Inc = ConstantInt::get(Type::getInt64Ty(*C), 1);
    487   ShadowValue = IRB.CreateAdd(ShadowValue, Inc);
    488   IRB.CreateStore(ShadowValue, ShadowAddr);
    489 }
    490 
    491 // Create the variable for the profile file name.
    492 void createProfileFileNameVar(Module &M) {
    493   const MDString *MemProfFilename =
    494       dyn_cast_or_null<MDString>(M.getModuleFlag("MemProfProfileFilename"));
    495   if (!MemProfFilename)
    496     return;
    497   assert(!MemProfFilename->getString().empty() &&
    498          "Unexpected MemProfProfileFilename metadata with empty string");
    499   Constant *ProfileNameConst = ConstantDataArray::getString(
    500       M.getContext(), MemProfFilename->getString(), true);
    501   GlobalVariable *ProfileNameVar = new GlobalVariable(
    502       M, ProfileNameConst->getType(), /*isConstant=*/true,
    503       GlobalValue::WeakAnyLinkage, ProfileNameConst, MemProfFilenameVar);
    504   Triple TT(M.getTargetTriple());
    505   if (TT.supportsCOMDAT()) {
    506     ProfileNameVar->setLinkage(GlobalValue::ExternalLinkage);
    507     ProfileNameVar->setComdat(M.getOrInsertComdat(MemProfFilenameVar));
    508   }
    509 }
    510 
    511 bool ModuleMemProfiler::instrumentModule(Module &M) {
    512   // Create a module constructor.
    513   std::string MemProfVersion = std::to_string(LLVM_MEM_PROFILER_VERSION);
    514   std::string VersionCheckName =
    515       ClInsertVersionCheck ? (MemProfVersionCheckNamePrefix + MemProfVersion)
    516                            : "";
    517   std::tie(MemProfCtorFunction, std::ignore) =
    518       createSanitizerCtorAndInitFunctions(M, MemProfModuleCtorName,
    519                                           MemProfInitName, /*InitArgTypes=*/{},
    520                                           /*InitArgs=*/{}, VersionCheckName);
    521 
    522   const uint64_t Priority = getCtorAndDtorPriority(TargetTriple);
    523   appendToGlobalCtors(M, MemProfCtorFunction, Priority);
    524 
    525   createProfileFileNameVar(M);
    526 
    527   return true;
    528 }
    529 
    530 void MemProfiler::initializeCallbacks(Module &M) {
    531   IRBuilder<> IRB(*C);
    532 
    533   for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
    534     const std::string TypeStr = AccessIsWrite ? "store" : "load";
    535 
    536     SmallVector<Type *, 3> Args2 = {IntptrTy, IntptrTy};
    537     SmallVector<Type *, 2> Args1{1, IntptrTy};
    538     MemProfMemoryAccessCallbackSized[AccessIsWrite] =
    539         M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + TypeStr + "N",
    540                               FunctionType::get(IRB.getVoidTy(), Args2, false));
    541 
    542     MemProfMemoryAccessCallback[AccessIsWrite] =
    543         M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + TypeStr,
    544                               FunctionType::get(IRB.getVoidTy(), Args1, false));
    545   }
    546   MemProfMemmove = M.getOrInsertFunction(
    547       ClMemoryAccessCallbackPrefix + "memmove", IRB.getInt8PtrTy(),
    548       IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy);
    549   MemProfMemcpy = M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + "memcpy",
    550                                         IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
    551                                         IRB.getInt8PtrTy(), IntptrTy);
    552   MemProfMemset = M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + "memset",
    553                                         IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
    554                                         IRB.getInt32Ty(), IntptrTy);
    555 }
    556 
    557 bool MemProfiler::maybeInsertMemProfInitAtFunctionEntry(Function &F) {
    558   // For each NSObject descendant having a +load method, this method is invoked
    559   // by the ObjC runtime before any of the static constructors is called.
    560   // Therefore we need to instrument such methods with a call to __memprof_init
    561   // at the beginning in order to initialize our runtime before any access to
    562   // the shadow memory.
    563   // We cannot just ignore these methods, because they may call other
    564   // instrumented functions.
    565   if (F.getName().find(" load]") != std::string::npos) {
    566     FunctionCallee MemProfInitFunction =
    567         declareSanitizerInitFunction(*F.getParent(), MemProfInitName, {});
    568     IRBuilder<> IRB(&F.front(), F.front().begin());
    569     IRB.CreateCall(MemProfInitFunction, {});
    570     return true;
    571   }
    572   return false;
    573 }
    574 
    575 bool MemProfiler::insertDynamicShadowAtFunctionEntry(Function &F) {
    576   IRBuilder<> IRB(&F.front().front());
    577   Value *GlobalDynamicAddress = F.getParent()->getOrInsertGlobal(
    578       MemProfShadowMemoryDynamicAddress, IntptrTy);
    579   if (F.getParent()->getPICLevel() == PICLevel::NotPIC)
    580     cast<GlobalVariable>(GlobalDynamicAddress)->setDSOLocal(true);
    581   DynamicShadowOffset = IRB.CreateLoad(IntptrTy, GlobalDynamicAddress);
    582   return true;
    583 }
    584 
    585 bool MemProfiler::instrumentFunction(Function &F) {
    586   if (F.getLinkage() == GlobalValue::AvailableExternallyLinkage)
    587     return false;
    588   if (ClDebugFunc == F.getName())
    589     return false;
    590   if (F.getName().startswith("__memprof_"))
    591     return false;
    592 
    593   bool FunctionModified = false;
    594 
    595   // If needed, insert __memprof_init.
    596   // This function needs to be called even if the function body is not
    597   // instrumented.
    598   if (maybeInsertMemProfInitAtFunctionEntry(F))
    599     FunctionModified = true;
    600 
    601   LLVM_DEBUG(dbgs() << "MEMPROF instrumenting:\n" << F << "\n");
    602 
    603   initializeCallbacks(*F.getParent());
    604 
    605   FunctionModified |= insertDynamicShadowAtFunctionEntry(F);
    606 
    607   SmallVector<Instruction *, 16> ToInstrument;
    608 
    609   // Fill the set of memory operations to instrument.
    610   for (auto &BB : F) {
    611     for (auto &Inst : BB) {
    612       if (isInterestingMemoryAccess(&Inst) || isa<MemIntrinsic>(Inst))
    613         ToInstrument.push_back(&Inst);
    614     }
    615   }
    616 
    617   int NumInstrumented = 0;
    618   for (auto *Inst : ToInstrument) {
    619     if (ClDebugMin < 0 || ClDebugMax < 0 ||
    620         (NumInstrumented >= ClDebugMin && NumInstrumented <= ClDebugMax)) {
    621       Optional<InterestingMemoryAccess> Access =
    622           isInterestingMemoryAccess(Inst);
    623       if (Access)
    624         instrumentMop(Inst, F.getParent()->getDataLayout(), *Access);
    625       else
    626         instrumentMemIntrinsic(cast<MemIntrinsic>(Inst));
    627     }
    628     NumInstrumented++;
    629   }
    630 
    631   if (NumInstrumented > 0)
    632     FunctionModified = true;
    633 
    634   LLVM_DEBUG(dbgs() << "MEMPROF done instrumenting: " << FunctionModified << " "
    635                     << F << "\n");
    636 
    637   return FunctionModified;
    638 }
    639