Home | History | Annotate | Line # | Download | only in CodeGen 
      1 //===-- MachineFunctionSplitter.cpp - Split machine functions //-----------===//
      2 //
      3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
      4 // See https://llvm.org/LICENSE.txt for license information.
      5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
      6 //
      7 //===----------------------------------------------------------------------===//
      8 //
      9 // \file
     10 // Uses profile information to split out cold blocks.
     11 //
     12 // This pass splits out cold machine basic blocks from the parent function. This
     13 // implementation leverages the basic block section framework. Blocks marked
     14 // cold by this pass are grouped together in a separate section prefixed with
     15 // ".text.unlikely.*". The linker can then group these together as a cold
     16 // section. The split part of the function is a contiguous region identified by
     17 // the symbol "foo.cold". Grouping all cold blocks across functions together
     18 // decreases fragmentation and improves icache and itlb utilization. Note that
     19 // the overall changes to the binary size are negligible; only a small number of
     20 // additional jump instructions may be introduced.
     21 //
     22 // For the original RFC of this pass please see
     23 // https://groups.google.com/d/msg/llvm-dev/RUegaMg-iqc/wFAVxa6fCgAJ
     24 //===----------------------------------------------------------------------===//
     25 
     26 #include "llvm/ADT/SmallVector.h"
     27 #include "llvm/ADT/Statistic.h"
     28 #include "llvm/Analysis/ProfileSummaryInfo.h"
     29 #include "llvm/CodeGen/BasicBlockSectionUtils.h"
     30 #include "llvm/CodeGen/MachineBasicBlock.h"
     31 #include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
     32 #include "llvm/CodeGen/MachineFunction.h"
     33 #include "llvm/CodeGen/MachineFunctionPass.h"
     34 #include "llvm/CodeGen/MachineModuleInfo.h"
     35 #include "llvm/CodeGen/Passes.h"
     36 #include "llvm/IR/Function.h"
     37 #include "llvm/IR/Module.h"
     38 #include "llvm/InitializePasses.h"
     39 #include "llvm/Support/CommandLine.h"
     40 
     41 using namespace llvm;
     42 
     43 // FIXME: This cutoff value is CPU dependent and should be moved to
     44 // TargetTransformInfo once we consider enabling this on other platforms.
     45 // The value is expressed as a ProfileSummaryInfo integer percentile cutoff.
     46 // Defaults to 999950, i.e. all blocks colder than 99.995 percentile are split.
     47 // The default was empirically determined to be optimal when considering cutoff
     48 // values between 99%-ile to 100%-ile with respect to iTLB and icache metrics on
     49 // Intel CPUs.
     50 static cl::opt<unsigned>
     51     PercentileCutoff("mfs-psi-cutoff",
     52                      cl::desc("Percentile profile summary cutoff used to "
     53                               "determine cold blocks. Unused if set to zero."),
     54                      cl::init(999950), cl::Hidden);
     55 
     56 static cl::opt<unsigned> ColdCountThreshold(
     57     "mfs-count-threshold",
     58     cl::desc(
     59         "Minimum number of times a block must be executed to be retained."),
     60     cl::init(1), cl::Hidden);
     61 
     62 namespace {
     63 
     64 class MachineFunctionSplitter : public MachineFunctionPass {
     65 public:
     66   static char ID;
     67   MachineFunctionSplitter() : MachineFunctionPass(ID) {
     68     initializeMachineFunctionSplitterPass(*PassRegistry::getPassRegistry());
     69   }
     70 
     71   StringRef getPassName() const override {
     72     return "Machine Function Splitter Transformation";
     73   }
     74 
     75   void getAnalysisUsage(AnalysisUsage &AU) const override;
     76 
     77   bool runOnMachineFunction(MachineFunction &F) override;
     78 };
     79 } // end anonymous namespace
     80 
     81 static bool isColdBlock(const MachineBasicBlock &MBB,
     82                         const MachineBlockFrequencyInfo *MBFI,
     83                         ProfileSummaryInfo *PSI) {
     84   Optional<uint64_t> Count = MBFI->getBlockProfileCount(&MBB);
     85   if (!Count.hasValue())
     86     return true;
     87 
     88   if (PercentileCutoff > 0) {
     89     return PSI->isColdCountNthPercentile(PercentileCutoff, *Count);
     90   }
     91   return (*Count < ColdCountThreshold);
     92 }
     93 
     94 bool MachineFunctionSplitter::runOnMachineFunction(MachineFunction &MF) {
     95   // TODO: We only target functions with profile data. Static information may
     96   // also be considered but we don't see performance improvements yet.
     97   if (!MF.getFunction().hasProfileData())
     98     return false;
     99 
    100   // TODO: We don't split functions where a section attribute has been set
    101   // since the split part may not be placed in a contiguous region. It may also
    102   // be more beneficial to augment the linker to ensure contiguous layout of
    103   // split functions within the same section as specified by the attribute.
    104   if (MF.getFunction().hasSection() ||
    105       MF.getFunction().hasFnAttribute("implicit-section-name"))
    106     return false;
    107 
    108   // We don't want to proceed further for cold functions
    109   // or functions of unknown hotness. Lukewarm functions have no prefix.
    110   Optional<StringRef> SectionPrefix = MF.getFunction().getSectionPrefix();
    111   if (SectionPrefix.hasValue() &&
    112       (SectionPrefix.getValue().equals("unlikely") ||
    113        SectionPrefix.getValue().equals("unknown"))) {
    114     return false;
    115   }
    116 
    117   // Renumbering blocks here preserves the order of the blocks as
    118   // sortBasicBlocksAndUpdateBranches uses the numeric identifier to sort
    119   // blocks. Preserving the order of blocks is essential to retaining decisions
    120   // made by prior passes such as MachineBlockPlacement.
    121   MF.RenumberBlocks();
    122   MF.setBBSectionsType(BasicBlockSection::Preset);
    123   auto *MBFI = &getAnalysis<MachineBlockFrequencyInfo>();
    124   auto *PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
    125 
    126   SmallVector<MachineBasicBlock *, 2> LandingPads;
    127   for (auto &MBB : MF) {
    128     if (MBB.isEntryBlock())
    129       continue;
    130 
    131     if (MBB.isEHPad())
    132       LandingPads.push_back(&MBB);
    133     else if (isColdBlock(MBB, MBFI, PSI))
    134       MBB.setSectionID(MBBSectionID::ColdSectionID);
    135   }
    136 
    137   // We only split out eh pads if all of them are cold.
    138   bool HasHotLandingPads = false;
    139   for (const MachineBasicBlock *LP : LandingPads) {
    140     if (!isColdBlock(*LP, MBFI, PSI))
    141       HasHotLandingPads = true;
    142   }
    143   if (!HasHotLandingPads) {
    144     for (MachineBasicBlock *LP : LandingPads)
    145       LP->setSectionID(MBBSectionID::ColdSectionID);
    146   }
    147 
    148   auto Comparator = [](const MachineBasicBlock &X, const MachineBasicBlock &Y) {
    149     return X.getSectionID().Type < Y.getSectionID().Type;
    150   };
    151   llvm::sortBasicBlocksAndUpdateBranches(MF, Comparator);
    152 
    153   return true;
    154 }
    155 
    156 void MachineFunctionSplitter::getAnalysisUsage(AnalysisUsage &AU) const {
    157   AU.addRequired<MachineModuleInfoWrapperPass>();
    158   AU.addRequired<MachineBlockFrequencyInfo>();
    159   AU.addRequired<ProfileSummaryInfoWrapperPass>();
    160 }
    161 
    162 char MachineFunctionSplitter::ID = 0;
    163 INITIALIZE_PASS(MachineFunctionSplitter, "machine-function-splitter",
    164                 "Split machine functions using profile information", false,
    165                 false)
    166 
    167 MachineFunctionPass *llvm::createMachineFunctionSplitterPass() {
    168   return new MachineFunctionSplitter();
    169 }
    170