Home | History | Annotate | Line # | Download | only in Vectorize 
      1 //===-- VPlanTransforms.cpp - Utility VPlan to VPlan transforms -----------===//
      2 //
      3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
      4 // See https://llvm.org/LICENSE.txt for license information.
      5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
      6 //
      7 //===----------------------------------------------------------------------===//
      8 ///
      9 /// \file
     10 /// This file implements a set of utility VPlan to VPlan transformations.
     11 ///
     12 //===----------------------------------------------------------------------===//
     13 
     14 #include "VPlanTransforms.h"
     15 #include "llvm/ADT/PostOrderIterator.h"
     16 
     17 using namespace llvm;
     18 
     19 void VPlanTransforms::VPInstructionsToVPRecipes(
     20     Loop *OrigLoop, VPlanPtr &Plan,
     21     LoopVectorizationLegality::InductionList &Inductions,
     22     SmallPtrSetImpl<Instruction *> &DeadInstructions, ScalarEvolution &SE) {
     23 
     24   auto *TopRegion = cast<VPRegionBlock>(Plan->getEntry());
     25   ReversePostOrderTraversal<VPBlockBase *> RPOT(TopRegion->getEntry());
     26 
     27   for (VPBlockBase *Base : RPOT) {
     28     // Do not widen instructions in pre-header and exit blocks.
     29     if (Base->getNumPredecessors() == 0 || Base->getNumSuccessors() == 0)
     30       continue;
     31 
     32     VPBasicBlock *VPBB = Base->getEntryBasicBlock();
     33     // Introduce each ingredient into VPlan.
     34     for (auto I = VPBB->begin(), E = VPBB->end(); I != E;) {
     35       VPRecipeBase *Ingredient = &*I++;
     36       VPValue *VPV = Ingredient->getVPSingleValue();
     37       Instruction *Inst = cast<Instruction>(VPV->getUnderlyingValue());
     38       if (DeadInstructions.count(Inst)) {
     39         VPValue DummyValue;
     40         VPV->replaceAllUsesWith(&DummyValue);
     41         Ingredient->eraseFromParent();
     42         continue;
     43       }
     44 
     45       VPRecipeBase *NewRecipe = nullptr;
     46       if (auto *VPPhi = dyn_cast<VPWidenPHIRecipe>(Ingredient)) {
     47         auto *Phi = cast<PHINode>(VPPhi->getUnderlyingValue());
     48         InductionDescriptor II = Inductions.lookup(Phi);
     49         if (II.getKind() == InductionDescriptor::IK_IntInduction ||
     50             II.getKind() == InductionDescriptor::IK_FpInduction) {
     51           VPValue *Start = Plan->getOrAddVPValue(II.getStartValue());
     52           NewRecipe = new VPWidenIntOrFpInductionRecipe(Phi, Start, nullptr);
     53         } else {
     54           Plan->addVPValue(Phi, VPPhi);
     55           continue;
     56         }
     57       } else {
     58         assert(isa<VPInstruction>(Ingredient) &&
     59                "only VPInstructions expected here");
     60         assert(!isa<PHINode>(Inst) && "phis should be handled above");
     61         // Create VPWidenMemoryInstructionRecipe for loads and stores.
     62         if (LoadInst *Load = dyn_cast<LoadInst>(Inst)) {
     63           NewRecipe = new VPWidenMemoryInstructionRecipe(
     64               *Load, Plan->getOrAddVPValue(getLoadStorePointerOperand(Inst)),
     65               nullptr /*Mask*/);
     66         } else if (StoreInst *Store = dyn_cast<StoreInst>(Inst)) {
     67           NewRecipe = new VPWidenMemoryInstructionRecipe(
     68               *Store, Plan->getOrAddVPValue(getLoadStorePointerOperand(Inst)),
     69               Plan->getOrAddVPValue(Store->getValueOperand()),
     70               nullptr /*Mask*/);
     71         } else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Inst)) {
     72           NewRecipe = new VPWidenGEPRecipe(
     73               GEP, Plan->mapToVPValues(GEP->operands()), OrigLoop);
     74         } else if (CallInst *CI = dyn_cast<CallInst>(Inst)) {
     75           NewRecipe = new VPWidenCallRecipe(
     76               *CI, Plan->mapToVPValues(CI->arg_operands()));
     77         } else if (SelectInst *SI = dyn_cast<SelectInst>(Inst)) {
     78           bool InvariantCond =
     79               SE.isLoopInvariant(SE.getSCEV(SI->getOperand(0)), OrigLoop);
     80           NewRecipe = new VPWidenSelectRecipe(
     81               *SI, Plan->mapToVPValues(SI->operands()), InvariantCond);
     82         } else {
     83           NewRecipe =
     84               new VPWidenRecipe(*Inst, Plan->mapToVPValues(Inst->operands()));
     85         }
     86       }
     87 
     88       NewRecipe->insertBefore(Ingredient);
     89       if (NewRecipe->getNumDefinedValues() == 1)
     90         VPV->replaceAllUsesWith(NewRecipe->getVPSingleValue());
     91       else
     92         assert(NewRecipe->getNumDefinedValues() == 0 &&
     93                "Only recpies with zero or one defined values expected");
     94       Ingredient->eraseFromParent();
     95       Plan->removeVPValueFor(Inst);
     96       for (auto *Def : NewRecipe->definedValues()) {
     97         Plan->addVPValue(Inst, Def);
     98       }
     99     }
    100   }
    101 }
    102