Home | History | Annotate | Line # | Download | only in CodeGen
CGLoopInfo.cpp revision 1.1.1.1.4.1 
      1 //===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===//
      2 //
      3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
      4 // See https://llvm.org/LICENSE.txt for license information.
      5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
      6 //
      7 //===----------------------------------------------------------------------===//
      8 
      9 #include "CGLoopInfo.h"
     10 #include "clang/AST/ASTContext.h"
     11 #include "clang/AST/Attr.h"
     12 #include "clang/AST/Expr.h"
     13 #include "clang/Basic/CodeGenOptions.h"
     14 #include "llvm/IR/BasicBlock.h"
     15 #include "llvm/IR/CFG.h"
     16 #include "llvm/IR/Constants.h"
     17 #include "llvm/IR/InstrTypes.h"
     18 #include "llvm/IR/Instructions.h"
     19 #include "llvm/IR/Metadata.h"
     20 using namespace clang::CodeGen;
     21 using namespace llvm;
     22 
     23 MDNode *
     24 LoopInfo::createLoopPropertiesMetadata(ArrayRef<Metadata *> LoopProperties) {
     25   LLVMContext &Ctx = Header->getContext();
     26   SmallVector<Metadata *, 4> NewLoopProperties;
     27   NewLoopProperties.push_back(nullptr);
     28   NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
     29 
     30   MDNode *LoopID = MDNode::getDistinct(Ctx, NewLoopProperties);
     31   LoopID->replaceOperandWith(0, LoopID);
     32   return LoopID;
     33 }
     34 
     35 MDNode *LoopInfo::createPipeliningMetadata(const LoopAttributes &Attrs,
     36                                            ArrayRef<Metadata *> LoopProperties,
     37                                            bool &HasUserTransforms) {
     38   LLVMContext &Ctx = Header->getContext();
     39 
     40   Optional<bool> Enabled;
     41   if (Attrs.PipelineDisabled)
     42     Enabled = false;
     43   else if (Attrs.PipelineInitiationInterval != 0)
     44     Enabled = true;
     45 
     46   if (Enabled != true) {
     47     SmallVector<Metadata *, 4> NewLoopProperties;
     48     if (Enabled == false) {
     49       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
     50       NewLoopProperties.push_back(
     51           MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.pipeline.disable"),
     52                             ConstantAsMetadata::get(ConstantInt::get(
     53                                 llvm::Type::getInt1Ty(Ctx), 1))}));
     54       LoopProperties = NewLoopProperties;
     55     }
     56     return createLoopPropertiesMetadata(LoopProperties);
     57   }
     58 
     59   SmallVector<Metadata *, 4> Args;
     60   Args.push_back(nullptr);
     61   Args.append(LoopProperties.begin(), LoopProperties.end());
     62 
     63   if (Attrs.PipelineInitiationInterval > 0) {
     64     Metadata *Vals[] = {
     65         MDString::get(Ctx, "llvm.loop.pipeline.initiationinterval"),
     66         ConstantAsMetadata::get(ConstantInt::get(
     67             llvm::Type::getInt32Ty(Ctx), Attrs.PipelineInitiationInterval))};
     68     Args.push_back(MDNode::get(Ctx, Vals));
     69   }
     70 
     71   // No follow-up: This is the last transformation.
     72 
     73   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
     74   LoopID->replaceOperandWith(0, LoopID);
     75   HasUserTransforms = true;
     76   return LoopID;
     77 }
     78 
     79 MDNode *
     80 LoopInfo::createPartialUnrollMetadata(const LoopAttributes &Attrs,
     81                                       ArrayRef<Metadata *> LoopProperties,
     82                                       bool &HasUserTransforms) {
     83   LLVMContext &Ctx = Header->getContext();
     84 
     85   Optional<bool> Enabled;
     86   if (Attrs.UnrollEnable == LoopAttributes::Disable)
     87     Enabled = false;
     88   else if (Attrs.UnrollEnable == LoopAttributes::Full)
     89     Enabled = None;
     90   else if (Attrs.UnrollEnable != LoopAttributes::Unspecified ||
     91            Attrs.UnrollCount != 0)
     92     Enabled = true;
     93 
     94   if (Enabled != true) {
     95     // createFullUnrollMetadata will already have added llvm.loop.unroll.disable
     96     // if unrolling is disabled.
     97     return createPipeliningMetadata(Attrs, LoopProperties, HasUserTransforms);
     98   }
     99 
    100   SmallVector<Metadata *, 4> FollowupLoopProperties;
    101 
    102   // Apply all loop properties to the unrolled loop.
    103   FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
    104 
    105   // Don't unroll an already unrolled loop.
    106   FollowupLoopProperties.push_back(
    107       MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
    108 
    109   bool FollowupHasTransforms = false;
    110   MDNode *Followup = createPipeliningMetadata(Attrs, FollowupLoopProperties,
    111                                               FollowupHasTransforms);
    112 
    113   SmallVector<Metadata *, 4> Args;
    114   Args.push_back(nullptr);
    115   Args.append(LoopProperties.begin(), LoopProperties.end());
    116 
    117   // Setting unroll.count
    118   if (Attrs.UnrollCount > 0) {
    119     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"),
    120                         ConstantAsMetadata::get(ConstantInt::get(
    121                             llvm::Type::getInt32Ty(Ctx), Attrs.UnrollCount))};
    122     Args.push_back(MDNode::get(Ctx, Vals));
    123   }
    124 
    125   // Setting unroll.full or unroll.disable
    126   if (Attrs.UnrollEnable == LoopAttributes::Enable) {
    127     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.enable")};
    128     Args.push_back(MDNode::get(Ctx, Vals));
    129   }
    130 
    131   if (FollowupHasTransforms)
    132     Args.push_back(MDNode::get(
    133         Ctx, {MDString::get(Ctx, "llvm.loop.unroll.followup_all"), Followup}));
    134 
    135   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
    136   LoopID->replaceOperandWith(0, LoopID);
    137   HasUserTransforms = true;
    138   return LoopID;
    139 }
    140 
    141 MDNode *
    142 LoopInfo::createUnrollAndJamMetadata(const LoopAttributes &Attrs,
    143                                      ArrayRef<Metadata *> LoopProperties,
    144                                      bool &HasUserTransforms) {
    145   LLVMContext &Ctx = Header->getContext();
    146 
    147   Optional<bool> Enabled;
    148   if (Attrs.UnrollAndJamEnable == LoopAttributes::Disable)
    149     Enabled = false;
    150   else if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable ||
    151            Attrs.UnrollAndJamCount != 0)
    152     Enabled = true;
    153 
    154   if (Enabled != true) {
    155     SmallVector<Metadata *, 4> NewLoopProperties;
    156     if (Enabled == false) {
    157       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
    158       NewLoopProperties.push_back(MDNode::get(
    159           Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
    160       LoopProperties = NewLoopProperties;
    161     }
    162     return createPartialUnrollMetadata(Attrs, LoopProperties,
    163                                        HasUserTransforms);
    164   }
    165 
    166   SmallVector<Metadata *, 4> FollowupLoopProperties;
    167   FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
    168   FollowupLoopProperties.push_back(
    169       MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
    170 
    171   bool FollowupHasTransforms = false;
    172   MDNode *Followup = createPartialUnrollMetadata(Attrs, FollowupLoopProperties,
    173                                                  FollowupHasTransforms);
    174 
    175   SmallVector<Metadata *, 4> Args;
    176   Args.push_back(nullptr);
    177   Args.append(LoopProperties.begin(), LoopProperties.end());
    178 
    179   // Setting unroll_and_jam.count
    180   if (Attrs.UnrollAndJamCount > 0) {
    181     Metadata *Vals[] = {
    182         MDString::get(Ctx, "llvm.loop.unroll_and_jam.count"),
    183         ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
    184                                                  Attrs.UnrollAndJamCount))};
    185     Args.push_back(MDNode::get(Ctx, Vals));
    186   }
    187 
    188   if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable) {
    189     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll_and_jam.enable")};
    190     Args.push_back(MDNode::get(Ctx, Vals));
    191   }
    192 
    193   if (FollowupHasTransforms)
    194     Args.push_back(MDNode::get(
    195         Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_outer"),
    196               Followup}));
    197 
    198   if (UnrollAndJamInnerFollowup)
    199     Args.push_back(MDNode::get(
    200         Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_inner"),
    201               UnrollAndJamInnerFollowup}));
    202 
    203   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
    204   LoopID->replaceOperandWith(0, LoopID);
    205   HasUserTransforms = true;
    206   return LoopID;
    207 }
    208 
    209 MDNode *
    210 LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs,
    211                                       ArrayRef<Metadata *> LoopProperties,
    212                                       bool &HasUserTransforms) {
    213   LLVMContext &Ctx = Header->getContext();
    214 
    215   Optional<bool> Enabled;
    216   if (Attrs.VectorizeEnable == LoopAttributes::Disable)
    217     Enabled = false;
    218   else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
    219            Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified ||
    220            Attrs.InterleaveCount != 0 || Attrs.VectorizeWidth != 0 ||
    221            Attrs.VectorizeScalable != LoopAttributes::Unspecified)
    222     Enabled = true;
    223 
    224   if (Enabled != true) {
    225     SmallVector<Metadata *, 4> NewLoopProperties;
    226     if (Enabled == false) {
    227       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
    228       NewLoopProperties.push_back(
    229           MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
    230                             ConstantAsMetadata::get(ConstantInt::get(
    231                                 llvm::Type::getInt1Ty(Ctx), 0))}));
    232       LoopProperties = NewLoopProperties;
    233     }
    234     return createUnrollAndJamMetadata(Attrs, LoopProperties, HasUserTransforms);
    235   }
    236 
    237   // Apply all loop properties to the vectorized loop.
    238   SmallVector<Metadata *, 4> FollowupLoopProperties;
    239   FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
    240 
    241   // Don't vectorize an already vectorized loop.
    242   FollowupLoopProperties.push_back(
    243       MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
    244 
    245   bool FollowupHasTransforms = false;
    246   MDNode *Followup = createUnrollAndJamMetadata(Attrs, FollowupLoopProperties,
    247                                                 FollowupHasTransforms);
    248 
    249   SmallVector<Metadata *, 4> Args;
    250   Args.push_back(nullptr);
    251   Args.append(LoopProperties.begin(), LoopProperties.end());
    252 
    253   // Setting vectorize.predicate when it has been specified and vectorization
    254   // has not been disabled.
    255   bool IsVectorPredicateEnabled = false;
    256   if (Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified) {
    257     IsVectorPredicateEnabled =
    258         (Attrs.VectorizePredicateEnable == LoopAttributes::Enable);
    259 
    260     Metadata *Vals[] = {
    261         MDString::get(Ctx, "llvm.loop.vectorize.predicate.enable"),
    262         ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt1Ty(Ctx),
    263                                                  IsVectorPredicateEnabled))};
    264     Args.push_back(MDNode::get(Ctx, Vals));
    265   }
    266 
    267   // Setting vectorize.width
    268   if (Attrs.VectorizeWidth > 0) {
    269     Metadata *Vals[] = {
    270         MDString::get(Ctx, "llvm.loop.vectorize.width"),
    271         ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
    272                                                  Attrs.VectorizeWidth))};
    273 
    274     Args.push_back(MDNode::get(Ctx, Vals));
    275   }
    276 
    277   if (Attrs.VectorizeScalable != LoopAttributes::Unspecified) {
    278     bool IsScalable = Attrs.VectorizeScalable == LoopAttributes::Enable;
    279     Metadata *Vals[] = {
    280         MDString::get(Ctx, "llvm.loop.vectorize.scalable.enable"),
    281         ConstantAsMetadata::get(
    282             ConstantInt::get(llvm::Type::getInt1Ty(Ctx), IsScalable))};
    283     Args.push_back(MDNode::get(Ctx, Vals));
    284   }
    285 
    286   // Setting interleave.count
    287   if (Attrs.InterleaveCount > 0) {
    288     Metadata *Vals[] = {
    289         MDString::get(Ctx, "llvm.loop.interleave.count"),
    290         ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
    291                                                  Attrs.InterleaveCount))};
    292     Args.push_back(MDNode::get(Ctx, Vals));
    293   }
    294 
    295   // vectorize.enable is set if:
    296   // 1) loop hint vectorize.enable is set, or
    297   // 2) it is implied when vectorize.predicate is set, or
    298   // 3) it is implied when vectorize.width is set to a value > 1
    299   // 4) it is implied when vectorize.scalable.enable is true
    300   // 5) it is implied when vectorize.width is unset (0) and the user
    301   //    explicitly requested fixed-width vectorization, i.e.
    302   //    vectorize.scalable.enable is false.
    303   if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
    304       (IsVectorPredicateEnabled && Attrs.VectorizeWidth != 1) ||
    305       Attrs.VectorizeWidth > 1 ||
    306       Attrs.VectorizeScalable == LoopAttributes::Enable ||
    307       (Attrs.VectorizeScalable == LoopAttributes::Disable &&
    308        Attrs.VectorizeWidth != 1)) {
    309     bool AttrVal = Attrs.VectorizeEnable != LoopAttributes::Disable;
    310     Args.push_back(
    311         MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
    312                           ConstantAsMetadata::get(ConstantInt::get(
    313                               llvm::Type::getInt1Ty(Ctx), AttrVal))}));
    314   }
    315 
    316   if (FollowupHasTransforms)
    317     Args.push_back(MDNode::get(
    318         Ctx,
    319         {MDString::get(Ctx, "llvm.loop.vectorize.followup_all"), Followup}));
    320 
    321   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
    322   LoopID->replaceOperandWith(0, LoopID);
    323   HasUserTransforms = true;
    324   return LoopID;
    325 }
    326 
    327 MDNode *
    328 LoopInfo::createLoopDistributeMetadata(const LoopAttributes &Attrs,
    329                                        ArrayRef<Metadata *> LoopProperties,
    330                                        bool &HasUserTransforms) {
    331   LLVMContext &Ctx = Header->getContext();
    332 
    333   Optional<bool> Enabled;
    334   if (Attrs.DistributeEnable == LoopAttributes::Disable)
    335     Enabled = false;
    336   if (Attrs.DistributeEnable == LoopAttributes::Enable)
    337     Enabled = true;
    338 
    339   if (Enabled != true) {
    340     SmallVector<Metadata *, 4> NewLoopProperties;
    341     if (Enabled == false) {
    342       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
    343       NewLoopProperties.push_back(
    344           MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.distribute.enable"),
    345                             ConstantAsMetadata::get(ConstantInt::get(
    346                                 llvm::Type::getInt1Ty(Ctx), 0))}));
    347       LoopProperties = NewLoopProperties;
    348     }
    349     return createLoopVectorizeMetadata(Attrs, LoopProperties,
    350                                        HasUserTransforms);
    351   }
    352 
    353   bool FollowupHasTransforms = false;
    354   MDNode *Followup =
    355       createLoopVectorizeMetadata(Attrs, LoopProperties, FollowupHasTransforms);
    356 
    357   SmallVector<Metadata *, 4> Args;
    358   Args.push_back(nullptr);
    359   Args.append(LoopProperties.begin(), LoopProperties.end());
    360 
    361   Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"),
    362                       ConstantAsMetadata::get(ConstantInt::get(
    363                           llvm::Type::getInt1Ty(Ctx),
    364                           (Attrs.DistributeEnable == LoopAttributes::Enable)))};
    365   Args.push_back(MDNode::get(Ctx, Vals));
    366 
    367   if (FollowupHasTransforms)
    368     Args.push_back(MDNode::get(
    369         Ctx,
    370         {MDString::get(Ctx, "llvm.loop.distribute.followup_all"), Followup}));
    371 
    372   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
    373   LoopID->replaceOperandWith(0, LoopID);
    374   HasUserTransforms = true;
    375   return LoopID;
    376 }
    377 
    378 MDNode *LoopInfo::createFullUnrollMetadata(const LoopAttributes &Attrs,
    379                                            ArrayRef<Metadata *> LoopProperties,
    380                                            bool &HasUserTransforms) {
    381   LLVMContext &Ctx = Header->getContext();
    382 
    383   Optional<bool> Enabled;
    384   if (Attrs.UnrollEnable == LoopAttributes::Disable)
    385     Enabled = false;
    386   else if (Attrs.UnrollEnable == LoopAttributes::Full)
    387     Enabled = true;
    388 
    389   if (Enabled != true) {
    390     SmallVector<Metadata *, 4> NewLoopProperties;
    391     if (Enabled == false) {
    392       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
    393       NewLoopProperties.push_back(
    394           MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
    395       LoopProperties = NewLoopProperties;
    396     }
    397     return createLoopDistributeMetadata(Attrs, LoopProperties,
    398                                         HasUserTransforms);
    399   }
    400 
    401   SmallVector<Metadata *, 4> Args;
    402   Args.push_back(nullptr);
    403   Args.append(LoopProperties.begin(), LoopProperties.end());
    404   Args.push_back(MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.full")));
    405 
    406   // No follow-up: there is no loop after full unrolling.
    407   // TODO: Warn if there are transformations after full unrolling.
    408 
    409   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
    410   LoopID->replaceOperandWith(0, LoopID);
    411   HasUserTransforms = true;
    412   return LoopID;
    413 }
    414 
    415 MDNode *LoopInfo::createMetadata(
    416     const LoopAttributes &Attrs,
    417     llvm::ArrayRef<llvm::Metadata *> AdditionalLoopProperties,
    418     bool &HasUserTransforms) {
    419   SmallVector<Metadata *, 3> LoopProperties;
    420 
    421   // If we have a valid start debug location for the loop, add it.
    422   if (StartLoc) {
    423     LoopProperties.push_back(StartLoc.getAsMDNode());
    424 
    425     // If we also have a valid end debug location for the loop, add it.
    426     if (EndLoc)
    427       LoopProperties.push_back(EndLoc.getAsMDNode());
    428   }
    429 
    430   LLVMContext &Ctx = Header->getContext();
    431   if (Attrs.MustProgress)
    432     LoopProperties.push_back(
    433         MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.mustprogress")));
    434 
    435   assert(!!AccGroup == Attrs.IsParallel &&
    436          "There must be an access group iff the loop is parallel");
    437   if (Attrs.IsParallel) {
    438     LoopProperties.push_back(MDNode::get(
    439         Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccGroup}));
    440   }
    441 
    442   LoopProperties.insert(LoopProperties.end(), AdditionalLoopProperties.begin(),
    443                         AdditionalLoopProperties.end());
    444   return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms);
    445 }
    446 
    447 LoopAttributes::LoopAttributes(bool IsParallel)
    448     : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),
    449       UnrollEnable(LoopAttributes::Unspecified),
    450       UnrollAndJamEnable(LoopAttributes::Unspecified),
    451       VectorizePredicateEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
    452       VectorizeScalable(LoopAttributes::Unspecified), InterleaveCount(0),
    453       UnrollCount(0), UnrollAndJamCount(0),
    454       DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false),
    455       PipelineInitiationInterval(0), MustProgress(false) {}
    456 
    457 void LoopAttributes::clear() {
    458   IsParallel = false;
    459   VectorizeWidth = 0;
    460   VectorizeScalable = LoopAttributes::Unspecified;
    461   InterleaveCount = 0;
    462   UnrollCount = 0;
    463   UnrollAndJamCount = 0;
    464   VectorizeEnable = LoopAttributes::Unspecified;
    465   UnrollEnable = LoopAttributes::Unspecified;
    466   UnrollAndJamEnable = LoopAttributes::Unspecified;
    467   VectorizePredicateEnable = LoopAttributes::Unspecified;
    468   DistributeEnable = LoopAttributes::Unspecified;
    469   PipelineDisabled = false;
    470   PipelineInitiationInterval = 0;
    471   MustProgress = false;
    472 }
    473 
    474 LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,
    475                    const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc,
    476                    LoopInfo *Parent)
    477     : Header(Header), Attrs(Attrs), StartLoc(StartLoc), EndLoc(EndLoc),
    478       Parent(Parent) {
    479 
    480   if (Attrs.IsParallel) {
    481     // Create an access group for this loop.
    482     LLVMContext &Ctx = Header->getContext();
    483     AccGroup = MDNode::getDistinct(Ctx, {});
    484   }
    485 
    486   if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
    487       Attrs.VectorizeScalable == LoopAttributes::Unspecified &&
    488       Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
    489       Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled &&
    490       Attrs.PipelineInitiationInterval == 0 &&
    491       Attrs.VectorizePredicateEnable == LoopAttributes::Unspecified &&
    492       Attrs.VectorizeEnable == LoopAttributes::Unspecified &&
    493       Attrs.UnrollEnable == LoopAttributes::Unspecified &&
    494       Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified &&
    495       Attrs.DistributeEnable == LoopAttributes::Unspecified && !StartLoc &&
    496       !EndLoc && !Attrs.MustProgress)
    497     return;
    498 
    499   TempLoopID = MDNode::getTemporary(Header->getContext(), None);
    500 }
    501 
    502 void LoopInfo::finish() {
    503   // We did not annotate the loop body instructions because there are no
    504   // attributes for this loop.
    505   if (!TempLoopID)
    506     return;
    507 
    508   MDNode *LoopID;
    509   LoopAttributes CurLoopAttr = Attrs;
    510   LLVMContext &Ctx = Header->getContext();
    511 
    512   if (Parent && (Parent->Attrs.UnrollAndJamEnable ||
    513                  Parent->Attrs.UnrollAndJamCount != 0)) {
    514     // Parent unroll-and-jams this loop.
    515     // Split the transformations in those that happens before the unroll-and-jam
    516     // and those after.
    517 
    518     LoopAttributes BeforeJam, AfterJam;
    519 
    520     BeforeJam.IsParallel = AfterJam.IsParallel = Attrs.IsParallel;
    521 
    522     BeforeJam.VectorizeWidth = Attrs.VectorizeWidth;
    523     BeforeJam.VectorizeScalable = Attrs.VectorizeScalable;
    524     BeforeJam.InterleaveCount = Attrs.InterleaveCount;
    525     BeforeJam.VectorizeEnable = Attrs.VectorizeEnable;
    526     BeforeJam.DistributeEnable = Attrs.DistributeEnable;
    527     BeforeJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
    528 
    529     switch (Attrs.UnrollEnable) {
    530     case LoopAttributes::Unspecified:
    531     case LoopAttributes::Disable:
    532       BeforeJam.UnrollEnable = Attrs.UnrollEnable;
    533       AfterJam.UnrollEnable = Attrs.UnrollEnable;
    534       break;
    535     case LoopAttributes::Full:
    536       BeforeJam.UnrollEnable = LoopAttributes::Full;
    537       break;
    538     case LoopAttributes::Enable:
    539       AfterJam.UnrollEnable = LoopAttributes::Enable;
    540       break;
    541     }
    542 
    543     AfterJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
    544     AfterJam.UnrollCount = Attrs.UnrollCount;
    545     AfterJam.PipelineDisabled = Attrs.PipelineDisabled;
    546     AfterJam.PipelineInitiationInterval = Attrs.PipelineInitiationInterval;
    547 
    548     // If this loop is subject of an unroll-and-jam by the parent loop, and has
    549     // an unroll-and-jam annotation itself, we have to decide whether to first
    550     // apply the parent's unroll-and-jam or this loop's unroll-and-jam. The
    551     // UnrollAndJam pass processes loops from inner to outer, so we apply the
    552     // inner first.
    553     BeforeJam.UnrollAndJamCount = Attrs.UnrollAndJamCount;
    554     BeforeJam.UnrollAndJamEnable = Attrs.UnrollAndJamEnable;
    555 
    556     // Set the inner followup metadata to process by the outer loop. Only
    557     // consider the first inner loop.
    558     if (!Parent->UnrollAndJamInnerFollowup) {
    559       // Splitting the attributes into a BeforeJam and an AfterJam part will
    560       // stop 'llvm.loop.isvectorized' (generated by vectorization in BeforeJam)
    561       // to be forwarded to the AfterJam part. We detect the situation here and
    562       // add it manually.
    563       SmallVector<Metadata *, 1> BeforeLoopProperties;
    564       if (BeforeJam.VectorizeEnable != LoopAttributes::Unspecified ||
    565           BeforeJam.VectorizePredicateEnable != LoopAttributes::Unspecified ||
    566           BeforeJam.InterleaveCount != 0 || BeforeJam.VectorizeWidth != 0 ||
    567           BeforeJam.VectorizeScalable == LoopAttributes::Enable)
    568         BeforeLoopProperties.push_back(
    569             MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
    570 
    571       bool InnerFollowupHasTransform = false;
    572       MDNode *InnerFollowup = createMetadata(AfterJam, BeforeLoopProperties,
    573                                              InnerFollowupHasTransform);
    574       if (InnerFollowupHasTransform)
    575         Parent->UnrollAndJamInnerFollowup = InnerFollowup;
    576     }
    577 
    578     CurLoopAttr = BeforeJam;
    579   }
    580 
    581   bool HasUserTransforms = false;
    582   LoopID = createMetadata(CurLoopAttr, {}, HasUserTransforms);
    583   TempLoopID->replaceAllUsesWith(LoopID);
    584 }
    585 
    586 void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc,
    587                          const llvm::DebugLoc &EndLoc) {
    588   Active.emplace_back(
    589       new LoopInfo(Header, StagedAttrs, StartLoc, EndLoc,
    590                    Active.empty() ? nullptr : Active.back().get()));
    591   // Clear the attributes so nested loops do not inherit them.
    592   StagedAttrs.clear();
    593 }
    594 
    595 void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
    596                          const clang::CodeGenOptions &CGOpts,
    597                          ArrayRef<const clang::Attr *> Attrs,
    598                          const llvm::DebugLoc &StartLoc,
    599                          const llvm::DebugLoc &EndLoc, bool MustProgress) {
    600   // Identify loop hint attributes from Attrs.
    601   for (const auto *Attr : Attrs) {
    602     const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr);
    603     const OpenCLUnrollHintAttr *OpenCLHint =
    604         dyn_cast<OpenCLUnrollHintAttr>(Attr);
    605 
    606     // Skip non loop hint attributes
    607     if (!LH && !OpenCLHint) {
    608       continue;
    609     }
    610 
    611     LoopHintAttr::OptionType Option = LoopHintAttr::Unroll;
    612     LoopHintAttr::LoopHintState State = LoopHintAttr::Disable;
    613     unsigned ValueInt = 1;
    614     // Translate opencl_unroll_hint attribute argument to
    615     // equivalent LoopHintAttr enums.
    616     // OpenCL v2.0 s6.11.5:
    617     // 0 - enable unroll (no argument).
    618     // 1 - disable unroll.
    619     // other positive integer n - unroll by n.
    620     if (OpenCLHint) {
    621       ValueInt = OpenCLHint->getUnrollHint();
    622       if (ValueInt == 0) {
    623         State = LoopHintAttr::Enable;
    624       } else if (ValueInt != 1) {
    625         Option = LoopHintAttr::UnrollCount;
    626         State = LoopHintAttr::Numeric;
    627       }
    628     } else if (LH) {
    629       auto *ValueExpr = LH->getValue();
    630       if (ValueExpr) {
    631         llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx);
    632         ValueInt = ValueAPS.getSExtValue();
    633       }
    634 
    635       Option = LH->getOption();
    636       State = LH->getState();
    637     }
    638     switch (State) {
    639     case LoopHintAttr::Disable:
    640       switch (Option) {
    641       case LoopHintAttr::Vectorize:
    642         // Disable vectorization by specifying a width of 1.
    643         setVectorizeWidth(1);
    644         setVectorizeScalable(LoopAttributes::Unspecified);
    645         break;
    646       case LoopHintAttr::Interleave:
    647         // Disable interleaving by speciyfing a count of 1.
    648         setInterleaveCount(1);
    649         break;
    650       case LoopHintAttr::Unroll:
    651         setUnrollState(LoopAttributes::Disable);
    652         break;
    653       case LoopHintAttr::UnrollAndJam:
    654         setUnrollAndJamState(LoopAttributes::Disable);
    655         break;
    656       case LoopHintAttr::VectorizePredicate:
    657         setVectorizePredicateState(LoopAttributes::Disable);
    658         break;
    659       case LoopHintAttr::Distribute:
    660         setDistributeState(false);
    661         break;
    662       case LoopHintAttr::PipelineDisabled:
    663         setPipelineDisabled(true);
    664         break;
    665       case LoopHintAttr::UnrollCount:
    666       case LoopHintAttr::UnrollAndJamCount:
    667       case LoopHintAttr::VectorizeWidth:
    668       case LoopHintAttr::InterleaveCount:
    669       case LoopHintAttr::PipelineInitiationInterval:
    670         llvm_unreachable("Options cannot be disabled.");
    671         break;
    672       }
    673       break;
    674     case LoopHintAttr::Enable:
    675       switch (Option) {
    676       case LoopHintAttr::Vectorize:
    677       case LoopHintAttr::Interleave:
    678         setVectorizeEnable(true);
    679         break;
    680       case LoopHintAttr::Unroll:
    681         setUnrollState(LoopAttributes::Enable);
    682         break;
    683       case LoopHintAttr::UnrollAndJam:
    684         setUnrollAndJamState(LoopAttributes::Enable);
    685         break;
    686       case LoopHintAttr::VectorizePredicate:
    687         setVectorizePredicateState(LoopAttributes::Enable);
    688         break;
    689       case LoopHintAttr::Distribute:
    690         setDistributeState(true);
    691         break;
    692       case LoopHintAttr::UnrollCount:
    693       case LoopHintAttr::UnrollAndJamCount:
    694       case LoopHintAttr::VectorizeWidth:
    695       case LoopHintAttr::InterleaveCount:
    696       case LoopHintAttr::PipelineDisabled:
    697       case LoopHintAttr::PipelineInitiationInterval:
    698         llvm_unreachable("Options cannot enabled.");
    699         break;
    700       }
    701       break;
    702     case LoopHintAttr::AssumeSafety:
    703       switch (Option) {
    704       case LoopHintAttr::Vectorize:
    705       case LoopHintAttr::Interleave:
    706         // Apply "llvm.mem.parallel_loop_access" metadata to load/stores.
    707         setParallel(true);
    708         setVectorizeEnable(true);
    709         break;
    710       case LoopHintAttr::Unroll:
    711       case LoopHintAttr::UnrollAndJam:
    712       case LoopHintAttr::VectorizePredicate:
    713       case LoopHintAttr::UnrollCount:
    714       case LoopHintAttr::UnrollAndJamCount:
    715       case LoopHintAttr::VectorizeWidth:
    716       case LoopHintAttr::InterleaveCount:
    717       case LoopHintAttr::Distribute:
    718       case LoopHintAttr::PipelineDisabled:
    719       case LoopHintAttr::PipelineInitiationInterval:
    720         llvm_unreachable("Options cannot be used to assume mem safety.");
    721         break;
    722       }
    723       break;
    724     case LoopHintAttr::Full:
    725       switch (Option) {
    726       case LoopHintAttr::Unroll:
    727         setUnrollState(LoopAttributes::Full);
    728         break;
    729       case LoopHintAttr::UnrollAndJam:
    730         setUnrollAndJamState(LoopAttributes::Full);
    731         break;
    732       case LoopHintAttr::Vectorize:
    733       case LoopHintAttr::Interleave:
    734       case LoopHintAttr::UnrollCount:
    735       case LoopHintAttr::UnrollAndJamCount:
    736       case LoopHintAttr::VectorizeWidth:
    737       case LoopHintAttr::InterleaveCount:
    738       case LoopHintAttr::Distribute:
    739       case LoopHintAttr::PipelineDisabled:
    740       case LoopHintAttr::PipelineInitiationInterval:
    741       case LoopHintAttr::VectorizePredicate:
    742         llvm_unreachable("Options cannot be used with 'full' hint.");
    743         break;
    744       }
    745       break;
    746     case LoopHintAttr::FixedWidth:
    747     case LoopHintAttr::ScalableWidth:
    748       switch (Option) {
    749       case LoopHintAttr::VectorizeWidth:
    750         setVectorizeScalable(State == LoopHintAttr::ScalableWidth
    751                                  ? LoopAttributes::Enable
    752                                  : LoopAttributes::Disable);
    753         if (LH->getValue())
    754           setVectorizeWidth(ValueInt);
    755         break;
    756       default:
    757         llvm_unreachable("Options cannot be used with 'scalable' hint.");
    758         break;
    759       }
    760       break;
    761     case LoopHintAttr::Numeric:
    762       switch (Option) {
    763       case LoopHintAttr::InterleaveCount:
    764         setInterleaveCount(ValueInt);
    765         break;
    766       case LoopHintAttr::UnrollCount:
    767         setUnrollCount(ValueInt);
    768         break;
    769       case LoopHintAttr::UnrollAndJamCount:
    770         setUnrollAndJamCount(ValueInt);
    771         break;
    772       case LoopHintAttr::PipelineInitiationInterval:
    773         setPipelineInitiationInterval(ValueInt);
    774         break;
    775       case LoopHintAttr::Unroll:
    776       case LoopHintAttr::UnrollAndJam:
    777       case LoopHintAttr::VectorizePredicate:
    778       case LoopHintAttr::Vectorize:
    779       case LoopHintAttr::VectorizeWidth:
    780       case LoopHintAttr::Interleave:
    781       case LoopHintAttr::Distribute:
    782       case LoopHintAttr::PipelineDisabled:
    783         llvm_unreachable("Options cannot be assigned a value.");
    784         break;
    785       }
    786       break;
    787     }
    788   }
    789 
    790   setMustProgress(MustProgress);
    791 
    792   if (CGOpts.OptimizationLevel > 0)
    793     // Disable unrolling for the loop, if unrolling is disabled (via
    794     // -fno-unroll-loops) and no pragmas override the decision.
    795     if (!CGOpts.UnrollLoops &&
    796         (StagedAttrs.UnrollEnable == LoopAttributes::Unspecified &&
    797          StagedAttrs.UnrollCount == 0))
    798       setUnrollState(LoopAttributes::Disable);
    799 
    800   /// Stage the attributes.
    801   push(Header, StartLoc, EndLoc);
    802 }
    803 
    804 void LoopInfoStack::pop() {
    805   assert(!Active.empty() && "No active loops to pop");
    806   Active.back()->finish();
    807   Active.pop_back();
    808 }
    809 
    810 void LoopInfoStack::InsertHelper(Instruction *I) const {
    811   if (I->mayReadOrWriteMemory()) {
    812     SmallVector<Metadata *, 4> AccessGroups;
    813     for (const auto &AL : Active) {
    814       // Here we assume that every loop that has an access group is parallel.
    815       if (MDNode *Group = AL->getAccessGroup())
    816         AccessGroups.push_back(Group);
    817     }
    818     MDNode *UnionMD = nullptr;
    819     if (AccessGroups.size() == 1)
    820       UnionMD = cast<MDNode>(AccessGroups[0]);
    821     else if (AccessGroups.size() >= 2)
    822       UnionMD = MDNode::get(I->getContext(), AccessGroups);
    823     I->setMetadata("llvm.access.group", UnionMD);
    824   }
    825 
    826   if (!hasInfo())
    827     return;
    828 
    829   const LoopInfo &L = getInfo();
    830   if (!L.getLoopID())
    831     return;
    832 
    833   if (I->isTerminator()) {
    834     for (BasicBlock *Succ : successors(I))
    835       if (Succ == L.getHeader()) {
    836         I->setMetadata(llvm::LLVMContext::MD_loop, L.getLoopID());
    837         break;
    838       }
    839     return;
    840   }
    841 }
    842