Home | History | Annotate | Line # | Download | only in CodeGen
CGLoopInfo.cpp revision 1.1.1.1 
      1 //===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===//
      2 //
      3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
      4 // See https://llvm.org/LICENSE.txt for license information.
      5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
      6 //
      7 //===----------------------------------------------------------------------===//
      8 
      9 #include "CGLoopInfo.h"
     10 #include "clang/AST/ASTContext.h"
     11 #include "clang/AST/Attr.h"
     12 #include "llvm/IR/BasicBlock.h"
     13 #include "llvm/IR/CFG.h"
     14 #include "llvm/IR/Constants.h"
     15 #include "llvm/IR/InstrTypes.h"
     16 #include "llvm/IR/Instructions.h"
     17 #include "llvm/IR/Metadata.h"
     18 using namespace clang::CodeGen;
     19 using namespace llvm;
     20 
     21 MDNode *
     22 LoopInfo::createLoopPropertiesMetadata(ArrayRef<Metadata *> LoopProperties) {
     23   LLVMContext &Ctx = Header->getContext();
     24   SmallVector<Metadata *, 4> NewLoopProperties;
     25   TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
     26   NewLoopProperties.push_back(TempNode.get());
     27   NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
     28 
     29   MDNode *LoopID = MDNode::getDistinct(Ctx, NewLoopProperties);
     30   LoopID->replaceOperandWith(0, LoopID);
     31   return LoopID;
     32 }
     33 
     34 MDNode *LoopInfo::createPipeliningMetadata(const LoopAttributes &Attrs,
     35                                            ArrayRef<Metadata *> LoopProperties,
     36                                            bool &HasUserTransforms) {
     37   LLVMContext &Ctx = Header->getContext();
     38 
     39   Optional<bool> Enabled;
     40   if (Attrs.PipelineDisabled)
     41     Enabled = false;
     42   else if (Attrs.PipelineInitiationInterval != 0)
     43     Enabled = true;
     44 
     45   if (Enabled != true) {
     46     SmallVector<Metadata *, 4> NewLoopProperties;
     47     if (Enabled == false) {
     48       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
     49       NewLoopProperties.push_back(
     50           MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.pipeline.disable"),
     51                             ConstantAsMetadata::get(ConstantInt::get(
     52                                 llvm::Type::getInt1Ty(Ctx), 1))}));
     53       LoopProperties = NewLoopProperties;
     54     }
     55     return createLoopPropertiesMetadata(LoopProperties);
     56   }
     57 
     58   SmallVector<Metadata *, 4> Args;
     59   TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
     60   Args.push_back(TempNode.get());
     61   Args.append(LoopProperties.begin(), LoopProperties.end());
     62 
     63   if (Attrs.PipelineInitiationInterval > 0) {
     64     Metadata *Vals[] = {
     65         MDString::get(Ctx, "llvm.loop.pipeline.initiationinterval"),
     66         ConstantAsMetadata::get(ConstantInt::get(
     67             llvm::Type::getInt32Ty(Ctx), Attrs.PipelineInitiationInterval))};
     68     Args.push_back(MDNode::get(Ctx, Vals));
     69   }
     70 
     71   // No follow-up: This is the last transformation.
     72 
     73   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
     74   LoopID->replaceOperandWith(0, LoopID);
     75   HasUserTransforms = true;
     76   return LoopID;
     77 }
     78 
     79 MDNode *
     80 LoopInfo::createPartialUnrollMetadata(const LoopAttributes &Attrs,
     81                                       ArrayRef<Metadata *> LoopProperties,
     82                                       bool &HasUserTransforms) {
     83   LLVMContext &Ctx = Header->getContext();
     84 
     85   Optional<bool> Enabled;
     86   if (Attrs.UnrollEnable == LoopAttributes::Disable)
     87     Enabled = false;
     88   else if (Attrs.UnrollEnable == LoopAttributes::Full)
     89     Enabled = None;
     90   else if (Attrs.UnrollEnable != LoopAttributes::Unspecified ||
     91            Attrs.UnrollCount != 0)
     92     Enabled = true;
     93 
     94   if (Enabled != true) {
     95     // createFullUnrollMetadata will already have added llvm.loop.unroll.disable
     96     // if unrolling is disabled.
     97     return createPipeliningMetadata(Attrs, LoopProperties, HasUserTransforms);
     98   }
     99 
    100   SmallVector<Metadata *, 4> FollowupLoopProperties;
    101 
    102   // Apply all loop properties to the unrolled loop.
    103   FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
    104 
    105   // Don't unroll an already unrolled loop.
    106   FollowupLoopProperties.push_back(
    107       MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
    108 
    109   bool FollowupHasTransforms = false;
    110   MDNode *Followup = createPipeliningMetadata(Attrs, FollowupLoopProperties,
    111                                               FollowupHasTransforms);
    112 
    113   SmallVector<Metadata *, 4> Args;
    114   TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
    115   Args.push_back(TempNode.get());
    116   Args.append(LoopProperties.begin(), LoopProperties.end());
    117 
    118   // Setting unroll.count
    119   if (Attrs.UnrollCount > 0) {
    120     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"),
    121                         ConstantAsMetadata::get(ConstantInt::get(
    122                             llvm::Type::getInt32Ty(Ctx), Attrs.UnrollCount))};
    123     Args.push_back(MDNode::get(Ctx, Vals));
    124   }
    125 
    126   // Setting unroll.full or unroll.disable
    127   if (Attrs.UnrollEnable == LoopAttributes::Enable) {
    128     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.enable")};
    129     Args.push_back(MDNode::get(Ctx, Vals));
    130   }
    131 
    132   if (FollowupHasTransforms)
    133     Args.push_back(MDNode::get(
    134         Ctx, {MDString::get(Ctx, "llvm.loop.unroll.followup_all"), Followup}));
    135 
    136   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
    137   LoopID->replaceOperandWith(0, LoopID);
    138   HasUserTransforms = true;
    139   return LoopID;
    140 }
    141 
    142 MDNode *
    143 LoopInfo::createUnrollAndJamMetadata(const LoopAttributes &Attrs,
    144                                      ArrayRef<Metadata *> LoopProperties,
    145                                      bool &HasUserTransforms) {
    146   LLVMContext &Ctx = Header->getContext();
    147 
    148   Optional<bool> Enabled;
    149   if (Attrs.UnrollAndJamEnable == LoopAttributes::Disable)
    150     Enabled = false;
    151   else if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable ||
    152            Attrs.UnrollAndJamCount != 0)
    153     Enabled = true;
    154 
    155   if (Enabled != true) {
    156     SmallVector<Metadata *, 4> NewLoopProperties;
    157     if (Enabled == false) {
    158       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
    159       NewLoopProperties.push_back(MDNode::get(
    160           Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
    161       LoopProperties = NewLoopProperties;
    162     }
    163     return createPartialUnrollMetadata(Attrs, LoopProperties,
    164                                        HasUserTransforms);
    165   }
    166 
    167   SmallVector<Metadata *, 4> FollowupLoopProperties;
    168   FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
    169   FollowupLoopProperties.push_back(
    170       MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
    171 
    172   bool FollowupHasTransforms = false;
    173   MDNode *Followup = createPartialUnrollMetadata(Attrs, FollowupLoopProperties,
    174                                                  FollowupHasTransforms);
    175 
    176   SmallVector<Metadata *, 4> Args;
    177   TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
    178   Args.push_back(TempNode.get());
    179   Args.append(LoopProperties.begin(), LoopProperties.end());
    180 
    181   // Setting unroll_and_jam.count
    182   if (Attrs.UnrollAndJamCount > 0) {
    183     Metadata *Vals[] = {
    184         MDString::get(Ctx, "llvm.loop.unroll_and_jam.count"),
    185         ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
    186                                                  Attrs.UnrollAndJamCount))};
    187     Args.push_back(MDNode::get(Ctx, Vals));
    188   }
    189 
    190   if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable) {
    191     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll_and_jam.enable")};
    192     Args.push_back(MDNode::get(Ctx, Vals));
    193   }
    194 
    195   if (FollowupHasTransforms)
    196     Args.push_back(MDNode::get(
    197         Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_outer"),
    198               Followup}));
    199 
    200   if (UnrollAndJamInnerFollowup)
    201     Args.push_back(MDNode::get(
    202         Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_inner"),
    203               UnrollAndJamInnerFollowup}));
    204 
    205   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
    206   LoopID->replaceOperandWith(0, LoopID);
    207   HasUserTransforms = true;
    208   return LoopID;
    209 }
    210 
    211 MDNode *
    212 LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs,
    213                                       ArrayRef<Metadata *> LoopProperties,
    214                                       bool &HasUserTransforms) {
    215   LLVMContext &Ctx = Header->getContext();
    216 
    217   Optional<bool> Enabled;
    218   if (Attrs.VectorizeEnable == LoopAttributes::Disable)
    219     Enabled = false;
    220   else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
    221            Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified ||
    222            Attrs.InterleaveCount != 0 || Attrs.VectorizeWidth != 0)
    223     Enabled = true;
    224 
    225   if (Enabled != true) {
    226     SmallVector<Metadata *, 4> NewLoopProperties;
    227     if (Enabled == false) {
    228       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
    229       NewLoopProperties.push_back(
    230           MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
    231                             ConstantAsMetadata::get(ConstantInt::get(
    232                                 llvm::Type::getInt1Ty(Ctx), 0))}));
    233       LoopProperties = NewLoopProperties;
    234     }
    235     return createUnrollAndJamMetadata(Attrs, LoopProperties, HasUserTransforms);
    236   }
    237 
    238   // Apply all loop properties to the vectorized loop.
    239   SmallVector<Metadata *, 4> FollowupLoopProperties;
    240   FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
    241 
    242   // Don't vectorize an already vectorized loop.
    243   FollowupLoopProperties.push_back(
    244       MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
    245 
    246   bool FollowupHasTransforms = false;
    247   MDNode *Followup = createUnrollAndJamMetadata(Attrs, FollowupLoopProperties,
    248                                                 FollowupHasTransforms);
    249 
    250   SmallVector<Metadata *, 4> Args;
    251   TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
    252   Args.push_back(TempNode.get());
    253   Args.append(LoopProperties.begin(), LoopProperties.end());
    254 
    255   // Setting vectorize.predicate
    256   bool IsVectorPredicateEnabled = false;
    257   if (Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified &&
    258       Attrs.VectorizeEnable != LoopAttributes::Disable &&
    259       Attrs.VectorizeWidth < 1) {
    260 
    261     IsVectorPredicateEnabled =
    262         (Attrs.VectorizePredicateEnable == LoopAttributes::Enable);
    263 
    264     Metadata *Vals[] = {
    265         MDString::get(Ctx, "llvm.loop.vectorize.predicate.enable"),
    266         ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt1Ty(Ctx),
    267                                                  IsVectorPredicateEnabled))};
    268     Args.push_back(MDNode::get(Ctx, Vals));
    269   }
    270 
    271   // Setting vectorize.width
    272   if (Attrs.VectorizeWidth > 0) {
    273     Metadata *Vals[] = {
    274         MDString::get(Ctx, "llvm.loop.vectorize.width"),
    275         ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
    276                                                  Attrs.VectorizeWidth))};
    277     Args.push_back(MDNode::get(Ctx, Vals));
    278   }
    279 
    280   // Setting interleave.count
    281   if (Attrs.InterleaveCount > 0) {
    282     Metadata *Vals[] = {
    283         MDString::get(Ctx, "llvm.loop.interleave.count"),
    284         ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
    285                                                  Attrs.InterleaveCount))};
    286     Args.push_back(MDNode::get(Ctx, Vals));
    287   }
    288 
    289   // Setting vectorize.enable
    290   if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
    291       IsVectorPredicateEnabled) {
    292     Metadata *Vals[] = {
    293         MDString::get(Ctx, "llvm.loop.vectorize.enable"),
    294         ConstantAsMetadata::get(ConstantInt::get(
    295             llvm::Type::getInt1Ty(Ctx),
    296             IsVectorPredicateEnabled
    297                 ? true
    298                 : (Attrs.VectorizeEnable == LoopAttributes::Enable)))};
    299     Args.push_back(MDNode::get(Ctx, Vals));
    300   }
    301 
    302   if (FollowupHasTransforms)
    303     Args.push_back(MDNode::get(
    304         Ctx,
    305         {MDString::get(Ctx, "llvm.loop.vectorize.followup_all"), Followup}));
    306 
    307   MDNode *LoopID = MDNode::get(Ctx, Args);
    308   LoopID->replaceOperandWith(0, LoopID);
    309   HasUserTransforms = true;
    310   return LoopID;
    311 }
    312 
    313 MDNode *
    314 LoopInfo::createLoopDistributeMetadata(const LoopAttributes &Attrs,
    315                                        ArrayRef<Metadata *> LoopProperties,
    316                                        bool &HasUserTransforms) {
    317   LLVMContext &Ctx = Header->getContext();
    318 
    319   Optional<bool> Enabled;
    320   if (Attrs.DistributeEnable == LoopAttributes::Disable)
    321     Enabled = false;
    322   if (Attrs.DistributeEnable == LoopAttributes::Enable)
    323     Enabled = true;
    324 
    325   if (Enabled != true) {
    326     SmallVector<Metadata *, 4> NewLoopProperties;
    327     if (Enabled == false) {
    328       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
    329       NewLoopProperties.push_back(
    330           MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.distribute.enable"),
    331                             ConstantAsMetadata::get(ConstantInt::get(
    332                                 llvm::Type::getInt1Ty(Ctx), 0))}));
    333       LoopProperties = NewLoopProperties;
    334     }
    335     return createLoopVectorizeMetadata(Attrs, LoopProperties,
    336                                        HasUserTransforms);
    337   }
    338 
    339   bool FollowupHasTransforms = false;
    340   MDNode *Followup =
    341       createLoopVectorizeMetadata(Attrs, LoopProperties, FollowupHasTransforms);
    342 
    343   SmallVector<Metadata *, 4> Args;
    344   TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
    345   Args.push_back(TempNode.get());
    346   Args.append(LoopProperties.begin(), LoopProperties.end());
    347 
    348   Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"),
    349                       ConstantAsMetadata::get(ConstantInt::get(
    350                           llvm::Type::getInt1Ty(Ctx),
    351                           (Attrs.DistributeEnable == LoopAttributes::Enable)))};
    352   Args.push_back(MDNode::get(Ctx, Vals));
    353 
    354   if (FollowupHasTransforms)
    355     Args.push_back(MDNode::get(
    356         Ctx,
    357         {MDString::get(Ctx, "llvm.loop.distribute.followup_all"), Followup}));
    358 
    359   MDNode *LoopID = MDNode::get(Ctx, Args);
    360   LoopID->replaceOperandWith(0, LoopID);
    361   HasUserTransforms = true;
    362   return LoopID;
    363 }
    364 
    365 MDNode *LoopInfo::createFullUnrollMetadata(const LoopAttributes &Attrs,
    366                                            ArrayRef<Metadata *> LoopProperties,
    367                                            bool &HasUserTransforms) {
    368   LLVMContext &Ctx = Header->getContext();
    369 
    370   Optional<bool> Enabled;
    371   if (Attrs.UnrollEnable == LoopAttributes::Disable)
    372     Enabled = false;
    373   else if (Attrs.UnrollEnable == LoopAttributes::Full)
    374     Enabled = true;
    375 
    376   if (Enabled != true) {
    377     SmallVector<Metadata *, 4> NewLoopProperties;
    378     if (Enabled == false) {
    379       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
    380       NewLoopProperties.push_back(
    381           MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
    382       LoopProperties = NewLoopProperties;
    383     }
    384     return createLoopDistributeMetadata(Attrs, LoopProperties,
    385                                         HasUserTransforms);
    386   }
    387 
    388   SmallVector<Metadata *, 4> Args;
    389   TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
    390   Args.push_back(TempNode.get());
    391   Args.append(LoopProperties.begin(), LoopProperties.end());
    392   Args.push_back(MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.full")));
    393 
    394   // No follow-up: there is no loop after full unrolling.
    395   // TODO: Warn if there are transformations after full unrolling.
    396 
    397   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
    398   LoopID->replaceOperandWith(0, LoopID);
    399   HasUserTransforms = true;
    400   return LoopID;
    401 }
    402 
    403 MDNode *LoopInfo::createMetadata(
    404     const LoopAttributes &Attrs,
    405     llvm::ArrayRef<llvm::Metadata *> AdditionalLoopProperties,
    406     bool &HasUserTransforms) {
    407   SmallVector<Metadata *, 3> LoopProperties;
    408 
    409   // If we have a valid start debug location for the loop, add it.
    410   if (StartLoc) {
    411     LoopProperties.push_back(StartLoc.getAsMDNode());
    412 
    413     // If we also have a valid end debug location for the loop, add it.
    414     if (EndLoc)
    415       LoopProperties.push_back(EndLoc.getAsMDNode());
    416   }
    417 
    418   assert(!!AccGroup == Attrs.IsParallel &&
    419          "There must be an access group iff the loop is parallel");
    420   if (Attrs.IsParallel) {
    421     LLVMContext &Ctx = Header->getContext();
    422     LoopProperties.push_back(MDNode::get(
    423         Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccGroup}));
    424   }
    425 
    426   LoopProperties.insert(LoopProperties.end(), AdditionalLoopProperties.begin(),
    427                         AdditionalLoopProperties.end());
    428   return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms);
    429 }
    430 
    431 LoopAttributes::LoopAttributes(bool IsParallel)
    432     : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),
    433       UnrollEnable(LoopAttributes::Unspecified),
    434       UnrollAndJamEnable(LoopAttributes::Unspecified),
    435       VectorizePredicateEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
    436       InterleaveCount(0), UnrollCount(0), UnrollAndJamCount(0),
    437       DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false),
    438       PipelineInitiationInterval(0) {}
    439 
    440 void LoopAttributes::clear() {
    441   IsParallel = false;
    442   VectorizeWidth = 0;
    443   InterleaveCount = 0;
    444   UnrollCount = 0;
    445   UnrollAndJamCount = 0;
    446   VectorizeEnable = LoopAttributes::Unspecified;
    447   UnrollEnable = LoopAttributes::Unspecified;
    448   UnrollAndJamEnable = LoopAttributes::Unspecified;
    449   VectorizePredicateEnable = LoopAttributes::Unspecified;
    450   DistributeEnable = LoopAttributes::Unspecified;
    451   PipelineDisabled = false;
    452   PipelineInitiationInterval = 0;
    453 }
    454 
    455 LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,
    456                    const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc,
    457                    LoopInfo *Parent)
    458     : Header(Header), Attrs(Attrs), StartLoc(StartLoc), EndLoc(EndLoc),
    459       Parent(Parent) {
    460 
    461   if (Attrs.IsParallel) {
    462     // Create an access group for this loop.
    463     LLVMContext &Ctx = Header->getContext();
    464     AccGroup = MDNode::getDistinct(Ctx, {});
    465   }
    466 
    467   if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
    468       Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
    469       Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled &&
    470       Attrs.PipelineInitiationInterval == 0 &&
    471       Attrs.VectorizePredicateEnable == LoopAttributes::Unspecified &&
    472       Attrs.VectorizeEnable == LoopAttributes::Unspecified &&
    473       Attrs.UnrollEnable == LoopAttributes::Unspecified &&
    474       Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified &&
    475       Attrs.DistributeEnable == LoopAttributes::Unspecified && !StartLoc &&
    476       !EndLoc)
    477     return;
    478 
    479   TempLoopID = MDNode::getTemporary(Header->getContext(), None);
    480 }
    481 
    482 void LoopInfo::finish() {
    483   // We did not annotate the loop body instructions because there are no
    484   // attributes for this loop.
    485   if (!TempLoopID)
    486     return;
    487 
    488   MDNode *LoopID;
    489   LoopAttributes CurLoopAttr = Attrs;
    490   LLVMContext &Ctx = Header->getContext();
    491 
    492   if (Parent && (Parent->Attrs.UnrollAndJamEnable ||
    493                  Parent->Attrs.UnrollAndJamCount != 0)) {
    494     // Parent unroll-and-jams this loop.
    495     // Split the transformations in those that happens before the unroll-and-jam
    496     // and those after.
    497 
    498     LoopAttributes BeforeJam, AfterJam;
    499 
    500     BeforeJam.IsParallel = AfterJam.IsParallel = Attrs.IsParallel;
    501 
    502     BeforeJam.VectorizeWidth = Attrs.VectorizeWidth;
    503     BeforeJam.InterleaveCount = Attrs.InterleaveCount;
    504     BeforeJam.VectorizeEnable = Attrs.VectorizeEnable;
    505     BeforeJam.DistributeEnable = Attrs.DistributeEnable;
    506     BeforeJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
    507 
    508     switch (Attrs.UnrollEnable) {
    509     case LoopAttributes::Unspecified:
    510     case LoopAttributes::Disable:
    511       BeforeJam.UnrollEnable = Attrs.UnrollEnable;
    512       AfterJam.UnrollEnable = Attrs.UnrollEnable;
    513       break;
    514     case LoopAttributes::Full:
    515       BeforeJam.UnrollEnable = LoopAttributes::Full;
    516       break;
    517     case LoopAttributes::Enable:
    518       AfterJam.UnrollEnable = LoopAttributes::Enable;
    519       break;
    520     }
    521 
    522     AfterJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
    523     AfterJam.UnrollCount = Attrs.UnrollCount;
    524     AfterJam.PipelineDisabled = Attrs.PipelineDisabled;
    525     AfterJam.PipelineInitiationInterval = Attrs.PipelineInitiationInterval;
    526 
    527     // If this loop is subject of an unroll-and-jam by the parent loop, and has
    528     // an unroll-and-jam annotation itself, we have to decide whether to first
    529     // apply the parent's unroll-and-jam or this loop's unroll-and-jam. The
    530     // UnrollAndJam pass processes loops from inner to outer, so we apply the
    531     // inner first.
    532     BeforeJam.UnrollAndJamCount = Attrs.UnrollAndJamCount;
    533     BeforeJam.UnrollAndJamEnable = Attrs.UnrollAndJamEnable;
    534 
    535     // Set the inner followup metadata to process by the outer loop. Only
    536     // consider the first inner loop.
    537     if (!Parent->UnrollAndJamInnerFollowup) {
    538       // Splitting the attributes into a BeforeJam and an AfterJam part will
    539       // stop 'llvm.loop.isvectorized' (generated by vectorization in BeforeJam)
    540       // to be forwarded to the AfterJam part. We detect the situation here and
    541       // add it manually.
    542       SmallVector<Metadata *, 1> BeforeLoopProperties;
    543       if (BeforeJam.VectorizeEnable != LoopAttributes::Unspecified ||
    544           BeforeJam.VectorizePredicateEnable != LoopAttributes::Unspecified ||
    545           BeforeJam.InterleaveCount != 0 || BeforeJam.VectorizeWidth != 0)
    546         BeforeLoopProperties.push_back(
    547             MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
    548 
    549       bool InnerFollowupHasTransform = false;
    550       MDNode *InnerFollowup = createMetadata(AfterJam, BeforeLoopProperties,
    551                                              InnerFollowupHasTransform);
    552       if (InnerFollowupHasTransform)
    553         Parent->UnrollAndJamInnerFollowup = InnerFollowup;
    554     }
    555 
    556     CurLoopAttr = BeforeJam;
    557   }
    558 
    559   bool HasUserTransforms = false;
    560   LoopID = createMetadata(CurLoopAttr, {}, HasUserTransforms);
    561   TempLoopID->replaceAllUsesWith(LoopID);
    562 }
    563 
    564 void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc,
    565                          const llvm::DebugLoc &EndLoc) {
    566   Active.emplace_back(
    567       new LoopInfo(Header, StagedAttrs, StartLoc, EndLoc,
    568                    Active.empty() ? nullptr : Active.back().get()));
    569   // Clear the attributes so nested loops do not inherit them.
    570   StagedAttrs.clear();
    571 }
    572 
    573 void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
    574                          ArrayRef<const clang::Attr *> Attrs,
    575                          const llvm::DebugLoc &StartLoc,
    576                          const llvm::DebugLoc &EndLoc) {
    577 
    578   // Identify loop hint attributes from Attrs.
    579   for (const auto *Attr : Attrs) {
    580     const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr);
    581     const OpenCLUnrollHintAttr *OpenCLHint =
    582         dyn_cast<OpenCLUnrollHintAttr>(Attr);
    583 
    584     // Skip non loop hint attributes
    585     if (!LH && !OpenCLHint) {
    586       continue;
    587     }
    588 
    589     LoopHintAttr::OptionType Option = LoopHintAttr::Unroll;
    590     LoopHintAttr::LoopHintState State = LoopHintAttr::Disable;
    591     unsigned ValueInt = 1;
    592     // Translate opencl_unroll_hint attribute argument to
    593     // equivalent LoopHintAttr enums.
    594     // OpenCL v2.0 s6.11.5:
    595     // 0 - enable unroll (no argument).
    596     // 1 - disable unroll.
    597     // other positive integer n - unroll by n.
    598     if (OpenCLHint) {
    599       ValueInt = OpenCLHint->getUnrollHint();
    600       if (ValueInt == 0) {
    601         State = LoopHintAttr::Enable;
    602       } else if (ValueInt != 1) {
    603         Option = LoopHintAttr::UnrollCount;
    604         State = LoopHintAttr::Numeric;
    605       }
    606     } else if (LH) {
    607       auto *ValueExpr = LH->getValue();
    608       if (ValueExpr) {
    609         llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx);
    610         ValueInt = ValueAPS.getSExtValue();
    611       }
    612 
    613       Option = LH->getOption();
    614       State = LH->getState();
    615     }
    616     switch (State) {
    617     case LoopHintAttr::Disable:
    618       switch (Option) {
    619       case LoopHintAttr::Vectorize:
    620         // Disable vectorization by specifying a width of 1.
    621         setVectorizeWidth(1);
    622         break;
    623       case LoopHintAttr::Interleave:
    624         // Disable interleaving by speciyfing a count of 1.
    625         setInterleaveCount(1);
    626         break;
    627       case LoopHintAttr::Unroll:
    628         setUnrollState(LoopAttributes::Disable);
    629         break;
    630       case LoopHintAttr::UnrollAndJam:
    631         setUnrollAndJamState(LoopAttributes::Disable);
    632         break;
    633       case LoopHintAttr::VectorizePredicate:
    634         setVectorizePredicateState(LoopAttributes::Disable);
    635         break;
    636       case LoopHintAttr::Distribute:
    637         setDistributeState(false);
    638         break;
    639       case LoopHintAttr::PipelineDisabled:
    640         setPipelineDisabled(true);
    641         break;
    642       case LoopHintAttr::UnrollCount:
    643       case LoopHintAttr::UnrollAndJamCount:
    644       case LoopHintAttr::VectorizeWidth:
    645       case LoopHintAttr::InterleaveCount:
    646       case LoopHintAttr::PipelineInitiationInterval:
    647         llvm_unreachable("Options cannot be disabled.");
    648         break;
    649       }
    650       break;
    651     case LoopHintAttr::Enable:
    652       switch (Option) {
    653       case LoopHintAttr::Vectorize:
    654       case LoopHintAttr::Interleave:
    655         setVectorizeEnable(true);
    656         break;
    657       case LoopHintAttr::Unroll:
    658         setUnrollState(LoopAttributes::Enable);
    659         break;
    660       case LoopHintAttr::UnrollAndJam:
    661         setUnrollAndJamState(LoopAttributes::Enable);
    662         break;
    663       case LoopHintAttr::VectorizePredicate:
    664         setVectorizePredicateState(LoopAttributes::Enable);
    665         break;
    666       case LoopHintAttr::Distribute:
    667         setDistributeState(true);
    668         break;
    669       case LoopHintAttr::UnrollCount:
    670       case LoopHintAttr::UnrollAndJamCount:
    671       case LoopHintAttr::VectorizeWidth:
    672       case LoopHintAttr::InterleaveCount:
    673       case LoopHintAttr::PipelineDisabled:
    674       case LoopHintAttr::PipelineInitiationInterval:
    675         llvm_unreachable("Options cannot enabled.");
    676         break;
    677       }
    678       break;
    679     case LoopHintAttr::AssumeSafety:
    680       switch (Option) {
    681       case LoopHintAttr::Vectorize:
    682       case LoopHintAttr::Interleave:
    683         // Apply "llvm.mem.parallel_loop_access" metadata to load/stores.
    684         setParallel(true);
    685         setVectorizeEnable(true);
    686         break;
    687       case LoopHintAttr::Unroll:
    688       case LoopHintAttr::UnrollAndJam:
    689       case LoopHintAttr::VectorizePredicate:
    690       case LoopHintAttr::UnrollCount:
    691       case LoopHintAttr::UnrollAndJamCount:
    692       case LoopHintAttr::VectorizeWidth:
    693       case LoopHintAttr::InterleaveCount:
    694       case LoopHintAttr::Distribute:
    695       case LoopHintAttr::PipelineDisabled:
    696       case LoopHintAttr::PipelineInitiationInterval:
    697         llvm_unreachable("Options cannot be used to assume mem safety.");
    698         break;
    699       }
    700       break;
    701     case LoopHintAttr::Full:
    702       switch (Option) {
    703       case LoopHintAttr::Unroll:
    704         setUnrollState(LoopAttributes::Full);
    705         break;
    706       case LoopHintAttr::UnrollAndJam:
    707         setUnrollAndJamState(LoopAttributes::Full);
    708         break;
    709       case LoopHintAttr::Vectorize:
    710       case LoopHintAttr::Interleave:
    711       case LoopHintAttr::UnrollCount:
    712       case LoopHintAttr::UnrollAndJamCount:
    713       case LoopHintAttr::VectorizeWidth:
    714       case LoopHintAttr::InterleaveCount:
    715       case LoopHintAttr::Distribute:
    716       case LoopHintAttr::PipelineDisabled:
    717       case LoopHintAttr::PipelineInitiationInterval:
    718       case LoopHintAttr::VectorizePredicate:
    719         llvm_unreachable("Options cannot be used with 'full' hint.");
    720         break;
    721       }
    722       break;
    723     case LoopHintAttr::Numeric:
    724       switch (Option) {
    725       case LoopHintAttr::VectorizeWidth:
    726         setVectorizeWidth(ValueInt);
    727         break;
    728       case LoopHintAttr::InterleaveCount:
    729         setInterleaveCount(ValueInt);
    730         break;
    731       case LoopHintAttr::UnrollCount:
    732         setUnrollCount(ValueInt);
    733         break;
    734       case LoopHintAttr::UnrollAndJamCount:
    735         setUnrollAndJamCount(ValueInt);
    736         break;
    737       case LoopHintAttr::PipelineInitiationInterval:
    738         setPipelineInitiationInterval(ValueInt);
    739         break;
    740       case LoopHintAttr::Unroll:
    741       case LoopHintAttr::UnrollAndJam:
    742       case LoopHintAttr::VectorizePredicate:
    743       case LoopHintAttr::Vectorize:
    744       case LoopHintAttr::Interleave:
    745       case LoopHintAttr::Distribute:
    746       case LoopHintAttr::PipelineDisabled:
    747         llvm_unreachable("Options cannot be assigned a value.");
    748         break;
    749       }
    750       break;
    751     }
    752   }
    753 
    754   /// Stage the attributes.
    755   push(Header, StartLoc, EndLoc);
    756 }
    757 
    758 void LoopInfoStack::pop() {
    759   assert(!Active.empty() && "No active loops to pop");
    760   Active.back()->finish();
    761   Active.pop_back();
    762 }
    763 
    764 void LoopInfoStack::InsertHelper(Instruction *I) const {
    765   if (I->mayReadOrWriteMemory()) {
    766     SmallVector<Metadata *, 4> AccessGroups;
    767     for (const auto &AL : Active) {
    768       // Here we assume that every loop that has an access group is parallel.
    769       if (MDNode *Group = AL->getAccessGroup())
    770         AccessGroups.push_back(Group);
    771     }
    772     MDNode *UnionMD = nullptr;
    773     if (AccessGroups.size() == 1)
    774       UnionMD = cast<MDNode>(AccessGroups[0]);
    775     else if (AccessGroups.size() >= 2)
    776       UnionMD = MDNode::get(I->getContext(), AccessGroups);
    777     I->setMetadata("llvm.access.group", UnionMD);
    778   }
    779 
    780   if (!hasInfo())
    781     return;
    782 
    783   const LoopInfo &L = getInfo();
    784   if (!L.getLoopID())
    785     return;
    786 
    787   if (I->isTerminator()) {
    788     for (BasicBlock *Succ : successors(I))
    789       if (Succ == L.getHeader()) {
    790         I->setMetadata(llvm::LLVMContext::MD_loop, L.getLoopID());
    791         break;
    792       }
    793     return;
    794   }
    795 }
    796