Home | History | Annotate | Line # | Download | only in Coroutines 
      1 //===- CoroInternal.h - Internal Coroutine interfaces ---------*- C++ -*---===//
      2 //
      3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
      4 // See https://llvm.org/LICENSE.txt for license information.
      5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
      6 //
      7 //===----------------------------------------------------------------------===//
      8 // Common definitions/declarations used internally by coroutine lowering passes.
      9 //===----------------------------------------------------------------------===//
     10 
     11 #ifndef LLVM_LIB_TRANSFORMS_COROUTINES_COROINTERNAL_H
     12 #define LLVM_LIB_TRANSFORMS_COROUTINES_COROINTERNAL_H
     13 
     14 #include "CoroInstr.h"
     15 #include "llvm/IR/IRBuilder.h"
     16 #include "llvm/Transforms/Coroutines.h"
     17 
     18 namespace llvm {
     19 
     20 class CallGraph;
     21 class CallGraphSCC;
     22 class PassRegistry;
     23 
     24 void initializeCoroEarlyLegacyPass(PassRegistry &);
     25 void initializeCoroSplitLegacyPass(PassRegistry &);
     26 void initializeCoroElideLegacyPass(PassRegistry &);
     27 void initializeCoroCleanupLegacyPass(PassRegistry &);
     28 
     29 // CoroEarly pass marks every function that has coro.begin with a string
     30 // attribute "coroutine.presplit"="0". CoroSplit pass processes the coroutine
     31 // twice. First, it lets it go through complete IPO optimization pipeline as a
     32 // single function. It forces restart of the pipeline by inserting an indirect
     33 // call to an empty function "coro.devirt.trigger" which is devirtualized by
     34 // CoroElide pass that triggers a restart of the pipeline by CGPassManager.
     35 // When CoroSplit pass sees the same coroutine the second time, it splits it up,
     36 // adds coroutine subfunctions to the SCC to be processed by IPO pipeline.
     37 // Async lowering similarily triggers a restart of the pipeline after it has
     38 // split the coroutine.
     39 #define CORO_PRESPLIT_ATTR "coroutine.presplit"
     40 #define UNPREPARED_FOR_SPLIT "0"
     41 #define PREPARED_FOR_SPLIT "1"
     42 #define ASYNC_RESTART_AFTER_SPLIT "2"
     43 
     44 #define CORO_DEVIRT_TRIGGER_FN "coro.devirt.trigger"
     45 
     46 namespace coro {
     47 
     48 bool declaresIntrinsics(const Module &M,
     49                         const std::initializer_list<StringRef>);
     50 void replaceCoroFree(CoroIdInst *CoroId, bool Elide);
     51 void updateCallGraph(Function &Caller, ArrayRef<Function *> Funcs,
     52                      CallGraph &CG, CallGraphSCC &SCC);
     53 /// Recover a dbg.declare prepared by the frontend and emit an alloca
     54 /// holding a pointer to the coroutine frame.
     55 void salvageDebugInfo(
     56     SmallDenseMap<llvm::Value *, llvm::AllocaInst *, 4> &DbgPtrAllocaCache,
     57     DbgVariableIntrinsic *DVI, bool ReuseFrameSlot);
     58 
     59 // Keeps data and helper functions for lowering coroutine intrinsics.
     60 struct LowererBase {
     61   Module &TheModule;
     62   LLVMContext &Context;
     63   PointerType *const Int8Ptr;
     64   FunctionType *const ResumeFnType;
     65   ConstantPointerNull *const NullPtr;
     66 
     67   LowererBase(Module &M);
     68   Value *makeSubFnCall(Value *Arg, int Index, Instruction *InsertPt);
     69 };
     70 
     71 enum class ABI {
     72   /// The "resume-switch" lowering, where there are separate resume and
     73   /// destroy functions that are shared between all suspend points.  The
     74   /// coroutine frame implicitly stores the resume and destroy functions,
     75   /// the current index, and any promise value.
     76   Switch,
     77 
     78   /// The "returned-continuation" lowering, where each suspend point creates a
     79   /// single continuation function that is used for both resuming and
     80   /// destroying.  Does not support promises.
     81   Retcon,
     82 
     83   /// The "unique returned-continuation" lowering, where each suspend point
     84   /// creates a single continuation function that is used for both resuming
     85   /// and destroying.  Does not support promises.  The function is known to
     86   /// suspend at most once during its execution, and the return value of
     87   /// the continuation is void.
     88   RetconOnce,
     89 
     90   /// The "async continuation" lowering, where each suspend point creates a
     91   /// single continuation function. The continuation function is available as an
     92   /// intrinsic.
     93   Async,
     94 };
     95 
     96 // Holds structural Coroutine Intrinsics for a particular function and other
     97 // values used during CoroSplit pass.
     98 struct LLVM_LIBRARY_VISIBILITY Shape {
     99   CoroBeginInst *CoroBegin;
    100   SmallVector<AnyCoroEndInst *, 4> CoroEnds;
    101   SmallVector<CoroSizeInst *, 2> CoroSizes;
    102   SmallVector<AnyCoroSuspendInst *, 4> CoroSuspends;
    103   SmallVector<CallInst*, 2> SwiftErrorOps;
    104 
    105   // Field indexes for special fields in the switch lowering.
    106   struct SwitchFieldIndex {
    107     enum {
    108       Resume,
    109       Destroy
    110 
    111       // The promise field is always at a fixed offset from the start of
    112       // frame given its type, but the index isn't a constant for all
    113       // possible frames.
    114 
    115       // The switch-index field isn't at a fixed offset or index, either;
    116       // we just work it in where it fits best.
    117     };
    118   };
    119 
    120   coro::ABI ABI;
    121 
    122   StructType *FrameTy;
    123   Align FrameAlign;
    124   uint64_t FrameSize;
    125   Instruction *FramePtr;
    126   BasicBlock *AllocaSpillBlock;
    127 
    128   /// This would only be true if optimization are enabled.
    129   bool ReuseFrameSlot;
    130 
    131   struct SwitchLoweringStorage {
    132     SwitchInst *ResumeSwitch;
    133     AllocaInst *PromiseAlloca;
    134     BasicBlock *ResumeEntryBlock;
    135     unsigned IndexField;
    136     unsigned IndexAlign;
    137     unsigned IndexOffset;
    138     bool HasFinalSuspend;
    139   };
    140 
    141   struct RetconLoweringStorage {
    142     Function *ResumePrototype;
    143     Function *Alloc;
    144     Function *Dealloc;
    145     BasicBlock *ReturnBlock;
    146     bool IsFrameInlineInStorage;
    147   };
    148 
    149   struct AsyncLoweringStorage {
    150     FunctionType *AsyncFuncTy;
    151     Value *Context;
    152     CallingConv::ID AsyncCC;
    153     unsigned ContextArgNo;
    154     uint64_t ContextHeaderSize;
    155     uint64_t ContextAlignment;
    156     uint64_t FrameOffset; // Start of the frame.
    157     uint64_t ContextSize; // Includes frame size.
    158     GlobalVariable *AsyncFuncPointer;
    159 
    160     Align getContextAlignment() const { return Align(ContextAlignment); }
    161   };
    162 
    163   union {
    164     SwitchLoweringStorage SwitchLowering;
    165     RetconLoweringStorage RetconLowering;
    166     AsyncLoweringStorage AsyncLowering;
    167   };
    168 
    169   CoroIdInst *getSwitchCoroId() const {
    170     assert(ABI == coro::ABI::Switch);
    171     return cast<CoroIdInst>(CoroBegin->getId());
    172   }
    173 
    174   AnyCoroIdRetconInst *getRetconCoroId() const {
    175     assert(ABI == coro::ABI::Retcon ||
    176            ABI == coro::ABI::RetconOnce);
    177     return cast<AnyCoroIdRetconInst>(CoroBegin->getId());
    178   }
    179 
    180   CoroIdAsyncInst *getAsyncCoroId() const {
    181     assert(ABI == coro::ABI::Async);
    182     return cast<CoroIdAsyncInst>(CoroBegin->getId());
    183   }
    184 
    185   unsigned getSwitchIndexField() const {
    186     assert(ABI == coro::ABI::Switch);
    187     assert(FrameTy && "frame type not assigned");
    188     return SwitchLowering.IndexField;
    189   }
    190   IntegerType *getIndexType() const {
    191     assert(ABI == coro::ABI::Switch);
    192     assert(FrameTy && "frame type not assigned");
    193     return cast<IntegerType>(FrameTy->getElementType(getSwitchIndexField()));
    194   }
    195   ConstantInt *getIndex(uint64_t Value) const {
    196     return ConstantInt::get(getIndexType(), Value);
    197   }
    198 
    199   PointerType *getSwitchResumePointerType() const {
    200     assert(ABI == coro::ABI::Switch);
    201   assert(FrameTy && "frame type not assigned");
    202   return cast<PointerType>(FrameTy->getElementType(SwitchFieldIndex::Resume));
    203   }
    204 
    205   FunctionType *getResumeFunctionType() const {
    206     switch (ABI) {
    207     case coro::ABI::Switch: {
    208       auto *FnPtrTy = getSwitchResumePointerType();
    209       return cast<FunctionType>(FnPtrTy->getPointerElementType());
    210     }
    211     case coro::ABI::Retcon:
    212     case coro::ABI::RetconOnce:
    213       return RetconLowering.ResumePrototype->getFunctionType();
    214     case coro::ABI::Async:
    215       // Not used. The function type depends on the active suspend.
    216       return nullptr;
    217     }
    218 
    219     llvm_unreachable("Unknown coro::ABI enum");
    220   }
    221 
    222   ArrayRef<Type*> getRetconResultTypes() const {
    223     assert(ABI == coro::ABI::Retcon ||
    224            ABI == coro::ABI::RetconOnce);
    225     auto FTy = CoroBegin->getFunction()->getFunctionType();
    226 
    227     // The safety of all this is checked by checkWFRetconPrototype.
    228     if (auto STy = dyn_cast<StructType>(FTy->getReturnType())) {
    229       return STy->elements().slice(1);
    230     } else {
    231       return ArrayRef<Type*>();
    232     }
    233   }
    234 
    235   ArrayRef<Type*> getRetconResumeTypes() const {
    236     assert(ABI == coro::ABI::Retcon ||
    237            ABI == coro::ABI::RetconOnce);
    238 
    239     // The safety of all this is checked by checkWFRetconPrototype.
    240     auto FTy = RetconLowering.ResumePrototype->getFunctionType();
    241     return FTy->params().slice(1);
    242   }
    243 
    244   CallingConv::ID getResumeFunctionCC() const {
    245     switch (ABI) {
    246     case coro::ABI::Switch:
    247       return CallingConv::Fast;
    248 
    249     case coro::ABI::Retcon:
    250     case coro::ABI::RetconOnce:
    251       return RetconLowering.ResumePrototype->getCallingConv();
    252     case coro::ABI::Async:
    253       return AsyncLowering.AsyncCC;
    254     }
    255     llvm_unreachable("Unknown coro::ABI enum");
    256   }
    257 
    258   AllocaInst *getPromiseAlloca() const {
    259     if (ABI == coro::ABI::Switch)
    260       return SwitchLowering.PromiseAlloca;
    261     return nullptr;
    262   }
    263 
    264   /// Allocate memory according to the rules of the active lowering.
    265   ///
    266   /// \param CG - if non-null, will be updated for the new call
    267   Value *emitAlloc(IRBuilder<> &Builder, Value *Size, CallGraph *CG) const;
    268 
    269   /// Deallocate memory according to the rules of the active lowering.
    270   ///
    271   /// \param CG - if non-null, will be updated for the new call
    272   void emitDealloc(IRBuilder<> &Builder, Value *Ptr, CallGraph *CG) const;
    273 
    274   Shape() = default;
    275   explicit Shape(Function &F, bool ReuseFrameSlot = false)
    276       : ReuseFrameSlot(ReuseFrameSlot) {
    277     buildFrom(F);
    278   }
    279   void buildFrom(Function &F);
    280 };
    281 
    282 void buildCoroutineFrame(Function &F, Shape &Shape);
    283 CallInst *createMustTailCall(DebugLoc Loc, Function *MustTailCallFn,
    284                              ArrayRef<Value *> Arguments, IRBuilder<> &);
    285 } // End namespace coro.
    286 } // End namespace llvm
    287 
    288 #endif
    289