Home | History | Annotate | Download | only in Syntax

Lines Matching refs:Tokens

30 #include "clang/Tooling/Syntax/Tokens.h"
356 /// Initially all tokens are added as pending nodes. When processing a clang AST
379   /// Populate children for \p New node, assuming it covers tokens from \p
401   /// Populate children for \p New list, assuming it covers tokens from a
438     auto Tokens = Arena.getTokenBuffer().expandedTokens();
439     assert(!Tokens.empty());
440     assert(Tokens.back().kind() == tok::eof);
443     Pending.foldChildren(Arena, Tokens.drop_back(),
454   /// Finds the syntax tokens corresponding to the \p SourceRange.
460   /// Finds the syntax tokens corresponding to the passed source locations.
474     auto Tokens = getRange(D->getSourceRange());
475     return maybeAppendSemicolon(Tokens, D);
507     ArrayRef<syntax::Token> Tokens;
510       Tokens = getRange(S->TypeDecl::getBeginLoc(), S->getEndLoc());
512       Tokens = getRange(D->getSourceRange());
513     return maybeAppendSemicolon(Tokens, D);
523     auto Tokens = getRange(S->getSourceRange());
525       return Tokens;
529     if (Tokens.back().kind() == tok::semi)
530       return Tokens;
531     return withTrailingSemicolon(Tokens);
535   ArrayRef<syntax::Token> maybeAppendSemicolon(ArrayRef<syntax::Token> Tokens,
538       return Tokens;
540       return Tokens;
543     return withTrailingSemicolon(Tokens);
547   withTrailingSemicolon(ArrayRef<syntax::Token> Tokens) const {
548     assert(!Tokens.empty());
549     assert(Tokens.back().kind() != tok::eof);
551     if (Tokens.back().kind() != tok::semi && Tokens.end()->kind() == tok::semi)
552       return llvm::makeArrayRef(Tokens.begin(), Tokens.end() + 1);
553     return Tokens;
561   /// A collection of trees covering the input tokens.
594     /// Shrink \p Range to a subrange that only contains tokens of a list.
623     /// Add \p Node to the forest and attach child nodes based on \p Tokens.
624     void foldChildren(const syntax::Arena &A, ArrayRef<syntax::Token> Tokens,
629       auto *FirstToken = Tokens.begin();
635       auto EndChildren = Trees.lower_bound(Tokens.end());
637           (EndChildren == Trees.end() || EndChildren->first == Tokens.end()) &&
650           A.getTokenBuffer().spelledForExpanded(Tokens).hasValue();
656     // EXPECTS: all tokens were consumed and are owned by a single root node.
673             formatv("- '{0}' covers '{1}'+{2} tokens\n", It->second->getKind(),
682     /// Keys in the map are pointers into the array of expanded tokens, so
683     /// pointer order corresponds to the order of preprocessor tokens.
691   /// To quickly find tokens by their start location.
1321     auto Tokens = Builder.getDeclarationRange(S);
1322     if (Tokens.front().kind() == tok::coloncolon) {
1328     Builder.foldNode(Tokens, new (allocator()) syntax::NamespaceDefinition, S);
1637     auto Tokens = llvm::makeArrayRef(Arrow, Return.end());
1642     Builder.foldNode(Tokens, R, L);