Cross Reference: /src/external/apache2/llvm/dist/clang/lib/Tooling/Syntax/Tokens.cpp

Lines Matching refs:Tokens
1 //===- Tokens.cpp - collect tokens from preprocessing ---------------------===//
8 #include "clang/Tooling/Syntax/Tokens.h"
58 // Finds the smallest expansion range that contains expanded tokens First and
70   // Keep traversing up the spelling chain as longs as tokens are part of the
85     // Update the result afterwards, as we want the tokens that triggered the
90   // we've already found some tokens expanded from the same macro argument, and
91   // they should map to a consecutive subset of spelled tokens. Unfortunately
94   // referring to same tokens, SourceManager might tell us that one is before
132   assert(F.file() == L.file() && "tokens from different files");
134          "wrong order of tokens");
241   // Find the first mapping that produced tokens after \p Expanded.
295     // This mapping applies to Spelled tokens.
297       // Spelled tokens don't cover the entire mapping, returning empty result.
300     // Spelled tokens start at the beginning of this mapping.
303     // Spelled tokens start after the mapping ends (they start in the hole
317     // This mapping applies to Spelled tokens.
319       // Spelled tokens don't cover the entire mapping, returning empty result.
324     // Spelled tokens end after the mapping ends.
356       llvm::formatv("spelled tokens: [{0},{1}), expanded tokens: [{2},{3})",
384   // If both tokens are coming from a macro argument expansion, try and map to
393     // It might be the case that tokens are arguments of different macro calls,
435          "Tokens not in spelled range");
478                               llvm::ArrayRef<syntax::Token> Tokens) {
482       Tokens, [&](const syntax::Token &Tok) { return Tok.location() < Loc; });
483   bool AcceptRight = Right != Tokens.end() && Right->location() <= Loc;
485       Right != Tokens.begin() && (Right - 1)->endLocation() >= Loc;
492                               const syntax::TokenBuffer &Tokens) {
494       Loc, Tokens.spelledTokens(Tokens.sourceManager().getFileID(Loc)));
499                                   llvm::ArrayRef<syntax::Token> Tokens) {
500   for (const syntax::Token &Tok : spelledTokensTouching(Loc, Tokens)) {
509                                   const syntax::TokenBuffer &Tokens) {
511       Loc, Tokens.spelledTokens(Tokens.sourceManager().getFileID(Loc)));
532   std::vector<syntax::Token> Tokens;
542     Tokens.push_back(syntax::Token(T));
559   return Tokens;
583     // Only record top-level expansions that directly produce expanded tokens.
592     // Both A and B produce expanded tokens, though the macro name 'B' comes
625 /// implementation tracks the tokens, macro expansions and directives coming
628 ///   spelled token stream or both. Stores the tokens appropriately.
642     DEBUG_WITH_TYPE("collect-tokens", llvm::dbgs()
658 /// Builds mappings and spelled tokens in the TokenBuffer based on the expanded
673     // Tokenize every file that contributed tokens to the expanded stream.
676     // The expanded token stream consists of runs of tokens that came from
678     // Between these runs are the logical positions of spelled tokens that
681       // Create empty mappings for spelled tokens that expanded to nothing here.
684       // Create mapping for a contiguous run of expanded tokens.
691     // If any tokens remain in any of the files, they didn't expand to anything.
713   // Consume a sequence of spelled tokens that didn't expand to anything.
714   // In the simplest case, skips spelled tokens until finding one that produced
716   // If Drain is provided, skips remaining tokens from that file instead.
728     // When dropping trailing tokens from a file, the empty mapping should
764   // (unless this is a run of file tokens, which we represent with no mapping).
773       // A run of file tokens continues while the expanded/spelled tokens match.
781       // We need no mapping for file tokens copied to the expanded stream.
791       // ... consumes spelled tokens within bounds we captured ...
795       // ... consumes expanded tokens rooted at the same expansion ...
818     llvm_unreachable("Couldn't map expanded token to spelled tokens!");
821   /// Initializes TokenBuffer::Files and fills spelled tokens and expanded
875                                         llvm::ArrayRef<syntax::Token> Tokens) {
876     if (Tokens.empty()) {
880     OS << Tokens[0].text(*SourceMgr);
881     for (unsigned I = 1; I < Tokens.size(); ++I) {
882       if (Tokens[I].kind() == tok::eof)
884       OS << " " << PrintToken(Tokens[I]);
891   OS << "expanded tokens:\n"
908        << "  spelled tokens:\n"
OpenGrok
