Lines Matching defs:Lexer
1 //===- Lexer.cpp - C Language Family Lexer --------------------------------===//
9 // This file implements the Lexer and Token interfaces.
13 #include "clang/Lex/Lexer.h"
74 // Lexer Class Implementation
77 void Lexer::anchor() {}
79 void Lexer::InitLexer(const char *BufStart, const char *BufPtr,
131 /// Lexer constructor - Create a new lexer object for the specified buffer
132 /// with the specified preprocessor managing the lexing process. This lexer
135 Lexer::Lexer(FileID FID, const llvm::MemoryBufferRef &InputFile,
146 /// Lexer constructor - Create a new raw lexer object. This object is only
147 /// suitable for calls to 'LexFromRawLexer'. This lexer assumes that the text
149 Lexer::Lexer(SourceLocation fileloc, const LangOptions &langOpts,
158 /// Lexer constructor - Create a new raw lexer object. This object is only
159 /// suitable for calls to 'LexFromRawLexer'. This lexer assumes that the text
161 Lexer::Lexer(FileID FID, const llvm::MemoryBufferRef &FromFile,
163 : Lexer(SM.getLocForStartOfFile(FID), langOpts, FromFile.getBufferStart(),
166 void Lexer::resetExtendedTokenMode() {
174 /// Create_PragmaLexer: Lexer constructor - Create a new lexer object for
176 /// sets up. It returns a new'd Lexer that must be delete'd when done.
184 /// normal lexer that remaps tokens as they fly by. This would require making
185 /// Preprocessor::Lex virtual. Given that, we could just dump in a magic lexer
187 /// out of the critical path of the lexer!
189 Lexer *Lexer::Create_PragmaLexer(SourceLocation SpellingLoc,
195 // Create the lexer as if we were going to lex the file normally.
198 Lexer *L = new Lexer(SpellingFID, InputFile, PP);
200 // Now that the lexer is created, change the start/end locations so that we
215 // Ensure that the lexer thinks it is inside a directive, so that end \n will
219 // This lexer really is for _Pragma.
224 bool Lexer::skipOver(unsigned NumBytes) {
258 std::string Lexer::Stringify(StringRef Str, bool Charify) {
265 void Lexer::Stringify(SmallVectorImpl<char> &Str) { StringifyImpl(Str, '"'); }
284 Spelling[Length++] = Lexer::getCharAndSizeNoWarn(BufPtr, Size, LangOpts);
313 Spelling[Length++] = Lexer::getCharAndSizeNoWarn(BufPtr, Size, LangOpts);
327 StringRef Lexer::getSpelling(SourceLocation loc,
346 Lexer lexer(SM.getLocForStartOfFile(locInfo.first), options,
349 lexer.LexFromRawLexer(token);
368 std::string Lexer::getSpelling(const Token &Tok, const SourceManager &SourceMgr,
400 unsigned Lexer::getSpelling(const Token &Tok, const char *&Buffer,
422 // Compute the start of the token in the input lexer buffer.
447 unsigned Lexer::MeasureTokenLength(SourceLocation Loc,
458 bool Lexer::getRawToken(SourceLocation Loc, Token &Result,
465 // Lexer::isObviouslySimpleCharacter for example to handle identifiers or
482 // Create a lexer starting at the beginning of this token.
483 Lexer TheLexer(SM.getLocForStartOfFile(LocInfo.first), LangOpts,
500 !Lexer::isNewLineEscaped(BufStart, LexStart)) {
529 // Create a lexer starting at the beginning of this token.
531 Lexer TheLexer(LexerStartLoc, LangOpts, Buffer.data(), LexStart,
541 // Lexing this token has taken the lexer past the source location we're
557 SourceLocation Lexer::GetBeginningOfToken(SourceLocation Loc,
585 PreambleBounds Lexer::ComputePreamble(StringRef Buffer,
588 // Create a lexer starting at the beginning of the file. Note that we use a
589 // "fake" file source location at offset 1 so that the lexer will track our
593 Lexer TheLexer(FileLoc, LangOpts, Buffer.begin(), Buffer.begin(),
721 unsigned Lexer::getTokenPrefixLength(SourceLocation TokStart, unsigned CharNo,
731 if (Invalid || (CharNo == 0 && Lexer::isObviouslySimpleCharacter(*TokPtr)))
739 while (Lexer::isObviouslySimpleCharacter(*TokPtr)) {
748 // lexer to parse it correctly.
751 Lexer::getCharAndSizeNoWarn(TokPtr, Size, LangOpts);
760 if (!Lexer::isObviouslySimpleCharacter(*TokPtr))
761 PhysOffset += Lexer::SkipEscapedNewLines(TokPtr)-TokPtr;
781 SourceLocation Lexer::getLocForEndOfToken(SourceLocation Loc, unsigned Offset,
792 unsigned Len = Lexer::MeasureTokenLength(Loc, SM, LangOpts);
803 bool Lexer::isAtStartOfMacroExpansion(SourceLocation loc,
825 bool Lexer::isAtEndOfMacroExpansion(SourceLocation loc,
858 End = Lexer::getLocForEndOfToken(End, 0, SM,LangOpts);
878 CharSourceRange Lexer::makeFileCharRange(CharSourceRange Range,
942 StringRef Lexer::getSourceText(CharSourceRange Range,
978 StringRef Lexer::getImmediateMacroName(SourceLocation Loc,
1020 unsigned MacroTokenLength = Lexer::MeasureTokenLength(Loc, SM, LangOpts);
1025 StringRef Lexer::getImmediateMacroNameForDiagnostics(
1045 unsigned MacroTokenLength = Lexer::MeasureTokenLength(Loc, SM, LangOpts);
1050 bool Lexer::isIdentifierBodyChar(char c, const LangOptions &LangOpts) {
1054 bool Lexer::isNewLineEscaped(const char *BufferStart, const char *Str) {
1074 StringRef Lexer::getIndentationForLine(SourceLocation Loc,
1100 /// lexer buffer was all expanded at a single point, perform the mapping.
1129 SourceLocation Lexer::getSourceLocation(const char *Loc,
1140 // Otherwise, this is the _Pragma lexer case, which pretends that all of the
1148 DiagnosticBuilder Lexer::Diag(const char *Loc, unsigned DiagID) const {
1177 static char DecodeTrigraphChar(const char *CP, Lexer *L) {
1195 unsigned Lexer::getEscapedNewLineSize(const char *Ptr) {
1218 const char *Lexer::SkipEscapedNewLines(const char *P) {
1234 unsigned NewLineSize = Lexer::getEscapedNewLineSize(AfterEscape);
1240 Optional<Token> Lexer::findNextToken(SourceLocation Loc,
1244 if (!Lexer::isAtEndOfMacroExpansion(Loc, SM, LangOpts, &Loc))
1247 Loc = Lexer::getLocForEndOfToken(Loc, 0, SM, LangOpts);
1261 Lexer lexer(SM.getLocForStartOfFile(LocInfo.first), LangOpts, File.begin(),
1265 lexer.LexFromRawLexer(Tok);
1273 SourceLocation Lexer::findLocationAfterToken(
1319 char Lexer::getCharAndSizeSlow(const char *Ptr, unsigned &Size,
1377 char Lexer::getCharAndSizeSlowNoWarn(const char *Ptr, unsigned &Size,
1423 void Lexer::SetByteOffset(unsigned Offset, bool StartOfLine) {
1471 static inline CharSourceRange makeCharRange(Lexer &L, const char *Begin,
1596 bool Lexer::tryConsumeIdentifierUCN(const char *&CurPtr, unsigned Size,
1618 bool Lexer::tryConsumeIdentifierUTF8Char(const char *&CurPtr) {
1642 bool Lexer::LexIdentifier(Token &Result, const char *CurPtr) {
1743 bool Lexer::isHexaLiteral(const char *Start, const LangOptions &LangOpts) {
1745 char C1 = Lexer::getCharAndSizeNoWarn(Start, Size, LangOpts);
1748 char C2 = Lexer::getCharAndSizeNoWarn(Start + Size, Size, LangOpts);
1755 bool Lexer::LexNumericConstant(Token &Result, const char *CurPtr) {
1820 const char *Lexer::LexUDSuffix(Token &Result, const char *CurPtr,
1911 bool Lexer::LexStringLiteral(Token &Result, const char *CurPtr,
1973 bool Lexer::LexRawStringLiteral(Token &Result, const char *CurPtr,
2000 // Search for the next '"' in hopes of salvaging the lexer. Unfortunately,
2053 bool Lexer::LexAngledStringLiteral(Token &Result, const char *CurPtr) {
2095 void Lexer::codeCompleteIncludedFile(const char *PathStart,
2130 bool Lexer::LexCharConstant(Token &Result, const char *CurPtr,
2197 bool Lexer::SkipWhitespace(Token &Result, const char *CurPtr,
2273 bool Lexer::SkipLineComment(Token &Result, const char *CurPtr,
2390 // return immediately, so that the lexer can return this as an EOD token.
2414 bool Lexer::SaveLineComment(Token &Result, const char *CurPtr) {
2443 Lexer *L) {
2516 bool Lexer::SkipBlockComment(Token &Result, const char *CurPtr,
2683 /// uninterpreted string. This switches the lexer out of directive mode.
2684 void Lexer::ReadToEndOfLine(SmallVectorImpl<char> *Result) {
2740 bool Lexer::LexEndOfFile(Token &Result, const char *CurPtr) {
2811 /// the specified lexer will return a tok::l_paren token, 0 if it is something
2813 /// lexer.
2814 unsigned Lexer::isNextPPTokenLParen() {
2839 // Restore the lexer back to non-skipping mode.
2871 bool Lexer::IsStartOfConflictMarker(const char *CurPtr) {
2915 bool Lexer::HandleEndOfConflictMarker(const char *CurPtr) {
2964 bool Lexer::lexEditorPlaceholder(Token &Result, const char *CurPtr) {
2983 bool Lexer::isCodeCompletionPoint(const char *CurPtr) const {
2992 uint32_t Lexer::tryReadUCN(const char *&StartPtr, const char *SlashLoc,
3103 bool Lexer::CheckUnicodeWhitespace(Token &Result, uint32_t C,
3118 bool Lexer::LexUnicode(Token &Result, uint32_t C, const char *CurPtr) {
3159 void Lexer::PropagateLineStartLeadingSpaceInfo(Token &Result) {
3166 bool Lexer::Lex(Token &Result) {
3191 // (After the LexTokenInternal call, the lexer might be destroyed.)
3196 /// LexTokenInternal - This implements a simple C family lexer. It is an
3201 bool Lexer::LexTokenInternal(Token &Result, bool TokAtPhysicalStartOfLine) {
3217 // skipped. The next lexer invocation will return the token after the
3258 // We know the lexer hasn't changed, so just try again with this lexer.
3304 // We only saw whitespace, so just try again with this lexer.
3334 // We only saw whitespace, so just try again with this lexer.
3653 // We only saw whitespace, so just try again with this lexer.
3904 // We only saw whitespace, so just try again with this lexer.
3937 // We only saw whitespace, so just try again with this lexer.
3958 // this lexer.
Indexes created Mon Apr 20 00:23:12 UTC 2026