Lines Matching defs:TOKEN
3065 ** complete if it ends with a semicolon token and is not a prefix of a
3068 ** independent tokens (they are part of the token in which they are
4284 ** ^If the most recent error references a specific token in the input
4286 ** of the start of that token. ^The byte offset returned by
4288 ** ^If the most recent error does not reference a specific token in the input
13169 ** to the column in which it occurs and *piOff the token offset of the
13170 ** first token of the phrase. SQLITE_OK is returned if successful, or an
13317 ** This is used to access token iToken of phrase iPhrase of the current
13319 ** to a buffer containing the requested token, and *pnToken to the
13329 ** token. It is the output of the tokenizer module. For tokendata=1
13333 ** This is used to access token iToken of phrase hit iIdx within the
13337 ** matching document token, and (*pnToken) to the size of that buffer in
13338 ** bytes. This API is not available if the specified token matches a
13451 ** followed by a "*" character, indicating that the last token
13452 ** returned by the tokenizer will be treated as a token prefix.
13460 ** For each token in the input string, the supplied callback xToken() must
13463 ** arguments are a pointer to a buffer containing the token text, and the
13464 ** size of the token in bytes. The 4th and 5th arguments are the byte offsets
13466 ** which the token is derived within the input.
13472 ** FTS5 assumes the xToken() callback is invoked for each token in the
13495 ** <ol><li> By mapping all synonyms to a single token. In this case, using
13497 ** same token for inputs "first" and "1st". Say that token is in
13514 ** first token in the MATCH query and FTS5 effectively runs a query
13526 ** provides multiple synonyms for each token. So that when a
13535 ** FTS index corresponding to both forms of the first token.
13540 ** is considered to supply a synonym for the previous token. For example,
13553 ** xToken() is called. Multiple synonyms may be specified for a single token
13556 ** single token.
13562 ** token "first" is substituted for "1st" by the tokenizer, then the query:
13567 ** will not match documents that contain the token "1st" (as the tokenizer
13578 ** token "1st", but not "first" (assuming the tokenizer is not able to
13602 const char *pToken, /* Pointer to buffer containing token */
13603 int nToken, /* Size of token in bytes */
13604 int iStart, /* Byte offset of token within input text */
13605 int iEnd /* Byte offset of end of token within input text */
13618 #define FTS5_TOKEN_COLOCATED 0x0001 /* Same position as prev. token */
15339 typedef struct Token Token;
16489 void *token; /* id that may be used to recursive triggers */
18749 #define SQLITE_TOKEN_QUOTED 0x1 /* Token is a quoted identifier. */
18750 #define SQLITE_TOKEN_KEYWORD 0x2 /* Token is a keyword. */
18753 ** Each token coming out of the lexer is an instance of
18758 ** the Token goes out of scope! Very often, the "z" points to some place
18762 struct Token {
18763 const char *z; /* Text of the token. Not NULL-terminated! */
18764 unsigned int n; /* Number of characters in this token */
18849 ** Expr.op is the opcode. The integer parser token codes are reused
18920 char *zToken; /* Token value. Zero terminated and dequoted */
18957 int iOfst; /* else: start of token from start of statement */
18984 #define EP_DblQuoted 0x000080 /* token.z was originally in "..." */
19112 char *zEName; /* Token associated with this expression */
19753 Token constraintName;/* Name of the constraint currently being parsed */
19793 Token sNameToken; /* Token with unqualified schema object name */
19802 Token sLastToken; /* The last token parsed */
19824 Token sArg; /* Complete text of a module argument */
20245 const Token *pName; /* Name of the container - used for error messages */
20423 SQLITE_PRIVATE Window *sqlite3WindowAssemble(Parse*, Window*, ExprList*, ExprList*, Token*);
20720 SQLITE_PRIVATE void sqlite3DequoteToken(Token*);
20721 SQLITE_PRIVATE void sqlite3TokenInit(Token*,char*);
20737 SQLITE_PRIVATE Expr *sqlite3ExprAlloc(sqlite3*,int,const Token*,int);
20744 SQLITE_PRIVATE Expr *sqlite3ExprFunction(Parse*,ExprList*, const Token*, int);
20757 SQLITE_PRIVATE void sqlite3ExprListSetName(Parse*,ExprList*,const Token*,int);
20766 SQLITE_PRIVATE void sqlite3Pragma(Parse*,Token*,Token*,Token*,int);
20793 SQLITE_PRIVATE void sqlite3StartTable(Parse*,Token*,Token*,int,int,int,int);
20799 SQLITE_PRIVATE void sqlite3AddColumn(Parse*,Token,Token);
20804 SQLITE_PRIVATE void sqlite3AddCollateType(Parse*, Token*);
20805 SQLITE_PRIVATE void sqlite3AddGenerated(Parse*,Expr*,Token*);
20806 SQLITE_PRIVATE void sqlite3EndTable(Parse*,Token*,Token*,u32,Select*);
20837 SQLITE_PRIVATE void sqlite3CreateView(Parse*,Token*,Token*,Token*,ExprList*,Select*,int,int);
20865 SQLITE_PRIVATE IdList *sqlite3IdListAppend(Parse*, IdList*, Token*);
20869 SQLITE_PRIVATE SrcList *sqlite3SrcListAppend(Parse*, SrcList*, Token*, Token*);
20870 SQLITE_PRIVATE SrcList *sqlite3SrcListAppendFromTerm(Parse*, SrcList*, Token*, Token*,
20871 Token*, Select*, OnOrUsing*);
20872 SQLITE_PRIVATE void sqlite3SrcListIndexedBy(Parse *, SrcList *, Token *);
20881 SQLITE_PRIVATE void sqlite3CreateIndex(Parse*,Token*,Token*,SrcList*,ExprList*,int,Token*,
20945 SQLITE_PRIVATE void sqlite3Vacuum(Parse*,Token*,Expr*);
20947 SQLITE_PRIVATE char *sqlite3NameFromToken(sqlite3*, const Token*);
20968 SQLITE_PRIVATE void sqlite3Savepoint(Parse*, int, Token*);
21035 SQLITE_PRIVATE void sqlite3BeginTrigger(Parse*, Token*,Token*,int,int,IdList*,SrcList*,
21037 SQLITE_PRIVATE void sqlite3FinishTrigger(Parse*, TriggerStep*, Token*);
21049 SQLITE_PRIVATE TriggerStep *sqlite3TriggerInsertStep(Parse*,Token*, IdList*,
21052 SQLITE_PRIVATE TriggerStep *sqlite3TriggerUpdateStep(Parse*,Token*,SrcList*,ExprList*,
21054 SQLITE_PRIVATE TriggerStep *sqlite3TriggerDeleteStep(Parse*,Token*, Expr*,
21076 SQLITE_PRIVATE int sqlite3JoinType(Parse*, Token*, Token*, Token*);
21080 SQLITE_PRIVATE void sqlite3CreateForeignKey(Parse*, ExprList*, Token*, ExprList*, int);
21097 SQLITE_PRIVATE void sqlite3FixInit(DbFixer*, Parse*, int, const char*, const Token*);
21166 SQLITE_PRIVATE int sqlite3TwoPartName(Parse *, Token *, Token *, Token **);
21188 SQLITE_PRIVATE Expr *sqlite3ExprAddCollateToken(const Parse *pParse, Expr*, const Token*, int);
21244 SQLITE_PRIVATE void sqlite3Reindex(Parse*, Token*, Token*);
21246 SQLITE_PRIVATE void sqlite3AlterRenameTable(Parse*, SrcList*, Token*);
21247 SQLITE_PRIVATE void sqlite3AlterRenameColumn(Parse*, SrcList*, Token*, Token*);
21271 SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *, Token *);
21273 SQLITE_PRIVATE void sqlite3AlterDropColumn(Parse*, SrcList*, const Token*);
21274 SQLITE_PRIVATE const void *sqlite3RenameTokenMap(Parse*, const void*, const Token*);
21280 SQLITE_PRIVATE void sqlite3Analyze(Parse*, Token*, Token*);
21282 SQLITE_PRIVATE int sqlite3FindDb(sqlite3*, Token*);
21357 SQLITE_PRIVATE void sqlite3Parser(void*, int, Token);
21427 SQLITE_PRIVATE void sqlite3VtabBeginParse(Parse*, Token*, Token*, Token*, int);
21428 SQLITE_PRIVATE void sqlite3VtabFinishParse(Parse*, Token*);
21430 SQLITE_PRIVATE void sqlite3VtabArgExtend(Parse*, Token*);
21458 SQLITE_PRIVATE Cte *sqlite3CteNew(Parse*,Token*,ExprList*,Select*,u8);
23208 void *token; /* Copy of SubProgram.token */
30870 #define etTOKEN 11 /* a pointer to a Token structure */
31669 /* %T means a Token pointer */
31670 Token *pToken = va_arg(ap, Token*);
31738 ** The z string points to the first character of a token that is
31761 token, record that as
32882 /* Expressions of the form: CAST(pLeft AS token) */
34915 ** If the input token p is quoted, try to adjust the token to remove
34925 SQLITE_PRIVATE void sqlite3DequoteToken(Token *p){
34937 ** Generate a Token object from a string
34939 SQLITE_PRIVATE void sqlite3TokenInit(Token *p, char *z){
84087 ** token (i.e. "5", "5.1", "'a string'"). If the expression can
92492 int n; /* Length of a token prefix */
92493 int nToken; /* Length of the parameter token */
99877 void *t; /* Token identifying trigger */
99892 ** single trigger all have the same value for the SubProgram.token
99895 t = pProgram->token;
99896 for(pFrame=p->pFrame; pFrame && pFrame->token!=t; pFrame=pFrame->pParent);
99943 pFrame->token = pProgram->token;
108565 const Token *pCollName, /* Name of collating sequence */
108583 Token s;
109302 ** If dequote is true, then the token (if it exists) is dequoted.
109304 ** parameter is ignored if pToken is NULL or if the token does not
109309 ** can be translated into a 32-bit integer, then the token is not
109317 const Token *pToken, /* Token argument. Might be NULL */
109359 ** Allocate a new expression node from a zero-terminated token that has
109365 const char *zToken /* Token argument. Might be NULL */
109367 Token x;
109549 const Token *pToken, /* Name of the function */
109858 ** dupedExprNodeSize() Size of Expr + space for token
109859 ** dupedExprSize() Expr + token + subtree components
110547 const Token *pName, /* Name to be added */
110561 ** statement handled by the parser. And so no token need be added
110562 ** to the token-map. */
113023 /* Expressions of the form: CAST(pLeft AS token) */
113919 ** This code depends on the fact that certain token values (ex: TK_EQ)
115537 Token *pName /* The new table name. */
115723 SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){
116009 Token *pOld, /* Name of column being changed */
116010 Token *pNew /* New column name */
116096 ** the token that generated that element. The parse tree element
116108 ** to look up the actual token value that created some element in
116112 const void *p; /* Parse tree element created by token t */
116113 Token t; /* The token that created parse tree element p */
116135 ** rename-token list.
116137 ** 2. Dereferences each pointer in the rename-token list.
116175 ** the token pToken.
116188 const Token *pToken
116208 ** with parse tree element pFrom. This function remaps the associated token
116493 ** corresponding rename-token from Parse object pParse and add it
116518 ** that matches the string in zOld, extract the corresponding rename-token
116588 ** This function edits SQL statement zSql, replacing each token identified
116600 const char *zNew, /* New token text */
116601 int bQuote /* True to always quote token */
116644 int iOff; /* Offset of token to replace in zOut */
116659 /* Dequote the double-quoted token. Then requote it again, this time
116661 ** original token within the input SQL was a single quote ('), then
116663 ** token. This is so that (SELECT "string"'alias') maps to
117562 ** table being edited, and token pName the name of the column to drop.
117564 SQLITE_PRIVATE void sqlite3AlterDropColumn(Parse *pParse, SrcList *pSrc, const Token *pName){
119162 SQLITE_PRIVATE void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){
119169 Token *pTableName;
120250 const Token *pName /* Name of the view, trigger, or index */
121518 ** Given a token, return a string that consists of the text of that
121519 ** token. Space to hold the returned string
121524 ** surround the body of the token are removed.
121530 SQLITE_PRIVATE char *sqlite3NameFromToken(sqlite3 *db, const Token *pName){
121575 ** The token *pName contains the name of a database (either "main" or
121580 SQLITE_PRIVATE int sqlite3FindDb(sqlite3 *db, Token *pName){
121601 ** This routine sets the *ppUnqual pointer to point at the token (pName1 or
121607 Token *pName1, /* The "xxx" in the name "xxx.yyy" or "xxx" */
121608 Token *pName2, /* The "yyy" in the name "xxx.yyy" */
121609 Token **pUnqual /* Write the unqualified object name here */
121831 Token *pName1, /* First part of the name of the table or view */
121832 Token
121843 Token *pName; /* Unqualified name of the table to create */
122108 SQLITE_PRIVATE void sqlite3AddColumn(Parse *pParse, Token sName, Token sType){
122539 Token t;
122557 SQLITE_PRIVATE void sqlite3AddCollateType(Parse *pParse, Token *pToken){
122590 SQLITE_PRIVATE void sqlite3AddGenerated(Parse *pParse, Expr *pExpr, Token *pType){
122997 Token ipkToken;
123244 Token *pCons, /* The ',' token after the last column defn. */
123245 Token *pEnd, /* The ')' before options in the CREATE TABLE */
123493 Token *pEnd2 = tabOpts ? &pParse->sLastToken : pEnd;
123597 Token *pBegin, /* The CREATE token that begins the statement */
123598 Token *pName1, /* The token that holds the name of the view */
123599 Token *pName2, /* The token that holds the name of the view */
123608 Token sEnd;
123610 Token *pName = 0;
123637 ** This will force all the Expr.token.z values to be dynamically
123899 ** token for additional information.
124218 Token *pTo, /* Name of the other table */
124539 Token *pName1, /* First part of index name. May be NULL */
124540 Token *pName2, /* Second part of index name. May be NULL */
124544 Token *pStart, /* The CREATE token that begins this statement */
124560 Token *pName = 0; /* Unqualified name of the index to create */
124734 Token prevCol;
125306 SQLITE_PRIVATE IdList *sqlite3IdListAppend(Parse *pParse, IdList *pList, Token *pToken){
125481 Token *pTable, /* Table to append */
125482 Token *pDatabase /* Database of the table */
125571 ** alias token. If the term is a subquery, then pSubquery is the
125582 Token *pTable, /* Name of the table to add to the FROM clause */
125583 Token *pDatabase, /* Name of the database containing pTable */
125584 Token *pAlias, /* The right-hand side of the AS subexpression */
125605 Token *pToken = (ALWAYS(pDatabase) && pDatabase->z) ? pDatabase : pTable;
125641 Token *pIndexedBy){
125806 SQLITE_PRIVATE void sqlite3Savepoint(Parse *pParse, int op, Token *pName){
126105 SQLITE_PRIVATE void sqlite3Reindex(Parse *pParse, Token *pName1, Token *pName2){
126113 Token *pObjName; /* Name of the table or index to be reindexed */
126212 Token *pName, /* Name of the common-table */
131904 Token tOld = { "old", 3 }; /* Literal "old" token */
131905 Token tNew = { "new", 3 }; /* Literal "new" token */
131906 Token tFromCol; /* Name of column in child table */
131907 Token tToCol; /* Name of column in parent table */
138146 Token *pId1, /* First part of [schema.]id field */
138147 Token *pId2, /* Second part of [schema.]id field, or NULL */
138148 Token *pValue, /* Token for <value>, or NULL */
138154 Token *pId; /* Pointer to <id> token */
142046 SQLITE_PRIVATE int sqlite3JoinType(Parse *pParse, Token *pA, Token *pB, Token *pC){
142048 Token *apAll[3];
142049 Token *p;
142373 static const Token tkCoalesce = { "coalesce", 8 };
147295 Token dummy;
150665 Token *pName1, /* The name of the trigger */
150666 Token *pName2, /* The name of the trigger */
150680 Token *pName; /* The unqualified db name */
150878 Token *pAll /* Token that describes the complete CREATE TRIGGER */
150885 Token nameToken; /* Trigger name for error reporting */
151021 Token *pName, /* The target name */
151053 Token *pTableName, /* Name of the table into which we insert */
151098 Token *pTableName, /* Name of the table to be updated */
151138 Token *pTableName, /* The table from which rows are deleted */
151419 TriggerStep *pStep /* The trigger containing the target token */
151437 Token as;
151817 pProgram->token = (void *)pTrigger;
153845 SQLITE_PRIVATE void sqlite3Vacuum(Parse *pParse, Token *pNm, Expr *pInto){
154543 Token *pName1, /* Name of new table, or database name */
154544 Token *pName2, /* Name of new table or NULL */
154545 Token *pModuleName, /* Name of the module for the virtual table */
154603 SQLITE_PRIVATE void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){
154683 ** The parser calls this routine when it sees the first token
154693 ** The parser calls this routine for each token after the first token
154696 SQLITE_PRIVATE void sqlite3VtabArgExtend(Parse *pParse, Token *p){
154697 Token *pArg = &pParse->sArg;
154800 ** columns of the table to see if any of them contain the token "hidden".
154801 ** If so, set the Column COLFLAG_HIDDEN flag and remove the token from
169108 Token *pBase
169124 ** Window *pWin has just been created from a WINDOW clause. Token pBase
171113 /* Construct a new Expr object from a single token */
171114 static Expr *tokenExpr(Parse *pParse, int op, Token t){
171167 Token *pIdToken,
171187 ***************** Begin token definitions *************************************/
171374 /**************** End token definitions ***************************************/
171391 ** token.
171435 #define sqlite3ParserTOKENTYPE Token
171501 ** current state and lookahead token. These tables are used to implement
171509 ** token onto the stack and goto state N.
172171 ** appears in the grammar, then ID becomes a fallback token for X, Y,
172173 ** but it does not parse, the type of the token is changed to ID and
172375 ** + The value of the token stored at this level of the stack.
172376 ** (In other words, the "major" token.)
172380 ** It is sometimes called the "minor" token.
172388 YYCODETYPE major; /* The major token value. This is the code
172389 ** number for the token at this stack level */
172390 YYMINORTYPE minor; /* The user-supplied minor token value. This
172391 ** is the value of the token */
173412 ** If there is a destructor routine associated with the token which
173474 ** is in state X and has a lookahead token Y. In a well-tested
173511 ** look-ahead token iLookAhead.
173514 YYCODETYPE iLookAhead, /* The look-ahead token */
173535 YYCODETYPE iFallback; /* Fallback token */
173576 ** look-ahead token iLookAhead.
173580 YYCODETYPE iLookAhead /* The look-ahead token */
173653 YYCODETYPE yyMajor, /* The major token to shift in */
173654 sqlite3ParserTOKENTYPE yyMinor /* The minor token to shift in */
174516 ** access to the lookahead token (if any). The yyLookahead will be YYNOCODE
174517 ** if the lookahead token has already been consumed. As this procedure is
174524 int yyLookahead, /* Lookahead token, or YYNOCODE if none */
174525 sqlite3ParserTOKENTYPE yyLookaheadToken /* Value of the lookahead token */
174870 Token x;
175183 Token as;
175308 Token t = yymsp[0].minor.yy0; /*A-overwrites-X*/
175660 Token all;
176130 int yymajor, /* The major type of the error token */
176131 sqlite3ParserTOKENTYPE yyminor /* The minor type of the error token */
176135 #define TOKEN yyminor
176139 if( TOKEN.z[0] ){
176140 sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &TOKEN);
176177 ** The second argument is the major token number. The third is
176178 ** the minor token. The fourth optional argument is whatever the
176185 ** <li> The major token number.
176186 ** <li> The minor token number.
176195 int yymajor, /* The major token code number */
176196 sqlite3ParserTOKENTYPE yyminor /* The value for the token */
176303 ** grammar defines an error token "ERROR".
176327 fprintf(yyTraceFILE,"%sDiscard input token %s\n",
176369 ** * Report an error message, and throw away the input token.
176371 ** * If the input token is $, then fail the parse.
176407 ** Return the fallback token corresponding to canonical token iToken, or
176560 ** it is a keyword. If it is a keyword, the token code of that keyword is
176862 ** return the integer n (the length of the token). */
177101 ** Return the id of the next token in string (*pz). Before returning, set
177102 ** (*pz) to point to the byte following the parsed token.
177106 int t; /* Token type to return */
177126 ** whether the token should be treated as a keyword or an SQL identifier.
177139 ** * the following token is an identifier, or a keyword that can fallback
177141 ** * the token after than one is TK_AS.
177145 ** * the previous token was TK_RP, and
177146 ** * the next token is either TK_LP or an identifier.
177150 ** * the previous token was TK_RP, and
177151 ** * the next token is TK_LP.
177177 ** Return the length (in bytes) of the token that begins at z[0].
177178 ** Store the token type in *tokenType before returning.
177183 ** of the token. See the comment on the CC_ defines
177418 /* This token started out using characters that can appear in keywords,
177478 int n = 0; /* Length of the next token token */
177479 int tokenType; /* type of the next token */
177480 int lastTokenParsed = -1; /* type of the previous token */
177569 Token x;
177572 sqlite3ErrorMsg(pParse, "unrecognized token: \"%T\"", &x);
177651 int n; /* length of current token */
177652 int tokenType; /* type of current token */
177653 int prevType = 0; /* Previous non-whitespace token */
177801 ** Token types used by the sqlite3_complete() routine. See the header
177854 ** (2) tkOTHER Any other SQL token.
177870 u8 token; /* Value of the next token */
177877 /* Token: */
177893 /* Token: */
177911 token = tkSEMI;
177919 token = tkWS;
177924 token = tkOTHER;
177931 token = tkWS;
177936 token = tkOTHER;
177941 token = tkWS;
177948 token = tkOTHER;
177958 token = tkOTHER;
177970 token = tkOTHER;
177975 token = tkCREATE;
177977 token = tkOTHER;
177983 token = tkTRIGGER;
177985 token = tkTEMP;
177987 token = tkTEMP;
177989 token = tkOTHER;
177995 token = tkEND;
177999 token = tkEXPLAIN;
178003 token = tkOTHER;
178008 token = tkOTHER;
178016 token = tkOTHER;
178021 state = trans[state][token];
183652 ** given term. Doclists hold docids and associated token positions.
183678 ** memory. A "position" is an index of a token in the token stream
184034 ** Retrieve the next token from the tokenizer cursor pCursor. This
184040 ** normalized version of the token (i.e. after any case-folding and/or
184042 ** of this buffer in bytes. The input text that generated the token is
184045 ** byte of the token in the input buffer. *piEndOffset should be set
184046 ** to the index of the first byte just past the end of the token in
184059 const char **ppToken, int *pnBytes, /* OUT: Normalized text for token */
184060 int *piStartOffset, /* OUT: Byte offset of token in input buffer */
184061 int *piEndOffset, /* OUT: Byte offset of end of token in input buffer */
184564 ** sequence. A single token is the base case and the most common case.
184569 char *z; /* Text of the token */
184571 int isPrefix; /* True if token ends with a "*" character */
184572 int bFirst; /* True if token must appear at position 0 */
184577 Fts3DeferredToken *pDeferred; /* Deferred token object for this token */
184578 Fts3MultiSegReader *pSegcsr; /* Segment-reader for this token */
184597 Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */
186845 ** Then the position list for this row for token 'b' would consist of:
186855 ** when the *pp1 token appears before the *pp2 token, but not more than nToken
186862 int nToken, /* Maximum difference in token positions */
186977 ** lists are the offsets of the last token in each phrase (tokens "1" and "2"
186986 int nRight, /* Maximum difference in token positions */
186987 int nLeft, /* Maximum difference in token positions */
187292 ** a token in position 0 (of any column). If so, it writes argument iDelta
187700 Fts3PhraseToken *pTok, /* Token to query for */
188738 ** Allocate an Fts3MultiSegReader for each token in the expression headed
188785 ** Arguments pList/nList contain the doclist for token iToken of phrase p.
188797 int iToken, /* Token pList/nList corresponds to */
188899 int iPrev = -1; /* Token number of previous deferred token */
189217 ** Token pToken is an incrementally loaded token that is part of a
189218 ** multi-token phrase. Advance it to the next matching document in the
189227 Fts3Phrase *pPhrase, /* Phrase to advance token of */
189228 int iToken, /* Specific token to advance */
189263 ** * features at least one token that uses an incremental doclist, and
189288 ** one incremental token. In which case the bIncr flag is set. */
189309 /* Advance the iterator for each token in the phrase once. */
189410 ** each phrase in the expression (subject to deferred token processing).
189445 ** token in the FTS expression.
189455 Fts3Phrase *pPhrase; /* The phrase the token belongs to */
189456 int iToken; /* Position of token in phrase */
189457 Fts3PhraseToken *pToken; /* The token itself */
189460 int iCol; /* The column the token must match */
189582 ** called on each token to defer. Otherwise, an SQLite error code is
189604 ** causing any problems, the deferred token optimization is completely
189612 ** only 1 token, exit early. No tokens to defer in this case. */
189628 ** to retrieve the entire doclist for the token from the full-text index.
189631 ** a. The cheapest token in the entire query (i.e. the one visited by the
189634 ** b. Part of a multi-token phrase.
189636 ** After each token doclist is loaded, merge it with the others from the
189639 ** any phrase doclist for which 1 or more token doclists have been loaded.
189643 ** Then, for each token, defer it if loading the doclist would result in
189650 Fts3TokenAndCost *pTC = 0; /* Set to cheapest remaining token. */
189652 /* Set pTC to point to the cheapest remaining token. */
189678 /* Either this is the cheapest token in the entire query, or it is
189679 ** part of a multi-token phrase. Either way, the entire doclist will
189724 token in the expression. */
189865 ** where "*" may match any single token. The position list in this case
190218 ** token. Then, see if this row is really a match, considering deferred
190230 /* Free the position-lists accumulated for each deferred token above. */
191315 ** c) The old syntax supports the "-" token qualifier. This is not
191337 ** notexpr ::= nearexpr (NOT nearexpr|-TOKEN)*.
191345 ** phrase ::= TOKEN.
191346 ** phrase ::= COLUMN:TOKEN.
191347 ** phrase ::= "TOKEN TOKEN TOKEN...".
191449 ** Extract the next token from buffer z (length n) using the tokenizer
191452 ** single token and set *ppExpr to point to it. If the end of the buffer is
191453 ** reached before a token is found, set *ppExpr to zero. It is the
191576 ** Fts3PhraseToken structures token buffers are all stored as a single
191757 ** user has supplied a token such as "ORacle". Continue.
191796 /* If control flows to this point, this must be a regular token, or
191797 ** the end of the input. Read a regular token using the sqlite3_tokenizer
191799 ** column specifier for the token.
191802 ** with a quoted phrase, only with a single token. Not sure if this was
193016 int iToken; /* index of next token to be returned */
193017 char *zToken; /* storage for current token */
193541 ** Characters that can be part of a token. We assume any character
193543 ** part of a token. In other words, delimiters all must have
193557 ** Extract the next token from a tokenization cursor. The cursor must
193562 const char **pzToken, /* OUT: *pzToken is the token text */
193563 int *pnBytes, /* OUT: Number of bytes in token */
193564 int *piStartOffset, /* OUT: Starting offset of token */
193565 int *piEndOffset, /* OUT: Ending offset of token */
193566 int *piPosition /* OUT: Position integer of token */
193763 /* Find the start of the next token. */
193879 ** list. For each token in the <input-string>, three elements are
193880 ** added to the returned list. The first is the token position, the
193881 ** second is the token text (folded, stemmed, etc.) and the third is the
193882 ** substring of <input-string> associated with the token. For example,
194199 int iToken; /* index of next token to be returned */
194200 char *pToken; /* storage for current token */
194308 ** Extract the next token from a tokenization cursor. The cursor must
194313 const char **ppToken, /* OUT: *ppToken is the token text */
194314 int *pnBytes, /* OUT: Number of bytes in token */
194315 int *piStartOffset, /* OUT: Starting offset of token */
194316 int *piEndOffset, /* OUT: Ending offset of token */
194317 int *piPosition /* OUT: Position integer of token */
194413 ** CREATE TABLE <tbl>(input, token, start, end, position)
194421 ** statement and returns one row for each token in the result. With
194425 ** token: A token from the input.
194426 ** start: Byte offset of the token within the input <string>.
194428 ** token within the input string.
194429 ** pos: Token offset of token within input.
194458 const char *zToken; /* Current 'token' value */
194537 #define FTS3_TOK_SCHEMA "CREATE TABLE x(input, token, start, end, position)"
194776 /* CREATE TABLE x(input, token, start, end, position) */
194967 Fts3PhraseToken *pToken; /* Pointer to corresponding expr token */
194968 int iCol; /* Column token must occur in */
195621 ** happen if there was no previous entry for this token.
195666 ** zText==0. In this case, add zero token entries to the hash table and
200199 char const *zToken; /* Buffer containing token */
200200 int nToken = 0; /* Number of bytes in token */
200202 int iPos = 0; /* Position of token in zText */
200240 ** The integrity-check works as follows. For each token and indexed token
200246 ** + The token (or token prefix) text itself,
200382 char const *zToken; /* Buffer containing token */
200383 int nToken = 0; /* Number of bytes in token */
200385 int iPos = 0; /* Position of token in zText */
200442 ** Add an entry for token pToken to the pCsr->pDeferred list.
200446 Fts3PhraseToken *pToken, /* Token to defer */
200447 int iCol /* Column that token must appear in (or -1) */
200753 int iCurrent; /* First token of current snippet */
200767 int iPos; /* Index of first token in snippet */
201104 int *piToken, /* OUT: First token of proposed snippet */
201109 int iStart = pIter->iCurrent; /* First token of snippet */
201346 int *piPos, /* IN/OUT: First token of snippet */
201370 int iCurrent = 0; /* Token counter */
201420 int iCurrent = 0; /* Current token number of document */
201421 int iEnd = 0; /* Byte offset of end of current token */
201423 int iPos = pFragment->iPos; /* First token of snippet */
201438 /* Open a token cursor on the document. */
201448 int iBegin = 0; /* Offset in zDoc of start of token */
201449 int iFin = 0; /* Offset in zDoc of end of token */
201466 /* Special case - the last token of the snippet is also the last token
201468 ** of the previous token and the end of the document to the output.
202357 int iMinPos = 0x7FFFFFFF; /* Position of next token */
202358 TermOffset *pTerm = 0; /* TermOffset associated with next token */
202538 int iToken; /* Index of next token to be returned */
202539 char *zToken; /* storage for current token */
202560 ** token characters (if bAlnum==1).
202651 ** considered a token character (not a separator).
202755 ** Extract the next token from a tokenization cursor. The cursor must
202760 const char **paToken, /* OUT: Token text */
202762 int *piStart, /* OUT: Starting offset of token */
202763 int *piEnd, /* OUT: Ending offset of token */
202764 int *piPos /* OUT: Position integer of token */
202775 /* Scan past any delimiter characters before the start of the next token.
212086 ** Return the length of a token
212954 ** return non-zero on success and zero if the next token is not a number.
215550 ** Extract the next token from a tokenization cursor.
215554 const char **ppToken, /* OUT: *ppToken is the token text */
215555 int *pnBytes, /* OUT: Number of bytes in token */
215556 int *piStartOffset, /* OUT: Starting offset of token */
215557 int *piEndOffset, /* OUT: Ending offset of token */
215558 int *piPosition /* OUT: Position integer of token */
221865 Token nm;
229643 ** to the column in which it occurs and *piOff the token offset of the
229644 ** first token of the phrase. SQLITE_OK is returned if successful, or an
229791 ** This is used to access token iToken of phrase iPhrase of the current
229793 ** to a buffer containing the requested token, and *pnToken to the
229803 ** token. It is the output of the tokenizer module. For tokendata=1
229807 ** This is used to access token iToken of phrase hit iIdx within the
229811 ** matching document token, and (*pnToken) to the size of that buffer in
229812 ** bytes. This API is not available if the specified token matches a
229925 ** followed by a "*" character, indicating that the last token
229926 ** returned by the tokenizer will be treated as a token prefix.
229934 ** For each token in the input string, the supplied callback xToken() must
229937 ** arguments are a pointer to a buffer containing the token text, and the
229938 ** size of the token in bytes. The 4th and 5th arguments are the byte offsets
229940 ** which the token is derived within the input.
229946 ** FTS5 assumes the xToken() callback is invoked for each token in the
229969 ** <ol><li> By mapping all synonyms to a single token. In this case, using
229971 ** same token for inputs "first" and "1st". Say that token is in
229988 ** first token in the MATCH query and FTS5 effectively runs a query
230000 ** provides multiple synonyms for each token. So that when a
230009 ** FTS index corresponding to both forms of the first token.
230014 ** is considered to supply a synonym for the previous token. For example,
230027 ** xToken() is called. Multiple synonyms may be specified for a single token
230030 ** single token.
230036 ** token "first" is substituted for "1st" by the tokenizer, then the query:
230041 ** will not match documents that contain the token "1st" (as the tokenizer
230052 ** token "1st", but not "first" (assuming the tokenizer is not able to
230076 const char *pToken, /* Pointer to buffer containing token */
230077 int nToken, /* Size of token in bytes */
230078 int iStart, /* Byte offset of token within input text */
230079 int iEnd /* Byte offset of end of token within input text */
230092 #define FTS5_TOKEN_COLOCATED 0x0001 /* Same position as prev. token */
230574 ** specified token or token prefix.
230578 const char *pToken, int nToken, /* Token (or prefix) to query for */
230585 ** The various operations on open token or token prefix iterators opened
230620 ** For an insert, it must be called once for each token in the new document.
230622 ** unique token in the document with an iCol value less than zero. The iPos
230627 int iCol, /* Column token appears in (-ve -> delete) */
230628 int iPos, /* Position of token within column */
230629 const char *pToken, int nToken /* Token to add or remove to or from index */
230771 int iCol, /* Column token appears in (-ve -> delete) */
230772 int iPos, /* Position of token within column */
230774 const char *pToken, int nToken /* Token to add or remove to or from index */
230874 const char *p; /* Token text (not NULL terminated) */
231113 ***************** Begin token definitions *************************************/
231131 /**************** End token definitions ***************************************/
231148 ** token.
231243 ** current state and lookahead token. These tables are used to implement
231251 ** token onto the stack and goto state N.
231350 ** appears in the grammar, then ID becomes a fallback token for X, Y,
231352 ** but it does not parse, the type of the token is changed to ID and
231369 ** + The value of the token stored at this level of the stack.
231370 ** (In other words, the "major" token.)
231374 ** It is sometimes called the "minor" token.
231382 fts5YYCODETYPE major; /* The major token value. This is the code
231383 ** number for the token at this stack level */
231384 fts5YYMINORTYPE minor; /* The user-supplied minor token value. This
231385 ** is the value of the token */
231673 ** If there is a destructor routine associated with the token which
231735 ** is in state X and has a lookahead token Y. In a well-tested
231772 ** look-ahead token iLookAhead.
231775 fts5YYCODETYPE iLookAhead, /* The look-ahead token */
231796 fts5YYCODETYPE iFallback; /* Fallback token */
231837 ** look-ahead token iLookAhead.
231841 fts5YYCODETYPE iLookAhead /* The look-ahead token */
231914 fts5YYCODETYPE fts5yyMajor, /* The major token to shift in */
231915 sqlite3Fts5ParserFTS5TOKENTYPE fts5yyMinor /* The minor token to shift in */
232023 ** access to the lookahead token (if any). The fts5yyLookahead will be fts5YYNOCODE
232024 ** if the lookahead token has already been consumed. As this procedure is
232031 int fts5yyLookahead, /* Lookahead token, or fts5YYNOCODE if none */
232032 sqlite3Fts5ParserFTS5TOKENTYPE fts5yyLookaheadToken /* Value of the lookahead token */
232248 int fts5yymajor, /* The major type of the error token */
232249 sqlite3Fts5ParserFTS5TOKENTYPE fts5yyminor /* The minor type of the error token */
232293 ** The second argument is the major token number. The third is
232294 ** the minor token. The fourth optional argument is whatever the
232301 ** <li> The major token number.
232302 ** <li> The minor token number.
232311 int fts5yymajor, /* The major token code number */
232312 sqlite3Fts5ParserFTS5TOKENTYPE fts5yyminor /* The value for the token */
232419 ** grammar defines an error token "ERROR".
232443 fprintf(fts5yyTraceFILE,"%sDiscard input token %s\n",
232485 ** * Report an error message, and throw away the input token.
232487 ** * If the input token is $, then fail the parse.
232523 ** Return the fallback token corresponding to canonical token iToken, or
232581 int iStart; /* First token in coalesced phrase instance */
232582 int iEnd; /* Last token in coalesced phrase instance */
232649 int iRangeStart; /* First token to include */
232650 int iRangeEnd; /* If non-zero, last token to include */
232658 int iPos; /* Current token offset in zIn[] */
232691 const char *pToken, /* Buffer containing token */
232692 int nToken, /* Size of token in bytes */
232693 int iStartOff, /* Start byte offset of token */
232694 int iEndOff /* End byte offset of token */
232710 /* If the parenthesis is open, and this token is not part of the current
232711 ** phrase, and the starting byte offset of this token is past the point
232824 int iPos; /* Current token position */
232827 int *aFirst; /* Array of first token in each sentence */
232853 ** For each such token, an entry is added to the SFinder.aFirst[] array.
232858 const char *pToken, /* Buffer containing token */
232859 int nToken, /* Size of token in bytes */
232860 int iStartOff, /* Start offset of token */
232861 int iEndOff /* End offset of token */
232962 int iBestStart = 0; /* First token of best snippet */
234442 ** The callback is invoked once for each token in the input text. The
234446 ** const char *pToken // Pointer to buffer containing token
234447 ** int nToken // Size of token in bytes
234448 ** int iStart // Byte offset of start of token within input text
234449 ** int iEnd // Byte offset of end of token within input text
234450 ** int iPos // Position of token in input (first token is 0)
234767 ** All token types in the generated fts5parse.h file are greater than 0.
234844 u8 bFirst; /* True if token must be first in column */
234864 ** One or more phrases that must appear within a certain token distance of
234929 ** Read the first token from the nul-terminated string at *pz.
235007 Fts5Token token;
235009 int t; /* Next token type */
235022 t = fts5ExprGetToken(&sParse, &z, &token);
235023 sqlite3Fts5Parser(pEngine, t, token, &sParse);
235832 int i, j; /* Phrase and token index, respectively */
236366 ** Set the "bFirst" flag on the first token of the phrase passed as the
236459 const char *pToken, /* Buffer containing token */
236460 int nToken, /* Size of token in bytes */
236461 int iUnused1, /* Start offset of token */
236462 int iUnused2 /* End offset of token */
236567 ** This function is called by the parser to process a string token. The
236610 /* This happens when parsing a token or quoted phrase that contains
236611 ** no token characters at all. (e.g ... MATCH '""'). */
236684 /* This happens when parsing a token or quoted phrase that contains
236685 ** no token characters at all. (e.g ... MATCH '""'). */
236722 ** Token pTok has appeared in a MATCH expression where the NEAR operator
236723 ** is expected. If token pTok does not contain "NEAR", store an error
236840 char *z; /* Dequoted copy of token p */
237697 ** fts5 tables. This implies that all phrases must be at most 1 token
237754 const char *pToken, /* Pointer to buffer containing token */
237755 int nToken, /* Size of token in bytes */
237756 int iUnused1, /* Byte offset of token within input text */
237757 int iUnused2 /* Byte offset of end of token within input text */
237988 ** Clear the token mappings for all Fts5IndexIter objects mannaged by
238046 ** followed by the term data. For example: "0token". There is no
238248 int iCol, /* Column token appears in (-ve -> delete) */
238249 int iPos, /* Position of token within column */
238250 char bByte, /* First byte of token */
238251 const char *pToken, int nToken /* Token to add or remove to or from index */
243624 ** too. This is done to ensure that if all instances of a token are
243626 ** the token itself remains in the database.
245117 ** For an insert, it must be called once for each token in the new document.
245119 ** unique token in the document with an iCol value less than zero. The iPos
245124 int iCol, /* Column token appears in (-ve -> delete) */
245125 int iPos, /* Position of token within column */
245126 const char *pToken, int nToken /* Token to add or remove to or from index */
245158 ** token pToken/nToken.
245191 i64 iRowid; /* Row this token is located in */
245192 i64 iPos; /* Position of token */
245193 int iIter; /* Iterator token was read from */
245264 ** Append a mapping to the token-map belonging to object pT.
245370 /* Ensure the token-mapping is large enough */
245604 ** specified token or token prefix.
245608 const char *pToken, int nToken, /* Token (or prefix) to query for */
245762 ** This is used by xInstToken() to access the token at offset iOff, column
245763 ** iCol of row iRowid. The token is returned via output variables *ppOut
245814 ** Clear any existing entries from the token-map associated with the
245825 ** Set a token-mapping for the iterator passed as the first argument. This
245826 ** is used in detail=column or detail=none mode when a token is requested
245828 ** current row and configures the token-mapping via multiple calls to this
248578 ** clear any token mappings accumulated at the fts5_index.c level. In
249766 const char *pUnused, /* Buffer containing token */
249767 int nUnused, /* Size of token in bytes */
249768 int iUnused1, /* Start offset of token */
249769 int iUnused2 /* End offset of token */
251153 const char *pToken, /* Buffer containing token */
251154 int nToken, /* Size of token in bytes */
251155 int iUnused1, /* Start offset of token */
251156 int iUnused2 /* End offset of token */
251647 const char *pToken, /* Buffer containing token */
251648 int nToken, /* Size of token in bytes */
251649 int iUnused1, /* Start offset of token */
251650 int iUnused2 /* End offset of token */
252019 ** For tokenizers with no "unicode" modifier, the set of token characters
252142 /* Count the token characters */
252161 /* Invoke the token callback */
252237 unsigned char aTokenChar[128]; /* ASCII range token characters */
252244 unsigned char aCategory[32]; /* True for token char categories */
252431 ** passed as the first argument, codepoint iCode is considered a token
252463 ** then the next token. */
252512 ** it is a token character, or break out of the loop if it is not. */
252522 /* An ascii-range separator character. End of token. */
252536 /* Invoke the token callback */
254232 aAscii[0] = 0; /* 0x00 is never a token character */
Indexes created Wed Mar 04 05:31:52 UTC 2026