diff -r 5f8e5adbbed9 -r 29cda98b007e engine/sqlite/src/tokenize.cpp --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/engine/sqlite/src/tokenize.cpp Thu Feb 25 14:29:19 2010 +0000 @@ -0,0 +1,462 @@ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** An tokenizer for SQL +** +** This file contains C code that splits an SQL input string up into +** individual tokens and sends those tokens one-by-one over to the +** parser for analysis. +** +** $Id: tokenize.cpp 1282 2008-11-13 09:31:33Z LarsPson $ +*/ +#include "sqliteInt.h" +#include +#include + +/* +** The charMap() macro maps alphabetic characters into their +** lower-case ASCII equivalent. On ASCII machines, this is just +** an upper-to-lower case map. On EBCDIC machines we also need +** to adjust the encoding. Only alphabetic characters and underscores +** need to be translated. +*/ +#ifdef SQLITE_ASCII +# define charMap(X) sqlite3UpperToLower[(unsigned char)X] +#endif +#ifdef SQLITE_EBCDIC +# define charMap(X) ebcdicToAscii[(unsigned char)X] +const unsigned char ebcdicToAscii[] = { +/* 0 1 2 3 4 5 6 7 8 9 A B C D E F */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 3x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 4x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 5x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, /* 6x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 7x */ + 0, 97, 98, 99,100,101,102,103,104,105, 0, 0, 0, 0, 0, 0, /* 8x */ + 0,106,107,108,109,110,111,112,113,114, 0, 0, 0, 0, 0, 0, /* 9x */ + 0, 0,115,116,117,118,119,120,121,122, 0, 0, 0, 0, 0, 0, /* Ax */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Bx */ + 0, 97, 98, 99,100,101,102,103,104,105, 0, 0, 0, 0, 0, 0, /* Cx */ + 0,106,107,108,109,110,111,112,113,114, 0, 0, 0, 0, 0, 0, /* Dx */ + 0, 0,115,116,117,118,119,120,121,122, 0, 0, 0, 0, 0, 0, /* Ex */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Fx */ +}; +#endif + +/* +** The sqlite3KeywordCode function looks up an identifier to determine if +** it is a keyword. If it is a keyword, the token code of that keyword is +** returned. If the input is not a keyword, TK_ID is returned. +** +** The implementation of this routine was generated by a program, +** mkkeywordhash.h, located in the tool subdirectory of the distribution. +** The output of the mkkeywordhash.c program is written into a file +** named keywordhash.h and then included into this source file by +** the #include below. +*/ +#include "keywordhash.h" + + + + +/* +** Return the length of the token that begins at z[0]. +** Store the token type in *tokenType before returning. +*/ +static int getToken(const unsigned char *z, int *tokenType){ + int i, c; + switch( *z ){ + case ' ': case '\t': case '\n': case '\f': case '\r': { + for(i=1; isspace(z[i]); i++){} + *tokenType = TK_SPACE; + return i; + } + case '-': { + if( z[1]=='-' ){ + for(i=2; (c=z[i])!=0 && c!='\n'; i++){} + *tokenType = TK_COMMENT; + return i; + } + *tokenType = TK_MINUS; + return 1; + } + case '(': { + *tokenType = TK_LP; + return 1; + } + case ')': { + *tokenType = TK_RP; + return 1; + } + case ';': { + *tokenType = TK_SEMI; + return 1; + } + case '+': { + *tokenType = TK_PLUS; + return 1; + } + case '*': { + *tokenType = TK_STAR; + return 1; + } + case '/': { + if( z[1]!='*' || z[2]==0 ){ + *tokenType = TK_SLASH; + return 1; + } + for(i=3, c=z[2]; (c!='*' || z[i]!='/') && (c=z[i])!=0; i++){} + if( c ) i++; + *tokenType = TK_COMMENT; + return i; + } + case '%': { + *tokenType = TK_REM; + return 1; + } + case '=': { + *tokenType = TK_EQ; + return 1 + (z[1]=='='); + } + case '<': { + if( (c=z[1])=='=' ){ + *tokenType = TK_LE; + return 2; + }else if( c=='>' ){ + *tokenType = TK_NE; + return 2; + }else if( c=='<' ){ + *tokenType = TK_LSHIFT; + return 2; + }else{ + *tokenType = TK_LT; + return 1; + } + } + case '>': { + if( (c=z[1])=='=' ){ + *tokenType = TK_GE; + return 2; + }else if( c=='>' ){ + *tokenType = TK_RSHIFT; + return 2; + }else{ + *tokenType = TK_GT; + return 1; + } + } + case '!': { + if( z[1]!='=' ){ + *tokenType = TK_ILLEGAL; + return 2; + }else{ + *tokenType = TK_NE; + return 2; + } + } + case '|': { + if( z[1]!='|' ){ + *tokenType = TK_BITOR; + return 1; + }else{ + *tokenType = TK_CONCAT; + return 2; + } + } + case ',': { + *tokenType = TK_COMMA; + return 1; + } + case '&': { + *tokenType = TK_BITAND; + return 1; + } + case '~': { + *tokenType = TK_BITNOT; + return 1; + } + case '`': + case '\'': + case '"': { + int delim = z[0]; + for(i=1; (c=z[i])!=0; i++){ + if( c==delim ){ + if( z[i+1]==delim ){ + i++; + }else{ + break; + } + } + } + if( c ){ + *tokenType = TK_STRING; + return i+1; + }else{ + *tokenType = TK_ILLEGAL; + return i; + } + } + case '.': { +#ifndef SQLITE_OMIT_FLOATING_POINT + if( !isdigit(z[1]) ) +#endif + { + *tokenType = TK_DOT; + return 1; + } + /* If the next character is a digit, this is a floating point + ** number that begins with ".". Fall thru into the next case */ + } + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': { + *tokenType = TK_INTEGER; + for(i=0; isdigit(z[i]); i++){} +#ifndef SQLITE_OMIT_FLOATING_POINT + if( z[i]=='.' ){ + i++; + while( isdigit(z[i]) ){ i++; } + *tokenType = TK_FLOAT; + } + if( (z[i]=='e' || z[i]=='E') && + ( isdigit(z[i+1]) + || ((z[i+1]=='+' || z[i+1]=='-') && isdigit(z[i+2])) + ) + ){ + i += 2; + while( isdigit(z[i]) ){ i++; } + *tokenType = TK_FLOAT; + } +#endif + while( IdChar(z[i]) ){ + *tokenType = TK_ILLEGAL; + i++; + } + return i; + } + case '[': { + for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){} + *tokenType = TK_ID; + return i; + } + case '?': { + *tokenType = TK_VARIABLE; + for(i=1; isdigit(z[i]); i++){} + return i; + } + case '#': { + for(i=1; isdigit(z[i]); i++){} + if( i>1 ){ + /* Parameters of the form #NNN (where NNN is a number) are used + ** internally by sqlite3NestedParse. */ + *tokenType = TK_REGISTER; + return i; + } + /* Fall through into the next case if the '#' is not followed by + ** a digit. Try to match #AAAA where AAAA is a parameter name. */ + } +#ifndef SQLITE_OMIT_TCL_VARIABLE + case '$': +#endif + case '@': /* For compatibility with MS SQL Server */ + case ':': { + int n = 0; + *tokenType = TK_VARIABLE; + for(i=1; (c=z[i])!=0; i++){ + if( IdChar(c) ){ + n++; +#ifndef SQLITE_OMIT_TCL_VARIABLE + }else if( c=='(' && n>0 ){ + do{ + i++; + }while( (c=z[i])!=0 && !isspace(c) && c!=')' ); + if( c==')' ){ + i++; + }else{ + *tokenType = TK_ILLEGAL; + } + break; + }else if( c==':' && z[i+1]==':' ){ + i++; +#endif + }else{ + break; + } + } + if( n==0 ) *tokenType = TK_ILLEGAL; + return i; + } +#ifndef SQLITE_OMIT_BLOB_LITERAL + case 'x': case 'X': { + if( (c=z[1])=='\'' || c=='"' ){ + int delim = c; + *tokenType = TK_BLOB; + for(i=2; (c=z[i])!=0; i++){ + if( c==delim ){ + if( i%2 ) *tokenType = TK_ILLEGAL; + break; + } + if( !isxdigit(c) ){ + *tokenType = TK_ILLEGAL; + return i; + } + } + if( c ) i++; + return i; + } + /* Otherwise fall through to the next case */ + } +#endif + default: { + if( !IdChar(*z) ){ + break; + } + for(i=1; IdChar(z[i]); i++){} + *tokenType = keywordCode((char*)z, i); + return i; + } + } + *tokenType = TK_ILLEGAL; + return 1; +} +int sqlite3GetToken(const unsigned char *z, int *tokenType){ + return getToken(z, tokenType); +} + +/* +** Run the parser on the given SQL string. The parser structure is +** passed in. An SQLITE_ status code is returned. If an error occurs +** and pzErrMsg!=NULL then an error message might be written into +** memory obtained from sqlite3_malloc() and *pzErrMsg made to point to that +** error message. Or maybe not. +*/ +int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){ + int nErr = 0; + int i; + void *pEngine; + int tokenType; + int lastTokenParsed = -1; + sqlite3 *db = pParse->db; + + if( db->activeVdbeCnt==0 ){ + db->u1.isInterrupted = 0; + } + pParse->rc = SQLITE_OK; + i = 0; + pEngine = sqlite3ParserAlloc((void*(*)(size_t))sqlite3_malloc); + if( pEngine==0 ){ + db->mallocFailed = 1; + return SQLITE_NOMEM; + } + assert( pParse->sLastToken.dyn==0 ); + assert( pParse->pNewTable==0 ); + assert( pParse->pNewTrigger==0 ); + assert( pParse->nVar==0 ); + assert( pParse->nVarExpr==0 ); + assert( pParse->nVarExprAlloc==0 ); + assert( pParse->apVarExpr==0 ); + pParse->zTail = pParse->zSql = zSql; + while( !db->mallocFailed && zSql[i]!=0 ){ + assert( i>=0 ); + pParse->sLastToken.z = (u8*)&zSql[i]; + assert( pParse->sLastToken.dyn==0 ); + pParse->sLastToken.n = getToken((unsigned char*)&zSql[i],&tokenType); + i += pParse->sLastToken.n; + if( i>SQLITE_MAX_SQL_LENGTH ){ + pParse->rc = SQLITE_TOOBIG; + break; + } + switch( tokenType ){ + case TK_SPACE: + case TK_COMMENT: { + if( db->u1.isInterrupted ){ + pParse->rc = SQLITE_INTERRUPT; + sqlite3SetString(pzErrMsg, "interrupt", (char*)0); + goto abort_parse; + } + break; + } + case TK_ILLEGAL: { + if( pzErrMsg ){ + sqlite3_free(*pzErrMsg); + *pzErrMsg = sqlite3MPrintf(db, "unrecognized token: \"%T\"", + &pParse->sLastToken); + } + nErr++; + goto abort_parse; + } + case TK_SEMI: { + pParse->zTail = &zSql[i]; + /* Fall thru into the default case */ + } + default: { + sqlite3Parser(pEngine, tokenType, pParse->sLastToken, pParse); + lastTokenParsed = tokenType; + if( pParse->rc!=SQLITE_OK ){ + goto abort_parse; + } + break; + } + } + } +abort_parse: + if( zSql[i]==0 && nErr==0 && pParse->rc==SQLITE_OK ){ + if( lastTokenParsed!=TK_SEMI ){ + sqlite3Parser(pEngine, TK_SEMI, pParse->sLastToken, pParse); + pParse->zTail = &zSql[i]; + } + sqlite3Parser(pEngine, 0, pParse->sLastToken, pParse); + } + sqlite3ParserFree(pEngine, sqlite3_free); + if( db->mallocFailed ){ + pParse->rc = SQLITE_NOMEM; + } + if( pParse->rc!=SQLITE_OK && pParse->rc!=SQLITE_DONE && pParse->zErrMsg==0 ){ + sqlite3SetString(&pParse->zErrMsg, sqlite3ErrStr(pParse->rc), (char*)0); + } + if( pParse->zErrMsg ){ + if( pzErrMsg && *pzErrMsg==0 ){ + *pzErrMsg = pParse->zErrMsg; + }else{ + sqlite3_free(pParse->zErrMsg); + } + pParse->zErrMsg = 0; + if( !nErr ) nErr++; + } + if( pParse->pVdbe && pParse->nErr>0 && pParse->nested==0 ){ + sqlite3VdbeDelete(pParse->pVdbe); + pParse->pVdbe = 0; + } +#ifndef SQLITE_OMIT_SHARED_CACHE + if( pParse->nested==0 ){ + sqlite3_free(pParse->aTableLock); + pParse->aTableLock = 0; + pParse->nTableLock = 0; + } +#endif + + if( !IN_DECLARE_VTAB ){ + /* If the pParse->declareVtab flag is set, do not delete any table + ** structure built up in pParse->pNewTable. The calling code (see vtab.c) + ** will take responsibility for freeing the Table structure. + */ + sqlite3DeleteTable(pParse->pNewTable); + } + + sqlite3DeleteTrigger(pParse->pNewTrigger); + sqlite3_free(pParse->apVarExpr); + if( nErr>0 && (pParse->rc==SQLITE_OK || pParse->rc==SQLITE_DONE) ){ + pParse->rc = SQLITE_ERROR; + } + return nErr; +}