--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/engine/sqlite/src/tokenize.cpp Thu Feb 25 14:29:19 2010 +0000
@@ -0,0 +1,462 @@
+/*
+** 2001 September 15
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+*************************************************************************
+** An tokenizer for SQL
+**
+** This file contains C code that splits an SQL input string up into
+** individual tokens and sends those tokens one-by-one over to the
+** parser for analysis.
+**
+** $Id: tokenize.cpp 1282 2008-11-13 09:31:33Z LarsPson $
+*/
+#include "sqliteInt.h"
+#include <ctype.h>
+#include <stdlib.h>
+
+/*
+** The charMap() macro maps alphabetic characters into their
+** lower-case ASCII equivalent. On ASCII machines, this is just
+** an upper-to-lower case map. On EBCDIC machines we also need
+** to adjust the encoding. Only alphabetic characters and underscores
+** need to be translated.
+*/
+#ifdef SQLITE_ASCII
+# define charMap(X) sqlite3UpperToLower[(unsigned char)X]
+#endif
+#ifdef SQLITE_EBCDIC
+# define charMap(X) ebcdicToAscii[(unsigned char)X]
+const unsigned char ebcdicToAscii[] = {
+/* 0 1 2 3 4 5 6 7 8 9 A B C D E F */
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 3x */
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 4x */
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 5x */
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, /* 6x */
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 7x */
+ 0, 97, 98, 99,100,101,102,103,104,105, 0, 0, 0, 0, 0, 0, /* 8x */
+ 0,106,107,108,109,110,111,112,113,114, 0, 0, 0, 0, 0, 0, /* 9x */
+ 0, 0,115,116,117,118,119,120,121,122, 0, 0, 0, 0, 0, 0, /* Ax */
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Bx */
+ 0, 97, 98, 99,100,101,102,103,104,105, 0, 0, 0, 0, 0, 0, /* Cx */
+ 0,106,107,108,109,110,111,112,113,114, 0, 0, 0, 0, 0, 0, /* Dx */
+ 0, 0,115,116,117,118,119,120,121,122, 0, 0, 0, 0, 0, 0, /* Ex */
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Fx */
+};
+#endif
+
+/*
+** The sqlite3KeywordCode function looks up an identifier to determine if
+** it is a keyword. If it is a keyword, the token code of that keyword is
+** returned. If the input is not a keyword, TK_ID is returned.
+**
+** The implementation of this routine was generated by a program,
+** mkkeywordhash.h, located in the tool subdirectory of the distribution.
+** The output of the mkkeywordhash.c program is written into a file
+** named keywordhash.h and then included into this source file by
+** the #include below.
+*/
+#include "keywordhash.h"
+
+
+
+
+/*
+** Return the length of the token that begins at z[0].
+** Store the token type in *tokenType before returning.
+*/
+static int getToken(const unsigned char *z, int *tokenType){
+ int i, c;
+ switch( *z ){
+ case ' ': case '\t': case '\n': case '\f': case '\r': {
+ for(i=1; isspace(z[i]); i++){}
+ *tokenType = TK_SPACE;
+ return i;
+ }
+ case '-': {
+ if( z[1]=='-' ){
+ for(i=2; (c=z[i])!=0 && c!='\n'; i++){}
+ *tokenType = TK_COMMENT;
+ return i;
+ }
+ *tokenType = TK_MINUS;
+ return 1;
+ }
+ case '(': {
+ *tokenType = TK_LP;
+ return 1;
+ }
+ case ')': {
+ *tokenType = TK_RP;
+ return 1;
+ }
+ case ';': {
+ *tokenType = TK_SEMI;
+ return 1;
+ }
+ case '+': {
+ *tokenType = TK_PLUS;
+ return 1;
+ }
+ case '*': {
+ *tokenType = TK_STAR;
+ return 1;
+ }
+ case '/': {
+ if( z[1]!='*' || z[2]==0 ){
+ *tokenType = TK_SLASH;
+ return 1;
+ }
+ for(i=3, c=z[2]; (c!='*' || z[i]!='/') && (c=z[i])!=0; i++){}
+ if( c ) i++;
+ *tokenType = TK_COMMENT;
+ return i;
+ }
+ case '%': {
+ *tokenType = TK_REM;
+ return 1;
+ }
+ case '=': {
+ *tokenType = TK_EQ;
+ return 1 + (z[1]=='=');
+ }
+ case '<': {
+ if( (c=z[1])=='=' ){
+ *tokenType = TK_LE;
+ return 2;
+ }else if( c=='>' ){
+ *tokenType = TK_NE;
+ return 2;
+ }else if( c=='<' ){
+ *tokenType = TK_LSHIFT;
+ return 2;
+ }else{
+ *tokenType = TK_LT;
+ return 1;
+ }
+ }
+ case '>': {
+ if( (c=z[1])=='=' ){
+ *tokenType = TK_GE;
+ return 2;
+ }else if( c=='>' ){
+ *tokenType = TK_RSHIFT;
+ return 2;
+ }else{
+ *tokenType = TK_GT;
+ return 1;
+ }
+ }
+ case '!': {
+ if( z[1]!='=' ){
+ *tokenType = TK_ILLEGAL;
+ return 2;
+ }else{
+ *tokenType = TK_NE;
+ return 2;
+ }
+ }
+ case '|': {
+ if( z[1]!='|' ){
+ *tokenType = TK_BITOR;
+ return 1;
+ }else{
+ *tokenType = TK_CONCAT;
+ return 2;
+ }
+ }
+ case ',': {
+ *tokenType = TK_COMMA;
+ return 1;
+ }
+ case '&': {
+ *tokenType = TK_BITAND;
+ return 1;
+ }
+ case '~': {
+ *tokenType = TK_BITNOT;
+ return 1;
+ }
+ case '`':
+ case '\'':
+ case '"': {
+ int delim = z[0];
+ for(i=1; (c=z[i])!=0; i++){
+ if( c==delim ){
+ if( z[i+1]==delim ){
+ i++;
+ }else{
+ break;
+ }
+ }
+ }
+ if( c ){
+ *tokenType = TK_STRING;
+ return i+1;
+ }else{
+ *tokenType = TK_ILLEGAL;
+ return i;
+ }
+ }
+ case '.': {
+#ifndef SQLITE_OMIT_FLOATING_POINT
+ if( !isdigit(z[1]) )
+#endif
+ {
+ *tokenType = TK_DOT;
+ return 1;
+ }
+ /* If the next character is a digit, this is a floating point
+ ** number that begins with ".". Fall thru into the next case */
+ }
+ case '0': case '1': case '2': case '3': case '4':
+ case '5': case '6': case '7': case '8': case '9': {
+ *tokenType = TK_INTEGER;
+ for(i=0; isdigit(z[i]); i++){}
+#ifndef SQLITE_OMIT_FLOATING_POINT
+ if( z[i]=='.' ){
+ i++;
+ while( isdigit(z[i]) ){ i++; }
+ *tokenType = TK_FLOAT;
+ }
+ if( (z[i]=='e' || z[i]=='E') &&
+ ( isdigit(z[i+1])
+ || ((z[i+1]=='+' || z[i+1]=='-') && isdigit(z[i+2]))
+ )
+ ){
+ i += 2;
+ while( isdigit(z[i]) ){ i++; }
+ *tokenType = TK_FLOAT;
+ }
+#endif
+ while( IdChar(z[i]) ){
+ *tokenType = TK_ILLEGAL;
+ i++;
+ }
+ return i;
+ }
+ case '[': {
+ for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){}
+ *tokenType = TK_ID;
+ return i;
+ }
+ case '?': {
+ *tokenType = TK_VARIABLE;
+ for(i=1; isdigit(z[i]); i++){}
+ return i;
+ }
+ case '#': {
+ for(i=1; isdigit(z[i]); i++){}
+ if( i>1 ){
+ /* Parameters of the form #NNN (where NNN is a number) are used
+ ** internally by sqlite3NestedParse. */
+ *tokenType = TK_REGISTER;
+ return i;
+ }
+ /* Fall through into the next case if the '#' is not followed by
+ ** a digit. Try to match #AAAA where AAAA is a parameter name. */
+ }
+#ifndef SQLITE_OMIT_TCL_VARIABLE
+ case '$':
+#endif
+ case '@': /* For compatibility with MS SQL Server */
+ case ':': {
+ int n = 0;
+ *tokenType = TK_VARIABLE;
+ for(i=1; (c=z[i])!=0; i++){
+ if( IdChar(c) ){
+ n++;
+#ifndef SQLITE_OMIT_TCL_VARIABLE
+ }else if( c=='(' && n>0 ){
+ do{
+ i++;
+ }while( (c=z[i])!=0 && !isspace(c) && c!=')' );
+ if( c==')' ){
+ i++;
+ }else{
+ *tokenType = TK_ILLEGAL;
+ }
+ break;
+ }else if( c==':' && z[i+1]==':' ){
+ i++;
+#endif
+ }else{
+ break;
+ }
+ }
+ if( n==0 ) *tokenType = TK_ILLEGAL;
+ return i;
+ }
+#ifndef SQLITE_OMIT_BLOB_LITERAL
+ case 'x': case 'X': {
+ if( (c=z[1])=='\'' || c=='"' ){
+ int delim = c;
+ *tokenType = TK_BLOB;
+ for(i=2; (c=z[i])!=0; i++){
+ if( c==delim ){
+ if( i%2 ) *tokenType = TK_ILLEGAL;
+ break;
+ }
+ if( !isxdigit(c) ){
+ *tokenType = TK_ILLEGAL;
+ return i;
+ }
+ }
+ if( c ) i++;
+ return i;
+ }
+ /* Otherwise fall through to the next case */
+ }
+#endif
+ default: {
+ if( !IdChar(*z) ){
+ break;
+ }
+ for(i=1; IdChar(z[i]); i++){}
+ *tokenType = keywordCode((char*)z, i);
+ return i;
+ }
+ }
+ *tokenType = TK_ILLEGAL;
+ return 1;
+}
+int sqlite3GetToken(const unsigned char *z, int *tokenType){
+ return getToken(z, tokenType);
+}
+
+/*
+** Run the parser on the given SQL string. The parser structure is
+** passed in. An SQLITE_ status code is returned. If an error occurs
+** and pzErrMsg!=NULL then an error message might be written into
+** memory obtained from sqlite3_malloc() and *pzErrMsg made to point to that
+** error message. Or maybe not.
+*/
+int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){
+ int nErr = 0;
+ int i;
+ void *pEngine;
+ int tokenType;
+ int lastTokenParsed = -1;
+ sqlite3 *db = pParse->db;
+
+ if( db->activeVdbeCnt==0 ){
+ db->u1.isInterrupted = 0;
+ }
+ pParse->rc = SQLITE_OK;
+ i = 0;
+ pEngine = sqlite3ParserAlloc((void*(*)(size_t))sqlite3_malloc);
+ if( pEngine==0 ){
+ db->mallocFailed = 1;
+ return SQLITE_NOMEM;
+ }
+ assert( pParse->sLastToken.dyn==0 );
+ assert( pParse->pNewTable==0 );
+ assert( pParse->pNewTrigger==0 );
+ assert( pParse->nVar==0 );
+ assert( pParse->nVarExpr==0 );
+ assert( pParse->nVarExprAlloc==0 );
+ assert( pParse->apVarExpr==0 );
+ pParse->zTail = pParse->zSql = zSql;
+ while( !db->mallocFailed && zSql[i]!=0 ){
+ assert( i>=0 );
+ pParse->sLastToken.z = (u8*)&zSql[i];
+ assert( pParse->sLastToken.dyn==0 );
+ pParse->sLastToken.n = getToken((unsigned char*)&zSql[i],&tokenType);
+ i += pParse->sLastToken.n;
+ if( i>SQLITE_MAX_SQL_LENGTH ){
+ pParse->rc = SQLITE_TOOBIG;
+ break;
+ }
+ switch( tokenType ){
+ case TK_SPACE:
+ case TK_COMMENT: {
+ if( db->u1.isInterrupted ){
+ pParse->rc = SQLITE_INTERRUPT;
+ sqlite3SetString(pzErrMsg, "interrupt", (char*)0);
+ goto abort_parse;
+ }
+ break;
+ }
+ case TK_ILLEGAL: {
+ if( pzErrMsg ){
+ sqlite3_free(*pzErrMsg);
+ *pzErrMsg = sqlite3MPrintf(db, "unrecognized token: \"%T\"",
+ &pParse->sLastToken);
+ }
+ nErr++;
+ goto abort_parse;
+ }
+ case TK_SEMI: {
+ pParse->zTail = &zSql[i];
+ /* Fall thru into the default case */
+ }
+ default: {
+ sqlite3Parser(pEngine, tokenType, pParse->sLastToken, pParse);
+ lastTokenParsed = tokenType;
+ if( pParse->rc!=SQLITE_OK ){
+ goto abort_parse;
+ }
+ break;
+ }
+ }
+ }
+abort_parse:
+ if( zSql[i]==0 && nErr==0 && pParse->rc==SQLITE_OK ){
+ if( lastTokenParsed!=TK_SEMI ){
+ sqlite3Parser(pEngine, TK_SEMI, pParse->sLastToken, pParse);
+ pParse->zTail = &zSql[i];
+ }
+ sqlite3Parser(pEngine, 0, pParse->sLastToken, pParse);
+ }
+ sqlite3ParserFree(pEngine, sqlite3_free);
+ if( db->mallocFailed ){
+ pParse->rc = SQLITE_NOMEM;
+ }
+ if( pParse->rc!=SQLITE_OK && pParse->rc!=SQLITE_DONE && pParse->zErrMsg==0 ){
+ sqlite3SetString(&pParse->zErrMsg, sqlite3ErrStr(pParse->rc), (char*)0);
+ }
+ if( pParse->zErrMsg ){
+ if( pzErrMsg && *pzErrMsg==0 ){
+ *pzErrMsg = pParse->zErrMsg;
+ }else{
+ sqlite3_free(pParse->zErrMsg);
+ }
+ pParse->zErrMsg = 0;
+ if( !nErr ) nErr++;
+ }
+ if( pParse->pVdbe && pParse->nErr>0 && pParse->nested==0 ){
+ sqlite3VdbeDelete(pParse->pVdbe);
+ pParse->pVdbe = 0;
+ }
+#ifndef SQLITE_OMIT_SHARED_CACHE
+ if( pParse->nested==0 ){
+ sqlite3_free(pParse->aTableLock);
+ pParse->aTableLock = 0;
+ pParse->nTableLock = 0;
+ }
+#endif
+
+ if( !IN_DECLARE_VTAB ){
+ /* If the pParse->declareVtab flag is set, do not delete any table
+ ** structure built up in pParse->pNewTable. The calling code (see vtab.c)
+ ** will take responsibility for freeing the Table structure.
+ */
+ sqlite3DeleteTable(pParse->pNewTable);
+ }
+
+ sqlite3DeleteTrigger(pParse->pNewTrigger);
+ sqlite3_free(pParse->apVarExpr);
+ if( nErr>0 && (pParse->rc==SQLITE_OK || pParse->rc==SQLITE_DONE) ){
+ pParse->rc = SQLITE_ERROR;
+ }
+ return nErr;
+}