1 /****************************************************************************

     2 **

     3 ** Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies).

     4 ** All rights reserved.

     5 ** Contact: Nokia Corporation (qt-info@nokia.com)

     6 **

     7 ** This file is part of the QtXmlPatterns module of the Qt Toolkit.

     8 **

     9 ** $QT_BEGIN_LICENSE:LGPL$

    10 ** No Commercial Usage

    11 ** This file contains pre-release code and may not be distributed.

    12 ** You may use this file in accordance with the terms and conditions

    13 ** contained in the Technology Preview License Agreement accompanying

    14 ** this package.

    15 **

    16 ** GNU Lesser General Public License Usage

    17 ** Alternatively, this file may be used under the terms of the GNU Lesser

    18 ** General Public License version 2.1 as published by the Free Software

    19 ** Foundation and appearing in the file LICENSE.LGPL included in the

    20 ** packaging of this file.  Please review the following information to

    21 ** ensure the GNU Lesser General Public License version 2.1 requirements

    22 ** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.

    23 **

    24 ** In addition, as a special exception, Nokia gives you certain additional

    25 ** rights.  These rights are described in the Nokia Qt LGPL Exception

    26 ** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.

    27 **

    28 ** If you have questions regarding the use of this file, please contact

    29 ** Nokia at qt-info@nokia.com.

    30 **

    31 **

    32 **

    33 **

    34 **

    35 **

    36 **

    37 **

    38 ** $QT_END_LICENSE$

    39 **

    40 ****************************************************************************/

    41 

    42 #include <QByteArray>

    43 

    44 #include "qquerytransformparser_p.h"

    45 

    46 #include "qxquerytokenizer_p.h"

    47 

    48 #include "qtokenlookup.cpp"

    49 

    50 QT_BEGIN_NAMESPACE

    51 

    52 namespace QPatternist

    53 {

    54 

    55 #define handleWhitespace()                      \

    56 {                                               \

    57     const TokenType t = consumeWhitespace();    \

    58     if(t != SUCCESS)                            \

    59         return Token(t);                        \

    60 }

    61 

    62 XQueryTokenizer::XQueryTokenizer(const QString &query,

    63                                  const QUrl &location,

    64                                  const State startingState) : Tokenizer(location)

    65                                                             , m_data(query)

    66                                                             , m_length(query.length())

    67                                                             , m_state(startingState)

    68                                                             , m_pos(0)

    69                                                             , m_line(1)

    70                                                             , m_columnOffset(0)

    71                                                             , m_scanOnly(false)

    72 {

    73     Q_ASSERT(location.isValid() || location.isEmpty());

    74 }

    75 

    76 const QChar XQueryTokenizer::current() const

    77 {

    78     if(m_pos < m_length)

    79         return m_data.at(m_pos);

    80     else

    81         return QChar();

    82 }

    83 

    84 char XQueryTokenizer::peekCurrent() const

    85 {

    86     return current().toAscii();

    87 }

    88 

    89 int XQueryTokenizer::peekForColonColon() const

    90 {

    91     /* Note, we don't modify m_pos in this function, so we need to do offset

    92      * calculations. */

    93     int pos = m_pos;

    94 

    95     while(pos < m_length)

    96     {

    97         switch(m_data.at(pos).toAscii())

    98         {

    99             /* Fallthrough these four. */

   100             case ' ':

   101             case '\t':

   102             case '\n':

   103             case '\r':

   104                 break;

   105             case ':':

   106             {

   107                 if(peekAhead((pos - m_pos) + 1) == ':')

   108                     return pos - m_pos;

   109                 /* Fallthrough. */

   110             }

   111             default:

   112                 return -1;

   113         }

   114         ++pos;

   115     }

   116 

   117     return -1;

   118 }

   119 

   120 Tokenizer::Token XQueryTokenizer::tokenAndChangeState(const TokenType code,

   121                                                       const State s,

   122                                                       const int advance)

   123 {

   124     Q_ASSERT(advance >= 0);

   125     m_pos += advance;

   126     setState(s);

   127     return Token(code);

   128 }

   129 

   130 Tokenizer::Token XQueryTokenizer::tokenAndChangeState(const TokenType code,

   131                                                       const QString &value,

   132                                                       const State s)

   133 {

   134     setState(s);

   135     return Token(code, value);

   136 }

   137 

   138 Tokenizer::Token XQueryTokenizer::tokenAndAdvance(const TokenType code,

   139                                                   const int advance)

   140 {

   141     Q_ASSERT(advance >= 0);

   142     m_pos += advance;

   143     return Token(code);

   144 }

   145 

   146 QString XQueryTokenizer::normalizeEOL(const QString &input,

   147                                       const CharacterSkips &characterSkips)

   148 {

   149     const int len = input.count();

   150     QString result;

   151 

   152     /* The likely hood is rather high it'll be the same content. */

   153     result.reserve(len);

   154 

   155     for(int i = 0; i < len; ++i)

   156     {

   157         const QChar &at = input.at(i);

   158 

   159         if(characterSkips.contains(i))

   160         {

   161             result.append(at);

   162             continue;

   163         }

   164         switch(input.at(i).unicode())

   165         {

   166             case '\r':

   167             {

   168                 if(i + 1 < len && input.at(i + 1) == QLatin1Char('\n'))

   169                     ++i;

   170 

   171                 /* Else, fallthrough. */

   172             }

   173             case '\n':

   174             {

   175                 result.append(QLatin1Char('\n'));

   176                 continue;

   177             }

   178             default:

   179             {

   180                 result.append(at);

   181             }

   182         }

   183     }

   184 

   185     return result;

   186 }

   187 

   188 Tokenizer::TokenType XQueryTokenizer::consumeComment()

   189 {

   190     /* Below, we return ERROR instead of END_OF_FILE such that the parser

   191      * sees an invalid comment. */

   192     while(m_pos < m_length)

   193     {

   194         switch(peekCurrent())

   195         {

   196             case ':':

   197             {

   198                 ++m_pos; /* Consume ':' */

   199                 if(atEnd())

   200                     return ERROR;

   201 

   202                 if(peekCurrent() == ')')

   203                 {

   204                     ++m_pos; /* Consume ')' */

   205                     return SUCCESS; /* The comment closed nicely. */

   206                 }

   207                 continue; /* We don't want to increment m_pos twice. */

   208             }

   209             case '(':

   210             { /* It looks like the start of a comment. */

   211                 ++m_pos;

   212 

   213                 if(atEnd())

   214                     return END_OF_FILE;

   215                 else if(peekCurrent() == ':')

   216                 {

   217                     /* And it is a nested comment -- parse it. */

   218                     const TokenType retval = consumeComment();

   219                     if(retval == SUCCESS)

   220                         continue; /* Continue with our "own" comment. */

   221                     else

   222                         return retval; /* Return the error in the nested comment. */

   223                 }

   224                 break;

   225             }

   226             case '\n':

   227             /* Fallthrough. */

   228             case '\r':

   229             {

   230                 /* We want to count \r\n as a single line break. */

   231                 if(peekAhead() == '\n')

   232                     ++m_pos;

   233 

   234                 m_columnOffset = m_pos;

   235                 ++m_line;

   236 

   237                 break;

   238             }

   239         }

   240         ++m_pos;

   241     }

   242 

   243     return ERROR; /* Error: we reached the end while inside a comment. */

   244 }

   245 

   246 bool XQueryTokenizer::consumeRawWhitespace()

   247 {

   248     while(m_pos < m_length)

   249     {

   250         switch(peekCurrent())

   251         {

   252             case ' ':

   253             case '\t':

   254                 break;

   255             case '\n':

   256             case '\r':

   257             {

   258                 if(peekAhead() == '\n')

   259                     ++m_pos;

   260 

   261                 m_columnOffset = m_pos;

   262                 ++m_line;

   263 

   264                 break;

   265             }

   266             default:

   267                 return false;

   268         }

   269         ++m_pos;

   270     }

   271     return true;

   272 }

   273 

   274 Tokenizer::TokenType XQueryTokenizer::consumeWhitespace()

   275 {

   276     while(m_pos < m_length)

   277     {

   278         switch(peekCurrent())

   279         {

   280             case ' ':

   281             case '\t':

   282                 break;

   283             case '\n':

   284             case '\r':

   285             {

   286                 /* We want to count \r\n as a single line break. */

   287                 if(peekAhead() == '\n')

   288                     ++m_pos;

   289 

   290                 m_columnOffset = m_pos;

   291                 ++m_line;

   292 

   293                 break;

   294             }

   295             case '(':

   296             {

   297                 if(peekAhead() == ':')

   298                 {

   299                     m_pos += 2; /* Consume "(:" */

   300 

   301                     const TokenType comment = consumeComment();

   302                     if(comment == SUCCESS)

   303                         continue;

   304                     else

   305                         return comment;

   306                 }

   307             }

   308             default:

   309                 return SUCCESS;

   310         }

   311         ++m_pos;

   312     }

   313 

   314     return END_OF_FILE;

   315 }

   316 

   317 char XQueryTokenizer::peekAhead(const int length) const

   318 {

   319     if(m_pos + length < m_length)

   320         return m_data.at(m_pos + length).toAscii();

   321     else

   322         return 0;

   323 }

   324 

   325 Tokenizer::Token XQueryTokenizer::error()

   326 {

   327     return Token(ERROR);

   328 }

   329 

   330 bool XQueryTokenizer::isDigit(const char ch)

   331 {

   332     return ch >= '0' && ch <= '9';

   333 }

   334 

   335 /* Replace with function in QXmlUtils. Write test cases for this. */

   336 bool XQueryTokenizer::isNCNameStart(const QChar ch)

   337 {

   338     if(ch == QLatin1Char('_'))

   339         return true;

   340 

   341     switch(ch.category())

   342     {

   343         case QChar::Letter_Lowercase:

   344         case QChar::Letter_Uppercase:

   345         case QChar::Letter_Other:

   346         case QChar::Letter_Titlecase:

   347         case QChar::Number_Letter:

   348             return true;

   349         default:

   350             return false;

   351     }

   352 }

   353 

   354 bool XQueryTokenizer::isNCNameBody(const QChar ch)

   355 {

   356     switch(ch.unicode())

   357     {

   358         case '.':

   359         case '_':

   360         case '-':

   361             return true;

   362     }

   363 

   364     switch(ch.category())

   365     {

   366         case QChar::Letter_Lowercase:

   367         case QChar::Letter_Uppercase:

   368         case QChar::Letter_Other:

   369         case QChar::Letter_Titlecase:

   370         case QChar::Number_Letter:

   371         case QChar::Mark_SpacingCombining:

   372         case QChar::Mark_Enclosing:

   373         case QChar::Mark_NonSpacing:

   374         case QChar::Letter_Modifier:

   375         case QChar::Number_DecimalDigit:

   376             return true;

   377         default:

   378             return false;

   379     }

   380 }

   381 

   382 bool XQueryTokenizer::isPhraseKeyword(const TokenType code)

   383 {

   384     switch(code)

   385     {

   386         /* Fallthrough all these. */

   387         case CASTABLE:

   388         case CAST:

   389         case COPY_NAMESPACES:

   390         case DECLARE:

   391         case EMPTY:

   392         case MODULE:

   393         case IMPORT:

   394         case INSTANCE:

   395         case ORDER:

   396         case ORDERING:

   397         case XQUERY:

   398         case STABLE:

   399         case TREAT:

   400             return true;

   401         default:

   402             return false;

   403     }

   404 }

   405 

   406 bool XQueryTokenizer::isOperatorKeyword(const TokenType code)

   407 {

   408     switch(code)

   409     {

   410         /* Fallthrough all these. */

   411         case AS:

   412         case ASCENDING:

   413         case AT:

   414         case CASE:

   415         case CAST:

   416         case CASTABLE:

   417         case EQ:

   418         case EXTERNAL:

   419         case GE:

   420         case G_EQ:

   421         case G_GT:

   422         case G_LT:

   423         case G_NE:

   424         case GT:

   425         case IN:

   426         case INHERIT:

   427         case INSTANCE:

   428         case IS:

   429         case ITEM:

   430         case LE:

   431         case LT:

   432         case NE:

   433         case NO_INHERIT:

   434         case NO_PRESERVE:

   435         case OF:

   436         case PRESERVE:

   437         case RETURN:

   438         case STABLE:

   439         case TO:

   440         case TREAT:

   441             return true;

   442         default:

   443             return false;

   444     };

   445 }

   446 

   447 bool XQueryTokenizer::isTypeToken(const TokenType t)

   448 {

   449     switch(t)

   450     {

   451         /* Fallthrough all these. */

   452         case ATTRIBUTE:

   453         case COMMENT:

   454         case DOCUMENT:

   455         case DOCUMENT_NODE:

   456         case ELEMENT:

   457         case ITEM:

   458         case NODE:

   459         case PROCESSING_INSTRUCTION:

   460         case SCHEMA_ATTRIBUTE:

   461         case SCHEMA_ELEMENT:

   462         case TEXT:

   463             return true;

   464         default:

   465             return false;

   466     }

   467 }

   468 

   469 Tokenizer::Token XQueryTokenizer::tokenizeNCNameOrQName()

   470 {

   471     const int start = m_pos;

   472 

   473     const Token t1 = tokenizeNCName();

   474     if(t1.hasError())

   475         return t1;

   476 

   477     if(peekCurrent() != ':' || peekAhead() == '=')

   478         return t1;

   479 

   480     ++m_pos;

   481 

   482     const Token t2 = tokenizeNCName();

   483     if(t2.hasError())

   484         return t2;

   485     else

   486         return Token(QNAME, m_data.mid(start, m_pos - start));

   487 }

   488 

   489 Tokenizer::Token XQueryTokenizer::tokenizeNumberLiteral()

   490 {

   491     setState(Operator);

   492     const int startPos = m_pos;

   493     bool hasDot = false;

   494     bool isXPath20 = false;

   495 

   496     for(; m_pos < m_length; ++m_pos)

   497     {

   498         QChar ch(current());

   499 

   500         char cell = ch.cell();

   501 

   502         if(cell == 'e' || cell == 'E')

   503         {

   504             isXPath20 = true;

   505             ++m_pos;

   506             ch = current();

   507 

   508             if(ch.row() != 0)

   509                 break;

   510 

   511             cell = ch.cell();

   512 

   513             if(cell == '+' || cell == '-')

   514                 continue;

   515         }

   516 

   517         if(isNCNameStart(ch))

   518             return error();

   519 

   520         if(cell < '0' || cell > '9')

   521         {

   522             if(cell == '.' && !hasDot)

   523                 hasDot = true;

   524             else

   525                 break;

   526         }

   527     }

   528 

   529     return Token(isXPath20 ? XPATH2_NUMBER : NUMBER, m_data.mid(startPos, m_pos - startPos));

   530 }

   531 

   532 QString XQueryTokenizer::tokenizeCharacterReference()

   533 {

   534     Q_ASSERT(peekCurrent() == '&');

   535 

   536     const int theEnd = m_data.indexOf(QLatin1Char(';'), m_pos + 1);

   537 

   538     if(theEnd == -1) /* No ';' found, a syntax error. i18n. */

   539         return QString();

   540 

   541     QString content(m_data.mid(m_pos + 1, (theEnd - m_pos) - 1));

   542     m_pos = theEnd;

   543 

   544     const QChar charRef(charForReference(content));

   545 

   546     if(!charRef.isNull())

   547         return charRef;

   548     else if(content.startsWith(QLatin1Char('#')))

   549     {

   550         int base;

   551 

   552         /* It is only '#' or '#x'. */

   553         if(content.length() < 2)

   554             return QString();

   555 

   556         /* We got a hex number if it starts with 'x', otherwise it's a decimal. */

   557         if(content.at(1) == QLatin1Char('x'))

   558         {

   559             base = 16;

   560             content = content.mid(2); /* Remove "#x". */

   561         }

   562         else

   563         {

   564             base = 10;

   565             content = content.mid(1); /* Remove "#". */

   566         }

   567 

   568         bool conversionOK = false;

   569         const int codepoint = content.toInt(&conversionOK, base);

   570 

   571         if(conversionOK)

   572         {

   573             const QChar ch(codepoint);

   574 

   575             if(ch.isNull())

   576             {

   577                 /* We likely have something which require surrogate pairs. */

   578                 QString result;

   579                 result += QChar(QChar::highSurrogate(codepoint));

   580                 result += QChar(QChar::lowSurrogate(codepoint));

   581                 return result;

   582             }

   583             else

   584                 return ch;

   585         }

   586         else

   587             return QString();

   588     }

   589     else

   590         return QString();

   591 }

   592 

   593 int XQueryTokenizer::scanUntil(const char *const content)

   594 {

   595     const int end = m_data.indexOf(QString::fromLatin1(content), m_pos);

   596 

   597     if(end == -1)

   598         return -1;

   599     else

   600     {

   601         const int len = end - m_pos;

   602         m_pos += len;

   603         return len;

   604     }

   605 }

   606 

   607 QChar XQueryTokenizer::charForReference(const QString &reference)

   608 {

   609     if(m_charRefs.isEmpty())

   610     {

   611         /* Initialize. */

   612         m_charRefs.reserve(5);

   613         m_charRefs.insert(QLatin1String("lt"),     QLatin1Char('<'));

   614         m_charRefs.insert(QLatin1String("gt"),     QLatin1Char('>'));

   615         m_charRefs.insert(QLatin1String("amp"),    QLatin1Char('&'));

   616         m_charRefs.insert(QLatin1String("quot"),   QLatin1Char('"'));

   617         m_charRefs.insert(QLatin1String("apos"),   QLatin1Char('\''));

   618     }

   619 

   620     return m_charRefs.value(reference);

   621 }

   622 

   623 Tokenizer::Token XQueryTokenizer::tokenizeStringLiteral()

   624 {

   625     const QChar delimiter(current());

   626     /* We cannot unfortunately just scan and then do mid(),

   627      * since we can encounter character references. */

   628     QString result;

   629 

   630     /* This is more likely than QString's default allocation. */

   631     result.reserve(8);

   632 

   633     CharacterSkips skipEOLNormalization;

   634 

   635     /* Advance over the initial quote character. */

   636     ++m_pos;

   637 

   638     for(; m_pos < m_length; ++m_pos)

   639     {

   640         const QChar c(current());

   641 

   642         if(c == QLatin1Char('&'))

   643         {

   644             const QString charRef(tokenizeCharacterReference());

   645 

   646             if(charRef.isNull())

   647                 return error();

   648             else

   649             {

   650                 skipEOLNormalization.insert(result.count());

   651                 result.append(charRef);

   652             }

   653 

   654         }

   655         else if(c == delimiter)

   656         {

   657             /* Maybe the escaping mechanism is used. For instance, "s""s"

   658              * has the value `s"s'. */

   659             ++m_pos;

   660 

   661             if(current() == delimiter) /* Double quote. */

   662                 result += delimiter;

   663             else

   664                 return Token(STRING_LITERAL, normalizeEOL(result, skipEOLNormalization));

   665         }

   666         else

   667             result += c;

   668     }

   669 

   670     return error();

   671 }

   672 

   673 Tokenizer::Token XQueryTokenizer::tokenizeNCName()

   674 {

   675     const int startPos = m_pos;

   676 

   677     if(m_pos < m_length && isNCNameStart(current()))

   678     {

   679         ++m_pos;

   680 

   681         for(; m_pos < m_length; ++m_pos)

   682         {

   683             if(!isNCNameBody(current()))

   684                 break;

   685         }

   686 

   687         return Token(NCNAME, m_data.mid(startPos, m_pos - startPos));

   688     }

   689     else

   690         return error();

   691 }

   692 

   693 bool XQueryTokenizer::aheadEquals(const char *const chs,

   694                                   const int len,

   695                                   const int offset) const

   696 {

   697     Q_ASSERT(len > 0);

   698     Q_ASSERT(qstrlen(chs) == uint(len));

   699 

   700     if(m_pos + len >= m_length)

   701         return false;

   702 

   703     for(int i = offset; i < (len + offset); ++i)

   704     {

   705         if(m_data.at(m_pos + i).toAscii() != chs[i - offset])

   706             return false;

   707     }

   708 

   709     return true;

   710 }

   711 

   712 const TokenMap *XQueryTokenizer::lookupKeyword(const QString &keyword)

   713 {

   714     return TokenLookup::value(keyword.toAscii().constData(), keyword.length());

   715 }

   716 

   717 XQueryTokenizer::State XQueryTokenizer::state() const

   718 {

   719     return m_state;

   720 }

   721 

   722 void XQueryTokenizer::setState(const State s)

   723 {

   724     m_state = s;

   725 }

   726 

   727 void XQueryTokenizer::pushState(const State s)

   728 {

   729     m_stateStack.push(s);

   730 }

   731 

   732 void XQueryTokenizer::pushState()

   733 {

   734     m_stateStack.push(m_state);

   735 }

   736 

   737 void XQueryTokenizer::popState()

   738 {

   739     /* QStack::pop() asserts if it's empty, so we need to check

   740      * it, since we might receive unbalanced curlies. */

   741     if(!m_stateStack.isEmpty())

   742         m_state = m_stateStack.pop();

   743 }

   744 

   745 Tokenizer::Token XQueryTokenizer::nextToken()

   746 {

   747     switch(state())

   748     {

   749         /* We want to skip or do special whitespace handling for these

   750          * states. So fallthrough all of the following. */

   751         case AposAttributeContent:

   752         case Axis:

   753         case ElementContent:

   754         case EndTag:

   755         case Pragma:

   756         case PragmaContent:

   757         case ProcessingInstructionName:

   758         case QuotAttributeContent:

   759         case StartTag:

   760         case XMLComment:

   761             break;

   762         default:

   763             handleWhitespace();

   764     }

   765 

   766     switch(state())

   767     {

   768         case XMLSpaceDecl:

   769         /* Fallthrough. */

   770         case NamespaceKeyword:

   771         {

   772             switch(peekCurrent())

   773             {

   774                 case ',':

   775                     return tokenAndAdvance(COMMA);

   776                 case '"':

   777                 /* Fallthrough. */

   778                 case '\'':

   779                 {

   780                     setState(NamespaceDecl);

   781                     return tokenizeStringLiteral();

   782                 }

   783             }

   784 

   785             const Token id(tokenizeNCName());

   786 

   787             if(id.type != NCNAME)

   788                 return id;

   789 

   790             const TokenMap *const keyword = lookupKeyword(id.value);

   791             if(keyword)

   792             {

   793                 switch(keyword->token)

   794                 {

   795                     case INHERIT:

   796                     /* Fallthrough. */

   797                     case NO_INHERIT:

   798                     {

   799                         setState(Default);

   800                         break;

   801                     }

   802                     case NAMESPACE:

   803                     {

   804                         setState(NamespaceDecl);

   805                         break;

   806                     }

   807                     case ORDERED:

   808                     /* Fallthrough. */

   809                     case UNORDERED:

   810                     /* Fallthrough. */

   811                     case STRIP:

   812                     {

   813                         setState(Default);

   814                         break;

   815                     }

   816                     case PRESERVE:

   817                     {

   818                         if(state() != NamespaceKeyword)

   819                             setState(Default);

   820                     }

   821                     default:

   822                         break;

   823                 }

   824 

   825                 return Token(keyword->token);

   826             }

   827             else

   828                 return id;

   829 

   830             Q_ASSERT(false);

   831         }

   832         case NamespaceDecl:

   833         {

   834             switch(peekCurrent())

   835             {

   836                 case '=':

   837                     return tokenAndAdvance(G_EQ);

   838                 case ';':

   839                     return tokenAndChangeState(SEMI_COLON, Default);

   840                 case '\'':

   841                 /* Fallthrough. */

   842                 case '\"':

   843                     return tokenizeStringLiteral();

   844             }

   845 

   846             const Token nc(tokenizeNCName());

   847 

   848             handleWhitespace();

   849 

   850             const char pc = peekCurrent();

   851             const TokenMap* const t = lookupKeyword(nc.value);

   852 

   853             if(pc == '\'' || (pc == '"' && t))

   854                 return tokenAndChangeState(t->token, Default, 0);

   855             else

   856                 return nc;

   857 

   858             Q_ASSERT(false);

   859         }

   860         case Axis:

   861         {

   862             if(peekCurrent() == ':')

   863             {

   864                 Q_ASSERT(peekAhead() == ':');

   865                 m_pos += 2;

   866                 setState(AfterAxisSeparator);

   867                 return Token(COLONCOLON);

   868             }

   869             /* Fallthrough. */

   870         }

   871         case AfterAxisSeparator:

   872         /* Fallthrough. */

   873         case Default:

   874            /* State Operator and state Default have a lot of tokens in common except

   875             * for minor differences. So we treat them the same way, and sprinkles logic

   876             * here and there to handle the small differences. */

   877         /* Fallthrough. */

   878         case Operator:

   879         {

   880             switch(peekCurrent())

   881             {

   882                 case '=':

   883                     return tokenAndChangeState(G_EQ, Default);

   884                 case '-':

   885                     return tokenAndChangeState(MINUS, Default);

   886                 case '+':

   887                     return tokenAndChangeState(PLUS, Default);

   888                 case '[':

   889                     return tokenAndChangeState(LBRACKET, Default);

   890                 case ']':

   891                     return tokenAndChangeState(RBRACKET, Operator);

   892                 case ',':

   893                     return tokenAndChangeState(COMMA, Default);

   894                 case ';':

   895                     return tokenAndChangeState(SEMI_COLON, Default);

   896                 case '$':

   897                     return tokenAndChangeState(DOLLAR, VarName);

   898                 case '|':

   899                     return tokenAndChangeState(BAR, Default);

   900                 case '?':

   901                     return tokenAndChangeState(QUESTION, Operator);

   902                 case ')':

   903                     return tokenAndChangeState(RPAREN, Operator);

   904                 case '@':

   905                     return tokenAndChangeState(AT_SIGN, Default);

   906                 /* Fallthrough all these. */

   907                 case '1':

   908                 case '2':

   909                 case '3':

   910                 case '4':

   911                 case '5':

   912                 case '6':

   913                 case '7':

   914                 case '8':

   915                 case '9':

   916                 case '0':

   917                     return tokenizeNumberLiteral();

   918                 case '.':

   919                 {

   920                     const char next = peekAhead();

   921                     if(next == '.')

   922                         return tokenAndChangeState(DOTDOT, Operator, 2);

   923                     /* .5 is allowed, as short form for 0.5:

   924                      * <tt>[142]     DecimalLiteral     ::=     ("." Digits) | (Digits "." [0-9]*)</tt>

   925                      */

   926                     else if(isDigit(next))

   927                         return tokenizeNumberLiteral();

   928                     else

   929                         return tokenAndChangeState(DOT, Operator);

   930                 }

   931                 case '\'':

   932                 /* Fallthrough. */

   933                 case '"':

   934                 {

   935                     setState(Operator);

   936                     return tokenizeStringLiteral();

   937 

   938                 }

   939                 case '(':

   940                 {

   941                     if(peekAhead() == '#')

   942                         return tokenAndChangeState(PRAGMA_START, Pragma, 2);

   943                     else

   944                         return tokenAndChangeState(LPAREN, Default);

   945                 }

   946                 case '*':

   947                 {

   948                     if(peekAhead() == ':')

   949                     {

   950                         m_pos += 2; /* Consume *:. */

   951                         const Token nc = tokenizeNCName();

   952 

   953                         if(nc.hasError())

   954                             return error();

   955                         else

   956                             return tokenAndChangeState(ANY_PREFIX, nc.value, Operator);

   957                     }

   958                     else

   959                         return tokenAndChangeState(STAR, state() == Default ? Operator : Default);

   960                 }

   961                 case ':':

   962                 {

   963                     switch(peekAhead())

   964                     {

   965                         case '=':

   966                             return tokenAndChangeState(ASSIGN, Default, 2);

   967                         case ':':

   968                             return tokenAndChangeState(COLONCOLON, Default, 2);

   969                         default:

   970                             return error();

   971                     }

   972                 }

   973                 case '!':

   974                 {

   975                     if(peekAhead() == '=')

   976                         return tokenAndChangeState(G_NE, Default, 2);

   977                     else

   978                         return error();

   979                 }

   980                 case '<':

   981                 {

   982                     switch(peekAhead())

   983                     {

   984                         case '=':

   985                             return tokenAndChangeState(G_LE, Default, 2);

   986                         case '<':

   987                             return tokenAndChangeState(PRECEDES, Default, 2);

   988                         case '?':

   989                         {

   990                             pushState(Operator);

   991                             return tokenAndChangeState(PI_START, ProcessingInstructionName, 2);

   992                         }

   993                         case '!':

   994                         {

   995                             if(aheadEquals("!--", 3))

   996                             {

   997                                 m_pos += 3; /* Consume "!--". */

   998                                 pushState(Operator);

   999                                 return tokenAndChangeState(COMMENT_START, XMLComment);

  1000                             }

  1001                             /* Fallthrough. It's a syntax error, and this is a good way to report it. */

  1002                         }

  1003                         default:

  1004                         {

  1005                             if((m_pos + 1) < m_length && isNCNameStart(m_data.at(m_pos + 1)))

  1006                             {

  1007                                 /* We assume it's an element constructor. */

  1008                                 pushState(Operator);

  1009                             }

  1010 

  1011                             return tokenAndChangeState(G_LT, state() == Operator ? Default : StartTag);

  1012                         }

  1013                     }

  1014                 }

  1015                 case '>':

  1016                 {

  1017                     switch(peekAhead())

  1018                     {

  1019                         case '=':

  1020                             return tokenAndChangeState(G_GE, Default, 2);

  1021                         case '>':

  1022                             return tokenAndChangeState(FOLLOWS, Default, 2);

  1023                         default:

  1024                             return tokenAndChangeState(G_GT, Default);

  1025                     }

  1026                 }

  1027                 case '/':

  1028                 {

  1029                     if(peekAhead() == '/')

  1030                         return tokenAndChangeState(SLASHSLASH, Default, 2);

  1031                     else

  1032                         return tokenAndChangeState(SLASH, Default);

  1033                 }

  1034                 case '{':

  1035                 {

  1036                     pushState(Operator);

  1037                     return tokenAndChangeState(CURLY_LBRACE, Default);

  1038                 }

  1039                 case '}':

  1040                 {

  1041                     popState();

  1042 

  1043                     return tokenAndAdvance(CURLY_RBRACE);

  1044                 }

  1045             }

  1046 

  1047             /* Ok. We're in state Default or Operator, and it wasn't a simple

  1048              * character. */

  1049 

  1050             const Token id(tokenizeNCName());

  1051 

  1052             if(id.type != NCNAME)

  1053                 return id;

  1054 

  1055             const TokenMap *const keyword = lookupKeyword(id.value);

  1056 

  1057             if(state() == Operator)

  1058             {

  1059                 if(keyword)

  1060                 {

  1061                     if(keyword->token == DEFAULT || keyword->token == ASCENDING || keyword->token == DESCENDING)

  1062                         setState(Operator);

  1063                     else if(keyword->token == RETURN)

  1064                         setState(Default);

  1065                     else if(isPhraseKeyword(keyword->token))

  1066                     {

  1067                         const TokenType ws = consumeWhitespace();

  1068                         if(ws == ERROR)

  1069                             return error();

  1070 

  1071                         const Token id2(tokenizeNCName());

  1072                         const TokenMap *const keyword2 = lookupKeyword(id2.value);

  1073 

  1074                         if(keyword2)

  1075                         {

  1076                             if(keyword->token == TREAT && keyword2->token == AS)

  1077                                 setState(ItemType);

  1078                             else if (keyword->token == CAST || (keyword->token == CASTABLE && keyword2->token == AS) || keyword2->token == BY)

  1079                                 setState(Default);

  1080 

  1081                             m_tokenStack.push(Token(keyword2->token));

  1082                         }

  1083                         else

  1084                             m_tokenStack.push(id2);

  1085 

  1086                         return Token(keyword->token);

  1087                     }

  1088                     else

  1089                     {

  1090                         /* Such that we tokenize the second token in "empty greatest". */

  1091                         if(keyword->token != EMPTY)

  1092                             setState(Default);

  1093                     }

  1094 

  1095                     if(keyword->token == AS || keyword->token == CASE)

  1096                         setState(ItemType);

  1097 

  1098                     return Token(keyword->token);

  1099                 }

  1100                 else

  1101                     return id;

  1102             }

  1103 

  1104             Q_ASSERT(state() == Default || state() == Axis || state() == AfterAxisSeparator);

  1105 

  1106             /*

  1107              * This is hard. Consider this:

  1108              *

  1109              * Valid:           child       ::nameTest

  1110              * Valid:           child::     nameTest

  1111              * Syntax Error:    child       :localName

  1112              * Syntax Error:    child:      localName

  1113              *

  1114              * Consider "child ::name". Right now, we're here:

  1115              *                ^

  1116              * We don't know whether "child" is a prefix and hence the whitespace is invalid,

  1117              * or whether it's an axis and hence skippable. */

  1118             {

  1119                 const int wsLength = peekForColonColon();

  1120                 /* We cannot call handleWhitespace() because it returns on

  1121                  * END_OF_FILE, and we have parsed up keyword, and we need to

  1122                  * deal with that.

  1123                  *

  1124                  * If we have a colon colon, which means the whitespace is

  1125                  * allowed, we skip it. */

  1126                 if(wsLength != -1)

  1127                     m_pos += wsLength;

  1128             }

  1129 

  1130             /* Handle name tests. */

  1131             if(peekCurrent() == ':')

  1132             {

  1133                 switch(peekAhead())

  1134                 {