1 /****************************************************************************

     2 **

     3 ** Copyright (C) 2010 Nokia Corporation and/or its subsidiary(-ies).

     4 ** All rights reserved.

     5 ** Contact: Nokia Corporation (qt-info@nokia.com)

     6 **

     7 ** This file is part of the Qt Mobility Components.

     8 **

     9 ** $QT_BEGIN_LICENSE:LGPL$

    10 ** No Commercial Usage

    11 ** This file contains pre-release code and may not be distributed.

    12 ** You may use this file in accordance with the terms and conditions

    13 ** contained in the Technology Preview License Agreement accompanying

    14 ** this package.

    15 **

    16 ** GNU Lesser General Public License Usage

    17 ** Alternatively, this file may be used under the terms of the GNU Lesser

    18 ** General Public License version 2.1 as published by the Free Software

    19 ** Foundation and appearing in the file LICENSE.LGPL included in the

    20 ** packaging of this file.  Please review the following information to

    21 ** ensure the GNU Lesser General Public License version 2.1 requirements

    22 ** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.

    23 **

    24 ** In addition, as a special exception, Nokia gives you certain additional

    25 ** rights.  These rights are described in the Nokia Qt LGPL Exception

    26 ** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.

    27 **

    28 ** If you have questions regarding the use of this file, please contact

    29 ** Nokia at qt-info@nokia.com.

    30 **

    31 **

    32 **

    33 **

    34 **

    35 **

    36 **

    37 **

    38 ** $QT_END_LICENSE$

    39 **

    40 ****************************************************************************/

    41 // Copyright (c) 2008 Roberto Raggi <roberto.raggi@gmail.com>

    42 //

    43 // Permission is hereby granted, free of charge, to any person obtaining a copy

    44 // of this software and associated documentation files (the "Software"), to deal

    45 // in the Software without restriction, including without limitation the rights

    46 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell

    47 // copies of the Software, and to permit persons to whom the Software is

    48 // furnished to do so, subject to the following conditions:

    49 //

    50 // The above copyright notice and this permission notice shall be included in

    51 // all copies or substantial portions of the Software.

    52 //

    53 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR

    54 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,

    55 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE

    56 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER

    57 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,

    58 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN

    59 // THE SOFTWARE.

    60 

    61 #include "Lexer.h"

    62 #include "Control.h"

    63 #include "TranslationUnit.h"

    64 #include "Literals.h"

    65 #include <cctype>

    66 #include <cassert>

    67 

    68 using namespace CPlusPlus;

    69 

    70 Lexer::Lexer(TranslationUnit *unit)

    71     : _translationUnit(unit),

    72       _state(State_Default),

    73       _flags(0),

    74       _currentLine(1)

    75 {

    76     f._scanKeywords = true;

    77     setSource(_translationUnit->firstSourceChar(),

    78               _translationUnit->lastSourceChar());

    79 }

    80 

    81 Lexer::Lexer(const char *firstChar, const char *lastChar)

    82     : _translationUnit(0),

    83       _state(State_Default),

    84       _flags(0),

    85       _currentLine(1)

    86 {

    87     f._scanKeywords = true;

    88     setSource(firstChar, lastChar);

    89 }

    90 

    91 Lexer::~Lexer()

    92 { }

    93 

    94 TranslationUnit *Lexer::translationUnit() const

    95 { return _translationUnit; }

    96 

    97 Control *Lexer::control() const

    98 {

    99     if (_translationUnit)

   100         return _translationUnit->control();

   101 

   102     return 0;

   103 }

   104 

   105 void Lexer::setSource(const char *firstChar, const char *lastChar)

   106 {

   107     _firstChar = firstChar;

   108     _lastChar = lastChar;

   109     _currentChar = _firstChar - 1;

   110     _tokenStart = _currentChar;

   111     _yychar = '\n';

   112 }

   113 

   114 void Lexer::setStartWithNewline(bool enabled)

   115 {

   116     if (enabled)

   117         _yychar = '\n';

   118     else

   119         _yychar = ' ';

   120 }

   121 

   122 int Lexer::state() const

   123 { return _state; }

   124 

   125 void Lexer::setState(int state)

   126 { _state = state; }

   127 

   128 bool Lexer::qtMocRunEnabled() const

   129 { return f._qtMocRunEnabled; }

   130 

   131 void Lexer::setQtMocRunEnabled(bool onoff)

   132 { f._qtMocRunEnabled = onoff; }

   133 

   134 bool Lexer::objCEnabled() const

   135 { return f._objCEnabled; }

   136 

   137 void Lexer::setObjCEnabled(bool onoff)

   138 { f._objCEnabled = onoff; }

   139 

   140 bool Lexer::isIncremental() const

   141 { return f._isIncremental; }

   142 

   143 void Lexer::setIncremental(bool isIncremental)

   144 { f._isIncremental = isIncremental; }

   145 

   146 bool Lexer::scanCommentTokens() const

   147 { return f._scanCommentTokens; }

   148 

   149 void Lexer::setScanCommentTokens(bool onoff)

   150 { f._scanCommentTokens = onoff; }

   151 

   152 bool Lexer::scanKeywords() const

   153 { return f._scanKeywords; }

   154 

   155 void Lexer::setScanKeywords(bool onoff)

   156 { f._scanKeywords = onoff; }

   157 

   158 void Lexer::setScanAngleStringLiteralTokens(bool onoff)

   159 { f._scanAngleStringLiteralTokens = onoff; }

   160 

   161 void Lexer::pushLineStartOffset()

   162 {

   163     ++_currentLine;

   164 

   165     if (_translationUnit)

   166         _translationUnit->pushLineOffset(_currentChar - _firstChar);

   167 }

   168 

   169 unsigned Lexer::tokenOffset() const

   170 { return _tokenStart - _firstChar; }

   171 

   172 unsigned Lexer::tokenLength() const

   173 { return _currentChar - _tokenStart; }

   174 

   175 const char *Lexer::tokenBegin() const

   176 { return _tokenStart; }

   177 

   178 const char *Lexer::tokenEnd() const

   179 { return _currentChar; }

   180 

   181 unsigned Lexer::currentLine() const

   182 { return _currentLine; }

   183 

   184 void Lexer::scan(Token *tok)

   185 {

   186     tok->reset();

   187     scan_helper(tok);

   188     tok->f.length = _currentChar - _tokenStart;

   189 }

   190 

   191 void Lexer::scan_helper(Token *tok)

   192 {

   193   _Lagain:

   194     while (_yychar && std::isspace(_yychar)) {

   195         if (_yychar == '\n')

   196             tok->f.newline = true;

   197         else

   198             tok->f.whitespace = true;

   199         yyinp();

   200     }

   201 

   202     if (! _translationUnit)

   203         tok->lineno = _currentLine;

   204 

   205     _tokenStart = _currentChar;

   206     tok->offset = _currentChar - _firstChar;

   207 

   208     if (_state == State_MultiLineComment || _state == State_MultiLineDoxyComment) {

   209         const int originalState = _state;

   210 

   211         if (! _yychar) {

   212             tok->f.kind = T_EOF_SYMBOL;

   213             return;

   214         }

   215 

   216         while (_yychar) {

   217             if (_yychar != '*')

   218                 yyinp();

   219             else {

   220                 yyinp();

   221                 if (_yychar == '/') {

   222                     yyinp();

   223                     _state = State_Default;

   224                     break;

   225                 }

   226             }

   227         }

   228 

   229         if (! f._scanCommentTokens)

   230             goto _Lagain;

   231 

   232         else if (originalState == State_MultiLineComment)

   233             tok->f.kind = T_COMMENT;

   234         else

   235             tok->f.kind = T_DOXY_COMMENT;

   236         return; // done

   237     }

   238 

   239     if (! _yychar) {

   240         tok->f.kind = T_EOF_SYMBOL;

   241         return;

   242     }

   243 

   244     unsigned char ch = _yychar;

   245     yyinp();

   246 

   247     switch (ch) {

   248     case '\\':

   249         while (_yychar != '\n' && std::isspace(_yychar))

   250             yyinp();

   251         // ### assert(! _yychar || _yychar == '\n');

   252         if (_yychar == '\n') {

   253             tok->f.joined = true;

   254             tok->f.newline = false;

   255             yyinp();

   256         }

   257         goto _Lagain;

   258 

   259     case '"': case '\'': {

   260         const char quote = ch;

   261 

   262         tok->f.kind = quote == '"'

   263             ? T_STRING_LITERAL

   264             : T_CHAR_LITERAL;

   265 

   266         const char *yytext = _currentChar;

   267 

   268         while (_yychar && _yychar != quote) {

   269             if (_yychar == '\n')

   270                 break;

   271             else if (_yychar != '\\')

   272                 yyinp();

   273             else {

   274                 yyinp(); // skip `\\'

   275 

   276                 if (_yychar)

   277                     yyinp();

   278             }

   279         }

   280         // assert(_yychar == quote);

   281 

   282         int yylen = _currentChar - yytext;

   283 

   284         if (_yychar == quote)

   285             yyinp();

   286 

   287         if (control())

   288             tok->string = control()->findOrInsertStringLiteral(yytext, yylen);

   289     } break;

   290 

   291     case '{':

   292         tok->f.kind = T_LBRACE;

   293         break;

   294 

   295     case '}':

   296         tok->f.kind = T_RBRACE;

   297         break;

   298 

   299     case '[':

   300         tok->f.kind = T_LBRACKET;

   301         break;

   302 

   303     case ']':

   304         tok->f.kind = T_RBRACKET;

   305         break;

   306 

   307     case '#':

   308         if (_yychar == '#') {

   309             tok->f.kind = T_POUND_POUND;

   310             yyinp();

   311         } else {

   312             tok->f.kind = T_POUND;

   313         }

   314         break;

   315 

   316     case '(':

   317         tok->f.kind = T_LPAREN;

   318         break;

   319 

   320     case ')':

   321         tok->f.kind = T_RPAREN;

   322         break;

   323 

   324     case ';':

   325         tok->f.kind = T_SEMICOLON;

   326         break;

   327 

   328     case ':':

   329         if (_yychar == ':') {

   330             yyinp();

   331             tok->f.kind = T_COLON_COLON;

   332         } else {

   333             tok->f.kind = T_COLON;

   334         }

   335         break;

   336 

   337     case '.':

   338         if (_yychar == '*') {

   339             yyinp();

   340             tok->f.kind = T_DOT_STAR;

   341         } else if (_yychar == '.') {

   342             yyinp();

   343             // ### assert(_yychar);

   344             if (_yychar == '.') {

   345                 yyinp();

   346                 tok->f.kind = T_DOT_DOT_DOT;

   347             } else {

   348                 tok->f.kind = T_ERROR;

   349             }

   350         } else if (std::isdigit(_yychar)) {

   351             const char *yytext = _currentChar - 2;

   352             do {

   353                 if (_yychar == 'e' || _yychar == 'E') {

   354                     yyinp();

   355                     if (_yychar == '-' || _yychar == '+') {

   356                         yyinp();

   357                         // ### assert(std::isdigit(_yychar));

   358                     }

   359                 } else if (std::isalnum(_yychar) || _yychar == '.') {

   360                     yyinp();

   361                 } else {

   362                     break;

   363                 }

   364             } while (_yychar);

   365             int yylen = _currentChar - yytext;

   366             tok->f.kind = T_NUMERIC_LITERAL;

   367             if (control())

   368                 tok->number = control()->findOrInsertNumericLiteral(yytext, yylen);

   369         } else {

   370             tok->f.kind = T_DOT;

   371         }

   372         break;

   373 

   374     case '?':

   375         tok->f.kind = T_QUESTION;

   376         break;

   377 

   378     case '+':

   379         if (_yychar == '+') {

   380             yyinp();

   381             tok->f.kind = T_PLUS_PLUS;

   382         } else if (_yychar == '=') {

   383             yyinp();

   384             tok->f.kind = T_PLUS_EQUAL;

   385         } else {

   386             tok->f.kind = T_PLUS;

   387         }

   388         break;

   389 

   390     case '-':

   391         if (_yychar == '-') {

   392             yyinp();

   393             tok->f.kind = T_MINUS_MINUS;

   394         } else if (_yychar == '=') {

   395             yyinp();

   396             tok->f.kind = T_MINUS_EQUAL;

   397         } else if (_yychar == '>') {

   398             yyinp();

   399             if (_yychar == '*') {

   400                 yyinp();

   401                 tok->f.kind = T_ARROW_STAR;

   402             } else {

   403                 tok->f.kind = T_ARROW;

   404             }

   405         } else {

   406             tok->f.kind = T_MINUS;

   407         }

   408         break;

   409 

   410     case '*':

   411         if (_yychar == '=') {

   412             yyinp();

   413             tok->f.kind = T_STAR_EQUAL;

   414         } else {

   415             tok->f.kind = T_STAR;

   416         }

   417         break;

   418 

   419     case '/':

   420         if (_yychar == '/') {

   421             yyinp();

   422 

   423             bool doxy = false;

   424 

   425             if (_yychar == '/' || _yychar == '!') {

   426                 yyinp();

   427 

   428                 if (_yychar == '<')

   429                     yyinp();

   430 

   431                 if (_yychar != '\n' && std::isspace(_yychar))

   432                     doxy = true;

   433             }

   434 

   435             while (_yychar && _yychar != '\n')

   436                 yyinp();

   437 

   438             if (! f._scanCommentTokens)

   439                 goto _Lagain;

   440 

   441             tok->f.kind = doxy ? T_CPP_DOXY_COMMENT : T_CPP_COMMENT;

   442 

   443         } else if (_yychar == '*') {

   444             yyinp();

   445 

   446             bool doxy = false;

   447 

   448             if (_yychar == '*' || _yychar == '!') {

   449                 const char ch = _yychar;

   450 

   451                 yyinp();

   452 

   453                 if (ch == '*' && _yychar == '/')

   454                     goto _Ldone;

   455 

   456                 if (_yychar == '<')

   457                     yyinp();

   458 

   459                 if (! _yychar || std::isspace(_yychar))

   460                     doxy = true;

   461             }

   462 

   463             while (_yychar) {

   464                 if (_yychar != '*') {

   465                     yyinp();

   466                 } else {

   467                     yyinp();

   468                     if (_yychar == '/')

   469                         break;

   470                 }

   471             }

   472 

   473         _Ldone:

   474             if (_yychar)

   475                 yyinp();

   476             else

   477                 _state = doxy ? State_MultiLineDoxyComment : State_MultiLineComment;

   478 

   479             if (! f._scanCommentTokens)

   480                 goto _Lagain;

   481 

   482             tok->f.kind = doxy ? T_DOXY_COMMENT : T_COMMENT;

   483 

   484         } else if (_yychar == '=') {

   485             yyinp();

   486             tok->f.kind = T_SLASH_EQUAL;

   487         } else {

   488             tok->f.kind = T_SLASH;

   489         }

   490         break;

   491 

   492     case '%':

   493         if (_yychar == '=') {

   494             yyinp();

   495             tok->f.kind = T_PERCENT_EQUAL;

   496         } else {

   497             tok->f.kind = T_PERCENT;

   498         }

   499         break;

   500 

   501     case '^':

   502         if (_yychar == '=') {

   503             yyinp();

   504             tok->f.kind = T_CARET_EQUAL;

   505         } else {

   506             tok->f.kind = T_CARET;

   507         }

   508         break;

   509 

   510     case '&':

   511         if (_yychar == '&') {

   512             yyinp();

   513             tok->f.kind = T_AMPER_AMPER;

   514         } else if (_yychar == '=') {

   515             yyinp();

   516             tok->f.kind = T_AMPER_EQUAL;

   517         } else {

   518             tok->f.kind = T_AMPER;

   519         }

   520         break;

   521 

   522     case '|':

   523         if (_yychar == '|') {

   524             yyinp();

   525             tok->f.kind = T_PIPE_PIPE;

   526         } else if (_yychar == '=') {

   527             yyinp();

   528             tok->f.kind = T_PIPE_EQUAL;

   529         } else {

   530             tok->f.kind = T_PIPE;

   531         }

   532         break;

   533 

   534     case '~':

   535         if (_yychar == '=') {

   536             yyinp();

   537             tok->f.kind = T_TILDE_EQUAL;

   538         } else {

   539             tok->f.kind = T_TILDE;

   540         }

   541         break;

   542 

   543     case '!':

   544         if (_yychar == '=') {

   545             yyinp();

   546             tok->f.kind = T_EXCLAIM_EQUAL;

   547         } else {

   548             tok->f.kind = T_EXCLAIM;

   549         }

   550         break;

   551 

   552     case '=':

   553         if (_yychar == '=') {

   554             yyinp();

   555             tok->f.kind = T_EQUAL_EQUAL;

   556         } else {

   557             tok->f.kind = T_EQUAL;

   558         }

   559         break;

   560 

   561     case '<':

   562         if (f._scanAngleStringLiteralTokens) {

   563             const char *yytext = _currentChar;

   564             while (_yychar && _yychar != '>')

   565                 yyinp();

   566             int yylen = _currentChar - yytext;

   567             // ### assert(_yychar == '>');

   568             if (_yychar == '>')

   569                 yyinp();

   570             if (control())

   571                 tok->string = control()->findOrInsertStringLiteral(yytext, yylen);

   572             tok->f.kind = T_ANGLE_STRING_LITERAL;

   573         } else if (_yychar == '<') {

   574             yyinp();

   575             if (_yychar == '=') {

   576                 yyinp();

   577                 tok->f.kind = T_LESS_LESS_EQUAL;

   578             } else

   579                 tok->f.kind = T_LESS_LESS;

   580         } else if (_yychar == '=') {

   581             yyinp();

   582             tok->f.kind = T_LESS_EQUAL;

   583         } else {

   584             tok->f.kind = T_LESS;

   585         }

   586         break;

   587 

   588     case '>':

   589         if (_yychar == '>') {

   590             yyinp();

   591             if (_yychar == '=') {

   592                 yyinp();

   593                 tok->f.kind = T_GREATER_GREATER_EQUAL;

   594             } else

   595                 tok->f.kind = T_LESS_LESS;

   596             tok->f.kind = T_GREATER_GREATER;

   597         } else if (_yychar == '=') {

   598             yyinp();

   599             tok->f.kind = T_GREATER_EQUAL;

   600         } else {

   601             tok->f.kind = T_GREATER;

   602         }

   603         break;

   604 

   605     case ',':

   606         tok->f.kind = T_COMMA;

   607         break;

   608 

   609     default: {

   610         if (f._objCEnabled) {

   611             if (ch == '@' && _yychar >= 'a' && _yychar <= 'z') {

   612                 const char *yytext = _currentChar;

   613 

   614                 do {

   615                     yyinp();

   616                     if (! (isalnum(_yychar) || _yychar == '_' || _yychar == '$'))

   617                         break;

   618                 } while (_yychar);

   619 

   620                 const int yylen = _currentChar - yytext;

   621                 tok->f.kind = classifyObjCAtKeyword(yytext, yylen);

   622                 break;

   623             } else if (ch == '@' && _yychar == '"') {

   624                 // objc @string literals

   625                 ch = _yychar;

   626                 yyinp();

   627                 tok->f.kind = T_AT_STRING_LITERAL;

   628 

   629                 const char *yytext = _currentChar;

   630 

   631                 while (_yychar && _yychar != '"') {

   632                     if (_yychar != '\\')

   633                         yyinp();

   634                     else {

   635                         yyinp(); // skip `\\'

   636 

   637                         if (_yychar)

   638                             yyinp();

   639                     }

   640                 }

   641                 // assert(_yychar == '"');

   642 

   643                 int yylen = _currentChar - yytext;

   644 

   645                 if (_yychar == '"')

   646                     yyinp();

   647 

   648                 if (control())

   649                     tok->string = control()->findOrInsertStringLiteral(yytext, yylen);

   650 

   651                 break;

   652             }

   653         }

   654 

   655         if (ch == 'L' && (_yychar == '"' || _yychar == '\'')) {

   656             // wide char/string literals

   657             ch = _yychar;

   658             yyinp();

   659 

   660             const char quote = ch;

   661 

   662             tok->f.kind = quote == '"'

   663                 ? T_WIDE_STRING_LITERAL

   664                 : T_WIDE_CHAR_LITERAL;

   665 

   666             const char *yytext = _currentChar;

   667 

   668             while (_yychar && _yychar != quote) {

   669                 if (_yychar != '\\')

   670                     yyinp();

   671                 else {

   672                     yyinp(); // skip `\\'

   673 

   674                     if (_yychar)

   675                         yyinp();

   676                 }

   677             }

   678             // assert(_yychar == quote);

   679 

   680             int yylen = _currentChar - yytext;

   681 

   682             if (_yychar == quote)

   683                 yyinp();

   684 

   685             if (control())

   686                 tok->string = control()->findOrInsertStringLiteral(yytext, yylen);

   687         } else if (std::isalpha(ch) || ch == '_' || ch == '$') {

   688             const char *yytext = _currentChar - 1;

   689             while (std::isalnum(_yychar) || _yychar == '_' || _yychar == '$')

   690                 yyinp();

   691             int yylen = _currentChar - yytext;

   692             if (f._scanKeywords)

   693                 tok->f.kind = classify(yytext, yylen, f._qtMocRunEnabled);

   694             else

   695                 tok->f.kind = T_IDENTIFIER;

   696 

   697             if (tok->f.kind == T_IDENTIFIER) {

   698                 tok->f.kind = classifyOperator(yytext, yylen);

   699 

   700                 if (control())

   701                     tok->identifier = control()->findOrInsertIdentifier(yytext, yylen);

   702             }

   703             break;

   704         } else if (std::isdigit(ch)) {

   705             const char *yytext = _currentChar - 1;

   706             while (_yychar) {

   707                 if (_yychar == 'e' || _yychar == 'E') {

   708                     yyinp();

   709                     if (_yychar == '-' || _yychar == '+') {

   710                         yyinp();

   711                         // ### assert(std::isdigit(_yychar));

   712                     }

   713                 } else if (std::isalnum(_yychar) || _yychar == '.') {

   714                     yyinp();

   715                 } else {

   716                     break;

   717                 }

   718             }

   719             int yylen = _currentChar - yytext;

   720             tok->f.kind = T_NUMERIC_LITERAL;

   721             if (control())

   722                 tok->number = control()->findOrInsertNumericLiteral(yytext, yylen);

   723             break;

   724         } else {

   725             tok->f.kind = T_ERROR;

   726             break;

   727         }

   728     } // default

   729 

   730     } // switch

   731 }

   732 

   733