1 /****************************************************************************

     2 **

     3 ** Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies).

     4 ** All rights reserved.

     5 ** Contact: Nokia Corporation (qt-info@nokia.com)

     6 **

     7 ** This file is part of the tools applications of the Qt Toolkit.

     8 **

     9 ** $QT_BEGIN_LICENSE:LGPL$

    10 ** No Commercial Usage

    11 ** This file contains pre-release code and may not be distributed.

    12 ** You may use this file in accordance with the terms and conditions

    13 ** contained in the Technology Preview License Agreement accompanying

    14 ** this package.

    15 **

    16 ** GNU Lesser General Public License Usage

    17 ** Alternatively, this file may be used under the terms of the GNU Lesser

    18 ** General Public License version 2.1 as published by the Free Software

    19 ** Foundation and appearing in the file LICENSE.LGPL included in the

    20 ** packaging of this file.  Please review the following information to

    21 ** ensure the GNU Lesser General Public License version 2.1 requirements

    22 ** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.

    23 **

    24 ** In addition, as a special exception, Nokia gives you certain additional

    25 ** rights.  These rights are described in the Nokia Qt LGPL Exception

    26 ** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.

    27 **

    28 ** If you have questions regarding the use of this file, please contact

    29 ** Nokia at qt-info@nokia.com.

    30 **

    31 **

    32 **

    33 **

    34 **

    35 **

    36 **

    37 **

    38 ** $QT_END_LICENSE$

    39 **

    40 ****************************************************************************/

    41 

    42 /*

    43   tokenizer.h

    44 */

    45 

    46 #ifndef TOKENIZER_H

    47 #define TOKENIZER_H

    48 

    49 #include <qstack.h>

    50 #include <qstring.h>

    51 

    52 #include <stdio.h>

    53 

    54 #include "location.h"

    55 

    56 QT_BEGIN_NAMESPACE

    57 

    58 /*

    59   Here come the C++ tokens we support.  The first part contains

    60   all-purpose tokens; then come keywords.

    61   

    62   If you add a keyword, make sure to modify the keyword array in

    63   tokenizer.cpp as well, and possibly adjust Tok_FirstKeyword and

    64   Tok_LastKeyword.

    65 */

    66 enum { Tok_Eoi, Tok_Ampersand, Tok_Aster, Tok_Caret, Tok_LeftParen, 

    67        Tok_RightParen, Tok_LeftParenAster, Tok_Equal, Tok_LeftBrace, 

    68        Tok_RightBrace, Tok_Semicolon, Tok_Colon, Tok_LeftAngle,

    69        Tok_RightAngle, Tok_Comma, Tok_Ellipsis, Tok_Gulbrandsen,

    70        Tok_LeftBracket, Tok_RightBracket, Tok_Tilde, Tok_SomeOperator,

    71        Tok_Number, Tok_String, Tok_Doc, Tok_Comment, Tok_Ident, Tok_At,

    72        Tok_char, Tok_class, Tok_const, Tok_double, Tok_enum, 

    73        Tok_explicit, Tok_friend, Tok_inline, Tok_int, Tok_long, 

    74        Tok_namespace, Tok_operator, Tok_private, Tok_protected,

    75        Tok_public, Tok_short, Tok_signals, Tok_signed, Tok_slots,

    76        Tok_static, Tok_struct, Tok_template, Tok_typedef, 

    77        Tok_typename, Tok_union, Tok_unsigned, Tok_using, Tok_virtual,

    78        Tok_void, Tok_volatile, Tok_int64, Tok_Q_OBJECT, Tok_Q_OVERRIDE, 

    79        Tok_Q_PROPERTY, Tok_Q_DECLARE_SEQUENTIAL_ITERATOR,

    80        Tok_Q_DECLARE_MUTABLE_SEQUENTIAL_ITERATOR, 

    81        Tok_Q_DECLARE_ASSOCIATIVE_ITERATOR,

    82        Tok_Q_DECLARE_MUTABLE_ASSOCIATIVE_ITERATOR, 

    83        Tok_Q_DECLARE_FLAGS, Tok_Q_SIGNALS, Tok_Q_SLOTS, Tok_QT_COMPAT, 

    84        Tok_QT_COMPAT_CONSTRUCTOR, Tok_QT_DEPRECATED, Tok_QT_MOC_COMPAT,

    85        Tok_QT_MODULE, Tok_QT3_SUPPORT, Tok_QT3_SUPPORT_CONSTRUCTOR, 

    86        Tok_QT3_MOC_SUPPORT, Tok_QDOC_PROPERTY,

    87        Tok_FirstKeyword = Tok_char, Tok_LastKeyword = Tok_QDOC_PROPERTY };

    88 

    89 /*

    90   The Tokenizer class implements lexical analysis of C++ source

    91   files.

    92 

    93   Not every operator or keyword of C++ is recognized; only those

    94   that are interesting to us. Some Qt keywords or macros are also

    95   recognized.

    96 */

    97 

    98 class Tokenizer

    99 {

   100  public:

   101     Tokenizer(const Location& loc, const QByteArray &in);

   102     Tokenizer(const Location& loc, FILE *in);

   103 

   104     ~Tokenizer();

   105 

   106     int getToken();

   107     void setParsingFnOrMacro(bool macro) { parsingMacro = macro; }

   108     bool parsingFnOrMacro() const { return parsingMacro; }

   109 

   110     const Location &location() const { return yyTokLoc; }

   111     QString previousLexeme() const { return QString(yyPrevLex); }

   112     QString lexeme() const { return QString(yyLex); }

   113     QString version() const { return yyVersion; }

   114     int braceDepth() const { return yyBraceDepth; }

   115     int parenDepth() const { return yyParenDepth; }

   116     int bracketDepth() const { return yyBracketDepth; }

   117 

   118     static void initialize(const Config &config);

   119     static void terminate();

   120     static bool isTrue(const QString &condition);

   121 

   122  private:

   123     void init();

   124     void start(const Location& loc);

   125     /*

   126       This limit on the length of a lexeme seems fairly high, but a

   127       doc comment can be arbitrarily long. The previous 65,536 limit

   128       was reached by Mark Summerfield.

   129     */

   130     enum { yyLexBufSize = 524288 };

   131 

   132     int getch()

   133     {

   134         return yyPos == yyIn.size() ? EOF : yyIn[yyPos++];

   135     }

   136 

   137     inline int getChar()

   138     {

   139         if (yyCh == EOF)

   140             return EOF;

   141         if (yyLexLen < yyLexBufSize - 1) {

   142             yyLex[yyLexLen++] = (char) yyCh;

   143             yyLex[yyLexLen] = '\0';

   144         }

   145         yyCurLoc.advance(yyCh);

   146         int ch = getch();

   147         if (ch == EOF)

   148             return EOF;

   149         // cast explicitely to make sure the value of ch 

   150         // is in range [0..255] to avoid assert messages 

   151         // when using debug CRT that checks its input.

   152         return int(uint(uchar(ch))); 

   153     }

   154 

   155     int getTokenAfterPreprocessor();

   156     void pushSkipping(bool skip);

   157     bool popSkipping();

   158 

   159     Location yyTokLoc;

   160     Location yyCurLoc;

   161     char *yyLexBuf1;

   162     char *yyLexBuf2;

   163     char *yyPrevLex;

   164     char *yyLex;

   165     size_t yyLexLen;

   166     QStack<bool> yyPreprocessorSkipping;

   167     int yyNumPreprocessorSkipping;

   168     int yyBraceDepth;

   169     int yyParenDepth;

   170     int yyBracketDepth;

   171     int yyCh;

   172 

   173     QString yyVersion;

   174     bool parsingMacro;

   175 

   176  protected:

   177     QByteArray yyIn;

   178     int yyPos;

   179 };

   180 

   181 QT_END_NAMESPACE

   182 

   183 #endif