1 /****************************************************************************

     2 **

     3 ** Copyright (C) 2001-2004 Roberto Raggi

     4 ** Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies).

     5 ** All rights reserved.

     6 ** Contact: Nokia Corporation (qt-info@nokia.com)

     7 **

     8 ** This file is part of the qt3to4 porting application of the Qt Toolkit.

     9 **

    10 ** $QT_BEGIN_LICENSE:LGPL$

    11 ** No Commercial Usage

    12 ** This file contains pre-release code and may not be distributed.

    13 ** You may use this file in accordance with the terms and conditions

    14 ** contained in the Technology Preview License Agreement accompanying

    15 ** this package.

    16 **

    17 ** GNU Lesser General Public License Usage

    18 ** Alternatively, this file may be used under the terms of the GNU Lesser

    19 ** General Public License version 2.1 as published by the Free Software

    20 ** Foundation and appearing in the file LICENSE.LGPL included in the

    21 ** packaging of this file.  Please review the following information to

    22 ** ensure the GNU Lesser General Public License version 2.1 requirements

    23 ** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.

    24 **

    25 ** In addition, as a special exception, Nokia gives you certain additional

    26 ** rights.  These rights are described in the Nokia Qt LGPL Exception

    27 ** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.

    28 **

    29 ** If you have questions regarding the use of this file, please contact

    30 ** Nokia at qt-info@nokia.com.

    31 **

    32 **

    33 **

    34 **

    35 **

    36 **

    37 **

    38 **

    39 ** $QT_END_LICENSE$

    40 **

    41 ****************************************************************************/

    42 

    43 #ifndef TOKENIZER_H

    44 #define TOKENIZER_H

    45 

    46 #include "tokenengine.h"

    47 #include <QVector>

    48 #include <QByteArray>

    49 

    50 QT_BEGIN_NAMESPACE

    51 

    52 class Tokenizer

    53 {

    54 public:

    55     Tokenizer();

    56     ~Tokenizer();

    57     typedef void (Tokenizer::*scan_fun_ptr)(int *kind);

    58     QVector<TokenEngine::Token> tokenize(QByteArray text);

    59 private:

    60     bool nextToken(TokenEngine::Token &tok);

    61 

    62     void scanChar(int *kind);

    63     void scanUnicodeChar(int *kind);

    64     void scanNewline(int *kind);

    65     void scanWhiteSpaces(int *kind);

    66     void scanCharLiteral(int *kind);

    67     void scanStringLiteral(int *kind);

    68     void scanNumberLiteral(int *kind);

    69     void scanIdentifier(int *kind);

    70     void scanComment(int *kind);

    71     void scanPreprocessor(int *kind);

    72     void scanOperator(int *kind);

    73 

    74     void setupScanTable();

    75 private:

    76     static bool s_initialized;

    77     static scan_fun_ptr s_scan_table[];

    78     static int s_attr_table[];

    79 

    80     const char *m_buffer;

    81     int m_ptr;

    82 

    83     QVector<TokenEngine::Token> m_tokens;

    84 };

    85 

    86 QT_END_NAMESPACE

    87 

    88 #endif