/******************************************************************************** Copyright (C) 2001-2004 Roberto Raggi** Copyright (C) 2010 Nokia Corporation and/or its subsidiary(-ies).** All rights reserved.** Contact: Nokia Corporation (qt-info@nokia.com)**** This file is part of the qt3to4 porting application of the Qt Toolkit.**** $QT_BEGIN_LICENSE:LGPL$** No Commercial Usage** This file contains pre-release code and may not be distributed.** You may use this file in accordance with the terms and conditions** contained in the Technology Preview License Agreement accompanying** this package.**** GNU Lesser General Public License Usage** Alternatively, this file may be used under the terms of the GNU Lesser** General Public License version 2.1 as published by the Free Software** Foundation and appearing in the file LICENSE.LGPL included in the** packaging of this file. Please review the following information to** ensure the GNU Lesser General Public License version 2.1 requirements** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.**** In addition, as a special exception, Nokia gives you certain additional** rights. These rights are described in the Nokia Qt LGPL Exception** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.**** If you have questions regarding the use of this file, please contact** Nokia at qt-info@nokia.com.****************** $QT_END_LICENSE$******************************************************************************/#ifndef TOKENIZER_H#define TOKENIZER_H#include "tokenengine.h"#include <QVector>#include <QByteArray>QT_BEGIN_NAMESPACEclass Tokenizer{public: Tokenizer(); ~Tokenizer(); typedef void (Tokenizer::*scan_fun_ptr)(int *kind); QVector<TokenEngine::Token> tokenize(QByteArray text);private: bool nextToken(TokenEngine::Token &tok); void scanChar(int *kind); void scanUnicodeChar(int *kind); void scanNewline(int *kind); void scanWhiteSpaces(int *kind); void scanCharLiteral(int *kind); void scanStringLiteral(int *kind); void scanNumberLiteral(int *kind); void scanIdentifier(int *kind); void scanComment(int *kind); void scanPreprocessor(int *kind); void scanOperator(int *kind); void setupScanTable();private: static bool s_initialized; static scan_fun_ptr s_scan_table[]; static int s_attr_table[]; const char *m_buffer; int m_ptr; QVector<TokenEngine::Token> m_tokens;};QT_END_NAMESPACE#endif