/****************************************************************************

**

** Copyright (C) 2010 Nokia Corporation and/or its subsidiary(-ies).

** All rights reserved.

** Contact: Nokia Corporation (qt-info@nokia.com)

**

** This file is part of the QtDeclarative module of the Qt Toolkit.

**

** $QT_BEGIN_LICENSE:LGPL$

** No Commercial Usage

** This file contains pre-release code and may not be distributed.

** You may use this file in accordance with the terms and conditions

** contained in the Technology Preview License Agreement accompanying

** this package.

**

** GNU Lesser General Public License Usage

** Alternatively, this file may be used under the terms of the GNU Lesser

** General Public License version 2.1 as published by the Free Software

** Foundation and appearing in the file LICENSE.LGPL included in the

** packaging of this file.  Please review the following information to

** ensure the GNU Lesser General Public License version 2.1 requirements

** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.

**

** In addition, as a special exception, Nokia gives you certain additional

** rights.  These rights are described in the Nokia Qt LGPL Exception

** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.

**

** If you have questions regarding the use of this file, please contact

** Nokia at qt-info@nokia.com.

**

**

**

**

**

**

**

**

** $QT_END_LICENSE$

**

****************************************************************************/

#ifdef HAVE_CONFIG_H

#include <config.h>

#endif

#include "private/qdeclarativejslexer_p.h"

#include "private/qdeclarativejsglobal_p.h"

#include "private/qdeclarativejsengine_p.h"

#include "private/qdeclarativejsgrammar_p.h"

#include <QtCore/qcoreapplication.h>

#include <ctype.h>

#include <stdlib.h>

#include <stdio.h>

#include <string.h>

QT_BEGIN_NAMESPACE

Q_CORE_EXPORT double qstrtod(const char *s00, char const **se, bool *ok);

QT_END_NAMESPACE

QT_QML_BEGIN_NAMESPACE

#define shiftWindowsLineBreak() \

    do { \

        if (((current == '\r') && (next1 == '\n')) \

            || ((current == '\n') && (next1 == '\r'))) { \

            shift(1); \

        } \

    } \

    while (0)

namespace QDeclarativeJS {

extern double integerFromString(const char *buf, int size, int radix);

}

using namespace QDeclarativeJS;

Lexer::Lexer(Engine *eng, bool tokenizeComments)

    : driver(eng),

      yylineno(0),

      done(false),

      size8(128), size16(128),

      pos8(0), pos16(0),

      terminator(false),

      restrKeyword(false),

      delimited(false),

      stackToken(-1),

      state(Start),

      pos(0),

      code(0), length(0),

      yycolumn(0),

      startpos(0),

      startlineno(0), startcolumn(0),

      bol(true),

      current(0), next1(0), next2(0), next3(0),

      err(NoError),

      wantRx(false),

      check_reserved(true),

      parenthesesState(IgnoreParentheses),

      parenthesesCount(0),

      prohibitAutomaticSemicolon(false),

      tokenizeComments(tokenizeComments)

{

    if (driver) driver->setLexer(this);

    // allocate space for read buffers

    buffer8 = new char[size8];

    buffer16 = new QChar[size16];

    pattern = 0;

    flags = 0;

}

Lexer::~Lexer()

{

    delete [] buffer8;

    delete [] buffer16;

}

void Lexer::setCode(const QString &c, int lineno)

{

    yylineno = lineno;

    yycolumn = 1;

    restrKeyword = false;

    delimited = false;

    stackToken = -1;

    pos = 0;

    code = c.unicode();

    length = c.length();

    bol = true;

    // read first characters

    current = (length > 0) ? code[0].unicode() : 0;

    next1 = (length > 1) ? code[1].unicode() : 0;

    next2 = (length > 2) ? code[2].unicode() : 0;

    next3 = (length > 3) ? code[3].unicode() : 0;

}

void Lexer::shift(uint p)

{

    while (p--) {

        ++pos;

        ++yycolumn;

        current = next1;

        next1 = next2;

        next2 = next3;

        next3 = (pos + 3 < length) ? code[pos+3].unicode() : 0;

    }

}

void Lexer::setDone(State s)

{

    state = s;

    done = true;

}

int Lexer::findReservedWord(const QChar *c, int size) const

{

    switch (size) {

    case 2: {

        if (c[0] == QLatin1Char('d') && c[1] == QLatin1Char('o'))

            return QDeclarativeJSGrammar::T_DO;

        else if (c[0] == QLatin1Char('i') && c[1] == QLatin1Char('f'))

            return QDeclarativeJSGrammar::T_IF;

        else if (c[0] == QLatin1Char('i') && c[1] == QLatin1Char('n'))

            return QDeclarativeJSGrammar::T_IN;

        else if (c[0] == QLatin1Char('a') && c[1] == QLatin1Char('s'))

            return QDeclarativeJSGrammar::T_AS;

        else if (c[0] == QLatin1Char('o') && c[1] == QLatin1Char('n'))

            return QDeclarativeJSGrammar::T_ON;

    }   break;

    case 3: {

        if (c[0] == QLatin1Char('f') && c[1] == QLatin1Char('o') && c[2] == QLatin1Char('r'))

            return QDeclarativeJSGrammar::T_FOR;

        else if (c[0] == QLatin1Char('n') && c[1] == QLatin1Char('e') && c[2] == QLatin1Char('w'))

            return QDeclarativeJSGrammar::T_NEW;

        else if (c[0] == QLatin1Char('t') && c[1] == QLatin1Char('r') && c[2] == QLatin1Char('y'))

            return QDeclarativeJSGrammar::T_TRY;

        else if (c[0] == QLatin1Char('v') && c[1] == QLatin1Char('a') && c[2] == QLatin1Char('r'))

            return QDeclarativeJSGrammar::T_VAR;

        else if (check_reserved) {

            if (c[0] == QLatin1Char('i') && c[1] == QLatin1Char('n') && c[2] == QLatin1Char('t'))

                return QDeclarativeJSGrammar::T_RESERVED_WORD;

        }

    }   break;

    case 4: {

        if (c[0] == QLatin1Char('c') && c[1] == QLatin1Char('a')

                && c[2] == QLatin1Char('s') && c[3] == QLatin1Char('e'))

            return QDeclarativeJSGrammar::T_CASE;

        else if (c[0] == QLatin1Char('e') && c[1] == QLatin1Char('l')

                && c[2] == QLatin1Char('s') && c[3] == QLatin1Char('e'))

            return QDeclarativeJSGrammar::T_ELSE;

        else if (c[0] == QLatin1Char('t') && c[1] == QLatin1Char('h')

                && c[2] == QLatin1Char('i') && c[3] == QLatin1Char('s'))

            return QDeclarativeJSGrammar::T_THIS;

        else if (c[0] == QLatin1Char('v') && c[1] == QLatin1Char('o')

                && c[2] == QLatin1Char('i') && c[3] == QLatin1Char('d'))

            return QDeclarativeJSGrammar::T_VOID;

        else if (c[0] == QLatin1Char('w') && c[1] == QLatin1Char('i')

                && c[2] == QLatin1Char('t') && c[3] == QLatin1Char('h'))

            return QDeclarativeJSGrammar::T_WITH;

        else if (c[0] == QLatin1Char('t') && c[1] == QLatin1Char('r')

                && c[2] == QLatin1Char('u') && c[3] == QLatin1Char('e'))

            return QDeclarativeJSGrammar::T_TRUE;

        else if (c[0] == QLatin1Char('n') && c[1] == QLatin1Char('u')

                && c[2] == QLatin1Char('l') && c[3] == QLatin1Char('l'))

            return QDeclarativeJSGrammar::T_NULL;

        else if (check_reserved) {

            if (c[0] == QLatin1Char('e') && c[1] == QLatin1Char('n')

                    && c[2] == QLatin1Char('u') && c[3] == QLatin1Char('m'))

                return QDeclarativeJSGrammar::T_RESERVED_WORD;

            else if (c[0] == QLatin1Char('b') && c[1] == QLatin1Char('y')

                    && c[2] == QLatin1Char('t') && c[3] == QLatin1Char('e'))

                return QDeclarativeJSGrammar::T_RESERVED_WORD;

            else if (c[0] == QLatin1Char('l') && c[1] == QLatin1Char('o')

                    && c[2] == QLatin1Char('n') && c[3] == QLatin1Char('g'))

                return QDeclarativeJSGrammar::T_RESERVED_WORD;

            else if (c[0] == QLatin1Char('c') && c[1] == QLatin1Char('h')

                    && c[2] == QLatin1Char('a') && c[3] == QLatin1Char('r'))

                return QDeclarativeJSGrammar::T_RESERVED_WORD;

            else if (c[0] == QLatin1Char('g') && c[1] == QLatin1Char('o')

                    && c[2] == QLatin1Char('t') && c[3] == QLatin1Char('o'))

                return QDeclarativeJSGrammar::T_RESERVED_WORD;

        }

    }   break;

    case 5: {

        if (c[0] == QLatin1Char('b') && c[1] == QLatin1Char('r')

                && c[2] == QLatin1Char('e') && c[3] == QLatin1Char('a')

                && c[4] == QLatin1Char('k'))

            return QDeclarativeJSGrammar::T_BREAK;

        else if (c[0] == QLatin1Char('c') && c[1] == QLatin1Char('a')

                && c[2] == QLatin1Char('t') && c[3] == QLatin1Char('c')

                && c[4] == QLatin1Char('h'))

            return QDeclarativeJSGrammar::T_CATCH;

        else if (c[0] == QLatin1Char('t') && c[1] == QLatin1Char('h')

                && c[2] == QLatin1Char('r') && c[3] == QLatin1Char('o')

                && c[4] == QLatin1Char('w'))

            return QDeclarativeJSGrammar::T_THROW;

        else if (c[0] == QLatin1Char('w') && c[1] == QLatin1Char('h')

                && c[2] == QLatin1Char('i') && c[3] == QLatin1Char('l')

                && c[4] == QLatin1Char('e'))

            return QDeclarativeJSGrammar::T_WHILE;

        else if (c[0] == QLatin1Char('c') && c[1] == QLatin1Char('o')

                && c[2] == QLatin1Char('n') && c[3] == QLatin1Char('s')

                && c[4] == QLatin1Char('t'))

            return QDeclarativeJSGrammar::T_CONST;

        else if (c[0] == QLatin1Char('f') && c[1] == QLatin1Char('a')

                && c[2] == QLatin1Char('l') && c[3] == QLatin1Char('s')

                && c[4] == QLatin1Char('e'))

            return QDeclarativeJSGrammar::T_FALSE;

        else if (check_reserved) {

            if (c[0] == QLatin1Char('s') && c[1] == QLatin1Char('h')

                    && c[2] == QLatin1Char('o') && c[3] == QLatin1Char('r')

                    && c[4] == QLatin1Char('t'))

                return QDeclarativeJSGrammar::T_RESERVED_WORD;

            else if (c[0] == QLatin1Char('s') && c[1] == QLatin1Char('u')

                    && c[2] == QLatin1Char('p') && c[3] == QLatin1Char('e')

                    && c[4] == QLatin1Char('r'))

                return QDeclarativeJSGrammar::T_RESERVED_WORD;

            else if (c[0] == QLatin1Char('f') && c[1] == QLatin1Char('i')

                    && c[2] == QLatin1Char('n') && c[3] == QLatin1Char('a')

                    && c[4] == QLatin1Char('l'))

                return QDeclarativeJSGrammar::T_RESERVED_WORD;

            else if (c[0] == QLatin1Char('c') && c[1] == QLatin1Char('l')

                    && c[2] == QLatin1Char('a') && c[3] == QLatin1Char('s')

                    && c[4] == QLatin1Char('s'))

                return QDeclarativeJSGrammar::T_RESERVED_WORD;

            else if (c[0] == QLatin1Char('f') && c[1] == QLatin1Char('l')

                    && c[2] == QLatin1Char('o') && c[3] == QLatin1Char('a')

                    && c[4] == QLatin1Char('t'))

                return QDeclarativeJSGrammar::T_RESERVED_WORD;

        }

    }   break;

    case 6: {

        if (c[0] == QLatin1Char('d') && c[1] == QLatin1Char('e')

                && c[2] == QLatin1Char('l') && c[3] == QLatin1Char('e')

                && c[4] == QLatin1Char('t') && c[5] == QLatin1Char('e'))

            return QDeclarativeJSGrammar::T_DELETE;

        else if (c[0] == QLatin1Char('r') && c[1] == QLatin1Char('e')

                && c[2] == QLatin1Char('t') && c[3] == QLatin1Char('u')

                && c[4] == QLatin1Char('r') && c[5] == QLatin1Char('n'))

            return QDeclarativeJSGrammar::T_RETURN;

        else if (c[0] == QLatin1Char('s') && c[1] == QLatin1Char('w')

                && c[2] == QLatin1Char('i') && c[3] == QLatin1Char('t')

                && c[4] == QLatin1Char('c') && c[5] == QLatin1Char('h'))

            return QDeclarativeJSGrammar::T_SWITCH;

        else if (c[0] == QLatin1Char('t') && c[1] == QLatin1Char('y')

                && c[2] == QLatin1Char('p') && c[3] == QLatin1Char('e')

                && c[4] == QLatin1Char('o') && c[5] == QLatin1Char('f'))

            return QDeclarativeJSGrammar::T_TYPEOF;

        else if (c[0] == QLatin1Char('i') && c[1] == QLatin1Char('m')

            && c[2] == QLatin1Char('p') && c[3] == QLatin1Char('o')

            && c[4] == QLatin1Char('r') && c[5] == QLatin1Char('t'))

            return QDeclarativeJSGrammar::T_IMPORT;

        else if (c[0] == QLatin1Char('s') && c[1] == QLatin1Char('i')

            && c[2] == QLatin1Char('g') && c[3] == QLatin1Char('n')

            && c[4] == QLatin1Char('a') && c[5] == QLatin1Char('l'))

            return QDeclarativeJSGrammar::T_SIGNAL;

        else if (check_reserved) {

            if (c[0] == QLatin1Char('e') && c[1] == QLatin1Char('x')

                    && c[2] == QLatin1Char('p') && c[3] == QLatin1Char('o')

                    && c[4] == QLatin1Char('r') && c[5] == QLatin1Char('t'))

                return QDeclarativeJSGrammar::T_RESERVED_WORD;

            else if (c[0] == QLatin1Char('s') && c[1] == QLatin1Char('t')

                    && c[2] == QLatin1Char('a') && c[3] == QLatin1Char('t')

                    && c[4] == QLatin1Char('i') && c[5] == QLatin1Char('c'))

                return QDeclarativeJSGrammar::T_RESERVED_WORD;

            else if (c[0] == QLatin1Char('d') && c[1] == QLatin1Char('o')

                    && c[2] == QLatin1Char('u') && c[3] == QLatin1Char('b')

                    && c[4] == QLatin1Char('l') && c[5] == QLatin1Char('e'))

                return QDeclarativeJSGrammar::T_RESERVED_WORD;

            else if (c[0] == QLatin1Char('i') && c[1] == QLatin1Char('m')

                    && c[2] == QLatin1Char('p') && c[3] == QLatin1Char('o')

                    && c[4] == QLatin1Char('r') && c[5] == QLatin1Char('t'))

                return QDeclarativeJSGrammar::T_RESERVED_WORD;

            else if (c[0] == QLatin1Char('p') && c[1] == QLatin1Char('u')

                    && c[2] == QLatin1Char('b') && c[3] == QLatin1Char('l')

                    && c[4] == QLatin1Char('i') && c[5] == QLatin1Char('c'))

                return QDeclarativeJSGrammar::T_PUBLIC;

            else if (c[0] == QLatin1Char('n') && c[1] == QLatin1Char('a')

                    && c[2] == QLatin1Char('t') && c[3] == QLatin1Char('i')

                    && c[4] == QLatin1Char('v') && c[5] == QLatin1Char('e'))

                return QDeclarativeJSGrammar::T_RESERVED_WORD;

            else if (c[0] == QLatin1Char('t') && c[1] == QLatin1Char('h')

                    && c[2] == QLatin1Char('r') && c[3] == QLatin1Char('o')

                    && c[4] == QLatin1Char('w') && c[5] == QLatin1Char('s'))

                return QDeclarativeJSGrammar::T_RESERVED_WORD;

        }

    }   break;

    case 7: {

        if (c[0] == QLatin1Char('d') && c[1] == QLatin1Char('e')

                && c[2] == QLatin1Char('f') && c[3] == QLatin1Char('a')

                && c[4] == QLatin1Char('u') && c[5] == QLatin1Char('l')

                && c[6] == QLatin1Char('t'))

            return QDeclarativeJSGrammar::T_DEFAULT;

        else if (c[0] == QLatin1Char('f') && c[1] == QLatin1Char('i')

                && c[2] == QLatin1Char('n') && c[3] == QLatin1Char('a')

                && c[4] == QLatin1Char('l') && c[5] == QLatin1Char('l')

                && c[6] == QLatin1Char('y'))

            return QDeclarativeJSGrammar::T_FINALLY;

        else if (check_reserved) {

            if (c[0] == QLatin1Char('b') && c[1] == QLatin1Char('o')

                    && c[2] == QLatin1Char('o') && c[3] == QLatin1Char('l')

                    && c[4] == QLatin1Char('e') && c[5] == QLatin1Char('a')

                    && c[6] == QLatin1Char('n'))

                return QDeclarativeJSGrammar::T_RESERVED_WORD;

            else if (c[0] == QLatin1Char('e') && c[1] == QLatin1Char('x')

                    && c[2] == QLatin1Char('t') && c[3] == QLatin1Char('e')

                    && c[4] == QLatin1Char('n') && c[5] == QLatin1Char('d')

                    && c[6] == QLatin1Char('s'))

                return QDeclarativeJSGrammar::T_RESERVED_WORD;

            else if (c[0] == QLatin1Char('p') && c[1] == QLatin1Char('a')

                    && c[2] == QLatin1Char('c') && c[3] == QLatin1Char('k')

                    && c[4] == QLatin1Char('a') && c[5] == QLatin1Char('g')

                    && c[6] == QLatin1Char('e'))

                return QDeclarativeJSGrammar::T_RESERVED_WORD;

            else if (c[0] == QLatin1Char('p') && c[1] == QLatin1Char('r')

                    && c[2] == QLatin1Char('i') && c[3] == QLatin1Char('v')

                    && c[4] == QLatin1Char('a') && c[5] == QLatin1Char('t')

                    && c[6] == QLatin1Char('e'))

                return QDeclarativeJSGrammar::T_RESERVED_WORD;

        }

    }   break;

    case 8: {

        if (c[0] == QLatin1Char('c') && c[1] == QLatin1Char('o')

                && c[2] == QLatin1Char('n') && c[3] == QLatin1Char('t')

                && c[4] == QLatin1Char('i') && c[5] == QLatin1Char('n')

                && c[6] == QLatin1Char('u') && c[7] == QLatin1Char('e'))

            return QDeclarativeJSGrammar::T_CONTINUE;

        else if (c[0] == QLatin1Char('f') && c[1] == QLatin1Char('u')

                && c[2] == QLatin1Char('n') && c[3] == QLatin1Char('c')

                && c[4] == QLatin1Char('t') && c[5] == QLatin1Char('i')

                && c[6] == QLatin1Char('o') && c[7] == QLatin1Char('n'))

            return QDeclarativeJSGrammar::T_FUNCTION;

        else if (c[0] == QLatin1Char('d') && c[1] == QLatin1Char('e')

                && c[2] == QLatin1Char('b') && c[3] == QLatin1Char('u')

                && c[4] == QLatin1Char('g') && c[5] == QLatin1Char('g')

                && c[6] == QLatin1Char('e') && c[7] == QLatin1Char('r'))

            return QDeclarativeJSGrammar::T_DEBUGGER;

        else if (c[0] == QLatin1Char('p') && c[1] == QLatin1Char('r')

                && c[2] == QLatin1Char('o') && c[3] == QLatin1Char('p')

                && c[4] == QLatin1Char('e') && c[5] == QLatin1Char('r')

                && c[6] == QLatin1Char('t') && c[7] == QLatin1Char('y'))

            return QDeclarativeJSGrammar::T_PROPERTY;

        else if (c[0] == QLatin1Char('r') && c[1] == QLatin1Char('e')

                && c[2] == QLatin1Char('a') && c[3] == QLatin1Char('d')

                && c[4] == QLatin1Char('o') && c[5] == QLatin1Char('n')

                && c[6] == QLatin1Char('l') && c[7] == QLatin1Char('y'))

            return QDeclarativeJSGrammar::T_READONLY;

        else if (check_reserved) {

            if (c[0] == QLatin1Char('a') && c[1] == QLatin1Char('b')

                    && c[2] == QLatin1Char('s') && c[3] == QLatin1Char('t')

                    && c[4] == QLatin1Char('r') && c[5] == QLatin1Char('a')

                    && c[6] == QLatin1Char('c') && c[7] == QLatin1Char('t'))

                return QDeclarativeJSGrammar::T_RESERVED_WORD;

            else if (c[0] == QLatin1Char('v') && c[1] == QLatin1Char('o')

                    && c[2] == QLatin1Char('l') && c[3] == QLatin1Char('a')

                    && c[4] == QLatin1Char('t') && c[5] == QLatin1Char('i')

                    && c[6] == QLatin1Char('l') && c[7] == QLatin1Char('e'))

                return QDeclarativeJSGrammar::T_RESERVED_WORD;

        }

    }   break;

    case 9: {

        if (check_reserved) {

            if (c[0] == QLatin1Char('i') && c[1] == QLatin1Char('n')

                    && c[2] == QLatin1Char('t') && c[3] == QLatin1Char('e')

                    && c[4] == QLatin1Char('r') && c[5] == QLatin1Char('f')

                    && c[6] == QLatin1Char('a') && c[7] == QLatin1Char('c')

                    && c[8] == QLatin1Char('e'))

                return QDeclarativeJSGrammar::T_RESERVED_WORD;

            else if (c[0] == QLatin1Char('t') && c[1] == QLatin1Char('r')

                    && c[2] == QLatin1Char('a') && c[3] == QLatin1Char('n')

                    && c[4] == QLatin1Char('s') && c[5] == QLatin1Char('i')

                    && c[6] == QLatin1Char('e') && c[7] == QLatin1Char('n')

                    && c[8] == QLatin1Char('t'))

                return QDeclarativeJSGrammar::T_RESERVED_WORD;

            else if (c[0] == QLatin1Char('p') && c[1] == QLatin1Char('r')

                    && c[2] == QLatin1Char('o') && c[3] == QLatin1Char('t')

                    && c[4] == QLatin1Char('e') && c[5] == QLatin1Char('c')

                    && c[6] == QLatin1Char('t') && c[7] == QLatin1Char('e')

                    && c[8] == QLatin1Char('d'))

                return QDeclarativeJSGrammar::T_RESERVED_WORD;

        }

    }   break;

    case 10: {

        if (c[0] == QLatin1Char('i') && c[1] == QLatin1Char('n')

                && c[2] == QLatin1Char('s') && c[3] == QLatin1Char('t')

                && c[4] == QLatin1Char('a') && c[5] == QLatin1Char('n')

                && c[6] == QLatin1Char('c') && c[7] == QLatin1Char('e')

                && c[8] == QLatin1Char('o') && c[9] == QLatin1Char('f'))

            return QDeclarativeJSGrammar::T_INSTANCEOF;

        else if (check_reserved) {

            if (c[0] == QLatin1Char('i') && c[1] == QLatin1Char('m')

                    && c[2] == QLatin1Char('p') && c[3] == QLatin1Char('l')

                    && c[4] == QLatin1Char('e') && c[5] == QLatin1Char('m')

                    && c[6] == QLatin1Char('e') && c[7] == QLatin1Char('n')

                    && c[8] == QLatin1Char('t') && c[9] == QLatin1Char('s'))

                return QDeclarativeJSGrammar::T_RESERVED_WORD;

        }

    }   break;

    case 12: {

        if (check_reserved) {

            if (c[0] == QLatin1Char('s') && c[1] == QLatin1Char('y')

                    && c[2] == QLatin1Char('n') && c[3] == QLatin1Char('c')

                    && c[4] == QLatin1Char('h') && c[5] == QLatin1Char('r')

                    && c[6] == QLatin1Char('o') && c[7] == QLatin1Char('n')

                    && c[8] == QLatin1Char('i') && c[9] == QLatin1Char('z')

                    && c[10] == QLatin1Char('e') && c[11] == QLatin1Char('d'))

                return QDeclarativeJSGrammar::T_RESERVED_WORD;

        }

    }   break;

    } // switch

    return -1;

}

int Lexer::lex()

{

    int token = 0;

    state = Start;

    ushort stringType = 0; // either single or double quotes

    bool multiLineString = false;

    pos8 = pos16 = 0;

    done = false;

    terminator = false;

    // did we push a token on the stack previously ?

    // (after an automatic semicolon insertion)

    if (stackToken >= 0) {

        setDone(Other);

        token = stackToken;

        stackToken = -1;

    }

    bool identifierWithEscapedUnicode = false;

    while (!done) {

        switch (state) {

        case Start:

            if (isWhiteSpace()) {

                // do nothing

            } else if (current == '/' && next1 == '/') {

                recordStartPos();

                shift(1);

                state = InSingleLineComment;

            } else if (current == '/' && next1 == '*') {

                recordStartPos();

                shift(1);

                state = InMultiLineComment;

            } else if (current == 0) {

                syncProhibitAutomaticSemicolon();

                if (!terminator && !delimited && !prohibitAutomaticSemicolon) {

                    // automatic semicolon insertion if program incomplete

                    token = QDeclarativeJSGrammar::T_SEMICOLON;

                    stackToken = 0;

                    setDone(Other);

                } else {

                    setDone(Eof);

                }

            } else if (isLineTerminator()) {

                shiftWindowsLineBreak();

                yylineno++;

                yycolumn = 0;

                bol = true;

                terminator = true;

                syncProhibitAutomaticSemicolon();

                if (restrKeyword) {

                    token = QDeclarativeJSGrammar::T_SEMICOLON;

                    setDone(Other);

                }

            } else if (current == '"' || current == '\'') {

                recordStartPos();

                state = InString;

                multiLineString = false;

                stringType = current;

            } else if (current == '\\' && next1 == 'u') {