// Copyright (c) 2007-2009 Nokia Corporation and/or its subsidiary(-ies).

// All rights reserved.

// This component and the accompanying materials are made available

// under the terms of "Eclipse Public License v1.0"

// which accompanies this distribution, and is available

// at the URL "http://www.eclipse.org/legal/epl-v10.html".

//

// Initial Contributors:

// Nokia Corporation - initial contribution.

//

// Contributors:

//

// Description:

//

#include <ctype.h>

#include <memory.h>

#include <string.h>

#include "messagedefparser\definitiontokenizer.h"

namespace Tokens

{

struct TTokenTypeText 

    {

    const char* iText;

    TTokenType iType;

    };

static const TTokenTypeText keywords[] =

    {

        { "enum", EEnum },

        { "message", EMessage },

        { "struct", EStruct },

        { "signature", ESignature }, 

        { "context", EContext },

        { "end", EEnd },

        { "const", EConst },

        { "include", EInclude },

        { "alias", EAlias },

        { "decimal", EDisplayDec },

        { "hex", EDisplayHex },

        { "uint8", EIntType },

        { "uint16", EIntType },

        { "uint32", EIntType },

        { "int8", EIntType },

        { "int16", EIntType },

        { "int32", EIntType },

        { "tmessageid", EMessageIdType },

        { "pad", EPadType },

        { "typeid", ETypeId },

        { "messageid", EMessageId },

        { NULL, EUnknown }

    };

static const TTokenTypeText tokenTypes[] =

    {

        { "EColon", EColon },

        { "EEquals", EEquals },

        { "EInclude", EInclude },

        { "EAlias", EAlias },

        { "EConst", EConst },

        { "EEnd", EEnd },

        { "EEnum", EEnum },

        { "EStruct", EStruct },

        { "ESignature", ESignature },

        { "EContext", EContext },

        { "EMessage", EMessage },

        { "EIdentifier", EIdentifier },

        { "ENumberDec", ENumberDec },

        { "ENumberHex", ENumberHex },

        { "EIntType", EIntType },

        { "EPadType", EPadType },

        { "EString", EString },

        { "ETypeId", ETypeId },

        { "EMessageId", EMessageId },

        { "EMessageIdType", EMessageIdType },

        { "EUnknown", EUnknown }

    };

const char* TokenTypeToString(Tokens::TTokenType aType)

    {

    int i = 0;

    while (tokenTypes[i].iType != EUnknown)

        {

        if (aType == tokenTypes[i].iType)

            {

            break;

            }

        ++i;

        }

    return tokenTypes[i].iText;

    }

CDefinitionTokenizer::CDefinitionTokenizer()

    {

    this->iToken = new char[KMaxTokenSize];

    this->iInputBuffer = new char[KBufferSize];

    this->iTokenOffset = 0;

    this->iLastBufferOffset = 0;

    this->iBufferValid = false;

    this->iBufferOffset = 0;

    this->iLine = 1;

    }

CDefinitionTokenizer::~CDefinitionTokenizer()

    {

    iInputFile.close();

    delete iToken;

    delete iInputBuffer;

    }

Tokens::TResult CDefinitionTokenizer::LoadDefinitionFile(const std::string& aFilename)

    {

    iInputFile.open(aFilename.c_str(), std::ios::in);

    iLine = 1;

    if (iInputFile.is_open())

        {

        RefillBuffer();

        iError = ENoError;

        }

    else

        {

        iError = EFileNotFound;

        }

    return iError;

    }

Tokens::TResult CDefinitionTokenizer::GetNextToken()

    {

    iTokenOffset = 0;

    iLastBufferOffset = iBufferOffset;

    iTokenType = EUnknown;

//    TTokenState state = EStateStartToken;

    iState = EStateStartToken;

    iError = EUnexpectedToken;

    while (iBufferValid)

        {

        // start to collect the token

        while (iBufferOffset < iBufferSize)

            {

            if (iTokenOffset == KMaxTokenSize)

                {

                iError = ETokenTooBig;

                return ETokenTooBig;

                }

            iToken[iTokenOffset] = iInputBuffer[iBufferOffset];

            if (iToken[iTokenOffset] == '\r')

                {

                ++iBufferOffset;

                continue;

                }

            switch (iState)

                {

                case EStateStartToken:

                    iState = ProcessStateStartToken();

                    break;

                case EStateMaybeComment:

                    iState = ProcessStateMaybeComment();

                    break;

                case EStateMultiLineComment:

                case EStateMaybeEndMultiLineComment:

                    iState = ProcessStateMultiLineComment();

                    break;

                case EStateComment:

                    iState = ProcessStateComment();

                    break;

                case EStateNumber:

                    iState = ProcessStateNumber();

                    break;

                case EStateDecimalNumber:

                    iState = ProcessStateDecimalNumber();

                    break;

                case EStateMaybeHexNumber:

                    iState = ProcessStateMaybeHexNumber();

                    break;

                case EStateHexNumber:

                    iState = ProcessStateHexNumber();

                    break;

                case EStateIdentifier:

                    iState = ProcessStateIdentifier();

                    break;

                case EStateMaybeString:

                    iState = ProcessStateMaybeString();

                    break;

                case EStateMaybeNegativeNumber:

                    iState = ProcessStateMaybeNegativeNumber();

                    break;

                default:

                    iState = EStateError;

                    iError = EUnknownState;

                    break;

                }

            // New state processing

            switch (iState)

                {

                case EStateError:

                    iTokenType = EUnknown;

                    iToken[++iTokenOffset] = 0; // include the invalid character in the token

                    iBufferOffset++;

                    return iError;

                    break;

                case EStateStartToken:

                    iTokenOffset = 0; 

                    iLastBufferOffset = iBufferOffset;

                    iBufferOffset++;

                    break;

                case EStateComplete:

                    // TODO: clear error

                    iLastBufferOffset = iBufferOffset;

                    iToken[iTokenOffset] = 0;

                    if (iTokenType == EIdentifier)

                        {

                        ExamineIdentifierForKeyword();

                        }

                    iError = ETokenFound;

                    return ETokenFound;

                    //break;

                case EStateComment:

                case EStateMultiLineComment:

                case EStateMaybeEndMultiLineComment:

                    iTokenOffset = 0;

                    iLastBufferOffset = iBufferOffset;

                    iBufferOffset++;

                    break;

                default:

                    ++iBufferOffset;

                    ++iTokenOffset;

                    break;

                }

            }

        RefillBuffer();

        }

    iToken[iTokenOffset] = 0;

    if (iTokenOffset == 0)

        {

        iError = EEndOfFile;

        }

    else

        {

        if (iTokenType == EIdentifier)

            {

            ExamineIdentifierForKeyword();

            }

        iError = ETokenFound;

        }

    return iError;

    }

void CDefinitionTokenizer::ExamineIdentifierForKeyword()

    {

    int i = 0;

    while (keywords[i].iText != NULL)

        {

        if (!_strcmpi(keywords[i].iText, iToken))

            {

            iTokenType = keywords[i].iType;

            break;

            }

        ++i;

        }

    }

void CDefinitionTokenizer::RefillBuffer()

    {

    int bufferSpace = KBufferSize;

    int bufferInUse = 0;

    if (iLastBufferOffset)

        {

        bufferInUse = iBufferSize - iLastBufferOffset;

        bufferSpace = KBufferSize - bufferInUse;

        memcpy(iInputBuffer, &iInputBuffer[iLastBufferOffset], bufferInUse);

        }

    memset(&iInputBuffer[bufferInUse], 0, bufferSpace);

    iInputFile.read(&iInputBuffer[bufferInUse], bufferSpace);

    iBufferSize = iInputFile.gcount();

    iBufferValid = (iBufferSize > 0);

    iBufferSize += bufferInUse;

    iBufferOffset = bufferInUse;

    iLastBufferOffset = 0;

    }

CDefinitionTokenizer::TTokenState CDefinitionTokenizer::ProcessStateStartToken()

    {

    TTokenState nextState = EStateError;

    iTokenType = EUnknown;

    switch (iToken[0])

        {

        // single char tokens

        case '=':

            iToken[++iTokenOffset] = 0;

            ++iBufferOffset;

            nextState = EStateComplete;

            iTokenType = EEquals;

            break;

        case ':':

            iToken[++iTokenOffset] = 0;

            ++iBufferOffset;

            nextState = EStateComplete;

            iTokenType = EColon;

            break;

        case ' ':

        case '\t':

            // consume leading whitespace

            nextState = EStateStartToken;

            break;

        case '\n':

            ++iLine;

            nextState = EStateStartToken;

            break;

        case '\"':

            nextState = EStateMaybeString;

            --iTokenOffset; // don't include the quotes in the token

            break;

        case '/':

            nextState = EStateMaybeComment; 

            break;

        case '-':

            nextState = EStateMaybeNegativeNumber;

            break;

        case '0':

            nextState = EStateNumber;

            iTokenType = ENumberDec;

            break;

        default:

            if (iToken[iTokenOffset] >= '1' && iToken[iTokenOffset] <= '9')

                {

                nextState = EStateDecimalNumber;

                iTokenType = ENumberDec;

                }

            else if (tolower(iToken[iTokenOffset]) >= 'a'&& tolower(iToken[iTokenOffset]) <= 'z')

                {

                nextState = EStateIdentifier;

                iTokenType = EIdentifier;

                }

            break;

        }

    return nextState;

    }

CDefinitionTokenizer::TTokenState CDefinitionTokenizer::ProcessStateMaybeNegativeNumber()

    {

    TTokenState nextState = EStateError;

    switch (iToken[iTokenOffset])

        {

        case '0':

            nextState = EStateNumber;

            iTokenType = ENumberDec;

            break;

        default:

            if (iToken[iTokenOffset] >= '1' && iToken[iTokenOffset] <= '9')

                {

                nextState = EStateDecimalNumber;

                iTokenType = ENumberDec;

                }

            break;

        }

    return nextState;

    }

CDefinitionTokenizer::TTokenState CDefinitionTokenizer::ProcessStateMaybeComment()

    {

    TTokenState nextState = EStateError;

    if (iToken[iTokenOffset] == '/')

        {

        nextState = EStateComment; 

        }

    else if (iToken[iTokenOffset] == '*')

        {

        nextState = EStateMultiLineComment;

        }

    return nextState;

    }

CDefinitionTokenizer::TTokenState CDefinitionTokenizer::ProcessStateComment()

    {

    TTokenState nextState = EStateComment;

    if (iToken[iTokenOffset] == '\n')

        {

        --iBufferOffset; // because the behaviour of moving to EStateStartToken is

                         // to progress to the next byte - we want to process the '\n'

        nextState = EStateStartToken; 

        iTokenType = EUnknown;

        }

    return nextState;

    }

CDefinitionTokenizer::TTokenState CDefinitionTokenizer::ProcessStateMultiLineComment()

    {

    TTokenState nextState = EStateMultiLineComment;

    if (iState == EStateMultiLineComment)

        {

        if (iToken[iTokenOffset] == '*')

            {

            // Started multi line comment

            nextState = EStateMaybeEndMultiLineComment;

            }

        }

    else if (iState == EStateMaybeEndMultiLineComment)

        {

        if (iToken[iTokenOffset] == '/')

            {

            nextState = EStateStartToken; 

            iTokenType = EUnknown;

            }

        }

    return nextState;

    }

CDefinitionTokenizer::TTokenState CDefinitionTokenizer::ProcessStateNumber()

    {

    TTokenState nextState = EStateError;

    if (IsTerminalChar())

        {

        nextState = EStateComplete;

        // iTokenType = ENumberDec; 

        }

    else

        {

        if (tolower(iToken[iTokenOffset]) == 'x')

            {

            nextState = EStateMaybeHexNumber;

            iTokenType = EUnknown;

            }

        else if (iToken[iTokenOffset] >= '0' && iToken[iTokenOffset] <= '9' )

            {

            nextState = EStateDecimalNumber;

            // iTokenType = ENumberDec;

            }

        }

    return nextState;

    }

CDefinitionTokenizer::TTokenState CDefinitionTokenizer::ProcessStateDecimalNumber()

    {

    TTokenState nextState = EStateError;

    if (IsTerminalChar())

        {

        nextState = EStateComplete;

        // iTokenType = ENumberDec;

        }

    else

        {

        if (iToken[iTokenOffset] >= '0' && iToken[iTokenOffset] <= '9')

            {

            nextState = EStateDecimalNumber;

            // iTokenType = ENumberDec;

            }

        }

    return nextState;

    }

CDefinitionTokenizer::TTokenState CDefinitionTokenizer::ProcessStateMaybeHexNumber()

    {

    TTokenState nextState = EStateError;

    char c = tolower(iToken[iTokenOffset]);

    if ((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f'))