/*

* Copyright (c) 2004-2006 Nokia Corporation and/or its subsidiary(-ies). 

* All rights reserved.

* This component and the accompanying materials are made available

* under the terms of "Eclipse Public License v1.0"

* which accompanies this distribution, and is available

* at the URL "http://www.eclipse.org/legal/epl-v10.html".

*

* Initial Contributors:

* Nokia Corporation - initial contribution.

*

* Contributors:

*

* Description:  Serialization functions for CSIGrammar and CSILexicon

*

*/

// INCLUDE FILES

#include "nsssidataserialize.h"

#include <s32mem.h> // RBuf[Read/Write]Stream

// LOCAL CONSTANTS AND MACROS

const TUint32 KNibble16Limit = 0xffff;

const TUint32 KNibble4Limit  = 0xf;

// LOCAL FUNCTION PROTOTYPES

void Externalize32bitArrayL( RWriteStream& aStream, RArray<TUint32>& aArray );

void Internalize32bitArrayL( RReadStream& aStream,  RArray<TUint32>& aArray );

TInt IndexOrderForPhonemes( const TLexiconPhoneme& p1, const TLexiconPhoneme& p2 );

TInt StringOrderForPhonemes( const TLexiconPhoneme& p1, const TLexiconPhoneme& p2 );

// ============================= LOCAL FUNCTIONS ===============================

// -----------------------------------------------------------------------------

// Externalize32bitArrayL

// Writes an RArray with 32-bit entries to the stream.

// Returns: None

// -----------------------------------------------------------------------------

//

void Externalize32bitArrayL(

    RWriteStream&    aStream,  // The stream where to store the array

    RArray<TUint32>& aArray )  // The array; the entries must be 32-bit.

    {

    TInt count = aArray.Count();

    aStream.WriteInt32L( count );

    if ( count > 0 )

        {

        TInt* startPtr = (TInt*)&aArray[0];

        aStream.WriteL( (TUint8*)startPtr, sizeof(TInt) * count );

        }

    }

// -----------------------------------------------------------------------------

// Internalize32bitArrayL

// Reads an array from the stream.

// Returns: None

// -----------------------------------------------------------------------------

//

void Internalize32bitArrayL(

    RReadStream& aStream,     // The stream to read from

    RArray<TUint32>& aArray ) // The array to be populated

    {

    TInt count = aStream.ReadInt32L();

    TInt err = KErrNone;

    for ( TInt k( 0 ); k < count; k++ )

        {

        TInt value = aStream.ReadInt32L();

        err |= aArray.Append( value );

        }

    User::LeaveIfError( err );

    }

// -----------------------------------------------------------------------------

// IndexOrderForPhonemes

// 

// -----------------------------------------------------------------------------

//

TInt IndexOrderForPhonemes( const TLexiconPhoneme& p1, const TLexiconPhoneme& p2 )

    {

    return( p1.iIndex - p2.iIndex );

    }

// -----------------------------------------------------------------------------

// StringOrderForPhonemes

// 

// -----------------------------------------------------------------------------

//

TInt StringOrderForPhonemes( const TLexiconPhoneme& p1, const TLexiconPhoneme& p2 )

    {

    return( p1.iPhoneme.Compare( p2.iPhoneme ) );

    }

// ============================ MEMBER FUNCTIONS ===============================

// -----------------------------------------------------------------------------

// Implementation of CSILexiconSerializer starts from here.

// -----------------------------------------------------------------------------

//

// -----------------------------------------------------------------------------

// CSILexiconSerializer::CSILexiconSerializer

// C++ default constructor can NOT contain any code, that

// might leave.

// -----------------------------------------------------------------------------

//

CSILexiconSerializer::CSILexiconSerializer()

    {

    iPronunIDs = NULL;

    iModelBankIDs = NULL;

    iParamBuf  = NULL;

    iCount     = 0; 

    iLexiconID = 0; // !!!! Change to KInvalidLexiconID.

    }

// -----------------------------------------------------------------------------

// CSILexiconSerializer::~CSILexiconSerializer

// Destructor.

// -----------------------------------------------------------------------------

//

CSILexiconSerializer::~CSILexiconSerializer()

    {

    iPronunciations.ResetAndDestroy();

    iIndexPronuns.ResetAndDestroy();

    iConversionTable.Close();

    delete iPronunIDs;

    delete iModelBankIDs;

    delete iParamBuf;

    }

// -----------------------------------------------------------------------------

// CSILexiconSerializer::NextPhonemeL

// Stores the next phoneme to phonemeBuf and increases the cursor.

// Returns true if there are still more phonemes.

// -----------------------------------------------------------------------------

//

bool CSILexiconSerializer::NextPhonemeL(

    TDes8& phonemeBuf,

    const TDesC8& phonemeSeq,

    TInt&  aReadIterator )

    {

    TInt length = phonemeSeq.Length();

    // Out of phonemes?

    if ( length == aReadIterator )

        {

        return( false );

        }

    // Cut those phonemes away, which have been processed already.

    TPtrC8 unfinishedPhonemes = phonemeSeq.Right( length - aReadIterator );

    // Make SeparatorLoc point to the end of the next phoneme.

    TInt separatorLoc = unfinishedPhonemes.Locate( KPhonemeSeparator );

    // Phoneme length is from [start of unprocessed string] to [separator]...

    TInt phonemeLength = separatorLoc;

    // ...unless we're dealing with the last phoneme. Then

    // phonemeLength is from [start of unprocessed string] to [end of string].

    if ( separatorLoc == KErrNotFound )

        phonemeLength = length - aReadIterator;

    // Check that the phoneme is not too long.

    if ( phonemeLength > KMaxPhonemeLength )

        {

        User::Leave( KErrArgument );

        }

    if ( phonemeLength == 0 )

        {

        return( false );

        }

    // Copy

    phonemeBuf.Copy( unfinishedPhonemes.Ptr(), phonemeLength );

    // Update iterator

    aReadIterator += phonemeLength; // The phoneme

    if ( aReadIterator < length )   // The separator

        {

        aReadIterator++;

        }

    return( true );

    }

// -----------------------------------------------------------------------------

// CSILexiconSerializer::SavePronunciationL

// Converts the ASCII phoneme strings to internal index strings.

// -----------------------------------------------------------------------------

//

void CSILexiconSerializer::SavePronunciationL(CSIPronunciation* aPronun)

    {

    TLinearOrder<TLexiconPhoneme> order( StringOrderForPhonemes );

    // This phoneme is used for searching

    TLexiconPhoneme searchPhoneme;

    TDes8& phonemeBuf = searchPhoneme.iPhoneme;

    const TDesC8& phonemeSeq = aPronun->PhonemeSequence();

    HBufC8* indexPronun = HBufC8::NewLC( phonemeSeq.Length() );

    TInt phonemeReadIter = 0;

    while( NextPhonemeL( phonemeBuf, phonemeSeq, phonemeReadIter ) )

        {

        // Search the phoneme from conversion table.

        TInt loc = iConversionTable.FindInOrder( searchPhoneme, order );

        // Not found? Add the phoneme as a new phoneme.

        if ( loc == KErrNotFound )

            {

            searchPhoneme.iIndex = iConversionTable.Count();

            User::LeaveIfError(

                iConversionTable.InsertInOrder( searchPhoneme, order )

                );

            // Can't fail: the phoneme has just been added.

            loc = iConversionTable.FindInOrder( searchPhoneme, order );

            // Make sure that phoneme count fits in TUint8

            if ( searchPhoneme.iIndex >= 250 )

                {

                User::Leave( KErrOverflow );

                }

            }

        // Phoneme to index

        // Case to TUint8 is safe - Checked 10 lines earlier.

        TUint8 index = (TUint8)iConversionTable[ loc ].iIndex;

        indexPronun->Des().Append( index );

        }

    // Add pronunciation to the list of "index-pronunciations"

    User::LeaveIfError( iIndexPronuns.Append( indexPronun ) );

    CleanupStack::Pop( indexPronun );

    }

// -----------------------------------------------------------------------------

// CSILexiconSerializer::Index2PhonemeLC

// Restores an internal index string into ASCII phonemes.

// -----------------------------------------------------------------------------

//

HBufC8* CSILexiconSerializer::Index2PhonemeLC(const TDesC8& aIndexSeq)

    {

    TLinearOrder<TLexiconPhoneme> order( IndexOrderForPhonemes );

    iConversionTable.Sort( order );

    TInt length = 0;

    TInt k( 0 );

    for( k = 0; k < aIndexSeq.Length(); k++ )

        {

        TInt index = aIndexSeq[ k ];

        length += iConversionTable[ index ].iPhoneme.Length();

        length++; // Separator ('-')

        }

    HBufC8* buf = HBufC8::NewLC( length );

    TPtr8 ptr = buf->Des();

    TInt lastK = aIndexSeq.Length() -1; // Index of the last character

    for( k = 0; k < aIndexSeq.Length(); k++ )

        {

        TInt index = aIndexSeq[ k ];

        ptr.Append( iConversionTable[ index ].iPhoneme );

        // Add separator, except for the last phoneme

        if ( k != lastK )

            {

            ptr.Append( KPhonemeSeparator );

            }

        }

    return( buf );

    }

// -----------------------------------------------------------------------------

// CSILexiconSerializer::NewLC

// Two-phased constructor.

// -----------------------------------------------------------------------------

//

CSILexiconSerializer* CSILexiconSerializer::NewLC( const CSILexicon& aLexicon )

    {

    CSILexiconSerializer* me = new (ELeave) CSILexiconSerializer;

    CleanupStack::PushL( me );

    me->ConstructL( aLexicon );

    return( me );

    }

// -----------------------------------------------------------------------------

// CSILexiconSerializer::NewLC

// Two-phased constructor.

// -----------------------------------------------------------------------------

//

CSILexiconSerializer* CSILexiconSerializer::NewLC( RReadStream& aStream )

    {

    CSILexiconSerializer* me = new (ELeave) CSILexiconSerializer;

    CleanupStack::PushL( me );

    me->InternalizeL( aStream );

    return( me );

    }

// -----------------------------------------------------------------------------

// CSILexiconSerializer::ConstructL

// Symbian 2nd phase constructor can leave.

// -----------------------------------------------------------------------------

//

void CSILexiconSerializer::ConstructL( const CSILexicon& aLexicon )

    {

    iLexiconID = aLexicon.LexiconID();

    iCount     = aLexicon.Count();

    iParamBuf = CBufFlat::NewL( 100 ); // Grow in 100 byte doses

    RArray<TSIModelBankID> modelBankIDs;

    RArray<TSIPronunciationID> pronunIDs;

    CleanupClosePushL( modelBankIDs );

    CleanupClosePushL( pronunIDs );

    RBufWriteStream paramStream;

    paramStream.Open( *iParamBuf );

    CleanupClosePushL( paramStream );

    // In serialization format, the 5 types of data in pronunciations

    // are put to 4 piles. Then, the common properties of the data is exploited.

    for ( TInt k = 0; k < aLexicon.Count(); k++ )

        {

        CSIPronunciation* pronun = &aLexicon.AtL( k );

        // 4 types of pronunciation data

        TInt err = KErrNone;

        // This is a hack to get the modelid aligned to 4-byte boundary,

        // for some reason this does not happen in winscw build if 

        // TSIModelBankID is taken from stack.

        TSIModelBankID* modelid = new (ELeave) TSIModelBankID;

        *modelid = pronun->ModelBankID();

        err |= modelBankIDs.Append( *modelid );

        // Delete temporary modelid 

        delete modelid;

        err |= pronunIDs.Append( pronun->PronunciationID() );

        User::LeaveIfError( err );

        SavePronunciationL( pronun );

        // Parameters

        pronun->CSIParameters::ExternalizeL( paramStream );

        }

    // Code the piled values

    // Casts are safe, as all variables are 32-bit (they are just called

    // with many different names - int, unsigned long, unsigned int)

    iPronunIDs = CNibble16Coder::NewL( pronunIDs );

    iModelBankIDs = CRLECoder::NewL( modelBankIDs );

    CleanupStack::PopAndDestroy( &paramStream );

    CleanupStack::PopAndDestroy( 2 ); // Close the RArrays

    }

// -----------------------------------------------------------------------------

// CSILexiconSerializer::ExternalizeL

// Stores the object to the stream.

// -----------------------------------------------------------------------------

//

void CSILexiconSerializer::ExternalizeL( RWriteStream& aStream )

    {

    aStream.WriteInt32L( KBinaryLexiconID );

    aStream.WriteInt32L( iLexiconID );

    aStream.WriteInt32L( iCount );

    // Write model bank IDs.

    iModelBankIDs->ExternalizeL( aStream );

    // Write pronunciation IDs

    iPronunIDs->ExternalizeL( aStream );

    // Write pronunciations (converted from phonemes to indices)

    aStream.WriteInt32L( iIndexPronuns.Count() );

    TInt k( 0 );

    for ( k = 0; k < iIndexPronuns.Count(); k++ )

        {

        HBufC8* pronun = iIndexPronuns[ k ];

        // There is no fixed limit on the length of a phoneme sequence.

        // It's not even guaranteed that the length fits in 32 bits, damnit!

        if ( pronun->Length() > 64000 )

            {

            User::Leave( KErrNotSupported );

            }

        aStream.WriteUint16L( pronun->Length() );

        aStream.WriteL( pronun->Ptr(), pronun->Length() );

        }

    // Write parameters

    aStream.WriteInt32L( iParamBuf->Size() );

    aStream.WriteL( iParamBuf->Ptr(0) );

    // Write phoneme<->index conversion table

    aStream.WriteInt32L( iConversionTable.Count() );

    for ( k = 0; k < iConversionTable.Count(); k++ )

        {

        aStream.WriteL( (TUint8*)&iConversionTable[k],

                        sizeof( TLexiconPhoneme ) );

        }

    // For corruption checking

    aStream.WriteInt32L( KBinaryLexiconID );

    }

// -----------------------------------------------------------------------------

// CSILexiconSerializer::InternalizeL

// Restores the object from a stream

// -----------------------------------------------------------------------------

//

void CSILexiconSerializer::InternalizeL(RReadStream& aStream)

    {

    if ( aStream.ReadInt32L() != KBinaryLexiconID )

        User::Leave( KErrArgument );

    iLexiconID = (TSILexiconID)aStream.ReadUint32L();

    iCount     = aStream.ReadInt32L();

    // Read model bank IDs

    iModelBankIDs = CRLECoder::NewL( aStream );

    // Read pronunciation IDs

    iPronunIDs = CNibble16Coder::NewL( aStream );

    // Read pronunciations (converted from phonemes to indices)

    TInt indexCount = aStream.ReadInt32L();

    if ( indexCount != iCount )

        {

        User::Leave( KErrCorrupt );

        }

    TInt k( 0 );

    for ( k = 0; k < indexCount; k++ )

        {

        HBufC8* pronun = 0;

        TUint16 length = aStream.ReadUint16L();

        pronun         = HBufC8::NewLC( length );

        TPtr8 pronunPtr = pronun->Des();

        aStream.ReadL( pronunPtr, length );

        User::LeaveIfError( iIndexPronuns.Append( pronun ) );

        CleanupStack::Pop( pronun );

        }

    // Read parameters

    TInt size = aStream.ReadInt32L();

    iParamBuf = CBufFlat::NewL( size+4 );

    iParamBuf->ExpandL( 0, size );

    TPtr8 paramBufPtr = iParamBuf->Ptr(0);

    aStream.ReadL( paramBufPtr, size );

    // Read phoneme<->index conversion table

    TInt conversionCount = aStream.ReadInt32L();

    for ( k = 0; k < conversionCount; k++ )

        {

        TLexiconPhoneme lexiconPhoneme;

        aStream.ReadL( (TUint8*)&lexiconPhoneme,

                        sizeof( TLexiconPhoneme ) );

        User::LeaveIfError( iConversionTable.Append( lexiconPhoneme ) );

        }

    // For corruption checking

    if ( aStream.ReadInt32L() != KBinaryLexiconID  )

        {

        User::Leave( KErrCorrupt );

        }

    }

// -----------------------------------------------------------------------------

// CSILexiconSerializer::RestoreL

// Populates CSILexicon with the newly internalized data.

// -----------------------------------------------------------------------------

//

void CSILexiconSerializer::RestoreL( CSILexicon& aLexicon, TSILexiconID& aLexiconID )

    {

    if ( aLexicon.Count() > 0 )

        {

        User::Leave( KErrInUse );

        }

    TSIModelBankID modelBankID = 0;

    TSIPronunciationID pronunID = 0;

    RBufReadStream paramStream;

    paramStream.Open( *iParamBuf );

    CleanupClosePushL( paramStream );

    iModelBankIDs->DecodeReset();

    iPronunIDs->DecodeReset();

    TInt iterPronunPos = 0;

    for ( TInt pronunK = 0; pronunK < iCount; pronunK++ )

        {

        modelBankID = iModelBankIDs->NextL();

        pronunID   = iPronunIDs->NextL();