1 /*

     2 * Copyright (c) 2004-2006 Nokia Corporation and/or its subsidiary(-ies). 

     3 * All rights reserved.

     4 * This component and the accompanying materials are made available

     5 * under the terms of "Eclipse Public License v1.0"

     6 * which accompanies this distribution, and is available

     7 * at the URL "http://www.eclipse.org/legal/epl-v10.html".

     8 *

     9 * Initial Contributors:

    10 * Nokia Corporation - initial contribution.

    11 *

    12 * Contributors:

    13 *

    14 * Description:  Serialization functions for CSIGrammar and CSILexicon

    15 *

    16 */

    17 

    18 

    19 // INCLUDE FILES

    20 #include "nsssidataserialize.h"

    21 #include <s32mem.h> // RBuf[Read/Write]Stream

    22 

    23 // LOCAL CONSTANTS AND MACROS

    24 const TUint32 KNibble16Limit = 0xffff;

    25 const TUint32 KNibble4Limit  = 0xf;

    26 

    27 // LOCAL FUNCTION PROTOTYPES

    28 void Externalize32bitArrayL( RWriteStream& aStream, RArray<TUint32>& aArray );

    29 void Internalize32bitArrayL( RReadStream& aStream,  RArray<TUint32>& aArray );

    30 TInt IndexOrderForPhonemes( const TLexiconPhoneme& p1, const TLexiconPhoneme& p2 );

    31 TInt StringOrderForPhonemes( const TLexiconPhoneme& p1, const TLexiconPhoneme& p2 );

    32 

    33 // ============================= LOCAL FUNCTIONS ===============================

    34 

    35 // -----------------------------------------------------------------------------

    36 // Externalize32bitArrayL

    37 // Writes an RArray with 32-bit entries to the stream.

    38 // Returns: None

    39 // -----------------------------------------------------------------------------

    40 //

    41 void Externalize32bitArrayL(

    42     RWriteStream&    aStream,  // The stream where to store the array

    43     RArray<TUint32>& aArray )  // The array; the entries must be 32-bit.

    44     {

    45     TInt count = aArray.Count();

    46 

    47     aStream.WriteInt32L( count );

    48 

    49     if ( count > 0 )

    50         {

    51         TInt* startPtr = (TInt*)&aArray[0];

    52         aStream.WriteL( (TUint8*)startPtr, sizeof(TInt) * count );

    53         }

    54     }

    55 

    56 // -----------------------------------------------------------------------------

    57 // Internalize32bitArrayL

    58 // Reads an array from the stream.

    59 // Returns: None

    60 // -----------------------------------------------------------------------------

    61 //

    62 void Internalize32bitArrayL(

    63     RReadStream& aStream,     // The stream to read from

    64     RArray<TUint32>& aArray ) // The array to be populated

    65     {

    66     TInt count = aStream.ReadInt32L();

    67     TInt err = KErrNone;

    68 

    69     for ( TInt k( 0 ); k < count; k++ )

    70         {

    71         TInt value = aStream.ReadInt32L();

    72         err |= aArray.Append( value );

    73         }

    74 

    75     User::LeaveIfError( err );

    76     }

    77 

    78 // -----------------------------------------------------------------------------

    79 // IndexOrderForPhonemes

    80 // 

    81 // -----------------------------------------------------------------------------

    82 //

    83 TInt IndexOrderForPhonemes( const TLexiconPhoneme& p1, const TLexiconPhoneme& p2 )

    84     {

    85     return( p1.iIndex - p2.iIndex );

    86     }

    87 

    88 // -----------------------------------------------------------------------------

    89 // StringOrderForPhonemes

    90 // 

    91 // -----------------------------------------------------------------------------

    92 //

    93 TInt StringOrderForPhonemes( const TLexiconPhoneme& p1, const TLexiconPhoneme& p2 )

    94     {

    95     return( p1.iPhoneme.Compare( p2.iPhoneme ) );

    96     }

    97 

    98 

    99 // ============================ MEMBER FUNCTIONS ===============================

   100 

   101 // -----------------------------------------------------------------------------

   102 // Implementation of CSILexiconSerializer starts from here.

   103 // -----------------------------------------------------------------------------

   104 //

   105 // -----------------------------------------------------------------------------

   106 // CSILexiconSerializer::CSILexiconSerializer

   107 // C++ default constructor can NOT contain any code, that

   108 // might leave.

   109 // -----------------------------------------------------------------------------

   110 //

   111 CSILexiconSerializer::CSILexiconSerializer()

   112     {

   113     iPronunIDs = NULL;

   114     iModelBankIDs = NULL;

   115     iParamBuf  = NULL;

   116     iCount     = 0; 

   117     iLexiconID = 0; // !!!! Change to KInvalidLexiconID.

   118     }

   119 

   120 // -----------------------------------------------------------------------------

   121 // CSILexiconSerializer::~CSILexiconSerializer

   122 // Destructor.

   123 // -----------------------------------------------------------------------------

   124 //

   125 CSILexiconSerializer::~CSILexiconSerializer()

   126     {

   127     iPronunciations.ResetAndDestroy();

   128     iIndexPronuns.ResetAndDestroy();

   129 

   130     iConversionTable.Close();

   131 

   132     delete iPronunIDs;

   133     delete iModelBankIDs;

   134     delete iParamBuf;

   135     }

   136 

   137 // -----------------------------------------------------------------------------

   138 // CSILexiconSerializer::NextPhonemeL

   139 // Stores the next phoneme to phonemeBuf and increases the cursor.

   140 // Returns true if there are still more phonemes.

   141 // -----------------------------------------------------------------------------

   142 //

   143 bool CSILexiconSerializer::NextPhonemeL(

   144     TDes8& phonemeBuf,

   145     const TDesC8& phonemeSeq,

   146     TInt&  aReadIterator )

   147     {

   148     TInt length = phonemeSeq.Length();

   149 

   150     // Out of phonemes?

   151     if ( length == aReadIterator )

   152         {

   153         return( false );

   154         }

   155 

   156     // Cut those phonemes away, which have been processed already.

   157     TPtrC8 unfinishedPhonemes = phonemeSeq.Right( length - aReadIterator );

   158 

   159     // Make SeparatorLoc point to the end of the next phoneme.

   160     TInt separatorLoc = unfinishedPhonemes.Locate( KPhonemeSeparator );

   161 

   162     // Phoneme length is from [start of unprocessed string] to [separator]...

   163     TInt phonemeLength = separatorLoc;

   164 

   165     // ...unless we're dealing with the last phoneme. Then

   166     // phonemeLength is from [start of unprocessed string] to [end of string].

   167     if ( separatorLoc == KErrNotFound )

   168         phonemeLength = length - aReadIterator;

   169 

   170     // Check that the phoneme is not too long.

   171     if ( phonemeLength > KMaxPhonemeLength )

   172         {

   173         User::Leave( KErrArgument );

   174         }

   175 

   176     if ( phonemeLength == 0 )

   177         {

   178         return( false );

   179         }

   180 

   181     // Copy

   182     phonemeBuf.Copy( unfinishedPhonemes.Ptr(), phonemeLength );

   183 

   184     // Update iterator

   185     aReadIterator += phonemeLength; // The phoneme

   186     if ( aReadIterator < length )   // The separator

   187         {

   188         aReadIterator++;

   189         }

   190 

   191     return( true );

   192     }

   193 

   194 // -----------------------------------------------------------------------------

   195 // CSILexiconSerializer::SavePronunciationL

   196 // Converts the ASCII phoneme strings to internal index strings.

   197 // -----------------------------------------------------------------------------

   198 //

   199 void CSILexiconSerializer::SavePronunciationL(CSIPronunciation* aPronun)

   200     {

   201     TLinearOrder<TLexiconPhoneme> order( StringOrderForPhonemes );

   202 

   203     // This phoneme is used for searching

   204     TLexiconPhoneme searchPhoneme;

   205     TDes8& phonemeBuf = searchPhoneme.iPhoneme;

   206 

   207     const TDesC8& phonemeSeq = aPronun->PhonemeSequence();

   208 

   209     HBufC8* indexPronun = HBufC8::NewLC( phonemeSeq.Length() );

   210 

   211     TInt phonemeReadIter = 0;

   212 

   213     while( NextPhonemeL( phonemeBuf, phonemeSeq, phonemeReadIter ) )

   214         {

   215         // Search the phoneme from conversion table.

   216         TInt loc = iConversionTable.FindInOrder( searchPhoneme, order );

   217 

   218         // Not found? Add the phoneme as a new phoneme.

   219         if ( loc == KErrNotFound )

   220             {

   221             searchPhoneme.iIndex = iConversionTable.Count();

   222             User::LeaveIfError(

   223                 iConversionTable.InsertInOrder( searchPhoneme, order )

   224                 );

   225 

   226             // Can't fail: the phoneme has just been added.

   227             loc = iConversionTable.FindInOrder( searchPhoneme, order );

   228 

   229             // Make sure that phoneme count fits in TUint8

   230             if ( searchPhoneme.iIndex >= 250 )

   231                 {

   232                 User::Leave( KErrOverflow );

   233                 }

   234             }

   235 

   236         // Phoneme to index

   237         // Case to TUint8 is safe - Checked 10 lines earlier.

   238         TUint8 index = (TUint8)iConversionTable[ loc ].iIndex;

   239         indexPronun->Des().Append( index );

   240         }

   241 

   242     // Add pronunciation to the list of "index-pronunciations"

   243     User::LeaveIfError( iIndexPronuns.Append( indexPronun ) );

   244     CleanupStack::Pop( indexPronun );

   245     }

   246 

   247 // -----------------------------------------------------------------------------

   248 // CSILexiconSerializer::Index2PhonemeLC

   249 // Restores an internal index string into ASCII phonemes.

   250 // -----------------------------------------------------------------------------

   251 //

   252 HBufC8* CSILexiconSerializer::Index2PhonemeLC(const TDesC8& aIndexSeq)

   253     {

   254     TLinearOrder<TLexiconPhoneme> order( IndexOrderForPhonemes );

   255     iConversionTable.Sort( order );

   256 

   257     TInt length = 0;

   258 

   259     TInt k( 0 );

   260     for( k = 0; k < aIndexSeq.Length(); k++ )

   261         {

   262         TInt index = aIndexSeq[ k ];

   263         length += iConversionTable[ index ].iPhoneme.Length();

   264         length++; // Separator ('-')

   265         }

   266 

   267     HBufC8* buf = HBufC8::NewLC( length );

   268     TPtr8 ptr = buf->Des();

   269 

   270     TInt lastK = aIndexSeq.Length() -1; // Index of the last character

   271 

   272     for( k = 0; k < aIndexSeq.Length(); k++ )

   273         {

   274         TInt index = aIndexSeq[ k ];

   275         ptr.Append( iConversionTable[ index ].iPhoneme );

   276 

   277         // Add separator, except for the last phoneme

   278         if ( k != lastK )

   279             {

   280             ptr.Append( KPhonemeSeparator );

   281             }

   282         }

   283 

   284     return( buf );

   285     }

   286 

   287 // -----------------------------------------------------------------------------

   288 // CSILexiconSerializer::NewLC

   289 // Two-phased constructor.

   290 // -----------------------------------------------------------------------------

   291 //

   292 CSILexiconSerializer* CSILexiconSerializer::NewLC( const CSILexicon& aLexicon )

   293     {

   294     CSILexiconSerializer* me = new (ELeave) CSILexiconSerializer;

   295     CleanupStack::PushL( me );

   296 

   297     me->ConstructL( aLexicon );

   298 

   299     return( me );

   300     }

   301 

   302 // -----------------------------------------------------------------------------

   303 // CSILexiconSerializer::NewLC

   304 // Two-phased constructor.

   305 // -----------------------------------------------------------------------------

   306 //

   307 CSILexiconSerializer* CSILexiconSerializer::NewLC( RReadStream& aStream )

   308     {

   309     CSILexiconSerializer* me = new (ELeave) CSILexiconSerializer;

   310     CleanupStack::PushL( me );

   311 

   312     me->InternalizeL( aStream );

   313 

   314     return( me );

   315     }

   316 

   317 // -----------------------------------------------------------------------------

   318 // CSILexiconSerializer::ConstructL

   319 // Symbian 2nd phase constructor can leave.

   320 // -----------------------------------------------------------------------------

   321 //

   322 void CSILexiconSerializer::ConstructL( const CSILexicon& aLexicon )

   323     {

   324     iLexiconID = aLexicon.LexiconID();

   325     iCount     = aLexicon.Count();

   326 

   327     iParamBuf = CBufFlat::NewL( 100 ); // Grow in 100 byte doses

   328 

   329     RArray<TSIModelBankID> modelBankIDs;

   330     RArray<TSIPronunciationID> pronunIDs;

   331 

   332     CleanupClosePushL( modelBankIDs );

   333     CleanupClosePushL( pronunIDs );

   334 

   335     RBufWriteStream paramStream;

   336     paramStream.Open( *iParamBuf );

   337     CleanupClosePushL( paramStream );

   338 

   339     // In serialization format, the 5 types of data in pronunciations

   340     // are put to 4 piles. Then, the common properties of the data is exploited.

   341     for ( TInt k = 0; k < aLexicon.Count(); k++ )

   342         {

   343         CSIPronunciation* pronun = &aLexicon.AtL( k );

   344 

   345         // 4 types of pronunciation data

   346         TInt err = KErrNone;

   347         

   348         // This is a hack to get the modelid aligned to 4-byte boundary,

   349         // for some reason this does not happen in winscw build if 

   350         // TSIModelBankID is taken from stack.

   351         TSIModelBankID* modelid = new (ELeave) TSIModelBankID;

   352         *modelid = pronun->ModelBankID();

   353         

   354         err |= modelBankIDs.Append( *modelid );

   355 

   356         // Delete temporary modelid 

   357         delete modelid;

   358         

   359         err |= pronunIDs.Append( pronun->PronunciationID() );

   360 

   361         User::LeaveIfError( err );

   362 

   363         SavePronunciationL( pronun );

   364 

   365         // Parameters

   366         pronun->CSIParameters::ExternalizeL( paramStream );

   367         }

   368 

   369     // Code the piled values

   370     // Casts are safe, as all variables are 32-bit (they are just called

   371     // with many different names - int, unsigned long, unsigned int)

   372     iPronunIDs = CNibble16Coder::NewL( pronunIDs );

   373     iModelBankIDs = CRLECoder::NewL( modelBankIDs );

   374 

   375     CleanupStack::PopAndDestroy( &paramStream );

   376 

   377     CleanupStack::PopAndDestroy( 2 ); // Close the RArrays

   378     }

   379 

   380 // -----------------------------------------------------------------------------

   381 // CSILexiconSerializer::ExternalizeL

   382 // Stores the object to the stream.

   383 // -----------------------------------------------------------------------------

   384 //

   385 void CSILexiconSerializer::ExternalizeL( RWriteStream& aStream )

   386     {

   387     aStream.WriteInt32L( KBinaryLexiconID );

   388 

   389     aStream.WriteInt32L( iLexiconID );

   390     aStream.WriteInt32L( iCount );

   391 

   392     // Write model bank IDs.

   393     iModelBankIDs->ExternalizeL( aStream );

   394     // Write pronunciation IDs

   395     iPronunIDs->ExternalizeL( aStream );

   396 

   397     // Write pronunciations (converted from phonemes to indices)

   398     aStream.WriteInt32L( iIndexPronuns.Count() );

   399 

   400     TInt k( 0 );

   401     for ( k = 0; k < iIndexPronuns.Count(); k++ )

   402         {

   403         HBufC8* pronun = iIndexPronuns[ k ];

   404 

   405         // There is no fixed limit on the length of a phoneme sequence.

   406         // It's not even guaranteed that the length fits in 32 bits, damnit!

   407         if ( pronun->Length() > 64000 )

   408             {

   409             User::Leave( KErrNotSupported );

   410             }

   411 

   412         aStream.WriteUint16L( pronun->Length() );

   413         aStream.WriteL( pronun->Ptr(), pronun->Length() );

   414         }

   415 

   416     // Write parameters

   417     aStream.WriteInt32L( iParamBuf->Size() );

   418     aStream.WriteL( iParamBuf->Ptr(0) );

   419 

   420     // Write phoneme<->index conversion table

   421     aStream.WriteInt32L( iConversionTable.Count() );

   422     for ( k = 0; k < iConversionTable.Count(); k++ )

   423         {

   424         aStream.WriteL( (TUint8*)&iConversionTable[k],

   425                         sizeof( TLexiconPhoneme ) );

   426         }

   427 

   428     // For corruption checking

   429     aStream.WriteInt32L( KBinaryLexiconID );

   430     }

   431 

   432 // -----------------------------------------------------------------------------

   433 // CSILexiconSerializer::InternalizeL

   434 // Restores the object from a stream

   435 // -----------------------------------------------------------------------------

   436 //

   437 void CSILexiconSerializer::InternalizeL(RReadStream& aStream)

   438     {

   439     if ( aStream.ReadInt32L() != KBinaryLexiconID )

   440         User::Leave( KErrArgument );

   441 

   442     iLexiconID = (TSILexiconID)aStream.ReadUint32L();

   443     iCount     = aStream.ReadInt32L();

   444 

   445     // Read model bank IDs

   446     iModelBankIDs = CRLECoder::NewL( aStream );

   447     // Read pronunciation IDs

   448     iPronunIDs = CNibble16Coder::NewL( aStream );

   449 

   450     // Read pronunciations (converted from phonemes to indices)

   451     TInt indexCount = aStream.ReadInt32L();

   452 

   453     if ( indexCount != iCount )

   454         {

   455         User::Leave( KErrCorrupt );

   456         }

   457 

   458     TInt k( 0 );

   459     for ( k = 0; k < indexCount; k++ )

   460         {

   461         HBufC8* pronun = 0;

   462 

   463         TUint16 length = aStream.ReadUint16L();

   464         pronun         = HBufC8::NewLC( length );

   465 

   466         TPtr8 pronunPtr = pronun->Des();

   467         aStream.ReadL( pronunPtr, length );

   468 

   469         User::LeaveIfError( iIndexPronuns.Append( pronun ) );

   470 

   471         CleanupStack::Pop( pronun );

   472         }

   473 

   474     // Read parameters

   475     TInt size = aStream.ReadInt32L();

   476     iParamBuf = CBufFlat::NewL( size+4 );

   477     iParamBuf->ExpandL( 0, size );

   478 

   479     TPtr8 paramBufPtr = iParamBuf->Ptr(0);

   480     aStream.ReadL( paramBufPtr, size );

   481 

   482     // Read phoneme<->index conversion table

   483     TInt conversionCount = aStream.ReadInt32L();

   484     for ( k = 0; k < conversionCount; k++ )

   485         {

   486         TLexiconPhoneme lexiconPhoneme;

   487 

   488         aStream.ReadL( (TUint8*)&lexiconPhoneme,

   489                         sizeof( TLexiconPhoneme ) );

   490 

   491         User::LeaveIfError( iConversionTable.Append( lexiconPhoneme ) );

   492         }

   493 

   494     // For corruption checking

   495     if ( aStream.ReadInt32L() != KBinaryLexiconID  )

   496         {

   497         User::Leave( KErrCorrupt );

   498         }

   499 

   500     }

   501 

   502 // -----------------------------------------------------------------------------

   503 // CSILexiconSerializer::RestoreL

   504 // Populates CSILexicon with the newly internalized data.

   505 // -----------------------------------------------------------------------------

   506 //

   507 void CSILexiconSerializer::RestoreL( CSILexicon& aLexicon, TSILexiconID& aLexiconID )

   508     {

   509     if ( aLexicon.Count() > 0 )

   510         {

   511         User::Leave( KErrInUse );

   512         }

   513 

   514     TSIModelBankID modelBankID = 0;

   515     TSIPronunciationID pronunID = 0;

   516 

   517     RBufReadStream paramStream;

   518     paramStream.Open( *iParamBuf );

   519     CleanupClosePushL( paramStream );

   520 

   521     iModelBankIDs->DecodeReset();

   522     iPronunIDs->DecodeReset();

   523 

   524     TInt iterPronunPos = 0;

   525 

   526     for ( TInt pronunK = 0; pronunK < iCount; pronunK++ )

   527         {

   528         modelBankID = iModelBankIDs->NextL();

   529         pronunID   = iPronunIDs->NextL();

   530 

   531         CSIPronunciation* pronun

   532             = CSIPronunciation::NewLC( pronunID, modelBankID ); 

   533 

   534         HBufC8* indexSequence = iIndexPronuns[ iterPronunPos ];

   535         iterPronunPos++;

   536 

   537         HBufC8* phonemeSequence = Index2PhonemeLC( *indexSequence );

   538         pronun->SetPhonemeSequenceL( *phonemeSequence );

   539         CleanupStack::PopAndDestroy( phonemeSequence );

   540 

   541         pronun->CSIParameters::InternalizeL( paramStream );

   542 

   543         aLexicon.AddL( pronun );

   544         CleanupStack::Pop( pronun );

   545         }

   546 

   547     CleanupStack::PopAndDestroy( &paramStream );

   548 

   549     // LexiconID can't be written to CSILexicon with public API.

   550     aLexiconID = iLexiconID;

   551     }

   552 

   553 

   554 // -----------------------------------------------------------------------------

   555 // Implementation of CSIGrammarSerializer starts from here.

   556 // -----------------------------------------------------------------------------

   557 //

   558 // -----------------------------------------------------------------------------

   559 // CSIGrammarSerializer::CSIGrammarSerializer

   560 // C++ default constructor can NOT contain any code, that

   561 // might leave.

   562 // -----------------------------------------------------------------------------

   563 //

   564 CSIGrammarSerializer::CSIGrammarSerializer()

   565     {

   566     iGrammarID = KInvalidGrammarID;

   567     iCount = 0;

   568     iRuleIDs                = NULL;

   569     iRuleVariantCounts      = NULL;

   570     iRuleVariantIDs         = NULL;

   571     iRuleVariantLexiconIDs  = NULL;

   572     iRuleVariantLanguages   = NULL;

   573     iPronunIDSeqLengths     = NULL;

   574     iPronunIDSequences      = NULL;

   575     iParamBuf               = NULL;

   576     }

   577 

   578 

   579 // -----------------------------------------------------------------------------

   580 // CSIGrammarSerializer::~CSIGrammarSerializer

   581 // Destructor.

   582 // -----------------------------------------------------------------------------

   583 //

   584 CSIGrammarSerializer::~CSIGrammarSerializer()

   585     {

   586     delete iRuleIDs;

   587     delete iRuleVariantCounts;

   588     delete iRuleVariantIDs;

   589     delete iRuleVariantLexiconIDs;

   590     delete iRuleVariantLanguages;

   591     delete iPronunIDSeqLengths;

   592     delete iPronunIDSequences;

   593     delete iParamBuf;

   594     }

   595 

   596 

   597 // -----------------------------------------------------------------------------

   598 // CSIGrammarSerializer::NewLC

   599 // Two-phased constructor.

   600 // -----------------------------------------------------------------------------

   601 //

   602 CSIGrammarSerializer* CSIGrammarSerializer::NewLC( const CSIGrammar& aGrammar )

   603     {

   604     CSIGrammarSerializer* me = new (ELeave) CSIGrammarSerializer();

   605     CleanupStack::PushL( me );

   606 

   607     me->ConstructL( aGrammar );

   608 

   609     return( me );

   610     }

   611 

   612 // -----------------------------------------------------------------------------

   613 // CSIGrammarSerializer::ConstructL

   614 // Symbian 2nd phase constructor can leave.

   615 // -----------------------------------------------------------------------------

   616 //

   617 void CSIGrammarSerializer::ConstructL( const CSIGrammar& aGrammar )

   618     {

   619     iParamBuf = CBufFlat::NewL( 100 ); // Grow in 100 byte doses

   620     RBufWriteStream paramStream;

   621     paramStream.Open( *iParamBuf );

   622     CleanupClosePushL( paramStream ); // Cleanp stack: param stream

   623 

   624     RArray<TSIRuleID> ruleIDs;

   625     RArray<TUint32> ruleVariantCounts;

   626     RArray<TUint32> ruleVariantIDs;

   627     RArray<TSILexiconID> ruleVariantLexiconIDs;

   628     RArray<TUint32> ruleVariantLanguages;

   629     RArray<TUint32> pronunIDSeqLengths;

   630     RArray<TSIPronunciationID> pronunIDSequences;

   631 

   632     CleanupClosePushL( ruleIDs );

   633     CleanupClosePushL( ruleVariantCounts );

   634     CleanupClosePushL( ruleVariantIDs );

   635     CleanupClosePushL( ruleVariantLexiconIDs );

   636     CleanupClosePushL( ruleVariantLanguages );

   637     CleanupClosePushL( pronunIDSeqLengths );

   638     CleanupClosePushL( pronunIDSequences  ); // Cleanup stack: param stream, 7 tmp arrays

   639 

   640     TInt err = KErrNone;

   641 

   642     // Pile the data according to type (8 types)

   643     for ( TInt ruleK( 0 ); ruleK < aGrammar.Count(); ruleK++ )

   644         {

   645         CSIRule& rule = aGrammar.AtL( ruleK );

   646 

   647         err |= ruleIDs.Append( rule.RuleID() );

   648         err |= ruleVariantCounts.Append( rule.Count() );

   649 

   650         for ( TInt variantK( 0 ); variantK < rule.Count(); variantK++ )

   651             {

   652             CSIRuleVariant& variant = rule.AtL( variantK );

   653 

   654             err |= ruleVariantIDs.Append( variant.RuleVariantID() );

   655             

   656             // This is a hack to get the lexid aligned to 4-byte boundary,

   657             // for some reason this does not happen in winscw build if 

   658             // TSILexiconID is taken from stack.

   659             TSILexiconID* lexid = new (ELeave) TSILexiconID;

   660             *lexid = variant.LexiconID();

   661            

   662             err |= ruleVariantLexiconIDs.Append( *lexid );

   663             

   664             delete lexid;

   665             

   666             err |= ruleVariantLanguages.Append( variant.Language() );

   667 

   668             TSIPronunciationIDSequence seq;

   669             CleanupClosePushL( seq ); // Cleanup stack: param stream, 7 tmp arrays, seq

   670             

   671             variant.GetPronunciationIDsL( seq );

   672 

   673             err |= pronunIDSeqLengths.Append( seq.Count() );

   674 

   675             for ( TInt k( 0 ); k < seq.Count(); k++ )

   676                 {

   677                 err |= pronunIDSequences.Append( seq[k] );

   678                 }

   679 

   680             CleanupStack::PopAndDestroy( &seq );

   681             // Cleanup stack: param stream, 7 tmp arrays

   682 

   683             // Store parameters to a buffer.

   684             variant.ExternalizeL( paramStream );

   685             }

   686 

   687         User::LeaveIfError( err );

   688         }

   689 

   690     // Code the arrays

   691     iGrammarID             = aGrammar.GrammarID();

   692     iCount                 = aGrammar.Count();

   693 

   694     iRuleIDs               = CNibble16Coder::NewL( ruleIDs );

   695     iRuleVariantCounts     = CNibble4Coder::NewL( ruleVariantCounts );

   696     iRuleVariantIDs        = CNibble4Coder::NewL( ruleVariantIDs );

   697     iRuleVariantLexiconIDs = CRLECoder::NewL( ruleVariantLexiconIDs );

   698     // Cast is safe, since TLanguage is 32-bit

   699     iRuleVariantLanguages  = CNibble4Coder::NewL( ruleVariantLanguages );

   700     iPronunIDSeqLengths    = CNibble4Coder::NewL( pronunIDSeqLengths );

   701     iPronunIDSequences     = CNibble16Coder::NewL( pronunIDSequences );

   702 

   703     CleanupStack::PopAndDestroy( 7 ); // Destory 7 tmp arrays

   704     CleanupStack::PopAndDestroy( &paramStream ); // Destroy parameter stream

   705     }

   706 

   707 // -----------------------------------------------------------------------------

   708 // CSIGrammarSerializer::ExternalizeL

   709 // Stores the object to the stream.

   710 // -----------------------------------------------------------------------------

   711 //

   712 void CSIGrammarSerializer::ExternalizeL( RWriteStream& aStream )

   713     {

   714     aStream.WriteUint32L( ( TUint32 ) KBinaryGrammarID );

   715     aStream.WriteUint32L( iGrammarID );

   716     aStream.WriteUint32L( iCount );

   717 

   718     iRuleIDs              ->ExternalizeL( aStream );

   719     iRuleVariantCounts    ->ExternalizeL( aStream );

   720     iRuleVariantIDs       ->ExternalizeL( aStream );

   721     iRuleVariantLexiconIDs->ExternalizeL( aStream );

   722     iRuleVariantLanguages ->ExternalizeL( aStream );

   723     iPronunIDSeqLengths   ->ExternalizeL( aStream );

   724     iPronunIDSequences    ->ExternalizeL( aStream );

   725 

   726     aStream.WriteInt32L( iParamBuf->Size() );

   727     aStream.WriteL( iParamBuf->Ptr(0) );

   728 

   729     aStream.WriteUint32L( ( TUint32 ) KBinaryGrammarID );

   730     }

   731 

   732 // -----------------------------------------------------------------------------

   733 // CSIGrammarSerializer::NewLC

   734 // Two-phased constructor.

   735 // -----------------------------------------------------------------------------

   736 //

   737 CSIGrammarSerializer* CSIGrammarSerializer::NewLC(RReadStream& aStream)

   738     {

   739     CSIGrammarSerializer* me = new (ELeave) CSIGrammarSerializer();

   740     CleanupStack::PushL( me );

   741 

   742     me->ConstructL( aStream );

   743 

   744     return( me );

   745     }

   746 

   747 // -----------------------------------------------------------------------------

   748 // CSIGrammarSerializer::ConstructL

   749 // Symbian 2nd phase constructor can leave.

   750 // -----------------------------------------------------------------------------

   751 //

   752 void CSIGrammarSerializer::ConstructL( RReadStream& aStream )

   753     {

   754     if ( aStream.ReadInt32L() != KBinaryGrammarID )

   755         {

   756         User::Leave( KErrCorrupt );

   757         }

   758 

   759     iGrammarID = (TSIGrammarID)aStream.ReadUint32L();

   760     iCount     = aStream.ReadUint32L();

   761 

   762     iRuleIDs              = CNibble16Coder::NewL( aStream );

   763     iRuleVariantCounts    = CNibble4Coder::NewL( aStream );

   764     iRuleVariantIDs       = CNibble4Coder::NewL( aStream );

   765     iRuleVariantLexiconIDs= CRLECoder::NewL( aStream );

   766     iRuleVariantLanguages = CNibble4Coder::NewL( aStream );

   767     iPronunIDSeqLengths   = CNibble4Coder::NewL( aStream );

   768     iPronunIDSequences    = CNibble16Coder::NewL( aStream );

   769 

   770     TInt32 paramSize = aStream.ReadInt32L();

   771     iParamBuf = CBufFlat::NewL( paramSize + 4 );

   772     iParamBuf->ExpandL( 0, paramSize );

   773     TPtr8 paramBufPtr = iParamBuf->Ptr(0);

   774     aStream.ReadL( paramBufPtr, paramSize );

   775 

   776     if ( aStream.ReadInt32L() != KBinaryGrammarID )

   777         {

   778         User::Leave( KErrCorrupt );

   779         }

   780     }

   781 

   782 // -----------------------------------------------------------------------------

   783 // CSIGrammarSerializer::RestoreL

   784 // Populates the CSIGrammar with newly internalized data.

   785 // -----------------------------------------------------------------------------

   786 //

   787 void CSIGrammarSerializer::RestoreL( CSIGrammar& aGrammar, TSIGrammarID& aGrammarID )

   788     {

   789     if ( aGrammar.Count() != 0 )

   790         {

   791         User::Leave( KErrInUse );

   792         }

   793 

   794     aGrammarID = iGrammarID;

   795 

   796     TBool ready = iRuleIDs && iRuleVariantCounts && iRuleVariantIDs &&

   797                   iRuleVariantLexiconIDs && iRuleVariantLanguages &&

   798                   iPronunIDSeqLengths && iPronunIDSequences;

   799 

   800     if ( !ready )

   801         {

   802         User::Leave( KErrNotReady );

   803         }

   804 

   805     iRuleIDs->DecodeReset();

   806     iRuleVariantCounts->DecodeReset();

   807     iRuleVariantIDs->DecodeReset();

   808     iRuleVariantLexiconIDs->DecodeReset();

   809     iRuleVariantLanguages->DecodeReset();

   810     iPronunIDSeqLengths->DecodeReset();

   811     iPronunIDSequences->DecodeReset();

   812 

   813     RBufReadStream paramStream;

   814     CleanupClosePushL( paramStream );

   815 

   816     paramStream.Open( *iParamBuf );

   817 

   818     for ( TInt ruleK( 0 ); ruleK < iCount; ruleK++ )

   819         {

   820         CSIRule* rule = CSIRule::NewLC( iRuleIDs->NextL() );

   821 

   822         TInt ruleVariantCount = iRuleVariantCounts->NextL();

   823 

   824         for ( TInt variantK( 0 ); variantK < ruleVariantCount; variantK++ )

   825             {

   826             CSIRuleVariant* variant = CSIRuleVariant::NewLC(

   827                 (TSIRuleVariantID)iRuleVariantIDs->NextL(),

   828                 iRuleVariantLexiconIDs->NextL()

   829                 );

   830 

   831             variant->SetLanguage(

   832                 (enum TLanguage)iRuleVariantLanguages->NextL() );

   833 

   834             RArray<TSIPronunciationID> pronunIDSeq;

   835             CleanupClosePushL( pronunIDSeq );

   836 

   837             TInt seqLength = iPronunIDSeqLengths->NextL();

   838             for ( TInt pronunIdK( 0 ); pronunIdK < seqLength; pronunIdK++ )

   839                 {

   840                 TInt err = pronunIDSeq.Append( iPronunIDSequences->NextL() );

   841                 User::LeaveIfError( err );

   842                 }

   843 

   844             variant->SetPronunciationIDsL( pronunIDSeq );

   845             CleanupStack::PopAndDestroy( &pronunIDSeq );

   846 

   847             variant->CSIParameters::InternalizeL( paramStream );

   848             rule->AddL( variant );

   849             CleanupStack::Pop( variant );

   850             }

   851 

   852         aGrammar.AddL( rule );

   853         CleanupStack::Pop( rule );

   854         }

   855 

   856     CleanupStack::PopAndDestroy( &paramStream );

   857     }

   858 

   859 

   860 // -----------------------------------------------------------------------------

   861 // Implementation of CNibble16Coder starts from here.

   862 // -----------------------------------------------------------------------------

   863 //

   864 // -----------------------------------------------------------------------------

   865 // CNibble16Coder::CNibble16Coder

   866 // C++ default constructor can NOT contain any code, that

   867 // might leave.

   868 // -----------------------------------------------------------------------------

   869 //

   870 CNibble16Coder::CNibble16Coder()

   871 : iMainArray(50),

   872   iOverflowArray(10)

   873     {

   874     DecodeReset();

   875     }

   876 

   877 // -----------------------------------------------------------------------------

   878 // CNibble16Coder::NewL

   879 // Two-phased constructor. Encodes the values given as the argument.

   880 // -----------------------------------------------------------------------------

   881 //

   882 CNibble16Coder* CNibble16Coder::NewL( const RArray<TUint32>& aPlainArray )

   883     {

   884     CNibble16Coder* me = new (ELeave) CNibble16Coder;

   885     CleanupStack::PushL( me );

   886 

   887     TInt count = aPlainArray.Count();;

   888     TInt sizeNeeded = (count+1) / 2;

   889 

   890     // Grow the array to be big enough

   891     TInt k( 0 );

   892     for ( k = 0; k < sizeNeeded; k++ )

   893         {

   894         User::LeaveIfError( me->iMainArray.Append( 0xffffffff ) );

   895         }

   896 

   897     TUint16* storagePtr = 0;

   898 

   899     if ( count > 0 )

   900         {

   901         storagePtr = (TUint16*)&me->iMainArray[0];

   902         }

   903 

   904     for( k = 0; k < count; k++ )

   905         {

   906         TSIPronunciationID id = aPlainArray[ k ];

   907         TBool overflow = false;

   908 

   909         // If the 32-bit value doesn't fit to 16 bits:

   910         //  * Put overflow sign to main array

   911         //  * Put the real ID to overflow array

   912         if ( id >= KNibble16Limit )

   913             {

   914             overflow = true;

   915             id = KNibble16Limit;

   916             }

   917 

   918         // Write to the main array

   919         // ([overflow sign] OR [id, which fits to 16 bits])

   920         *(storagePtr++) = (TUint16)id;

   921 

   922         // If it didn't fit to 16 bits, save it to the overflow array.

   923         if ( overflow )

   924             {

   925             TInt err = me->iOverflowArray.Append( aPlainArray[k] );

   926             User::LeaveIfError( err );

   927             }

   928         }

   929 

   930     CleanupStack::Pop( me );

   931     return( me );

   932     }

   933 

   934 // -----------------------------------------------------------------------------

   935 // CNibble16Coder::NewL

   936 // Two-phased constructor. Internalizes an array from stream.

   937 // -----------------------------------------------------------------------------

   938 //

   939 CNibble16Coder* CNibble16Coder::NewL( RReadStream& aStream )

   940     {

   941     CNibble16Coder* me = new (ELeave) CNibble16Coder;

   942     CleanupStack::PushL( me );

   943 

   944     if ( aStream.ReadInt32L() != KBinaryNibble16ID )

   945         {

   946         User::Leave( KErrCorrupt );

   947         }

   948 

   949     Internalize32bitArrayL( aStream, me->iMainArray     );

   950     Internalize32bitArrayL( aStream, me->iOverflowArray );

   951 

   952     CleanupStack::Pop( me );

   953     return( me );

   954     }

   955 

   956 // -----------------------------------------------------------------------------

   957 // CNibble16Coder::~CNibble16Coder

   958 // Destructor

   959 // -----------------------------------------------------------------------------

   960 //

   961 CNibble16Coder::~CNibble16Coder()

   962     {

   963     iMainArray.Close();

   964     iOverflowArray.Close();

   965     }

   966 

   967 // -----------------------------------------------------------------------------

   968 // CNibble16Coder::DecodeReset

   969 // Moves cursor to the beginning of the array.

   970 // -----------------------------------------------------------------------------

   971 //

   972 void CNibble16Coder::DecodeReset()

   973     {

   974     iPosition = 0;

   975     iOverflowPos = 0;

   976     }

   977 

   978 // -----------------------------------------------------------------------------

   979 // CNibble16Coder::NextL

   980 // Returns a value in the array and moves the cursor forward.

   981 // -----------------------------------------------------------------------------

   982 //

   983 TUint32 CNibble16Coder::NextL()

   984     {

   985     if ( iPosition >= 2*iMainArray.Count() )

   986         User::Leave( KErrOverflow );

   987 

   988     TUint16* storagePtr = (TUint16*)&iMainArray[0];

   989 

   990     // Get the [pronunciation ID] OR [overflow sign] from main array.

   991     TSIPronunciationID id = storagePtr[ iPosition ];

   992 

   993     // It was overflow sign; get whole 32-bit ID from overflow array

   994     if ( id == KNibble16Limit ) // It was pronunciation ID

   995         {

   996         if ( iOverflowPos == iOverflowArray.Count() )

   997             {

   998             User::Leave( KErrOverflow );

   999             }

  1000 

  1001         id = iOverflowArray[ iOverflowPos ];

  1002         iOverflowPos++;

  1003         }

  1004 

  1005     // Update iterator

  1006     iPosition++;

  1007 

  1008     return( id );

  1009     }

  1010 

  1011 // -----------------------------------------------------------------------------

  1012 // CNibble16Coder::ExternalizeL

  1013 // Externalizes the array into a stream.

  1014 // -----------------------------------------------------------------------------

  1015 //

  1016 void CNibble16Coder::ExternalizeL( RWriteStream& aStream )

  1017     {

  1018     aStream.WriteInt32L( KBinaryNibble16ID );

  1019     Externalize32bitArrayL( aStream, iMainArray );

  1020     Externalize32bitArrayL( aStream, iOverflowArray );

  1021     }

  1022 

  1023 

  1024 // -----------------------------------------------------------------------------

  1025 // Implementation of CNibble4Coder starts from here.

  1026 // -----------------------------------------------------------------------------

  1027 //

  1028 // -----------------------------------------------------------------------------

  1029 // CNibble4Coder::NewL

  1030 // Two-phased constructor.

  1031 // -----------------------------------------------------------------------------

  1032 //

  1033 CNibble4Coder* CNibble4Coder::NewL( const RArray<TUint32>& aPlainArray )

  1034     {

  1035     CNibble4Coder* me = new (ELeave) CNibble4Coder;

  1036     CleanupStack::PushL( me );

  1037     me->DecodeReset();

  1038 

  1039     TInt count = aPlainArray.Count();

  1040 

  1041     for ( TInt k( 0 ); k < count; k++ )

  1042         {

  1043         TUint compactValue = aPlainArray[ k ];

  1044         TBool overflow = EFalse;

  1045 

  1046         // If the value does not fit to 4 bytes, put an overflow sign to main

  1047         // array, and save the value to a separate overflow array.

  1048         if ( compactValue >= KNibble4Limit )

  1049             {

  1050             compactValue = KNibble4Limit;

  1051             overflow = true;

  1052             }

  1053 

  1054         // Save the compact value

  1055         if ( me->iSlot == 0 )

  1056             {

  1057             User::LeaveIfError( me->iMainArray.Append( compactValue ) );

  1058             me->iSlot++;

  1059             }

  1060         else{

  1061             // We save 4-bit values to 32-bit variable. So, there are 8 slots

  1062             // in each 32-bit entry.

  1063 

  1064             // Shift the value to correct slot. 

  1065             // Earlier, we have made sure, that compactValue < 16.

  1066             compactValue = compactValue << (4 * me->iSlot);

  1067 

  1068             // Put it to the array.

  1069             me->iMainArray[ me->iPosition ]

  1070                 = compactValue | me->iMainArray[ me->iPosition ];

  1071 

  1072             // Update iterator

  1073             me->iSlot++;

  1074             if ( me->iSlot == 8 )

  1075                 {

  1076                 me->iPosition++;

  1077                 me->iSlot = 0;

  1078                 }

  1079             }

  1080 

  1081         // If the value didn't fit to 4 bits, save it to 32-bit overflow array.

  1082         if ( overflow )

  1083             {

  1084             TInt ret = me->iOverflowArray.Append( aPlainArray[ k ] );

  1085             User::LeaveIfError( ret );

  1086             }

  1087         }

  1088 

  1089     // If the number of entries was not divisible by 8, we have

  1090     // uninitialized slots in the end. Initialize them with 0xf.

  1091 

  1092     // If the decoder finds one of these 0xf's, it goes to overflow array,

  1093     // notices that all entries in the overflow array have been processed,

  1094     // and leaves.

  1095     if ( me->iSlot != 0 )

  1096         {

  1097         TUint unusedValue = 0xffffffff;

  1098         unusedValue = unusedValue << (4 * me->iSlot);

  1099 

  1100         me->iMainArray[ me->iPosition ] 

  1101             = unusedValue | me->iMainArray[ me->iPosition ];

  1102         }

  1103 

  1104     me->DecodeReset();

  1105     CleanupStack::Pop( me );

  1106     return me;

  1107     }

  1108 

  1109 // -----------------------------------------------------------------------------

  1110 // CNibble4Coder::NewL

  1111 // Two-phased constructor.

  1112 // -----------------------------------------------------------------------------

  1113 //

  1114 CNibble4Coder* CNibble4Coder::NewL( RReadStream& aStream )

  1115     {

  1116     CNibble4Coder* me = new (ELeave) CNibble4Coder;

  1117     CleanupStack::PushL( me );

  1118     me->DecodeReset();

  1119 

  1120     if ( aStream.ReadInt32L() != KBinaryNibble4ID )

  1121         {

  1122         User::Leave( KErrCorrupt );

  1123         }

  1124 

  1125     Internalize32bitArrayL( aStream, me->iMainArray     );

  1126     Internalize32bitArrayL( aStream, me->iOverflowArray );

  1127 

  1128     CleanupStack::Pop( me );

  1129     return( me );

  1130     }

  1131 

  1132 // -----------------------------------------------------------------------------

  1133 // CNibble4Coder::CNibble4Coder

  1134 // C++ default constructor can NOT contain any code, that