/*
* Copyright (c) 2003-2005 Nokia Corporation and/or its subsidiary(-ies). 
* All rights reserved.
* This component and the accompanying materials are made available
* under the terms of "Eclipse Public License v1.0"
* which accompanies this distribution, and is available
* at the URL "http://www.eclipse.org/legal/epl-v10.html".
*
* Initial Contributors:
* Nokia Corporation - initial contribution.
*
* Contributors:
*
* Description:      
*
*/








#include <e32std.h>
#include <e32base.h>
#include <charconv.h>
#include <convutils.h>
                   
const TInt KNoPreviousCharacterSet=-1;
const TInt KDefaultCharacterSet = 0;
const TUint KControlCharacterEscape=0x1b;

#if defined(_DEBUG)
const TInt KMaximumLengthOfIntermediateBuffer=5;
#else
const TInt KMaximumLengthOfIntermediateBuffer=150;
#endif

struct SCnvConversionData;

_LIT(KLitPanicText, "CONVUTILS");

enum TPanic
	{
	EPanicBadInputConversionFlags1=1,
	EPanicBadInputConversionFlags2,
	EPanicBadInputConversionFlags3,
	EPanicBadNumberOfUnicodeElementsConsumed,
	EPanicAppendFlagViolated,
	EPanicBadNumberOfUnicodeCharactersConverted,
	EPanicBadNumberOfCharactersThatDroppedOut,
	EPanicLoopCounterOverRun1,
	EPanicLoopCounterOverRun2,
	EPanicDescriptorNotWholeNumberOfCharacters1,
	EPanicDescriptorNotWholeNumberOfCharacters2,
	EPanicDescriptorNotWholeNumberOfCharacters3,
	EPanicDescriptorNotWholeNumberOfCharacters4,
	EPanicBadStartOfNextEscapeSequence,
	EPanicInconsistentNumberOfForeignBytesRemaining,
	EPanicBadLengthOfRunToConvert1,
	EPanicBadLengthOfRunToConvert2,
	EPanicBadMethodPointer,
	EPanicBadMethodData1,
	EPanicBadMethodData2,
	EPanicBadMethodData3,
	EPanicBadMethodData4,
	EPanicBadNumberOfCharacterSets,
	EPanicBadConversionDataPointer1,
	EPanicBadConversionDataPointer2,
	EPanicBadConversionDataPointer3,
	EPanicBadFunctionPointer1,
	EPanicBadFunctionPointer2,
	EPanicBadFunctionPointer3,
	EPanicBadEscapeSequencePointer1,
	EPanicBadEscapeSequencePointer2,
	EPanicBadNumberOfStates,
	EPanicBadEscapeSequenceStart,
	EPanicBadNumberOfMethods,
	EPanicBadSurrogatePair1,
	EPanicBadSurrogatePair2,
	EPanicBadRemainderOfForeign,
	EPanicOutputDescriptorTooShortEvenToHoldEscapeSequenceToDefaultCharacterSet
	};

LOCAL_C void Panic(TPanic aPanic)
	{
	User::Panic(KLitPanicText, aPanic);
	}

/** Converts Unicode text into a complex foreign character set encoding. This 
is an encoding which cannot be converted simply by calling 
CCnvCharacterSetConverter::DoConvertFromUnicode(). It may be modal (e.g. JIS) 
or non-modal (e.g. Shift-JIS).

The Unicode text specified in aUnicode is converted using the array of 
conversion data objects (aArrayOfCharacterSets) provided by the plug-in for 
the complex character set encoding, and the converted text is returned in 
aForeign. Any existing contents in aForeign are overwritten.

Unlike CCnvCharacterSetConverter::DoConvertFromUnicode(), multiple character 
sets can be specified. aUnicode is converted using the first character conversion 
data object in the array. When a character is found which cannot be converted 
using that data, each character set in the array is tried in turn. If it cannot 
be converted using any object in the array, the index of the character is 
appended to aIndicesOfUnconvertibleCharacters and the character is replaced 
by aReplacementForUnconvertibleUnicodeCharacters.

If it can be converted using another object in the array, that object is used 
to convert all subsequent characters until another unconvertible character 
is found.

@param aDefaultEndiannessOfForeignCharacters The default endian-ness to use 
when writing the characters in the foreign character set. If an endian-ness 
for foreign characters is specified in the current conversion data object, 
then that is used instead and the value of 
aDefaultEndiannessOfForeignCharacters is ignored. 
@param aReplacementForUnconvertibleUnicodeCharacters The single character (one 
or more byte values) which is used to replace unconvertible characters. 
@param aForeign On return, contains the converted text in the non-Unicode 
character set. 
@param aUnicode The source Unicode text to be converted. 
@param aIndicesOfUnconvertibleCharacters On return, holds an ascending array 
of the indices of each Unicode character in the source text which could not 
be converted (because none of the target character sets have an equivalent 
character). 
@param aArrayOfCharacterSets Array of character conversion data objects, 
representing the character sets which comprise a complex character set 
encoding. These are used in sequence to convert the Unicode text. There must 
be at least one character set in this array and no character set may have any 
NULL member data, or a panic occurs. 
@return The number of unconverted characters left at the end of the input 
descriptor (e.g. because aForeign was not long enough to hold all the text), 
or a negative error value, as defined in CCnvCharacterSetConverter::TError. */
EXPORT_C TInt CnvUtilities::ConvertFromUnicode(
		CCnvCharacterSetConverter::TEndianness aDefaultEndiannessOfForeignCharacters, 
		const TDesC8& aReplacementForUnconvertibleUnicodeCharacters, 
		TDes8& aForeign, 
		const TDesC16& aUnicode, 
		CCnvCharacterSetConverter::TArrayOfAscendingIndices& aIndicesOfUnconvertibleCharacters, 
		const TArray<SCharacterSet>& aArrayOfCharacterSets)
	{
	TUint notUsed;
	return ConvertFromUnicode(aDefaultEndiannessOfForeignCharacters, 
								aReplacementForUnconvertibleUnicodeCharacters, 
								aForeign, 
								aUnicode, 
								aIndicesOfUnconvertibleCharacters, 
								aArrayOfCharacterSets, 
								notUsed, 
								0);
	}

/** Converts Unicode text into a complex foreign character set encoding. This is 
an encoding which cannot be converted simply by a call to 
CCnvCharacterSetConverter::DoConvertFromUnicode(). It may be modal (e.g. JIS) 
or non-modal (e.g. Shift-JIS).

The Unicode text specified in aUnicode is converted using the array of conversion 
data objects (aArrayOfCharacterSets) provided by the plug-in for the complex 
character set encoding and the converted text is returned in aForeign. The 
function can either append to aForeign or overwrite its contents (if any).

Unlike CCnvCharacterSetConverter::DoConvertFromUnicode(), multiple character 
sets can be specified. aUnicode is converted using the first character conversion 
data object in the array. When a character is found which cannot be converted 
using that data, each character set in the array is tried in turn. If it cannot 
be converted using any object in the array, the index of the character is 
appended to aIndicesOfUnconvertibleCharacters and the character is replaced 
by aReplacementForUnconvertibleUnicodeCharacters.

If it can be converted using another object in the array, that object is used 
to convert all subsequent characters until another unconvertible character 
is found.

@param aDefaultEndiannessOfForeignCharacters The default endian-ness to use 
when writing the characters in the foreign character set. If an endian-ness 
for foreign characters is specified in the current conversion data object, 
then that is used instead and the value of 
aDefaultEndiannessOfForeignCharacters is ignored. 
@param aReplacementForUnconvertibleUnicodeCharacters The single character (one 
or more byte values) which is used to replace unconvertible characters. 
@param aForeign On return, contains the converted text in the non-Unicode 
character set. This may already contain some text. If it does, and if 
aInputConversionFlags specifies EInputConversionFlagAppend, then the converted 
text is appended to this descriptor. 
@param aUnicode The source Unicode text to be converted. 
@param aIndicesOfUnconvertibleCharacters On return, holds an ascending array 
of the indices of each Unicode character in the source text which could not 
be converted (because none of the target character sets have an equivalent 
character). 
@param aArrayOfCharacterSets Array of character set data objects. These are 
used in sequence to convert the Unicode text. There must be at least one 
character set in this array and no character set may have any NULL member 
data, or a panic occurs. 
@param aOutputConversionFlags If the input descriptor ended in a truncated 
sequence, e.g. the first half only of a Unicode surrogate pair, this returns 
with the EOutputConversionFlagInputIsTruncated flag set. 
@param aInputConversionFlags Specify 
CCnvCharacterSetConverter::EInputConversionFlagAppend to append the text to 
aForeign. Specify CCnvCharacterSetConverter::EInputConversionFlagAllowTruncatedInputNotEvenPartlyConsumable 
to prevent the function from returning the error-code EErrorIllFormedInput 
when the input descriptor consists of nothing but a truncated sequence. The 
CCnvCharacterSetConverter::EInputConversionFlagStopAtFirstUnconvertibleCharacter 
flag must not be set, otherwise a panic occurs. 
@return The number of unconverted characters left at the end of the input descriptor 
(e.g. because aForeign was not long enough to hold all the text), or a negative 
error value, as defined in CCnvCharacterSetConverter::TError. */
EXPORT_C TInt CnvUtilities::ConvertFromUnicode(
				CCnvCharacterSetConverter::TEndianness aDefaultEndiannessOfForeignCharacters, 
				const TDesC8& aReplacementForUnconvertibleUnicodeCharacters, 
				TDes8& aForeign, 
				const TDesC16& aUnicode, 
				CCnvCharacterSetConverter::TArrayOfAscendingIndices& aIndicesOfUnconvertibleCharacters, 
				const TArray<SCharacterSet>& aArrayOfCharacterSets, 
				TUint& aOutputConversionFlags, 
				TUint aInputConversionFlags)
	{
	__ASSERT_ALWAYS(~aInputConversionFlags&CCnvCharacterSetConverter::EInputConversionFlagStopAtFirstUnconvertibleCharacter, Panic(EPanicBadInputConversionFlags1));
	CheckArrayOfCharacterSets(aArrayOfCharacterSets);
	aOutputConversionFlags=0;
	TUint internalInputConversionFlags=aInputConversionFlags;
	if (~aInputConversionFlags&CCnvCharacterSetConverter::EInputConversionFlagAppend)
		{
		aForeign.SetLength(0);
		internalInputConversionFlags|=CCnvCharacterSetConverter::EInputConversionFlagAppend;
		}
	if (aUnicode.Length()==0)
		{
		return 0;
		}
	if (aForeign.MaxLength()==aForeign.Length()) // relies on the fact that aForeign's length has been set to zero if aInputConversionFlags does not have CCnvCharacterSetConverter::EInputConversionFlagAppend set
		{
		return aUnicode.Length();
		}
	TDes8* foreign=&aForeign;
	TPtr8 dummyForeign(NULL, 0, 0);
	if (aInputConversionFlags&CCnvCharacterSetConverter::EInputConversionFlagMustEndInDefaultCharacterSet)
		{
		TInt dummyMaximumLength =
			aForeign.MaxLength() - aArrayOfCharacterSets[KDefaultCharacterSet].iEscapeSequence->Length();
		__ASSERT_ALWAYS(dummyMaximumLength >= 0, 
			Panic(EPanicOutputDescriptorTooShortEvenToHoldEscapeSequenceToDefaultCharacterSet));
		dummyForeign.Set(const_cast <TUint8*> (aForeign.Ptr()),
						 aForeign.Length(), 
						 dummyMaximumLength);
		foreign=&dummyForeign;
		}
	const TInt numberOfCharacterSets=aArrayOfCharacterSets.Count();
	TInt numberOfUnicodeElementsConsumed=0;
	internalInputConversionFlags|=CCnvCharacterSetConverter::EInputConversionFlagStopAtFirstUnconvertibleCharacter; // this is not just an optimization - it ensures that "foreign" doesn't get filled up too much each time CCnvCharacterSetConverter::DoConvertFromUnicode is called
    TInt previousCharacterSet = aInputConversionFlags&CCnvCharacterSetConverter::EInputConversionFlagAssumeStartInDefaultCharacterSet?
        KDefaultCharacterSet : KNoPreviousCharacterSet;
	FOREVER
		{
		for (TInt presentCharacterSet=KDefaultCharacterSet;;)
			{
			__ASSERT_DEBUG(numberOfUnicodeElementsConsumed<=aUnicode.Length(), Panic(EPanicBadNumberOfUnicodeElementsConsumed));
			if (numberOfUnicodeElementsConsumed>=aUnicode.Length())
				{
				goto end;
				}
			const SCharacterSet& characterSet=aArrayOfCharacterSets[presentCharacterSet];
			const TInt oldNumberOfBytesInForeign=foreign->Length();
			if (numberOfUnicodeElementsConsumed>0)
				{
				internalInputConversionFlags|=CCnvCharacterSetConverter::EInputConversionFlagAllowTruncatedInputNotEvenPartlyConsumable;
				}
			CCnvCharacterSetConverter::TArrayOfAscendingIndices indicesOfUnconvertibleCharacters;
			const TInt returnValue=CCnvCharacterSetConverter::DoConvertFromUnicode(*characterSet.iConversionData, aDefaultEndiannessOfForeignCharacters, KNullDesC8, *foreign, aUnicode.Mid(numberOfUnicodeElementsConsumed), indicesOfUnconvertibleCharacters, aOutputConversionFlags, internalInputConversionFlags);
			if (returnValue<0)
				{
				return returnValue; // this is an error-code
				}
			__ASSERT_DEBUG(foreign->Length()>=oldNumberOfBytesInForeign, Panic(EPanicAppendFlagViolated));
			TInt indexOfFirstUnconvertibleCharacter;
			if (indicesOfUnconvertibleCharacters.NumberOfIndices()==0)
				{
				indexOfFirstUnconvertibleCharacter=-1;
				numberOfUnicodeElementsConsumed=aUnicode.Length()-returnValue;
				}
			else
				{
				indexOfFirstUnconvertibleCharacter=indicesOfUnconvertibleCharacters[0];
				numberOfUnicodeElementsConsumed+=indexOfFirstUnconvertibleCharacter;
				__ASSERT_DEBUG(numberOfUnicodeElementsConsumed+LengthOfUnicodeCharacter(aUnicode, numberOfUnicodeElementsConsumed)==aUnicode.Length()-returnValue, Panic(EPanicBadNumberOfUnicodeCharactersConverted));
				}
			if (indexOfFirstUnconvertibleCharacter!=0) // if at least one Unicode character at the start of CCnvCharacterSetConverter::DoConvertFromUnicode's input descriptor was convertible...
				{
				TBool gotoEnd = EFalse;
				if (foreign->Length()>oldNumberOfBytesInForeign)
					{
					TInt numberOfCharactersThatDroppedOut=0;
                     // Insert an escape sequence if this character set is different from the last one.
                     if (presentCharacterSet  != previousCharacterSet)
                        {
                        // Insert escape sequence (if requred) in front of the last encoded run of text.
                        // Note that this may cause some characters to drop out at the end.
                        (*characterSet.iConvertFromIntermediateBufferInPlace)(oldNumberOfBytesInForeign, *foreign, numberOfCharactersThatDroppedOut);
                        if (oldNumberOfBytesInForeign < foreign->Length())
                			previousCharacterSet = presentCharacterSet;
                        }
					numberOfUnicodeElementsConsumed-=numberOfCharactersThatDroppedOut;
					if (numberOfCharactersThatDroppedOut>0 )// if "foreign" has been filled to as much as it will hold...
						{
						gotoEnd = ETrue;
						}
					}
				if (indexOfFirstUnconvertibleCharacter<0) // if we've successfully converted up to the end of aUnicode (using *characterSet.iConversionData)...
					{
					gotoEnd = ETrue;
					}
				if (gotoEnd)
					{
					if ( aInputConversionFlags&CCnvCharacterSetConverter::EInputConversionFlagMustEndInDefaultCharacterSet
					    && previousCharacterSet != KDefaultCharacterSet
					    && previousCharacterSet != KNoPreviousCharacterSet)
					    {
					    aForeign.SetLength(foreign->Length());
    				    aForeign.Append(*aArrayOfCharacterSets[KDefaultCharacterSet].iEscapeSequence);
    				    foreign=NULL;
					    }
					goto end;
					}
				break;
				}
			__ASSERT_DEBUG(presentCharacterSet<numberOfCharacterSets, Panic(EPanicLoopCounterOverRun1));
			++presentCharacterSet;
			if (presentCharacterSet>=numberOfCharacterSets)
				{
				if ((foreign->MaxLength()-foreign->Length()<aReplacementForUnconvertibleUnicodeCharacters.Length()) ||
					(aIndicesOfUnconvertibleCharacters.AppendIndex(numberOfUnicodeElementsConsumed)!=CCnvCharacterSetConverter::TArrayOfAscendingIndices::EAppendSuccessful)) // the tests must be done in this order as AppendIndex must only be called if there is room for aReplacementForUnconvertibleUnicodeCharacters
					{
					goto end;
					}
				numberOfUnicodeElementsConsumed+=LengthOfUnicodeCharacter(aUnicode, numberOfUnicodeElementsConsumed);
				foreign->Append(aReplacementForUnconvertibleUnicodeCharacters);
				break;
				}
			}
		}
end:
	if (foreign!=NULL)
		{
		aForeign.SetLength(foreign->Length());
		foreign=NULL;
		}
	if ((numberOfUnicodeElementsConsumed==0) && (aOutputConversionFlags&CCnvCharacterSetConverter::EOutputConversionFlagInputIsTruncated) && (~aInputConversionFlags&CCnvCharacterSetConverter::EInputConversionFlagAllowTruncatedInputNotEvenPartlyConsumable))
		{
		return CCnvCharacterSetConverter::EErrorIllFormedInput;
		}
	return aUnicode.Length()-numberOfUnicodeElementsConsumed;
	}


/** Inserts an escape sequence into the descriptor.

This function is provided to help in the implementation of
ConvertFromUnicode() for modal character set encodings.
Each SCharacterSet object in the array passed to
ConvertFromUnicode() must have its
iConvertFromIntermediateBufferInPlace member assigned. To
do this for a modal character set encoding, implement a function whose
signature matches that of FConvertFromIntermediateBufferInPlace 
and which calls this function, passing all arguments unchanged, and 
specifying the character set's escape sequence and the number of bytes per 
character.

@param aStartPositionInDescriptor The byte position in aDescriptor at which 
the escape sequence is inserted. If the character set uses more than one byte 
per character, this position must be the start of a character, otherwise a 
panic occurs. 
@param aDescriptor The descriptor into which the escape sequence is inserted. 
@param aNumberOfCharactersThatDroppedOut The escape sequence is inserted into 
the start of aDescriptor and any characters that need to drop out to make 
room for the escape sequence (because the descriptor's maximum length was 
not long enough) drop out from the end of the buffer. This parameter indicates 
the number of characters that needed to drop out.
@param aEscapeSequence The escape sequence for the character set. 
@param aNumberOfBytesPerCharacter The number of bytes per character. */
EXPORT_C void CnvUtilities::ConvertFromIntermediateBufferInPlace(
					TInt aStartPositionInDescriptor, 
					TDes8& aDescriptor, 
					TInt& aNumberOfCharactersThatDroppedOut, 
					const TDesC8& aEscapeSequence, 
					TInt aNumberOfBytesPerCharacter)
	{
	const TInt lengthOfDescriptor=aDescriptor.Length();
	__ASSERT_ALWAYS((lengthOfDescriptor-aStartPositionInDescriptor)%aNumberOfBytesPerCharacter==0, Panic(EPanicDescriptorNotWholeNumberOfCharacters1));
	aNumberOfCharactersThatDroppedOut=(Max(0, aEscapeSequence.Length()-(aDescriptor.MaxLength()-lengthOfDescriptor))+(aNumberOfBytesPerCharacter-1))/aNumberOfBytesPerCharacter;
	const TInt lengthOfRunInCharacters=(lengthOfDescriptor-aStartPositionInDescriptor)/aNumberOfBytesPerCharacter;
	if (aNumberOfCharactersThatDroppedOut>=lengthOfRunInCharacters) // ">=" is correct (rather than ">") as if there's only room for the escape sequence we don't want to have it in the descriptor
		{
		aNumberOfCharactersThatDroppedOut=lengthOfRunInCharacters;
		aDescriptor.SetLength(aStartPositionInDescriptor);
		}
	else
		{
		aDescriptor.SetLength(lengthOfDescriptor-(aNumberOfCharactersThatDroppedOut*aNumberOfBytesPerCharacter));
		aDescriptor.Insert(aStartPositionInDescriptor, aEscapeSequence);
		}
	}


/**  Converts text from a modal foreign character set encoding into Unicode.

The non-Unicode text specified in aForeign is converted using 
the array of character set conversion objects (aArrayOfStates) 
provided by the plug-in, and the converted text is returned in 
aUnicode. The function can either append to aUnicode 
or overwrite its contents (if any), depending on the input conversion flags 
specified. The first element in aArrayOfStates is taken to be 
the default mode (i.e. the mode to assume by default if there is no preceding 
escape sequence).
 
@param aDefaultEndiannessOfForeignCharacters The default endian-ness of the 
foreign characters. If an endian-ness for foreign characters is specified 
in the conversion data, then that is used instead and the value of 
aDefaultEndiannessOfForeignCharacters is ignored. 
@param aUnicode On return, contains the text converted into Unicode. 
@param aForeign The non-Unicode source text to be converted. 
@param aState Used to store a modal character set encoding's current mode across 
multiple calls to ConvertToUnicode() on the same input descriptor. This argument 
should be passed the same object as passed to the plug-in's ConvertToUnicode() 
exported function.
@param aNumberOfUnconvertibleCharacters On return, contains the number of 
characters in aForeign which were not converted. Characters which cannot be 
converted are output as Unicode replacement characters (0xfffd). 
@param aIndexOfFirstByteOfFirstUnconvertibleCharacter On return, the index 
of the first byte of the first unconvertible character. For instance if the 
first character in the input descriptor (aForeign) could not be converted, 
then this parameter is set to the first byte of that character, i.e. zero. 
A negative value is returned if all the characters were converted. 
@param aArrayOfStates Array of character set conversion data objects, and their 
escape sequences ("modes"). There must be one or more modes in this array, 
none of the modes can have any NULL member data, and each mode's escape sequence 
must begin with KControlCharacterEscape (0x1b) or a panic occurs. 
@return The number of unconverted bytes left at the end of the input descriptor, 
or a negative error value, as defined in TError. */
EXPORT_C TInt CnvUtilities::ConvertToUnicodeFromModalForeign(
					CCnvCharacterSetConverter::TEndianness aDefaultEndiannessOfForeignCharacters, 
					TDes16& aUnicode, 
					const TDesC8& aForeign, 
					TInt& aState, 
					TInt& aNumberOfUnconvertibleCharacters, 
					TInt& aIndexOfFirstByteOfFirstUnconvertibleCharacter, 
					const TArray<SState>& aArrayOfStates)
	{
	TUint notUsed;
	return ConvertToUnicodeFromModalForeign(aDefaultEndiannessOfForeignCharacters, 
											aUnicode, 
											aForeign, 
											aState, 
											aNumberOfUnconvertibleCharacters, 
											aIndexOfFirstByteOfFirstUnconvertibleCharacter, 
											aArrayOfStates, 
											notUsed, 
											0);
	}

/** @param aDefaultEndiannessOfForeignCharacters The default endian-ness for 
the foreign characters. If an endian-ness for foreign characters is specified 
in the conversion data, then that is used instead and the value of 
aDefaultEndiannessOfForeignCharacters is ignored. 
@param aUnicode On return, contains the text converted into Unicode. 
@param aForeign The non-Unicode source text to be converted. 
@param aState Used to store a modal character set encoding's current mode 
across multiple calls to ConvertToUnicode() on the same input descriptor. This 
argument should be passed the same object as passed to the plug-in's 
ConvertToUnicode() exported function. 
@param aNumberOfUnconvertibleCharacters On return, contains the number of 
characters in aForeign which were not converted. Characters which cannot be 
converted are output as Unicode replacement characters (0xfffd). 
@param aIndexOfFirstByteOfFirstUnconvertibleCharacter On return, the index 
of the first byte of the first unconvertible character. For instance if the 
first character in the input descriptor (aForeign) could not be converted, 
then this parameter is set to the first byte of that character, i.e. zero. 
A negative value is returned if all the characters were converted. 
@param aArrayOfStates Array of character set conversion data objects, and their 
escape sequences. There must be one or more modes in this array, none of the 
modes can have any NULL member data, and each mode's escape sequence must 
begin with KControlCharacterEscape (0x1b) or a panic occurs. 
@param aOutputConversionFlags If the input descriptor ended in a truncated 
sequence, e.g. an incomplete multi-byte character, aOutputConversionFlags 
returns with the EOutputConversionFlagInputIsTruncated flag set. 
@param aInputConversionFlags Specify 
CCnvCharacterSetConverter::EInputConversionFlagAppend to append the text to 
aUnicode. Specify EInputConversionFlagAllowTruncatedInputNotEvenPartlyConsumable 
to prevent the function from returning the error-code EErrorIllFormedInput 
when the input descriptor consists of nothing but a truncated sequence. The 
CCnvCharacterSetConverter::EInputConversionFlagStopAtFirstUnconvertibleCharacter 
flag must not be set, otherwise a panic occurs. 
@return The number of unconverted bytes left at the end of the input descriptor, 
or a negative error value, as defined in TError. */
EXPORT_C TInt CnvUtilities::ConvertToUnicodeFromModalForeign(
								CCnvCharacterSetConverter::TEndianness aDefaultEndiannessOfForeignCharacters, 
								TDes16& aUnicode, 
								const TDesC8& aForeign, 
								TInt& aState, 
								TInt& aNumberOfUnconvertibleCharacters, 
								TInt& aIndexOfFirstByteOfFirstUnconvertibleCharacter, 
								const TArray<SState>& aArrayOfStates, 
								TUint& aOutputConversionFlags, 
								TUint aInputConversionFlags)
	{
	__ASSERT_ALWAYS(~aInputConversionFlags&CCnvCharacterSetConverter::EInputConversionFlagStopAtFirstUnconvertibleCharacter, Panic(EPanicBadInputConversionFlags2));
	CheckArrayOfStates(aArrayOfStates);
	aNumberOfUnconvertibleCharacters=0;
	aIndexOfFirstByteOfFirstUnconvertibleCharacter=-1;
	aOutputConversionFlags=0;
	TUint internalInputConversionFlags=aInputConversionFlags;
	if (~aInputConversionFlags&CCnvCharacterSetConverter::EInputConversionFlagAppend)
		{
		aUnicode.SetLength(0);
		internalInputConversionFlags|=CCnvCharacterSetConverter::EInputConversionFlagAppend;
		}
	if (aForeign.Length()==0)
		{
		return 0;
		}
	if (aUnicode.MaxLength()==aUnicode.Length()) // relies on the fact that aUnicode's length has been set to zero if aInputConversionFlags does not have CCnvCharacterSetConverter::EInputConversionFlagAppend set
		{
		return aForeign.Length();
		}
	TPtrC8 remainderOfForeign(aForeign);
	TPtrC8 homogeneousRun;
	TInt numberOfForeignBytesConsumed=0;
	const SCnvConversionData* conversionData = NULL;
	const TInt startOfNextEscapeSequence=aForeign.Locate(KControlCharacterEscape);
	if (startOfNextEscapeSequence!=0) // if aForeign doesn't start with an escape sequence...
		{
		conversionData=(aState!=CCnvCharacterSetConverter::KStateDefault)? REINTERPRET_CAST(const SCnvConversionData*, aState): aArrayOfStates[0].iConversionData;
		if (startOfNextEscapeSequence==KErrNotFound)
			{
			homogeneousRun.Set(remainderOfForeign);
			remainderOfForeign.Set(NULL, 0);
			}
		else
			{
			__ASSERT_DEBUG(startOfNextEscapeSequence>0, Panic(EPanicBadStartOfNextEscapeSequence));
			homogeneousRun.Set(remainderOfForeign.Left(startOfNextEscapeSequence));
			remainderOfForeign.Set(remainderOfForeign.Mid(startOfNextEscapeSequence));
			}
		goto handleHomogeneousRun;
		}
	FOREVER
		{
		if (!NextHomogeneousForeignRun(conversionData, numberOfForeignBytesConsumed, homogeneousRun, remainderOfForeign, aArrayOfStates, aOutputConversionFlags))
			{
			goto end;
			}
handleHomogeneousRun:
		if (conversionData==NULL)
			{
			return CCnvCharacterSetConverter::EErrorIllFormedInput;
			}
		TInt numberOfUnconvertibleCharacters;
		TInt indexOfFirstByteOfFirstUnconvertibleCharacter;
		const TInt returnValue=CCnvCharacterSetConverter::DoConvertToUnicode(*conversionData, aDefaultEndiannessOfForeignCharacters, aUnicode, homogeneousRun, numberOfUnconvertibleCharacters, indexOfFirstByteOfFirstUnconvertibleCharacter, aOutputConversionFlags, internalInputConversionFlags);
		if (returnValue<0)
			{
			return returnValue; // this is an error-code
			}
		if (numberOfUnconvertibleCharacters>0)
			{
			if (aNumberOfUnconvertibleCharacters==0)
				{
				aIndexOfFirstByteOfFirstUnconvertibleCharacter=numberOfForeignBytesConsumed+indexOfFirstByteOfFirstUnconvertibleCharacter;
				}
			aNumberOfUnconvertibleCharacters+=numberOfUnconvertibleCharacters;
			}
		numberOfForeignBytesConsumed+=homogeneousRun.Length();
		if (returnValue>0)
			{
			numberOfForeignBytesConsumed-=returnValue;
			goto end;
			}
		if (numberOfForeignBytesConsumed>0)
			{
			internalInputConversionFlags|=CCnvCharacterSetConverter::EInputConversionFlagAllowTruncatedInputNotEvenPartlyConsumable;
			}
		__ASSERT_DEBUG(remainderOfForeign==aForeign.Mid(numberOfForeignBytesConsumed), Panic(EPanicInconsistentNumberOfForeignBytesRemaining));
		}
end:
	if ((numberOfForeignBytesConsumed==0) && (aOutputConversionFlags&CCnvCharacterSetConverter::EOutputConversionFlagInputIsTruncated) && (~aInputConversionFlags&CCnvCharacterSetConverter::EInputConversionFlagAllowTruncatedInputNotEvenPartlyConsumable))
		{
		return CCnvCharacterSetConverter::EErrorIllFormedInput;
		}
	aState=REINTERPRET_CAST(TInt, conversionData);
	return aForeign.Length()-numberOfForeignBytesConsumed;
	}


/**  Converts text from a non-modal complex character set encoding (e.g. 
Shift-JIS or EUC-JP) into Unicode.The non-Unicode text specified in
aForeign is converted using the array of character set
conversion methods (aArrayOfMethods) provided by the
plug-in, and the converted text is returned in aUnicode.
Overwrites the contents, if any, of aUnicode.

@param aDefaultEndiannessOfForeignCharacters The default endian-ness of the 
foreign characters. If an endian-ness for foreign characters is specified 
in the conversion data, then that is used instead and the value of 
aDefaultEndiannessOfForeignCharacters is ignored. 
@param aUnicode On return, contains the text converted into Unicode. 
@param aForeign The non-Unicode source text to be converted. 
@param aNumberOfUnconvertibleCharacters On return, contains the number of 
characters in aForeign which were not converted. Characters which cannot be 
converted are output as Unicode replacement characters (0xfffd). 
@param aIndexOfFirstByteOfFirstUnconvertibleCharacter On return, the index 
of the first byte of the first unconvertible character. For instance if the 
first character in the input descriptor (aForeign) could not be converted, 
then this parameter is set to the first byte of that character, i.e. zero. 
A negative value is returned if all the characters were converted. 
@param aArrayOfMethods Array of conversion methods. There must be one or more 
methods in this array and none of the methods in the array can have any NULL 
member data or a panic occurs. 
@return The number of unconverted bytes left at the end of the input descriptor, 
or a negative error value, as defined in TError. */
EXPORT_C TInt CnvUtilities::ConvertToUnicodeFromHeterogeneousForeign(
					CCnvCharacterSetConverter::TEndianness aDefaultEndiannessOfForeignCharacters, 
					TDes16& aUnicode, 
					const TDesC8& aForeign, 
					TInt& aNumberOfUnconvertibleCharacters, 
					TInt& aIndexOfFirstByteOfFirstUnconvertibleCharacter, 
					const TArray<SMethod>& aArrayOfMethods)
	{
	TUint notUsed;
	return ConvertToUnicodeFromHeterogeneousForeign(
				aDefaultEndiannessOfForeignCharacters, 
				aUnicode, 
				aForeign, 
				aNumberOfUnconvertibleCharacters, 
				aIndexOfFirstByteOfFirstUnconvertibleCharacter, 
				aArrayOfMethods, 
				notUsed, 
				0);
	}

/** @param aDefaultEndiannessOfForeignCharacters The default endian-ness for the 
foreign characters. If an endian-ness for foreign characters is specified 
in the conversion data, then that is used instead and the value of 
aDefaultEndiannessOfForeignCharacters is ignored. 
@param aUnicode On return, contains the text converted into Unicode. 
@param aForeign The non-Unicode source text to be converted. 
@param aNumberOfUnconvertibleCharacters On return, contains the number of 
characters in aForeign which were not converted. Characters which cannot be 
converted are output as Unicode replacement characters (0xfffd). 
@param aIndexOfFirstByteOfFirstUnconvertibleCharacter On return, the index 
of the first byte of the first unconvertible character. For instance if the 
first character in the input descriptor (aForeign) could not be converted, 
then this parameter is set to the first byte of that character, i.e. zero. 
A negative value is returned if all the characters were converted. 
@param aArrayOfMethods Array of conversion methods. There must be one or more 
methods in this array and none of the methods in the array can have any NULL 
member data or a panic occurs. 
@param aOutputConversionFlags If the input descriptor ended in a truncated 
sequence, e.g. an incomplete multi-byte character, aOutputConversionFlags 
returns with the EOutputConversionFlagInputIsTruncated flag set. 
@param aInputConversionFlags Specify 
CCnvCharacterSetConverter::EInputConversionFlagAppend to append the text to 
aUnicode. Specify EInputConversionFlagAllowTruncatedInputNotEvenPartlyConsumable 
to prevent the function from returning the error-code EErrorIllFormedInput 
when the input descriptor consists of nothing but a truncated sequence. The 
CCnvCharacterSetConverter::EInputConversionFlagStopAtFirstUnconvertibleCharacter 
flag must not be set, otherwise a panic occurs. 
@return The number of unconverted bytes left at the end of the input descriptor, 
or a negative error value, as defined in TError. */
EXPORT_C TInt CnvUtilities::ConvertToUnicodeFromHeterogeneousForeign(
						CCnvCharacterSetConverter::TEndianness aDefaultEndiannessOfForeignCharacters, 
						TDes16& aUnicode, 
						const TDesC8& aForeign, 
						TInt& aNumberOfUnconvertibleCharacters, 
						TInt& aIndexOfFirstByteOfFirstUnconvertibleCharacter, 
						const TArray<SMethod>& aArrayOfMethods, 
						TUint& aOutputConversionFlags, 
						TUint aInputConversionFlags)
	{
	__ASSERT_ALWAYS(~aInputConversionFlags&CCnvCharacterSetConverter::EInputConversionFlagStopAtFirstUnconvertibleCharacter, Panic(EPanicBadInputConversionFlags3));
	CheckArrayOfMethods(aArrayOfMethods);
	aNumberOfUnconvertibleCharacters=0;
	aIndexOfFirstByteOfFirstUnconvertibleCharacter=-1;
	aOutputConversionFlags=0;
	TUint internalInputConversionFlags=aInputConversionFlags;
	if (~aInputConversionFlags&CCnvCharacterSetConverter::EInputConversionFlagAppend)
		{
		aUnicode.SetLength(0);
		internalInputConversionFlags|=CCnvCharacterSetConverter::EInputConversionFlagAppend;
		}
	if (aForeign.Length()==0)
		{
		return 0;
		}
	if (aUnicode.MaxLength()==aUnicode.Length()) // relies on the fact that aUnicode's length has been set to zero if aInputConversionFlags does not have CCnvCharacterSetConverter::EInputConversionFlagAppend set
		{
		return aForeign.Length();
		}
	const TInt numberOfMethods=aArrayOfMethods.Count();
	TPtrC8 remainderOfForeign(aForeign);
	TInt numberOfForeignBytesConsumed=0;
	FOREVER
		{
		TInt lengthOfRunToConvert=0;
		const SMethod* method=NULL;
		for (TInt i=0;;)
			{
			method=&aArrayOfMethods[i];
			lengthOfRunToConvert=(*method->iNumberOfBytesAbleToConvert)(remainderOfForeign);
			if (lengthOfRunToConvert<0)
				{
				return lengthOfRunToConvert; // this is an error-code
				}
			if (lengthOfRunToConvert>0)
				{
				break;
				}
			__ASSERT_DEBUG(i<numberOfMethods, Panic(EPanicLoopCounterOverRun2));
			++i;
			if (i>=numberOfMethods)
				{
				aOutputConversionFlags|=CCnvCharacterSetConverter::EOutputConversionFlagInputIsTruncated;
				goto end;
				}
			}
		__ASSERT_DEBUG(lengthOfRunToConvert>0, Panic(EPanicBadLengthOfRunToConvert1));
		__ASSERT_DEBUG(method!=NULL, Panic(EPanicBadMethodPointer));
		TBuf8<KMaximumLengthOfIntermediateBuffer> intermediateBuffer;
		const TInt maximumUsableLengthOfIntermediateBuffer=ReduceToNearestMultipleOf(KMaximumLengthOfIntermediateBuffer, method->iNumberOfBytesPerCharacter);
		FOREVER
			{
			const TInt numberOfForeignBytesConsumedThisTime=Min(lengthOfRunToConvert, maximumUsableLengthOfIntermediateBuffer);
			intermediateBuffer=remainderOfForeign.Left(numberOfForeignBytesConsumedThisTime);
			__ASSERT_DEBUG((numberOfForeignBytesConsumedThisTime%method->iNumberOfBytesPerCharacter)==0, Panic(EPanicDescriptorNotWholeNumberOfCharacters2));
			(*method->iConvertToIntermediateBufferInPlace)(intermediateBuffer);
			__ASSERT_DEBUG((intermediateBuffer.Length()%method->iNumberOfCoreBytesPerCharacter)==0, Panic(EPanicDescriptorNotWholeNumberOfCharacters3));
			__ASSERT_DEBUG((intermediateBuffer.Length()/method->iNumberOfCoreBytesPerCharacter)*method->iNumberOfBytesPerCharacter==numberOfForeignBytesConsumedThisTime, Panic(EPanicBadMethodData1));
			TInt numberOfUnconvertibleCharacters;
			TInt indexOfFirstByteOfFirstUnconvertibleCharacter;
			const TInt returnValue=CCnvCharacterSetConverter::DoConvertToUnicode(*method->iConversionData, aDefaultEndiannessOfForeignCharacters, aUnicode, intermediateBuffer, numberOfUnconvertibleCharacters, indexOfFirstByteOfFirstUnconvertibleCharacter, aOutputConversionFlags, internalInputConversionFlags);
			if (returnValue<0)
				{
				return returnValue; // this is an error-code
				}
			if (numberOfUnconvertibleCharacters>0)
				{
				if (aNumberOfUnconvertibleCharacters==0)
					{
					aIndexOfFirstByteOfFirstUnconvertibleCharacter=numberOfForeignBytesConsumed+indexOfFirstByteOfFirstUnconvertibleCharacter;
					}
				aNumberOfUnconvertibleCharacters+=numberOfUnconvertibleCharacters;
				}
			numberOfForeignBytesConsumed+=numberOfForeignBytesConsumedThisTime;
			if (returnValue>0)
				{
				__ASSERT_DEBUG((returnValue%method->iNumberOfCoreBytesPerCharacter)==0, Panic(EPanicDescriptorNotWholeNumberOfCharacters4));
				numberOfForeignBytesConsumed-=(returnValue/method->iNumberOfCoreBytesPerCharacter)*method->iNumberOfBytesPerCharacter;
				goto end;
				}
			if (numberOfForeignBytesConsumed>0)
				{
				internalInputConversionFlags|=CCnvCharacterSetConverter::EInputConversionFlagAllowTruncatedInputNotEvenPartlyConsumable;
				}
			remainderOfForeign.Set(aForeign.Mid(numberOfForeignBytesConsumed));
			lengthOfRunToConvert-=numberOfForeignBytesConsumedThisTime;
			__ASSERT_DEBUG(lengthOfRunToConvert>=0, Panic(EPanicBadLengthOfRunToConvert2));
			if (lengthOfRunToConvert<=0)
				{
				break;
				}
			}
		}
end:
	if ((numberOfForeignBytesConsumed==0) && (aOutputConversionFlags&CCnvCharacterSetConverter::EOutputConversionFlagInputIsTruncated) && (~aInputConversionFlags&CCnvCharacterSetConverter::EInputConversionFlagAllowTruncatedInputNotEvenPartlyConsumable))
		{
		return CCnvCharacterSetConverter::EErrorIllFormedInput;
		}
	return aForeign.Length()-numberOfForeignBytesConsumed;
	}

void CnvUtilities::CheckArrayOfCharacterSets(const TArray<SCharacterSet>& aArrayOfCharacterSets)
	{
	const TInt numberOfCharacterSets=aArrayOfCharacterSets.Count();
	__ASSERT_ALWAYS(numberOfCharacterSets>0, Panic(EPanicBadNumberOfCharacterSets));
	for (TInt i=0; i<numberOfCharacterSets; ++i)
		{
		const SCharacterSet& characterSet=aArrayOfCharacterSets[i];
		__ASSERT_ALWAYS(characterSet.iConversionData!=NULL, Panic(EPanicBadConversionDataPointer1));
		__ASSERT_ALWAYS(characterSet.iConvertFromIntermediateBufferInPlace!=NULL, Panic(EPanicBadFunctionPointer1));
		__ASSERT_ALWAYS(characterSet.iEscapeSequence!=NULL, Panic(EPanicBadEscapeSequencePointer1));
		}
	}

void CnvUtilities::CheckArrayOfStates(const TArray<SState>& aArrayOfStates)
	{
	const TInt numberOfStates=aArrayOfStates.Count();
	__ASSERT_ALWAYS(numberOfStates>0, Panic(EPanicBadNumberOfStates));
	for (TInt i=0; i<numberOfStates; ++i)
		{
		const SState& state=aArrayOfStates[i];
		__ASSERT_ALWAYS(state.iEscapeSequence!=NULL, Panic(EPanicBadEscapeSequencePointer2));
		__ASSERT_ALWAYS((*state.iEscapeSequence)[0]==KControlCharacterEscape, Panic(EPanicBadEscapeSequenceStart));
		__ASSERT_ALWAYS(state.iConversionData!=NULL, Panic(EPanicBadConversionDataPointer2));
		}
	}

void CnvUtilities::CheckArrayOfMethods(const TArray<SMethod>& aArrayOfMethods)
	{
	const TInt numberOfMethods=aArrayOfMethods.Count();
	__ASSERT_ALWAYS(numberOfMethods>0, Panic(EPanicBadNumberOfMethods));
	for (TInt i=0; i<numberOfMethods; ++i)
		{
		const SMethod& method=aArrayOfMethods[i];
		__ASSERT_ALWAYS(method.iNumberOfBytesAbleToConvert!=NULL, Panic(EPanicBadFunctionPointer2));
		__ASSERT_ALWAYS(method.iConvertToIntermediateBufferInPlace!=NULL, Panic(EPanicBadFunctionPointer3));
		__ASSERT_ALWAYS(method.iConversionData!=NULL, Panic(EPanicBadConversionDataPointer3));
		__ASSERT_ALWAYS(method.iNumberOfBytesPerCharacter>0, Panic(EPanicBadMethodData2));
		__ASSERT_ALWAYS(method.iNumberOfCoreBytesPerCharacter>0, Panic(EPanicBadMethodData3));
		__ASSERT_ALWAYS(method.iNumberOfCoreBytesPerCharacter<=method.iNumberOfBytesPerCharacter, Panic(EPanicBadMethodData4));
		}
	}

TInt CnvUtilities::LengthOfUnicodeCharacter(const TDesC16& aUnicode, TInt aIndex)
	{
	const TUint unicodeCharacter=aUnicode[aIndex];
	if ((unicodeCharacter>=0xd800) && (unicodeCharacter<=0xdbff)) // if the unicode character is the first half of a surrogate-pair...
		{
		__ASSERT_DEBUG(aIndex+1<aUnicode.Length(), Panic(EPanicBadSurrogatePair1));
#if defined(_DEBUG)
		const TUint secondHalfOfSurrogatePair=aUnicode[aIndex+1];
#endif
		__ASSERT_DEBUG((secondHalfOfSurrogatePair>=0xdc00) && (secondHalfOfSurrogatePair<=0xdfff), Panic(EPanicBadSurrogatePair2)); // this can be asserted as CCnvCharacterSetConverter::DoConvertFromUnicode should have returned an error value if this was a bad surrogate pair
		return 2;
		}
	return 1;
	}

TBool CnvUtilities::NextHomogeneousForeignRun(const SCnvConversionData*& aConversionData, TInt& aNumberOfForeignBytesConsumed, TPtrC8& aHomogeneousRun, TPtrC8& aRemainderOfForeign, const TArray<SState>& aArrayOfStates, TUint& aOutputConversionFlags)
	{
	__ASSERT_DEBUG((aRemainderOfForeign.Length()==0) || (aRemainderOfForeign[0]==KControlCharacterEscape), Panic(EPanicBadRemainderOfForeign));
	FOREVER
		{
		if (aRemainderOfForeign.Length()==0)
			{
			return EFalse;
			}
		const TInt numberOfStates=aArrayOfStates.Count();
		TInt i;
		for (i=0; i<numberOfStates; ++i)
			{
			const SState& state=aArrayOfStates[i];
			if (MatchesEscapeSequence(aNumberOfForeignBytesConsumed, aHomogeneousRun, aRemainderOfForeign, *state.iEscapeSequence))
				{
				aConversionData=state.iConversionData;
				goto foundState;
				}
			}
		for (i=0; i<numberOfStates; ++i)
			{
			if (IsStartOf(aRemainderOfForeign, *aArrayOfStates[i].iEscapeSequence))
				{
				// aRemainderOfForeign ends with a truncated escape sequence, so ConvertToUnicode cannot convert any more
				aOutputConversionFlags|=CCnvCharacterSetConverter::EOutputConversionFlagInputIsTruncated;
				return EFalse;
				}
			}
		// force ConvertToUnicode to return CCnvCharacterSetConverter::EErrorIllFormedInput
		aConversionData=NULL;
		return ETrue;
foundState:
		if (aHomogeneousRun.Length()>0)
			{
			return ETrue;
			}
		}
	}

TBool CnvUtilities::MatchesEscapeSequence(TInt& aNumberOfForeignBytesConsumed, TPtrC8& aHomogeneousRun, TPtrC8& aRemainderOfForeign, const TDesC8& aEscapeSequence)
	{
	const TInt lengthOfEscapeSequence=aEscapeSequence.Length();
	if (IsStartOf(aEscapeSequence, aRemainderOfForeign))
		{
		aRemainderOfForeign.Set(aRemainderOfForeign.Mid(lengthOfEscapeSequence));
		const TInt startOfNextEscapeSequence=aRemainderOfForeign.Locate(KControlCharacterEscape);
		if (startOfNextEscapeSequence==KErrNotFound)
			{
			aHomogeneousRun.Set(aRemainderOfForeign);
			aRemainderOfForeign.Set(NULL, 0);
			}
		else
			{
			aHomogeneousRun.Set(aRemainderOfForeign.Left(startOfNextEscapeSequence));
			aRemainderOfForeign.Set(aRemainderOfForeign.Mid(startOfNextEscapeSequence));
			}
		aNumberOfForeignBytesConsumed+=lengthOfEscapeSequence;
		return ETrue;
		}
	return EFalse;
	}

TBool CnvUtilities::IsStartOf(const TDesC8& aStart, const TDesC8& aPotentiallyLongerDescriptor)
	{
	const TInt lengthOfStart=aStart.Length();
	return (aPotentiallyLongerDescriptor.Length()>=lengthOfStart) && (aPotentiallyLongerDescriptor.Left(lengthOfStart)==aStart);
	}

