/*
* Copyright (c) 2003-2005 Nokia Corporation and/or its subsidiary(-ies).
* All rights reserved.
* This component and the accompanying materials are made available
* under the terms of the License "Eclipse Public License v1.0"
* which accompanies this distribution, and is available
* at the URL "http://www.eclipse.org/legal/epl-v10.html".
*
* Initial Contributors:
* Nokia Corporation - initial contribution.
*
* Contributors:
*
* Description:
*
*/
#include <e32std.h>
#include <e32base.h>
#include <charconv.h>
#include <convutils.h>
const TInt KNoPreviousCharacterSet=-1;
const TInt KDefaultCharacterSet = 0;
const TUint KControlCharacterEscape=0x1b;
#if defined(_DEBUG)
const TInt KMaximumLengthOfIntermediateBuffer=5;
#else
const TInt KMaximumLengthOfIntermediateBuffer=150;
#endif
struct SCnvConversionData;
_LIT(KLitPanicText, "CONVUTILS");
enum TPanic
{
EPanicBadInputConversionFlags1=1,
EPanicBadInputConversionFlags2,
EPanicBadInputConversionFlags3,
EPanicBadNumberOfUnicodeElementsConsumed,
EPanicAppendFlagViolated,
EPanicBadNumberOfUnicodeCharactersConverted,
EPanicBadNumberOfCharactersThatDroppedOut,
EPanicLoopCounterOverRun1,
EPanicLoopCounterOverRun2,
EPanicDescriptorNotWholeNumberOfCharacters1,
EPanicDescriptorNotWholeNumberOfCharacters2,
EPanicDescriptorNotWholeNumberOfCharacters3,
EPanicDescriptorNotWholeNumberOfCharacters4,
EPanicBadStartOfNextEscapeSequence,
EPanicInconsistentNumberOfForeignBytesRemaining,
EPanicBadLengthOfRunToConvert1,
EPanicBadLengthOfRunToConvert2,
EPanicBadMethodPointer,
EPanicBadMethodData1,
EPanicBadMethodData2,
EPanicBadMethodData3,
EPanicBadMethodData4,
EPanicBadNumberOfCharacterSets,
EPanicBadConversionDataPointer1,
EPanicBadConversionDataPointer2,
EPanicBadConversionDataPointer3,
EPanicBadFunctionPointer1,
EPanicBadFunctionPointer2,
EPanicBadFunctionPointer3,
EPanicBadEscapeSequencePointer1,
EPanicBadEscapeSequencePointer2,
EPanicBadNumberOfStates,
EPanicBadEscapeSequenceStart,
EPanicBadNumberOfMethods,
EPanicBadSurrogatePair1,
EPanicBadSurrogatePair2,
EPanicBadRemainderOfForeign,
EPanicOutputDescriptorTooShortEvenToHoldEscapeSequenceToDefaultCharacterSet
};
LOCAL_C void Panic(TPanic aPanic)
{
User::Panic(KLitPanicText, aPanic);
}
/** Converts Unicode text into a complex foreign character set encoding. This
is an encoding which cannot be converted simply by calling
CCnvCharacterSetConverter::DoConvertFromUnicode(). It may be modal (e.g. JIS)
or non-modal (e.g. Shift-JIS).
The Unicode text specified in aUnicode is converted using the array of
conversion data objects (aArrayOfCharacterSets) provided by the plug-in for
the complex character set encoding, and the converted text is returned in
aForeign. Any existing contents in aForeign are overwritten.
Unlike CCnvCharacterSetConverter::DoConvertFromUnicode(), multiple character
sets can be specified. aUnicode is converted using the first character conversion
data object in the array. When a character is found which cannot be converted
using that data, each character set in the array is tried in turn. If it cannot
be converted using any object in the array, the index of the character is
appended to aIndicesOfUnconvertibleCharacters and the character is replaced
by aReplacementForUnconvertibleUnicodeCharacters.
If it can be converted using another object in the array, that object is used
to convert all subsequent characters until another unconvertible character
is found.
@param aDefaultEndiannessOfForeignCharacters The default endian-ness to use
when writing the characters in the foreign character set. If an endian-ness
for foreign characters is specified in the current conversion data object,
then that is used instead and the value of
aDefaultEndiannessOfForeignCharacters is ignored.
@param aReplacementForUnconvertibleUnicodeCharacters The single character (one
or more byte values) which is used to replace unconvertible characters.
@param aForeign On return, contains the converted text in the non-Unicode
character set.
@param aUnicode The source Unicode text to be converted.
@param aIndicesOfUnconvertibleCharacters On return, holds an ascending array
of the indices of each Unicode character in the source text which could not
be converted (because none of the target character sets have an equivalent
character).
@param aArrayOfCharacterSets Array of character conversion data objects,
representing the character sets which comprise a complex character set
encoding. These are used in sequence to convert the Unicode text. There must
be at least one character set in this array and no character set may have any
NULL member data, or a panic occurs.
@return The number of unconverted characters left at the end of the input
descriptor (e.g. because aForeign was not long enough to hold all the text),
or a negative error value, as defined in CCnvCharacterSetConverter::TError. */
EXPORT_C TInt CnvUtilities::ConvertFromUnicode(
CCnvCharacterSetConverter::TEndianness aDefaultEndiannessOfForeignCharacters,
const TDesC8& aReplacementForUnconvertibleUnicodeCharacters,
TDes8& aForeign,
const TDesC16& aUnicode,
CCnvCharacterSetConverter::TArrayOfAscendingIndices& aIndicesOfUnconvertibleCharacters,
const TArray<SCharacterSet>& aArrayOfCharacterSets)
{
TUint notUsed;
return ConvertFromUnicode(aDefaultEndiannessOfForeignCharacters,
aReplacementForUnconvertibleUnicodeCharacters,
aForeign,
aUnicode,
aIndicesOfUnconvertibleCharacters,
aArrayOfCharacterSets,
notUsed,
0);
}
/** Converts Unicode text into a complex foreign character set encoding. This is
an encoding which cannot be converted simply by a call to
CCnvCharacterSetConverter::DoConvertFromUnicode(). It may be modal (e.g. JIS)
or non-modal (e.g. Shift-JIS).
The Unicode text specified in aUnicode is converted using the array of conversion
data objects (aArrayOfCharacterSets) provided by the plug-in for the complex
character set encoding and the converted text is returned in aForeign. The
function can either append to aForeign or overwrite its contents (if any).
Unlike CCnvCharacterSetConverter::DoConvertFromUnicode(), multiple character
sets can be specified. aUnicode is converted using the first character conversion
data object in the array. When a character is found which cannot be converted
using that data, each character set in the array is tried in turn. If it cannot
be converted using any object in the array, the index of the character is
appended to aIndicesOfUnconvertibleCharacters and the character is replaced
by aReplacementForUnconvertibleUnicodeCharacters.
If it can be converted using another object in the array, that object is used
to convert all subsequent characters until another unconvertible character
is found.
@param aDefaultEndiannessOfForeignCharacters The default endian-ness to use
when writing the characters in the foreign character set. If an endian-ness
for foreign characters is specified in the current conversion data object,
then that is used instead and the value of
aDefaultEndiannessOfForeignCharacters is ignored.
@param aReplacementForUnconvertibleUnicodeCharacters The single character (one
or more byte values) which is used to replace unconvertible characters.
@param aForeign On return, contains the converted text in the non-Unicode
character set. This may already contain some text. If it does, and if
aInputConversionFlags specifies EInputConversionFlagAppend, then the converted
text is appended to this descriptor.
@param aUnicode The source Unicode text to be converted.
@param aIndicesOfUnconvertibleCharacters On return, holds an ascending array
of the indices of each Unicode character in the source text which could not
be converted (because none of the target character sets have an equivalent
character).
@param aArrayOfCharacterSets Array of character set data objects. These are
used in sequence to convert the Unicode text. There must be at least one
character set in this array and no character set may have any NULL member
data, or a panic occurs.
@param aOutputConversionFlags If the input descriptor ended in a truncated
sequence, e.g. the first half only of a Unicode surrogate pair, this returns
with the EOutputConversionFlagInputIsTruncated flag set.
@param aInputConversionFlags Specify
CCnvCharacterSetConverter::EInputConversionFlagAppend to append the text to
aForeign. Specify CCnvCharacterSetConverter::EInputConversionFlagAllowTruncatedInputNotEvenPartlyConsumable
to prevent the function from returning the error-code EErrorIllFormedInput
when the input descriptor consists of nothing but a truncated sequence. The
CCnvCharacterSetConverter::EInputConversionFlagStopAtFirstUnconvertibleCharacter
flag must not be set, otherwise a panic occurs.
@return The number of unconverted characters left at the end of the input descriptor
(e.g. because aForeign was not long enough to hold all the text), or a negative
error value, as defined in CCnvCharacterSetConverter::TError. */
EXPORT_C TInt CnvUtilities::ConvertFromUnicode(
CCnvCharacterSetConverter::TEndianness aDefaultEndiannessOfForeignCharacters,
const TDesC8& aReplacementForUnconvertibleUnicodeCharacters,
TDes8& aForeign,
const TDesC16& aUnicode,
CCnvCharacterSetConverter::TArrayOfAscendingIndices& aIndicesOfUnconvertibleCharacters,
const TArray<SCharacterSet>& aArrayOfCharacterSets,
TUint& aOutputConversionFlags,
TUint aInputConversionFlags)
{
__ASSERT_ALWAYS(~aInputConversionFlags&CCnvCharacterSetConverter::EInputConversionFlagStopAtFirstUnconvertibleCharacter, Panic(EPanicBadInputConversionFlags1));
CheckArrayOfCharacterSets(aArrayOfCharacterSets);
aOutputConversionFlags=0;
TUint internalInputConversionFlags=aInputConversionFlags;
if (~aInputConversionFlags&CCnvCharacterSetConverter::EInputConversionFlagAppend)
{
aForeign.SetLength(0);
internalInputConversionFlags|=CCnvCharacterSetConverter::EInputConversionFlagAppend;
}
if (aUnicode.Length()==0)
{
return 0;
}
if (aForeign.MaxLength()==aForeign.Length()) // relies on the fact that aForeign's length has been set to zero if aInputConversionFlags does not have CCnvCharacterSetConverter::EInputConversionFlagAppend set
{
return aUnicode.Length();
}
TDes8* foreign=&aForeign;
TPtr8 dummyForeign(NULL, 0, 0);
if (aInputConversionFlags&CCnvCharacterSetConverter::EInputConversionFlagMustEndInDefaultCharacterSet)
{
TInt dummyMaximumLength =
aForeign.MaxLength() - aArrayOfCharacterSets[KDefaultCharacterSet].iEscapeSequence->Length();
__ASSERT_ALWAYS(dummyMaximumLength >= 0,
Panic(EPanicOutputDescriptorTooShortEvenToHoldEscapeSequenceToDefaultCharacterSet));
dummyForeign.Set(const_cast <TUint8*> (aForeign.Ptr()),
aForeign.Length(),
dummyMaximumLength);
foreign=&dummyForeign;
}
const TInt numberOfCharacterSets=aArrayOfCharacterSets.Count();
TInt numberOfUnicodeElementsConsumed=0;
internalInputConversionFlags|=CCnvCharacterSetConverter::EInputConversionFlagStopAtFirstUnconvertibleCharacter; // this is not just an optimization - it ensures that "foreign" doesn't get filled up too much each time CCnvCharacterSetConverter::DoConvertFromUnicode is called
TInt previousCharacterSet = aInputConversionFlags&CCnvCharacterSetConverter::EInputConversionFlagAssumeStartInDefaultCharacterSet?
KDefaultCharacterSet : KNoPreviousCharacterSet;
FOREVER
{
for (TInt presentCharacterSet=KDefaultCharacterSet;;)
{
__ASSERT_DEBUG(numberOfUnicodeElementsConsumed<=aUnicode.Length(), Panic(EPanicBadNumberOfUnicodeElementsConsumed));
if (numberOfUnicodeElementsConsumed>=aUnicode.Length())
{
goto end;
}
const SCharacterSet& characterSet=aArrayOfCharacterSets[presentCharacterSet];
const TInt oldNumberOfBytesInForeign=foreign->Length();
if (numberOfUnicodeElementsConsumed>0)
{
internalInputConversionFlags|=CCnvCharacterSetConverter::EInputConversionFlagAllowTruncatedInputNotEvenPartlyConsumable;
}
CCnvCharacterSetConverter::TArrayOfAscendingIndices indicesOfUnconvertibleCharacters;
const TInt returnValue=CCnvCharacterSetConverter::DoConvertFromUnicode(*characterSet.iConversionData, aDefaultEndiannessOfForeignCharacters, KNullDesC8, *foreign, aUnicode.Mid(numberOfUnicodeElementsConsumed), indicesOfUnconvertibleCharacters, aOutputConversionFlags, internalInputConversionFlags);
if (returnValue<0)
{
return returnValue; // this is an error-code
}
__ASSERT_DEBUG(foreign->Length()>=oldNumberOfBytesInForeign, Panic(EPanicAppendFlagViolated));
TInt indexOfFirstUnconvertibleCharacter;
if (indicesOfUnconvertibleCharacters.NumberOfIndices()==0)
{
indexOfFirstUnconvertibleCharacter=-1;
numberOfUnicodeElementsConsumed=aUnicode.Length()-returnValue;
}
else
{
indexOfFirstUnconvertibleCharacter=indicesOfUnconvertibleCharacters[0];
numberOfUnicodeElementsConsumed+=indexOfFirstUnconvertibleCharacter;
__ASSERT_DEBUG(numberOfUnicodeElementsConsumed+LengthOfUnicodeCharacter(aUnicode, numberOfUnicodeElementsConsumed)==aUnicode.Length()-returnValue, Panic(EPanicBadNumberOfUnicodeCharactersConverted));
}
if (indexOfFirstUnconvertibleCharacter!=0) // if at least one Unicode character at the start of CCnvCharacterSetConverter::DoConvertFromUnicode's input descriptor was convertible...
{
TBool gotoEnd = EFalse;
if (foreign->Length()>oldNumberOfBytesInForeign)
{
TInt numberOfCharactersThatDroppedOut=0;
// Insert an escape sequence if this character set is different from the last one.
if (presentCharacterSet != previousCharacterSet)
{
// Insert escape sequence (if requred) in front of the last encoded run of text.
// Note that this may cause some characters to drop out at the end.
(*characterSet.iConvertFromIntermediateBufferInPlace)(oldNumberOfBytesInForeign, *foreign, numberOfCharactersThatDroppedOut);
if (oldNumberOfBytesInForeign < foreign->Length())
previousCharacterSet = presentCharacterSet;
}
numberOfUnicodeElementsConsumed-=numberOfCharactersThatDroppedOut;
if (numberOfCharactersThatDroppedOut>0 )// if "foreign" has been filled to as much as it will hold...
{
gotoEnd = ETrue;
}
}
if (indexOfFirstUnconvertibleCharacter<0) // if we've successfully converted up to the end of aUnicode (using *characterSet.iConversionData)...
{
gotoEnd = ETrue;
}
if (gotoEnd)
{
if ( aInputConversionFlags&CCnvCharacterSetConverter::EInputConversionFlagMustEndInDefaultCharacterSet
&& previousCharacterSet != KDefaultCharacterSet
&& previousCharacterSet != KNoPreviousCharacterSet)
{
aForeign.SetLength(foreign->Length());
aForeign.Append(*aArrayOfCharacterSets[KDefaultCharacterSet].iEscapeSequence);
foreign=NULL;
}
goto end;
}
break;
}
__ASSERT_DEBUG(presentCharacterSet<numberOfCharacterSets, Panic(EPanicLoopCounterOverRun1));
++presentCharacterSet;
if (presentCharacterSet>=numberOfCharacterSets)
{
if ((foreign->MaxLength()-foreign->Length()<aReplacementForUnconvertibleUnicodeCharacters.Length()) ||
(aIndicesOfUnconvertibleCharacters.AppendIndex(numberOfUnicodeElementsConsumed)!=CCnvCharacterSetConverter::TArrayOfAscendingIndices::EAppendSuccessful)) // the tests must be done in this order as AppendIndex must only be called if there is room for aReplacementForUnconvertibleUnicodeCharacters
{
goto end;
}
numberOfUnicodeElementsConsumed+=LengthOfUnicodeCharacter(aUnicode, numberOfUnicodeElementsConsumed);
foreign->Append(aReplacementForUnconvertibleUnicodeCharacters);
break;
}
}
}
end:
if (foreign!=NULL)
{
aForeign.SetLength(foreign->Length());
foreign=NULL;
}
if ((numberOfUnicodeElementsConsumed==0) && (aOutputConversionFlags&CCnvCharacterSetConverter::EOutputConversionFlagInputIsTruncated) && (~aInputConversionFlags&CCnvCharacterSetConverter::EInputConversionFlagAllowTruncatedInputNotEvenPartlyConsumable))
{
return CCnvCharacterSetConverter::EErrorIllFormedInput;
}
return aUnicode.Length()-numberOfUnicodeElementsConsumed;
}
/** Inserts an escape sequence into the descriptor.
This function is provided to help in the implementation of
ConvertFromUnicode() for modal character set encodings.
Each SCharacterSet object in the array passed to
ConvertFromUnicode() must have its
iConvertFromIntermediateBufferInPlace member assigned. To
do this for a modal character set encoding, implement a function whose
signature matches that of FConvertFromIntermediateBufferInPlace
and which calls this function, passing all arguments unchanged, and
specifying the character set's escape sequence and the number of bytes per
character.
@param aStartPositionInDescriptor The byte position in aDescriptor at which
the escape sequence is inserted. If the character set uses more than one byte
per character, this position must be the start of a character, otherwise a
panic occurs.
@param aDescriptor The descriptor into which the escape sequence is inserted.
@param aNumberOfCharactersThatDroppedOut The escape sequence is inserted into
the start of aDescriptor and any characters that need to drop out to make
room for the escape sequence (because the descriptor's maximum length was
not long enough) drop out from the end of the buffer. This parameter indicates
the number of characters that needed to drop out.
@param aEscapeSequence The escape sequence for the character set.
@param aNumberOfBytesPerCharacter The number of bytes per character. */
EXPORT_C void CnvUtilities::ConvertFromIntermediateBufferInPlace(
TInt aStartPositionInDescriptor,
TDes8& aDescriptor,
TInt& aNumberOfCharactersThatDroppedOut,
const TDesC8& aEscapeSequence,
TInt aNumberOfBytesPerCharacter)
{
const TInt lengthOfDescriptor=aDescriptor.Length();
__ASSERT_ALWAYS((lengthOfDescriptor-aStartPositionInDescriptor)%aNumberOfBytesPerCharacter==0, Panic(EPanicDescriptorNotWholeNumberOfCharacters1));
aNumberOfCharactersThatDroppedOut=(Max(0, aEscapeSequence.Length()-(aDescriptor.MaxLength()-lengthOfDescriptor))+(aNumberOfBytesPerCharacter-1))/aNumberOfBytesPerCharacter;
const TInt lengthOfRunInCharacters=(lengthOfDescriptor-aStartPositionInDescriptor)/aNumberOfBytesPerCharacter;
if (aNumberOfCharactersThatDroppedOut>=lengthOfRunInCharacters) // ">=" is correct (rather than ">") as if there's only room for the escape sequence we don't want to have it in the descriptor
{
aNumberOfCharactersThatDroppedOut=lengthOfRunInCharacters;
aDescriptor.SetLength(aStartPositionInDescriptor);
}
else
{
aDescriptor.SetLength(lengthOfDescriptor-(aNumberOfCharactersThatDroppedOut*aNumberOfBytesPerCharacter));
aDescriptor.Insert(aStartPositionInDescriptor, aEscapeSequence);
}
}
/** Converts text from a modal foreign character set encoding into Unicode.
The non-Unicode text specified in aForeign is converted using
the array of character set conversion objects (aArrayOfStates)
provided by the plug-in, and the converted text is returned in
aUnicode. The function can either append to aUnicode
or overwrite its contents (if any), depending on the input conversion flags
specified. The first element in aArrayOfStates is taken to be
the default mode (i.e. the mode to assume by default if there is no preceding
escape sequence).
@param aDefaultEndiannessOfForeignCharacters The default endian-ness of the
foreign characters. If an endian-ness for foreign characters is specified
in the conversion data, then that is used instead and the value of
aDefaultEndiannessOfForeignCharacters is ignored.
@param aUnicode On return, contains the text converted into Unicode.
@param aForeign The non-Unicode source text to be converted.
@param aState Used to store a modal character set encoding's current mode across
multiple calls to ConvertToUnicode() on the same input descriptor. This argument
should be passed the same object as passed to the plug-in's ConvertToUnicode()
exported function.
@param aNumberOfUnconvertibleCharacters On return, contains the number of
characters in aForeign which were not converted. Characters which cannot be
converted are output as Unicode replacement characters (0xfffd).
@param aIndexOfFirstByteOfFirstUnconvertibleCharacter On return, the index
of the first byte of the first unconvertible character. For instance if the
first character in the input descriptor (aForeign) could not be converted,
then this parameter is set to the first byte of that character, i.e. zero.
A negative value is returned if all the characters were converted.
@param aArrayOfStates Array of character set conversion data objects, and their
escape sequences ("modes"). There must be one or more modes in this array,
none of the modes can have any NULL member data, and each mode's escape sequence
must begin with KControlCharacterEscape (0x1b) or a panic occurs.
@return The number of unconverted bytes left at the end of the input descriptor,
or a negative error value, as defined in TError. */
EXPORT_C TInt CnvUtilities::ConvertToUnicodeFromModalForeign(
CCnvCharacterSetConverter::TEndianness aDefaultEndiannessOfForeignCharacters,
TDes16& aUnicode,
const TDesC8& aForeign,
TInt& aState,
TInt& aNumberOfUnconvertibleCharacters,
TInt& aIndexOfFirstByteOfFirstUnconvertibleCharacter,
const TArray<SState>& aArrayOfStates)
{
TUint notUsed;
return ConvertToUnicodeFromModalForeign(aDefaultEndiannessOfForeignCharacters,
aUnicode,
aForeign,
aState,
aNumberOfUnconvertibleCharacters,
aIndexOfFirstByteOfFirstUnconvertibleCharacter,
aArrayOfStates,
notUsed,
0);
}
/** @param aDefaultEndiannessOfForeignCharacters The default endian-ness for
the foreign characters. If an endian-ness for foreign characters is specified
in the conversion data, then that is used instead and the value of
aDefaultEndiannessOfForeignCharacters is ignored.
@param aUnicode On return, contains the text converted into Unicode.
@param aForeign The non-Unicode source text to be converted.
@param aState Used to store a modal character set encoding's current mode
across multiple calls to ConvertToUnicode() on the same input descriptor. This
argument should be passed the same object as passed to the plug-in's
ConvertToUnicode() exported function.
@param aNumberOfUnconvertibleCharacters On return, contains the number of
characters in aForeign which were not converted. Characters which cannot be
converted are output as Unicode replacement characters (0xfffd).
@param aIndexOfFirstByteOfFirstUnconvertibleCharacter On return, the index
of the first byte of the first unconvertible character. For instance if the
first character in the input descriptor (aForeign) could not be converted,
then this parameter is set to the first byte of that character, i.e. zero.
A negative value is returned if all the characters were converted.
@param aArrayOfStates Array of character set conversion data objects, and their
escape sequences. There must be one or more modes in this array, none of the
modes can have any NULL member data, and each mode's escape sequence must
begin with KControlCharacterEscape (0x1b) or a panic occurs.
@param aOutputConversionFlags If the input descriptor ended in a truncated
sequence, e.g. an incomplete multi-byte character, aOutputConversionFlags
returns with the EOutputConversionFlagInputIsTruncated flag set.
@param aInputConversionFlags Specify
CCnvCharacterSetConverter::EInputConversionFlagAppend to append the text to
aUnicode. Specify EInputConversionFlagAllowTruncatedInputNotEvenPartlyConsumable
to prevent the function from returning the error-code EErrorIllFormedInput
when the input descriptor consists of nothing but a truncated sequence. The
CCnvCharacterSetConverter::EInputConversionFlagStopAtFirstUnconvertibleCharacter
flag must not be set, otherwise a panic occurs.
@return The number of unconverted bytes left at the end of the input descriptor,
or a negative error value, as defined in TError. */
EXPORT_C TInt CnvUtilities::ConvertToUnicodeFromModalForeign(
CCnvCharacterSetConverter::TEndianness aDefaultEndiannessOfForeignCharacters,
TDes16& aUnicode,
const TDesC8& aForeign,
TInt& aState,
TInt& aNumberOfUnconvertibleCharacters,
TInt& aIndexOfFirstByteOfFirstUnconvertibleCharacter,
const TArray<SState>& aArrayOfStates,
TUint& aOutputConversionFlags,
TUint aInputConversionFlags)
{
__ASSERT_ALWAYS(~aInputConversionFlags&CCnvCharacterSetConverter::EInputConversionFlagStopAtFirstUnconvertibleCharacter, Panic(EPanicBadInputConversionFlags2));
CheckArrayOfStates(aArrayOfStates);
aNumberOfUnconvertibleCharacters=0;
aIndexOfFirstByteOfFirstUnconvertibleCharacter=-1;
aOutputConversionFlags=0;
TUint internalInputConversionFlags=aInputConversionFlags;
if (~aInputConversionFlags&CCnvCharacterSetConverter::EInputConversionFlagAppend)
{
aUnicode.SetLength(0);
internalInputConversionFlags|=CCnvCharacterSetConverter::EInputConversionFlagAppend;
}
if (aForeign.Length()==0)
{
return 0;
}
if (aUnicode.MaxLength()==aUnicode.Length()) // relies on the fact that aUnicode's length has been set to zero if aInputConversionFlags does not have CCnvCharacterSetConverter::EInputConversionFlagAppend set
{
return aForeign.Length();
}
TPtrC8 remainderOfForeign(aForeign);
TPtrC8 homogeneousRun;
TInt numberOfForeignBytesConsumed=0;
const SCnvConversionData* conversionData = NULL;
const TInt startOfNextEscapeSequence=aForeign.Locate(KControlCharacterEscape);
if (startOfNextEscapeSequence!=0) // if aForeign doesn't start with an escape sequence...
{
conversionData=(aState!=CCnvCharacterSetConverter::KStateDefault)? REINTERPRET_CAST(const SCnvConversionData*, aState): aArrayOfStates[0].iConversionData;
if (startOfNextEscapeSequence==KErrNotFound)
{
homogeneousRun.Set(remainderOfForeign);
remainderOfForeign.Set(NULL, 0);
}
else
{
__ASSERT_DEBUG(startOfNextEscapeSequence>0, Panic(EPanicBadStartOfNextEscapeSequence));
homogeneousRun.Set(remainderOfForeign.Left(startOfNextEscapeSequence));
remainderOfForeign.Set(remainderOfForeign.Mid(startOfNextEscapeSequence));
}
goto handleHomogeneousRun;
}
FOREVER
{
if (!NextHomogeneousForeignRun(conversionData, numberOfForeignBytesConsumed, homogeneousRun, remainderOfForeign, aArrayOfStates, aOutputConversionFlags))
{
goto end;
}
handleHomogeneousRun:
if (conversionData==NULL)
{
return CCnvCharacterSetConverter::EErrorIllFormedInput;
}
TInt numberOfUnconvertibleCharacters;
TInt indexOfFirstByteOfFirstUnconvertibleCharacter;
const TInt returnValue=CCnvCharacterSetConverter::DoConvertToUnicode(*conversionData, aDefaultEndiannessOfForeignCharacters, aUnicode, homogeneousRun, numberOfUnconvertibleCharacters, indexOfFirstByteOfFirstUnconvertibleCharacter, aOutputConversionFlags, internalInputConversionFlags);
if (returnValue<0)
{
return returnValue; // this is an error-code
}
if (numberOfUnconvertibleCharacters>0)
{
if (aNumberOfUnconvertibleCharacters==0)
{
aIndexOfFirstByteOfFirstUnconvertibleCharacter=numberOfForeignBytesConsumed+indexOfFirstByteOfFirstUnconvertibleCharacter;
}
aNumberOfUnconvertibleCharacters+=numberOfUnconvertibleCharacters;
}
numberOfForeignBytesConsumed+=homogeneousRun.Length();
if (returnValue>0)
{
numberOfForeignBytesConsumed-=returnValue;
goto end;
}
if (numberOfForeignBytesConsumed>0)
{
internalInputConversionFlags|=CCnvCharacterSetConverter::EInputConversionFlagAllowTruncatedInputNotEvenPartlyConsumable;
}
__ASSERT_DEBUG(remainderOfForeign==aForeign.Mid(numberOfForeignBytesConsumed), Panic(EPanicInconsistentNumberOfForeignBytesRemaining));
}
end:
if ((numberOfForeignBytesConsumed==0) && (aOutputConversionFlags&CCnvCharacterSetConverter::EOutputConversionFlagInputIsTruncated) && (~aInputConversionFlags&CCnvCharacterSetConverter::EInputConversionFlagAllowTruncatedInputNotEvenPartlyConsumable))
{
return CCnvCharacterSetConverter::EErrorIllFormedInput;
}
aState=REINTERPRET_CAST(TInt, conversionData);
return aForeign.Length()-numberOfForeignBytesConsumed;
}
/** Converts text from a non-modal complex character set encoding (e.g.
Shift-JIS or EUC-JP) into Unicode.The non-Unicode text specified in
aForeign is converted using the array of character set
conversion methods (aArrayOfMethods) provided by the
plug-in, and the converted text is returned in aUnicode.
Overwrites the contents, if any, of aUnicode.
@param aDefaultEndiannessOfForeignCharacters The default endian-ness of the
foreign characters. If an endian-ness for foreign characters is specified
in the conversion data, then that is used instead and the value of
aDefaultEndiannessOfForeignCharacters is ignored.
@param aUnicode On return, contains the text converted into Unicode.
@param aForeign The non-Unicode source text to be converted.
@param aNumberOfUnconvertibleCharacters On return, contains the number of
characters in aForeign which were not converted. Characters which cannot be
converted are output as Unicode replacement characters (0xfffd).
@param aIndexOfFirstByteOfFirstUnconvertibleCharacter On return, the index
of the first byte of the first unconvertible character. For instance if the
first character in the input descriptor (aForeign) could not be converted,
then this parameter is set to the first byte of that character, i.e. zero.
A negative value is returned if all the characters were converted.
@param aArrayOfMethods Array of conversion methods. There must be one or more
methods in this array and none of the methods in the array can have any NULL
member data or a panic occurs.
@return The number of unconverted bytes left at the end of the input descriptor,
or a negative error value, as defined in TError. */
EXPORT_C TInt CnvUtilities::ConvertToUnicodeFromHeterogeneousForeign(
CCnvCharacterSetConverter::TEndianness aDefaultEndiannessOfForeignCharacters,
TDes16& aUnicode,
const TDesC8& aForeign,
TInt& aNumberOfUnconvertibleCharacters,
TInt& aIndexOfFirstByteOfFirstUnconvertibleCharacter,
const TArray<SMethod>& aArrayOfMethods)
{
TUint notUsed;
return ConvertToUnicodeFromHeterogeneousForeign(
aDefaultEndiannessOfForeignCharacters,
aUnicode,
aForeign,
aNumberOfUnconvertibleCharacters,
aIndexOfFirstByteOfFirstUnconvertibleCharacter,
aArrayOfMethods,
notUsed,
0);
}
/** @param aDefaultEndiannessOfForeignCharacters The default endian-ness for the
foreign characters. If an endian-ness for foreign characters is specified
in the conversion data, then that is used instead and the value of
aDefaultEndiannessOfForeignCharacters is ignored.
@param aUnicode On return, contains the text converted into Unicode.
@param aForeign The non-Unicode source text to be converted.
@param aNumberOfUnconvertibleCharacters On return, contains the number of
characters in aForeign which were not converted. Characters which cannot be
converted are output as Unicode replacement characters (0xfffd).
@param aIndexOfFirstByteOfFirstUnconvertibleCharacter On return, the index
of the first byte of the first unconvertible character. For instance if the
first character in the input descriptor (aForeign) could not be converted,
then this parameter is set to the first byte of that character, i.e. zero.
A negative value is returned if all the characters were converted.
@param aArrayOfMethods Array of conversion methods. There must be one or more
methods in this array and none of the methods in the array can have any NULL
member data or a panic occurs.
@param aOutputConversionFlags If the input descriptor ended in a truncated
sequence, e.g. an incomplete multi-byte character, aOutputConversionFlags
returns with the EOutputConversionFlagInputIsTruncated flag set.
@param aInputConversionFlags Specify
CCnvCharacterSetConverter::EInputConversionFlagAppend to append the text to
aUnicode. Specify EInputConversionFlagAllowTruncatedInputNotEvenPartlyConsumable
to prevent the function from returning the error-code EErrorIllFormedInput
when the input descriptor consists of nothing but a truncated sequence. The
CCnvCharacterSetConverter::EInputConversionFlagStopAtFirstUnconvertibleCharacter
flag must not be set, otherwise a panic occurs.
@return The number of unconverted bytes left at the end of the input descriptor,
or a negative error value, as defined in TError. */
EXPORT_C TInt CnvUtilities::ConvertToUnicodeFromHeterogeneousForeign(
CCnvCharacterSetConverter::TEndianness aDefaultEndiannessOfForeignCharacters,
TDes16& aUnicode,
const TDesC8& aForeign,
TInt& aNumberOfUnconvertibleCharacters,
TInt& aIndexOfFirstByteOfFirstUnconvertibleCharacter,
const TArray<SMethod>& aArrayOfMethods,
TUint& aOutputConversionFlags,
TUint aInputConversionFlags)
{
__ASSERT_ALWAYS(~aInputConversionFlags&CCnvCharacterSetConverter::EInputConversionFlagStopAtFirstUnconvertibleCharacter, Panic(EPanicBadInputConversionFlags3));
CheckArrayOfMethods(aArrayOfMethods);
aNumberOfUnconvertibleCharacters=0;
aIndexOfFirstByteOfFirstUnconvertibleCharacter=-1;
aOutputConversionFlags=0;
TUint internalInputConversionFlags=aInputConversionFlags;
if (~aInputConversionFlags&CCnvCharacterSetConverter::EInputConversionFlagAppend)
{
aUnicode.SetLength(0);
internalInputConversionFlags|=CCnvCharacterSetConverter::EInputConversionFlagAppend;
}
if (aForeign.Length()==0)
{
return 0;
}
if (aUnicode.MaxLength()==aUnicode.Length()) // relies on the fact that aUnicode's length has been set to zero if aInputConversionFlags does not have CCnvCharacterSetConverter::EInputConversionFlagAppend set
{
return aForeign.Length();
}
const TInt numberOfMethods=aArrayOfMethods.Count();
TPtrC8 remainderOfForeign(aForeign);
TInt numberOfForeignBytesConsumed=0;
FOREVER
{
TInt lengthOfRunToConvert=0;
const SMethod* method=NULL;
for (TInt i=0;;)
{
method=&aArrayOfMethods[i];
lengthOfRunToConvert=(*method->iNumberOfBytesAbleToConvert)(remainderOfForeign);
if (lengthOfRunToConvert<0)
{
return lengthOfRunToConvert; // this is an error-code
}
if (lengthOfRunToConvert>0)
{
break;
}
__ASSERT_DEBUG(i<numberOfMethods, Panic(EPanicLoopCounterOverRun2));
++i;
if (i>=numberOfMethods)
{
aOutputConversionFlags|=CCnvCharacterSetConverter::EOutputConversionFlagInputIsTruncated;
goto end;
}
}
__ASSERT_DEBUG(lengthOfRunToConvert>0, Panic(EPanicBadLengthOfRunToConvert1));
__ASSERT_DEBUG(method!=NULL, Panic(EPanicBadMethodPointer));
TBuf8<KMaximumLengthOfIntermediateBuffer> intermediateBuffer;
const TInt maximumUsableLengthOfIntermediateBuffer=ReduceToNearestMultipleOf(KMaximumLengthOfIntermediateBuffer, method->iNumberOfBytesPerCharacter);
FOREVER
{
const TInt numberOfForeignBytesConsumedThisTime=Min(lengthOfRunToConvert, maximumUsableLengthOfIntermediateBuffer);
intermediateBuffer=remainderOfForeign.Left(numberOfForeignBytesConsumedThisTime);
__ASSERT_DEBUG((numberOfForeignBytesConsumedThisTime%method->iNumberOfBytesPerCharacter)==0, Panic(EPanicDescriptorNotWholeNumberOfCharacters2));
(*method->iConvertToIntermediateBufferInPlace)(intermediateBuffer);
__ASSERT_DEBUG((intermediateBuffer.Length()%method->iNumberOfCoreBytesPerCharacter)==0, Panic(EPanicDescriptorNotWholeNumberOfCharacters3));
__ASSERT_DEBUG((intermediateBuffer.Length()/method->iNumberOfCoreBytesPerCharacter)*method->iNumberOfBytesPerCharacter==numberOfForeignBytesConsumedThisTime, Panic(EPanicBadMethodData1));
TInt numberOfUnconvertibleCharacters;
TInt indexOfFirstByteOfFirstUnconvertibleCharacter;
const TInt returnValue=CCnvCharacterSetConverter::DoConvertToUnicode(*method->iConversionData, aDefaultEndiannessOfForeignCharacters, aUnicode, intermediateBuffer, numberOfUnconvertibleCharacters, indexOfFirstByteOfFirstUnconvertibleCharacter, aOutputConversionFlags, internalInputConversionFlags);
if (returnValue<0)
{
return returnValue; // this is an error-code
}
if (numberOfUnconvertibleCharacters>0)
{
if (aNumberOfUnconvertibleCharacters==0)
{
aIndexOfFirstByteOfFirstUnconvertibleCharacter=numberOfForeignBytesConsumed+indexOfFirstByteOfFirstUnconvertibleCharacter;
}
aNumberOfUnconvertibleCharacters+=numberOfUnconvertibleCharacters;
}
numberOfForeignBytesConsumed+=numberOfForeignBytesConsumedThisTime;
if (returnValue>0)
{
__ASSERT_DEBUG((returnValue%method->iNumberOfCoreBytesPerCharacter)==0, Panic(EPanicDescriptorNotWholeNumberOfCharacters4));
numberOfForeignBytesConsumed-=(returnValue/method->iNumberOfCoreBytesPerCharacter)*method->iNumberOfBytesPerCharacter;
goto end;
}
if (numberOfForeignBytesConsumed>0)
{
internalInputConversionFlags|=CCnvCharacterSetConverter::EInputConversionFlagAllowTruncatedInputNotEvenPartlyConsumable;
}
remainderOfForeign.Set(aForeign.Mid(numberOfForeignBytesConsumed));
lengthOfRunToConvert-=numberOfForeignBytesConsumedThisTime;
__ASSERT_DEBUG(lengthOfRunToConvert>=0, Panic(EPanicBadLengthOfRunToConvert2));
if (lengthOfRunToConvert<=0)
{
break;
}
}
}
end:
if ((numberOfForeignBytesConsumed==0) && (aOutputConversionFlags&CCnvCharacterSetConverter::EOutputConversionFlagInputIsTruncated) && (~aInputConversionFlags&CCnvCharacterSetConverter::EInputConversionFlagAllowTruncatedInputNotEvenPartlyConsumable))
{
return CCnvCharacterSetConverter::EErrorIllFormedInput;
}
return aForeign.Length()-numberOfForeignBytesConsumed;
}
void CnvUtilities::CheckArrayOfCharacterSets(const TArray<SCharacterSet>& aArrayOfCharacterSets)
{
const TInt numberOfCharacterSets=aArrayOfCharacterSets.Count();
__ASSERT_ALWAYS(numberOfCharacterSets>0, Panic(EPanicBadNumberOfCharacterSets));
for (TInt i=0; i<numberOfCharacterSets; ++i)
{
const SCharacterSet& characterSet=aArrayOfCharacterSets[i];
__ASSERT_ALWAYS(characterSet.iConversionData!=NULL, Panic(EPanicBadConversionDataPointer1));
__ASSERT_ALWAYS(characterSet.iConvertFromIntermediateBufferInPlace!=NULL, Panic(EPanicBadFunctionPointer1));
__ASSERT_ALWAYS(characterSet.iEscapeSequence!=NULL, Panic(EPanicBadEscapeSequencePointer1));
}
}
void CnvUtilities::CheckArrayOfStates(const TArray<SState>& aArrayOfStates)
{
const TInt numberOfStates=aArrayOfStates.Count();
__ASSERT_ALWAYS(numberOfStates>0, Panic(EPanicBadNumberOfStates));
for (TInt i=0; i<numberOfStates; ++i)
{
const SState& state=aArrayOfStates[i];
__ASSERT_ALWAYS(state.iEscapeSequence!=NULL, Panic(EPanicBadEscapeSequencePointer2));
__ASSERT_ALWAYS((*state.iEscapeSequence)[0]==KControlCharacterEscape, Panic(EPanicBadEscapeSequenceStart));
__ASSERT_ALWAYS(state.iConversionData!=NULL, Panic(EPanicBadConversionDataPointer2));
}
}
void CnvUtilities::CheckArrayOfMethods(const TArray<SMethod>& aArrayOfMethods)
{
const TInt numberOfMethods=aArrayOfMethods.Count();
__ASSERT_ALWAYS(numberOfMethods>0, Panic(EPanicBadNumberOfMethods));
for (TInt i=0; i<numberOfMethods; ++i)
{
const SMethod& method=aArrayOfMethods[i];
__ASSERT_ALWAYS(method.iNumberOfBytesAbleToConvert!=NULL, Panic(EPanicBadFunctionPointer2));
__ASSERT_ALWAYS(method.iConvertToIntermediateBufferInPlace!=NULL, Panic(EPanicBadFunctionPointer3));
__ASSERT_ALWAYS(method.iConversionData!=NULL, Panic(EPanicBadConversionDataPointer3));
__ASSERT_ALWAYS(method.iNumberOfBytesPerCharacter>0, Panic(EPanicBadMethodData2));
__ASSERT_ALWAYS(method.iNumberOfCoreBytesPerCharacter>0, Panic(EPanicBadMethodData3));
__ASSERT_ALWAYS(method.iNumberOfCoreBytesPerCharacter<=method.iNumberOfBytesPerCharacter, Panic(EPanicBadMethodData4));
}
}
TInt CnvUtilities::LengthOfUnicodeCharacter(const TDesC16& aUnicode, TInt aIndex)
{
const TUint unicodeCharacter=aUnicode[aIndex];
if ((unicodeCharacter>=0xd800) && (unicodeCharacter<=0xdbff)) // if the unicode character is the first half of a surrogate-pair...
{
__ASSERT_DEBUG(aIndex+1<aUnicode.Length(), Panic(EPanicBadSurrogatePair1));
#if defined(_DEBUG)
const TUint secondHalfOfSurrogatePair=aUnicode[aIndex+1];
#endif
__ASSERT_DEBUG((secondHalfOfSurrogatePair>=0xdc00) && (secondHalfOfSurrogatePair<=0xdfff), Panic(EPanicBadSurrogatePair2)); // this can be asserted as CCnvCharacterSetConverter::DoConvertFromUnicode should have returned an error value if this was a bad surrogate pair
return 2;
}
return 1;
}
TBool CnvUtilities::NextHomogeneousForeignRun(const SCnvConversionData*& aConversionData, TInt& aNumberOfForeignBytesConsumed, TPtrC8& aHomogeneousRun, TPtrC8& aRemainderOfForeign, const TArray<SState>& aArrayOfStates, TUint& aOutputConversionFlags)
{
__ASSERT_DEBUG((aRemainderOfForeign.Length()==0) || (aRemainderOfForeign[0]==KControlCharacterEscape), Panic(EPanicBadRemainderOfForeign));
FOREVER
{
if (aRemainderOfForeign.Length()==0)
{
return EFalse;
}
const TInt numberOfStates=aArrayOfStates.Count();
TInt i;
for (i=0; i<numberOfStates; ++i)
{
const SState& state=aArrayOfStates[i];
if (MatchesEscapeSequence(aNumberOfForeignBytesConsumed, aHomogeneousRun, aRemainderOfForeign, *state.iEscapeSequence))
{
aConversionData=state.iConversionData;
goto foundState;
}
}
for (i=0; i<numberOfStates; ++i)
{
if (IsStartOf(aRemainderOfForeign, *aArrayOfStates[i].iEscapeSequence))
{
// aRemainderOfForeign ends with a truncated escape sequence, so ConvertToUnicode cannot convert any more
aOutputConversionFlags|=CCnvCharacterSetConverter::EOutputConversionFlagInputIsTruncated;
return EFalse;
}
}
// force ConvertToUnicode to return CCnvCharacterSetConverter::EErrorIllFormedInput
aConversionData=NULL;
return ETrue;
foundState:
if (aHomogeneousRun.Length()>0)
{
return ETrue;
}
}
}
TBool CnvUtilities::MatchesEscapeSequence(TInt& aNumberOfForeignBytesConsumed, TPtrC8& aHomogeneousRun, TPtrC8& aRemainderOfForeign, const TDesC8& aEscapeSequence)
{
const TInt lengthOfEscapeSequence=aEscapeSequence.Length();
if (IsStartOf(aEscapeSequence, aRemainderOfForeign))
{
aRemainderOfForeign.Set(aRemainderOfForeign.Mid(lengthOfEscapeSequence));
const TInt startOfNextEscapeSequence=aRemainderOfForeign.Locate(KControlCharacterEscape);
if (startOfNextEscapeSequence==KErrNotFound)
{
aHomogeneousRun.Set(aRemainderOfForeign);
aRemainderOfForeign.Set(NULL, 0);
}
else
{
aHomogeneousRun.Set(aRemainderOfForeign.Left(startOfNextEscapeSequence));
aRemainderOfForeign.Set(aRemainderOfForeign.Mid(startOfNextEscapeSequence));
}
aNumberOfForeignBytesConsumed+=lengthOfEscapeSequence;
return ETrue;
}
return EFalse;
}
TBool CnvUtilities::IsStartOf(const TDesC8& aStart, const TDesC8& aPotentiallyLongerDescriptor)
{
const TInt lengthOfStart=aStart.Length();
return (aPotentiallyLongerDescriptor.Length()>=lengthOfStart) && (aPotentiallyLongerDescriptor.Left(lengthOfStart)==aStart);
}