/*

* Copyright (c) 2005 Nokia Corporation and/or its subsidiary(-ies). 

* All rights reserved.

* This component and the accompanying materials are made available

* which accompanies this distribution, and is available

* at the URL "http://www.eclipse.org/legal/epl-v10.html".

*

* Initial Contributors:

* Nokia Corporation - initial contribution.

*

* Contributors:

*

* Description:      

*

*/

#include <e32std.h>

#include <charconv.h>

#include <ecom/implementationproxy.h>

#include <utf.h>

#include "charactersetconverter.h"

#include "convutils.h"

#include "shiftjis.h"

#include "JISBASE.H"

#include "J5.H"

#include "jisx0201.h"

#include "jisx0208.h"

#include "jisx0212.h"

/**

 J5 will use up to KMaxSizeAutoDetectSample to try to deterine the format of data.

 */

const TInt KMaxSizeAutoDetectSample = 1000;

const TUint8 KEscape = 0x1b;

const TInt KByteOrderMark = 0xfeff;

const TDesC8& CJ5Converter::ReplacementForUnconvertibleUnicodeCharacters()

	{

	return CnvShiftJis::ReplacementForUnconvertibleUnicodeCharacters();

	}

/**

 This API should not be used as it is ambiguous as to what encoding is required.  

 The user should instead call the specific plug-in for the appropriate conversion.

 J5 ConvertFromUnicode() will convert to UTF8 as default.

@internalTechnology 

 */

TInt CJ5Converter::ConvertFromUnicode(

		CCnvCharacterSetConverter::TEndianness /* aDefaultEndiannessOfForeignCharacters */, 

		const TDesC8& /* aReplacementForUnconvertibleUnicodeCharacters */, 

		TDes8& aForeign, 

		const TDesC16& aUnicode, 

		CCnvCharacterSetConverter::TArrayOfAscendingIndices& /* aIndicesOfUnconvertibleCharacters */)

	{

	return CnvUtfConverter::ConvertFromUnicodeToUtf8(aForeign, aUnicode);

	}

/**

 This will automatically determine one of the five supported encodings 

 to use and convert accordingly.  This plugin method is available to the 

 user though the CCnvCharacterSetConverter::ConvertToUnicode() method.  

 There is no way for the caller to determine which encoding has been used.

 NOTE: For debugging the selected character set is returned in the state.

  @released  9.1

  @param     aDefaultEndiannessOfForeignCharacters The default endian-ness to use when reading characters

             in the foreign character set.

  @param     aUnicode On return, contains the text converted into Unicode.

  @param     aForeign The non-Unicode source text to be converted.

  @param     aState Used to save state information across multiple calls

             to <code>ConvertToUnicode()</code>.

  @param     aNumberOfUnconvertibleCharacters On return, contains the number of bytes which were not

             converted.

  @param     aIndexOfFirstByteOfFirstUnconvertibleCharacter On return, contains the index of the first bytein the

             input text that could not be converted. A negative

             value indicates that all the characters were

             converted.

  @return 	 The number of unconverted bytes left at the end of the input descriptor 

 		     (e.g. because the output descriptor is not long enough to hold all the text), 

 		     or one of the error values defined in TError. 

  @internalTechnology 

*/

TInt CJ5Converter::ConvertToUnicode(

		CCnvCharacterSetConverter::TEndianness aDefaultEndiannessOfForeignCharacters, 

		TDes16& aUnicode, 

		const TDesC8& aForeign, 

		TInt& aState, 

		TInt& aNumberOfUnconvertibleCharacters, 

		TInt& aIndexOfFirstByteOfFirstUnconvertibleCharacter)

	{

	// As the aState parameter is used to pass back the detected value

	// use a "hidden" internal state variable.

	TInt internalState = CCnvCharacterSetConverter::KStateDefault;

	// determine the encoding type and then decode appropriatly

	switch ( DetectEncoding(aDefaultEndiannessOfForeignCharacters, aForeign))

		{

		case EShiftjis:

			aState = EShiftjis;

			return CnvShiftJis::ConvertToUnicode(aDefaultEndiannessOfForeignCharacters, aUnicode, aForeign, 

					aNumberOfUnconvertibleCharacters, aIndexOfFirstByteOfFirstUnconvertibleCharacter);

		case EIso2022jp1: 

			aState = EIso2022jp1;

			return CnvJisBase::ConvertToUnicode(aDefaultEndiannessOfForeignCharacters, aUnicode, aForeign, internalState,

					aNumberOfUnconvertibleCharacters, aIndexOfFirstByteOfFirstUnconvertibleCharacter);

		case EEucjp: 

			aState = EEucjp;

			return ConvertEEucjpToUnicode(

					aDefaultEndiannessOfForeignCharacters, aUnicode, aForeign, internalState,

					aNumberOfUnconvertibleCharacters, aIndexOfFirstByteOfFirstUnconvertibleCharacter);	

		case EUcs2:

			aState = EUcs2;

			return ConvertUcs2ToUnicode( aDefaultEndiannessOfForeignCharacters, aUnicode, aForeign, 

					aNumberOfUnconvertibleCharacters, aIndexOfFirstByteOfFirstUnconvertibleCharacter);

		case EUtf8: 

			aState = EUtf8;

			return CnvUtfConverter::ConvertToUnicodeFromUtf8(aUnicode, aForeign);

		default:

			// fall though to the default, which is decode as UTF8

			aState = EUnknown;

			break;

		}

	// decode as UTF8

	return CnvUtfConverter::ConvertToUnicodeFromUtf8(aUnicode, aForeign);

	}

/**

 This API is used by CCnvCharacterSetConverter::AutoDetectCharacterSetL(). 

 This method returns a value between 0 and 100, indicating how likely it 

 is that this is the correct converter, for the text supplied.  As J5 is 

 NOT intended to be used with the existing auto-detect mechanism, it will 

 always return 0

 @internalTechnology 

 */

TBool CJ5Converter::IsInThisCharacterSetL(

		TBool& aSetToTrue, 

		TInt& aConfidenceLevel, 

		const TDesC8& /* aSample */)

	{

  	/*

  	aSetToTrue - This value should be set to ETrue. It is used to indicate to 

  	CCnvCharacterSetConverter::AutoDetectCharacterSetL() that the plug-in DLL 

  	is implementing a function of this signature and is therefore not the empty 

  	*/

  	aSetToTrue=ETrue;

 	/* no need to look at the sample as this always returns 0 

 	   as the autodetect feature is not supported by the J5 plug-in

 	*/

 	aConfidenceLevel=0;

	return ETrue;

	}

CJ5Converter* CJ5Converter::NewL()

	{

	CJ5Converter* self = new(ELeave) CJ5Converter();

	CleanupStack::PushL(self);

	self->ConstructL();

	CleanupStack::Pop(self);

	return self;

	}

CJ5Converter::~CJ5Converter()

	{

	}

CJ5Converter::CJ5Converter()

	{

	}

void CJ5Converter::ConstructL()

	{

	}

const TImplementationProxy ImplementationTable[] = 

	{

#ifdef DOCOMO_TEST

		// for the test build use a special test UID

		IMPLEMENTATION_PROXY_ENTRY(0x01000002,	CJ5Converter::NewL)

#else

		IMPLEMENTATION_PROXY_ENTRY(KCharacterSetIdentifierJ5,	CJ5Converter::NewL)

#endif

	};

EXPORT_C const TImplementationProxy* ImplementationGroupProxy(TInt& aTableCount)

	{

	aTableCount = sizeof(ImplementationTable) / sizeof(TImplementationProxy);

	return ImplementationTable;

	}

/**

 DetectEncoding determine the characterset encoding.

 The logic for this detection is based on the information in CJKV by Ken Lunde.

 A detailed diagram of this logic is in the J5 how to document section 2.4

 @return The detected character set as a enum CJ5Converter.

 @internalTechnology 

 */

enum CJ5Converter::TJ5Encoding CJ5Converter::DetectEncoding(

		CCnvCharacterSetConverter::TEndianness& aDefaultEndiannessOfForeignCharacters , 

		const TDesC8& aForeign)

	{

	// first check for UCS2

	CCnvCharacterSetConverter::TEndianness ucs2Endianness = CCnvCharacterSetConverter::ELittleEndian;

	if ( DetectUcs2(aForeign, ucs2Endianness ))

		{

		// if ucs2 is detected pass back the detected endianess

		aDefaultEndiannessOfForeignCharacters = ucs2Endianness;

		return EUcs2;

		}

	// next try EUC_JP

	TInt eucJpValidBytes = 0;

	CJ5Converter::TDectectCharacterSet result = DetectEucJp( aForeign, eucJpValidBytes );

	if ( result == EIsCharacterSet )

		{

		return EEucjp;

		}

	// next try Iso 2020JP

	if ( DetectIso2022( aForeign ) == EIsCharacterSet )

		{

		return EIso2022jp1;

		}

	// next try Utf8

	if ( DetectUtf8( aForeign ) == EIsCharacterSet )

		{

		return EUtf8;

		}

	// shiftjis

	TInt shiftjisValidBytes = 0;

	result = DetectShiftJis( aForeign, shiftjisValidBytes );

	if ( result == EIsCharacterSet )

		{

		return EShiftjis;

		}

	// no clear winner so go for the best 

	TInt sampleLength = Min(aForeign.Length(), KMaxSizeAutoDetectSample);

	// if more than half is shiftjis and more shiftjis than EUC_JP, 	

	if ((shiftjisValidBytes >  eucJpValidBytes ) && (shiftjisValidBytes * 2> sampleLength))

		return EShiftjis;

	// if more than half is EUC_JP and more EUC_JP than shiftjis, 	

	if ((eucJpValidBytes >  shiftjisValidBytes ) && (eucJpValidBytes * 2> sampleLength))

		return EEucjp;

	// return the default

	return EUcs2;

	}

/**

 Check if UCS2.

 If the first two bytes are the Unicode Endian Specifiers (0xfffe or 0xfeff)

 then this must be UCS2. Otherwise try lookiing for  0x**00 or 0x00**

 @param A sample of data to be checked

 @param The Endianness if USC2 is detected

 @return ETrue if UCS2 else EFalse

 @internalTechnology 

 */

TBool CJ5Converter::DetectUcs2( const TDesC8& aForeign, 

	CCnvCharacterSetConverter::TEndianness& aTEndianness )

	{

	// if the sample is not big enough

	if (aForeign.Length() < 2)

		{

		return EFalse;

		}

	else if (aForeign[0]==0xff && aForeign[1]==0xfe )

		{ 

		// we have found a Little Endian Byte order mark

		aTEndianness = CCnvCharacterSetConverter::ELittleEndian;

		return ETrue;

		}

	else if (aForeign[0]==0xfe && aForeign[1]==0xff )

		{ 

		// we have found a Big Endian Byte order mark 

		aTEndianness = CCnvCharacterSetConverter::EBigEndian;

		return ETrue;

		}

	// Next check for sequences of 0x**00 or 0x00** as UCS-2 is the only charset that 

	// specifies 0x**00 or 0x00** (according to endianness) for the ASCII range of characters. 

	// NB: This will fail if there are no ASCII characters in the text.

	TInt sampleLength = aForeign.Length();

	sampleLength = Min(aForeign.Length(), KMaxSizeAutoDetectSample);;

	// check the sample for sequences of 0x**00	or 0x00**

	TInt bigEndianConfidence = 0;

	TInt littleEndianConfidence = 0;

	TInt i=0;

	for(;i< (sampleLength-1); i+=2)

		{

		if( aForeign[i] == 0x00)

			{

			bigEndianConfidence +=2;

			}

		else if ( aForeign[i+1] == 0x00)

			{

	 		littleEndianConfidence +=2;

			}

		}

	// which occurs most BE or LE	

	TInt confidenceLevel = 0;

	if (bigEndianConfidence > littleEndianConfidence)

		{

		aTEndianness = CCnvCharacterSetConverter::EBigEndian;

		confidenceLevel = bigEndianConfidence;

		}

	else

		{

		aTEndianness = CCnvCharacterSetConverter::ELittleEndian;

		confidenceLevel = littleEndianConfidence;

		}

	// if more than 97% count as UCS2

	if ( confidenceLevel * 100/sampleLength > 97) 

		return ETrue;

	return EFalse;

	}	

/**

 Check if ShiftJis (reference CJKV by Ken Lunde page 175)

 @param A sample of data to be checked

 @param The number of input bytes that can be converted

 @return The result of the check as either EIsCharacterSet, EIsNotCharacterSet or EMaybeCharacterSet

 @internalTechnology 

 */

enum CJ5Converter::TDectectCharacterSet CJ5Converter::DetectShiftJis( const TDesC8& aForeign,TInt &aNumberOfBytesConverted )

	{

	// Get the sample length

	TInt sampleLength = Min(aForeign.Length(), KMaxSizeAutoDetectSample);;

	TInt i=0;

	aNumberOfBytesConverted = 0;

	TText8 character;

	TText8 characterPlus1;

	TText8 characterPlus2;

	// scan the sample text looking for valid shiftjis data

	while ( i < sampleLength )

		{

		// get the next few characters, use 0 if there is no more sample

		// as this will not match any test.

		character = aForeign[i];

		characterPlus1 = ( i < (sampleLength-1) ? aForeign[i+1]:0);

		characterPlus2 = ( i < (sampleLength-2) ? aForeign[i+2]:0);

		// SHIFTJIS	- 0x8e to 0x9f followed by 0x40 to 0xfc  

		if ((character >= 0x81) && (character <= 0x9f) &&

				(characterPlus1 >= 0x40) && (characterPlus1 <= 0xfc) ) 

			{

			// this is SHIFTJIS unless it is EUC JP code set 2 or 3

			if ((character == 0x8E) && (characterPlus1 >= 0xA1) && (characterPlus1 <= 0xDF))

				{

				// this could be EUC JP code set 2 (or shiftjis)

				aNumberOfBytesConverted+=2;

				i++;

				}

			else if ((character == 0x8F) && 

				(characterPlus1 >= 0xA1) && (characterPlus1 <= 0xDF) &&

					(characterPlus2 >= 0xA1) && (characterPlus2 <= 0xDF))

				{

				// this could be EUC JP code set 3 (or shiftjis)

				aNumberOfBytesConverted+=3;

				i+=2;

				}

			else

				{

				// this can only be shift jis 

				return EIsCharacterSet;

				}

			}

		// SHIFTJIS	- 0xE0 to 0xEF followed by .....

		else if ((character >= 0xE0) && (character <= 0xEF))

			{

			// 0x40 to 0xFC which overlaps UTF8 between 0x80 and 0xBF  

			// including Mopera extension to shiftjis from 0xEF80 to 0xEFFC

			if ( (characterPlus1 >= 0x40) && (characterPlus1 <= 0x7E) ) 

				{

				// this can only be shift jis 

				return EIsCharacterSet;

				}

			else if ( (characterPlus1 >= 0xC0) && (characterPlus1 <= 0xFC) ) 

				{

				// this could be EUC JP code set 1

				aNumberOfBytesConverted+=2;

				i++;

				}

			// problem here is the overlap between the UTF8 and shiftjis

			else if ( (characterPlus1 >= 0x80) && (characterPlus1 <= 0xBF) )

				{

				// this could be shiftjis or utf8

				aNumberOfBytesConverted+=2;

				i++;

				}		

			}

		// half width katakana A1-DF	

		else if ((character >= 0xA1) && (character <= 0xDF))

			{

			aNumberOfBytesConverted+=1;

			}

		// ASCII or JIS-Roman 20-7e	

		else if ( ((character >= 0x20) && (character <= 0x7E)) || (character == 0x0A) || (character == 0x0D))

			{

			aNumberOfBytesConverted+=1;

			}

		else

			{

			// This is not decoding as shiftjis, so reject

			aNumberOfBytesConverted =0;

			return EIsNotCharacterSet;

			}

		i++;

		}

	// if all the characters could be converted

	if (aNumberOfBytesConverted == sampleLength)

		{

		return EIsCharacterSet;

		}

	else if (aNumberOfBytesConverted == 0)

		{

		return EIsNotCharacterSet;

		}

	else

		{

		return EMaybeCharacterSet;

		}

	}

/**

 Check if UTF8 (reference CJKV by Ken Lunde page 189)

 @param A sample of data to be checked

 @param The number of input bytes that can be converted

 @return The result of the check as either EIsCharacterSet, EIsNotCharacterSet or EMaybeCharacterSet

 @internalTechnology 

 */

enum CJ5Converter::TDectectCharacterSet CJ5Converter::DetectUtf8( const TDesC8& aForeign )

	{

	// Get the sample length

	TInt sampleLength = Min(aForeign.Length(), KMaxSizeAutoDetectSample);;

	TInt i=0;	

	TText8 character;

	TText8 characterPlus1;

	TText8 characterPlus2;

	TText8 characterPlus3;

	// scan the sample text looking for valid UTF8

	while ( i < sampleLength )

		{

		// get the next few characters, use 0 if there is no more sample

		// as this will not match any test.

		character = aForeign[i];

		characterPlus1 = ( i < (sampleLength-1) ? aForeign[i+1]:0);

		characterPlus2 = ( i < (sampleLength-2) ? aForeign[i+2]:0);

		characterPlus3 = ( i < (sampleLength-3) ? aForeign[i+3]:0);

		// UTF8 range 110xxxxx followed by one valid UTF8 bytes

		if(((character & 0xe0)==0xc0) && (( characterPlus1 & 0xc0)==0x80) )

			{

			// two bytes of valid UTF8 found

			i+=2;

			}

		// UTF8 range 1110xxxx followed by two valid UTF8 bytes

		else if(((character & 0xf0)==0xe0) && (( characterPlus1 & 0xc0)==0x80) && (( characterPlus2 & 0xc0)==0x80))

			{

			// three bytes of valid UTF8 found

			i+=3;

			}

		// UTF8 range 11110xxx followed by three valid UTF8 bytes

		else if(((character & 0xf8)==0xf0) && (( characterPlus1 & 0xc0)==0x80) 

				&& (( characterPlus2 & 0xc0)==0x80) && (( characterPlus3 & 0xc0)==0x80) )

			{

			// four bytes of valid UTF8 found

			i+=4;

			}

		// ascii range 0 to 0x7F	

		else if((character & 0x80)==0x00)

			{

			// The value of character is in the range 0x00-0x7f

			// UTF8 maintains ASCII transparency. So it's a valid UTF8.

			i++;

			}

		// if the sample data is longer than KMaxSizeAutoDetectSample then except anything

		// for the last two bytes as they may not appear valid without more data	

		else if( i >= (KMaxSizeAutoDetectSample -2) )

			{

			i++;

			}

		else

			{

			// This is not decoding as UTF8 so reject

			return EIsNotCharacterSet;

			}

		}	

	// All the characters could be converted