/*

* Copyright (c) 1997-2009 Nokia Corporation and/or its subsidiary(-ies).

* All rights reserved.

* This component and the accompanying materials are made available

* under the terms of "Eclipse Public License v1.0"

* which accompanies this distribution, and is available

* at the URL "http://www.eclipse.org/legal/epl-v10.html".

*

* Initial Contributors:

* Nokia Corporation - initial contribution.

*

* Contributors:

*

* Description: 

*

*/

#include <e32std.h>

#include <e32base.h>

#include <f32file.h>

#include <charconv.h>

#include <convdata.h>

#include <utf.h>

#include <ecom/ecom.h>

#include <charactersetconverter.h>

#include "chcnvpanic.h"

#include "charconv_tls.h"

#if defined(_DEBUG)

#define NON_DEBUG_INLINE

#else

#define NON_DEBUG_INLINE inline

#endif

GLREF_D const SCnvConversionData codePage1252ConversionData;

GLREF_D const SCnvConversionData iso88591ConversionData;

GLREF_D const SCnvConversionData asciiConversionData;

GLREF_D const SCnvConversionData sms7BitConversionData;

GLREF_D const SCnvConversionData unicodeConversionDataLittle;

GLREF_D const SCnvConversionData unicodeConversionDataBig;

GLREF_C void IsCharacterSetAscii(TInt& aConfidenceLevel, const TDesC8& aSample);

GLREF_C void IsCharacterSetSMS7Bit(TInt& aConfidenceLevel, const TDesC8& aSample);

GLREF_C void IsCharacterSetISO88591(TInt& aConfidenceLevel, const TDesC8& aSample);

GLREF_C void IsCharacterSetCP1252(TInt& aConfidenceLevel, const TDesC8& aSample);

GLREF_C void IsCharacterSetUTF8(TInt& aConfidenceLevel, const TDesC8& aSample);

GLREF_C void IsCharacterSetUTF7(TInt& aConfidenceLevel, const TDesC8& aSample);

GLREF_C void IsCharacterSetUnicodeLittle(TInt& aConfidenceLevel, const TDesC8& aSample);

GLREF_C void IsCharacterSetUnicodeBig(TInt& aConfidenceLevel, const TDesC8& aSample);

_LIT(KLitWildCard, "*");

_LIT(KLitSystemCharconvDirectory, "\\resource\\charconv\\");

_LIT(KLitROMSystemCharconvDirectory, "z:\\resource\\charconv\\");

_LIT(KLitCharacterSetNameUtf7, "UTF-7");

_LIT(KLitCharacterSetNameUtf8, "UTF-8");

_LIT(KLitCharacterSetNameImapUtf7, "IMAP UTF-7");

_LIT(KLitCharacterSetNameJavaConformantUtf8, "Java UTF-8");

_LIT(KLitCharacterSetNameCodePage1252, "Code Page 1252");

_LIT(KLitCharacterSetNameIso88591, "ISO-8859-1");

_LIT(KLitCharacterSetNameAscii, "ASCII");

_LIT(KLitCharacterSetNameUnicodeLittle, "Little-Endian UNICODE");

_LIT(KLitCharacterSetNameUnicodeBig, "Big-Endian UNICODE");

_LIT(KLitCharacterSetNameSms7Bit, "SMS 7-bit");

_LIT8(KLit8AsciiSubstituteCharacter, "\x1a");

_LIT8(KLit8Sms7BitSubstituteCharacter, "\x3f");

const TUint KNoConversionAvailable=KMaxTUint;

const TUint KCharacterSetIdentifierWin1252=0x101F873B; // same as built-in cp1252, just to be compliance with S60 CP1252

enum

	{

	EConversionDataFilePositionOfName=24,

	EConversionDataFileMaximumLengthOfName=KMaxTUint8

	};

LOCAL_C TBool StandardNamesMatch(const TDesC8& aStandardName1, const TDesC8& aStandardName2)

	{

	return aStandardName1.CompareF(aStandardName2)==0; // "no distinction is made between use of upper and lower case letters" as stated by ftp.isi.edu/in-notes/iana/assignments/character-sets

	}

LOCAL_C void ResetAndDestroyRImplInfoPtrArray(TAny* aPtr)

	{

	RImplInfoPtrArray* array = reinterpret_cast <RImplInfoPtrArray*> (aPtr);

	array->ResetAndDestroy();

	}

LOCAL_C void CloseECOMSession(TAny*)

	{

	REComSession::FinalClose();

	}

static void IsBuiltInCharacterSet(const TUint& aCharacterSetIdentifier, 

								  TInt& aConfidenceLevel, 

								  const TDesC8& aSample)

    {

	switch(aCharacterSetIdentifier)

	    {

		case KCharacterSetIdentifierUtf7:

		    IsCharacterSetUTF7(aConfidenceLevel,aSample);

			break;

		case KCharacterSetIdentifierUtf8:

			IsCharacterSetUTF8(aConfidenceLevel,aSample);

			break;

		case KCharacterSetIdentifierImapUtf7:

		case KCharacterSetIdentifierJavaConformantUtf8:

			break;

		case KCharacterSetIdentifierCodePage1252:

		case KCharacterSetIdentifierWin1252:

			IsCharacterSetCP1252(aConfidenceLevel,aSample);

			break;

		case KCharacterSetIdentifierIso88591:

			IsCharacterSetISO88591(aConfidenceLevel,aSample);

			break;

		case KCharacterSetIdentifierAscii:

			IsCharacterSetAscii(aConfidenceLevel,aSample);

			break;

		case KCharacterSetIdentifierSms7Bit:

			IsCharacterSetSMS7Bit(aConfidenceLevel,aSample);

			break;

		case KCharacterSetIdentifierUnicodeLittle:

			IsCharacterSetUnicodeLittle(aConfidenceLevel,aSample);

			break;

		case KCharacterSetIdentifierUnicodeBig:

			IsCharacterSetUnicodeBig(aConfidenceLevel,aSample);

			break;

#if defined(_DEBUG)

		default:

			Panic(EPanicCharacterSetNotPresent);

			break;

#endif

		}

    }

void TTlsData::CharacterSetConverterIsBeingCreatedL()

	{

	TTlsData* tlsData=STATIC_CAST(TTlsData*, Dll::Tls());

	if (tlsData!=NULL)

		{

		++tlsData->iReferenceCount;

		}

	else

		{

		tlsData=new(ELeave) TTlsData;

		CleanupStack::PushL(tlsData);

		User::LeaveIfError(Dll::SetTls(tlsData));

		CleanupStack::Pop(); // tlsData

		}

	}

void TTlsData::CharacterSetConverterIsBeingDestroyed()

	{

	TTlsData* tlsData=STATIC_CAST(TTlsData*, Dll::Tls());

	if (tlsData!=NULL)

		{

		__ASSERT_DEBUG(tlsData->iCurrentCharacterSetConverter==NULL, Panic(EPanicDestructionDuringConversion));

		--tlsData->iReferenceCount;

		__ASSERT_DEBUG(tlsData->iReferenceCount>=0, Panic(EPanicBadTlsDataReferenceCount));

		if (tlsData->iReferenceCount<=0)

			{

			delete tlsData;

			Dll::FreeTls();

			}

		}

	}

void TTlsData::SetCurrentCharacterSetConverter(const CCnvCharacterSetConverter* aCharacterSetConverter)

	{

	TTlsData* tlsData=STATIC_CAST(TTlsData*, Dll::Tls());

	__ASSERT_ALWAYS(tlsData!=NULL, Panic(EPanicNoTlsData));

	__ASSERT_ALWAYS((tlsData->iCurrentCharacterSetConverter==NULL)!=(aCharacterSetConverter==NULL), Panic(EPanicBadToggleOfCurrentCharacterSetConverter));

	tlsData->iCurrentCharacterSetConverter=aCharacterSetConverter;

	}

EXPORT_C const CCnvCharacterSetConverter* TTlsData::CurrentCharacterSetConverter()

	{

	TTlsData* tlsData=STATIC_CAST(TTlsData*, Dll::Tls());

	return (tlsData!=NULL)? tlsData->iCurrentCharacterSetConverter: NULL;

	}

// CDeepDestructingArrayOfCharactersSets

NONSHARABLE_CLASS(CDeepDestructingArrayOfCharactersSets) : public CArrayFixFlat<CCnvCharacterSetConverter::SCharacterSet>

	{

public:

	static CDeepDestructingArrayOfCharactersSets* NewLC(TInt aGranularity);

	virtual ~CDeepDestructingArrayOfCharactersSets();

private:

	CDeepDestructingArrayOfCharactersSets(TInt aGranularity);

	};

CDeepDestructingArrayOfCharactersSets* CDeepDestructingArrayOfCharactersSets::NewLC(TInt aGranularity)

	{

	CDeepDestructingArrayOfCharactersSets* deepDestructingArrayOfCharactersSets=new(ELeave) CDeepDestructingArrayOfCharactersSets(aGranularity);

	CleanupStack::PushL(deepDestructingArrayOfCharactersSets);

	return deepDestructingArrayOfCharactersSets;

	}

CDeepDestructingArrayOfCharactersSets::~CDeepDestructingArrayOfCharactersSets()

	{

	for (TInt i=Count()-1; i>=0; --i)

		{

		delete (*this)[i].iName;

		}

	}

CDeepDestructingArrayOfCharactersSets::CDeepDestructingArrayOfCharactersSets(TInt aGranularity)

	:CArrayFixFlat<CCnvCharacterSetConverter::SCharacterSet>(aGranularity)

	{

	}

// CFileReader

NONSHARABLE_CLASS(CFileReader) : public CBase

	{

public:

	static CFileReader* NewLC(RFile& aFile);

	static CFileReader* NewLC(const TUint8* aRomFile, TInt aLengthOfRomFile);

	virtual ~CFileReader();

	void SkipL(TInt aNumberOfBytes);

	TInt ReadUint8L();

	TInt ReadUint16L();

	TUint ReadUint32L();

	TInt ReadPositiveIntegerCompacted15L();

	TInt ReadPositiveIntegerCompacted30L();

	TInt ReadSignedIntegerCompacted29L();

	void ReadBufferL(TDes8& aBuffer, TInt aBufferLength);

	HBufC8* ReadBufferL(TInt aBufferLength);

	HBufC8* ReadBufferLC(TInt aBufferLength);

private:

	enum {ENumberOfBytesToConsumeBetweenEachReAllocation=1000};

private:

	CFileReader();

	CFileReader(const TUint8* aRomFile, TInt aLengthOfRomFile);

	void ConstructForNonRomFileL(RFile& aFile);

	NON_DEBUG_INLINE void ReAllocateTheBuffer();

#if defined(_DEBUG)

	inline void CheckPointers(TPanic aPanic) {__ASSERT_DEBUG((iNextByteToConsume!=NULL) && (iOnePastEndOfBuffer!=NULL) && (iNextByteToConsume<=iOnePastEndOfBuffer), Panic(aPanic)); if (iBuffer==NULL) {__ASSERT_DEBUG(iFlagPoleForReAllocation==NULL, Panic(aPanic));} else {__ASSERT_DEBUG(iNextByteToConsume<iFlagPoleForReAllocation, Panic(aPanic));}}

#else

	inline void CheckPointers(TPanic) {}

#endif

private:

	TUint8* iBuffer;

	const TUint8* iNextByteToConsume;

	const TUint8* iOnePastEndOfBuffer;

	const TUint8* iFlagPoleForReAllocation;

	};

void CFileReader::ReAllocateTheBuffer() // put this function first so that the compiler does actually inline it for non-DEBUG builds

	{

	__ASSERT_DEBUG((iBuffer!=NULL) && (iNextByteToConsume>=iFlagPoleForReAllocation), Panic(EPanicNotPastFlagPoleForReAllocation));

	const TInt lengthOfBuffer=iOnePastEndOfBuffer-iNextByteToConsume;

	Mem::Copy(STATIC_CAST(TAny*, iBuffer), iNextByteToConsume, lengthOfBuffer);

	const TAny* reAllocatedCell=

	__ASSERT_DEBUG(reAllocatedCell==iBuffer, Panic(EPanicReAllocatedCellMoved));

	iNextByteToConsume=iBuffer;

	iOnePastEndOfBuffer=iBuffer+lengthOfBuffer;

	// iFlagPoleForReAllocation can stay as it is

	}

CFileReader* CFileReader::NewLC(RFile& aFile)

	{

	CFileReader* fileReader=new(ELeave) CFileReader;

	CleanupStack::PushL(fileReader);

	fileReader->ConstructForNonRomFileL(aFile);

	return fileReader;

	}

CFileReader* CFileReader::NewLC(const TUint8* aRomFile, TInt aLengthOfRomFile)

	{

	CFileReader* fileReader=new(ELeave) CFileReader(aRomFile, aLengthOfRomFile);

	CleanupStack::PushL(fileReader);

	return fileReader;

	}

CFileReader::~CFileReader()

	{

	User::Free(iBuffer);

	}

void CFileReader::SkipL(TInt aNumberOfBytes)

	{

	__ASSERT_DEBUG(aNumberOfBytes>=0, Panic(EPanicNegativeNumberOfBytes));

	CheckPointers(EPanicInconsistentFileReader1);

	const TUint8* newNextByteToConsume=iNextByteToConsume+aNumberOfBytes;

	if (newNextByteToConsume>iOnePastEndOfBuffer)

		{

		User::Leave(KErrCorrupt);

		}

	iNextByteToConsume=newNextByteToConsume;

	if ((iBuffer!=NULL) && (iNextByteToConsume>=iFlagPoleForReAllocation))

		{

		ReAllocateTheBuffer();

		}

	CheckPointers(EPanicInconsistentFileReader2);

	}

TInt CFileReader::ReadUint8L()

	{

	CheckPointers(EPanicInconsistentFileReader3);

	const TUint8* newNextByteToConsume=iNextByteToConsume+sizeof(TUint8);

	if (newNextByteToConsume>iOnePastEndOfBuffer)

		{

		User::Leave(KErrCorrupt);

		}

	const TInt integer=*iNextByteToConsume;

	iNextByteToConsume=newNextByteToConsume;

	if ((iBuffer!=NULL) && (iNextByteToConsume>=iFlagPoleForReAllocation))

		{

		ReAllocateTheBuffer();

		}

	CheckPointers(EPanicInconsistentFileReader4);

	return integer;

	}

TInt CFileReader::ReadUint16L() // little-endian

	{

	CheckPointers(EPanicInconsistentFileReader5);

	const TUint8* newNextByteToConsume=iNextByteToConsume+sizeof(TUint16);

	if (newNextByteToConsume>iOnePastEndOfBuffer)

		{

		User::Leave(KErrCorrupt);

		}

	const TInt integer=(*iNextByteToConsume|(*(iNextByteToConsume+1)<<8));

	iNextByteToConsume=newNextByteToConsume;

	if ((iBuffer!=NULL) && (iNextByteToConsume>=iFlagPoleForReAllocation))

		{

		ReAllocateTheBuffer();

		}

	CheckPointers(EPanicInconsistentFileReader6);

	return integer;

	}

TUint CFileReader::ReadUint32L() // little-endian

	{

	CheckPointers(EPanicInconsistentFileReader7);

	const TUint8* newNextByteToConsume=iNextByteToConsume+sizeof(TUint32);

	if (newNextByteToConsume>iOnePastEndOfBuffer)

		{

		User::Leave(KErrCorrupt);

		}

	const TInt integer=(*iNextByteToConsume|(*(iNextByteToConsume+1)<<8)|(*(iNextByteToConsume+2)<<16)|(*(iNextByteToConsume+3)<<24));

	iNextByteToConsume=newNextByteToConsume;

	if ((iBuffer!=NULL) && (iNextByteToConsume>=iFlagPoleForReAllocation))

		{

		ReAllocateTheBuffer();

		}

	CheckPointers(EPanicInconsistentFileReader8);

	return integer;

	}

TInt CFileReader::ReadPositiveIntegerCompacted15L() // big-endian

	{

	CheckPointers(EPanicInconsistentFileReader9);

	const TUint8* bytePointer=iNextByteToConsume;

	if (bytePointer>=iOnePastEndOfBuffer)

		{

		User::Leave(KErrCorrupt);

		}

	TInt integer=*bytePointer;

	if (integer&0x00000080)

		{

		integer&=~0x00000080;

		++bytePointer;

		if (bytePointer>=iOnePastEndOfBuffer)

			{

			User::Leave(KErrCorrupt);

			}

		integer<<=8;

		integer|=*bytePointer;

		}

	++bytePointer;

	iNextByteToConsume=bytePointer;

	if ((iBuffer!=NULL) && (iNextByteToConsume>=iFlagPoleForReAllocation))

		{

		ReAllocateTheBuffer();

		}

	CheckPointers(EPanicInconsistentFileReader10);

	return integer;

	}

TInt CFileReader::ReadPositiveIntegerCompacted30L() // big-endian

	{

	CheckPointers(EPanicInconsistentFileReader11);

	const TUint8* bytePointer=iNextByteToConsume;

	if (bytePointer>=iOnePastEndOfBuffer)

		{

		User::Leave(KErrCorrupt);

		}

	TInt integer=*bytePointer;

	if (integer&0x00000080)

		{

		integer&=~0x00000080;

		++bytePointer;

		if (bytePointer>=iOnePastEndOfBuffer)

			{

			User::Leave(KErrCorrupt);

			}

		integer<<=8;

		integer|=*bytePointer;

		if (integer&0x00004000)

			{

			integer&=~0x00004000;

			if (bytePointer+2>=iOnePastEndOfBuffer)

				{

				User::Leave(KErrCorrupt);

				}

			++bytePointer;

			integer<<=8;

			integer|=*bytePointer;

			++bytePointer;

			integer<<=8;

			integer|=*bytePointer;

			}

		}

	++bytePointer;

	iNextByteToConsume=bytePointer;

	if ((iBuffer!=NULL) && (iNextByteToConsume>=iFlagPoleForReAllocation))

		{

		ReAllocateTheBuffer();

		}

	CheckPointers(EPanicInconsistentFileReader12);

	return integer;

	}

TInt CFileReader::ReadSignedIntegerCompacted29L() // big-endian

	{

	CheckPointers(EPanicInconsistentFileReader13);

	const TUint8* bytePointer=iNextByteToConsume;

	if (bytePointer>=iOnePastEndOfBuffer)

		{

		User::Leave(KErrCorrupt);

		}

	TInt integer=*bytePointer;

	const TBool isNegative=(integer&0x00000080);

	integer&=~0x00000080;

	if (integer&0x00000040)

		{

		integer&=~0x00000040;

		++bytePointer;

		if (bytePointer>=iOnePastEndOfBuffer)

			{

			User::Leave(KErrCorrupt);

			}

		integer<<=8;

		integer|=*bytePointer;

		if (integer&0x00002000)

			{

			integer&=~0x00002000;

			if (bytePointer+2>=iOnePastEndOfBuffer)

				{

				User::Leave(KErrCorrupt);

				}

			++bytePointer;

			integer<<=8;

			integer|=*bytePointer;

			++bytePointer;

			integer<<=8;

			integer|=*bytePointer;

			}

		}

	++bytePointer;

	if (isNegative)

		{

		integer=-integer;

		}

	iNextByteToConsume=bytePointer;

	if ((iBuffer!=NULL) && (iNextByteToConsume>=iFlagPoleForReAllocation))

		{

		ReAllocateTheBuffer();

		}

	CheckPointers(EPanicInconsistentFileReader14);

	return isNegative? -integer: integer;

	}

void CFileReader::ReadBufferL(TDes8& aBuffer, TInt aBufferLength)

	{

	__ASSERT_DEBUG(aBufferLength>=0, Panic(EPanicNegativeBufferLength1));

	CheckPointers(EPanicInconsistentFileReader15);

	const TUint8* newNextByteToConsume=iNextByteToConsume+aBufferLength;

	if (newNextByteToConsume>iOnePastEndOfBuffer)

		{

		User::Leave(KErrCorrupt);

		}

	aBuffer=TPtrC8(iNextByteToConsume, aBufferLength);

	iNextByteToConsume=newNextByteToConsume;

	if ((iBuffer!=NULL) && (iNextByteToConsume>=iFlagPoleForReAllocation))

		{

		ReAllocateTheBuffer();

		}

	CheckPointers(EPanicInconsistentFileReader16);

	}

HBufC8* CFileReader::ReadBufferL(TInt aBufferLength)

	{

	__ASSERT_DEBUG(aBufferLength>=0, Panic(EPanicNegativeBufferLength2));

	CheckPointers(EPanicInconsistentFileReader17);

	const TUint8* newNextByteToConsume=iNextByteToConsume+aBufferLength;

	if (newNextByteToConsume>iOnePastEndOfBuffer)

		{

		User::Leave(KErrCorrupt);

		}

	HBufC8* buffer=TPtrC8(iNextByteToConsume, aBufferLength).AllocL();

	iNextByteToConsume=newNextByteToConsume;

	if ((iBuffer!=NULL) && (iNextByteToConsume>=iFlagPoleForReAllocation))

		{

		ReAllocateTheBuffer();

		}

	CheckPointers(EPanicInconsistentFileReader18);

	return buffer;

	}

HBufC8* CFileReader::ReadBufferLC(TInt aBufferLength)

	{

	HBufC8* buffer=ReadBufferL(aBufferLength);

	CleanupStack::PushL(buffer);

	return buffer;

	}

CFileReader::CFileReader()