/*

* Copyright (c) 2005 Nokia Corporation and/or its subsidiary(-ies). 

* All rights reserved.

* This component and the accompanying materials are made available

* which accompanies this distribution, and is available

* at the URL "http://www.eclipse.org/legal/epl-v10.html".

*

* Initial Contributors:

* Nokia Corporation - initial contribution.

*

* Contributors:

*

* Description:      

*

*/

#include <e32std.h>

#include <e32base.h>

#include <e32test.h>

#include <f32file.h>

#include <charconv.h>

#include <shiftjis.h>

#include <utf.h>

// test data files

#ifdef __WINS__

	// on C drive for emulator

	_LIT16(KAnsiTxt,"c:\\test\\data\\Ansi.test");

	_LIT16(KImodeTxt,"c:\\test\\data\\imode.test");

	_LIT16(KJpEUCjTxt,"c:\\test\\data\\JpEUC-J.test");

	_LIT16(KJpJISTxt,"c:\\test\\data\\JpJIS.test");

	_LIT16(KJpSJISTxt,"c:\\test\\data\\JpSJIS.test");

	_LIT16(KJpUnicodeTxt,"c:\\test\\data\\JpUnicode.test");

	_LIT16(KJpUTF8Txt,"c:\\test\\data\\JpUTF8.test");

	_LIT16(KmoperaTxt,"c:\\test\\data\\mopera.test");

	_LIT16(KMScodeFontTxt,"c:\\test\\data\\MScodefont.test");

	_LIT16(KSJISTxt,"c:\\test\\data\\S-JIS.test");

	_LIT16(KAnsiUni,"c:\\test\\data\\Ansi.uni");

	_LIT16(KImodeUni,"c:\\test\\data\\imode.uni");

	_LIT16(KJpEUCjUni,"c:\\test\\data\\JpEUC-J.uni");

	_LIT16(KJpJISUni,"c:\\test\\data\\JpJIS.uni");

	_LIT16(KJpSJISUni,"c:\\test\\data\\JpSJIS.uni");

	_LIT16(KJpUnicodeUni,"c:\\test\\data\\JpUnicode.uni");

	_LIT16(KJpUTF8Uni,"c:\\test\\data\\JpUTF8.uni");

	_LIT16(KmoperaUni,"c:\\test\\data\\mopera.uni");

	_LIT16(KMScodeFontUni,"c:\\test\\data\\MScodefont.uni");

	_LIT16(KSJISUni,"c:\\test\\data\\S-JIS_docomo.uni");

#else

	// on Z drive for target ROM

	_LIT16(KAnsiTxt,"z:\\test\\data\\Ansi.test");

	_LIT16(KImodeTxt,"z:\\test\\data\\imode.test");

	_LIT16(KJpEUCjTxt,"z:\\test\\data\\JpEUC-J.test");

	_LIT16(KJpJISTxt,"z:\\test\\data\\JpJIS.test");

	_LIT16(KJpSJISTxt,"z:\\test\\data\\JpSJIS.test");

	_LIT16(KJpUnicodeTxt,"z:\\test\\data\\JpUnicode.test");

	_LIT16(KJpUTF8Txt,"z:\\test\\data\\JpUTF8.test");

	_LIT16(KmoperaTxt,"z:\\test\\data\\mopera.test");

	_LIT16(KMScodeFontTxt,"z:\\test\\data\\MScodefont.test");

	_LIT16(KSJISTxt,"z:\\test\\data\\S-JIS.test");

	_LIT16(KAnsiUni,"z:\\test\\data\\Ansi.uni");

	_LIT16(KImodeUni,"z:\\test\\data\\imode.uni");

	_LIT16(KJpEUCjUni,"z:\\test\\data\\JpEUC-J.uni");

	_LIT16(KJpJISUni,"z:\\test\\data\\JpJIS.uni");

	_LIT16(KJpSJISUni,"z:\\test\\data\\JpSJIS.uni");

	_LIT16(KJpUnicodeUni,"z:\\test\\data\\JpUnicode.uni");

	_LIT16(KJpUTF8Uni,"z:\\test\\data\\JpUTF8.uni");

	_LIT16(KmoperaUni,"z:\\test\\data\\mopera.uni");

	_LIT16(KMScodeFontUni,"z:\\test\\data\\MScodefont.uni");

	_LIT16(KSJISUni,"z:\\test\\data\\S-JIS_docomo.uni");

#endif

	_LIT16(KAnsiOut,"c:\\test\\data\\Ansi.out");

	_LIT16(KImodeOut,"c:\\test\\data\\imode.out");

	_LIT16(KJpEUCjOut,"c:\\test\\data\\JpEUC-J.out");

	_LIT16(KJpJISOut,"c:\\test\\data\\JpJIS.out");

	_LIT16(KJpSJISOut,"c:\\test\\data\\JpSJIS.out");

	_LIT16(KJpUnicodeOut,"c:\\test\\data\\JpUnicode.out");

	_LIT16(KJpUTF8Out,"c:\\test\\data\\JpUTF8.out");

	_LIT16(KmoperaOut,"c:\\test\\data\\mopera.out");

	_LIT16(KMScodeFontOut,"c:\\test\\data\\MScodefont.out");

	_LIT16(KSJISOut,"c:\\test\\data\\S-JIS_docomo.out");

// this has been copied from J5, as this not exported

// and only needed for testing   

enum TJ5Encoding

	{

	EShiftjis = 1,

	EIso2022jp1, 

	EEucjp, 

	EUtf8, 

	EUcs2,

	EUnknown

	};

// this is the test plug in rather than KCharacterSetIdentifierJ5

const TUint KCharacterSetTestJ5Docomo=0x01000002;

/**

  For development only! - used for generating unicode files

  Save the converted file as unicode

  */

#ifdef __WINS__

LOCAL_C void SaveConvertedUnicodeAsFile( RFs &fileServerSession, 

		const TDesC &aFileName, 

		const TDesC &aConvertedUnicode)

	{

 	RFile file;

	User::LeaveIfError( file.Replace( fileServerSession, aFileName, EFileWrite ));

	// get a pointer to 8bit data 	

	const TUint16* p16 = aConvertedUnicode.Ptr();

 	TPtrC8 p8( (TUint8 *)p16, aConvertedUnicode.Size() );

	file.Write( p8, aConvertedUnicode.Size());

	file.Close();

	}

#endif

/**

@SYMTestCaseID SYSLIB-CharConv-J5-U-0209-CP

@SYMTestCaseDesc Check that the J5 converter is available for testing

@SYMTestStatus Implemented

@SYMTestPriority Critical

@SYMTestActions The character conversion framework is called to check J5 is loaded.

@SYMTestExpectedResults J5 is in the list of avalable converters.

@SYMTestType Unit

@SYMREQ REQ4828

*/

LOCAL_C void checkJ5Available(RTest aTest, RFs &aFileServerSession)

	{	

	TBool J5found = EFalse;

	CArrayFix<CCnvCharacterSetConverter::SCharacterSet>* arrayOfCharacterSetsAvailable=CCnvCharacterSetConverter::CreateArrayOfCharacterSetsAvailableLC(aFileServerSession);

	// check the list of available converters looking for J5

	aTest.Printf(_L("charconv plugins available:\n"));

	for (TInt i=arrayOfCharacterSetsAvailable->Count()-1; i>=0; --i)

		{

		const CCnvCharacterSetConverter::SCharacterSet& charactersSet=(*arrayOfCharacterSetsAvailable)[i];

		TPtrC charactersSetName(charactersSet.Name());

		if (charactersSet.NameIsFileName())

			{

			charactersSetName.Set(TParsePtrC(charactersSetName).Name());

			if ( charactersSetName.Compare(_L("J5_DOCOMO")) == 0)

				J5found = ETrue;	

			}

		aTest.Printf(_L("    %S\n"), &charactersSetName);

		}

	if (J5found)

		aTest.Printf(_L("test_J5_DOCOMO available\n") );

	else

		{

		aTest.Printf(_L("Test failed test_J5_DOCOMO not available\n") );

		}

	aTest(J5found);

	// PopAndDestroy characterSetConverter 

	CleanupStack::PopAndDestroy(1);

	}

/**

@SYMTestCaseID SYSLIB-CharConv-J5-U-0210-CP

@SYMTestCaseDesc Convert the file supplied (aForeign) to unicode and then 

				 check the converted data against aExpectedUnicode

@SYMTestStatus Implemented

@SYMTestPriority Critical

@SYMTestActions The J5 character converter is used convert ansi.test.

@SYMTestExpectedResults The result is checked against the expected result imode.uni.

@SYMTestType Unit

@SYMREQ REQ4828

*/

LOCAL_C void convertFileUsingJ5L(RTest& aTest, 

	RFs fileServerSession,

	CCnvCharacterSetConverter*aJ5converter, 

	const TDesC &aForeign, 

	const TDesC &aExpectedUnicode,

	const TDesC &aOutputUnicode )

	{

	aTest.Printf(_L("Read %S check that it converts to %S \n"),&aForeign, &aExpectedUnicode);

  	// open the file & read into sourceBuffer

 	RFile source, expected;

 	User::LeaveIfError( source.Open( fileServerSession, aForeign, EFileRead ));

 	User::LeaveIfError( expected.Open( fileServerSession, aExpectedUnicode, EFileRead ));

 	TInt sourceSize = 0, expectedSize = 0;

 	source.Size(sourceSize);

 	expected.Size(expectedSize);

 	// create buffers for source file and expected unicode file

 	HBufC8 * sourceBuffer = HBufC8::NewLC(sourceSize);

 	TPtr8 sourcePtr(sourceBuffer->Des());

 	HBufC8 * expectedBuffer = HBufC8::NewLC(expectedSize);

 	TPtr8 expectedPtr(expectedBuffer->Des());

 	// buffer to convert into

 	HBufC * outputBuffer = HBufC::NewLC(expectedSize);

 	TPtr outputPtr(outputBuffer->Des());

	// read the whole source & output file into the buffer 	

	User::LeaveIfError( source.Read( sourcePtr, sourceSize ));

	User::LeaveIfError( expected.Read( expectedPtr, expectedSize ));

     // measure the time taken to convert.

    TTime startTime, stopTime;

    startTime.UniversalTime();

 	// now convert using J5 into unicode in the output buffer

	TInt state = CCnvCharacterSetConverter::KStateDefault;

	TInt error = KErrNone;

	error = aJ5converter->ConvertToUnicode(outputPtr, sourcePtr, state);

	if (error != KErrNone )

		{

		aTest.Printf(_L("ConvertToUnicode returned %d\n"), error);

		}

    stopTime.UniversalTime();

    TTimeIntervalMicroSeconds timeTaken = stopTime.MicroSecondsFrom(startTime);

 	TInt result = timeTaken.Int64();

    aTest.Printf(_L("Time for autodetect and convert to unicode %S using J5 %d microseconds\n"), 

    	&aForeign, result );

	TInt outputLength = outputBuffer->Length() ; // 16 bit words

	TInt expectedLength = expectedPtr.Length() / 2 ; // Bytes (hence divide by 2!)

	TInt checkLength = Min(outputLength, expectedLength) ;

	if (outputLength != expectedLength)

		{

		aTest.Printf(_L("Fail: Output length incorrect (expected %d bytes after conversion, got %d\n"),

		              expectedLength, outputLength) ;

		}

	// check that the output buffer contains the same as the expected buffer

	TInt j=0;

	TUint8 firstByte =0;

	TUint8 secondbyte =0;

	for ( TInt i=0; i < checkLength; i++, j+=2)

		{

		// two bytes in the file - one 16bit unicode value

		firstByte = TUint8(outputPtr[i] & 0x00ff);

		secondbyte = (outputPtr[i] & 0xff00) >> 8 ;

		if ( ( firstByte == expectedPtr[j] ) && ( secondbyte == expectedPtr[j+1] ) )

			{

			// useful for development

 			/*aTest.Printf(_L(" %02x%02x = %02x%02x ok\n"), 

				secondbyte, firstByte, expectedPtr[j+1], expectedPtr[j]);*/

			}

		else

			{

			aTest.Printf(_L("line:%d %02x%02x (output) != %02x%02x (expected) fail\n"), 

				i, secondbyte, firstByte, expectedPtr[j+1], expectedPtr[j]);

			}

		}

#ifdef __WINS__

	// for debugging save as file

	SaveConvertedUnicodeAsFile( fileServerSession, aOutputUnicode, outputPtr );

#endif

	source.Close();

	expected.Close();

	// sourceBuffer, expectedBufffer, outputBuffer

	CleanupStack::PopAndDestroy(3);

	}

/**

@SYMTestCaseID SYSLIB-CharConv-J5-U-0211-CP

@SYMTestCaseDesc Check that J5 converts to UTF8 when ConvertFromUnicode() is called

@SYMTestStatus Implemented

@SYMTestPriority Critical

@SYMTestActions The J5 character converter is used to convert unicode to UTF8

@SYMTestExpectedResults The result is checked against the expected UTF8.

@SYMTestType Unit

@SYMREQ REQ4828

*/

LOCAL_C void convertFromUnicodeUsingJ5(RTest& aTest, 

	RFs fileServerSession,

	CCnvCharacterSetConverter*aJ5converter )

	{

	/* sample UTF8 for testing */	

	_LIT16(KsampleUnicode,"\x0069\x002d\x006d\x006f\x0064\x0065\xFFFD");

	_LIT8(KsampleAsUTF8,"\x69\x2D\x6D\x6F\x64\x65\xEF\xBF\xBD");

	TBuf8<100> utf8buffer;

	aTest.Printf(_L("Check J5 convertd from Unicode to UTF8 \n"));

    CCnvCharacterSetConverter::TAvailability available = aJ5converter->PrepareToConvertToOrFromL(KCharacterSetTestJ5Docomo, fileServerSession);

    if (  available != CCnvCharacterSetConverter::EAvailable)

        User::Leave(KErrNotSupported);

 	// J5 convert from unicode

	TInt state = CCnvCharacterSetConverter::KStateDefault;

	TInt numberNotConverted =0;

	numberNotConverted = aJ5converter->ConvertFromUnicode(utf8buffer, KsampleUnicode, state);

	if (numberNotConverted != 0)

		{

		aTest.Printf(_L("numberNotConverted = %d"),	numberNotConverted );

		}

	// compare the UTF8 created to that supplied

	aTest(utf8buffer==KsampleAsUTF8);

	}

/**

@SYMTestCaseID SYSLIB-CharConv-J5-U-0212-CP

@SYMTestCaseDesc Check that the J5 converter autodetection

@SYMTestStatus Implemented

@SYMTestPriority Critical

@SYMTestActions The J5 character converter is used convert sample files.

@SYMTestExpectedResults The result is passed back for testing in the state paramater.

@SYMTestType Unit

@SYMREQ REQ4828

*/

LOCAL_C void checkDetection(RTest& aTest, 

	RFs fileServerSession,

	const TDesC &aForeign, 

	TInt aExpectedCode )

	{

	__UHEAP_MARK;

	aTest.Printf(_L("Read %S check that it converts to %d \n"),&aForeign, aExpectedCode);

  	// open the file & read into sourceBuffer

 	RFile source;

 	User::LeaveIfError( source.Open( fileServerSession, aForeign, EFileRead ));

 	TInt sourceSize = 0;

 	source.Size(sourceSize);

 	// create buffers for source file 

 	HBufC8 * sourceBuffer = HBufC8::NewLC(sourceSize);

 	TPtr8 sourcePtr(sourceBuffer->Des());

 	// buffer to convert into

 	HBufC * outputBuffer = HBufC::NewLC(sourceSize * 2);

 	TPtr outputPtr(outputBuffer->Des());

	// read the whole source & output file into the buffer 	

	User::LeaveIfError( source.Read( sourcePtr, sourceSize ));

 	source.Close();

    // load the converter ready for testing

 	CCnvCharacterSetConverter* J5Converter=CCnvCharacterSetConverter::NewLC();

    CCnvCharacterSetConverter::TAvailability available = CCnvCharacterSetConverter::EAvailable;

    available = J5Converter->PrepareToConvertToOrFromL(KCharacterSetTestJ5Docomo, fileServerSession);

    if (  available != CCnvCharacterSetConverter::EAvailable)

        User::Leave(KErrNotSupported);

 	// now convert using J5 into unicode in the output buffer

	TInt state = CCnvCharacterSetConverter::KStateDefault;

	TInt numberCharsNotConverted = 0;

 	numberCharsNotConverted = J5Converter->ConvertToUnicode(outputPtr, sourcePtr, state);

	if ( numberCharsNotConverted != 0)

		aTest.Printf(_L("number of characters not converted:%d of %d\n"), 

			numberCharsNotConverted, sourceSize );

	// check that J5 has been able to convert all the data	

 	aTest(J5Converter->ConvertToUnicode(outputPtr, sourcePtr, state)==0);

	// look at the code returned in the state

	if ( aExpectedCode == state )

		{

		aTest.Printf(_L(" expected:%d found:%d ok\n"), aExpectedCode , state  );

		}

	else

		{

		aTest.Printf(_L(" expected:%d found:%d failed \n"), aExpectedCode , state );

	 	aTest( aExpectedCode == state);

		}

	// sourceBuffer, expectedBufffer, J5Converter

	CleanupStack::PopAndDestroy(3);

	__UHEAP_MARKEND;

	}

/**

@SYMTestCaseID SYSLIB-CharConv-J5-U-0213-CP

@SYMTestCaseDesc Check the J5 on some small data samples

@SYMTestStatus Implemented

@SYMTestPriority Critical

@SYMTestActions The sample data is converted to unicode by J5.

@SYMTestExpectedResults The converted unicode data is checked against expected data.

@SYMTestType Unit

@SYMREQ REQ4828

*/

LOCAL_C void convertDataUsingJ5(RTest& aTest, 

	RFs &fileServerSession,

	const TDesC8 &aForeign, 

	const TDesC16 &aUnicode )

	{

    // for the unicode created

    TBuf16<512> unicode;

    // load the converter ready for testing

 	CCnvCharacterSetConverter* J5Converter=CCnvCharacterSetConverter::NewLC();

    CCnvCharacterSetConverter::TAvailability available = CCnvCharacterSetConverter::EAvailable;

    available = J5Converter->PrepareToConvertToOrFromL(KCharacterSetTestJ5Docomo, fileServerSession);

    if (  available != CCnvCharacterSetConverter::EAvailable)

        User::Leave(KErrNotSupported);

 	// J5 decode to unicode

	TInt state = CCnvCharacterSetConverter::KStateDefault;

	TInt numberNotConverted =0;

	numberNotConverted = J5Converter->ConvertToUnicode(unicode, aForeign, state);

	if (numberNotConverted != 0)

		{

		aTest.Printf(_L("numberNotConverted = %d"),	numberNotConverted );

		}

	// compare the unicode created to that supplied

	aTest(unicode==aUnicode);

	// tidy up J5Converter

	CleanupStack::PopAndDestroy(1);

	}

/**

@SYMTestCaseID SYSLIB-CharConv-J5-U-0214-CP

@SYMTestCaseDesc Check that J5 does not interfere with the normal operation of Autodetect

				 And make a rough timing measurement.

@SYMTestStatus Implemented

@SYMTestPriority Critical

@SYMTestActions Autodetect using other plugins is tested when J5 is available

@SYMTestExpectedResults Autodetect still works and does NOT select J5.

@SYMTestType Unit

@SYMREQ REQ4828

*/

LOCAL_C void checkNormalAutodetectL( RTest& aTest, RFs &aFileServerSession, const TDesC &aForeign )

	{

    aTest.Printf(_L("Test normal autodetect \n") );

	CCnvCharacterSetConverter* characterSetConverter=CCnvCharacterSetConverter::NewLC();

	CArrayFix<CCnvCharacterSetConverter::SCharacterSet>* arrayOfCharacterSetsAvailable=CCnvCharacterSetConverter::CreateArrayOfCharacterSetsAvailableLC(aFileServerSession);

  	// open the file & read into sourceBuffer

 	RFile source;

 	User::LeaveIfError( source.Open( aFileServerSession, aForeign, EFileRead ));

 	TInt sourceSize = 0;

 	source.Size(sourceSize);

 	// create buffers for source file 

 	HBufC8 * sourceBuffer = HBufC8::NewLC(sourceSize);

 	TPtr8 sourcePtr(sourceBuffer->Des());

	// read the whole source file into the buffer 	

	User::LeaveIfError( source.Read( sourcePtr, sourceSize ));

	source.Close();

    // measure the time taken to autodetect.

    TTime startTime, stopTime;

    startTime.UniversalTime();

	// do the autodetect

	TInt confidenceLevel =0;

	TUint characterSetIdentifier = 0;

	characterSetConverter->AutoDetectCharSetL( confidenceLevel, 

		characterSetIdentifier, *arrayOfCharacterSetsAvailable, sourcePtr);

	aTest(characterSetIdentifier == KCharacterSetIdentifierShiftJis);

    stopTime.UniversalTime();

    TTimeIntervalMicroSeconds timeTaken = stopTime.MicroSecondsFrom(startTime);

 	TInt result = timeTaken.Int64();

    aTest.Printf(_L("Time taken for old autodetect %S = %d microseconds\n"), 

    	&aForeign, result );

	// characterSetConverter,arrayOfCharacterSetsAvailable, sourceBuffer

	CleanupStack::PopAndDestroy(3);

	}

/**

@SYMTestCaseID SYSLIB-CharConv-J5-U-0215-CP

@SYMTestCaseDesc Check the conversion speed of shiftjis

@SYMTestStatus Implemented

@SYMTestPriority Critical

@SYMTestActions The sample data is converted to unicode by J5.

@SYMTestExpectedResults The converted unicode data is checked against expected data.

@SYMTestType Unit

@SYMREQ REQ4828

*/

LOCAL_C void checkConversionSpeedL(RTest& aTest, RFs &aFileServerSession, const TDesC &aForeign)

	{

    aTest.Printf(_L("Test conversion speed \n") );

  	// open the file & read into sourceBuffer

 	RFile source;

 	User::LeaveIfError( source.Open( aFileServerSession, aForeign, EFileRead ));

 	TInt sourceSize = 0, expectedSize = 0;

 	source.Size(sourceSize);

	// assume that every byte coverts to 2 bytes of unicode and the buffer will be big enough

	expectedSize = sourceSize * 2;

 	// create buffer for source file 

 	HBufC8 * sourceBuffer = HBufC8::NewLC(sourceSize);

 	TPtr8 sourcePtr(sourceBuffer->Des());

    // for the unicode created

 	HBufC * outputBuffer = HBufC::NewLC(expectedSize);

 	TPtr outputPtr(outputBuffer->Des());

    // load the converter ready for testing

 	CCnvCharacterSetConverter* charConverter=CCnvCharacterSetConverter::NewLC();

    CCnvCharacterSetConverter::TAvailability available = CCnvCharacterSetConverter::EAvailable;

    available = charConverter->PrepareToConvertToOrFromL(KCharacterSetIdentifierShiftJis, aFileServerSession);

    if (  available != CCnvCharacterSetConverter::EAvailable)

        User::Leave(KErrNotSupported);

	// read the whole source file into the buffer