--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/charconvfw/Charconv/ongoing/test/source/main/TJ5.CPP Fri Jun 04 10:37:54 2010 +0100
@@ -0,0 +1,769 @@
+/*
+* Copyright (c) 2005 Nokia Corporation and/or its subsidiary(-ies).
+* All rights reserved.
+* This component and the accompanying materials are made available
+* under the terms of the License "Eclipse Public License v1.0"
+* which accompanies this distribution, and is available
+* at the URL "http://www.eclipse.org/legal/epl-v10.html".
+*
+* Initial Contributors:
+* Nokia Corporation - initial contribution.
+*
+* Contributors:
+*
+* Description:
+*
+*/
+
+
+
+
+
+
+
+
+#include <e32std.h>
+#include <e32base.h>
+#include <e32test.h>
+#include <f32file.h>
+#include <charconv.h>
+#include <shiftjis.h>
+#include <utf.h>
+
+// test data files
+#ifdef __WINS__
+ // on C drive for emulator
+ _LIT16(KAnsiTxt,"c:\\test\\data\\Ansi.test");
+ _LIT16(KImodeTxt,"c:\\test\\data\\imode.test");
+ _LIT16(KJpEUCjTxt,"c:\\test\\data\\JpEUC-J.test");
+ _LIT16(KJpJISTxt,"c:\\test\\data\\JpJIS.test");
+ _LIT16(KJpSJISTxt,"c:\\test\\data\\JpSJIS.test");
+ _LIT16(KJpUnicodeTxt,"c:\\test\\data\\JpUnicode.test");
+ _LIT16(KJpUTF8Txt,"c:\\test\\data\\JpUTF8.test");
+ _LIT16(KmoperaTxt,"c:\\test\\data\\mopera.test");
+ _LIT16(KMScodeFontTxt,"c:\\test\\data\\MScodefont.test");
+ _LIT16(KSJISTxt,"c:\\test\\data\\S-JIS.test");
+
+ _LIT16(KAnsiUni,"c:\\test\\data\\Ansi.uni");
+ _LIT16(KImodeUni,"c:\\test\\data\\imode_not_mapped.uni");
+ _LIT16(KJpEUCjUni,"c:\\test\\data\\JpEUC-J.uni");
+ _LIT16(KJpJISUni,"c:\\test\\data\\JpJIS.uni");
+ _LIT16(KJpSJISUni,"c:\\test\\data\\JpSJIS.uni");
+ _LIT16(KJpUnicodeUni,"c:\\test\\data\\JpUnicode.uni");
+ _LIT16(KJpUTF8Uni,"c:\\test\\data\\JpUTF8.uni");
+ _LIT16(KmoperaUni,"c:\\test\\data\\mopera_not_mapped.uni");
+ _LIT16(KMScodeFontUni,"c:\\test\\data\\MScodefont.uni");
+ _LIT16(KSJISUni,"c:\\test\\data\\S-JIS.uni");
+#else
+ // on Z drive for target ROM
+ _LIT16(KAnsiTxt,"z:\\test\\data\\Ansi.test");
+ _LIT16(KImodeTxt,"z:\\test\\data\\imode.test");
+ _LIT16(KJpEUCjTxt,"z:\\test\\data\\JpEUC-J.test");
+ _LIT16(KJpJISTxt,"z:\\test\\data\\JpJIS.test");
+ _LIT16(KJpSJISTxt,"z:\\test\\data\\JpSJIS.test");
+ _LIT16(KJpUnicodeTxt,"z:\\test\\data\\JpUnicode.test");
+ _LIT16(KJpUTF8Txt,"z:\\test\\data\\JpUTF8.test");
+ _LIT16(KmoperaTxt,"z:\\test\\data\\mopera.test");
+ _LIT16(KMScodeFontTxt,"z:\\test\\data\\MScodefont.test");
+ _LIT16(KSJISTxt,"z:\\test\\data\\S-JIS.test");
+
+ _LIT16(KAnsiUni,"z:\\test\\data\\Ansi.uni");
+ _LIT16(KImodeUni,"z:\\test\\data\\imode_not_mapped.uni");
+ _LIT16(KJpEUCjUni,"z:\\test\\data\\JpEUC-J.uni");
+ _LIT16(KJpJISUni,"z:\\test\\data\\JpJIS.uni");
+ _LIT16(KJpSJISUni,"z:\\test\\data\\JpSJIS.uni");
+ _LIT16(KJpUnicodeUni,"z:\\test\\data\\JpUnicode.uni");
+ _LIT16(KJpUTF8Uni,"z:\\test\\data\\JpUTF8.uni");
+ _LIT16(KmoperaUni,"z:\\test\\data\\mopera_not_mapped.uni");
+ _LIT16(KMScodeFontUni,"z:\\test\\data\\MScodefont.uni");
+ _LIT16(KSJISUni,"z:\\test\\data\\S-JIS.uni");
+#endif
+
+ _LIT16(KAnsiOut,"c:\\test\\data\\Ansi.out");
+ _LIT16(KImodeOut,"c:\\test\\data\\imode_not_mapped.out");
+ _LIT16(KJpEUCjOut,"c:\\test\\data\\JpEUC-J.out");
+ _LIT16(KJpJISOut,"c:\\test\\data\\JpJIS.out");
+ _LIT16(KJpSJISOut,"c:\\test\\data\\JpSJIS.out");
+ _LIT16(KJpUnicodeOut,"c:\\test\\data\\JpUnicode.out");
+ _LIT16(KJpUTF8Out,"c:\\test\\data\\JpUTF8.out");
+ _LIT16(KmoperaOut,"c:\\test\\data\\mopera_not_mapped.out");
+ _LIT16(KMScodeFontOut,"c:\\test\\data\\MScodefont.out");
+ _LIT16(KSJISOut,"c:\\test\\data\\S-JIS.out");
+
+// this has been copied from J5, as this not exported
+// and only needed for testing
+enum TJ5Encoding
+ {
+ EShiftjis = 1,
+ EIso2022jp1,
+ EEucjp,
+ EUtf8,
+ EUcs2,
+ EUnknown
+ };
+
+
+#ifdef __WINS__
+/**
+ For development only! - used for generating unicode files
+ Save the converted file as unicode
+ */
+LOCAL_C void SaveConvertedUnicodeAsFile( RFs &fileServerSession,
+ const TDesC &aFileName,
+ const TDesC &aConvertedUnicode)
+ {
+ RFile file;
+ User::LeaveIfError( file.Replace( fileServerSession, aFileName, EFileWrite ));
+
+ // get a pointer to 8bit data
+ const TUint16* p16 = aConvertedUnicode.Ptr();
+ TPtrC8 p8( (TUint8 *)p16, aConvertedUnicode.Size() );
+
+
+ file.Write( p8, aConvertedUnicode.Size());
+ file.Close();
+ }
+#endif
+
+
+/**
+@SYMTestCaseID SYSLIB-CharConv-J5-U-0209-CP
+@SYMTestCaseDesc Check that the J5 converter is available for testing
+@SYMTestStatus Implemented
+@SYMTestPriority Critical
+@SYMTestActions The character conversion framework is called to check J5 is loaded.
+@SYMTestExpectedResults J5 is in the list of avalable converters.
+@SYMTestType Unit
+@SYMREQ REQ4828
+*/
+LOCAL_C void checkJ5Available(RTest aTest, RFs &aFileServerSession)
+ {
+ TBool J5found = EFalse;
+
+ CArrayFix<CCnvCharacterSetConverter::SCharacterSet>* arrayOfCharacterSetsAvailable=CCnvCharacterSetConverter::CreateArrayOfCharacterSetsAvailableLC(aFileServerSession);
+
+ // check the list of available converters looking for J5
+ aTest.Printf(_L("charconv plugins available:\n"));
+ for (TInt i=arrayOfCharacterSetsAvailable->Count()-1; i>=0; --i)
+ {
+ const CCnvCharacterSetConverter::SCharacterSet& charactersSet=(*arrayOfCharacterSetsAvailable)[i];
+ TPtrC charactersSetName(charactersSet.Name());
+ if (charactersSet.NameIsFileName())
+ {
+ charactersSetName.Set(TParsePtrC(charactersSetName).Name());
+
+ if ( charactersSetName.Compare(_L("J5")) == 0)
+ J5found = ETrue;
+ }
+ aTest.Printf(_L(" %S\n"), &charactersSetName);
+ }
+
+ if (J5found)
+ aTest.Printf(_L("J5 available\n") );
+ else
+ {
+ aTest.Printf(_L("Test failed J5 not available\n") );
+ }
+
+ aTest(J5found);
+
+ // PopAndDestroy arrayOfCharacterSetsAvailable
+ CleanupStack::PopAndDestroy(1);
+ }
+
+/**
+@SYMTestCaseID SYSLIB-CharConv-J5-U-0210-CP
+@SYMTestCaseDesc Convert the file supplied (aForeign) to unicode and then
+ check the converted data against aExpectedUnicode
+@SYMTestStatus Implemented
+@SYMTestPriority Critical
+@SYMTestActions The J5 character converter is used convert ansi.test.
+@SYMTestExpectedResults The result is checked against the expected result imode.uni.
+@SYMTestType Unit
+@SYMREQ REQ4828
+*/
+LOCAL_C void convertFileUsingJ5L(RTest& aTest,
+ RFs fileServerSession,
+ CCnvCharacterSetConverter*aJ5converter,
+ const TDesC &aForeign,
+ const TDesC &aExpectedUnicode,
+ const TDesC &aOutputUnicode )
+ {
+ aTest.Printf(_L("Read %S check that it converts to %S \n"),&aForeign, &aExpectedUnicode);
+
+ // open the file & read into sourceBuffer
+ RFile source, expected;
+ User::LeaveIfError( source.Open( fileServerSession, aForeign, EFileRead ));
+ User::LeaveIfError( expected.Open( fileServerSession, aExpectedUnicode, EFileRead ));
+ TInt sourceSize = 0, expectedSize = 0;
+ source.Size(sourceSize);
+ expected.Size(expectedSize);
+
+ // create buffers for source file and expected unicode file
+ HBufC8 * sourceBuffer = HBufC8::NewLC(sourceSize);
+ TPtr8 sourcePtr(sourceBuffer->Des());
+ HBufC8 * expectedBuffer = HBufC8::NewLC(expectedSize);
+ TPtr8 expectedPtr(expectedBuffer->Des());
+
+ // buffer to convert into
+ HBufC * outputBuffer = HBufC::NewLC(expectedSize);
+ TPtr outputPtr(outputBuffer->Des());
+
+ // read the whole source & output file into the buffer
+ User::LeaveIfError( source.Read( sourcePtr, sourceSize ));
+ User::LeaveIfError( expected.Read( expectedPtr, expectedSize ));
+
+ // measure the time taken to convert.
+ TTime startTime, stopTime;
+ startTime.UniversalTime();
+
+ // now convert using J5 into unicode in the output buffer
+ TInt state = CCnvCharacterSetConverter::KStateDefault;
+ TInt error = KErrNone;
+ error = aJ5converter->ConvertToUnicode(outputPtr, sourcePtr, state);
+ if (error != KErrNone )
+ {
+ aTest.Printf(_L("ConvertToUnicode returned %d\n"), error);
+ }
+
+ stopTime.UniversalTime();
+ TTimeIntervalMicroSeconds timeTaken = stopTime.MicroSecondsFrom(startTime);
+ TInt result = timeTaken.Int64();
+ aTest.Printf(_L("Time for autodetect and convert to unicode %S using J5 %d microseconds\n"),
+ &aForeign, result );
+
+ TInt outputLength = outputBuffer->Length() ; // 16 bit words
+ TInt expectedLength = expectedPtr.Length() / 2 ; // Bytes (hence divide by 2!)
+ TInt checkLength = Min(outputLength, expectedLength) ;
+
+ if (outputLength != expectedLength)
+ {
+ aTest.Printf(_L("Fail: Output length incorrect (expected %d bytes after conversion, got %d\n"),
+ expectedLength, outputLength) ;
+ }
+ // check that the output buffer contains the same as the expected buffer
+ TInt j=0;
+ TUint8 firstByte =0;
+ TUint8 secondbyte =0;
+
+ for ( TInt i=0; i < checkLength; i++, j+=2)
+ {
+ // two bytes in the file - one 16bit unicode value
+ firstByte = TUint8(outputPtr[i] & 0x00ff);
+ secondbyte = (outputPtr[i] & 0xff00) >> 8 ;
+
+ if ( ( firstByte == expectedPtr[j] ) && ( secondbyte == expectedPtr[j+1] ) )
+ {
+ // useful for development
+ /*aTest.Printf(_L(" %02x%02x = %02x%02x ok\n"),
+ secondbyte, firstByte, expectedPtr[j+1], expectedPtr[j]);*/
+ }
+ else
+ {
+ aTest.Printf(_L("line:%d %02x%02x (output) != %02x%02x (expected) fail\n"),
+ i, secondbyte, firstByte, expectedPtr[j+1], expectedPtr[j]);
+ }
+ }
+
+#ifdef __WINS__
+ // for debugging save as file
+ SaveConvertedUnicodeAsFile( fileServerSession, aOutputUnicode, outputPtr );
+#endif
+
+ source.Close();
+ expected.Close();
+
+ // sourceBuffer, expectedBufffer, outputBuffer
+ CleanupStack::PopAndDestroy(3);
+ }
+
+
+/**
+@SYMTestCaseID SYSLIB-CharConv-J5-U-0211-CP
+@SYMTestCaseDesc Check that J5 converts to UTF8 when ConvertFromUnicode() is called
+@SYMTestStatus Implemented
+@SYMTestPriority Critical
+@SYMTestActions The J5 character converter is used to convert unicode to UTF8
+@SYMTestExpectedResults The result is checked against the expected UTF8.
+@SYMTestType Unit
+@SYMREQ REQ4828
+*/
+LOCAL_C void convertFromUnicodeUsingJ5(RTest& aTest,
+ RFs fileServerSession,
+ CCnvCharacterSetConverter*aJ5converter )
+ {
+ /* sample UTF8 for testing */
+ _LIT16(KsampleUnicode,"\x0069\x002d\x006d\x006f\x0064\x0065\xFFFD");
+ _LIT8(KsampleAsUTF8,"\x69\x2D\x6D\x6F\x64\x65\xEF\xBF\xBD");
+ TBuf8<100> utf8buffer;
+
+ aTest.Printf(_L("Check J5 convertd from Unicode to UTF8 \n"));
+
+ CCnvCharacterSetConverter::TAvailability available = aJ5converter->PrepareToConvertToOrFromL(KCharacterSetIdentifierJ5, fileServerSession);
+ if ( available != CCnvCharacterSetConverter::EAvailable)
+ User::Leave(KErrNotSupported);
+
+ // J5 convert from unicode
+ TInt state = CCnvCharacterSetConverter::KStateDefault;
+ TInt numberNotConverted =0;
+ numberNotConverted = aJ5converter->ConvertFromUnicode(utf8buffer, KsampleUnicode, state);
+ if (numberNotConverted != 0)
+ {
+ aTest.Printf(_L("numberNotConverted = %d"), numberNotConverted );
+ }
+
+ // compare the UTF8 created to that supplied
+ aTest(utf8buffer==KsampleAsUTF8);
+
+ }
+
+/**
+@SYMTestCaseID SYSLIB-CharConv-J5-U-0212-CP
+@SYMTestCaseDesc Check that the J5 converter autodetection
+@SYMTestStatus Implemented
+@SYMTestPriority Critical
+@SYMTestActions The J5 character converter is used convert sample files.
+@SYMTestExpectedResults The result is passed back for testing in the state paramater.
+@SYMTestType Unit
+@SYMREQ REQ4828
+*/
+LOCAL_C void checkDetection(RTest& aTest,
+ RFs fileServerSession,
+ const TDesC &aForeign,
+ TInt aExpectedCode )
+ {
+ __UHEAP_MARK;
+ aTest.Printf(_L("Read %S check that it converts to %d \n"),&aForeign, aExpectedCode);
+
+ // open the file & read into sourceBuffer
+ RFile source;
+ User::LeaveIfError( source.Open( fileServerSession, aForeign, EFileRead ));
+ TInt sourceSize = 0;
+ source.Size(sourceSize);
+
+ // create buffers for source file
+ HBufC8 * sourceBuffer = HBufC8::NewLC(sourceSize);
+ TPtr8 sourcePtr(sourceBuffer->Des());
+
+ // buffer to convert into
+ HBufC * outputBuffer = HBufC::NewLC(sourceSize * 2);
+ TPtr outputPtr(outputBuffer->Des());
+
+ // read the whole source & output file into the buffer
+ User::LeaveIfError( source.Read( sourcePtr, sourceSize ));
+ source.Close();
+
+ // load the converter ready for testing
+ CCnvCharacterSetConverter* J5Converter=CCnvCharacterSetConverter::NewLC();
+ CCnvCharacterSetConverter::TAvailability available = CCnvCharacterSetConverter::EAvailable;
+ available = J5Converter->PrepareToConvertToOrFromL(KCharacterSetIdentifierJ5, fileServerSession);
+ if ( available != CCnvCharacterSetConverter::EAvailable)
+ User::Leave(KErrNotSupported);
+
+ // now convert using J5 into unicode in the output buffer
+ TInt state = CCnvCharacterSetConverter::KStateDefault;
+ TInt numberCharsNotConverted = 0;
+ numberCharsNotConverted = J5Converter->ConvertToUnicode(outputPtr, sourcePtr, state);
+ if ( numberCharsNotConverted != 0)
+ aTest.Printf(_L("number of characters not converted:%d of %d\n"),
+ numberCharsNotConverted, sourceSize );
+
+ // check that J5 has been able to convert all the data
+ aTest(J5Converter->ConvertToUnicode(outputPtr, sourcePtr, state)==0);
+
+ // look at the code returned in the state
+ if ( aExpectedCode == state )
+ {
+ aTest.Printf(_L(" expected:%d found:%d ok\n"), aExpectedCode , state );
+ }
+ else
+ {
+ aTest.Printf(_L(" expected:%d found:%d failed \n"), aExpectedCode , state );
+ aTest( aExpectedCode == state);
+ }
+
+ // sourceBuffer, expectedBufffer, J5Converter
+ CleanupStack::PopAndDestroy(3);
+ __UHEAP_MARKEND;
+ }
+
+
+/**
+@SYMTestCaseID SYSLIB-CharConv-J5-U-0213-CP
+@SYMTestCaseDesc Check the J5 on some small data samples
+@SYMTestStatus Implemented
+@SYMTestPriority Critical
+@SYMTestActions The sample data is converted to unicode by J5.
+@SYMTestExpectedResults The converted unicode data is checked against expected data.
+@SYMTestType Unit
+@SYMREQ REQ4828
+*/
+LOCAL_C void convertDataUsingJ5(RTest& aTest,
+ RFs &fileServerSession,
+ const TDesC8 &aForeign,
+ const TDesC16 &aUnicode )
+ {
+ // for the unicode created
+ TBuf16<512> unicode;
+
+ // load the converter ready for testing
+ CCnvCharacterSetConverter* J5Converter=CCnvCharacterSetConverter::NewLC();
+ CCnvCharacterSetConverter::TAvailability available = CCnvCharacterSetConverter::EAvailable;
+ available = J5Converter->PrepareToConvertToOrFromL(KCharacterSetIdentifierJ5, fileServerSession);
+ if ( available != CCnvCharacterSetConverter::EAvailable)
+ User::Leave(KErrNotSupported);
+
+ // J5 decode to unicode
+ TInt state = CCnvCharacterSetConverter::KStateDefault;
+ TInt numberNotConverted =0;
+ numberNotConverted = J5Converter->ConvertToUnicode(unicode, aForeign, state);
+ if (numberNotConverted != 0)
+ {
+ aTest.Printf(_L("numberNotConverted = %d"), numberNotConverted );
+ }
+
+ // compare the unicode created to that supplied
+ aTest(unicode==aUnicode);
+
+ // tidy up J5Converter
+ CleanupStack::PopAndDestroy(1);
+ }
+
+
+/**
+@SYMTestCaseID SYSLIB-CharConv-J5-U-0214-CP
+@SYMTestCaseDesc Check that J5 does not interfere with the normal operation of Autodetect
+ And make a rough timing measurement.
+@SYMTestStatus Implemented
+@SYMTestPriority Critical
+@SYMTestActions Autodetect using other plugins is tested when J5 is available
+@SYMTestExpectedResults Autodetect still works and does NOT select J5.
+@SYMTestType Unit
+@SYMREQ REQ4828
+*/
+LOCAL_C void checkNormalAutodetectL( RTest& aTest, RFs &aFileServerSession, const TDesC &aForeign )
+ {
+ aTest.Printf(_L("Test normal autodetect \n") );
+ CCnvCharacterSetConverter* characterSetConverter=CCnvCharacterSetConverter::NewLC();
+ CArrayFix<CCnvCharacterSetConverter::SCharacterSet>* arrayOfCharacterSetsAvailable=CCnvCharacterSetConverter::CreateArrayOfCharacterSetsAvailableLC(aFileServerSession);
+
+ // open the file & read into sourceBuffer
+ RFile source;
+ User::LeaveIfError( source.Open( aFileServerSession, aForeign, EFileRead ));
+ TInt sourceSize = 0;
+ source.Size(sourceSize);
+
+ // create buffers for source file
+ HBufC8 * sourceBuffer = HBufC8::NewLC(sourceSize);
+ TPtr8 sourcePtr(sourceBuffer->Des());
+
+ // read the whole source file into the buffer
+ User::LeaveIfError( source.Read( sourcePtr, sourceSize ));
+ source.Close();
+
+ // measure the time taken to autodetect.
+ TTime startTime, stopTime;
+ startTime.UniversalTime();
+
+ // do the autodetect
+ TInt confidenceLevel =0;
+ TUint characterSetIdentifier = 0;
+ characterSetConverter->AutoDetectCharSetL( confidenceLevel,
+ characterSetIdentifier, *arrayOfCharacterSetsAvailable, sourcePtr);
+
+ aTest(characterSetIdentifier == KCharacterSetIdentifierShiftJis);
+
+ stopTime.UniversalTime();
+ TTimeIntervalMicroSeconds timeTaken = stopTime.MicroSecondsFrom(startTime);
+ TInt result = timeTaken.Int64();
+ aTest.Printf(_L("Time taken for old autodetect %S = %d microseconds\n"),
+ &aForeign, result );
+
+ // characterSetConverter,arrayOfCharacterSetsAvailable, sourceBuffer
+ CleanupStack::PopAndDestroy(3);
+ }
+
+
+/**
+@SYMTestCaseID SYSLIB-CharConv-J5-U-0215-CP
+@SYMTestCaseDesc Check the conversion speed of shiftjis
+@SYMTestStatus Implemented
+@SYMTestPriority Critical
+@SYMTestActions The sample data is converted to unicode by J5.
+@SYMTestExpectedResults The converted unicode data is checked against expected data.
+@SYMTestType Unit
+@SYMREQ REQ4828
+*/
+LOCAL_C void checkConversionSpeedL(RTest& aTest, RFs &aFileServerSession, const TDesC &aForeign)
+ {
+ aTest.Printf(_L("Test conversion speed \n") );
+
+ // open the file & read into sourceBuffer
+ RFile source;
+ User::LeaveIfError( source.Open( aFileServerSession, aForeign, EFileRead ));
+ TInt sourceSize = 0, expectedSize = 0;
+ source.Size(sourceSize);
+
+ // assume that every byte coverts to 2 bytes of unicode and the buffer will be big enough
+ expectedSize = sourceSize * 2;
+
+ // create buffer for source file
+ HBufC8 * sourceBuffer = HBufC8::NewLC(sourceSize);
+ TPtr8 sourcePtr(sourceBuffer->Des());
+
+ // for the unicode created
+ HBufC * outputBuffer = HBufC::NewLC(expectedSize);
+ TPtr outputPtr(outputBuffer->Des());
+
+ // load the converter ready for testing
+ CCnvCharacterSetConverter* charConverter=CCnvCharacterSetConverter::NewLC();
+ CCnvCharacterSetConverter::TAvailability available = CCnvCharacterSetConverter::EAvailable;
+ available = charConverter->PrepareToConvertToOrFromL(KCharacterSetIdentifierShiftJis, aFileServerSession);
+ if ( available != CCnvCharacterSetConverter::EAvailable)
+ User::Leave(KErrNotSupported);
+
+ // read the whole source file into the buffer
+ User::LeaveIfError( source.Read( sourcePtr, sourceSize ));
+ source.Close();
+
+ // measure the time taken to convert.
+ TTime startTime, stopTime;
+ startTime.UniversalTime();
+
+ // now convert using J5 into unicode in the output buffer
+ TInt state = CCnvCharacterSetConverter::KStateDefault;
+ TInt numberCharsNotConverted = 0;
+
+ // repeat this to get a more accurate measurement
+ const int scale =10;
+ TInt i=0;
+ for ( i=0; i<scale; i++)
+ {
+ numberCharsNotConverted = charConverter->ConvertToUnicode(outputPtr, sourcePtr, state);
+ if ( numberCharsNotConverted != 0)
+ aTest.Printf(_L("number of characters not converted:%d \n"),
+ numberCharsNotConverted );
+ }
+
+ stopTime.UniversalTime();
+ TTimeIntervalMicroSeconds timeTaken = stopTime.MicroSecondsFrom(startTime);
+ TInt result = timeTaken.Int64()/scale;
+ aTest.Printf(_L("Time taken for converting %S to unicode using shiftjis:%d microseconds\n"),
+ &aForeign, result );
+
+
+ // characterSetConverter, sourceBuffer, outputBuffer
+ CleanupStack::PopAndDestroy(3);
+ }
+
+/**
+@SYMTestCaseID SYSLIB-CharConv-J5-U-0216-CP
+@SYMTestCaseDesc Check some UCS2 conversions
+@SYMTestStatus Implemented
+@SYMTestPriority Critical
+@SYMTestActions The UCS2 sample data is converted to unicode by J5.
+@SYMTestExpectedResults The converted unicode data is checked against expected data.
+@SYMTestType Unit
+@SYMREQ REQ4828
+*/
+LOCAL_C void checkUCS2converstion(RTest& aTest, CCnvCharacterSetConverter *aJ5converter )
+ {
+ // buffer to convert into
+ TBuf16<512> unicode;
+ TInt numberOfUnconvertibleCharacters;
+ TInt indexOfFirstByteOfFirstUnconvertibleCharacter;
+
+ // small sample of EBigEndian UCS2 with no Byte order markers
+ aTest.Printf(_L("Check J5 conversion of EBigEndian UCS2 with no Byte order markers to unicode \n"));
+ _LIT8(Kucs2sample1,"\x0\x69\x0\x2d\x0\x6d\x0\x6f\x0\x64\x0\x65");
+ _LIT16(KucsAsUnicode1,"\x0069\x002d\x006d\x006f\x0064\x0065");
+
+ // J5 convert from unicode
+ TInt state = CCnvCharacterSetConverter::KStateDefault;
+ TInt numberNotConverted =0;
+ numberNotConverted = aJ5converter->ConvertToUnicode(unicode, Kucs2sample1, state, numberOfUnconvertibleCharacters, indexOfFirstByteOfFirstUnconvertibleCharacter );
+ aTest(numberNotConverted == 0);
+ aTest(numberOfUnconvertibleCharacters == 0);
+ aTest(indexOfFirstByteOfFirstUnconvertibleCharacter == -1);
+
+ // compare the unicode created to that supplied
+ aTest(unicode==KucsAsUnicode1);
+
+ // small sample of ELittleEndian UCS2 with no Byte order markers
+ aTest.Printf(_L("Check J5 conversion of ELittleEndian UCS2 with no Byte order markers to unicode \n"));
+ _LIT8(Kucs2sample2,"\x69\x0\x2d\x0\x6d\x0\x6f\x0\x64\x0\x65\x0");
+ _LIT16(KucsAsUnicode2,"\x0069\x002d\x006d\x006f\x0064\x0065");
+ state = CCnvCharacterSetConverter::KStateDefault;
+ numberNotConverted = aJ5converter->ConvertToUnicode(unicode, Kucs2sample2, state);
+ aTest(numberNotConverted == 0);
+
+ // compare the unicode created to that supplied
+ aTest(unicode==KucsAsUnicode2);
+
+ // check UCS2 converter handles output buffer too small case
+ aTest.Printf(_L("Check J5 conversion of UCS2 with small output buffer \n"));
+ TBuf16<4> smallOutputBuffer;
+ numberNotConverted = aJ5converter->ConvertToUnicode(smallOutputBuffer, Kucs2sample2, state);
+ // check numberNotConverted = size of KucsAsUnicode2(12bytes) - smallOutputBuffer(8bytes) = 4
+ aTest.Printf(_L("numberNotConverted %d should be 4 \n"), numberNotConverted );
+ aTest( numberNotConverted == 4);
+
+ // check UCS2 converter handles a odd number of input bytes
+ aTest.Printf(_L("Check J5 conversion of UCS2 handles a odd number of input bytes \n"));
+ _LIT8(Kucs2sample3,"\xff\xfe\x69\x0\x2d\x0\x6d\x0\x6f\x0\x64\x0\x65\x0\x23");
+ _LIT16(KucsAsUnicode3,"\x0069\x002d\x006d\x006f\x0064\x0065");
+ numberNotConverted = aJ5converter->ConvertToUnicode(unicode, Kucs2sample3, state, numberOfUnconvertibleCharacters, indexOfFirstByteOfFirstUnconvertibleCharacter);
+ aTest.Printf(_L("numberOfUnconvertibleCharacters %d should be the 1 odd byte at the end\n"), numberOfUnconvertibleCharacters );
+ aTest(unicode==KucsAsUnicode3);
+ aTest(numberNotConverted == 1);
+ aTest(numberOfUnconvertibleCharacters == 0);
+ aTest(indexOfFirstByteOfFirstUnconvertibleCharacter == -1);
+ }
+
+
+/**
+ test code starts here...
+ */
+LOCAL_C void DoE32MainL()
+ {
+ RTest test=_L("TJ5");
+ CleanupClosePushL(test);
+ test.Start(_L("TJ5 start"));
+ RFs fileServerSession;
+ CleanupClosePushL(fileServerSession);
+ User::LeaveIfError(fileServerSession.Connect());
+
+ // check the J5 converter is available
+ checkJ5Available( test, fileServerSession );
+
+ // Convert some sample data to unicode using J5 and check result
+ _LIT8(KShiftJis,"\x82\xB1\x82\xF1\x82\xC9\x82\xBF\x82\xCD\x82\xC9\x82\xBF\x82\xCD");
+ _LIT16(KUnicode,"\x3053\x3093\x306B\x3061\x306F\x306B\x3061\x306F");
+ // test J5 on simple example shift jis test data,
+ convertDataUsingJ5(test, fileServerSession, KShiftJis, KUnicode );
+
+ // small sample of code set 1&3 EUC_JP
+ _LIT8(KCodeSet2EucJp,"\x8f\xa2\xc2\xa1\xf1\xa1\xf2\x8f\xa2\xc2\x5c\x8f\xa2\xc3\xa1\xf8");
+ // should convert to ¡¢£¤¥¦§
+ _LIT16(KCodeSet2EucJpAsUnicode,"\x00A1\x00A2\x00A3\x00A1\x00A5\x00A6\x00A7");
+ convertDataUsingJ5( test, fileServerSession, KCodeSet2EucJp, KCodeSet2EucJpAsUnicode);
+
+ /* example of code set 3 EUC_JP 8faba98fa9c18fabae8fabb28fabb18fabb48fabb38fabc0 converts to åæçèéêëì*/
+ /* Note this could also be decoded as shiftjis */
+ _LIT8(KCodeSet3EucJp,"\x8f\xab\xa9\x8f\xa9\xc1\x8f\xab\xae\x8f\xab\xb2\x8f\xab\xb1\x8f\xab\xb4\x8f\xab\xb3\x8f\xab\xc0");
+ _LIT16(KCodeSet3EucJpAsUnicode,"\x00E5\x00E6\x00E7\x00E8\x00E9\x00EA\x00EB\x00EC");
+ convertDataUsingJ5( test, fileServerSession, KCodeSet3EucJp, KCodeSet3EucJpAsUnicode);
+
+ // small sample of UTF8 ®¯°±²³
+ _LIT8(Kutf8,"\xc2\x8e\xc2\xaf\xc2\xb0\xc2\xb1\xc2\xb2\xc2\xb3");
+ _LIT16(Kutf8AsUnicode,"\x008E\x00AF\x00B0\x00B1\x00B2\x00B3");
+ convertDataUsingJ5( test, fileServerSession, Kutf8, Kutf8AsUnicode);
+
+ // small sample of ISO2022jp
+ _LIT8(Kiso2022jp,"\x1B\x24\x42\x24\x33\x24\x73\x24");
+ _LIT16(Kiso2022jpAsUnicode,"\x3053\x3093");
+ convertDataUsingJ5( test, fileServerSession, Kiso2022jp, Kiso2022jpAsUnicode);
+
+ // some shiftjis which starts with two bytes that could be EUC 0xe1,0x81
+ _LIT8(KShiftJisWithEucStart,"\xE1\x81\x82\xB1\x82\xF1\x82\xC9\x82\xBF\x82\xCD\x82\xC9\x82\xBF\x82\xCD");
+ _LIT16(KUnicodeShiftJisWithEucStart,"\x7601\x3053\x3093\x306B\x3061\x306F\x306B\x3061\x306F");
+ convertDataUsingJ5(test, fileServerSession, KShiftJisWithEucStart, KUnicodeShiftJisWithEucStart );
+
+ // some shiftjis which starts with two bytes that could be UTF8 0xc2,0x8e
+ _LIT8(KShiftJisWithUtf8Start,"\xC2\x8E\x82\xB1\x82\xF1\x82\xC9\x82\xBF\x82\xCD\x82\xC9\x82\xBF\x82\xCD");
+ _LIT16(KUnicodeShiftJisWithUtf8Start,"\xFF82\x7345\xFF71\x3093\x306B\x3061\x306F\x306B\x3061\x306F");
+ convertDataUsingJ5(test, fileServerSession, KShiftJisWithUtf8Start, KUnicodeShiftJisWithUtf8Start );
+
+ // Small shiftjis sample with some half width Katakana
+ _LIT8(KShiftJisHalfWidthKatakana,"\xB0\xE0\x70\xB0\x70");
+ _LIT16(KUnicodeShiftJisHalfWidthKatakana,"\xFF70\x703E\xFF70\x0070");
+ convertDataUsingJ5(test, fileServerSession, KShiftJisHalfWidthKatakana, KUnicodeShiftJisHalfWidthKatakana );
+
+ /* Read in a text file and convert to unicode using J5
+ then check the correct converstion was used by checking
+ returned state variable */
+ // Note ansi.txt is valid for several conversions EUtf8, Eucjp
+ checkDetection( test, fileServerSession, KAnsiTxt, EEucjp);
+ checkDetection( test, fileServerSession, KImodeTxt, EShiftjis);
+ checkDetection( test, fileServerSession, KJpEUCjTxt, EEucjp);
+ checkDetection( test, fileServerSession, KJpJISTxt, EIso2022jp1);
+ checkDetection( test, fileServerSession, KJpSJISTxt, EShiftjis);
+ checkDetection( test, fileServerSession, KJpUnicodeTxt, EUcs2);
+ checkDetection( test, fileServerSession, KJpUTF8Txt, EUtf8);
+ checkDetection( test, fileServerSession, KmoperaTxt, EShiftjis);
+ checkDetection( test, fileServerSession, KMScodeFontTxt,EShiftjis );
+ checkDetection( test, fileServerSession, KSJISTxt, EShiftjis );
+
+ // load the converter ready for testing
+ CCnvCharacterSetConverter* J5Converter=CCnvCharacterSetConverter::NewLC();
+ CCnvCharacterSetConverter::TAvailability available = CCnvCharacterSetConverter::EAvailable;
+ available = J5Converter->PrepareToConvertToOrFromL(KCharacterSetIdentifierJ5, fileServerSession);
+ if ( available != CCnvCharacterSetConverter::EAvailable)
+ User::Leave(KErrNotSupported);
+
+ // check some UCS2 examples
+ checkUCS2converstion( test, J5Converter );
+
+ // Convert the file Ansi.test supplied by Symbian Japan to unicode
+ convertFileUsingJ5L( test, fileServerSession, J5Converter, KAnsiTxt, KAnsiUni, KAnsiOut );
+
+ // Convert the file imode.test supplied by Symbian Japan to unicode
+ convertFileUsingJ5L( test, fileServerSession, J5Converter, KImodeTxt, KImodeUni, KImodeOut );
+
+ // Convert the file JpEUC-J.test supplied by Symbian Japan to unicode
+ convertFileUsingJ5L( test, fileServerSession, J5Converter, KJpEUCjTxt, KJpEUCjUni, KJpEUCjOut );
+
+ // Convert the file JpJIS.test supplied by Symbian Japan to unicode
+ convertFileUsingJ5L( test, fileServerSession, J5Converter, KJpJISTxt, KJpJISUni, KJpJISOut);
+
+ // Convert the file JpSJIS.test supplied by Symbian Japan to unicode
+ convertFileUsingJ5L( test, fileServerSession, J5Converter, KJpSJISTxt, KJpSJISUni, KJpSJISOut );
+
+ // Convert the file JpUnicode.test supplied by Symbian Japan to unicode
+ convertFileUsingJ5L( test, fileServerSession, J5Converter, KJpUnicodeTxt, KJpUnicodeUni, KJpUnicodeOut );
+
+ // Convert the file JpUTF8.test supplied by Symbian Japan to unicode
+ convertFileUsingJ5L( test, fileServerSession, J5Converter, KJpUTF8Txt, KJpUTF8Uni, KJpUTF8Out);
+
+ // Convert the file mopera.test supplied by Symbian Japan to unicode
+ convertFileUsingJ5L( test, fileServerSession, J5Converter, KmoperaTxt, KmoperaUni, KmoperaOut);
+
+ // Convert the file MScodefont.test supplied by Symbian Japan to unicode
+ convertFileUsingJ5L( test, fileServerSession, J5Converter, KMScodeFontTxt, KMScodeFontUni, KMScodeFontOut);
+
+ // Convert the file S-JIS.test supplied by Symbian Japan to unicode
+ convertFileUsingJ5L( test, fileServerSession, J5Converter, KSJISTxt, KSJISUni, KSJISOut);
+
+ // Check that J5 converts to UTF8 when ConvertFromUnicode() is called
+ convertFromUnicodeUsingJ5( test, fileServerSession, J5Converter );
+
+ // autodetect speed benchmark test
+ checkNormalAutodetectL( test, fileServerSession, KMScodeFontTxt );
+ checkNormalAutodetectL( test, fileServerSession, KSJISTxt );
+ checkNormalAutodetectL( test, fileServerSession, KImodeTxt );
+
+ // conversion speed benchmark test
+ checkConversionSpeedL(test, fileServerSession, KMScodeFontTxt);
+ checkConversionSpeedL(test, fileServerSession, KSJISTxt);
+ checkConversionSpeedL(test, fileServerSession, KImodeTxt);
+
+ // J5Converter
+ CleanupStack::PopAndDestroy(1 /*, &J5Converter*/);
+
+ // tidy up after text
+ test.End();
+ User::After(3000000);
+ // test, fileServerSession
+ CleanupStack::PopAndDestroy(2, &test);
+ }
+
+GLDEF_C TInt E32Main()
+ {
+ __UHEAP_MARK;
+ CTrapCleanup* trapCleanup=CTrapCleanup::New();
+ TRAPD(error, DoE32MainL());
+ delete trapCleanup;
+ __UHEAP_MARKEND;
+ return error;
+ }
+