|
1 /* |
|
2 * Copyright (c) 2005 Nokia Corporation and/or its subsidiary(-ies). |
|
3 * All rights reserved. |
|
4 * This component and the accompanying materials are made available |
|
5 * under the terms of the License "Eclipse Public License v1.0" |
|
6 * which accompanies this distribution, and is available |
|
7 * at the URL "http://www.eclipse.org/legal/epl-v10.html". |
|
8 * |
|
9 * Initial Contributors: |
|
10 * Nokia Corporation - initial contribution. |
|
11 * |
|
12 * Contributors: |
|
13 * |
|
14 * Description: |
|
15 * |
|
16 */ |
|
17 |
|
18 |
|
19 |
|
20 |
|
21 |
|
22 |
|
23 |
|
24 |
|
25 #include <e32std.h> |
|
26 #include <e32base.h> |
|
27 #include <e32test.h> |
|
28 #include <f32file.h> |
|
29 #include <charconv.h> |
|
30 #include <shiftjis.h> |
|
31 #include <utf.h> |
|
32 |
|
33 // test data files |
|
34 #ifdef __WINS__ |
|
35 // on C drive for emulator |
|
36 _LIT16(KAnsiTxt,"c:\\test\\data\\Ansi.test"); |
|
37 _LIT16(KImodeTxt,"c:\\test\\data\\imode.test"); |
|
38 _LIT16(KJpEUCjTxt,"c:\\test\\data\\JpEUC-J.test"); |
|
39 _LIT16(KJpJISTxt,"c:\\test\\data\\JpJIS.test"); |
|
40 _LIT16(KJpSJISTxt,"c:\\test\\data\\JpSJIS.test"); |
|
41 _LIT16(KJpUnicodeTxt,"c:\\test\\data\\JpUnicode.test"); |
|
42 _LIT16(KJpUTF8Txt,"c:\\test\\data\\JpUTF8.test"); |
|
43 _LIT16(KmoperaTxt,"c:\\test\\data\\mopera.test"); |
|
44 _LIT16(KMScodeFontTxt,"c:\\test\\data\\MScodefont.test"); |
|
45 _LIT16(KSJISTxt,"c:\\test\\data\\S-JIS.test"); |
|
46 |
|
47 _LIT16(KAnsiUni,"c:\\test\\data\\Ansi.uni"); |
|
48 _LIT16(KImodeUni,"c:\\test\\data\\imode_not_mapped.uni"); |
|
49 _LIT16(KJpEUCjUni,"c:\\test\\data\\JpEUC-J.uni"); |
|
50 _LIT16(KJpJISUni,"c:\\test\\data\\JpJIS.uni"); |
|
51 _LIT16(KJpSJISUni,"c:\\test\\data\\JpSJIS.uni"); |
|
52 _LIT16(KJpUnicodeUni,"c:\\test\\data\\JpUnicode.uni"); |
|
53 _LIT16(KJpUTF8Uni,"c:\\test\\data\\JpUTF8.uni"); |
|
54 _LIT16(KmoperaUni,"c:\\test\\data\\mopera_not_mapped.uni"); |
|
55 _LIT16(KMScodeFontUni,"c:\\test\\data\\MScodefont.uni"); |
|
56 _LIT16(KSJISUni,"c:\\test\\data\\S-JIS.uni"); |
|
57 #else |
|
58 // on Z drive for target ROM |
|
59 _LIT16(KAnsiTxt,"z:\\test\\data\\Ansi.test"); |
|
60 _LIT16(KImodeTxt,"z:\\test\\data\\imode.test"); |
|
61 _LIT16(KJpEUCjTxt,"z:\\test\\data\\JpEUC-J.test"); |
|
62 _LIT16(KJpJISTxt,"z:\\test\\data\\JpJIS.test"); |
|
63 _LIT16(KJpSJISTxt,"z:\\test\\data\\JpSJIS.test"); |
|
64 _LIT16(KJpUnicodeTxt,"z:\\test\\data\\JpUnicode.test"); |
|
65 _LIT16(KJpUTF8Txt,"z:\\test\\data\\JpUTF8.test"); |
|
66 _LIT16(KmoperaTxt,"z:\\test\\data\\mopera.test"); |
|
67 _LIT16(KMScodeFontTxt,"z:\\test\\data\\MScodefont.test"); |
|
68 _LIT16(KSJISTxt,"z:\\test\\data\\S-JIS.test"); |
|
69 |
|
70 _LIT16(KAnsiUni,"z:\\test\\data\\Ansi.uni"); |
|
71 _LIT16(KImodeUni,"z:\\test\\data\\imode_not_mapped.uni"); |
|
72 _LIT16(KJpEUCjUni,"z:\\test\\data\\JpEUC-J.uni"); |
|
73 _LIT16(KJpJISUni,"z:\\test\\data\\JpJIS.uni"); |
|
74 _LIT16(KJpSJISUni,"z:\\test\\data\\JpSJIS.uni"); |
|
75 _LIT16(KJpUnicodeUni,"z:\\test\\data\\JpUnicode.uni"); |
|
76 _LIT16(KJpUTF8Uni,"z:\\test\\data\\JpUTF8.uni"); |
|
77 _LIT16(KmoperaUni,"z:\\test\\data\\mopera_not_mapped.uni"); |
|
78 _LIT16(KMScodeFontUni,"z:\\test\\data\\MScodefont.uni"); |
|
79 _LIT16(KSJISUni,"z:\\test\\data\\S-JIS.uni"); |
|
80 #endif |
|
81 |
|
82 _LIT16(KAnsiOut,"c:\\test\\data\\Ansi.out"); |
|
83 _LIT16(KImodeOut,"c:\\test\\data\\imode_not_mapped.out"); |
|
84 _LIT16(KJpEUCjOut,"c:\\test\\data\\JpEUC-J.out"); |
|
85 _LIT16(KJpJISOut,"c:\\test\\data\\JpJIS.out"); |
|
86 _LIT16(KJpSJISOut,"c:\\test\\data\\JpSJIS.out"); |
|
87 _LIT16(KJpUnicodeOut,"c:\\test\\data\\JpUnicode.out"); |
|
88 _LIT16(KJpUTF8Out,"c:\\test\\data\\JpUTF8.out"); |
|
89 _LIT16(KmoperaOut,"c:\\test\\data\\mopera_not_mapped.out"); |
|
90 _LIT16(KMScodeFontOut,"c:\\test\\data\\MScodefont.out"); |
|
91 _LIT16(KSJISOut,"c:\\test\\data\\S-JIS.out"); |
|
92 |
|
93 // this has been copied from J5, as this not exported |
|
94 // and only needed for testing |
|
95 enum TJ5Encoding |
|
96 { |
|
97 EShiftjis = 1, |
|
98 EIso2022jp1, |
|
99 EEucjp, |
|
100 EUtf8, |
|
101 EUcs2, |
|
102 EUnknown |
|
103 }; |
|
104 |
|
105 |
|
106 #ifdef __WINS__ |
|
107 /** |
|
108 For development only! - used for generating unicode files |
|
109 Save the converted file as unicode |
|
110 */ |
|
111 LOCAL_C void SaveConvertedUnicodeAsFile( RFs &fileServerSession, |
|
112 const TDesC &aFileName, |
|
113 const TDesC &aConvertedUnicode) |
|
114 { |
|
115 RFile file; |
|
116 User::LeaveIfError( file.Replace( fileServerSession, aFileName, EFileWrite )); |
|
117 |
|
118 // get a pointer to 8bit data |
|
119 const TUint16* p16 = aConvertedUnicode.Ptr(); |
|
120 TPtrC8 p8( (TUint8 *)p16, aConvertedUnicode.Size() ); |
|
121 |
|
122 |
|
123 file.Write( p8, aConvertedUnicode.Size()); |
|
124 file.Close(); |
|
125 } |
|
126 #endif |
|
127 |
|
128 |
|
129 /** |
|
130 @SYMTestCaseID SYSLIB-CharConv-J5-U-0209-CP |
|
131 @SYMTestCaseDesc Check that the J5 converter is available for testing |
|
132 @SYMTestStatus Implemented |
|
133 @SYMTestPriority Critical |
|
134 @SYMTestActions The character conversion framework is called to check J5 is loaded. |
|
135 @SYMTestExpectedResults J5 is in the list of avalable converters. |
|
136 @SYMTestType Unit |
|
137 @SYMREQ REQ4828 |
|
138 */ |
|
139 LOCAL_C void checkJ5Available(RTest aTest, RFs &aFileServerSession) |
|
140 { |
|
141 TBool J5found = EFalse; |
|
142 |
|
143 CArrayFix<CCnvCharacterSetConverter::SCharacterSet>* arrayOfCharacterSetsAvailable=CCnvCharacterSetConverter::CreateArrayOfCharacterSetsAvailableLC(aFileServerSession); |
|
144 |
|
145 // check the list of available converters looking for J5 |
|
146 aTest.Printf(_L("charconv plugins available:\n")); |
|
147 for (TInt i=arrayOfCharacterSetsAvailable->Count()-1; i>=0; --i) |
|
148 { |
|
149 const CCnvCharacterSetConverter::SCharacterSet& charactersSet=(*arrayOfCharacterSetsAvailable)[i]; |
|
150 TPtrC charactersSetName(charactersSet.Name()); |
|
151 if (charactersSet.NameIsFileName()) |
|
152 { |
|
153 charactersSetName.Set(TParsePtrC(charactersSetName).Name()); |
|
154 |
|
155 if ( charactersSetName.Compare(_L("J5")) == 0) |
|
156 J5found = ETrue; |
|
157 } |
|
158 aTest.Printf(_L(" %S\n"), &charactersSetName); |
|
159 } |
|
160 |
|
161 if (J5found) |
|
162 aTest.Printf(_L("J5 available\n") ); |
|
163 else |
|
164 { |
|
165 aTest.Printf(_L("Test failed J5 not available\n") ); |
|
166 } |
|
167 |
|
168 aTest(J5found); |
|
169 |
|
170 // PopAndDestroy arrayOfCharacterSetsAvailable |
|
171 CleanupStack::PopAndDestroy(1); |
|
172 } |
|
173 |
|
174 /** |
|
175 @SYMTestCaseID SYSLIB-CharConv-J5-U-0210-CP |
|
176 @SYMTestCaseDesc Convert the file supplied (aForeign) to unicode and then |
|
177 check the converted data against aExpectedUnicode |
|
178 @SYMTestStatus Implemented |
|
179 @SYMTestPriority Critical |
|
180 @SYMTestActions The J5 character converter is used convert ansi.test. |
|
181 @SYMTestExpectedResults The result is checked against the expected result imode.uni. |
|
182 @SYMTestType Unit |
|
183 @SYMREQ REQ4828 |
|
184 */ |
|
185 LOCAL_C void convertFileUsingJ5L(RTest& aTest, |
|
186 RFs fileServerSession, |
|
187 CCnvCharacterSetConverter*aJ5converter, |
|
188 const TDesC &aForeign, |
|
189 const TDesC &aExpectedUnicode, |
|
190 const TDesC &aOutputUnicode ) |
|
191 { |
|
192 aTest.Printf(_L("Read %S check that it converts to %S \n"),&aForeign, &aExpectedUnicode); |
|
193 |
|
194 // open the file & read into sourceBuffer |
|
195 RFile source, expected; |
|
196 User::LeaveIfError( source.Open( fileServerSession, aForeign, EFileRead )); |
|
197 User::LeaveIfError( expected.Open( fileServerSession, aExpectedUnicode, EFileRead )); |
|
198 TInt sourceSize = 0, expectedSize = 0; |
|
199 source.Size(sourceSize); |
|
200 expected.Size(expectedSize); |
|
201 |
|
202 // create buffers for source file and expected unicode file |
|
203 HBufC8 * sourceBuffer = HBufC8::NewLC(sourceSize); |
|
204 TPtr8 sourcePtr(sourceBuffer->Des()); |
|
205 HBufC8 * expectedBuffer = HBufC8::NewLC(expectedSize); |
|
206 TPtr8 expectedPtr(expectedBuffer->Des()); |
|
207 |
|
208 // buffer to convert into |
|
209 HBufC * outputBuffer = HBufC::NewLC(expectedSize); |
|
210 TPtr outputPtr(outputBuffer->Des()); |
|
211 |
|
212 // read the whole source & output file into the buffer |
|
213 User::LeaveIfError( source.Read( sourcePtr, sourceSize )); |
|
214 User::LeaveIfError( expected.Read( expectedPtr, expectedSize )); |
|
215 |
|
216 // measure the time taken to convert. |
|
217 TTime startTime, stopTime; |
|
218 startTime.UniversalTime(); |
|
219 |
|
220 // now convert using J5 into unicode in the output buffer |
|
221 TInt state = CCnvCharacterSetConverter::KStateDefault; |
|
222 TInt error = KErrNone; |
|
223 error = aJ5converter->ConvertToUnicode(outputPtr, sourcePtr, state); |
|
224 if (error != KErrNone ) |
|
225 { |
|
226 aTest.Printf(_L("ConvertToUnicode returned %d\n"), error); |
|
227 } |
|
228 |
|
229 stopTime.UniversalTime(); |
|
230 TTimeIntervalMicroSeconds timeTaken = stopTime.MicroSecondsFrom(startTime); |
|
231 TInt result = timeTaken.Int64(); |
|
232 aTest.Printf(_L("Time for autodetect and convert to unicode %S using J5 %d microseconds\n"), |
|
233 &aForeign, result ); |
|
234 |
|
235 TInt outputLength = outputBuffer->Length() ; // 16 bit words |
|
236 TInt expectedLength = expectedPtr.Length() / 2 ; // Bytes (hence divide by 2!) |
|
237 TInt checkLength = Min(outputLength, expectedLength) ; |
|
238 |
|
239 if (outputLength != expectedLength) |
|
240 { |
|
241 aTest.Printf(_L("Fail: Output length incorrect (expected %d bytes after conversion, got %d\n"), |
|
242 expectedLength, outputLength) ; |
|
243 } |
|
244 // check that the output buffer contains the same as the expected buffer |
|
245 TInt j=0; |
|
246 TUint8 firstByte =0; |
|
247 TUint8 secondbyte =0; |
|
248 |
|
249 for ( TInt i=0; i < checkLength; i++, j+=2) |
|
250 { |
|
251 // two bytes in the file - one 16bit unicode value |
|
252 firstByte = TUint8(outputPtr[i] & 0x00ff); |
|
253 secondbyte = (outputPtr[i] & 0xff00) >> 8 ; |
|
254 |
|
255 if ( ( firstByte == expectedPtr[j] ) && ( secondbyte == expectedPtr[j+1] ) ) |
|
256 { |
|
257 // useful for development |
|
258 /*aTest.Printf(_L(" %02x%02x = %02x%02x ok\n"), |
|
259 secondbyte, firstByte, expectedPtr[j+1], expectedPtr[j]);*/ |
|
260 } |
|
261 else |
|
262 { |
|
263 aTest.Printf(_L("line:%d %02x%02x (output) != %02x%02x (expected) fail\n"), |
|
264 i, secondbyte, firstByte, expectedPtr[j+1], expectedPtr[j]); |
|
265 } |
|
266 } |
|
267 |
|
268 #ifdef __WINS__ |
|
269 // for debugging save as file |
|
270 SaveConvertedUnicodeAsFile( fileServerSession, aOutputUnicode, outputPtr ); |
|
271 #endif |
|
272 |
|
273 source.Close(); |
|
274 expected.Close(); |
|
275 |
|
276 // sourceBuffer, expectedBufffer, outputBuffer |
|
277 CleanupStack::PopAndDestroy(3); |
|
278 } |
|
279 |
|
280 |
|
281 /** |
|
282 @SYMTestCaseID SYSLIB-CharConv-J5-U-0211-CP |
|
283 @SYMTestCaseDesc Check that J5 converts to UTF8 when ConvertFromUnicode() is called |
|
284 @SYMTestStatus Implemented |
|
285 @SYMTestPriority Critical |
|
286 @SYMTestActions The J5 character converter is used to convert unicode to UTF8 |
|
287 @SYMTestExpectedResults The result is checked against the expected UTF8. |
|
288 @SYMTestType Unit |
|
289 @SYMREQ REQ4828 |
|
290 */ |
|
291 LOCAL_C void convertFromUnicodeUsingJ5(RTest& aTest, |
|
292 RFs fileServerSession, |
|
293 CCnvCharacterSetConverter*aJ5converter ) |
|
294 { |
|
295 /* sample UTF8 for testing */ |
|
296 _LIT16(KsampleUnicode,"\x0069\x002d\x006d\x006f\x0064\x0065\xFFFD"); |
|
297 _LIT8(KsampleAsUTF8,"\x69\x2D\x6D\x6F\x64\x65\xEF\xBF\xBD"); |
|
298 TBuf8<100> utf8buffer; |
|
299 |
|
300 aTest.Printf(_L("Check J5 convertd from Unicode to UTF8 \n")); |
|
301 |
|
302 CCnvCharacterSetConverter::TAvailability available = aJ5converter->PrepareToConvertToOrFromL(KCharacterSetIdentifierJ5, fileServerSession); |
|
303 if ( available != CCnvCharacterSetConverter::EAvailable) |
|
304 User::Leave(KErrNotSupported); |
|
305 |
|
306 // J5 convert from unicode |
|
307 TInt state = CCnvCharacterSetConverter::KStateDefault; |
|
308 TInt numberNotConverted =0; |
|
309 numberNotConverted = aJ5converter->ConvertFromUnicode(utf8buffer, KsampleUnicode, state); |
|
310 if (numberNotConverted != 0) |
|
311 { |
|
312 aTest.Printf(_L("numberNotConverted = %d"), numberNotConverted ); |
|
313 } |
|
314 |
|
315 // compare the UTF8 created to that supplied |
|
316 aTest(utf8buffer==KsampleAsUTF8); |
|
317 |
|
318 } |
|
319 |
|
320 /** |
|
321 @SYMTestCaseID SYSLIB-CharConv-J5-U-0212-CP |
|
322 @SYMTestCaseDesc Check that the J5 converter autodetection |
|
323 @SYMTestStatus Implemented |
|
324 @SYMTestPriority Critical |
|
325 @SYMTestActions The J5 character converter is used convert sample files. |
|
326 @SYMTestExpectedResults The result is passed back for testing in the state paramater. |
|
327 @SYMTestType Unit |
|
328 @SYMREQ REQ4828 |
|
329 */ |
|
330 LOCAL_C void checkDetection(RTest& aTest, |
|
331 RFs fileServerSession, |
|
332 const TDesC &aForeign, |
|
333 TInt aExpectedCode ) |
|
334 { |
|
335 __UHEAP_MARK; |
|
336 aTest.Printf(_L("Read %S check that it converts to %d \n"),&aForeign, aExpectedCode); |
|
337 |
|
338 // open the file & read into sourceBuffer |
|
339 RFile source; |
|
340 User::LeaveIfError( source.Open( fileServerSession, aForeign, EFileRead )); |
|
341 TInt sourceSize = 0; |
|
342 source.Size(sourceSize); |
|
343 |
|
344 // create buffers for source file |
|
345 HBufC8 * sourceBuffer = HBufC8::NewLC(sourceSize); |
|
346 TPtr8 sourcePtr(sourceBuffer->Des()); |
|
347 |
|
348 // buffer to convert into |
|
349 HBufC * outputBuffer = HBufC::NewLC(sourceSize * 2); |
|
350 TPtr outputPtr(outputBuffer->Des()); |
|
351 |
|
352 // read the whole source & output file into the buffer |
|
353 User::LeaveIfError( source.Read( sourcePtr, sourceSize )); |
|
354 source.Close(); |
|
355 |
|
356 // load the converter ready for testing |
|
357 CCnvCharacterSetConverter* J5Converter=CCnvCharacterSetConverter::NewLC(); |
|
358 CCnvCharacterSetConverter::TAvailability available = CCnvCharacterSetConverter::EAvailable; |
|
359 available = J5Converter->PrepareToConvertToOrFromL(KCharacterSetIdentifierJ5, fileServerSession); |
|
360 if ( available != CCnvCharacterSetConverter::EAvailable) |
|
361 User::Leave(KErrNotSupported); |
|
362 |
|
363 // now convert using J5 into unicode in the output buffer |
|
364 TInt state = CCnvCharacterSetConverter::KStateDefault; |
|
365 TInt numberCharsNotConverted = 0; |
|
366 numberCharsNotConverted = J5Converter->ConvertToUnicode(outputPtr, sourcePtr, state); |
|
367 if ( numberCharsNotConverted != 0) |
|
368 aTest.Printf(_L("number of characters not converted:%d of %d\n"), |
|
369 numberCharsNotConverted, sourceSize ); |
|
370 |
|
371 // check that J5 has been able to convert all the data |
|
372 aTest(J5Converter->ConvertToUnicode(outputPtr, sourcePtr, state)==0); |
|
373 |
|
374 // look at the code returned in the state |
|
375 if ( aExpectedCode == state ) |
|
376 { |
|
377 aTest.Printf(_L(" expected:%d found:%d ok\n"), aExpectedCode , state ); |
|
378 } |
|
379 else |
|
380 { |
|
381 aTest.Printf(_L(" expected:%d found:%d failed \n"), aExpectedCode , state ); |
|
382 aTest( aExpectedCode == state); |
|
383 } |
|
384 |
|
385 // sourceBuffer, expectedBufffer, J5Converter |
|
386 CleanupStack::PopAndDestroy(3); |
|
387 __UHEAP_MARKEND; |
|
388 } |
|
389 |
|
390 |
|
391 /** |
|
392 @SYMTestCaseID SYSLIB-CharConv-J5-U-0213-CP |
|
393 @SYMTestCaseDesc Check the J5 on some small data samples |
|
394 @SYMTestStatus Implemented |
|
395 @SYMTestPriority Critical |
|
396 @SYMTestActions The sample data is converted to unicode by J5. |
|
397 @SYMTestExpectedResults The converted unicode data is checked against expected data. |
|
398 @SYMTestType Unit |
|
399 @SYMREQ REQ4828 |
|
400 */ |
|
401 LOCAL_C void convertDataUsingJ5(RTest& aTest, |
|
402 RFs &fileServerSession, |
|
403 const TDesC8 &aForeign, |
|
404 const TDesC16 &aUnicode ) |
|
405 { |
|
406 // for the unicode created |
|
407 TBuf16<512> unicode; |
|
408 |
|
409 // load the converter ready for testing |
|
410 CCnvCharacterSetConverter* J5Converter=CCnvCharacterSetConverter::NewLC(); |
|
411 CCnvCharacterSetConverter::TAvailability available = CCnvCharacterSetConverter::EAvailable; |
|
412 available = J5Converter->PrepareToConvertToOrFromL(KCharacterSetIdentifierJ5, fileServerSession); |
|
413 if ( available != CCnvCharacterSetConverter::EAvailable) |
|
414 User::Leave(KErrNotSupported); |
|
415 |
|
416 // J5 decode to unicode |
|
417 TInt state = CCnvCharacterSetConverter::KStateDefault; |
|
418 TInt numberNotConverted =0; |
|
419 numberNotConverted = J5Converter->ConvertToUnicode(unicode, aForeign, state); |
|
420 if (numberNotConverted != 0) |
|
421 { |
|
422 aTest.Printf(_L("numberNotConverted = %d"), numberNotConverted ); |
|
423 } |
|
424 |
|
425 // compare the unicode created to that supplied |
|
426 aTest(unicode==aUnicode); |
|
427 |
|
428 // tidy up J5Converter |
|
429 CleanupStack::PopAndDestroy(1); |
|
430 } |
|
431 |
|
432 |
|
433 /** |
|
434 @SYMTestCaseID SYSLIB-CharConv-J5-U-0214-CP |
|
435 @SYMTestCaseDesc Check that J5 does not interfere with the normal operation of Autodetect |
|
436 And make a rough timing measurement. |
|
437 @SYMTestStatus Implemented |
|
438 @SYMTestPriority Critical |
|
439 @SYMTestActions Autodetect using other plugins is tested when J5 is available |
|
440 @SYMTestExpectedResults Autodetect still works and does NOT select J5. |
|
441 @SYMTestType Unit |
|
442 @SYMREQ REQ4828 |
|
443 */ |
|
444 LOCAL_C void checkNormalAutodetectL( RTest& aTest, RFs &aFileServerSession, const TDesC &aForeign ) |
|
445 { |
|
446 aTest.Printf(_L("Test normal autodetect \n") ); |
|
447 CCnvCharacterSetConverter* characterSetConverter=CCnvCharacterSetConverter::NewLC(); |
|
448 CArrayFix<CCnvCharacterSetConverter::SCharacterSet>* arrayOfCharacterSetsAvailable=CCnvCharacterSetConverter::CreateArrayOfCharacterSetsAvailableLC(aFileServerSession); |
|
449 |
|
450 // open the file & read into sourceBuffer |
|
451 RFile source; |
|
452 User::LeaveIfError( source.Open( aFileServerSession, aForeign, EFileRead )); |
|
453 TInt sourceSize = 0; |
|
454 source.Size(sourceSize); |
|
455 |
|
456 // create buffers for source file |
|
457 HBufC8 * sourceBuffer = HBufC8::NewLC(sourceSize); |
|
458 TPtr8 sourcePtr(sourceBuffer->Des()); |
|
459 |
|
460 // read the whole source file into the buffer |
|
461 User::LeaveIfError( source.Read( sourcePtr, sourceSize )); |
|
462 source.Close(); |
|
463 |
|
464 // measure the time taken to autodetect. |
|
465 TTime startTime, stopTime; |
|
466 startTime.UniversalTime(); |
|
467 |
|
468 // do the autodetect |
|
469 TInt confidenceLevel =0; |
|
470 TUint characterSetIdentifier = 0; |
|
471 characterSetConverter->AutoDetectCharSetL( confidenceLevel, |
|
472 characterSetIdentifier, *arrayOfCharacterSetsAvailable, sourcePtr); |
|
473 |
|
474 aTest(characterSetIdentifier == KCharacterSetIdentifierShiftJis); |
|
475 |
|
476 stopTime.UniversalTime(); |
|
477 TTimeIntervalMicroSeconds timeTaken = stopTime.MicroSecondsFrom(startTime); |
|
478 TInt result = timeTaken.Int64(); |
|
479 aTest.Printf(_L("Time taken for old autodetect %S = %d microseconds\n"), |
|
480 &aForeign, result ); |
|
481 |
|
482 // characterSetConverter,arrayOfCharacterSetsAvailable, sourceBuffer |
|
483 CleanupStack::PopAndDestroy(3); |
|
484 } |
|
485 |
|
486 |
|
487 /** |
|
488 @SYMTestCaseID SYSLIB-CharConv-J5-U-0215-CP |
|
489 @SYMTestCaseDesc Check the conversion speed of shiftjis |
|
490 @SYMTestStatus Implemented |
|
491 @SYMTestPriority Critical |
|
492 @SYMTestActions The sample data is converted to unicode by J5. |
|
493 @SYMTestExpectedResults The converted unicode data is checked against expected data. |
|
494 @SYMTestType Unit |
|
495 @SYMREQ REQ4828 |
|
496 */ |
|
497 LOCAL_C void checkConversionSpeedL(RTest& aTest, RFs &aFileServerSession, const TDesC &aForeign) |
|
498 { |
|
499 aTest.Printf(_L("Test conversion speed \n") ); |
|
500 |
|
501 // open the file & read into sourceBuffer |
|
502 RFile source; |
|
503 User::LeaveIfError( source.Open( aFileServerSession, aForeign, EFileRead )); |
|
504 TInt sourceSize = 0, expectedSize = 0; |
|
505 source.Size(sourceSize); |
|
506 |
|
507 // assume that every byte coverts to 2 bytes of unicode and the buffer will be big enough |
|
508 expectedSize = sourceSize * 2; |
|
509 |
|
510 // create buffer for source file |
|
511 HBufC8 * sourceBuffer = HBufC8::NewLC(sourceSize); |
|
512 TPtr8 sourcePtr(sourceBuffer->Des()); |
|
513 |
|
514 // for the unicode created |
|
515 HBufC * outputBuffer = HBufC::NewLC(expectedSize); |
|
516 TPtr outputPtr(outputBuffer->Des()); |
|
517 |
|
518 // load the converter ready for testing |
|
519 CCnvCharacterSetConverter* charConverter=CCnvCharacterSetConverter::NewLC(); |
|
520 CCnvCharacterSetConverter::TAvailability available = CCnvCharacterSetConverter::EAvailable; |
|
521 available = charConverter->PrepareToConvertToOrFromL(KCharacterSetIdentifierShiftJis, aFileServerSession); |
|
522 if ( available != CCnvCharacterSetConverter::EAvailable) |
|
523 User::Leave(KErrNotSupported); |
|
524 |
|
525 // read the whole source file into the buffer |
|
526 User::LeaveIfError( source.Read( sourcePtr, sourceSize )); |
|
527 source.Close(); |
|
528 |
|
529 // measure the time taken to convert. |
|
530 TTime startTime, stopTime; |
|
531 startTime.UniversalTime(); |
|
532 |
|
533 // now convert using J5 into unicode in the output buffer |
|
534 TInt state = CCnvCharacterSetConverter::KStateDefault; |
|
535 TInt numberCharsNotConverted = 0; |
|
536 |
|
537 // repeat this to get a more accurate measurement |
|
538 const int scale =10; |
|
539 TInt i=0; |
|
540 for ( i=0; i<scale; i++) |
|
541 { |
|
542 numberCharsNotConverted = charConverter->ConvertToUnicode(outputPtr, sourcePtr, state); |
|
543 if ( numberCharsNotConverted != 0) |
|
544 aTest.Printf(_L("number of characters not converted:%d \n"), |
|
545 numberCharsNotConverted ); |
|
546 } |
|
547 |
|
548 stopTime.UniversalTime(); |
|
549 TTimeIntervalMicroSeconds timeTaken = stopTime.MicroSecondsFrom(startTime); |
|
550 TInt result = timeTaken.Int64()/scale; |
|
551 aTest.Printf(_L("Time taken for converting %S to unicode using shiftjis:%d microseconds\n"), |
|
552 &aForeign, result ); |
|
553 |
|
554 |
|
555 // characterSetConverter, sourceBuffer, outputBuffer |
|
556 CleanupStack::PopAndDestroy(3); |
|
557 } |
|
558 |
|
559 /** |
|
560 @SYMTestCaseID SYSLIB-CharConv-J5-U-0216-CP |
|
561 @SYMTestCaseDesc Check some UCS2 conversions |
|
562 @SYMTestStatus Implemented |
|
563 @SYMTestPriority Critical |
|
564 @SYMTestActions The UCS2 sample data is converted to unicode by J5. |
|
565 @SYMTestExpectedResults The converted unicode data is checked against expected data. |
|
566 @SYMTestType Unit |
|
567 @SYMREQ REQ4828 |
|
568 */ |
|
569 LOCAL_C void checkUCS2converstion(RTest& aTest, CCnvCharacterSetConverter *aJ5converter ) |
|
570 { |
|
571 // buffer to convert into |
|
572 TBuf16<512> unicode; |
|
573 TInt numberOfUnconvertibleCharacters; |
|
574 TInt indexOfFirstByteOfFirstUnconvertibleCharacter; |
|
575 |
|
576 // small sample of EBigEndian UCS2 with no Byte order markers |
|
577 aTest.Printf(_L("Check J5 conversion of EBigEndian UCS2 with no Byte order markers to unicode \n")); |
|
578 _LIT8(Kucs2sample1,"\x0\x69\x0\x2d\x0\x6d\x0\x6f\x0\x64\x0\x65"); |
|
579 _LIT16(KucsAsUnicode1,"\x0069\x002d\x006d\x006f\x0064\x0065"); |
|
580 |
|
581 // J5 convert from unicode |
|
582 TInt state = CCnvCharacterSetConverter::KStateDefault; |
|
583 TInt numberNotConverted =0; |
|
584 numberNotConverted = aJ5converter->ConvertToUnicode(unicode, Kucs2sample1, state, numberOfUnconvertibleCharacters, indexOfFirstByteOfFirstUnconvertibleCharacter ); |
|
585 aTest(numberNotConverted == 0); |
|
586 aTest(numberOfUnconvertibleCharacters == 0); |
|
587 aTest(indexOfFirstByteOfFirstUnconvertibleCharacter == -1); |
|
588 |
|
589 // compare the unicode created to that supplied |
|
590 aTest(unicode==KucsAsUnicode1); |
|
591 |
|
592 // small sample of ELittleEndian UCS2 with no Byte order markers |
|
593 aTest.Printf(_L("Check J5 conversion of ELittleEndian UCS2 with no Byte order markers to unicode \n")); |
|
594 _LIT8(Kucs2sample2,"\x69\x0\x2d\x0\x6d\x0\x6f\x0\x64\x0\x65\x0"); |
|
595 _LIT16(KucsAsUnicode2,"\x0069\x002d\x006d\x006f\x0064\x0065"); |
|
596 state = CCnvCharacterSetConverter::KStateDefault; |
|
597 numberNotConverted = aJ5converter->ConvertToUnicode(unicode, Kucs2sample2, state); |
|
598 aTest(numberNotConverted == 0); |
|
599 |
|
600 // compare the unicode created to that supplied |
|
601 aTest(unicode==KucsAsUnicode2); |
|
602 |
|
603 // check UCS2 converter handles output buffer too small case |
|
604 aTest.Printf(_L("Check J5 conversion of UCS2 with small output buffer \n")); |
|
605 TBuf16<4> smallOutputBuffer; |
|
606 numberNotConverted = aJ5converter->ConvertToUnicode(smallOutputBuffer, Kucs2sample2, state); |
|
607 // check numberNotConverted = size of KucsAsUnicode2(12bytes) - smallOutputBuffer(8bytes) = 4 |
|
608 aTest.Printf(_L("numberNotConverted %d should be 4 \n"), numberNotConverted ); |
|
609 aTest( numberNotConverted == 4); |
|
610 |
|
611 // check UCS2 converter handles a odd number of input bytes |
|
612 aTest.Printf(_L("Check J5 conversion of UCS2 handles a odd number of input bytes \n")); |
|
613 _LIT8(Kucs2sample3,"\xff\xfe\x69\x0\x2d\x0\x6d\x0\x6f\x0\x64\x0\x65\x0\x23"); |
|
614 _LIT16(KucsAsUnicode3,"\x0069\x002d\x006d\x006f\x0064\x0065"); |
|
615 numberNotConverted = aJ5converter->ConvertToUnicode(unicode, Kucs2sample3, state, numberOfUnconvertibleCharacters, indexOfFirstByteOfFirstUnconvertibleCharacter); |
|
616 aTest.Printf(_L("numberOfUnconvertibleCharacters %d should be the 1 odd byte at the end\n"), numberOfUnconvertibleCharacters ); |
|
617 aTest(unicode==KucsAsUnicode3); |
|
618 aTest(numberNotConverted == 1); |
|
619 aTest(numberOfUnconvertibleCharacters == 0); |
|
620 aTest(indexOfFirstByteOfFirstUnconvertibleCharacter == -1); |
|
621 } |
|
622 |
|
623 |
|
624 /** |
|
625 test code starts here... |
|
626 */ |
|
627 LOCAL_C void DoE32MainL() |
|
628 { |
|
629 RTest test=_L("TJ5"); |
|
630 CleanupClosePushL(test); |
|
631 test.Start(_L("TJ5 start")); |
|
632 RFs fileServerSession; |
|
633 CleanupClosePushL(fileServerSession); |
|
634 User::LeaveIfError(fileServerSession.Connect()); |
|
635 |
|
636 // check the J5 converter is available |
|
637 checkJ5Available( test, fileServerSession ); |
|
638 |
|
639 // Convert some sample data to unicode using J5 and check result |
|
640 _LIT8(KShiftJis,"\x82\xB1\x82\xF1\x82\xC9\x82\xBF\x82\xCD\x82\xC9\x82\xBF\x82\xCD"); |
|
641 _LIT16(KUnicode,"\x3053\x3093\x306B\x3061\x306F\x306B\x3061\x306F"); |
|
642 // test J5 on simple example shift jis test data, |
|
643 convertDataUsingJ5(test, fileServerSession, KShiftJis, KUnicode ); |
|
644 |
|
645 // small sample of code set 1&3 EUC_JP |
|
646 _LIT8(KCodeSet2EucJp,"\x8f\xa2\xc2\xa1\xf1\xa1\xf2\x8f\xa2\xc2\x5c\x8f\xa2\xc3\xa1\xf8"); |
|
647 // should convert to ¡¢£¤¥¦§ |
|
648 _LIT16(KCodeSet2EucJpAsUnicode,"\x00A1\x00A2\x00A3\x00A1\x00A5\x00A6\x00A7"); |
|
649 convertDataUsingJ5( test, fileServerSession, KCodeSet2EucJp, KCodeSet2EucJpAsUnicode); |
|
650 |
|
651 /* example of code set 3 EUC_JP 8faba98fa9c18fabae8fabb28fabb18fabb48fabb38fabc0 converts to åæçèéêëì*/ |
|
652 /* Note this could also be decoded as shiftjis */ |
|
653 _LIT8(KCodeSet3EucJp,"\x8f\xab\xa9\x8f\xa9\xc1\x8f\xab\xae\x8f\xab\xb2\x8f\xab\xb1\x8f\xab\xb4\x8f\xab\xb3\x8f\xab\xc0"); |
|
654 _LIT16(KCodeSet3EucJpAsUnicode,"\x00E5\x00E6\x00E7\x00E8\x00E9\x00EA\x00EB\x00EC"); |
|
655 convertDataUsingJ5( test, fileServerSession, KCodeSet3EucJp, KCodeSet3EucJpAsUnicode); |
|
656 |
|
657 // small sample of UTF8 ®¯°±²³ |
|
658 _LIT8(Kutf8,"\xc2\x8e\xc2\xaf\xc2\xb0\xc2\xb1\xc2\xb2\xc2\xb3"); |
|
659 _LIT16(Kutf8AsUnicode,"\x008E\x00AF\x00B0\x00B1\x00B2\x00B3"); |
|
660 convertDataUsingJ5( test, fileServerSession, Kutf8, Kutf8AsUnicode); |
|
661 |
|
662 // small sample of ISO2022jp |
|
663 _LIT8(Kiso2022jp,"\x1B\x24\x42\x24\x33\x24\x73\x24"); |
|
664 _LIT16(Kiso2022jpAsUnicode,"\x3053\x3093"); |
|
665 convertDataUsingJ5( test, fileServerSession, Kiso2022jp, Kiso2022jpAsUnicode); |
|
666 |
|
667 // some shiftjis which starts with two bytes that could be EUC 0xe1,0x81 |
|
668 _LIT8(KShiftJisWithEucStart,"\xE1\x81\x82\xB1\x82\xF1\x82\xC9\x82\xBF\x82\xCD\x82\xC9\x82\xBF\x82\xCD"); |
|
669 _LIT16(KUnicodeShiftJisWithEucStart,"\x7601\x3053\x3093\x306B\x3061\x306F\x306B\x3061\x306F"); |
|
670 convertDataUsingJ5(test, fileServerSession, KShiftJisWithEucStart, KUnicodeShiftJisWithEucStart ); |
|
671 |
|
672 // some shiftjis which starts with two bytes that could be UTF8 0xc2,0x8e |
|
673 _LIT8(KShiftJisWithUtf8Start,"\xC2\x8E\x82\xB1\x82\xF1\x82\xC9\x82\xBF\x82\xCD\x82\xC9\x82\xBF\x82\xCD"); |
|
674 _LIT16(KUnicodeShiftJisWithUtf8Start,"\xFF82\x7345\xFF71\x3093\x306B\x3061\x306F\x306B\x3061\x306F"); |
|
675 convertDataUsingJ5(test, fileServerSession, KShiftJisWithUtf8Start, KUnicodeShiftJisWithUtf8Start ); |
|
676 |
|
677 // Small shiftjis sample with some half width Katakana |
|
678 _LIT8(KShiftJisHalfWidthKatakana,"\xB0\xE0\x70\xB0\x70"); |
|
679 _LIT16(KUnicodeShiftJisHalfWidthKatakana,"\xFF70\x703E\xFF70\x0070"); |
|
680 convertDataUsingJ5(test, fileServerSession, KShiftJisHalfWidthKatakana, KUnicodeShiftJisHalfWidthKatakana ); |
|
681 |
|
682 /* Read in a text file and convert to unicode using J5 |
|
683 then check the correct converstion was used by checking |
|
684 returned state variable */ |
|
685 // Note ansi.txt is valid for several conversions EUtf8, Eucjp |
|
686 checkDetection( test, fileServerSession, KAnsiTxt, EEucjp); |
|
687 checkDetection( test, fileServerSession, KImodeTxt, EShiftjis); |
|
688 checkDetection( test, fileServerSession, KJpEUCjTxt, EEucjp); |
|
689 checkDetection( test, fileServerSession, KJpJISTxt, EIso2022jp1); |
|
690 checkDetection( test, fileServerSession, KJpSJISTxt, EShiftjis); |
|
691 checkDetection( test, fileServerSession, KJpUnicodeTxt, EUcs2); |
|
692 checkDetection( test, fileServerSession, KJpUTF8Txt, EUtf8); |
|
693 checkDetection( test, fileServerSession, KmoperaTxt, EShiftjis); |
|
694 checkDetection( test, fileServerSession, KMScodeFontTxt,EShiftjis ); |
|
695 checkDetection( test, fileServerSession, KSJISTxt, EShiftjis ); |
|
696 |
|
697 // load the converter ready for testing |
|
698 CCnvCharacterSetConverter* J5Converter=CCnvCharacterSetConverter::NewLC(); |
|
699 CCnvCharacterSetConverter::TAvailability available = CCnvCharacterSetConverter::EAvailable; |
|
700 available = J5Converter->PrepareToConvertToOrFromL(KCharacterSetIdentifierJ5, fileServerSession); |
|
701 if ( available != CCnvCharacterSetConverter::EAvailable) |
|
702 User::Leave(KErrNotSupported); |
|
703 |
|
704 // check some UCS2 examples |
|
705 checkUCS2converstion( test, J5Converter ); |
|
706 |
|
707 // Convert the file Ansi.test supplied by Symbian Japan to unicode |
|
708 convertFileUsingJ5L( test, fileServerSession, J5Converter, KAnsiTxt, KAnsiUni, KAnsiOut ); |
|
709 |
|
710 // Convert the file imode.test supplied by Symbian Japan to unicode |
|
711 convertFileUsingJ5L( test, fileServerSession, J5Converter, KImodeTxt, KImodeUni, KImodeOut ); |
|
712 |
|
713 // Convert the file JpEUC-J.test supplied by Symbian Japan to unicode |
|
714 convertFileUsingJ5L( test, fileServerSession, J5Converter, KJpEUCjTxt, KJpEUCjUni, KJpEUCjOut ); |
|
715 |
|
716 // Convert the file JpJIS.test supplied by Symbian Japan to unicode |
|
717 convertFileUsingJ5L( test, fileServerSession, J5Converter, KJpJISTxt, KJpJISUni, KJpJISOut); |
|
718 |
|
719 // Convert the file JpSJIS.test supplied by Symbian Japan to unicode |
|
720 convertFileUsingJ5L( test, fileServerSession, J5Converter, KJpSJISTxt, KJpSJISUni, KJpSJISOut ); |
|
721 |
|
722 // Convert the file JpUnicode.test supplied by Symbian Japan to unicode |
|
723 convertFileUsingJ5L( test, fileServerSession, J5Converter, KJpUnicodeTxt, KJpUnicodeUni, KJpUnicodeOut ); |
|
724 |
|
725 // Convert the file JpUTF8.test supplied by Symbian Japan to unicode |
|
726 convertFileUsingJ5L( test, fileServerSession, J5Converter, KJpUTF8Txt, KJpUTF8Uni, KJpUTF8Out); |
|
727 |
|
728 // Convert the file mopera.test supplied by Symbian Japan to unicode |
|
729 convertFileUsingJ5L( test, fileServerSession, J5Converter, KmoperaTxt, KmoperaUni, KmoperaOut); |
|
730 |
|
731 // Convert the file MScodefont.test supplied by Symbian Japan to unicode |
|
732 convertFileUsingJ5L( test, fileServerSession, J5Converter, KMScodeFontTxt, KMScodeFontUni, KMScodeFontOut); |
|
733 |
|
734 // Convert the file S-JIS.test supplied by Symbian Japan to unicode |
|
735 convertFileUsingJ5L( test, fileServerSession, J5Converter, KSJISTxt, KSJISUni, KSJISOut); |
|
736 |
|
737 // Check that J5 converts to UTF8 when ConvertFromUnicode() is called |
|
738 convertFromUnicodeUsingJ5( test, fileServerSession, J5Converter ); |
|
739 |
|
740 // autodetect speed benchmark test |
|
741 checkNormalAutodetectL( test, fileServerSession, KMScodeFontTxt ); |
|
742 checkNormalAutodetectL( test, fileServerSession, KSJISTxt ); |
|
743 checkNormalAutodetectL( test, fileServerSession, KImodeTxt ); |
|
744 |
|
745 // conversion speed benchmark test |
|
746 checkConversionSpeedL(test, fileServerSession, KMScodeFontTxt); |
|
747 checkConversionSpeedL(test, fileServerSession, KSJISTxt); |
|
748 checkConversionSpeedL(test, fileServerSession, KImodeTxt); |
|
749 |
|
750 // J5Converter |
|
751 CleanupStack::PopAndDestroy(1 /*, &J5Converter*/); |
|
752 |
|
753 // tidy up after text |
|
754 test.End(); |
|
755 User::After(3000000); |
|
756 // test, fileServerSession |
|
757 CleanupStack::PopAndDestroy(2, &test); |
|
758 } |
|
759 |
|
760 GLDEF_C TInt E32Main() |
|
761 { |
|
762 __UHEAP_MARK; |
|
763 CTrapCleanup* trapCleanup=CTrapCleanup::New(); |
|
764 TRAPD(error, DoE32MainL()); |
|
765 delete trapCleanup; |
|
766 __UHEAP_MARKEND; |
|
767 return error; |
|
768 } |
|
769 |