|
1 /**************************************************************************** |
|
2 ** |
|
3 ** Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). |
|
4 ** All rights reserved. |
|
5 ** Contact: Nokia Corporation (qt-info@nokia.com) |
|
6 ** |
|
7 ** This file is part of the test suite of the Qt Toolkit. |
|
8 ** |
|
9 ** $QT_BEGIN_LICENSE:LGPL$ |
|
10 ** No Commercial Usage |
|
11 ** This file contains pre-release code and may not be distributed. |
|
12 ** You may use this file in accordance with the terms and conditions |
|
13 ** contained in the Technology Preview License Agreement accompanying |
|
14 ** this package. |
|
15 ** |
|
16 ** GNU Lesser General Public License Usage |
|
17 ** Alternatively, this file may be used under the terms of the GNU Lesser |
|
18 ** General Public License version 2.1 as published by the Free Software |
|
19 ** Foundation and appearing in the file LICENSE.LGPL included in the |
|
20 ** packaging of this file. Please review the following information to |
|
21 ** ensure the GNU Lesser General Public License version 2.1 requirements |
|
22 ** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. |
|
23 ** |
|
24 ** In addition, as a special exception, Nokia gives you certain additional |
|
25 ** rights. These rights are described in the Nokia Qt LGPL Exception |
|
26 ** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. |
|
27 ** |
|
28 ** If you have questions regarding the use of this file, please contact |
|
29 ** Nokia at qt-info@nokia.com. |
|
30 ** |
|
31 ** |
|
32 ** |
|
33 ** |
|
34 ** |
|
35 ** |
|
36 ** |
|
37 ** |
|
38 ** $QT_END_LICENSE$ |
|
39 ** |
|
40 ****************************************************************************/ |
|
41 #include <QtTest/QtTest> |
|
42 |
|
43 #include <qtextcodec.h> |
|
44 #include <qsharedpointer.h> |
|
45 |
|
46 //TESTED_CLASS= |
|
47 //TESTED_FILES= |
|
48 |
|
49 static const char utf8bom[] = "\xEF\xBB\xBF"; |
|
50 |
|
51 class tst_Utf8 : public QObject |
|
52 { |
|
53 Q_OBJECT |
|
54 |
|
55 public: |
|
56 // test data: |
|
57 QTextCodec *codec; |
|
58 QString (*from8BitPtr)(const char *, int); |
|
59 QByteArray (QString:: *to8Bit)() const; |
|
60 |
|
61 inline QString from8Bit(const QByteArray &ba) |
|
62 { return from8BitPtr(ba.constData(), ba.length()); } |
|
63 public slots: |
|
64 void initTestCase(); |
|
65 void init(); |
|
66 |
|
67 private slots: |
|
68 void roundTrip_data(); |
|
69 void roundTrip(); |
|
70 |
|
71 void charByChar_data(); |
|
72 void charByChar(); |
|
73 |
|
74 void invalidUtf8_data(); |
|
75 void invalidUtf8(); |
|
76 }; |
|
77 |
|
78 void tst_Utf8::initTestCase() |
|
79 { |
|
80 QTest::addColumn<bool>("useLocale"); |
|
81 QTest::newRow("utf8codec") << false; |
|
82 |
|
83 // is the locale UTF-8? |
|
84 if (QString(QChar(QChar::ReplacementCharacter)).toLocal8Bit() == "\xEF\xBF\xBD") { |
|
85 QTest::newRow("localecodec") << true; |
|
86 qDebug() << "locale is utf8"; |
|
87 } |
|
88 } |
|
89 |
|
90 void tst_Utf8::init() |
|
91 { |
|
92 QFETCH_GLOBAL(bool, useLocale); |
|
93 if (useLocale) { |
|
94 codec = QTextCodec::codecForLocale(); |
|
95 from8BitPtr = &QString::fromLocal8Bit; |
|
96 to8Bit = &QString::toLocal8Bit; |
|
97 } else { |
|
98 codec = QTextCodec::codecForMib(106); |
|
99 from8BitPtr = &QString::fromUtf8; |
|
100 to8Bit = &QString::toUtf8; |
|
101 } |
|
102 } |
|
103 |
|
104 void tst_Utf8::roundTrip_data() |
|
105 { |
|
106 QTest::addColumn<QByteArray>("utf8"); |
|
107 QTest::addColumn<QString>("utf16"); |
|
108 |
|
109 QTest::newRow("empty") << QByteArray() << QString(); |
|
110 QTest::newRow("nul") << QByteArray("", 1) << QString(QChar(QChar::Null)); |
|
111 |
|
112 static const char ascii[] = "This is a standard US-ASCII message"; |
|
113 QTest::newRow("ascii") << QByteArray(ascii) << ascii; |
|
114 |
|
115 static const char ascii2[] = "\1This\2is\3an\4US-ASCII\020 message interspersed with control chars"; |
|
116 QTest::newRow("ascii2") << QByteArray(ascii2) << ascii2; |
|
117 |
|
118 static const char utf8_1[] = "\302\240"; // NBSP |
|
119 QTest::newRow("utf8_1") << QByteArray(utf8_1) << QString(QChar(QChar::Nbsp)); |
|
120 |
|
121 static const char utf8_2[] = "\342\202\254"; // Euro symbol |
|
122 QTest::newRow("utf8_2") << QByteArray(utf8_2) << QString(QChar(0x20AC)); |
|
123 |
|
124 #if 0 |
|
125 // Can't test this because QString::fromUtf8 consumes it |
|
126 static const char utf8_3[] = "\357\273\277"; // byte order mark |
|
127 QTest::newRow("utf8_3") << QByteArray(utf8_3) << QString(QChar(QChar::ByteOrderMark)); |
|
128 #endif |
|
129 |
|
130 static const char utf8_4[] = "\357\277\275"; // replacement char |
|
131 QTest::newRow("utf8_4") << QByteArray(utf8_4) << QString(QChar(QChar::ReplacementCharacter)); |
|
132 |
|
133 static const char utf8_5[] = "\360\220\210\203"; // U+010203 |
|
134 static const uint utf32_5[] = { 0x010203 }; |
|
135 QTest::newRow("utf8_5") << QByteArray(utf8_5) << QString::fromUcs4(utf32_5, 1); |
|
136 |
|
137 static const char utf8_6[] = "\364\217\277\277"; // U+10FFFF |
|
138 static const uint utf32_6[] = { 0x10FFFF }; |
|
139 QTest::newRow("utf8_6") << QByteArray(utf8_6) << QString::fromUcs4(utf32_6, 1); |
|
140 |
|
141 static const char utf8_7[] = "abc\302\240\303\241\303\251\307\275 \342\202\254def"; |
|
142 static const ushort utf16_7[] = { 'a', 'b', 'c', 0x00A0, |
|
143 0x00E1, 0x00E9, 0x01FD, |
|
144 ' ', 0x20AC, 'd', 'e', 'f', 0 }; |
|
145 QTest::newRow("utf8_7") << QByteArray(utf8_7) << QString::fromUtf16(utf16_7); |
|
146 |
|
147 static const char utf8_8[] = "abc\302\240\303\241\303\251\307\275 \364\217\277\277 \342\202\254def"; |
|
148 static const uint utf32_8[] = { 'a', 'b', 'c', 0x00A0, |
|
149 0x00E1, 0x00E9, 0x01FD, |
|
150 ' ', 0x10FFFF, ' ', |
|
151 0x20AC, 'd', 'e', 'f', 0 }; |
|
152 QTest::newRow("utf8_8") << QByteArray(utf8_8) << QString::fromUcs4(utf32_8); |
|
153 } |
|
154 |
|
155 void tst_Utf8::roundTrip() |
|
156 { |
|
157 QFETCH(QByteArray, utf8); |
|
158 QFETCH(QString, utf16); |
|
159 |
|
160 QCOMPARE((utf16.*to8Bit)(), utf8); |
|
161 QCOMPARE(from8Bit(utf8), utf16); |
|
162 |
|
163 QCOMPARE((from8Bit(utf8).*to8Bit)(), utf8); |
|
164 QCOMPARE(from8Bit((utf16.*to8Bit)()), utf16); |
|
165 } |
|
166 |
|
167 void tst_Utf8::charByChar_data() |
|
168 { |
|
169 roundTrip_data(); |
|
170 } |
|
171 |
|
172 void tst_Utf8::charByChar() |
|
173 { |
|
174 QFETCH(QByteArray, utf8); |
|
175 QFETCH(QString, utf16); |
|
176 |
|
177 { |
|
178 // from utf16 to utf8 char by char: |
|
179 QSharedPointer<QTextEncoder> encoder = QSharedPointer<QTextEncoder>(codec->makeEncoder()); |
|
180 QByteArray encoded; |
|
181 |
|
182 for (int i = 0; i < utf16.length(); ++i) { |
|
183 encoded += encoder->fromUnicode(utf16.constData() + i, 1); |
|
184 QVERIFY(!encoder->hasFailure()); |
|
185 } |
|
186 |
|
187 if (encoded.startsWith(utf8bom)) |
|
188 encoded = encoded.mid(strlen(utf8bom)); |
|
189 QCOMPARE(encoded, utf8); |
|
190 } |
|
191 { |
|
192 // from utf8 to utf16 char by char: |
|
193 QSharedPointer<QTextDecoder> decoder = QSharedPointer<QTextDecoder>(codec->makeDecoder()); |
|
194 QString decoded; |
|
195 |
|
196 for (int i = 0; i < utf8.length(); ++i) { |
|
197 decoded += decoder->toUnicode(utf8.constData() + i, 1); |
|
198 QVERIFY(!decoder->hasFailure()); |
|
199 } |
|
200 |
|
201 QCOMPARE(decoded, utf16); |
|
202 } |
|
203 } |
|
204 |
|
205 void tst_Utf8::invalidUtf8_data() |
|
206 { |
|
207 QTest::addColumn<QByteArray>("utf8"); |
|
208 |
|
209 QTest::newRow("1char") << QByteArray("\x80"); |
|
210 QTest::newRow("2chars") << QByteArray("\xC2\xC0"); |
|
211 QTest::newRow("3chars-1") << QByteArray("\xE0\xA0\xC0"); |
|
212 QTest::newRow("3chars-2") << QByteArray("\xE0\xC0\xA0"); |
|
213 QTest::newRow("4chars-1") << QByteArray("\xF0\x90\x80\xC0"); |
|
214 QTest::newRow("4chars-2") << QByteArray("\xF0\x90\xC0\x80"); |
|
215 QTest::newRow("4chars-3") << QByteArray("\xF0\xC0\x80\x80"); |
|
216 |
|
217 // U+FFFE and U+FFFF are non-characters and must not be present |
|
218 // U+FFFE: 1111 11 1111 11 1110 |
|
219 // encoding: xxxz:1111 xz11:1111 xz11:1110 |
|
220 QTest::newRow("fffe") << QByteArray("\xEF\xBF\xBE"); |
|
221 // U+FFFF: 1111 11 1111 11 1111 |
|
222 // encoding: xxxz:1111 xz11:1111 xz11:1111 |
|
223 QTest::newRow("ffff") << QByteArray("\xEF\xBF\xBF"); |
|
224 |
|
225 // Surrogate pairs must now be present either |
|
226 // U+D800: 1101 10 0000 00 0000 |
|
227 // encoding: xxxz:1101 xz10:0000 xz00:0000 |
|
228 QTest::newRow("hi-surrogate") << QByteArray("\xED\xA0\x80"); |
|
229 // U+DC00: 1101 11 0000 00 0000 |
|
230 // encoding: xxxz:1101 xz11:0000 xz00:0000 |
|
231 QTest::newRow("lo-surrogate") << QByteArray("\xED\xB0\x80"); |
|
232 |
|
233 // not even in pair: |
|
234 QTest::newRow("surrogate-pair") << QByteArray("\xED\xA0\x80\xED\xB0\x80"); |
|
235 |
|
236 // Characters outside the Unicode range: |
|
237 // 0x110000: 00 0100 01 0000 00 0000 00 0000 |
|
238 // encoding: xxxx:z100 xz01:0000 xz00:0000 xz00:0000 |
|
239 QTest::newRow("non-unicode-1") << QByteArray("\xF4\x90\x80\x80"); |
|
240 // 0x200000: 00 1000 00 0000 00 0000 00 0000 |
|
241 // encoding: xxxx:xz00 xz00:1000 xz00:0000 xz00:0000 xz00:0000 |
|
242 QTest::newRow("non-unicode-2") << QByteArray("\xF8\x88\x80\x80\x80"); |
|
243 // 0x04000000: 0100 00 0000 00 0000 00 0000 00 0000 |
|
244 // encoding: xxxx:xxz0 xz00:0100 xz00:0000 xz00:0000 xz00:0001 xz00:0001 |
|
245 QTest::newRow("non-unicode-3") << QByteArray("\xFC\x84\x80\x80\x80\x80"); |
|
246 // 0x7fffffff: 1 11 1111 11 1111 11 1111 11 1111 11 1111 |
|
247 // encoding: xxxx:xxz0 xz00:0100 xz00:0000 xz00:0000 xz00:0001 xz00:0001 |
|
248 QTest::newRow("non-unicode-3") << QByteArray("\xFD\xBF\xBF\xBF\xBF\xBF"); |
|
249 |
|
250 // As seen above, 0xFE and 0xFF never appear: |
|
251 QTest::newRow("fe") << QByteArray("\xFE"); |
|
252 QTest::newRow("fe-bis") << QByteArray("\xFE\xBF\xBF\xBF\xBF\xBF\xBF"); |
|
253 QTest::newRow("ff") << QByteArray("\xFF"); |
|
254 QTest::newRow("ff-bis") << QByteArray("\xFF\xBF\xBF\xBF\xBF\xBF\xBF\xBF"); |
|
255 |
|
256 // some combinations in UTF-8 are invalid even though they have the proper bits set |
|
257 // these are known as overlong sequences |
|
258 |
|
259 // "A": U+0041: 01 00 0001 |
|
260 // overlong 2: xxz0:0001 xz00:0001 |
|
261 QTest::newRow("overlong-1-2") << QByteArray("\xC1\x81"); |
|
262 // overlong 3: xxxz:0000 xz00:0001 xz00:0001 |
|
263 QTest::newRow("overlong-1-3") << QByteArray("\xE0\x81\x81"); |
|
264 // overlong 4: xxxx:z000 xz00:0000 xz00:0001 xz00:0001 |
|
265 QTest::newRow("overlong-1-4") << QByteArray("\xF0\x80\x81\x81"); |
|
266 // overlong 5: xxxx:xz00 xz00:0000 xz00:0000 xz00:0001 xz00:0001 |
|
267 QTest::newRow("overlong-1-5") << QByteArray("\xF8\x80\x80\x81\x81"); |
|
268 // overlong 6: xxxx:xxz0 xz00:0000 xz00:0000 xz00:0000 xz00:0001 xz00:0001 |
|
269 QTest::newRow("overlong-1-6") << QByteArray("\xFC\x80\x80\x80\x81\x81"); |
|
270 |
|
271 // NBSP: U+00A0: 10 00 0000 |
|
272 // proper encoding: xxz0:0010 xz00:0000 |
|
273 // overlong 3: xxxz:0000 xz00:0010 xz00:0000 |
|
274 QTest::newRow("overlong-2-3") << QByteArray("\xC0\x82\x80"); |
|
275 // overlong 4: xxxx:z000 xz00:0000 xz00:0010 xz00:0000 |
|
276 QTest::newRow("overlong-2-4") << QByteArray("\xF0\x80\x82\x80"); |
|
277 // overlong 5: xxxx:xz00 xz00:0000 xz00:0000 xz00:0010 xz00:0000 |
|
278 QTest::newRow("overlong-2-4") << QByteArray("\xF8\x80\x80\x82\x80"); |
|
279 // overlong 6: xxxx:xxz0 xz00:0000 xz00:0000 xz00:0000 xz00:0010 xz00:0000 |
|
280 QTest::newRow("overlong-2-4") << QByteArray("\xFC\x80\x80\x80\x82\x80"); |
|
281 |
|
282 // U+0800: 10 0000 00 0000 |
|
283 // proper encoding: xxxz:0000 xz10:0000 xz00:0000 |
|
284 // overlong 4: xxxx:z000 xz00:0000 xz10:0000 xz00:0000 |
|
285 QTest::newRow("overlong-3-4") << QByteArray("\xF0\x80\xA0\x80"); |
|
286 // overlong 5: xxxx:xz00 xz00:0000 xz00:0000 xz10:0000 xz00:0000 |
|
287 QTest::newRow("overlong-3-5") << QByteArray("\xF8\x80\x80\xA0\x80"); |
|
288 // overlong 6: xxxx:xxz0 xz00:0000 xz00:0000 xz00:0000 xz10:0000 xz00:0000 |
|
289 QTest::newRow("overlong-3-6") << QByteArray("\xFC\x80\x80\x80\xA0\x80"); |
|
290 |
|
291 // U+010000: 00 0100 00 0000 00 0000 |
|
292 // proper encoding: xxxx:z000 xz00:0100 xz00:0000 xz00:0000 |
|
293 // overlong 5: xxxx:xz00 xz00:0000 xz00:0100 xz00:0000 xz00:0000 |
|
294 QTest::newRow("overlong-4-5") << QByteArray("\xF8\x80\x84\x80\x80"); |
|
295 // overlong 6: xxxx:xxz0 xz00:0000 xz00:0000 xz00:0100 xz00:0000 xz00:0000 |
|
296 QTest::newRow("overlong-4-6") << QByteArray("\xFC\x80\x80\x84\x80\x80"); |
|
297 } |
|
298 |
|
299 void tst_Utf8::invalidUtf8() |
|
300 { |
|
301 QFETCH(QByteArray, utf8); |
|
302 QFETCH_GLOBAL(bool, useLocale); |
|
303 |
|
304 QSharedPointer<QTextDecoder> decoder = QSharedPointer<QTextDecoder>(codec->makeDecoder()); |
|
305 QString decoded = decoder->toUnicode(utf8); |
|
306 |
|
307 // Only enforce correctness on our UTF-8 decoder |
|
308 // The system's UTF-8 codec is sometimes buggy |
|
309 // GNU libc's iconv is known to accept U+FFFF and U+FFFE encoded as UTF-8 |
|
310 // OS X's iconv is known to accept those, plus surrogates and codepoints above U+10FFFF |
|
311 if (!useLocale) |
|
312 QVERIFY(decoder->hasFailure()); |
|
313 else if (!decoder->hasFailure()) |
|
314 qWarning("System codec does not report failure when it should. Should report bug upstream."); |
|
315 } |
|
316 |
|
317 QTEST_MAIN(tst_Utf8) |
|
318 #include "tst_utf8.moc" |