FCL/sf/mw/qt: comparison util/unicode/main.cpp

equal deleted inserted replaced

-:b72c6db6890b
+:5dc02b23752f
 **
 **
 ** $QT_END_LICENSE$
 **
 ****************************************************************************/
 #include <qlist.h>
 #include <qhash.h>
 #include <qfile.h>
+#include <qbytearray.h>
 #include <qstring.h>
 #include <qchar.h>
-#include <private/qunicodetables_p.h>
 #include <qvector.h>
 #include <qdebug.h>
+#if 0
+#include <private/qunicodetables_p.h>
-static struct AgeMap {
+#endif
-const char *age;
-const QChar::UnicodeVersion version;
+#define DATA_VERSION_S "5.0"
-} ageMap [] = {
+#define DATA_VERSION_STR "QChar::Unicode_5_0"
-{ "1.1", QChar::Unicode_1_1 },
-{ "2.0", QChar::Unicode_2_0 },
+#define LAST_CODEPOINT 0x10ffff
-{ "2.1", QChar::Unicode_2_1_2 },
+#define LAST_CODEPOINT_STR "0x10ffff"
-{ "3.0", QChar::Unicode_3_0 },
-{ "3.1", QChar::Unicode_3_1 },
-{ "3.2", QChar::Unicode_3_2 },
+static QHash<QByteArray, QChar::UnicodeVersion> age_map;
-{ "4.0", QChar::Unicode_4_0 },
-{ "4.1", QChar::Unicode_4_1 },
+static void initAgeMap()
-{ "5.0", QChar::Unicode_5_0 },
+{
-{ 0, QChar::Unicode_Unassigned }
+struct AgeMap {
-};
+const QChar::UnicodeVersion version;
-#define CURRENT_UNICODE_VERSION "QChar::Unicode_5_0"
+const char *age;
+} ageMap[] = {
+{ QChar::Unicode_1_1,   "1.1" },
+{ QChar::Unicode_2_0,   "2.0" },
+{ QChar::Unicode_2_1_2, "2.1" },
+{ QChar::Unicode_3_0,   "3.0" },
+{ QChar::Unicode_3_1,   "3.1" },
+{ QChar::Unicode_3_2,   "3.2" },
+{ QChar::Unicode_4_0,   "4.0" },
+{ QChar::Unicode_4_1,   "4.1" },
+{ QChar::Unicode_5_0,   "5.0" },
+{ QChar::Unicode_Unassigned, 0 }
+};
+AgeMap *d = ageMap;
+while (d->age) {
+age_map.insert(d->age, d->version);
+++d;
+}
+}
 static const char *grapheme_break_string =
 "    enum GraphemeBreak {\n"
-"        GraphemeBreakOther, \n"
+"        GraphemeBreakOther,\n"
 "        GraphemeBreakCR,\n"
 "        GraphemeBreakLF,\n"
 "        GraphemeBreakControl,\n"
 "        GraphemeBreakExtend,\n"
 "        GraphemeBreakL,\n"
 GraphemeBreakL,
 GraphemeBreakV,
 GraphemeBreakT,
 GraphemeBreakLV,
 GraphemeBreakLVT
+, GraphemeBreak_Unassigned
 };
-QHash<QByteArray, GraphemeBreak> grapheme_break_map;
+static QHash<QByteArray, GraphemeBreak> grapheme_break_map;
 static void initGraphemeBreak()
 {
 struct GraphemeBreakList {
 GraphemeBreak brk;
 { GraphemeBreakL, "L" },
 { GraphemeBreakV, "V" },
 { GraphemeBreakT, "T" },
 { GraphemeBreakLV, "LV" },
 { GraphemeBreakLVT, "LVT" },
-{ GraphemeBreakOther, 0 }
+{ GraphemeBreak_Unassigned, 0 }
 };
 GraphemeBreakList *d = breaks;
 while (d->name) {
 grapheme_break_map.insert(d->name, d->brk);
 ++d;
 }
 }
-const char *word_break_string =
+static const char *word_break_string =
 "    enum WordBreak {\n"
 "        WordBreakOther,\n"
 "        WordBreakFormat,\n"
 "        WordBreakKatakana,\n"
 "        WordBreakALetter,\n"
 WordBreakALetter,
 WordBreakMidLetter,
 WordBreakMidNum,
 WordBreakNumeric,
 WordBreakExtendNumLet
+, WordBreak_Unassigned
 };
+static QHash<QByteArray, WordBreak> word_break_map;
-QHash<QByteArray, WordBreak> word_break_map;
 static void initWordBreak()
 {
 struct WordBreakList {
 WordBreak brk;
 { WordBreakALetter, "ALetter" },
 { WordBreakMidLetter, "MidLetter" },
 { WordBreakMidNum, "MidNum" },
 { WordBreakNumeric, "Numeric" },
 { WordBreakExtendNumLet, "ExtendNumLet" },
-{ WordBreakFormat,  0 }
+{ WordBreak_Unassigned, 0 }
 };
 WordBreakList *d = breaks;
 while (d->name) {
 word_break_map.insert(d->name, d->brk);
 ++d;
 SentenceBreakOLetter,
 SentenceBreakNumeric,
 SentenceBreakATerm,
 SentenceBreakSTerm,
 SentenceBreakClose
+, SentenceBreak_Unassigned
 };
+static QHash<QByteArray, SentenceBreak> sentence_break_map;
-QHash<QByteArray, SentenceBreak> sentence_break_map;
 static void initSentenceBreak()
 {
 struct SentenceBreakList {
 SentenceBreak brk;
 { SentenceBreakOLetter, "OLetter" },
 { SentenceBreakNumeric, "Numeric" },
 { SentenceBreakATerm, "ATerm" },
 { SentenceBreakSTerm, "STerm" },
 { SentenceBreakClose, "Close" },
-{ SentenceBreakOther,  0 }
+{ SentenceBreak_Unassigned, 0 }
 };
 SentenceBreakList *d = breaks;
 while (d->name) {
 sentence_break_map.insert(d->name, d->brk);
 ++d;
 }
 }
-// Keep this one in sync with the code in createPropertyInfo
+static const char *lineBreakClass =
-const char *property_string =
-"    struct Properties {\n"
-"        ushort category : 8;\n"
-"        ushort line_break_class : 8;\n"
-"        ushort direction : 8;\n"
-"        ushort combiningClass :8;\n"
-"        ushort joining : 2;\n"
-"        signed short digitValue : 6; /* 5 needed */\n"
-"        ushort unicodeVersion : 4;\n"
-"        ushort lowerCaseSpecial : 1;\n"
-"        ushort upperCaseSpecial : 1;\n"
-"        ushort titleCaseSpecial : 1;\n"
-"        ushort caseFoldSpecial : 1; /* currently unused */\n"
-"        signed short mirrorDiff : 16;\n"
-"        signed short lowerCaseDiff : 16;\n"
-"        signed short upperCaseDiff : 16;\n"
-"        signed short titleCaseDiff : 16;\n"
-"        signed short caseFoldDiff : 16;\n"
-"        ushort graphemeBreak : 8;\n"
-"        ushort wordBreak : 8;\n"
-"        ushort sentenceBreak : 8;\n"
-"    };\n"
-"    Q_CORE_EXPORT const Properties * QT_FASTCALL properties(uint ucs4);\n"
-"    Q_CORE_EXPORT const Properties * QT_FASTCALL properties(ushort ucs2);\n";
-const char *lineBreakClass =
 "    // see http://www.unicode.org/reports/tr14/tr14-19.html\n"
 "    // we don't use the XX, AI and CB properties and map them to AL instead.\n"
 "    // as we don't support any EBDIC based OS'es, NL is ignored and mapped to AL as well.\n"
 "    enum LineBreakClass {\n"
 "        LineBreak_OP, LineBreak_CL, LineBreak_QU, LineBreak_GL, LineBreak_NS,\n"
 "        LineBreak_WJ, LineBreak_H2, LineBreak_H3, LineBreak_JL, LineBreak_JV,\n"
 "        LineBreak_JT, LineBreak_SA, LineBreak_SG,\n"
 "        LineBreak_SP, LineBreak_CR, LineBreak_LF, LineBreak_BK\n"
 "    };\n\n";
-const char *methods =
+enum LineBreakClass {
+LineBreak_OP, LineBreak_CL, LineBreak_QU, LineBreak_GL, LineBreak_NS,
+LineBreak_EX, LineBreak_SY, LineBreak_IS, LineBreak_PR, LineBreak_PO,
+LineBreak_NU, LineBreak_AL, LineBreak_ID, LineBreak_IN, LineBreak_HY,
+LineBreak_BA, LineBreak_BB, LineBreak_B2, LineBreak_ZW, LineBreak_CM,
+LineBreak_WJ, LineBreak_H2, LineBreak_H3, LineBreak_JL, LineBreak_JV,
+LineBreak_JT, LineBreak_SA, LineBreak_SG,
+LineBreak_SP, LineBreak_CR, LineBreak_LF, LineBreak_BK
+, LineBreak_Unassigned
+};
+static QHash<QByteArray, LineBreakClass> line_break_map;
+static void initLineBreak()
+{
+// ### Classes XX and AI are left out and mapped to AL for now;
+// ### Class NL is ignored and mapped to AL as well.
+struct LineBreakList {
+LineBreakClass brk;
+const char *name;
+} breaks[] = {
+{ LineBreak_BK, "BK" },
+{ LineBreak_CR, "CR" },
+{ LineBreak_LF, "LF" },
+{ LineBreak_CM, "CM" },
+{ LineBreak_AL, "NL" },
+{ LineBreak_SG, "SG" },
+{ LineBreak_WJ, "WJ" },
+{ LineBreak_ZW, "ZW" },
+{ LineBreak_GL, "GL" },
+{ LineBreak_SP, "SP" },
+{ LineBreak_B2, "B2" },
+{ LineBreak_BA, "BA" },
+{ LineBreak_BB, "BB" },
+{ LineBreak_HY, "HY" },
+{ LineBreak_AL, "CB" }, // ###
+{ LineBreak_CL, "CL" },
+{ LineBreak_EX, "EX" },
+{ LineBreak_IN, "IN" },
+{ LineBreak_NS, "NS" },
+{ LineBreak_OP, "OP" },
+{ LineBreak_QU, "QU" },
+{ LineBreak_IS, "IS" },
+{ LineBreak_NU, "NU" },
+{ LineBreak_PO, "PO" },
+{ LineBreak_PR, "PR" },
+{ LineBreak_SY, "SY" },
+{ LineBreak_AL, "AI" },
+{ LineBreak_AL, "AL" },
+{ LineBreak_H2, "H2" },
+{ LineBreak_H3, "H3" },
+{ LineBreak_ID, "ID" },
+{ LineBreak_JL, "JL" },
+{ LineBreak_JV, "JV" },
+{ LineBreak_JT, "JT" },
+{ LineBreak_SA, "SA" },
+{ LineBreak_AL, "XX" },
+{ LineBreak_Unassigned, 0 }
+};
+LineBreakList *d = breaks;
+while (d->name) {
+line_break_map.insert(d->name, d->brk);
+++d;
+}
+}
+// Keep this one in sync with the code in createPropertyInfo
+static const char *property_string =
+"    struct Properties {\n"
+"        ushort category         : 8; /* 5 needed */\n"
+"        ushort line_break_class : 8; /* 6 needed */\n"
+"        ushort direction        : 8; /* 5 needed */\n"
+"        ushort combiningClass   : 8;\n"
+"        ushort joining          : 2;\n"
+"        signed short digitValue : 6; /* 5 needed */\n"
+"        ushort unicodeVersion   : 4;\n"
+"        ushort lowerCaseSpecial : 1;\n"
+"        ushort upperCaseSpecial : 1;\n"
+"        ushort titleCaseSpecial : 1;\n"
+"        ushort caseFoldSpecial  : 1; /* currently unused */\n"
+"        signed short mirrorDiff    : 16;\n"
+"        signed short lowerCaseDiff : 16;\n"
+"        signed short upperCaseDiff : 16;\n"
+"        signed short titleCaseDiff : 16;\n"
+"        signed short caseFoldDiff  : 16;\n"
+"        ushort graphemeBreak    : 8; /* 4 needed */\n"
+"        ushort wordBreak        : 8; /* 4 needed */\n"
+"        ushort sentenceBreak    : 8; /* 4 needed */\n"
+"    };\n"
+"    Q_CORE_EXPORT const Properties * QT_FASTCALL properties(uint ucs4);\n"
+"    Q_CORE_EXPORT const Properties * QT_FASTCALL properties(ushort ucs2);\n";
+static const char *methods =
 "    Q_CORE_EXPORT QUnicodeTables::LineBreakClass QT_FASTCALL lineBreakClass(uint ucs4);\n"
-"    inline int lineBreakClass(const QChar &ch) {\n"
+"    inline int lineBreakClass(const QChar &ch)\n"
-"        return QUnicodeTables::lineBreakClass(ch.unicode());\n"
+"    { return lineBreakClass(ch.unicode()); }\n"
-"    }\n"
 "\n"
 "    Q_CORE_EXPORT int QT_FASTCALL script(uint ucs4);\n"
-"    Q_CORE_EXPORT_INLINE int QT_FASTCALL script(const QChar &ch) {\n"
+"    inline int script(const QChar &ch)\n"
-"        return script(ch.unicode());\n"
+"    { return script(ch.unicode()); }\n\n";
-"    }\n\n";
 struct PropertyFlags {
 bool operator ==(const PropertyFlags &o) {
 return (combiningClass == o.combiningClass
 // from ArabicShaping.txt
 QChar::Joining joining : 2;
 // from DerivedAge.txt
 QChar::UnicodeVersion age : 4;
 int digitValue;
-uint line_break_class : 5;
+uint line_break_class : 6;
 int mirrorDiff : 16;
 int lowerCaseDiff;
 int upperCaseDiff;
 GraphemeBreak graphemeBreak;
 WordBreak wordBreak;
 SentenceBreak sentenceBreak;
 };
-QList<int> specialCaseMap;
-int specialCaseMaxLen = 0;
+static QList<int> specialCaseMap;
+static int specialCaseMaxLen = 0;
 static int appendToSpecialCaseMap(const QList<int> &map)
 {
 QList<int> utf16map;
 for (int i = 0; i < map.size(); ++i) {
 }
 }
 specialCaseMaxLen = qMax(specialCaseMaxLen, utf16map.size());
 utf16map << 0;
-for (int i = 0; i < specialCaseMap.size() - utf16map.size() - 1; ++i) {
+for (int i = 0; i < specialCaseMap.size() - utf16map.size() + 1; ++i) {
 int j;
 for (j = 0; j < utf16map.size(); ++j) {
 if (specialCaseMap.at(i+j) != utf16map.at(j))
 break;
 }
 return pos;
 }
 struct UnicodeData {
 UnicodeData(int codepoint = 0) {
-p.category = QChar::NoCategory;
+p.category = QChar::Other_NotAssigned; // Cn
 p.combiningClass = 0;
 p.direction = QChar::DirL;
 // DirR for:  U+0590..U+05FF, U+07C0..U+08FF, U+FB1D..U+FB4F, U+10800..U+10FFF
 if ((codepoint >= 0x590 && codepoint <= 0x5ff)
 decompositionType = QChar::NoDecomposition;
 p.joining = QChar::OtherJoining;
 p.age = QChar::Unicode_Unassigned;
 p.mirrorDiff = 0;
 p.digitValue = -1;
-p.line_break_class = QUnicodeTables::LineBreak_AL;
+p.line_break_class = LineBreak_AL; // XX -> AL
 p.lowerCaseDiff = 0;
 p.upperCaseDiff = 0;
 p.titleCaseDiff = 0;
 p.caseFoldDiff = 0;
 p.lowerCaseSpecial = 0;
 UD_UpperCase,
 UD_LowerCase,
 UD_TitleCase
 };
-QHash<QByteArray, QChar::Category> categoryMap;
+static QHash<QByteArray, QChar::Category> categoryMap;
 static void initCategoryMap()
 {
 struct Cat {
 QChar::Category cat;
 const char *name;
-} categories [] = {
+} categories[] = {
 { QChar::Mark_NonSpacing,          "Mn" },
 { QChar::Mark_SpacingCombining,    "Mc" },
 { QChar::Mark_Enclosing,           "Me" },
 { QChar::Number_DecimalDigit,      "Nd" },
 { QChar::Symbol_Modifier,          "Sk" },
 { QChar::Symbol_Other,             "So" },
 { QChar::NoCategory, 0 }
 };
 Cat *c = categories;
-while (c->cat != QChar::NoCategory) {
+while (c->name) {
 categoryMap.insert(c->name, c->cat);
 ++c;
 }
 }
-QHash<QByteArray, QChar::Direction> directionMap;
+static QHash<QByteArray, QChar::Direction> directionMap;
 static void initDirectionMap()
 {
 struct Dir {
 QChar::Direction dir;
 ++d;
 }
 }
-QHash<QByteArray, QChar::Decomposition> decompositionMap;
+static QHash<QByteArray, QChar::Decomposition> decompositionMap;
 static void initDecompositionMap()
 {
 struct Dec {
 QChar::Decomposition dec;
 { QChar::Narrow, "<narrow>" },
 { QChar::Small, "<small>" },
 { QChar::Square, "<square>" },
 { QChar::Compat, "<compat>" },
 { QChar::Fraction, "<fraction>" },
-{ QChar::NoDecomposition,  0 }
+{ QChar::NoDecomposition, 0 }
 };
 Dec *d = decompositions;
 while (d->name) {
 decompositionMap.insert(d->name, d->dec);
 ++d;
 }
 }
-QHash<int, UnicodeData> unicodeData;
+static QHash<int, UnicodeData> unicodeData;
-QList<PropertyFlags> uniqueProperties;
+static QList<PropertyFlags> uniqueProperties;
-QHash<int, int> decompositionLength;
+static QHash<int, int> decompositionLength;
-int highestComposedCharacter = 0;
+static int highestComposedCharacter = 0;
-int numLigatures = 0;
+static int numLigatures = 0;
-int highestLigature = 0;
+static int highestLigature = 0;
-struct Ligature {ushort u1; ushort u2; ushort ligature;};
+struct Ligature {
+ushort u1;
+ushort u2;
+ushort ligature;
+};
 // we need them sorted after the first component for fast lookup
-bool operator < (const Ligature &l1, const Ligature &l2) {
+bool operator < (const Ligature &l1, const Ligature &l2)
-return l1.u1 < l2.u1;
+{ return l1.u1 < l2.u1; }
-}
+static QHash<ushort, QList<Ligature> > ligatureHashes;
-QHash<ushort, QList<Ligature> > ligatureHashes;
+static QHash<int, int> combiningClassUsage;
-QHash<int, int> combiningClassUsage;
+static int maxLowerCaseDiff = 0;
-int maxLowerCaseDiff = 0;
+static int maxUpperCaseDiff = 0;
-int maxUpperCaseDiff = 0;
+static int maxTitleCaseDiff = 0;
-int maxTitleCaseDiff = 0;
 static void readUnicodeData()
 {
 QFile f("data/UnicodeData.txt");
 if (!f.exists())
 continue;
 QList<QByteArray> properties = line.split(';');
 bool ok;
 int codepoint = properties[UD_Value].toInt(&ok, 16);
+Q_ASSERT(ok);
+Q_ASSERT(codepoint <= LAST_CODEPOINT);
 int lastCodepoint = codepoint;
 QByteArray name = properties[UD_Name];
 if (name.startsWith('<') && name.contains("First")) {
 QByteArray nextLine;
 nextLine.resize(1024);
 f.readLine(nextLine.data(), 1024);
 QList<QByteArray> properties = nextLine.split(';');
+Q_ASSERT(properties[UD_Name].startsWith('<') && properties[UD_Name].contains("Last"));
 lastCodepoint = properties[UD_Value].toInt(&ok, 16);
+Q_ASSERT(ok);
+Q_ASSERT(lastCodepoint <= LAST_CODEPOINT);
 }
 UnicodeData data(codepoint);
 data.p.category = categoryMap.value(properties[UD_Category], QChar::NoCategory);
+if (data.p.category == QChar::NoCategory)
+qFatal("unassigned char category: %s", properties[UD_Category].constData());
 data.p.combiningClass = properties[UD_CombiningClass].toInt();
 if (!combiningClassUsage.contains(data.p.combiningClass))
 combiningClassUsage[data.p.combiningClass] = 1;
 else
 data.p.direction = directionMap.value(properties[UD_BidiCategory], data.p.direction);
 if (!properties[UD_UpperCase].isEmpty()) {
 int upperCase = properties[UD_UpperCase].toInt(&ok, 16);
 Q_ASSERT(ok);
+if (qAbs(upperCase - codepoint) >= (1<<14))
+qWarning() << "upperCaseDiff exceeded (" << hex << codepoint << "->" << upperCase << ")";
 data.p.upperCaseDiff = upperCase - codepoint;
 maxUpperCaseDiff = qMax(maxUpperCaseDiff, qAbs(data.p.upperCaseDiff));
 if (codepoint > 0xffff) {
 // if the condition below doesn't hold anymore we need to modify our case folding code
 //qDebug() << codepoint << QChar::highSurrogate(codepoint) << QChar::highSurrogate(foldMap.at(0));
 Q_ASSERT(QChar::highSurrogate(codepoint) == QChar::highSurrogate(upperCase));
 }
 }
 if (!properties[UD_LowerCase].isEmpty()) {
 int lowerCase = properties[UD_LowerCase].toInt(&ok, 16);
-Q_ASSERT (ok);
+Q_ASSERT(ok);
+if (qAbs(lowerCase - codepoint) >= (1<<14))
+qWarning() << "lowerCaseDiff exceeded (" << hex << codepoint << "->" << lowerCase << ")";
 data.p.lowerCaseDiff = lowerCase - codepoint;
 maxLowerCaseDiff = qMax(maxLowerCaseDiff, qAbs(data.p.lowerCaseDiff));
 if (codepoint > 0xffff) {
 // if the condition below doesn't hold anymore we need to modify our case folding code
 //qDebug() << codepoint << QChar::highSurrogate(codepoint) << QChar::highSurrogate(foldMap.at(0));
 // we want toTitleCase to map to ToUpper in case we don't have any titlecase.
 if (properties[UD_TitleCase].isEmpty())
 properties[UD_TitleCase] = properties[UD_UpperCase];
 if (!properties[UD_TitleCase].isEmpty()) {
 int titleCase = properties[UD_TitleCase].toInt(&ok, 16);
-Q_ASSERT (ok);
+Q_ASSERT(ok);
+if (qAbs(titleCase - codepoint) >= (1<<14))
+qWarning() << "titleCaseDiff exceeded (" << hex << codepoint << "->" << titleCase << ")";
 data.p.titleCaseDiff = titleCase - codepoint;
 maxTitleCaseDiff = qMax(maxTitleCaseDiff, qAbs(data.p.titleCaseDiff));
 if (codepoint > 0xffff) {
 // if the condition below doesn't hold anymore we need to modify our case folding code
 //qDebug() << codepoint << QChar::highSurrogate(codepoint) << QChar::highSurrogate(foldMap.at(0));
 QByteArray decomposition = properties[UD_Decomposition];
 if (!decomposition.isEmpty()) {
 highestComposedCharacter = qMax(highestComposedCharacter, codepoint);
 QList<QByteArray> d = decomposition.split(' ');
 if (d[0].contains('<')) {
-data.decompositionType = decompositionMap.value(d[0], QChar::Canonical);
+data.decompositionType = decompositionMap.value(d[0], QChar::NoDecomposition);
+if (data.decompositionType == QChar::NoDecomposition)
+qFatal("unassigned decomposition type: %s", d[0].constData());
 d.takeFirst();
 } else {
 data.decompositionType = QChar::Canonical;
 }
-for (int i = 0; i < d.size(); ++i)
+for (int i = 0; i < d.size(); ++i) {
 data.decomposition.append(d[i].toInt(&ok, 16));
+Q_ASSERT(ok);
+}
 if (!decompositionLength.contains(data.decomposition.size()))
 decompositionLength[data.decomposition.size()] = 1;
 else
 ++decompositionLength[data.decomposition.size()];
 }
 QList<QByteArray> pair = line.split(';');
 Q_ASSERT(pair.size() == 2);
 bool ok;
 int codepoint = pair[0].toInt(&ok, 16);
+Q_ASSERT(ok);
 int mirror = pair[1].toInt(&ok, 16);
+Q_ASSERT(ok);
 UnicodeData d = unicodeData.value(codepoint, UnicodeData(codepoint));
 d.mirroredChar = mirror;
-if (qAbs(codepoint-d.mirroredChar) > maxMirroredDiff)
-maxMirroredDiff = qAbs(codepoint - d.mirroredChar);
 d.p.mirrorDiff = d.mirroredChar - codepoint;
+maxMirroredDiff = qMax(maxMirroredDiff, qAbs(d.p.mirrorDiff));
 unicodeData.insert(codepoint, d);
 }
 }
 static void readArabicShaping()
 QList<QByteArray> shaping = line.split(';');
 Q_ASSERT(shaping.size() == 4);
 bool ok;
 int codepoint = shaping[0].toInt(&ok, 16);
+Q_ASSERT(ok);
 QChar::Joining j = QChar::OtherJoining;
 QByteArray shape = shaping[2].trimmed();
 if (shape == "R")
 j = QChar::Right;
 else if (shape == "D")
 codes.replace("..", ".");
 QList<QByteArray> cl = codes.split('.');
 bool ok;
 int from = cl[0].toInt(&ok, 16);
+Q_ASSERT(ok);
 int to = from;
-if (cl.size() == 2)
+if (cl.size() == 2) {
 to = cl[1].toInt(&ok, 16);
+Q_ASSERT(ok);
-QChar::UnicodeVersion age = QChar::Unicode_Unassigned;
+}
-QByteArray ba = l[1];
-AgeMap *map = ageMap;
+QChar::UnicodeVersion age = age_map.value(l[1].trimmed(), QChar::Unicode_Unassigned);
-while (map->age) {
-if (ba == map->age) {
-age = map->version;
-break;
-}
-++map;
-}
 //qDebug() << hex << from << ".." << to << ba << age;
-Q_ASSERT(age != QChar::Unicode_Unassigned);
+if (age == QChar::Unicode_Unassigned)
+qFatal("unassigned or unhandled age value: %s", l[1].constData());
 for (int codepoint = from; codepoint <= to; ++codepoint) {
 UnicodeData d = unicodeData.value(codepoint, UnicodeData(codepoint));
 d.p.age = age;
 unicodeData.insert(codepoint, d);
 }
 }
 }
-static void readCompositionExclusion()
+static void readDerivedNormalizationProps()
 {
-QFile f("data/CompositionExclusions.txt");
+QFile f("data/DerivedNormalizationProps.txt");
 if (!f.exists())
-qFatal("Couldn't find CompositionExclusions.txt");
+qFatal("Couldn't find DerivedNormalizationProps.txt");
 f.open(QFile::ReadOnly);
 while (!f.atEnd()) {
 QByteArray line;
 line.resize(len-1);
 int comment = line.indexOf('#');
 if (comment >= 0)
 line = line.left(comment);
-line.replace(" ", "");
+if (line.trimmed().isEmpty())
-if (line.isEmpty())
 continue;
-Q_ASSERT(!line.contains(".."));
+QList<QByteArray> l = line.split(';');
+Q_ASSERT(l.size() >= 2);
+QByteArray propName = l[1].trimmed();
+if (propName != "Full_Composition_Exclusion")
+// ###
+continue;
+QByteArray codes = l[0].trimmed();
+codes.replace("..", ".");
+QList<QByteArray> cl = codes.split('.');
 bool ok;
-int codepoint = line.toInt(&ok, 16);
+int from = cl[0].toInt(&ok, 16);
+Q_ASSERT(ok);
+int to = from;
+if (cl.size() == 2) {
+to = cl[1].toInt(&ok, 16);
+Q_ASSERT(ok);
+}
+for (int codepoint = from; codepoint <= to; ++codepoint) {
+UnicodeData d = unicodeData.value(codepoint, UnicodeData(codepoint));
+d.excludedComposition = true;
+unicodeData.insert(codepoint, d);
+}
+}
+for (int codepoint = 0; codepoint <= LAST_CODEPOINT; ++codepoint) {
 UnicodeData d = unicodeData.value(codepoint, UnicodeData(codepoint));
-d.excludedComposition = true;
+if (!d.excludedComposition
-unicodeData.insert(codepoint, d);
+&& d.decompositionType == QChar::Canonical
-}
+&& d.decomposition.size() > 1) {
+Q_ASSERT(d.decomposition.size() == 2);
-for (int i = 0; i < 0x110000; ++i) {
-UnicodeData data = unicodeData.value(i, UnicodeData(i));
+uint part1 = d.decomposition.at(0);
-if (!data.excludedComposition
+uint part2 = d.decomposition.at(1);
-&& data.decompositionType == QChar::Canonical
-&& data.decomposition.size() > 1) {
+// all non-starters are listed in DerivedNormalizationProps.txt
-Q_ASSERT(data.decomposition.size() == 2);
+// and already excluded from composition
+Q_ASSERT(unicodeData.value(part1, UnicodeData(part1)).p.combiningClass == 0);
-uint part1 = data.decomposition.at(0);
-uint part2 = data.decomposition.at(1);
-UnicodeData first = unicodeData.value(part1, UnicodeData(part1));
-if (first.p.combiningClass != 0)
-continue;
 ++numLigatures;
 highestLigature = qMax(highestLigature, (int)part1);
-Ligature l = {(ushort)part1, (ushort)part2, i};
+Ligature l = {(ushort)part1, (ushort)part2, codepoint};
 ligatureHashes[part2].append(l);
 }
 }
 }
 struct NormalizationCorrection {
 uint codepoint;
 uint mapped;
 uint version;
 Q_ASSERT(!line.contains(".."));
 QList<QByteArray> fields = line.split(';');
 Q_ASSERT(fields.size() == 4);
-NormalizationCorrection c;
+NormalizationCorrection c = { 0, 0, 0 };
 bool ok;
 c.codepoint = fields.at(0).toInt(&ok, 16);
+Q_ASSERT(ok);
 c.mapped = fields.at(1).toInt(&ok, 16);
+Q_ASSERT(ok);
 if (fields.at(3) == "3.2.0")
 c.version = QChar::Unicode_3_2;
 else if (fields.at(3) == "4.0.0")
 c.version = QChar::Unicode_4_0;
 else
 out += "};\n\n"
 "enum { NumNormalizationCorrections = " + QByteArray::number(numCorrections) + " };\n\n";
 return out;
 }
 static void computeUniqueProperties()
 {
 qDebug("computeUniqueProperties:");
-for (int uc = 0; uc < 0x110000; ++uc) {
+for (int uc = 0; uc <= LAST_CODEPOINT; ++uc) {
 UnicodeData d = unicodeData.value(uc, UnicodeData(uc));
 int index = uniqueProperties.indexOf(d.p);
 if (index == -1) {
 index = uniqueProperties.size();
 uniqueProperties.append(d.p);
 }
 d.propertyIndex = index;
 unicodeData.insert(uc, d);
 }
-qDebug("    %d unicode properties found", uniqueProperties.size());
+qDebug("    %d unique unicode properties found", uniqueProperties.size());
 }
 static void readLineBreak()
 {
 codes.replace("..", ".");
 QList<QByteArray> cl = codes.split('.');
 bool ok;
 int from = cl[0].toInt(&ok, 16);
+Q_ASSERT(ok);
 int to = from;
-if (cl.size() == 2)
+if (cl.size() == 2) {
 to = cl[1].toInt(&ok, 16);
+Q_ASSERT(ok);
-// ### Classes XX and AI are left out and mapped to AL for now
+}
-QUnicodeTables::LineBreakClass lb = QUnicodeTables::LineBreak_AL;
-QByteArray ba = l[1];
+LineBreakClass lb = line_break_map.value(l[1].trimmed(), LineBreak_Unassigned);
+if (lb == LineBreak_Unassigned)
-if (ba == "AI") lb = QUnicodeTables::LineBreak_AL;
+qFatal("unassigned line break class: %s", l[1].constData());
-else if (ba == "XX") lb = QUnicodeTables::LineBreak_AL;
-else if (ba == "NL") lb = QUnicodeTables::LineBreak_AL;
-else if (ba == "OP") lb = QUnicodeTables::LineBreak_OP;
-else if (ba == "CL") lb = QUnicodeTables::LineBreak_CL;
-else if (ba == "QU") lb = QUnicodeTables::LineBreak_QU;
-else if (ba == "GL") lb = QUnicodeTables::LineBreak_GL;
-else if (ba == "NS") lb = QUnicodeTables::LineBreak_NS;
-else if (ba == "EX") lb = QUnicodeTables::LineBreak_EX;
-else if (ba == "SY") lb = QUnicodeTables::LineBreak_SY;
-else if (ba == "IS") lb = QUnicodeTables::LineBreak_IS;
-else if (ba == "PR") lb = QUnicodeTables::LineBreak_PR;
-else if (ba == "PO") lb = QUnicodeTables::LineBreak_PO;
-else if (ba == "NU") lb = QUnicodeTables::LineBreak_NU;
-else if (ba == "AL") lb = QUnicodeTables::LineBreak_AL;
-else if (ba == "ID") lb = QUnicodeTables::LineBreak_ID;
-else if (ba == "IN") lb = QUnicodeTables::LineBreak_IN;
-else if (ba == "HY") lb = QUnicodeTables::LineBreak_HY;
-else if (ba == "BA") lb = QUnicodeTables::LineBreak_BA;
-else if (ba == "BB") lb = QUnicodeTables::LineBreak_BB;
-else if (ba == "B2") lb = QUnicodeTables::LineBreak_B2;
-else if (ba == "ZW") lb = QUnicodeTables::LineBreak_ZW;
-else if (ba == "CM") lb = QUnicodeTables::LineBreak_CM;
-else if (ba == "SA") lb = QUnicodeTables::LineBreak_SA;
-else if (ba == "BK") lb = QUnicodeTables::LineBreak_BK;
-else if (ba == "CR") lb = QUnicodeTables::LineBreak_CR;
-else if (ba == "LF") lb = QUnicodeTables::LineBreak_LF;
-else if (ba == "SG") lb = QUnicodeTables::LineBreak_SG;
-else if (ba == "CB") lb = QUnicodeTables::LineBreak_AL;
-else if (ba == "SP") lb = QUnicodeTables::LineBreak_SP;
-else if (ba == "WJ") lb = QUnicodeTables::LineBreak_WJ;
-else if (ba == "H2") lb = QUnicodeTables::LineBreak_H2;
-else if (ba == "H3") lb = QUnicodeTables::LineBreak_H3;
-else if (ba == "JL") lb = QUnicodeTables::LineBreak_JL;
-else if (ba == "JV") lb = QUnicodeTables::LineBreak_JV;
-else if (ba == "JT") lb = QUnicodeTables::LineBreak_JT;
-else {
-qDebug() << "unhandled line break class:" << ba;
-}
 for (int codepoint = from; codepoint <= to; ++codepoint) {
 UnicodeData d = unicodeData.value(codepoint, UnicodeData(codepoint));
 d.p.line_break_class = lb;
 unicodeData.insert(codepoint, d);
 }
 static void readSpecialCasing()
 {
-//     qDebug() << "Reading SpecialCasing.txt";
+qDebug() << "Reading SpecialCasing.txt";
 QFile f("data/SpecialCasing.txt");
 if (!f.exists())
 qFatal("Couldn't find SpecialCasing.txt");
 f.open(QFile::ReadOnly);
 QList<QByteArray> title = l[2].trimmed().split(' ');
 QList<int> titleMap;
 for (int i = 0; i < title.size(); ++i) {
 bool ok;
 titleMap.append(title.at(i).toInt(&ok, 16));
-if (!ok)
-qDebug() << line << title.at(i);
 Q_ASSERT(ok);
 }
 QList<QByteArray> upper = l[3].trimmed().split(' ');
 QList<int> upperMap;
 unicodeData.insert(codepoint, ud);
 }
 }
-int maxCaseFoldDiff = 0;
+static int maxCaseFoldDiff = 0;
 static void readCaseFolding()
 {
 qDebug() << "Reading CaseFolding.txt";
 QFile f("data/CaseFolding.txt");
 continue;
 QList<QByteArray> l = line.split(';');
 bool ok;
-uint codepoint = l[0].trimmed().toInt(&ok, 16);
+int codepoint = l[0].trimmed().toInt(&ok, 16);
 Q_ASSERT(ok);
 l[1] = l[1].trimmed();
 if (l[1] == "F" || l[1] == "T")
 Q_ASSERT(ok);
 }
 UnicodeData ud = unicodeData.value(codepoint, UnicodeData(codepoint));
 if (foldMap.size() == 1) {
+if (qAbs(foldMap.at(0) - codepoint) >= (1<<14))
+qWarning() << "caseFoldDiff exceeded (" << hex << codepoint << "->" << foldMap.at(0) << ")";
 ud.p.caseFoldDiff = foldMap.at(0) - codepoint;
-maxCaseFoldDiff = qMax(maxCaseFoldDiff, ud.p.caseFoldDiff);
+maxCaseFoldDiff = qMax(maxCaseFoldDiff, qAbs(ud.p.caseFoldDiff));
 if (codepoint > 0xffff) {
 // if the condition below doesn't hold anymore we need to modify our case folding code
 //qDebug() << codepoint << QChar::highSurrogate(codepoint) << QChar::highSurrogate(foldMap.at(0));
 Q_ASSERT(QChar::highSurrogate(codepoint) == QChar::highSurrogate(foldMap.at(0)));
 }
 if (foldMap.at(0) != codepoint + ud.p.lowerCaseDiff)
 qDebug() << hex << codepoint;
 } else {
-Q_ASSERT(false); // we currently don't support full case foldings
+qFatal("we currently don't support full case foldings");
 //             qDebug() << "special" << hex << foldMap;
 ud.p.caseFoldSpecial = true;
 ud.p.caseFoldDiff = appendToSpecialCaseMap(foldMap);
 }
 unicodeData.insert(codepoint, ud);
 if (cl.size() == 2) {
 to = cl[1].toInt(&ok, 16);
 Q_ASSERT(ok);
 }
-GraphemeBreak brk = grapheme_break_map.value(l[1].trimmed(), GraphemeBreakOther);
+GraphemeBreak brk = grapheme_break_map.value(l[1].trimmed(), GraphemeBreak_Unassigned);
+if (brk == GraphemeBreak_Unassigned)
+qFatal("unassigned grapheme break class: %s", l[1].constData());
 for (int codepoint = from; codepoint <= to; ++codepoint) {
 UnicodeData ud = unicodeData.value(codepoint, UnicodeData(codepoint));
 ud.p.graphemeBreak = brk;
 unicodeData.insert(codepoint, ud);
 if (cl.size() == 2) {
 to = cl[1].toInt(&ok, 16);
 Q_ASSERT(ok);
 }
-WordBreak brk = word_break_map.value(l[1].trimmed(), WordBreakOther);
+WordBreak brk = word_break_map.value(l[1].trimmed(), WordBreak_Unassigned);
-Q_ASSERT(brk != WordBreakOther);
+if (brk == WordBreak_Unassigned)
+qFatal("unassigned word break class: %s", l[1].constData());
 for (int codepoint = from; codepoint <= to; ++codepoint) {
 UnicodeData ud = unicodeData.value(codepoint, UnicodeData(codepoint));
 ud.p.wordBreak = brk;
 unicodeData.insert(codepoint, ud);
 if (cl.size() == 2) {
 to = cl[1].toInt(&ok, 16);
 Q_ASSERT(ok);
 }
-SentenceBreak brk = sentence_break_map.value(l[1].trimmed(), SentenceBreakOther);
+SentenceBreak brk = sentence_break_map.value(l[1].trimmed(), SentenceBreak_Unassigned);
-Q_ASSERT(brk != SentenceBreakOther);
+if (brk == SentenceBreak_Unassigned)
+qFatal("unassigned sentence break class: %s", l[1].constData());
 for (int codepoint = from; codepoint <= to; ++codepoint) {
 UnicodeData ud = unicodeData.value(codepoint, UnicodeData(codepoint));
 ud.p.sentenceBreak = brk;
 unicodeData.insert(codepoint, ud);
 "Kannada",
 "Khmer",
 "Lao",
 "Malayalam",
 "Myanmar",
+"Nko",
 "Ogham",
 "Oriya",
 "Runic",
 "Sinhala",
 "Syriac",
 const int specialScriptsCount = sizeof(specialScripts) / sizeof(const char *);
 // generate script enum
 QByteArray declaration;
-declaration += "    // See http://www.unicode.org/reports/tr24/tr24-5.html\n\n";
+declaration += "    // See http://www.unicode.org/reports/tr24/tr24-5.html\n";
 declaration += "    enum Script {\n        Common";
 int uniqueScripts = 1; // Common
 // output the ones with special processing first
 for (int i = 1; i < scriptNames.size(); ++i) {
 QByteArray scriptName = scriptNames.at(i);
 // does the script require special processing?
 bool special = false;
-for (int s = 0; !special && s < specialScriptsCount; ++s) {
+for (int s = 0; s < specialScriptsCount; ++s) {
-if (scriptName == specialScripts[s])
+if (scriptName == specialScripts[s]) {
 special = true;
+break;
+}
 }
 if (!special) {
-scriptHash[i] =  0; // alias for 'Common'
+scriptHash[i] = 0; // alias for 'Common'
 continue;
 } else {
 ++uniqueScripts;
 scriptHash[i] = i;
 }
-declaration += ",\n        ";
+if (scriptName != "Inherited") {
-declaration += scriptName;
+declaration += ",\n        ";
-}
+declaration += scriptName;
+}
+}
+declaration += ",\n        Inherited";
 declaration += ",\n        ScriptCount = Inherited";
 // output the ones that are an alias for 'Common'
 for (int i = 1; i < scriptNames.size(); ++i) {
 if (scriptHash.value(i) != 0)
 continue;
-QByteArray scriptName = scriptNames.at(i);
-scriptName += " = Common";
 declaration += ",\n        ";
-declaration += scriptName;
+declaration += scriptNames.at(i);
+declaration += " = Common";
 }
 declaration += "\n    };\n";
 scriptSentinel = ((uniqueScripts + 16) / 32) * 32; // a multiple of 32
 struct PropertyBlock {
 PropertyBlock() { index = -1; }
 int index;
 QList<int> properties;
-bool operator ==(const PropertyBlock &other) { return properties == other.properties; }
+bool operator==(const PropertyBlock &other)
+{ return properties == other.properties; }
 };
 static QByteArray createPropertyInfo()
 {
 qDebug("createPropertyInfo:");
-const int BMP_BLOCKSIZE=32;
+const int BMP_BLOCKSIZE = 32;
 const int BMP_SHIFT = 5;
 const int BMP_END = 0x11000;
 const int SMP_END = 0x110000;
 const int SMP_BLOCKSIZE = 256;
 const int SMP_SHIFT = 8;
 }
 int bmp_block_data = bmp_blocks*BMP_BLOCKSIZE*2;
 int bmp_trie = BMP_END/BMP_BLOCKSIZE*2;
 int bmp_mem = bmp_block_data + bmp_trie;
-qDebug("    %d unique blocks in BMP.",blocks.size());
+qDebug("    %d unique blocks in BMP.", blocks.size());
 qDebug("        block data uses: %d bytes", bmp_block_data);
 qDebug("        trie data uses : %d bytes", bmp_trie);
-int smp_block_data = (blocks.size()- bmp_blocks)*SMP_BLOCKSIZE*2;
+int smp_block_data = (blocks.size() - bmp_blocks)*SMP_BLOCKSIZE*2;
 int smp_trie = (SMP_END-BMP_END)/SMP_BLOCKSIZE*2;
 int smp_mem = smp_block_data + smp_trie;
-qDebug("    %d unique blocks in SMP.",blocks.size()-bmp_blocks);
+qDebug("    %d unique blocks in SMP.", blocks.size()-bmp_blocks);
 qDebug("        block data uses: %d bytes", smp_block_data);
 qDebug("        trie data uses : %d bytes", smp_trie);
 qDebug("\n        properties use : %d bytes", uniqueProperties.size()*20);
 qDebug("    memory usage: %d bytes", bmp_mem+smp_mem + uniqueProperties.size()*20);
 QByteArray out;
 out += "static const unsigned short uc_property_trie[] = {\n";
 // first write the map
-out += "    // 0x" + QByteArray::number(BMP_END, 16);
+out += "    // 0 - 0x" + QByteArray::number(BMP_END, 16);
 for (int i = 0; i < BMP_END/BMP_BLOCKSIZE; ++i) {
 if (!(i % 8)) {
 if (out.endsWith(' '))
 out.chop(1);
 if (!((i*BMP_BLOCKSIZE) % 0x1000))
 "#define GET_PROP_INDEX_UCS2(ucs2) \\\n"
 "(uc_property_trie[uc_property_trie[ucs2>>" + QByteArray::number(BMP_SHIFT) +
 "] + (ucs2 & 0x" + QByteArray::number(BMP_BLOCKSIZE-1, 16)+ ")])\n\n"
-"static const QUnicodeTables::Properties uc_properties [] = {\n";
+"static const QUnicodeTables::Properties uc_properties[] = {\n";
 // keep in sync with the property declaration
 for (int i = 0; i < uniqueProperties.size(); ++i) {
 PropertyFlags p = uniqueProperties.at(i);
 out += "    { ";
 out += QByteArray::number( p.graphemeBreak );
 out += ", ";
 out += QByteArray::number( p.wordBreak );
 out += ", ";
 out += QByteArray::number( p.sentenceBreak );
-out += "},\n";
+out += " },\n";
 }
 out += "};\n\n";
 out += "static inline const QUnicodeTables::Properties *qGetProp(uint ucs4)\n"
 "{\n"
 "{\n"
 "    int index = GET_PROP_INDEX_UCS2(ucs2);\n"
 "    return uc_properties + index;\n"
 "}\n\n";
-out += "#define CURRENT_VERSION "CURRENT_UNICODE_VERSION"\n\n";
+out += "static const ushort specialCaseMap[] = {\n   ";
-out += "static const ushort specialCaseMap [] = {";
 for (int i = 0; i < specialCaseMap.size(); ++i) {
-if (!(i % 16))
-out += "\n   ";
 out += QByteArray(" 0x") + QByteArray::number(specialCaseMap.at(i), 16);
 if (i < specialCaseMap.size() - 1)
 out += ",";
+if (!specialCaseMap.at(i))
+out += "\n   ";
 }
 out += "\n};\n";
 out += "#define SPECIAL_CASE_MAX_LEN " + QByteArray::number(specialCaseMaxLen) + "\n\n";
-qDebug() << "Special case map uses " << specialCaseMap.size()*2 << "bytes";
+qDebug("Special case map uses : %d bytes", specialCaseMap.size()*2);
 return out;
 }
 struct DecompositionBlock {
 DecompositionBlock() { index = -1; }
 int index;
 QList<int> decompositionPositions;
 bool operator ==(const DecompositionBlock &other)
 { return decompositionPositions == other.decompositionPositions; }
 };
 static QByteArray createCompositionInfo()
 {
 qDebug("createCompositionInfo:");
-const int BMP_BLOCKSIZE=16;
+const int BMP_BLOCKSIZE = 16;
 const int BMP_SHIFT = 4;
 const int BMP_END = 0x3400; // start of Han
 const int SMP_END = 0x30000;
 const int SMP_BLOCKSIZE = 256;
 const int SMP_SHIFT = 8;
 int uc = block*BMP_BLOCKSIZE + i;
 UnicodeData d = unicodeData.value(uc, UnicodeData(uc));
 if (!d.decomposition.isEmpty()) {
 int utf16Chars = 0;
 for (int j = 0; j < d.decomposition.size(); ++j)
-utf16Chars += d.decomposition.at(j) > 0x10000 ? 2 : 1;
+utf16Chars += d.decomposition.at(j) >= 0x10000 ? 2 : 1;
 decompositions.append(d.decompositionType + (utf16Chars<<8));
 for (int j = 0; j < d.decomposition.size(); ++j) {
 int code = d.decomposition.at(j);
-if (code > 0x10000) {
+if (code >= 0x10000) {
 // save as surrogate pair
-code -= 0x10000;
+ushort high = QChar::highSurrogate(code);
-ushort high = code/0x400 + 0xd800;
+ushort low = QChar::lowSurrogate(code);
-ushort low = code%0x400 + 0xdc00;
 decompositions.append(high);
 decompositions.append(low);
 } else {
 decompositions.append(code);
 }
 int uc = block*SMP_BLOCKSIZE + i;
 UnicodeData d = unicodeData.value(uc, UnicodeData(uc));
 if (!d.decomposition.isEmpty()) {
 int utf16Chars = 0;
 for (int j = 0; j < d.decomposition.size(); ++j)
-utf16Chars += d.decomposition.at(j) > 0x10000 ? 2 : 1;
+utf16Chars += d.decomposition.at(j) >= 0x10000 ? 2 : 1;
 decompositions.append(d.decompositionType + (utf16Chars<<8));
 for (int j = 0; j < d.decomposition.size(); ++j) {
 int code = d.decomposition.at(j);
-if (code > 0x10000) {
+if (code >= 0x10000) {
 // save as surrogate pair
-code -= 0x10000;
+ushort high = QChar::highSurrogate(code);
-ushort high = code/0x400 + 0xd800;
+ushort low = QChar::lowSurrogate(code);
-ushort low = code%0x400 + 0xdc00;
 decompositions.append(high);
 decompositions.append(low);
 } else {
 decompositions.append(code);
 }
 }
 int bmp_block_data = bmp_blocks*BMP_BLOCKSIZE*2;
 int bmp_trie = BMP_END/BMP_BLOCKSIZE*2;
 int bmp_mem = bmp_block_data + bmp_trie;
-qDebug("    %d unique blocks in BMP.",blocks.size());
+qDebug("    %d unique blocks in BMP.", blocks.size());
 qDebug("        block data uses: %d bytes", bmp_block_data);
 qDebug("        trie data uses : %d bytes", bmp_trie);
 qDebug("        memory usage: %d bytes", bmp_mem);
-int smp_block_data = (blocks.size()- bmp_blocks)*SMP_BLOCKSIZE*2;
+int smp_block_data = (blocks.size() - bmp_blocks)*SMP_BLOCKSIZE*2;
 int smp_trie = (SMP_END-BMP_END)/SMP_BLOCKSIZE*2;
 int smp_mem = smp_block_data + smp_trie;
-qDebug("    %d unique blocks in SMP.",blocks.size()-bmp_blocks);
+qDebug("    %d unique blocks in SMP.", blocks.size()-bmp_blocks);
 qDebug("        block data uses: %d bytes", smp_block_data);
 qDebug("        trie data uses : %d bytes", smp_trie);
 qDebug("\n        decomposition table use : %d bytes", decompositions.size()*2);
 qDebug("    memory usage: %d bytes", bmp_mem+smp_mem + decompositions.size()*2);
 Q_ASSERT(blockMap.size() == BMP_END/BMP_BLOCKSIZE);
 int bmp_block_data = bmp_blocks*BMP_BLOCKSIZE*2;
 int bmp_trie = BMP_END/BMP_BLOCKSIZE*2;
 int bmp_mem = bmp_block_data + bmp_trie;
-qDebug("    %d unique blocks in BMP.",blocks.size());
+qDebug("    %d unique blocks in BMP.", blocks.size());
 qDebug("        block data uses: %d bytes", bmp_block_data);
 qDebug("        trie data uses : %d bytes", bmp_trie);
 qDebug("        ligature data uses : %d bytes", ligatures.size()*2);
 qDebug("        memory usage: %d bytes", bmp_mem + ligatures.size() * 2);
 "#define GET_LIGATURE_INDEX(u2) "
 "(u2 < 0x" + QByteArray::number(BMP_END, 16) + " ? "
 "uc_ligature_trie[uc_ligature_trie[u2>>" + QByteArray::number(BMP_SHIFT) +
 "] + (u2 & 0x" + QByteArray::number(BMP_BLOCKSIZE-1, 16)+ ")] : 0xffff);\n\n"
-"static const unsigned short uc_ligature_map [] = {\n";
+"static const unsigned short uc_ligature_map[] = {\n";
 for (int i = 0; i < ligatures.size(); ++i) {
 if (!(i % 8)) {
 if (out.endsWith(' '))
 out.chop(1);
 return out;
 }
 int main(int, char **)
 {
+initAgeMap();
 initCategoryMap();
 initDirectionMap();
 initDecompositionMap();
 initGraphemeBreak();
 initWordBreak();
 initSentenceBreak();
+initLineBreak();
 readUnicodeData();
 readBidiMirroring();
 readArabicShaping();
 readDerivedAge();
-readCompositionExclusion();
+readDerivedNormalizationProps();
-readLineBreak();
 readSpecialCasing();
 readCaseFolding();
 // readBlocks();
 readScripts();
 readGraphemeBreak();
 readWordBreak();
 readSentenceBreak();
+readLineBreak();
 computeUniqueProperties();
 QByteArray properties = createPropertyInfo();
 QByteArray compositions = createCompositionInfo();
 QByteArray ligatures = createLigatureInfo();
 QByteArray normalizationCorrections = createNormalizationCorrections();
 QByteArray scriptEnumDeclaration = createScriptEnumDeclaration();
 QByteArray scriptTableDeclaration = createScriptTableDeclaration();
-QFile f("../../src/corelib/tools/qunicodetables.cpp");
-f.open(QFile::WriteOnly|QFile::Truncate);
 QByteArray header =
 "/****************************************************************************\n"
 "**\n"
 "** Copyright (C) 2010 Nokia Corporation and/or its subsidiary(-ies).\n"
 "**\n"
 "**\n"
 "**\n"
 "** $QT_END_LICENSE$\n"
 "**\n"
-"****************************************************************************/\n\n"
+"****************************************************************************/\n\n";
-"/* This file is autogenerated from the Unicode 5.0 database. Do not edit */\n\n";
+QByteArray note =
+"/* This file is autogenerated from the Unicode "DATA_VERSION_S" database. Do not edit */\n\n";
 QByteArray warning =
 "//\n"
 "//  W A R N I N G\n"
 "//  -------------\n"
 "// without notice, or even be removed.\n"
 "//\n"
 "// We mean it.\n"
 "//\n\n";
+QFile f("../../src/corelib/tools/qunicodetables.cpp");
+f.open(QFile::WriteOnly|QFile::Truncate);
 f.write(header);
+f.write(note);
 f.write("QT_BEGIN_NAMESPACE\n\n");
 f.write(properties);
 f.write(compositions);
 f.write(ligatures);
 f.write(normalizationCorrections);
 f.write(scriptTableDeclaration);
-f.write("\nQT_END_NAMESPACE\n");
+f.write("QT_END_NAMESPACE\n");
 f.close();
 f.setFileName("../../src/corelib/tools/qunicodetables_p.h");
 f.open(QFile::WriteOnly | QFile::Truncate);
 f.write(header);
+f.write(note);
 f.write(warning);
 f.write("#ifndef QUNICODETABLES_P_H\n"
 "#define QUNICODETABLES_P_H\n\n"
 "#include <QtCore/qchar.h>\n\n"
 "QT_BEGIN_NAMESPACE\n\n");
-f.write("namespace QUnicodeTables {\n");
+f.write("#define UNICODE_DATA_VERSION "DATA_VERSION_STR"\n\n");
+f.write("#define UNICODE_LAST_CODEPOINT "LAST_CODEPOINT_STR"\n\n");
+f.write("namespace QUnicodeTables {\n\n");
 f.write(property_string);
 f.write("\n");
 f.write(scriptEnumDeclaration);
 f.write("\n");
 f.write(lineBreakClass);
 f.write("\n");
-f.write(methods);
-f.write("\n");
 f.write(grapheme_break_string);
 f.write("\n");
 f.write(word_break_string);
 f.write("\n");
 f.write(sentence_break_string);
-f.write("\n}\n\n"
+f.write("\n");
+f.write(methods);
+f.write("} // namespace QUnicodeTables\n\n"
 "QT_END_NAMESPACE\n\n"
-"#endif\n");
+"#endif // QUNICODETABLES_P_H\n");
 f.close();
 qDebug() << "maxMirroredDiff  = " << hex << maxMirroredDiff;
 qDebug() << "maxLowerCaseDiff = " << hex << maxLowerCaseDiff;
 qDebug() << "maxUpperCaseDiff = " << hex << maxUpperCaseDiff;
 qDebug("    length %d used %d times", i, decompositionLength.value(i, 0));
 totalcompositions += i*decompositionLength.value(i, 0);
 sum += decompositionLength.value(i, 0);
 }
 qDebug("    len decomposition map %d, average length %f, num composed chars %d",
-totalcompositions, (float)totalcompositions/(float)sum,  sum);
+totalcompositions, (float)totalcompositions/(float)sum, sum);
 qDebug("highest composed character %x", highestComposedCharacter);
 qDebug("num ligatures = %d highest=%x, maxLength=%d", numLigatures, highestLigature, longestLigature);
 qBubbleSort(ligatures);
 for (int i = 0; i < ligatures.size(); ++i)
 //     }
 //     qDebug("total of %d combining classes used", numClasses);
 #endif
 }

changeset 30	5dc02b23752f
parent 18	2f34d5167611