1 /****************************************************************************

     2 **

     3 ** Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies).

     4 ** All rights reserved.

     5 ** Contact: Nokia Corporation (qt-info@nokia.com)

     6 **

     7 ** This file is part of the utils of the Qt Toolkit.

     8 **

     9 ** $QT_BEGIN_LICENSE:LGPL$

    10 ** No Commercial Usage

    11 ** This file contains pre-release code and may not be distributed.

    12 ** You may use this file in accordance with the terms and conditions

    13 ** contained in the Technology Preview License Agreement accompanying

    14 ** this package.

    15 **

    16 ** GNU Lesser General Public License Usage

    17 ** Alternatively, this file may be used under the terms of the GNU Lesser

    18 ** General Public License version 2.1 as published by the Free Software

    19 ** Foundation and appearing in the file LICENSE.LGPL included in the

    20 ** packaging of this file.  Please review the following information to

    21 ** ensure the GNU Lesser General Public License version 2.1 requirements

    22 ** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.

    23 **

    24 ** In addition, as a special exception, Nokia gives you certain additional

    25 ** rights.  These rights are described in the Nokia Qt LGPL Exception

    26 ** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.

    27 **

    28 ** If you have questions regarding the use of this file, please contact

    29 ** Nokia at qt-info@nokia.com.

    30 **

    31 **

    32 **

    33 **

    34 **

    35 **

    36 **

    37 **

    38 ** $QT_END_LICENSE$

    39 **

    40 ****************************************************************************/

    41 #include <qlist.h>

    42 #include <qhash.h>

    43 #include <qfile.h>

    44 #include <qstring.h>

    45 #include <qchar.h>

    46 #include <private/qunicodetables_p.h>

    47 #include <qvector.h>

    48 #include <qdebug.h>

    49 

    50 

    51 static struct AgeMap {

    52     const char *age;

    53     const QChar::UnicodeVersion version;

    54 } ageMap [] = {

    55     { "1.1", QChar::Unicode_1_1 },

    56     { "2.0", QChar::Unicode_2_0 },

    57     { "2.1", QChar::Unicode_2_1_2 },

    58     { "3.0", QChar::Unicode_3_0 },

    59     { "3.1", QChar::Unicode_3_1 },

    60     { "3.2", QChar::Unicode_3_2 },

    61     { "4.0", QChar::Unicode_4_0 },

    62     { "4.1", QChar::Unicode_4_1 },

    63     { "5.0", QChar::Unicode_5_0 },

    64     { 0, QChar::Unicode_Unassigned }

    65 };

    66 #define CURRENT_UNICODE_VERSION "QChar::Unicode_5_0"

    67 

    68 static const char *grapheme_break_string =

    69     "    enum GraphemeBreak {\n"

    70     "        GraphemeBreakOther, \n"

    71     "        GraphemeBreakCR,\n"

    72     "        GraphemeBreakLF,\n"

    73     "        GraphemeBreakControl,\n"

    74     "        GraphemeBreakExtend,\n"

    75     "        GraphemeBreakL,\n"

    76     "        GraphemeBreakV,\n"

    77     "        GraphemeBreakT,\n"

    78     "        GraphemeBreakLV,\n"

    79     "        GraphemeBreakLVT\n"

    80     "    };\n\n";

    81 

    82 enum GraphemeBreak {

    83     GraphemeBreakOther,

    84     GraphemeBreakCR,

    85     GraphemeBreakLF,

    86     GraphemeBreakControl,

    87     GraphemeBreakExtend,

    88     GraphemeBreakL,

    89     GraphemeBreakV,

    90     GraphemeBreakT,

    91     GraphemeBreakLV,

    92     GraphemeBreakLVT

    93 };

    94 

    95 QHash<QByteArray, GraphemeBreak> grapheme_break_map;

    96 

    97 static void initGraphemeBreak()

    98 {

    99     struct GraphemeBreakList {

   100         GraphemeBreak brk;

   101         const char *name;

   102     } breaks[] = {

   103         { GraphemeBreakOther, "Other" },

   104         { GraphemeBreakCR, "CR" },

   105         { GraphemeBreakLF, "LF" },

   106         { GraphemeBreakControl, "Control" },

   107         { GraphemeBreakExtend, "Extend" },

   108         { GraphemeBreakL, "L" },

   109         { GraphemeBreakV, "V" },

   110         { GraphemeBreakT, "T" },

   111         { GraphemeBreakLV, "LV" },

   112         { GraphemeBreakLVT, "LVT" },

   113         { GraphemeBreakOther, 0 }

   114     };

   115     GraphemeBreakList *d = breaks;

   116     while (d->name) {

   117         grapheme_break_map.insert(d->name, d->brk);

   118         ++d;

   119     }

   120 }

   121 

   122 const char *word_break_string =

   123     "    enum WordBreak {\n"

   124     "        WordBreakOther,\n"

   125     "        WordBreakFormat,\n"

   126     "        WordBreakKatakana,\n"

   127     "        WordBreakALetter,\n"

   128     "        WordBreakMidLetter,\n"

   129     "        WordBreakMidNum,\n"

   130     "        WordBreakNumeric,\n"

   131     "        WordBreakExtendNumLet\n"

   132     "    };\n\n";

   133 

   134 enum WordBreak {

   135     WordBreakOther,

   136     WordBreakFormat,

   137     WordBreakKatakana,

   138     WordBreakALetter,

   139     WordBreakMidLetter,

   140     WordBreakMidNum,

   141     WordBreakNumeric,

   142     WordBreakExtendNumLet

   143 };

   144 

   145 

   146 QHash<QByteArray, WordBreak> word_break_map;

   147 

   148 static void initWordBreak()

   149 {

   150     struct WordBreakList {

   151         WordBreak brk;

   152         const char *name;

   153     } breaks[] = {

   154         { WordBreakFormat, "Format" },

   155         { WordBreakFormat, "Extend" }, // these are copied in from GraphemeBreakProperty.txt

   156         { WordBreakKatakana, "Katakana" },

   157         { WordBreakALetter, "ALetter" },

   158         { WordBreakMidLetter, "MidLetter" },

   159         { WordBreakMidNum, "MidNum" },

   160         { WordBreakNumeric, "Numeric" },

   161         { WordBreakExtendNumLet, "ExtendNumLet" },

   162         { WordBreakFormat,  0 }

   163     };

   164     WordBreakList *d = breaks;

   165     while (d->name) {

   166         word_break_map.insert(d->name, d->brk);

   167         ++d;

   168     }

   169 }

   170 

   171 

   172 static const char *sentence_break_string =

   173     "    enum SentenceBreak {\n"

   174     "        SentenceBreakOther,\n"

   175     "        SentenceBreakSep,\n"

   176     "        SentenceBreakFormat,\n"

   177     "        SentenceBreakSp,\n"

   178     "        SentenceBreakLower,\n"

   179     "        SentenceBreakUpper,\n"

   180     "        SentenceBreakOLetter,\n"

   181     "        SentenceBreakNumeric,\n"

   182     "        SentenceBreakATerm,\n"

   183     "        SentenceBreakSTerm,\n"

   184     "        SentenceBreakClose\n"

   185     "    };\n\n";

   186 

   187 enum SentenceBreak {

   188     SentenceBreakOther,

   189     SentenceBreakSep,

   190     SentenceBreakFormat,

   191     SentenceBreakSp,

   192     SentenceBreakLower,

   193     SentenceBreakUpper,

   194     SentenceBreakOLetter,

   195     SentenceBreakNumeric,

   196     SentenceBreakATerm,

   197     SentenceBreakSTerm,

   198     SentenceBreakClose

   199 };

   200 

   201 

   202 QHash<QByteArray, SentenceBreak> sentence_break_map;

   203 

   204 static void initSentenceBreak()

   205 {

   206     struct SentenceBreakList {

   207         SentenceBreak brk;

   208         const char *name;

   209     } breaks[] = {

   210         { SentenceBreakOther, "Other" },

   211         { SentenceBreakSep, "Sep" },

   212         { SentenceBreakFormat, "Format" },

   213         { SentenceBreakSp, "Sp" },

   214         { SentenceBreakLower, "Lower" },

   215         { SentenceBreakUpper, "Upper" },

   216         { SentenceBreakOLetter, "OLetter" },

   217         { SentenceBreakNumeric, "Numeric" },

   218         { SentenceBreakATerm, "ATerm" },

   219         { SentenceBreakSTerm, "STerm" },

   220         { SentenceBreakClose, "Close" },

   221         { SentenceBreakOther,  0 }

   222     };

   223     SentenceBreakList *d = breaks;

   224     while (d->name) {

   225         sentence_break_map.insert(d->name, d->brk);

   226         ++d;

   227     }

   228 }

   229 

   230 

   231 // Keep this one in sync with the code in createPropertyInfo

   232 const char *property_string =

   233     "    struct Properties {\n"

   234     "        ushort category : 8;\n"

   235     "        ushort line_break_class : 8;\n"

   236     "        ushort direction : 8;\n"

   237     "        ushort combiningClass :8;\n"

   238     "        ushort joining : 2;\n"

   239     "        signed short digitValue : 6; /* 5 needed */\n"

   240     "        ushort unicodeVersion : 4;\n"

   241     "        ushort lowerCaseSpecial : 1;\n"

   242     "        ushort upperCaseSpecial : 1;\n"

   243     "        ushort titleCaseSpecial : 1;\n"

   244     "        ushort caseFoldSpecial : 1; /* currently unused */\n"

   245     "        signed short mirrorDiff : 16;\n"

   246     "        signed short lowerCaseDiff : 16;\n"

   247     "        signed short upperCaseDiff : 16;\n"

   248     "        signed short titleCaseDiff : 16;\n"

   249     "        signed short caseFoldDiff : 16;\n"

   250     "        ushort graphemeBreak : 8;\n"

   251     "        ushort wordBreak : 8;\n"

   252     "        ushort sentenceBreak : 8;\n"

   253     "    };\n"

   254     "    Q_CORE_EXPORT const Properties * QT_FASTCALL properties(uint ucs4);\n"

   255     "    Q_CORE_EXPORT const Properties * QT_FASTCALL properties(ushort ucs2);\n";

   256 

   257 const char *lineBreakClass =

   258     "    // see http://www.unicode.org/reports/tr14/tr14-19.html\n"

   259     "    // we don't use the XX, AI and CB properties and map them to AL instead.\n"

   260     "    // as we don't support any EBDIC based OS'es, NL is ignored and mapped to AL as well.\n"

   261     "    enum LineBreakClass {\n"

   262     "        LineBreak_OP, LineBreak_CL, LineBreak_QU, LineBreak_GL, LineBreak_NS,\n"

   263     "        LineBreak_EX, LineBreak_SY, LineBreak_IS, LineBreak_PR, LineBreak_PO,\n"

   264     "        LineBreak_NU, LineBreak_AL, LineBreak_ID, LineBreak_IN, LineBreak_HY,\n"

   265     "        LineBreak_BA, LineBreak_BB, LineBreak_B2, LineBreak_ZW, LineBreak_CM,\n"

   266     "        LineBreak_WJ, LineBreak_H2, LineBreak_H3, LineBreak_JL, LineBreak_JV,\n"

   267     "        LineBreak_JT, LineBreak_SA, LineBreak_SG,\n"

   268     "        LineBreak_SP, LineBreak_CR, LineBreak_LF, LineBreak_BK\n"

   269     "    };\n\n";

   270 

   271 const char *methods =

   272     "    Q_CORE_EXPORT QUnicodeTables::LineBreakClass QT_FASTCALL lineBreakClass(uint ucs4);\n"

   273     "    inline int lineBreakClass(const QChar &ch) {\n"

   274     "        return QUnicodeTables::lineBreakClass(ch.unicode());\n"

   275     "    }\n"

   276     "\n"

   277     "    Q_CORE_EXPORT int QT_FASTCALL script(uint ucs4);\n"

   278     "    Q_CORE_EXPORT_INLINE int QT_FASTCALL script(const QChar &ch) {\n"

   279     "        return script(ch.unicode());\n"

   280     "    }\n\n";

   281 

   282 

   283 struct PropertyFlags {

   284     bool operator ==(const PropertyFlags &o) {

   285         return (combiningClass == o.combiningClass

   286                 && category == o.category

   287                 && direction == o.direction

   288                 && joining == o.joining

   289                 && age == o.age

   290                 && digitValue == o.digitValue

   291                 && line_break_class == o.line_break_class

   292                 && mirrorDiff == o.mirrorDiff

   293                 && lowerCaseDiff == o.lowerCaseDiff

   294                 && upperCaseDiff == o.upperCaseDiff

   295                 && titleCaseDiff == o.titleCaseDiff

   296                 && caseFoldDiff == o.caseFoldDiff

   297                 && lowerCaseSpecial == o.lowerCaseSpecial

   298                 && upperCaseSpecial == o.upperCaseSpecial

   299                 && titleCaseSpecial == o.titleCaseSpecial

   300                 && caseFoldSpecial == o.caseFoldSpecial

   301                 && graphemeBreak == o.graphemeBreak

   302                 && wordBreak == o.wordBreak

   303                 && sentenceBreak == o.sentenceBreak

   304             );

   305     }

   306     // from UnicodeData.txt

   307     uchar combiningClass : 8;

   308     QChar::Category category : 5;

   309     QChar::Direction direction : 5;

   310     // from ArabicShaping.txt

   311     QChar::Joining joining : 2;

   312     // from DerivedAge.txt

   313     QChar::UnicodeVersion age : 4;

   314     int digitValue;

   315     uint line_break_class : 5;

   316 

   317     int mirrorDiff : 16;

   318 

   319     int lowerCaseDiff;

   320     int upperCaseDiff;

   321     int titleCaseDiff;

   322     int caseFoldDiff;

   323     bool lowerCaseSpecial;

   324     bool upperCaseSpecial;

   325     bool titleCaseSpecial;

   326     bool caseFoldSpecial;

   327     GraphemeBreak graphemeBreak;

   328     WordBreak wordBreak;

   329     SentenceBreak sentenceBreak;

   330 };

   331 

   332 QList<int> specialCaseMap;

   333 int specialCaseMaxLen = 0;

   334 

   335 static int appendToSpecialCaseMap(const QList<int> &map)

   336 {

   337     QList<int> utf16map;

   338     for (int i = 0; i < map.size(); ++i) {

   339         int val = map.at(i);

   340         if (val > 0xffff) {

   341             utf16map << QChar::highSurrogate(val);

   342             utf16map << QChar::lowSurrogate(val);

   343         } else {

   344             utf16map << val;

   345         }

   346     }

   347     specialCaseMaxLen = qMax(specialCaseMaxLen, utf16map.size());

   348     utf16map << 0;

   349 

   350     for (int i = 0; i < specialCaseMap.size() - utf16map.size() - 1; ++i) {

   351         int j;

   352         for (j = 0; j < utf16map.size(); ++j) {

   353             if (specialCaseMap.at(i+j) != utf16map.at(j))

   354                 break;

   355         }

   356         if (j == utf16map.size())

   357             return i;

   358     }

   359 

   360     int pos = specialCaseMap.size();

   361     specialCaseMap << utf16map;

   362     return pos;

   363 }

   364 

   365 struct UnicodeData {

   366     UnicodeData(int codepoint = 0) {

   367         p.category = QChar::NoCategory;

   368         p.combiningClass = 0;

   369 

   370         p.direction = QChar::DirL;

   371         // DirR for:  U+0590..U+05FF, U+07C0..U+08FF, U+FB1D..U+FB4F, U+10800..U+10FFF

   372         if ((codepoint >= 0x590 && codepoint <= 0x5ff)

   373             || (codepoint >= 0x7c0 && codepoint <= 0x8ff)

   374             || (codepoint >= 0xfb1d && codepoint <= 0xfb4f)

   375             || (codepoint >= 0x10800 && codepoint <= 0x10fff))

   376             p.direction = QChar::DirR;

   377         // DirAL for: U+0600..U+07BF, U+FB50..U+FDCF, U+FDF0..U+FDFF, U+FE70..U+FEFE

   378         if ((codepoint >= 0x600 && codepoint <= 0x7bf)

   379             || (codepoint >= 0xfb50 && codepoint <= 0xfdcf)

   380             || (codepoint >= 0xfdf0 && codepoint <= 0xfdff)

   381             || (codepoint >= 0xfe70 && codepoint <= 0xfefe))

   382             p.direction = QChar::DirAL;

   383 

   384         mirroredChar = 0;

   385         decompositionType = QChar::NoDecomposition;

   386         p.joining = QChar::OtherJoining;

   387         p.age = QChar::Unicode_Unassigned;

   388         p.mirrorDiff = 0;

   389         p.digitValue = -1;

   390         p.line_break_class = QUnicodeTables::LineBreak_AL;

   391         p.lowerCaseDiff = 0;

   392         p.upperCaseDiff = 0;

   393         p.titleCaseDiff = 0;

   394         p.caseFoldDiff = 0;

   395         p.lowerCaseSpecial = 0;

   396         p.upperCaseSpecial = 0;

   397         p.titleCaseSpecial = 0;

   398         p.caseFoldSpecial = 0;

   399         p.graphemeBreak = GraphemeBreakOther;

   400         p.wordBreak = WordBreakOther;

   401         p.sentenceBreak = SentenceBreakOther;

   402         propertyIndex = -1;

   403         excludedComposition = false;

   404     }

   405     PropertyFlags p;

   406 

   407     // from UnicodeData.txt

   408     QChar::Decomposition decompositionType;

   409     QList<int> decomposition;

   410 

   411     QList<int> specialFolding;

   412 

   413     // from BidiMirroring.txt

   414     int mirroredChar;

   415 

   416     // CompositionExclusions.txt

   417     bool excludedComposition;

   418 

   419     // computed position of unicode property set

   420     int propertyIndex;

   421 };

   422 

   423 enum UniDataFields {

   424     UD_Value,

   425     UD_Name,

   426     UD_Category,

   427     UD_CombiningClass,

   428     UD_BidiCategory,

   429     UD_Decomposition,

   430     UD_DecimalDigitValue,

   431     UD_DigitValue,

   432     UD_NumericValue,

   433     UD_Mirrored,

   434     UD_OldName,

   435     UD_Comment,

   436     UD_UpperCase,

   437     UD_LowerCase,

   438     UD_TitleCase

   439 };

   440 

   441 QHash<QByteArray, QChar::Category> categoryMap;

   442 

   443 static void initCategoryMap()

   444 {

   445     struct Cat {

   446         QChar::Category cat;

   447         const char *name;

   448     } categories [] = {

   449         { QChar::Mark_NonSpacing,          "Mn" },

   450         { QChar::Mark_SpacingCombining,    "Mc" },

   451         { QChar::Mark_Enclosing,           "Me" },

   452 

   453         { QChar::Number_DecimalDigit,      "Nd" },

   454         { QChar::Number_Letter,            "Nl" },

   455         { QChar::Number_Other,             "No" },

   456 

   457         { QChar::Separator_Space,          "Zs" },

   458         { QChar::Separator_Line,           "Zl" },

   459         { QChar::Separator_Paragraph,      "Zp" },

   460 

   461         { QChar::Other_Control,            "Cc" },

   462         { QChar::Other_Format,             "Cf" },

   463         { QChar::Other_Surrogate,          "Cs" },

   464         { QChar::Other_PrivateUse,         "Co" },

   465         { QChar::Other_NotAssigned,        "Cn" },

   466 

   467         { QChar::Letter_Uppercase,         "Lu" },

   468         { QChar::Letter_Lowercase,         "Ll" },

   469         { QChar::Letter_Titlecase,         "Lt" },

   470         { QChar::Letter_Modifier,          "Lm" },

   471         { QChar::Letter_Other,             "Lo" },

   472 

   473         { QChar::Punctuation_Connector,    "Pc" },

   474         { QChar::Punctuation_Dash,         "Pd" },

   475         { QChar::Punctuation_Open,         "Ps" },

   476         { QChar::Punctuation_Close,        "Pe" },

   477         { QChar::Punctuation_InitialQuote, "Pi" },

   478         { QChar::Punctuation_FinalQuote,   "Pf" },

   479         { QChar::Punctuation_Other,        "Po" },

   480 

   481         { QChar::Symbol_Math,              "Sm" },

   482         { QChar::Symbol_Currency,          "Sc" },

   483         { QChar::Symbol_Modifier,          "Sk" },

   484         { QChar::Symbol_Other,             "So" },

   485         { QChar::NoCategory, 0 }

   486     };

   487     Cat *c = categories;

   488     while (c->cat != QChar::NoCategory) {

   489         categoryMap.insert(c->name, c->cat);

   490         ++c;

   491     }

   492 }

   493 

   494 QHash<QByteArray, QChar::Direction> directionMap;

   495 

   496 static void initDirectionMap()

   497 {

   498     struct Dir {

   499         QChar::Direction dir;

   500         const char *name;

   501     } directions[] = {

   502         { QChar::DirL, "L" },

   503         { QChar::DirR, "R" },

   504         { QChar::DirEN, "EN" },

   505         { QChar::DirES, "ES" },

   506         { QChar::DirET, "ET" },

   507         { QChar::DirAN, "AN" },

   508         { QChar::DirCS, "CS" },

   509         { QChar::DirB, "B" },

   510         { QChar::DirS, "S" },

   511         { QChar::DirWS, "WS" },

   512         { QChar::DirON, "ON" },

   513         { QChar::DirLRE, "LRE" },

   514         { QChar::DirLRO, "LRO" },

   515         { QChar::DirAL, "AL" },

   516         { QChar::DirRLE, "RLE" },

   517         { QChar::DirRLO, "RLO" },

   518         { QChar::DirPDF, "PDF" },

   519         { QChar::DirNSM, "NSM" },

   520         { QChar::DirBN, "BN" },

   521         { QChar::DirL, 0 }

   522     };

   523     Dir *d = directions;

   524     while (d->name) {

   525         directionMap.insert(d->name, d->dir);

   526         ++d;

   527     }

   528 }

   529 

   530 

   531 QHash<QByteArray, QChar::Decomposition> decompositionMap;

   532 

   533 static void initDecompositionMap()

   534 {

   535     struct Dec {

   536         QChar::Decomposition dec;

   537         const char *name;

   538     } decompositions[] = {

   539         { QChar::Canonical, "<canonical>" },

   540         { QChar::Font, "<font>" },

   541         { QChar::NoBreak, "<noBreak>" },

   542         { QChar::Initial, "<initial>" },

   543         { QChar::Medial, "<medial>" },

   544         { QChar::Final, "<final>" },

   545         { QChar::Isolated, "<isolated>" },

   546         { QChar::Circle, "<circle>" },

   547         { QChar::Super, "<super>" },

   548         { QChar::Sub, "<sub>" },

   549         { QChar::Vertical, "<vertical>" },

   550         { QChar::Wide, "<wide>" },

   551         { QChar::Narrow, "<narrow>" },

   552         { QChar::Small, "<small>" },

   553         { QChar::Square, "<square>" },

   554         { QChar::Compat, "<compat>" },

   555         { QChar::Fraction, "<fraction>" },

   556         { QChar::NoDecomposition,  0 }

   557     };

   558     Dec *d = decompositions;

   559     while (d->name) {

   560         decompositionMap.insert(d->name, d->dec);

   561         ++d;

   562     }

   563 }

   564 

   565 

   566 QHash<int, UnicodeData> unicodeData;

   567 QList<PropertyFlags> uniqueProperties;

   568 

   569 

   570 QHash<int, int> decompositionLength;

   571 int highestComposedCharacter = 0;

   572 int numLigatures = 0;

   573 int highestLigature = 0;

   574 

   575 struct Ligature {ushort u1; ushort u2; ushort ligature;};

   576 // we need them sorted after the first component for fast lookup

   577 bool operator < (const Ligature &l1, const Ligature &l2) {

   578     return l1.u1 < l2.u1;

   579 }

   580 

   581 QHash<ushort, QList<Ligature> > ligatureHashes;

   582 

   583 QHash<int, int> combiningClassUsage;

   584 

   585 int maxLowerCaseDiff = 0;

   586 int maxUpperCaseDiff = 0;

   587 int maxTitleCaseDiff = 0;

   588 

   589 static void readUnicodeData()

   590 {

   591     QFile f("data/UnicodeData.txt");

   592     if (!f.exists())

   593         qFatal("Couldn't find UnicodeData.txt");

   594 

   595     f.open(QFile::ReadOnly);

   596 

   597     while (!f.atEnd()) {

   598         QByteArray line;

   599         line.resize(1024);

   600         int len = f.readLine(line.data(), 1024);

   601         line.truncate(len-1);

   602 

   603         int comment = line.indexOf('#');

   604         if (comment >= 0)

   605             line = line.left(comment);

   606         if (line.isEmpty())

   607             continue;

   608 

   609         QList<QByteArray> properties = line.split(';');

   610         bool ok;

   611         int codepoint = properties[UD_Value].toInt(&ok, 16);

   612         int lastCodepoint = codepoint;

   613 

   614         QByteArray name = properties[UD_Name];

   615         if (name.startsWith('<') && name.contains("First")) {

   616             QByteArray nextLine;

   617             nextLine.resize(1024);

   618             f.readLine(nextLine.data(), 1024);

   619             QList<QByteArray> properties = nextLine.split(';');

   620             lastCodepoint = properties[UD_Value].toInt(&ok, 16);

   621         }

   622 

   623         UnicodeData data(codepoint);

   624         data.p.category = categoryMap.value(properties[UD_Category], QChar::NoCategory);

   625         data.p.combiningClass = properties[UD_CombiningClass].toInt();

   626 

   627         if (!combiningClassUsage.contains(data.p.combiningClass))

   628             combiningClassUsage[data.p.combiningClass] = 1;

   629         else

   630             ++combiningClassUsage[data.p.combiningClass];

   631 

   632         data.p.direction = directionMap.value(properties[UD_BidiCategory], data.p.direction);

   633 

   634         if (!properties[UD_UpperCase].isEmpty()) {

   635             int upperCase = properties[UD_UpperCase].toInt(&ok, 16);

   636             Q_ASSERT(ok);

   637             data.p.upperCaseDiff = upperCase - codepoint;

   638             maxUpperCaseDiff = qMax(maxUpperCaseDiff, qAbs(data.p.upperCaseDiff));

   639             if (codepoint > 0xffff) {

   640                 // if the condition below doesn't hold anymore we need to modify our case folding code

   641                 //qDebug() << codepoint << QChar::highSurrogate(codepoint) << QChar::highSurrogate(foldMap.at(0));

   642                 Q_ASSERT(QChar::highSurrogate(codepoint) == QChar::highSurrogate(upperCase));

   643             }

   644         }

   645         if (!properties[UD_LowerCase].isEmpty()) {

   646             int lowerCase = properties[UD_LowerCase].toInt(&ok, 16);

   647             Q_ASSERT (ok);

   648             data.p.lowerCaseDiff = lowerCase - codepoint;

   649             maxLowerCaseDiff = qMax(maxLowerCaseDiff, qAbs(data.p.lowerCaseDiff));

   650             if (codepoint > 0xffff) {

   651                 // if the condition below doesn't hold anymore we need to modify our case folding code

   652                 //qDebug() << codepoint << QChar::highSurrogate(codepoint) << QChar::highSurrogate(foldMap.at(0));

   653                 Q_ASSERT(QChar::highSurrogate(codepoint) == QChar::highSurrogate(lowerCase));

   654             }

   655         }

   656         // we want toTitleCase to map to ToUpper in case we don't have any titlecase.

   657         if (properties[UD_TitleCase].isEmpty())

   658             properties[UD_TitleCase] = properties[UD_UpperCase];

   659         if (!properties[UD_TitleCase].isEmpty()) {

   660             int titleCase = properties[UD_TitleCase].toInt(&ok, 16);

   661             Q_ASSERT (ok);

   662             data.p.titleCaseDiff = titleCase - codepoint;

   663             maxTitleCaseDiff = qMax(maxTitleCaseDiff, qAbs(data.p.titleCaseDiff));

   664             if (codepoint > 0xffff) {

   665                 // if the condition below doesn't hold anymore we need to modify our case folding code

   666                 //qDebug() << codepoint << QChar::highSurrogate(codepoint) << QChar::highSurrogate(foldMap.at(0));

   667                 Q_ASSERT(QChar::highSurrogate(codepoint) == QChar::highSurrogate(titleCase));

   668             }

   669         }

   670 

   671         if (!properties[UD_DigitValue].isEmpty())

   672             data.p.digitValue = properties[UD_DigitValue].toInt();

   673 

   674         // decompositition

   675         QByteArray decomposition = properties[UD_Decomposition];

   676         if (!decomposition.isEmpty()) {

   677             highestComposedCharacter = qMax(highestComposedCharacter, codepoint);

   678             QList<QByteArray> d = decomposition.split(' ');

   679             if (d[0].contains('<')) {

   680                 data.decompositionType = decompositionMap.value(d[0], QChar::Canonical);

   681                 d.takeFirst();

   682             } else {

   683                 data.decompositionType = QChar::Canonical;

   684             }

   685             for (int i = 0; i < d.size(); ++i)

   686                 data.decomposition.append(d[i].toInt(&ok, 16));

   687             if (!decompositionLength.contains(data.decomposition.size()))

   688                 decompositionLength[data.decomposition.size()] = 1;

   689             else

   690                 ++decompositionLength[data.decomposition.size()];

   691         }

   692 

   693         for (int i = codepoint; i <= lastCodepoint; ++i)

   694             unicodeData.insert(i, data);

   695     }

   696 

   697 }

   698 

   699 static int maxMirroredDiff = 0;

   700 

   701 static void readBidiMirroring()

   702 {

   703     QFile f("data/BidiMirroring.txt");

   704     if (!f.exists())

   705         qFatal("Couldn't find BidiMirroring.txt");

   706 

   707     f.open(QFile::ReadOnly);

   708 

   709     while (!f.atEnd()) {

   710         QByteArray line;

   711         line.resize(1024);

   712         int len = f.readLine(line.data(), 1024);

   713         line.resize(len-1);

   714 

   715         int comment = line.indexOf('#');

   716         if (comment >= 0)

   717             line = line.left(comment);

   718 

   719         if (line.isEmpty())

   720             continue;

   721         line = line.replace(" ", "");

   722 

   723         QList<QByteArray> pair = line.split(';');

   724         Q_ASSERT(pair.size() == 2);

   725 

   726         bool ok;

   727         int codepoint = pair[0].toInt(&ok, 16);

   728         int mirror = pair[1].toInt(&ok, 16);

   729 

   730         UnicodeData d = unicodeData.value(codepoint, UnicodeData(codepoint));

   731         d.mirroredChar = mirror;

   732         if (qAbs(codepoint-d.mirroredChar) > maxMirroredDiff)

   733             maxMirroredDiff = qAbs(codepoint - d.mirroredChar);

   734 

   735         d.p.mirrorDiff = d.mirroredChar - codepoint;

   736         unicodeData.insert(codepoint, d);

   737     }

   738 }

   739 

   740 static void readArabicShaping()

   741 {

   742     QFile f("data/ArabicShaping.txt");

   743     if (!f.exists())

   744         qFatal("Couldn't find ArabicShaping.txt");

   745 

   746     f.open(QFile::ReadOnly);

   747 

   748     while (!f.atEnd()) {

   749         QByteArray line;

   750         line.resize(1024);

   751         int len = f.readLine(line.data(), 1024);

   752         line.resize(len-1);

   753 

   754         int comment = line.indexOf('#');

   755         if (comment >= 0)

   756             line = line.left(comment);

   757         line = line.trimmed();

   758 

   759         if (line.isEmpty())

   760             continue;

   761 

   762         QList<QByteArray> shaping = line.split(';');

   763         Q_ASSERT(shaping.size() == 4);

   764 

   765         bool ok;

   766         int codepoint = shaping[0].toInt(&ok, 16);

   767         QChar::Joining j = QChar::OtherJoining;

   768         QByteArray shape = shaping[2].trimmed();

   769         if (shape == "R")

   770             j = QChar::Right;

   771         else if (shape == "D")

   772             j = QChar::Dual;

   773         else if (shape == "C")

   774             j = QChar::Center;

   775 

   776         UnicodeData d = unicodeData.value(codepoint, UnicodeData(codepoint));

   777         d.p.joining = j;

   778         unicodeData.insert(codepoint, d);

   779     }

   780 }

   781 

   782 static void readDerivedAge()

   783 {

   784     QFile f("data/DerivedAge.txt");

   785     if (!f.exists())

   786         qFatal("Couldn't find DerivedAge.txt");

   787 

   788     f.open(QFile::ReadOnly);

   789 

   790     while (!f.atEnd()) {

   791         QByteArray line;

   792         line.resize(1024);

   793         int len = f.readLine(line.data(), 1024);

   794         line.resize(len-1);

   795 

   796         int comment = line.indexOf('#');

   797         if (comment >= 0)

   798             line = line.left(comment);

   799         line.replace(" ", "");

   800 

   801         if (line.isEmpty())

   802             continue;

   803 

   804         QList<QByteArray> l = line.split(';');

   805         Q_ASSERT(l.size() == 2);

   806 

   807         QByteArray codes = l[0];

   808         codes.replace("..", ".");

   809         QList<QByteArray> cl = codes.split('.');

   810 

   811         bool ok;

   812         int from = cl[0].toInt(&ok, 16);

   813         int to = from;

   814         if (cl.size() == 2)

   815             to = cl[1].toInt(&ok, 16);

   816 

   817         QChar::UnicodeVersion age = QChar::Unicode_Unassigned;

   818         QByteArray ba = l[1];

   819         AgeMap *map = ageMap;

   820         while (map->age) {

   821             if (ba == map->age) {

   822                 age = map->version;

   823                 break;

   824             }

   825             ++map;

   826         }

   827         //qDebug() << hex << from << ".." << to << ba << age;

   828         Q_ASSERT(age != QChar::Unicode_Unassigned);

   829 

   830         for (int codepoint = from; codepoint <= to; ++codepoint) {

   831             UnicodeData d = unicodeData.value(codepoint, UnicodeData(codepoint));

   832             d.p.age = age;

   833             unicodeData.insert(codepoint, d);

   834         }

   835     }

   836 }

   837 

   838 

   839 static void readCompositionExclusion()

   840 {

   841     QFile f("data/CompositionExclusions.txt");

   842     if (!f.exists())

   843         qFatal("Couldn't find CompositionExclusions.txt");

   844 

   845     f.open(QFile::ReadOnly);

   846 

   847     while (!f.atEnd()) {

   848         QByteArray line;

   849         line.resize(1024);

   850         int len = f.readLine(line.data(), 1024);

   851         line.resize(len-1);

   852 

   853         int comment = line.indexOf('#');

   854         if (comment >= 0)

   855             line = line.left(comment);

   856         line.replace(" ", "");

   857 

   858         if (line.isEmpty())

   859             continue;

   860 

   861         Q_ASSERT(!line.contains(".."));

   862 

   863         bool ok;

   864         int codepoint = line.toInt(&ok, 16);

   865 

   866         UnicodeData d = unicodeData.value(codepoint, UnicodeData(codepoint));

   867         d.excludedComposition = true;

   868         unicodeData.insert(codepoint, d);

   869     }

   870 

   871     for (int i = 0; i < 0x110000; ++i) {

   872         UnicodeData data = unicodeData.value(i, UnicodeData(i));

   873         if (!data.excludedComposition

   874             && data.decompositionType == QChar::Canonical

   875             && data.decomposition.size() > 1) {

   876             Q_ASSERT(data.decomposition.size() == 2);

   877 

   878             uint part1 = data.decomposition.at(0);

   879             uint part2 = data.decomposition.at(1);

   880             UnicodeData first = unicodeData.value(part1, UnicodeData(part1));

   881             if (first.p.combiningClass != 0)

   882                 continue;

   883 

   884             ++numLigatures;

   885             highestLigature = qMax(highestLigature, (int)part1);

   886             Ligature l = {(ushort)part1, (ushort)part2, i};

   887             ligatureHashes[part2].append(l);

   888         }

   889     }

   890 }

   891 

   892 struct NormalizationCorrection {

   893     uint codepoint;

   894     uint mapped;

   895     uint version;

   896 };

   897 

   898 static QByteArray createNormalizationCorrections()

   899 {

   900     QFile f("data/NormalizationCorrections.txt");

   901     if (!f.exists())

   902         qFatal("Couldn't find NormalizationCorrections.txt");

   903 

   904     f.open(QFile::ReadOnly);

   905 

   906     QByteArray out;

   907 

   908     out += "struct NormalizationCorrection {\n"

   909            "    uint ucs4;\n"

   910            "    uint old_mapping;\n"

   911            "    int version;\n"

   912            "};\n\n"

   913 

   914            "static const NormalizationCorrection uc_normalization_corrections[] = {\n";

   915 

   916     int numCorrections = 0;

   917     while (!f.atEnd()) {

   918         QByteArray line;

   919         line.resize(1024);

   920         int len = f.readLine(line.data(), 1024);

   921         line.resize(len-1);

   922 

   923         int comment = line.indexOf('#');

   924         if (comment >= 0)

   925             line = line.left(comment);

   926         line.replace(" ", "");

   927 

   928         if (line.isEmpty())

   929             continue;

   930 

   931         Q_ASSERT(!line.contains(".."));

   932 

   933         QList<QByteArray> fields = line.split(';');

   934         Q_ASSERT(fields.size() == 4);

   935 

   936         NormalizationCorrection c;

   937         bool ok;

   938         c.codepoint = fields.at(0).toInt(&ok, 16);

   939         c.mapped = fields.at(1).toInt(&ok, 16);

   940         if (fields.at(3) == "3.2.0")

   941             c.version = QChar::Unicode_3_2;

   942         else if (fields.at(3) == "4.0.0")

   943             c.version = QChar::Unicode_4_0;

   944         else

   945             qFatal("unknown unicode version in NormalizationCorrection.txt");

   946 

   947         out += "    { 0x" + QByteArray::number(c.codepoint, 16) + ", 0x" + QByteArray::number(c.mapped, 16)

   948              + ", " + QString::number(c.version) + " },\n";

   949         ++numCorrections;

   950     }

   951 

   952     out += "};\n\n"

   953 

   954            "enum { NumNormalizationCorrections = " + QByteArray::number(numCorrections) + " };\n\n";

   955 

   956 

   957     return out;

   958 }

   959 

   960 

   961 static void computeUniqueProperties()

   962 {

   963     qDebug("computeUniqueProperties:");

   964     for (int uc = 0; uc < 0x110000; ++uc) {

   965         UnicodeData d = unicodeData.value(uc, UnicodeData(uc));

   966 

   967         int index = uniqueProperties.indexOf(d.p);

   968         if (index == -1) {

   969             index = uniqueProperties.size();

   970             uniqueProperties.append(d.p);

   971         }

   972         d.propertyIndex = index;

   973         unicodeData.insert(uc, d);

   974     }

   975     qDebug("    %d unicode properties found", uniqueProperties.size());

   976 }

   977 

   978 

   979 static void readLineBreak()

   980 {

   981     QFile f("data/LineBreak.txt");

   982     if (!f.exists())

   983         qFatal("Couldn't find LineBreak.txt");

   984 

   985     f.open(QFile::ReadOnly);

   986 

   987     while (!f.atEnd()) {

   988         QByteArray line;

   989         line.resize(1024);

   990         int len = f.readLine(line.data(), 1024);

   991         line.resize(len-1);

   992 

   993         int comment = line.indexOf('#');

   994         if (comment >= 0)

   995             line = line.left(comment);

   996         line.replace(" ", "");

   997 

   998         if (line.isEmpty())

   999             continue;

  1000 

  1001         QList<QByteArray> l = line.split(';');

  1002         Q_ASSERT(l.size() == 2);

  1003 

  1004         QByteArray codes = l[0];

  1005         codes.replace("..", ".");

  1006         QList<QByteArray> cl = codes.split('.');

  1007 

  1008         bool ok;

  1009         int from = cl[0].toInt(&ok, 16);

  1010         int to = from;

  1011         if (cl.size() == 2)

  1012             to = cl[1].toInt(&ok, 16);

  1013 

  1014         // ### Classes XX and AI are left out and mapped to AL for now

  1015         QUnicodeTables::LineBreakClass lb = QUnicodeTables::LineBreak_AL;

  1016         QByteArray ba = l[1];

  1017 

  1018         if (ba == "AI") lb = QUnicodeTables::LineBreak_AL;

  1019         else if (ba == "XX") lb = QUnicodeTables::LineBreak_AL;

  1020         else if (ba == "NL") lb = QUnicodeTables::LineBreak_AL;

  1021         else if (ba == "OP") lb = QUnicodeTables::LineBreak_OP;

  1022         else if (ba == "CL") lb = QUnicodeTables::LineBreak_CL;

  1023         else if (ba == "QU") lb = QUnicodeTables::LineBreak_QU;

  1024         else if (ba == "GL") lb = QUnicodeTables::LineBreak_GL;

  1025         else if (ba == "NS") lb = QUnicodeTables::LineBreak_NS;

  1026         else if (ba == "EX") lb = QUnicodeTables::LineBreak_EX;

  1027         else if (ba == "SY") lb = QUnicodeTables::LineBreak_SY;

  1028         else if (ba == "IS") lb = QUnicodeTables::LineBreak_IS;

  1029         else if (ba == "PR") lb = QUnicodeTables::LineBreak_PR;

  1030         else if (ba == "PO") lb = QUnicodeTables::LineBreak_PO;

  1031         else if (ba == "NU") lb = QUnicodeTables::LineBreak_NU;

  1032         else if (ba == "AL") lb = QUnicodeTables::LineBreak_AL;

  1033         else if (ba == "ID") lb = QUnicodeTables::LineBreak_ID;

  1034         else if (ba == "IN") lb = QUnicodeTables::LineBreak_IN;

  1035         else if (ba == "HY") lb = QUnicodeTables::LineBreak_HY;

  1036         else if (ba == "BA") lb = QUnicodeTables::LineBreak_BA;

  1037         else if (ba == "BB") lb = QUnicodeTables::LineBreak_BB;

  1038         else if (ba == "B2") lb = QUnicodeTables::LineBreak_B2;

  1039         else if (ba == "ZW") lb = QUnicodeTables::LineBreak_ZW;

  1040         else if (ba == "CM") lb = QUnicodeTables::LineBreak_CM;

  1041         else if (ba == "SA") lb = QUnicodeTables::LineBreak_SA;

  1042         else if (ba == "BK") lb = QUnicodeTables::LineBreak_BK;

  1043         else if (ba == "CR") lb = QUnicodeTables::LineBreak_CR;

  1044         else if (ba == "LF") lb = QUnicodeTables::LineBreak_LF;

  1045         else if (ba == "SG") lb = QUnicodeTables::LineBreak_SG;

  1046         else if (ba == "CB") lb = QUnicodeTables::LineBreak_AL;

  1047         else if (ba == "SP") lb = QUnicodeTables::LineBreak_SP;

  1048         else if (ba == "WJ") lb = QUnicodeTables::LineBreak_WJ;

  1049         else if (ba == "H2") lb = QUnicodeTables::LineBreak_H2;

  1050         else if (ba == "H3") lb = QUnicodeTables::LineBreak_H3;

  1051         else if (ba == "JL") lb = QUnicodeTables::LineBreak_JL;

  1052         else if (ba == "JV") lb = QUnicodeTables::LineBreak_JV;

  1053         else if (ba == "JT") lb = QUnicodeTables::LineBreak_JT;

  1054         else {

  1055             qDebug() << "unhandled line break class:" << ba;

  1056         }

  1057 

  1058         for (int codepoint = from; codepoint <= to; ++codepoint) {

  1059             UnicodeData d = unicodeData.value(codepoint, UnicodeData(codepoint));

  1060             d.p.line_break_class = lb;

  1061             unicodeData.insert(codepoint, d);

  1062         }

  1063     }

  1064 }

  1065 

  1066 

  1067 static void readSpecialCasing()

  1068 {

  1069 //     qDebug() << "Reading SpecialCasing.txt";

  1070     QFile f("data/SpecialCasing.txt");

  1071     if (!f.exists())

  1072         qFatal("Couldn't find SpecialCasing.txt");

  1073 

  1074     f.open(QFile::ReadOnly);

  1075 

  1076     while (!f.atEnd()) {

  1077         QByteArray line;

  1078         line.resize(1024);

  1079         int len = f.readLine(line.data(), 1024);

  1080         line.resize(len-1);

  1081 

  1082         int comment = line.indexOf('#');

  1083         if (comment >= 0)

  1084             line = line.left(comment);

  1085 

  1086         if (line.isEmpty())

  1087             continue;

  1088 

  1089         QList<QByteArray> l = line.split(';');

  1090 

  1091         QByteArray condition = l.size() < 5 ? QByteArray() : l[4].trimmed();

  1092         if (!condition.isEmpty())

  1093             // #####

  1094             continue;

  1095 

  1096         bool ok;

  1097         int codepoint = l[0].trimmed().toInt(&ok, 16);

  1098         Q_ASSERT(ok);

  1099         Q_ASSERT(codepoint <= 0xffff);

  1100 

  1101 //         qDebug() << "codepoint" << hex << codepoint;

  1102 //         qDebug() << line;

  1103 

  1104         QList<QByteArray> lower = l[1].trimmed().split(' ');

  1105         QList<int> lowerMap;

  1106         for (int i = 0; i < lower.size(); ++i) {

  1107             bool ok;

  1108             lowerMap.append(lower.at(i).toInt(&ok, 16));

  1109             Q_ASSERT(ok);

  1110         }

  1111 

  1112         QList<QByteArray> title = l[2].trimmed().split(' ');

  1113         QList<int> titleMap;

  1114         for (int i = 0; i < title.size(); ++i) {

  1115             bool ok;

  1116             titleMap.append(title.at(i).toInt(&ok, 16));

  1117             if (!ok)

  1118                 qDebug() << line << title.at(i);

  1119             Q_ASSERT(ok);

  1120         }

  1121 

  1122         QList<QByteArray> upper = l[3].trimmed().split(' ');

  1123         QList<int> upperMap;

  1124         for (int i = 0; i < upper.size(); ++i) {

  1125             bool ok;

  1126             upperMap.append(upper.at(i).toInt(&ok, 16));

  1127             Q_ASSERT(ok);

  1128         }

  1129 

  1130 

  1131         UnicodeData ud = unicodeData.value(codepoint, UnicodeData(codepoint));

  1132 

  1133         Q_ASSERT(lowerMap.size() > 1 || lowerMap.at(0) == codepoint + ud.p.lowerCaseDiff);

  1134         Q_ASSERT(titleMap.size() > 1 || titleMap.at(0) == codepoint + ud.p.titleCaseDiff);