--- a/localisation/localesupport/coltab/COLTAB.CPP Mon Dec 21 16:14:42 2009 +0000
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,1278 +0,0 @@
-// Copyright (c) 1999-2009 Nokia Corporation and/or its subsidiary(-ies).
-// All rights reserved.
-// This component and the accompanying materials are made available
-// under the terms of "Eclipse Public License v1.0"
-// which accompanies this distribution, and is available
-// at the URL "http://www.eclipse.org/legal/epl-v10.html".
-//
-// Initial Contributors:
-// Nokia Corporation - initial contribution.
-//
-// Contributors:
-//
-// Description:
-// Reads and parses the Unicode collation value table and writes out a C++ source file
-// containing the data in a form that can be used by the EPOC collation system.
-//
-// The program reads three files or one compositive files:
-//
-// Three files (by default):
-// 1. Base keys (maps single Unicode values to single collation key values): must be in the same format as
-// basekeys.txt, supplied with the Standard Unicode Collation system
-//
-// 2. Composite keys (maps single Unicode values to strings of collation keys): must be in the same format as
-// compkeys.txt, supplied with the Standard Unicode Collation system
-//
-// 3. Strings (maps strings of Unicode values to single collation keys OR strings of collation keys): must be in the
-// same format as compkeys.txt, except that there can be any number of Unicode characters at the start of the line,
-// space-separated and each exactly 4 hex digits.
-//
-// One compositive files (with option /a):
-// 1. All Keys (combine above three files into one file): must be in the same format as allkeys.txt, supplied with the Standard Unicode Collation system (after Unicode 3.0).
-//
-//
-
-
-#include <assert.h>
-#include <ctype.h>
-
-#ifdef __MSVCDOTNET__
-#include <fstream>
-#include <iostream>
-using namespace std;
-#else //!__MSVCDOTNET__
-#include <fstream.h>
-#include <iostream.h>
-#endif //__MSVCDOTNET__
-
-#include <stdlib.h>
-#include <string.h>
-#include <stdio.h>
-
-/*
-Constants constraining the range of level-1 and level-2 keys so that they can be packed.
-Non-zero values are reduced by one less than the minimum value.
-*/
-const unsigned int KLevel1Bits = 8;
-const unsigned int KLevel1Min = 0x20;
-const unsigned int KLevel1Max = KLevel1Min + (1 << KLevel1Bits) - 2;
-const unsigned int KLevel2Bits = 6;
-const unsigned int KLevel2Min = 1;
-const unsigned int KLevel2Max = KLevel2Min + (1 << KLevel2Bits) - 2;
-
-/*
-Table of characters in the WGL4 set, plus characters in canonical decompositions of
-those characters, plus commonly used control characters and space characters,
-given as ranges of Unicode characters. In each pair, the first code is the first in the range,
-and the second is the first code NOT in the range.
-
-The extra characters are added mainly to ensure that control characters and spaces are
-normally ignored. The extra characters are:
-
-0x0000-0x001F: ASCII control characters
-0x2000-0x2012: spaces, hyphen variants, figure dash
-0x2028-0x202E: line and paragraph separator, bidirectional control characters
-0xFEFF : byte-order mark
-0xFFFC-0xFFFD: object replacement character, replacement character
-*/
-const unsigned int Wgl4Range[] =
- {
- 0x00, 0x7f, // All ASCII
- 0xa0, 0x180, // Non-breaking space, Latin-1, Latin Extended-A
- 0x192,0x193, // Latin f with hook
- 0x1fa,0x200, // A-ring, a-ring, AE, ae, O slash, o slash all with acute accent
- 0x2c6,0x2c8, // non-combining circumflex and caron
- 0x2c9,0x2ca, // non-combining macron
- 0x2d8,0x2dc, // non-combining breve, dot above, ring above, ogonek
- 0x2dd,0x2de, // non-combining double acute
- 0x300,0x305, // combining grave, acute, circumflex, tilde, macron
- 0x306,0x309, // combining breve, dot above, double dot above
- 0x30a,0x30e, // combining ring above, double acute, caron, vertical line above
- 0x327,0x329, // combining cedilla, ogonek
- 0x384,0x38b, // Greek
- 0x38c,0x38d, // Greek
- 0x38e,0x3a2, // Greek
- 0x3a3,0x3cf, // Greek
- 0x401,0x40d, // Cyrillic
- 0x40e,0x450, // Cyrillic
- 0x451,0x45d, // Cyrillic
- 0x45e,0x460, // Cyrillic
- 0x490,0x492, // Cyrillic
- 0x1e80,0x1e86, // Both W and w with each of grave, acute and diaeresis
- 0x1ef2,0x1ef4, // Y with grave, y with grave
- 0x2000,0x2016, // various space and horizontal lines
- 0x2017,0x201f, //double vertical line, double low line, various quotation marks
- 0x2020,0x2023, // dagger, double dagger, bullet
- 0x2026,0x2027, //ellipsis
- 0x2028,0x202F, // line & paragraph separators and directional formatting
- 0x2030,0x2031, // per mille
- 0x2032,0x2034, // prime
- 0x2039,0x203b, // single angle quotation marks
- 0x203c,0x203d, // double exclamation mark
- 0x203e,0x203f, // non-combining overscore
- 0x2044,0x2045, // fraction slash
- 0x207f,0x2080, // superscript n
- 0x20a3,0x20a5, // French Franc, Italian/Turkish Lira
- 0x20a7,0x20a8, // Spanish Peseta
- 0x20ac,0x20ad, // Euro symbol
- 0x2105,0x2106, // care of
- 0x2113,0x2114, // script l
- 0x2116,0x2117, // numero
- 0x2122,0x2123, // trade mark
- 0x2126,0x2127, // ohm
- 0x212e,0x212f, // estimated (net weight)
- 0x215b,0x215f, // 1/8, 3/8, 5/8, 7/8
- 0x2190,0x2196, // horizontal and vertical arrows
- 0x21a8,0x21a9, // up down arrow with base
- 0x2202,0x2203, // partial differential
- 0x2206,0x2207, // increment (delta)
- 0x220f,0x2210, // n-ary product (pi)
- 0x2211,0x2213, // n-ary sum (sigma), minus
- 0x2215,0x2216, // division (slash)
- 0x2219,0x221b, // bullet operator, square root
- 0x221e,0x2220, // infinity, right angle
- 0x2229,0x222a, // intersection
- 0x222b,0x222c, // union
- 0x2248,0x2249, // almost equal to
- 0x2260,0x2262, // not equal to, identical to
- 0x2264,0x2266, // less-than-or-equal-to, greater-than-or-equal-to
- 0x2302,0x2303, // house
- 0x2310,0x2311, // rversed not sign
- 0x2320,0x2322, // top and bottom of integral
- 0x2500,0x2501, // box drawing
- 0x2502,0x2503, // box drawing
- 0x250c,0x250d, // box drawing
- 0x2510,0x2511, // box drawing
- 0x2514,0x2515, // box drawing
- 0x2518,0x2519, // box drawing
- 0x251c,0x251d, // box drawing
- 0x2524,0x2525, // box drawing
- 0x252c,0x252d, // box drawing
- 0x2534,0x2535, // box drawing
- 0x253c,0x253d, // box drawing
- 0x2550,0x256d, // box drawing
- 0x2580,0x2581, // block element
- 0x2584,0x2585, // block element
- 0x2588,0x2589, // block element
- 0x258c,0x258d, // block element
- 0x2590,0x2594, // block element
- 0x25a0,0x25a2, // geometric shapes
- 0x25aa,0x25ad, // geometric shapes
- 0x25b2,0x25b3, // geometric shapes
- 0x25ba,0x25bb, // geometric shapes
- 0x25bc,0x25bd, // geometric shapes
- 0x25c4,0x25c5, // geometric shapes
- 0x25ca,0x25cc, // geometric shapes
- 0x25cf,0x25d0, // geometric shapes
- 0x25d8,0x25da, // geometric shapes
- 0x25e6,0x25e7, // geometric shapes
- 0x263a,0x263d, // smilies, sun
- 0x2640,0x2641, // female
- 0x2642,0x2643, // male
- 0x2660,0x2661, // spade
- 0x2663,0x2664, // club
- 0x2665,0x2667, // heart
- 0x266a,0x266c, // quaver, beamed quavers
- 0xfb01,0xfb03, // fi, fl ligatures
- 0xfeff,0xff00, // zero-width non-breaking space
- 0xfffc, 0xfffe // object replacement character and replacement character
- };
-const int Wgl4Ranges = sizeof(Wgl4Range) / sizeof(Wgl4Range[0]) / 2;
-
-int CompareWgl4Ranges(const void* aRange1,const void* aRange2)
- {
- unsigned int* p = (unsigned int*)aRange1;
- unsigned int* q = (unsigned int*)aRange2;
- if (q[0] == q[1])
- {
- unsigned int* temp = p;
- p = q;
- q = temp;
- }
- if (*p < *q)
- return -1;
- else if (*p >= q[1])
- return 1;
- else
- return 0;
- }
-
-// Determine if a character is in the WGL4 character repertoire.
-static bool InWgl4(unsigned int aChar)
- {
- unsigned int key[2];
- key[0] = key[1] = aChar;
- return bsearch(key,Wgl4Range,Wgl4Ranges,sizeof(Wgl4Range[0]) * 2,CompareWgl4Ranges) != NULL;
- }
-
-// A collation key.
-class CollationKey
- {
-public:
- bool operator==(const CollationKey& k) const
- { return iLevel[0] == k.iLevel[0] && iLevel[1] == k.iLevel[1] && iLevel[2] == k.iLevel[2] &&
- iIgnorable == k.iIgnorable && iStop == k.iStop; }
-
- enum
- {
- ELevels = 3
- };
- int iLevel[ELevels];// the keys at the various levels
- bool iIgnorable; // TRUE if this key can normally be ignored
- bool iStop; // TRUE if this is the last key in a string of keys
- };
-
-// The collation index for a single Unicode value.
-class CollationIndex
- {
-public:
- static int Compare(const void* aIndex1,const void* aIndex2);
-
- int iCode; // Unicode value
- int iIndex; // index into the key table
- };
-
-class Reader
- {
-public:
- Reader(bool aWgl4,bool aStandard,const char* aLocaleName, const char* aUidString);
- ~Reader();
- void ReadBaseKeys(const char* aFileName);
- void ReadCompKeys(const char* aFileName);
- void ReadStrings(const char* aFileName);
- void ReadAllKeys(const char* aFileName);
- void WriteOutput(const char* aFileName, bool aCopyrightMessage);
- int CompareStringIndices(int aIndex1,int aIndex2) const;
-
-private:
- Reader(const Reader&);
- int Hex(const char *aString, int &aCharConsumed, bool aTolerate = false);
- void GetCollationKey(const char* aString, int& aCharConsumed, CollationKey* aKey=NULL);
- void GetMultipleCollationKeys(const char* aString);
- unsigned int PackKey(const CollationKey& aValue);
- int PackIndex(const CollationIndex& aValue, unsigned int result[2]);
- bool ParseLine(const char* aLine, int aCode[16], int& aCodeCount, int& aKeyStart, int& aKeyCount);
- void AddKeyOneToOne(const char* aLine, const int aCode, const int aKeyStart);
- void AddKeyOneToMuch(const char* aLine, const int aCode, const int aKeyStart);
- void AddKeyMuchToMuch(const char* aLine, const int aCode[16], const int aCodeCount, const int aKeyStart);
-
- enum
- {
- EMaxCollationKeys = 0x110000 * 2, /*more elements considering composite keys */
- EMaxCollationIndices = 0x110000,
- EMaxStringElements = 65536,
- EMaxStringIndices = 65536
- };
- CollationKey iCollationKey[EMaxCollationKeys];
- int iKeys;
- CollationIndex iCollationIndex[EMaxCollationIndices];
- int iIndices;
- int iStringElement[EMaxStringElements];
- int iStringElements;
- unsigned int iStringIndex[EMaxStringIndices];
- int iStringIndices;
- const char* iInputFileName;
- int iLineNumber;
- bool iSuppressCanonseqWarning; // have we issued the canonseq warning yet?
- bool iWgl4; // true if writing keys for wgl4 characters only
- bool iStandard; // true if reading standard files, not tailoring files
- const char* iLocaleName;
- const char* iUidString;
- char* iCPlusPlusIdentifier; // iLocaleName in title case with difficult characters removed
- };
-
-bool isValidHexDigit(char c)
- {
- if ('0' <= c && c <= '9')
- return true;
- if ('a' <= c && c <= 'f')
- return true;
- if ('A' <= c && c <= 'F')
- return true;
- return false;
- }
-
-void PrintUsage()
- {
- cout << "Usage: coltab [/u<uid>] [/c] [/a] [/h<topic>] <locale>\n";
- cout << "By Default (without /a option), for the locales 'standard' and 'wgl4' coltab reads basekeys.txt & compkeys.txt\n";
- cout << "For any other locale name <name> coltab reads <name>_basekeys.txt,\n";
- cout << "<name>_compkeys.txt and <name>_strings.txt.\n";
- cout << "Use the /a option, for the locales 'standard' and 'wgl4' coltab reads allkeys.txt\n";
- cout << "For any other locale name <name> coltab reads <name>_allkeys.txt.\n";
- cout << "The output file is always ls_<name>.cpp.\n";
- cout << "Use the /u option to specify the UID that the collation table should have.\n";
- cout << "A hex number must follow /u immediately, for example /u800ACBDE\n";
- cout << "this hex number must not exceed eight digits. If this is not specified,\n";
- cout << "the output file will have to be edited to make it compilable.\n";
- cout << "Specify /c to prefix the output with a Nokia copyright message.\n";
- cout << "Specify /h for in-depth help.";
- }
-
-void UsageError()
- {
- PrintUsage();
- exit(1);
- }
-
-void PrintHelp(char* aTopic)
- {
- int topic = 0;
- while ('0' <= *aTopic && *aTopic <= '9')
- {
- topic = topic * 10 + (*aTopic - '0');
- ++aTopic;
- }
- switch(topic)
- {
- case 1:
- cout << "How Coltab interprets CANONSEQ:\n\n"\
- "If the CANONSEQ specifier is used in a line, Coltab will ignore the mapping.\n"\
- "This because, on the Symbian platform, any canonically composed character is\n"\
- "decomposed before the key mapping is applied, so characters with canonical\n"\
- "decompositions do not need keys. In files supplied by the Unicode Consortium,\n"\
- "all mappings for composed characters are flagged by CANONSEQ, so it is useful\n"\
- "if Coltab can just ignore these so that Unicode Consortium files can be used\n"\
- "unedited.\n\n"\
- "This can cause problems if a localizer copies a line from a Unicode file into,\n"\
- "say, the <lang>_strings.txt file, in order to give a mapping for an accented\n"\
- "character. The localizer replaces the composed character code with the\n"\
- "decomposition and changes the keys but forgets to remove the CANONSEQ\n"\
- "specifier. In this case the key would be ignored. Coltab provides a warning so\n"\
- "that this can be put right.\n\n"\
- "Coltab will only warn about the first CANONSEQ in each file, and does not warn\n"\
- "if the 'standard' or 'wgl4' options are used.";
- exit(1);
- break;
- case 2:
- cout << "How to ensure coltab's output files are compilable.\n\n"\
- "By default, Coltab's files for locales need to be edited before they are\n"\
- "compilable. The UID for the collation method needs to be filled in. This UID\n"\
- "is added so that the collation table can be searched for later. At present,\n"\
- "this UID is not necessary for the correct functioning of the Symbian platform\n"\
- "and so a value of 0 can be safely used.\n\n"\
- "To insert this value into the file directly, use the /u option, for example\n"\
- "coltab /u0 french\n"\
- "If the /u option is used, the file should be compilable as is. If it is not,\n"\
- "please raise it as a defect with Symbian's internationalization team,\n"\
- "supplying the files that caused the problem if this is possible.\n"\
- "If the 'standard' or 'wgl4' options are used, no UID is output, so the /u\n"\
- "option is not required.";
- exit(1);
- break;
- case 3:
- cout << "How to ensure collation key values are inside the supported range. \n\n"\
- "According to Unicode Standard, the range suppored by tool COLTAB:\n"\
- " Level 0 (primary): 0000 - FFFF, \n"\
- " Level 1 (Secondary): 0020 - 011E, \n"\
- " Level 2 (Tertiary): 0001 - 003F. \n"\
- "Please edit your collation files and make sure key values are inside the above range";
- exit(1);
- break;
- default:
- PrintUsage();
- cout << "\n\nSpecify /h1 for help on the use of CANONSEQ\n";
- cout << "Specify /h2 for help on making compilable files that do not need editing\n";
- exit(1);
- break;
- }
- }
-
-short HighSurrogate(int aCode)
- {
- return static_cast<short>(0xD7C0 + (aCode >> 10));
- }
-
-short LowSurrogate(int aCode)
- {
- return static_cast<short>(0xDC00 | (aCode & 0x3FF));
- }
-
-int main(int argc,char** argv)
- {
- bool copyright = false;
- bool wgl4 = false;
- bool allKeys = false;
- const char* prefix = "";
- const char* infix = "";
- const char* locale = "";
- char* localeArg = 0;
- char* uidArg = 0;
- for (int i = 1; i < argc; ++i)
- {
- if (argv[i][0] == '/' || argv[i][0] == '-')
- {
- switch (argv[i][1])
- {
- case 'u':
- case 'U':
- {
- uidArg = argv[i] + 2;
- const char* uidCheck = uidArg;
- while (*uidCheck)
- {
- if (!isValidHexDigit(*uidCheck))
- UsageError();
- ++uidCheck;
- }
- if (uidCheck == uidArg || 8 < uidCheck - uidArg)
- UsageError();
- break;
- }
- case 'c':
- case 'C':
- copyright = true;
- break;
- case 'a':
- allKeys = true;
- break;
- case 'h':
- case 'H':
- PrintHelp(argv[i] + 2);
- break;
- default:
- UsageError();
- break;
- }
- }
- else if (!localeArg)
- localeArg = argv[i];
- else
- UsageError();
- }
- if (!localeArg)
- UsageError();
- bool standard = false;
- if (!_stricmp(localeArg, "standard"))
- {
- locale = "Standard";
- standard = true;
- }
- else if (!_stricmp(localeArg, "wgl4"))
- {
- locale = "Wgl4";
- wgl4 = true;
- standard = true;
- }
- else
- {
- locale = prefix = localeArg;
- infix = "_";
- }
-
- Reader* reader = new Reader(wgl4, standard, locale, uidArg);
- if (!reader)
- {
- cout << "out of memory\n";
- exit(1);
- }
- char* filename = new char[strlen(prefix) + strlen(infix) + 64];
- if (allKeys == false)
- {
- sprintf(filename,"%s%scompkeys.txt",prefix,infix);
- reader->ReadCompKeys(filename);
- if (!standard)
- {
- sprintf(filename,"%s%sstrings.txt",prefix,infix);
- reader->ReadStrings(filename);
- }
- sprintf(filename,"%s%sbasekeys.txt",prefix,infix);
- reader->ReadBaseKeys(filename);
- }
- else
- {
- sprintf(filename,"%s%sAllKeys.txt",prefix,infix);
- reader->ReadAllKeys(filename);
- }
- sprintf(filename,"ls_%s.cpp", localeArg);
- reader->WriteOutput(filename, copyright);
-
- delete reader;
- delete [] filename;
- return 0;
- }
-
-Reader::Reader(bool aWgl4, bool aStandard,
- const char* aLocaleName, const char* aUidString):
- iKeys(0),
- iIndices(0),
- iStringElements(0),
- iStringIndices(0),
- iInputFileName(NULL),
- iLineNumber(0),
- iSuppressCanonseqWarning(false),
- iWgl4(aWgl4),
- iStandard(aStandard),
- iLocaleName(aLocaleName),
- iUidString(aUidString)
- {
- if (iStandard)
- {
- iCPlusPlusIdentifier = new char[9];
- strcpy(iCPlusPlusIdentifier, "Standard");
- return;
- }
- char* p = iCPlusPlusIdentifier = new char[strlen(aLocaleName) + 2];
- int current = toupper(aLocaleName[0]);
- if (current < 'A' || 'Z' < current)
- *p++ = 'C';
- else
- {
- *p++ = static_cast<char>(current);
- ++aLocaleName;
- }
- bool inUnderScore = false;
- while (*aLocaleName)
- {
- current = tolower(*aLocaleName++);
- if (current < 'a' || 'z' < current)
- {
- if (!inUnderScore)
- {
- inUnderScore = true;
- *p++ = '_';
- }
- }
- else
- {
- inUnderScore = false;
- *p++ = static_cast<char>(current);
- }
- }
- *p = 0;
- }
-
-Reader::~Reader()
- {
- delete [] iCPlusPlusIdentifier;
- }
-
-// Get a hex number of exactly four digits from aString. Return -1 if none is found and aTolerate is true.
-int Reader::Hex(const char *aString, int &aCharConsumed, bool aTolerate)
- {
- char *end;
- unsigned long x = strtoul(aString,&end,16);
- aCharConsumed = end - aString;
- if ((aCharConsumed != 4) && (aCharConsumed != 5) && (aCharConsumed != 6))
- {
- if (!aTolerate)
- {
- cout << "bad hex number on line " << iLineNumber << " of file " << iInputFileName << '\n';
- exit(1);
- }
- return -1;
- }
- return x;
- }
-
-// Get a collation value from a string of the form [.xxxx.xxxx.xxxx.xxxx]
-void Reader::GetCollationKey(const char* aString, int& aCharConsumed, CollationKey* aKey)
- {
- aCharConsumed = 0;
- const char *end = strchr(aString, ']');
- if (end != NULL){
- aCharConsumed = end - aString;
- }
-
- if (aString[0] != '[' || (aCharConsumed != 21 && aCharConsumed != 22 && aCharConsumed != 23))
- {
- cout << "syntax error on line " << iLineNumber << " of file " << iInputFileName << '\n';
- exit(1);
- }
- if (aKey == NULL)
- {
- if (iKeys >= EMaxCollationKeys)
- {
- cout << "too many keys";
- exit(1);
- }
- aKey = &iCollationKey[iKeys++];
- }
- aKey->iIgnorable = aString[1] == '*'; // asterisk means that this character is normally ignored
- int charConsumed = 0;
- for (int i = 0; i < CollationKey::ELevels; i++)
- aKey->iLevel[i] = Hex(aString + 2 + i * 5, charConsumed);
-
- if (aKey->iLevel[1] > 0 && (aKey->iLevel[1] < KLevel1Min || aKey->iLevel[1] > KLevel1Max))
- {
- aKey->iLevel[1] = KLevel1Max;
- cout << "illegal level-1 key value on line " << iLineNumber << "; outside the range " << KLevel1Min << ".." << KLevel1Max << "\n";
- cout << "Error: illegal key value in file, please see coltab /h3 for details.\n";
- exit(1);
- }
-
- if (aKey->iLevel[2] > 0 && (aKey->iLevel[2] < KLevel2Min || aKey->iLevel[2] > KLevel2Max))
- {
- cout << "illegal level-2 key value on line " << iLineNumber << "; outside the range " << KLevel2Min << ".." << KLevel2Max << "\n";
- cout << "Error: illegal key value in file, please see coltab /h3 for details.\n";
- exit(1);
- }
-
- aKey->iStop = true;
- }
-
-void Reader::GetMultipleCollationKeys(const char* aString)
- {
- int keyCount = 0;
- int charConsumed =0;
- while (aString[0] == '[')
- {
- GetCollationKey(aString, charConsumed);
-
- keyCount++;
- iCollationKey[iKeys - 1].iStop = false;
- int length = strlen(aString);
- if (length <= charConsumed + 1)
- break;
- aString += charConsumed + 1;
-
- if (aString[0] == ' ') //a space is put between collation keys in keys files provided by previous Unicode Standard (i.e 3.1)
- aString++;
-
- }
- iCollationKey[iKeys - 1].iStop = true;
- }
-
-/*
-Partially parse a line, returning its key code and the start of its first block of key data.
-Return false if it is not a data line, or not relevant.
-*/
-bool Reader::ParseLine(const char* aLine, int aCode[16], int& aCodeCount, int& aKeyStart, int& aKeyCount)
- {
- int lineLength = strlen(aLine);
- int charConsumed = 0;
- aCodeCount = 0;
- aCode[0] = Hex(aLine,charConsumed,true);
-
- /*
- A data line must start with a hex number and be at least 27 characters long.
- Canonically decomposable Unicode characters are skipped.
- Skip non-WGL4 characters if doing WGL4 only.
- */
- if (aCode[0] != -1)
- {
- aCodeCount = 1;
- if (!strcmp(aLine + lineLength - 8,"CANONSEQ"))
- {
- if (!iSuppressCanonseqWarning)
- {
- cout << "Warning: CANONSEQ used in file " << iInputFileName
- << " on line " << iLineNumber << ".\nWarning: All mappings specifying CANONSEQ are ignored.\n"
- << "Warning: Use coltab /h1 for more details.";
- iSuppressCanonseqWarning = true;
- }
- aCodeCount = 0;
- }
- else if (lineLength < 27 ||
- (iWgl4 && !InWgl4((unsigned int)aCode)))
- aCodeCount = 0;
- }
-
- if (aCode[0] != -1)
- {
- // find '['
- aKeyStart = charConsumed;
- while (aKeyStart < lineLength && aLine[aKeyStart] != '[')
- aKeyStart++;
-
- // read all hex before '['
- int index = charConsumed + 1;
- while (index < aKeyStart)
- {
- aCode[aCodeCount] = Hex(aLine+index, charConsumed, true);
- if (aCode[aCodeCount] == -1)
- break;
-
- index += charConsumed + 1;
- aCodeCount++;
- }
-
- // find number of collation keys
- aKeyCount = 0;
- index = aKeyStart;
- while (index < lineLength && aLine[index] != '%' && aLine[index] != '#')
- {
- if (aLine[index] == '[')
- aKeyCount++;
- index++;
- }
- }
-
- return aCodeCount > 0;
- }
-
-void Reader::AddKeyOneToOne(const char* aLine, const int aCode, const int aKeyStart)
- {
- if (iIndices >= EMaxCollationIndices)
- {
- cout << "too many Unicode values";
- exit(1);
- }
- CollationIndex& index = iCollationIndex[iIndices++];
- index.iCode = aCode;
- index.iIndex = -1;
-
- /*
- First try to find the key in the array of keys found so far.
- Search backwards to use the fact that runs of the same key occur together.
- */
- CollationKey key;
- int charConsumed = 0;
- GetCollationKey(aLine + aKeyStart, charConsumed, &key);
- for (int i = iKeys - 1; i >= 0 && index.iIndex == -1; i--)
- if (iCollationKey[i] == key)
- index.iIndex = i;
-
- // If that fails, add a new key.
- if (index.iIndex == -1)
- {
- index.iIndex = iKeys++;
- if (iKeys > EMaxCollationKeys)
- {
- cout << "too many keys";
- exit(1);
- }
- iCollationKey[index.iIndex] = key;
- }
- }
-/*
-Read 1-to-1 mapping. Sample:
-02B9 ; [*02A5.0020.0002.02B9] % MODIFIER LETTER PRIME
-
-aCombinedFile = true: aFileName is combined file, which contains base keys, comp keys, and string keys.
-*/
-void Reader::ReadBaseKeys(const char* aFileName)
- {
- iSuppressCanonseqWarning = iStandard || iWgl4;
- iLineNumber = 0;
- iInputFileName = aFileName;
- ifstream input_file;
-
-#ifdef __MSVCDOTNET__
- input_file.open(iInputFileName, ios::in);
-#else //!__MSVCDOTNET__
- input_file.open(iInputFileName, ios::in | ios::nocreate);
-#endif //__MSVCDOTNET__
-
- if (input_file.fail())
- {
- cout << "cannot open input file '" << iInputFileName << "'\n";
- exit(1);
- }
- cout << "reading base keys from '" << iInputFileName << "'\n";
-
- char line[1024];
- for (;;)
- {
- input_file.getline(line,sizeof(line));
- if (input_file.eof())
- break;
- iLineNumber++;
- // line number counting
- if (iLineNumber % 100 == 0)
- {
- cout << "line " << iLineNumber << '\n';
- cout.flush();
- }
- int code[16];
- int codeCount = 0;
- int key_start = 0;
- int keyCount = 0;
- if (ParseLine(line, code, codeCount, key_start, keyCount))
- {
- if (codeCount != 1 || keyCount != 1)
- continue; // goto next line
- AddKeyOneToOne(line, code[0], key_start);
- }
- }
-
- input_file.close();
- }
-
-void Reader::AddKeyOneToMuch(const char* aLine, const int aCode, const int aKeyStart)
- {
- if (iIndices >= EMaxCollationIndices)
- {
- cout << "too many Unicode values";
- exit(1);
- }
- CollationIndex& index = iCollationIndex[iIndices++];
- index.iCode = aCode;
- index.iIndex = iKeys;
- GetMultipleCollationKeys(aLine + aKeyStart);
- }
-/*
-Read 1-to-much mapping.
-3303 ; [.279F.0020.001C.3303][.1114.0020.001C.3303][.27C7.0020.001F.3303] # SQUARE AARU; QQKN
-*/
-void Reader::ReadCompKeys(const char* aFileName)
- {
- iSuppressCanonseqWarning = iStandard || iWgl4;
- iLineNumber = 0;
- iInputFileName = aFileName;
- ifstream input_file;
-
-#ifdef __MSVCDOTNET__
- input_file.open(iInputFileName, ios::in);
-#else //!__MSVCDOTNET__
- input_file.open(iInputFileName, ios::in | ios::nocreate);
-#endif //__MSVCDOTNET__
-
- if (input_file.fail())
- {
- cout << "there are no composite keys; '" << iInputFileName << "' not found\n";
- return;
- }
- cout << "reading composite keys from '" << iInputFileName << "'\n";
-
- char line[1024];
- for (;;)
- {
- input_file.getline(line,sizeof(line));
- if (input_file.eof())
- break;
- iLineNumber++;
- // line number counting
- if (iLineNumber % 100 == 0)
- {
- cout << "line " << iLineNumber << '\n';
- cout.flush();
- }
- int code[16];
- int codeCount = 0;
- int key_start = 0;
- int keyCount = 0;
- if (ParseLine(line, code, codeCount, key_start, keyCount))
- {
- if (codeCount != 1 || keyCount < 2)
- continue; // goto next line
- AddKeyOneToMuch(line, code[0], key_start);
- }
- }
-
- input_file.close();
- }
-
-
-void Reader::AddKeyMuchToMuch(const char* aLine, const int aCode[16], const int aCodeCount, const int aKeyStart)
- {
-
- // Store the index to the Unicode string and the key sequence.
- if (iStringIndices > EMaxStringIndices)
- {
- cout << "too many string indices";
- exit(1);
- }
- iStringIndex[iStringIndices++] = (iStringElements << 16) | iKeys;
-
- // Reserve space for the length.
- if (iStringElements >= EMaxStringElements)
- {
- cout << "too many string elements";
- exit(1);
- }
- iStringElements++;
-
- // Read the Unicode string.
- int length = 0; // in unit of int16
- int charCount = 0; // in unit of char. for debug.
-
- for (int i=0; i<aCodeCount; i++)
- {
- if (iStringElements >= EMaxStringElements)
- {
- cout << "too many string elements";
- exit(1);
- }
-
- if (aCode[i] > 0xFFFF)
- {
- // UCS4 --> UTF-16
- iStringElement[iStringElements++] = 0xD7C0 + (aCode[i] >> 10);
- iStringElement[iStringElements++] = 0xDC00 | (aCode[i] & 0x3FF);
- length += 2;
- }
- else
- {
- iStringElement[iStringElements++] = aCode[i];
- length++;
- }
- charCount++;
- }
-
- iStringElement[iStringElements - length - 1] = (unsigned int)length;
-
- // Read the key sequence.
- GetMultipleCollationKeys(aLine + aKeyStart);
- }
-/*
-Read much-to-much mapping. Sample:
-004F 0338 [.08EA.0020.0008.00D8] % capital O-stroke
-0E40 0E08 ; [.1E2B.0020.0002.0E08][.1E5E.0020.001F.0E40] # <THAI CHARACTER SARA E, THAI CHARACTER CHO CHAN>
-*/
-void Reader::ReadStrings(const char* aFileName)
- {
- iSuppressCanonseqWarning = iStandard || iWgl4;
- iLineNumber = 0;
- iInputFileName = aFileName;
- ifstream input_file;
-
-#ifdef __MSVCDOTNET__
- input_file.open(iInputFileName, ios::in);
-#else //!__MSVCDOTNET__
- input_file.open(iInputFileName, ios::in | ios::nocreate);
-#endif //__MSVCDOTNET__
-
- if (input_file.fail())
- {
- cout << "there are no strings; '" << iInputFileName << "' not found\n";
- return;
- }
- cout << "reading strings from '" << iInputFileName << "'\n";
-
- char line[1024];
- for (;;)
- {
- input_file.getline(line,sizeof(line));
- if (input_file.eof())
- break;
- iLineNumber++;
- // line number counting
- if (iLineNumber % 100 == 0)
- {
- cout << "line " << iLineNumber << '\n';
- cout.flush();
- }
- int code[16];
- int codeCount = 0;
- int key_start = 0;
- int keyCount = 0;
- if (ParseLine(line, code, codeCount, key_start, keyCount))
- {
- if (codeCount < 2 || keyCount < 1)
- continue; // goto next line
- AddKeyMuchToMuch(line, code, codeCount, key_start);
- }
- }
-
- input_file.close();
- }
-
-/*
-Read combined key table. Sample:
-1-to-1 mapping:
-02B9 ; [*02A5.0020.0002.02B9] % MODIFIER LETTER PRIME
-
-1-to-much mapping:
-3303 ; [.279F.0020.001C.3303][.1114.0020.001C.3303][.27C7.0020.001F.3303] # SQUARE AARU; QQKN
-
-much-to-much mapping:
-004F 0338 [.08EA.0020.0008.00D8] % capital O-stroke
-0E40 0E08 ; [.1E2B.0020.0002.0E08][.1E5E.0020.001F.0E40] # <THAI CHARACTER SARA E, THAI CHARACTER CHO CHAN>
-*/
-void Reader::ReadAllKeys(const char* aFileName)
- {
- iSuppressCanonseqWarning = iStandard || iWgl4;
- iLineNumber = 0;
- iInputFileName = aFileName;
- ifstream input_file;
-
-#ifdef __MSVCDOTNET__
- input_file.open(iInputFileName, ios::in);
-#else //!__MSVCDOTNET__
- input_file.open(iInputFileName, ios::in | ios::nocreate);
-#endif //__MSVCDOTNET__
-
- if (input_file.fail())
- {
- cout << "there are no keys; '" << iInputFileName << "' not found\n";
- return;
- }
- cout << "reading all keys from '" << iInputFileName << "'\n";
-
- char line[1024];
- for (;;)
- {
- if (input_file.eof())
- break;
- input_file.getline(line,sizeof(line));
- iLineNumber++;
-
- int code[16];
- int codeCount = 0;
- int key_start = 0;
- int keyCount = 0;
- if (ParseLine(line, code, codeCount, key_start, keyCount))
- {
- if (codeCount == 1 && keyCount == 1)
- AddKeyOneToOne(line, code[0], key_start);
- else if (codeCount == 1 && keyCount > 1)
- AddKeyOneToMuch(line, code[0], key_start);
- else if (codeCount > 1 && keyCount > 0)
- AddKeyMuchToMuch(line, code, codeCount, key_start);
- else
- cout << "ignore line: " << line << "\n";
- }
- }
-
- input_file.close();
- }
-
-
-// Pack the 3 collation key levels into a single 32-bit integer.
-unsigned int Reader::PackKey(const CollationKey& aValue)
- {
- unsigned int level0 = aValue.iLevel[0];
- unsigned int level1 = aValue.iLevel[1];
- if (level1 > 0)
- level1 -= (KLevel1Min - 1);
- unsigned int level2 = aValue.iLevel[2];
- if (level2 > 0)
- level2 -= (KLevel2Min - 1);
- unsigned int key = level0 << 16 | level1 << 8 | level2 << 2;
- if (aValue.iIgnorable)
- key |= 2;
- if (aValue.iStop)
- key |= 1;
- return key;
- }
-
-// Pack a collation index value into a single 32-bit integer.
-int Reader::PackIndex(const CollationIndex& aValue, unsigned int result[2])
- {
- unsigned int code = aValue.iCode;
- unsigned int index = aValue.iIndex;
- if (code <= 0xFFFF)
- {
- result[0] = (code << 16 | index);
- return 1;
- }
- else
- {
- result[0] = (::HighSurrogate(code) << 16 | index);
- result[1] = (::LowSurrogate(code) << 16 | index);
- return 2;
- }
- }
-
-const Reader* TheReader;
-static int CompareStringIndices(const void* aIndex1,const void* aIndex2)
- {
- return TheReader->CompareStringIndices(*(unsigned int*)aIndex1 >> 16,*(unsigned int*)aIndex2 >> 16);
- }
-
-int CompareUnicodeStrings(const int *aString1,int aLength1,const int *aString2,int aLength2)
- {
- for (int i = 0; i < aLength1 || i < aLength2; i++, aString1++, aString2++)
- {
- int x = i < aLength1 ? *aString1 : -1;
- int y = i < aLength2 ? *aString2 : -1;
- if (x != y)
- return x - y;
- }
- return 0;
- }
-
-int Reader::CompareStringIndices(int aIndex1,int aIndex2) const
- {
- return CompareUnicodeStrings(iStringElement + aIndex1 + 1,iStringElement[aIndex1],
- iStringElement + aIndex2 + 1,iStringElement[aIndex2]);
- }
-
-void Reader::WriteOutput(const char* aFileName, bool aCopyright)
- {
- int i;
- ofstream output_file;
- output_file.open(aFileName);
- if (output_file.fail())
- {
- cout << "cannot open output file '" << aFileName << "'\n";
- exit(1);
- }
- cout << "writing output to '" << aFileName << "'\n";
-
- char *locale = NULL;
- if (iStandard)
- locale = _strdup("Standard");
- else
- locale = _strdup(iLocaleName);
-
- if (!iStandard)
- {
- _strlwr(locale);
- locale[0] = (char)toupper(locale[0]);
- if (aCopyright)
- {
- char* capsFileName = new char[strlen(aFileName) + 1];
- strcpy(capsFileName, aFileName);
- _strupr(capsFileName);
- output_file << "/*\n" << capsFileName << "\n\nCopyright (C) 2000-2009 Nokia Corporation and/or its subsidiary(-ies). All rights reserved.\n*/\n";
- delete [] capsFileName;
- output_file << "\n/*\nThe LCharSet object used by the " << locale << " locale.\n";
- output_file << "Generated by COLTAB.\n*/\n";
- }
-
- output_file << "\n#include \"ls_std.h\"\n#include <collate.h>\n";
- output_file << "\nconst TUint KUid" << iCPlusPlusIdentifier << "CollationMethod = ";
- if (iUidString)
- output_file << "0x" << iUidString << ";\n";
- else
- {
- output_file << "/* FILL THIS IN */;\n";
- cout << "Warning: File will need editing\nWarning: see coltab /h2 for details.\n";
- }
- }
-
- /*
- Write the unique collation keys.
- Each one has the format, going from highest to lowest bit:
-
- 16 bits: level-0 key
- 8 bits: level-1 key
- 6 bits: level-2 key
- 1 bit: set if this key is optionally ignorable
- 1 bit: set if this is the last key in the string of keys for a single Unicode value
-
- */
- if (iKeys != 0)
- {
- output_file << "\nstatic const TUint32 The" << iCPlusPlusIdentifier << "Key[] = \n\t{";
- CollationKey* ck = iCollationKey;
- output_file << "\t // " << iKeys << " keys";
- output_file << hex;
- for (i = 0; i < iKeys; i++, ck++)
- {
- unsigned int key = PackKey(*ck);
- if (i % 8 == 0)
- output_file << "\n\t";
- output_file << "0x";
- output_file << key << ",";
- }
- output_file << dec;
- output_file << "\n\t};\n\n";
- }
-
- if (iIndices != 0)
- {
- // Sort then write the collation index values - these relate Unicode values to collation keys.
- qsort(iCollationIndex,iIndices,sizeof(CollationIndex),CollationIndex::Compare);
- output_file << "static const TUint32 The" << iCPlusPlusIdentifier << "Index[] = \n\t{";
- CollationIndex* ci = iCollationIndex;
- int entry=0;
- output_file << "\t // " << iIndices << " indices";
- output_file << hex;
- for (i = 0; i < iIndices; i++, ci++, entry++)
- {
- unsigned int key[2];
- int bytecount = PackIndex(*ci, key);
-
- if (entry % 8 == 0)
- output_file << "\n\t";
- output_file << "0x";
- output_file << key[0] << ",";
-
- if (bytecount == 2)
- {
- entry++;
- if (entry % 8 == 0)
- output_file << "\n\t";
- output_file << "0x";
- output_file << key[1] << ",";
- }
- }
- output_file << dec;
- output_file << "\n\t};";
- output_file << "\t // " << entry << " entries";
- output_file << "\n\n";
- iIndices = entry; //One surrogate pair occupies 2 entries
- }
-
- if (iStringElements)
- {
- // Write the Unicode strings; these are preceded by their lengths.
- output_file << "static const TUint16 The" << iCPlusPlusIdentifier << "StringElement[] = \n\t{";
- output_file << hex;
- for (i = 0; i < iStringElements; i++)
- {
- if (i % 8 == 0)
- output_file << "\n\t";
- output_file << "0x" << iStringElement[i] << ",";
- }
- output_file << dec;
- if (iStringElements==0)
- output_file << "0";
- output_file << "\n\t};\n\n";
-
- /*
- Sort then write the string index values - these relate Unicode strings to collation keys.
- Each one has the string index in the upper word and the key index in the lower word.
- */
- TheReader = this;
- qsort(iStringIndex,iStringIndices,sizeof(iStringIndex[0]),::CompareStringIndices);
- output_file << "static const TUint32 The" << iCPlusPlusIdentifier << "StringIndex[] = \n\t{";
- output_file << hex;
- for (i = 0; i < iStringIndices; i++)
- {
- if (i % 8 == 0)
- output_file << "\n\t";
- output_file << "0x" << iStringIndex[i] << ",";
- }
- output_file << dec;
- if (iStringIndices ==0)
- output_file << "0";
- output_file << "\n\t};\n\n";
- }
-
- // Write the collation table structure.
- output_file << "static const TCollationKeyTable The" << iCPlusPlusIdentifier << "Table = \n\t{ ";
- if (iKeys)
- output_file << "The" << iCPlusPlusIdentifier << "Key";
- else
- output_file << "0";
- if (iIndices)
- output_file << ", The" << iCPlusPlusIdentifier << "Index, " << iIndices;
- else
- output_file << ", 0, 0";
- if (iStringElements)
- output_file << ", The" << iCPlusPlusIdentifier << "StringElement, The" << iCPlusPlusIdentifier << "StringIndex, " << iStringIndices << " };\n";
- else
- output_file << ", 0, 0, 0 };\n";
-
- if (!iStandard)
- output_file << "\nstatic const TCollationMethod TheCollationMethod[] = \n"\
- " {\n"\
- " {\n"\
- " KUid" << iCPlusPlusIdentifier << "CollationMethod, // the method for the locale\n"\
- " NULL, // use the standard table as the main table\n"\
- " &The" << iCPlusPlusIdentifier << "Table, // the locale values override the standard values\n"\
- " 0 // the flags are standard\n"\
- " },\n"\
- " {\n"\
- " KUidBasicCollationMethod, // the standard unlocalised method\n"\
- " NULL, // null means use the standard table\n"\
- " NULL, // there's no override table\n"\
- " 0 // the flags are standard\n"\
- " }\n"\
- " };\n"\
- "\n"\
- "static const TCollationDataSet TheCollationDataSet =\n"\
- " {\n"\
- " TheCollationMethod,\n"\
- " 2\n"\
- " };"\
- "\n\n"\
- "// The one and only locale character set object.\n"\
- "const LCharSet TheCharSet =\n"\
- " {\n"\
- " NULL,\n"\
- " &TheCollationDataSet\n"\
- " };\n";
-
- output_file.close();
- delete [] locale;
- }
-
-int CollationIndex::Compare(const void* aIndex1,const void* aIndex2)
- {
- return ((CollationIndex*)aIndex1)->iCode - ((CollationIndex*)aIndex2)->iCode;
- }