// Copyright (c) 1999-2009 Nokia Corporation and/or its subsidiary(-ies).

// All rights reserved.

// This component and the accompanying materials are made available

// under the terms of "Eclipse Public License v1.0"

// which accompanies this distribution, and is available

// at the URL "http://www.eclipse.org/legal/epl-v10.html".

//

// Initial Contributors:

// Nokia Corporation - initial contribution.

//

// Contributors:

//

// Description:

// Reads and parses the Unicode collation value table and writes out a C++ source file

// containing the data in a form that can be used by the EPOC collation system.

//

// The program reads three files or one compositive files:

//

// Three files (by default):

// 1. Base keys (maps single Unicode values to single collation key values): must be in the same format as

// basekeys.txt, supplied with the Standard Unicode Collation system

//

// 2. Composite keys (maps single Unicode values to strings of collation keys): must be in the same format as

// compkeys.txt, supplied with the Standard Unicode Collation system

//

// 3. Strings (maps strings of Unicode values to single collation keys OR strings of collation keys): must be in the

// same format as compkeys.txt, except that there can be any number of Unicode characters at the start of the line,

// space-separated and each exactly 4 hex digits.

//

// One compositive files (with option /a):

// 1. All Keys (combine above three files into one file): must be in the same format as allkeys.txt, supplied with the Standard Unicode Collation system (after Unicode 3.0).

//

//

#include <assert.h>

#include <ctype.h>

#ifdef __MSVCDOTNET__

#include <fstream>

#include <iostream>

using namespace std;

#else //!__MSVCDOTNET__

#include <fstream.h>

#include <iostream.h>

#endif //__MSVCDOTNET__

#include <stdlib.h>

#include <string.h>

#include <stdio.h>

/*

Constants constraining the range of level-1 and level-2 keys so that they can be packed.

Non-zero values are reduced by one less than the minimum value.

*/

const unsigned int KLevel1Bits = 8;

const unsigned int KLevel1Min = 0x20;

const unsigned int KLevel1Max = KLevel1Min + (1 << KLevel1Bits) - 2;

const unsigned int KLevel2Bits = 6;

const unsigned int KLevel2Min = 1;

const unsigned int KLevel2Max = KLevel2Min + (1 << KLevel2Bits) - 2;

/*

Table of characters in the WGL4 set, plus characters in canonical decompositions of

those characters, plus commonly used control characters and space characters,

given as ranges of Unicode characters. In each pair, the first code is the first in the range,

and the second is the first code NOT in the range.

The extra characters are added mainly to ensure that control characters and spaces are

normally ignored. The extra characters are:

0x0000-0x001F: ASCII control characters

0x2000-0x2012: spaces, hyphen variants, figure dash

0x2028-0x202E: line and paragraph separator, bidirectional control characters

0xFEFF		 : byte-order mark

0xFFFC-0xFFFD: object replacement character, replacement character

*/

const unsigned int Wgl4Range[] =

	{

	0x00, 0x7f,		// All ASCII

	0xa0, 0x180,		// Non-breaking space, Latin-1, Latin Extended-A

	0x192,0x193,		// Latin f with hook

	0x1fa,0x200,		// A-ring, a-ring, AE, ae, O slash, o slash all with acute accent

	0x2c6,0x2c8,		// non-combining circumflex and caron

	0x2c9,0x2ca,		// non-combining macron

	0x2d8,0x2dc,		// non-combining breve, dot above, ring above, ogonek

	0x2dd,0x2de,		// non-combining double acute

	0x300,0x305,		// combining grave, acute, circumflex, tilde, macron

	0x306,0x309,		// combining breve, dot above, double dot above

	0x30a,0x30e,		// combining ring above, double acute, caron, vertical line above

	0x327,0x329,		// combining cedilla, ogonek

	0x384,0x38b,		// Greek

	0x38c,0x38d,		// Greek

	0x38e,0x3a2,		// Greek

	0x3a3,0x3cf,		// Greek

	0x401,0x40d,		// Cyrillic

	0x40e,0x450,		// Cyrillic

	0x451,0x45d,		// Cyrillic

	0x45e,0x460,		// Cyrillic

	0x490,0x492,		// Cyrillic

	0x1e80,0x1e86,		// Both W and w with each of grave, acute and diaeresis

	0x1ef2,0x1ef4,		// Y with grave, y with grave

	0x2000,0x2016,		// various space and horizontal lines

	0x2017,0x201f,		//double vertical line, double low line, various quotation marks

	0x2020,0x2023,		// dagger, double dagger, bullet

	0x2026,0x2027,		//ellipsis

	0x2028,0x202F,		// line & paragraph separators and directional formatting

	0x2030,0x2031,		// per mille

	0x2032,0x2034,		// prime

	0x2039,0x203b,		// single angle quotation marks

	0x203c,0x203d,		// double exclamation mark

	0x203e,0x203f,		// non-combining overscore

	0x2044,0x2045,		// fraction slash

	0x207f,0x2080,		// superscript n

	0x20a3,0x20a5,		// French Franc, Italian/Turkish Lira

	0x20a7,0x20a8,		// Spanish Peseta

	0x20ac,0x20ad,		// Euro symbol

	0x2105,0x2106,		// care of

	0x2113,0x2114,		// script l

	0x2116,0x2117,		// numero

	0x2122,0x2123,		// trade mark

	0x2126,0x2127,		// ohm

	0x212e,0x212f,		// estimated (net weight)

	0x215b,0x215f,		// 1/8, 3/8, 5/8, 7/8

	0x2190,0x2196,		// horizontal and vertical arrows

	0x21a8,0x21a9,		// up down arrow with base

	0x2202,0x2203,		// partial differential

	0x2206,0x2207,		// increment (delta)

	0x220f,0x2210,		// n-ary product (pi)

	0x2211,0x2213,		// n-ary sum (sigma), minus

	0x2215,0x2216,		// division (slash)

	0x2219,0x221b,		// bullet operator, square root

	0x221e,0x2220,		// infinity, right angle

	0x2229,0x222a,		// intersection

	0x222b,0x222c,		// union

	0x2248,0x2249,		// almost equal to

	0x2260,0x2262,		// not equal to, identical to

	0x2264,0x2266,		// less-than-or-equal-to, greater-than-or-equal-to

	0x2302,0x2303,		// house

	0x2310,0x2311,		// rversed not sign

	0x2320,0x2322,		// top and bottom of integral

	0x2500,0x2501,		// box drawing

	0x2502,0x2503,		// box drawing

	0x250c,0x250d,		// box drawing

	0x2510,0x2511,		// box drawing

	0x2514,0x2515,		// box drawing

	0x2518,0x2519,		// box drawing

	0x251c,0x251d,		// box drawing

	0x2524,0x2525,		// box drawing

	0x252c,0x252d,		// box drawing

	0x2534,0x2535,		// box drawing

	0x253c,0x253d,		// box drawing

	0x2550,0x256d,		// box drawing

	0x2580,0x2581,		// block element

	0x2584,0x2585,		// block element

	0x2588,0x2589,		// block element

	0x258c,0x258d,		// block element

	0x2590,0x2594,		// block element

	0x25a0,0x25a2,		// geometric shapes

	0x25aa,0x25ad,		// geometric shapes

	0x25b2,0x25b3,		// geometric shapes

	0x25ba,0x25bb,		// geometric shapes

	0x25bc,0x25bd,		// geometric shapes

	0x25c4,0x25c5,		// geometric shapes

	0x25ca,0x25cc,		// geometric shapes

	0x25cf,0x25d0,		// geometric shapes

	0x25d8,0x25da,		// geometric shapes

	0x25e6,0x25e7,		// geometric shapes

	0x263a,0x263d,		// smilies, sun

	0x2640,0x2641,		// female

	0x2642,0x2643,		// male

	0x2660,0x2661,		// spade

	0x2663,0x2664,		// club

	0x2665,0x2667,		// heart

	0x266a,0x266c,		// quaver, beamed quavers

	0xfb01,0xfb03,		// fi, fl ligatures

	0xfeff,0xff00,		// zero-width non-breaking space

	0xfffc, 0xfffe		// object replacement character and replacement character

	};

const int Wgl4Ranges = sizeof(Wgl4Range) / sizeof(Wgl4Range[0]) / 2;

int CompareWgl4Ranges(const void* aRange1,const void* aRange2)

	{

	unsigned int* p = (unsigned int*)aRange1;

	unsigned int* q = (unsigned int*)aRange2;

	if (q[0] == q[1])

		{

		unsigned int* temp = p;

		p = q;

		q = temp;

		}

	if (*p < *q)

		return -1;

	else if (*p >= q[1])

		return 1;

	else

		return 0;

	}

// Determine if a character is in the WGL4 character repertoire.

static bool InWgl4(unsigned int aChar)

	{

	unsigned int key[2];

	key[0] = key[1] = aChar;

	return bsearch(key,Wgl4Range,Wgl4Ranges,sizeof(Wgl4Range[0]) * 2,CompareWgl4Ranges) != NULL;

	}

// A collation key.

class CollationKey

	{

public:

	bool operator==(const CollationKey& k) const

		{ return iLevel[0] == k.iLevel[0] && iLevel[1] == k.iLevel[1] && iLevel[2] == k.iLevel[2] &&

		  iIgnorable == k.iIgnorable && iStop == k.iStop; }

	enum

		{

		ELevels = 3

		};

	int iLevel[ELevels];// the keys at the various levels

	bool iIgnorable;	// TRUE if this key can normally be ignored

	bool iStop;			// TRUE if this is the last key in a string of keys

	};

// The collation index for a single Unicode value.

class CollationIndex

	{

public:

	static int Compare(const void* aIndex1,const void* aIndex2);

	int iCode;			// Unicode value

	int iIndex;			// index into the key table

	};

class Reader

	{

public:

	Reader(bool aWgl4,bool aStandard,const char* aLocaleName, const char* aUidString);

	~Reader();

	void ReadBaseKeys(const char* aFileName);

	void ReadCompKeys(const char* aFileName);

	void ReadStrings(const char* aFileName);

	void ReadAllKeys(const char* aFileName);

	void WriteOutput(const char* aFileName, bool aCopyrightMessage);

	int CompareStringIndices(int aIndex1,int aIndex2) const;

private:

	Reader(const Reader&);

	int Hex(const char *aString, int &aCharConsumed, bool aTolerate = false);

	void GetCollationKey(const char* aString, int& aCharConsumed, CollationKey* aKey=NULL);

	void GetMultipleCollationKeys(const char* aString);

	unsigned int PackKey(const CollationKey& aValue);

	int PackIndex(const CollationIndex& aValue, unsigned int result[2]);

	bool ParseLine(const char* aLine, int aCode[16], int& aCodeCount, int& aKeyStart, int& aKeyCount);

	void AddKeyOneToOne(const char* aLine, const int aCode, const int aKeyStart);

	void AddKeyOneToMuch(const char* aLine, const int aCode, const int aKeyStart);

	void AddKeyMuchToMuch(const char* aLine, const int aCode[16], const int aCodeCount, const int aKeyStart);

	enum

		{

		EMaxCollationKeys = 0x110000 * 2, /*more elements considering composite keys */

		EMaxCollationIndices = 0x110000,

		EMaxStringElements = 65536,

		EMaxStringIndices = 65536

		};

	CollationKey iCollationKey[EMaxCollationKeys];

	int iKeys;

	CollationIndex iCollationIndex[EMaxCollationIndices];

	int iIndices;

	int iStringElement[EMaxStringElements];

	int iStringElements;

	unsigned int iStringIndex[EMaxStringIndices];

	int iStringIndices;

	const char* iInputFileName;

	int iLineNumber;

	bool iSuppressCanonseqWarning;		// have we issued the canonseq warning yet?

	bool iWgl4;				// true if writing keys for wgl4 characters only

	bool iStandard;			// true if reading standard files, not tailoring files

	const char* iLocaleName;

	const char* iUidString;

	char* iCPlusPlusIdentifier;		// iLocaleName in title case with difficult characters removed

	};

bool isValidHexDigit(char c)

	{

	if ('0' <= c && c <= '9')

		return true;

	if ('a' <= c && c <= 'f')

		return true;

	if ('A' <= c && c <= 'F')

		return true;

	return false;

	}

void PrintUsage()

	{

	cout << "Usage: coltab [/u<uid>] [/c] [/a] [/h<topic>] <locale>\n";

	cout << "By Default (without /a option), for the locales 'standard' and 'wgl4' coltab reads basekeys.txt & compkeys.txt\n";

	cout << "For any other locale name <name> coltab reads <name>_basekeys.txt,\n";

	cout << "<name>_compkeys.txt and <name>_strings.txt.\n";

	cout << "Use the /a option, for the locales 'standard' and 'wgl4' coltab reads allkeys.txt\n";

	cout << "For any other locale name <name> coltab reads <name>_allkeys.txt.\n"; 

	cout << "The output file is always ls_<name>.cpp.\n";

	cout << "Use the /u option to specify the UID that the collation table should have.\n";

	cout << "A hex number must follow /u immediately, for example /u800ACBDE\n";

	cout << "this hex number must not exceed eight digits. If this is not specified,\n";

	cout << "the output file will have to be edited to make it compilable.\n";

	cout << "Specify /c to prefix the output with a Nokia copyright message.\n";

	cout << "Specify /h for in-depth help.";

	}

void UsageError()

	{

	PrintUsage();

	exit(1);

	}

void PrintHelp(char* aTopic)

	{

	int topic = 0;

	while ('0' <= *aTopic && *aTopic <= '9')

		{

		topic = topic * 10 + (*aTopic - '0');

		++aTopic;

		}

	switch(topic)

		{

	case 1:

		cout << "How Coltab interprets CANONSEQ:\n\n"\

			"If the CANONSEQ specifier is used in a line, Coltab will ignore the mapping.\n"\

			"This because, on the Symbian platform, any canonically composed character is\n"\

			"decomposed before the key mapping is applied, so characters with canonical\n"\

			"decompositions do not need keys. In files supplied by the Unicode Consortium,\n"\

			"all mappings for composed characters are flagged by CANONSEQ, so it is useful\n"\

			"if Coltab can just ignore these so that Unicode Consortium files can be used\n"\

			"unedited.\n\n"\

			"This can cause problems if a localizer copies a line from a Unicode file into,\n"\

			"say, the <lang>_strings.txt file, in order to give a mapping for an accented\n"\

			"character. The localizer replaces the composed character code with the\n"\

			"decomposition and changes the keys but forgets to remove the CANONSEQ\n"\

			"specifier. In this case the key would be ignored. Coltab provides a warning so\n"\

			"that this can be put right.\n\n"\

			"Coltab will only warn about the first CANONSEQ in each file, and does not warn\n"\

			"if the 'standard' or 'wgl4' options are used.";

		exit(1);

		break;

	case 2:

		cout << "How to ensure coltab's output files are compilable.\n\n"\

			"By default, Coltab's files for locales need to be edited before they are\n"\

			"compilable. The UID for the collation method needs to be filled in. This UID\n"\

			"is added so that the collation table can be searched for later. At present,\n"\

			"this UID is not necessary for the correct functioning of the Symbian platform\n"\

			"and so a value of 0 can be safely used.\n\n"\

			"To insert this value into the file directly, use the /u option, for example\n"\

			"coltab /u0 french\n"\

			"If the /u option is used, the file should be compilable as is. If it is not,\n"\

			"please raise it as a defect with Symbian's internationalization team,\n"\

			"supplying the files that caused the problem if this is possible.\n"\

			"If the 'standard' or 'wgl4' options are used, no UID is output, so the /u\n"\

			"option is not required.";

		exit(1);

		break;

	case 3:

		cout << "How to ensure collation key values are inside the supported range. \n\n"\

			"According to Unicode Standard, the range suppored by tool COLTAB:\n"\

			" Level 0 (primary):   0000 - FFFF, \n"\

			" Level 1 (Secondary): 0020 - 011E, \n"\

			" Level 2 (Tertiary):  0001 - 003F. \n"\

			"Please edit your collation files and make sure key values are inside the above range";

		exit(1);

		break;

	default:

		PrintUsage();

		cout << "\n\nSpecify /h1 for help on the use of CANONSEQ\n";

		cout << "Specify /h2 for help on making compilable files that do not need editing\n";

		exit(1);

		break;

		}

	}

short HighSurrogate(int aCode)

	{

	return static_cast<short>(0xD7C0 + (aCode >> 10));

	}

short LowSurrogate(int aCode)

	{

	return static_cast<short>(0xDC00 | (aCode & 0x3FF));

	}

int main(int argc,char** argv)

	{

	bool copyright = false;

	bool wgl4 = false;

	bool allKeys = false;

	const char* prefix = "";

	const char* infix = "";

	const char* locale = "";

	char* localeArg = 0;

	char* uidArg = 0;

	for (int i = 1; i < argc; ++i)

		{

		if (argv[i][0] == '/' || argv[i][0] == '-')

			{

			switch (argv[i][1])

				{

			case 'u':

			case 'U':

				{

				uidArg = argv[i] + 2;

				const char* uidCheck = uidArg;

				while (*uidCheck)

					{

					if (!isValidHexDigit(*uidCheck))

						UsageError();

					++uidCheck;

					}

				if (uidCheck == uidArg || 8 < uidCheck - uidArg)

					UsageError();

				break;

				}

			case 'c':

			case 'C':

				copyright = true;

				break;

			case 'a':

				allKeys = true;

				break;

			case 'h':

			case 'H':

				PrintHelp(argv[i] + 2);

				break;

			default:

				UsageError();

				break;

				}

			}

		else if (!localeArg)

			localeArg = argv[i];

		else

			UsageError();

		}

	if (!localeArg)

		UsageError();

	bool standard = false;

	if (!_stricmp(localeArg, "standard"))

		{

		locale = "Standard";

		standard = true;

		}

	else if (!_stricmp(localeArg, "wgl4"))

		{

		locale = "Wgl4";

		wgl4 = true;

		standard = true;

		}

	else

		{

		locale = prefix = localeArg;

		infix = "_";

		}

	Reader* reader = new Reader(wgl4, standard, locale, uidArg);

	if (!reader)

		{

		cout << "out of memory\n";

		exit(1);

		}

	char* filename = new char[strlen(prefix) + strlen(infix) + 64];

	if (allKeys == false)

		{

		sprintf(filename,"%s%scompkeys.txt",prefix,infix);

		reader->ReadCompKeys(filename);

		if (!standard)

			{

			sprintf(filename,"%s%sstrings.txt",prefix,infix);

			reader->ReadStrings(filename);

			}

		sprintf(filename,"%s%sbasekeys.txt",prefix,infix);

		reader->ReadBaseKeys(filename);

		}

	else

		{

		sprintf(filename,"%s%sAllKeys.txt",prefix,infix);

		reader->ReadAllKeys(filename);

		}

	sprintf(filename,"ls_%s.cpp", localeArg);

	reader->WriteOutput(filename, copyright);

	delete reader;

	delete [] filename;

	return 0;

	}

Reader::Reader(bool aWgl4, bool aStandard,

	const char* aLocaleName, const char* aUidString):

	iKeys(0),

	iIndices(0),

	iStringElements(0),

	iStringIndices(0),

	iInputFileName(NULL),

	iLineNumber(0),

	iSuppressCanonseqWarning(false),

	iWgl4(aWgl4),

	iStandard(aStandard),

	iLocaleName(aLocaleName),

	iUidString(aUidString)

	{

	if (iStandard)

		{

		iCPlusPlusIdentifier = new char[9];

		strcpy(iCPlusPlusIdentifier, "Standard");

		return;

		}

	char* p = iCPlusPlusIdentifier = new char[strlen(aLocaleName) + 2];

	int current = toupper(aLocaleName[0]);

	if (current < 'A' || 'Z' < current)

		*p++ = 'C';

	else

		{

		*p++ = static_cast<char>(current);

		++aLocaleName;

		}

	bool inUnderScore = false;

	while (*aLocaleName)

		{

		current = tolower(*aLocaleName++);

		if (current < 'a' || 'z' < current)