// Copyright (c) 1997-2009 Nokia Corporation and/or its subsidiary(-ies).

// All rights reserved.

// This component and the accompanying materials are made available

// under the terms of the License "Eclipse Public License v1.0"

// which accompanies this distribution, and is available

// at the URL "http://www.eclipse.org/legal/epl-v10.html".

//

// Initial Contributors:

// Nokia Corporation - initial contribution.

//

// Contributors:

//

// Description:

// Reads a Unicode character type data file (such as UnicodeData-3.0.0.txt or a file containing locale-specific overrides)

// and writes C++ definitions of tables containing the information.

// Usage: readtype <input-file> <output-file> { <locale-name> }.

// <input-file>: either the standard Unicode character data file (e.g., UnicodeData-3.0.0.txt) or a file containing

// overriding information for a certain locale, in the same format as the standard file, but with ranges for which

// there is no data given in the form:

// 0041;;;;;;;;;;;;;;

// 006A;<No Data First>;;;;;;;;;;;;;

// FFFF;<No Data Last>;;;;;;;;;;;;;

// (in this example, these entries show that there is no overriding data for the character 0041 and range

// 006A..FFFF inclusive).

// Both single entries with no data and ranges with no data must have nothing in the third field (category).

// <output-file>: the C++ source file to be output: this file becomes \e32\unicode\unitable.cpp, or an overriding

// file in \e32\lsrc; there are none of these yet.

// <locale-name>: a an optional name to be inserted into identifiers in the output file: omit this for the standard

// data set; use names like 'Turkish', 'Japanese', etc., for locales.

// 

//

#include <stdio.h>

#include <stdlib.h>

#include <string.h>

#ifndef _UNICODE

#define _UNICODE

#endif

#include <unicode.h>

// don't use unicode.h::TUnicodeDataRange, since its for 16-bit, and deprecated

struct TUnicodeDataRange32	// Only used inside this cpp.

	{

	TUint32 iRangeStart;	// Unicode value of the start of the range of characters

	TInt16 iIndex;			// index into an array of character information structures (-1 means data no available)

	};

const int PlaneCount = 17;

TUnicodePlane ThePlanesInReadType[PlaneCount];

// Tables to convert names used in the data file to categories defined in TChar.

struct CatInfo

	{

	const char* iName;

	TChar::TCategory iCat;

	};

static const CatInfo TheCatInfo[] =

	{

	{ "Lu", TChar::ELuCategory },

	{ "Ll", TChar::ELlCategory },

	{ "Lt", TChar::ELtCategory },

	{ "Lo", TChar::ELoCategory },

	{ "Lm", TChar::ELmCategory },

	{ "Mn", TChar::EMnCategory },

	{ "Mc", TChar::EMcCategory },

	{ "Me", TChar::EMeCategory },

	{ "Nd", TChar::ENdCategory },

	{ "Nl", TChar::ENlCategory },

	{ "No", TChar::ENoCategory },

	{ "Pc", TChar::EPcCategory },

	{ "Pd", TChar::EPdCategory },

	{ "Ps", TChar::EPsCategory },

	{ "Pe", TChar::EPeCategory },

	{ "Pi", TChar::EPiCategory },

	{ "Pf", TChar::EPfCategory },

	{ "Po", TChar::EPoCategory },

	{ "Sm", TChar::ESmCategory },

	{ "Sc", TChar::EScCategory },

	{ "Sk", TChar::ESkCategory },

	{ "So", TChar::ESoCategory },

	{ "Zs", TChar::EZsCategory },

	{ "Zl", TChar::EZlCategory },

	{ "Zp", TChar::EZpCategory },

	{ "Cc", TChar::ECcCategory },

	{ "Cf", TChar::ECfCategory },

	{ "Cs", TChar::ECsCategory },

	{ "Co", TChar::ECoCategory },

	{ "Cn", TChar::ECnCategory }

	};

const int TheCategories = sizeof(TheCatInfo) / sizeof(TheCatInfo[0]);

struct BdCatInfo

	{

	const char* iName;

	TChar::TBdCategory iBdCat;

	};

static const BdCatInfo TheBdCatInfo[] =

	{

	{ "L", TChar::ELeftToRight },

	{ "LRE", TChar::ELeftToRightEmbedding },

	{ "LRO", TChar::ELeftToRightOverride },

	{ "R", TChar::ERightToLeft },

	{ "AL", TChar::ERightToLeftArabic }, 

	{ "RLE", TChar::ERightToLeftEmbedding },

	{ "RLO", TChar::ERightToLeftOverride },

	{ "PDF", TChar::EPopDirectionalFormat },

	{ "EN", TChar::EEuropeanNumber }, 

	{ "ES", TChar::EEuropeanNumberSeparator }, 

	{ "ET", TChar::EEuropeanNumberTerminator },  

	{ "AN", TChar::EArabicNumber }, 

	{ "CS", TChar::ECommonNumberSeparator }, 

	{ "NSM", TChar::ENonSpacingMark },

	{ "BN", TChar::EBoundaryNeutral }, 

	{ "B", TChar::EParagraphSeparator },

	{ "S", TChar::ESegmentSeparator },

	{ "WS", TChar::EWhitespace }, 

	{ "ON", TChar::EOtherNeutral }, 

	};

const int TheBdCategories = sizeof(TheBdCatInfo) / sizeof(TheBdCatInfo[0]);

// Class derived from TUnicodeData to provide constructor etc.

class Data: public TUnicodeData

	{

	public:

	Data();

	TBool operator==(const Data& c) const;

	TBool operator!=(const Data& c) const { return !(*this == c); }

	void Write();

	};

// The character information table.

const int MaxDatas = 1000;

Data TheData[MaxDatas];

int Datas = 0;

// The range table, containing indices to the character information table.

const int MaxRanges = 4000;

TUnicodeDataRange32 TheRange[MaxRanges];

int Ranges = 0;

// The exhaustive index table, containing indices from every 16-bit value to the character information table.

int TheIndex[0x110000];

// The special tables for characters in the range 0..255.

TUint16 LowerCaseTable[256];

TUint16 FoldTable[256];

// The special table for characters in the range 0xFF00..0xFFFF

TUint16 CjkWidthFoldTable[256];

/*

The composition table. The compositions are stored as a word made up from the composition tag (high byte) and

the number of components (low byte), the Unicode value of the composed character, then the Unicode values of

the components.

Two tables are created containing the indices of compositions. One of these is sorted by

composed character, one by decomposition. This enables quick conversions to be made in both directions.

*/

const int MaxCompositionWords = 14000;

TUint32 CompositionBuffer[MaxCompositionWords];

int CompositionWords = 0;

const int MaxCompositions = 8000;

TInt16 Compose[MaxCompositions];		// composition buffer indices, sorted by composed character

TInt16 Decompose[MaxCompositions];		// composition buffer indices, sorted by decomposition

int Compositions = 0;

int trie_data[0x110000];					// used to build the trie

FILE *input_file;

FILE *output_file;

const char *input_filename;

const char *output_filename;

// Convert a hex string to an integer.

static int hex(const char *s)

	{

	int x = 0;

	while (*s)

		{

		int n = *s;

		if (n >= '0' && n <= '9')

			n -= '0';

		else if (n >= 'A' && n <= 'F')

			n -= 'A' - 10;

		else if (n >= 'a' && n <= 'f')

			n -= 'a' - 10;

		else

			break;

		x = x * 16 + n;

		s++;

		}

	return x;

	}

static TChar::TCategory Category(const char* aName,bool aWarn)

	{

	for (int i = 0; i < TheCategories; i++)

		if (!strcmp(aName,TheCatInfo[i].iName))

			return TheCatInfo[i].iCat;

	if (aWarn)

		fprintf(stderr,"unknown category %s\n",aName);

	return (TChar::TCategory)(-1);

	}

static TChar::TBdCategory BdCategory(const char* aName,bool aWarn)

	{

	for (int i = 0; i < TheBdCategories; i++)

		if (!strcmp(aName,TheBdCatInfo[i].iName))

			return TheBdCatInfo[i].iBdCat;

	if (aWarn)

		fprintf(stderr,"unknown bidirectional category %s\n",aName);

	return (TChar::TBdCategory)(-1);

	}

// Write an aggregate initialiser for a Data object to the output file.

void Data::Write()

	{

	fprintf(output_file,"{ %d, %d, %d, %d, %d, %d }",

			(int)iCategory,

			(int)iBdCategory,

			(int)iCombiningClass,

			(int)iDigitOffset,

			(int)iCaseOffset,

			(int)iFlags);

	}

/*

Add a new entry to the range table. If the category is the illegal value -1 store -1 as the

index; this feature is used when creating character data for specific locales, which mostly

consists of ranges for which the data is held in the main table, and is marked in this way

as unspecified in the locale table.

*/

void add_range(Data& info,TInt code)

	{

	// Get an index to the character info; add a new entry if necessary.

	int index = -1;

	if (info.iCategory != TChar::TCategory(0xFF))

		{

		for (int i = 0; i < Datas && index == -1; i++)

			if (TheData[i] == info)

				index = i;

		if (index == -1)

			{

			if (Datas >= MaxDatas)

				{

				fprintf(stderr,"too many Datas: > %d\n",MaxDatas);

				exit(1);

				}

			TheData[index = Datas++] = info;

			}

		}

	// Add the entry to the range table.

	if (Ranges >= MaxRanges)

		{

		fprintf(stderr,"too many Ranges: > %d, when processing U+%x\n", MaxRanges, code);

		exit(1);

		}

	TheRange[Ranges].iRangeStart = code;

	TheRange[Ranges].iIndex = (TInt16)index;

	Ranges++;

	}

// Write a table of "entries" integers each of "entry_size" bytes.

int write_table(const void *table,const char *name,

				int entries,int input_entry_size,int output_entry_size,

				int entry_signed,int entries_per_row,int write_array_size)

	{

	const char *type = entry_signed ? "TInt" : "TUint";

	const int bits = output_entry_size * 8;

	/*

	There is a choice here whether or not the number of entries in the array is written:

	either <name>[<size>] or <name>[] is written. The latter method is used where the header

	says <name>[] so that compilers like GCC don't moan about type mismatches.

	*/

	if (entries == 0)

		{

		// In case that given plane has no character.

		fprintf(output_file,"const %s%d * const %s = NULL;\n",type,bits,name);

		return 0;

		}

	if (write_array_size)

		fprintf(output_file,"const %s%d %s[%d] = \n\t{",type,bits,name,entries);

	else

		fprintf(output_file,"const %s%d %s[] = \n\t{ // %d entries",type,bits,name,entries);

	const unsigned char *p = (const unsigned char *)table;

	for (int i = 0; i < entries; i++, p += input_entry_size)

		{

		if (i % entries_per_row == 0)

			fprintf(output_file,"\n\t");

		if (output_entry_size == 1)

			fprintf(output_file,"0x%02x",(int)(*p));

		else if (output_entry_size == 2)

			fprintf(output_file,"0x%04x",(int)(*((TUint16 *)p)));

		else if (output_entry_size == 4)

			fprintf(output_file,"0x%08x",(int)(*((TUint32 *)p)));

		else

			{

			fprintf(stderr,"illegal output entry size: %d\n",output_entry_size);

			exit(1);

			}

		if (i < entries - 1)

			fputc(',',output_file);

		// comment for easy read

		//if ((i+1) % entries_per_row == 0)

		//	fprintf(output_file, "\t// U+%X-U+%X (%d-%d)", i+1-entries_per_row, i, i+1-entries_per_row, i);

		}

	fprintf(output_file,"\n\t};\n");

	return entries * output_entry_size;

	}

/*

Create and write a trie representing the data in 'aTheIndex'

The trie is of two levels, the first level indexed by the high 'aBlockBits' bits of the

character code, the second by the low bits. There is one wrinkle; if the index value, which is 16 bits,

has its top bit set, it is not an index but the actual data value for all entries in that block.

Thus the way to get the value for a code is:

int index = trie_index[code >> aBlockBits];

if (index & 0x8000)

	value = index & ~0x8000;

else

	value = aTrieData[code & (1 << (16 - aBlockBits))];

The data size in bytes is returned.

The argument 'aWrite' determines whether the data is written or not.

The arguments 'aTrie1Name' and 'aTrie2Name' are used as variable names in generated unitable.cpp.

*/

int write_trie(int aOutputEntrySize,int aBlockBits,bool aWrite, int *aTheIndex, int *aTrieData, char *aTrie1Name, char *aTrie2Name)

	{

	int n = 0; // number of entries used in trie_data

	int block_size = 1 << aBlockBits;

	int blocks = 1 << (16 - aBlockBits);

	int* trie_index = new int[blocks];

	int* block = new int[block_size];

	for (int block_index = 0; block_index < blocks; block_index++)

		{

		// Write the data for the current block.

		int block_start = block_index * block_size;

		bool all_the_same = true;

		for (int code = 0; code < block_size; code++)

			{

			block[code] = aTheIndex[block_start + code];

			if (block[code] != block[0])

				all_the_same = false;

			}

		// Try to find a match for it.

		int insert_at;

		if (all_the_same)

			trie_index[block_index] = block[0] | 0x8000;

		else

			{

			for (insert_at = 0; insert_at < n; insert_at++)

				{

				int entries = n - insert_at;

				if (entries > block_size)

					entries = block_size;

				int bytes = entries * sizeof(int);

				if (memcmp(block,aTrieData + insert_at,bytes) == 0)

					break;

				}

			memcpy(aTrieData + insert_at,block,block_size * sizeof(int));

			if (insert_at + block_size > n)

				n = insert_at + block_size;

			trie_index[block_index] = insert_at;

			}

		}

	if (aWrite)

		{

		write_table(trie_index,aTrie1Name,blocks,4,2,false,16,true);

		write_table(aTrieData,aTrie2Name,n,4,aOutputEntrySize,false,32,true);

		}

	delete [] trie_index;

	delete [] block;

	return blocks * 2 + n * aOutputEntrySize;

	}

// Write the best possible 2-level trie for all planes, trying block sizes of 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096 and 8192

// @return Data size in bytes.

int write_trie()

	{

	int byteCount = 0;

	for (int plane=0; plane<PlaneCount; plane++)

		{

		int best_data_size = 1 << 30;

		int best_bits = 0;

		int outputEntrySize = 2;

		char trie1Name[255];

		char trie2Name[255];

		sprintf(trie1Name, "ThePlane%02dTrieIndex1", plane);

		sprintf(trie2Name, "ThePlane%02dTrieIndex2", plane);

		int *theIndex = TheIndex + plane * 0x10000;

		int *trieData = trie_data + plane * 0x10000;

		for (int cur_bits = 3; cur_bits < 14; cur_bits++)

			{

			int cur_data_size = write_trie(outputEntrySize, cur_bits, false, theIndex, trieData, trie1Name, trie2Name);

			if (cur_data_size < best_data_size)

				{

				best_bits = cur_bits;

				best_data_size = cur_data_size;

				}

			}

		byteCount += write_trie(outputEntrySize, best_bits, true, theIndex, trieData, trie1Name, trie2Name);

		ThePlanesInReadType[plane].iCodesPerBlock = (TUint8) best_bits;

		ThePlanesInReadType[plane].iMaskForCodePoint = (TUint16) ((1 << (best_bits)) - 1);

		ThePlanesInReadType[plane].iMaskForBlock = (TUint16) (~(ThePlanesInReadType[plane].iMaskForCodePoint));

		}

	return byteCount;

	}

/*

Compare entries in the decompose table for the purpose of sorting them. The entries are indices

into the starting words of compositions stored in the composition buffer.

*/

int compare_decompositions(const void *p,const void *q)

	{

	// Get the indexes.

	TInt16 index1 = *((const TInt16 *)p);

	TInt16 index2 = *((const TInt16 *)q);

	// Compare the two composition strings.

	return TUnicode::Compare((TUint16 *)&CompositionBuffer[index1 + 2], CompositionBuffer[index1 + 1]*2,

							 (TUint16 *)&CompositionBuffer[index2 + 2], CompositionBuffer[index2 + 1]*2);

	}

// Write the output file.

void write_output()

	{

	int data_bytes = 0;

	// Write the comment at the top of the file

	fprintf(output_file, "// Copyright (c) 2007-2009 Nokia Corporation and/or its subsidiary(-ies).\n");

	fprintf(output_file, "// All rights reserved.\n");

	fprintf(output_file, "// This component and the accompanying materials are made available\n");

	fprintf(output_file, "// under the terms of the License \"Eclipse Public License v1.0\"\n");

	fprintf(output_file, "// which accompanies this distribution, and is available\n");

	fprintf(output_file, "// at the URL \"http://www.eclipse.org/legal/epl-v10.html\".\n");

	fprintf(output_file, "//\n");

	fprintf(output_file, "// Initial Contributors:\n");

	fprintf(output_file, "// Nokia Corporation - initial contribution.\n");

	fprintf(output_file, "//\n");

	fprintf(output_file, "// Contributors:\n");

	fprintf(output_file, "//\n");

	fprintf(output_file, "// Description:\n");

	fprintf(output_file,

			"// Unicode character information tables.\n"

			"// Written by the READTYPE program.\n"

			"// Please read the 'Unicode Character Data and Line Break data Update History.doc' file for detailed history of updates to this file.\n"

			"// This file was generated by the READTYPE tool using UCD 5.0.\n"

			"// The contents of this file were generated automatically. Please do not edit this manually.\n"

			"//\n"

			"//\n"

			"\n");

	// Write the directive to include the header file.

	fprintf(output_file,"#include <unicode.h>\n\n");

	// Export two variables for unicode.cpp.

	fprintf(output_file, "\n");

	fprintf(output_file, "// Declarations for tables held in unitable.cpp and used by unicode.cpp.\n");

	fprintf(output_file, "extern const TStandardUnicodeDataSet TheStandardUnicodeDataSet[];\n");

	fprintf(output_file, "extern const TUnicodePlane ThePlanes[17];\n\n\n");

	// Write the trie data.

	data_bytes += write_trie();

	// Write the character information table.

	fprintf(output_file,"static const TUnicodeData TheUnicodeData[] =\n\t{ // %d entries\n", Datas);

	int i;

	for (i = 0; i < Datas; i++)

		{

		fputc('\t',output_file);

		TheData[i].Write();

		if (i < Datas - 1)

			fputc(',',output_file);

		fprintf(output_file, "\t// 0x%X (%d)", i, i);

		fputc('\n',output_file);

		}

	fprintf(output_file,"\t};\n\n");

	data_bytes += Datas * sizeof(Data);

	// write plane properties

	fprintf(output_file, "const TUnicodePlane ThePlanes[%d] =\n\t{\n", PlaneCount);

	int plane;

	for (plane=0; plane<=16; plane++)

		{

		fprintf(output_file, "\t{%d, 0x%04X, 0x%04X }",

			ThePlanesInReadType[plane].iCodesPerBlock, ThePlanesInReadType[plane].iMaskForBlock, ThePlanesInReadType[plane].iMaskForCodePoint);

		if (plane < 16)

			fprintf(output_file, ",\n");

		}

	fprintf(output_file, "\n\t};\n\n");

	data_bytes += 5*PlaneCount;

	// Write a data structure referring to the trie data.

	fprintf(output_file,"const TStandardUnicodeDataSet TheStandardUnicodeDataSet[] =\n\t{ // %d entries\n", PlaneCount);

	for (plane=0; plane<=16; plane++)

		{

		fprintf(output_file,"\t{ ThePlane%02dTrieIndex1, ThePlane%02dTrieIndex2, TheUnicodeData }", plane, plane);

		if (plane < 16)

			fprintf(output_file, ",\n");

		}

	fprintf(output_file, "\n\t};\n\n");

	data_bytes += 12*PlaneCount;

	// Convert the fold table to lower case.

	for (i = 0; i < 256; i++)