/*

* Copyright (c) 2010 Nokia Corporation and/or its subsidiary(-ies).

* All rights reserved.

* This component and the accompanying materials are made available

* under the terms of "Eclipse Public License v1.0"

* which accompanies this distribution, and is available

* at the URL "http://www.eclipse.org/legal/epl-v10.html".

*

* Initial Contributors:

* Nokia Corporation - initial contribution.

*

* Contributors:

*

* Description: 

*

*/

#include "CLucene/StdHeader.h"

#include "CJKAnalyzer.h"

CL_NS_DEF2(analysis,cjk)

CL_NS_USE(analysis)

CL_NS_USE(util)

const TCHAR* CJKTokenizer::tokenTypeSingle = _T("single");

const TCHAR* CJKTokenizer::tokenTypeDouble = _T("double");

CJKTokenizer::CJKTokenizer(Reader* in):

	Tokenizer(in)

{

	tokenType = Token::defaultType;

	offset = 0;

	bufferIndex = 0;

	dataLen = 0;

	preIsTokened = false;

	ignoreSurrogates = true;

}

bool CJKTokenizer::next(Token* token){

	while (true) {

		/** how many character(s) has been stored in buffer */

		int32_t length = 0;

		/** the position used to create Token */

		int32_t start = offset;

		while (true) {

			/** current character */

			clunichar c;

			int charlen = 1;

			offset++;

			if (bufferIndex >= dataLen) {

				dataLen = input->read(ioBuffer);

				bufferIndex = 0;

			}

			if (dataLen == -1) {

				if (length > 0) {

					if (preIsTokened == true) {

						length = 0;

						preIsTokened = false;

					} else {

						offset--;

					}

					break;

				} else {

					offset--;

					return false;

				}

			} else {

				//get current character

				c = ioBuffer[bufferIndex++];

			}

			//to support surrogates, we'll need to convert the incoming utf16 into

			//ucs4(c variable). however, gunichartables doesn't seem to classify

			//any of the surrogates as alpha, so they are skipped anyway...

			//so for now we just convert to ucs4 so that we dont corrupt the input.

			if ( c >= 0xd800 || c <= 0xdfff ){

				clunichar c2 = ioBuffer[bufferIndex];

				if ( c2 >= 0xdc00 && c2 <= 0xdfff ){

					bufferIndex++;

					offset++;

					charlen=2;

					c = (((c & 0x03ffL) << 10) | ((c2 & 0x03ffL) <<  0)) + 0x00010000L;

				}

			}

			//if the current character is ASCII or Extend ASCII

			if ((c <= 0xFF) //is BASIC_LATIN

				|| (c>=0xFF00 && c<=0xFFEF) //ascii >0x74 cast to unsigned...

			   ) {

				if (c >= 0xFF00) {

					//todo: test this... only happens on platforms where char is signed, i think...

					/** convert  HALFWIDTH_AND_FULLWIDTH_FORMS to BASIC_LATIN */

					c -= 0xFEE0;

				}

				// if the current character is a letter or "_" "+" "#"

				if (_istalnum(c) || ((c == '_') || (c == '+') || (c == '#')) ) {

					if (length == 0) {

						// "javaC1C2C3C4linux" <br>

						//      ^--: the current character begin to token the ASCII

						// letter

						start = offset - 1;

					} else if (tokenType == tokenTypeDouble) {

						// "javaC1C2C3C4linux" <br>

						//              ^--: the previous non-ASCII

						// : the current character

						offset-=charlen;

						bufferIndex-=charlen;

						tokenType = tokenTypeSingle;

						if (preIsTokened == true) {

							// there is only one non-ASCII has been stored

							length = 0;

							preIsTokened = false;

							break;

						} else {

							break;

						}

					}

					// store the LowerCase(c) in the buffer

					buffer[length++] = _totlower((TCHAR)c);

					tokenType = tokenTypeSingle;

					// break the procedure if buffer overflowed!

					if (length == LUCENE_MAX_WORD_LEN) {

						break;

					}

				} else if (length > 0) {

					if (preIsTokened == true) {

						length = 0;

						preIsTokened = false;

					} else {

						break;

					}

				}

			} else {

				// non-ASCII letter, eg."C1C2C3C4"

				if ( _istalpha(c) || (!ignoreSurrogates && c >= 0x10000) ) {

					if (length == 0) {

						start = offset - 1;

						if ( c < 0x00010000L )

							buffer[length++] = (TCHAR)c;

						else{

							clunichar ucs4 = c - 0x00010000L;

							buffer[length++] = (TCHAR)((ucs4 >> 10) & 0x3ff) | 0xd800;

							buffer[length++] = (TCHAR)((ucs4 >>  0) & 0x3ff) | 0xdc00;

						}

						tokenType = tokenTypeDouble;

					} else {

						if (tokenType == tokenTypeSingle) {

							offset-=charlen;

							bufferIndex-=charlen;

							//return the previous ASCII characters

							break;

						} else {

							if ( c < 0x00010000L )

								buffer[length++] = (TCHAR)c;

							else{

								clunichar ucs4 = c - 0x00010000L;

								buffer[length++] = (TCHAR)((ucs4 >> 10) & 0x3ff) | 0xd800;

								buffer[length++] = (TCHAR)((ucs4 >>  0) & 0x3ff) | 0xdc00;

							}

							tokenType = tokenTypeDouble;

							if (length >= 2) {

								offset-=charlen;

								bufferIndex-=charlen;

								preIsTokened = true;

								break;

							}

						}

					}

				} else if (length > 0) {

					if (preIsTokened == true) {

						// empty the buffer

						length = 0;

						preIsTokened = false;

					} else {

						break;

					}

				}

			}

		}

		if (length > 0) {

			buffer[length]='\0';

			token->set(buffer,start, start+length, tokenType);

			return true; 

		} else if (dataLen == -1) {

			offset--; 

			return false;

		}

	}

}

TokenStream* CJKAnalyzer::tokenStream(const TCHAR* fieldName, CL_NS(util)::Reader* reader) {

	return new CJKTokenizer(reader);

}

CL_NS_END2