/*------------------------------------------------------------------------------

* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team

* 

* Distributable under the terms of either the Apache License (Version 2.0) or 

* the GNU Lesser General Public License, as specified in the COPYING file.

------------------------------------------------------------------------------*/

#include "clucene/stdheader.h"

#include "Analyzers.h"

#include "clucene/util/stringbuffer.h"

CL_NS_USE(util)

CL_NS_DEF(analysis)

CharTokenizer::CharTokenizer(Reader* in) :

	Tokenizer(in),

	offset(0),

	bufferIndex(0),

	dataLen(0),

	ioBuffer(NULL)

{

	buffer[0]=0;

}

TCHAR CharTokenizer::normalize(const TCHAR c) const 

{ 

	return c; 

}

bool CharTokenizer::next(Token* token){

	int32_t length = 0;

	int32_t start = offset;

	while (true) {

		TCHAR c;

		offset++;

		if (bufferIndex >= dataLen) {

			dataLen = input->read(ioBuffer, LUCENE_IO_BUFFER_SIZE);

			if (dataLen == -1)

				dataLen = 0;

			bufferIndex = 0;

		}

		if (dataLen <= 0 ) {

			if (length > 0)

				break;

			else

				return false;

		}else

			c = ioBuffer[bufferIndex++];

		if (isTokenChar(c)) {                       // if it's a token TCHAR

			if (length == 0)			  // start of token

				start = offset-1;

			buffer[length++] = normalize(c);          // buffer it, normalized

			if (length == LUCENE_MAX_WORD_LEN)		  // buffer overflow!

				break;

		} else if (length > 0)			  // at non-Letter w/ chars

			break;					  // return 'em

	}

	buffer[length]=0;

	token->set( buffer, start, start+length);

	return true;

}

bool LetterTokenizer::isTokenChar(const TCHAR c) const {

	return _istalpha(c)!=0;

}

TCHAR LowerCaseTokenizer::normalize(const TCHAR chr) const {

	return _totlower(chr);

}

bool WhitespaceTokenizer::isTokenChar(const TCHAR c)  const{

	return _istspace(c)==0; //(return true if NOT a space)

}

TokenStream* WhitespaceAnalyzer::tokenStream(const TCHAR* , Reader* reader) {

	return _CLNEW WhitespaceTokenizer(reader);

}

TokenStream* SimpleAnalyzer::tokenStream(const TCHAR* , Reader* reader) {

	return _CLNEW LowerCaseTokenizer(reader);

}

bool LowerCaseFilter::next(Token* t){

	if (!input->next(t))

		return false;

 	stringCaseFold( t->_termText );

	return true;

}

StopFilter::StopFilter(TokenStream* in, bool deleteTokenStream, const TCHAR** stopWords):

	TokenFilter(in, deleteTokenStream),

	table(_CLNEW CLSetList<const TCHAR*>(false)),

	ownTable(true)

{

	fillStopTable( table,stopWords );

}

StopFilter::~StopFilter()

{

	if (ownTable) { 

		_CLDELETE( table ); 

	}

}

void StopFilter::fillStopTable(CLSetList<const TCHAR*>* stopTable,

								  const TCHAR** stopWords) {

	for (int32_t i = 0; stopWords[i]!=NULL; i++)

		stopTable->insert(stopWords[i]);

}

bool StopFilter::next(Token* token) {

	// return the first non-stop word found

	while (input->next(token)){

		if (table->find(token->_termText)==table->end()){

			return true;

		}

	}

	// reached EOS -- return nothing

	return false;

}

StopAnalyzer::StopAnalyzer():stopTable(false)

{

	StopFilter::fillStopTable(&stopTable,ENGLISH_STOP_WORDS);

}

StopAnalyzer::~StopAnalyzer()

{

}

StopAnalyzer::StopAnalyzer( const TCHAR** stopWords) {

	StopFilter::fillStopTable(&stopTable,stopWords);

}

TokenStream* StopAnalyzer::tokenStream(const TCHAR* , Reader* reader) {

	return _CLNEW StopFilter(_CLNEW LowerCaseTokenizer(reader),true, &stopTable);

}

const TCHAR* StopAnalyzer::ENGLISH_STOP_WORDS[]  = 

{

	_T("a"), _T("an"), _T("and"), _T("are"), _T("as"), _T("at"), _T("be"), _T("but"), _T("by"),

	_T("for"), _T("if"), _T("in"), _T("into"), _T("is"), _T("it"),

	_T("no"), _T("not"), _T("of"), _T("on"), _T("or"), _T("s"), _T("such"),

	_T("t"), _T("that"), _T("the"), _T("their"), _T("then"), _T("there"), _T("these"),

	_T("they"), _T("this"), _T("to"), _T("was"), _T("will"), _T("with"), NULL

};

PerFieldAnalyzerWrapper::PerFieldAnalyzerWrapper(Analyzer* defaultAnalyzer):

    analyzerMap(true,true)

{

    this->defaultAnalyzer = defaultAnalyzer;

}

PerFieldAnalyzerWrapper::~PerFieldAnalyzerWrapper(){

    analyzerMap.clear();

    _CLDELETE(defaultAnalyzer);

}

void PerFieldAnalyzerWrapper::addAnalyzer(const TCHAR* fieldName, Analyzer* analyzer) {

    analyzerMap.put(STRDUP_TtoT(fieldName), analyzer);

}

TokenStream* PerFieldAnalyzerWrapper::tokenStream(const TCHAR* fieldName, Reader* reader) {

    Analyzer* analyzer = (fieldName==NULL?defaultAnalyzer:analyzerMap.get(fieldName));

    if (analyzer == NULL) {

      analyzer = defaultAnalyzer;

    }

    return analyzer->tokenStream(fieldName, reader);

}

bool ISOLatin1AccentFilter::next(Token* token){

	if ( input->next(token) ){

		int32_t l = token->termTextLength();

		const TCHAR* chars = token->termText();

		bool doProcess = false;

		for (int32_t i = 0; i < l; ++i) {

			#ifdef _UCS2

			if ( chars[i] >= 0xC0 && chars[i] <= 0x178 ) {

			#else

			if ( (chars[i] >= 0xC0 && chars[i] <= 0xFF) || chars[i] < 0 ) {

			#endif

				doProcess = true;

				break;

			}

		}

		if ( !doProcess ) {

			return true;

		}

		StringBuffer output(l*2);

		for (int32_t j = 0; j < l; j++) {

			#ifdef _UCS2

			TCHAR c = chars[j];

			#else

			unsigned char c = chars[j];

			#endif

			switch (c) {

				case 0xC0 : // À

				case 0xC1 : // Á

				case 0xC2 : // Â

				case 0xC3 : // Ã

				case 0xC4 : // Ä

				case 0xC5 : // Å

					output.appendChar('A');

					break;

				case 0xC6 : // Æ

					output.append(_T("AE"));

					break;

				case 0xC7 : // Ç

					output.appendChar('C');

					break;

				case 0xC8 : // È

				case 0xC9 : // É

				case 0xCA : // Ê

				case 0xCB : // Ë

					output.appendChar('E');

					break;

				case 0xCC : // Ì

				case 0xCD : // Í

				case 0xCE : // Î

				case 0xCF : // Ï

					output.appendChar('I');

					break;

				case 0xD0 : // Ð

					output.appendChar('D');

					break;

				case 0xD1 : // Ñ

					output.appendChar('N');

					break;

				case 0xD2 : // Ò

				case 0xD3 : // Ó

				case 0xD4 : // Ô

				case 0xD5 : // Õ

				case 0xD6 : // Ö

				case 0xD8 : // Ø

					output.appendChar('O');

					break;

				case 0xDE : // Þ

					output.append(_T("TH"));

					break;

				case 0xD9 : // Ù

				case 0xDA : // Ú

				case 0xDB : // Û

				case 0xDC : // Ü

					output.appendChar('U');

					break;

				case 0xDD : // Ý

					output.appendChar('Y');

					break;

				case 0xE0 : // à

				case 0xE1 : // á

				case 0xE2 : // â

				case 0xE3 : // ã

				case 0xE4 : // ä

				case 0xE5 : // å

					output.appendChar('a');

					break;

				case 0xE6 : // æ

					output.append(_T("ae"));

					break;

				case 0xE7 : // ç

					output.appendChar('c');

					break;

				case 0xE8 : // è

				case 0xE9 : // é

				case 0xEA : // ê

				case 0xEB : // ë

					output.appendChar('e');

					break;

				case 0xEC : // ì

				case 0xED : // í

				case 0xEE : // î

				case 0xEF : // ï

					output.appendChar('i');

					break;

				case 0xF0 : // ð

					output.appendChar('d');

					break;

				case 0xF1 : // ñ

					output.appendChar('n');

					break;

				case 0xF2 : // ò

				case 0xF3 : // ó

				case 0xF4 : // ô

				case 0xF5 : // õ

				case 0xF6 : // ö

				case 0xF8 : // ø

					output.appendChar('o');

					break;

				case 0xDF : // ß

					output.append(_T("ss"));

					break;

				case 0xFE : // þ

					output.append(_T("th"));

					break;

				case 0xF9 : // ù

				case 0xFA : // ú

				case 0xFB : // û

				case 0xFC : // ü

					output.appendChar('u');

					break;

				case 0xFD : // ý

				case 0xFF : // ÿ

					output.appendChar('y');

					break;

				#ifdef _UCS2

				case 0x152 : // Œ

					output.append(_T("OE"));

					break;

				case 0x153 : // œ

					output.append(_T("oe"));

					break;

				case 0x178 : // Ÿ

					output.appendChar('Y');

					break;

				#endif

				default :

					output.appendChar(c);

					break;

			}

		}

		token->setText(output.getBuffer());

		return true;

	}

	return false;

}

TokenStream* KeywordAnalyzer::tokenStream(const TCHAR* fieldName, CL_NS(util)::Reader* reader){

    return _CLNEW KeywordTokenizer(reader);

}

KeywordTokenizer::KeywordTokenizer(CL_NS(util)::Reader* input, int bufferSize):

	Tokenizer(input)

{

    this->done = false;

	if ( bufferSize < 0 )

		this->bufferSize = DEFAULT_BUFFER_SIZE;

}

KeywordTokenizer::~KeywordTokenizer(){

}

bool KeywordTokenizer::next(Token* token){

    if (!done) {

      done = true;

	  int32_t rd;

	  const TCHAR* buffer=0;

      while (true) {

        rd = input->read(buffer, bufferSize);

        if (rd == -1) 

			break;

		token->growBuffer(token->_termTextLen +rd+1);

		int32_t cp = rd;

		if ( token->_termTextLen + cp > token->bufferLength() )

			cp = token->bufferLength() -  token->_termTextLen;

		_tcsncpy(token->_termText+token->_termTextLen,buffer,cp);

		token->_termTextLen+=rd;

      }

	  token->_termText[token->_termTextLen]=0;

	  token->set(token->_termText,0,token->_termTextLen);

	  return true;

    }

    return false;

}

LengthFilter::LengthFilter(TokenStream* in, int _min, int _max):

    TokenFilter(in)

{

    this->_min = _min;

    this->_max = _max;

}

LengthFilter::LengthFilter(TokenStream* in, bool deleteTs, int _min, int _max):

    TokenFilter(in, deleteTs)

{

    this->_min = _min;

    this->_max = _max;

}

bool LengthFilter::next(Token* token)

{

    // return the first non-stop word found

    while ( input->next(token) )

    {

        size_t len = token->termTextLength();

        if (len >= _min && len <= _max)

            return true;

        // note: else we ignore it but should we index each part of it?

    }

    // reached EOS -- return null

    return false;

}

CL_NS_END