/*

* Copyright (c) 2010 Nokia Corporation and/or its subsidiary(-ies).

* All rights reserved.

* This component and the accompanying materials are made available

* under the terms of "Eclipse Public License v1.0"

* which accompanies this distribution, and is available

* at the URL "http://www.eclipse.org/legal/epl-v10.html".

*

* Initial Contributors:

* Nokia Corporation - initial contribution.

*

* Contributors:

*

* Description: 

*

*/

// system library

#include "wchar.h"

#include <string>

#include <vector>

#include <sstream>

#include <iostream>

#include <glib.h>

// clucene 

#include "CLucene.h"

#include "CLucene/analysis/AnalysisHeader.h"

#include "CLucene/analysis/Analyzers.h"

// local libary

#include "thaianalysis.h"

#include "ngram.h"

#include "koreananalyzer.h"

#include "cjkanalyzer.h"

#include "cpixparsetools.h"

#include "prefixfilter.h"

// cpix internal

#include "customanalyzer.h"

#include "cpixanalyzer.h"

#include "analyzer.h"

#include "cluceneext.h"

#include "analyzerexp.h"

#include "indevicecfg.h"

#include "cpixexc.h"

#include "localization.h"

namespace Cpix {

	//

	// Following sections provide the glue code for connecting the 

	// analyzer definition syntax with analyzer, tokenizers and filter 

	// implementations. 

	//

	// The glue code is template heavy with the indent of providing 

	// automation for associating specific keywords with specific

	// analyzers, tokenizers and filters implementing corresponding 

	// CLucene abstractions. Additional classes are needed only if 

	// filters, tokenizers, etc. accept parameters.

	//

	// NOTE: To understand the analyzers, it is sufficient to understand

	// that an analyzer transforms characters stream into specific token streams 

	// (e.g. character stream 'foobarmetawords' can be transformed into token 

	// stream 'foo', 'bar' 'meta' 'words'). Analysis consist of two main

	// parts which are tokenization and filtering. Tokenization converts

	// the character stream into token stream (e.g. 'FoO bAr' -> 'FoO' 'bAr')

	// and filtering modifies the tokens (e.g. lowercase filtering 'FoO' -> 

	// 'foo', 'bAr' -> 'bar'). Analyzer as an object is responsible for

	// constructing a tokenizer and a sequence of filters to perform

	// these required tasks.  

	// 

	// See the documentation around TokenizerClassEntries and 

	// FilterClassEntries to see how implementations not taking parameters

	// can be easily added.  

	// 

	using namespace Cpix::AnalyzerExp;

// Safe assumption

#define MAX_LANGCODE_LENGTH 256

	class LocaleSwitchStreamFactory : public TokenStreamFactory {

	public: 

		LocaleSwitchStreamFactory(const AnalyzerExp::LocaleSwitch& sw, const wchar_t* config);

		~LocaleSwitchStreamFactory();

		virtual lucene::analysis::TokenStream* tokenStream(const wchar_t        * fieldName, 

														   lucene::util::Reader * reader);

		lucene::analysis::TokenStream* tokenStream(std::vector<std::wstring>& languages, 

												   const wchar_t            * fieldName, 

												   lucene::util::Reader     * reader);

	private: 

		std::map<std::wstring, CustomAnalyzer*> analyzers_;

		std::auto_ptr<CustomAnalyzer> default_;  

	};

	TokenStreamFactory::~TokenStreamFactory() {};

	LocaleSwitchStreamFactory::LocaleSwitchStreamFactory(const LocaleSwitch& sw, const wchar_t* config) {

		for (int i = 0; i < sw.cases().size(); i++) {

			const Case& cs = *sw.cases()[i];

			for (int j = 0; j < cs.cases().size(); j++) {

				std::wstring c = cs.cases()[j]; 

				if (analyzers_.count(c)) delete analyzers_[c]; 

				analyzers_[c] = new CustomAnalyzer(cs.piping(), config);

			}

		}

		default_.reset(new CustomAnalyzer(sw.def())); 

	}

	LocaleSwitchStreamFactory::~LocaleSwitchStreamFactory() {

		typedef std::map<std::wstring, CustomAnalyzer*>::iterator iter;

		for (iter i = analyzers_.begin(); i != analyzers_.end(); i++) {

			delete i->second;

		}

	}

	lucene::analysis::TokenStream* 

		LocaleSwitchStreamFactory::tokenStream(const wchar_t        * fieldName, 

											   lucene::util::Reader * reader) {

		std::vector<std::wstring> languages = 

				Localization::instance().getLanguageNames();

		return tokenStream(languages, fieldName, reader); 

	}

	lucene::analysis::TokenStream* 

		LocaleSwitchStreamFactory::tokenStream(std::vector<std::wstring>& languages, 

											   const wchar_t            * fieldName, 

											   lucene::util::Reader     * reader) {

		for (int i = 0; i < languages.size(); i++) {

			if ( analyzers_.count(languages[i]) ) {

				return analyzers_[languages[i]]->tokenStream( fieldName, reader );

			}

		}		

		return default_->tokenStream( fieldName, reader ); 

	}

	class DefaultTokenStreamFactory : public TokenStreamFactory {

	public:

		enum Target {

			NORMAL, 

			INDEXING,

			QUERY,

			PREFIX

		};

		DefaultTokenStreamFactory(const Invokation& invokation) {

			if (invokation.params().size() == 1) {

				const Identifier* id = dynamic_cast<const Identifier*>( invokation.params()[0] ); 

				if ( id ) {

					if ( id->id() == CPIX_ID_INDEXING ) {

						target_ = INDEXING;

					} else if ( id->id() == CPIX_ID_QUERY ) {

						target_ = QUERY;

					} else if ( id->id() == CPIX_ID_PREFIX ) {

						target_ = PREFIX;

					} else {

						THROW_CPIXEXC(L"Default analyzer does not accept %S for parameter", id->id().c_str());

					}

				} else {

					THROW_CPIXEXC(L"Default accepts only identifier as a parameter.");

				}

			} else if (invokation.params().size() > 1) {

				THROW_CPIXEXC(L"Default analyzer does not accept more than one parameter");

			} else {

				target_ = NORMAL;

			}

		}

		virtual lucene::analysis::TokenStream* tokenStream(const TCHAR          * fieldName, 

														   lucene::util::Reader * reader) {

			switch (target_) {

				case QUERY: 

					return Analysis::getQueryAnalyzer().tokenStream( fieldName, reader );

				case PREFIX: 

					return Analysis::getPrefixAnalyzer().tokenStream( fieldName, reader );

			}

			return Analysis::getDefaultAnalyzer().tokenStream( fieldName, reader );

		}

	private:

		Target target_;

	};

	/**

	 * Template class used to create CLucene tokenizers. Template

	 * parameter T must implement lucene::analysis::Tokenizer abstraction.  

	 */    

	template<class T>

	class TokenizerFactory : public TokenStreamFactory 

	{

	public:

		TokenizerFactory(const Invokation& invokation) {

			if (invokation.params().size() > 0) {

				THROW_CPIXEXC(L"Tokenizer %S does not accept parameters",

							  invokation.id().c_str());

			}

		}

		virtual lucene::analysis::TokenStream* tokenStream(const TCHAR          * /*fieldName*/, 

														   lucene::util::Reader * reader) {

			return _CLNEW T(reader); 

		}

	};

	template<>

    class TokenizerFactory<analysis::CjkNGramTokenizer> : public TokenStreamFactory 

    {   

    public:

        static const int DefaultNgramSize = 1;

        TokenizerFactory(const Invokation& invokation) {

            using namespace Cpix::AnalyzerExp;

            if (invokation.params().size() > 1) {

                THROW_CPIXEXC(L"Cjk Ngram tokenizer does not accept more than one parameter",

                              invokation.id().c_str());

            }

            if (invokation.params().size() == DefaultNgramSize) {

                IntegerLit* ngramSize = dynamic_cast<IntegerLit*>(invokation.params()[0]);

                if ( ngramSize ) {

                    ngramSize_ = ngramSize->value();

                } else {

                    THROW_CPIXEXC(L"Cjk Ngram tokenizer parameter must be an integer");

                }

            } else {

                ngramSize_ = 1;

            }

        }

        virtual lucene::analysis::TokenStream* tokenStream(const TCHAR          * /*fieldName*/, 

                                                           lucene::util::Reader * reader) {

            return _CLNEW analysis::CjkNGramTokenizer(reader, ngramSize_); 

        }

    private:

        int ngramSize_;

    };

	/**

	 * Template class wrapping CLucene analyzers. Template parameter T must 

	 * implement lucene::analysis::Analyzer abstraction.  

	 */    

	template<class T>

	class AnalyzerWrap : public TokenStreamFactory 

	{

	public:

		AnalyzerWrap(const Invokation& invokation) : analyzer_() {

			if (invokation.params().size() > 0) {

				THROW_CPIXEXC(L"Tokenizer %S does not accept parameters",

							  invokation.id().c_str());

			}

		}

		virtual lucene::analysis::TokenStream* tokenStream(const TCHAR          * fieldName, 

														   lucene::util::Reader * reader) {

			return analyzer_.tokenStream(fieldName, reader); 

		}

	private: 

		T analyzer_;

	};

	/**

	 * Template class associated with CLucene filter and a TokenStreamFactory. 

	 * Uses TokenStreamFactory to transform given character stream into tokenstream

	 * and then applies the given Clucene filter to the token stream. 

	 * The template parameter T must implement lucene::analysis::Filter abstraction.     

	 */    

	template<class T>

	class FilterFactory : public TokenStreamFactory 

	{

	public:

		FilterFactory(const Invokation& invokation, auto_ptr<TokenStreamFactory> factory) : factory_(factory) {

			if (invokation.params().size() > 0) {

				THROW_CPIXEXC(L"Filter %S does not accept parameters",

							  invokation.id().c_str());

			}

		}

		virtual lucene::analysis::TokenStream* tokenStream(const TCHAR          * fieldName, 

														   lucene::util::Reader * reader) {

			return _CLNEW T(factory_->tokenStream(fieldName, reader), true); 

		}

	private: 

		std::auto_ptr<TokenStreamFactory> factory_; 

	};

	/**

	 * Specialized Analyzer wrap for CLucene's PerFieldAnalyzer. Specialized

	 * template is needed because perfield analyzer accepts parameters

	 * (specific analyzers for different field plus default analyzer)

	 */

	template<>

	class AnalyzerWrap<lucene::analysis::PerFieldAnalyzerWrapper> : public TokenStreamFactory {

	public:

		AnalyzerWrap(const Switch& sw, const wchar_t* config) : analyzer_(0) {

			using namespace Cpt::Parser;

			using namespace lucene::analysis;

			analyzer_ = _CLNEW PerFieldAnalyzerWrapper(_CLNEW CustomAnalyzer(sw.def()));

			for (int i = 0; i < sw.cases().size(); i++) {

				const Case& cs = *sw.cases()[i];

				for (int j = 0; j < cs.cases().size(); j++) {

					analyzer_->addAnalyzer( cs.cases()[j].c_str(), _CLNEW CustomAnalyzer( cs.piping(), config ) );

				}

			}

		}

		virtual ~AnalyzerWrap() {

			_CLDELETE(analyzer_);

		}

		virtual lucene::analysis::TokenStream* tokenStream(const TCHAR          * fieldName, 

														   lucene::util::Reader * reader) {

			return analyzer_->tokenStream(fieldName, reader); 

		}

	private: 

		lucene::analysis::PerFieldAnalyzerWrapper* analyzer_;

	};

	/**

	 * Specialized StopFilter factory. Specialized filter is needed

	 * because StopFilter needs parameters (stop word list or a language) 

	 */

	template<>

	class FilterFactory<lucene::analysis::StopFilter> : public TokenStreamFactory 

	{

	public:

		FilterFactory(const Invokation& invokation,

					  auto_ptr<TokenStreamFactory> factory)

			:words_(0),  ownWords_(0), factory_(factory) {

			using namespace Cpt::Parser;

			if (invokation.params().size() == 1 && dynamic_cast<Identifier*>(invokation.params()[0])) {

				Identifier* id = dynamic_cast<Identifier*>(invokation.params()[0]);

				//cpix_LangCode lang; 

				if (id->id() == CPIX_WLANG_EN) {

					words_ = lucene::analysis::StopAnalyzer::ENGLISH_STOP_WORDS;

                } else if (id->id() == CPIX_WLANG_FR) {

                    words_ = analysis::NonEnglishStopWords::FRENCH_STOP_WORDS;

				} else {

					THROW_CPIXEXC(L"No prepared stopword list for language code '%S'",

								  id->id().c_str());

				}

			} else {

				ownWords_ = new wchar_t*[invokation.params().size()+1];

				memset(ownWords_, 0, sizeof(wchar_t*)*(invokation.params().size()+1)); 

				// FIXE: args may leak

				for (int i = 0; i < invokation.params().size(); i++) {

					StringLit* lit = dynamic_cast<StringLit*>(invokation.params()[i]);

					if (lit) {

						const wstring& str = lit->text(); 

						ownWords_[i] = new wchar_t[str.length()+1]; 

						wcscpy(ownWords_[i], str.c_str());

					} else {

						THROW_CPIXEXC(L"StopFilter accepts only language identifer or list of strings as a parameters.");

					}

				}

			}

		}

		virtual ~FilterFactory() { 

			if (ownWords_) {

				for (int i = 0; ownWords_[i]; i++) {

					delete[] ownWords_[i]; 

				}

				delete[] ownWords_;

			}

		}

		virtual lucene::analysis::TokenStream* tokenStream(const TCHAR          * fieldName, 

														   lucene::util::Reader * reader) {

			return _CLNEW lucene::analysis::StopFilter(factory_->tokenStream(fieldName, reader), true, ownWords_ ? const_cast<const wchar_t**>(ownWords_) : words_); 

		}

	private: 

		const wchar_t **words_;

		wchar_t **ownWords_; // owned

		std::auto_ptr<TokenStreamFactory> factory_; 

	};

	/**

	 * Specialized SnowballFilter factory is needed, because SnowballFilter

	 * accepts parameters (the language). 

	 */

	template<>

	class FilterFactory<lucene::analysis::SnowballFilter> : public TokenStreamFactory 

	{

	public:

		FilterFactory(const Invokation& invokation, 		

					  auto_ptr<TokenStreamFactory> factory)

			: factory_(factory) {

			using namespace Cpt::Parser;

			if (invokation.params().size() != 1 || !dynamic_cast<Identifier*>(invokation.params()[0])) {

				THROW_CPIXEXC(L"Snowball filter takes exactly one identifier as a parameter." );

			}

			Identifier* id = dynamic_cast<Identifier*>(invokation.params()[0]);

			if (id->id() == CPIX_WLANG_EN) {

				lang_ = cpix_LANG_EN; 

			} else {

				THROW_CPIXEXC(L"Language identifier %S is not supported for stemming",

							  id->id().c_str());

			}

		}

		virtual lucene::analysis::TokenStream* tokenStream(const TCHAR          * fieldName, 

														   lucene::util::Reader * reader) {

			return _CLNEW lucene::analysis::SnowballFilter(factory_->tokenStream(fieldName, reader), true, lang_); 

		}

	private: 

		cpix_LangCode lang_;

		std::auto_ptr<TokenStreamFactory> factory_; 

	};

	/**

	 * Specialized LengthFilter factory is needed, because length filter 

	 * accepts parameters (minimum length and maximum length)

	 */

	template<>

	class FilterFactory<lucene::analysis::LengthFilter> : public TokenStreamFactory 

	{

	public:

		FilterFactory(const Invokation& invokation, 

					  auto_ptr<TokenStreamFactory> factory) 

			: factory_(factory) {

			using namespace Cpt::Parser;

			if (invokation.params().size() != 2 || 

				!dynamic_cast<IntegerLit*>(invokation.params()[0]) || 

				!dynamic_cast<IntegerLit*>(invokation.params()[1])) {

				THROW_CPIXEXC("Length filter takes exactly two integer parameters");

			}

			min_ = dynamic_cast<IntegerLit*>(invokation.params()[0])->value();

			max_ = dynamic_cast<IntegerLit*>(invokation.params()[1])->value();

		}

		virtual lucene::analysis::TokenStream* tokenStream(const TCHAR          * fieldName, 

														   lucene::util::Reader * reader) {

			return _CLNEW lucene::analysis::LengthFilter(factory_->tokenStream(fieldName, reader), true, min_, max_ ); 

		}

	private: 

		int min_, max_;

		std::auto_ptr<TokenStreamFactory> factory_; 

	};

	/**

	 * Specialized PrefixGenerator factory is needed, because PrefixGenerator

	 * requires the max prefix size. 

	 */

	template<>

	class FilterFactory<PrefixGenerator> : public TokenStreamFactory 

	{

	public:

		FilterFactory(const Invokation& invokation, 

					  auto_ptr<TokenStreamFactory> factory) 

			: factory_(factory) {

			using namespace Cpt::Parser;

			if (invokation.params().size() != 1 || 

				!dynamic_cast<IntegerLit*>(invokation.params()[0])) {

				THROW_CPIXEXC("Prefix generator takes exactly one integer parameter");

			}

			maxPrefixLength_ = dynamic_cast<IntegerLit*>(invokation.params()[0])->value();

		}

		virtual lucene::analysis::TokenStream* tokenStream(const TCHAR          * fieldName, 

														   lucene::util::Reader * reader) {

			return _CLNEW PrefixGenerator(factory_->tokenStream(fieldName, reader), true, maxPrefixLength_ ); 

		}

	private: 

		int maxPrefixLength_;

		std::auto_ptr<TokenStreamFactory> factory_; 

	};

	/**

	 * Specialized PrefixFilter factory is needed, because prefix filter 

	 * accepts parameters (language set or prefixes)

	 */

	template<>

	class FilterFactory<analysis::PrefixFilter> : public TokenStreamFactory 

	{

	public:

		FilterFactory(const Invokation& invokation,

					  auto_ptr<TokenStreamFactory> factory)

			:	prefixes_(0),  ownPrefixes_(0), factory_(factory) {

			using namespace Cpt::Parser;

			if (invokation.params().size() == 1 && 

				dynamic_cast<Identifier*>(invokation.params()[0])) {

				Identifier* id = dynamic_cast<Identifier*>(invokation.params()[0]);

				//cpix_LangCode lang; 

				if (id->id() == CPIX_WLANG_HE) {

					prefixes_ = analysis::HebrewPrefixes;

				} else {

					THROW_CPIXEXC(L"No prepared prefix list for language code '%S'",

								  id->id().c_str());

				}

			} else {

				ownPrefixes_ = new wchar_t*[invokation.params().size()+1];

				memset(ownPrefixes_, 0, sizeof(wchar_t*)*(invokation.params().size()+1)); 

				// FIXE: args may leak

				for (int i = 0; i < invokation.params().size(); i++) {

					StringLit* lit = dynamic_cast<StringLit*>(invokation.params()[i]);

					if (lit) {

						const wstring& str = lit->text(); 

						ownPrefixes_[i] = new wchar_t[str.length()+1]; 

						wcscpy(ownPrefixes_[i], str.c_str());

					} else {

						THROW_CPIXEXC(L"PrefixFilter accepts only language identifer or list of strings as a parameters.");

					}

				}

			}

		}

		virtual ~FilterFactory() { 

			if (ownPrefixes_) {

				for (int i = 0; ownPrefixes_[i]; i++) {

					delete[] ownPrefixes_[i]; 

				}

				delete[] ownPrefixes_;

			}

		}

		virtual lucene::analysis::TokenStream* tokenStream(const TCHAR          * fieldName, 

														   lucene::util::Reader * reader) {

			return _CLNEW analysis::PrefixFilter(factory_->tokenStream(fieldName, reader), true, ownPrefixes_ ? const_cast<const wchar_t**>(ownPrefixes_) : prefixes_); 

		}

	private: 

		const wchar_t **prefixes_;

		wchar_t **ownPrefixes_; // owned

		std::auto_ptr<TokenStreamFactory> factory_;