/*

* Copyright (c) 2010 Nokia Corporation and/or its subsidiary(-ies).

* All rights reserved.

* This component and the accompanying materials are made available

* under the terms of "Eclipse Public License v1.0"

* which accompanies this distribution, and is available

* at the URL "http://www.eclipse.org/legal/epl-v10.html".

*

* Initial Contributors:

* Nokia Corporation - initial contribution.

*

* Contributors:

*

* Description: 

*

*/

// general utilities

#include "wchar.h"

#include <string>

#include <vector>

#include <sstream>

#include <iostream>

#include <fstream>

#include <algorithm>

// clucene

#include "CLucene.h"

// support

#include "cpixparsetools.h"

#include "cpixfstools.h"

// internal

#include "analyzer.h"

#include "cpixanalyzer.h"

#include "cpixexc.h"

#include "document.h"

#include "cluceneext.h"

#include "indevicecfg.h"

#include "initparams.h"

#include "thaianalysis.h"

#include "analyzerexp.h"

#include "customanalyzer.h"

#include "common/cpixlog.h"

namespace

{

    const char AGGR_NONFILEREADERPROXY_ERR[] 

    = "Aggregated reader field should be FileReaderProxy instance";

    const char AGGR_STREAMREADER_ERR[] 

    = "Aggregating streamValue-fields not implemented";

    const char THAI_LANGUAGE_FILE[] 

    = "thaidict.sm";

    const char ANALYZER_FILE[]

    = "analyzer.loc";

    const wchar_t DEFAULT_ANALYZER_CONFIG[]

        = L"default";

    const wchar_t QUERY_ANALYZER_CONFIG[]

        = L"query";

    const wchar_t PREFIX_ANALYZER_CONFIG[]

        = L"prefix";

//    const wchar_t CPIX_ANALYZER_FALLBACK[]

//    = CPIX_ANALYZER_STANDARD;

//

//    const wchar_t CPIX_PREFIX_ANALYZER_FALLBACK[]

//    = CPIX_TOKENIZER_LETTER L">" CPIX_FILTER_LOWERCASE;

}

namespace Cpix {

Analysis* Analysis::theInstance_ = NULL; 

	void Analysis::init(InitParams& ip) {

		// Init thai analysis with thai dictionary

		std::string thai( Cpt::appendpath(ip.getResourceDir(),

										  THAI_LANGUAGE_FILE) );

		if ( Cpt::filesize( thai.c_str() ) ) {

			analysis::InitThaiAnalysis(thai.c_str());

		} else {

			logMsg(CPIX_LL_WARNING,

				   "Thai dictionary could not be found. Thai analysis will NOT work.");

		}

		// Setup the analysis instance

		theInstance_ = new Analysis(ip);

	}

	Analysis::Analysis(InitParams& ip) 

	:	defaultAnalyzer_(),

	 	queryAnalyzer_(), 

		prefixAnalyzer_() {

		auto_ptr<AnalyzerExp::Piping> p = parse( Cpt::appendpath( ip.getResourceDir(), ANALYZER_FILE ) );

		defaultAnalyzer_.reset( new CustomAnalyzer( *p, DEFAULT_ANALYZER_CONFIG ) ); 

		queryAnalyzer_.reset( new CustomAnalyzer( *p, QUERY_ANALYZER_CONFIG ) ); 

		prefixAnalyzer_.reset( new CustomAnalyzer( *p, PREFIX_ANALYZER_CONFIG ) ); 

	}

	auto_ptr<AnalyzerExp::Piping> Analysis::parse(std::string path) {

		std::wifstream in(path.c_str());

		auto_ptr<AnalyzerExp::Piping> ret; 

		if ( in ) {

			// Reserve constant size buffer and populate it with definition

			//

			int filesize = Cpt::filesize(path.c_str()); 

			Cpt::auto_array<wchar_t> buf( new wchar_t[filesize+1] );

			in.read(buf.get(), filesize);

			buf.get()[filesize] = '\0'; 

			if ( !in.fail() ) {

				try {

					ret = AnalyzerExp::ParsePiping( buf.get() );

				} catch (...) {}

			} 

			in.close();

		} 

		if ( !ret.get() ) { 

			THROW_CPIXEXC("Analyzer definition not found. %s could not be opened. ", path.c_str()); 

		}

		return ret; 

	}

	void Analysis::shutdown() {

		analysis::ShutdownThaiAnalysis(); 

		delete theInstance_;

		theInstance_ = NULL; 

	}

	lucene::analysis::Analyzer& Analysis::getDefaultAnalyzer() {

		// TODO: Assert( theInstance_ );

		return *theInstance_->defaultAnalyzer_; 

	}

	lucene::analysis::Analyzer& Analysis::getQueryAnalyzer() {

		// TODO: Assert( theInstance_ );

		return *theInstance_->queryAnalyzer_; 

	}

	lucene::analysis::Analyzer& Analysis::getPrefixAnalyzer() {

		// TODO: Assert( theInstance_ );

		return *theInstance_->prefixAnalyzer_; 

	}

	PrefixGenerator::PrefixGenerator(

		lucene::analysis::TokenStream* in, 

		bool deleteTS, 

		size_t maxPrefixLength) 

	: 	TokenFilter(in, deleteTS),

	  	token_(), 

	  	prefixLength_(0),

	  	maxPrefixLength_(maxPrefixLength) {}

	PrefixGenerator::~PrefixGenerator() {

	}

	bool PrefixGenerator::next(lucene::analysis::Token* token) {

		token_.setPositionIncrement(0); 

		while (prefixLength_ == 0) {

			token_.setPositionIncrement(1); // default position increment

			if (!input->next(&token_)) {

				return false;

			}

			prefixLength_ = std::min(token_.termTextLength(), maxPrefixLength_);

		}

		// Clip token

		std::wstring clipped; 

		clipped = token_.termText();

		token_.setText(clipped.substr(0, prefixLength_).c_str());

		// Copy

		token->set(token_.termText(), token_.startOffset(), token_.endOffset(), token_.type());

		token->setPositionIncrement(token_.getPositionIncrement());

		// Reduce prefixLength_

		prefixLength_--;

		return true; 

	}

    AggregateFieldTokenStream::AggregateFieldTokenStream(lucene::analysis::Analyzer& analyzer, 

                                                         DocumentFieldIterator* fields) 

	: stream_(), analyzer_( analyzer ), reader_(), fields_( fields ) {

        getNextStream(); 

    }

    AggregateFieldTokenStream::~AggregateFieldTokenStream() {

        _CLDELETE( stream_ ); 

        delete fields_; 

    }

    bool AggregateFieldTokenStream::next(lucene::analysis::Token* token) {

        while ( stream_ ) {

            if ( stream_->next( token ) ) {

                return true;

            }

            getNextStream();

        }

        return false;

    }

    void AggregateFieldTokenStream::close() {

        if (stream_) stream_->close(); 

        _CLDELETE( stream_ ); 

        _CLDELETE( reader_ ); 

    }

    void AggregateFieldTokenStream::getNextStream()

    {

        using namespace lucene::document;

        using namespace lucene::util; 

        if ( stream_ ) stream_->close(); 

        _CLDELETE( stream_ ); 

        _CLDELETE( reader_ );

        Field* field = 0; 

        while (*fields_ && field == NULL)

            {

                field = (*fields_)++;

                if (!field->isAggregated()) 

                    {

						field = 0;

                    }

            }

        if (field) {

            if (field->stringValue() != NULL)

                {

                    reader_ = _CLNEW CL_NS(util)::StringReader(field->stringValue(),_tcslen(field->stringValue()),false);

                }

            else if (field->native().readerValue() != NULL)

                {

                    Reader* r = field->native().readerValue();

                    FileReaderProxy

                        * frp = 

                        dynamic_cast<FileReaderProxy*>(r);

                    if (frp == NULL)

                        {

                            _CLTHROWA(CL_ERR_IO, AGGR_NONFILEREADERPROXY_ERR);

                        }

                    else

                        {

                            reader_ = frp->clone();

                        }

                }

            else

                {

                    _CLTHROWA(CL_ERR_IO, AGGR_STREAMREADER_ERR);

                }

            if(field->isFreeText())

                {

                    using namespace lucene::analysis;

                    stream_ = _CLNEW standard::StandardTokenizer(reader_);

                    stream_ = _CLNEW standard::StandardFilter(stream_,true);

                    stream_ = _CLNEW LowerCaseFilter(stream_,true);

                }

            else

            stream_ = analyzer_.tokenStream( field->name(), reader_ ); 

        }

    }

    AggregateFieldAnalyzer::AggregateFieldAnalyzer(Cpix::Document& document, 

                                                   lucene::analysis::Analyzer& analyzer) 

	:	analyzer_(analyzer), document_(document)

    {

    }

    lucene::analysis::TokenStream* AggregateFieldAnalyzer::tokenStream(const TCHAR     * fieldName, 

                                                                       lucene::util::Reader * reader) {

        if ( wcscmp( fieldName, LCPIX_DEFAULT_FIELD ) == 0 ) {

            return new AggregateFieldTokenStream( analyzer_, document_.fields()); 

        } else if ( wcscmp( fieldName, LCPIX_DEFAULT_PREFIX_FIELD ) == 0 ) {

            return

				new PrefixGenerator(

					new AggregateFieldTokenStream( analyzer_, document_.fields()),

					true,

					OPTIMIZED_PREFIX_MAX_LENGTH);

        } else {

            return analyzer_.tokenStream( fieldName, reader ); 

        }

    }

    SystemAnalyzer::SystemAnalyzer(lucene::analysis::Analyzer* analyzer) : analyzer_(analyzer) {} 

    SystemAnalyzer::~SystemAnalyzer() { _CLDELETE(analyzer_); }

    lucene::analysis::TokenStream* SystemAnalyzer::tokenStream(const TCHAR      	* fieldName, 

                                                               lucene::util::Reader * reader) {

        using namespace lucene::analysis; 

        if ( wcscmp( fieldName, LCPIX_DEFAULT_FIELD ) == 0 ) {

            // Use standard analyzer without stop filter for this task

            TokenStream* ret = _CLNEW standard::StandardTokenizer(reader);

            ret = _CLNEW standard::StandardFilter(ret,true);

            ret = _CLNEW LowerCaseFilter(ret,true);

            return ret;

        } else if (wcscmp( fieldName, LCPIX_DOCUID_FIELD) == 0){

            // Use standard analyzer without stop filter for this task

            return  _CLNEW KeywordTokenizer(reader);

        } else if (wcscmp( fieldName, LCPIX_APPCLASS_FIELD )  == 0){

            // Use standard analyzer without stop filter for this task

            TokenStream* ret = _CLNEW WhitespaceTokenizer(reader);

            ret = _CLNEW LowerCaseFilter(ret,true);

            return ret;

        } else if (wcscmp( fieldName, LCPIX_MIMETYPE_FIELD ) == 0) {

            TokenStream* ret = _CLNEW KeywordTokenizer(reader);

            return ret;

        } else {

            return analyzer_->tokenStream( fieldName, reader ); 

        }									 

    }

}