/*
* Copyright (c) 2010 Nokia Corporation and/or its subsidiary(-ies).
* All rights reserved.
* This component and the accompanying materials are made available
* under the terms of "Eclipse Public License v1.0"
* which accompanies this distribution, and is available
* at the URL "http://www.eclipse.org/legal/epl-v10.html".
*
* Initial Contributors:
* Nokia Corporation - initial contribution.
*
* Contributors:
*
* Description: 
*
*/

#ifndef THAIWORDFILTER_H_
#define THAIWORDFILTER_H_

#include <memory>

#include "cpixstrtools.h"

#include "Clucene.h"

namespace analysis 
{
	// Forward declarations
	class BreakIterator;

	
	/**
	 * Thai analysis uses compiled dictionary and it needs to 
	 * know the dictionary's location in the file system. This dictionary 
	 * is then loaded when thai analysis is used for first time.
	 * The dictionary is kept in memory until shutdown.
	 */ 
	void InitThaiAnalysis(const char* thaiDataFile);
	
	/**
	 * Releases the thai dictionary, if loaded to memory 
	 */
	void ShutdownThaiAnalysis(); 
	
	/**
	 * Dictionary based token filtering. Reads tokens from the token stream. 
	 * If token beginning with Thai character is encountered, it breaks
	 * the token into a number of thai word tokens based on the dictionary
	 * with the longest matching algorithm.
	 */
	class ThaiWordFilter : public lucene::analysis::TokenFilter 
	{
	public:
	
		ThaiWordFilter( lucene::analysis::TokenStream* input, bool deleteTs );
		
		~ThaiWordFilter(); 
		
		bool next(lucene::analysis::Token* token); 
	
	private: 

		std::auto_ptr<BreakIterator> breaks_;
		
		lucene::analysis::Token thaiToken_;
		
	};
	
	/**
	 * Analyzer for thai language. Uses StandardAnalyzer to make preliminary
	 * tokenization and ThaiWordFilter to split Thai sentence tokens 
	 * into a number of Thai word tokens.  
	 */
	class ThaiAnalyzer : public lucene::analysis::Analyzer 
	{
	public:
	
		ThaiAnalyzer(); 
		
	public: 
		
		lucene::analysis::TokenStream* tokenStream(const wchar_t* fieldName, 
												   lucene::util::Reader* reader);
		
	private: 
		
		CL_NS(util)::CLSetList<const TCHAR*> stopWords_;

	};

}

#endif /* THAIWORDFILTER_H_ */