/*
* Copyright (c) 2010 Nokia Corporation and/or its subsidiary(-ies).
* All rights reserved.
* This component and the accompanying materials are made available
* under the terms of "Eclipse Public License v1.0"
* which accompanies this distribution, and is available
* at the URL "http://www.eclipse.org/legal/epl-v10.html".
*
* Initial Contributors:
* Nokia Corporation - initial contribution.
*
* Contributors:
*
* Description: 
*
*/

#ifndef KOREANANALYZER_H_
#define KOREANANALYZER_H_

#include "Clucene.h"

#include "ngram.h"

#include "tinyanalysis.h"
#include "tinyutf16.h"
#include "tinyunicode.h"

namespace analysis 
{
    // Forward declarations

	/**
	 * Special Korean analyzer that is designed so, that Cpix can 
	 * update the result list, when each individual Jamu character is
	 * entered.
	 * 
	 * The analyzer tries to first convert given character stream into 
	 * a form, where all Jamu characters are composed into Hangul form.
	 * This means, that character sequences of form LV and LVT are 
	 * eliminated and replaced with hangul syllables (L is leading Jamu
	 * consonant, V is for vocal and T is for trailing consonant).  
	 * 
	 * The idea behind the analyzer is that it produces up to 3 alternative 
	 * tokens for each hangul syllabic. All of these alternatives are returned
	 * to be located at the same position. Let's have some Hangul syllabic H1
	 * consisting of Jamu characters so that H1=J1J2J3. If H2=J1J2, then first
	 * returned token is H1, second token is H2 and third token is J1. This 
	 * means, that when user enters H1, H2 or J1, the term H1 will be found. 
	 * Also, if user enters J1J2J3 or J1J2, term will be found, because
	 * J1J2J3 is automatically turned to H1 and J1J2 is turned to H2.
	 * 
	 * NOTE: This analyzer MUST NOT be used, when searching, because 
	 * CLuceneQueryParser will break, when it faces tokens with zero 
	 * increment. Use KoreanQueryAnalyzer for searching material indexed
	 * with this analyzer. 
	 */
    class KoreanTokenizer : public lucene::analysis::Tokenizer {
   
        public:
            
			/** Used to read from buffer */
            typedef tiny::cl::ReaderBuffer<512>::iterator buffer_iterator;

            /** Turns utf16 code points into unicode */
            typedef tiny::Utf16Iterator<buffer_iterator> utf16_iterator;
            
            /** Turns Jamu alphabets into Hangul syllables */
            typedef tiny::HangulIterator<utf16_iterator> iterator;
            
            KoreanTokenizer(lucene::util::Reader* reader);
            
            virtual bool next(lucene::analysis::Token* token);

        private:
            
            /** Jamu form of last consumed hangul syllable */
            wchar_t jamu_[4];
            
            /** offsets of last consumed hangul syllable  */
            int begin_, end_;
            
            /**
             * The amount of jamu characters left in buffer. 
             * If this is non-zero, hangul syllable is being processed. 
             */
            int state_;
            
            /**
             * Tiny CJK tokenizer is used to construct 1-grams out of 
             * chinese and japanese characters and to turn latin script
             * into terms.
             */
            TinyCjkTokenizer<iterator> t_;
            
            /** 512 byte buffer for storing characters read with reader */
            tiny::cl::ReaderBuffer<512> in_;
            
            /** 
             * Reads utf16 from in_ buffer, turns it into unicode and 
             * then composes jamu alphabets into hangul syllables. 
             */
            iterator i_;
            
    
    };
    
    /** Korean tokenizer plus lowercase filter */
    typedef TemplateAnalyzer1F<KoreanTokenizer, lucene::analysis::LowerCaseFilter> 
        KoreanAnalyzer;
    
    /**
     * Turns Jamu characters into Hangul syllables and generates 1-grams for
     * all Chinese, Korean and Japanese text. 
     */
    class KoreanQueryTokenizer : public lucene::analysis::Tokenizer {
   
		public:
    
		    /** Used to read from buffer */
			typedef tiny::cl::ReaderBuffer<512>::iterator buffer_iterator;

			/** Turns utf16 code points into unicode */
            typedef tiny::Utf16Iterator<buffer_iterator> utf16_iterator;
            
            /** Turns Hangul syllables into Jamu alphabets */
            typedef tiny::HangulIterator<utf16_iterator> iterator;
            
		public:
       
            KoreanQueryTokenizer( lucene::util::Reader* reader );
            
            virtual bool next( lucene::analysis::Token* token );            
    
		private:

            /** Buffer for storing characters read with reader */
            TinyCjkTokenizer<iterator> t_;
       
            /** Buffer for storing characters read with reader */
            tiny::cl::ReaderBuffer<512> in_;
            
            /** 
             * Reads utf16 from in_ buffer, turns it into unicode and 
             * then composes jamu alphabets into hangul syllables. 
             */
            iterator i_; 
           
    };

    /** Korean query analyzer plus lowercase filter */
    typedef TemplateAnalyzer1F<KoreanQueryTokenizer, lucene::analysis::LowerCaseFilter> 
        KoreanQueryAnalyzer;

}

#endif /* KOREANANALYZER_H_ */