/*

* Copyright (c) 2010 Nokia Corporation and/or its subsidiary(-ies).

* All rights reserved.

* This component and the accompanying materials are made available

* under the terms of "Eclipse Public License v1.0"

* which accompanies this distribution, and is available

* at the URL "http://www.eclipse.org/legal/epl-v10.html".

*

* Initial Contributors:

* Nokia Corporation - initial contribution.

*

* Contributors:

*

* Description: 

*

*/

#ifndef NGRAM_H_

#define NGRAM_H_

#include "tinyanalysis.h"

#include "tinyunicode.h"

#include "clutil.h"

namespace analysis {

	/**

	 * Returns true, if the character is non-cjk letter

	 */

    int IsNonCjk(int c);

    /**

     * TinyCjkTokenizer. Contains tiny analysis classes, that are 

     * used to turn Chinese, Korean and Japanese into 1-grams, while

     * using letter analyzer for other kinds of text (western, cyrillic,

     * etc.)   

     * 

     * @tparam I the iterator, that is used to read characters

     */

    template<typename I>

    struct TinyCjkTokenizer {

    	/** Deals with cjk */

        tiny::NGramTokenizer<I> cjk_;

    	/** Letter tokenizer for space separated language */

        tiny::CustomTokenizer<I> noncjk_; 

    	/** Combines cjk with noncjk */

        tiny::PairTokenizer<I> pair_;

        /** Moves forward, if tokenization fails */

        tiny::RelaxedTokenizer<I> t_;

        /** 

         * Constructs the tiny cjk tokenizer with given ngram size

         *

         * @param ngramsize cjk text is treated with n-gram analyzer of this size   

         */

        TinyCjkTokenizer(int ngramsize) 

        : cjk_(ngramsize, &unicode::IsCjk),

          noncjk_(&IsNonCjk),

          pair_(cjk_, noncjk_),

          t_(pair_) {}          

        /**

         * Consumes a token from given iterator. Returns n-grams

         * for cjk text, letter tokenized words for non-cjk text. 

         * Always returns something unless EOS has been reached.

         */

        inline tiny::Token<I> consume(I& i) {

            return t_.consume(i);

        }

    };

    /**

     * Constructs n-grams of Chinese, Korean and Japanese text. Uses

     * letter tokenization for other kinds of texts. 

     */

	class CjkNGramTokenizer : public lucene::analysis::Tokenizer {

		public: 

			/** Reads from buffer */

			typedef tiny::cl::ReaderBuffer<512>::iterator buffer_iterator;

			/** Turns utf16 to unicode */

	        typedef tiny::Utf16Iterator<buffer_iterator> iterator;

		public:

			CjkNGramTokenizer( lucene::util::Reader* reader, int gramSize );

			virtual bool next( lucene::analysis::Token* token );			

		private:

			/** The tokenizer */

            TinyCjkTokenizer<iterator> t_;

            /** Buffer */

			tiny::cl::ReaderBuffer<512> in_;

			/** Reads utf16 from buffer and transforms it to unicode*/

			iterator i_; 

	};

	/**

	 * The great difference of this class compared to CJK ngram, 

	 * that it decomposes Hangul syllables into Hangul Jamu letters.

	 * 

	 * This analyzer appeared to have bad performance in testing. 

	 */

	class JamuNGramTokenizer : public lucene::analysis::Tokenizer {

       public:

           typedef tiny::cl::ReaderBuffer<512>::iterator buffer_iterator;

           typedef tiny::Utf16Iterator<buffer_iterator> utf16_iterator;

           typedef tiny::JamuIterator<utf16_iterator> iterator;

       public:

           JamuNGramTokenizer( lucene::util::Reader* reader, int gramSize );

           virtual bool next( lucene::analysis::Token* token );            

       private:

           TinyCjkTokenizer<iterator> t_;

           tiny::cl::ReaderBuffer<512> in_;

           iterator i_; 

    };

	// Analyzers using the tokenizers

	//   * Provided mainly for testing

	//

	/** CjkNGramTokenizer plus lowercase filter */

    typedef TemplateAnalyzer1A1F<CjkNGramTokenizer, int, lucene::analysis::LowerCaseFilter> 

        CjkNGramAnalyzer;

	/** JamuNGramTokenizer plus lowercase filter */

    typedef TemplateAnalyzer1A1F<JamuNGramTokenizer, int, lucene::analysis::LowerCaseFilter> 

        JamuNGramAnalyzer;

}

#endif /* NGRAM_H_ */