/*

* Copyright (c) 2010 Nokia Corporation and/or its subsidiary(-ies).

* All rights reserved.

* This component and the accompanying materials are made available

* under the terms of "Eclipse Public License v1.0"

* which accompanies this distribution, and is available

* at the URL "http://www.eclipse.org/legal/epl-v10.html".

*

* Initial Contributors:

* Nokia Corporation - initial contribution.

*

* Contributors:

*

* Description: 

*

*/

#include "ngram.h"

#include "tinyunicode.h"

#include "tinyanalysis.inl"

namespace analysis {

	using namespace unicode; 

	int IsNonCjk(int c) {

		return iswalnum(c) && !IsCjk(c); 

	}

	CjkNGramTokenizer::CjkNGramTokenizer( 

			lucene::util::Reader* reader, 

			int gramSize ) 

		: lucene::analysis::Tokenizer(reader),

          t_( gramSize ),

		  in_( *reader ),

	      i_( buffer_iterator( in_ ) ){

	}

	bool CjkNGramTokenizer::next( lucene::analysis::Token* token ) {

		using namespace tiny;

		Token<iterator> t = t_.consume(i_);

        if ( t ) {

            t.copyTo( token );

            return true; 

        } 

		return false; 

	}

	JamuNGramTokenizer::JamuNGramTokenizer( lucene::util::Reader* reader, 

                                            int gramSize ) 

    : lucene::analysis::Tokenizer( reader ),

       t_( gramSize ),

       in_( *reader ), 

       i_( utf16_iterator( buffer_iterator( in_ ) ) ) {}

    bool JamuNGramTokenizer::next( lucene::analysis::Token* token ) {

        using namespace tiny;

        Token<iterator> t = t_.consume(i_);

        if ( t ) {

            t.copyTo( token );

            return true; 

        } 

        return false; 

    }

}