/*

* Copyright (c) 2010 Nokia Corporation and/or its subsidiary(-ies).

* All rights reserved.

* This component and the accompanying materials are made available

* under the terms of "Eclipse Public License v1.0"

* which accompanies this distribution, and is available

* at the URL "http://www.eclipse.org/legal/epl-v10.html".

*

* Initial Contributors:

* Nokia Corporation - initial contribution.

*

* Contributors:

*

* Description: 

*

*/

#include "koreananalyzer.h"

#include "tinyanalysis.inl"

namespace analysis {

    KoreanTokenizer::KoreanTokenizer(lucene::util::Reader* reader) :

        begin_(0),

        end_(0),

        state_(0),

        t_(1), 

        in_(*reader),

        i_(iterator(utf16_iterator(in_.begin()))) {}

    bool KoreanTokenizer::next(lucene::analysis::Token* token) {

        using namespace unicode;

        using namespace tiny;

        if ( state_ ) {

            jamu_[state_--] = '\0';

            const wchar_t buf[] = { ComposeJamu(jamu_), '\0' }; 

            token->set( buf, begin_, end_);

            token->setPositionIncrement(0);

            return true;

        } else {

            while ( *i_ ) {

                if ( IsHangulSyllable( *i_ ) ) {

                    DecomposeHangul( IteratorOutput<wchar_t*>(jamu_), *i_ );

                    state_ = wcslen(jamu_)-1;

					wchar_t buf[] = {*i_, '\0'};

                    begin_ = i_;

                    end_ = ++i_;

                    token->set( buf, begin_, end_ );

                    return true;

                } else {

                    Token<iterator> t = t_.consume( i_ );

                    if ( t ) {

                        t.copyTo(token);

                        return true;

                    }

                }

                ++i_;

            }

            return false;

        }

    }

	KoreanQueryTokenizer::KoreanQueryTokenizer( lucene::util::Reader* reader ) 

    : lucene::analysis::Tokenizer( reader ),

       t_( 1 ),

       in_( *reader ), 

       i_( utf16_iterator( buffer_iterator( in_ ) ) ) {}

    bool KoreanQueryTokenizer::next( lucene::analysis::Token* token ) {

        using namespace tiny;

        Token<iterator> t = t_.consume(i_);

        if ( t ) {

            t.copyTo( token );

            return true; 

        } 

        return false; 

    }   

}