/*

* Copyright (c) 2010 Nokia Corporation and/or its subsidiary(-ies).

* All rights reserved.

* This component and the accompanying materials are made available

* under the terms of "Eclipse Public License v1.0"

* which accompanies this distribution, and is available

* at the URL "http://www.eclipse.org/legal/epl-v10.html".

*

* Initial Contributors:

* Nokia Corporation - initial contribution.

*

* Contributors:

*

* Description: 

*

*/

#include <map>

#include <memory>

#include <vector>

#include <queue>

#include <iostream>

#include <sstream>

#include "CLucene.h"

#include "CLucene/queryParser/MultiFieldQueryParser.h"

#include "cpixtools.h"

#include "cpixexc.h"

#include "cpixsearch.h"

#include "cpixidxdb.h"

#include "idxdb.h"

#include "cpixhits.h"

#include "initparams.h"

#include "cpixutil.h"

#include "cpixdoc.h"

#include "ifieldfilter.h"

#include "cluceneext.h"

#include "rotlogger.h"

#include "idxdbmgr.h"

#include "document.h"

#include "cpixmaindefs.h"

#include "analyzer.h"

#include "common/cpixlog.h"

namespace

{

    /**

     * Process a filter by its id. The filter id field is not

     * (anymore) on the document.

     *

     * NOTE: Recent changes allow filter's to install new filters, so

     * they need to be exhaustively and iteratively processed as long

     * as there is anything to process.

     */

    bool processFilter(Cpix::Document * doc,

                       const wchar_t              * filterId)

    {

        using namespace Cpix;

        bool

            goOn = true;

        std::wstring

            dummy;

        while (goOn && filterId != NULL)

            {

                FieldFilterSentry

                    ffs(FieldFilterPool::getFieldFilter(filterId));

                goOn = ffs.get()->process(doc);

                if (goOn)

                    {

                        filterId = NULL;

                        filterId = doc->get(LCPIX_FILTERID_FIELD);

                        if (filterId != NULL)

                            {

                                // re-point filterId: same content but

                                // not destroyed when the field is

                                // removed

                                dummy = filterId;

                                filterId = dummy.c_str();

                                // remove the field

                                doc->removeField(LCPIX_FILTERID_FIELD);

                            }

                    }

            }

        return goOn;

    }

    /**

     * Process a filter, by its id, if it is defined, removing the

     * given filter id field.

     */

    bool processFilterIfNecessary(Cpix::Document * doc)

    {

        bool

            rv = true;

        const wchar_t

            * filterId = doc->get(LCPIX_FILTERID_FIELD);

        if (filterId != NULL)

            {

                std::wstring

                    wfid(filterId);

                doc->removeField(LCPIX_FILTERID_FIELD);

                rv = processFilter(doc,

                                   wfid.c_str());

            }

        return rv;

    }

}

// GetTerms implementation details

namespace {

    // Following definitions are needed for memory management and sorting of 

    // getTerms operations: 

    // Document entry definition. First integer contains the document frequence

    typedef std::pair<int, lucene::document::Document*> DocumentEntry;

    // Comparation method for sorting document entries

    /*bool operator<(DocumentEntry first, DocumentEntry second) {

        return first.first < second.first; 

    }

*/

    typedef std::vector<DocumentEntry> DocumentEntryVector;

    // Cleans out documents in case of exceptions

    class DocumentEntryQueue : public std::priority_queue<DocumentEntry>

    {

    public:

        DocumentEntryQueue() : std::priority_queue<DocumentEntry>() {}

        ~DocumentEntryQueue() {

            while (!empty()) 

                {

                    delete top().second; 

                    pop(); 

                }

        }

    };

}

Cpix::IHits * GetTerms(lucene::index::IndexReader * reader,

                       const wchar_t              * field,

                       const wchar_t              * wildcard,

                       const wchar_t 		  * appclassPrefix, 

                       int                          maxHits)

{

    using namespace std; 

    using namespace lucene::document; 

    using namespace lucene::index;

    using namespace lucene::search;

    if (field == NULL || wildcard == NULL || reader == NULL)

        {

            THROW_CPIXEXC("Null args to ::GetTerms");

        }

    /* TEMP

    logDbgMsg("::GetTerms BEGIN %S:%S",

              field,

              wildcard);

    */

    // Check that wilcard string is valid, WildcardTermEnum will panic otherwise

    const wchar_t* sindex = wcschr( wildcard, 

                                    LUCENE_WILDCARDTERMENUM_WILDCARD_STRING );

    const wchar_t* cindex = wcschr( wildcard, 

                                    LUCENE_WILDCARDTERMENUM_WILDCARD_CHAR );

    if (!sindex && !cindex)

        {

            THROW_CPIXEXC("missing wildcard in term '%S'",

                          wildcard); 

        }

    typedef std::vector<int> IntVector;

    typedef std::greater<int> GreaterInt;

    DocumentEntryQueue documents; 

    std::priority_queue<int, IntVector, GreaterInt> docFreqList; 

    // Setup wildcard term enumeration

    Term

        * term = _CLNEW Term( field, wildcard ); // increases reference

    // Appclass filtering infra

    std::auto_ptr<TermDocs> docs;

    FieldCacheAuto * cache=NULL; 

    // Prepare for filtering only if appclass filter is set

    if ( appclassPrefix ) 

        {

            docs.reset( reader->termDocs() );

            cache = FieldCache::DEFAULT->getStrings( reader, 

                                                     LCPIX_UNTOKENIZED_APPCLASS_FIELD ); // not owned.

        }

    /* TEMP

    logDbgMsg("::GetTerms 0 (%p)",

              cache);

    */

    try 

        {

            WildcardTermEnum terms( reader, term ); // increases reference

            do {

                Term

                    * t = terms.term(false);

                if (t == NULL)

                    {

                        break;

                    }

                // First step: Measure document frequence

                int 

                    docFreq = 0; 

                if ( appclassPrefix && cache ) 

                    {

                        // If appclass filter is defined, document

                        // frequence has to be counted from documents

                        // matching the filter

                        docs->seek(t); 

                        while (docs->next()) {

                            int doc = docs->doc();

                            const wchar_t* docField = cache->stringArray[doc];

                            if (docField) {

                                int i = 0; 

                                for (;appclassPrefix[i]; i++) {

                                    if (docField[i] != appclassPrefix[i]) break;

                                }

                                if (!appclassPrefix[i]) {

                                    docFreq++; 

                                }

                            }

                        }

                    } else {

                    // All documents go

                    docFreq = terms.docFreq();

                }

                if (docFreq > 0) 

                    {

                        // Extract information out of current term. Document

                        // frequence needs to be converted into as string

                        const wchar_t

                            * termText = t->text();

                        if (documents.size() < maxHits || docFreq > docFreqList.top()) 

                            {   // filter out all documents with low document frequence 

                                wostringstream toDocFreqText;

                                toDocFreqText<<docFreq;

                                // setup and populate

                                auto_ptr<Document>

                                    doc(new Document());

                                auto_ptr<Field>

                                    newField(new Field(LTERM_TEXT_FIELD, 

                                                       termText, 

                                                       Field::STORE_YES | Field::INDEX_NO));

                                doc->add(*newField.get()); 

                                newField.release();

                                newField.reset(new Field(LTERM_DOCFREQ_FIELD, 

                                                         toDocFreqText.str().c_str(), 

                                                         Field::STORE_YES | Field::INDEX_NO));

                                doc->add(*newField.get());

                                newField.release();

                                documents.push( DocumentEntry( docFreq, doc.get() ) ); 

                                doc.release();

                                // maintain knowledge of the last document frequence

                                if (documents.size() >= maxHits) docFreqList.pop();

                                docFreqList.push(docFreq); 

                            } 

                    }

            } while (terms.next());

        }

    catch (...)

        {

            // who knows if auto_ptr is compatible with _CL??? macros

            // - doing it manually

            _CLDECDELETE( term );

            throw;

        }

    // TEMP logDbgMsg("::GetTerms 1");

    _CLDECDELETE( term );

    // extract the sorted documents into hit document list safely

    auto_ptr<Cpix::HitDocumentList> 

        hits(new Cpix::HitDocumentList); 

    while (!documents.empty() && maxHits--) 

        {

            // Pick documents one by one. This will reverse the order

            Document

                * doc = documents.top().second; 

            documents.pop(); // releases ownership

            hits->add( doc ); // grants ownership

        }

    // TEMP logDbgMsg("::GetTerms END");

    return hits.release(); 

}

namespace Cpix

{

    using namespace lucene::analysis;

    using namespace lucene::document;

    using namespace lucene::index;

    using namespace lucene::queryParser;

    using namespace lucene::search; 

    using namespace lucene::store; 

    using namespace lucene::util;

    //////////////////////////////////////////////////////

    //

    //  InsertBuf

    //

    InsertBuf::InsertBuf(size_t maxDocs)

        : maxDocs_(maxDocs),

          writer_(NULL),

          ramDir_(NULL)

    {

        ;

    }

    void InsertBuf::close()

    {

        getRidOfWriter();

        if (ramDir_ != NULL)

            {

                Impl::CallCloseLogging(*ramDir_);

                _CLDECDELETE(ramDir_);

                ramDir_ = NULL;

            }

    }

    bool InsertBuf::isEmpty() const

    {

        return ramDir_ == NULL;

    }

    lucene::store::TransactionalRAMDirectory * InsertBuf::getRAMDir()

    {

        if (ramDir_ == NULL)

            {

                THROW_CPIXEXC(PL_ERROR "accessing NULL ramDir");

            }

        getRidOfWriter();

        return ramDir_;

    }

    InsertBuf::~InsertBuf()

    {

        close();

    }

    void InsertBuf::add(Cpix::Document             * doc,

                        lucene::analysis::Analyzer * analyzer)

    {

        lazyInit();

        std::wstring

            docUid(doc->get(LCPIX_DOCUID_FIELD));

        writer_->addDocument(&doc->native(),

                             analyzer);

    }

    int32_t InsertBuf::deleteDocuments(lucene::index::Term * term)

    {

        int32_t

            rv = 0;

        if (!isEmpty()) // ramDir_ is not NULL but writer_ may be

            {

                using namespace lucene::index;

                getRidOfWriter();

                // no we do the deletion on the ramDir_ using

                // a temporary reader

                Impl::cl_auto_ptr<IndexReader>

                    tmpReader(IndexReader::open(ramDir_,

                                                false)); // don't close

                rv = tmpReader->deleteDocuments(term);

                // close()-ing tmpReader implied by cl_auto_ptr

                tmpReader.reset();

            }

        return rv;

    }

    size_t InsertBuf::size()

    {

        size_t

            rv = 0;

        if (!isEmpty()) // ramDir_ not NULL but writer_ may be

            {

                lazyInit();

                std::vector<std::string>

                    files;

                ramDir_->list(&files);

                rv = writer_->ramSizeInBytes(); 

                std::vector<std::string>::iterator 

                    i = files.begin(),

                    end = files.end();

                for (; i != end; ++i) 

                    {

                        rv += ramDir_->fileLength(i->c_str());

                    }

            }

        return rv; 

    }

    void InsertBuf::lazyInit()

    {

        bool

            create = false;

        if (ramDir_ == NULL)

            {

                ramDir_ = _CLNEW TransactionalRAMDirectory();

                create = true;

            }

        if (writer_ == NULL)

            {

                writer_ = new IndexWriter(ramDir_,

                                          NULL,

                                          create,

                                          false);

                writer_->setMaxBufferedDocs(maxDocs_);

            }

    }

    void InsertBuf::getRidOfWriter()

    {

        if (writer_ != NULL)

            {

                Impl::CallCloseLogging(*writer_);

                // writer_->close();

                delete writer_;

                writer_ = NULL;

            }

    }

    //////////////////////////////////////////////////////

    //

    //  VersionedReader

    //

    void VersionedReader::open(const char * clIdxPath,

                               Cpt::Mutex & dirMutex)

    {

        using namespace lucene::index;

        logDbgMsg("VersionedReader::open (%s) BEGIN",

                  clIdxPath);

        Cpt::SyncRegion

            sr(mutex_);

        Cpt::SyncRegion

            sr2(dirMutex);

        if (reader_ != NULL)