/*
* Copyright (c) 2010 Nokia Corporation and/or its subsidiary(-ies).
* All rights reserved.
* This component and the accompanying materials are made available
* under the terms of "Eclipse Public License v1.0"
* which accompanies this distribution, and is available
* at the URL "http://www.eclipse.org/legal/epl-v10.html".
*
* Initial Contributors:
* Nokia Corporation - initial contribution.
*
* Contributors:
*
* Description: 
*
*/

#include <unistd.h>

#include "cpixexc.h"

#include "common/cpixlog.h"

#include "idxdbdelta.h"
#include "fwdtypes.h"
#include "iidxdb.h"
#include "idxdbmgr.h"


namespace
{

    /**
     * TODO comment
     */
    bool isSuffixOf(const char * little,
                    const char * big)
    {
        bool
            rv = false;

        size_t
            littleSize = strlen(little),
            bigSize = strlen(big);

        if (littleSize <= bigSize)
            {
                const char
                    * littleCur = little + littleSize - 1,
                    * bigCur = big + bigSize - 1;

                for (; 
                     littleCur >= little 
                         && bigCur >= big
                         && *littleCur == *bigCur; 
                     --littleCur, --bigCur)
                    {
                        ;
                    }

                rv = (littleCur == (little - 1));
            }

        return rv;
    }



    /**
     * Removes everything under a certain path except an
     * exception-path.
     */
    class RemoverWithException : public Cpt::IFileVisitor
    {
    private:
        int          success_;
        const char * startPath_;
        const char * exception_;

        std::string  newerMarkerFile_;

    public:
        /**
         * Arguments are not copied, only stored, so make sure
         * those paths exists while this instance is alive.
         */
        RemoverWithException(const char * path,
                             const char * exception)
            : success_(0),
              startPath_(path),
              exception_(exception)
        {
            ;
        }

            
        //
        // from interface Cpt::IFileVisitor
        //
        int success() const
        {
            return success_;
        }


        virtual bool visitFile(const char * path)
        {
            if (isSuffixOf(Cpix::Impl::VERSION_MARKER_SUFFIX, path))
                {
                    // if path is a version marker file then defer its
                    // deletion
                    newerMarkerFile_ = path;
                }
            else
                {
                    removePath(path);
                }

            return success_ == 0;
        }


        virtual DirVisitResult visitDirPre(const char * path)
        {
            DirVisitResult
                rv = IFV_CONTINUE;

            if (exception_ != NULL
                && strcmp(path, exception_) == 0)
                {
                    rv = IFV_SKIPTHISDIR;
                }

            return rv;
        }


        virtual bool visitDirPost(const char * path)
        {
            bool
                rv = true;

            bool
                doesNotMatchStartPath = 
                startPath_ == NULL || strcmp(path, startPath_) != 0;

            bool
                doesNotMatchException =
                exception_ == NULL || strcmp(path, exception_) != 0;
                
            if (doesNotMatchStartPath && doesNotMatchException)
                {
                    rv = visitFile(path);
                }

            if (startPath_) {
            if (rv
                && strcmp(path, startPath_) == 0
                && newerMarkerFile_.length() > 0)
                {
                    // the version marker file, if any, as the last
                    // item
                    removePath(newerMarkerFile_.c_str());
                    rv = success_ == 0;
                }
            }

            return rv;
        }
            

    private:
        //
        // private members
        //
        void removePath(const char * path)
        {
            if (success_ != 0)
                {
                    logMsg(CPIX_LL_ERROR,
                           PL_ERROR "removing %s, when already broken.",
                           path);
                }

            // 'remove' works for files and directories
            Cpt_EINTR_RETRY(success_,remove(path));

            if (success_ != 0)
                {
                    logMsg(CPIX_LL_ERROR,
                           "RemoverWithException::removePath Failed, errno: %d, %s",
                           errno,
                           path);
                }
        }
    };




}



namespace Cpix
{

    namespace Impl
    {
        /////////////////////////////////////////////////////
        // IInsertBuf
        //
        IInsertBuf::~IInsertBuf()
        {
            ;
        }


        /////////////////////////////////////////////////////
        // IIdxReader
        //
        IIdxReader::~IIdxReader()
        {
            ;
        }


        /////////////////////////////////////////////////////
        // CpixPaths
        //
        CpixPaths::CpixPaths(const char * basePath)
            : oneDir_(basePath),
              oneDirNewerMarkerFile_(basePath),
              otherDir_(basePath),
              otherDirNewerMarkerFile_(basePath),
              delsMarkerFile_(basePath),
              ibufCommittingDir_(basePath),
              ibufCommittedDir_(basePath),
              mergingDir_(basePath),
              idxDirsExistence_(IDXE_NEITHER),
              curDir_(NULL),
              newDir_(NULL)
        {
            using namespace Cpt;

            pathappend(oneDir_,
                       ONE_COMPLETE_SUBDIR);
            pathappend(oneDirNewerMarkerFile_,
                       ONE_SUBDIR_NEWER_MARKER);
            pathappend(otherDir_,
                       OTHER_COMPLETE_SUBDIR);
            pathappend(otherDirNewerMarkerFile_,
                       OTHER_SUBDIR_NEWER_MARKER);
            pathappend(delsMarkerFile_,
                       DELS_COMMITTED_MARKER);
            pathappend(ibufCommittingDir_,
                       IBUF_COMMITTING_SUBDIR);
            pathappend(ibufCommittedDir_,
                       IBUF_COMMITTED_SUBDIR);
            pathappend(mergingDir_,
                       MERGING_SUBDIR);

            bool
                oneDirExists = directoryexists(oneDir_.c_str()),
                otherDirExists = directoryexists(otherDir_.c_str());

            if (oneDirExists && otherDirExists)
                {
                    idxDirsExistence_ = IDXE_BOTH;

                    bool
                        oneDirNewer = true;

                    // first, we try to decide using the marker
                    if (isfile(oneDirNewerMarkerFile_.c_str()))
                        {
                            oneDirNewer = true;
                        }
                    else if (isfile(otherDirNewerMarkerFile_.c_str()))
                        {
                            oneDirNewer = false;
                        }
                    else
                        {
                            // if there is no marker (failed to create
                            // it?), we try to look at modification dates
                            time_t
                                oneModTime = filemodified(oneDir_.c_str()),
                                otherModTime = filemodified(otherDir_.c_str());
                            
                            // the later one is the newer, if they
                            // happen to have the exact same date
                            // (second granurality allows it), then we
                            // just pick the other, but that's
                            // arbitrary (the best we can do)
                            oneDirNewer = oneModTime > otherModTime;

                            if (oneModTime == otherModTime)
                                {
                                    logMsg(CPIX_LL_WARNING,
                                           "No marker file and identical mod dates in cpix dir %s: picking arbitrary version - information may be lost",
                                           basePath);
                                }
                        }

                    if (oneDirNewer)
                        {
                            curDir_ = &otherDir_;
                            newDir_ = &oneDir_;
                        }
                    else
                        {
                            curDir_ = &oneDir_;
                            newDir_ = &otherDir_;
                        }
                }
            else if (oneDirExists)
                {
                    idxDirsExistence_ = IDXE_ONE;

                    curDir_ = &oneDir_;
                    newDir_ = &otherDir_;
                }
            else if (otherDirExists)
                {
                    idxDirsExistence_ = IDXE_ONE;

                    curDir_ = &otherDir_;
                    newDir_ = &oneDir_;
                }
            else
                {
                    curDir_ = &oneDir_;
                    newDir_ = &otherDir_;
                }
        }


        bool CpixPaths::HasValidIndex(const std::string & path)
        {
            bool
                rv = false;

            try
                {
                    using namespace std;
                    using namespace lucene::index;

                    cl_auto_ptr<IndexReader>
                        reader(IndexReader::open(path.c_str()));

                    // close()-ing reader is implied by cl_auto_ptr dtor
                    reader.reset();
                    rv = true;
                }
            catch (...)
                {
                    ;
                }

            return rv;
        }



        void CpixPaths::RemoveExcept(const char * path,
                                     const char * exception)
        {
            RemoverWithException
                rwe(path,
                    exception);

            Cpt::traverse(path,
                          &rwe);

            if (rwe.success() != 0)
                {
                    logMsg(CPIX_LL_ERROR,
                           "Could not cleanup '%s' (leaving '%s' out)",
                           path,
                           exception);
                    THROW_CPIXEXC("Could not cleanup '%s' (leaving '%s' out)",
                                  path,
                                  exception);
                }
        }



        bool CpixPaths::IsCleanCpixDir(const char * basePath)
        {
            using namespace Cpt;
            using namespace std;

            string
                oneDir(basePath),
                oneDirNewerMarkerFile(basePath),
                otherDir(basePath),
                otherDirNewerMarkerFile(basePath),
                delsMarkerFile(basePath),
                ibufCommittingDir(basePath),
                ibufCommittedDir(basePath),
                mergingDir(basePath);

            pathappend(oneDir,
                       ONE_COMPLETE_SUBDIR);
            pathappend(oneDirNewerMarkerFile,
                       ONE_SUBDIR_NEWER_MARKER);
            pathappend(otherDir,
                       OTHER_COMPLETE_SUBDIR);
            pathappend(otherDirNewerMarkerFile,
                       OTHER_SUBDIR_NEWER_MARKER);
            pathappend(delsMarkerFile,
                       DELS_COMMITTED_MARKER);
            pathappend(ibufCommittingDir,
                       IBUF_COMMITTING_SUBDIR);
            pathappend(ibufCommittedDir,
                       IBUF_COMMITTED_SUBDIR);
            pathappend(mergingDir,
                       MERGING_SUBDIR);

            bool
                oneDirExists = directoryexists(oneDir.c_str()),
                otherDirExists = directoryexists(otherDir.c_str());

            bool
                rv = false;

            if (!oneDirExists || !otherDirExists)
                {
                    rv =   !directoryexists(ibufCommittingDir.c_str())
                        && !directoryexists(ibufCommittedDir.c_str())
                        && !directoryexists(mergingDir.c_str())
                        && !isfile(oneDirNewerMarkerFile.c_str())
                        && !isfile(otherDirNewerMarkerFile.c_str())
                        && !isfile(delsMarkerFile.c_str());
                }

            return rv;
        }
        



        /////////////////////////////////////////////////////
        // Path suffixes
        //
        const char ONE_COMPLETE_SUBDIR[]       = "_0";
        const char ONE_SUBDIR_NEWER_MARKER[]   = "_0newer.mrk";
        const char OTHER_COMPLETE_SUBDIR[]     = "_1";
        const char OTHER_SUBDIR_NEWER_MARKER[] = "_1newer.mrk";
        const char DELS_COMMITTED_MARKER[]     = "_dels.mrk";
        const char IBUF_COMMITTING_SUBDIR[]    = "_ibuftmp";
        const char IBUF_COMMITTED_SUBDIR[]     = "_ibufdone";
        const char MERGING_SUBDIR[]            = "_mergetmp";
        const char VERSION_MARKER_SUFFIX[]     = "newer.mrk";



        /////////////////////////////////////////////////////
        // Delta function implementations
        //
        void IdxDbDelta::RecoverReader(const char * basePath,
                                       Cpt::Mutex & idxMutex,
                                       IIdxReader * reader)
        {
            using namespace std;

            string
                idxDbPath;

            { // SYNC
                Cpt::SyncRegion
                    sr(idxMutex);

                CommitStage
                    commitStage = GuessCommitStage_(basePath);

                MergeOnDisk_(basePath,
                             NULL, // no insert buffer
                             commitStage,
                             DefaultCommitStageAction(),
                             idxDbPath);

                CleanupCPixDir_(basePath,
                                idxDbPath.c_str());
            } // SYNC

            if (reader != NULL)
                {
                    reader->open(idxDbPath.c_str(),
                                 idxMutex);
                }
        }



        void IdxDbDelta::CommitToDisk(const char                  * basePath,
                                      IInsertBuf                  * insertBuf,
                                      IIdxReader                  & reader,
                                      bool                          reRead,
                                      Cpt::Mutex                  & idxMutex)
        {
            CommitToDisk_(basePath,
                          insertBuf,
                          reader,
                          reRead,
                          idxMutex,
                          DefaultCommitStageAction());
        }


        void DefaultCommitStageAction::operator()(CommitStage commitStage)
        {
            ;
            /* TMP
            logDbgMsg("DefaultCommitStageAction:: - stage at %d",
                      commitStage);
            */
        }


        CommitStage GuessCommitStage_(const char * basePath)
        {
            using namespace lucene::index;
            using namespace std;
            using namespace Cpt;

            CpixPaths
                paths(basePath);

            CommitStage
                rv;

            if (paths.idxDirsExistence_ == CpixPaths::IDXE_BOTH)
                {
                    rv = CS_MERGED;
                }
            else if (paths.idxDirsExistence_ == CpixPaths::IDXE_NEITHER)
                {
                    int
                        result = removeunder(basePath);

                    if (result != 0)
                        {
                            THROW_CPIXEXC("Could not clean dir '%s'",
                                          basePath);
                        }

                    result = mkdirs(paths.curDir_->c_str(),
                                    0766);

                    if (result != 0)
                        {
                            THROW_CPIXEXC("Could not create idx dir '%s'",
                                          paths.curDir_->c_str());
                        }

                    IdxDbMgr::RecreateFsClIdx(paths.curDir_->c_str());

                    rv = CS_COMPLETE;
                }
            else if (CpixPaths::HasValidIndex(paths.mergingDir_))
                {
                    rv = CS_MERGING;
                }
            else if (directoryexists(paths.ibufCommittedDir_.c_str()))
                {
                    rv = CS_IBUF_COMMITTED;
                }
            else if (CpixPaths::HasValidIndex(paths.ibufCommittingDir_.c_str()))
                {
                    rv = CS_IBUF_COMMITTING;
                }
            else if (isfile(paths.delsMarkerFile_.c_str()))
                {
                    rv = CS_DELS_COMMITTED;
                }
            else
                {
                    rv = CS_COMPLETE;
                }

            return rv;
        }



        void CommitInsertBuffer_(CpixPaths                   & paths,
                                 IInsertBuf                  * insertBuf)
        {
            using namespace Cpt;
            using namespace lucene::index;

            if (insertBuf != NULL && !insertBuf->isEmpty())
                {
                    // create target directory (a temp directory)
                    int
                        result = mkdirs(paths.ibufCommittingDir_.c_str(),
                                        0766);

                    if (result != 0)
                        {
                            THROW_CPIXEXC("Could not create dir '%s'",
                                          paths.ibufCommittingDir_.c_str());
                        }

                    // create index writer to write to target dir
                    lucene::analysis::SimpleAnalyzer
                        dummyAnalyzer;

                    cl_auto_ptr<IndexWriter>
                        target(new IndexWriter(paths.ibufCommittingDir_.c_str(),
                                               &dummyAnalyzer,
                                               true,   // create
                                               true));
                    
                    Directory
                        * dirsToAdd[] = {
                        insertBuf->getRAMDir(),
                        NULL
                    };

                    // merge (add) ramDir's contents to target
                    target->addIndexes(dirsToAdd);

                    // close()-ing target - implied by cl_auto_ptr dtor
                    target.reset();

                    // clean up all insert state in insert buffer
                    //
                    // NOTE: this close() is to be called only if
                    // previous operations were successful
                    insertBuf->close();
                }
        }



        void RenameInsertBufferCommittingDir_(CpixPaths & paths)
        {
            using namespace Cpt;

            if (directoryexists(paths.ibufCommittingDir_.c_str()))
                {
                    int
                        result = rename(paths.ibufCommittingDir_.c_str(),
                                        paths.ibufCommittedDir_.c_str());

                    if (result != 0)
                        {
                            THROW_CPIXEXC("Could not rename '%s' to '%s': %d",
                                          paths.ibufCommittingDir_.c_str(),
                                          paths.ibufCommittedDir_.c_str(),
                                          result);
                        }
                }
        }


        void MergeParts_(CpixPaths & paths)
        {
            using namespace Cpt;
            using namespace lucene::store;

            bool
                iBufDirExists = directoryexists(paths.ibufCommittedDir_.c_str());

            if (isfile(paths.delsMarkerFile_.c_str())
                || iBufDirExists)
                {
                    int
                        result = mkdirs(paths.mergingDir_.c_str(),
                                        0766);
                    if (result != 0)
                        {
                            THROW_CPIXEXC("Could not create '%s'",
                                          paths.mergingDir_.c_str());
                        }

                    // readers to merge, at maximum 2 (current index
                    // and the insertbuffer, if any), plus the third
                    // item is just a placefolder for NULL
                    Directory
                        * dirsToMerge[] = {
                        NULL,
                        NULL,
                        NULL
                    };

                    const char
                        * path = paths.curDir_->c_str();
                    const bool
                        dontCreate = false;

                    Directory
                        * curIdxDir = FSDirectory::getDirectory(path,
                                                                dontCreate),
                        * iBufDir = NULL;
                    
                    dirsToMerge[0] = curIdxDir;
                    
                    try
                        {
                            
                            if (iBufDirExists)
                                {
                                    path = paths.ibufCommittedDir_.c_str();
                                    
                                    iBufDir = FSDirectory::getDirectory(path,
                                                                        dontCreate);
                                    dirsToMerge[1] = iBufDir;
                                }
                            
                            lucene::analysis::SimpleAnalyzer
                                dummyAnalyzer;
                            
                            cl_auto_ptr<IndexWriter>
                                writer(new IndexWriter(paths.mergingDir_.c_str(),
                                                       & dummyAnalyzer,
                                                       true,    // create
                                                       true));
                            
                            writer->addIndexes(dirsToMerge);

                            // close()-ing writer is implied by cl_auto_ptr
                            writer.reset();

                            // see comments below (in catch clause)
                            CallCloseLogging(*curIdxDir);
                            _CLDECDELETE(curIdxDir);
                            if (iBufDir != NULL)
                                {
                                    CallCloseLogging(*iBufDir);
                                    _CLDECDELETE(iBufDir);
                                }
                        }
                    catch (...)
                        {
                            // we release manually these instances,
                            // but why don't we use std::auto_ptr?
                            //
                            // These FSDirectory instances are aquired
                            // by the FSDirectory::getDirectory call,
                            // which creates them with clucene's own
                            // _CLNEW macro. Now, Directory instances
                            // are also reference counted, so if they
                            // were created by clucene, they should
                            // also be released the way clucene
                            // releases them, and that's _CLDECDELETE
                            // macro. To make it worse, FSDirectory
                            // instances are collected in global cache
                            // (clucene internal) and to de-register
                            // from that cache, the close() member
                            // function has to be called, otherwise
                            // leak occurs.
                            //
                            // In short, an FSDirectory instance
                            // (acquired through getDirectory() call)
                            // must be close()-d and _CLDECDELETE-ed
                            // to get rid of properly.
                            CallCloseLogging(*curIdxDir);
                            _CLDECDELETE(curIdxDir);
                            if (iBufDir != NULL)
                                {
                                    CallCloseLogging(*iBufDir);
                                    _CLDECDELETE(iBufDir);
                                }

                            throw;
                        }
                }
        }



        void RenameMergingDir_(CpixPaths & paths)
        {
            using namespace std;
            using namespace Cpt;

            if (directoryexists(paths.mergingDir_.c_str()))
                {
                    string
                        newDirMarker(paths.newDir_->c_str());
                    newDirMarker += VERSION_MARKER_SUFFIX;

                    int
                        result = touch(newDirMarker.c_str(),
                                       0666);

                    if (result != 0)
                        {
                            // failure to create the marker file is
                            // not considered an error, as CpixPaths
                            // constructor uses a fall-back method
                            // (checking modification dates) to decide
                            // which index version is newer - see
                            // above
                            logMsg(CPIX_LL_WARNING,
                                   "Could not create '%s': %d",
                                   newDirMarker.c_str(),
                                   result);
                        }

                    
                    result = rename(paths.mergingDir_.c_str(),
                                    paths.newDir_->c_str());
                    
                    if (result != 0)
                        {
                            THROW_CPIXEXC("Could not rename '%s' to '%s': %d",
                                          paths.mergingDir_.c_str(),
                                          paths.newDir_->c_str(),
                                          result);
                        }
                }
        }

        

        void CleanupCPixDir_(const char * basePath,
                             const char * exception)
        {
            CpixPaths::RemoveExcept(basePath,
                                    exception);
        }


    } // ns

} // ns
