/*
* Copyright (c) 2010 Nokia Corporation and/or its subsidiary(-ies).
* All rights reserved.
* This component and the accompanying materials are made available
* under the terms of "Eclipse Public License v1.0"
* which accompanies this distribution, and is available
* at the URL "http://www.eclipse.org/legal/epl-v10.html".
*
* Initial Contributors:
* Nokia Corporation - initial contribution.
*
* Contributors:
*
* Description:
*
*/
#include <fstream>
#include <map>
#include <vector>
#include <list>
#include <string>
#include "CLucene.h"
#include "CLucene/queryParser/MultiFieldQueryParser.h"
#include "cpixtools.h"
#include "cpixexc.h"
#include "cpixidxdb.h"
#include "idxdb.h"
#include "iidxdb.h"
#include "multiidxdb.h"
#include "indevicecfg.h"
#include "idxdbmgr.h"
#include "idxdbdelta.h"
#include "common/cpixlog.h"
namespace
{
const char VOLID_PREFIX = '@';
const char VOLID_DELIMITER = ':';
const char DOMSEL_DELIMITER = ',';
const char APPHIERARCHY_DELIMITER = ' ';
// TODO this is a very suspicious candidate to move to cpixtool lib,
// like "cpixbittools.h" or something
/**
* Produces a bitmask for an integer type that has the highest bit
* set. Compile time thingy, does not involve runtime cost.
*/
template<typename INTEGER>
struct HighestBit
{
enum { Value = (1 << (8*sizeof(INTEGER) - 1)) };
};
}
namespace Cpix
{
/*********************************************
*
* IIdxDbInfo implementation
*
*/
void IIdxDbInfo::incRefCount()
{
++refCount_;
}
void IIdxDbInfo::decRefCount()
{
if (refCount_ == 0)
{
THROW_CPIXEXC(PL_ERROR "Decreasing zero refcount");
}
--refCount_;
if (refCount_ == 0)
{
if (ptr_ != NULL)
{
Impl::cl_auto_ptr<IIdxDb>
tmp(ptr_);
ptr_ = NULL;
// close()-ing and destroying former ptr_ is
// implied by cl_auto_ptr dtor
}
}
}
int IIdxDbInfo::refCount() const
{
return refCount_;
}
IIdxDb * IIdxDbInfo::ptr() const
{
return ptr_;
}
IIdxDb * IIdxDbInfo::getPtrForUse() const
{
accessed();
return ptr_;
}
void IIdxDbInfo::setPtr(IIdxDb * ptr)
{
if (ptr_ != NULL)
{
// at this point the current index is thrown away, any
// rambuffers and stuff need not be properly put to
// disk - hence brutal close
//
// NOTE: void brutalClose() throw(); - no-throw
ptr_->brutalClose();
delete ptr_;
ptr_ = NULL;
}
ptr_ = ptr;
accessed();
}
IIdxDb * IIdxDbInfo::release()
{
IIdxDb
* rv = ptr_;
ptr_ = NULL;
return rv;
}
/* OBS
void IIdxDbInfo::destroy(bool resetRefCount)
{
setPtr(NULL);
if (resetRefCount)
{
refCount_ = 0;
}
}
*/
void IIdxDbInfo::restIfIdle(long nowSec,
size_t maxIdleSec)
{
if (ptr_ != NULL)
{
if ((nowSec - lastAccessedSecs_) > maxIdleSec)
{
Impl::CallCloseLogging(*ptr_);
}
else
{
ptr_->doHousekeeping();
}
}
}
IIdxDbInfo::IIdxDbInfo()
: refCount_(0),
ptr_(NULL),
lastAccessedSecs_(0)
{
;
}
long IIdxDbInfo::lastAccessedSecs() const
{
return lastAccessedSecs_;
}
void IIdxDbInfo::dbgDumpState(size_t idx) const
{
logTestMsg(CPIX_LL_TRACE,
" o entry [%d] -> (%s, %d)",
idx,
ptr() == NULL ? "NULL" : "@?",
refCount());
logTestMsg(CPIX_LL_TRACE,
" (last accessed: %ld)",
lastAccessedSecs());
if (ptr() != NULL)
{
ptr()->dbgDumpState();
}
}
void IIdxDbInfo::accessed() const
{
lastAccessedSecs_ = Cpt::GetTimeOfDay();
}
/*********************************************
*
* IdxDbMgr implementation
*
*/
// IdxDbMgr::IIdxDbToDestroy constructor
IdxDbMgr::IIdxDbToDestroy::IIdxDbToDestroy(IIdxDb * ptr,
HousekeepCounter housekeepCounter)
: ptr_(ptr),
housekeepCounter_(housekeepCounter)
{
;
}
// IdxDbMgr proper
void IdxDbMgr::init(InitParams & ip) {
if (instance_ == NULL)
{
instance_ = new IdxDbMgr(ip);
}
else
{
THROW_CPIXEXC(PL_ERROR "Re-initing (IdxDbMgr singleton)");
}
}
IdxDbMgr * IdxDbMgr::instance()
{
return instance_;
}
bool isEmpty(std::vector<IIdxDbInfo> & idxDbs)
{
using namespace std;
vector<IIdxDbInfo>::iterator
found = find_if(idxDbs.begin(),
idxDbs.end(),
compose1(bind2nd(not_equal_to<int>(),
0),
mem_fun_ref(&IIdxDbInfo::refCount)));
return found == idxDbs.end();
}
void IdxDbMgr::scrapAll()
{
using namespace Cpix;
IdxDbMgr
* mgr = instance();
mgr->deleteAllScheduledIdxDbs();
Cpt::SyncRegion
sr(mgr->mutex_);
// checking if IIdxDb instances are still in use (they should not be)
if (!isEmpty(mgr->idxDbs_))
{
THROW_CPIXEXC("Indexes are still in use");
}
else if (!isEmpty(mgr->multiIdxDbs_))
{
THROW_CPIXEXC("(Multi) indexes are still in use");
}
int
result = unlink(mgr->regFilePath_.c_str());
if (result != 0)
{
THROW_CPIXEXC("Could not delete CPix reg file %s",
mgr->regFilePath_.c_str());
}
mgr->idxDbs_.clear();
mgr->multiIdxDbs_.clear();
mgr->qbacs_.clear();
}
void IdxDbMgr::shutdownAll()
{
delete instance_;
instance_ = NULL;
}
IdxDbHndl IdxDbMgr::create(const char * qualBaseAppClass)
{
using namespace std;
using namespace Cpix;
IdxDbHndl
rv = 0;
string
qbac(qualBaseAppClass);
Cpt::SyncRegion
sr(mutex_);
map<QualBaseAppClass, PathAndHndl>::iterator
qbacIt = qbacs_.find(qbac);
if (qbacIt == qbacs_.end())
{
// qbacIt should point to a meaningful entry, as a
// previously called defineVolume should either throw
// or make sure that the association exists
THROW_CPIXEXC(PL_ERROR "unknown qbac '%s'",
qualBaseAppClass);
}
PathAndHndl
pathAndHndl = qbacIt->second;
// Any previous instance is to be thrown away
rv = pathAndHndl.second;
// we are re-creating the index without its refCount_ reset ...
createIdxDbInstance(rv,
pathAndHndl.first.c_str(),
true, // re-create
qbac.c_str());
idxDbs_[rv].incRefCount(); // ... we just increment it
return rv;
}
bool IdxDbMgr::isQualBaseAppClass(const char * domainSelector)
{
if (domainSelector == NULL
|| strlen(domainSelector) == 0)
{
THROW_CPIXEXC("NULL or empty domain selector '%s'",
domainSelector);
}
bool
rv = false;
if (domainSelector[0] == VOLID_PREFIX)
{
const char
* delimiter = strchr(domainSelector,
VOLID_DELIMITER);
if (delimiter == NULL
|| (delimiter - domainSelector) == strlen(domainSelector))
{
THROW_CPIXEXC("Garbled domain selector '%s'",
domainSelector);
}
rv = strchr(domainSelector, DOMSEL_DELIMITER) == NULL;
}
return rv;
}
// see comments for match().
bool IdxDbMgr::matchAppClass(const char * appClass,
const char * baseAppClass)
{
bool
rv = false;
if (strlen(appClass) == 0)
{
THROW_CPIXEXC("Emptry string for app class");
}
if (appClass[0] != VOLID_PREFIX
&& baseAppClass[0] == VOLID_PREFIX)
{
// base app class is qualified, we will find the unq part
baseAppClass = getBaseAppClassPart(baseAppClass);
}
rv = Cpt::issubstroforequal(appClass,
baseAppClass);
return rv;
}
//
// Things that should match
//
// DOMAIN SELECTOR | BASE APP CLASS
// (what a searcher asks for) | (what you have as physical index)
// ----------------------------------+----------------------------------
//
//
// (multiple volumes use case)
//
// "root file" "@c:root file"
// "root file" "@d:root file"
// "@c:root file" "@c:root file"
// "@d:root file" "@d:root file"
//
//
// (generic/widening search use case)
//
// "root" "@c:root file"
// "root" "@d:root file"
// "root" "@0:root msg phone sms"
// "root msg" "@0:root msg phone sms"
// "@c:root" "@c:root file"
// "@d:root" "@d:root file"
//
//
// (selecting arbitrary indexes use case)
//
// "@FIN:root maps,@GBR:root maps" "@FIN:root maps"
// "@FIN:root maps,@GBR:root maps" "@GBR:root maps"
// "root file,root msg phone sms" "@c:root file"
// "root file,root msg phone sms" "@d:root file"
// "root file,root msg phone sms" "@0:root msg phone sms"
// "@c:root file,root msg phone sms" "@c:root file"
// "@c:root file,root msg phone sms" "@0:root msg phone sms"
//
//
// Things that should NEVER match
//
// "root file" "@0:root msg phone sms"
// "@c:root file" "@0:root msg phone sms"
// "root file" "@FIN:root maps"
// "@c:root file" "@FIN:root maps"
//
//
// TODO ??? UNIMPLEMENTED USE CASE: narrowing searches
//
// "root file media" "@c:root file" (TODO)
// "root file media" "@d:root file" (TODO)
// "@c:root file media" "@c:root file" (TODO)
// "@d:root file media" "@d:root file" (TODO)
//
// ... and in these cases search/suggest needs to implement the
// extra clause of " AND _appclass:"root file media".
//
bool IdxDbMgr::match(const char * domainSelector,
const char * baseAppClass)
{
bool
rv = false;
if (domainSelector == NULL)
{
THROW_CPIXEXC("NULL string for domain selector");
}
else if (strlen(domainSelector) == 0)
{
THROW_CPIXEXC(L"Emptry string for domain selector");
}
using namespace std;
list<string>
appClasses;
static const char domainSelectorDelimiter[] = { DOMSEL_DELIMITER, 0};
Cpt::splitstring(domainSelector,
domainSelectorDelimiter,
appClasses);
list<string>::const_iterator
i = appClasses.begin(),
end = appClasses.end();
for (; i != end && !rv; ++i)
{
rv = matchAppClass(i->c_str(),
baseAppClass);
}
return rv;
}
Version IdxDbMgr::getNextVersion()
{
Cpt::SyncRegion
sr(mutex_);
Version
rv = version_;
++version_;
return rv;
}
IdxDbHndl IdxDbMgr::getHndl(const char * domainSelector,
bool allowMultiSearch)
{
using namespace std;
using namespace Cpix;
Cpt::SyncRegion
sr(mutex_);
if (qbacs_.empty())
{
THROW_CPIXEXC("Unregistered qual base app class '%s' (empty registry).",
domainSelector);
}
IdxDbHndl
rv = 0;
if (!isQualBaseAppClass(domainSelector) && allowMultiSearch)
{
// multi search case: when it is allowed (coming
// through cpix_IdxSearch API object) and the base app
// class is unqalified.
rv = createMultiIdxDbHndl(domainSelector);
}
else
{
// normal case: when not allowing multi search (coming
// through cpix_IdxDb API object) or when qualified
// with volume id even if multi search is allowed
// (coming through cpix_IdxSearcher API object)
string
qbac(domainSelector);
map<QualBaseAppClass, PathAndHndl>::iterator
qbacIt = qbacs_.find(qbac);
if (qbacIt == qbacs_.end())
{
THROW_CPIXEXC("Unregistered qual base app class '%s'",
domainSelector);
}
else
{
PathAndHndl
pathAndHndl = qbacIt->second;
rv = pathAndHndl.second;
incIdxDbRefCount(rv);
}
}
return rv;
}
Cpix::IIdxDb * IdxDbMgr::get(IdxDbHndl handle)
{
Cpt::SyncRegion
sr(mutex_);
checkHndlSanity(handle,
false); // TODO is the 2nd parameter to checkHndlSanity needed?
std::vector<IIdxDbInfo>
& idxDbs(isMultiIdxDbHndl(handle) ? multiIdxDbs_ : idxDbs_);
handle &= ~HighestBit<IdxDbHndl>::Value;
if (idxDbs[handle].ptr() == NULL)
{
if (idxDbs[handle].refCount() == 0)
{
THROW_CPIXEXC("Accessing closed index.");
}
else
{
// TODO CHECK : what guarantees that idxDbs is
// idxDbs_ and not multiIdxDbs_ ? That
// MultiIdxDb instances are destroyed
// (null-ed) only when they are completely
// released?
if (&idxDbs != &idxDbs_)
{
THROW_CPIXEXC(PL_ERROR "NULL but non-zero refcount multiidxdb handle");
}
createIdxDbInstance(handle,
false); // don't recreate
}
}
return idxDbs[handle].getPtrForUse();
}
void IdxDbMgr::incRefHndl(IdxDbHndl handle)
{
Cpt::SyncRegion
sr(mutex_);
checkHndlSanity(handle,
false);
std::vector<IIdxDbInfo>
& idxDbs(isMultiIdxDbHndl(handle) ? multiIdxDbs_ : idxDbs_);
handle &= ~HighestBit<IdxDbHndl>::Value;
idxDbs[handle].incRefCount();
}
void IdxDbMgr::releaseHndl(IdxDbHndl handle)
{
Cpt::SyncRegion
sr(mutex_);
checkHndlSanity(handle,
false);
std::vector<IIdxDbInfo>
& idxDbs(isMultiIdxDbHndl(handle) ? multiIdxDbs_ : idxDbs_);
handle &= ~HighestBit<IdxDbHndl>::Value;
idxDbs[handle].decRefCount();
}
void IdxDbMgr::defineVolume(const char * qualBaseAppClass,
const char * indexDbPath)
{
if (!isQualBaseAppClass(qualBaseAppClass))
{
THROW_CPIXEXC("Not a qualified base app class '%s'",
qualBaseAppClass);
}
string
qbac(qualBaseAppClass),
idp;
Cpt::SyncRegion
sr(mutex_);
if (indexDbPath == NULL || *indexDbPath == 0)
{
idp = getAutomaticIdxDbPath(qualBaseAppClass);
}
else
{
idp = indexDbPath;
}
map<QualBaseAppClass, PathAndHndl>::iterator
qbacIt = qbacs_.find(qbac);
if (qbacIt != qbacs_.end())
{
if (qbacIt->second.first == idp)
{
// already defined with same path - nothing to do
return;
}
else
{
THROW_CPIXEXC("Volume (qbac %s) already defined with different path",
qbac.c_str());
}
}
sanityCheck(qbac,
idp,
NULL); // throw if problem found
InitFileSystem(idp.c_str(),
false);
qbacIt = qbacs_.find(qbac);
if (qbacIt == qbacs_.end())
{
idxDbs_.push_back(IIdxDbInfo());
size_t
newHandle = idxDbs_.size() - 1;
qbacs_[qbac] = PathAndHndl(idp.c_str(),
newHandle);
storeReg();
std::vector<IIdxDbInfo>::iterator
i = multiIdxDbs_.begin(),
end = multiIdxDbs_.end();
for (; i!= end; ++i)
{
if (i->ptr() != NULL)
{
MultiIdxDb
* m = dynamic_cast<MultiIdxDb*>(i->ptr());
m->suggestHndl(newHandle,
qualBaseAppClass);
}
}
}
}
void IdxDbMgr::undefineVolume(const char * qualBaseAppClass)
{
using namespace std;
using namespace Cpix;
IdxDbHndl
hndl = 0;
string
qbac(qualBaseAppClass);
IIdxDb
* toDestroy = NULL;
{ // SYNC
Cpt::SyncRegion
sr(mutex_);
map<QualBaseAppClass, PathAndHndl>::iterator
qbacIt = qbacs_.find(qbac);
// it's okay if the base app class does not exist
if (qbacIt != qbacs_.end())
{
PathAndHndl
pathAndHndl = qbacIt->second;
// Any previous instance is to be thrown away
hndl = pathAndHndl.second;
qbacs_.erase(qbacIt);
// At this point, the handle cannot be looked up
// by its qbac. Also, if the ptr associated with
// the handle is NULL, it cannot be
// re-instantiated, because qbac/path information
// has been deleted
storeReg();
vector<IIdxDbInfo>::iterator
i = multiIdxDbs_.begin(),
end = multiIdxDbs_.end();
for (; i != end; ++i)
{
if (i->ptr() != NULL)
{
MultiIdxDb
* m = dynamic_cast<MultiIdxDb*>(i->ptr());
bool
used = m->removeHndl(hndl);
if (used)
{
idxDbs_[hndl].decRefCount();
}
}
}
// At this point, all multiidx-es have lost their
// reference to the index in question
//
// At worst, there may be some search / index
// operations still ongoing using the current
// IIdxDb instance
//
// We have to store the potential pointer value
// because now IdxDbMgr / IIdxDbInfo structures do
// not own it anymore
toDestroy = idxDbs_[hndl].release();
}
} // SYNC
if (toDestroy != NULL)
{
if (idxDbs_[hndl].refCount() == 0)
{
delete toDestroy;
}
else
{
// we can't delete it now, there may be some active
// jobs using it
scheduleForDeletion(toDestroy);
}
}
}
void IdxDbMgr::doHousekeepingOnAll()
{
using namespace std;
long
now = Cpt::GetTimeOfDay();
vector<IIdxDbInfo>::size_type
count;
{ // SYNC
Cpt::SyncRegion
sr(mutex_);
count = idxDbs_.size();
} // SYNC
// We don't want the mutex held while we are calling
// restIfIdle, as they may result in file i/o. We just check
// the size of idxDbs_, and that gives us the possible handle
// range: [0, size-of-idxDbs).
//
// Some of these handles are invalid (undefined), that's ok.
//
// Note, that this here should not overindex the vector as
// long as there is no operation where the idxDbs_ vector is
// decreased. Currently, there is no such operation, except
// the dbg-scrapAll(), which is invoked only from debug code
// and only when all handles have been released. So once
// idxDbs_ reaches a size, it can't get smaller.
for (vector<IIdxDbInfo>::size_type handle = 0;
handle < count;
++handle)
{
idxDbs_[handle].restIfIdle(now,
maxIdleSec_);
}
deleteSomeScheduledIdxDbs();
++housekeepCounter_;
}
IdxDbMgr::IdxDbMgr(InitParams & ip)
: cpixDir_(ip.getCpixDir()),
regFilePath_(ip.getCpixDir()),
maxIdleSec_(ip.getMaxIdleSec()),
mutex_(true), // recursive
version_(0),
initParams_(ip),
housekeepCounter_(0)
{
regFilePath_ += "cpixreg.txt";
// TODO this should be removed at some point - now it is only
// here to have the indexes and the cpixreg.txt file in the
// legacy locations even with this new logic
cpixDir_ += "indexing\\indexdb";
loadReg();
}
void destroyIdxDbs(std::vector<IIdxDbInfo> & idxDbs)
{
using namespace std;
vector<Cpix::IIdxDbInfo>::iterator
i = idxDbs.begin(),
end = idxDbs.end();
for (; i != end; ++i)
{
if (i->ptr() != NULL)
{
Impl::cl_auto_ptr<IIdxDb>
tmp(i->ptr());
// close()-ing and destroying the IIdxDb
// instance is implied by cl_auto_ptr dtor
}
}
idxDbs.clear();
}
IdxDbMgr::~IdxDbMgr()
{
destroyIdxDbs(multiIdxDbs_);
destroyIdxDbs(idxDbs_);
deleteAllScheduledIdxDbs();
}
void IdxDbMgr::sanityCheck(const std::string & baseAppClass,
const std::string & idxDbPath,
bool * succeeded)
{
using namespace std;
string
msg;
map<QualBaseAppClass, PathAndHndl>::const_iterator
i = qbacs_.begin(),
end = qbacs_.end();
for (; i != end; ++i)
{
if (i->second.first == idxDbPath
&& i->first != baseAppClass)
{
// (1) We don't allow multiple
// baseAppClass-es pointing to the same
// physical path
msg = "Both ";
msg += i->first;
msg += " and ";
msg += baseAppClass;
msg += " point to the same phyisical idx db path ";
msg += idxDbPath;
break;
}
else if (i->first == baseAppClass
&& i->second.first != idxDbPath)
{
// (2) We don't allow the same
// baseAppClass to have multiple physical
// paths associated.
msg = "A base app class ";
msg += baseAppClass;
msg += " cannot have multiple index db paths.";
break;
}
}
if (succeeded == NULL)
{
if (msg.length() > 0)
{
THROW_CPIXEXC(msg.c_str());
}
}
else
{
if (msg.length() > 0)
{
*succeeded = false;
logMsg(CPIX_LL_ERROR,
msg.c_str());
}
else
{
*succeeded = true;
}
}
}
void IdxDbMgr::InitFileSystem(const char * path,
bool recreate)
{
// making sure that the directory exists
if (!Cpt::directoryexists(path))
{
int
result = Cpt::mkdirs(path, 0777);
if (result == -1)
{
THROW_CPIXEXC("Could not create folder(s): '%s'",
path);
}
}
// if only opening, ...
if (!recreate)
{
try
{
// ... then we just try to open the physical
// index ...
OpenFsIdx(path);
}
catch (LuceneError & clErr)
{
// ... and if it failed, we must re-create it
recreate = true;
}
}
// if creating or re-creating
if (recreate)
{
RecreateFsCpixIdx(path);
}
}
void IdxDbMgr::createIdxDbInstance(IdxDbHndl handle,
const char * path,
bool recreate,
const char * qualBaseAppClass)
{
try
{
if (idxDbs_[handle].ptr() == NULL)
{
// there is no instance in use, create one
InitFileSystem(path,
recreate);
IIdxDb
* idx = new IdxDb(path,
initParams_);
idxDbs_[handle].setPtr(idx);
}
else
{
// there is an instance already, we can't
// destroy/recreate it without the danger of
// crashing another thread still working on
// that object
if (recreate)
{
idxDbs_[handle].ptr()->recreateIdx();
}
else
{
; // nothing to do - we already have
// the instance and it's index does
// not need to be reset
}
}
}
catch (...)
{
logMsg(CPIX_LL_WARNING,
"Unable to load index for %s from %s, deleting from CPix registry.",
qualBaseAppClass,
path);
// getting rid of stale stuff
qbacs_.erase(qualBaseAppClass);
storeReg();
throw;
}
}
void IdxDbMgr::createIdxDbInstance(IdxDbHndl handle,
bool recreate)
{
using namespace std;
map<QualBaseAppClass, PathAndHndl>::iterator
i = qbacs_.begin(),
end = qbacs_.end();
for (; i != end && i->second.second != handle; ++i)
;
if (i == end)
{
THROW_CPIXEXC(PL_ERROR "Handle %d does not correspond to existing qbac",
handle);
}
const char
* qbac = i->first.c_str(),
* path = i->second.first.c_str();
createIdxDbInstance(handle,
path,
recreate,
qbac);
}
void IdxDbMgr::loadReg()
{
using namespace std;
bool
succeeded = false;
// format: <base application type>=<index database path>
// e.g. "root file usrdoc text=c:/this/and/that/path"
ifstream
reg(regFilePath_.c_str());
if (reg.is_open())
{
string line;
succeeded = true;
while (getline(reg, line))
{
size_t
delimiterPos = line.find_first_of(delimiter_);
if (delimiterPos == line.npos)
{
logMsg(CPIX_LL_ERROR,
"Garbled idx db config");
succeeded = false;
break;
}
string
baseAppClass(line.substr(0,
delimiterPos)),
idxDbPath(line.substr(delimiterPos + 1));
sanityCheck(baseAppClass,
idxDbPath,
&succeeded); // warn, and report back only
if (!succeeded)
{
break;
}
idxDbs_.push_back(IIdxDbInfo());
PathAndHndl
pathAndHndl(idxDbPath,
idxDbs_.size() - 1);
qbacs_[baseAppClass] = pathAndHndl;
}
reg.close();
}
if (!succeeded)
{
logMsg(CPIX_LL_WARNING,
"Could not read CPix registry %s, (re-)creating it.\n",
regFilePath_.c_str());
remove(regFilePath_.c_str());
// this call is made here only to check that we can
// create the file, otherwise we should fail at this
// point
storeReg();
}
}
void IdxDbMgr::storeReg()
{
using namespace std;
ofstream
reg(regFilePath_.c_str());
if (reg.is_open())
{
map<string,PathAndHndl>::iterator
i = qbacs_.begin(),
end = qbacs_.end();
for (; i != end; ++i)
{
const string
& baseAppClass = i->first;
const string
& idxDbPath = i->second.first;
reg
<< baseAppClass
<< delimiter_
<< idxDbPath
<< endl;
}
}
else
{
THROW_CPIXEXC("Cannot open idx db config file '%s'",
regFilePath_.c_str());
}
}
std::string IdxDbMgr::getAutomaticIdxDbPath(const char * qualBaseAppClass)
{
std::string
rv(cpixDir_);
const char
* ptr = getBaseAppClassPart(qualBaseAppClass);
while (ptr != NULL && *ptr != 0)
{
const char
* end = strchr(ptr,
APPHIERARCHY_DELIMITER);
size_t
wordSize = (end == NULL) ? strlen(ptr) : end - ptr;
rv += '\\';
rv += std::string(ptr,
wordSize);
ptr = end;
if (ptr != NULL)
{
++ptr;
}
}
return rv;
}
const char * IdxDbMgr::getBaseAppClassPart(const char * qualBaseAppClass)
{
const char
* delimiter = strchr(qualBaseAppClass,
VOLID_DELIMITER);
if (delimiter == NULL
|| (delimiter - qualBaseAppClass) == strlen(qualBaseAppClass))
{
THROW_CPIXEXC("Garbled qualified base app class '%s'",
qualBaseAppClass);
}
return delimiter + 1;
}
void IdxDbMgr::incIdxDbRefCount(IdxDbHndl handle)
{
idxDbs_[handle].incRefCount();
}
IdxDbHndl IdxDbMgr::createMultiIdxDbHndl(const char * domainSelector)
{
using namespace std;
IdxDbHndl
rv = 0;
vector<IIdxDbInfo>::iterator
firstEmpty = find_if(multiIdxDbs_.begin(),
multiIdxDbs_.end(),
compose1(bind2nd(equal_to<IIdxDb*>(),
NULL),
mem_fun_ref(&IIdxDbInfo::ptr)));
if (firstEmpty == multiIdxDbs_.end())
{
firstEmpty = multiIdxDbs_.insert(multiIdxDbs_.end(),
IIdxDbInfo());
}
rv = firstEmpty - multiIdxDbs_.begin();
set<IdxDbHndl>
matchedHandles;
map<QualBaseAppClass, PathAndHndl>::iterator
i = qbacs_.begin(),
end = qbacs_.end();
for (; i != end; ++i)
{
if (match(domainSelector, i->first.c_str()))
{
matchedHandles.insert(i->second.second);
incIdxDbRefCount(i->second.second);
}
}
MultiIdxDb
* multi = new MultiIdxDb(matchedHandles,
domainSelector);
*firstEmpty = IIdxDbInfo();
firstEmpty->setPtr(multi);
firstEmpty->incRefCount();
// this is how we mark that this handle is a multi idx db handle
rv |= (IdxDbHndl)HighestBit<IdxDbHndl>::Value;
return rv;
}
bool IdxDbMgr::isMultiIdxDbHndl(IdxDbHndl handle)
{
const IdxDbHndl
mask = (IdxDbHndl)HighestBit<IdxDbHndl>::Value;
return (handle & mask) != 0;
}
void IdxDbMgr::checkHndlSanity(IdxDbHndl handle,
bool allowUndefinedIdxDb)
{
bool
isMulti = isMultiIdxDbHndl(handle);
std::vector<IIdxDbInfo>
& idxDbs(isMulti ? multiIdxDbs_ : idxDbs_);
handle &= ~HighestBit<IdxDbHndl>::Value;
if (handle > idxDbs.size())
{
THROW_CPIXEXC(PL_ERROR "Invalid idxdb handle %d",
handle);
}
if (idxDbs[handle].ptr() == NULL)
{
if (isMulti)
{
THROW_CPIXEXC(PL_ERROR "MultiIdxDb instance is NULL for handle '%d'",
handle);
}
else if (!allowUndefinedIdxDb)
{
/* TODO check if this should be allowed to throw
THROW_CPIXEXC(PL_ERROR "IdxDb instance is NULL for handle '%d'",
handle);
*/
}
}
}
void IdxDbMgr::dbgDumpState()
{
logTestMsg(CPIX_LL_TRACE,
"DUMPING IdxDbMgr singleton instance: BEGIN");
logTestMsg(CPIX_LL_TRACE,
" o cpixDir_: %s",
cpixDir_.c_str());
logTestMsg(CPIX_LL_TRACE,
" o regFilePath_: %s",
regFilePath_.c_str());
Cpt::SyncRegion
sr(mutex_);
{
logTestMsg(CPIX_LL_TRACE,
" o qbacs_:");
std::map<QualBaseAppClass,PathAndHndl>::iterator
i = qbacs_.begin(),
end = qbacs_.end();
for (; i != end; ++i)
{
logTestMsg(CPIX_LL_TRACE,
" o entry: [%s] -> (%s, %d)",
i->first.c_str(),
i->second.first.c_str(),
i->second.second);
}
}
{
std::vector<IIdxDbInfo>::iterator
i = idxDbs_.begin(),
end = idxDbs_.end();
size_t
idx = 0;
logTestMsg(CPIX_LL_TRACE,
" o idxDb_:");
for (; i != end; ++i)
{
i->dbgDumpState(idx);
++idx;
}
i = multiIdxDbs_.begin(),
end = multiIdxDbs_.end();
idx = 0;
logTestMsg(CPIX_LL_TRACE,
" o multiIdxDb_:");
for (; i != end; ++i)
{
i->dbgDumpState(idx);
++idx;
}
}
logTestMsg(CPIX_LL_TRACE,
"DUMPING IdxDbMgr singleton instance: END.",
reinterpret_cast<long>(this));
}
size_t IdxDbMgr::getClHitsPageSize() const
{
return initParams_.getClHitsPageSize();
}
void IdxDbMgr::RecreateFsClIdx(const char * cluceneDir)
{
using namespace lucene::index;
Impl::cl_auto_ptr<IndexWriter>
tmpIdxWriter(new IndexWriter(cluceneDir,
NULL, // &dummyAnalyzer,
true, // create
true));
// close()-ing tmpIdxWriter is implied by cl_auto_ptr dtor
}
void IdxDbMgr::OpenFsIdx(const char * cpixDir)
{
using namespace lucene::index;
using namespace lucene::analysis;
Cpix::Impl::cl_auto_ptr<IndexReader>
dummyReader(NULL);
Cpix::Impl::CpixPaths
paths(cpixDir);
dummyReader.reset(IndexReader::open(paths.curDir_->c_str()));
// close()-ing dummyReader is implied by cl_auto_ptr dtor
}
void IdxDbMgr::RecreateFsCpixIdx(const char * cpixDir)
{
Cpt::removeunder(cpixDir);
using namespace lucene::index;
using namespace lucene::analysis;
const char
* cluceneDir = cpixDir;
// old machinery: cpix dir and clucene dir ARE the same
// new machinery: cpix dir and clucene dir are NOT the same thing
Cpix::Impl::CpixPaths
paths(cpixDir);
cluceneDir = paths.curDir_->c_str();
Cpix::IdxDbMgr::RecreateFsClIdx(cluceneDir);
}
void IdxDbMgr::scheduleForDeletion(IIdxDb * toDestroy)
{
Cpt::SyncRegion
sr(mutex_);
idxDbsToDestroy_.push_back(IIdxDbToDestroy(toDestroy,
housekeepCounter_));
}
void IdxDbMgr::deleteAllScheduledIdxDbs()
{
Cpt::SyncRegion
sr(mutex_);
std::list<IIdxDbToDestroy>::iterator
i = idxDbsToDestroy_.begin(),
end = idxDbsToDestroy_.end();
for (; i != end; ++i)
{
delete i->ptr_;
}
}
void IdxDbMgr::deleteSomeScheduledIdxDbs()
{
Cpt::SyncRegion
sr(mutex_);
while (!idxDbsToDestroy_.empty())
{
IIdxDbToDestroy
tmp = idxDbsToDestroy_.front();
if (tmp.housekeepCounter_ + DISCARD_SAFETY_PERIOD
< housekeepCounter_)
{
// after tmp.ptr_ has been scheduled for
// destruction by undefineVolume, we have
// completed DISCARD_SAFETY_PERIOD number of
// housekeep rounds, so we can be confident
// that no active jobs use tmp.ptr_ anymore
tmp.ptr_->brutalClose();
delete tmp.ptr_;
idxDbsToDestroy_.pop_front();
}
else
{
// idxDbsToDestroy_ is used as a queue (added
// to at the end, taken from at the front),
// therefore if a scheduled pointer is not
// ready for desstruction, the ones added
// later won't be ready either at this point
break;
}
}
}
//
// This is for single threaded use only
//
IdxDbMgr * IdxDbMgr::instance_ = NULL;
const char IdxDbMgr::delimiter_ = '=';
}