--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/searchengine/cpix/tsrc/cpixunittest/src/whiteboxtests.cpp Mon Apr 19 14:40:16 2010 +0300
@@ -0,0 +1,1395 @@
+/*
+* Copyright (c) 2010 Nokia Corporation and/or its subsidiary(-ies).
+* All rights reserved.
+* This component and the accompanying materials are made available
+* under the terms of "Eclipse Public License v1.0"
+* which accompanies this distribution, and is available
+* at the URL "http://www.eclipse.org/legal/epl-v10.html".
+*
+* Initial Contributors:
+* Nokia Corporation - initial contribution.
+*
+* Contributors:
+*
+* Description:
+*
+*/
+#include <wchar.h>
+#include <stddef.h>
+
+#include <iostream>
+
+#include "cpixidxdb.h"
+
+#include "itk.h"
+
+#include "config.h"
+#include "testutils.h"
+#include "setupsentry.h"
+
+// internal (inc/private) headers from cpix - that is why these tests
+// are whitebox tests
+#include "cpixtools.h"
+#include "iidxdb.h"
+#include "iqrytype.h"
+#include "cpixexc.h"
+#include "cpixhits.h"
+#include "idxdbdelta.h"
+
+void TestBaseAppClassCollision(Itk::TestMgr * testMgr)
+{
+ cpix_Result
+ result;
+
+ cpix_IdxDb_defineVolume(&result,
+ SMS_QBASEAPPCLASS,
+ NULL);
+
+ ITK_ASSERT(testMgr,
+ cpix_Succeeded(&result),
+ "Definition of volume (%s, %s) failed.",
+ SMS_QBASEAPPCLASS,
+ "<default>");
+
+ cpix_IdxDb_defineVolume(&result,
+ SMS_QBASEAPPCLASS,
+ NULL);
+
+ ITK_ASSERT(testMgr,
+ cpix_Succeeded(&result),
+ "Re-definition of identical volume (%s, %s) failed.",
+ SMS_QBASEAPPCLASS,
+ "<default>");
+
+ const char
+ * dummyPath = "\\dummy\\path";
+
+ cpix_IdxDb_defineVolume(&result,
+ SMS_QBASEAPPCLASS,
+ dummyPath);
+
+ ITK_ASSERT(testMgr,
+ cpix_Failed(&result),
+ "Volume (%s, %s) definition should have failed.",
+ SMS_QBASEAPPCLASS,
+ dummyPath);
+
+ cpix_IdxDb_undefineVolume("@0:root foo bar");
+}
+
+
+#define DUMMY_QBASEAPPCLASS "@0:root dummy"
+#define DUMMY2_QBASEAPPCLASS "@0:root dummy2"
+#define DUMMY_IDXDBPATH "c:\\Data\\indexing\\indexdb\\root\\dummy"
+
+
+void TestIdxDbPathCollision(Itk::TestMgr * testMgr)
+{
+ cpix_Result
+ result;
+
+ cpix_IdxDb_defineVolume(&result,
+ DUMMY_QBASEAPPCLASS,
+ DUMMY_IDXDBPATH);
+ ITK_ASSERT(testMgr,
+ cpix_Succeeded(&result),
+ "Defining volume %s, %s failed.",
+ DUMMY_QBASEAPPCLASS,
+ DUMMY_IDXDBPATH);
+
+ cpix_IdxDb_defineVolume(&result,
+ DUMMY2_QBASEAPPCLASS,
+ DUMMY_IDXDBPATH);
+
+ ITK_ASSERT(testMgr,
+ cpix_Failed(&result),
+ "Defining volume %s, %s should have failed.",
+ DUMMY2_QBASEAPPCLASS,
+ DUMMY_IDXDBPATH);
+}
+
+
+
+void TestScrapAll(Itk::TestMgr * testMgr)
+{
+ cpix_Result
+ result;
+
+ cpix_IdxDb_defineVolume(&result,
+ SMS_QBASEAPPCLASS,
+ NULL);
+ if (cpix_Succeeded(&result))
+ {
+ cpix_IdxDb
+ * idxDb = cpix_IdxDb_openDb(&result,
+ SMS_QBASEAPPCLASS,
+ cpix_IDX_CREATE);
+
+ if (cpix_Succeeded(&result))
+ {
+ cpix_IdxDb_releaseDb(idxDb);
+ }
+ }
+
+ ITK_ASSERT(testMgr,
+ cpix_Succeeded(&result),
+ "Definition and creation of IdxDb (%s, %s) failed",
+ SMS_QBASEAPPCLASS,
+ "<default>");
+
+ cpix_IdxDb_dbgScrapAll(&result);
+
+ cpix_IdxDb
+ * idxDb = cpix_IdxDb_openDb(&result,
+ SMS_QBASEAPPCLASS,
+ cpix_IDX_OPEN);
+
+ if (cpix_Succeeded(&result))
+ {
+ cpix_IdxDb_releaseDb(idxDb);
+ }
+
+ ITK_ASSERT(testMgr,
+ cpix_Failed(&result),
+ "Opening IdxDb(%s) should have failed now.",
+ SMS_QBASEAPPCLASS);
+}
+
+const wchar_t * QryStrings[] = {
+
+ // should pass:
+ L"$foo",
+ L"$bar<1.2,'blabla',3>",
+ L" $ bar < 1.2 , 'blabla' , 3 > ",
+ L"$barf(inner query)",
+ L" $ barf ( inner query ) ",
+ L"$barfology<'bloblo',3.1415>(inner query)",
+ L" $ barfology < 'bloblo' , 3.1415 > ( inner query ) ",
+ L"some query",
+ L"*",
+ L"* AND some more criteria", // special case: "innery query" is the whole
+
+ // should fail:
+ L"$barf<1.23(blo)",
+ L"$farb<1.23>(blo",
+ L"$farb<1.23>(blo ) oops more data",
+
+
+ // end of test data
+ NULL
+};
+
+
+void TestUnifiedSearchParse(Itk::TestMgr * )
+{
+ using namespace std;
+
+ printf("Whitebox testing parsing of unified search syntax\n\n");
+
+ const wchar_t
+ ** p = QryStrings;
+
+ for (; *p != NULL; ++p)
+ {
+ printf("Trying to parse %S:\n",
+ *p);
+
+ try {
+ Cpix::QryCall
+ qc(*p);
+
+ printf("\to qry type id: %S\n",
+ qc.qryTypeId_.c_str());
+
+ printf("\to qry arguments: ");
+ list<wstring>::const_iterator
+ i = qc.args_.begin(),
+ end = qc.args_.end();
+ for (; i != end; ++i)
+ {
+ printf("%S ",
+ i->c_str());
+ }
+ printf("\n");
+
+ printf("\to inner qry: '%S'\n",
+ qc.innerQryStr_.c_str());
+
+ } catch (CpixExc & cpixExc) {
+ printf("Failed to parse: %S\n",
+ cpixExc.wWhat());
+ } catch (...) {
+ printf("Failed t parse: for unknown reasons\n");
+ }
+ }
+}
+
+
+class PageContext : public Itk::ITestContext
+{
+private:
+ //
+ // private members
+ //
+ SmsIdxUtil * util_;
+ cpix_Analyzer * analyzer_;
+ cpix_Query * query_;
+ cpix_QueryParser * queryParser_;
+
+ size_t pageSize_;
+ size_t expectedHitCount_;
+
+public:
+ //
+ // from interface Itk::ITestContext
+ //
+ virtual void setup() throw (Itk::PanicExc)
+ {
+ SetupSentry
+ ss(*this);
+
+ util_ = new SmsIdxUtil;
+ util_->init();
+
+ cpix_Result
+ result;
+
+ analyzer_ = cpix_CreateSimpleAnalyzer(&result);
+
+ if (analyzer_ == NULL)
+ {
+ ITK_PANIC("Could not create analyzer");
+ }
+
+ queryParser_ = cpix_QueryParser_create(&result,
+ LBODY_FIELD,
+ analyzer_);
+
+ if (queryParser_ == NULL)
+ {
+ ITK_PANIC("Could not create query parser");
+ }
+
+ query_ = cpix_QueryParser_parse(queryParser_,
+ L"abc");
+
+ if (cpix_Failed(queryParser_) || query_ == NULL)
+ {
+ ITK_PANIC("Could not parse query string");
+ }
+
+ ss.setupComplete();
+ }
+
+
+ virtual void tearDown() throw ()
+ {
+ cleanup();
+ }
+
+
+ virtual ~PageContext()
+ {
+ cleanup();
+ }
+
+
+ //
+ // public operations
+ //
+ PageContext()
+ : util_(NULL),
+ analyzer_(NULL),
+ query_(NULL),
+ queryParser_(NULL),
+ pageSize_(Cpix::IdxDbMgr::instance()->getClHitsPageSize())
+ {
+ expectedHitCount_ = 4 * pageSize_ - 3;
+ }
+
+
+ void testAddSome(Itk::TestMgr * mgr)
+ {
+ std::wstring
+ firstBody,
+ secondBody;
+
+ for (int i = 0; i < expectedHitCount_; ++i)
+ {
+ generateTwoSmsBodies(i,
+ firstBody,
+ secondBody);
+
+ util_->indexSms(2*i,
+ firstBody.c_str(),
+ analyzer_,
+ mgr);
+
+ util_->indexSms(2*i + 1,
+ secondBody.c_str(),
+ analyzer_,
+ mgr);
+ if ((i % 5) == 0)
+ {
+ ITK_DBGMSG(mgr,
+ ".");
+ }
+ }
+ util_->flush();
+ }
+
+
+ void testSearchPages(Itk::TestMgr * mgr)
+ {
+ cpix_Hits
+ * hits = cpix_IdxDb_search(util_->idxDb(),
+ query_);
+
+ if (cpix_Failed(util_->idxDb()))
+ {
+ ITK_EXPECT(mgr,
+ false,
+ "Failed to search index");
+ cpix_ClearError(util_->idxDb());
+ }
+ else
+ {
+ int32_t
+ hitsLength = cpix_Hits_length(hits);
+
+ ITK_EXPECT(mgr,
+ expectedHitCount_ == hitsLength,
+ "Did not get expected hitcounts (%d), but %d",
+ expectedHitCount_,
+ hitsLength);
+
+
+ printf("Got %d hits\n",
+ hitsLength);
+
+ search(hits,
+ 0,
+ 2 * pageSize_,
+ mgr);
+
+ search(hits,
+ 3 * pageSize_,
+ 4 * pageSize_,
+ mgr);
+
+ search(hits,
+ 0,
+ pageSize_,
+ mgr);
+
+ search(hits,
+ 2 * pageSize_,
+ 4 * pageSize_,
+ mgr);
+
+ }
+
+ cpix_Hits_destroy( hits );
+ }
+
+
+private:
+ //
+ // private methods
+ //
+ void cleanup()
+ {
+ delete util_;
+ util_ = NULL;
+
+ cpix_Analyzer_destroy(analyzer_);
+ analyzer_ = NULL;
+
+ cpix_Query_destroy(query_);
+ query_ = NULL;;
+
+ cpix_QueryParser_destroy(queryParser_);
+ queryParser_ = NULL;
+ }
+
+
+ void generateTwoSmsBodies(int idx,
+ std::wstring & firstBody,
+ std::wstring & secondBody)
+ {
+ static wchar_t
+ firstBuf[] = L"hello abc ",
+ secondBuf[] = L"hola, xyz ";
+ // ^
+ enum // |
+ { // |
+ VARIANT_POS = 10
+ };
+
+ wchar_t
+ * p1 = firstBuf + VARIANT_POS,
+ * p2 = secondBuf + VARIANT_POS;
+
+ static const int
+ radix = 'z' - 'a' + 1;
+
+ int
+ length = sizeof(firstBuf) / sizeof(wchar_t) - VARIANT_POS;
+
+ bool
+ firstRun = true;
+
+ while (length > 0)
+ {
+ if (!firstRun && idx == 0)
+ {
+ *p1 = L' ';
+ *p2 = L' ';
+ break;
+ }
+
+ int
+ digit = idx % radix;
+
+ wchar_t
+ charDigit = L'a' + digit;
+
+ *p1 = charDigit;
+ *p2 = charDigit;
+
+ ++p1;
+ ++p2;
+
+ idx = idx / radix;
+ firstRun = false;
+ }
+
+ firstBody = firstBuf;
+ secondBody = secondBuf;
+ }
+
+
+ void search(cpix_Hits * hits,
+ int32_t from,
+ int32_t to,
+ Itk::TestMgr * mgr)
+ {
+ printf("Printing hit docs from hit doc idx %d to %d\n",
+ from,
+ to);
+
+ for (int32_t hitDocIdx = from; hitDocIdx < to; ++hitDocIdx)
+ {
+ cpix_Document
+ doc;
+
+ cpix_Hits_doc(hits,
+ hitDocIdx,
+ &doc);
+
+ if (cpix_Failed(hits))
+ {
+ wchar_t
+ buf[92];
+ cpix_Error_report(hits->err_,
+ buf,
+ sizeof(buf) / sizeof(wchar_t));
+ printf("Failed to get hit doc %d: %S\n",
+ hitDocIdx,
+ buf);
+ cpix_ClearError(hits);
+ break;
+ }
+
+ util_->printHit(&doc,
+ mgr);
+ }
+ }
+
+
+};
+
+
+Itk::TesterBase * CreatePageTests()
+{
+ using namespace Itk;
+
+ PageContext
+ * pageContext = new PageContext;
+ ContextTester
+ * ctxtTester = new ContextTester("page",
+ pageContext);
+
+#define TEST "addSome"
+ ctxtTester->add(TEST,
+ pageContext,
+ &PageContext::testAddSome,
+ TEST);
+#undef TEST
+
+#define TEST "searchPages"
+ ctxtTester->add(TEST,
+ pageContext,
+ &PageContext::testSearchPages,
+ TEST);
+#undef TEST
+
+ return ctxtTester;
+}
+
+
+
+
+const wchar_t DELTA_QRY_TERM[] = L"hello";
+
+struct DeltaSms
+{
+ const wchar_t * id_;
+ const wchar_t * body_;
+};
+
+
+static DeltaSms DeltaSmsesToStartWith[] = {
+ { L"0", L"Hello it is a nice day" },
+ { L"1", L"Goodbye and fare you well" },
+ { L"2", L"Did you say hello to her?" }, // to delete
+ { L"3", L"No I said hola" }, // to delete
+};
+
+
+const wchar_t * DeltaSmsesToDelete[] = {
+ L"2",
+ L"3"
+};
+
+
+static DeltaSms DeltaSmsesToAdd[] = {
+ { L"4", L"Do hello, hallo and hullo mean the same?" },
+ { L"5", L"Yes, they just spell different" },
+};
+
+
+const char DeltaIdxDbPathBase[] = "c:\\data\\cpixunittest\\_testidx";
+
+const wchar_t DELTA_ID_FIELD[] = L"id";
+const wchar_t DELTA_BODY_FIELD[] = L"body";
+
+
+class TestInsertBuf : public Cpix::Impl::IInsertBuf
+{
+private:
+ lucene::index::IndexWriter * writer_;
+ lucene::store::TransactionalRAMDirectory * ramDir_;
+
+public:
+ //
+ // from interface Cpix::Impl::IInsertBuf
+ //
+ virtual void close()
+ {
+ if (writer_ != NULL)
+ {
+ writer_->close();
+ delete writer_;
+ writer_ = NULL;
+ }
+
+ if (ramDir_ != NULL)
+ {
+ ramDir_->close();
+ _CLDECDELETE(ramDir_);
+ ramDir_ = NULL;
+ }
+ }
+
+
+ virtual bool isEmpty() const
+ {
+ return ramDir_ == NULL;
+ }
+
+
+ virtual lucene::store::TransactionalRAMDirectory * getRAMDir()
+ {
+ if (isEmpty())
+ {
+ THROW_CPIXEXC(PL_ERROR "Accessing empty ram dir");
+ }
+
+ if (writer_ != NULL)
+ {
+ writer_->close();
+ delete writer_;
+ writer_ = NULL;
+ }
+
+ return ramDir_;
+ }
+
+
+ virtual ~TestInsertBuf()
+ {
+ close();
+ }
+
+
+ TestInsertBuf(lucene::analysis::standard::StandardAnalyzer * analyzer)
+ : writer_(NULL),
+ ramDir_(NULL)
+ {
+ ramDir_ = _CLNEW lucene::store::TransactionalRAMDirectory();
+ writer_ = new lucene::index::IndexWriter(ramDir_,
+ analyzer,
+ true, // create
+ false);
+ }
+
+
+ lucene::index::IndexWriter * getWriter()
+ {
+ if (writer_ == NULL)
+ {
+ THROW_CPIXEXC(PL_ERROR "messed up test case");
+ }
+
+ return writer_;
+ }
+};
+
+
+class TestIdxReader : public Cpix::Impl::IIdxReader
+{
+private:
+ lucene::index::IndexReader * reader_;
+
+public:
+ //
+ // from Cpix::Impl::IIdxReader interface
+ //
+ virtual void open(const char * clIdxPath,
+ Cpt::Mutex & dirMutex)
+ {
+ if (reader_ != NULL)
+ {
+ THROW_CPIXEXC(PL_ERROR "re-opening a reader");
+ }
+
+ Cpt::SyncRegion
+ sr(dirMutex);
+
+ reader_ = lucene::index::IndexReader::open(clIdxPath);
+ }
+
+
+ virtual void reopen(Cpt::Mutex & dirMutex,
+ const char * clIdxPath,
+ bool reRead)
+ {
+ if (reader_ != NULL)
+ {
+ Cpt::SyncRegion
+ sr(dirMutex);
+
+ reader_->close();
+ delete reader_;
+ reader_ = NULL;
+
+ if (reRead)
+ {
+ reader_ = lucene::index::IndexReader::open(clIdxPath);
+ }
+ }
+ }
+
+
+ virtual bool commitIfNecessary(Cpt::Mutex & dirMutex)
+ {
+ bool
+ rv = false;
+
+ if (reader_ != NULL && reader_->hasDeletions())
+ {
+ Cpt::SyncRegion
+ sr(dirMutex);
+
+ reader_->commit();
+
+ rv = true;
+ }
+
+ return rv;
+ }
+
+ //
+ // Lifetime mgmt
+ //
+ virtual ~TestIdxReader()
+ {
+ close(NULL);
+ }
+
+
+ TestIdxReader()
+ : reader_(NULL)
+ {
+ ;
+ }
+
+
+ //
+ // Own operations
+ //
+ void close(Cpt::Mutex * dirMutex)
+ {
+ if (reader_ != NULL)
+ {
+ // lock if we have something to lock on
+ std::auto_ptr<Cpt::SyncRegion>
+ sr;
+
+ if (dirMutex != NULL)
+ {
+ sr.reset(new Cpt::SyncRegion(*dirMutex));
+ }
+
+ reader_->close();
+ delete reader_;
+ reader_ = NULL;
+ }
+ }
+
+
+ bool isOpen() const
+ {
+ return reader_ != NULL;
+ }
+
+
+ lucene::index::IndexReader * getReader()
+ {
+ if (!isOpen())
+ {
+ THROW_CPIXEXC(PL_ERROR "accessing a closed reader");
+ }
+
+ return reader_;
+ }
+};
+
+
+
+class IdxDbDeltaContext : public Itk::ITestContext
+{
+private:
+ std::string test_;
+
+ bool haveInsertBuffer_;
+ bool haveReader_;
+ bool haveDels_;
+ Cpix::Impl::CommitStage targetCommitStage_;
+
+ TestInsertBuf * insertBuf_;
+ TestIdxReader reader_;
+
+ lucene::analysis::standard::StandardAnalyzer analyzer_;
+
+ std::string deltaIdxDbPath_;
+ Cpt::Mutex idxMutex_;
+
+public:
+
+ /**
+ * @param test syntax: [nw][nr][nd]-(dels_committed|ibuf_committing|ibuf_committed|merging|merged|complete)
+ *
+ * Where
+ *
+ * (a) n,w tell if there is an insertbuffer (writer + ramdir) or not
+ * (b) n,r tell if there is a reader or not
+ * (c) n,d tell if there were deletions on reader or not
+ * (d) the keyword after dash tell the target commit stage (that
+ * should be reached)
+ *
+ *
+ */
+ IdxDbDeltaContext(const char * test)
+ : test_(test),
+ haveInsertBuffer_(false),
+ haveReader_(false),
+ haveDels_(false),
+ targetCommitStage_(Cpix::Impl::CS_DELS_COMMITTED),
+ insertBuf_(NULL),
+ deltaIdxDbPath_(DeltaIdxDbPathBase)
+ {
+ if (test == NULL)
+ {
+ ITK_PANIC("NULL as test specification for IdxDbDeltaContext");
+ }
+
+ if (strlen(test) < 5)
+ {
+ ITK_PANIC("Test specification '%s' too short",
+ test);
+ }
+
+ if (test[0] == 'w')
+ {
+ haveInsertBuffer_ = true;
+ }
+ else if (test[0] != 'n')
+ {
+ ITK_PANIC("Test char in position %d: '%c' is wrong",
+ 0,
+ test[0]);
+ }
+
+ if (test[1] == 'r')
+ {
+ haveReader_ = true;
+ }
+ else if (test[1] != 'n')
+ {
+ ITK_PANIC("Test char in position %d: '%c' is wrong",
+ 1,
+ test[1]);
+ }
+
+ if (test[2] == 'd')
+ {
+ haveDels_ = true;
+ }
+ else if (test[2] != 'n')
+ {
+ ITK_PANIC("Test char in position %d: '%c' is wrong",
+ 2,
+ test[2]);
+ }
+
+ if (test[3] != '-')
+ {
+ ITK_PANIC("Test char in position %d: '%c' is wrong",
+ 3,
+ test[3]);
+ }
+
+ const char
+ * stageStr = test + 4;
+
+ if (strcmp("dels_committed", stageStr) == 0)
+ {
+ targetCommitStage_ = Cpix::Impl::CS_DELS_COMMITTED;
+ }
+ else if (strcmp("ibuf_committing", stageStr) == 0)
+ {
+ targetCommitStage_ = Cpix::Impl::CS_IBUF_COMMITTING;
+ }
+ else if (strcmp("ibuf_committed", stageStr) == 0)
+ {
+ targetCommitStage_ = Cpix::Impl::CS_IBUF_COMMITTED;
+ }
+ else if (strcmp("merging", stageStr) == 0)
+ {
+ targetCommitStage_ = Cpix::Impl::CS_MERGING;
+ }
+ else if (strcmp("merged", stageStr) == 0)
+ {
+ targetCommitStage_ = Cpix::Impl::CS_MERGED;
+ }
+ else if (strcmp("complete", stageStr) == 0)
+ {
+ targetCommitStage_ = Cpix::Impl::CS_COMPLETE;
+ }
+ else
+ {
+ ITK_PANIC("Test stage string '%s' is wrong",
+ stageStr);
+ }
+
+ deltaIdxDbPath_ += '\\';
+ deltaIdxDbPath_ += Cpix::Impl::ONE_COMPLETE_SUBDIR;
+ }
+
+
+ virtual void setup() throw (Itk::PanicExc)
+ {
+ using namespace std;
+ using namespace lucene::index;
+ using namespace lucene::store;
+
+ int
+ result;
+
+ if (Cpt::directoryexists(DeltaIdxDbPathBase))
+ {
+
+ result = Cpt::removeall(DeltaIdxDbPathBase);
+
+ if (result != 0)
+ {
+ ITK_PANIC("could not remove dir recursively: %s",
+ DeltaIdxDbPathBase);
+ }
+ }
+
+ result = Cpt::mkdirs(deltaIdxDbPath_.c_str(),
+ 0666);
+
+ if (result != 0)
+ {
+ ITK_PANIC("Could not create dir %s",
+ deltaIdxDbPath_.c_str());
+ }
+
+ try
+ {
+ auto_ptr<IndexWriter>
+ writer(new IndexWriter(deltaIdxDbPath_.c_str(),
+ &analyzer_,
+ true));
+
+ addDocs(writer.get(),
+ DeltaSmsesToStartWith,
+ sizeof(DeltaSmsesToStartWith) / sizeof(DeltaSms));
+
+ writer->optimize();
+ writer->close();
+ }
+ catch (...)
+ {
+ ITK_PANIC("Setting up start stage index failed");
+ }
+
+ if (haveInsertBuffer_)
+ {
+ insertBuf_ = new TestInsertBuf(&analyzer_);
+ }
+
+ if (haveReader_)
+ {
+ // reader_ = IndexReader::open(deltaIdxDbPath_.c_str());
+ reader_.open(deltaIdxDbPath_.c_str(),
+ idxMutex_);
+ }
+ }
+
+
+ virtual void tearDown() throw ()
+ {
+ cleanup();
+ }
+
+
+ ~IdxDbDeltaContext()
+ {
+ cleanup();
+ }
+
+
+
+ void testStartStage(Itk::TestMgr * mgr)
+ {
+ using namespace lucene::index;
+ using namespace std;
+
+ printf("Start state of index:\n");
+
+ {
+ auto_ptr<IndexReader>
+ reader(IndexReader::open(deltaIdxDbPath_.c_str()));
+
+ searchAndPrint(reader.get());
+
+ reader->close();
+ }
+
+ if (haveInsertBuffer_)
+ {
+ addDocs(insertBuf_->getWriter(),
+ DeltaSmsesToAdd,
+ sizeof(DeltaSmsesToAdd) / sizeof(DeltaSms));
+
+ printf("... added extra docs to insert buffer\n");
+ }
+
+ if (haveDels_)
+ {
+ if (!reader_.isOpen())
+ {
+ ITK_PANIC("Reader should be open");
+ }
+
+ for (size_t i = 0;
+ i < sizeof(DeltaSmsesToDelete) / sizeof(wchar_t*);
+ ++i)
+ {
+ auto_ptr<Term>
+ term(new Term(DELTA_ID_FIELD,
+ DeltaSmsesToDelete[i]));
+
+ int
+ result = reader_.getReader()->deleteDocuments(term.get());
+
+ ITK_EXPECT(mgr,
+ result == 1,
+ "Should have deleted exactly one doc");
+
+ printf("... deleted doc %S\n",
+ DeltaSmsesToDelete[i]);
+ }
+ }
+ }
+
+
+
+ struct TestCommitStageExc
+ {
+ Cpix::Impl::CommitStage commitStage_;
+
+ TestCommitStageExc(Cpix::Impl::CommitStage commitStage)
+ : commitStage_(commitStage)
+ {
+ ;
+ }
+ };
+
+
+ /**
+ * Throws a TestCommitStageExc (with the current commit stage
+ * info) if the current commit stage info is the same as the
+ * target commit stage info we want to reach, except if the target
+ * commit stage is the "complete" one.
+ */
+ struct TestCommitStageAction
+ {
+ Cpix::Impl::CommitStage targetCommitStage_;
+
+
+ void operator()(Cpix::Impl::CommitStage commitStage)
+ {
+ if (commitStage >= targetCommitStage_
+ && targetCommitStage_ != Cpix::Impl::CS_COMPLETE)
+ {
+ throw TestCommitStageExc(commitStage);
+ }
+ }
+
+
+ TestCommitStageAction(Cpix::Impl::CommitStage targetCommitStage)
+ : targetCommitStage_(targetCommitStage)
+ {
+ ;
+ }
+ };
+
+
+ void testCommitToDisk(Itk::TestMgr * mgr)
+ {
+ using namespace lucene::search;
+ using namespace Cpix::Impl;
+
+ // The TestCommitStageAction instance we give is to emulate
+ // interrupting the committing-to-disk process at different
+ // stages, and see if we can recover whatever information can
+ // be from whatever has been committed to disk up to that
+ // point. The interrupt is done by throwing a test exception
+ // and catching it.
+
+ /*
+ CommitStage
+ commitStage = targetCommitStage_;
+ */
+ bool
+ properlyInterrupted = false;
+
+ try
+ {
+ CommitToDisk_(DeltaIdxDbPathBase,
+ insertBuf_,
+ reader_,
+ true, // re-read reader_ if there was file i/o
+ idxMutex_,
+ TestCommitStageAction(targetCommitStage_));
+
+ if (targetCommitStage_ == CS_COMPLETE)
+ {
+ properlyInterrupted = true;
+
+ // if there was any file i/o operation, we
+ // must have a new version of idx now
+ if (haveDels_ || haveInsertBuffer_)
+ {
+ std::string
+ deltaIdxDbPath(DeltaIdxDbPathBase);
+ deltaIdxDbPath += '\\';
+ deltaIdxDbPath += Cpix::Impl::OTHER_COMPLETE_SUBDIR;
+
+ ITK_EXPECT(mgr,
+ Cpt::directoryexists(deltaIdxDbPath.c_str()),
+ "Complete, committed, updated idx dir should exist (%s): %s",
+ test_.c_str(),
+ deltaIdxDbPath.c_str());
+ }
+ }
+ else
+ {
+ ITK_EXPECT(mgr,
+ false,
+ "Should have been interrupted at stage %d",
+ targetCommitStage_);
+ }
+ }
+ catch (TestCommitStageExc & exc)
+ {
+ if (exc.commitStage_ == targetCommitStage_)
+ {
+ properlyInterrupted = true;
+ }
+ }
+
+ ITK_EXPECT(mgr,
+ properlyInterrupted,
+ "Did not test-interrupt committing to disk at the required point");
+
+ ITK_EXPECT(mgr,
+ !haveReader_ || reader_.isOpen(),
+ "Expected an open reader back");
+
+ ITK_EXPECT(mgr,
+ (insertBuf_ == NULL
+ || insertBuf_->isEmpty()
+ || targetCommitStage_ < CS_IBUF_COMMITTING),
+ "Expected the (test) insert buffer to be closed");
+
+ }
+
+
+ void testRecoveredStage(Itk::TestMgr * )
+ {
+ if (reader_.isOpen())
+ {
+ printf("Reader (either the original or the re-created):\n");
+ searchAndPrint(reader_.getReader());
+ reader_.close(NULL);
+ }
+
+ printf("\n\nRecovering (most of the) state of the index:\n");
+
+ Cpix::Impl::IdxDbDelta::RecoverReader(DeltaIdxDbPathBase,
+ idxMutex_,
+ &reader_);
+
+ printf("The recovered state of index:\n");
+
+ searchAndPrint(reader_.getReader());
+ }
+
+
+private:
+ void cleanup()
+ {
+ if (insertBuf_ != NULL)
+ {
+ insertBuf_->close();
+ delete insertBuf_;
+ insertBuf_ = NULL;
+ }
+
+ reader_.close(NULL);
+ }
+
+
+ void addDocs(lucene::index::IndexWriter * writer,
+ const DeltaSms * p,
+ size_t size)
+ {
+ using namespace lucene::document;
+ using namespace std;
+
+ const DeltaSms
+ * pEnd = p + size;
+
+ for (; p < pEnd; ++p)
+ {
+ auto_ptr<Document>
+ doc(new Document());
+
+ doc->add(* new Field(DELTA_ID_FIELD,
+ p->id_,
+ Field::STORE_YES | Field::INDEX_UNTOKENIZED));
+ doc->add(* new Field(DELTA_BODY_FIELD,
+ p->body_,
+ Field::STORE_YES | Field::INDEX_TOKENIZED));
+ writer->addDocument(doc.get());
+ doc.reset();
+ }
+ }
+
+
+ void searchAndPrint(lucene::index::IndexReader * reader)
+ {
+ using namespace lucene::document;
+ using namespace lucene::index;
+ using namespace lucene::queryParser;
+ using namespace lucene::search;
+ using namespace std;
+
+ auto_ptr<QueryParser>
+ queryParser(new QueryParser(DELTA_BODY_FIELD,
+ &analyzer_));
+ auto_ptr<Query>
+ query(queryParser->parse(DELTA_QRY_TERM));
+
+ auto_ptr<IndexSearcher>
+ searcher(new IndexSearcher(reader));
+
+ auto_ptr<Hits>
+ hits(searcher->search(query.get()));
+
+ int32_t
+ length = hits->length();
+
+ printf("SEARCHING for '%S' resulted in %d hits:\n",
+ DELTA_QRY_TERM,
+ length);
+
+ for (int32_t i = 0; i < length; ++i)
+ {
+ Document
+ & doc(hits->doc(i));
+
+ printf(" o DOC %S : %S\n",
+ doc.get(DELTA_ID_FIELD),
+ doc.get(DELTA_BODY_FIELD));
+ }
+
+ searcher->close();
+ }
+
+};
+
+
+
+Itk::TesterBase * CreateIdxDbDeltaTest(const char * test)
+{
+ using namespace Itk;
+
+ IdxDbDeltaContext
+ * context = new IdxDbDeltaContext(test);
+ ContextTester
+ * tester = new ContextTester(test,
+ context);
+
+#define TEST "startStage"
+ tester->add(TEST,
+ context,
+ &IdxDbDeltaContext::testStartStage,
+ TEST);
+#undef TEST
+
+#define TEST "commitToDisk"
+ tester->add(TEST,
+ context,
+ &IdxDbDeltaContext::testCommitToDisk);
+#undef TEST
+
+#define TEST "recoveredStage"
+ tester->add(TEST,
+ context,
+ &IdxDbDeltaContext::testRecoveredStage,
+ TEST);
+#undef TEST
+
+ return tester;
+}
+
+
+Itk::TesterBase * CreateIdxDbDeltaTests()
+{
+ using namespace Itk;
+
+ SuiteTester
+ * deltaTests = new SuiteTester("delta");
+
+ const char
+ * deltaTestSpecs[] = {
+ "nnn-dels_committed",
+ "nnn-ibuf_committing",
+ "nnn-ibuf_committed",
+ "nnn-merging",
+ "nnn-merged",
+ "nnn-complete",
+
+ "nrn-dels_committed",
+ "nrn-ibuf_committing",
+ "nrn-ibuf_committed",
+ "nrn-merging",
+ "nrn-merged",
+ "nrn-complete",
+
+ "nrd-dels_committed",
+ "nrd-ibuf_committing",
+ "nrd-ibuf_committed",
+ "nrd-merging",
+ "nrd-merged",
+ "nrd-complete",
+
+ "wnn-dels_committed",
+ "wnn-ibuf_committing",
+ "wnn-ibuf_committed",
+ "wnn-merging",
+ "wnn-merged",
+ "wnn-complete",
+
+ "wrn-dels_committed",
+ "wrn-ibuf_committing",
+ "wrn-ibuf_committed",
+ "wrn-merging",
+ "wrn-merged",
+ "wrn-complete",
+
+ "wrd-dels_committed",
+ "wrd-ibuf_committing",
+ "wrd-ibuf_committed",
+ "wrd-merging",
+ "wrd-merged",
+ "wrd-complete"
+ };
+
+ for (size_t i = 0; i < sizeof(deltaTestSpecs) / sizeof(char*); ++i)
+ {
+ deltaTests->add(CreateIdxDbDeltaTest(deltaTestSpecs[i]));
+ }
+
+ return deltaTests;
+}
+
+
+
+
+
+
+Itk::TesterBase * CreateWhiteBoxTests()
+{
+ using namespace Itk;
+
+ SuiteTester
+ * whiteBox = new SuiteTester("whitebox");
+
+ whiteBox->add("baccoll",
+ &TestBaseAppClassCollision);
+ whiteBox->add("idpcoll",
+ &TestIdxDbPathCollision);
+ whiteBox->add("scrapall",
+ &TestScrapAll);
+
+ whiteBox->add("unifiedSearchParse",
+ &TestUnifiedSearchParse,
+ "unifiedSearchParse");
+
+ whiteBox->add(CreatePageTests());
+
+ whiteBox->add(CreateIdxDbDeltaTests());
+
+ // TODO add more
+
+ return whiteBox;
+}