|
1 /* |
|
2 * Copyright (c) 2010 Nokia Corporation and/or its subsidiary(-ies). |
|
3 * All rights reserved. |
|
4 * This component and the accompanying materials are made available |
|
5 * under the terms of "Eclipse Public License v1.0" |
|
6 * which accompanies this distribution, and is available |
|
7 * at the URL "http://www.eclipse.org/legal/epl-v10.html". |
|
8 * |
|
9 * Initial Contributors: |
|
10 * Nokia Corporation - initial contribution. |
|
11 * |
|
12 * Contributors: |
|
13 * |
|
14 * Description: |
|
15 * |
|
16 */ |
|
17 #include <wchar.h> |
|
18 #include <stddef.h> |
|
19 |
|
20 #include <iostream> |
|
21 |
|
22 #include "cpixidxdb.h" |
|
23 |
|
24 #include "itk.h" |
|
25 |
|
26 #include "config.h" |
|
27 #include "testutils.h" |
|
28 #include "setupsentry.h" |
|
29 |
|
30 // internal (inc/private) headers from cpix - that is why these tests |
|
31 // are whitebox tests |
|
32 #include "cpixtools.h" |
|
33 #include "iidxdb.h" |
|
34 #include "iqrytype.h" |
|
35 #include "cpixexc.h" |
|
36 #include "cpixhits.h" |
|
37 #include "idxdbdelta.h" |
|
38 |
|
39 void TestBaseAppClassCollision(Itk::TestMgr * testMgr) |
|
40 { |
|
41 cpix_Result |
|
42 result; |
|
43 |
|
44 cpix_IdxDb_defineVolume(&result, |
|
45 SMS_QBASEAPPCLASS, |
|
46 NULL); |
|
47 |
|
48 ITK_ASSERT(testMgr, |
|
49 cpix_Succeeded(&result), |
|
50 "Definition of volume (%s, %s) failed.", |
|
51 SMS_QBASEAPPCLASS, |
|
52 "<default>"); |
|
53 |
|
54 cpix_IdxDb_defineVolume(&result, |
|
55 SMS_QBASEAPPCLASS, |
|
56 NULL); |
|
57 |
|
58 ITK_ASSERT(testMgr, |
|
59 cpix_Succeeded(&result), |
|
60 "Re-definition of identical volume (%s, %s) failed.", |
|
61 SMS_QBASEAPPCLASS, |
|
62 "<default>"); |
|
63 |
|
64 const char |
|
65 * dummyPath = "\\dummy\\path"; |
|
66 |
|
67 cpix_IdxDb_defineVolume(&result, |
|
68 SMS_QBASEAPPCLASS, |
|
69 dummyPath); |
|
70 |
|
71 ITK_ASSERT(testMgr, |
|
72 cpix_Failed(&result), |
|
73 "Volume (%s, %s) definition should have failed.", |
|
74 SMS_QBASEAPPCLASS, |
|
75 dummyPath); |
|
76 |
|
77 cpix_IdxDb_undefineVolume("@0:root foo bar"); |
|
78 } |
|
79 |
|
80 |
|
81 #define DUMMY_QBASEAPPCLASS "@0:root dummy" |
|
82 #define DUMMY2_QBASEAPPCLASS "@0:root dummy2" |
|
83 #define DUMMY_IDXDBPATH "c:\\Data\\indexing\\indexdb\\root\\dummy" |
|
84 |
|
85 |
|
86 void TestIdxDbPathCollision(Itk::TestMgr * testMgr) |
|
87 { |
|
88 cpix_Result |
|
89 result; |
|
90 |
|
91 cpix_IdxDb_defineVolume(&result, |
|
92 DUMMY_QBASEAPPCLASS, |
|
93 DUMMY_IDXDBPATH); |
|
94 ITK_ASSERT(testMgr, |
|
95 cpix_Succeeded(&result), |
|
96 "Defining volume %s, %s failed.", |
|
97 DUMMY_QBASEAPPCLASS, |
|
98 DUMMY_IDXDBPATH); |
|
99 |
|
100 cpix_IdxDb_defineVolume(&result, |
|
101 DUMMY2_QBASEAPPCLASS, |
|
102 DUMMY_IDXDBPATH); |
|
103 |
|
104 ITK_ASSERT(testMgr, |
|
105 cpix_Failed(&result), |
|
106 "Defining volume %s, %s should have failed.", |
|
107 DUMMY2_QBASEAPPCLASS, |
|
108 DUMMY_IDXDBPATH); |
|
109 } |
|
110 |
|
111 |
|
112 |
|
113 void TestScrapAll(Itk::TestMgr * testMgr) |
|
114 { |
|
115 cpix_Result |
|
116 result; |
|
117 |
|
118 cpix_IdxDb_defineVolume(&result, |
|
119 SMS_QBASEAPPCLASS, |
|
120 NULL); |
|
121 if (cpix_Succeeded(&result)) |
|
122 { |
|
123 cpix_IdxDb |
|
124 * idxDb = cpix_IdxDb_openDb(&result, |
|
125 SMS_QBASEAPPCLASS, |
|
126 cpix_IDX_CREATE); |
|
127 |
|
128 if (cpix_Succeeded(&result)) |
|
129 { |
|
130 cpix_IdxDb_releaseDb(idxDb); |
|
131 } |
|
132 } |
|
133 |
|
134 ITK_ASSERT(testMgr, |
|
135 cpix_Succeeded(&result), |
|
136 "Definition and creation of IdxDb (%s, %s) failed", |
|
137 SMS_QBASEAPPCLASS, |
|
138 "<default>"); |
|
139 |
|
140 cpix_IdxDb_dbgScrapAll(&result); |
|
141 |
|
142 cpix_IdxDb |
|
143 * idxDb = cpix_IdxDb_openDb(&result, |
|
144 SMS_QBASEAPPCLASS, |
|
145 cpix_IDX_OPEN); |
|
146 |
|
147 if (cpix_Succeeded(&result)) |
|
148 { |
|
149 cpix_IdxDb_releaseDb(idxDb); |
|
150 } |
|
151 |
|
152 ITK_ASSERT(testMgr, |
|
153 cpix_Failed(&result), |
|
154 "Opening IdxDb(%s) should have failed now.", |
|
155 SMS_QBASEAPPCLASS); |
|
156 } |
|
157 |
|
158 const wchar_t * QryStrings[] = { |
|
159 |
|
160 // should pass: |
|
161 L"$foo", |
|
162 L"$bar<1.2,'blabla',3>", |
|
163 L" $ bar < 1.2 , 'blabla' , 3 > ", |
|
164 L"$barf(inner query)", |
|
165 L" $ barf ( inner query ) ", |
|
166 L"$barfology<'bloblo',3.1415>(inner query)", |
|
167 L" $ barfology < 'bloblo' , 3.1415 > ( inner query ) ", |
|
168 L"some query", |
|
169 L"*", |
|
170 L"* AND some more criteria", // special case: "innery query" is the whole |
|
171 |
|
172 // should fail: |
|
173 L"$barf<1.23(blo)", |
|
174 L"$farb<1.23>(blo", |
|
175 L"$farb<1.23>(blo ) oops more data", |
|
176 |
|
177 |
|
178 // end of test data |
|
179 NULL |
|
180 }; |
|
181 |
|
182 |
|
183 void TestUnifiedSearchParse(Itk::TestMgr * ) |
|
184 { |
|
185 using namespace std; |
|
186 |
|
187 printf("Whitebox testing parsing of unified search syntax\n\n"); |
|
188 |
|
189 const wchar_t |
|
190 ** p = QryStrings; |
|
191 |
|
192 for (; *p != NULL; ++p) |
|
193 { |
|
194 printf("Trying to parse %S:\n", |
|
195 *p); |
|
196 |
|
197 try { |
|
198 Cpix::QryCall |
|
199 qc(*p); |
|
200 |
|
201 printf("\to qry type id: %S\n", |
|
202 qc.qryTypeId_.c_str()); |
|
203 |
|
204 printf("\to qry arguments: "); |
|
205 list<wstring>::const_iterator |
|
206 i = qc.args_.begin(), |
|
207 end = qc.args_.end(); |
|
208 for (; i != end; ++i) |
|
209 { |
|
210 printf("%S ", |
|
211 i->c_str()); |
|
212 } |
|
213 printf("\n"); |
|
214 |
|
215 printf("\to inner qry: '%S'\n", |
|
216 qc.innerQryStr_.c_str()); |
|
217 |
|
218 } catch (CpixExc & cpixExc) { |
|
219 printf("Failed to parse: %S\n", |
|
220 cpixExc.wWhat()); |
|
221 } catch (...) { |
|
222 printf("Failed t parse: for unknown reasons\n"); |
|
223 } |
|
224 } |
|
225 } |
|
226 |
|
227 |
|
228 class PageContext : public Itk::ITestContext |
|
229 { |
|
230 private: |
|
231 // |
|
232 // private members |
|
233 // |
|
234 SmsIdxUtil * util_; |
|
235 cpix_Analyzer * analyzer_; |
|
236 cpix_Query * query_; |
|
237 cpix_QueryParser * queryParser_; |
|
238 |
|
239 size_t pageSize_; |
|
240 size_t expectedHitCount_; |
|
241 |
|
242 public: |
|
243 // |
|
244 // from interface Itk::ITestContext |
|
245 // |
|
246 virtual void setup() throw (Itk::PanicExc) |
|
247 { |
|
248 SetupSentry |
|
249 ss(*this); |
|
250 |
|
251 util_ = new SmsIdxUtil; |
|
252 util_->init(); |
|
253 |
|
254 cpix_Result |
|
255 result; |
|
256 |
|
257 analyzer_ = cpix_CreateSimpleAnalyzer(&result); |
|
258 |
|
259 if (analyzer_ == NULL) |
|
260 { |
|
261 ITK_PANIC("Could not create analyzer"); |
|
262 } |
|
263 |
|
264 queryParser_ = cpix_QueryParser_create(&result, |
|
265 LBODY_FIELD, |
|
266 analyzer_); |
|
267 |
|
268 if (queryParser_ == NULL) |
|
269 { |
|
270 ITK_PANIC("Could not create query parser"); |
|
271 } |
|
272 |
|
273 query_ = cpix_QueryParser_parse(queryParser_, |
|
274 L"abc"); |
|
275 |
|
276 if (cpix_Failed(queryParser_) || query_ == NULL) |
|
277 { |
|
278 ITK_PANIC("Could not parse query string"); |
|
279 } |
|
280 |
|
281 ss.setupComplete(); |
|
282 } |
|
283 |
|
284 |
|
285 virtual void tearDown() throw () |
|
286 { |
|
287 cleanup(); |
|
288 } |
|
289 |
|
290 |
|
291 virtual ~PageContext() |
|
292 { |
|
293 cleanup(); |
|
294 } |
|
295 |
|
296 |
|
297 // |
|
298 // public operations |
|
299 // |
|
300 PageContext() |
|
301 : util_(NULL), |
|
302 analyzer_(NULL), |
|
303 query_(NULL), |
|
304 queryParser_(NULL), |
|
305 pageSize_(Cpix::IdxDbMgr::instance()->getClHitsPageSize()) |
|
306 { |
|
307 expectedHitCount_ = 4 * pageSize_ - 3; |
|
308 } |
|
309 |
|
310 |
|
311 void testAddSome(Itk::TestMgr * mgr) |
|
312 { |
|
313 std::wstring |
|
314 firstBody, |
|
315 secondBody; |
|
316 |
|
317 for (int i = 0; i < expectedHitCount_; ++i) |
|
318 { |
|
319 generateTwoSmsBodies(i, |
|
320 firstBody, |
|
321 secondBody); |
|
322 |
|
323 util_->indexSms(2*i, |
|
324 firstBody.c_str(), |
|
325 analyzer_, |
|
326 mgr); |
|
327 |
|
328 util_->indexSms(2*i + 1, |
|
329 secondBody.c_str(), |
|
330 analyzer_, |
|
331 mgr); |
|
332 if ((i % 5) == 0) |
|
333 { |
|
334 ITK_DBGMSG(mgr, |
|
335 "."); |
|
336 } |
|
337 } |
|
338 util_->flush(); |
|
339 } |
|
340 |
|
341 |
|
342 void testSearchPages(Itk::TestMgr * mgr) |
|
343 { |
|
344 cpix_Hits |
|
345 * hits = cpix_IdxDb_search(util_->idxDb(), |
|
346 query_); |
|
347 |
|
348 if (cpix_Failed(util_->idxDb())) |
|
349 { |
|
350 ITK_EXPECT(mgr, |
|
351 false, |
|
352 "Failed to search index"); |
|
353 cpix_ClearError(util_->idxDb()); |
|
354 } |
|
355 else |
|
356 { |
|
357 int32_t |
|
358 hitsLength = cpix_Hits_length(hits); |
|
359 |
|
360 ITK_EXPECT(mgr, |
|
361 expectedHitCount_ == hitsLength, |
|
362 "Did not get expected hitcounts (%d), but %d", |
|
363 expectedHitCount_, |
|
364 hitsLength); |
|
365 |
|
366 |
|
367 printf("Got %d hits\n", |
|
368 hitsLength); |
|
369 |
|
370 search(hits, |
|
371 0, |
|
372 2 * pageSize_, |
|
373 mgr); |
|
374 |
|
375 search(hits, |
|
376 3 * pageSize_, |
|
377 4 * pageSize_, |
|
378 mgr); |
|
379 |
|
380 search(hits, |
|
381 0, |
|
382 pageSize_, |
|
383 mgr); |
|
384 |
|
385 search(hits, |
|
386 2 * pageSize_, |
|
387 4 * pageSize_, |
|
388 mgr); |
|
389 |
|
390 } |
|
391 |
|
392 cpix_Hits_destroy( hits ); |
|
393 } |
|
394 |
|
395 |
|
396 private: |
|
397 // |
|
398 // private methods |
|
399 // |
|
400 void cleanup() |
|
401 { |
|
402 delete util_; |
|
403 util_ = NULL; |
|
404 |
|
405 cpix_Analyzer_destroy(analyzer_); |
|
406 analyzer_ = NULL; |
|
407 |
|
408 cpix_Query_destroy(query_); |
|
409 query_ = NULL;; |
|
410 |
|
411 cpix_QueryParser_destroy(queryParser_); |
|
412 queryParser_ = NULL; |
|
413 } |
|
414 |
|
415 |
|
416 void generateTwoSmsBodies(int idx, |
|
417 std::wstring & firstBody, |
|
418 std::wstring & secondBody) |
|
419 { |
|
420 static wchar_t |
|
421 firstBuf[] = L"hello abc ", |
|
422 secondBuf[] = L"hola, xyz "; |
|
423 // ^ |
|
424 enum // | |
|
425 { // | |
|
426 VARIANT_POS = 10 |
|
427 }; |
|
428 |
|
429 wchar_t |
|
430 * p1 = firstBuf + VARIANT_POS, |
|
431 * p2 = secondBuf + VARIANT_POS; |
|
432 |
|
433 static const int |
|
434 radix = 'z' - 'a' + 1; |
|
435 |
|
436 int |
|
437 length = sizeof(firstBuf) / sizeof(wchar_t) - VARIANT_POS; |
|
438 |
|
439 bool |
|
440 firstRun = true; |
|
441 |
|
442 while (length > 0) |
|
443 { |
|
444 if (!firstRun && idx == 0) |
|
445 { |
|
446 *p1 = L' '; |
|
447 *p2 = L' '; |
|
448 break; |
|
449 } |
|
450 |
|
451 int |
|
452 digit = idx % radix; |
|
453 |
|
454 wchar_t |
|
455 charDigit = L'a' + digit; |
|
456 |
|
457 *p1 = charDigit; |
|
458 *p2 = charDigit; |
|
459 |
|
460 ++p1; |
|
461 ++p2; |
|
462 |
|
463 idx = idx / radix; |
|
464 firstRun = false; |
|
465 } |
|
466 |
|
467 firstBody = firstBuf; |
|
468 secondBody = secondBuf; |
|
469 } |
|
470 |
|
471 |
|
472 void search(cpix_Hits * hits, |
|
473 int32_t from, |
|
474 int32_t to, |
|
475 Itk::TestMgr * mgr) |
|
476 { |
|
477 printf("Printing hit docs from hit doc idx %d to %d\n", |
|
478 from, |
|
479 to); |
|
480 |
|
481 for (int32_t hitDocIdx = from; hitDocIdx < to; ++hitDocIdx) |
|
482 { |
|
483 cpix_Document |
|
484 doc; |
|
485 |
|
486 cpix_Hits_doc(hits, |
|
487 hitDocIdx, |
|
488 &doc); |
|
489 |
|
490 if (cpix_Failed(hits)) |
|
491 { |
|
492 wchar_t |
|
493 buf[92]; |
|
494 cpix_Error_report(hits->err_, |
|
495 buf, |
|
496 sizeof(buf) / sizeof(wchar_t)); |
|
497 printf("Failed to get hit doc %d: %S\n", |
|
498 hitDocIdx, |
|
499 buf); |
|
500 cpix_ClearError(hits); |
|
501 break; |
|
502 } |
|
503 |
|
504 util_->printHit(&doc, |
|
505 mgr); |
|
506 } |
|
507 } |
|
508 |
|
509 |
|
510 }; |
|
511 |
|
512 |
|
513 Itk::TesterBase * CreatePageTests() |
|
514 { |
|
515 using namespace Itk; |
|
516 |
|
517 PageContext |
|
518 * pageContext = new PageContext; |
|
519 ContextTester |
|
520 * ctxtTester = new ContextTester("page", |
|
521 pageContext); |
|
522 |
|
523 #define TEST "addSome" |
|
524 ctxtTester->add(TEST, |
|
525 pageContext, |
|
526 &PageContext::testAddSome, |
|
527 TEST); |
|
528 #undef TEST |
|
529 |
|
530 #define TEST "searchPages" |
|
531 ctxtTester->add(TEST, |
|
532 pageContext, |
|
533 &PageContext::testSearchPages, |
|
534 TEST); |
|
535 #undef TEST |
|
536 |
|
537 return ctxtTester; |
|
538 } |
|
539 |
|
540 |
|
541 |
|
542 |
|
543 const wchar_t DELTA_QRY_TERM[] = L"hello"; |
|
544 |
|
545 struct DeltaSms |
|
546 { |
|
547 const wchar_t * id_; |
|
548 const wchar_t * body_; |
|
549 }; |
|
550 |
|
551 |
|
552 static DeltaSms DeltaSmsesToStartWith[] = { |
|
553 { L"0", L"Hello it is a nice day" }, |
|
554 { L"1", L"Goodbye and fare you well" }, |
|
555 { L"2", L"Did you say hello to her?" }, // to delete |
|
556 { L"3", L"No I said hola" }, // to delete |
|
557 }; |
|
558 |
|
559 |
|
560 const wchar_t * DeltaSmsesToDelete[] = { |
|
561 L"2", |
|
562 L"3" |
|
563 }; |
|
564 |
|
565 |
|
566 static DeltaSms DeltaSmsesToAdd[] = { |
|
567 { L"4", L"Do hello, hallo and hullo mean the same?" }, |
|
568 { L"5", L"Yes, they just spell different" }, |
|
569 }; |
|
570 |
|
571 |
|
572 const char DeltaIdxDbPathBase[] = "c:\\data\\cpixunittest\\_testidx"; |
|
573 |
|
574 const wchar_t DELTA_ID_FIELD[] = L"id"; |
|
575 const wchar_t DELTA_BODY_FIELD[] = L"body"; |
|
576 |
|
577 |
|
578 class TestInsertBuf : public Cpix::Impl::IInsertBuf |
|
579 { |
|
580 private: |
|
581 lucene::index::IndexWriter * writer_; |
|
582 lucene::store::TransactionalRAMDirectory * ramDir_; |
|
583 |
|
584 public: |
|
585 // |
|
586 // from interface Cpix::Impl::IInsertBuf |
|
587 // |
|
588 virtual void close() |
|
589 { |
|
590 if (writer_ != NULL) |
|
591 { |
|
592 writer_->close(); |
|
593 delete writer_; |
|
594 writer_ = NULL; |
|
595 } |
|
596 |
|
597 if (ramDir_ != NULL) |
|
598 { |
|
599 ramDir_->close(); |
|
600 _CLDECDELETE(ramDir_); |
|
601 ramDir_ = NULL; |
|
602 } |
|
603 } |
|
604 |
|
605 |
|
606 virtual bool isEmpty() const |
|
607 { |
|
608 return ramDir_ == NULL; |
|
609 } |
|
610 |
|
611 |
|
612 virtual lucene::store::TransactionalRAMDirectory * getRAMDir() |
|
613 { |
|
614 if (isEmpty()) |
|
615 { |
|
616 THROW_CPIXEXC(PL_ERROR "Accessing empty ram dir"); |
|
617 } |
|
618 |
|
619 if (writer_ != NULL) |
|
620 { |
|
621 writer_->close(); |
|
622 delete writer_; |
|
623 writer_ = NULL; |
|
624 } |
|
625 |
|
626 return ramDir_; |
|
627 } |
|
628 |
|
629 |
|
630 virtual ~TestInsertBuf() |
|
631 { |
|
632 close(); |
|
633 } |
|
634 |
|
635 |
|
636 TestInsertBuf(lucene::analysis::standard::StandardAnalyzer * analyzer) |
|
637 : writer_(NULL), |
|
638 ramDir_(NULL) |
|
639 { |
|
640 ramDir_ = _CLNEW lucene::store::TransactionalRAMDirectory(); |
|
641 writer_ = new lucene::index::IndexWriter(ramDir_, |
|
642 analyzer, |
|
643 true, // create |
|
644 false); |
|
645 } |
|
646 |
|
647 |
|
648 lucene::index::IndexWriter * getWriter() |
|
649 { |
|
650 if (writer_ == NULL) |
|
651 { |
|
652 THROW_CPIXEXC(PL_ERROR "messed up test case"); |
|
653 } |
|
654 |
|
655 return writer_; |
|
656 } |
|
657 }; |
|
658 |
|
659 |
|
660 class TestIdxReader : public Cpix::Impl::IIdxReader |
|
661 { |
|
662 private: |
|
663 lucene::index::IndexReader * reader_; |
|
664 |
|
665 public: |
|
666 // |
|
667 // from Cpix::Impl::IIdxReader interface |
|
668 // |
|
669 virtual void open(const char * clIdxPath, |
|
670 Cpt::Mutex & dirMutex) |
|
671 { |
|
672 if (reader_ != NULL) |
|
673 { |
|
674 THROW_CPIXEXC(PL_ERROR "re-opening a reader"); |
|
675 } |
|
676 |
|
677 Cpt::SyncRegion |
|
678 sr(dirMutex); |
|
679 |
|
680 reader_ = lucene::index::IndexReader::open(clIdxPath); |
|
681 } |
|
682 |
|
683 |
|
684 virtual void reopen(Cpt::Mutex & dirMutex, |
|
685 const char * clIdxPath, |
|
686 bool reRead) |
|
687 { |
|
688 if (reader_ != NULL) |
|
689 { |
|
690 Cpt::SyncRegion |
|
691 sr(dirMutex); |
|
692 |
|
693 reader_->close(); |
|
694 delete reader_; |
|
695 reader_ = NULL; |
|
696 |
|
697 if (reRead) |
|
698 { |
|
699 reader_ = lucene::index::IndexReader::open(clIdxPath); |
|
700 } |
|
701 } |
|
702 } |
|
703 |
|
704 |
|
705 virtual bool commitIfNecessary(Cpt::Mutex & dirMutex) |
|
706 { |
|
707 bool |
|
708 rv = false; |
|
709 |
|
710 if (reader_ != NULL && reader_->hasDeletions()) |
|
711 { |
|
712 Cpt::SyncRegion |
|
713 sr(dirMutex); |
|
714 |
|
715 reader_->commit(); |
|
716 |
|
717 rv = true; |
|
718 } |
|
719 |
|
720 return rv; |
|
721 } |
|
722 |
|
723 // |
|
724 // Lifetime mgmt |
|
725 // |
|
726 virtual ~TestIdxReader() |
|
727 { |
|
728 close(NULL); |
|
729 } |
|
730 |
|
731 |
|
732 TestIdxReader() |
|
733 : reader_(NULL) |
|
734 { |
|
735 ; |
|
736 } |
|
737 |
|
738 |
|
739 // |
|
740 // Own operations |
|
741 // |
|
742 void close(Cpt::Mutex * dirMutex) |
|
743 { |
|
744 if (reader_ != NULL) |
|
745 { |
|
746 // lock if we have something to lock on |
|
747 std::auto_ptr<Cpt::SyncRegion> |
|
748 sr; |
|
749 |
|
750 if (dirMutex != NULL) |
|
751 { |
|
752 sr.reset(new Cpt::SyncRegion(*dirMutex)); |
|
753 } |
|
754 |
|
755 reader_->close(); |
|
756 delete reader_; |
|
757 reader_ = NULL; |
|
758 } |
|
759 } |
|
760 |
|
761 |
|
762 bool isOpen() const |
|
763 { |
|
764 return reader_ != NULL; |
|
765 } |
|
766 |
|
767 |
|
768 lucene::index::IndexReader * getReader() |
|
769 { |
|
770 if (!isOpen()) |
|
771 { |
|
772 THROW_CPIXEXC(PL_ERROR "accessing a closed reader"); |
|
773 } |
|
774 |
|
775 return reader_; |
|
776 } |
|
777 }; |
|
778 |
|
779 |
|
780 |
|
781 class IdxDbDeltaContext : public Itk::ITestContext |
|
782 { |
|
783 private: |
|
784 std::string test_; |
|
785 |
|
786 bool haveInsertBuffer_; |
|
787 bool haveReader_; |
|
788 bool haveDels_; |
|
789 Cpix::Impl::CommitStage targetCommitStage_; |
|
790 |
|
791 TestInsertBuf * insertBuf_; |
|
792 TestIdxReader reader_; |
|
793 |
|
794 lucene::analysis::standard::StandardAnalyzer analyzer_; |
|
795 |
|
796 std::string deltaIdxDbPath_; |
|
797 Cpt::Mutex idxMutex_; |
|
798 |
|
799 public: |
|
800 |
|
801 /** |
|
802 * @param test syntax: [nw][nr][nd]-(dels_committed|ibuf_committing|ibuf_committed|merging|merged|complete) |
|
803 * |
|
804 * Where |
|
805 * |
|
806 * (a) n,w tell if there is an insertbuffer (writer + ramdir) or not |
|
807 * (b) n,r tell if there is a reader or not |
|
808 * (c) n,d tell if there were deletions on reader or not |
|
809 * (d) the keyword after dash tell the target commit stage (that |
|
810 * should be reached) |
|
811 * |
|
812 * |
|
813 */ |
|
814 IdxDbDeltaContext(const char * test) |
|
815 : test_(test), |
|
816 haveInsertBuffer_(false), |
|
817 haveReader_(false), |
|
818 haveDels_(false), |
|
819 targetCommitStage_(Cpix::Impl::CS_DELS_COMMITTED), |
|
820 insertBuf_(NULL), |
|
821 deltaIdxDbPath_(DeltaIdxDbPathBase) |
|
822 { |
|
823 if (test == NULL) |
|
824 { |
|
825 ITK_PANIC("NULL as test specification for IdxDbDeltaContext"); |
|
826 } |
|
827 |
|
828 if (strlen(test) < 5) |
|
829 { |
|
830 ITK_PANIC("Test specification '%s' too short", |
|
831 test); |
|
832 } |
|
833 |
|
834 if (test[0] == 'w') |
|
835 { |
|
836 haveInsertBuffer_ = true; |
|
837 } |
|
838 else if (test[0] != 'n') |
|
839 { |
|
840 ITK_PANIC("Test char in position %d: '%c' is wrong", |
|
841 0, |
|
842 test[0]); |
|
843 } |
|
844 |
|
845 if (test[1] == 'r') |
|
846 { |
|
847 haveReader_ = true; |
|
848 } |
|
849 else if (test[1] != 'n') |
|
850 { |
|
851 ITK_PANIC("Test char in position %d: '%c' is wrong", |
|
852 1, |
|
853 test[1]); |
|
854 } |
|
855 |
|
856 if (test[2] == 'd') |
|
857 { |
|
858 haveDels_ = true; |
|
859 } |
|
860 else if (test[2] != 'n') |
|
861 { |
|
862 ITK_PANIC("Test char in position %d: '%c' is wrong", |
|
863 2, |
|
864 test[2]); |
|
865 } |
|
866 |
|
867 if (test[3] != '-') |
|
868 { |
|
869 ITK_PANIC("Test char in position %d: '%c' is wrong", |
|
870 3, |
|
871 test[3]); |
|
872 } |
|
873 |
|
874 const char |
|
875 * stageStr = test + 4; |
|
876 |
|
877 if (strcmp("dels_committed", stageStr) == 0) |
|
878 { |
|
879 targetCommitStage_ = Cpix::Impl::CS_DELS_COMMITTED; |
|
880 } |
|
881 else if (strcmp("ibuf_committing", stageStr) == 0) |
|
882 { |
|
883 targetCommitStage_ = Cpix::Impl::CS_IBUF_COMMITTING; |
|
884 } |
|
885 else if (strcmp("ibuf_committed", stageStr) == 0) |
|
886 { |
|
887 targetCommitStage_ = Cpix::Impl::CS_IBUF_COMMITTED; |
|
888 } |
|
889 else if (strcmp("merging", stageStr) == 0) |
|
890 { |
|
891 targetCommitStage_ = Cpix::Impl::CS_MERGING; |
|
892 } |
|
893 else if (strcmp("merged", stageStr) == 0) |
|
894 { |
|
895 targetCommitStage_ = Cpix::Impl::CS_MERGED; |
|
896 } |
|
897 else if (strcmp("complete", stageStr) == 0) |
|
898 { |
|
899 targetCommitStage_ = Cpix::Impl::CS_COMPLETE; |
|
900 } |
|
901 else |
|
902 { |
|
903 ITK_PANIC("Test stage string '%s' is wrong", |
|
904 stageStr); |
|
905 } |
|
906 |
|
907 deltaIdxDbPath_ += '\\'; |
|
908 deltaIdxDbPath_ += Cpix::Impl::ONE_COMPLETE_SUBDIR; |
|
909 } |
|
910 |
|
911 |
|
912 virtual void setup() throw (Itk::PanicExc) |
|
913 { |
|
914 using namespace std; |
|
915 using namespace lucene::index; |
|
916 using namespace lucene::store; |
|
917 |
|
918 int |
|
919 result; |
|
920 |
|
921 if (Cpt::directoryexists(DeltaIdxDbPathBase)) |
|
922 { |
|
923 |
|
924 result = Cpt::removeall(DeltaIdxDbPathBase); |
|
925 |
|
926 if (result != 0) |
|
927 { |
|
928 ITK_PANIC("could not remove dir recursively: %s", |
|
929 DeltaIdxDbPathBase); |
|
930 } |
|
931 } |
|
932 |
|
933 result = Cpt::mkdirs(deltaIdxDbPath_.c_str(), |
|
934 0666); |
|
935 |
|
936 if (result != 0) |
|
937 { |
|
938 ITK_PANIC("Could not create dir %s", |
|
939 deltaIdxDbPath_.c_str()); |
|
940 } |
|
941 |
|
942 try |
|
943 { |
|
944 auto_ptr<IndexWriter> |
|
945 writer(new IndexWriter(deltaIdxDbPath_.c_str(), |
|
946 &analyzer_, |
|
947 true)); |
|
948 |
|
949 addDocs(writer.get(), |
|
950 DeltaSmsesToStartWith, |
|
951 sizeof(DeltaSmsesToStartWith) / sizeof(DeltaSms)); |
|
952 |
|
953 writer->optimize(); |
|
954 writer->close(); |
|
955 } |
|
956 catch (...) |
|
957 { |
|
958 ITK_PANIC("Setting up start stage index failed"); |
|
959 } |
|
960 |
|
961 if (haveInsertBuffer_) |
|
962 { |
|
963 insertBuf_ = new TestInsertBuf(&analyzer_); |
|
964 } |
|
965 |
|
966 if (haveReader_) |
|
967 { |
|
968 // reader_ = IndexReader::open(deltaIdxDbPath_.c_str()); |
|
969 reader_.open(deltaIdxDbPath_.c_str(), |
|
970 idxMutex_); |
|
971 } |
|
972 } |
|
973 |
|
974 |
|
975 virtual void tearDown() throw () |
|
976 { |
|
977 cleanup(); |
|
978 } |
|
979 |
|
980 |
|
981 ~IdxDbDeltaContext() |
|
982 { |
|
983 cleanup(); |
|
984 } |
|
985 |
|
986 |
|
987 |
|
988 void testStartStage(Itk::TestMgr * mgr) |
|
989 { |
|
990 using namespace lucene::index; |
|
991 using namespace std; |
|
992 |
|
993 printf("Start state of index:\n"); |
|
994 |
|
995 { |
|
996 auto_ptr<IndexReader> |
|
997 reader(IndexReader::open(deltaIdxDbPath_.c_str())); |
|
998 |
|
999 searchAndPrint(reader.get()); |
|
1000 |
|
1001 reader->close(); |
|
1002 } |
|
1003 |
|
1004 if (haveInsertBuffer_) |
|
1005 { |
|
1006 addDocs(insertBuf_->getWriter(), |
|
1007 DeltaSmsesToAdd, |
|
1008 sizeof(DeltaSmsesToAdd) / sizeof(DeltaSms)); |
|
1009 |
|
1010 printf("... added extra docs to insert buffer\n"); |
|
1011 } |
|
1012 |
|
1013 if (haveDels_) |
|
1014 { |
|
1015 if (!reader_.isOpen()) |
|
1016 { |
|
1017 ITK_PANIC("Reader should be open"); |
|
1018 } |
|
1019 |
|
1020 for (size_t i = 0; |
|
1021 i < sizeof(DeltaSmsesToDelete) / sizeof(wchar_t*); |
|
1022 ++i) |
|
1023 { |
|
1024 auto_ptr<Term> |
|
1025 term(new Term(DELTA_ID_FIELD, |
|
1026 DeltaSmsesToDelete[i])); |
|
1027 |
|
1028 int |
|
1029 result = reader_.getReader()->deleteDocuments(term.get()); |
|
1030 |
|
1031 ITK_EXPECT(mgr, |
|
1032 result == 1, |
|
1033 "Should have deleted exactly one doc"); |
|
1034 |
|
1035 printf("... deleted doc %S\n", |
|
1036 DeltaSmsesToDelete[i]); |
|
1037 } |
|
1038 } |
|
1039 } |
|
1040 |
|
1041 |
|
1042 |
|
1043 struct TestCommitStageExc |
|
1044 { |
|
1045 Cpix::Impl::CommitStage commitStage_; |
|
1046 |
|
1047 TestCommitStageExc(Cpix::Impl::CommitStage commitStage) |
|
1048 : commitStage_(commitStage) |
|
1049 { |
|
1050 ; |
|
1051 } |
|
1052 }; |
|
1053 |
|
1054 |
|
1055 /** |
|
1056 * Throws a TestCommitStageExc (with the current commit stage |
|
1057 * info) if the current commit stage info is the same as the |
|
1058 * target commit stage info we want to reach, except if the target |
|
1059 * commit stage is the "complete" one. |
|
1060 */ |
|
1061 struct TestCommitStageAction |
|
1062 { |
|
1063 Cpix::Impl::CommitStage targetCommitStage_; |
|
1064 |
|
1065 |
|
1066 void operator()(Cpix::Impl::CommitStage commitStage) |
|
1067 { |
|
1068 if (commitStage >= targetCommitStage_ |
|
1069 && targetCommitStage_ != Cpix::Impl::CS_COMPLETE) |
|
1070 { |
|
1071 throw TestCommitStageExc(commitStage); |
|
1072 } |
|
1073 } |
|
1074 |
|
1075 |
|
1076 TestCommitStageAction(Cpix::Impl::CommitStage targetCommitStage) |
|
1077 : targetCommitStage_(targetCommitStage) |
|
1078 { |
|
1079 ; |
|
1080 } |
|
1081 }; |
|
1082 |
|
1083 |
|
1084 void testCommitToDisk(Itk::TestMgr * mgr) |
|
1085 { |
|
1086 using namespace lucene::search; |
|
1087 using namespace Cpix::Impl; |
|
1088 |
|
1089 // The TestCommitStageAction instance we give is to emulate |
|
1090 // interrupting the committing-to-disk process at different |
|
1091 // stages, and see if we can recover whatever information can |
|
1092 // be from whatever has been committed to disk up to that |
|
1093 // point. The interrupt is done by throwing a test exception |
|
1094 // and catching it. |
|
1095 |
|
1096 /* |
|
1097 CommitStage |
|
1098 commitStage = targetCommitStage_; |
|
1099 */ |
|
1100 bool |
|
1101 properlyInterrupted = false; |
|
1102 |
|
1103 try |
|
1104 { |
|
1105 CommitToDisk_(DeltaIdxDbPathBase, |
|
1106 insertBuf_, |
|
1107 reader_, |
|
1108 true, // re-read reader_ if there was file i/o |
|
1109 idxMutex_, |
|
1110 TestCommitStageAction(targetCommitStage_)); |
|
1111 |
|
1112 if (targetCommitStage_ == CS_COMPLETE) |
|
1113 { |
|
1114 properlyInterrupted = true; |
|
1115 |
|
1116 // if there was any file i/o operation, we |
|
1117 // must have a new version of idx now |
|
1118 if (haveDels_ || haveInsertBuffer_) |
|
1119 { |
|
1120 std::string |
|
1121 deltaIdxDbPath(DeltaIdxDbPathBase); |
|
1122 deltaIdxDbPath += '\\'; |
|
1123 deltaIdxDbPath += Cpix::Impl::OTHER_COMPLETE_SUBDIR; |
|
1124 |
|
1125 ITK_EXPECT(mgr, |
|
1126 Cpt::directoryexists(deltaIdxDbPath.c_str()), |
|
1127 "Complete, committed, updated idx dir should exist (%s): %s", |
|
1128 test_.c_str(), |
|
1129 deltaIdxDbPath.c_str()); |
|
1130 } |
|
1131 } |
|
1132 else |
|
1133 { |
|
1134 ITK_EXPECT(mgr, |
|
1135 false, |
|
1136 "Should have been interrupted at stage %d", |
|
1137 targetCommitStage_); |
|
1138 } |
|
1139 } |
|
1140 catch (TestCommitStageExc & exc) |
|
1141 { |
|
1142 if (exc.commitStage_ == targetCommitStage_) |
|
1143 { |
|
1144 properlyInterrupted = true; |
|
1145 } |
|
1146 } |
|
1147 |
|
1148 ITK_EXPECT(mgr, |
|
1149 properlyInterrupted, |
|
1150 "Did not test-interrupt committing to disk at the required point"); |
|
1151 |
|
1152 ITK_EXPECT(mgr, |
|
1153 !haveReader_ || reader_.isOpen(), |
|
1154 "Expected an open reader back"); |
|
1155 |
|
1156 ITK_EXPECT(mgr, |
|
1157 (insertBuf_ == NULL |
|
1158 || insertBuf_->isEmpty() |
|
1159 || targetCommitStage_ < CS_IBUF_COMMITTING), |
|
1160 "Expected the (test) insert buffer to be closed"); |
|
1161 |
|
1162 } |
|
1163 |
|
1164 |
|
1165 void testRecoveredStage(Itk::TestMgr * ) |
|
1166 { |
|
1167 if (reader_.isOpen()) |
|
1168 { |
|
1169 printf("Reader (either the original or the re-created):\n"); |
|
1170 searchAndPrint(reader_.getReader()); |
|
1171 reader_.close(NULL); |
|
1172 } |
|
1173 |
|
1174 printf("\n\nRecovering (most of the) state of the index:\n"); |
|
1175 |
|
1176 Cpix::Impl::IdxDbDelta::RecoverReader(DeltaIdxDbPathBase, |
|
1177 idxMutex_, |
|
1178 &reader_); |
|
1179 |
|
1180 printf("The recovered state of index:\n"); |
|
1181 |
|
1182 searchAndPrint(reader_.getReader()); |
|
1183 } |
|
1184 |
|
1185 |
|
1186 private: |
|
1187 void cleanup() |
|
1188 { |
|
1189 if (insertBuf_ != NULL) |
|
1190 { |
|
1191 insertBuf_->close(); |
|
1192 delete insertBuf_; |
|
1193 insertBuf_ = NULL; |
|
1194 } |
|
1195 |
|
1196 reader_.close(NULL); |
|
1197 } |
|
1198 |
|
1199 |
|
1200 void addDocs(lucene::index::IndexWriter * writer, |
|
1201 const DeltaSms * p, |
|
1202 size_t size) |
|
1203 { |
|
1204 using namespace lucene::document; |
|
1205 using namespace std; |
|
1206 |
|
1207 const DeltaSms |
|
1208 * pEnd = p + size; |
|
1209 |
|
1210 for (; p < pEnd; ++p) |
|
1211 { |
|
1212 auto_ptr<Document> |
|
1213 doc(new Document()); |
|
1214 |
|
1215 doc->add(* new Field(DELTA_ID_FIELD, |
|
1216 p->id_, |
|
1217 Field::STORE_YES | Field::INDEX_UNTOKENIZED)); |
|
1218 doc->add(* new Field(DELTA_BODY_FIELD, |
|
1219 p->body_, |
|
1220 Field::STORE_YES | Field::INDEX_TOKENIZED)); |
|
1221 writer->addDocument(doc.get()); |
|
1222 doc.reset(); |
|
1223 } |
|
1224 } |
|
1225 |
|
1226 |
|
1227 void searchAndPrint(lucene::index::IndexReader * reader) |
|
1228 { |
|
1229 using namespace lucene::document; |
|
1230 using namespace lucene::index; |
|
1231 using namespace lucene::queryParser; |
|
1232 using namespace lucene::search; |
|
1233 using namespace std; |
|
1234 |
|
1235 auto_ptr<QueryParser> |
|
1236 queryParser(new QueryParser(DELTA_BODY_FIELD, |
|
1237 &analyzer_)); |
|
1238 auto_ptr<Query> |
|
1239 query(queryParser->parse(DELTA_QRY_TERM)); |
|
1240 |
|
1241 auto_ptr<IndexSearcher> |
|
1242 searcher(new IndexSearcher(reader)); |
|
1243 |
|
1244 auto_ptr<Hits> |
|
1245 hits(searcher->search(query.get())); |
|
1246 |
|
1247 int32_t |
|
1248 length = hits->length(); |
|
1249 |
|
1250 printf("SEARCHING for '%S' resulted in %d hits:\n", |
|
1251 DELTA_QRY_TERM, |
|
1252 length); |
|
1253 |
|
1254 for (int32_t i = 0; i < length; ++i) |
|
1255 { |
|
1256 Document |
|
1257 & doc(hits->doc(i)); |
|
1258 |
|
1259 printf(" o DOC %S : %S\n", |
|
1260 doc.get(DELTA_ID_FIELD), |
|
1261 doc.get(DELTA_BODY_FIELD)); |
|
1262 } |
|
1263 |
|
1264 searcher->close(); |
|
1265 } |
|
1266 |
|
1267 }; |
|
1268 |
|
1269 |
|
1270 |
|
1271 Itk::TesterBase * CreateIdxDbDeltaTest(const char * test) |
|
1272 { |
|
1273 using namespace Itk; |
|
1274 |
|
1275 IdxDbDeltaContext |
|
1276 * context = new IdxDbDeltaContext(test); |
|
1277 ContextTester |
|
1278 * tester = new ContextTester(test, |
|
1279 context); |
|
1280 |
|
1281 #define TEST "startStage" |
|
1282 tester->add(TEST, |
|
1283 context, |
|
1284 &IdxDbDeltaContext::testStartStage, |
|
1285 TEST); |
|
1286 #undef TEST |
|
1287 |
|
1288 #define TEST "commitToDisk" |
|
1289 tester->add(TEST, |
|
1290 context, |
|
1291 &IdxDbDeltaContext::testCommitToDisk); |
|
1292 #undef TEST |
|
1293 |
|
1294 #define TEST "recoveredStage" |
|
1295 tester->add(TEST, |
|
1296 context, |
|
1297 &IdxDbDeltaContext::testRecoveredStage, |
|
1298 TEST); |
|
1299 #undef TEST |
|
1300 |
|
1301 return tester; |
|
1302 } |
|
1303 |
|
1304 |
|
1305 Itk::TesterBase * CreateIdxDbDeltaTests() |
|
1306 { |
|
1307 using namespace Itk; |
|
1308 |
|
1309 SuiteTester |
|
1310 * deltaTests = new SuiteTester("delta"); |
|
1311 |
|
1312 const char |
|
1313 * deltaTestSpecs[] = { |
|
1314 "nnn-dels_committed", |
|
1315 "nnn-ibuf_committing", |
|
1316 "nnn-ibuf_committed", |
|
1317 "nnn-merging", |
|
1318 "nnn-merged", |
|
1319 "nnn-complete", |
|
1320 |
|
1321 "nrn-dels_committed", |
|
1322 "nrn-ibuf_committing", |
|
1323 "nrn-ibuf_committed", |
|
1324 "nrn-merging", |
|
1325 "nrn-merged", |
|
1326 "nrn-complete", |
|
1327 |
|
1328 "nrd-dels_committed", |
|
1329 "nrd-ibuf_committing", |
|
1330 "nrd-ibuf_committed", |
|
1331 "nrd-merging", |
|
1332 "nrd-merged", |
|
1333 "nrd-complete", |
|
1334 |
|
1335 "wnn-dels_committed", |
|
1336 "wnn-ibuf_committing", |
|
1337 "wnn-ibuf_committed", |
|
1338 "wnn-merging", |
|
1339 "wnn-merged", |
|
1340 "wnn-complete", |
|
1341 |
|
1342 "wrn-dels_committed", |
|
1343 "wrn-ibuf_committing", |
|
1344 "wrn-ibuf_committed", |
|
1345 "wrn-merging", |
|
1346 "wrn-merged", |
|
1347 "wrn-complete", |
|
1348 |
|
1349 "wrd-dels_committed", |
|
1350 "wrd-ibuf_committing", |
|
1351 "wrd-ibuf_committed", |
|
1352 "wrd-merging", |
|
1353 "wrd-merged", |
|
1354 "wrd-complete" |
|
1355 }; |
|
1356 |
|
1357 for (size_t i = 0; i < sizeof(deltaTestSpecs) / sizeof(char*); ++i) |
|
1358 { |
|
1359 deltaTests->add(CreateIdxDbDeltaTest(deltaTestSpecs[i])); |
|
1360 } |
|
1361 |
|
1362 return deltaTests; |
|
1363 } |
|
1364 |
|
1365 |
|
1366 |
|
1367 |
|
1368 |
|
1369 |
|
1370 Itk::TesterBase * CreateWhiteBoxTests() |
|
1371 { |
|
1372 using namespace Itk; |
|
1373 |
|
1374 SuiteTester |
|
1375 * whiteBox = new SuiteTester("whitebox"); |
|
1376 |
|
1377 whiteBox->add("baccoll", |
|
1378 &TestBaseAppClassCollision); |
|
1379 whiteBox->add("idpcoll", |
|
1380 &TestIdxDbPathCollision); |
|
1381 whiteBox->add("scrapall", |
|
1382 &TestScrapAll); |
|
1383 |
|
1384 whiteBox->add("unifiedSearchParse", |
|
1385 &TestUnifiedSearchParse, |
|
1386 "unifiedSearchParse"); |
|
1387 |
|
1388 whiteBox->add(CreatePageTests()); |
|
1389 |
|
1390 whiteBox->add(CreateIdxDbDeltaTests()); |
|
1391 |
|
1392 // TODO add more |
|
1393 |
|
1394 return whiteBox; |
|
1395 } |