# HG changeset patch
# User Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
# Date 1276256627 -10800
# Node ID a5fbfefd615f449c2297c3399562df9cdd0cd2af
# Parent  ae3f1779f6da95757d7c4db879d08a5a0305e385
Revision: 201021
Kit: 2010123

diff -r ae3f1779f6da -r a5fbfefd615f cenrep/2001f6fb.cre
Binary file cenrep/2001f6fb.cre has changed
diff -r ae3f1779f6da -r a5fbfefd615f cenrep/2001f6fb.txt
Binary file cenrep/2001f6fb.txt has changed
diff -r ae3f1779f6da -r a5fbfefd615f conf/s60/cpixhscommon.confml
Binary file conf/s60/cpixhscommon.confml has changed
diff -r ae3f1779f6da -r a5fbfefd615f conf/s60/cpixhscommon_2001f6fb.crml
Binary file conf/s60/cpixhscommon_2001f6fb.crml has changed
diff -r ae3f1779f6da -r a5fbfefd615f group/bld.inf
--- a/group/bld.inf	Thu May 27 13:59:44 2010 +0300
+++ b/group/bld.inf	Fri Jun 11 14:43:47 2010 +0300
@@ -33,6 +33,8 @@
 PRJ_EXPORTS
 ../rom/CPix_mw.iby	CORE_MW_LAYER_IBY_EXPORT_PATH(CPix_mw.iby)
 ../sis/cpixsearch_stub.sis        /epoc32/data/z/system/install/cpixsearch_stub.sis
+../cenrep/2001f6fb.cre                         /epoc32/winscw/c/private/10202be9/2001f6fb.cre
+../cenrep/2001f6fb.cre                         /epoc32/data/z/private/10202be9/2001f6fb.cre
 
 PRJ_TESTMMPFILES
 
diff -r ae3f1779f6da -r a5fbfefd615f harvester/harvesterserver/group/harvesterserver.mmp
--- a/harvester/harvesterserver/group/harvesterserver.mmp	Thu May 27 13:59:44 2010 +0300
+++ b/harvester/harvesterserver/group/harvesterserver.mmp	Fri Jun 11 14:43:47 2010 +0300
@@ -50,6 +50,7 @@
 LIBRARY			  CPixHarvesterPluginInterface.lib
 LIBRARY           edbms.lib
 LIBRARY           sqldb.lib
+LIBRARY           centralrepository.lib
 
 // Logging
 LIBRARY           flogger.lib 
diff -r ae3f1779f6da -r a5fbfefd615f harvester/harvesterserver/inc/cblacklistdb.h
--- a/harvester/harvesterserver/inc/cblacklistdb.h	Thu May 27 13:59:44 2010 +0300
+++ b/harvester/harvesterserver/inc/cblacklistdb.h	Fri Jun 11 14:43:47 2010 +0300
@@ -96,7 +96,30 @@
     * Leaves in case of errors.
     */
 	TBool FindL(TInt32 aPluginUid);
+	
+	/*
+     * @description Adds the given uid of a plugin to the unloadlist table.
+     * @param aPluginUid Uid of the plugin
+     * @return sysmbian error code
+     * Leaves in case of errors.
+     */  
+	TInt AddtoUnloadListL( TInt32 aPluginUid );
     
+	/*
+     * @description remove the given uid of a plugin to the unloadlist table.
+     * @param aPluginUid Uid of the plugin
+     * Leaves in case of errors.
+     */  
+    void RemoveFromUnloadListL( TInt32 aPluginUid );
+    
+    /*
+     * @description Find the given uid of a plugin to the unloadlist table.
+     * @param aPluginUid Uid of the plugin
+     * @return ETrue if exists else returns EFalse
+     * Leaves in case of errors.
+     */  
+    TBool FindFromUnloadListL( TInt32 aPluginUid );
+        
 private :
     /*
     * @description Creates the Blacklist database.
@@ -110,6 +133,13 @@
     * Leaves in case of errors.
     */
     CDbColSet* CreateColumnSetLC();
+    
+    /*
+    * @description Creates Column set for unload table.
+    * @return CDbColSet database column set
+    * Leaves in case of errors.
+    */
+    CDbColSet* CreateUnloadColumnSetLC();
 private:    
     /*
      * A handle to a file server session.Owned
diff -r ae3f1779f6da -r a5fbfefd615f harvester/harvesterserver/inc/cblacklistmgr.h
--- a/harvester/harvesterserver/inc/cblacklistmgr.h	Thu May 27 13:59:44 2010 +0300
+++ b/harvester/harvesterserver/inc/cblacklistmgr.h	Fri Jun 11 14:43:47 2010 +0300
@@ -87,6 +87,29 @@
      */ 
     TBool FindL(TUid aPluginUid , TInt aVersion);
     
+    /*
+    * @description Adds the given uid of a plugin to the unload list of Blacklist DB.    
+    * @param aPluginUid Uid of the plugin
+    * @return sysmbian error code
+    * Leaves in case of errors.
+    */    
+   TInt  AddtoUnloadListL( TUid aPluginUid );
+   
+   /*
+   * @description removes the given uid of a plugin from the unload list of Blacklist DB.    
+   * @param aPluginUid Uid of the plugin
+   * Leaves in case of errors.
+   */    
+  void  RemoveFromUnloadListL( TUid aPluginUid );
+  
+  /*
+   * @description Checks wether the plugin uid is exists in the database unload list or not.
+   * @param aPluginUid Uid of the plugin
+   * @return ETrue if uid exists else returns EFalse
+   * Leaves in case of errors.
+   */ 
+  TBool FindfromUnloadListL(TUid aPluginUid );
+    
 private:    
     /*
      * Interface to Blacklist database.Owned
diff -r ae3f1779f6da -r a5fbfefd615f harvester/harvesterserver/inc/cindexingmanager.h
--- a/harvester/harvesterserver/inc/cindexingmanager.h	Thu May 27 13:59:44 2010 +0300
+++ b/harvester/harvesterserver/inc/cindexingmanager.h	Fri Jun 11 14:43:47 2010 +0300
@@ -100,6 +100,14 @@
      * saving the state of the Plugins
      */
 	void SaveL();
+	/**
+     * Update content info Db with the plugin details
+     */
+	void UpdateContentInfoDbL( const TDesC& aPluginName);
+	/**
+     * Update the unload list in a separate table in blacklist database
+     */
+	void UpdateUnloadListL();
 	
 private:
 	CIndexingManager();
diff -r ae3f1779f6da -r a5fbfefd615f harvester/harvesterserver/src/cblacklistdb.cpp
--- a/harvester/harvesterserver/src/cblacklistdb.cpp	Thu May 27 13:59:44 2010 +0300
+++ b/harvester/harvesterserver/src/cblacklistdb.cpp	Fri Jun 11 14:43:47 2010 +0300
@@ -33,7 +33,7 @@
 //column name for version numbers of plugins in blacklist database
 _LIT( Kversion , "version" );
 // The max length for creating sql query for KBlistSqlFormatSeek format
-const TInt KBlistSqlStringMaxLength(40);
+const TInt KBlistSqlStringMaxLength(50);
 //SQL query to fetch the records with given uid
 _LIT(KBlistSqlFormatSeek , "SELECT * FROM table WHERE uid=%d");
 //SQL query to delete the records with given uid
@@ -41,6 +41,14 @@
 //SQL query to fetch all the records in database
 _LIT(KBlistSqlFormatAll , "SELECT * FROM table");
 _LIT(KDriveC, "c:");
+//Unload plugins Table name in blacklist database
+_LIT( KBLUnloadTableName , "unloadtable" );
+//SQL query to fetch all the records in unload table
+_LIT(KUnloadlistSqlFormatAll , "SELECT * FROM unloadtable");
+//SQL query to delete the records with given uid in unload table
+_LIT(KunloadlistSqlDelete, "DELETE FROM unloadtable WHERE uid=%d");
+//SQL query to fetch the records with given uid from unload table
+_LIT(KUnloadlistSqlFormatSeek , "SELECT * FROM unloadtable WHERE uid=%d");
 // -----------------------------------------------------------------------------
 // CBlacklistDb::NewL()
 // -----------------------------------------------------------------------------
@@ -372,8 +380,11 @@
     User::LeaveIfError( iDatabase.Create( iFs , datafile ) );
     CDbColSet* columns = CreateColumnSetLC();//creates the columns and push to cleanupstack
     User::LeaveIfError( iDatabase.CreateTable( KBlacklistTableName , *columns ) );
+    //Add table to store the unload plugins
+    CDbColSet* unloadcolumns = CreateUnloadColumnSetLC(); //creates the columns and push to cleanupstack
+    User::LeaveIfError( iDatabase.CreateTable( KBLUnloadTableName , *unloadcolumns ) );
     //clean up of variables (columns and dataFile)
-    CleanupStack::PopAndDestroy( columns );
+    CleanupStack::PopAndDestroy( 2 );
     
     CPIXLOGSTRING("CBlacklistDb::CreateDBL(): Exit");
     OstTraceFunctionExit0( CBLACKLISTDB_CREATEDBL_EXIT );
@@ -406,3 +417,127 @@
     OstTraceFunctionExit0( CBLACKLISTDB_CREATECOLUMNSETLC_EXIT );
     return columns; // columns stays on CleanupStack
     }
+
+// -----------------------------------------------------------------------------
+// CBlacklistDb::CreateUnloadColumnSetLC
+// -----------------------------------------------------------------------------
+//
+CDbColSet* CBlacklistDb::CreateUnloadColumnSetLC()
+    {
+    OstTraceFunctionEntry0( CBLACKLISTDB_CREATEUNLOADCOLUMNSETLC_ENTRY );
+    CDbColSet* columns = CDbColSet::NewLC();
+        
+    //Add uid column
+    TDbCol col( Kuid , EDbColInt32 );
+    col.iAttributes = TDbCol::ENotNull ;
+    columns->AddL( col );        
+    
+    OstTraceFunctionExit0( CBLACKLISTDB_CREATEUNLOADCOLUMNSETLC_EXIT );
+    return columns; // columns stays on CleanupStack
+    }
+
+// -----------------------------------------------------------------------------
+// CBlacklistDb::AddtoUnloadListL
+// -----------------------------------------------------------------------------
+//
+TInt CBlacklistDb::AddtoUnloadListL( TInt32 aPluginUid )
+    {
+    OstTraceFunctionEntry0( CBLACKLISTDB_ADDTOUNLOADLISTL_ENTRY );
+    if ( !iOpened )
+            return KErrNotReady;
+        
+    TInt err; 
+    //Prepare the view
+    RDbView dbView;
+    CleanupClosePushL( dbView );
+
+    err = dbView.Prepare( iDatabase , TDbQuery( KUnloadlistSqlFormatAll ) ) ;
+        
+    if ( err == KErrNone )
+       {
+        TRAP( err , dbView.InsertL() );
+        CDbColSet* colSet = dbView.ColSetL();
+        TDbColNo uidcolno = colSet->ColNo( Kuid );        
+        dbView.SetColL( uidcolno , aPluginUid );
+        dbView.PutL();
+        //If addition failed, rollback
+        if(err != KErrNone)
+            {
+            iDatabase.Rollback();
+            }            
+       }
+    CleanupStack::PopAndDestroy( &dbView ); // dbView/    
+    User::LeaveIfError( iDatabase.Compact() );    
+    
+    OstTraceFunctionExit0( CBLACKLISTDB_ADDTOUNLOADLISTL_EXIT );
+    return err;
+    }
+// -----------------------------------------------------------------------------
+// CBlacklistDb::RemoveFromUnloadListL
+// -----------------------------------------------------------------------------
+//
+void CBlacklistDb::RemoveFromUnloadListL( TInt32 aPluginUid )
+    {
+    OstTraceFunctionEntry0( CBLACKLISTDB_REMOVEFROMUNLOADLISTL_ENTRY );
+    if ( !iOpened )
+           return ;
+        
+    //Remove the item record to database
+    // Create the sql statement.  KBlistSqlDelete
+    TBuf<KBlistSqlStringMaxLength> sql;
+    sql.Format( KunloadlistSqlDelete , aPluginUid );
+    
+    //delete the row.
+    TInt rowCount( iDatabase.Execute(sql) );
+    if(rowCount > 0)
+        {       
+        OstTrace0( TRACE_NORMAL, CBLACKLISTDB_REMOVEFROMUNLOADLISTL, "CBlacklistDb::RemoveFromUnloadListL :: removed UID succesfully" );
+        CPIXLOGSTRING("CBlacklistDb::RemoveFromUnloadListL(): Removed UID succesfully");
+        }
+    else
+        {        
+        OstTrace0( TRACE_NORMAL, DUP1_CBLACKLISTDB_REMOVEFROMUNLOADLISTL, "CBlacklistDb::RemoveFromUnloadListL:: UID not found" );
+        CPIXLOGSTRING("CBlacklistDb::RemoveFromUnloadListL(): UID not found");
+        }
+    CPIXLOGSTRING("CBlacklistDb::RemoveFromUnloadListL(): Exit");
+    
+    OstTraceFunctionExit0( CBLACKLISTDB_REMOVEFROMUNLOADLISTL_EXIT );
+    return ;
+    }
+
+// -----------------------------------------------------------------------------
+// CBlacklistDb::FindFromUnloadListL
+// -----------------------------------------------------------------------------
+//
+TBool CBlacklistDb::FindFromUnloadListL( TInt32 aPluginUid )
+    {
+    OstTraceFunctionEntry0( CBLACKLISTDB_FINDFROMUNLOADLISTL_ENTRY );
+    CPIXLOGSTRING2("CBlacklistDb::FindFromUnloadListL(): Uid = %x " , aPluginUid );
+        
+    if ( !iOpened )
+            return EFalse;
+    
+    //Check if the item is available in database
+    //Prepare the sql
+    TBuf<KBlistSqlStringMaxLength> sql;
+    sql.Format( KUnloadlistSqlFormatSeek , aPluginUid );
+    TBool found = EFalse;
+    //Prepare the view
+    RDbView dbView;
+    CleanupClosePushL( dbView );
+
+    User::LeaveIfError( dbView.Prepare( iDatabase , TDbQuery(sql) , RDbView::EReadOnly ) );
+    User::LeaveIfError( dbView.EvaluateAll() );
+
+    TInt isAtRow( dbView.FirstL() );
+    
+    if ( isAtRow )
+       {
+        OstTrace0( TRACE_NORMAL, CBLACKLISTDB_FINDFROMUNLOADLISTL, "CBlacklistDb::FindFromUnloadListL::UID found" );
+        CPIXLOGSTRING("CBlacklistDb::FindFromUnloadListL(): UID found");
+        found = ETrue;                 
+       }    
+    CleanupStack::PopAndDestroy( &dbView ); // dbView/
+    OstTraceFunctionExit0( CBLACKLISTDB_FINDFROMUNLOADLISTL_EXIT );
+    return found;
+    }
diff -r ae3f1779f6da -r a5fbfefd615f harvester/harvesterserver/src/cblacklistmgr.cpp
--- a/harvester/harvesterserver/src/cblacklistmgr.cpp	Thu May 27 13:59:44 2010 +0300
+++ b/harvester/harvesterserver/src/cblacklistmgr.cpp	Fri Jun 11 14:43:47 2010 +0300
@@ -129,7 +129,7 @@
     }
 
 // -----------------------------------------------------------------------------
-// CBlacklistMgr::iSAvailableL()
+// CBlacklistMgr::FindL()
 // -----------------------------------------------------------------------------
 //
 TBool CBlacklistMgr::FindL( TUid aPluginUid , TInt aVersion )
@@ -151,3 +151,54 @@
         }
     return found;
     }
+
+// -----------------------------------------------------------------------------
+// CBlacklistMgr::AddtoUnloadListL()
+// -----------------------------------------------------------------------------
+//
+TInt CBlacklistMgr::AddtoUnloadListL( TUid aPluginUid )
+    {
+    OstTraceFunctionEntry0( CBLACKLISTMGR_ADDTOUNLOADLISTL_ENTRY );
+    OstTrace1( TRACE_NORMAL, CBLACKLISTMGR_ADDTOUNLOADLISTL, "CBlacklistMgr::AddtoUnloadListL;Uid=%x", aPluginUid.iUid );
+    CPIXLOGSTRING2("CBlacklistMgr::AddtoUnloadListL(): Uid = %x " , aPluginUid.iUid );
+    //Check if the record with given plugin uid is already available in database or not
+    //If available just ignore the addition
+    //If there is no record found in database with given uid, add new record with given uid    
+    TInt err = KErrNone;
+    
+    if( !(iBlacklistDb->FindFromUnloadListL( aPluginUid.iUid )) )    
+        {
+        err = iBlacklistDb->AddtoUnloadListL( aPluginUid.iUid );
+        }
+      
+    CPIXLOGSTRING("CBlacklistMgr::AddtoUnloadListL(): Exit");    
+    OstTraceFunctionExit0( CBLACKLISTMGR_ADDTOUNLOADLISTL_EXIT );
+    return err;
+    }
+
+// -----------------------------------------------------------------------------
+// CBlacklistMgr::RemoveFromUnloadListL()
+// -----------------------------------------------------------------------------
+//
+void CBlacklistMgr::RemoveFromUnloadListL( TUid aPluginUid )
+    {
+    OstTraceFunctionEntry0( CBLACKLISTMGR_REMOVEFROMUNLOADLISTL_ENTRY );
+    OstTrace1( TRACE_NORMAL, CBLACKLISTMGR_REMOVEFROMUNLOADLISTL, "CBlacklistMgr::RemoveFromUnloadListL;Uid=%x", aPluginUid.iUid );
+    CPIXLOGSTRING2("CBlacklistMgr::RemoveFromUnloadListL(): Uid = %x " , aPluginUid.iUid );
+    //Remove the item record to database
+    iBlacklistDb->RemoveFromUnloadListL( aPluginUid.iUid );
+    
+    CPIXLOGSTRING("CBlacklistMgr::RemoveFromUnloadListL(): Exit");  
+    OstTraceFunctionExit0( CBLACKLISTMGR_REMOVEFROMUNLOADLISTL_EXIT );
+    }
+
+// -----------------------------------------------------------------------------
+// CBlacklistMgr::FindfromUnloadListL()
+// -----------------------------------------------------------------------------
+//
+TBool CBlacklistMgr::FindfromUnloadListL(TUid aPluginUid )
+    {
+    CPIXLOGSTRING2("CBlacklistMgr::FindfromUnloadListL(): Uid = %x " , aPluginUid.iUid );
+    OstTrace1( TRACE_NORMAL, CBLACKLISTMGR_FINDFROMUNLOADLISTL, "CBlacklistMgr::FindfromUnloadListL;Uid=%x", aPluginUid.iUid );
+    return ( iBlacklistDb->FindFromUnloadListL( aPluginUid.iUid ) );    
+    }
diff -r ae3f1779f6da -r a5fbfefd615f harvester/harvesterserver/src/cindexingmanager.cpp
--- a/harvester/harvesterserver/src/cindexingmanager.cpp	Thu May 27 13:59:44 2010 +0300
+++ b/harvester/harvesterserver/src/cindexingmanager.cpp	Fri Jun 11 14:43:47 2010 +0300
@@ -17,6 +17,7 @@
 
 #include <f32file.h>
 #include <s32file.h>
+#include <centralrepository.h>
 #include "CIndexingManager.h"
 #include "HarvesterServerLogger.h"
 #include "CBlacklistMgr.h"
@@ -48,6 +49,11 @@
 //constants for enable and disable status
 const TInt KEnable = 1;
 const TInt KDisable = 0;
+
+const TUid KCPIXHSrepoUidMenu = {0x2001f6fb};
+
+//Length of uid string in cenrep
+const TInt KuidStringLength = 8;
 // -----------------------------------------------------------------------------
 // CHarvesterServer::NewL()
 // -----------------------------------------------------------------------------
@@ -134,6 +140,8 @@
 	iBlacklistMgr = CBlacklistMgr::NewL();
 	//Instantiate Contentinfo manager
 	iContentInfoMgr = CContentInfoMgr::NewL();
+	
+	UpdateUnloadListL();
 
 	// Load plugins
 	LoadPluginsL();
@@ -322,28 +330,28 @@
         TInt version = infoArray[i]->Version();
         //FFLOGSTRING2( "CFastFindHarvesterPluginControl:: PLUGINS UID %x", uid );
         plugin = NULL;
-        TBool pluginblacklisted = EFalse;
-        
-        pluginblacklisted = iBlacklistMgr->FindL( uid , version );
         
-        TBool iscontentfound = iContentInfoMgr->FindL( infoArray[i]->DisplayName() );
+        UpdateContentInfoDbL( infoArray[i]->DisplayName() );
+        TBool loadplugin = ETrue;
+        //status of plugin in blacklist table
+        TBool pluginblacklisted = iBlacklistMgr->FindL( uid, version);
+        //status of plugin in unload table
+        TBool loadstatus =  iBlacklistMgr->FindfromUnloadListL( uid );  
+        //Check the Uid in both the tables of the blacklist db 
+        if ( loadstatus || pluginblacklisted )
+            loadplugin = EFalse;
         
-        if( !iscontentfound )
+        if ( loadstatus )
             {
-            //Add the content details to database
-            CContentInfo* contentinfo = CContentInfo::NewL();
-            contentinfo->SetNameL( infoArray[i]->DisplayName() );
-            contentinfo->SetBlacklistStatus( KEnable );
-            contentinfo->SetIndexStatus( KEnable );
-            iContentInfoMgr->AddL( contentinfo );
-            delete contentinfo;
+            //Found in unload list.Update the indexing and blacklist status in contentinfo DB
+            iContentInfoMgr->UpdatePluginIndexStatusL( infoArray[i]->DisplayName() , KDisable );
+            iContentInfoMgr->UpdateBlacklistStatusL( infoArray[i]->DisplayName() , KDisable );
             }
-        else
-            {
+        if ( pluginblacklisted )
+            //Update the blacklist status in content info db
             iContentInfoMgr->UpdateBlacklistStatusL( infoArray[i]->DisplayName() , KEnable );
-            }
         
-        if ( !pluginblacklisted )
+        if ( loadplugin )
             {
             // Plugin is not black listed. Add it to database and try to load the plugin
             iBlacklistMgr->AddL( uid , version );
@@ -614,3 +622,65 @@
 	CleanupStack::PopAndDestroy(2, &file);
 }
 
+// -----------------------------------------------------------------------------
+// CIndexingManager::UpdateContentInfoDbL()
+// -----------------------------------------------------------------------------
+//
+void CIndexingManager::UpdateContentInfoDbL( const TDesC& aPluginName)
+{
+    OstTraceFunctionEntry0( CINDEXINGMANAGER_UPDATECONTENTINFODBL_ENTRY );
+    TBool iscontentfound = iContentInfoMgr->FindL( aPluginName );
+            
+    if( !iscontentfound )
+        {
+        //Add the content details to database
+        CContentInfo* contentinfo = CContentInfo::NewL();
+        contentinfo->SetNameL( aPluginName );
+        contentinfo->SetBlacklistStatus( KEnable );
+        contentinfo->SetIndexStatus( KEnable );
+        iContentInfoMgr->AddL( contentinfo );
+        delete contentinfo;
+        }
+    else
+        {
+        iContentInfoMgr->UpdateBlacklistStatusL( aPluginName , KEnable );
+        }
+    OstTraceFunctionExit0( CINDEXINGMANAGER_UPDATECONTENTINFODBL_EXIT );
+}
+
+// -----------------------------------------------------------------------------
+// CIndexingManager::UpdateUnloadList()
+// -----------------------------------------------------------------------------
+//
+void CIndexingManager::UpdateUnloadListL()
+    {
+    OstTraceFunctionEntry0( CINDEXINGMANAGER_UPDATEUNLOADLISTL_ENTRY );
+    CPIXLOGSTRING("CIndexingManager::UpdateUnloadList : Start");
+    //Read the list of Uid's from the cenrep and update blacklist database
+    //Open the unload list common repository
+    CRepository* unloadrepo = NULL;
+    TRAPD( cerror , unloadrepo = CRepository::NewL( KCPIXHSrepoUidMenu ));
+    if ( cerror != KErrNone)
+        return;
+    RArray<TUint32> uidlist;    
+    //Read all the key list
+    TInt error = unloadrepo->FindL( 0, 0, uidlist);
+    if ( error == KErrNone )
+        {
+        TBuf<KuidStringLength> temp;
+        //get the Uid of each and every plugin and add it to blacklist database
+        TInt count = uidlist.Count();
+        for (int i = 0; i < count; i++ )
+            {
+            TUid uid;
+            TInt64 value;
+            unloadrepo->Get( uidlist[i], temp );
+            TLex uidvalue(temp);
+            TInt xerr = uidvalue.Val( value,EHex );
+            uid.iUid = value;
+            (void)iBlacklistMgr->AddtoUnloadListL( uid );            
+            }
+        }
+    CPIXLOGSTRING("CIndexingManager::UpdateUnloadList : End");
+    OstTraceFunctionExit0( CINDEXINGMANAGER_UPDATEUNLOADLISTL_EXIT );
+    }
diff -r ae3f1779f6da -r a5fbfefd615f harvester/harvesterserver/traces/CBlacklistDbTraces.h
--- a/harvester/harvesterserver/traces/CBlacklistDbTraces.h	Thu May 27 13:59:44 2010 +0300
+++ b/harvester/harvesterserver/traces/CBlacklistDbTraces.h	Fri Jun 11 14:43:47 2010 +0300
@@ -1,4 +1,4 @@
-// Created by TraceCompiler 2.1.2
+// Created by TraceCompiler 2.2.2
 // DO NOT EDIT, CHANGES WILL BE LOST
 
 #ifndef __CBLACKLISTDBTRACES_H__
@@ -20,6 +20,14 @@
 #define CBLACKLISTDB_CREATEDBL_EXIT 0x8a000c
 #define CBLACKLISTDB_CREATECOLUMNSETLC_ENTRY 0x8a000d
 #define CBLACKLISTDB_CREATECOLUMNSETLC_EXIT 0x8a000e
+#define CBLACKLISTDB_CREATEUNLOADCOLUMNSETLC_ENTRY 0x8a003b
+#define CBLACKLISTDB_CREATEUNLOADCOLUMNSETLC_EXIT 0x8a003c
+#define CBLACKLISTDB_ADDTOUNLOADLISTL_ENTRY 0x8a003d
+#define CBLACKLISTDB_ADDTOUNLOADLISTL_EXIT 0x8a003e
+#define CBLACKLISTDB_REMOVEFROMUNLOADLISTL_ENTRY 0x8a003f
+#define CBLACKLISTDB_REMOVEFROMUNLOADLISTL_EXIT 0x8a0040
+#define CBLACKLISTDB_FINDFROMUNLOADLISTL_ENTRY 0x8a0041
+#define CBLACKLISTDB_FINDFROMUNLOADLISTL_EXIT 0x8a0042
 #define CBLACKLISTDB_CONSTRUCTL 0x860001
 #define CBLACKLISTDB_ADDL 0x860002
 #define CBLACKLISTDB_REMOVE 0x860003
@@ -30,6 +38,9 @@
 #define CBLACKLISTDB_UPDATEL 0x860008
 #define CBLACKLISTDB_FINDL 0x860009
 #define DUP1_CBLACKLISTDB_FINDL 0x86000a
+#define CBLACKLISTDB_REMOVEFROMUNLOADLISTL 0x860027
+#define DUP1_CBLACKLISTDB_REMOVEFROMUNLOADLISTL 0x860028
+#define CBLACKLISTDB_FINDFROMUNLOADLISTL 0x860029
 
 
 inline TBool OstTraceGen2( TUint32 aTraceID, TUint aParam1, TInt aParam2 )
diff -r ae3f1779f6da -r a5fbfefd615f harvester/harvesterserver/traces/CBlacklistMgrTraces.h
--- a/harvester/harvesterserver/traces/CBlacklistMgrTraces.h	Thu May 27 13:59:44 2010 +0300
+++ b/harvester/harvesterserver/traces/CBlacklistMgrTraces.h	Fri Jun 11 14:43:47 2010 +0300
@@ -1,4 +1,4 @@
-// Created by TraceCompiler 2.1.2
+// Created by TraceCompiler 2.2.2
 // DO NOT EDIT, CHANGES WILL BE LOST
 
 #ifndef __CBLACKLISTMGRTRACES_H__
@@ -14,11 +14,18 @@
 #define CBLACKLISTMGR_ADDL_EXIT 0x8a0014
 #define CBLACKLISTMGR_REMOVE_ENTRY 0x8a0015
 #define CBLACKLISTMGR_REMOVE_EXIT 0x8a0016
+#define CBLACKLISTMGR_ADDTOUNLOADLISTL_ENTRY 0x8a0043
+#define CBLACKLISTMGR_ADDTOUNLOADLISTL_EXIT 0x8a0044
+#define CBLACKLISTMGR_REMOVEFROMUNLOADLISTL_ENTRY 0x8a0045
+#define CBLACKLISTMGR_REMOVEFROMUNLOADLISTL_EXIT 0x8a0046
 #define CBLACKLISTMGR_ADDL 0x86000b
 #define CBLACKLISTMGR_REMOVE 0x86000c
 #define CBLACKLISTMGR_FINDL 0x86000d
 #define DUP1_CBLACKLISTMGR_FINDL 0x86000e
 #define DUP2_CBLACKLISTMGR_FINDL 0x86000f
+#define CBLACKLISTMGR_ADDTOUNLOADLISTL 0x86002a
+#define CBLACKLISTMGR_REMOVEFROMUNLOADLISTL 0x86002b
+#define CBLACKLISTMGR_FINDFROMUNLOADLISTL 0x86002c
 
 
 inline TBool OstTraceGen2( TUint32 aTraceID, TUint aParam1, TInt aParam2 )
diff -r ae3f1779f6da -r a5fbfefd615f harvester/harvesterserver/traces/CIndexingManagerTraces.h
--- a/harvester/harvesterserver/traces/CIndexingManagerTraces.h	Thu May 27 13:59:44 2010 +0300
+++ b/harvester/harvesterserver/traces/CIndexingManagerTraces.h	Fri Jun 11 14:43:47 2010 +0300
@@ -1,4 +1,4 @@
-// Created by TraceCompiler 2.1.2
+// Created by TraceCompiler 2.2.2
 // DO NOT EDIT, CHANGES WILL BE LOST
 
 #ifndef __CINDEXINGMANAGERTRACES_H__
@@ -6,6 +6,10 @@
 
 #define KOstTraceComponentID 0x2001f6fb
 
+#define CINDEXINGMANAGER_UPDATECONTENTINFODBL_ENTRY 0x8a0047
+#define CINDEXINGMANAGER_UPDATECONTENTINFODBL_EXIT 0x8a0048
+#define CINDEXINGMANAGER_UPDATEUNLOADLISTL_ENTRY 0x8a0049
+#define CINDEXINGMANAGER_UPDATEUNLOADLISTL_EXIT 0x8a004a
 #define CINDEXINGMANAGER_RUNL 0x860010
 #define DUP1_CINDEXINGMANAGER_RUNL 0x860011
 #define DUP2_CINDEXINGMANAGER_RUNL 0x860012
diff -r ae3f1779f6da -r a5fbfefd615f harvester/harvesterserver/traces/OstTraceDefinitions.h
--- a/harvester/harvesterserver/traces/OstTraceDefinitions.h	Thu May 27 13:59:44 2010 +0300
+++ b/harvester/harvesterserver/traces/OstTraceDefinitions.h	Fri Jun 11 14:43:47 2010 +0300
@@ -1,19 +1,3 @@
-/*
-* Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).
-* All rights reserved.
-* This component and the accompanying materials are made available
-* under the terms of "Eclipse Public License v1.0"
-* which accompanies this distribution, and is available
-* at the URL "http://www.eclipse.org/legal/epl-v10.html".
-*
-* Initial Contributors:
-* Nokia Corporation - initial contribution.
-*
-* Contributors:
-*
-* Description: 
-*
-*/
 #ifndef __OSTTRACEDEFINITIONS_H__
 #define __OSTTRACEDEFINITIONS_H__
 // OST_TRACE_COMPILER_IN_USE flag has been added by Trace Compiler
diff -r ae3f1779f6da -r a5fbfefd615f harvester/harvesterserver/traces/ccontentinfoTraces.h
--- a/harvester/harvesterserver/traces/ccontentinfoTraces.h	Thu May 27 13:59:44 2010 +0300
+++ b/harvester/harvesterserver/traces/ccontentinfoTraces.h	Fri Jun 11 14:43:47 2010 +0300
@@ -1,4 +1,4 @@
-// Created by TraceCompiler 2.1.2
+// Created by TraceCompiler 2.2.2
 // DO NOT EDIT, CHANGES WILL BE LOST
 
 #ifndef __CCONTENTINFOTRACES_H__
diff -r ae3f1779f6da -r a5fbfefd615f harvester/harvesterserver/traces/ccontentinfodbTraces.h
--- a/harvester/harvesterserver/traces/ccontentinfodbTraces.h	Thu May 27 13:59:44 2010 +0300
+++ b/harvester/harvesterserver/traces/ccontentinfodbTraces.h	Fri Jun 11 14:43:47 2010 +0300
@@ -1,4 +1,4 @@
-// Created by TraceCompiler 2.1.2
+// Created by TraceCompiler 2.2.2
 // DO NOT EDIT, CHANGES WILL BE LOST
 
 #ifndef __CCONTENTINFODBTRACES_H__
diff -r ae3f1779f6da -r a5fbfefd615f harvester/harvesterserver/traces/contentinfomgrTraces.h
--- a/harvester/harvesterserver/traces/contentinfomgrTraces.h	Thu May 27 13:59:44 2010 +0300
+++ b/harvester/harvesterserver/traces/contentinfomgrTraces.h	Fri Jun 11 14:43:47 2010 +0300
@@ -1,4 +1,4 @@
-// Created by TraceCompiler 2.1.2
+// Created by TraceCompiler 2.2.2
 // DO NOT EDIT, CHANGES WILL BE LOST
 
 #ifndef __CONTENTINFOMGRTRACES_H__
diff -r ae3f1779f6da -r a5fbfefd615f harvester/harvesterserver/traces/fixed_id.definitions
--- a/harvester/harvesterserver/traces/fixed_id.definitions	Thu May 27 13:59:44 2010 +0300
+++ b/harvester/harvesterserver/traces/fixed_id.definitions	Fri Jun 11 14:43:47 2010 +0300
@@ -3,24 +3,36 @@
 [GROUP]TRACE_NORMAL=0x86
 [TRACE]TRACE_FLOW[0x8A]_CBLACKLISTDB_ADDL_ENTRY=0x5
 [TRACE]TRACE_FLOW[0x8A]_CBLACKLISTDB_ADDL_EXIT=0x6
+[TRACE]TRACE_FLOW[0x8A]_CBLACKLISTDB_ADDTOUNLOADLISTL_ENTRY=0x3d
+[TRACE]TRACE_FLOW[0x8A]_CBLACKLISTDB_ADDTOUNLOADLISTL_EXIT=0x3e
 [TRACE]TRACE_FLOW[0x8A]_CBLACKLISTDB_CONSTRUCTL_ENTRY=0x3
 [TRACE]TRACE_FLOW[0x8A]_CBLACKLISTDB_CONSTRUCTL_EXIT=0x4
 [TRACE]TRACE_FLOW[0x8A]_CBLACKLISTDB_CREATECOLUMNSETLC_ENTRY=0xd
 [TRACE]TRACE_FLOW[0x8A]_CBLACKLISTDB_CREATECOLUMNSETLC_EXIT=0xe
 [TRACE]TRACE_FLOW[0x8A]_CBLACKLISTDB_CREATEDBL_ENTRY=0xb
 [TRACE]TRACE_FLOW[0x8A]_CBLACKLISTDB_CREATEDBL_EXIT=0xc
+[TRACE]TRACE_FLOW[0x8A]_CBLACKLISTDB_CREATEUNLOADCOLUMNSETLC_ENTRY=0x3b
+[TRACE]TRACE_FLOW[0x8A]_CBLACKLISTDB_CREATEUNLOADCOLUMNSETLC_EXIT=0x3c
+[TRACE]TRACE_FLOW[0x8A]_CBLACKLISTDB_FINDFROMUNLOADLISTL_ENTRY=0x41
+[TRACE]TRACE_FLOW[0x8A]_CBLACKLISTDB_FINDFROMUNLOADLISTL_EXIT=0x42
 [TRACE]TRACE_FLOW[0x8A]_CBLACKLISTDB_NEWL_ENTRY=0x1
 [TRACE]TRACE_FLOW[0x8A]_CBLACKLISTDB_NEWL_EXIT=0x2
+[TRACE]TRACE_FLOW[0x8A]_CBLACKLISTDB_REMOVEFROMUNLOADLISTL_ENTRY=0x3f
+[TRACE]TRACE_FLOW[0x8A]_CBLACKLISTDB_REMOVEFROMUNLOADLISTL_EXIT=0x40
 [TRACE]TRACE_FLOW[0x8A]_CBLACKLISTDB_REMOVE_ENTRY=0x7
 [TRACE]TRACE_FLOW[0x8A]_CBLACKLISTDB_REMOVE_EXIT=0x8
 [TRACE]TRACE_FLOW[0x8A]_CBLACKLISTDB_UPDATEL_ENTRY=0x9
 [TRACE]TRACE_FLOW[0x8A]_CBLACKLISTDB_UPDATEL_EXIT=0xa
 [TRACE]TRACE_FLOW[0x8A]_CBLACKLISTMGR_ADDL_ENTRY=0x13
 [TRACE]TRACE_FLOW[0x8A]_CBLACKLISTMGR_ADDL_EXIT=0x14
+[TRACE]TRACE_FLOW[0x8A]_CBLACKLISTMGR_ADDTOUNLOADLISTL_ENTRY=0x43
+[TRACE]TRACE_FLOW[0x8A]_CBLACKLISTMGR_ADDTOUNLOADLISTL_EXIT=0x44
 [TRACE]TRACE_FLOW[0x8A]_CBLACKLISTMGR_CONSTRUCTL_ENTRY=0x11
 [TRACE]TRACE_FLOW[0x8A]_CBLACKLISTMGR_CONSTRUCTL_EXIT=0x12
 [TRACE]TRACE_FLOW[0x8A]_CBLACKLISTMGR_NEWL_ENTRY=0xf
 [TRACE]TRACE_FLOW[0x8A]_CBLACKLISTMGR_NEWL_EXIT=0x10
+[TRACE]TRACE_FLOW[0x8A]_CBLACKLISTMGR_REMOVEFROMUNLOADLISTL_ENTRY=0x45
+[TRACE]TRACE_FLOW[0x8A]_CBLACKLISTMGR_REMOVEFROMUNLOADLISTL_EXIT=0x46
 [TRACE]TRACE_FLOW[0x8A]_CBLACKLISTMGR_REMOVE_ENTRY=0x15
 [TRACE]TRACE_FLOW[0x8A]_CBLACKLISTMGR_REMOVE_EXIT=0x16
 [TRACE]TRACE_FLOW[0x8A]_CCONTENTINFODB_ADDL_ENTRY=0x25
@@ -59,15 +71,24 @@
 [TRACE]TRACE_FLOW[0x8A]_CCONTENTINFO_NEWL_EXIT=0x18
 [TRACE]TRACE_FLOW[0x8A]_CCONTENTINFO_SETNAMEL_ENTRY=0x1d
 [TRACE]TRACE_FLOW[0x8A]_CCONTENTINFO_SETNAMEL_EXIT=0x1e
+[TRACE]TRACE_FLOW[0x8A]_CINDEXINGMANAGER_UPDATECONTENTINFODBL_ENTRY=0x47
+[TRACE]TRACE_FLOW[0x8A]_CINDEXINGMANAGER_UPDATECONTENTINFODBL_EXIT=0x48
+[TRACE]TRACE_FLOW[0x8A]_CINDEXINGMANAGER_UPDATEUNLOADLISTL_ENTRY=0x49
+[TRACE]TRACE_FLOW[0x8A]_CINDEXINGMANAGER_UPDATEUNLOADLISTL_EXIT=0x4a
 [TRACE]TRACE_NORMAL[0x86]_CBLACKLISTDB_ADDL=0x2
 [TRACE]TRACE_NORMAL[0x86]_CBLACKLISTDB_CONSTRUCTL=0x1
+[TRACE]TRACE_NORMAL[0x86]_CBLACKLISTDB_FINDFROMUNLOADLISTL=0x29
 [TRACE]TRACE_NORMAL[0x86]_CBLACKLISTDB_FINDL=0x9
 [TRACE]TRACE_NORMAL[0x86]_CBLACKLISTDB_FINDWITHVERSIONL=0x6
 [TRACE]TRACE_NORMAL[0x86]_CBLACKLISTDB_REMOVE=0x3
+[TRACE]TRACE_NORMAL[0x86]_CBLACKLISTDB_REMOVEFROMUNLOADLISTL=0x27
 [TRACE]TRACE_NORMAL[0x86]_CBLACKLISTDB_UPDATEL=0x8
 [TRACE]TRACE_NORMAL[0x86]_CBLACKLISTMGR_ADDL=0xb
+[TRACE]TRACE_NORMAL[0x86]_CBLACKLISTMGR_ADDTOUNLOADLISTL=0x2a
+[TRACE]TRACE_NORMAL[0x86]_CBLACKLISTMGR_FINDFROMUNLOADLISTL=0x2c
 [TRACE]TRACE_NORMAL[0x86]_CBLACKLISTMGR_FINDL=0xd
 [TRACE]TRACE_NORMAL[0x86]_CBLACKLISTMGR_REMOVE=0xc
+[TRACE]TRACE_NORMAL[0x86]_CBLACKLISTMGR_REMOVEFROMUNLOADLISTL=0x2b
 [TRACE]TRACE_NORMAL[0x86]_CINDEXINGMANAGER_ADDHARVESTINGQUEUE=0x1d
 [TRACE]TRACE_NORMAL[0x86]_CINDEXINGMANAGER_HARVESTINGCOMPLETED=0x24
 [TRACE]TRACE_NORMAL[0x86]_CINDEXINGMANAGER_LOADPLUGINSL=0x1a
@@ -76,6 +97,7 @@
 [TRACE]TRACE_NORMAL[0x86]_DUP1_CBLACKLISTDB_FINDL=0xa
 [TRACE]TRACE_NORMAL[0x86]_DUP1_CBLACKLISTDB_FINDWITHVERSIONL=0x7
 [TRACE]TRACE_NORMAL[0x86]_DUP1_CBLACKLISTDB_REMOVE=0x4
+[TRACE]TRACE_NORMAL[0x86]_DUP1_CBLACKLISTDB_REMOVEFROMUNLOADLISTL=0x28
 [TRACE]TRACE_NORMAL[0x86]_DUP1_CBLACKLISTMGR_FINDL=0xe
 [TRACE]TRACE_NORMAL[0x86]_DUP1_CINDEXINGMANAGER_ADDHARVESTINGQUEUE=0x1e
 [TRACE]TRACE_NORMAL[0x86]_DUP1_CINDEXINGMANAGER_HARVESTINGCOMPLETED=0x25
diff -r ae3f1779f6da -r a5fbfefd615f qcpix/tsrc/orbitsearch/searchhelper.cpp
--- a/qcpix/tsrc/orbitsearch/searchhelper.cpp	Thu May 27 13:59:44 2010 +0300
+++ b/qcpix/tsrc/orbitsearch/searchhelper.cpp	Fri Jun 11 14:43:47 2010 +0300
@@ -37,17 +37,16 @@
     resultsBox->setPlainText("Search button clicked!");
     
     int hits = 0;
+    
     QString resultString("");
     resultsBox->setPlainText( resultString );
     searchTime.restart();
     QString searchString;
-    
-#if PREFIX_SEARCH
     searchString = "$prefix(\""; 
     searchString += searchBox->text();
     searchString += "\")";
 
-#elif STAR_SEARCH
+#if STAR_SEARCH
     searchString += searchBox->text();
     searchString += "*";
 #elif NO_STAR_SEARCH
diff -r ae3f1779f6da -r a5fbfefd615f rom/cpix_mw.iby
--- a/rom/cpix_mw.iby	Thu May 27 13:59:44 2010 +0300
+++ b/rom/cpix_mw.iby	Fri Jun 11 14:43:47 2010 +0300
@@ -1,22 +1,25 @@
 /*
-* Copyright (c) 2002 Nokia Corporation and/or its subsidiary(-ies).
-* All rights reserved.
-* This component and the accompanying materials are made available
-* under the terms of "Eclipse Public License v1.0"
-* which accompanies this distribution, and is available
-* at the URL "http://www.eclipse.org/legal/epl-v10.html".
+* ============================================================================
+*  Name     : cpix.iby
+*  Part of  : S605.0, PF52.50, ivalo
+*
+*  Description: Includes additional application IBYs for PF52.50/ivalo
+*       
+*  Version:
 *
-* Initial Contributors:
-* Nokia Corporation - initial contribution.
-*
-* Contributors:
+*  Copyright (C) 2002 Nokia Corporation.
+*  This material, including documentation and any related 
+*  computer programs, is protected by copyright controlled by 
+*  Nokia Corporation. All rights are reserved. Copying, 
+*  including reproducing, storing,  adapting or translating, any 
+*  or all of this material requires the prior written consent of 
+*  Nokia Corporation. This material also contains confidential 
+*  information which may not be disclosed to others without the 
+*  prior written consent of Nokia Corporation.
 *
-* Description:  Includes additional application IBYs for PF52.50/ivalo
-*       
-*
+* ============================================================================
 */
 
-
 #ifndef __CPIXENGINE_IBY__
 #define __CPIXENGINE_IBY__
 
@@ -53,6 +56,8 @@
 
 data=ZSYSTEM\install\cpixsearch_stub.sis    System\Install\cpixsearch_stub.sis
 
+data=DATAZ_\private\10202be9\2001f6fb.cre       private\10202be9\2001f6fb.cre
+
 #endif //FF_SEARCH_SW
 
 #endif // __CPIXENGINE_IBY__
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/cpix/cpix/inc/private/document.h
--- a/searchengine/cpix/cpix/inc/private/document.h	Thu May 27 13:59:44 2010 +0300
+++ b/searchengine/cpix/cpix/inc/private/document.h	Fri Jun 11 14:43:47 2010 +0300
@@ -74,7 +74,9 @@
 
         int isIndexed() const;
 
-        bool isAggregated()  const; 
+        bool isAggregated()  const;
+        
+        bool isFreeText()  const; 
 		
         float_t boost() const;
 
@@ -93,6 +95,8 @@
         lucene::document::Field* field_;
 		
         bool aggregate_;
+        
+        bool freeText_;
 	
     };
 	
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/cpix/cpix/inc/public/cpixdoc.h
--- a/searchengine/cpix/cpix/inc/public/cpixdoc.h	Thu May 27 13:59:44 2010 +0300
+++ b/searchengine/cpix/cpix/inc/public/cpixdoc.h	Fri Jun 11 14:43:47 2010 +0300
@@ -73,6 +73,10 @@
             // index the value of the field without an analyzer and
             // disable storing of norms
             cpix_INDEX_UNTOKENIZED = 64,
+            
+            // index the value of the field without the stop word analyzer and
+            // store it in _aggregate
+            cpix_FREE_TEXT = 128
         };
 
     typedef enum cpix_Index_ cpix_Index;
@@ -89,6 +93,7 @@
             // Expose the value to be searchable throught the
             // aggregate field
             cpix_AGGREGATE_YES = 1<<31
+
     	};
     
     typedef enum cpix_Aggregate_ cpix_Aggregate;   
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/cpix/cpix/src/analyzer.cpp
--- a/searchengine/cpix/cpix/src/analyzer.cpp	Thu May 27 13:59:44 2010 +0300
+++ b/searchengine/cpix/cpix/src/analyzer.cpp	Fri Jun 11 14:43:47 2010 +0300
@@ -160,6 +160,14 @@
                 {
                     _CLTHROWA(CL_ERR_IO, AGGR_STREAMREADER_ERR);
                 }
+            if(field->isFreeText())
+                {
+                    using namespace lucene::analysis;
+                    stream_ = _CLNEW standard::StandardTokenizer(reader_);
+                    stream_ = _CLNEW standard::StandardFilter(stream_,true);
+                    stream_ = _CLNEW LowerCaseFilter(stream_,true);
+                }
+            else
             stream_ = analyzer_.tokenStream( field->name(), reader_ ); 
         }
     }
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/cpix/cpix/src/analyzerexp.cpp
--- a/searchengine/cpix/cpix/src/analyzerexp.cpp	Thu May 27 13:59:44 2010 +0300
+++ b/searchengine/cpix/cpix/src/analyzerexp.cpp	Fri Jun 11 14:43:47 2010 +0300
@@ -16,6 +16,7 @@
 */
 
 
+
 #include "analyzerexp.h"
 
 #include "indevicecfg.h" 
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/cpix/cpix/src/document.cpp
--- a/searchengine/cpix/cpix/src/document.cpp	Thu May 27 13:59:44 2010 +0300
+++ b/searchengine/cpix/cpix/src/document.cpp	Fri Jun 11 14:43:47 2010 +0300
@@ -35,6 +35,7 @@
                  const wchar_t * value,
                  int             configs) 
         :	own_(true), field_(0) {
+        freeText_  = false;
         resolveConfig(configs); 
         field_ = _CLNEW lucene::document::Field(name, value, configs); 		 
     }
@@ -44,6 +45,7 @@
                  lucene::util::Reader* stream,
                  int             configs) 
 	:  own_(true), field_(0) {
+        freeText_  = false;
         resolveConfig(configs); 
         field_ = _CLNEW lucene::document::Field(name, stream, configs); 		 
     }
@@ -63,6 +65,11 @@
             // Aggregate indexed fields by default. 
             aggregate_ = !(configs & cpix_INDEX_NO);
         }
+        
+        if(configs & cpix_FREE_TEXT){
+            freeText_ = true;
+            configs &= (~cpix_FREE_TEXT);
+        }
     }
 				    
 
@@ -73,6 +80,7 @@
       field_( field ), 
 	  aggregate_( aggregate ) 
 	   {
+        freeText_  = false;
     }
 	
     Field::~Field() {
@@ -105,7 +113,11 @@
     int Field::isIndexed() const {
         return field_->isIndexed(); 
     }
-
+    
+    bool Field::isFreeText() const {
+        return freeText_;
+    }
+    
     bool Field::isAggregated() const {
         return aggregate_;
     }
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/cpix/tsrc/cpixsample/sis/cpixsample_s60_3_x_v_1_0_0.pkg
--- a/searchengine/cpix/tsrc/cpixsample/sis/cpixsample_s60_3_x_v_1_0_0.pkg	Thu May 27 13:59:44 2010 +0300
+++ b/searchengine/cpix/tsrc/cpixsample/sis/cpixsample_s60_3_x_v_1_0_0.pkg	Fri Jun 11 14:43:47 2010 +0300
@@ -1,18 +1,3 @@
-;
-; Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).
-; All rights reserved.
-; This component and the accompanying materials are made available
-; under the terms of "Eclipse Public License v1.0"
-; which accompanies this distribution, and is available
-; at the URL "http://www.eclipse.org/legal/epl-v10.html".
-;
-; Initial Contributors:
-; Nokia Corporation - initial contribution.
-;
-; Contributors:
-;
-; Description: 
-;
 ; Installation file for cpixsample application
 ;
 ; This is an auto-generated PKG file by Carbide.
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/cpix/tsrc/cpixsample/src/cpixsample.cpp
--- a/searchengine/cpix/tsrc/cpixsample/src/cpixsample.cpp	Thu May 27 13:59:44 2010 +0300
+++ b/searchengine/cpix/tsrc/cpixsample/src/cpixsample.cpp	Fri Jun 11 14:43:47 2010 +0300
@@ -1,19 +1,3 @@
-/*
-* Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).
-* All rights reserved.
-* This component and the accompanying materials are made available
-* under the terms of "Eclipse Public License v1.0"
-* which accompanies this distribution, and is available
-* at the URL "http://www.eclipse.org/legal/epl-v10.html".
-*
-* Initial Contributors:
-* Nokia Corporation - initial contribution.
-*
-* Contributors:
-*
-* Description: 
-*
-*/
 
 // INCLUDE FILES
 #include <stdio.h>
@@ -41,14 +25,15 @@
 
 
 #define FIELD_ALPHA L"Alpha"
+#define FIELD_ALPHA1 L"Alpha1"
 
 
 #define DOC1CONTENT L"mary had little lamb issue its anirban fleece was black as coal"
-#define DOC2CONTENT L"sri rama jeyam shankar.rajendran@yahoo.co.in www.google.com U.S.A. file.txt"
+#define DOC2CONTENT L"sri rama jeyam and it  3gpp_70.jpg 170(kb).jpg is shankar.rajendran@yahoo.co.in then www.google.com U.S.A. file.txt"
 
 
 // The term that will be present in multiple documents.
-#define SEARCH_TERM L"$prefix(\"a\")"
+#define SEARCH_TERM L"$prefix(\"had\")"
 
 
 int testInit(cpix_Analyzer **analyzer_, cpix_IdxDb **idxDb_)
@@ -103,11 +88,15 @@
     {
     cpix_Document *doc;
     cpix_Field field;
+    cpix_Field field1;
+    cpix_Field field2;
+    cpix_Field field3;
     cpix_Result result;
     
     doc = cpix_Document_create(&result,docUid,NULL,      // app class
             NULL,      // excerpt
             NULL);     // mime type
+    
     if (cpix_Failed(&result))
         {
         printf("Failed to create a document\n");
@@ -117,7 +106,24 @@
     cpix_Field_initialize(&field,
             FIELD_ALPHA,
             data, 
-            cpix_STORE_YES |cpix_INDEX_TOKENIZED);
+            cpix_STORE_YES |cpix_INDEX_UNTOKENIZED | cpix_AGGREGATE_YES | cpix_FREE_TEXT );
+    
+    cpix_Field_initialize(&field1,
+                FIELD_ALPHA1,
+                data, 
+                cpix_STORE_YES |cpix_INDEX_UNTOKENIZED | cpix_AGGREGATE_YES);
+    
+    
+    cpix_Field_initialize(&field2,
+                L"ALPHA2",
+                L"This is shankar and I am working for nokia", 
+                cpix_STORE_YES |cpix_INDEX_TOKENIZED | cpix_AGGREGATE_YES | cpix_FREE_TEXT );
+    
+    
+    cpix_Field_initialize(&field3,
+                L"ALPHA3",
+                L"This is shankar and I am working for nokia", 
+                cpix_STORE_NO |cpix_INDEX_TOKENIZED | cpix_AGGREGATE_YES);
 
     if (cpix_Failed(&field))
         {
@@ -126,6 +132,9 @@
         return 0;
         }
     cpix_Document_add(doc,&field);
+    cpix_Document_add(doc,&field1);
+    cpix_Document_add(doc,&field2);
+    cpix_Document_add(doc,&field3);
     cpix_IdxDb_add(*idxDb_,doc,*analyzer_);
 
     cpix_Document_destroy(doc);
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/cpix/tsrc/cpixunittest/data/cpixunittestcorpus/pdf/start_enter.pdf
Binary file searchengine/cpix/tsrc/cpixunittest/data/cpixunittestcorpus/pdf/start_enter.pdf has changed
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/cpix/tsrc/cpixunittest/group/bld.inf
--- a/searchengine/cpix/tsrc/cpixunittest/group/bld.inf	Thu May 27 13:59:44 2010 +0300
+++ b/searchengine/cpix/tsrc/cpixunittest/group/bld.inf	Fri Jun 11 14:43:47 2010 +0300
@@ -360,5 +360,11 @@
 "../data/cpixunittestcorpus/query/query8.txt"                            "/epoc32/winscw/c/Data/cpixunittestcorpus/query/query8.txt"
 "../data/cpixunittestcorpus/query/query9.txt"                            "/epoc32/winscw/c/Data/cpixunittestcorpus/query/query9.txt"
 
+"../data/cpixunittestcorpus/pdf/ctutor.pdf"                             "/epoc32/winscw/c/Data/cpixunittestcorpus/pdf/ctutor.pdf"
+"../data/cpixunittestcorpus/pdf/Empty.pdf"                              "/epoc32/winscw/c/Data/cpixunittestcorpus/pdf/Empty.pdf"
+"../data/cpixunittestcorpus/pdf/geology.pdf"                            "/epoc32/winscw/c/Data/cpixunittestcorpus/pdf/geology.pdf"
+"../data/cpixunittestcorpus/pdf/samplepdf.pdf"                          "/epoc32/winscw/c/Data/cpixunittestcorpus/pdf/samplepdf.pdf"
+"../data/cpixunittestcorpus/pdf/windjack.pdf"                           "/epoc32/winscw/c/Data/cpixunittestcorpus/pdf/windjack.PDF"
+
 PRJ_TESTMMPFILES
 cpixunittest.mmp
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/cpix/tsrc/cpixunittest/group/cpixunittest.mmp
--- a/searchengine/cpix/tsrc/cpixunittest/group/cpixunittest.mmp	Thu May 27 13:59:44 2010 +0300
+++ b/searchengine/cpix/tsrc/cpixunittest/group/cpixunittest.mmp	Fri Jun 11 14:43:47 2010 +0300
@@ -71,7 +71,7 @@
 SOURCE          config.cpp
 SOURCE          clq/uxqry.cpp
 SOURCE          clq/clqry.cpp 
-SOURCE		pdftests.cpp querytest.cpp std_log_result.cpp
+SOURCE		pdftests.cpp querytest.cpp std_log_result.cpp misc.cpp
 
 USERINCLUDE     ../../../../../searchsrv_plat/cpix_utility_api/inc
 USERINCLUDE     ../inc
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/cpix/tsrc/cpixunittest/inc/std_log_result.h
--- a/searchengine/cpix/tsrc/cpixunittest/inc/std_log_result.h	Thu May 27 13:59:44 2010 +0300
+++ b/searchengine/cpix/tsrc/cpixunittest/inc/std_log_result.h	Fri Jun 11 14:43:47 2010 +0300
@@ -1,20 +1,15 @@
-/*
-* Copyright (c) 2008 Nokia Corporation and/or its subsidiary(-ies).
-* All rights reserved.
-* This component and the accompanying materials are made available
-* under the terms of "Eclipse Public License v1.0"
-* which accompanies this distribution, and is available
-* at the URL "http://www.eclipse.org/legal/epl-v10.html".
-*
-* Initial Contributors:
-* Nokia Corporation - initial contribution.
-*
-* Contributors:
-*
-* Description: 
-*
-*/
-
+ /*
+ *  Copyright © 2008 Nokia Corporation.
+ *  This material, including documentation and any related 
+ *  computer programs, is protected by copyright controlled by 
+ *  Nokia Corporation. All rights are reserved. Copying, 
+ *  including reproducing, storing, adapting or translating, any 
+ *  or all of this material requires the prior written consent of 
+ *  Nokia Corporation. This material also contains confidential 
+ *  information which may not be disclosed to others without the 
+ *  prior written consent of Nokia Corporation.
+ * ============================================================================
+ */
 
 
 #ifndef _STD_LOG_FILE_H__
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/cpix/tsrc/cpixunittest/src/analysis.cpp
--- a/searchengine/cpix/tsrc/cpixunittest/src/analysis.cpp	Thu May 27 13:59:44 2010 +0300
+++ b/searchengine/cpix/tsrc/cpixunittest/src/analysis.cpp	Fri Jun 11 14:43:47 2010 +0300
@@ -31,6 +31,7 @@
 #include "cpixdoc.h"
 
 #include "std_log_result.h"
+
 const char * AnalysisTestDocsToIndex[5] = {
     FILE_TEST_CORPUS_PATH "\\en\\1.txt",
     FILE_TEST_CORPUS_PATH "\\en\\2.txt",
@@ -83,14 +84,14 @@
 
 	// bad syntaxes
 	TestAnalyzerParsing(testMgr, L"letter><lowercase" ,0); 
-	TestAnalyzerParsing(testMgr, L"38j_d fad23 4?q ca'wRA", 0 ); 
+	TestAnalyzerParsing(testMgr, L"38j_d fad23 4?q ca'wRA", 0 );
+	TestAnalyzerParsing(testMgr, L"38.45_d fd23<ca'wRA", 0 ); 
 	// parsing failures
 	TestAnalyzerParsing(testMgr, L"letter>>lowercase", 0 ); 
 	TestAnalyzerParsing(testMgr, L">letter>>lowercase lowercase", 0 ); 
 	TestAnalyzerParsing(testMgr, L"letter lowercase", 0 );
 	testResultXml(xml_file);
 }
-
 void TestSwitchParsing(Itk::TestMgr * testMgr) 
 {
     char *xml_file = (char*)__FUNCTION__;
@@ -105,14 +106,13 @@
 	TestAnalyzerParsing(testMgr, L"switch{ case '_qnr': whitespace; default: standard; }>lowercase");
 	TestAnalyzerParsing(testMgr, L"switch{ default: 	standard; }");
 	TestAnalyzerParsing(testMgr, L"switch{ case '_qnr': switch{ case '_docuid': keyword; default: whitespace; }; default: standard; }");
+	TestAnalyzerParsing(testMgr, L"switch{ case '_mimetype': standard; default: whitespace; }; default: standard; }");
 	testResultXml(xml_file);
 }
 
 void TestAnalyzerUsage(Itk::TestMgr * testMgr, const wchar_t* definition) 
 {
 	printf("Indexing and searching with %S\n", definition); 
-	char *xml_file = (char*)__FUNCTION__;
-	    assert_failed = 0;
 	cpix_Result
         result;
 
@@ -205,7 +205,7 @@
 	}
 	cpix_QueryParser_destroy(queryParser);
 	cpix_Analyzer_destroy( analyzer ); 
-	testResultXml(xml_file);
+	
 }
 
 void TestAnalyzersUsage(Itk::TestMgr * testMgr) 
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/cpix/tsrc/cpixunittest/src/analysiswhitebox.cpp
--- a/searchengine/cpix/tsrc/cpixunittest/src/analysiswhitebox.cpp	Thu May 27 13:59:44 2010 +0300
+++ b/searchengine/cpix/tsrc/cpixunittest/src/analysiswhitebox.cpp	Fri Jun 11 14:43:47 2010 +0300
@@ -18,6 +18,7 @@
 #include <wchar.h>
 #include <stddef.h>
 
+
 #include <iostream>
 
 #include "cpixidxdb.h"
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/cpix/tsrc/cpixunittest/src/cpixunittest.cpp
--- a/searchengine/cpix/tsrc/cpixunittest/src/cpixunittest.cpp	Thu May 27 13:59:44 2010 +0300
+++ b/searchengine/cpix/tsrc/cpixunittest/src/cpixunittest.cpp	Fri Jun 11 14:43:47 2010 +0300
@@ -68,6 +68,8 @@
 
 Itk::TesterBase * CreateASyncTests();
 
+Itk::TesterBase * CreateMiscTests();
+
 // Avoid media Testcases if MDS not present. 
 #ifdef MDS_PRESENT
 // Spi 
@@ -98,7 +100,7 @@
     SuiteTester
         * all = new SuiteTester("all");
     
-//
+
     all->add(CreateSuiteTesterRandom()); 
     all->add(CreateDestructiveTests());
     all->add(CreatePrematureQryTests());
@@ -116,12 +118,12 @@
     all->add(CreateUtf8PathTests());
     all->add(CreateAnalysisTests());		
 		 
-
+    all->add(CreatePdfSearchTests());
     all->add(CreateDocumentTests());		
     all->add(CreateFlushTests());			
     all->add(CreateHeartbeatTests());	    
     all->add(CreateTermSearchTests()); 		
-    all->add(CreatePdfSearchTests());
+    
     
     // TODO enable later Itk::TesterBase * CreateClQryHierarchy();
 
@@ -134,6 +136,7 @@
     // all->add(CreateSpiHierarchy());  // Commented for JPG/EXIF files checks
 #endif
 	all->add(CreateQueryTests()); 
+	all->add(CreateMiscTests());
 
     // add more top level test suites here
     // ...
@@ -173,6 +176,39 @@
                         printf("Failed to initialize Cpix\n");
                         return -1;
                     }
+//                /* Added for decision coverage Test case */
+                const char *cpix_Dir = cpix_InitParams_getCpixDir(initParams);
+                
+                const char *logFilebase = cpix_InitParams_getLogFileBase(initParams);
+                
+                size_t logSizeLimit = cpix_InitParams_getLogSizeLimit(initParams);
+                
+                size_t logSizeRecurrency = cpix_InitParams_getLogSizeCheckRecurrency(initParams);
+                
+                size_t maxIdleSec = cpix_InitParams_getMaxIdleSec(initParams);
+                
+                //size_t maxInsertBufferSize = cpix_InitParams_getMaxInsertBufSize(initParams);
+                //cpix_InitParams_setMaxInsertBufferSize(initParams, maxInsertBufferSize);
+                
+                size_t insertBufMaxDocs = cpix_InitParams_getInsertBufMaxDocs(initParams);
+                cpix_InitParams_setInsertBufMaxDocs(initParams, insertBufMaxDocs);
+                
+                size_t idxJobQueueSize = cpix_InitParams_getIdxJobQueueSize(initParams);
+                cpix_InitParams_setIdxJobQueueSize(initParams, idxJobQueueSize);
+                
+                size_t qryJobQueueSize = cpix_InitParams_getQryJobQueueSize(initParams);
+                cpix_InitParams_setQryJobQueueSize(initParams, qryJobQueueSize);
+                
+                int idxThreadPriorityDelta = cpix_InitParams_getIdxThreadPriorityDelta(initParams);
+                cpix_InitParams_setIdxThreadPriorityDelta(initParams, idxThreadPriorityDelta);
+                
+                int qryThreadPriorityDelta = cpix_InitParams_getQryThreadPriorityDelta(initParams);
+                size_t clHitsPageSize = cpix_InitParams_getClHitsPageSize(initParams);
+                
+                cpix_InitParams_setClHitsPageSize(initParams, clHitsPageSize);
+                
+                
+                
                 cpix_InitParams_setMaxIdleSec(initParams,
                                               MaxIdleSec);
                 if (cpix_Failed(initParams))
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/cpix/tsrc/cpixunittest/src/documenttest.cpp
--- a/searchengine/cpix/tsrc/cpixunittest/src/documenttest.cpp	Thu May 27 13:59:44 2010 +0300
+++ b/searchengine/cpix/tsrc/cpixunittest/src/documenttest.cpp	Fri Jun 11 14:43:47 2010 +0300
@@ -36,6 +36,8 @@
 #include "setupsentry.h"
 
 #include "std_log_result.h"
+#include "..\..\..\cpix\src\qrytypes\prefixqrytype.cpp"
+#include "..\..\..\cpix\src\qrytypes\termsqrytype.cpp"
 
 #define TEST_DOCUMENT_QBASEAPPCLASS "@0:root test document"
 #define TEST_DOCUMENT_INDEXDB_PATH "c:\\Data\\indexing\\indexdb\\root\\test\\document"
@@ -293,6 +295,61 @@
     {
     }
 
+    
+
+    void TestPrefixQryType(Itk::TestMgr *testMgr )
+        {
+        char *xml_file = (char *)__FUNCTION__;
+        assert_failed = 0;
+        tearDown();
+        setup();
+        cpix_Result  result;
+        addDocument(testMgr,
+                    LDOCUID1,
+                    DOC1CONTENT);
+        addDocument(testMgr,
+                    LDOCUID2,
+                    DOC2CONTENT);
+        
+        cpix_IdxDb_flush(idxDb_);
+        ITK_EXPECT(testMgr,
+                   cpix_Succeeded(idxDb_),
+                   "Flushing index has failed");
+        if(!cpix_Succeeded(idxDb_))
+            {
+            assert_failed = 1;
+            }
+        Cpix::PrefixQryType *qryType = new Cpix::PrefixQryType;
+        std::list<std::wstring> list(3,L"");
+        std::list<std::wstring> list1;
+        qryType->setUp(queryParser_, list, SEARCH_TERM);
+        qryType->setUp(queryParser_, list1, SEARCH_TERM);
+        cpix_IdxSearcher *
+        searcher = cpix_IdxSearcher_openDb(&result,
+                TEST_DOCUMENT_QBASEAPPCLASS);
+        if (searcher == NULL)
+            {
+                ITK_PANIC("Could not create searcher");
+            }
+        cpix_Hits *Hits1 = qryType->search(searcher);
+        cpix_Hits *Hits2 = qryType->search(idxDb_);
+        testResultXml(xml_file);
+        }
+    
+    void TestTermsQryType(Itk::TestMgr * )
+        {
+        char *xml_file = (char *)__FUNCTION__;
+        assert_failed = 0;
+        Cpix::TermsQryType qrytype;
+        tearDown();
+        setup();
+        std::list<std::wstring> list(3, L"term");
+        std::list<std::wstring> list1;
+        qrytype.setUp(queryParser_, list, SEARCH_TERM);
+        qrytype.setUp(queryParser_, list1, SEARCH_TERM);
+        testResultXml(xml_file);
+        }
+    
     void testNoBoostingFields(Itk::TestMgr * testMgr)
     {
         // Don't boost Field Alpha in doc1
@@ -652,6 +709,18 @@
             documentContext,
                        &DocumentContext::testBoostQuery);
 #undef TEST
-        
+    // Both test throws the exception so need not to cover.
+//#define TEST "perfixqrytype"
+//    contextTester->add(TEST,
+//            documentContext,
+//                &DocumentContext::TestPrefixQryType);
+//#undef TEST    
+//        
+//#define TEST "termqrytype"
+//    contextTester->add(TEST,
+//            documentContext,
+//                &DocumentContext::TestTermsQryType);
+//#undef TEST 
+    
     return contextTester;
 }
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/cpix/tsrc/cpixunittest/src/misc.cpp
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/searchengine/cpix/tsrc/cpixunittest/src/misc.cpp	Fri Jun 11 14:43:47 2010 +0300
@@ -0,0 +1,159 @@
+/*
+* Copyright (c) 2010 Nokia Corporation and/or its subsidiary(-ies).
+* All rights reserved.
+* This component and the accompanying materials are made available
+* under the terms of "Eclipse Public License v1.0"
+* which accompanies this distribution, and is available
+* at the URL "http://www.eclipse.org/legal/epl-v10.html".
+*
+* Initial Contributors:
+* Nokia Corporation - initial contribution.
+*
+* Contributors:
+*
+* Description: 
+*
+*/
+
+
+#include <wchar.h>
+#include <stddef.h>
+#include <iostream>
+#include "itk.h"
+
+#include "config.h"
+#include "testutils.h"
+
+#include "std_log_result.h"
+
+#include "cpixexc.h"
+#include "cpixhits.h"
+#include "cpixidxdb.h"
+
+
+#include "..\..\..\cpix\src\cpixerror.cpp"
+#include "..\..\..\cpix\src\cpixanalyzer.cpp"
+#include "..\..\..\cpix\src\fileparser\fileparser.cpp"
+#include "..\..\..\cpix\src\fileparser\pdffileparser.cpp"
+#include "..\..\..\cpix\src\qrytypes\cluceneqrytype.cpp"
+
+const cpix_LangCode cpix_LANG_JA = { "ja" };
+void TestAnalyzer(Itk::TestMgr *)
+    {
+    const cpix_LangCode **C_Lang = cpix_supportedLanguages();
+    const wchar_t* toWchar = cpix_ToWideLangCode(cpix_LANG_EN);
+    cpix_Result result;
+    cpix_Analyzer * c_analyzer = cpix_CreateSnowballAnalyzer(&result,cpix_LANG_JA);
+    }
+void TestSetError(Itk::TestMgr *)
+    {
+    struct ErrorInfo serrorinfo;
+    serrorinfo.setInfo(ET_CPIX_EXC);
+    serrorinfo.setInfo(ET_CPIX_EXC, "");
+    cpix_Error *err1 = CreateError(ET_OS_EXC, L"");
+    cpix_Error *err2 = CreateError(ET_CPIX_EXC, "");
+    cpix_Error_report(NULL,L"",0);
+    }
+
+void TestCpixExc(Itk::TestMgr * )
+    {
+    char *xml_file = (char*)__FUNCTION__;
+    assert_failed = 0;
+    CpixExc *exc1 = new CpixExc(/*(const wchar_t *)NULL*/L"", "misc.cpp", __LINE__);
+    CpixExc *exc2 = new CpixExc("", "misc.cpp", __LINE__);
+    CpixExc exc3 = *exc1;
+    exc3 = *exc2;
+    exc2->file();
+    exc2->line();
+    exc2->wWhat();
+    free(exc1);
+    free(exc2);
+    testResultXml(xml_file);
+    }
+
+void TestHitDocumentList(Itk::TestMgr * )
+    {
+    Cpix::HitDocumentList *hitdoclist = new Cpix::HitDocumentList;
+    hitdoclist->remove(0);
+    free(hitdoclist);
+    }
+
+void TestTermCreateDestroy(Itk::TestMgr * )
+    {
+    cpix_Result result;
+    cpix_Term *term = cpix_Term_create(&result, CONTENTS_FIELD, L"Hello");
+    cpix_Term_destroy(term);
+    }
+
+void TestPdfFileParser(Itk::TestMgr * )
+    {
+    bool isParse1 = Cpix::isFileAllowedToParse(L"");
+    bool isParse2 = Cpix::isFileAllowedToParse(L"c:\\data\\cpixunittestcorpus\\pdf\\ctutor");
+//     getPDFExcerpt is unnamed namespace so cant call ...
+//    std::wstring line;
+//    int result = getPDFExcerpt(L"c:\\data\\cpixunittestcorpus\\pdf\\ctutor\\empty.pdf", &line);
+    
+    char *buffer = "Hello this is test for find string";
+
+    Cpix::FindStringInBuffer(buffer, "find", strlen(buffer));
+    
+    Cpix::FindStringInBuffer(buffer, "no", strlen(buffer));
+    
+    Cpix::seen2("find", "If its for find");
+    
+    Cpix::ExtractNumber("hello calculate 123   ", strlen("hello calculate 123   "));
+    
+    //Cpix::convertPDFToText("c:\\data\\cpixunittestcorpus\\pdf\\ctutor.pdf");
+    
+    
+    }
+void TestLuceneQryType(Itk::TestMgr * )
+    {
+    cpix_Result result;
+    
+    cpix_Analyzer* analyzer = cpix_CreateSimpleAnalyzer(&result);
+    if ( cpix_Failed( &result) ) 
+        {
+            ITK_PANIC("Analyzer could not be created");
+            assert_failed = 1;
+        }
+    cpix_QueryParser
+        * queryParser = cpix_QueryParser_create(&result,
+                                                LCPIX_DEFAULT_FIELD,
+                                                analyzer );
+    if (queryParser == NULL)
+        {
+            cpix_Analyzer_destroy( analyzer );
+            ITK_PANIC("Could not create query parser");
+        }
+    Cpix::LuceneQryType *QryType = new Cpix::LuceneQryType();
+    std::list<std::wstring> list (2, L"no argumnet");
+    //QryType->setUp(queryParser,list,L"Find this");
+    
+    }
+
+
+Itk::TesterBase * CreateMiscTests()
+    {
+    
+    using namespace Itk;
+
+    SuiteTester
+        * misc = new SuiteTester("misc");
+
+    misc->add("testanalyzer",&TestAnalyzer,"testanalyzer");
+    
+    misc->add("SetError",&TestSetError,"SetError");
+    
+    misc->add("CpixExc", &TestCpixExc, "CpixExc");
+
+    misc->add("doclist", &TestHitDocumentList, "doclist");
+    
+    misc->add("termCD", &TestTermCreateDestroy, "termCD");
+
+    misc->add("pdfparser", &TestPdfFileParser, "pdfparser");
+    
+    misc->add("lucenetype", &TestLuceneQryType, "lucenetype");
+    
+    return misc;
+    }
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/cpix/tsrc/cpixunittest/src/pdftests.cpp
--- a/searchengine/cpix/tsrc/cpixunittest/src/pdftests.cpp	Thu May 27 13:59:44 2010 +0300
+++ b/searchengine/cpix/tsrc/cpixunittest/src/pdftests.cpp	Fri Jun 11 14:43:47 2010 +0300
@@ -38,13 +38,14 @@
 #include "suggestion.h"
 #include "std_log_result.h"
 
-const char * PdfDocsToIndex[7] = {
+const char * PdfDocsToIndex[8] = {
     "c:\\data\\cpixunittestcorpus\\pdf\\ctutor.pdf",
-	"c:\\data\\cpixunittestcorpus\\stem\\pdf\\geology.pdf",
-	"c:\\data\\cpixunittestcorpus\\stem\\pdf\\samplepdf.pdf",
-	"c:\\data\\cpixunittestcorpus\\stem\\pdf\\windjack.pdf",
-	"c:\\data\\cpixunittestcorpus\\stem\\pdf\\DCTDecode.pdf",
-	"c:\\data\\cpixunittestcorpus\\stem\\pdf\\Empty.pdf",
+	"c:\\data\\cpixunittestcorpus\\pdf\\geology.pdf",
+	"c:\\data\\cpixunittestcorpus\\pdf\\samplepdf.pdf",
+	"c:\\data\\cpixunittestcorpus\\pdf\\windjack.pdf",
+	"c:\\data\\cpixunittestcorpus\\pdf\\DCTDecode.pdf",
+	"c:\\data\\cpixunittestcorpus\\pdf\\Empty.pdf",
+	"c:\\data\\cpixunittestcorpus\\pdf\\start_enter.pdf",
     NULL
 };
 
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/cpix/tsrc/cpixunittest/src/std_log_result.cpp
--- a/searchengine/cpix/tsrc/cpixunittest/src/std_log_result.cpp	Thu May 27 13:59:44 2010 +0300
+++ b/searchengine/cpix/tsrc/cpixunittest/src/std_log_result.cpp	Fri Jun 11 14:43:47 2010 +0300
@@ -1,19 +1,3 @@
-/*
-* Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).
-* All rights reserved.
-* This component and the accompanying materials are made available
-* under the terms of "Eclipse Public License v1.0"
-* which accompanies this distribution, and is available
-* at the URL "http://www.eclipse.org/legal/epl-v10.html".
-*
-* Initial Contributors:
-* Nokia Corporation - initial contribution.
-*
-* Contributors:
-*
-* Description: 
-*
-*/
 #include "std_log_result.h"
 #ifdef __SYMBIAN32__
 #define LOG_FILE "c:\\logs\\std_test_log.txt"
@@ -124,7 +108,8 @@
     // create the xml file name
     FILE *fp_result;
     sprintf(xmlfilename, "%s%s.%s", LOG_DIR, filename, LOG_FILE_EXT);
-    strftime(time_buf,50,"%c",tm1);
+    //strftime(time_buf,50,"%c",tm1);
+    sprintf(time_buf,"%s","");
 
     if(assert_failed )
         strcpy(result,"FAILED");
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/oss/cl/clucene/group/clucene.mmp
--- a/searchengine/oss/cl/clucene/group/clucene.mmp	Thu May 27 13:59:44 2010 +0300
+++ b/searchengine/oss/cl/clucene/group/clucene.mmp	Fri Jun 11 14:43:47 2010 +0300
@@ -165,6 +165,22 @@
 SOURCE StringIntern.cpp
 SOURCE ThreadLocal.cpp
 
+// highlighter
+SOURCEPATH	  ../src/CLucene/highlighter
+SOURCE Encoder.cpp 
+SOURCE Formatter.cpp
+SOURCE Fragmenter.cpp
+SOURCE Highlighter.cpp
+SOURCE QueryScorer.cpp
+SOURCE QueryTermExtractor.cpp
+SOURCE SimpleFragmenter.cpp
+SOURCE SimpleHTMLEncoder.cpp
+SOURCE SimpleHTMLFormatter.cpp
+SOURCE TextFragment.cpp
+SOURCE TokenSources.cpp
+SOURCE WeightedTerm.cpp
+SOURCE TokenGroup.cpp
+
 #endif // !MONOLITHIC
 
 //CAPABILITY 	  NONE
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/oss/cl/clucene/src/clucene.h
--- a/searchengine/oss/cl/clucene/src/clucene.h	Thu May 27 13:59:44 2010 +0300
+++ b/searchengine/oss/cl/clucene/src/clucene.h	Fri Jun 11 14:43:47 2010 +0300
@@ -35,5 +35,6 @@
 #include "CLucene/analysis/standard/StandardAnalyzer.h"
 #include "CLucene/analysis/Analyzers.h"
 #include "CLucene/util/Reader.h"
+#include "CLucene/highlighter/Highlighter.h"
 
 #endif
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/oss/cl/clucene/src/clucene/highlighter/Encoder.cpp
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/searchengine/oss/cl/clucene/src/clucene/highlighter/Encoder.cpp	Fri Jun 11 14:43:47 2010 +0300
@@ -0,0 +1,3 @@
+#include "CLucene/StdHeader.h"
+#include "Encoder.h"
+
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/oss/cl/clucene/src/clucene/highlighter/Encoder.h
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/searchengine/oss/cl/clucene/src/clucene/highlighter/Encoder.h	Fri Jun 11 14:43:47 2010 +0300
@@ -0,0 +1,61 @@
+/**
+ * Copyright 2002-2004 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef _lucene_search_highlight_encoder_
+#define _lucene_search_highlight_encoder_
+
+#if defined(_LUCENE_PRAGMA_ONCE)
+# pragma once
+#endif
+
+#include "CLucene/util/StringBuffer.h"
+
+CL_NS_DEF2(search,highlight)
+
+/**
+ * Encodes original text. The Encoder works with the Formatter to generate the output.
+ *
+ */
+class Encoder:LUCENE_BASE
+{
+public:
+	/** Virtual destructor */
+	virtual ~Encoder(){
+	}
+
+	/**
+	 * @param originalText The section of text being output
+	 */
+	virtual TCHAR* encodeText(TCHAR* originalText) = 0;
+};
+
+/**
+ * Simple {@link Encoder} implementation that does not modify the output
+ * @author Nicko Cadell
+ *
+ */
+class DefaultEncoder: public Encoder
+{
+public:
+	TCHAR* encodeText(TCHAR* originalText)
+	{
+		return STRDUP_TtoT(originalText);
+	}
+};
+
+
+CL_NS_END2
+
+#endif
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/oss/cl/clucene/src/clucene/highlighter/Formatter.cpp
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/searchengine/oss/cl/clucene/src/clucene/highlighter/Formatter.cpp	Fri Jun 11 14:43:47 2010 +0300
@@ -0,0 +1,2 @@
+#include "CLucene/StdHeader.h"
+#include "Formatter.h"
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/oss/cl/clucene/src/clucene/highlighter/Formatter.h
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/searchengine/oss/cl/clucene/src/clucene/highlighter/Formatter.h	Fri Jun 11 14:43:47 2010 +0300
@@ -0,0 +1,54 @@
+/**
+ * Copyright 2002-2004 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _lucene_search_highlight_formatter_
+#define _lucene_search_highlight_formatter_
+
+#if defined(_LUCENE_PRAGMA_ONCE)
+# pragma once
+#endif
+
+#include "CLucene/highlighter/TokenGroup.h"
+
+CL_NS_DEF2(search,highlight)
+
+/**
+ * Processes terms found in the original text, typically by applying some form 
+ * of mark-up to highlight terms in HTML search results pages.
+ *
+ */
+class Formatter:LUCENE_BASE
+{
+public:
+
+	/** Virtual destructor */
+	virtual ~Formatter(){
+	}
+
+  /**
+	 * @param originalText The section of text being considered for markup
+	 * @param tokenGroup contains one or several overlapping Tokens along with
+	 * their scores and positions.
+	 */
+  virtual TCHAR* highlightTerm(const TCHAR* originalTermText, const TokenGroup* tokenGroup) = 0;
+};
+
+CL_NS_END2
+
+#endif
+
+
+
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/oss/cl/clucene/src/clucene/highlighter/Fragmenter.cpp
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/searchengine/oss/cl/clucene/src/clucene/highlighter/Fragmenter.cpp	Fri Jun 11 14:43:47 2010 +0300
@@ -0,0 +1,3 @@
+#include "CLucene/StdHeader.h"
+#include "Fragmenter.h"
+
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/oss/cl/clucene/src/clucene/highlighter/Fragmenter.h
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/searchengine/oss/cl/clucene/src/clucene/highlighter/Fragmenter.h	Fri Jun 11 14:43:47 2010 +0300
@@ -0,0 +1,55 @@
+/**
+ * Copyright 2002-2004 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _lucene_search_highlight_fragmenter_
+#define _lucene_search_highlight_fragmenter_
+
+#if defined(_LUCENE_PRAGMA_ONCE)
+# pragma once
+#endif
+
+#include "CLucene/analysis/AnalysisHeader.h"
+
+CL_NS_DEF2(search,highlight)
+
+/**
+ * Implements the policy for breaking text into multiple fragments for consideration
+ * by the {@link Highlighter} class. A sophisticated implementation may do this on the basis
+ * of detecting end of sentences in the text. 
+ */
+class Fragmenter:LUCENE_BASE
+{
+public:
+	/** Virtual destructor */
+	virtual ~Fragmenter(){
+	}
+
+	/**
+	 * Initializes the Fragmenter
+	 * @param originalText
+	 */
+	virtual void start(const TCHAR* originalText) = 0;
+
+	/**
+	 * Test to see if this token from the stream should be held in a new TextFragment
+	 * @param nextToken
+	 */
+	virtual bool isNewFragment(const CL_NS(analysis)::Token * nextToken) = 0;
+};
+
+CL_NS_END2
+
+#endif
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/oss/cl/clucene/src/clucene/highlighter/HighlightScorer.h
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/searchengine/oss/cl/clucene/src/clucene/highlighter/HighlightScorer.h	Fri Jun 11 14:43:47 2010 +0300
@@ -0,0 +1,65 @@
+/**
+ * Copyright 2002-2004 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _lucene_search_highlight_highlighterscorer_
+#define _lucene_search_highlight_highlighterscorer_
+
+#if defined(_LUCENE_PRAGMA_ONCE)
+# pragma once
+#endif
+
+#include "CLucene/highlighter/TextFragment.h"
+#include "CLucene/analysis/AnalysisHeader.h"
+
+CL_NS_DEF2(search,highlight)
+
+/**
+ * Adds to the score for a fragment based on its tokens
+ */
+class HighlightScorer:LUCENE_BASE
+{
+public:
+	virtual ~HighlightScorer(){
+	}
+
+	/**
+	 * called when a new fragment is started for consideration
+	 * @param newFragment
+	 */
+	virtual void startFragment(TextFragment* newFragment) = 0;
+
+	/**
+	 * Called for each token in the current fragment
+	 * @param token The token to be scored
+	 * @return a score which is passed to the Highlighter class to influence the mark-up of the text
+	 * (this return value is NOT used to score the fragment)
+	 */
+	virtual float_t getTokenScore(CL_NS(analysis)::Token* token) = 0;
+	
+
+	/**
+	 * Called when the highlighter has no more tokens for the current fragment - the scorer returns
+	 * the weighting it has derived for the most recent fragment, typically based on the tokens
+	 * passed to getTokenScore(). 
+	 *
+	 */	
+	virtual float_t getFragmentScore() = 0;
+};
+
+CL_NS_END2
+#endif
+
+
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/oss/cl/clucene/src/clucene/highlighter/Highlighter.cpp
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/searchengine/oss/cl/clucene/src/clucene/highlighter/Highlighter.cpp	Fri Jun 11 14:43:47 2010 +0300
@@ -0,0 +1,439 @@
+#include "CLucene/StdHeader.h"
+#include "Highlighter.h"
+
+CL_NS_DEF2(search,highlight)
+CL_NS_USE(analysis)
+CL_NS_USE(util)
+
+	class FragmentQueue : public CL_NS(util)::PriorityQueue<TextFragment*, CL_NS(util)::Deletor::Object<TextFragment> >
+	{
+	public:
+		FragmentQueue(int32_t size)
+		{
+			initialize(size, true);
+		}
+
+	protected:
+		bool lessThan(TextFragment * fragA, TextFragment * fragB)
+		{
+			if (fragA->getScore() == fragB->getScore())
+				return fragA->getFragNum() > fragB->getFragNum();
+			else
+				return fragA->getScore() < fragB->getScore();
+		}
+	};
+
+
+	Highlighter::Highlighter(HighlightScorer * fragmentScorer):
+        delete_formatter(true),
+        delete_encoder(true),
+		delete_textFragmenter(true),
+		delete_fragmentScorer(false)		
+	{
+		maxDocBytesToAnalyze = DEFAULT_MAX_DOC_BYTES_TO_ANALYZE;
+		
+		_textFragmenter = _CLNEW SimpleFragmenter();
+		_fragmentScorer = fragmentScorer;
+		_formatter = _CLNEW SimpleHTMLFormatter();
+		_encoder = _CLNEW DefaultEncoder();
+	}
+
+	Highlighter::Highlighter(Formatter * formatter, HighlightScorer * fragmentScorer):
+        delete_formatter(false),
+        delete_encoder(true),
+		delete_textFragmenter(true),
+		delete_fragmentScorer(false)	
+	{
+		maxDocBytesToAnalyze = DEFAULT_MAX_DOC_BYTES_TO_ANALYZE;
+		
+		_textFragmenter = _CLNEW SimpleFragmenter();
+		_fragmentScorer = fragmentScorer;
+		_formatter = formatter;
+		_encoder = _CLNEW DefaultEncoder();
+	}
+
+	Highlighter::Highlighter(Formatter * formatter, Encoder* encoder, HighlightScorer * fragmentScorer):
+        delete_formatter(false),
+        delete_encoder(true),
+        delete_textFragmenter(true),
+        delete_fragmentScorer(false)    
+	{
+		maxDocBytesToAnalyze = DEFAULT_MAX_DOC_BYTES_TO_ANALYZE;
+		_textFragmenter = _CLNEW SimpleFragmenter();
+		_fragmentScorer = fragmentScorer;
+		_formatter = formatter;
+		_encoder = encoder;
+	}
+
+	Highlighter::~Highlighter()
+	{
+		if ( delete_textFragmenter )
+			_CLDELETE ( _textFragmenter );
+
+		if ( delete_fragmentScorer )
+			_CLDELETE(_fragmentScorer);
+
+		if( delete_formatter )
+			_CLDELETE(_formatter);
+
+		if ( delete_encoder )
+			_CLDELETE(_encoder);
+	}
+
+	TCHAR* Highlighter::getBestFragment(TokenStream * tokenStream, const TCHAR* text)
+	{
+		TCHAR** results = getBestFragments(tokenStream,text, 1);
+		TCHAR* result = 0;
+
+		if (results[0] != NULL )
+			result = stringDuplicate(results[0]);
+
+		_CLDELETE_CARRAY_ALL(results);
+
+		return result;
+	}
+
+	/**
+  	* Highlights chosen terms in a text, extracting the most relevant section.
+  	* This is a convenience method that calls
+  	* {@link #getBestFragment(TokenStream, const TCHAR*)}
+  	*
+  	* @param analyzer   the analyzer that will be used to split <code>text</code>
+  	* into chunks
+  	* @param text text to highlight terms in
+  	* @param fieldName Name of field used to influence analyzer's tokenization policy
+  	*
+  	* @return highlighted text fragment or NULL if no terms found
+  	*/
+  	TCHAR* Highlighter::getBestFragment(Analyzer* analyzer, const TCHAR* fieldName, const TCHAR* text)
+  	{
+  	    TokenStream* tokenStream = analyzer->tokenStream(fieldName, _CLNEW StringReader(text));
+  	    return getBestFragment(tokenStream, text);
+  	}
+
+	TCHAR** Highlighter::getBestFragments(
+		TokenStream * tokenStream,	
+		const TCHAR* text,
+		int32_t maxNumFragments)
+	{
+		maxNumFragments = max((int32_t)1, maxNumFragments); //sanity check
+		
+		StringBuffer buffer;
+		TextFragment** frags = getBestTextFragments(&buffer,tokenStream,text, true,maxNumFragments);
+
+		//Get text
+		CL_NS(util)::StringArray fragTexts;
+		for (uint32_t i=0; frags[i]!=NULL; i++)
+		{
+			TextFragment* f = frags[i];
+			if ((f != NULL) && (f->getScore() > 0))
+			{
+				 fragTexts.push_back(f->toString(&buffer));
+			}
+			_CLDELETE(f);
+		}
+
+		_CLDELETE_ARRAY(frags);
+
+		TCHAR** ret = _CL_NEWARRAY(TCHAR*,fragTexts.size()+1);
+		fragTexts.toArray(ret);
+		return ret;
+	}
+
+	TCHAR* Highlighter::getBestFragments(
+		TokenStream * tokenStream,	
+		const TCHAR* text,
+		int32_t maxNumFragments,
+		const TCHAR* separator)
+	{
+		TCHAR** sections = getBestFragments(tokenStream,text, maxNumFragments);
+		StringBuffer result;
+
+		for (int32_t i = 0; sections[i]!=NULL; i++)
+		{
+			if (i > 0)
+			{
+				result.append(separator);
+			}
+			result.append(sections[i]);
+		}
+
+		_CLDELETE_CARRAY_ALL(sections);
+		return result.toString();
+	}
+
+	TextFragment** Highlighter::getBestTextFragments(
+		StringBuffer* writeTo,
+		TokenStream * tokenStream,	
+		const TCHAR* text,
+		bool mergeContiguousFragments,
+		int32_t maxNumFragments)
+	{
+		CLArrayList<TextFragment*> docFrags(false);
+		TextFragment* currentFrag = _CLNEW TextFragment(writeTo->length(), docFrags.size());
+		_fragmentScorer->startFragment(currentFrag);
+		docFrags.push_back(currentFrag);
+
+		FragmentQueue fragQueue(maxNumFragments);
+
+		try
+		{
+			int32_t startOffset;
+			int32_t endOffset;
+			int32_t lastEndOffset = 0;
+			_textFragmenter->start(text);
+			TCHAR substringBuffer[LUCENE_MAX_WORD_LEN];
+
+			TokenGroup* tokenGroup=_CLNEW TokenGroup();
+
+			TCHAR buffer[LUCENE_MAX_FIELD_LEN+1];
+			Token token;
+			while ( tokenStream->next(&token) )
+			{
+				if((tokenGroup->getNumTokens()>0)&&(tokenGroup->isDistinct(&token))){
+					//the current token is distinct from previous tokens -
+					// markup the cached token group info
+					 startOffset = tokenGroup->getStartOffset();
+					 endOffset = tokenGroup->getEndOffset();
+
+					 _tcsncpy(substringBuffer,text+startOffset,endOffset-startOffset);
+					 substringBuffer[endOffset-startOffset]=_T('\0');
+
+					 TCHAR* encoded = _encoder->encodeText(substringBuffer);
+					 const TCHAR* markedUpText=_formatter->highlightTerm(encoded, tokenGroup);
+					 _CLDELETE_CARRAY(encoded);
+
+					 //store any whitespace etc from between this and last group
+					 if (startOffset > lastEndOffset){
+						 int len = startOffset-lastEndOffset;
+						 if ( len > LUCENE_MAX_FIELD_LEN )
+							 len = LUCENE_MAX_FIELD_LEN;
+						 _tcsncpy(buffer,text+lastEndOffset,len);
+						 buffer[len]=_T('\0');
+
+						 TCHAR* encoded = _encoder->encodeText(buffer);
+						 writeTo->append(encoded);
+						 _CLDELETE_CARRAY(encoded);
+					 }
+					 writeTo->append(markedUpText);
+					 lastEndOffset=endOffset;
+					 tokenGroup->clear();
+					 _CLDELETE_CARRAY(markedUpText);
+
+					//check if current token marks the start of a new fragment
+					if (_textFragmenter->isNewFragment(&token))
+					{
+						currentFrag->setScore(_fragmentScorer->getFragmentScore());
+						//record stats for a new fragment
+						currentFrag->setTextEndPos( writeTo->length() );
+						currentFrag =_CLNEW TextFragment(writeTo->length(), docFrags.size());
+						_fragmentScorer->startFragment(currentFrag);
+						docFrags.push_back(currentFrag);
+					}
+				}
+
+				// does query contain current token?
+				float_t score=_fragmentScorer->getTokenScore(&token);			
+				//TCHAR* highlightedTerm = _formatter->highlightTerm(&substringBuffer, token->termText(), score, startOffset);
+				//newText->append(highlightedTerm);
+				//_CLDELETE_CARRAY(highlightedTerm);
+				//_CLDELETE(token);
+
+				tokenGroup->addToken(&token,_fragmentScorer->getTokenScore(&token));
+
+				if(lastEndOffset>maxDocBytesToAnalyze)
+				{
+					break;
+				}
+			}
+			currentFrag->setScore(_fragmentScorer->getFragmentScore());
+
+			if(tokenGroup->getNumTokens()>0)
+  	        {
+  	            //flush the accumulated text (same code as in above loop)
+  	            startOffset = tokenGroup->getStartOffset();
+  	            endOffset = tokenGroup->getEndOffset();
+
+				_tcsncpy(substringBuffer,text+startOffset,endOffset-startOffset);
+				substringBuffer[endOffset-startOffset]=_T('\0');
+
+				TCHAR* encoded = _encoder->encodeText(substringBuffer);
+  	            const TCHAR* markedUpText=_formatter->highlightTerm(encoded, tokenGroup);
+				_CLDELETE_CARRAY(encoded);
+
+  	            //store any whitespace etc from between this and last group
+				if (startOffset > lastEndOffset){
+					int len = startOffset-lastEndOffset;
+					if ( len > LUCENE_MAX_FIELD_LEN )
+						len = LUCENE_MAX_FIELD_LEN;
+					_tcsncpy(buffer,text+lastEndOffset,len);
+					buffer[len]=_T('\0');
+
+					TCHAR* encoded = _encoder->encodeText(buffer);
+  					writeTo->append(encoded);
+					_CLDELETE_CARRAY(encoded);
+				}
+  	            writeTo->append(markedUpText);
+  	            lastEndOffset=endOffset;
+
+				_CLDELETE_CARRAY(markedUpText);
+  	        }
+
+			// append text after end of last token
+			//if (lastEndOffset < (int32_t)_tcslen(text))
+			//newText->append(text+lastEndOffset);
+
+			currentFrag->setTextEndPos(writeTo->length());
+
+			//sort the most relevant sections of the text
+			while (docFrags.size() > 0) {
+			//for (TextFragmentList::iterator i = docFrags.begin(); i != docFrags.end(); i++)
+			//{
+				currentFrag = (TextFragment*) docFrags[0];
+				docFrags.remove(0);
+
+				//If you are running with a version of Lucene before 11th Sept 03
+				// you do not have PriorityQueue.insert() - so uncomment the code below					
+
+				/*if (currentFrag->getScore() >= minScore)
+				{
+					fragQueue.put(currentFrag);
+					if (fragQueue.size() > maxNumFragments)
+					{ // if hit queue overfull
+						_CLLDELETE(fragQueue.pop()); // remove lowest in hit queue
+						minScore = ((TextFragment *) fragQueue.top())->getScore(); // reset minScore
+					}
+
+
+				} else {
+					_CLDELETE(currentFrag);
+				}*/
+
+				//The above code caused a problem as a result of Christoph Goller's 11th Sept 03
+				//fix to PriorityQueue. The correct method to use here is the new "insert" method
+				// USE ABOVE CODE IF THIS DOES NOT COMPILE!
+				if ( !fragQueue.insert(currentFrag) )
+					_CLDELETE(currentFrag);
+
+				//todo: check this
+			}
+
+			//return the most relevant fragments
+			int32_t fragsLen = fragQueue.size();
+			TextFragment** frags = _CL_NEWARRAY(TextFragment*,fragsLen+1);
+			for ( int32_t i=0;i<fragsLen;i++ )
+				frags[i] = fragQueue.pop();
+			frags[fragsLen]=NULL;
+
+			//merge any contiguous fragments to improve readability
+  	        if(mergeContiguousFragments)
+  	        {
+  	            _mergeContiguousFragments(frags,fragsLen);
+  	            CLArrayList<TextFragment*> fragTexts;
+  	            for (int32_t i = 0; i < fragsLen; i++)
+  	            {
+					TextFragment* tf = frags[i];
+  	                if ((tf != NULL) && (tf->getScore() > 0))
+  						fragTexts.push_back(tf);
+  	                else
+						_CLDELETE(tf);
+  	            }
+				_CLDELETE_ARRAY(frags);
+				frags = _CL_NEWARRAY(TextFragment*,fragTexts.size()+1);
+				fragTexts.toArray(frags);
+  	        }
+
+			_CLDELETE(tokenGroup);
+			//_CLDELETE(newText);
+			return frags;
+
+		}
+		_CLFINALLY(
+			if (tokenStream)
+			{
+				try
+				{
+					tokenStream->close();
+				}
+				catch (...)
+				{
+				}
+			}
+		)
+	}
+
+
+	void Highlighter::_mergeContiguousFragments(TextFragment** frag, int32_t fragsLen)
+	{
+		bool mergingStillBeingDone;
+		if ( frag[0] != NULL )
+			do
+			{
+				mergingStillBeingDone = false; //initialise loop control flag
+				//for each fragment, scan other frags looking for contiguous blocks
+				for (int32_t i=0; i<fragsLen; i++)
+				{
+					if (frag[i] == NULL)
+					{
+						continue;
+					}
+					//merge any contiguous blocks 
+					for (int32_t x=0; x<fragsLen; x++)
+					{
+					   if ( x==i )
+					      continue; //bug 1072183. don't try and merge with self
+
+						if (frag[x] == NULL)
+							continue;
+						if (frag[i] == NULL)
+							break;
+
+						TextFragment * frag1 = NULL;
+						TextFragment * frag2 = NULL;
+						int32_t frag1Num = 0;
+						int32_t frag2Num = 0;
+						int32_t bestScoringFragNum;
+						int32_t worstScoringFragNum;
+						//if blocks are contiguous....
+						if (frag[i]->follows(frag[x]))
+						{
+							frag1 = frag[x];
+							frag1Num = x;
+							frag2 = frag[i];
+							frag2Num = i;
+						}
+						else if (frag[x]->follows(frag[i]))
+						{
+							frag1 = frag[i];
+							frag1Num = i;
+							frag2 = frag[x];
+							frag2Num = x;
+						}
+						//merging required..
+						if (frag1 != NULL)
+						{
+							if (frag1->getScore() > frag2->getScore())
+							{
+								bestScoringFragNum = frag1Num;
+								worstScoringFragNum = frag2Num;
+							}
+							else
+							{
+								bestScoringFragNum = frag2Num;
+								worstScoringFragNum = frag1Num;
+							}
+							frag1->merge(frag2);
+							frag[worstScoringFragNum]= NULL;
+							mergingStillBeingDone = true;
+							frag[bestScoringFragNum]=frag1;
+							_CLDELETE(frag2);
+						}
+					}
+				}
+			}
+			while (mergingStillBeingDone);
+	}
+
+
+
+CL_NS_END2
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/oss/cl/clucene/src/clucene/highlighter/Highlighter.h
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/searchengine/oss/cl/clucene/src/clucene/highlighter/Highlighter.h	Fri Jun 11 14:43:47 2010 +0300
@@ -0,0 +1,265 @@
+/**
+ * Copyright 2002-2004 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _lucene_search_highlight_highlighter_
+#define _lucene_search_highlight_highlighter_
+
+#if defined(_LUCENE_PRAGMA_ONCE)
+# pragma once
+#endif
+
+#include "CLucene/util/StringBuffer.h"
+#include "CLucene/util/PriorityQueue.h"
+#include "CLucene/util/VoidList.h"
+#include "CLucene/highlighter/Formatter.h"
+#include "CLucene/highlighter/Encoder.h"
+#include "CLucene/highlighter/SimpleHTMLFormatter.h"
+#include "CLucene/highlighter/Fragmenter.h"
+#include "CLucene/highlighter/HighlightScorer.h"
+#include "CLucene/highlighter/SimpleFragmenter.h"
+#include "CLucene/highlighter/TextFragment.h"
+
+CL_NS_DEF2(search,highlight)
+
+/**
+* Class used to markup highlighted terms found in the best sections of a
+* text, using configurable {@link Fragmenter}, {@link Scorer}, {@link Formatter},
+* and tokenizers. 	  
+* {@link Encoder} and tokenizers.
+*/
+class Highlighter :LUCENE_BASE
+{
+private:
+	int32_t maxDocBytesToAnalyze;
+
+	Formatter * _formatter;
+	bool delete_formatter;
+	
+	Encoder* _encoder;
+	bool delete_encoder;
+
+	Fragmenter * _textFragmenter;
+	bool delete_textFragmenter;
+
+	HighlightScorer * _fragmentScorer;
+	bool delete_fragmentScorer;
+
+	/** Improves readability of a score-sorted list of TextFragments by merging any fragments 
+	 * that were contiguous in the original text into one larger fragment with the correct order.
+	 * This will leave a "null" in the array entry for the lesser scored fragment. 
+	 * 
+	 * @param frag An array of document fragments in descending score
+	 */
+	void _mergeContiguousFragments(TextFragment** frag, int32_t fragsLen);
+	
+public:
+	LUCENE_STATIC_CONSTANT(int32_t, DEFAULT_MAX_DOC_BYTES_TO_ANALYZE=50*1024);
+
+	/**
+	 * Constructs a Highlighter object with the provided scorer. The HighlightScorer object is owned
+	 * by the Highlighter object, and it will freed in the destructor.
+	 */
+	Highlighter(HighlightScorer * fragmentScorer);
+
+	Highlighter(Formatter * formatter, HighlightScorer * fragmentScorer);
+
+	Highlighter(Formatter * formatter, Encoder* encoder, HighlightScorer * fragmentScorer);
+
+
+	/**
+	 * Destructor for Highlighter. It deletes the owned HighlightScorer, formatter and textFragmenter.
+	 */
+	~Highlighter();
+
+	/**
+	 * Highlights chosen terms in a text, extracting the most relevant section.
+	 * The document text is analysed in chunks to record hit statistics
+	 * across the document. After accumulating stats, the fragment with the highest score
+	 * is returned
+	 *
+	 * @param tokenStream   a stream of tokens identified in the text parameter, including offset information. 
+	 * This is typically produced by an analyzer re-parsing a document's 
+	 * text. Some work may be done on retrieving TokenStreams more efficently 
+	 * by adding support for storing original text position data in the Lucene
+	 * index but this support is not currently available (as of Lucene 1.4 rc2).  
+	 * @param text text to highlight terms in
+	 *
+	 * @return highlighted text fragment or null if no terms found
+	 */
+	TCHAR* getBestFragment(CL_NS(analysis)::TokenStream * tokenStream, const TCHAR* text);
+
+	/**
+	 * Highlights chosen terms in a text, extracting the most relevant section.
+	 * This is a convenience method that calls
+	 * {@link #getBestFragment(TokenStream, const TCHAR*)}
+	 *
+	 * @param analyzer   the analyzer that will be used to split <code>text</code>
+	 * into chunks  
+	 * @param text text to highlight terms in
+	 * @param fieldName Name of field used to influence analyzer's tokenization policy 
+	 *
+	 * @return highlighted text fragment or null if no terms found
+	 */
+	TCHAR* getBestFragment(CL_NS(analysis)::Analyzer* analyzer, const TCHAR* fieldName, const TCHAR* text);
+
+	/**
+	 * Highlights chosen terms in a text, extracting the most relevant sections.
+	 * This is a convenience method that calls
+	 * {@link #getBestFragments(TokenStream, const TCHAR*, int)}
+	 *
+	 * @param analyzer   the analyzer that will be used to split <code>text</code>
+	 * into chunks  
+	 * @param text        	text to highlight terms in
+	 * @param maxNumFragments  the maximum number of fragments.
+	 *
+	 * @return highlighted text fragments (between 0 and maxNumFragments number of fragments)
+	 */
+	TCHAR** getBestFragments(
+		CL_NS(analysis)::Analyzer* analyzer,	
+		const TCHAR* text,
+		int32_t maxNumFragments);
+
+	/**
+	 * Highlights chosen terms in a text, extracting the most relevant sections.
+	 * The document text is analysed in chunks to record hit statistics
+	 * across the document. After accumulating stats, the fragments with the highest scores
+	 * are returned as an array of strings in order of score (contiguous fragments are merged into 
+	 * one in their original order to improve readability)
+	 *
+	 * @param text        	text to highlight terms in
+	 * @param maxNumFragments  the maximum number of fragments.
+	 *
+	 * @return highlighted text fragments (between 0 and maxNumFragments number of fragments)
+	 */
+	 TCHAR** getBestFragments(
+		CL_NS(analysis)::TokenStream * tokenStream,	
+		const TCHAR* text,
+		int32_t maxNumFragments);
+
+	/**
+    * Low level api to get the most relevant (formatted) sections of the document.
+  	* This method has been made public to allow visibility of score information held in TextFragment objects.
+  	* Thanks to Jason Calabrese for help in redefining the interface.
+    * @param tokenStream
+    * @param text
+    * @param maxNumFragments
+    * @param mergeContiguousFragments
+    */
+	TextFragment** getBestTextFragments(
+		CL_NS(util)::StringBuffer* writeTo,
+		CL_NS(analysis)::TokenStream * tokenStream,	
+		const TCHAR* text,
+		bool mergeContiguousFragments,
+		int32_t maxNumFragments);
+
+	/**
+	 * Highlights terms in the  text , extracting the most relevant sections
+	 * and concatenating the chosen fragments with a separator (typically "...").
+	 * The document text is analysed in chunks to record hit statistics
+	 * across the document. After accumulating stats, the fragments with the highest scores
+	 * are returned in order as "separator" delimited strings.
+	 *
+	 * @param text        text to highlight terms in
+	 * @param maxNumFragments  the maximum number of fragments.
+	 * @param separator  the separator used to intersperse the document fragments (typically "...")
+	 *
+	 * @return highlighted text
+	 */
+	TCHAR* getBestFragments(
+		CL_NS(analysis)::TokenStream * tokenStream,	
+		const TCHAR* text,
+		int32_t maxNumFragments,
+		const TCHAR* separator);
+
+	/**
+	 * @return the maximum number of bytes to be tokenized per doc 
+	 */
+	int32_t getMaxDocBytesToAnalyze()
+	{
+		return maxDocBytesToAnalyze;
+	}
+
+	/**
+	 * @param byteCount the maximum number of bytes to be tokenized per doc
+	 * (This can improve performance with large documents)
+	 */
+	void setMaxDocBytesToAnalyze(int32_t byteCount)
+	{
+		maxDocBytesToAnalyze = byteCount;
+	}
+
+	/**
+	 */
+	Fragmenter * getTextFragmenter()
+	{
+		return _textFragmenter;
+	}
+
+	/**
+	 * @param fragmenter
+	 */
+	void setTextFragmenter(Fragmenter * fragmenter)
+	{
+		if ( delete_textFragmenter ){
+			_CLDELETE(_textFragmenter);
+			delete_textFragmenter = false;
+		}
+		_textFragmenter = fragmenter;
+	}
+
+	/**
+	 * @return Object used to score each text fragment 
+	 */
+	HighlightScorer * getFragmentScorer()
+	{
+		return _fragmentScorer;
+	}
+
+
+	/**
+	 * @param HighlightScorer
+	 */
+	void setFragmentScorer(HighlightScorer * scorer)
+	{
+		if ( delete_fragmentScorer ){
+			delete_fragmentScorer = false;
+			_CLDELETE(scorer);
+		}
+		_fragmentScorer = scorer;
+	}
+
+	
+    Encoder* getEncoder()
+    {
+        return _encoder;
+    }
+    void setEncoder(Encoder* encoder)
+    {
+		if ( delete_encoder ){
+			_CLDELETE(encoder);
+			delete_encoder = false;
+		}
+        this->_encoder = encoder;
+    }
+
+
+};
+
+
+CL_NS_END2
+
+#endif
+
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/oss/cl/clucene/src/clucene/highlighter/Makefile.am
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/searchengine/oss/cl/clucene/src/clucene/highlighter/Makefile.am	Fri Jun 11 14:43:47 2010 +0300
@@ -0,0 +1,19 @@
+## Makefile.am -- Process this file with automake to produce Makefile.in
+highlighterdir = $(lsrcdir)/highlighter
+highlighterhdir = $(includedir)/CLucene/highlighter
+ 
+libclucene_contrib_la_SOURCES  += $(highlighterdir)/Encoder.cpp
+libclucene_contrib_la_SOURCES  += $(highlighterdir)/Formatter.cpp
+libclucene_contrib_la_SOURCES  += $(highlighterdir)/Fragmenter.cpp
+libclucene_contrib_la_SOURCES  += $(highlighterdir)/Highlighter.cpp
+libclucene_contrib_la_SOURCES  += $(highlighterdir)/QueryScorer.cpp
+libclucene_contrib_la_SOURCES  += $(highlighterdir)/QueryTermExtractor.cpp
+libclucene_contrib_la_SOURCES  += $(highlighterdir)/SimpleFragmenter.cpp
+libclucene_contrib_la_SOURCES  += $(highlighterdir)/SimpleHTMLEncoder.cpp
+libclucene_contrib_la_SOURCES  += $(highlighterdir)/SimpleHTMLFormatter.cpp
+libclucene_contrib_la_SOURCES  += $(highlighterdir)/TextFragment.cpp
+libclucene_contrib_la_SOURCES  += $(highlighterdir)/TokenGroup.cpp
+libclucene_contrib_la_SOURCES  += $(highlighterdir)/TokenSources.cpp
+libclucene_contrib_la_SOURCES  += $(highlighterdir)/WeightedTerm.cpp
+
+highlighterh_HEADERS = $(highlighterdir)/*.h
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/oss/cl/clucene/src/clucene/highlighter/QueryScorer.cpp
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/searchengine/oss/cl/clucene/src/clucene/highlighter/QueryScorer.cpp	Fri Jun 11 14:43:47 2010 +0300
@@ -0,0 +1,100 @@
+#include "clucene/stdheader.h"
+#include "queryscorer.h"
+
+CL_NS_DEF2(search,highlight)
+CL_NS_USE(index)
+CL_NS_USE(analysis)
+
+	QueryScorer::QueryScorer(const Query * query):
+        _uniqueTermsInFragment(true),
+		_termsToFind(false,true)
+		
+	 {
+		 WeightedTerm** _weighted_terms = QueryTermExtractor::getTerms(query);
+		 initialize(_weighted_terms);
+		 _CLDELETE_ARRAY(_weighted_terms);
+	 }
+	 QueryScorer::~QueryScorer()
+	 {
+	 }
+
+/*	 QueryScorer(Query* query, CL_NS(index)::IndexReader* reader, const TCHAR* fieldName)
+	 {
+		 WeightedTerm** _weighted_terms = QueryTermExtractor.getIdfWeightedTerms(query, reader, fieldName);
+		 initialize(_weighted_terms);
+	 }*/
+
+
+	QueryScorer::QueryScorer(WeightedTerm** weightedTerms)
+	{
+		 initialize(weightedTerms);
+	}
+	
+	void QueryScorer::initialize(WeightedTerm** weightedTerms)
+	{
+		_currentTextFragment = NULL;
+		_totalScore = 0;
+		_maxTermWeight = 0;
+
+		// Copy external weighted terms
+		 int i=0;
+		 while ( weightedTerms[i] != NULL ){
+			const WeightedTerm* existingTerm=_termsToFind.get(weightedTerms[i]->getTerm());
+			if( (existingTerm==NULL) ||(existingTerm->getWeight()<weightedTerms[i]->getWeight()) )
+  	        {
+  				//if a term is defined more than once, always use the highest scoring weight
+				WeightedTerm* term = weightedTerms[i];
+				_termsToFind.put(term->getTerm(), term);
+
+				_maxTermWeight=max(_maxTermWeight,weightedTerms[i]->getWeight());
+  	        }else
+				_CLDELETE(weightedTerms[i]);
+
+			i++;
+		 }
+	}
+
+	void QueryScorer::startFragment(TextFragment * newFragment)
+	{
+		_uniqueTermsInFragment.clear();
+		_currentTextFragment=newFragment;
+		_totalScore=0;
+		
+	}
+	
+	float_t QueryScorer::getTokenScore(Token * token)
+	{
+		const TCHAR* termText=token->termText();
+		
+		const WeightedTerm* queryTerm = _termsToFind.get(termText);
+		if(queryTerm==NULL)
+		{
+			//not a query term - return
+			return 0;
+		}
+		//found a query term - is it unique in this doc?
+		if(_uniqueTermsInFragment.find(termText)==_uniqueTermsInFragment.end())
+		{
+			_totalScore+=queryTerm->getWeight();
+			TCHAR* owned_term = stringDuplicate(termText);
+			_uniqueTermsInFragment.insert(owned_term);
+		}
+		return queryTerm->getWeight();
+	}
+	
+	/**
+  	*
+  	* @return The highest weighted term (useful for passing to GradientFormatter to set
+  	* top end of coloring scale.
+  	*/
+	float_t QueryScorer::getMaxTermWeight()
+	{
+  		return _maxTermWeight;
+	}
+
+
+	float_t QueryScorer::getFragmentScore(){
+		return _totalScore;
+	}
+
+CL_NS_END2
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/oss/cl/clucene/src/clucene/highlighter/QueryScorer.h
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/searchengine/oss/cl/clucene/src/clucene/highlighter/QueryScorer.h	Fri Jun 11 14:43:47 2010 +0300
@@ -0,0 +1,114 @@
+/**
+ * Copyright 2002-2004 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _lucene_search_highlighter_queryscorer_
+#define _lucene_search_highlighter_queryscorer_
+
+#if defined(_LUCENE_PRAGMA_ONCE)
+# pragma once
+#endif
+
+#include <math.h> // float_t
+#include "CLucene/analysis/AnalysisHeader.h"
+#include "CLucene/search/SearchHeader.h"
+#include "CLucene/index/IndexReader.h"
+#include "CLucene/highlighter/WeightedTerm.h"
+#include "CLucene/highlighter/HighlightScorer.h"
+#include "CLucene/highlighter/QueryTermExtractor.h"
+#include "CLucene/highlighter/TextFragment.h"
+
+CL_NS_DEF2(search,highlight)
+
+/**
+ * {@link Scorer} implementation which scores text fragments by the number of unique query terms found.
+ * This class uses the {@link QueryTermExtractor} class to process determine the query terms and 
+ * their boosts to be used. 
+ */
+//TODO: provide option to boost score of fragments near beginning of document 
+// based on fragment.getFragNum()
+class QueryScorer : public HighlightScorer
+{
+private:
+	TextFragment * _currentTextFragment;
+	CL_NS(util)::CLHashSet<const TCHAR*,
+		CL_NS(util)::Compare::TChar,
+		CL_NS(util)::Deletor::tcArray> _uniqueTermsInFragment;
+	double _totalScore;
+	double _maxTermWeight;
+	CL_NS(util)::LHashMap<const TCHAR*, const WeightedTerm *,
+		CL_NS(util)::Compare::TChar,
+		CL_NS(util)::Equals::TChar,
+		CL_NS(util)::Deletor::tcArray,
+		CL_NS(util)::Deletor::Object<const WeightedTerm> > _termsToFind;
+
+public:
+	/**
+	* 
+	* @param query a Lucene query (ideally rewritten using query.rewrite 
+	* before being passed to this class and the searcher)
+	*/
+	QueryScorer(const Query * query);
+
+	/**
+	* 
+	* @param query a Lucene query (ideally rewritten using query.rewrite 
+	* before being passed to this class and the searcher)
+	* @param reader used to compute IDF which can be used to a) score selected fragments better 
+	* b) use graded highlights eg set font color intensity
+	* @param fieldName the field on which Inverse Document Frequency (IDF) calculations are based
+	*/
+	QueryScorer(const Query* query, CL_NS(index)::IndexReader* reader, const TCHAR* fieldName);
+
+	QueryScorer(WeightedTerm** weightedTerms);
+
+	~QueryScorer();
+
+	/* (non-Javadoc)
+	 * @see org.apache.lucene.search.highlight.FragmentScorer#startFragment(org.apache.lucene.search.highlight.TextFragment)
+	 */
+	void startFragment(TextFragment* newFragment);
+	
+	/* (non-Javadoc)
+	 * @see org.apache.lucene.search.highlight.FragmentScorer#scoreToken(org.apache.lucene.analysis.Token)
+	 */
+	float_t getTokenScore(CL_NS(analysis)::Token * token);
+	
+	/* (non-Javadoc)
+	 * @see org.apache.lucene.search.highlight.FragmentScorer#endFragment(org.apache.lucene.search.highlight.TextFragment)
+	 */
+	float_t getFragmentScore();
+
+	/* (non-Javadoc)
+	 * @see org.apache.lucene.search.highlight.FragmentScorer#allFragmentsProcessed()
+	 */
+	void allFragmentsProcessed();
+
+	/**
+	 * 
+	 * @return The highest weighted term (useful for passing to GradientFormatter to set
+	 * top end of coloring scale.  
+		*/
+	float_t getMaxTermWeight();
+
+private:
+	void initialize(WeightedTerm** weightedTerms);
+
+};
+
+CL_NS_END2
+
+#endif
+
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/oss/cl/clucene/src/clucene/highlighter/QueryTermExtractor.cpp
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/searchengine/oss/cl/clucene/src/clucene/highlighter/QueryTermExtractor.cpp	Fri Jun 11 14:43:47 2010 +0300
@@ -0,0 +1,136 @@
+#include "CLucene/StdHeader.h"
+#include "QueryTermExtractor.h"
+
+CL_NS_DEF2(search,highlight)
+CL_NS_USE(index)
+
+	WeightedTerm** QueryTermExtractor::getTerms(const Query *query) 
+	{
+		WeightedTerm** ret = getTerms(query,false);
+		return ret;
+	}
+
+	WeightedTerm** QueryTermExtractor::getTerms(const Query * query, bool prohibited) 
+	{
+		WeightedTermList terms(false);
+		getTerms(query,&terms,prohibited);
+
+		// Return extracted terms
+		WeightedTerm** ret = _CL_NEWARRAY(WeightedTerm*,terms.size()+1);
+		terms.toArray(ret);
+
+		return ret;
+	}
+
+	void QueryTermExtractor::getTerms(const Query * query, WeightedTermList * terms,bool prohibited) 
+	{
+		if (query->instanceOf( BooleanQuery::getClassName() ))
+			getTermsFromBooleanQuery((BooleanQuery *) query, terms, prohibited);
+		else if (query->instanceOf( PhraseQuery::getClassName() ))
+			getTermsFromPhraseQuery((PhraseQuery *) query, terms);
+		else if (query->instanceOf( TermQuery::getClassName() ))
+			getTermsFromTermQuery((TermQuery *) query, terms);
+		//else if(query->instanceOf(_T("SpanNearQuery"))
+		//	getTermsFromSpanNearQuery((SpanNearQuery*) query, terms);
+	}
+
+	/**
+  	* Extracts all terms texts of a given Query into an array of WeightedTerms
+  	*
+  	* @param query      Query to extract term texts from
+  	* @param reader used to compute IDF which can be used to a) score selected fragments better
+  	* b) use graded highlights eg chaning intensity of font color
+  	* @param fieldName the field on which Inverse Document Frequency (IDF) calculations are based
+  	* @return an array of the terms used in a query, plus their weights.
+  	*/
+  	WeightedTerm** QueryTermExtractor::getIdfWeightedTerms(const Query* query, IndexReader* reader, const TCHAR* fieldName)
+  	{
+  	    WeightedTermList terms(true);
+		getTerms(query,&terms,false);
+
+  	    int32_t totalNumDocs=reader->numDocs();
+		
+		WeightedTermList::iterator itr = terms.begin();
+  	    while ( itr != terms.end() )
+  		{
+  			try
+  			{
+				Term* term = _CLNEW Term(fieldName,(*itr)->getTerm());
+  				int32_t docFreq=reader->docFreq(term);
+				_CLDECDELETE(term);
+
+  				//IDF algorithm taken from DefaultSimilarity class
+  				float_t idf=(float_t)(log(totalNumDocs/(float_t)(docFreq+1)) + 1.0);
+  				(*itr)->setWeight((*itr)->getWeight() * idf);
+  			}catch (LuceneError& e){
+  				if ( e.number()!=CL_ERR_IO )
+					throw e;
+  			}
+
+			itr++;
+  		}
+  	   
+		// Return extracted terms
+		WeightedTerm** ret = _CL_NEWARRAY(WeightedTerm*,terms.size()+1);
+		terms.toArray(ret);
+
+		return ret;
+  	}
+
+	void QueryTermExtractor::getTermsFromBooleanQuery(const BooleanQuery * query, WeightedTermList * terms, bool prohibited)
+	{
+		// TODO: change Query to get the queryclauses and their number in one function call
+		BooleanClause** queryClauses = query->getClauses();
+		uint32_t numClauses = query->getClauseCount();
+
+		for (uint32_t i = 0; i < numClauses; i++)
+		{
+			if (prohibited || !queryClauses[i]->prohibited){
+				Query* qry = queryClauses[i]->query;
+				getTerms(qry, terms, prohibited);
+			}
+		}
+
+		_CLDELETE_ARRAY(queryClauses);
+	}
+
+	void QueryTermExtractor::getTermsFromPhraseQuery(const PhraseQuery * query, WeightedTermList * terms)
+	{
+		Term** queryTerms = query->getTerms();
+		int32_t i = 0;
+		while ( queryTerms[i] != NULL ){
+			WeightedTerm * pWT = _CLNEW WeightedTerm(query->getBoost(),queryTerms[i]->text());
+			if (terms->find(pWT)==terms->end()) // possible memory leak if key already present
+				terms->insert(pWT);
+			else
+				_CLDELETE(pWT);
+
+			i++;
+		}
+		_CLDELETE_ARRAY(queryTerms);
+	}
+
+	void QueryTermExtractor::getTermsFromTermQuery(const TermQuery * query, WeightedTermList * terms)
+	{
+		Term * term = query->getTerm();
+		WeightedTerm * pWT = _CLNEW WeightedTerm(query->getBoost(),term->text());
+		_CLDECDELETE(term);
+		if (terms->find(pWT)==terms->end()) // possible memory leak if key already present
+			terms->insert(pWT);
+		else
+			_CLDELETE(pWT);
+	}
+
+	//todo: implement this when span queries are implemented
+	/*void getTermsFromSpanNearQuery(SpanNearQuery* query, WeightedTermList* terms){
+  	    Collection queryTerms = query.getTerms();
+
+  	    for(Iterator iterator = queryTerms.iterator(); iterator.hasNext();){
+  	        // break it out for debugging.
+  	        Term term = (Term) iterator.next();
+  	        const TCHAR* text = term.text();
+  	        terms.add(_CLNEW WeightedTerm(query.getBoost(), text));
+  	    }
+  	}*/
+
+CL_NS_END2
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/oss/cl/clucene/src/clucene/highlighter/QueryTermExtractor.h
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/searchengine/oss/cl/clucene/src/clucene/highlighter/QueryTermExtractor.h	Fri Jun 11 14:43:47 2010 +0300
@@ -0,0 +1,86 @@
+/**
+ * Copyright 2002-2004 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef _lucene_search_highlight_querytermextractor_
+#define _lucene_search_highlight_querytermextractor_
+
+#if defined(_LUCENE_PRAGMA_ONCE)
+# pragma once
+#endif
+
+#include "CLucene/util/VoidList.h"
+#include "CLucene/search/SearchHeader.h"
+#include "CLucene/index/IndexReader.h"
+#include "CLucene/search/BooleanQuery.h"
+#include "CLucene/search/PhraseQuery.h"
+#include "CLucene/search/TermQuery.h"
+#include "CLucene/highlighter/WeightedTerm.h"
+
+CL_NS_DEF2(search,highlight)
+
+/**
+ * Utility class used to extract the terms used in a query, plus any weights.
+ * This class will not find terms for MultiTermQuery, RangeQuery and PrefixQuery classes
+ * so the caller must pass a rewritten query (see Query.rewrite) to obtain a list of 
+ * expanded terms. 
+ * 
+ */
+class QueryTermExtractor
+{
+	QueryTermExtractor(){
+	}
+public:
+
+	/**
+	 * Extracts all terms texts of a given Query into an array of WeightedTerms
+	 *
+	 * @param query      Query to extract term texts from
+	 * @return an array of the terms used in a query, plus their weights.
+	 */
+	static WeightedTerm** getTerms(const Query *query);
+
+	/**
+	 * Extracts all terms texts of a given Query into an array of WeightedTerms
+	 *
+	 * @param query      Query to extract term texts from
+	 * @param reader used to compute IDF which can be used to a) score selected fragments better 
+	 * b) use graded highlights eg chaning intensity of font color
+	 * @param fieldName the field on which Inverse Document Frequency (IDF) calculations are based
+	 * @return an array of the terms used in a query, plus their weights.
+	 */
+	 static WeightedTerm** getIdfWeightedTerms(const Query* query, CL_NS(index)::IndexReader* reader, const TCHAR* fieldName);
+
+	/**
+	 * Extracts all terms texts of a given Query into an array of WeightedTerms
+	 *
+	 * @param query      Query to extract term texts from
+	 * @param prohibited <code>true</code> to extract "prohibited" terms, too
+     * @return an array of the terms used in a query, plus their weights.Memory owned by the caller
+     */
+	static WeightedTerm** getTerms(const Query * query, bool prohibited);
+
+
+	static void getTerms(const Query * query, WeightedTermList* terms,bool prohibited);
+	static void getTermsFromBooleanQuery(const BooleanQuery * query, WeightedTermList* terms, bool prohibited);
+	static void getTermsFromPhraseQuery(const PhraseQuery * query, WeightedTermList* terms);
+	static void getTermsFromTermQuery(const TermQuery * query, WeightedTermList* terms);
+//	static void getTermsFromSpanNearQuery(SpanNearQuery* query, WeightedTermList* terms);
+};
+
+CL_NS_END2
+
+
+
+#endif
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/oss/cl/clucene/src/clucene/highlighter/Scorer.h
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/searchengine/oss/cl/clucene/src/clucene/highlighter/Scorer.h	Fri Jun 11 14:43:47 2010 +0300
@@ -0,0 +1,62 @@
+#ifndef _lucene_search_highlight_scorer_h_
+#define _lucene_search_highlight_scorer_h_
+
+/**
+ * Copyright 2002-2004 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <CLucene\StdHeader.h>
+#include <CLucene\Analysis\AnalysisHeader.h>
+
+using namespace lucene::analysis;
+
+/**
+ * Adds to the score for a fragment based on its tokens
+ */
+namespace lucene { namespace search { namespace highlight {
+
+	class TextFragment;
+
+class Scorer
+{
+public:
+	virtual ~Scorer() = 0;
+	/**
+	 * called when a new fragment is started for consideration
+	 * @param newFragment
+	 */
+	virtual void startFragment(TextFragment * newFragment) = 0;
+
+	/**
+	 * Called for each token in the current fragment
+	 * @param token The token to be scored
+	 * @return a score which is passed to the TermHighlighter class to influence the mark-up of the text
+	 * (this return value is NOT used to score the fragment)
+	 */
+	virtual float_t getTokenScore(Token * token) = 0;
+	
+
+	/**
+	 * Called when the highlighter has no more tokens for the current fragment - the scorer will typically
+	 * call setScore() on the fragment passed in startFragment to record total info
+	 *
+	 */	
+	virtual float_t getFragmentScore() = 0;
+
+};
+
+}}}
+
+#endif
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/oss/cl/clucene/src/clucene/highlighter/SimpleFragmenter.cpp
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/searchengine/oss/cl/clucene/src/clucene/highlighter/SimpleFragmenter.cpp	Fri Jun 11 14:43:47 2010 +0300
@@ -0,0 +1,38 @@
+#include "CLucene/StdHeader.h"
+#include "SimpleFragmenter.h"
+
+CL_NS_DEF2(search,highlight)
+CL_NS_USE(analysis)
+
+SimpleFragmenter::SimpleFragmenter(int32_t fragmentSize)
+	: _currentNumFrags(0), _fragmentSize(fragmentSize)
+{
+}
+SimpleFragmenter::~SimpleFragmenter(){
+}
+
+void SimpleFragmenter::start(const TCHAR*)
+{
+	_currentNumFrags=1;
+}
+
+bool SimpleFragmenter::isNewFragment(const Token * token)
+{
+	bool isNewFrag= token->endOffset()>=(_fragmentSize*_currentNumFrags);
+	if (isNewFrag) {
+		_currentNumFrags++;
+	}
+	return isNewFrag;
+}
+
+int32_t SimpleFragmenter::getFragmentSize() const
+{
+	return _fragmentSize;
+}
+
+void SimpleFragmenter::setFragmentSize(int32_t size)
+{
+	_fragmentSize = size;
+}
+
+CL_NS_END2
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/oss/cl/clucene/src/clucene/highlighter/SimpleFragmenter.h
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/searchengine/oss/cl/clucene/src/clucene/highlighter/SimpleFragmenter.h	Fri Jun 11 14:43:47 2010 +0300
@@ -0,0 +1,74 @@
+/**
+ * Copyright 2002-2004 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _lucene_search_highlight_simplefragmenter_
+#define _lucene_search_highlight_simplefragmenter_
+
+#if defined(_LUCENE_PRAGMA_ONCE)
+# pragma once
+#endif
+
+#include "CLucene/analysis/AnalysisHeader.h"
+#include "CLucene/highlighter/Fragmenter.h"
+
+CL_NS_DEF2(search,highlight)
+
+/**
+ * {@link Fragmenter} implementation which breaks text up into same-size 
+ * fragments with no concerns over spotting sentence boundaries.
+ */
+
+class SimpleFragmenter:public Fragmenter
+{
+private:
+	LUCENE_STATIC_CONSTANT(int32_t, DEFAULT_FRAGMENT_SIZE =100 );
+	int32_t _currentNumFrags;
+	int32_t _fragmentSize;
+
+public:
+	/**
+	 * 
+	 * @param fragmentSize size in bytes of each fragment
+	 */
+	SimpleFragmenter(int32_t fragmentSize = DEFAULT_FRAGMENT_SIZE);
+
+	~SimpleFragmenter();
+
+	/* (non-Javadoc)
+	 * @see org.apache.lucene.search.highlight.TextFragmenter#start(const TCHAR*)
+	 */
+	void start(const TCHAR* originalText);
+
+	/* (non-Javadoc)
+	 * @see org.apache.lucene.search.highlight.TextFragmenter#isNewFragment(org.apache.lucene.analysis.Token)
+	 */
+	bool isNewFragment(const CL_NS(analysis)::Token * token);
+
+	/**
+	 * @return size in bytes of each fragment
+	 */
+	int32_t getFragmentSize() const;
+
+	/**
+	 * @param size size in bytes of each fragment
+	 */
+	void setFragmentSize(int32_t size);
+
+};
+
+CL_NS_END2
+
+#endif
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/oss/cl/clucene/src/clucene/highlighter/SimpleHTMLEncoder.cpp
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/searchengine/oss/cl/clucene/src/clucene/highlighter/SimpleHTMLEncoder.cpp	Fri Jun 11 14:43:47 2010 +0300
@@ -0,0 +1,64 @@
+#include "CLucene/StdHeader.h"
+#include "SimpleHTMLEncoder.h"
+CL_NS_DEF2(search,highlight)
+
+SimpleHTMLEncoder::SimpleHTMLEncoder(void)
+{
+}
+
+SimpleHTMLEncoder::~SimpleHTMLEncoder(void)
+{
+}
+
+TCHAR* SimpleHTMLEncoder::encodeText(TCHAR* originalText)
+{
+	return htmlEncode(originalText);
+}
+
+TCHAR* SimpleHTMLEncoder::htmlEncode(TCHAR* plainText) 
+{
+	size_t plainTextLen = _tcslen(plainText);
+	if (plainText == NULL || plainTextLen == 0)
+	{
+		return STRDUP_TtoT(_T(""));
+	}
+
+	CL_NS(util)::StringBuffer result(plainTextLen);
+
+	for (int32_t index=0; index<plainTextLen; index++) 
+	{
+		TCHAR ch = plainText[index];
+
+		switch (ch) 
+		{
+		case '"':
+			result.append(_T("&quot;"));
+			break;
+
+		case '&':
+			result.append(_T("&amp;"));
+			break;
+
+		case '<':
+			result.append(_T("&lt;"));
+			break;
+
+		case '>':
+			result.append(_T("&gt;"));
+			break;
+
+		default:
+			if (ch < 128)
+				result.appendChar(ch);
+			else{
+  	            result.append(_T("&#"));
+				result.appendInt(ch);
+				result.append(_T(";"));
+			}
+		}
+	}
+
+	return result.toString();
+}
+
+CL_NS_END2
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/oss/cl/clucene/src/clucene/highlighter/SimpleHTMLEncoder.h
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/searchengine/oss/cl/clucene/src/clucene/highlighter/SimpleHTMLEncoder.h	Fri Jun 11 14:43:47 2010 +0300
@@ -0,0 +1,49 @@
+/**
+ * Copyright 2002-2004 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _lucene_search_highlight_simplehtmlencoder_
+#define _lucene_search_highlight_simplehtmlencoder_
+
+#if defined(_LUCENE_PRAGMA_ONCE)
+# pragma once
+#endif
+
+#include "CLucene/util/StringBuffer.h"
+#include "Encoder.h"
+
+CL_NS_DEF2(search,highlight)
+
+/**
+ * Simple {@link Encoder} implementation to escape text for HTML output
+ *
+ */
+class SimpleHTMLEncoder:public Encoder
+{
+public:
+	SimpleHTMLEncoder(void);
+	~SimpleHTMLEncoder(void);
+	
+	TCHAR* encodeText(TCHAR* originalText);
+	
+	/**
+	 * Encode string into HTML
+	 */
+	static TCHAR* htmlEncode(TCHAR* plainText) ;
+};
+
+CL_NS_END2
+
+#endif
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/oss/cl/clucene/src/clucene/highlighter/SimpleHTMLFormatter.cpp
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/searchengine/oss/cl/clucene/src/clucene/highlighter/SimpleHTMLFormatter.cpp	Fri Jun 11 14:43:47 2010 +0300
@@ -0,0 +1,38 @@
+#include "CLucene/StdHeader.h"
+#include "SimpleHTMLFormatter.h"
+
+
+CL_NS_DEF2(search,highlight)
+CL_NS_USE(util)
+
+SimpleHTMLFormatter::SimpleHTMLFormatter(const TCHAR* preTag, const TCHAR* postTag):
+	_preTag(stringDuplicate(preTag)),
+	_postTag(stringDuplicate(postTag))
+{
+}
+
+SimpleHTMLFormatter::SimpleHTMLFormatter()
+{
+	_preTag = stringDuplicate(_T("<B>"));
+	_postTag = stringDuplicate(_T("</B>"));
+}
+
+SimpleHTMLFormatter::~SimpleHTMLFormatter() 
+{
+	_CLDELETE_CARRAY(_preTag);
+	_CLDELETE_CARRAY(_postTag);
+}
+
+TCHAR* SimpleHTMLFormatter::highlightTerm(const TCHAR* originalText, const TokenGroup* tokenGroup)
+{
+	if(tokenGroup->getTotalScore()>0){
+		StringBuffer sb;
+		sb.append(_preTag);
+		sb.append(originalText);
+		sb.append(_postTag);
+		return sb.toString();
+	}
+	return stringDuplicate(originalText);
+}
+
+CL_NS_END2
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/oss/cl/clucene/src/clucene/highlighter/SimpleHTMLFormatter.h
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/searchengine/oss/cl/clucene/src/clucene/highlighter/SimpleHTMLFormatter.h	Fri Jun 11 14:43:47 2010 +0300
@@ -0,0 +1,63 @@
+/**
+ * Copyright 2002-2004 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _lucene_search_highlight_simplehtmlformatter_
+#define _lucene_search_highlight_simplehtmlformatter_
+
+#if defined(_LUCENE_PRAGMA_ONCE)
+# pragma once
+#endif
+
+#include "CLucene/util/StringBuffer.h"
+#include "CLucene/highlighter/Formatter.h"
+
+CL_NS_DEF2(search,highlight)
+
+/**
+ * Simple {@link Formatter} implementation to highlight terms with a pre and post tag
+ *
+ */
+class SimpleHTMLFormatter :public Formatter
+{
+private:
+	const TCHAR* _preTag;
+	const TCHAR* _postTag;
+
+public:
+	~SimpleHTMLFormatter(); 
+
+
+	SimpleHTMLFormatter(const TCHAR* preTag, const TCHAR* postTag);
+
+	/**
+	 * Default constructor uses HTML: &lt;B&gt; tags to markup terms
+	 * 
+	 **/
+	SimpleHTMLFormatter();
+
+	
+	/**
+	* Returns the original text enclosed in _preTag and _postTag, if the score is greater 
+	* than 0. Otherwise, it returns the original text.
+	* It doesn't use the stemmed text nor the startOffset. 
+	* It allocates memory for the returned text, and it has to be freed by the caller.
+	*/
+	TCHAR* highlightTerm(const TCHAR* originalText, const TokenGroup* tokenGroup);
+};
+
+CL_NS_END2
+
+#endif
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/oss/cl/clucene/src/clucene/highlighter/TextFragment.cpp
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/searchengine/oss/cl/clucene/src/clucene/highlighter/TextFragment.cpp	Fri Jun 11 14:43:47 2010 +0300
@@ -0,0 +1,61 @@
+#include "CLucene/StdHeader.h"
+#include "TextFragment.h"
+
+CL_NS_DEF2(search,highlight)
+CL_NS_USE(util)
+
+	TextFragment::TextFragment(int32_t textStartPos, int32_t fragNum)
+	{
+		_textStartPos = textStartPos;
+		_textEndPos=0;
+		_fragNum = fragNum;
+	}
+	TextFragment::~TextFragment(){
+	}
+
+	void TextFragment::setScore(float_t score)
+	{
+		_score=score;
+	}
+
+	float_t TextFragment::getScore() const
+	{
+		return _score;
+	}
+
+	/**
+	 * @param frag2 Fragment to be merged into this one
+	 */
+	void TextFragment::merge(const TextFragment * frag2)
+	{
+		_textEndPos = frag2->_textEndPos;
+		_score=max(_score,frag2->_score);
+	}
+	/**
+	 * @param fragment 
+	 * @return true if this fragment follows the one passed
+	 */
+	bool TextFragment::follows(const TextFragment * fragment) const
+	{
+		return _textStartPos == fragment->_textEndPos;
+	}
+
+	/**
+	 * @return the fragment sequence number
+	 */
+	int32_t TextFragment::getFragNum() const
+	{
+		return _fragNum;
+	}
+
+	/* Returns the marked-up text for this text fragment 
+	 */
+	TCHAR* TextFragment::toString(StringBuffer* buffer) {
+		TCHAR* ret = _CL_NEWARRAY(TCHAR,_textEndPos-_textStartPos+1);
+		_tcsncpy(ret,buffer->getBuffer()+_textStartPos,_textEndPos-_textStartPos);
+		ret[_textEndPos-_textStartPos]=_T('\0');
+		
+		return ret;
+	}
+
+CL_NS_END2
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/oss/cl/clucene/src/clucene/highlighter/TextFragment.h
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/searchengine/oss/cl/clucene/src/clucene/highlighter/TextFragment.h	Fri Jun 11 14:43:47 2010 +0300
@@ -0,0 +1,90 @@
+/**
+ * Copyright 2002-2004 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _lucene_search_highlight_textfragment_
+#define _lucene_search_highlight_textfragment_
+
+#if defined(_LUCENE_PRAGMA_ONCE)
+# pragma once
+#endif
+
+#include "CLucene/util/StringBuffer.h"
+#include "CLucene/util/VoidList.h"
+
+CL_NS_DEF2(search,highlight)
+
+/**
+ * Low-level class used to record information about a section of a document 
+ * with a score.
+ */
+class TextFragment:LUCENE_BASE
+{
+	int32_t _fragNum;
+	int32_t _textStartPos;
+	int32_t _textEndPos;
+	float_t _score;
+
+public:
+	TextFragment(int32_t textStartPos, int32_t fragNum);
+	~TextFragment();
+
+	void setScore(float_t score);
+	float_t getScore() const;
+
+	int32_t textEndPos(){ return _textEndPos; }
+	void setTextEndPos(int32_t val){ _textEndPos = val; }
+
+	/**
+	 * @param frag2 Fragment to be merged into this one
+	 */
+	void merge(const TextFragment * frag2);
+
+	/**
+	 * @param fragment 
+	 * @return true if this fragment follows the one passed
+	 */
+	bool follows(const TextFragment * fragment) const;
+
+	/**
+	 * @return the fragment sequence number
+	 */
+	int32_t getFragNum() const;
+
+	/* Returns the marked-up text for this text fragment 
+	 */
+	TCHAR* toString(CL_NS(util)::StringBuffer* buffer);
+
+	/**
+	 * Compare weighted terms, according to the term text.
+	 * @todo Do we have to take boost factors into account
+	 */
+	class Compare:LUCENE_BASE, public CL_NS(util)::Compare::_base //<TextFragment*>
+	{
+	public:
+	//todo: this should be more efficient, but will be using a hash table soon, anyway
+		bool operator()( TextFragment* t1, TextFragment* t2 ) const;
+		size_t operator()( TextFragment* t ) const;
+	};
+};
+
+/**
+ * Text fragment list.
+ */
+//typedef CL_NS(util)::CLSetList<TextFragment*,TextFragment::Compare,CL_NS(util)::Deletor::Object<TextFragment> > TextFragmentList;
+
+CL_NS_END2
+
+#endif
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/oss/cl/clucene/src/clucene/highlighter/TokenGroup.cpp
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/searchengine/oss/cl/clucene/src/clucene/highlighter/TokenGroup.cpp	Fri Jun 11 14:43:47 2010 +0300
@@ -0,0 +1,104 @@
+#include "CLucene/StdHeader.h"
+#include "TokenGroup.h"
+
+CL_NS_DEF2(search,highlight)
+CL_NS_USE(analysis)
+
+TokenGroup::TokenGroup(void)
+{
+	numTokens=0;
+	startOffset=0;
+	endOffset=0;
+}
+
+TokenGroup::~TokenGroup(void)
+{
+}
+
+void TokenGroup::addToken(Token* token, float_t score)
+{
+	if(numTokens < MAX_NUM_TOKENS_PER_GROUP)
+    {	    
+		if(numTokens==0)
+		{
+			startOffset=token->startOffset();		
+			endOffset=token->endOffset();		
+		}
+		else
+		{
+			startOffset=min(startOffset,token->startOffset());		
+			endOffset=max(endOffset,token->endOffset());		
+		}
+		tokens[numTokens].set(token->termText(),token->startOffset(),token->endOffset(),token->type());;
+		scores[numTokens]=score;
+		numTokens++;
+    }
+}
+
+CL_NS(analysis)::Token& TokenGroup::getToken(int32_t index)
+{
+	return tokens[index];
+}
+
+float_t TokenGroup::getScore(int32_t index) const
+{
+	return scores[index];
+}
+
+int32_t TokenGroup::getEndOffset() const
+{
+	return endOffset;
+}
+
+int32_t TokenGroup::getNumTokens() const
+{
+	return numTokens;
+}
+
+int32_t TokenGroup::getStartOffset() const
+{
+	return startOffset;
+}
+
+float_t TokenGroup::getTotalScore() const
+{
+	float_t total=0;
+	for (int32_t i = 0; i < numTokens; i++)
+	{
+		total+=scores[i];
+	}
+	return total;
+}
+
+/*void addToken(CL_NS(analysis)::Token* token, float_t score)
+{
+	if(numTokens < MAX_NUM_TOKENS_PER_GROUP)
+  		{
+		if(numTokens==0)
+		{
+			startOffset=token->startOffset();		
+			endOffset=token->endOffset();		
+		}
+		else
+		{
+			startOffset=min(startOffset,token->startOffset());		
+			endOffset=max(endOffset,token->endOffset());		
+		}
+		tokens[numTokens]=token;
+		scores[numTokens]=score;
+		numTokens++;
+	}
+}*/
+
+bool TokenGroup::isDistinct(CL_NS(analysis)::Token* token) const
+{
+	return token->startOffset() > endOffset;
+}
+
+
+void TokenGroup::clear()
+{
+	numTokens=0;
+}
+
+CL_NS_END2
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/oss/cl/clucene/src/clucene/highlighter/TokenGroup.h
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/searchengine/oss/cl/clucene/src/clucene/highlighter/TokenGroup.h	Fri Jun 11 14:43:47 2010 +0300
@@ -0,0 +1,86 @@
+/**
+ * Copyright 2002-2004 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _lucene_search_highlight_tokengroup_
+#define _lucene_search_highlight_tokengroup_
+
+#if defined(_LUCENE_PRAGMA_ONCE)
+# pragma once
+#endif
+
+#include "CLucene/analysis/AnalysisHeader.h"
+
+CL_NS_DEF2(search,highlight)
+
+/**
+ * One, or several overlapping tokens, along with the score(s) and the
+ * scope of the original text
+ */
+class TokenGroup: LUCENE_BASE
+{
+	LUCENE_STATIC_CONSTANT(int32_t,MAX_NUM_TOKENS_PER_GROUP=50);
+	CL_NS(analysis)::Token tokens[MAX_NUM_TOKENS_PER_GROUP];
+	float_t scores[MAX_NUM_TOKENS_PER_GROUP];
+	int32_t numTokens;
+	int32_t startOffset;
+	int32_t endOffset;
+
+public:
+	TokenGroup(void);
+	~TokenGroup(void);
+
+	void addToken(CL_NS(analysis)::Token* token, float_t score);
+
+	/**
+	 * 
+	 * @param index a value between 0 and numTokens -1
+	 * @return the "n"th token
+	 */
+	CL_NS(analysis)::Token& getToken(int32_t index);
+
+	/**
+	 * 
+	 * @param index a value between 0 and numTokens -1
+	 * @return the "n"th score
+	 */
+	float_t getScore(int32_t index) const;
+
+	/**
+	 * @return the end position in the original text
+	 */
+	int32_t getEndOffset() const;
+
+	/**
+	 * @return the number of tokens in this group
+	 */
+	int32_t getNumTokens() const;
+
+	/**
+	 * @return the start position in the original text
+	 */
+	int32_t getStartOffset() const;
+
+	/**
+	 * @return all tokens' scores summed up
+	 */
+	float_t getTotalScore() const;
+
+	bool isDistinct(CL_NS(analysis)::Token* token)  const;
+	void clear();
+};
+
+CL_NS_END2
+#endif
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/oss/cl/clucene/src/clucene/highlighter/TokenSources.cpp
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/searchengine/oss/cl/clucene/src/clucene/highlighter/TokenSources.cpp	Fri Jun 11 14:43:47 2010 +0300
@@ -0,0 +1,219 @@
+#include "CLucene/StdHeader.h"
+#include "TokenSources.h"
+
+#include "CLucene/util/VoidList.h"
+
+CL_NS_DEF2(search,highlight)
+CL_NS_USE(analysis)
+CL_NS_USE(index)
+CL_NS_USE(util)
+
+TokenSources::TokenSources(void)
+{
+}
+
+TokenSources::~TokenSources(void)
+{
+}
+
+TokenStream* TokenSources::getAnyTokenStream(IndexReader* reader,int32_t docId, TCHAR* field, Analyzer* analyzer)
+{
+	TokenStream* ts=NULL;
+
+	TermFreqVector* tfv=reader->getTermFreqVector(docId,field);
+	if(tfv!=NULL)
+	{
+//		todo: this is actually very dodgy... we try casting
+//		to TermPositionVector, we take the token stream
+//		only if the cast works... should have a way of
+//		knowing what type this is
+		TermPositionVector* tmp = NULL;
+		try{
+             tmp = dynamic_cast<TermPositionVector *> (tfv);
+		}catch(...){
+			//ignore
+		}
+		if ( tmp != NULL )
+		    ts=getTokenStream(tmp);
+	}
+	//No token info stored so fall back to analyzing raw content
+	if(ts==NULL)
+	{
+		ts=getTokenStream(reader,docId,field,analyzer);
+	}
+	return ts;
+}
+
+
+TokenStream* TokenSources::getTokenStream(TermPositionVector* tpv)
+{
+    //assumes the worst and makes no assumptions about token position sequences.
+    return getTokenStream(tpv,false);   
+}
+
+TokenStream* TokenSources::getTokenStream(TermPositionVector* tpv, bool tokenPositionsGuaranteedContiguous)
+{
+    //an object used to iterate across an array of tokens
+    /*class StoredTokenStream extends TokenStream
+    {
+        Token tokens[];
+        int32_t currentToken=0;
+        StoredTokenStream(Token tokens[])
+        {
+            this.tokens=tokens;
+        }
+        public Token next()
+        {
+            if(currentToken>=tokens.length)
+            {
+                return NULL;
+            }
+            return tokens[currentToken++];
+        }            
+    }     */   
+    //code to reconstruct the original sequence of Tokens
+    const TCHAR** terms=tpv->getTerms();          
+    const int32_t* freq= (int32_t *)tpv->getTermFrequencies();
+	int32_t freqLen = tpv->size();
+
+    size_t totalTokens=0;
+	{
+		for (int32_t t = 0; t < freqLen; t++)
+			totalTokens+=freq[t];
+	}
+
+    Token** tokensInOriginalOrder=NULL;
+	CLSetList<Token*,Token::OrderCompare>* unsortedTokens = NULL;
+    for (int32_t t = 0; t < freqLen; t++)
+    {
+        TermVectorOffsetInfo** offsets=(TermVectorOffsetInfo**)tpv->getOffsets(t);
+        if(offsets==NULL)
+            return NULL;
+        
+        int32_t* pos=NULL;
+		int32_t posLen=0;
+        if(tokenPositionsGuaranteedContiguous)
+        {
+            //try get the token position info to speed up assembly of tokens into sorted sequence
+            pos=(int32_t *)tpv->getTermPositions(t);
+			posLen=1;//todo
+        }
+
+		if ( tokensInOriginalOrder != NULL )
+			tokensInOriginalOrder = _CL_NEWARRAY(Token*, totalTokens+1);
+
+        if(pos==NULL)
+        {	
+            //tokens NOT stored with positions or not guaranteed contiguous - must add to list and sort later
+            if(unsortedTokens==NULL)
+                unsortedTokens=_CLNEW CLSetList<Token*,Token::OrderCompare>(false);
+            for (int32_t tp=0; offsets[tp]!=NULL; tp++)
+            {
+                unsortedTokens->insert(_CLNEW Token(terms[t],
+                    offsets[tp]->getStartOffset(),
+                    offsets[tp]->getEndOffset()));
+            }
+        }
+        else
+        {
+            //We have positions stored and a guarantee that the token position information is contiguous
+            
+            // This may be fast BUT wont work if Tokenizers used which create >1 token in same position or
+            // creates jumps in position numbers - this code would fail under those circumstances
+            
+            //tokens stored with positions - can use this to index straight into sorted array
+            for (int32_t tp = 0; tp < posLen; tp++)
+            {
+                if ( tokensInOriginalOrder )
+                    {
+                    tokensInOriginalOrder[pos[tp]]=_CLNEW Token(terms[t],
+                        offsets[tp]->getStartOffset(),
+                        offsets[tp]->getEndOffset());
+                    }
+            }                
+        }
+    }
+    //If the field has been stored without position data we must perform a sort        
+    if(unsortedTokens!=NULL)
+    {
+		if ( totalTokens<unsortedTokens->size() ){
+			_CLDELETE_ARRAY(tokensInOriginalOrder);
+			tokensInOriginalOrder = _CL_NEWARRAY(Token*,unsortedTokens->size()+1);
+		}
+		//the list has already sorted our items //todo:check that this is true...
+		if ( tokensInOriginalOrder )
+		   unsortedTokens->toArray(tokensInOriginalOrder);
+		
+		return _CLNEW StoredTokenStream(tokensInOriginalOrder,unsortedTokens->size());
+    }else
+		return _CLNEW StoredTokenStream(tokensInOriginalOrder,totalTokens);
+}
+
+TokenStream* TokenSources::getTokenStream(IndexReader* reader,int32_t docId, TCHAR* field)
+{
+	TermFreqVector* tfv=reader->getTermFreqVector(docId,field);
+	if(tfv==NULL)
+	{
+		TCHAR buf[250];
+		_sntprintf(buf,250,_T("%s in doc #%d does not have any term position data stored"),field,docId);
+		_CLTHROWT(CL_ERR_IllegalArgument,buf);
+		return NULL;
+	}
+
+	//todo:bad way of doing this...
+	TermPositionVector* tmp = NULL;
+	try{
+		tmp = dynamic_cast<TermPositionVector *> (tfv); //check to see if tfv is a Tpv
+	}catch(...){}
+	TokenStream* stream = NULL;
+	if ( tmp != NULL ){
+		TermPositionVector* tpv = dynamic_cast<TermPositionVector *> (reader->getTermFreqVector(docId,field));
+		if ( tpv )
+		    stream = getTokenStream(tpv);  
+	    //return getTokenStream(tpv);	        
+	}else{
+		TCHAR buf[250];
+		_sntprintf(buf,250,_T("%s in doc #%d does not have any term position data stored"),field,docId);
+		_CLTHROWT(CL_ERR_IllegalArgument,buf);
+		//return NULL;
+	}
+	return stream;
+}
+
+//convenience method
+TokenStream* TokenSources::getTokenStream(IndexReader* reader,int32_t docId, TCHAR* field,Analyzer* analyzer)
+{
+	CL_NS(document)::Document* doc=reader->document(docId);
+	const TCHAR* contents=doc->get(field);
+	if(contents==NULL)
+	{
+		TCHAR buf[250];
+		_sntprintf(buf,250,_T("Field %s in document #%d is not stored and cannot be analyzed"),field,docId);
+		_CLTHROWT(CL_ERR_IllegalArgument,buf);
+		return NULL;
+	}
+    return analyzer->tokenStream(field,_CLNEW StringReader(contents));
+}
+
+TokenSources::StoredTokenStream::StoredTokenStream(CL_NS(analysis)::Token** tokens, size_t len)
+{
+	currentToken = 0;
+    this->tokens=tokens;
+	this->length = len;
+}
+bool TokenSources::StoredTokenStream::next(CL_NS(analysis)::Token* token)
+{
+    if(currentToken>=length)
+    {
+        return false;
+    }
+	Token* t = tokens[currentToken++];
+
+	token->set(t->termText(),t->startOffset(),t->endOffset(),t->type());;
+    return true;
+}
+void TokenSources::StoredTokenStream::close(){
+	
+}
+
+CL_NS_END2
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/oss/cl/clucene/src/clucene/highlighter/TokenSources.h
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/searchengine/oss/cl/clucene/src/clucene/highlighter/TokenSources.h	Fri Jun 11 14:43:47 2010 +0300
@@ -0,0 +1,98 @@
+/**
+ * Copyright 2002-2004 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _lucene_search_highlight_tokensources_
+#define _lucene_search_highlight_tokensources_
+
+#if defined(_LUCENE_PRAGMA_ONCE)
+# pragma once
+#endif
+
+#include "CLucene/analysis/AnalysisHeader.h"
+#include "CLucene/index/IndexReader.h"
+#include "CLucene/index/TermVector.h"
+#include "CLucene/analysis/AnalysisHeader.h"
+
+CL_NS_DEF2(search,highlight)
+
+class TokenSources: LUCENE_BASE
+{
+	//an object used to iterate across an array of tokens
+	class StoredTokenStream:public CL_NS(analysis)::TokenStream
+    {
+	public:
+        CL_NS(analysis)::Token** tokens;
+		size_t length;
+        int32_t currentToken;
+        StoredTokenStream(CL_NS(analysis)::Token** tokens, size_t len);
+		bool next(CL_NS(analysis)::Token* token);
+		void close();
+    };
+public:
+	TokenSources(void);
+	~TokenSources(void);
+
+	/**
+     * A convenience method that tries a number of approaches to getting a token stream.
+     * The cost of finding there are no termVectors in the index is minimal (1000 invocations still 
+     * registers 0 ms). So this "lazy" (flexible?) approach to coding is probably acceptable
+     * @param reader
+     * @param docId
+     * @param field
+     * @param analyzer
+     * @return null if field not stored correctly 
+     * @throws IOException
+     */
+	static CL_NS(analysis)::TokenStream* getAnyTokenStream(CL_NS(index)::IndexReader* reader,int32_t docId, TCHAR* field, CL_NS(analysis)::Analyzer* analyzer);
+    
+    static CL_NS(analysis)::TokenStream* getTokenStream(CL_NS(index)::TermPositionVector* tpv);
+
+    /**
+     * Low level api.
+     * Returns a token stream or null if no offset info available in index.
+     * This can be used to feed the highlighter with a pre-parsed token stream 
+     * 
+     * In my tests the speeds to recreate 1000 token streams using this method are:
+     * - with TermVector offset only data stored - 420  milliseconds 
+     * - with TermVector offset AND position data stored - 271 milliseconds
+     *  (nb timings for TermVector with position data are based on a tokenizer with contiguous
+     *  positions - no overlaps or gaps)
+     * The cost of not using TermPositionVector to store
+     * pre-parsed content and using an analyzer to re-parse the original content: 
+     * - reanalyzing the original content - 980 milliseconds
+     * 
+     * The re-analyze timings will typically vary depending on -
+     * 	1) The complexity of the analyzer code (timings above were using a 
+     * 	   stemmer/lowercaser/stopword combo)
+     *  2) The  number of other fields (Lucene reads ALL fields off the disk 
+     *     when accessing just one document field - can cost dear!)
+     *  3) Use of compression on field storage - could be faster cos of compression (less disk IO)
+     *     or slower (more CPU burn) depending on the content.
+     *
+     * @param tpv
+     * @param tokenPositionsGuaranteedContiguous true if the token position numbers have no overlaps or gaps. If looking
+     * to eek out the last drops of performance, set to true. If in doubt, set to false.
+     */
+    static CL_NS(analysis)::TokenStream* getTokenStream(CL_NS(index)::TermPositionVector* tpv, bool tokenPositionsGuaranteedContiguous);
+
+	static CL_NS(analysis)::TokenStream* getTokenStream(CL_NS(index)::IndexReader* reader,int32_t docId, TCHAR* field);
+
+    //convenience method
+	static CL_NS(analysis)::TokenStream* getTokenStream(CL_NS(index)::IndexReader* reader,int32_t docId, TCHAR* field,CL_NS(analysis)::Analyzer* analyzer);
+};
+
+CL_NS_END2
+#endif
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/oss/cl/clucene/src/clucene/highlighter/WeightedTerm.cpp
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/searchengine/oss/cl/clucene/src/clucene/highlighter/WeightedTerm.cpp	Fri Jun 11 14:43:47 2010 +0300
@@ -0,0 +1,84 @@
+#include "CLucene/StdHeader.h"
+#include "WeightedTerm.h"
+#include "CLucene/search/Similarity.h"
+
+CL_NS_DEF2(search,highlight)
+
+	WeightedTerm::WeightedTerm (float_t weight,const TCHAR* term)
+	{
+		_weight=weight;
+		_term = stringDuplicate(term);
+		cachedHashCode = 0;
+	}
+	
+	WeightedTerm::~WeightedTerm()
+	{
+		_CLDELETE_CARRAY(_term);
+	}
+
+	WeightedTerm::WeightedTerm(const WeightedTerm& other) 
+	{
+		_weight = other.getWeight();
+		_term = STRDUP_TtoT(other.getTerm());
+	}
+	
+	WeightedTerm* WeightedTerm::clone() const{
+		return _CLNEW WeightedTerm(*this);
+	}
+
+	/**
+	 * @return the term value (stemmed)
+	 */
+	const TCHAR* WeightedTerm::getTerm() const
+	{
+		return _term;
+	}
+
+	/**
+	 * @return the weight associated with this term
+	 */
+	float_t WeightedTerm::getWeight() const 
+	{
+		return _weight;
+	}
+
+	/**
+	 * @param term the term value (stemmed)
+	 */
+	void WeightedTerm::setTerm(TCHAR* term)
+	{
+		_CLDELETE_CARRAY(this->_term);
+		this->_term = STRDUP_TtoT(_term);
+		cachedHashCode = 0;
+	}
+
+	/**
+	 * @param weight the weight associated with this term
+	 */
+	void WeightedTerm::setWeight(float_t aweight) {
+		this->_weight = aweight;
+		cachedHashCode = 0;
+	}
+
+	size_t WeightedTerm::hashCode(){
+		if ( cachedHashCode == 0 ){
+			cachedHashCode = ( CL_NS(util)::Misc::thashCode(this->_term) ^ CL_NS(search)::Similarity::floatToByte(_weight) );
+		}
+
+		return cachedHashCode;
+	}
+
+	bool WeightedTerm::Compare::operator()( WeightedTerm* t1, WeightedTerm* t2 ) const{
+		int r = _tcscmp(t1->getTerm(), t2->getTerm());
+		if ( r < 0 )
+			return true;
+		else if ( r == 0 )
+			return t1->getWeight() < t2->getWeight();
+		else
+			return false;
+	}
+	size_t WeightedTerm::Compare::operator()( WeightedTerm* t ) const{
+		return t->hashCode();
+	}
+
+CL_NS_END2
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/oss/cl/clucene/src/clucene/highlighter/WeightedTerm.h
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/searchengine/oss/cl/clucene/src/clucene/highlighter/WeightedTerm.h	Fri Jun 11 14:43:47 2010 +0300
@@ -0,0 +1,82 @@
+/**
+ * Copyright 2002-2004 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _lucene_search_highlight_weightedterm_
+#define _lucene_search_highlight_weightedterm_
+
+#if defined(_LUCENE_PRAGMA_ONCE)
+# pragma once
+#endif
+
+#include "CLucene/util/VoidList.h"
+
+CL_NS_DEF2(search,highlight)
+
+/** Lightweight class to hold term and a weight value used for scoring this term 
+ */
+class WeightedTerm:LUCENE_BASE
+{
+private:
+	float_t _weight; // multiplier
+	TCHAR* _term; //stemmed form
+	size_t cachedHashCode;
+	WeightedTerm(const WeightedTerm& other);
+public:
+	WeightedTerm (float_t weight,const TCHAR* term);
+	~WeightedTerm();
+
+	/**
+	 * @return the term value (stemmed)
+	 */
+	const TCHAR* getTerm() const;
+
+	/**
+	 * @return the weight associated with this term
+	 */
+	float_t getWeight() const ;
+
+	/**
+	 * @param term the term value (stemmed)
+	 */
+	void setTerm(TCHAR* term);
+	/**
+	 * @param weight the weight associated with this term
+	 */
+	void setWeight(float_t aweight);
+
+	size_t hashCode();
+	WeightedTerm* clone() const;
+
+	/**
+	 * Compare weighted terms, according to the term text.
+	 * @todo Do we have to take boost factors into account
+	 */
+	class Compare:LUCENE_BASE, public CL_NS(util)::Compare::_base //<WeightedTerm*>
+	{
+	public:
+	//todo: this should be more efficient, but will be using a hash table soon, anyway
+		bool operator()( WeightedTerm* t1, WeightedTerm* t2 ) const;
+		size_t operator()( WeightedTerm* t ) const;
+	};
+};
+
+/** CLHashSet of WeightedTerm */
+typedef CL_NS(util)::CLHashSet<WeightedTerm*, WeightedTerm::Compare, CL_NS(util)::Deletor::Object<WeightedTerm> > WeightedTermList;
+
+CL_NS_END2
+
+#endif
+
diff -r ae3f1779f6da -r a5fbfefd615f searchengine/oss/cl/clucene/src/clucene/util/equators.h
--- a/searchengine/oss/cl/clucene/src/clucene/util/equators.h	Thu May 27 13:59:44 2010 +0300
+++ b/searchengine/oss/cl/clucene/src/clucene/util/equators.h	Fri Jun 11 14:43:47 2010 +0300
@@ -188,7 +188,7 @@
 	class Object{
 	public:
 		static void doDelete(_kt* obj){
-			_CLLDELETE(obj);
+			_CLVDELETE(obj);
 		}
 	};
 	template<typename _kt>
diff -r ae3f1779f6da -r a5fbfefd615f searcher/searchclient/traces/OstTraceDefinitions.h
--- a/searcher/searchclient/traces/OstTraceDefinitions.h	Thu May 27 13:59:44 2010 +0300
+++ b/searcher/searchclient/traces/OstTraceDefinitions.h	Fri Jun 11 14:43:47 2010 +0300
@@ -1,19 +1,3 @@
-/*
-* Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).
-* All rights reserved.
-* This component and the accompanying materials are made available
-* under the terms of "Eclipse Public License v1.0"
-* which accompanies this distribution, and is available
-* at the URL "http://www.eclipse.org/legal/epl-v10.html".
-*
-* Initial Contributors:
-* Nokia Corporation - initial contribution.
-*
-* Contributors:
-*
-* Description: 
-*
-*/
 #ifndef __OSTTRACEDEFINITIONS_H__
 #define __OSTTRACEDEFINITIONS_H__
 // OST_TRACE_COMPILER_IN_USE flag has been added by Trace Compiler
diff -r ae3f1779f6da -r a5fbfefd615f searcher/searchserver/traces/OstTraceDefinitions.h
--- a/searcher/searchserver/traces/OstTraceDefinitions.h	Thu May 27 13:59:44 2010 +0300
+++ b/searcher/searchserver/traces/OstTraceDefinitions.h	Fri Jun 11 14:43:47 2010 +0300
@@ -1,19 +1,3 @@
-/*
-* Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).
-* All rights reserved.
-* This component and the accompanying materials are made available
-* under the terms of "Eclipse Public License v1.0"
-* which accompanies this distribution, and is available
-* at the URL "http://www.eclipse.org/legal/epl-v10.html".
-*
-* Initial Contributors:
-* Nokia Corporation - initial contribution.
-*
-* Contributors:
-*
-* Description: 
-*
-*/
 #ifndef __OSTTRACEDEFINITIONS_H__
 #define __OSTTRACEDEFINITIONS_H__
 // OST_TRACE_COMPILER_IN_USE flag has been added by Trace Compiler
diff -r ae3f1779f6da -r a5fbfefd615f searchsrv_plat/cpix_framework_api/inc/cdocumentfield.h
--- a/searchsrv_plat/cpix_framework_api/inc/cdocumentfield.h	Thu May 27 13:59:44 2010 +0300
+++ b/searchsrv_plat/cpix_framework_api/inc/cdocumentfield.h	Fri Jun 11 14:43:47 2010 +0300
@@ -60,7 +60,8 @@
 		{
 		EIndexNo = 16,
 		EIndexTokenized = 32,
-		EIndexUnTokenized = 64
+		EIndexUnTokenized = 64,
+		EIndexFreeText = 128
 		};
 
 	/**
diff -r ae3f1779f6da -r a5fbfefd615f sis/centrep.pkg
--- a/sis/centrep.pkg	Thu May 27 13:59:44 2010 +0300
+++ b/sis/centrep.pkg	Fri Jun 11 14:43:47 2010 +0300
@@ -1,18 +1,3 @@
-;
-; Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).
-; All rights reserved.
-; This component and the accompanying materials are made available
-; under the terms of "Eclipse Public License v1.0"
-; which accompanies this distribution, and is available
-; at the URL "http://www.eclipse.org/legal/epl-v10.html".
-;
-; Initial Contributors:
-; Nokia Corporation - initial contribution.
-;
-; Contributors:
-;
-; Description: 
-;
 ; Languages
 &EN
 
@@ -28,4 +13,7 @@
 ; CenRep ini file
 ;"12345678.txt" - "c:\private\10202be9\12345678.txt"
 "\epoc32\winscw\c\private\10202be9\20029ac7.cre"
--"!:\private\10202be9\20029ac7.cre"
\ No newline at end of file
+-"!:\private\10202be9\20029ac7.cre"
+
+"\epoc32\winscw\c\private\10202be9\2001f6fb.cre"
+-"!:\private\10202be9\2001f6fb.cre"
\ No newline at end of file
diff -r ae3f1779f6da -r a5fbfefd615f sis/cpixsearch_stub.pkg
--- a/sis/cpixsearch_stub.pkg	Thu May 27 13:59:44 2010 +0300
+++ b/sis/cpixsearch_stub.pkg	Fri Jun 11 14:43:47 2010 +0300
@@ -1,18 +1,3 @@
-;
-; Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).
-; All rights reserved.
-; This component and the accompanying materials are made available
-; under the terms of "Eclipse Public License v1.0"
-; which accompanies this distribution, and is available
-; at the URL "http://www.eclipse.org/legal/epl-v10.html".
-;
-; Initial Contributors:
-; Nokia Corporation - initial contribution.
-;
-; Contributors:
-;
-; Description: 
-;
 
 
 
diff -r ae3f1779f6da -r a5fbfefd615f sis/makeme.bat
--- a/sis/makeme.bat	Thu May 27 13:59:44 2010 +0300
+++ b/sis/makeme.bat	Fri Jun 11 14:43:47 2010 +0300
@@ -1,18 +1,3 @@
-@rem
-@rem Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).
-@rem All rights reserved.
-@rem This component and the accompanying materials are made available
-@rem under the terms of "Eclipse Public License v1.0"
-@rem which accompanies this distribution, and is available
-@rem at the URL "http://www.eclipse.org/legal/epl-v10.html".
-@rem
-@rem Initial Contributors:
-@rem Nokia Corporation - initial contribution.
-@rem
-@rem Contributors:
-@rem
-@rem Description: 
-@rem
 REM echo %EPOCROOT%
 set FOOBAR=%EPOCROOT%
 REM echo %FOOBAR%
diff -r ae3f1779f6da -r a5fbfefd615f sis/makeme.pl
--- a/sis/makeme.pl	Thu May 27 13:59:44 2010 +0300
+++ b/sis/makeme.pl	Fri Jun 11 14:43:47 2010 +0300
@@ -1,19 +1,3 @@
-#
-# Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).
-# All rights reserved.
-# This component and the accompanying materials are made available
-# under the terms of "Eclipse Public License v1.0"
-# which accompanies this distribution, and is available
-# at the URL "http://www.eclipse.org/legal/epl-v10.html".
-#
-# Initial Contributors:
-# Nokia Corporation - initial contribution.
-#
-# Contributors:
-#
-# Description: 
-#
-
 if (!$ARGV[0]) {
 	printf("ERROR: Target (udeb/urel) needs to be specified.\n"); 
 	printf("Use either 'makeme udeb' or 'makeme urel'\n"); 
diff -r ae3f1779f6da -r a5fbfefd615f sis/makesis.mk
--- a/sis/makesis.mk	Thu May 27 13:59:44 2010 +0300
+++ b/sis/makesis.mk	Fri Jun 11 14:43:47 2010 +0300
@@ -1,18 +1,3 @@
-#
-# Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).
-# All rights reserved.
-# This component and the accompanying materials are made available
-# under the terms of "Eclipse Public License v1.0"
-# which accompanies this distribution, and is available
-# at the URL "http://www.eclipse.org/legal/epl-v10.html".
-#
-# Initial Contributors:
-# Nokia Corporation - initial contribution.
-#
-# Contributors:
-#
-# Description:  
-#
 # ============================================================================
 #  Name	 : makeSis.mk
 #  Part of  : 
diff -r ae3f1779f6da -r a5fbfefd615f tsrc/cpixmwtester/conf/cpixmwtester.cfg
--- a/tsrc/cpixmwtester/conf/cpixmwtester.cfg	Thu May 27 13:59:44 2010 +0300
+++ b/tsrc/cpixmwtester/conf/cpixmwtester.cfg	Fri Jun 11 14:43:47 2010 +0300
@@ -63,4 +63,18 @@
 create CPixMWTester foobar 
 foobar TestUpdateINStatus contacts
 delete foobar
+[Endtest]
+
+[Test]
+title TestAddUnloadlist
+create CPixMWTester foobar 
+foobar TestAddUnloadlist
+delete foobar
+[Endtest]
+
+[Test]
+title TestRemovefromUnloadlist
+create CPixMWTester foobar 
+foobar TestRemovefromUnloadlist
+delete foobar
 [Endtest]
\ No newline at end of file
diff -r ae3f1779f6da -r a5fbfefd615f tsrc/cpixmwtester/inc/cpixmwtester.h
--- a/tsrc/cpixmwtester/inc/cpixmwtester.h	Thu May 27 13:59:44 2010 +0300
+++ b/tsrc/cpixmwtester/inc/cpixmwtester.h	Fri Jun 11 14:43:47 2010 +0300
@@ -176,6 +176,10 @@
         
         virtual TInt TestUpdateINStatusL( CStifItemParser& aItem );
         
+        virtual TInt TestAddUnloadlistL( CStifItemParser& aItem );
+        
+        virtual TInt TestRemovefromUnloadlistL( CStifItemParser& aItem );
+        
         /**
          * Method used to log version of test class
          */
diff -r ae3f1779f6da -r a5fbfefd615f tsrc/cpixmwtester/src/cpixmwtesterblocks.cpp
--- a/tsrc/cpixmwtester/src/cpixmwtesterblocks.cpp	Thu May 27 13:59:44 2010 +0300
+++ b/tsrc/cpixmwtester/src/cpixmwtesterblocks.cpp	Fri Jun 11 14:43:47 2010 +0300
@@ -167,6 +167,8 @@
         ENTRY( "TestResetContent",CCPixMWTester::TestResetContentL ),
         ENTRY( "TestUpdateBLStatus",CCPixMWTester::TestUpdateBLStatusL ),
         ENTRY( "TestUpdateINStatus",CCPixMWTester::TestUpdateINStatusL ),
+        ENTRY( "TestAddUnloadlist",CCPixMWTester::TestAddUnloadlistL ),
+        ENTRY( "TestRemovefromUnloadlist",CCPixMWTester::TestRemovefromUnloadlistL ),
         //ADD NEW ENTRY HERE
         // [test cases entries] - Do not remove
 
@@ -435,6 +437,54 @@
     doLog( iLog, err, KNoErrorString );
     return err;
     }
+
+// -----------------------------------------------------------------------------
+// CCPixMWTester::TestAddUnloadlistL
+// -----------------------------------------------------------------------------
+//
+TInt CCPixMWTester::TestAddUnloadlistL( CStifItemParser& aItem)
+    {
+    TInt err = KErrNone;        
+    CBlacklistMgr* blacklistmanager = CBlacklistMgr::NewL();
+    CleanupStack::PushL( blacklistmanager );
+    //Add an Uid to Blacklist DB
+    blacklistmanager->AddtoUnloadListL( KTestUid );
+    //Check if the Uid is added to database or not
+    TBool found = blacklistmanager->FindfromUnloadListL(KTestUid );
+    
+    if(!found) err = KErrNotFound;
+    //clear the UID from the database
+    blacklistmanager->RemoveFromUnloadListL(KTestUid);
+    CleanupStack::PopAndDestroy( blacklistmanager ); 
+    doLog( iLog, err, KNoErrorString );        
+    return err;
+    }
+
+// -----------------------------------------------------------------------------
+// CCPixMWTester::TestRemovefromUnloadlistL
+// -----------------------------------------------------------------------------
+//
+TInt CCPixMWTester::TestRemovefromUnloadlistL( CStifItemParser& aItem)
+    {
+    TInt err = KErrNotFound;        
+    CBlacklistMgr* blacklistmanager = CBlacklistMgr::NewL();
+    CleanupStack::PushL( blacklistmanager );
+    //Add an Uid to Blacklist DB
+    blacklistmanager->AddtoUnloadListL( KTestUid );
+    //Check if the Uid is added to database or not
+    TBool found = blacklistmanager->FindfromUnloadListL(KTestUid );
+    
+    if(found)
+        {
+         //clear the UID from the database
+        blacklistmanager->RemoveFromUnloadListL(KTestUid);
+        found = blacklistmanager->FindfromUnloadListL(KTestUid );
+        if ( !found )  err = KErrNone;
+        }
+    CleanupStack::PopAndDestroy( blacklistmanager ); 
+    doLog( iLog, err, KNoErrorString );        
+    return err;
+    }
 // -----------------------------------------------------------------------------
 // CCPixMWTester::?member_function
 // ?implementation_description
diff -r ae3f1779f6da -r a5fbfefd615f watchdog/traces/OstTraceDefinitions.h
--- a/watchdog/traces/OstTraceDefinitions.h	Thu May 27 13:59:44 2010 +0300
+++ b/watchdog/traces/OstTraceDefinitions.h	Fri Jun 11 14:43:47 2010 +0300
@@ -1,19 +1,3 @@
-/*
-* Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).
-* All rights reserved.
-* This component and the accompanying materials are made available
-* under the terms of "Eclipse Public License v1.0"
-* which accompanies this distribution, and is available
-* at the URL "http://www.eclipse.org/legal/epl-v10.html".
-*
-* Initial Contributors:
-* Nokia Corporation - initial contribution.
-*
-* Contributors:
-*
-* Description: 
-*
-*/
 #ifndef __OSTTRACEDEFINITIONS_H__
 #define __OSTTRACEDEFINITIONS_H__
 // OST_TRACE_COMPILER_IN_USE flag has been added by Trace Compiler