searchengine/oss/cl/clucene/src/clucene/highlighter/QueryScorer.cpp
author hgs
Fri, 15 Oct 2010 12:09:28 +0530
changeset 24 65456528cac2
parent 7 a5fbfefd615f
permissions -rw-r--r--
201041
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
7
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
     1
#include "clucene/stdheader.h"
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
     2
#include "queryscorer.h"
24
hgs
parents: 7
diff changeset
     3
#include <wchar.h>
7
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
     4
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
     5
CL_NS_DEF2(search,highlight)
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
     6
CL_NS_USE(index)
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
     7
CL_NS_USE(analysis)
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
     8
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
     9
	QueryScorer::QueryScorer(const Query * query):
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    10
        _uniqueTermsInFragment(true),
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    11
		_termsToFind(false,true)
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    12
		
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    13
	 {
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    14
		 WeightedTerm** _weighted_terms = QueryTermExtractor::getTerms(query);
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    15
		 initialize(_weighted_terms);
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    16
		 _CLDELETE_ARRAY(_weighted_terms);
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    17
	 }
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    18
	 QueryScorer::~QueryScorer()
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    19
	 {
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    20
	 }
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    21
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    22
/*	 QueryScorer(Query* query, CL_NS(index)::IndexReader* reader, const TCHAR* fieldName)
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    23
	 {
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    24
		 WeightedTerm** _weighted_terms = QueryTermExtractor.getIdfWeightedTerms(query, reader, fieldName);
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    25
		 initialize(_weighted_terms);
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    26
	 }*/
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    27
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    28
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    29
	QueryScorer::QueryScorer(WeightedTerm** weightedTerms)
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    30
	{
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    31
		 initialize(weightedTerms);
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    32
	}
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    33
	
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    34
	void QueryScorer::initialize(WeightedTerm** weightedTerms)
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    35
	{
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    36
		_currentTextFragment = NULL;
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    37
		_totalScore = 0;
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    38
		_maxTermWeight = 0;
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    39
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    40
		// Copy external weighted terms
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    41
		 int i=0;
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    42
		 while ( weightedTerms[i] != NULL ){
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    43
			const WeightedTerm* existingTerm=_termsToFind.get(weightedTerms[i]->getTerm());
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    44
			if( (existingTerm==NULL) ||(existingTerm->getWeight()<weightedTerms[i]->getWeight()) )
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    45
  	        {
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    46
  				//if a term is defined more than once, always use the highest scoring weight
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    47
				WeightedTerm* term = weightedTerms[i];
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    48
				_termsToFind.put(term->getTerm(), term);
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    49
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    50
				_maxTermWeight=max(_maxTermWeight,weightedTerms[i]->getWeight());
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    51
  	        }else
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    52
				_CLDELETE(weightedTerms[i]);
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    53
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    54
			i++;
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    55
		 }
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    56
	}
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    57
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    58
	void QueryScorer::startFragment(TextFragment * newFragment)
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    59
	{
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    60
		_uniqueTermsInFragment.clear();
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    61
		_currentTextFragment=newFragment;
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    62
		_totalScore=0;
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    63
		
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
    64
	}
24
hgs
parents: 7
diff changeset
    65
	/*
hgs
parents: 7
diff changeset
    66
	 * Compares the Query term to Field text token if match, returns 1
hgs
parents: 7
diff changeset
    67
	 *
hgs
parents: 7
diff changeset
    68
	 */
hgs
parents: 7
diff changeset
    69
hgs
parents: 7
diff changeset
    70
	int QueryScorer::matchQuryText(const TCHAR* termText)
hgs
parents: 7
diff changeset
    71
	{
hgs
parents: 7
diff changeset
    72
			CL_NS(util)::LHashMap<const TCHAR*, const WeightedTerm *,
hgs
parents: 7
diff changeset
    73
		CL_NS(util)::Compare::TChar,
hgs
parents: 7
diff changeset
    74
		CL_NS(util)::Equals::TChar,
hgs
parents: 7
diff changeset
    75
		CL_NS(util)::Deletor::tcArray,
hgs
parents: 7
diff changeset
    76
		CL_NS(util)::Deletor::Object<const WeightedTerm> >::iterator i = _termsToFind.begin();
hgs
parents: 7
diff changeset
    77
			for(; i != _termsToFind.end() ; i ++)
hgs
parents: 7
diff changeset
    78
			{ 
hgs
parents: 7
diff changeset
    79
				const TCHAR * temp = i->first;
hgs
parents: 7
diff changeset
    80
				int tempLen =  wcslen(temp);
hgs
parents: 7
diff changeset
    81
				int matchFlag = 1;
hgs
parents: 7
diff changeset
    82
				if(tempLen <= wcslen(termText))
hgs
parents: 7
diff changeset
    83
				{
hgs
parents: 7
diff changeset
    84
					for(int j = tempLen - 1 ; j >=0 ; j--)
hgs
parents: 7
diff changeset
    85
					{
hgs
parents: 7
diff changeset
    86
						if(temp[j] != termText[j])
hgs
parents: 7
diff changeset
    87
						{
hgs
parents: 7
diff changeset
    88
							matchFlag = 0;
hgs
parents: 7
diff changeset
    89
							break;
hgs
parents: 7
diff changeset
    90
						}
hgs
parents: 7
diff changeset
    91
					}
hgs
parents: 7
diff changeset
    92
				}
hgs
parents: 7
diff changeset
    93
				else
hgs
parents: 7
diff changeset
    94
				{
hgs
parents: 7
diff changeset
    95
					matchFlag = 0;
hgs
parents: 7
diff changeset
    96
				}
hgs
parents: 7
diff changeset
    97
				if(matchFlag)
hgs
parents: 7
diff changeset
    98
					return 1;
hgs
parents: 7
diff changeset
    99
				
hgs
parents: 7
diff changeset
   100
     		}
hgs
parents: 7
diff changeset
   101
			return 0;
hgs
parents: 7
diff changeset
   102
	}
hgs
parents: 7
diff changeset
   103
7
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
   104
	
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
   105
	float_t QueryScorer::getTokenScore(Token * token)
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
   106
	{
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
   107
		const TCHAR* termText=token->termText();
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
   108
		
24
hgs
parents: 7
diff changeset
   109
	//	const WeightedTerm* queryTerm = _termsToFind.get(termText);
hgs
parents: 7
diff changeset
   110
	// Instead of checking for weighted terms directly match qurey text to field text.
hgs
parents: 7
diff changeset
   111
	    int isQueryTerm = matchQuryText(termText);
hgs
parents: 7
diff changeset
   112
		if(isQueryTerm==0)
7
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
   113
		{
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
   114
			//not a query term - return
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
   115
			return 0;
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
   116
		}
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
   117
		//found a query term - is it unique in this doc?
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
   118
		if(_uniqueTermsInFragment.find(termText)==_uniqueTermsInFragment.end())
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
   119
		{
24
hgs
parents: 7
diff changeset
   120
			//_totalScore+=queryTerm->getWeight();
hgs
parents: 7
diff changeset
   121
			/*
hgs
parents: 7
diff changeset
   122
			 * Keeping the Score value to 1 
hgs
parents: 7
diff changeset
   123
			 *
hgs
parents: 7
diff changeset
   124
			 */
hgs
parents: 7
diff changeset
   125
				_totalScore+=1;
7
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
   126
			TCHAR* owned_term = stringDuplicate(termText);
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
   127
			_uniqueTermsInFragment.insert(owned_term);
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
   128
		}
24
hgs
parents: 7
diff changeset
   129
		//return queryTerm->getWeight();
hgs
parents: 7
diff changeset
   130
			return 1;
7
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
   131
	}
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
   132
	
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
   133
	/**
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
   134
  	*
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
   135
  	* @return The highest weighted term (useful for passing to GradientFormatter to set
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
   136
  	* top end of coloring scale.
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
   137
  	*/
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
   138
	float_t QueryScorer::getMaxTermWeight()
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
   139
	{
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
   140
  		return _maxTermWeight;
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
   141
	}
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
   142
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
   143
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
   144
	float_t QueryScorer::getFragmentScore(){
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
   145
		return _totalScore;
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
   146
	}
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
   147
a5fbfefd615f Revision: 201021
Dremov Kirill (Nokia-D-MSW/Tampere) <kirill.dremov@nokia.com>
parents:
diff changeset
   148
CL_NS_END2