--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/searchengine/oss/cl/clucene/src/clucene/highlighter/TokenGroup.cpp	Fri Jun 11 14:43:47 2010 +0300
@@ -0,0 +1,104 @@
+#include "CLucene/StdHeader.h"
+#include "TokenGroup.h"
+
+CL_NS_DEF2(search,highlight)
+CL_NS_USE(analysis)
+
+TokenGroup::TokenGroup(void)
+{
+	numTokens=0;
+	startOffset=0;
+	endOffset=0;
+}
+
+TokenGroup::~TokenGroup(void)
+{
+}
+
+void TokenGroup::addToken(Token* token, float_t score)
+{
+	if(numTokens < MAX_NUM_TOKENS_PER_GROUP)
+    {	    
+		if(numTokens==0)
+		{
+			startOffset=token->startOffset();		
+			endOffset=token->endOffset();		
+		}
+		else
+		{
+			startOffset=min(startOffset,token->startOffset());		
+			endOffset=max(endOffset,token->endOffset());		
+		}
+		tokens[numTokens].set(token->termText(),token->startOffset(),token->endOffset(),token->type());;
+		scores[numTokens]=score;
+		numTokens++;
+    }
+}
+
+CL_NS(analysis)::Token& TokenGroup::getToken(int32_t index)
+{
+	return tokens[index];
+}
+
+float_t TokenGroup::getScore(int32_t index) const
+{
+	return scores[index];
+}
+
+int32_t TokenGroup::getEndOffset() const
+{
+	return endOffset;
+}
+
+int32_t TokenGroup::getNumTokens() const
+{
+	return numTokens;
+}
+
+int32_t TokenGroup::getStartOffset() const
+{
+	return startOffset;
+}
+
+float_t TokenGroup::getTotalScore() const
+{
+	float_t total=0;
+	for (int32_t i = 0; i < numTokens; i++)
+	{
+		total+=scores[i];
+	}
+	return total;
+}
+
+/*void addToken(CL_NS(analysis)::Token* token, float_t score)
+{
+	if(numTokens < MAX_NUM_TOKENS_PER_GROUP)
+  		{
+		if(numTokens==0)
+		{
+			startOffset=token->startOffset();		
+			endOffset=token->endOffset();		
+		}
+		else
+		{
+			startOffset=min(startOffset,token->startOffset());		
+			endOffset=max(endOffset,token->endOffset());		
+		}
+		tokens[numTokens]=token;
+		scores[numTokens]=score;
+		numTokens++;
+	}
+}*/
+
+bool TokenGroup::isDistinct(CL_NS(analysis)::Token* token) const
+{
+	return token->startOffset() > endOffset;
+}
+
+
+void TokenGroup::clear()
+{
+	numTokens=0;
+}
+
+CL_NS_END2