|
1 #include "CLucene/StdHeader.h" |
|
2 #include "TokenGroup.h" |
|
3 |
|
4 CL_NS_DEF2(search,highlight) |
|
5 CL_NS_USE(analysis) |
|
6 |
|
7 TokenGroup::TokenGroup(void) |
|
8 { |
|
9 numTokens=0; |
|
10 startOffset=0; |
|
11 endOffset=0; |
|
12 } |
|
13 |
|
14 TokenGroup::~TokenGroup(void) |
|
15 { |
|
16 } |
|
17 |
|
18 void TokenGroup::addToken(Token* token, float_t score) |
|
19 { |
|
20 if(numTokens < MAX_NUM_TOKENS_PER_GROUP) |
|
21 { |
|
22 if(numTokens==0) |
|
23 { |
|
24 startOffset=token->startOffset(); |
|
25 endOffset=token->endOffset(); |
|
26 } |
|
27 else |
|
28 { |
|
29 startOffset=min(startOffset,token->startOffset()); |
|
30 endOffset=max(endOffset,token->endOffset()); |
|
31 } |
|
32 tokens[numTokens].set(token->termText(),token->startOffset(),token->endOffset(),token->type());; |
|
33 scores[numTokens]=score; |
|
34 numTokens++; |
|
35 } |
|
36 } |
|
37 |
|
38 CL_NS(analysis)::Token& TokenGroup::getToken(int32_t index) |
|
39 { |
|
40 return tokens[index]; |
|
41 } |
|
42 |
|
43 float_t TokenGroup::getScore(int32_t index) const |
|
44 { |
|
45 return scores[index]; |
|
46 } |
|
47 |
|
48 int32_t TokenGroup::getEndOffset() const |
|
49 { |
|
50 return endOffset; |
|
51 } |
|
52 |
|
53 int32_t TokenGroup::getNumTokens() const |
|
54 { |
|
55 return numTokens; |
|
56 } |
|
57 |
|
58 int32_t TokenGroup::getStartOffset() const |
|
59 { |
|
60 return startOffset; |
|
61 } |
|
62 |
|
63 float_t TokenGroup::getTotalScore() const |
|
64 { |
|
65 float_t total=0; |
|
66 for (int32_t i = 0; i < numTokens; i++) |
|
67 { |
|
68 total+=scores[i]; |
|
69 } |
|
70 return total; |
|
71 } |
|
72 |
|
73 /*void addToken(CL_NS(analysis)::Token* token, float_t score) |
|
74 { |
|
75 if(numTokens < MAX_NUM_TOKENS_PER_GROUP) |
|
76 { |
|
77 if(numTokens==0) |
|
78 { |
|
79 startOffset=token->startOffset(); |
|
80 endOffset=token->endOffset(); |
|
81 } |
|
82 else |
|
83 { |
|
84 startOffset=min(startOffset,token->startOffset()); |
|
85 endOffset=max(endOffset,token->endOffset()); |
|
86 } |
|
87 tokens[numTokens]=token; |
|
88 scores[numTokens]=score; |
|
89 numTokens++; |
|
90 } |
|
91 }*/ |
|
92 |
|
93 bool TokenGroup::isDistinct(CL_NS(analysis)::Token* token) const |
|
94 { |
|
95 return token->startOffset() > endOffset; |
|
96 } |
|
97 |
|
98 |
|
99 void TokenGroup::clear() |
|
100 { |
|
101 numTokens=0; |
|
102 } |
|
103 |
|
104 CL_NS_END2 |