|
24
|
1 |
/*
|
|
|
2 |
* Copyright (c) 2010 Nokia Corporation and/or its subsidiary(-ies).
|
|
|
3 |
* All rights reserved.
|
|
|
4 |
* This component and the accompanying materials are made available
|
|
|
5 |
* under the terms of "Eclipse Public License v1.0"
|
|
|
6 |
* which accompanies this distribution, and is available
|
|
|
7 |
* at the URL "http://www.eclipse.org/legal/epl-v10.html".
|
|
|
8 |
*
|
|
|
9 |
* Initial Contributors:
|
|
|
10 |
* Nokia Corporation - initial contribution.
|
|
|
11 |
*
|
|
|
12 |
* Contributors:
|
|
|
13 |
*
|
|
|
14 |
* Description:
|
|
|
15 |
*
|
|
|
16 |
*/
|
|
|
17 |
|
|
|
18 |
#include "prefixfilter.h"
|
|
|
19 |
|
|
|
20 |
#include "Clucene.h"
|
|
|
21 |
|
|
|
22 |
namespace analysis {
|
|
|
23 |
|
|
|
24 |
using namespace lucene::analysis;
|
|
|
25 |
using namespace lucene::util;
|
|
|
26 |
|
|
|
27 |
const wchar_t* HebrewPrefixes[] = {
|
|
|
28 |
L"\x05d0", // aleph
|
|
|
29 |
L"\x05d1", // bet
|
|
|
30 |
L"\x05d4", // he
|
|
|
31 |
L"\x05d5", // vav
|
|
|
32 |
L"\x05d9", // yud
|
|
|
33 |
L"\x05db", // kaf
|
|
|
34 |
L"\x05dc", // lamed
|
|
|
35 |
L"\x05de", // mem
|
|
|
36 |
L"\x05e0", // nun
|
|
|
37 |
L"\x05e9", // shin
|
|
|
38 |
L"\x05ea", // tav
|
|
|
39 |
NULL
|
|
|
40 |
};
|
|
|
41 |
|
|
|
42 |
PrefixFilter::PrefixFilter(TokenStream* input, bool deleteTs, const wchar_t** prefixes)
|
|
|
43 |
: TokenFilter(input, deleteTs), prefixes_(prefixes), token_(), prefixFound_( false ) {}
|
|
|
44 |
|
|
|
45 |
bool PrefixFilter::next(Token* token) {
|
|
|
46 |
// 1. Either use reduced form of last token or fetch a new token
|
|
|
47 |
if ( prefixFound_ ) {
|
|
|
48 |
token->set(token_.termText(),
|
|
|
49 |
token_.startOffset(),
|
|
|
50 |
token_.endOffset(),
|
|
|
51 |
token_.type());
|
|
|
52 |
token->setPositionIncrement(0);
|
|
|
53 |
prefixFound_ = false;
|
|
|
54 |
} else if (!input->next( token )) {
|
|
|
55 |
// no token found
|
|
|
56 |
return false;
|
|
|
57 |
}
|
|
|
58 |
|
|
|
59 |
// 2. Try to match prefixes with the token
|
|
|
60 |
const wchar_t* text = token->_termText;
|
|
|
61 |
int sz = 0;
|
|
|
62 |
for (int i = 0; prefixes_[i] && !sz; i++) {
|
|
|
63 |
for (; prefixes_[i][sz]; sz++) {
|
|
|
64 |
if (text[sz] != prefixes_[i][sz]) {
|
|
|
65 |
sz = 0;
|
|
|
66 |
break;
|
|
|
67 |
}
|
|
|
68 |
}
|
|
|
69 |
}
|
|
|
70 |
// 3. If prefix found, cut prefix and store cut form of token,
|
|
|
71 |
// if cut form is non-empty
|
|
|
72 |
if ( sz && token->termTextLength() > sz) {
|
|
|
73 |
token_.set(token->termText() + sz,
|
|
|
74 |
token->startOffset(),
|
|
|
75 |
token->endOffset(),
|
|
|
76 |
token->type());
|
|
|
77 |
prefixFound_ = true;
|
|
|
78 |
}
|
|
|
79 |
return true;
|
|
|
80 |
}
|
|
|
81 |
|
|
|
82 |
#define APOSTROPHE1 L'\''
|
|
|
83 |
#define APOSTROPHE2 L'\x2019'
|
|
|
84 |
|
|
|
85 |
const wchar_t* FrenchArticles[] = {
|
|
|
86 |
L"l",
|
|
|
87 |
L"m",
|
|
|
88 |
L"t",
|
|
|
89 |
L"qu",
|
|
|
90 |
L"n",
|
|
|
91 |
L"s",
|
|
|
92 |
L"j",
|
|
|
93 |
NULL
|
|
|
94 |
};
|
|
|
95 |
|
|
|
96 |
// This list should not be relied on
|
|
|
97 |
// Consult language experts!
|
|
|
98 |
const wchar_t* ItalianArticles[] = {
|
|
|
99 |
L"l", // the
|
|
|
100 |
L"d", // from
|
|
|
101 |
L"un", // one, a
|
|
|
102 |
L"dell", // to
|
|
|
103 |
L"all",
|
|
|
104 |
L"e",
|
|
|
105 |
L"quest",
|
|
|
106 |
L"quell",
|
|
|
107 |
L"buon", // means good, should it be here?
|
|
|
108 |
NULL
|
|
|
109 |
};
|
|
|
110 |
|
|
|
111 |
ElisionFilter::ElisionFilter(TokenStream* input, bool deleteTs, const wchar_t** articles)
|
|
|
112 |
: TokenFilter(input, deleteTs), articles_(articles) {}
|
|
|
113 |
|
|
|
114 |
bool ElisionFilter::next(Token* token) {
|
|
|
115 |
if ( input->next(token) ) {
|
|
|
116 |
wchar_t* text = token->_termText;
|
|
|
117 |
int i = 0;
|
|
|
118 |
for (; text[i]; i++) {
|
|
|
119 |
if ( text[i] == APOSTROPHE1
|
|
|
120 |
|| text[i] == APOSTROPHE2 ) {
|
|
|
121 |
break;
|
|
|
122 |
}
|
|
|
123 |
}
|
|
|
124 |
if ( text[i] ) {
|
|
|
125 |
for (int j = 0; articles_[j]; j++) {
|
|
|
126 |
if ( memcmp( articles_[j], text, i ) ) {
|
|
|
127 |
for (int k = 0; ; k++) {
|
|
|
128 |
text[k] = text[k+i+1];
|
|
|
129 |
if (!text[k+i+1]) break; // end of string
|
|
|
130 |
}
|
|
|
131 |
token->resetTermTextLen();
|
|
|
132 |
return true;
|
|
|
133 |
}
|
|
|
134 |
}
|
|
|
135 |
}
|
|
|
136 |
return true;
|
|
|
137 |
}
|
|
|
138 |
return false;
|
|
|
139 |
}
|
|
|
140 |
|
|
|
141 |
using namespace lucene::analysis::standard;
|
|
|
142 |
|
|
|
143 |
TokenStream* HebrewAnalyzer::tokenStream(const wchar_t* fieldName, Reader* reader) {
|
|
|
144 |
auto_ptr<TokenStream> ret( new StandardTokenizer(reader) );
|
|
|
145 |
|
|
|
146 |
ret.reset( new LowerCaseFilter( ret.release(), true ) );
|
|
|
147 |
ret.reset( new StandardFilter( ret.release(), true ) );
|
|
|
148 |
ret.reset( new PrefixFilter( ret.release(), true, HebrewPrefixes ) );
|
|
|
149 |
|
|
|
150 |
return ret.release();
|
|
|
151 |
}
|
|
|
152 |
|
|
|
153 |
TokenStream* HebrewQueryAnalyzer::tokenStream(const wchar_t* fieldName, Reader* reader) {
|
|
|
154 |
auto_ptr<TokenStream> ret( new StandardTokenizer(reader) );
|
|
|
155 |
|
|
|
156 |
ret.reset( new LowerCaseFilter( ret.release(), true ) );
|
|
|
157 |
ret.reset( new StandardFilter( ret.release(), true ) );
|
|
|
158 |
|
|
|
159 |
return ret.release();
|
|
|
160 |
}
|
|
|
161 |
|
|
|
162 |
TokenStream* FrenchAnalyzer::tokenStream(const wchar_t* fieldName, Reader* reader) {
|
|
|
163 |
auto_ptr<TokenStream> ret( new StandardTokenizer(reader) );
|
|
|
164 |
|
|
|
165 |
ret.reset( new LowerCaseFilter( ret.release(), true ) );
|
|
|
166 |
ret.reset( new StandardFilter( ret.release(), true ) );
|
|
|
167 |
ret.reset( new ElisionFilter( ret.release(), true, FrenchArticles ) );
|
|
|
168 |
|
|
|
169 |
return ret.release();
|
|
|
170 |
}
|
|
|
171 |
|
|
|
172 |
const TCHAR* NonEnglishStopWords::FRENCH_STOP_WORDS[] = {
|
|
|
173 |
_T("a"), _T("afin"), _T("ai"), _T("ainsi"), _T("après"), _T("attendu"), _T("au"), _T("aujourd"),
|
|
|
174 |
_T("auquel"), _T("aussi"), _T("autre"), _T("autres"), _T("aux"), _T("auxquelles"), _T("auxquels"),
|
|
|
175 |
_T("avait"), _T("avant"), _T("avec"), _T("avoir"), _T("c"), _T("car"), _T("ce"), _T("ceci"), _T("cela"), _T("celle"), _T("celles"), _T("celui"), _T("cependant"), _T("certain"),
|
|
|
176 |
_T("certaine"), _T("certaines"), _T("certains"), _T("ces"), _T("cet"), _T("cette"), _T("ceux"), _T("chez"), _T("ci"),
|
|
|
177 |
_T("combien"), _T("comme"), _T("comment"), _T("concernant"), _T("contre"), _T("d"), _T("dans"), _T("de"), _T("debout"),
|
|
|
178 |
_T("dedans"), _T("dehors"), _T("delà"), _T("depuis"), _T("derrière"), _T("des"), _T("désormais"), _T("desquelles"),
|
|
|
179 |
_T("desquels"), _T("dessous"), _T("dessus"), _T("devant"), _T("devers"), _T("devra"), _T("divers"), _T("diverse"),
|
|
|
180 |
_T("diverses"), _T("doit"), _T("donc"), _T("dont"), _T("du"), _T("duquel"), _T("durant"), _T("dès"), _T("elle"), _T("elles"),
|
|
|
181 |
_T("en"), _T("entre"), _T("environ"), _T("est"), _T("et"), _T("etc"), _T("etre"), _T("eu"), _T("eux"), _T("excepté"), _T("hormis"),
|
|
|
182 |
_T("hors"), _T("hélas"), _T("hui"), _T("il"), _T("ils"), _T("j"), _T("je"), _T("jusqu"), _T("jusque"), _T("l"), _T("la"), _T("laquelle"),
|
|
|
183 |
_T("le"), _T("lequel"), _T("les"), _T("lesquelles"), _T("lesquels"), _T("leur"), _T("leurs"), _T("lorsque"), _T("lui"), _T("là"),
|
|
|
184 |
_T("ma"), _T("mais"), _T("malgré"), _T("me"), _T("merci"), _T("mes"), _T("mien"), _T("mienne"), _T("miennes"), _T("miens"), _T("moi"),
|
|
|
185 |
_T("moins"), _T("mon"), _T("moyennant"), _T("même"), _T("mêmes"), _T("n"), _T("ne"), _T("ni"), _T("non"), _T("nos"), _T("notre"),
|
|
|
186 |
_T("nous"), _T("néanmoins"), _T("nôtre"), _T("nôtres"), _T("on"), _T("ont"), _T("ou"), _T("outre"), _T("où"), _T("par"), _T("parmi"),
|
|
|
187 |
_T("partant"), _T("pas"), _T("passé"), _T("pendant"), _T("plein"), _T("plus"), _T("plusieurs"), _T("pour"), _T("pourquoi"),
|
|
|
188 |
_T("proche"), _T("près"), _T("puisque"), _T("qu"), _T("quand"), _T("que"), _T("quel"), _T("quelle"), _T("quelles"), _T("quels"),
|
|
|
189 |
_T("qui"), _T("quoi"), _T("quoique"), _T("revoici"), _T("revoilà"), _T("s"), _T("sa"), _T("sans"), _T("sauf"), _T("se"), _T("selon"),
|
|
|
190 |
_T("seront"), _T("ses"), _T("si"), _T("sien"), _T("sienne"), _T("siennes"), _T("siens"), _T("sinon"), _T("soi"), _T("soit"),
|
|
|
191 |
_T("son"), _T("sont"), _T("sous"), _T("suivant"), _T("sur"), _T("ta"), _T("te"), _T("tes"), _T("tien"), _T("tienne"), _T("tiennes"),
|
|
|
192 |
_T("tiens"), _T("toi"), _T("ton"), _T("tous"), _T("tout"), _T("toute"), _T("toutes"), _T("tu"), _T("un"), _T("une"), _T("va"), _T("vers"),
|
|
|
193 |
_T("voici"), _T("voilà"), _T("vos"), _T("votre"), _T("vous"), _T("vu"), _T("vôtre"), _T("vôtres"), _T("y"), _T("à"), _T("ça"), _T("ès"),
|
|
|
194 |
_T("été"), _T("être"), _T("ô"), NULL
|
|
|
195 |
};
|
|
|
196 |
|
|
|
197 |
const TCHAR* NonEnglishStopWords::BRAZILIAN_STOP_WORDS[] = {
|
|
|
198 |
_T("a"),_T("ainda"),_T("alem"), _T("ambas"), _T("ambos"), _T("antes"),
|
|
|
199 |
_T("ao"), _T("aonde"), _T("aos"), _T("apos"), _T("aquele"), _T("aqueles"),
|
|
|
200 |
_T("as"), _T("assim"), _T("com"), _T("como"), _T("contra"), _T("contudo"),
|
|
|
201 |
_T("cuja"), _T("cujas"), _T("cujo"), _T("cujos"), _T("da"), _T("das"), _T("de"),
|
|
|
202 |
_T("dela"), _T("dele"), _T("deles"), _T("demais"), _T("depois"), _T("desde"),
|
|
|
203 |
_T("desta"), _T("deste"), _T("dispoe"), _T("dispoem"), _T("diversa"),
|
|
|
204 |
_T("diversas"), _T("diversos"), _T("do"), _T("dos"), _T("durante"), _T("e"),
|
|
|
205 |
_T("ela"), _T("elas"), _T("ele"), _T("eles"), _T("em"), _T("entao"), _T("entre"),
|
|
|
206 |
_T("essa"), _T("essas"), _T("esse"), _T("esses"), _T("esta"), _T("estas"),
|
|
|
207 |
_T("este"), _T("estes"), _T("ha"), _T("isso"), _T("isto"), _T("logo"), _T("mais"),
|
|
|
208 |
_T("mas"), _T("mediante"), _T("menos"), _T("mesma"), _T("mesmas"), _T("mesmo"),
|
|
|
209 |
_T("mesmos"), _T("na"), _T("nas"), _T("nao"), _T("nas"), _T("nem"), _T("nesse"), _T("neste"),
|
|
|
210 |
_T("nos"), _T("o"), _T("os"), _T("ou"), _T("outra"), _T("outras"), _T("outro"), _T("outros"),
|
|
|
211 |
_T("pelas"), _T("pelas"), _T("pelo"), _T("pelos"), _T("perante"), _T("pois"), _T("por"),
|
|
|
212 |
_T("porque"), _T("portanto"), _T("proprio"), _T("propios"), _T("quais"), _T("qual"),
|
|
|
213 |
_T("qualquer"), _T("quando"), _T("quanto"), _T("que"), _T("quem"), _T("quer"),_T("se"),
|
|
|
214 |
_T("seja"), _T("sem"), _T("sendo"), _T("seu"), _T("seus"), _T("sob"), _T("sobre"), _T("sua"),
|
|
|
215 |
_T("suas"), _T("tal"), _T("tambem"), _T("teu"), _T("teus"), _T("toda"), _T("todas"), _T("todo"),
|
|
|
216 |
_T("todos"), _T("tua"), _T("tuas"), _T("tudo"), _T("um"), _T("uma"), _T("umas"), _T("uns"),
|
|
|
217 |
NULL
|
|
|
218 |
};
|
|
|
219 |
|
|
|
220 |
const TCHAR* NonEnglishStopWords::CZECH_STOP_WORDS[] = {
|
|
|
221 |
_T("a"),_T("s"),_T("k"),_T("o"),_T("i"),_T("u"),_T("v"), _T("z"),_T("dnes"),
|
|
|
222 |
_T("cz"),_T("t\u00edmto"),_T("bude\u0161"),_T("budem"), _T("byli"), _T("jse\u0161"), _T("m\u016fj"), _T("sv\u00fdm"), _T("ta"), _T("tomto"), _T("tohle"), _T("tuto"), _T("tyto"),
|
|
|
223 |
_T("jej"), _T("zda"), _T("pro\u010d"), _T("m\u00e1te"), _T("tato"), _T("kam"), _T("tohoto"), _T("kdo"), _T("kte\u0159\u00ed"),
|
|
|
224 |
_T("mi"), _T("n\u00e1m"), _T("tom"), _T("tomuto"), _T("m\u00edt"), _T("nic"), _T("proto"), _T("kterou"), _T("byla"),
|
|
|
225 |
_T("toho"), _T("proto\u017ee"), _T("asi"), _T("ho"), _T("na\u0161i"), _T("napi\u0161te"), _T("re"), _T("co\u017e"), _T("t\u00edm"),
|
|
|
226 |
_T("tak\u017ee"), _T("sv\u00fdch"), _T("jej\u00ed"), _T("sv\u00fdmi"), _T("jste"), _T("aj"), _T("tu"), _T("tedy"), _T("teto"),
|
|
|
227 |
_T("bylo"), _T("kde"), _T("ke"), _T("prav\u00e9"), _T("ji"), _T("nad"), _T("nejsou"), _T("\u010di"), _T("pod"), _T("t\u00e9ma"),
|
|
|
228 |
_T("mezi"), _T("p\u0159es"), _T("ty"), _T("pak"), _T("v\u00e1m"), _T("ani"), _T("kdy\u017e"), _T("v\u0161ak"), _T("neg"), _T("jsem"),
|
|
|
229 |
_T("tento"), _T("\u010dl\u00e1nku"), _T("\u010dl\u00e1nky"), _T("aby"), _T("jsme"), _T("p\u0159ed"), _T("pta"), _T("jejich"),
|
|
|
230 |
_T("byl"), _T("je\u0161t\u011b"), _T("a\u017e"), _T("bez"), _T("tak\u00e9"), _T("pouze"), _T("prvn\u00ed"), _T("va\u0161e"), _T("kter\u00e1"),
|
|
|
231 |
_T("n\u00e1s"), _T("nov\u00fd"), _T("tipy"), _T("pokud"), _T("m\u016f\u017ee"), _T("strana"), _T("jeho"), _T("sv\u00e9"), _T("jin\u00e9"),
|
|
|
232 |
_T("zpr\u00e1vy"), _T("nov\u00e9"), _T("nen\u00ed"), _T("v\u00e1s"), _T("jen"), _T("podle"), _T("zde"), _T("u\u017e"), _T("b\u00fdt"), _T("v\u00edce"),
|
|
|
233 |
_T("bude"), _T("ji\u017e"), _T("ne\u017e"), _T("kter\u00fd"), _T("by"), _T("kter\u00e9"), _T("co"), _T("nebo"), _T("ten"), _T("tak"),
|
|
|
234 |
_T("m\u00e1"), _T("p\u0159i"), _T("od"), _T("po"), _T("jsou"), _T("jak"), _T("dal\u0161\u00ed"), _T("ale"), _T("si"), _T("se"), _T("ve"),
|
|
|
235 |
_T("to"), _T("jako"), _T("za"), _T("zp\u011bt"), _T("ze"), _T("do"), _T("pro"), _T("je"), _T("na"), _T("atd"), _T("atp"),
|
|
|
236 |
_T("jakmile"), _T("p\u0159i\u010dem\u017e"), _T("j\u00e1"), _T("on"), _T("ona"), _T("ono"), _T("oni"), _T("ony"), _T("my"), _T("vy"),
|
|
|
237 |
_T( "j\u00ed"), _T("ji"), _T("m\u011b"), _T("mne"), _T("jemu"), _T("tomu"), _T("t\u011bm"), _T("t\u011bmu"), _T("n\u011bmu"), _T("n\u011bmu\u017e"),
|
|
|
238 |
_T("jeho\u017e"), _T("j\u00ed\u017e"), _T("jeliko\u017e"), _T("je\u017e"), _T("jako\u017e"), _T("na\u010de\u017e"),
|
|
|
239 |
NULL
|
|
|
240 |
};
|
|
|
241 |
|
|
|
242 |
const TCHAR* NonEnglishStopWords::GERMAN_STOP_WORDS[] = {
|
|
|
243 |
_T("einer"), _T( "eine"), _T( "eines"), _T( "einem"), _T( "einen"),
|
|
|
244 |
_T("der"), _T( "die"), _T( "das"), _T( "dass"), _T( "daß"),
|
|
|
245 |
_T("du"), _T( "er"), _T( "sie"), _T( "es"),
|
|
|
246 |
_T("was"), _T( "wer"), _T( "wie"), _T( "wir"),
|
|
|
247 |
_T("und"), _T( "oder"), _T( "ohne"), _T( "mit"),
|
|
|
248 |
_T("am"), _T( "im"), _T( "in"), _T( "aus"), _T( "auf"),
|
|
|
249 |
_T("ist"), _T( "sein"), _T( "war"), _T( "wird"),
|
|
|
250 |
_T("ihr"), _T( "ihre"), _T( "ihres"),
|
|
|
251 |
_T("als"), _T( "für"), _T( "von"), _T( "mit"),
|
|
|
252 |
_T("dich"), _T( "dir"), _T( "mich"), _T( "mir"),
|
|
|
253 |
_T("mein"), _T( "sein"), _T( "kein"),
|
|
|
254 |
_T("durch"), _T( "wegen"), _T( "wird"),
|
|
|
255 |
NULL
|
|
|
256 |
};
|
|
|
257 |
|
|
|
258 |
const TCHAR* NonEnglishStopWords::GREEK_STOP_WORDS[] = {
|
|
|
259 |
_T( "ο" ), _T( "η" ), _T( "το" ), _T( "οι" ), _T( "τα" ), _T( "του" ), _T( "τησ" ), _T( "των" ), _T( "τον" ), _T( "την" ), _T( "και" ),
|
|
|
260 |
_T( "κι" ), _T( "κ" ), _T( "ειμαι" ), _T( "εισαι" ), _T( "ειναι" ), _T( "ειμαστε" ), _T( "ειστε" ), _T( "στο" ), _T( "στον" ),
|
|
|
261 |
_T( "στη" ), _T( "στην" ), _T( "μα" ), _T( "αλλα" ), _T( "απο" ), _T( "για" ), _T( "π?οσ" ), _T( "με" ), _T( "σε" ), _T( "ωσ" ),
|
|
|
262 |
_T( "πα?α" ), _T( "αντι" ), _T( "κατα" ), _T( "μετα" ), _T( "θα" ), _T( "να" ), _T( "δε" ), _T( "δεν" ), _T( "μη" ), _T( "μην" ),
|
|
|
263 |
_T( "επι" ), _T( "ενω" ), _T( "εαν" ), _T( "αν" ), _T( "τοτε" ), _T( "που" ), _T( "πωσ" ), _T( "ποιοσ" ), _T( "ποια" ), _T( "ποιο" ),
|
|
|
264 |
_T( "ποιοι" ), _T( "ποιεσ" ), _T( "ποιων" ), _T( "ποιουσ" ), _T( "αυτοσ" ), _T( "αυτη" ), _T( "αυτο" ), _T( "αυτοι" ),
|
|
|
265 |
_T( "αυτων" ), _T( "αυτουσ" ), _T( "αυτεσ" ), _T( "αυτα" ), _T( "εκεινοσ" ), _T( "εκεινη" ), _T( "εκεινο" ),
|
|
|
266 |
_T( "εκεινοι" ), _T( "εκεινεσ" ), _T( "εκεινα" ), _T( "εκεινων" ), _T( "εκεινουσ" ), _T( "οπωσ" ), _T( "ομωσ" ),
|
|
|
267 |
_T( "ισωσ" ), _T( "οσο" ), _T( "οτι" ),
|
|
|
268 |
NULL
|
|
|
269 |
};
|
|
|
270 |
|
|
|
271 |
const TCHAR* NonEnglishStopWords::DUTCH_STOP_WORDS[] = {
|
|
|
272 |
_T("de"), _T("en"), _T("van"), _T("ik"), _T("te"), _T("dat"), _T("die"), _T("in"), _T("een"),
|
|
|
273 |
_T("hij"), _T("het"), _T("niet"), _T("zijn"), _T("is"), _T("was"), _T("op"), _T("aan"), _T("met"), _T("als"), _T("voor"), _T("had"),
|
|
|
274 |
_T("er"), _T("maar"), _T("om"), _T("hem"), _T("dan"), _T("zou"), _T("of"), _T("wat"), _T("mijn"), _T("men"), _T("dit"), _T("zo"),
|
|
|
275 |
_T("door"), _T("over"), _T("ze"), _T("zich"), _T("bij"), _T("ook"), _T("tot"), _T("je"), _T("mij"), _T("uit"), _T("der"), _T("daar"),
|
|
|
276 |
_T("haar"), _T("naar"), _T("heb"), _T("hoe"), _T("heeft"), _T("hebben"), _T("deze"), _T("u"), _T("want"), _T("nog"), _T("zal"),
|
|
|
277 |
_T("me"), _T("zij"), _T("nu"), _T("ge"), _T("geen"), _T("omdat"), _T("iets"), _T("worden"), _T("toch"), _T("al"), _T("waren"),
|
|
|
278 |
_T("veel"), _T("meer"), _T("doen"), _T("toen"), _T("moet"), _T("ben"), _T("zonder"), _T("kan"), _T("hun"), _T("dus"),
|
|
|
279 |
_T("alles"), _T("onder"), _T("ja"), _T("eens"), _T("hier"), _T("wie"), _T("werd"), _T("altijd"), _T("doch"), _T("wordt"),
|
|
|
280 |
_T("wezen"), _T("kunnen"), _T("ons"), _T("zelf"), _T("tegen"), _T("na"), _T("reeds"), _T("wil"), _T("kon"), _T("niets"),
|
|
|
281 |
_T("uw"), _T("iemand"), _T("geweest"), _T("andere"),
|
|
|
282 |
NULL
|
|
|
283 |
};
|
|
|
284 |
|
|
|
285 |
const TCHAR* NonEnglishStopWords::RUSSIAN_STOP_WORDS[] = {
|
|
|
286 |
_T("а"), _T("без"), _T("более"), _T("бы"), _T("был"), _T("была"), _T("были"), _T("было"), _T("быть"), _T("в"),
|
|
|
287 |
_T("вам"), _T("вас"), _T("весь"), _T("во"), _T("вот"), _T("все"), _T("всего"), _T("всех"), _T("вы"), _T("где"), _T(
|
|
|
288 |
"да"), _T("даже"), _T("для"), _T("до"), _T("его"), _T("ее"), _T("ей"), _T("ею"), _T("если"), _T("есть"), _T(
|
|
|
289 |
"еще"), _T("же"), _T("за"), _T("здесь"), _T("и"), _T("из"), _T("или"), _T("им"), _T("их"), _T("к"), _T("как"),
|
|
|
290 |
_T("ко"), _T("когда"), _T("кто"), _T("ли"), _T("либо"), _T("мне"), _T("может"), _T("мы"), _T("на"), _T("надо"), _T(
|
|
|
291 |
"наш"), _T("не"), _T("него"), _T("нее"), _T("нет"), _T("ни"), _T("них"), _T("но"), _T("ну"), _T("о"), _T("об"), _T(
|
|
|
292 |
"однако"), _T("он"), _T("она"), _T("они"), _T("оно"), _T("от"), _T("очень"), _T("по"), _T("под"), _T("при"), _T(
|
|
|
293 |
"с"), _T("со"), _T("так"), _T("также"), _T("такой"), _T("там"), _T("те"), _T("тем"), _T("то"), _T("того"), _T(
|
|
|
294 |
"тоже"), _T("той"), _T("только"), _T("том"), _T("ты"), _T("у"), _T("уже"), _T("хотя"), _T("чего"), _T("чей"), _T(
|
|
|
295 |
"чем"), _T("что"), _T("чтобы"), _T("чье"), _T("чья"), _T("эта"), _T("эти"), _T("это"), _T("я"),
|
|
|
296 |
NULL
|
|
|
297 |
};
|
|
|
298 |
|
|
|
299 |
const TCHAR* NonEnglishStopWords::EXTENDED_ENGLISH_STOP_WORDS[] = {
|
|
|
300 |
_T("a"), _T("about"), _T("above"), _T("across"), _T("adj"), _T("after"), _T("afterwards"),
|
|
|
301 |
_T("again"), _T("against"), _T("albeit"), _T("all"), _T("almost"), _T("alone"), _T("along"),
|
|
|
302 |
_T("already"), _T("also"), _T("although"), _T("always"), _T("among"), _T("amongst"), _T("an"),
|
|
|
303 |
_T("and"), _T("another"), _T("any"), _T("anyhow"), _T("anyone"), _T("anything"),
|
|
|
304 |
_T("anywhere"), _T("are"), _T("around"), _T("as"), _T("at"), _T("be"), _T("became"), _T("because"),
|
|
|
305 |
_T("become"), _T("becomes"), _T("becoming"), _T("been"), _T("before"), _T("beforehand"),
|
|
|
306 |
_T("behind"), _T("being"), _T("below"), _T("beside"), _T("besides"), _T("between"),
|
|
|
307 |
_T("beyond"), _T("both"), _T("but"), _T("by"), _T("can"), _T("cannot"), _T("co"), _T("could"),
|
|
|
308 |
_T("down"), _T("during"), _T("each"), _T("eg"), _T("either"), _T("else"), _T("elsewhere"),
|
|
|
309 |
_T("enough"), _T("etc"), _T("even"), _T("ever"), _T("every"), _T("everyone"), _T("everything"),
|
|
|
310 |
_T("everywhere"), _T("except"), _T("few"), _T("first"), _T("for"), _T("former"),
|
|
|
311 |
_T("formerly"), _T("from"), _T("further"), _T("had"), _T("has"), _T("have"), _T("he"), _T("hence"),
|
|
|
312 |
_T("her"), _T("here"), _T("hereafter"), _T("hereby"), _T("herein"), _T("hereupon"), _T("hers"),
|
|
|
313 |
_T("herself"), _T("him"), _T("himself"), _T("his"), _T("how"), _T("however"), _T("i"), _T("ie"), _T("if"),
|
|
|
314 |
_T("in"), _T("inc"), _T("indeed"), _T("into"), _T("is"), _T("it"), _T("its"), _T("itself"), _T("last"),
|
|
|
315 |
_T("latter"), _T("latterly"), _T("least"), _T("less"), _T("ltd"), _T("many"), _T("may"), _T("me"),
|
|
|
316 |
_T("meanwhile"), _T("might"), _T("more"), _T("moreover"), _T("most"), _T("mostly"), _T("much"),
|
|
|
317 |
_T("must"), _T("my"), _T("myself"), _T("namely"), _T("neither"), _T("never"),
|
|
|
318 |
_T("nevertheless"), _T("next"), _T("no"), _T("nobody"), _T("none"), _T("noone"), _T("nor"),
|
|
|
319 |
_T("not"), _T("nothing"), _T("now"), _T("nowhere"), _T("of"), _T("off"), _T("often"), _T("on"),
|
|
|
320 |
_T("once one"), _T("only"), _T("onto"), _T("or"), _T("other"), _T("others"), _T("otherwise"),
|
|
|
321 |
_T("our"), _T("ours"), _T("ourselves"), _T("out"), _T("over"), _T("own"), _T("per"), _T("perhaps"),
|
|
|
322 |
_T("rather"), _T("s"), _T("same"), _T("seem"), _T("seemed"), _T("seeming"), _T("seems"),
|
|
|
323 |
_T("several"), _T("she"), _T("should"), _T("since"), _T("so"), _T("some"), _T("somehow"),
|
|
|
324 |
_T("someone"), _T("something"), _T("sometime"), _T("sometimes"), _T("somewhere"),
|
|
|
325 |
_T("still"), _T("such"), _T("t"), _T("than"), _T("that"), _T("the"), _T("their"), _T("them"),
|
|
|
326 |
_T("themselves"), _T("then"), _T("thence"), _T("there"), _T("thereafter"), _T("thereby"),
|
|
|
327 |
_T("therefor"), _T("therein"), _T("thereupon"), _T("these"), _T("they"), _T("this"),
|
|
|
328 |
_T("those"), _T("though"), _T("through"), _T("throughout"), _T("thru"), _T("thus"), _T("to"),
|
|
|
329 |
_T("together"), _T("too"), _T("toward"), _T("towards"), _T("under"), _T("until"), _T("up"),
|
|
|
330 |
_T("upon"), _T("us"), _T("very"), _T("via"), _T("was"), _T("we"), _T("well"), _T("were"), _T("what"),
|
|
|
331 |
_T("whatever"), _T("whatsoever"), _T("when"), _T("whence"), _T("whenever"),
|
|
|
332 |
_T("whensoever"), _T("where"), _T("whereafter"), _T("whereas"), _T("whereat"),
|
|
|
333 |
_T("whereby"), _T("wherefrom"), _T("wherein"), _T("whereinto"), _T("whereof"),
|
|
|
334 |
_T("whereon"), _T("whereto"), _T("whereunto"), _T("whereupon"), _T("wherever"),
|
|
|
335 |
_T("wherewith"), _T("whether"), _T("which"), _T("whichever"), _T("whichsoever"),
|
|
|
336 |
_T("while"), _T("whilst"), _T("whither"), _T("who"), _T("whoever"), _T("whole"), _T("whom"),
|
|
|
337 |
_T("whomever"), _T("whomsoever"), _T("whose"), _T("whosoever"), _T("why"), _T("will"),
|
|
|
338 |
_T("with"), _T("within"), _T("without"), _T("would"), _T("xsubj"), _T("xcal"), _T("xauthor"),
|
|
|
339 |
_T("xother "), _T("xnote"), _T("yet"), _T("you"), _T("your"), _T("yours"), _T("yourself"),
|
|
|
340 |
_T("yourselves"),
|
|
|
341 |
NULL
|
|
|
342 |
};
|
|
|
343 |
}
|
|
|
344 |
/*TODO:
|
|
|
345 |
* extented english stop word set can be merged to main english stop words set
|
|
|
346 |
* chinese thai, cjk and may be others that use english stop word list may be done as
|
|
|
347 |
* const TCHAR* NonEnglishStopWords::THAI_STOP_WORDS = NonEnglishStopWords::ENGLISH_STOP_WORDS
|
|
|
348 |
* so as to safely use CustomAnalyzer(L"thai>stop(th)")
|
|
|
349 |
*/
|