/*------------------------------------------------------------------------------
* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team
*
* Distributable under the terms of either the Apache License (Version 2.0) or
* the GNU Lesser General Public License, as specified in the COPYING file.
------------------------------------------------------------------------------*/
#include "clucene/stdheader.h"
#include "StandardTokenizer.h"
CL_NS_USE(analysis)
CL_NS_USE(util)
CL_NS_DEF2(analysis,standard)
const TCHAR* tokenImageArray[] = {
_T("<EOF>"),
_T("<UNKNOWN>"),
_T("<ALPHANUM>"),
_T("<APOSTROPHE>"),
_T("<ACRONYM>"),
_T("<COMPANY>"),
_T("<EMAIL>"),
_T("<HOST>"),
_T("<NUM>"),
_T("<CJK>")
};
const TCHAR** tokenImage = tokenImageArray;
/* A bunch of shortcut macros, many of which make assumptions about variable
** names. These macros enhance readability, not just convenience! */
#define EOS (ch==-1 || rd->Eos())
#define SPACE (_istspace((TCHAR)ch) != 0)
#define ALPHA (_istalpha((TCHAR)ch) != 0)
#define ALNUM (_istalnum(ch) != 0)
#define DIGIT (_istdigit(ch) != 0)
#define UNDERSCORE (ch == '_')
#define _CJK ( (ch>=0x3040 && ch<=0x318f) || \
(ch>=0x3300 && ch<=0x337f) || \
(ch>=0x3400 && ch<=0x3d2d) || \
(ch>=0x4e00 && ch<=0x9fff) || \
(ch>=0xf900 && ch<=0xfaff) || \
(ch>=0xac00 && ch<=0xd7af) ) //korean
// CHANGED Thai support ->
#define _THAI ( ch >= 0x0E00 && ch <= 0x0Eff )
// CHANGED
#define DASH (ch == '-')
#define NEGATIVE_SIGN_ DASH
//#define POSITIVE_SIGN_ (ch == '+')
//#define SIGN (NEGATIVE_SIGN_ || POSITIVE_SIGN_)
#define DOT (ch == '.')
#define DECIMAL DOT
//freebsd seems to have a problem with defines over multiple lines, so this has to be one long line
#define _CONSUME_AS_LONG_AS(conditionFails) while (true) { ch = readChar(); if (ch==-1 || (!(conditionFails) || str.len >= LUCENE_MAX_WORD_LEN)) { break; } str.appendChar(ch);}
#define CONSUME_ALPHAS _CONSUME_AS_LONG_AS(ALPHA)
#define CONSUME_DIGITS _CONSUME_AS_LONG_AS(DIGIT)
/* otherMatches is a condition (possibly compound) under which a character
** that's not an ALNUM or UNDERSCORE can be considered not to break the
** span. Callers should pass false if only ALNUM/UNDERSCORE are acceptable. */
#define CONSUME_WORD _CONSUME_AS_LONG_AS(ALNUM /*|| UNDERSCORE*/)
/*
** Consume CJK characters
*/
#define CONSUME_CJK _CONSUME_AS_LONG_AS(_CJK)
// CHANGED Thai support ->
/**
* Consume Thai
*/
#define CONSUME_THAI _CONSUME_AS_LONG_AS(_THAI)
// CHANGED
/* It is considered that "nothing of value" has been read if:
** a) The "read head" hasn't moved since specialCharPos was established.
** or
** b) The "read head" has moved by one character, but that character was
** either whitespace or not among the characters found in the body of
** a token (deliberately doesn't include the likes of '@'/'&'). */
#define CONSUMED_NOTHING_OF_VALUE (rdPos == specialCharPos || (rdPos == specialCharPos+1 && ( SPACE || !(ALNUM || DOT || DASH || UNDERSCORE) )))
#define RIGHTMOST(sb) (sb.getBuffer()[sb.len-1])
#define RIGHTMOST_IS(sb, c) (RIGHTMOST(sb) == c)
/* To discard the last character in a StringBuffer, we decrement the buffer's
** length indicator and move the terminator back by one character. */
#define SHAVE_RIGHTMOST(sb) (sb.getBuffer()[--sb.len] = '\0')
//#define REMOVE_TRAILING_CHARS(sb, charMatchesCondition) { TCHAR* sbBuf = sb.getBuffer(); for (int32_t i = sb.len-1; i >= 0; i--) { TCHAR c = sbBuf[i]; if (charMatchesCondition) { sbBuf[--sb.len] = '\0'; } else {break;}}}
/* Does StringBuffer sb contain any of the characters in string ofThese? */
#define CONTAINS_ANY(sb, ofThese) (_tcscspn(sb.getBuffer(), _T(ofThese)) != static_cast<size_t>(sb.len))
StandardTokenizer::StandardTokenizer(Reader* reader):
/* rdPos is zero-based. It starts at -1, and will advance to the first
** position when readChar() is first called. */
rdPos(-1),
tokenStart(-1),
rd(_CLNEW FastCharStream(reader))
{
}
StandardTokenizer::~StandardTokenizer() {
_CLDELETE(rd);
}
int StandardTokenizer::readChar() {
/* Increment by 1 because we're speaking in terms of characters, not
** necessarily bytes: */
rdPos++;
return rd->GetNext();
}
void StandardTokenizer::unReadChar() {
rd->UnGet();
rdPos--;
}
inline bool StandardTokenizer::setToken(Token* t, StringBuffer* sb, TokenTypes tokenCode) {
t->setStartOffset(tokenStart);
t->setEndOffset(tokenStart+sb->length());
t->setType(tokenImage[tokenCode]);
sb->getBuffer(); //null terminates the buffer
t->resetTermTextLen();
t->setPositionIncrement(1);
return true;
}
bool StandardTokenizer::next(Token* t) {
int ch=0;
while (!EOS) {
ch = readChar();
if ( ch == 0 || ch == -1 ){
continue;
// CHANGED Thai Support ->
} else if ( _THAI ) {
if ( ReadThai(ch,t) )
return true;
// CHANGED
} else if (SPACE) {
continue;
} else if (ALPHA) {
tokenStart = rdPos;
return ReadAlphaNum(ch,t);
} else if (DIGIT || NEGATIVE_SIGN_ || DECIMAL) {
tokenStart = rdPos;
/* ReadNumber returns NULL if it fails to extract a valid number; in
** that case, we just continue. */
if (ReadNumber(NULL, ch,t))
return true;
} else if ( _CJK ){
if ( ReadCJK(ch,t) )
return true;
}
}
return false;
}
bool StandardTokenizer::ReadNumber(const TCHAR* previousNumber, const TCHAR prev,Token* t) {
/* previousNumber is only non-NULL if this function already read a complete
** number in a previous recursion, yet has been asked to read additional
** numeric segments. For example, in the HOST "192.168.1.3", "192.168" is
** a complete number, but this function will recurse to read the "1.3",
** generating a single HOST token "192.168.1.3". */
t->growBuffer(LUCENE_MAX_WORD_LEN+1);//make sure token can hold the next word
StringBuffer str(t->_termText,t->bufferLength(),true); //use stringbuffer to read data onto the termText
TokenTypes tokenType;
bool decExhausted;
if (previousNumber != NULL) {
str.prepend(previousNumber);
tokenType = CL_NS2(analysis,standard)::HOST;
decExhausted = false;
} else {
tokenType = CL_NS2(analysis,standard)::NUM;
decExhausted = (prev == '.');
}
if ( str.len >= LUCENE_MAX_WORD_LEN ){
//if a number is too long, i would say there is no point
//storing it, because its going to be the wrong number anyway?
//what do people think?
return false;
}
str.appendChar(prev);
const bool signExhausted = (prev == '-');
int ch = prev;
CONSUME_DIGITS;
if (str.len < 2 /* CONSUME_DIGITS didn't find any digits. */
&& (
(signExhausted && !DECIMAL)
|| (decExhausted /* && !DIGIT is implied, since CONSUME_DIGITS stopped on a non-digit. */)
)
)
{
/* We have either:
** a) a negative sign that's not followed by either digit(s) or a decimal
** b) a decimal that's not followed by digit(s)
** so this is not a valid number. */
if (!EOS) {
/* Unread the character that stopped CONSUME_DIGITS: */
unReadChar();
}
return false;
}
/* We just read a group of digits. Is it followed by a decimal symbol,
** implying that there might be another group of digits available? */
if (!EOS) {
if (DECIMAL) {
if ( str.len >= LUCENE_MAX_WORD_LEN )
return false; //read above for rationale
str.appendChar(ch);
} else {
unReadChar();
goto SUCCESSFULLY_EXTRACTED_NUMBER;
}
CONSUME_DIGITS;
if (!DIGIT && !DECIMAL) {
unReadChar();
} else if (!EOS && DECIMAL && _istdigit(rd->Peek())) {
/* We just read the fractional digit group, but it's also followed by
** a decimal symbol and at least one more digit, so this must be a
** HOST rather than a real number. */
return ReadNumber(str.getBuffer(), '.',t);
}
}
SUCCESSFULLY_EXTRACTED_NUMBER:
TCHAR rightmost = RIGHTMOST(str);
/* Don't including a trailing decimal point. */
if(ALPHA){
CONSUME_WORD;
}
if (rightmost == '.') {
SHAVE_RIGHTMOST(str);
unReadChar();
rightmost = RIGHTMOST(str);
}
/* If all we have left is a negative sign, it's not a valid number. */
if (rightmost == '-') {
CND_PRECONDITION (str.len == 1, "Number is invalid");
return false;
}
return setToken(t,&str,tokenType);
}
bool StandardTokenizer::ReadAlphaNum(const TCHAR prev, Token* t) {
t->growBuffer(LUCENE_MAX_WORD_LEN+1);//make sure token can hold the next word
StringBuffer str(t->_termText,t->bufferLength(),true); //use stringbuffer to read data onto the termText
if ( str.len < LUCENE_MAX_WORD_LEN ){
str.appendChar(prev);
int ch = prev;
CONSUME_WORD;
if (!EOS && str.len < LUCENE_MAX_WORD_LEN-1 ) { //still have space for 1 more character?
switch(ch) { /* What follows the first alphanum segment? */
// case '.':
// str.appendChar('.');
// return ReadDotted(&str, CL_NS2(analysis,standard)::UNKNOWN,t);
case '\'':
str.appendChar('\'');
return ReadApostrophe(&str,t);
// case '@':
// str.appendChar('@');
// return ReadAt(&str,t);
case '&':
str.appendChar('&');
return ReadCompany(&str,t);
/* default: fall through to end of this function. */
}
}
}
return setToken(t,&str,CL_NS2(analysis,standard)::ALPHANUM);
}
bool StandardTokenizer::ReadCJK(const TCHAR prev, Token* t) {
t->growBuffer(LUCENE_MAX_WORD_LEN+1);//make sure token can hold the next word
StringBuffer str(t->_termText,t->bufferLength(),true); //use stringbuffer to read data onto the termText
if ( str.len < LUCENE_MAX_WORD_LEN ){
str.appendChar(prev);
int ch = prev;
CONSUME_CJK;
}
return setToken(t,&str,CL_NS2(analysis,standard)::CJK);
}
// CHANGED Thai Support
bool StandardTokenizer::ReadThai(const TCHAR prev, Token* t) {
t->growBuffer(LUCENE_MAX_WORD_LEN+1);//make sure token can hold the next word
StringBuffer str(t->_termText,t->bufferLength(),true); //use stringbuffer to read data onto the termText
if ( str.len < LUCENE_MAX_WORD_LEN ){
str.appendChar(prev);
int ch = prev;
CONSUME_THAI;
}
return setToken(t,&str,CL_NS2(analysis,standard)::Thai);
}
// CHANGED
bool StandardTokenizer::ReadDotted(StringBuffer* _str, TokenTypes forcedType, Token* t) {
const int32_t specialCharPos = rdPos;
StringBuffer& str=*_str;
/* A segment of a "dotted" is not allowed to begin with another dot or a dash.
** Even though hosts, e-mail addresses, etc., could have a dotted-segment
** that begins with a dot or a dash, it's far more common in source text
** for a pattern like "abc.--def" to be intended as two tokens. */
int ch = rd->Peek();
if (!(DOT || DASH)) {
bool prevWasDot;
bool prevWasDash;
if (str.len == 0) {
prevWasDot = false;
prevWasDash = false;
} else {
prevWasDot = RIGHTMOST(str) == '.';
prevWasDash = RIGHTMOST(str) == '-';
}
while (!EOS && str.len < LUCENE_MAX_WORD_LEN-1 ) {
ch = readChar();
const bool dot = ch == '.';
const bool dash = ch == '-';
//const bool at = ch == '@';
if (!(ALNUM || UNDERSCORE || dot || dash)) {
break;
}
/* Multiple dots or dashes in succession end the token.
** Consider the following inputs:
** "Visit windowsupdate.microsoft.com--update today!"
** "In the U.S.A.--yes, even there!" */
if ((dot || dash) && (prevWasDot || prevWasDash)) {
/* We're not going to append the character we just read, in any case.
** As to the character before it (which is currently RIGHTMOST(str)):
** Unless RIGHTMOST(str) is a dot, in which we need to save it so the
** acronym-versus-host detection can work, we want to get rid of it. */
if (!prevWasDot) {
SHAVE_RIGHTMOST(str);
}
break;
}
str.appendChar(ch);
prevWasDot = dot;
prevWasDash = dash;
}
}
/* There's a potential StringBuffer.append call in the code above, which
** could cause str to reallocate its internal buffer. We must wait to
** obtain the optimization-oriented strBuf pointer until after the initial
** potentially realloc-triggering operations on str.
** Because there can be other such ops much later in this function, strBuf
** is guarded within a block to prevent its use during or after the calls
** that would potentially invalidate it. */
{ /* Begin block-guard of strBuf */
TCHAR* strBuf = str.getBuffer();
bool rightmostIsDot = RIGHTMOST_IS(str, '.');
if (CONSUMED_NOTHING_OF_VALUE) {
/* No more alphanums available for this token; shave trailing dot, if any. */
if (rightmostIsDot) {
SHAVE_RIGHTMOST(str);
}
/* If there are no dots remaining, this is a generic ALPHANUM. */
if (_tcschr(strBuf, '.') == NULL) {
forcedType = CL_NS2(analysis,standard)::ALPHANUM;
}
/* Check the token to see if it's an acronym. An acronym must have a
** letter in every even slot and a dot in every odd slot, including the
** last slot (for example, "U.S.A."). */
} else if (rightmostIsDot) {
bool isAcronym = true;
const int32_t upperCheckLimit = str.len - 1; /* -1 b/c we already checked the last slot. */
for (int32_t i = 0; i < upperCheckLimit; i++) {
const bool even = (i % 2 == 0);
ch = strBuf[i];
if ( (even && !ALPHA) || (!even && !DOT) ) {
isAcronym = false;
break;
}
}
if (isAcronym) {
forcedType = CL_NS2(analysis,standard)::ACRONYM;
} else {
/* If it's not an acronym, we don't want the trailing dot. */
SHAVE_RIGHTMOST(str);
/* If there are no dots remaining, this is a generic ALPHANUM. */
if (_tcschr(strBuf, '.') == NULL) {
forcedType = CL_NS2(analysis,standard)::ALPHANUM;
}
}
}
} /* End block-guard of strBuf */
// if (!EOS) {
// if (ch == '@' && str.len < LUCENE_MAX_WORD_LEN-1) {
// str.appendChar('@');
// return ReadAt(&str,t);
// } else {
// unReadChar();
// }
if (!EOS) {
unReadChar();
}
return setToken(t,&str,forcedType);
// return setToken(t,&str,CL_NS2(analysis,standard)::UNKNOWN
// ? forcedType : CL_NS2(analysis,standard)::HOST);
}
bool StandardTokenizer::ReadApostrophe(StringBuffer* _str, Token* t) {
StringBuffer& str=*_str;
TokenTypes tokenType = CL_NS2(analysis,standard)::APOSTROPHE;
const int32_t specialCharPos = rdPos;
int ch=0;
CONSUME_ALPHAS;
if (RIGHTMOST_IS(str, '\'') || CONSUMED_NOTHING_OF_VALUE) {
/* After the apostrophe, no more alphanums were available within this
** token; shave trailing apostrophe and revert to generic ALPHANUM. */
SHAVE_RIGHTMOST(str);
tokenType = CL_NS2(analysis,standard)::ALPHANUM;
}
if (!EOS) {
unReadChar();
}
return setToken(t,&str,tokenType);
}
bool StandardTokenizer::ReadAt(StringBuffer* str, Token* t) {
ReadDotted(str, CL_NS2(analysis,standard)::EMAIL,t);
/* JLucene grammar indicates dots/digits not allowed in company name: */
if (!CONTAINS_ANY((*str), ".0123456789")) {
setToken(t,str,CL_NS2(analysis,standard)::COMPANY);
}
return true;
}
bool StandardTokenizer::ReadCompany(StringBuffer* _str, Token* t) {
StringBuffer& str = *_str;
const int32_t specialCharPos = rdPos;
int ch=0;
CONSUME_WORD;
if (CONSUMED_NOTHING_OF_VALUE) {
/* After the ampersand, no more alphanums were available within this
** token; shave trailing ampersand and revert to ALPHANUM. */
CND_PRECONDITION(RIGHTMOST_IS(str, '&'),"ReadCompany failed");
SHAVE_RIGHTMOST(str);
return setToken(t,&str,CL_NS2(analysis,standard)::ALPHANUM);
}
if (!EOS) {
unReadChar();
}
return setToken(t,&str,CL_NS2(analysis,standard)::COMPANY);
}
CL_NS_END2