/*

** 2006 Oct 10

**

** The author disclaims copyright to this source code.  In place of

** a legal notice, here is a blessing:

**

**    May you do good and not evil.

**    May you find forgiveness for yourself and forgive others.

**    May you share freely, never taking more than you give.

**

******************************************************************************

**

** Implementation of the "simple" full-text-search tokenizer.

*/

/*

** The code in this file is only compiled if:

**

**     * The FTS3 module is being built as an extension

**       (in which case SQLITE_CORE is not defined), or

**

**     * The FTS3 module is being built into the core of

**       SQLite (in which case SQLITE_ENABLE_FTS3 is defined).

*/

#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)

#include <assert.h>

#include <stdlib.h>

#include <stdio.h>

#include <string.h>

#include <ctype.h>

#include "fts3_tokenizer.h"

typedef struct simple_tokenizer {

  sqlite3_tokenizer base;

  char delim[128];             /* flag ASCII delimiters */

} simple_tokenizer;

typedef struct simple_tokenizer_cursor {

  sqlite3_tokenizer_cursor base;

  const char *pInput;          /* input we are tokenizing */

  int nBytes;                  /* size of the input */

  int iOffset;                 /* current position in pInput */

  int iToken;                  /* index of next token to be returned */

  char *pToken;                /* storage for current token */

  int nTokenAllocated;         /* space allocated to zToken buffer */

} simple_tokenizer_cursor;

/* Forward declaration */

static const sqlite3_tokenizer_module simpleTokenizerModule;

static int simpleDelim(simple_tokenizer *t, unsigned char c){

  return c<0x80 && t->delim[c];

}

/*

** Create a new tokenizer instance.

*/

static int simpleCreate(

  int argc, const char * const *argv,

  sqlite3_tokenizer **ppTokenizer

){

  simple_tokenizer *t;

  t = (simple_tokenizer *) sqlite3_malloc(sizeof(*t));

  if( t==NULL ) return SQLITE_NOMEM;

  memset(t, 0, sizeof(*t));

  /* TODO(shess) Delimiters need to remain the same from run to run,

  ** else we need to reindex.  One solution would be a meta-table to

  ** track such information in the database, then we'd only want this

  ** information on the initial create.

  */

  if( argc>1 ){

    int i, n = strlen(argv[1]);

    for(i=0; i<n; i++){

      unsigned char ch = argv[1][i];

      /* We explicitly don't support UTF-8 delimiters for now. */

      if( ch>=0x80 ){

        sqlite3_free(t);

        return SQLITE_ERROR;

      }

      t->delim[ch] = 1;

    }

  } else {

    /* Mark non-alphanumeric ASCII characters as delimiters */

    int i;

    for(i=1; i<0x80; i++){

      t->delim[i] = !isalnum(i);

    }

  }

  *ppTokenizer = &t->base;

  return SQLITE_OK;

}

/*

** Destroy a tokenizer

*/

static int simpleDestroy(sqlite3_tokenizer *pTokenizer){

  sqlite3_free(pTokenizer);

  return SQLITE_OK;

}

/*

** Prepare to begin tokenizing a particular string.  The input

** string to be tokenized is pInput[0..nBytes-1].  A cursor

** used to incrementally tokenize this string is returned in 

** *ppCursor.

*/

static int simpleOpen(

  sqlite3_tokenizer *pTokenizer,         /* The tokenizer */

  const char *pInput, int nBytes,        /* String to be tokenized */

  sqlite3_tokenizer_cursor **ppCursor    /* OUT: Tokenization cursor */

){

  simple_tokenizer_cursor *c;

  c = (simple_tokenizer_cursor *) sqlite3_malloc(sizeof(*c));

  if( c==NULL ) return SQLITE_NOMEM;

  c->pInput = pInput;

  if( pInput==0 ){

    c->nBytes = 0;

  }else if( nBytes<0 ){

    c->nBytes = (int)strlen(pInput);

  }else{

    c->nBytes = nBytes;

  }

  c->iOffset = 0;                 /* start tokenizing at the beginning */

  c->iToken = 0;

  c->pToken = NULL;               /* no space allocated, yet. */

  c->nTokenAllocated = 0;

  *ppCursor = &c->base;

  return SQLITE_OK;

}

/*

** Close a tokenization cursor previously opened by a call to

** simpleOpen() above.

*/

static int simpleClose(sqlite3_tokenizer_cursor *pCursor){

  simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;

  sqlite3_free(c->pToken);

  sqlite3_free(c);

  return SQLITE_OK;

}

/*

** Extract the next token from a tokenization cursor.  The cursor must

** have been opened by a prior call to simpleOpen().

*/

static int simpleNext(

  sqlite3_tokenizer_cursor *pCursor,  /* Cursor returned by simpleOpen */

  const char **ppToken,               /* OUT: *ppToken is the token text */

  int *pnBytes,                       /* OUT: Number of bytes in token */

  int *piStartOffset,                 /* OUT: Starting offset of token */

  int *piEndOffset,                   /* OUT: Ending offset of token */

  int *piPosition                     /* OUT: Position integer of token */

){

  simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;

  simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer;

  unsigned char *p = (unsigned char *)c->pInput;

  while( c->iOffset<c->nBytes ){

    int iStartOffset;

    /* Scan past delimiter characters */

    while( c->iOffset<c->nBytes && simpleDelim(t, p[c->iOffset]) ){

      c->iOffset++;

    }

    /* Count non-delimiter characters. */

    iStartOffset = c->iOffset;

    while( c->iOffset<c->nBytes && !simpleDelim(t, p[c->iOffset]) ){

      c->iOffset++;

    }

    if( c->iOffset>iStartOffset ){

      int i, n = c->iOffset-iStartOffset;

      if( n>c->nTokenAllocated ){

        c->nTokenAllocated = n+20;

        c->pToken = sqlite3_realloc(c->pToken, c->nTokenAllocated);

        if( c->pToken==NULL ) return SQLITE_NOMEM;

      }

      for(i=0; i<n; i++){

        /* TODO(shess) This needs expansion to handle UTF-8

        ** case-insensitivity.

        */

        unsigned char ch = p[iStartOffset+i];

        c->pToken[i] = ch<0x80 ? tolower(ch) : ch;

      }

      *ppToken = c->pToken;

      *pnBytes = n;

      *piStartOffset = iStartOffset;

      *piEndOffset = c->iOffset;

      *piPosition = c->iToken++;

      return SQLITE_OK;

    }

  }

  return SQLITE_DONE;

}

/*

** The set of routines that implement the simple tokenizer

*/

static const sqlite3_tokenizer_module simpleTokenizerModule = {

  0,

  simpleCreate,

  simpleDestroy,

  simpleOpen,

  simpleClose,

  simpleNext,

};

/*

** Allocate a new simple tokenizer.  Return a pointer to the new

** tokenizer in *ppModule

*/

void sqlite3Fts3SimpleTokenizerModule(

  sqlite3_tokenizer_module const**ppModule

){

  *ppModule = &simpleTokenizerModule;

}

#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */