1 /*

     2 ** The author disclaims copyright to this source code.

     3 **

     4 *************************************************************************

     5 ** Implementation of the "simple" full-text-search tokenizer.

     6 */

     7 

     8 /*

     9 ** The code in this file is only compiled if:

    10 **

    11 **     * The FTS1 module is being built as an extension

    12 **       (in which case SQLITE_CORE is not defined), or

    13 **

    14 **     * The FTS1 module is being built into the core of

    15 **       SQLite (in which case SQLITE_ENABLE_FTS1 is defined).

    16 */

    17 #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS1)

    18 

    19 

    20 #include <assert.h>

    21 #include <stdlib.h>

    22 #include <stdio.h>

    23 #include <string.h>

    24 #include <ctype.h>

    25 

    26 #include "fts1_tokenizer.h"

    27 

    28 typedef struct simple_tokenizer {

    29   sqlite3_tokenizer base;

    30   char delim[128];             /* flag ASCII delimiters */

    31 } simple_tokenizer;

    32 

    33 typedef struct simple_tokenizer_cursor {

    34   sqlite3_tokenizer_cursor base;

    35   const char *pInput;          /* input we are tokenizing */

    36   int nBytes;                  /* size of the input */

    37   int iOffset;                 /* current position in pInput */

    38   int iToken;                  /* index of next token to be returned */

    39   char *pToken;                /* storage for current token */

    40   int nTokenAllocated;         /* space allocated to zToken buffer */

    41 } simple_tokenizer_cursor;

    42 

    43 

    44 /* Forward declaration */

    45 static const sqlite3_tokenizer_module simpleTokenizerModule;

    46 

    47 static int isDelim(simple_tokenizer *t, unsigned char c){

    48   return c<0x80 && t->delim[c];

    49 }

    50 

    51 /*

    52 ** Create a new tokenizer instance.

    53 */

    54 static int simpleCreate(

    55   int argc, const char * const *argv,

    56   sqlite3_tokenizer **ppTokenizer

    57 ){

    58   simple_tokenizer *t;

    59 

    60   t = (simple_tokenizer *) calloc(sizeof(*t), 1);

    61   if( t==NULL ) return SQLITE_NOMEM;

    62 

    63   /* TODO(shess) Delimiters need to remain the same from run to run,

    64   ** else we need to reindex.  One solution would be a meta-table to

    65   ** track such information in the database, then we'd only want this

    66   ** information on the initial create.

    67   */

    68   if( argc>1 ){

    69     int i, n = strlen(argv[1]);

    70     for(i=0; i<n; i++){

    71       unsigned char ch = argv[1][i];

    72       /* We explicitly don't support UTF-8 delimiters for now. */

    73       if( ch>=0x80 ){

    74         free(t);

    75         return SQLITE_ERROR;

    76       }

    77       t->delim[ch] = 1;

    78     }

    79   } else {

    80     /* Mark non-alphanumeric ASCII characters as delimiters */

    81     int i;

    82     for(i=1; i<0x80; i++){

    83       t->delim[i] = !isalnum(i);

    84     }

    85   }

    86 

    87   *ppTokenizer = &t->base;

    88   return SQLITE_OK;

    89 }

    90 

    91 /*

    92 ** Destroy a tokenizer

    93 */

    94 static int simpleDestroy(sqlite3_tokenizer *pTokenizer){

    95   free(pTokenizer);

    96   return SQLITE_OK;

    97 }

    98 

    99 /*

   100 ** Prepare to begin tokenizing a particular string.  The input

   101 ** string to be tokenized is pInput[0..nBytes-1].  A cursor

   102 ** used to incrementally tokenize this string is returned in 

   103 ** *ppCursor.

   104 */

   105 static int simpleOpen(

   106   sqlite3_tokenizer *pTokenizer,         /* The tokenizer */

   107   const char *pInput, int nBytes,        /* String to be tokenized */

   108   sqlite3_tokenizer_cursor **ppCursor    /* OUT: Tokenization cursor */

   109 ){

   110   simple_tokenizer_cursor *c;

   111 

   112   c = (simple_tokenizer_cursor *) malloc(sizeof(*c));

   113   if( c==NULL ) return SQLITE_NOMEM;

   114 

   115   c->pInput = pInput;

   116   if( pInput==0 ){

   117     c->nBytes = 0;

   118   }else if( nBytes<0 ){

   119     c->nBytes = (int)strlen(pInput);

   120   }else{

   121     c->nBytes = nBytes;

   122   }

   123   c->iOffset = 0;                 /* start tokenizing at the beginning */

   124   c->iToken = 0;

   125   c->pToken = NULL;               /* no space allocated, yet. */

   126   c->nTokenAllocated = 0;

   127 

   128   *ppCursor = &c->base;

   129   return SQLITE_OK;

   130 }

   131 

   132 /*

   133 ** Close a tokenization cursor previously opened by a call to

   134 ** simpleOpen() above.

   135 */

   136 static int simpleClose(sqlite3_tokenizer_cursor *pCursor){

   137   simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;

   138   free(c->pToken);

   139   free(c);

   140   return SQLITE_OK;

   141 }

   142 

   143 /*

   144 ** Extract the next token from a tokenization cursor.  The cursor must

   145 ** have been opened by a prior call to simpleOpen().

   146 */

   147 static int simpleNext(

   148   sqlite3_tokenizer_cursor *pCursor,  /* Cursor returned by simpleOpen */

   149   const char **ppToken,               /* OUT: *ppToken is the token text */

   150   int *pnBytes,                       /* OUT: Number of bytes in token */

   151   int *piStartOffset,                 /* OUT: Starting offset of token */

   152   int *piEndOffset,                   /* OUT: Ending offset of token */

   153   int *piPosition                     /* OUT: Position integer of token */

   154 ){

   155   simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;

   156   simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer;

   157   unsigned char *p = (unsigned char *)c->pInput;

   158 

   159   while( c->iOffset<c->nBytes ){

   160     int iStartOffset;

   161 

   162     /* Scan past delimiter characters */

   163     while( c->iOffset<c->nBytes && isDelim(t, p[c->iOffset]) ){

   164       c->iOffset++;

   165     }

   166 

   167     /* Count non-delimiter characters. */

   168     iStartOffset = c->iOffset;

   169     while( c->iOffset<c->nBytes && !isDelim(t, p[c->iOffset]) ){

   170       c->iOffset++;

   171     }

   172 

   173     if( c->iOffset>iStartOffset ){

   174       int i, n = c->iOffset-iStartOffset;

   175       if( n>c->nTokenAllocated ){

   176         c->nTokenAllocated = n+20;

   177         c->pToken = realloc(c->pToken, c->nTokenAllocated);

   178         if( c->pToken==NULL ) return SQLITE_NOMEM;

   179       }

   180       for(i=0; i<n; i++){

   181         /* TODO(shess) This needs expansion to handle UTF-8

   182         ** case-insensitivity.

   183         */

   184         unsigned char ch = p[iStartOffset+i];

   185         c->pToken[i] = ch<0x80 ? tolower(ch) : ch;

   186       }

   187       *ppToken = c->pToken;

   188       *pnBytes = n;

   189       *piStartOffset = iStartOffset;

   190       *piEndOffset = c->iOffset;

   191       *piPosition = c->iToken++;

   192 

   193       return SQLITE_OK;

   194     }

   195   }

   196   return SQLITE_DONE;

   197 }

   198 

   199 /*

   200 ** The set of routines that implement the simple tokenizer

   201 */

   202 static const sqlite3_tokenizer_module simpleTokenizerModule = {

   203   0,

   204   simpleCreate,

   205   simpleDestroy,

   206   simpleOpen,

   207   simpleClose,

   208   simpleNext,

   209 };

   210 

   211 /*

   212 ** Allocate a new simple tokenizer.  Return a pointer to the new

   213 ** tokenizer in *ppModule

   214 */

   215 void sqlite3Fts1SimpleTokenizerModule(

   216   sqlite3_tokenizer_module const**ppModule

   217 ){

   218   *ppModule = &simpleTokenizerModule;

   219 }

   220 

   221 #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS1) */