--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/persistentstorage/sqlite3api/SQLite/fts3_icu.c Fri Jan 22 11:06:30 2010 +0200
@@ -0,0 +1,260 @@
+/*
+** 2007 June 22
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file implements a tokenizer for fts3 based on the ICU library.
+**
+** $Id: fts3_icu.c,v 1.3 2008/09/01 18:34:20 danielk1977 Exp $
+*/
+
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
+#ifdef SQLITE_ENABLE_ICU
+
+#include <assert.h>
+#include <string.h>
+#include "fts3_tokenizer.h"
+
+#include <unicode/ubrk.h>
+#include <unicode/ucol.h>
+#include <unicode/ustring.h>
+#include <unicode/utf16.h>
+
+typedef struct IcuTokenizer IcuTokenizer;
+typedef struct IcuCursor IcuCursor;
+
+struct IcuTokenizer {
+ sqlite3_tokenizer base;
+ char *zLocale;
+};
+
+struct IcuCursor {
+ sqlite3_tokenizer_cursor base;
+
+ UBreakIterator *pIter; /* ICU break-iterator object */
+ int nChar; /* Number of UChar elements in pInput */
+ UChar *aChar; /* Copy of input using utf-16 encoding */
+ int *aOffset; /* Offsets of each character in utf-8 input */
+
+ int nBuffer;
+ char *zBuffer;
+
+ int iToken;
+};
+
+/*
+** Create a new tokenizer instance.
+*/
+static int icuCreate(
+ int argc, /* Number of entries in argv[] */
+ const char * const *argv, /* Tokenizer creation arguments */
+ sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */
+){
+ IcuTokenizer *p;
+ int n = 0;
+
+ if( argc>0 ){
+ n = strlen(argv[0])+1;
+ }
+ p = (IcuTokenizer *)sqlite3_malloc(sizeof(IcuTokenizer)+n);
+ if( !p ){
+ return SQLITE_NOMEM;
+ }
+ memset(p, 0, sizeof(IcuTokenizer));
+
+ if( n ){
+ p->zLocale = (char *)&p[1];
+ memcpy(p->zLocale, argv[0], n);
+ }
+
+ *ppTokenizer = (sqlite3_tokenizer *)p;
+
+ return SQLITE_OK;
+}
+
+/*
+** Destroy a tokenizer
+*/
+static int icuDestroy(sqlite3_tokenizer *pTokenizer){
+ IcuTokenizer *p = (IcuTokenizer *)pTokenizer;
+ sqlite3_free(p);
+ return SQLITE_OK;
+}
+
+/*
+** Prepare to begin tokenizing a particular string. The input
+** string to be tokenized is pInput[0..nBytes-1]. A cursor
+** used to incrementally tokenize this string is returned in
+** *ppCursor.
+*/
+static int icuOpen(
+ sqlite3_tokenizer *pTokenizer, /* The tokenizer */
+ const char *zInput, /* Input string */
+ int nInput, /* Length of zInput in bytes */
+ sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */
+){
+ IcuTokenizer *p = (IcuTokenizer *)pTokenizer;
+ IcuCursor *pCsr;
+
+ const int32_t opt = U_FOLD_CASE_DEFAULT;
+ UErrorCode status = U_ZERO_ERROR;
+ int nChar;
+
+ UChar32 c;
+ int iInput = 0;
+ int iOut = 0;
+
+ *ppCursor = 0;
+
+ if( nInput<0 ){
+ nInput = strlen(zInput);
+ }
+ nChar = nInput+1;
+ pCsr = (IcuCursor *)sqlite3_malloc(
+ sizeof(IcuCursor) + /* IcuCursor */
+ nChar * sizeof(UChar) + /* IcuCursor.aChar[] */
+ (nChar+1) * sizeof(int) /* IcuCursor.aOffset[] */
+ );
+ if( !pCsr ){
+ return SQLITE_NOMEM;
+ }
+ memset(pCsr, 0, sizeof(IcuCursor));
+ pCsr->aChar = (UChar *)&pCsr[1];
+ pCsr->aOffset = (int *)&pCsr->aChar[nChar];
+
+ pCsr->aOffset[iOut] = iInput;
+ U8_NEXT(zInput, iInput, nInput, c);
+ while( c>0 ){
+ int isError = 0;
+ c = u_foldCase(c, opt);
+ U16_APPEND(pCsr->aChar, iOut, nChar, c, isError);
+ if( isError ){
+ sqlite3_free(pCsr);
+ return SQLITE_ERROR;
+ }
+ pCsr->aOffset[iOut] = iInput;
+
+ if( iInput<nInput ){
+ U8_NEXT(zInput, iInput, nInput, c);
+ }else{
+ c = 0;
+ }
+ }
+
+ pCsr->pIter = ubrk_open(UBRK_WORD, p->zLocale, pCsr->aChar, iOut, &status);
+ if( !U_SUCCESS(status) ){
+ sqlite3_free(pCsr);
+ return SQLITE_ERROR;
+ }
+ pCsr->nChar = iOut;
+
+ ubrk_first(pCsr->pIter);
+ *ppCursor = (sqlite3_tokenizer_cursor *)pCsr;
+ return SQLITE_OK;
+}
+
+/*
+** Close a tokenization cursor previously opened by a call to icuOpen().
+*/
+static int icuClose(sqlite3_tokenizer_cursor *pCursor){
+ IcuCursor *pCsr = (IcuCursor *)pCursor;
+ ubrk_close(pCsr->pIter);
+ sqlite3_free(pCsr->zBuffer);
+ sqlite3_free(pCsr);
+ return SQLITE_OK;
+}
+
+/*
+** Extract the next token from a tokenization cursor.
+*/
+static int icuNext(
+ sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by simpleOpen */
+ const char **ppToken, /* OUT: *ppToken is the token text */
+ int *pnBytes, /* OUT: Number of bytes in token */
+ int *piStartOffset, /* OUT: Starting offset of token */
+ int *piEndOffset, /* OUT: Ending offset of token */
+ int *piPosition /* OUT: Position integer of token */
+){
+ IcuCursor *pCsr = (IcuCursor *)pCursor;
+
+ int iStart = 0;
+ int iEnd = 0;
+ int nByte = 0;
+
+ while( iStart==iEnd ){
+ UChar32 c;
+
+ iStart = ubrk_current(pCsr->pIter);
+ iEnd = ubrk_next(pCsr->pIter);
+ if( iEnd==UBRK_DONE ){
+ return SQLITE_DONE;
+ }
+
+ while( iStart<iEnd ){
+ int iWhite = iStart;
+ U8_NEXT(pCsr->aChar, iWhite, pCsr->nChar, c);
+ if( u_isspace(c) ){
+ iStart = iWhite;
+ }else{
+ break;
+ }
+ }
+ assert(iStart<=iEnd);
+ }
+
+ do {
+ UErrorCode status = U_ZERO_ERROR;
+ if( nByte ){
+ char *zNew = sqlite3_realloc(pCsr->zBuffer, nByte);
+ if( !zNew ){
+ return SQLITE_NOMEM;
+ }
+ pCsr->zBuffer = zNew;
+ pCsr->nBuffer = nByte;
+ }
+
+ u_strToUTF8(
+ pCsr->zBuffer, pCsr->nBuffer, &nByte, /* Output vars */
+ &pCsr->aChar[iStart], iEnd-iStart, /* Input vars */
+ &status /* Output success/failure */
+ );
+ } while( nByte>pCsr->nBuffer );
+
+ *ppToken = pCsr->zBuffer;
+ *pnBytes = nByte;
+ *piStartOffset = pCsr->aOffset[iStart];
+ *piEndOffset = pCsr->aOffset[iEnd];
+ *piPosition = pCsr->iToken++;
+
+ return SQLITE_OK;
+}
+
+/*
+** The set of routines that implement the simple tokenizer
+*/
+static const sqlite3_tokenizer_module icuTokenizerModule = {
+ 0, /* iVersion */
+ icuCreate, /* xCreate */
+ icuDestroy, /* xCreate */
+ icuOpen, /* xOpen */
+ icuClose, /* xClose */
+ icuNext, /* xNext */
+};
+
+/*
+** Set *ppModule to point at the implementation of the ICU tokenizer.
+*/
+void sqlite3Fts3IcuTokenizerModule(
+ sqlite3_tokenizer_module const**ppModule
+){
+ *ppModule = &icuTokenizerModule;
+}
+
+#endif /* defined(SQLITE_ENABLE_ICU) */
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */