|
1 /* |
|
2 ** 2006 July 10 |
|
3 ** |
|
4 ** The author disclaims copyright to this source code. |
|
5 ** |
|
6 ************************************************************************* |
|
7 ** Defines the interface to tokenizers used by fulltext-search. There |
|
8 ** are three basic components: |
|
9 ** |
|
10 ** sqlite3_tokenizer_module is a singleton defining the tokenizer |
|
11 ** interface functions. This is essentially the class structure for |
|
12 ** tokenizers. |
|
13 ** |
|
14 ** sqlite3_tokenizer is used to define a particular tokenizer, perhaps |
|
15 ** including customization information defined at creation time. |
|
16 ** |
|
17 ** sqlite3_tokenizer_cursor is generated by a tokenizer to generate |
|
18 ** tokens from a particular input. |
|
19 */ |
|
20 #ifndef _FTS1_TOKENIZER_H_ |
|
21 #define _FTS1_TOKENIZER_H_ |
|
22 |
|
23 /* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time. |
|
24 ** If tokenizers are to be allowed to call sqlite3_*() functions, then |
|
25 ** we will need a way to register the API consistently. |
|
26 */ |
|
27 #include "sqlite3.h" |
|
28 |
|
29 /* |
|
30 ** Structures used by the tokenizer interface. |
|
31 */ |
|
32 typedef struct sqlite3_tokenizer sqlite3_tokenizer; |
|
33 typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor; |
|
34 typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module; |
|
35 |
|
36 struct sqlite3_tokenizer_module { |
|
37 int iVersion; /* currently 0 */ |
|
38 |
|
39 /* |
|
40 ** Create and destroy a tokenizer. argc/argv are passed down from |
|
41 ** the fulltext virtual table creation to allow customization. |
|
42 */ |
|
43 int (*xCreate)(int argc, const char *const*argv, |
|
44 sqlite3_tokenizer **ppTokenizer); |
|
45 int (*xDestroy)(sqlite3_tokenizer *pTokenizer); |
|
46 |
|
47 /* |
|
48 ** Tokenize a particular input. Call xOpen() to prepare to |
|
49 ** tokenize, xNext() repeatedly until it returns SQLITE_DONE, then |
|
50 ** xClose() to free any internal state. The pInput passed to |
|
51 ** xOpen() must exist until the cursor is closed. The ppToken |
|
52 ** result from xNext() is only valid until the next call to xNext() |
|
53 ** or until xClose() is called. |
|
54 */ |
|
55 /* TODO(shess) current implementation requires pInput to be |
|
56 ** nul-terminated. This should either be fixed, or pInput/nBytes |
|
57 ** should be converted to zInput. |
|
58 */ |
|
59 int (*xOpen)(sqlite3_tokenizer *pTokenizer, |
|
60 const char *pInput, int nBytes, |
|
61 sqlite3_tokenizer_cursor **ppCursor); |
|
62 int (*xClose)(sqlite3_tokenizer_cursor *pCursor); |
|
63 int (*xNext)(sqlite3_tokenizer_cursor *pCursor, |
|
64 const char **ppToken, int *pnBytes, |
|
65 int *piStartOffset, int *piEndOffset, int *piPosition); |
|
66 }; |
|
67 |
|
68 struct sqlite3_tokenizer { |
|
69 const sqlite3_tokenizer_module *pModule; /* The module for this tokenizer */ |
|
70 /* Tokenizer implementations will typically add additional fields */ |
|
71 }; |
|
72 |
|
73 struct sqlite3_tokenizer_cursor { |
|
74 sqlite3_tokenizer *pTokenizer; /* Tokenizer for this cursor. */ |
|
75 /* Tokenizer implementations will typically add additional fields */ |
|
76 }; |
|
77 |
|
78 /* |
|
79 ** Get the module for a tokenizer which generates tokens based on a |
|
80 ** set of non-token characters. The default is to break tokens at any |
|
81 ** non-alnum character, though the set of delimiters can also be |
|
82 ** specified by the first argv argument to xCreate(). |
|
83 */ |
|
84 /* TODO(shess) This doesn't belong here. Need some sort of |
|
85 ** registration process. |
|
86 */ |
|
87 void sqlite3Fts1SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule); |
|
88 void sqlite3Fts1PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule); |
|
89 |
|
90 #endif /* _FTS1_TOKENIZER_H_ */ |