|
1 /* |
|
2 ********************************************************************** |
|
3 * Copyright (C) 1999-2005 IBM Corp. All rights reserved. |
|
4 ********************************************************************** |
|
5 * Date Name Description |
|
6 * 12/1/99 rgillam Complete port from Java. |
|
7 * 01/13/2000 helena Added UErrorCode to ctors. |
|
8 ********************************************************************** |
|
9 */ |
|
10 |
|
11 #ifndef DBBI_H |
|
12 #define DBBI_H |
|
13 |
|
14 #include "unicode/rbbi.h" |
|
15 |
|
16 #if !UCONFIG_NO_BREAK_ITERATION |
|
17 |
|
18 /** |
|
19 * \file |
|
20 * \brief C++ API: Dictionary Based Break Iterator |
|
21 */ |
|
22 |
|
23 U_NAMESPACE_BEGIN |
|
24 |
|
25 /* forward declaration */ |
|
26 class DictionaryBasedBreakIteratorTables; |
|
27 |
|
28 /** |
|
29 * A subclass of RuleBasedBreakIterator that adds the ability to use a dictionary |
|
30 * to further subdivide ranges of text beyond what is possible using just the |
|
31 * state-table-based algorithm. This is necessary, for example, to handle |
|
32 * word and line breaking in Thai, which doesn't use spaces between words. The |
|
33 * state-table-based algorithm used by RuleBasedBreakIterator is used to divide |
|
34 * up text as far as possible, and then contiguous ranges of letters are |
|
35 * repeatedly compared against a list of known words (i.e., the dictionary) |
|
36 * to divide them up into words. |
|
37 * |
|
38 * <p>Applications do not normally need to include this header.</p> |
|
39 * |
|
40 * <p>This class will probably be deprecated in a future release of ICU, and replaced |
|
41 * with a more flexible and capable dictionary based break iterator. This change |
|
42 * should be invisible to applications, because creation and use of instances of |
|
43 * DictionaryBasedBreakIterator is through the factories and abstract |
|
44 * API on class BreakIterator, which will remain stable.</p> |
|
45 * |
|
46 * <p>This class is not intended to be subclassed.</p> |
|
47 * |
|
48 * |
|
49 * DictionaryBasedBreakIterator uses the same rule language as RuleBasedBreakIterator, |
|
50 * but adds one more special substitution name: <dictionary>. This substitution |
|
51 * name is used to identify characters in words in the dictionary. The idea is that |
|
52 * if the iterator passes over a chunk of text that includes two or more characters |
|
53 * in a row that are included in <dictionary>, it goes back through that range and |
|
54 * derives additional break positions (if possible) using the dictionary. |
|
55 * |
|
56 * DictionaryBasedBreakIterator is also constructed with the filename of a dictionary |
|
57 * file. It follows a prescribed search path to locate the dictionary (right now, |
|
58 * it looks for it in /com/ibm/text/resources in each directory in the classpath, |
|
59 * and won't find it in JAR files, but this location is likely to change). The |
|
60 * dictionary file is in a serialized binary format. We have a very primitive (and |
|
61 * slow) BuildDictionaryFile utility for creating dictionary files, but aren't |
|
62 * currently making it public. Contact us for help. |
|
63 * <p> |
|
64 * <b> NOTE </b> The DictionaryBasedIterator class is still under development. The |
|
65 * APIs are not in stable condition yet. |
|
66 */ |
|
67 class U_COMMON_API DictionaryBasedBreakIterator : public RuleBasedBreakIterator { |
|
68 |
|
69 private: |
|
70 |
|
71 /** |
|
72 * when a range of characters is divided up using the dictionary, the break |
|
73 * positions that are discovered are stored here, preventing us from having |
|
74 * to use either the dictionary or the state table again until the iterator |
|
75 * leaves this range of text |
|
76 */ |
|
77 int32_t* cachedBreakPositions; |
|
78 |
|
79 /** |
|
80 * The number of elements in cachedBreakPositions |
|
81 */ |
|
82 int32_t numCachedBreakPositions; |
|
83 |
|
84 /** |
|
85 * if cachedBreakPositions is not null, this indicates which item in the |
|
86 * cache the current iteration position refers to |
|
87 */ |
|
88 int32_t positionInCache; |
|
89 |
|
90 DictionaryBasedBreakIteratorTables *fTables; |
|
91 |
|
92 /**======================================================================= |
|
93 * Create a dictionary based break boundary detection iterator. |
|
94 * @param tablesImage The location for the dictionary to be loaded into memory |
|
95 * @param dictionaryFilename The name of the dictionary file |
|
96 * @param status the error code status |
|
97 * @return A dictionary based break detection iterator. The UErrorCode& status |
|
98 * parameter is used to return status information to the user. |
|
99 * To check whether the construction succeeded or not, you should check |
|
100 * the value of U_SUCCESS(err). If you wish more detailed information, you |
|
101 * can check for informational error results which still indicate success. For example, |
|
102 * U_FILE_ACCESS_ERROR will be returned if the file does not exist. |
|
103 * The caller owns the returned object and is responsible for deleting it. |
|
104 ======================================================================= */ |
|
105 DictionaryBasedBreakIterator(UDataMemory* tablesImage, const char* dictionaryFilename, UErrorCode& status); |
|
106 |
|
107 public: |
|
108 //======================================================================= |
|
109 // boilerplate |
|
110 //======================================================================= |
|
111 |
|
112 /** |
|
113 * Destructor |
|
114 * @stable ICU 2.0 |
|
115 */ |
|
116 virtual ~DictionaryBasedBreakIterator(); |
|
117 |
|
118 /** |
|
119 * Default constructor. Creates an "empty" break iterator. |
|
120 * Such an iterator can subsequently be assigned to. |
|
121 * @return the newly created DictionaryBaseBreakIterator. |
|
122 * @stable ICU 2.0 |
|
123 */ |
|
124 DictionaryBasedBreakIterator(); |
|
125 |
|
126 /** |
|
127 * Copy constructor. |
|
128 * @param other The DictionaryBasedBreakIterator to be copied. |
|
129 * @return the newly created DictionaryBasedBreakIterator. |
|
130 * @stable ICU 2.0 |
|
131 */ |
|
132 DictionaryBasedBreakIterator(const DictionaryBasedBreakIterator &other); |
|
133 |
|
134 /** |
|
135 * Assignment operator. |
|
136 * @param that The object to be copied. |
|
137 * @return the newly set DictionaryBasedBreakIterator. |
|
138 * @stable ICU 2.0 |
|
139 */ |
|
140 DictionaryBasedBreakIterator& operator=(const DictionaryBasedBreakIterator& that); |
|
141 |
|
142 /** |
|
143 * Returns a newly-constructed RuleBasedBreakIterator with the same |
|
144 * behavior, and iterating over the same text, as this one. |
|
145 * @return Returns a newly-constructed RuleBasedBreakIterator. |
|
146 * @stable ICU 2.0 |
|
147 */ |
|
148 virtual BreakIterator* clone(void) const; |
|
149 |
|
150 //======================================================================= |
|
151 // BreakIterator overrides |
|
152 //======================================================================= |
|
153 /** |
|
154 * Advances the iterator backwards, to the last boundary preceding this one. |
|
155 * @return The position of the last boundary position preceding this one. |
|
156 * @stable ICU 2.0 |
|
157 */ |
|
158 virtual int32_t previous(void); |
|
159 |
|
160 /** |
|
161 * Sets the iterator to refer to the first boundary position following |
|
162 * the specified position. |
|
163 * @param offset The position from which to begin searching for a break position. |
|
164 * @return The position of the first break after the current position. |
|
165 * @stable ICU 2.0 |
|
166 */ |
|
167 virtual int32_t following(int32_t offset); |
|
168 |
|
169 /** |
|
170 * Sets the iterator to refer to the last boundary position before the |
|
171 * specified position. |
|
172 * @param offset The position to begin searching for a break from. |
|
173 * @return The position of the last boundary before the starting position. |
|
174 * @stable ICU 2.0 |
|
175 */ |
|
176 virtual int32_t preceding(int32_t offset); |
|
177 |
|
178 /** |
|
179 * Returns the class ID for this class. This is useful only for |
|
180 * comparing to a return value from getDynamicClassID(). For example: |
|
181 * |
|
182 * Base* polymorphic_pointer = createPolymorphicObject(); |
|
183 * if (polymorphic_pointer->getDynamicClassID() == |
|
184 * Derived::getStaticClassID()) ... |
|
185 * |
|
186 * @return The class ID for all objects of this class. |
|
187 * @stable ICU 2.0 |
|
188 */ |
|
189 static UClassID U_EXPORT2 getStaticClassID(void); |
|
190 |
|
191 /** |
|
192 * Returns a unique class ID POLYMORPHICALLY. Pure virtual override. |
|
193 * This method is to implement a simple version of RTTI, since not all |
|
194 * C++ compilers support genuine RTTI. Polymorphic operator==() and |
|
195 * clone() methods call this method. |
|
196 * |
|
197 * @return The class ID for this object. All objects of a |
|
198 * given class have the same class ID. Objects of |
|
199 * other classes have different class IDs. |
|
200 * @stable ICU 2.0 |
|
201 */ |
|
202 virtual UClassID getDynamicClassID(void) const; |
|
203 |
|
204 protected: |
|
205 //======================================================================= |
|
206 // implementation |
|
207 //======================================================================= |
|
208 /** |
|
209 * This method is the actual implementation of the next() method. All iteration |
|
210 * vectors through here. This method initializes the state machine to state 1 |
|
211 * and advances through the text character by character until we reach the end |
|
212 * of the text or the state machine transitions to state 0. We update our return |
|
213 * value every time the state machine passes through a possible end state. |
|
214 * @internal |
|
215 */ |
|
216 virtual int32_t handleNext(void); |
|
217 |
|
218 /** |
|
219 * removes the cache of break positions (usually in response to a change in |
|
220 * position of some sort) |
|
221 * @internal |
|
222 */ |
|
223 virtual void reset(void); |
|
224 |
|
225 /** |
|
226 * init Initialize a dbbi. Common routine for use by constructors. |
|
227 * @internal |
|
228 */ |
|
229 void init(); |
|
230 |
|
231 /** |
|
232 * @param stackBuffer user allocated space for the new clone. If NULL new memory will be allocated. |
|
233 * If buffer is not large enough, new memory will be allocated. |
|
234 * @param BufferSize reference to size of allocated space. |
|
235 * If BufferSize == 0, a sufficient size for use in cloning will |
|
236 * be returned ('pre-flighting') |
|
237 * If BufferSize is not enough for a stack-based safe clone, |
|
238 * new memory will be allocated. |
|
239 * @param status to indicate whether the operation went on smoothly or there were errors |
|
240 * An informational status value, U_SAFECLONE_ALLOCATED_ERROR, is used if any allocations were |
|
241 * necessary. |
|
242 * @return pointer to the new clone |
|
243 * @internal |
|
244 */ |
|
245 virtual BreakIterator * createBufferClone(void *stackBuffer, |
|
246 int32_t &BufferSize, |
|
247 UErrorCode &status); |
|
248 |
|
249 |
|
250 private: |
|
251 /** |
|
252 * This is the function that actually implements the dictionary-based |
|
253 * algorithm. Given the endpoints of a range of text, it uses the |
|
254 * dictionary to determine the positions of any boundaries in this |
|
255 * range. It stores all the boundary positions it discovers in |
|
256 * cachedBreakPositions so that we only have to do this work once |
|
257 * for each time we enter the range. |
|
258 * @param startPos The start position of a range of text |
|
259 * @param endPos The end position of a range of text |
|
260 * @param status The error code status |
|
261 */ |
|
262 void divideUpDictionaryRange(int32_t startPos, int32_t endPos, UErrorCode &status); |
|
263 |
|
264 |
|
265 /* |
|
266 * HSYS : Please revisit with Rich, the ctors of the DBBI class is currently |
|
267 * marked as private. |
|
268 */ |
|
269 friend class DictionaryBasedBreakIteratorTables; |
|
270 friend class BreakIterator; |
|
271 }; |
|
272 |
|
273 U_NAMESPACE_END |
|
274 |
|
275 #endif /* #if !UCONFIG_NO_BREAK_ITERATION */ |
|
276 |
|
277 #endif |