1 /*

     2 **********************************************************************

     3 *   Copyright (C) 1999-2004, International Business Machines

     4 *   Corporation and others.  All Rights Reserved.

     5 **********************************************************************

     6 *

     7 *   uconv_cnv.h:

     8 *   defines all the low level conversion functions

     9 *   T_UnicodeConverter_{to,from}Unicode_$ConversionType

    10 *

    11 * Modification History:

    12 *

    13 *   Date        Name        Description

    14 *   05/09/00    helena      Added implementation to handle fallback mappings.

    15 *   06/29/2000  helena      Major rewrite of the callback APIs.

    16 */

    17 

    18 #ifndef UCNV_CNV_H

    19 #define UCNV_CNV_H

    20 

    21 #include "unicode/utypes.h"

    22 

    23 #if !UCONFIG_NO_CONVERSION

    24 

    25 #include "unicode/ucnv.h"

    26 #include "unicode/ucnv_err.h"

    27 #include "unicode/uset.h"

    28 #include "uset_imp.h"

    29 

    30 U_CDECL_BEGIN

    31 

    32 /* this is used in fromUnicode DBCS tables as an "unassigned" marker */

    33 #define missingCharMarker 0xFFFF

    34 

    35 /*

    36  * #define missingUCharMarker 0xfffe

    37  *

    38  * commented out because there are actually two values used in toUnicode tables:

    39  * U+fffe "unassigned"

    40  * U+ffff "illegal"

    41  */

    42 

    43 /** Forward declaration, see ucnv_bld.h */

    44 struct UConverterSharedData;

    45 typedef struct UConverterSharedData UConverterSharedData;

    46 

    47 /* function types for UConverterImpl ---------------------------------------- */

    48 

    49 /* struct with arguments for UConverterLoad and ucnv_load() */

    50 typedef struct {

    51     int32_t size;               /* sizeof(UConverterLoadArgs) */

    52     int32_t nestedLoads;        /* count nested ucnv_load() calls */

    53     int32_t reserved;           /* reserved - for good alignment of the pointers */

    54     uint32_t options;

    55     const char *pkg, *name;

    56 } UConverterLoadArgs;

    57 

    58 typedef void (*UConverterLoad) (UConverterSharedData *sharedData,

    59                                 UConverterLoadArgs *pArgs,

    60                                 const uint8_t *raw, UErrorCode *pErrorCode);

    61 typedef void (*UConverterUnload) (UConverterSharedData *sharedData);

    62 

    63 typedef void (*UConverterOpen) (UConverter *cnv, const char *name, const char *locale,uint32_t options, UErrorCode *pErrorCode);

    64 typedef void (*UConverterClose) (UConverter *cnv);

    65 

    66 typedef enum UConverterResetChoice {

    67     UCNV_RESET_BOTH,

    68     UCNV_RESET_TO_UNICODE,

    69     UCNV_RESET_FROM_UNICODE

    70 } UConverterResetChoice;

    71 

    72 typedef void (*UConverterReset) (UConverter *cnv, UConverterResetChoice choice);

    73 

    74 /*

    75  * Converter implementation function(s) for ucnv_toUnicode().

    76  * If the toUnicodeWithOffsets function pointer is NULL,

    77  * then the toUnicode function will be used and the offsets will be set to -1.

    78  *

    79  * Must maintain state across buffers. Use toUBytes[toULength] for partial input

    80  * sequences; it will be checked in ucnv.c at the end of the input stream

    81  * to detect truncated input.

    82  * Some converters may need additional detection and may then set U_TRUNCATED_CHAR_FOUND.

    83  *

    84  * The toUnicodeWithOffsets must write exactly as many offset values as target

    85  * units. Write offset values of -1 for when the source index corresponding to

    86  * the output unit is not known (e.g., the character started in an earlier buffer).

    87  * The pArgs->offsets pointer need not be moved forward.

    88  *

    89  * At function return, either one of the following conditions must be true:

    90  * - U_BUFFER_OVERFLOW_ERROR and the target is full: target==targetLimit

    91  * - another error code with toUBytes[toULength] set to the offending input

    92  * - no error, and the source is consumed: source==sourceLimit

    93  *

    94  * The ucnv.c code will handle the end of the input (reset)

    95  * (reset, and truncation detection) and callbacks.

    96  */

    97 typedef void (*UConverterToUnicode) (UConverterToUnicodeArgs *, UErrorCode *);

    98 

    99 /*

   100  * Same rules as for UConverterToUnicode.

   101  * A lead surrogate is kept in fromUChar32 across buffers, and if an error

   102  * occurs, then the offending input code point must be put into fromUChar32

   103  * as well.

   104  */

   105 typedef void (*UConverterFromUnicode) (UConverterFromUnicodeArgs *, UErrorCode *);

   106 

   107 /*

   108  * Converter implementation function for ucnv_getNextUChar().

   109  * If the function pointer is NULL, then the toUnicode function will be used.

   110  *

   111  * Will be called at a character boundary (toULength==0).

   112  * May return with

   113  * - U_INDEX_OUTOFBOUNDS_ERROR if there was no output for the input

   114  *   (the return value will be ignored)

   115  * - U_TRUNCATED_CHAR_FOUND or another error code (never U_BUFFER_OVERFLOW_ERROR!)

   116  *   with toUBytes[toULength] set to the offending input

   117  *   (the return value will be ignored)

   118  * - return UCNV_GET_NEXT_UCHAR_USE_TO_U, without moving the source pointer,

   119  *   to indicate that the ucnv.c code shall call the toUnicode function instead

   120  * - return a real code point result

   121  *

   122  * Unless UCNV_GET_NEXT_UCHAR_USE_TO_U is returned, the source bytes must be consumed.

   123  *

   124  * The ucnv.c code will handle the end of the input (reset)

   125  * (except for truncation detection!) and callbacks.

   126  */

   127 typedef UChar32 (*UConverterGetNextUChar) (UConverterToUnicodeArgs *, UErrorCode *);

   128 

   129 typedef void (*UConverterGetStarters)(const UConverter* converter,

   130                                       UBool starters[256],

   131                                       UErrorCode *pErrorCode);

   132 

   133 /* If this function pointer is null or if the function returns null

   134  * the name field in static data struct should be returned by 

   135  * ucnv_getName() API function

   136  */

   137 typedef const char * (*UConverterGetName) (const UConverter *cnv);

   138 

   139 /**

   140  * Write the codepage substitution character.

   141  * If this function is not set, then ucnv_cbFromUWriteSub() writes

   142  * the substitution character from UConverter.

   143  * For stateful converters, it is typically necessary to handle this

   144  * specificially for the converter in order to properly maintain the state.

   145  */

   146 typedef void (*UConverterWriteSub) (UConverterFromUnicodeArgs *pArgs, int32_t offsetIndex, UErrorCode *pErrorCode);

   147 

   148 /**

   149  * For converter-specific safeClone processing

   150  * If this function is not set, then ucnv_safeClone assumes that the converter has no private data that changes

   151  * after the converter is done opening.

   152  * If this function is set, then it is called just after a memcpy() of

   153  * converter data to the new, empty converter, and is expected to set up

   154  * the initial state of the converter.  It is not expected to increment the

   155  * reference counts of the standard data types such as the shared data.

   156  */

   157 typedef UConverter * (*UConverterSafeClone) (const UConverter   *cnv, 

   158                                              void               *stackBuffer,

   159                                              int32_t            *pBufferSize, 

   160                                              UErrorCode         *status);

   161 

   162 /**

   163  * Fills the set of Unicode code points that can be converted by an ICU converter.

   164  * The API function ucnv_getUnicodeSet() clears the USet before calling

   165  * the converter's getUnicodeSet() implementation; the converter should only

   166  * add the appropriate code points to allow recursive use.

   167  * For example, the ISO-2022-JP converter will call each subconverter's

   168  * getUnicodeSet() implementation to consecutively add code points to

   169  * the same USet, which will result in a union of the sets of all subconverters.

   170  *

   171  * For more documentation, see ucnv_getUnicodeSet() in ucnv.h.

   172  */

   173 typedef void (*UConverterGetUnicodeSet) (const UConverter *cnv,

   174                                          const USetAdder *sa,

   175                                          UConverterUnicodeSet which,

   176                                          UErrorCode *pErrorCode);

   177 

   178 UBool CONVERSION_U_SUCCESS (UErrorCode err);

   179 

   180 /**

   181  * UConverterImpl contains all the data and functions for a converter type.

   182  * Its function pointers work much like a C++ vtable.

   183  * Many converter types need to define only a subset of the functions;

   184  * when a function pointer is NULL, then a default action will be performed.

   185  *

   186  * Every converter type must implement toUnicode, fromUnicode, and getNextUChar,

   187  * otherwise the converter may crash.

   188  * Every converter type that has variable-length codepage sequences should

   189  * also implement toUnicodeWithOffsets and fromUnicodeWithOffsets for

   190  * correct offset handling.

   191  * All other functions may or may not be implemented - it depends only on

   192  * whether the converter type needs them.

   193  *

   194  * When open() fails, then close() will be called, if present.

   195  */

   196 struct UConverterImpl {

   197     UConverterType type;

   198 

   199     UConverterLoad load;

   200     UConverterUnload unload;

   201 

   202     UConverterOpen open;

   203     UConverterClose close;

   204     UConverterReset reset;

   205 

   206     UConverterToUnicode toUnicode;

   207     UConverterToUnicode toUnicodeWithOffsets;

   208     UConverterFromUnicode fromUnicode;

   209     UConverterFromUnicode fromUnicodeWithOffsets;

   210     UConverterGetNextUChar getNextUChar;

   211 

   212     UConverterGetStarters getStarters;

   213     UConverterGetName getName;

   214     UConverterWriteSub writeSub;

   215     UConverterSafeClone safeClone;

   216     UConverterGetUnicodeSet getUnicodeSet;

   217 };

   218 

   219 extern const UConverterSharedData

   220     _MBCSData, _Latin1Data,

   221     _UTF8Data, _UTF16BEData, _UTF16LEData, _UTF32BEData, _UTF32LEData,

   222     _ISO2022Data, 

   223     _LMBCSData1,_LMBCSData2, _LMBCSData3, _LMBCSData4, _LMBCSData5, _LMBCSData6,

   224     _LMBCSData8,_LMBCSData11,_LMBCSData16,_LMBCSData17,_LMBCSData18,_LMBCSData19,

   225     _HZData,_ISCIIData, _SCSUData, _ASCIIData,

   226     _UTF7Data, _Bocu1Data, _UTF16Data, _UTF32Data, _CESU8Data, _IMAPData;

   227 

   228 U_CDECL_END

   229 

   230 /** Always use fallbacks from codepage to Unicode */

   231 #define TO_U_USE_FALLBACK(useFallback) TRUE

   232 #define UCNV_TO_U_USE_FALLBACK(cnv) TRUE

   233 

   234 /** Use fallbacks from Unicode to codepage when cnv->useFallback or for private-use code points */

   235 #define IS_PRIVATE_USE(c) ((uint32_t)((c)-0xe000)<0x1900 || (uint32_t)((c)-0xf0000)<0x20000)

   236 #define FROM_U_USE_FALLBACK(useFallback, c) ((useFallback) || IS_PRIVATE_USE(c))

   237 #define UCNV_FROM_U_USE_FALLBACK(cnv, c) FROM_U_USE_FALLBACK((cnv)->useFallback, c)

   238 

   239 /**

   240  * Magic number for ucnv_getNextUChar(), returned by a

   241  * getNextUChar() implementation to indicate to use the converter's toUnicode()

   242  * instead of the native function.

   243  * @internal

   244  */

   245 #define UCNV_GET_NEXT_UCHAR_USE_TO_U -9

   246 

   247 U_CFUNC void

   248 ucnv_getCompleteUnicodeSet(const UConverter *cnv,

   249                    const USetAdder *sa,

   250                    UConverterUnicodeSet which,

   251                    UErrorCode *pErrorCode);

   252 

   253 U_CFUNC void

   254 ucnv_getNonSurrogateUnicodeSet(const UConverter *cnv,

   255                                const USetAdder *sa,

   256                                UConverterUnicodeSet which,

   257                                UErrorCode *pErrorCode);

   258 

   259 U_CFUNC void

   260 ucnv_fromUWriteBytes(UConverter *cnv,

   261                      const char *bytes, int32_t length,

   262                      char **target, const char *targetLimit,

   263                      int32_t **offsets,

   264                      int32_t sourceIndex,

   265                      UErrorCode *pErrorCode);

   266 U_CFUNC void

   267 ucnv_toUWriteUChars(UConverter *cnv,

   268                     const UChar *uchars, int32_t length,

   269                     UChar **target, const UChar *targetLimit,

   270                     int32_t **offsets,

   271                     int32_t sourceIndex,

   272                     UErrorCode *pErrorCode);

   273 

   274 U_CFUNC void

   275 ucnv_toUWriteCodePoint(UConverter *cnv,

   276                        UChar32 c,

   277                        UChar **target, const UChar *targetLimit,

   278                        int32_t **offsets,

   279                        int32_t sourceIndex,

   280                        UErrorCode *pErrorCode);

   281 

   282 #endif

   283 

   284 #endif /* UCNV_CNV */