1 /*

     2 *******************************************************************************

     3 *

     4 *   Copyright (C) 1999-2004, International Business Machines

     5 *   Corporation and others.  All Rights Reserved.

     6 *

     7 *******************************************************************************

     8 *   file name:  utf.h

     9 *   encoding:   US-ASCII

    10 *   tab size:   8 (not used)

    11 *   indentation:4

    12 *

    13 *   created on: 1999sep09

    14 *   created by: Markus W. Scherer

    15 */

    16 

    17 /**

    18  * \file

    19  * \brief C API: Code point macros

    20  *

    21  * This file defines macros for checking whether a code point is

    22  * a surrogate or a non-character etc.

    23  *

    24  * The UChar and UChar32 data types for Unicode code units and code points

    25  * are defined in umachines.h because they can be machine-dependent.

    26  *

    27  * utf.h is included by utypes.h and itself includes utf8.h and utf16.h after some

    28  * common definitions. Those files define macros for efficiently getting code points

    29  * in and out of UTF-8/16 strings.

    30  * utf16.h macros have "U16_" prefixes.

    31  * utf8.h defines similar macros with "U8_" prefixes for UTF-8 string handling.

    32  *

    33  * ICU processes 16-bit Unicode strings.

    34  * Most of the time, such strings are well-formed UTF-16.

    35  * Single, unpaired surrogates must be handled as well, and are treated in ICU

    36  * like regular code points where possible.

    37  * (Pairs of surrogate code points are indistinguishable from supplementary

    38  * code points encoded as pairs of supplementary code units.)

    39  *

    40  * In fact, almost all Unicode code points in normal text (>99%)

    41  * are on the BMP (<=U+ffff) and even <=U+d7ff.

    42  * ICU functions handle supplementary code points (U+10000..U+10ffff)

    43  * but are optimized for the much more frequently occurring BMP code points.

    44  *

    45  * utf.h defines UChar to be an unsigned 16-bit integer. If this matches wchar_t, then

    46  * UChar is defined to be exactly wchar_t, otherwise uint16_t.

    47  *

    48  * UChar32 is defined to be a signed 32-bit integer (int32_t), large enough for a 21-bit

    49  * Unicode code point (Unicode scalar value, 0..0x10ffff).

    50  * Before ICU 2.4, the definition of UChar32 was similarly platform-dependent as

    51  * the definition of UChar. For details see the documentation for UChar32 itself.

    52  *

    53  * utf.h also defines a small number of C macros for single Unicode code points.

    54  * These are simple checks for surrogates and non-characters.

    55  * For actual Unicode character properties see uchar.h.

    56  *

    57  * By default, string operations must be done with error checking in case

    58  * a string is not well-formed UTF-16.

    59  * The macros will detect if a surrogate code unit is unpaired

    60  * (lead unit without trail unit or vice versa) and just return the unit itself

    61  * as the code point.

    62  * (It is an accidental property of Unicode and UTF-16 that all

    63  * malformed sequences can be expressed unambiguously with a distinct subrange

    64  * of Unicode code points.)

    65  *

    66  * When it is safe to assume that text is well-formed UTF-16

    67  * (does not contain single, unpaired surrogates), then one can use

    68  * U16_..._UNSAFE macros.

    69  * These do not check for proper code unit sequences or truncated text and may

    70  * yield wrong results or even cause a crash if they are used with "malformed"

    71  * text.

    72  * In practice, U16_..._UNSAFE macros will produce slightly less code but

    73  * should not be faster because the processing is only different when a

    74  * surrogate code unit is detected, which will be rare.

    75  *

    76  * Similarly for UTF-8, there are "safe" macros without a suffix,

    77  * and U8_..._UNSAFE versions.

    78  * The performance differences are much larger here because UTF-8 provides so

    79  * many opportunities for malformed sequences.

    80  * The unsafe UTF-8 macros are entirely implemented inside the macro definitions

    81  * and are fast, while the safe UTF-8 macros call functions for all but the

    82  * trivial (ASCII) cases.

    83  *

    84  * Unlike with UTF-16, malformed sequences cannot be expressed with distinct

    85  * code point values (0..U+10ffff). They are indicated with negative values instead.

    86  *

    87  * For more information see the ICU User Guide Strings chapter

    88  * (http://oss.software.ibm.com/icu/userguide/).

    89  *

    90  * <em>Usage:</em>

    91  * ICU coding guidelines for if() statements should be followed when using these macros.

    92  * Compound statements (curly braces {}) must be used  for if-else-while... 

    93  * bodies and all macro statements should be terminated with semicolon.

    94  *

    95  * @stable ICU 2.4

    96  */

    97 

    98 #ifndef __UTF_H__

    99 #define __UTF_H__

   100 

   101 #include "unicode/utypes.h"

   102 /* include the utfXX.h after the following definitions */

   103 

   104 /* single-code point definitions -------------------------------------------- */

   105 

   106 /**

   107  * This value is intended for sentinel values for APIs that

   108  * (take or) return single code points (UChar32).

   109  * It is outside of the Unicode code point range 0..0x10ffff.

   110  * 

   111  * For example, a "done" or "error" value in a new API

   112  * could be indicated with U_SENTINEL.

   113  *

   114  * ICU APIs designed before ICU 2.4 usually define service-specific "done"

   115  * values, mostly 0xffff.

   116  * Those may need to be distinguished from

   117  * actual U+ffff text contents by calling functions like

   118  * CharacterIterator::hasNext() or UnicodeString::length().

   119  *

   120  * @return -1

   121  * @see UChar32

   122  * @stable ICU 2.4

   123  */

   124 #define U_SENTINEL (-1)

   125 

   126 /**

   127  * Is this code point a Unicode noncharacter?

   128  * @param c 32-bit code point

   129  * @return TRUE or FALSE

   130  * @stable ICU 2.4

   131  */

   132 #define U_IS_UNICODE_NONCHAR(c) \

   133     ((c)>=0xfdd0 && \

   134      ((uint32_t)(c)<=0xfdef || ((c)&0xfffe)==0xfffe) && \

   135      (uint32_t)(c)<=0x10ffff)

   136 

   137 /**

   138  * Is c a Unicode code point value (0..U+10ffff)

   139  * that can be assigned a character?

   140  *

   141  * Code points that are not characters include:

   142  * - single surrogate code points (U+d800..U+dfff, 2048 code points)

   143  * - the last two code points on each plane (U+__fffe and U+__ffff, 34 code points)

   144  * - U+fdd0..U+fdef (new with Unicode 3.1, 32 code points)

   145  * - the highest Unicode code point value is U+10ffff

   146  *

   147  * This means that all code points below U+d800 are character code points,

   148  * and that boundary is tested first for performance.

   149  *

   150  * @param c 32-bit code point

   151  * @return TRUE or FALSE

   152  * @stable ICU 2.4

   153  */

   154 #define U_IS_UNICODE_CHAR(c) \

   155     ((uint32_t)(c)<0xd800 || \

   156         ((uint32_t)(c)>0xdfff && \

   157          (uint32_t)(c)<=0x10ffff && \

   158          !U_IS_UNICODE_NONCHAR(c)))

   159 

   160 #ifndef U_HIDE_DRAFT_API

   161 

   162 /**

   163  * Is this code point a BMP code point (U+0000..U+ffff)?

   164  * @param c 32-bit code point

   165  * @return TRUE or FALSE

   166  * @draft ICU 2.8

   167  */

   168 #define U_IS_BMP(c) ((uint32_t)(c)<=0xffff)

   169 

   170 /**

   171  * Is this code point a supplementary code point (U+10000..U+10ffff)?

   172  * @param c 32-bit code point

   173  * @return TRUE or FALSE

   174  * @draft ICU 2.8

   175  */

   176 #define U_IS_SUPPLEMENTARY(c) ((uint32_t)((c)-0x10000)<=0xfffff)

   177 

   178 #endif /*U_HIDE_DRAFT_API*/

   179  

   180 /**

   181  * Is this code point a lead surrogate (U+d800..U+dbff)?

   182  * @param c 32-bit code point

   183  * @return TRUE or FALSE

   184  * @stable ICU 2.4

   185  */

   186 #define U_IS_LEAD(c) (((c)&0xfffffc00)==0xd800)

   187 

   188 /**

   189  * Is this code point a trail surrogate (U+dc00..U+dfff)?

   190  * @param c 32-bit code point

   191  * @return TRUE or FALSE

   192  * @stable ICU 2.4

   193  */

   194 #define U_IS_TRAIL(c) (((c)&0xfffffc00)==0xdc00)

   195 

   196 /**

   197  * Is this code point a surrogate (U+d800..U+dfff)?

   198  * @param c 32-bit code point

   199  * @return TRUE or FALSE

   200  * @stable ICU 2.4

   201  */

   202 #define U_IS_SURROGATE(c) (((c)&0xfffff800)==0xd800)

   203 

   204 /**

   205  * Assuming c is a surrogate code point (U_IS_SURROGATE(c)),

   206  * is it a lead surrogate?

   207  * @param c 32-bit code point

   208  * @return TRUE or FALSE

   209  * @stable ICU 2.4

   210  */

   211 #define U_IS_SURROGATE_LEAD(c) (((c)&0x400)==0)

   212 

   213 /* include the utfXX.h ------------------------------------------------------ */

   214 

   215 #include "unicode/utf8.h"

   216 #include "unicode/utf16.h"

   217 

   218 /* utf_old.h contains deprecated, pre-ICU 2.4 definitions */

   219 #include "unicode/utf_old.h"

   220 

   221 #endif