/*

*******************************************************************************

*

*   Copyright (C) 2001-2004, International Business Machines

*   Corporation and others.  All Rights Reserved.

*

*******************************************************************************

*   file name:  unormimp.h

*   encoding:   US-ASCII

*   tab size:   8 (not used)

*   indentation:4

*

*   created on: 2001may25

*   created by: Markus W. Scherer

*/

#ifndef __UNORMIMP_H__

#define __UNORMIMP_H__

#include "unicode/utypes.h"

#if !UCONFIG_NO_NORMALIZATION

#ifdef XP_CPLUSPLUS

#include "unicode/uniset.h"

#endif

#include "unicode/uiter.h"

#include "unicode/unorm.h"

#include "unicode/uset.h"

#include "utrie.h"

#include "ustr_imp.h"

#include "udataswp.h"

/*

 * This new implementation of the normalization code loads its data from

 * unorm.icu, which is generated with the gennorm tool.

 * The format of that file is described at the end of this file.

 */

/* norm32 value constants */

enum {

    /* quick check flags 0..3 set mean "no" for their forms */

    _NORM_QC_NFC=0x11,          /* no|maybe */

    _NORM_QC_NFKC=0x22,         /* no|maybe */

    _NORM_QC_NFD=4,             /* no */

    _NORM_QC_NFKD=8,            /* no */

    _NORM_QC_ANY_NO=0xf,

    /* quick check flags 4..5 mean "maybe" for their forms; test flags>=_NORM_QC_MAYBE */

    _NORM_QC_MAYBE=0x10,

    _NORM_QC_ANY_MAYBE=0x30,

    _NORM_QC_MASK=0x3f,

    _NORM_COMBINES_FWD=0x40,

    _NORM_COMBINES_BACK=0x80,

    _NORM_COMBINES_ANY=0xc0,

    _NORM_CC_SHIFT=8,           /* UnicodeData.txt combining class in bits 15..8 */

    _NORM_CC_MASK=0xff00,

    _NORM_EXTRA_SHIFT=16,               /* 16 bits for the index to UChars and other extra data */

    _NORM_EXTRA_INDEX_TOP=0xfc00,       /* start of surrogate specials after shift */

    _NORM_EXTRA_SURROGATE_MASK=0x3ff,

    _NORM_EXTRA_SURROGATE_TOP=0x3f0,    /* hangul etc. */

    _NORM_EXTRA_HANGUL=_NORM_EXTRA_SURROGATE_TOP,

    _NORM_EXTRA_JAMO_L,

    _NORM_EXTRA_JAMO_V,

    _NORM_EXTRA_JAMO_T

};

/* norm32 value constants using >16 bits */

#define _NORM_MIN_SPECIAL       0xfc000000

#define _NORM_SURROGATES_TOP    0xfff00000

#define _NORM_MIN_HANGUL        0xfff00000

#define _NORM_MIN_JAMO_V        0xfff20000

#define _NORM_JAMO_V_TOP        0xfff30000

/* value constants for auxTrie */

enum {

    _NORM_AUX_COMP_EX_SHIFT=10,

    _NORM_AUX_UNSAFE_SHIFT=11,

    _NORM_AUX_NFC_SKIPPABLE_F_SHIFT=12

};

#define _NORM_AUX_MAX_FNC           ((int32_t)1<<_NORM_AUX_COMP_EX_SHIFT)

#define _NORM_AUX_FNC_MASK          (uint32_t)(_NORM_AUX_MAX_FNC-1)

#define _NORM_AUX_COMP_EX_MASK      ((uint32_t)1<<_NORM_AUX_COMP_EX_SHIFT)

#define _NORM_AUX_UNSAFE_MASK       ((uint32_t)1<<_NORM_AUX_UNSAFE_SHIFT)

#define _NORM_AUX_NFC_SKIP_F_MASK   ((uint32_t)1<<_NORM_AUX_NFC_SKIPPABLE_F_SHIFT)

/* canonStartSets[0..31] contains indexes for what is in the array */

enum {

    _NORM_SET_INDEX_CANON_SETS_LENGTH,      /* number of uint16_t in canonical starter sets */

    _NORM_SET_INDEX_CANON_BMP_TABLE_LENGTH, /* number of uint16_t in the BMP search table (contains pairs) */

    _NORM_SET_INDEX_CANON_SUPP_TABLE_LENGTH,/* number of uint16_t in the supplementary search table (contains triplets) */

    /* from formatVersion 2.3: */

    _NORM_SET_INDEX_NX_CJK_COMPAT_OFFSET,   /* uint16_t offset from canonStartSets[0] to the

                                               exclusion set for CJK compatibility characters */

    _NORM_SET_INDEX_NX_UNICODE32_OFFSET,    /* uint16_t offset from canonStartSets[0] to the

                                               exclusion set for Unicode 3.2 characters */

    _NORM_SET_INDEX_NX_RESERVED_OFFSET,     /* uint16_t offset from canonStartSets[0] to the

                                               end of the previous exclusion set */

    _NORM_SET_INDEX_TOP=32                  /* changing this requires a new formatVersion */

};

/* more constants for canonical starter sets */

/* 14 bit indexes to canonical USerializedSets */

#define _NORM_MAX_CANON_SETS            0x4000

/* single-code point BMP sets are encoded directly in the search table except if result=0x4000..0x7fff */

#define _NORM_CANON_SET_BMP_MASK        0xc000

#define _NORM_CANON_SET_BMP_IS_INDEX    0x4000

/* indexes[] value names */

enum {

    _NORM_INDEX_TRIE_SIZE,              /* number of bytes in normalization trie */

    _NORM_INDEX_UCHAR_COUNT,            /* number of UChars in extra data */

    _NORM_INDEX_COMBINE_DATA_COUNT,     /* number of uint16_t words for combining data */

    _NORM_INDEX_COMBINE_FWD_COUNT,      /* number of code points that combine forward */

    _NORM_INDEX_COMBINE_BOTH_COUNT,     /* number of code points that combine forward and backward */

    _NORM_INDEX_COMBINE_BACK_COUNT,     /* number of code points that combine backward */

    _NORM_INDEX_MIN_NFC_NO_MAYBE,       /* first code point with quick check NFC NO/MAYBE */

    _NORM_INDEX_MIN_NFKC_NO_MAYBE,      /* first code point with quick check NFKC NO/MAYBE */

    _NORM_INDEX_MIN_NFD_NO_MAYBE,       /* first code point with quick check NFD NO/MAYBE */

    _NORM_INDEX_MIN_NFKD_NO_MAYBE,      /* first code point with quick check NFKD NO/MAYBE */

    _NORM_INDEX_FCD_TRIE_SIZE,          /* number of bytes in FCD trie */

    _NORM_INDEX_AUX_TRIE_SIZE,          /* number of bytes in the auxiliary trie */

    _NORM_INDEX_CANON_SET_COUNT,        /* number of uint16_t in the array of serialized USet */

    _NORM_INDEX_TOP=32                  /* changing this requires a new formatVersion */

};

enum {

    /* FCD check: everything below this code point is known to have a 0 lead combining class */

    _NORM_MIN_WITH_LEAD_CC=0x300

};

enum {

    /**

     * Bit 7 of the length byte for a decomposition string in extra data is

     * a flag indicating whether the decomposition string is

     * preceded by a 16-bit word with the leading and trailing cc

     * of the decomposition (like for A-umlaut);

     * if not, then both cc's are zero (like for compatibility ideographs).

     */

    _NORM_DECOMP_FLAG_LENGTH_HAS_CC=0x80,

    /**

     * Bits 6..0 of the length byte contain the actual length.

     */

    _NORM_DECOMP_LENGTH_MASK=0x7f

};

#endif /* #if !UCONFIG_NO_NORMALIZATION */

/* Korean Hangul and Jamo constants */

enum {

    JAMO_L_BASE=0x1100,     /* "lead" jamo */

    JAMO_V_BASE=0x1161,     /* "vowel" jamo */

    JAMO_T_BASE=0x11a7,     /* "trail" jamo */

    HANGUL_BASE=0xac00,

    JAMO_L_COUNT=19,

    JAMO_V_COUNT=21,

    JAMO_T_COUNT=28,

    HANGUL_COUNT=JAMO_L_COUNT*JAMO_V_COUNT*JAMO_T_COUNT

};

#if !UCONFIG_NO_NORMALIZATION

/* Constants for options flags for normalization. @draft ICU 2.6 */

enum {

    /** Options bit 0, do not decompose Hangul syllables. @draft ICU 2.6 */

    UNORM_NX_HANGUL=1,

    /** Options bit 1, do not decompose CJK compatibility characters. @draft ICU 2.6 */

    UNORM_NX_CJK_COMPAT=2,

    /**

     * Options bit 8, use buggy recomposition described in

     * Unicode Public Review Issue #29

     * at http://www.unicode.org/review/resolved-pri.html#pri29

     *

     * Used in IDNA implementation according to strict interpretation

     * of IDNA definition based on Unicode 3.2 which predates PRI #29.

     */

    UNORM_BEFORE_PRI_29=0x100

};

/**

 * Is the normalizer data loaded?

 * This is used internally before other internal normalizer functions

 * are called.

 * It saves this check in each of many normalization calls that

 * are made for, e.g., collation.

 *

 * @param pErrorCode as usual

 * @return boolean value for whether the normalization data is loaded

 *

 * @internal

 */

U_CAPI UBool U_EXPORT2

unorm_haveData(UErrorCode *pErrorCode);

/**

 * Internal API for normalizing.

 * Does not check for bad input.

 * @internal

 */

U_CAPI int32_t U_EXPORT2

unorm_internalNormalize(UChar *dest, int32_t destCapacity,

                        const UChar *src, int32_t srcLength,

                        UNormalizationMode mode, int32_t options,

                        UErrorCode *pErrorCode);

#ifdef XP_CPLUSPLUS

/**

 * Internal API for normalizing.

 * Does not check for bad input.

 * Requires _haveData() to be true.

 * @internal

 */

U_CFUNC int32_t

unorm_internalNormalizeWithNX(UChar *dest, int32_t destCapacity,

                              const UChar *src, int32_t srcLength,

                              UNormalizationMode mode, int32_t options, const UnicodeSet *nx,

                              UErrorCode *pErrorCode);

#endif

/**

 * internal API, used by normlzr.cpp

 * @internal

 */

U_CAPI int32_t U_EXPORT2

unorm_decompose(UChar *dest, int32_t destCapacity,

                const UChar *src, int32_t srcLength,

                UBool compat, int32_t options,

                UErrorCode *pErrorCode);

/**

 * internal API, used by normlzr.cpp

 * @internal

 */

U_CAPI int32_t U_EXPORT2

unorm_compose(UChar *dest, int32_t destCapacity,

              const UChar *src, int32_t srcLength,

              UBool compat, int32_t options,

              UErrorCode *pErrorCode);

#ifdef XP_CPLUSPLUS

/**

 * internal API, used by unormcmp.cpp

 * @internal

 */

U_CFUNC UNormalizationCheckResult

unorm_internalQuickCheck(const UChar *src,

                         int32_t srcLength,

                         UNormalizationMode mode,

                         UBool allowMaybe,

                         const UnicodeSet *nx,

                         UErrorCode *pErrorCode);

#endif

#endif /* #if !UCONFIG_NO_NORMALIZATION */

/**

 * Internal option for unorm_cmpEquivFold() for decomposing.

 * If not set, just do strcasecmp().

 * @internal

 */

#define _COMPARE_EQUIV 0x80000

#ifndef U_COMPARE_IGNORE_CASE

/* see also unorm.h */

/**

 * Option bit for unorm_compare:

 * Perform case-insensitive comparison.

 * @draft ICU 2.2

 */

#define U_COMPARE_IGNORE_CASE       0x10000

#endif

/**

 * Internal option for unorm_cmpEquivFold() for strncmp style.

 * If set, checks for both string length and terminating NUL.

 * @internal

 */

#define _STRNCMP_STYLE 0x1000

#if !UCONFIG_NO_NORMALIZATION

/**

 * Internal API to get the 16-bit FCD value (lccc + tccc) for c,

 * for u_getIntPropertyValue().

 * @internal

 */

U_CAPI uint16_t U_EXPORT2

unorm_getFCD16FromCodePoint(UChar32 c);

/**

 * Internal API, used by collation code.

 * Get access to the internal FCD trie table to be able to perform

 * incremental, per-code unit, FCD checks in collation.

 * One pointer is sufficient because the trie index values are offset

 * by the index size, so that the same pointer is used to access the trie data.

 * @internal

 */

U_CAPI const uint16_t * U_EXPORT2

unorm_getFCDTrie(UErrorCode *pErrorCode);

#ifdef XP_CPLUSPLUS

U_NAMESPACE_BEGIN

/**

 * Internal API, used by collation code.

 * Get the FCD value for a code unit, with

 * bits 15..8   lead combining class

 * bits  7..0   trail combining class

 *

 * If c is a lead surrogate and the value is not 0,

 * then instead of combining classes the value

 * is used in unorm_getFCD16FromSurrogatePair() to get the real value

 * of the supplementary code point.

 *

 * @internal

 */

inline uint16_t

unorm_getFCD16(const uint16_t *fcdTrieIndex, UChar c) {

    return

        fcdTrieIndex[

            (fcdTrieIndex[

                c>>UTRIE_SHIFT

            ]<<UTRIE_INDEX_SHIFT)+

            (c&UTRIE_MASK)

        ];

}

/**

 * Internal API, used by collation code.

 * Get the FCD value for a supplementary code point, with

 * bits 15..8   lead combining class

 * bits  7..0   trail combining class

 *

 * @param fcd16  The FCD value for the lead surrogate, not 0.

 * @param c2     The trail surrogate code unit.

 *

 * @internal

 */

inline uint16_t

unorm_getFCD16FromSurrogatePair(const uint16_t *fcdTrieIndex, uint16_t fcd16, UChar c2) {

    return

        fcdTrieIndex[

            (fcdTrieIndex[

                (int32_t)fcd16+((c2&0x3ff)>>UTRIE_SHIFT)

            ]<<UTRIE_INDEX_SHIFT)+

            (c2&UTRIE_MASK)

        ];

}

U_NAMESPACE_END

#endif

/**

 * internal API, used by StringPrep

 * @internal

 */

U_CAPI void U_EXPORT2

unorm_getUnicodeVersion(UVersionInfo *versionInfo, UErrorCode *pErrorCode);

/**

 * Get the canonical decomposition for one code point.

 * Requires unorm_haveData() and buffer!=NULL and pLength!=NULL.

 * @param c code point

 * @param buffer out-only buffer for algorithmic decompositions of Hangul

 * @param length out-only, takes the length of the decomposition, if any

 * @return pointer to decomposition, or 0 if none

 * @internal

 */

U_CFUNC const UChar *

unorm_getCanonicalDecomposition(UChar32 c, UChar buffer[4], int32_t *pLength);

/**

 * internal API, used by the canonical iterator

 * TODO Consider using signature similar to unorm_getCanonicalDecomposition()

 * for more efficiency

 * @internal

 */

U_CAPI int32_t U_EXPORT2

unorm_getDecomposition(UChar32 c, UBool compat,

                       UChar *dest, int32_t destCapacity);

/**

 * internal API, used by uprops.cpp

 * @internal

 */

U_CAPI UBool U_EXPORT2

unorm_internalIsFullCompositionExclusion(UChar32 c);

/**

 * Internal API, used by enumeration of canonically equivalent strings

 * @internal

 */

U_CAPI UBool U_EXPORT2

unorm_isCanonSafeStart(UChar32 c);

/**

 * Internal API, used by enumeration of canonically equivalent strings

 * @internal

 */

U_CAPI UBool U_EXPORT2

unorm_getCanonStartSet(UChar32 c, USerializedSet *fillSet);

/**

 * Is c an NF<mode>-skippable code point? See unormimp.h.

 * @internal

 */

U_CAPI UBool U_EXPORT2

unorm_isNFSkippable(UChar32 c, UNormalizationMode mode);

#ifdef XP_CPLUSPLUS

/**

 * Get normalization exclusion set for the options.

 * Requires unorm_haveData().

 * @internal

 */

U_CFUNC const UnicodeSet *

unorm_getNX(int32_t options, UErrorCode *pErrorCode);

#endif

/**

 * Enumerate each normalization data trie and add the

 * start of each range of same properties to the set.

 * @internal

 */

U_CAPI void U_EXPORT2

unorm_addPropertyStarts(const USetAdder *sa, UErrorCode *pErrorCode);

/**

 * Swap unorm.icu. See udataswp.h.

 * @internal

 */

U_CAPI int32_t U_EXPORT2

unorm_swap(const UDataSwapper *ds,

           const void *inData, int32_t length, void *outData,

           UErrorCode *pErrorCode);

/**

 * Get the NF*_QC property for a code point, for u_getIntPropertyValue().

 * @internal

 */

U_CAPI UNormalizationCheckResult U_EXPORT2

unorm_getQuickCheck(UChar32 c, UNormalizationMode mode);

/**

 * Description of the format of unorm.icu version 2.3.

 *

 * Main change from version 1 to version 2:

 * Use of new, common UTrie instead of normalization-specific tries.

 * Change to version 2.1: add third/auxiliary trie with associated data.

 * Change to version 2.2: add skippable (f) flag data (_NORM_AUX_NFC_SKIP_F_MASK).

 * Change to version 2.3: add serialized sets for normalization exclusions

 *                        stored inside canonStartSets[]

 *

 * For more details of how to use the data structures see the code

 * in unorm.cpp (runtime normalization code) and

 * in gennorm.c and gennorm/store.c (build-time data generation).

 *

 * For the serialized format of UTrie see utrie.c/UTrieHeader.

 *

 * - Overall partition

 *

 * unorm.dat customarily begins with a UDataInfo structure, see udata.h and .c.

 * After that there are the following structures:

 *

 * int32_t indexes[_NORM_INDEX_TOP];            -- _NORM_INDEX_TOP=32, see enum in this file

 *

 * UTrie normTrie;                              -- size in bytes=indexes[_NORM_INDEX_TRIE_SIZE]

 * 

 * uint16_t extraData[extraDataTop];            -- extraDataTop=indexes[_NORM_INDEX_UCHAR_COUNT]

 *                                                 extraData[0] contains the number of units for

 *                                                 FC_NFKC_Closure (formatVersion>=2.1)

 *

 * uint16_t combiningTable[combiningTableTop];  -- combiningTableTop=indexes[_NORM_INDEX_COMBINE_DATA_COUNT]

 *                                                 combiningTableTop may include one 16-bit padding unit

 *                                                 to make sure that fcdTrie is 32-bit-aligned

 *

 * UTrie fcdTrie;                               -- size in bytes=indexes[_NORM_INDEX_FCD_TRIE_SIZE]

 *

 * UTrie auxTrie;                               -- size in bytes=indexes[_NORM_INDEX_AUX_TRIE_SIZE]

 *

 * uint16_t canonStartSets[canonStartSetsTop]   -- canonStartSetsTop=indexes[_NORM_INDEX_CANON_SET_COUNT]

 *                                                 serialized USets and binary search tables, see below

 *

 *

 * The indexes array contains lengths and sizes of the following arrays and structures

 * as well as the following values:

 *  indexes[_NORM_INDEX_COMBINE_FWD_COUNT]=combineFwdTop

 *      -- one more than the highest combining index computed for forward-only-combining characters

 *  indexes[_NORM_INDEX_COMBINE_BOTH_COUNT]=combineBothTop-combineFwdTop

 *      -- number of combining indexes computed for both-ways-combining characters

 *  indexes[_NORM_INDEX_COMBINE_BACK_COUNT]=combineBackTop-combineBothTop

 *      -- number of combining indexes computed for backward-only-combining characters

 *

 *  indexes[_NORM_INDEX_MIN_NF*_NO_MAYBE] (where *={ C, D, KC, KD })

 *      -- first code point with a quick check NF* value of NO/MAYBE

 *

 *

 * - Tries

 *