/*

*******************************************************************************

* Copyright (c) 1996-2004, International Business Machines Corporation

*               and others. All Rights Reserved.

*******************************************************************************

* File unorm.h

*

* Created by: Vladimir Weinstein 12052000

*

* Modification history :

*

* Date        Name        Description

* 02/01/01    synwee      Added normalization quickcheck enum and method.

*/

#ifndef UNORM_H

#define UNORM_H

#include "unicode/utypes.h"

#if !UCONFIG_NO_NORMALIZATION

#include "unicode/uiter.h"

/**

 * \file

 * \brief C API: Unicode Normalization 

 *

 * <h2>Unicode normalization API</h2>

 *

 * <code>unorm_normalize</code> transforms Unicode text into an equivalent composed or

 * decomposed form, allowing for easier sorting and searching of text.

 * <code>unorm_normalize</code> supports the standard normalization forms described in

 * <a href="http://www.unicode.org/unicode/reports/tr15/" target="unicode">

 * Unicode Standard Annex #15 &#8212; Unicode Normalization Forms</a>.

 *

 * Characters with accents or other adornments can be encoded in

 * several different ways in Unicode.  For example, take the character A-acute.

 * In Unicode, this can be encoded as a single character (the

 * "composed" form):

 *

 * \code

 *      00C1    LATIN CAPITAL LETTER A WITH ACUTE

 * \endcode

 *

 * or as two separate characters (the "decomposed" form):

 *

 * \code

 *      0041    LATIN CAPITAL LETTER A

 *      0301    COMBINING ACUTE ACCENT

 * \endcode

 *

 * To a user of your program, however, both of these sequences should be

 * treated as the same "user-level" character "A with acute accent".  When you are searching or

 * comparing text, you must ensure that these two sequences are treated 

 * equivalently.  In addition, you must handle characters with more than one

 * accent.  Sometimes the order of a character's combining accents is

 * significant, while in other cases accent sequences in different orders are

 * really equivalent.

 *

 * Similarly, the string "ffi" can be encoded as three separate letters:

 *

 * \code

 *      0066    LATIN SMALL LETTER F

 *      0066    LATIN SMALL LETTER F

 *      0069    LATIN SMALL LETTER I

 * \endcode

 *

 * or as the single character

 *

 * \code

 *      FB03    LATIN SMALL LIGATURE FFI

 * \endcode

 *

 * The ffi ligature is not a distinct semantic character, and strictly speaking

 * it shouldn't be in Unicode at all, but it was included for compatibility

 * with existing character sets that already provided it.  The Unicode standard

 * identifies such characters by giving them "compatibility" decompositions

 * into the corresponding semantic characters.  When sorting and searching, you

 * will often want to use these mappings.

 *

 * <code>unorm_normalize</code> helps solve these problems by transforming text into the

 * canonical composed and decomposed forms as shown in the first example above.  

 * In addition, you can have it perform compatibility decompositions so that 

 * you can treat compatibility characters the same as their equivalents.

 * Finally, <code>unorm_normalize</code> rearranges accents into the proper canonical

 * order, so that you do not have to worry about accent rearrangement on your

 * own.

 *

 * Form FCD, "Fast C or D", is also designed for collation.

 * It allows to work on strings that are not necessarily normalized

 * with an algorithm (like in collation) that works under "canonical closure", i.e., it treats precomposed

 * characters and their decomposed equivalents the same.

 *

 * It is not a normalization form because it does not provide for uniqueness of representation. Multiple strings

 * may be canonically equivalent (their NFDs are identical) and may all conform to FCD without being identical

 * themselves.

 *

 * The form is defined such that the "raw decomposition", the recursive canonical decomposition of each character,

 * results in a string that is canonically ordered. This means that precomposed characters are allowed for as long

 * as their decompositions do not need canonical reordering.

 *

 * Its advantage for a process like collation is that all NFD and most NFC texts - and many unnormalized texts -

 * already conform to FCD and do not need to be normalized (NFD) for such a process. The FCD quick check will

 * return UNORM_YES for most strings in practice.

 *

 * unorm_normalize(UNORM_FCD) may be implemented with UNORM_NFD.

 *

 * For more details on FCD see the collation design document:

 * http://oss.software.ibm.com/cvs/icu/~checkout~/icuhtml/design/collation/ICU_collation_design.htm

 *

 * ICU collation performs either NFD or FCD normalization automatically if normalization

 * is turned on for the collator object.

 * Beyond collation and string search, normalized strings may be useful for string equivalence comparisons,

 * transliteration/transcription, unique representations, etc.

 *

 * The W3C generally recommends to exchange texts in NFC.

 * Note also that most legacy character encodings use only precomposed forms and often do not

 * encode any combining marks by themselves. For conversion to such character encodings the

 * Unicode text needs to be normalized to NFC.

 * For more usage examples, see the Unicode Standard Annex.

 */

/**

 * Constants for normalization modes.

 * @stable ICU 2.0

 */

typedef enum {

  /** No decomposition/composition. @stable ICU 2.0 */

  UNORM_NONE = 1, 

  /** Canonical decomposition. @stable ICU 2.0 */

  UNORM_NFD = 2,

  /** Compatibility decomposition. @stable ICU 2.0 */

  UNORM_NFKD = 3,

  /** Canonical decomposition followed by canonical composition. @stable ICU 2.0 */

  UNORM_NFC = 4,

  /** Default normalization. @stable ICU 2.0 */

  UNORM_DEFAULT = UNORM_NFC, 

  /** Compatibility decomposition followed by canonical composition. @stable ICU 2.0 */

  UNORM_NFKC =5,

  /** "Fast C or D" form. @stable ICU 2.0 */

  UNORM_FCD = 6,

  /** One more than the highest normalization mode constant. @stable ICU 2.0 */

  UNORM_MODE_COUNT

} UNormalizationMode;

/**

 * Constants for options flags for normalization.

 * Use 0 for default options,

 * including normalization according to the Unicode version

 * that is currently supported by ICU (see u_getUnicodeVersion).

 * @stable ICU 2.6

 */

enum {

    /**

     * Options bit set value to select Unicode 3.2 normalization

     * (except NormalizationCorrections).

     * At most one Unicode version can be selected at a time.

     * @stable ICU 2.6

     */

    UNORM_UNICODE_3_2=0x20

};

/**

 * Lowest-order bit number of unorm_compare() options bits corresponding to

 * normalization options bits.

 *

 * The options parameter for unorm_compare() uses most bits for

 * itself and for various comparison and folding flags.

 * The most significant bits, however, are shifted down and passed on

 * to the normalization implementation.

 * (That is, from unorm_compare(..., options, ...),

 * options>>UNORM_COMPARE_NORM_OPTIONS_SHIFT will be passed on to the

 * internal normalization functions.)

 *

 * @see unorm_compare

 * @stable ICU 2.6

 */

#define UNORM_COMPARE_NORM_OPTIONS_SHIFT 20

/**

 * Normalize a string.

 * The string will be normalized according the specified normalization mode

 * and options.

 *

 * @param source The string to normalize.

 * @param sourceLength The length of source, or -1 if NUL-terminated.

 * @param mode The normalization mode; one of UNORM_NONE, 

 *             UNORM_NFD, UNORM_NFC, UNORM_NFKC, UNORM_NFKD, UNORM_DEFAULT.

 * @param options The normalization options, ORed together (0 for no options).

 * @param result A pointer to a buffer to receive the result string.

 *               The result string is NUL-terminated if possible.

 * @param resultLength The maximum size of result.

 * @param status A pointer to a UErrorCode to receive any errors.

 * @return The total buffer size needed; if greater than resultLength,

 *         the output was truncated, and the error code is set to U_BUFFER_OVERFLOW_ERROR.

 * @stable ICU 2.0

 */

U_STABLE int32_t U_EXPORT2 

unorm_normalize(const UChar *source, int32_t sourceLength,

                UNormalizationMode mode, int32_t options,

                UChar *result, int32_t resultLength,

                UErrorCode *status);

#endif

/**

 * Result values for unorm_quickCheck().

 * For details see Unicode Technical Report 15.

 * @stable ICU 2.0

 */

typedef enum UNormalizationCheckResult {

  /** 

   * Indicates that string is not in the normalized format

   */

  UNORM_NO,

  /** 

   * Indicates that string is in the normalized format

   */

  UNORM_YES,

  /** 

   * Indicates that string cannot be determined if it is in the normalized 

   * format without further thorough checks.

   */

  UNORM_MAYBE

} UNormalizationCheckResult;

#if !UCONFIG_NO_NORMALIZATION

/**

 * Performing quick check on a string, to quickly determine if the string is 

 * in a particular normalization format.

 * Three types of result can be returned UNORM_YES, UNORM_NO or

 * UNORM_MAYBE. Result UNORM_YES indicates that the argument

 * string is in the desired normalized format, UNORM_NO determines that

 * argument string is not in the desired normalized format. A 

 * UNORM_MAYBE result indicates that a more thorough check is required, 

 * the user may have to put the string in its normalized form and compare the 

 * results.

 *

 * @param source       string for determining if it is in a normalized format

 * @param sourcelength length of source to test, or -1 if NUL-terminated

 * @param mode         which normalization form to test for

 * @param status       a pointer to a UErrorCode to receive any errors

 * @return UNORM_YES, UNORM_NO or UNORM_MAYBE

 *

 * @see unorm_isNormalized

 * @stable ICU 2.0

 */

U_STABLE UNormalizationCheckResult U_EXPORT2

unorm_quickCheck(const UChar *source, int32_t sourcelength,

                 UNormalizationMode mode,

                 UErrorCode *status);

/**

 * Performing quick check on a string; same as unorm_quickCheck but

 * takes an extra options parameter like most normalization functions.

 *

 * @param src        String that is to be tested if it is in a normalization format.

 * @param srcLength  Length of source to test, or -1 if NUL-terminated.

 * @param mode       Which normalization form to test for.

 * @param options    The normalization options, ORed together (0 for no options).

 * @param pErrorCode ICU error code in/out parameter.

 *                   Must fulfill U_SUCCESS before the function call.

 * @return UNORM_YES, UNORM_NO or UNORM_MAYBE

 *

 * @see unorm_quickCheck

 * @see unorm_isNormalized

 * @stable ICU 2.6

 */

U_STABLE UNormalizationCheckResult U_EXPORT2

unorm_quickCheckWithOptions(const UChar *src, int32_t srcLength, 

                            UNormalizationMode mode, int32_t options,

                            UErrorCode *pErrorCode);

/**

 * Test if a string is in a given normalization form.

 * This is semantically equivalent to source.equals(normalize(source, mode)) .

 *

 * Unlike unorm_quickCheck(), this function returns a definitive result,

 * never a "maybe".

 * For NFD, NFKD, and FCD, both functions work exactly the same.

 * For NFC and NFKC where quickCheck may return "maybe", this function will

 * perform further tests to arrive at a TRUE/FALSE result.

 *

 * @param src        String that is to be tested if it is in a normalization format.

 * @param srcLength  Length of source to test, or -1 if NUL-terminated.

 * @param mode       Which normalization form to test for.

 * @param pErrorCode ICU error code in/out parameter.

 *                   Must fulfill U_SUCCESS before the function call.

 * @return Boolean value indicating whether the source string is in the

 *         "mode" normalization form.

 *

 * @see unorm_quickCheck

 * @stable ICU 2.2

 */

U_STABLE UBool U_EXPORT2

unorm_isNormalized(const UChar *src, int32_t srcLength,

                   UNormalizationMode mode,

                   UErrorCode *pErrorCode);

/**

 * Test if a string is in a given normalization form; same as unorm_isNormalized but

 * takes an extra options parameter like most normalization functions.

 *

 * @param src        String that is to be tested if it is in a normalization format.

 * @param srcLength  Length of source to test, or -1 if NUL-terminated.

 * @param mode       Which normalization form to test for.

 * @param options    The normalization options, ORed together (0 for no options).

 * @param pErrorCode ICU error code in/out parameter.

 *                   Must fulfill U_SUCCESS before the function call.

 * @return Boolean value indicating whether the source string is in the

 *         "mode/options" normalization form.

 *

 * @see unorm_quickCheck

 * @see unorm_isNormalized

 * @stable ICU 2.6

 */

U_STABLE UBool U_EXPORT2

unorm_isNormalizedWithOptions(const UChar *src, int32_t srcLength,

                              UNormalizationMode mode, int32_t options,

                              UErrorCode *pErrorCode);

/**

 * Iterative normalization forward.

 * This function (together with unorm_previous) is somewhat

 * similar to the C++ Normalizer class (see its non-static functions).

 *

 * Iterative normalization is useful when only a small portion of a longer

 * string/text needs to be processed.

 *

 * For example, the likelihood may be high that processing the first 10% of some

 * text will be sufficient to find certain data.

 * Another example: When one wants to concatenate two normalized strings and get a

 * normalized result, it is much more efficient to normalize just a small part of

 * the result around the concatenation place instead of re-normalizing everything.

 *

 * The input text is an instance of the C character iteration API UCharIterator.

 * It may wrap around a simple string, a CharacterIterator, a Replaceable, or any

 * other kind of text object.

 *

 * If a buffer overflow occurs, then the caller needs to reset the iterator to the

 * old index and call the function again with a larger buffer - if the caller cares

 * for the actual output.

 * Regardless of the output buffer, the iterator will always be moved to the next

 * normalization boundary.

 *

 * This function (like unorm_previous) serves two purposes:

 *

 * 1) To find the next boundary so that the normalization of the part of the text

 * from the current position to that boundary does not affect and is not affected

 * by the part of the text beyond that boundary.

 *

 * 2) To normalize the text up to the boundary.

 *

 * The second step is optional, per the doNormalize parameter.

 * It is omitted for operations like string concatenation, where the two adjacent

 * string ends need to be normalized together.

 * In such a case, the output buffer will just contain a copy of the text up to the

 * boundary.

 *

 * pNeededToNormalize is an output-only parameter. Its output value is only defined

 * if normalization was requested (doNormalize) and successful (especially, no

 * buffer overflow).

 * It is useful for operations like a normalizing transliterator, where one would

 * not want to replace a piece of text if it is not modified.

 *

 * If doNormalize==TRUE and pNeededToNormalize!=NULL then *pNeeded... is set TRUE

 * if the normalization was necessary.

 *

 * If doNormalize==FALSE then *pNeededToNormalize will be set to FALSE.

 *

 * If the buffer overflows, then *pNeededToNormalize will be undefined;

 * essentially, whenever U_FAILURE is true (like in buffer overflows), this result

 * will be undefined.

 *

 * @param src The input text in the form of a C character iterator.

 * @param dest The output buffer; can be NULL if destCapacity==0 for pure preflighting.

 * @param destCapacity The number of UChars that fit into dest.

 * @param mode The normalization mode.

 * @param options The normalization options, ORed together (0 for no options).

 * @param doNormalize Indicates if the source text up to the next boundary

 *                    is to be normalized (TRUE) or just copied (FALSE).

 * @param pNeededToNormalize Output flag indicating if the normalization resulted in

 *                           different text from the input.

 *                           Not defined if an error occurs including buffer overflow.

 *                           Always FALSE if !doNormalize.

 * @param pErrorCode ICU error code in/out parameter.

 *                   Must fulfill U_SUCCESS before the function call.

 * @return Length of output (number of UChars) when successful or buffer overflow.

 *

 * @see unorm_previous

 * @see unorm_normalize

 *

 * @stable ICU 2.1

 */

U_STABLE int32_t U_EXPORT2

unorm_next(UCharIterator *src,

           UChar *dest, int32_t destCapacity,

           UNormalizationMode mode, int32_t options,

           UBool doNormalize, UBool *pNeededToNormalize,

           UErrorCode *pErrorCode);

/**

 * Iterative normalization backward.

 * This function (together with unorm_next) is somewhat

 * similar to the C++ Normalizer class (see its non-static functions).

 * For all details see unorm_next.

 *

 * @param src The input text in the form of a C character iterator.

 * @param dest The output buffer; can be NULL if destCapacity==0 for pure preflighting.

 * @param destCapacity The number of UChars that fit into dest.

 * @param mode The normalization mode.

 * @param options The normalization options, ORed together (0 for no options).

 * @param doNormalize Indicates if the source text up to the next boundary

 *                    is to be normalized (TRUE) or just copied (FALSE).

 * @param pNeededToNormalize Output flag indicating if the normalization resulted in

 *                           different text from the input.

 *                           Not defined if an error occurs including buffer overflow.

 *                           Always FALSE if !doNormalize.

 * @param pErrorCode ICU error code in/out parameter.

 *                   Must fulfill U_SUCCESS before the function call.

 * @return Length of output (number of UChars) when successful or buffer overflow.

 *

 * @see unorm_next

 * @see unorm_normalize

 *

 * @stable ICU 2.1

 */

U_STABLE int32_t U_EXPORT2

unorm_previous(UCharIterator *src,

               UChar *dest, int32_t destCapacity,

               UNormalizationMode mode, int32_t options,

               UBool doNormalize, UBool *pNeededToNormalize,

               UErrorCode *pErrorCode);

/**

 * Concatenate normalized strings, making sure that the result is normalized as well.

 *

 * If both the left and the right strings are in

 * the normalization form according to "mode/options",

 * then the result will be

 *

 * \code

 *     dest=normalize(left+right, mode, options)

 * \endcode

 *

 * With the input strings already being normalized,

 * this function will use unorm_next() and unorm_previous()

 * to find the adjacent end pieces of the input strings.

 * Only the concatenation of these end pieces will be normalized and

 * then concatenated with the remaining parts of the input strings.

 *

 * It is allowed to have dest==left to avoid copying the entire left string.

 *

 * @param left Left source string, may be same as dest.

 * @param leftLength Length of left source string, or -1 if NUL-terminated.

 * @param right Right source string.

 * @param rightLength Length of right source string, or -1 if NUL-terminated.

 * @param dest The output buffer; can be NULL if destCapacity==0 for pure preflighting.

 * @param destCapacity The number of UChars that fit into dest.

 * @param mode The normalization mode.

 * @param options The normalization options, ORed together (0 for no options).

 * @param pErrorCode ICU error code in/out parameter.

 *                   Must fulfill U_SUCCESS before the function call.

 * @return Length of output (number of UChars) when successful or buffer overflow.

 *

 * @see unorm_normalize

 * @see unorm_next

 * @see unorm_previous

 *

 * @stable ICU 2.1

 */

U_STABLE int32_t U_EXPORT2

unorm_concatenate(const UChar *left, int32_t leftLength,

                  const UChar *right, int32_t rightLength,

                  UChar *dest, int32_t destCapacity,

                  UNormalizationMode mode, int32_t options,

                  UErrorCode *pErrorCode);

/**

 * Option bit for unorm_compare:

 * Both input strings are assumed to fulfill FCD conditions.

 * @stable ICU 2.2

 */

#define UNORM_INPUT_IS_FCD          0x20000

/**

 * Option bit for unorm_compare:

 * Perform case-insensitive comparison.

 * @stable ICU 2.2

 */

#define U_COMPARE_IGNORE_CASE       0x10000

#ifndef U_COMPARE_CODE_POINT_ORDER

/* see also unistr.h and ustring.h */

/**

 * Option bit for u_strCaseCompare, u_strcasecmp, unorm_compare, etc:

 * Compare strings in code point order instead of code unit order.

 * @stable ICU 2.2

 */

#define U_COMPARE_CODE_POINT_ORDER  0x8000

#endif

/**

 * Compare two strings for canonical equivalence.

 * Further options include case-insensitive comparison and

 * code point order (as opposed to code unit order).

 *

 * Canonical equivalence between two strings is defined as their normalized

 * forms (NFD or NFC) being identical.

 * This function compares strings incrementally instead of normalizing

 * (and optionally case-folding) both strings entirely,

 * improving performance significantly.

 *

 * Bulk normalization is only necessary if the strings do not fulfill the FCD

 * conditions. Only in this case, and only if the strings are relatively long,

 * is memory allocated temporarily.

 * For FCD strings and short non-FCD strings there is no memory allocation.

 *

 * Semantically, this is equivalent to

 *   strcmp[CodePointOrder](NFD(foldCase(NFD(s1))), NFD(foldCase(NFD(s2))))

 * where code point order and foldCase are all optional.

 *

 * UAX 21 2.5 Caseless Matching specifies that for a canonical caseless match

 * the case folding must be performed first, then the normalization.

 *

 * @param s1 First source string.

 * @param length1 Length of first source string, or -1 if NUL-terminated.

 *

 * @param s2 Second source string.

 * @param length2 Length of second source string, or -1 if NUL-terminated.