1 /*

     2 ** 2004 April 13

     3 **

     4 ** The author disclaims copyright to this source code.  In place of

     5 ** a legal notice, here is a blessing:

     6 **

     7 **    May you do good and not evil.

     8 **    May you find forgiveness for yourself and forgive others.

     9 **    May you share freely, never taking more than you give.

    10 **

    11 *************************************************************************

    12 ** This file contains routines used to translate between UTF-8, 

    13 ** UTF-16, UTF-16BE, and UTF-16LE.

    14 **

    15 ** $Id: utf.c,v 1.40 2006/07/26 14:57:30 drh Exp $

    16 **

    17 ** Notes on UTF-8:

    18 **

    19 **   Byte-0    Byte-1    Byte-2    Byte-3    Value

    20 **  0xxxxxxx                                 00000000 00000000 0xxxxxxx

    21 **  110yyyyy  10xxxxxx                       00000000 00000yyy yyxxxxxx

    22 **  1110zzzz  10yyyyyy  10xxxxxx             00000000 zzzzyyyy yyxxxxxx

    23 **  11110uuu  10uuzzzz  10yyyyyy  10xxxxxx   000uuuuu zzzzyyyy yyxxxxxx

    24 **

    25 **

    26 ** Notes on UTF-16:  (with wwww+1==uuuuu)

    27 **

    28 **      Word-0               Word-1          Value

    29 **  110110ww wwzzzzyy   110111yy yyxxxxxx    000uuuuu zzzzyyyy yyxxxxxx

    30 **  zzzzyyyy yyxxxxxx                        00000000 zzzzyyyy yyxxxxxx

    31 **

    32 **

    33 ** BOM or Byte Order Mark:

    34 **     0xff 0xfe   little-endian utf-16 follows

    35 **     0xfe 0xff   big-endian utf-16 follows

    36 **

    37 **

    38 ** Handling of malformed strings:

    39 **

    40 ** SQLite accepts and processes malformed strings without an error wherever

    41 ** possible. However this is not possible when converting between UTF-8 and

    42 ** UTF-16.

    43 **

    44 ** When converting malformed UTF-8 strings to UTF-16, one instance of the

    45 ** replacement character U+FFFD for each byte that cannot be interpeted as

    46 ** part of a valid unicode character.

    47 **

    48 ** When converting malformed UTF-16 strings to UTF-8, one instance of the

    49 ** replacement character U+FFFD for each pair of bytes that cannot be

    50 ** interpeted as part of a valid unicode character.

    51 **

    52 ** This file contains the following public routines:

    53 **

    54 ** sqlite3VdbeMemTranslate() - Translate the encoding used by a Mem* string.

    55 ** sqlite3VdbeMemHandleBom() - Handle byte-order-marks in UTF16 Mem* strings.

    56 ** sqlite3utf16ByteLen()     - Calculate byte-length of a void* UTF16 string.

    57 ** sqlite3utf8CharLen()      - Calculate char-length of a char* UTF8 string.

    58 ** sqlite3utf8LikeCompare()  - Do a LIKE match given two UTF8 char* strings.

    59 **

    60 */

    61 #include "sqliteInt.h"

    62 #include <assert.h>

    63 #include "vdbeInt.h"

    64 

    65 /*

    66 ** This table maps from the first byte of a UTF-8 character to the number

    67 ** of trailing bytes expected. A value '255' indicates that the table key

    68 ** is not a legal first byte for a UTF-8 character.

    69 */

    70 static const u8 xtra_utf8_bytes[256]  = {

    71 /* 0xxxxxxx */

    72 0, 0, 0, 0, 0, 0, 0, 0,     0, 0, 0, 0, 0, 0, 0, 0,

    73 0, 0, 0, 0, 0, 0, 0, 0,     0, 0, 0, 0, 0, 0, 0, 0,

    74 0, 0, 0, 0, 0, 0, 0, 0,     0, 0, 0, 0, 0, 0, 0, 0,

    75 0, 0, 0, 0, 0, 0, 0, 0,     0, 0, 0, 0, 0, 0, 0, 0,

    76 0, 0, 0, 0, 0, 0, 0, 0,     0, 0, 0, 0, 0, 0, 0, 0,

    77 0, 0, 0, 0, 0, 0, 0, 0,     0, 0, 0, 0, 0, 0, 0, 0,

    78 0, 0, 0, 0, 0, 0, 0, 0,     0, 0, 0, 0, 0, 0, 0, 0,

    79 0, 0, 0, 0, 0, 0, 0, 0,     0, 0, 0, 0, 0, 0, 0, 0,

    80 

    81 /* 10wwwwww */

    82 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,

    83 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,

    84 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,

    85 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,

    86 

    87 /* 110yyyyy */

    88 1, 1, 1, 1, 1, 1, 1, 1,     1, 1, 1, 1, 1, 1, 1, 1,

    89 1, 1, 1, 1, 1, 1, 1, 1,     1, 1, 1, 1, 1, 1, 1, 1,

    90 

    91 /* 1110zzzz */

    92 2, 2, 2, 2, 2, 2, 2, 2,     2, 2, 2, 2, 2, 2, 2, 2,

    93 

    94 /* 11110yyy */

    95 3, 3, 3, 3, 3, 3, 3, 3,     255, 255, 255, 255, 255, 255, 255, 255,

    96 };

    97 

    98 /*

    99 ** This table maps from the number of trailing bytes in a UTF-8 character

   100 ** to an integer constant that is effectively calculated for each character

   101 ** read by a naive implementation of a UTF-8 character reader. The code

   102 ** in the READ_UTF8 macro explains things best.

   103 */

   104 static const int xtra_utf8_bits[4] =  {

   105 0,

   106 12416,          /* (0xC0 << 6) + (0x80) */

   107 925824,         /* (0xE0 << 12) + (0x80 << 6) + (0x80) */

   108 63447168        /* (0xF0 << 18) + (0x80 << 12) + (0x80 << 6) + 0x80 */

   109 };

   110 

   111 #define READ_UTF8(zIn, c) { \

   112   int xtra;                                            \

   113   c = *(zIn)++;                                        \

   114   xtra = xtra_utf8_bytes[c];                           \

   115   switch( xtra ){                                      \

   116     case 255: c = (int)0xFFFD; break;                  \

   117     case 3: c = (c<<6) + *(zIn)++;                     \

   118     case 2: c = (c<<6) + *(zIn)++;                     \

   119     case 1: c = (c<<6) + *(zIn)++;                     \

   120     c -= xtra_utf8_bits[xtra];                         \

   121   }                                                    \

   122 }

   123 int sqlite3ReadUtf8(const unsigned char *z){

   124   int c;

   125   READ_UTF8(z, c);

   126   return c;

   127 }

   128 

   129 #define SKIP_UTF8(zIn) {                               \

   130   zIn += (xtra_utf8_bytes[*(u8 *)zIn] + 1);            \

   131 }

   132 

   133 #define WRITE_UTF8(zOut, c) {                          \

   134   if( c<0x00080 ){                                     \

   135     *zOut++ = (c&0xFF);                                \

   136   }                                                    \

   137   else if( c<0x00800 ){                                \

   138     *zOut++ = 0xC0 + ((c>>6)&0x1F);                    \

   139     *zOut++ = 0x80 + (c & 0x3F);                       \

   140   }                                                    \

   141   else if( c<0x10000 ){                                \

   142     *zOut++ = 0xE0 + ((c>>12)&0x0F);                   \

   143     *zOut++ = 0x80 + ((c>>6) & 0x3F);                  \

   144     *zOut++ = 0x80 + (c & 0x3F);                       \

   145   }else{                                               \

   146     *zOut++ = 0xF0 + ((c>>18) & 0x07);                 \

   147     *zOut++ = 0x80 + ((c>>12) & 0x3F);                 \

   148     *zOut++ = 0x80 + ((c>>6) & 0x3F);                  \

   149     *zOut++ = 0x80 + (c & 0x3F);                       \

   150   }                                                    \

   151 }

   152 

   153 #define WRITE_UTF16LE(zOut, c) {                                \

   154   if( c<=0xFFFF ){                                              \

   155     *zOut++ = (c&0x00FF);                                       \

   156     *zOut++ = ((c>>8)&0x00FF);                                  \

   157   }else{                                                        \

   158     *zOut++ = (((c>>10)&0x003F) + (((c-0x10000)>>10)&0x00C0));  \

   159     *zOut++ = (0x00D8 + (((c-0x10000)>>18)&0x03));              \

   160     *zOut++ = (c&0x00FF);                                       \

   161     *zOut++ = (0x00DC + ((c>>8)&0x03));                         \

   162   }                                                             \

   163 }

   164 

   165 #define WRITE_UTF16BE(zOut, c) {                                \

   166   if( c<=0xFFFF ){                                              \

   167     *zOut++ = ((c>>8)&0x00FF);                                  \

   168     *zOut++ = (c&0x00FF);                                       \

   169   }else{                                                        \

   170     *zOut++ = (0x00D8 + (((c-0x10000)>>18)&0x03));              \

   171     *zOut++ = (((c>>10)&0x003F) + (((c-0x10000)>>10)&0x00C0));  \

   172     *zOut++ = (0x00DC + ((c>>8)&0x03));                         \

   173     *zOut++ = (c&0x00FF);                                       \

   174   }                                                             \

   175 }

   176 

   177 #define READ_UTF16LE(zIn, c){                                         \

   178   c = (*zIn++);                                                       \

   179   c += ((*zIn++)<<8);                                                 \

   180   if( c>=0xD800 && c<=0xE000 ){                                       \

   181     int c2 = (*zIn++);                                                \

   182     c2 += ((*zIn++)<<8);                                              \

   183     c = (c2&0x03FF) + ((c&0x003F)<<10) + (((c&0x03C0)+0x0040)<<10);   \

   184   }                                                                   \

   185 }

   186 

   187 #define READ_UTF16BE(zIn, c){                                         \

   188   c = ((*zIn++)<<8);                                                  \

   189   c += (*zIn++);                                                      \

   190   if( c>=0xD800 && c<=0xE000 ){                                       \

   191     int c2 = ((*zIn++)<<8);                                           \

   192     c2 += (*zIn++);                                                   \

   193     c = (c2&0x03FF) + ((c&0x003F)<<10) + (((c&0x03C0)+0x0040)<<10);   \

   194   }                                                                   \

   195 }

   196 

   197 #define SKIP_UTF16BE(zIn){                                            \

   198   if( *zIn>=0xD8 && (*zIn<0xE0 || (*zIn==0xE0 && *(zIn+1)==0x00)) ){  \

   199     zIn += 4;                                                         \

   200   }else{                                                              \

   201     zIn += 2;                                                         \

   202   }                                                                   \

   203 }

   204 #define SKIP_UTF16LE(zIn){                                            \

   205   zIn++;                                                              \

   206   if( *zIn>=0xD8 && (*zIn<0xE0 || (*zIn==0xE0 && *(zIn-1)==0x00)) ){  \

   207     zIn += 3;                                                         \

   208   }else{                                                              \

   209     zIn += 1;                                                         \

   210   }                                                                   \

   211 }

   212 

   213 #define RSKIP_UTF16LE(zIn){                                            \

   214   if( *zIn>=0xD8 && (*zIn<0xE0 || (*zIn==0xE0 && *(zIn-1)==0x00)) ){  \

   215     zIn -= 4;                                                         \

   216   }else{                                                              \

   217     zIn -= 2;                                                         \

   218   }                                                                   \

   219 }

   220 #define RSKIP_UTF16BE(zIn){                                            \

   221   zIn--;                                                              \

   222   if( *zIn>=0xD8 && (*zIn<0xE0 || (*zIn==0xE0 && *(zIn+1)==0x00)) ){  \

   223     zIn -= 3;                                                         \

   224   }else{                                                              \

   225     zIn -= 1;                                                         \

   226   }                                                                   \

   227 }

   228 

   229 /*

   230 ** If the TRANSLATE_TRACE macro is defined, the value of each Mem is

   231 ** printed on stderr on the way into and out of sqlite3VdbeMemTranslate().

   232 */ 

   233 /* #define TRANSLATE_TRACE 1 */

   234 

   235 #ifndef SQLITE_OMIT_UTF16

   236 /*

   237 ** This routine transforms the internal text encoding used by pMem to

   238 ** desiredEnc. It is an error if the string is already of the desired

   239 ** encoding, or if *pMem does not contain a string value.

   240 */

   241 int sqlite3VdbeMemTranslate(Mem *pMem, u8 desiredEnc){

   242   unsigned char zShort[NBFS]; /* Temporary short output buffer */

   243   int len;                    /* Maximum length of output string in bytes */

   244   unsigned char *zOut;                  /* Output buffer */

   245   unsigned char *zIn;                   /* Input iterator */

   246   unsigned char *zTerm;                 /* End of input */

   247   unsigned char *z;                     /* Output iterator */

   248   int c;

   249 

   250   assert( pMem->flags&MEM_Str );

   251   assert( pMem->enc!=desiredEnc );

   252   assert( pMem->enc!=0 );

   253   assert( pMem->n>=0 );

   254 

   255 #if defined(TRANSLATE_TRACE) && defined(SQLITE_DEBUG)

   256   {

   257     char zBuf[100];

   258     sqlite3VdbeMemPrettyPrint(pMem, zBuf);

   259     fprintf(stderr, "INPUT:  %s\n", zBuf);

   260   }

   261 #endif

   262 

   263   /* If the translation is between UTF-16 little and big endian, then 

   264   ** all that is required is to swap the byte order. This case is handled

   265   ** differently from the others.

   266   */

   267   if( pMem->enc!=SQLITE_UTF8 && desiredEnc!=SQLITE_UTF8 ){

   268     u8 temp;

   269     int rc;

   270     rc = sqlite3VdbeMemMakeWriteable(pMem);

   271     if( rc!=SQLITE_OK ){

   272       assert( rc==SQLITE_NOMEM );

   273       return SQLITE_NOMEM;

   274     }

   275     zIn = (u8*)pMem->z;

   276     zTerm = &zIn[pMem->n];

   277     while( zIn<zTerm ){

   278       temp = *zIn;

   279       *zIn = *(zIn+1);

   280       zIn++;

   281       *zIn++ = temp;

   282     }

   283     pMem->enc = desiredEnc;

   284     goto translate_out;

   285   }

   286 

   287   /* Set len to the maximum number of bytes required in the output buffer. */

   288   if( desiredEnc==SQLITE_UTF8 ){

   289     /* When converting from UTF-16, the maximum growth results from

   290     ** translating a 2-byte character to a 4-byte UTF-8 character.

   291     ** A single byte is required for the output string

   292     ** nul-terminator.

   293     */

   294     len = pMem->n * 2 + 1;

   295   }else{

   296     /* When converting from UTF-8 to UTF-16 the maximum growth is caused

   297     ** when a 1-byte UTF-8 character is translated into a 2-byte UTF-16

   298     ** character. Two bytes are required in the output buffer for the

   299     ** nul-terminator.

   300     */

   301     len = pMem->n * 2 + 2;

   302   }

   303 

   304   /* Set zIn to point at the start of the input buffer and zTerm to point 1

   305   ** byte past the end.

   306   **

   307   ** Variable zOut is set to point at the output buffer. This may be space

   308   ** obtained from malloc(), or Mem.zShort, if it large enough and not in

   309   ** use, or the zShort array on the stack (see above).

   310   */

   311   zIn = (u8*)pMem->z;

   312   zTerm = &zIn[pMem->n];

   313   if( len>NBFS ){

   314     zOut = sqliteMallocRaw(len);

   315     if( !zOut ) return SQLITE_NOMEM;

   316   }else{

   317     zOut = zShort;

   318   }

   319   z = zOut;

   320 

   321   if( pMem->enc==SQLITE_UTF8 ){

   322     if( desiredEnc==SQLITE_UTF16LE ){

   323       /* UTF-8 -> UTF-16 Little-endian */

   324       while( zIn<zTerm ){

   325         READ_UTF8(zIn, c); 

   326         WRITE_UTF16LE(z, c);

   327       }

   328     }else{

   329       assert( desiredEnc==SQLITE_UTF16BE );

   330       /* UTF-8 -> UTF-16 Big-endian */

   331       while( zIn<zTerm ){

   332         READ_UTF8(zIn, c); 

   333         WRITE_UTF16BE(z, c);

   334       }

   335     }

   336     pMem->n = z - zOut;

   337     *z++ = 0;

   338   }else{

   339     assert( desiredEnc==SQLITE_UTF8 );

   340     if( pMem->enc==SQLITE_UTF16LE ){

   341       /* UTF-16 Little-endian -> UTF-8 */

   342       while( zIn<zTerm ){

   343         READ_UTF16LE(zIn, c); 

   344         WRITE_UTF8(z, c);

   345       }

   346     }else{

   347       /* UTF-16 Little-endian -> UTF-8 */

   348       while( zIn<zTerm ){

   349         READ_UTF16BE(zIn, c); 

   350         WRITE_UTF8(z, c);

   351       }

   352     }

   353     pMem->n = z - zOut;

   354   }

   355   *z = 0;

   356   assert( (pMem->n+(desiredEnc==SQLITE_UTF8?1:2))<=len );

   357 

   358   sqlite3VdbeMemRelease(pMem);

   359   pMem->flags &= ~(MEM_Static|MEM_Dyn|MEM_Ephem|MEM_Short);

   360   pMem->enc = desiredEnc;

   361   if( zOut==zShort ){

   362     memcpy(pMem->zShort, zOut, len);

   363     zOut = (u8*)pMem->zShort;

   364     pMem->flags |= (MEM_Term|MEM_Short);

   365   }else{

   366     pMem->flags |= (MEM_Term|MEM_Dyn);

   367   }

   368   pMem->z = (char*)zOut;

   369 

   370 translate_out:

   371 #if defined(TRANSLATE_TRACE) && defined(SQLITE_DEBUG)

   372   {

   373     char zBuf[100];

   374     sqlite3VdbeMemPrettyPrint(pMem, zBuf);

   375     fprintf(stderr, "OUTPUT: %s\n", zBuf);

   376   }

   377 #endif

   378   return SQLITE_OK;

   379 }

   380 

   381 /*

   382 ** This routine checks for a byte-order mark at the beginning of the 

   383 ** UTF-16 string stored in *pMem. If one is present, it is removed and

   384 ** the encoding of the Mem adjusted. This routine does not do any

   385 ** byte-swapping, it just sets Mem.enc appropriately.

   386 **

   387 ** The allocation (static, dynamic etc.) and encoding of the Mem may be

   388 ** changed by this function.

   389 */

   390 int sqlite3VdbeMemHandleBom(Mem *pMem){

   391   int rc = SQLITE_OK;

   392   u8 bom = 0;

   393 

   394   if( pMem->n<0 || pMem->n>1 ){

   395     u8 b1 = *(u8 *)pMem->z;

   396     u8 b2 = *(((u8 *)pMem->z) + 1);

   397     if( b1==0xFE && b2==0xFF ){

   398       bom = SQLITE_UTF16BE;

   399     }

   400     if( b1==0xFF && b2==0xFE ){

   401       bom = SQLITE_UTF16LE;

   402     }

   403   }

   404   

   405   if( bom ){

   406     /* This function is called as soon as a string is stored in a Mem*,

   407     ** from within sqlite3VdbeMemSetStr(). At that point it is not possible

   408     ** for the string to be stored in Mem.zShort, or for it to be stored

   409     ** in dynamic memory with no destructor.

   410     */

   411     assert( !(pMem->flags&MEM_Short) );

   412     assert( !(pMem->flags&MEM_Dyn) || pMem->xDel );

   413     if( pMem->flags & MEM_Dyn ){

   414       void (*xDel)(void*) = pMem->xDel;

   415       char *z = pMem->z;

   416       pMem->z = 0;

   417       pMem->xDel = 0;

   418       rc = sqlite3VdbeMemSetStr(pMem, &z[2], pMem->n-2, bom, SQLITE_TRANSIENT);

   419       xDel(z);

   420     }else{

   421       rc = sqlite3VdbeMemSetStr(pMem, &pMem->z[2], pMem->n-2, bom, 

   422           SQLITE_TRANSIENT);

   423     }

   424   }

   425   return rc;

   426 }

   427 #endif /* SQLITE_OMIT_UTF16 */

   428 

   429 /*

   430 ** pZ is a UTF-8 encoded unicode string. If nByte is less than zero,

   431 ** return the number of unicode characters in pZ up to (but not including)

   432 ** the first 0x00 byte. If nByte is not less than zero, return the

   433 ** number of unicode characters in the first nByte of pZ (or up to 

   434 ** the first 0x00, whichever comes first).

   435 */

   436 int sqlite3utf8CharLen(const char *z, int nByte){

   437   int r = 0;

   438   const char *zTerm;

   439   if( nByte>=0 ){

   440     zTerm = &z[nByte];

   441   }else{

   442     zTerm = (const char *)(-1);

   443   }

   444   assert( z<=zTerm );

   445   while( *z!=0 && z<zTerm ){

   446     SKIP_UTF8(z);

   447     r++;

   448   }

   449   return r;

   450 }

   451 

   452 #ifndef SQLITE_OMIT_UTF16

   453 /*

   454 ** Convert a UTF-16 string in the native encoding into a UTF-8 string.

   455 ** Memory to hold the UTF-8 string is obtained from malloc and must be

   456 ** freed by the calling function.

   457 **

   458 ** NULL is returned if there is an allocation error.

   459 */

   460 char *sqlite3utf16to8(const void *z, int nByte){

   461   Mem m;

   462   memset(&m, 0, sizeof(m));

   463   sqlite3VdbeMemSetStr(&m, z, nByte, SQLITE_UTF16NATIVE, SQLITE_STATIC);

   464   sqlite3VdbeChangeEncoding(&m, SQLITE_UTF8);

   465   assert( (m.flags & MEM_Term)!=0 || sqlite3MallocFailed() );

   466   assert( (m.flags & MEM_Str)!=0 || sqlite3MallocFailed() );

   467   return (m.flags & MEM_Dyn)!=0 ? m.z : sqliteStrDup(m.z);

   468 }

   469 

   470 /*

   471 ** pZ is a UTF-16 encoded unicode string. If nChar is less than zero,

   472 ** return the number of bytes up to (but not including), the first pair

   473 ** of consecutive 0x00 bytes in pZ. If nChar is not less than zero,

   474 ** then return the number of bytes in the first nChar unicode characters

   475 ** in pZ (or up until the first pair of 0x00 bytes, whichever comes first).

   476 */

   477 int sqlite3utf16ByteLen(const void *zIn, int nChar){

   478   int c = 1;

   479   char const *z = zIn;

   480   int n = 0;

   481   if( SQLITE_UTF16NATIVE==SQLITE_UTF16BE ){

   482     /* Using an "if (SQLITE_UTF16NATIVE==SQLITE_UTF16BE)" construct here

   483     ** and in other parts of this file means that at one branch will

   484     ** not be covered by coverage testing on any single host. But coverage

   485     ** will be complete if the tests are run on both a little-endian and 

   486     ** big-endian host. Because both the UTF16NATIVE and SQLITE_UTF16BE

   487     ** macros are constant at compile time the compiler can determine

   488     ** which branch will be followed. It is therefore assumed that no runtime

   489     ** penalty is paid for this "if" statement.

   490     */

   491     while( c && ((nChar<0) || n<nChar) ){

   492       READ_UTF16BE(z, c);

   493       n++;

   494     }

   495   }else{

   496     while( c && ((nChar<0) || n<nChar) ){

   497       READ_UTF16LE(z, c);

   498       n++;

   499     }

   500   }

   501   return (z-(char const *)zIn)-((c==0)?2:0);

   502 }

   503 

   504 /*

   505 ** UTF-16 implementation of the substr()

   506 */

   507 void sqlite3utf16Substr(

   508   sqlite3_context *context,

   509   int argc,

   510   sqlite3_value **argv

   511 ){

   512   int y, z;

   513   unsigned char const *zStr;

   514   unsigned char const *zStrEnd;

   515   unsigned char const *zStart;

   516   unsigned char const *zEnd;

   517   int i;

   518 

   519   zStr = (unsigned char const *)sqlite3_value_text16(argv[0]);

   520   zStrEnd = &zStr[sqlite3_value_bytes16(argv[0])];

   521   y = sqlite3_value_int(argv[1]);

   522   z = sqlite3_value_int(argv[2]);

   523 

   524   if( y>0 ){

   525     y = y-1;

   526     zStart = zStr;

   527     if( SQLITE_UTF16BE==SQLITE_UTF16NATIVE ){

   528       for(i=0; i<y && zStart<zStrEnd; i++) SKIP_UTF16BE(zStart);

   529     }else{

   530       for(i=0; i<y && zStart<zStrEnd; i++) SKIP_UTF16LE(zStart);

   531     }

   532   }else{

   533     zStart = zStrEnd;

   534     if( SQLITE_UTF16BE==SQLITE_UTF16NATIVE ){

   535       for(i=y; i<0 && zStart>zStr; i++) RSKIP_UTF16BE(zStart);

   536     }else{

   537       for(i=y; i<0 && zStart>zStr; i++) RSKIP_UTF16LE(zStart);

   538     }

   539     for(; i<0; i++) z -= 1;

   540   }

   541 

   542   zEnd = zStart;

   543   if( SQLITE_UTF16BE==SQLITE_UTF16NATIVE ){

   544     for(i=0; i<z && zEnd<zStrEnd; i++) SKIP_UTF16BE(zEnd);

   545   }else{

   546     for(i=0; i<z && zEnd<zStrEnd; i++) SKIP_UTF16LE(zEnd);

   547   }

   548 

   549   sqlite3_result_text16(context, zStart, zEnd-zStart, SQLITE_TRANSIENT);

   550 }

   551 

   552 #if defined(SQLITE_TEST)

   553 /*

   554 ** This routine is called from the TCL test function "translate_selftest".

   555 ** It checks that the primitives for serializing and deserializing

   556 ** characters in each encoding are inverses of each other.

   557 */

   558 void sqlite3utfSelfTest(){

   559   int i;

   560   unsigned char zBuf[20];

   561   unsigned char *z;

   562   int n;

   563   int c;

   564 

   565   for(i=0; i<0x00110000; i++){

   566     z = zBuf;

   567     WRITE_UTF8(z, i);

   568     n = z-zBuf;

   569     z = zBuf;

   570     READ_UTF8(z, c);

   571     assert( c==i );

   572     assert( (z-zBuf)==n );

   573   }

   574   for(i=0; i<0x00110000; i++){

   575     if( i>=0xD800 && i<=0xE000 ) continue;

   576     z = zBuf;

   577     WRITE_UTF16LE(z, i);

   578     n = z-zBuf;

   579     z = zBuf;

   580     READ_UTF16LE(z, c);

   581     assert( c==i );

   582     assert( (z-zBuf)==n );

   583   }

   584   for(i=0; i<0x00110000; i++){

   585     if( i>=0xD800 && i<=0xE000 ) continue;

   586     z = zBuf;

   587     WRITE_UTF16BE(z, i);

   588     n = z-zBuf;

   589     z = zBuf;

   590     READ_UTF16BE(z, c);

   591     assert( c==i );

   592     assert( (z-zBuf)==n );

   593   }

   594 }

   595 #endif /* SQLITE_TEST */

   596 #endif /* SQLITE_OMIT_UTF16 */