1 /****************************************************************************

     2 **

     3 ** Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies).

     4 ** All rights reserved.

     5 ** Contact: Nokia Corporation (qt-info@nokia.com)

     6 **

     7 ** This file is part of the QtCore module of the Qt Toolkit.

     8 **

     9 ** $QT_BEGIN_LICENSE:LGPL$

    10 ** No Commercial Usage

    11 ** This file contains pre-release code and may not be distributed.

    12 ** You may use this file in accordance with the terms and conditions

    13 ** contained in the Technology Preview License Agreement accompanying

    14 ** this package.

    15 **

    16 ** GNU Lesser General Public License Usage

    17 ** Alternatively, this file may be used under the terms of the GNU Lesser

    18 ** General Public License version 2.1 as published by the Free Software

    19 ** Foundation and appearing in the file LICENSE.LGPL included in the

    20 ** packaging of this file.  Please review the following information to

    21 ** ensure the GNU Lesser General Public License version 2.1 requirements

    22 ** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.

    23 **

    24 ** In addition, as a special exception, Nokia gives you certain additional

    25 ** rights.  These rights are described in the Nokia Qt LGPL Exception

    26 ** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.

    27 **

    28 ** If you have questions regarding the use of this file, please contact

    29 ** Nokia at qt-info@nokia.com.

    30 **

    31 **

    32 **

    33 **

    34 **

    35 **

    36 **

    37 **

    38 ** $QT_END_LICENSE$

    39 **

    40 ****************************************************************************/

    41 

    42 #include "qiconvcodec_p.h"

    43 #include "qtextcodec_p.h"

    44 #include <qlibrary.h>

    45 #include <qdebug.h>

    46 #include <qthreadstorage.h>

    47 

    48 #include <errno.h>

    49 #include <locale.h>

    50 #include <stdio.h>

    51 #include <dlfcn.h>

    52 

    53 // unistd.h is needed for the _XOPEN_UNIX macro

    54 #include <unistd.h>

    55 #if defined(_XOPEN_UNIX) && !defined(Q_OS_QNX) && !defined(Q_OS_OSF)

    56 #  include <langinfo.h>

    57 #endif

    58 

    59 #if defined(Q_OS_HPUX)

    60 #  define NO_BOM

    61 #  define UTF16 "ucs2"

    62 #elif defined(Q_OS_AIX)

    63 #  define NO_BOM

    64 #  define UTF16 "UCS-2"

    65 #elif defined(Q_OS_FREEBSD) || defined(Q_OS_MAC)

    66 #  define NO_BOM

    67 #  if Q_BYTE_ORDER == Q_BIG_ENDIAN

    68 #    define UTF16 "UTF-16BE"

    69 #  else

    70 #    define UTF16 "UTF-16LE"

    71 #  endif

    72 #else

    73 #  define UTF16 "UTF-16"

    74 #endif

    75 

    76 #if defined(Q_OS_MAC)

    77 #ifndef GNU_LIBICONV

    78 #define GNU_LIBICONV

    79 #endif

    80 typedef iconv_t (*Ptr_iconv_open) (const char*, const char*);

    81 typedef size_t (*Ptr_iconv) (iconv_t, const char **, size_t *, char **, size_t *);

    82 typedef int (*Ptr_iconv_close) (iconv_t);

    83 

    84 static Ptr_iconv_open ptr_iconv_open = 0;

    85 static Ptr_iconv ptr_iconv = 0;

    86 static Ptr_iconv_close ptr_iconv_close = 0;

    87 #endif

    88 

    89 QT_BEGIN_NAMESPACE

    90 

    91 extern bool qt_locale_initialized;

    92 

    93 QIconvCodec::QIconvCodec()

    94     : utf16Codec(0)

    95 {

    96     utf16Codec = QTextCodec::codecForMib(1015);

    97     Q_ASSERT_X(utf16Codec != 0,

    98                "QIconvCodec::convertToUnicode",

    99                "internal error, UTF-16 codec not found");

   100     if (!utf16Codec) {

   101         fprintf(stderr, "QIconvCodec::convertToUnicode: internal error, UTF-16 codec not found\n");

   102         utf16Codec = reinterpret_cast<QTextCodec *>(~0);

   103     }

   104 #if defined(Q_OS_MAC)

   105     if (ptr_iconv_open == 0) {

   106         QLibrary libiconv(QLatin1String("/usr/lib/libiconv"));

   107         libiconv.setLoadHints(QLibrary::ExportExternalSymbolsHint);

   108 

   109         ptr_iconv_open = reinterpret_cast<Ptr_iconv_open>(libiconv.resolve("libiconv_open"));

   110         if (!ptr_iconv_open)

   111             ptr_iconv_open = reinterpret_cast<Ptr_iconv_open>(libiconv.resolve("iconv_open"));

   112         ptr_iconv = reinterpret_cast<Ptr_iconv>(libiconv.resolve("libiconv"));

   113         if (!ptr_iconv)

   114             ptr_iconv = reinterpret_cast<Ptr_iconv>(libiconv.resolve("iconv"));

   115         ptr_iconv_close = reinterpret_cast<Ptr_iconv_close>(libiconv.resolve("libiconv_close"));

   116         if (!ptr_iconv_close)

   117             ptr_iconv_close = reinterpret_cast<Ptr_iconv_close>(libiconv.resolve("iconv_close"));

   118 

   119         Q_ASSERT_X(ptr_iconv_open && ptr_iconv && ptr_iconv_close,

   120         "QIconvCodec::QIconvCodec()",

   121         "internal error, could not resolve the iconv functions");

   122 

   123 #       undef iconv_open

   124 #       define iconv_open ptr_iconv_open

   125 #       undef iconv

   126 #       define iconv ptr_iconv

   127 #       undef iconv_close

   128 #       define iconv_close ptr_iconv_close

   129     }

   130 #endif

   131 }

   132 

   133 QIconvCodec::~QIconvCodec()

   134 {

   135 }

   136 

   137 QIconvCodec::IconvState::IconvState(iconv_t x)

   138     : buffer(array), bufferLen(sizeof array), cd(x)

   139 {

   140 }

   141 

   142 QIconvCodec::IconvState::~IconvState()

   143 {

   144     if (cd != reinterpret_cast<iconv_t>(-1))

   145         iconv_close(cd);

   146     if (buffer != array)

   147         delete[] buffer;

   148 }

   149 

   150 void QIconvCodec::IconvState::saveChars(const char *c, int count)

   151 {

   152     if (count > bufferLen) {

   153         if (buffer != array)

   154             delete[] buffer;

   155         buffer = new char[bufferLen = count];

   156     }

   157 

   158     memcpy(buffer, c, count);

   159 }

   160 

   161 static void qIconvCodecStateFree(QTextCodec::ConverterState *state)

   162 {

   163     delete reinterpret_cast<QIconvCodec::IconvState *>(state->d);

   164 }

   165 

   166 Q_GLOBAL_STATIC(QThreadStorage<QIconvCodec::IconvState *>, toUnicodeState)

   167 

   168 QString QIconvCodec::convertToUnicode(const char* chars, int len, ConverterState *convState) const

   169 {

   170     if (utf16Codec == reinterpret_cast<QTextCodec *>(~0))

   171         return QString::fromAscii(chars, len);

   172 

   173     int invalidCount = 0;

   174     int remainingCount = 0;

   175     char *remainingBuffer = 0;

   176     IconvState **pstate;

   177 

   178     if (convState) {

   179         // stateful conversion

   180         pstate = reinterpret_cast<IconvState **>(&convState->d);

   181         if (convState->d) {

   182             // restore state

   183             remainingCount = convState->remainingChars;

   184             remainingBuffer = (*pstate)->buffer;

   185         } else {

   186             // first time

   187             convState->flags |= FreeFunction;

   188             QTextCodecUnalignedPointer::encode(convState->state_data, qIconvCodecStateFree);

   189         }

   190     } else {

   191         QThreadStorage<QIconvCodec::IconvState *> *ts = toUnicodeState();

   192         if (!qt_locale_initialized || !ts) {

   193             // we're running after the Q_GLOBAL_STATIC has been deleted

   194             // or before the QCoreApplication initialization

   195             // bad programmer, no cookie for you

   196             return QString::fromLatin1(chars, len);

   197         }

   198 

   199         // stateless conversion -- use thread-local data

   200         pstate = &toUnicodeState()->localData();

   201     }

   202 

   203     if (!*pstate) {

   204         // first time, create the state

   205         iconv_t cd = QIconvCodec::createIconv_t(UTF16, 0);

   206         if (cd == reinterpret_cast<iconv_t>(-1)) {

   207             static int reported = 0;

   208             if (!reported++) {

   209                 fprintf(stderr,

   210                         "QIconvCodec::convertToUnicode: using ASCII for conversion, iconv_open failed\n");

   211             }

   212             return QString::fromAscii(chars, len);

   213         }

   214 

   215         *pstate = new IconvState(cd);

   216     }

   217 

   218     IconvState *state = *pstate;

   219     size_t inBytesLeft = len;

   220     // best case assumption, each byte is converted into one UTF-16 character, plus 2 bytes for the BOM

   221 #ifdef GNU_LIBICONV

   222     // GNU doesn't disagree with POSIX :/

   223     const char *inBytes = chars;

   224 #else

   225     char *inBytes = const_cast<char *>(chars);

   226 #endif

   227 

   228     QByteArray in;

   229     if (remainingCount) {

   230         // we have to prepend the remaining bytes from the previous conversion

   231         inBytesLeft += remainingCount;

   232         in.resize(inBytesLeft);

   233         inBytes = in.data();

   234 

   235         memcpy(in.data(), remainingBuffer, remainingCount);

   236         memcpy(in.data() + remainingCount, chars, len);

   237 

   238         remainingCount = 0;

   239     }

   240 

   241     size_t outBytesLeft = len * 2 + 2;

   242     QByteArray ba(outBytesLeft, Qt::Uninitialized);

   243     char *outBytes = ba.data();

   244     do {

   245         size_t ret = iconv(state->cd, &inBytes, &inBytesLeft, &outBytes, &outBytesLeft);

   246         if (ret == (size_t) -1) {

   247             if (errno == E2BIG) {

   248                 int offset = ba.size() - outBytesLeft;

   249                 ba.resize(ba.size() * 2);

   250                 outBytes = ba.data() + offset;

   251                 outBytesLeft = ba.size() - offset;

   252 

   253                 continue;

   254             }

   255 

   256             if (errno == EILSEQ) {

   257                 // conversion stopped because of an invalid character in the sequence

   258                 ++invalidCount;

   259             } else if (errno == EINVAL && convState) {

   260                 // conversion stopped because the remaining inBytesLeft make up

   261                 // an incomplete multi-byte sequence; save them for later

   262                 state->saveChars(inBytes, inBytesLeft);

   263                 remainingCount = inBytesLeft;

   264                 break;

   265             }

   266 

   267             if (errno == EILSEQ || errno == EINVAL) {

   268                 // skip the next character

   269                 ++inBytes;

   270                 --inBytesLeft;

   271                 continue;

   272             }

   273 

   274             // some other error

   275             // note, cannot use qWarning() since we are implementing the codecForLocale :)

   276             perror("QIconvCodec::convertToUnicode: using ASCII for conversion, iconv failed");

   277 

   278             if (!convState) {

   279                 // reset state

   280                 iconv(state->cd, 0, &inBytesLeft, 0, &outBytesLeft);

   281             }

   282 

   283             return QString::fromAscii(chars, len);

   284         }

   285     } while (inBytesLeft != 0);

   286 

   287     QString s = utf16Codec->toUnicode(ba.constData(), ba.size() - outBytesLeft);

   288 

   289     if (convState) {

   290         convState->invalidChars = invalidCount;

   291         convState->remainingChars = remainingCount;

   292     } else {

   293         // reset state

   294         iconv(state->cd, 0, &inBytesLeft, 0, &outBytesLeft);

   295     }

   296 

   297     return s;

   298 }

   299 

   300 Q_GLOBAL_STATIC(QThreadStorage<QIconvCodec::IconvState *>, fromUnicodeState)

   301 

   302 QByteArray QIconvCodec::convertFromUnicode(const QChar *uc, int len, ConverterState *convState) const

   303 {

   304     char *inBytes;

   305     char *outBytes;

   306     size_t inBytesLeft;

   307 

   308 #if defined(GNU_LIBICONV)

   309     const char **inBytesPtr = const_cast<const char **>(&inBytes);

   310 #else

   311     char **inBytesPtr = &inBytes;

   312 #endif

   313 

   314     QThreadStorage<QIconvCodec::IconvState *> *ts = fromUnicodeState();

   315     if (!qt_locale_initialized || !ts) {

   316         // we're running after the Q_GLOBAL_STATIC has been deleted

   317         // or before the QCoreApplication initialization

   318         // bad programmer, no cookie for you

   319         if (!len)

   320             // this is a special case - zero-sized string should be

   321             // translated to empty but not-null QByteArray.

   322             return QByteArray("");

   323         return QString::fromRawData(uc, len).toLatin1();

   324     }

   325     IconvState *&state = ts->localData();

   326     if (!state) {

   327         state = new IconvState(QIconvCodec::createIconv_t(0, UTF16));

   328         if (state->cd != reinterpret_cast<iconv_t>(-1)) {

   329             size_t outBytesLeft = len + 3; // +3 for the BOM

   330             QByteArray ba(outBytesLeft, Qt::Uninitialized);

   331             outBytes = ba.data();

   332 

   333 #if !defined(NO_BOM)

   334             // give iconv() a BOM

   335             QChar bom[] = { QChar(QChar::ByteOrderMark) };

   336             inBytes = reinterpret_cast<char *>(bom);

   337             inBytesLeft = sizeof(bom);

   338             if (iconv(state->cd, inBytesPtr, &inBytesLeft, &outBytes, &outBytesLeft) == (size_t) -1) {

   339                 perror("QIconvCodec::convertFromUnicode: using ASCII for conversion, iconv failed for BOM");

   340 

   341                 iconv_close(state->cd);

   342                 state->cd = reinterpret_cast<iconv_t>(-1);

   343 

   344                 return QString(uc, len).toAscii();

   345             }

   346 #endif // NO_BOM

   347         }

   348     }

   349     if (state->cd == reinterpret_cast<iconv_t>(-1)) {

   350         static int reported = 0;

   351         if (!reported++) {

   352             fprintf(stderr,

   353                     "QIconvCodec::convertFromUnicode: using ASCII for conversion, iconv_open failed\n");

   354         }

   355         return QString(uc, len).toAscii();

   356     }

   357  

   358     size_t outBytesLeft = len;

   359     QByteArray ba(outBytesLeft, Qt::Uninitialized);

   360     outBytes = ba.data();

   361 

   362     // now feed iconv() the real data

   363     inBytes = const_cast<char *>(reinterpret_cast<const char *>(uc));

   364     inBytesLeft = len * sizeof(QChar);

   365 

   366     QByteArray in;

   367     if (convState && convState->remainingChars) {

   368         // we have one surrogate char to be prepended

   369         in.resize(sizeof(QChar) + len);

   370         inBytes = in.data();

   371 

   372         QChar remaining = convState->state_data[0];

   373         memcpy(in.data(), &remaining, sizeof(QChar));

   374         memcpy(in.data() + sizeof(QChar), uc, inBytesLeft);

   375 

   376         inBytesLeft += sizeof(QChar);

   377         convState->remainingChars = 0;

   378     }

   379 

   380     int invalidCount = 0;

   381     do {

   382         if (iconv(state->cd, inBytesPtr, &inBytesLeft, &outBytes, &outBytesLeft) == (size_t) -1) {

   383             if (errno == EINVAL && convState) {

   384                 // buffer ends in a surrogate

   385                 Q_ASSERT(inBytesLeft == 2);

   386                 convState->remainingChars = 1;

   387                 convState->state_data[0] = uc[len - 1].unicode();

   388                 break;

   389             }

   390 

   391             switch (errno) {

   392             case EILSEQ:

   393                 ++invalidCount;

   394                 // fall through

   395             case EINVAL:

   396                 {

   397                     inBytes += sizeof(QChar);

   398                     inBytesLeft -= sizeof(QChar);

   399                     break;

   400                 }

   401             case E2BIG:

   402                 {

   403                     int offset = ba.size() - outBytesLeft;

   404                     ba.resize(ba.size() * 2);

   405                     outBytes = ba.data() + offset;

   406                     outBytesLeft = ba.size() - offset;

   407                     break;

   408                 }

   409             default:

   410                 {

   411                     // note, cannot use qWarning() since we are implementing the codecForLocale :)

   412                     perror("QIconvCodec::convertFromUnicode: using ASCII for conversion, iconv failed");

   413 

   414                     // reset to initial state

   415                     iconv(state->cd, 0, &inBytesLeft, 0, &outBytesLeft);

   416 

   417                     return QString(uc, len).toAscii();

   418                 }

   419             }

   420         }

   421     } while (inBytesLeft != 0);

   422 

   423     // reset to initial state

   424     iconv(state->cd, 0, &inBytesLeft, 0, &outBytesLeft);

   425 

   426     ba.resize(ba.size() - outBytesLeft);

   427 

   428     if (convState)

   429         convState->invalidChars = invalidCount;

   430 

   431     return ba;

   432 }

   433 

   434 QByteArray QIconvCodec::name() const

   435 {

   436     return "System";

   437 }

   438 

   439 int QIconvCodec::mibEnum() const

   440 {

   441     return 0;

   442 }

   443 

   444 iconv_t QIconvCodec::createIconv_t(const char *to, const char *from)

   445 {

   446     Q_ASSERT((to == 0 && from != 0) || (to != 0 && from == 0));

   447 

   448     iconv_t cd = (iconv_t) -1;

   449 #if defined(__GLIBC__) || defined(GNU_LIBICONV)

   450     // both GLIBC and libgnuiconv will use the locale's encoding if from or to is an empty string

   451     static const char empty_codeset[] = "";

   452     const char *codeset = empty_codeset;

   453     cd = iconv_open(to ? to : codeset, from ? from : codeset);

   454 #else

   455     char *codeset = 0;

   456 #endif

   457 

   458 #if defined(_XOPEN_UNIX) && !defined(Q_OS_QNX) && !defined(Q_OS_OSF)

   459     if (cd == (iconv_t) -1) {

   460         codeset = nl_langinfo(CODESET);

   461         if (codeset)

   462             cd = iconv_open(to ? to : codeset, from ? from : codeset);

   463     }

   464 #endif

   465 

   466     if (cd == (iconv_t) -1) {

   467         // Very poorly defined and followed standards causes lots of

   468         // code to try to get all the cases... This logic is

   469         // duplicated in QTextCodec, so if you change it here, change

   470         // it there too.

   471 

   472         // Try to determine locale codeset from locale name assigned to

   473         // LC_CTYPE category.

   474 

   475         // First part is getting that locale name.  First try setlocale() which

   476         // definitely knows it, but since we cannot fully trust it, get ready

   477         // to fall back to environment variables.

   478         char * ctype = qstrdup(setlocale(LC_CTYPE, 0));

   479 

   480         // Get the first nonempty value from $LC_ALL, $LC_CTYPE, and $LANG

   481         // environment variables.

   482         char * lang = qstrdup(qgetenv("LC_ALL").constData());

   483         if (!lang || lang[0] == 0 || strcmp(lang, "C") == 0) {

   484             if (lang) delete [] lang;

   485             lang = qstrdup(qgetenv("LC_CTYPE").constData());

   486         }

   487         if (!lang || lang[0] == 0 || strcmp(lang, "C") == 0) {

   488             if (lang) delete [] lang;

   489             lang = qstrdup(qgetenv("LANG").constData());

   490         }

   491 

   492         // Now try these in order:

   493         // 1. CODESET from ctype if it contains a .CODESET part (e.g. en_US.ISO8859-15)

   494         // 2. CODESET from lang if it contains a .CODESET part

   495         // 3. ctype (maybe the locale is named "ISO-8859-1" or something)

   496         // 4. locale (ditto)

   497         // 5. check for "@euro"

   498 

   499         // 1. CODESET from ctype if it contains a .CODESET part (e.g. en_US.ISO8859-15)

   500         codeset = ctype ? strchr(ctype, '.') : 0;

   501         if (codeset && *codeset == '.') {

   502             ++codeset;

   503             cd = iconv_open(to ? to : codeset, from ? from : codeset);

   504         }

   505 

   506         // 2. CODESET from lang if it contains a .CODESET part

   507         codeset = lang ? strchr(lang, '.') : 0;

   508         if (cd == (iconv_t) -1 && codeset && *codeset == '.') {

   509             ++codeset;

   510             cd = iconv_open(to ? to : codeset, from ? from : codeset);

   511         }

   512 

   513         // 3. ctype (maybe the locale is named "ISO-8859-1" or something)

   514         if (cd == (iconv_t) -1 && ctype && *ctype != 0 && strcmp (ctype, "C") != 0)

   515             cd = iconv_open(to ? to : ctype, from ? from : ctype);

   516 

   517 

   518         // 4. locale (ditto)

   519         if (cd == (iconv_t) -1 && lang && *lang != 0)

   520             cd = iconv_open(to ? to : lang, from ? from : lang);

   521 

   522         // 5. "@euro"

   523         if ((cd == (iconv_t) -1 && ctype && strstr(ctype, "@euro")) || (lang && strstr(lang, "@euro")))

   524             cd = iconv_open(to ? to : "ISO8859-15", from ? from : "ISO8859-15");

   525 

   526         delete [] ctype;

   527         delete [] lang;

   528     }

   529 

   530     return cd;

   531 }

   532 

   533 QT_END_NAMESPACE