1 /****************************************************************************

     2 ** 

     3 **

     4 ** Implementation of QTextCodec class

     5 **

     6 ** Created : 981015

     7 **

     8 ** Copyright (C)1998-2000 Trolltech AS.  All rights reserved.

     9 **

    10 ** This file is part of the tools module of the Qt GUI Toolkit.

    11 **

    12 ** This file may be distributed under the terms of the Q Public License

    13 ** as defined by Trolltech AS of Norway and appearing in the file

    14 ** LICENSE.QPL included in the packaging of this file.

    15 **

    16 ** This file may be distributed and/or modified under the terms of the

    17 ** GNU General Public License version 2 as published by the Free Software

    18 ** Foundation and appearing in the file LICENSE.GPL included in the

    19 ** packaging of this file.

    20 **

    21 ** Licensees holding valid Qt Enterprise Edition or Qt Professional Edition

    22 ** licenses may use this file in accordance with the Qt Commercial License

    23 ** Agreement provided with the Software.

    24 **

    25 ** This file is provided AS IS with NO WARRANTY OF ANY KIND, INCLUDING THE

    26 ** WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.

    27 **

    28 ** See http://www.trolltech.com/pricing.html or email sales@trolltech.com for

    29 **   information about Qt Commercial License Agreements.

    30 ** See http://www.trolltech.com/qpl/ for QPL licensing information.

    31 ** See http://www.trolltech.com/gpl/ for GPL licensing information.

    32 **

    33 ** Contact info@trolltech.com if any conditions of this licensing are

    34 ** not clear to you.

    35 **

    36 **********************************************************************/

    37 

    38 #include "qtextcodec.h"

    39 #ifndef QT_NO_TEXTCODEC

    40 

    41 #include "qlist.h"

    42 #ifndef QT_NO_CODECS

    43 #include "qutfcodec.h"

    44 #include "qgbkcodec.h"

    45 #include "qeucjpcodec.h"

    46 #include "qjiscodec.h"

    47 #include "qsjiscodec.h"

    48 #include "qeuckrcodec.h"

    49 #include "qbig5codec.h"

    50 #include "qrtlcodec.h"

    51 #include "qtsciicodec.h"

    52 #endif

    53 

    54 #include "qfile.h"

    55 #include "qstrlist.h"

    56 #include "qstring.h"

    57 

    58 #include <stdlib.h>

    59 #include <ctype.h>

    60 #include <locale.h>

    61 

    62 

    63 static QList<QTextCodec> * all = 0;

    64 static bool destroying_is_ok; // starts out as 0

    65 

    66 /*!  Deletes all the created codecs.

    67 

    68   \warning Do not call this function.

    69 

    70   QApplication calls this just before exiting, to delete any

    71   QTextCodec objects that may be lying around.  Since various other

    72   classes hold pointers to QTextCodec objects, it is not safe to call

    73   this function earlier.

    74 

    75   If you are using the utility classes (like QString) but not using

    76   QApplication, calling this function at the very end of your

    77   application can be helpful to chasing down memory leaks, as

    78   QTextCodec objects will not show up.

    79 */

    80 

    81 void QTextCodec::deleteAllCodecs()

    82 {

    83     if ( !all )

    84         return;

    85 

    86     destroying_is_ok = TRUE;

    87     QList<QTextCodec> * ball = all;

    88     all = 0;

    89     ball->clear();

    90     delete ball;

    91     destroying_is_ok = FALSE;

    92 }

    93 

    94 

    95 static void setupBuiltinCodecs();

    96 

    97 

    98 static void realSetup()

    99 {

   100 #if defined(CHECK_STATE)

   101     if ( destroying_is_ok )

   102         qWarning( "creating new codec during codec cleanup" );

   103 #endif

   104     all = new QList<QTextCodec>;

   105     all->setAutoDelete( TRUE );

   106     setupBuiltinCodecs();

   107 }

   108 

   109 

   110 static inline void setup()

   111 {

   112     if ( !all )

   113         realSetup();

   114 }

   115 

   116 

   117 class QTextStatelessEncoder: public QTextEncoder {

   118     const QTextCodec* codec;

   119 public:

   120     QTextStatelessEncoder(const QTextCodec*);

   121     QCString fromUnicode(const QString& uc, int& lenInOut);

   122 };

   123 

   124 

   125 class QTextStatelessDecoder : public QTextDecoder {

   126     const QTextCodec* codec;

   127 public:

   128     QTextStatelessDecoder(const QTextCodec*);

   129     QString toUnicode(const char* chars, int len);

   130 };

   131 

   132 QTextStatelessEncoder::QTextStatelessEncoder(const QTextCodec* c) :

   133     codec(c)

   134 {

   135 }

   136 

   137 

   138 QCString QTextStatelessEncoder::fromUnicode(const QString& uc, int& lenInOut)

   139 {

   140     return codec->fromUnicode(uc,lenInOut);

   141 }

   142 

   143 

   144 QTextStatelessDecoder::QTextStatelessDecoder(const QTextCodec* c) :

   145     codec(c)

   146 {

   147 }

   148 

   149 

   150 QString QTextStatelessDecoder::toUnicode(const char* chars, int len)

   151 {

   152     return codec->toUnicode(chars,len);

   153 }

   154 

   155 

   156 

   157 // NOT REVISED

   158 /*!

   159   \class QTextCodec qtextcodec.h

   160   \brief Provides conversion between text encodings.

   161 

   162   By making objects of subclasses of QTextCodec, support for

   163   new text encodings can be added to Qt.

   164 

   165   The abstract virtual functions describe the encoder to the

   166   system and the coder is used as required in the different

   167   text file formats supported QTextStream and, under X11 for the

   168   locale-specific character input and output (under Windows NT

   169   codecs are not needed for GUI I/O since the system works

   170   with Unicode already, and Windows 95/98 has built-in convertors

   171   for the 8-bit local encoding).

   172 

   173   More recently created QTextCodec objects take precedence

   174   over earlier ones.

   175 

   176   To add support for another 8-bit encoding to Qt, make a subclass

   177   or QTextCodec and implement at least the following methods:

   178   <dl>

   179    <dt>\c const char* name() const

   180     <dd>Return the official name for the encoding.

   181    <dt>\c int mibEnum() const

   182     <dd>Return the MIB enum for the encoding if it is listed in the

   183       <a href=ftp://ftp.isi.edu/in-notes/iana/assignments/character-sets>

   184       IANA character-sets encoding file</a>.

   185   </dl>

   186   If the encoding is multi-byte then it will have "state"; that is,

   187   the interpretation of some bytes will be dependent on some preceding

   188   bytes.  For such an encoding, you will need to implement

   189   <dl>

   190    <dt> \c QTextDecoder* makeDecoder() const

   191     <dd>Return a QTextDecoder that remembers incomplete multibyte

   192         sequence prefixes or other required state.

   193   </dl>

   194   If the encoding does \e not require state, you should implement:

   195   <dl>

   196    <dt> \c QString toUnicode(const char* chars, int len) const

   197     <dd>Converts \e len characters from \e chars to Unicode.

   198   </dl>

   199   The base QTextCodec class has default implementations of the above

   200   two functions, <i>but they are mutually recursive</i>, so you must

   201   re-implement at least one of them, or both for improved efficiency.

   202 

   203   For conversion from Unicode to 8-bit encodings, it is rarely necessary

   204   to maintain state.  However, two functions similar to the two above

   205   are used for encoding:

   206   <dl>

   207    <dt> \c QTextEncoder* makeEncoder() const

   208     <dd>Return a QTextDecoder.

   209    <dt> \c QCString fromUnicode(const QString& uc, int& lenInOut ) const;

   210     <dd>Converts \e lenInOut characters (of type QChar) from the start

   211         of the string \a uc, returning a QCString result, and also returning

   212         the \link QCString::length() length\endlink

   213         of the result in lenInOut.

   214   </dl>

   215   Again, these are mutually recursive so only one needs to be implemented,

   216   or both if better efficiency is possible.

   217 

   218   Finally, you must implement:

   219   <dl>

   220    <dt> \c int heuristicContentMatch(const char* chars, int len) const

   221     <dd>Gives a value indicating how likely it is that \e len characters

   222         from \e chars are in the encoding.

   223   </dl>

   224   A good model for this function is the

   225   QWindowsLocalCodec::heuristicContentMatch function found in the Qt sources.

   226 

   227   A QTextCodec subclass might have improved performance if you also

   228   re-implement:

   229   <dl>

   230    <dt> \c bool canEncode( QChar ) const

   231     <dd>Test if a Unicode character can be encoded.

   232    <dt> \c bool canEncode( const QString& ) const

   233     <dd>Test if a string of Unicode characters can be encoded.

   234    <dt> \c int heuristicNameMatch(const char* hint) const

   235     <dd>Test if a possibly non-standard name is referring to the codec.

   236   </dl>

   237 */

   238 

   239 

   240 /*!

   241   Constructs a QTextCodec, making it of highest precedence.

   242   The QTextCodec should always be constructed on the heap

   243   (with new), and once constructed it becomes the responsibility

   244   of Qt to delete it (which is done at QApplication destruction).

   245 */

   246 QTextCodec::QTextCodec()

   247 {

   248     setup();

   249     all->insert(0,this);

   250 }

   251 

   252 

   253 /*!

   254   Destructs the QTextCodec.  Note that you should not delete

   255   codecs yourself - once created they become the responsibility

   256   of Qt to delete.

   257 */

   258 QTextCodec::~QTextCodec()

   259 {

   260     if ( !destroying_is_ok )

   261         qWarning("QTextCodec::~QTextCodec() called by application");

   262     if ( all )

   263         all->remove( this );

   264 }

   265 

   266 

   267 /*!

   268   Returns a value indicating how likely this decoder is

   269   for decoding some format that has the given name.

   270 

   271   A good match returns a positive number around

   272   the length of the string.  A bad match is negative.

   273 

   274   The default implementation calls simpleHeuristicNameMatch()

   275   with the name of the codec.

   276 */

   277 int QTextCodec::heuristicNameMatch(const char* hint) const

   278 {

   279     return simpleHeuristicNameMatch(name(),hint);

   280 }

   281 

   282 

   283 // returns a string cotnaining the letters and numbers from input,

   284 // with a space separating run of a character class.  e.g. "iso8859-1"

   285 // becomes "iso 8859 1"

   286 static QString lettersAndNumbers( const char * input )

   287 {

   288     QString result;

   289     QChar c;

   290 

   291     while( input && *input ) {

   292         c = *input;

   293         if ( c.isLetter() || c.isNumber() )

   294             result += c.lower();

   295         if ( input[1] ) {

   296             // add space at character class transition, except

   297             // transition from upper-case to lower-case letter

   298             QChar n( input[1] );

   299             if ( c.isLetter() && n.isLetter() ) {

   300                 if ( c == c.lower() && n == n.upper() )

   301                     result += ' ';

   302             } else if ( c.category() != n.category() ) {

   303                 result += ' ';

   304             }

   305         }

   306         input++;

   307     }

   308     return result.simplifyWhiteSpace();

   309 }

   310 

   311 /*!

   312   A simple utility function for heuristicNameMatch() - it

   313   does some very minor character-skipping

   314   so that almost-exact matches score high.

   315 */

   316 int QTextCodec::simpleHeuristicNameMatch(const char* name, const char* hint)

   317 {

   318     // if they're the same, return a perfect score.

   319     if ( name && hint && qstrcmp( name, hint ) == 0 )

   320         return qstrlen( hint );

   321 

   322     // if the letters and numbers are the same, we have an "almost"

   323     // perfect match.

   324     QString h( lettersAndNumbers( hint ) );

   325     QString n( lettersAndNumbers( name ) );

   326     if ( h == n )

   327         return qstrlen( hint )-1;

   328 

   329     if ( h.stripWhiteSpace() == n.stripWhiteSpace() )

   330         return qstrlen( hint )-2;

   331 

   332     // could do some more here, but I don't think it's worth it

   333 

   334     return 0;

   335 }

   336 

   337 

   338 /*!

   339   Returns the QTextCodec \a i places from the more recently

   340   inserted, or NULL if there is no such QTextCodec.  Thus,

   341   codecForIndex(0) returns the most recently created QTextCodec.

   342 */

   343 QTextCodec* QTextCodec::codecForIndex(int i)

   344 {

   345     setup();

   346     return (uint)i >= all->count() ? 0 : all->at(i);

   347 }

   348 

   349 

   350 /*!

   351   Returns the QTextCodec which matches the

   352   \link QTextCodec::mibEnum() MIBenum\endlink \a mib.

   353 */

   354 QTextCodec* QTextCodec::codecForMib(int mib)

   355 {

   356     setup();

   357     QListIterator<QTextCodec> i(*all);

   358     QTextCodec* result;

   359     for ( ; (result=i); ++i ) {

   360         if ( result->mibEnum()==mib )

   361             break;

   362     }

   363     return result;

   364 }

   365 

   366 

   367 

   368 

   369 

   370 #ifdef _OS_WIN32_

   371 class QWindowsLocalCodec: public QTextCodec

   372 {

   373 public:

   374     QWindowsLocalCodec();

   375     ~QWindowsLocalCodec();

   376 

   377     QString toUnicode(const char* chars, int len) const;

   378     QCString fromUnicode(const QString& uc, int& lenInOut ) const;

   379 

   380     const char* name() const;

   381     int mibEnum() const;

   382 

   383     int heuristicContentMatch(const char* chars, int len) const;

   384 };

   385 

   386 QWindowsLocalCodec::QWindowsLocalCodec()

   387 {

   388 }

   389 

   390 QWindowsLocalCodec::~QWindowsLocalCodec()

   391 {

   392 }

   393 

   394 

   395 QString QWindowsLocalCodec::toUnicode(const char* chars, int len) const

   396 {

   397     if ( len == 1 && chars ) {          // Optimization; avoids allocation

   398         char c[2];

   399         c[0] = *chars;

   400         c[1] = 0;

   401         return qt_winMB2QString( c, 2 );

   402     }

   403     if ( len < 0 )

   404         return qt_winMB2QString( chars );

   405     QCString s(chars,len+1);

   406     return qt_winMB2QString(s);

   407 }

   408 

   409 QCString QWindowsLocalCodec::fromUnicode(const QString& uc, int& lenInOut ) const

   410 {

   411     QCString r = qt_winQString2MB( uc, lenInOut );

   412     lenInOut = r.length();

   413     return r;

   414 }

   415 

   416 

   417 const char* QWindowsLocalCodec::name() const

   418 {

   419     return "System";

   420 }

   421 

   422 int QWindowsLocalCodec::mibEnum() const

   423 {

   424     return 0;

   425 }

   426 

   427 

   428 int QWindowsLocalCodec::heuristicContentMatch(const char* chars, int len) const

   429 {

   430     // ### Not a bad default implementation?

   431     QString t = toUnicode(chars,len);

   432     int l = t.length();

   433     QCString mb = fromUnicode(t,l);

   434     int i=0;

   435     while ( i < len )

   436         if ( chars[i] == mb[i] )

   437             i++;

   438     return i;

   439 }

   440 

   441 #else

   442 

   443 /* locale names mostly copied from XFree86 */

   444 static const char * const iso8859_2locales[] = {

   445     "croatian", "cs", "cs_CS", "cs_CZ","cz", "cz_CZ", "czech", "hr",

   446     "hr_HR", "hu", "hu_HU", "hungarian", "pl", "pl_PL", "polish", "ro",

   447     "ro_RO", "rumanian", "serbocroatian", "sh", "sh_SP", "sh_YU", "sk",

   448     "sk_SK", "sl", "sl_CS", "sl_SI", "slovak", "slovene", "sr_SP", 0 };

   449 

   450 static const char * const iso8859_3locales[] = {

   451     "eo", 0 };

   452 

   453 static const char * const iso8859_4locales[] = {

   454     "ee", "ee_EE", "lt", "lt_LT", "lv", "lv_LV", 0 };

   455 

   456 static const char * const iso8859_5locales[] = {

   457     "bg", "bg_BG", "bulgarian", "mk", "mk_MK",

   458     "sp", "sp_YU", 0 };

   459 

   460 static const char * const iso8859_6locales[] = {

   461     "ar_AA", "ar_SA", "arabic", 0 };

   462 

   463 static const char * const iso8859_7locales[] = {

   464     "el", "el_GR", "greek", 0 };

   465 

   466 static const char * const iso8859_8locales[] = {

   467     "hebrew", "he", "he_IL", "iw", "iw_IL", 0 };

   468 

   469 static const char * const iso8859_9locales[] = {

   470     "tr", "tr_TR", "turkish", 0 };

   471 

   472 static const char * const iso8859_15locales[] = {

   473     "fr", "fi", "french", "finnish", "et", "et_EE", 0 };

   474 

   475 static const char * const koi8_ulocales[] = {

   476     "uk", "uk_UA", "ru_UA", "ukrainian", 0 };

   477 

   478 static const char * const tis_620locales[] = {

   479     "th", "th_TH", "thai", 0 };

   480 

   481 

   482 static bool try_locale_list( const char * const locale[], const char * lang )

   483 {

   484     int i;

   485     for( i=0; locale[i] && qstrcmp(locale[i], lang); i++ )

   486     { }

   487     return locale[i] != 0;

   488 }

   489 

   490 // For the probably_koi8_locales we have to look. the standard says

   491 // these are 8859-5, but almsot all Russion users uses KOI8-R and

   492 // incorrectly set $LANG to ru_RU. We'll check tolower() to see what

   493 // tolower() thinks ru_RU means.

   494 

   495 // If you read the history, it seems that many Russians blame ISO and

   496 // Peristroika for the confusion.

   497 //

   498 // The real bug is that some programs break if the user specifies

   499 // ru_RU.KOI8-R.

   500 

   501 static const char * const probably_koi8_rlocales[] = {

   502     "ru", "ru_SU", "ru_RU", "russian", 0 };

   503 

   504 // this means ANY of these locale aliases. if they're aliases for

   505 // different locales, the code breaks.

   506 static QTextCodec * ru_RU_codec = 0;

   507 

   508 static QTextCodec * ru_RU_hack( const char * i ) {

   509     if ( ! ru_RU_codec ) {

   510         QCString origlocale = setlocale( LC_CTYPE, i );

   511         // unicode   koi8r   latin5   name

   512         // 0x044E    0xC0    0xEE     CYRILLIC SMALL LETTER YU

   513         // 0x042E    0xE0    0xCE     CYRILLIC CAPITAL LETTER YU

   514         int latin5 = tolower( 0xCE );

   515         int koi8r = tolower( 0xE0 );

   516         if ( koi8r == 0xC0 && latin5 != 0xEE ) {

   517             ru_RU_codec = QTextCodec::codecForName( "KOI8-R" );

   518         } else if ( koi8r != 0xC0 && latin5 == 0xEE ) {

   519             ru_RU_codec = QTextCodec::codecForName( "ISO 8859-5" );

   520         } else {

   521             // something else again... let's assume... *throws dice*

   522             ru_RU_codec = QTextCodec::codecForName( "KOI8-R" );

   523             qWarning( "QTextCodec: using KOI8-R, probe failed (%02x %02x %s)",

   524                       koi8r, latin5, i );

   525         }

   526         setlocale( LC_CTYPE, origlocale.data() );

   527     }

   528     return ru_RU_codec;

   529 }

   530 

   531 #endif

   532 

   533 static QTextCodec * localeMapper = 0;

   534 

   535 void qt_set_locale_codec( QTextCodec *codec )

   536 {

   537     localeMapper = codec;

   538 }

   539 

   540 /*!  Returns a pointer to the codec most suitable for this locale. */

   541 

   542 QTextCodec* QTextCodec::codecForLocale()

   543 {

   544     if ( localeMapper )

   545         return localeMapper;

   546 

   547     setup();

   548 

   549 #ifdef _OS_WIN32_

   550     localeMapper = new QWindowsLocalCodec;

   551 #else

   552     // Very poorly defined and followed standards causes lots of code

   553     // to try to get all the cases...

   554 

   555     char * lang = qstrdup( getenv("LANG") );

   556 

   557     char * p = lang ? strchr( lang, '.' ) : 0;

   558     if ( !p || *p != '.' ) {

   559         // Some versions of setlocale return encoding, others not.

   560         char *ctype = qstrdup( setlocale( LC_CTYPE, 0 ) );

   561         // Some Linux distributions have broken locales which will return

   562         // "C" for LC_CTYPE

   563         if ( qstrcmp( ctype, "C" ) == 0 ) {

   564             delete [] ctype;

   565         } else {

   566             if ( lang )

   567                 delete [] lang;

   568             lang = ctype;

   569             p = lang ? strchr( lang, '.' ) : 0;

   570         }

   571     }

   572 

   573     if( p && *p == '.' ) {

   574         // if there is an encoding and we don't know it, we return 0

   575         // User knows what they are doing.  Codecs will believe them.

   576         localeMapper = codecForName( lang );

   577         if ( !localeMapper ) {

   578             // Use or codec disagree.

   579             localeMapper = codecForName( p+1 );

   580         }

   581     }

   582     if ( !localeMapper || !(p && *p == '.') ) {

   583         // if there is none, we default to 8859-1

   584         // We could perhaps default to 8859-15.

   585         if ( try_locale_list( iso8859_2locales, lang ) )

   586             localeMapper = codecForName( "ISO 8859-2" );

   587         else if ( try_locale_list( iso8859_3locales, lang ) )

   588             localeMapper = codecForName( "ISO 8859-3" );

   589         else if ( try_locale_list( iso8859_4locales, lang ) )

   590             localeMapper = codecForName( "ISO 8859-4" );

   591         else if ( try_locale_list( iso8859_5locales, lang ) )

   592             localeMapper = codecForName( "ISO 8859-5" );

   593         else if ( try_locale_list( iso8859_6locales, lang ) )

   594             localeMapper = codecForName( "ISO 8859-6-I" );

   595         else if ( try_locale_list( iso8859_7locales, lang ) )

   596             localeMapper = codecForName( "ISO 8859-7" );

   597         else if ( try_locale_list( iso8859_8locales, lang ) )

   598             localeMapper = codecForName( "ISO 8859-8-I" );

   599         else if ( try_locale_list( iso8859_9locales, lang ) )

   600             localeMapper = codecForName( "ISO 8859-9" );

   601         else if ( try_locale_list( iso8859_15locales, lang ) )

   602             localeMapper = codecForName( "ISO 8859-15" );

   603         else if ( try_locale_list( tis_620locales, lang ) )

   604             localeMapper = codecForName( "ISO 8859-11" );

   605         else if ( try_locale_list( koi8_ulocales, lang ) )

   606             localeMapper = codecForName( "KOI8-U" );

   607          else if ( try_locale_list( probably_koi8_rlocales, lang ) )

   608             localeMapper = ru_RU_hack( lang );

   609         else if (!lang || !(localeMapper = codecForName(lang) ))

   610             localeMapper = codecForName( "ISO 8859-1" );

   611     }

   612     delete[] lang;

   613 #endif

   614 

   615     return localeMapper;

   616 }

   617 

   618 

   619 /*!

   620   Searches all installed QTextCodec objects, returning the one

   621   which best matches given name.  Returns NULL if no codec has

   622   a match closeness above \a accuracy.

   623 

   624   \sa heuristicNameMatch()

   625 */

   626 QTextCodec* QTextCodec::codecForName(const char* hint, int accuracy)

   627 {

   628     setup();

   629     QListIterator<QTextCodec> i(*all);

   630     QTextCodec* result = 0;

   631     int best=accuracy;

   632     for ( QTextCodec* cursor; (cursor=i); ++i ) {

   633         int s = cursor->heuristicNameMatch(hint);

   634         if ( s > best ) {

   635             best = s;

   636             result = cursor;

   637         }

   638     }

   639     return result;

   640 }

   641 

   642 

   643 /*!

   644   Searches all installed QTextCodec objects, returning the one

   645   which most recognizes the given content.  May return 0.

   646 

   647   Note that this is often a poor choice, since character

   648   encodings often use most of the available character sequences,

   649   and so only by linguistic analysis could a true match be made.

   650 

   651   \sa heuristicContentMatch()

   652 */

   653 QTextCodec* QTextCodec::codecForContent(const char* chars, int len)

   654 {

   655     setup();

   656     QListIterator<QTextCodec> i(*all);

   657     QTextCodec* result = 0;

   658     int best=0;

   659     for ( QTextCodec* cursor; (cursor=i); ++i ) {

   660         int s = cursor->heuristicContentMatch(chars,len);

   661         if ( s > best ) {

   662             best = s;

   663             result = cursor;

   664         }

   665     }

   666     return result;

   667 }

   668 

   669 

   670 /*!

   671   \fn const char* QTextCodec::name() const

   672   Subclasses of QTextCodec must reimplement this function.  It returns

   673   the name of the encoding supported by the subclass.  When choosing

   674   a name for an encoding, consider these points:

   675   <ul>

   676     <li>On X11, heuristicNameMatch( const char * hint )

   677         is used to test if a the QTextCodec

   678         can convert between Unicode and the encoding of a font

   679         with encoding \e hint, such as "iso8859-1" for Latin-1 fonts,

   680         "koi8-r" for Russian KOI8 fonts.

   681         The default algorithm of heuristicNameMatch() uses name().

   682     <li>Some applications may use this function to present

   683         encodings to the end user.

   684   </ul>

   685 */

   686 

   687 /*!

   688   \fn int QTextCodec::mibEnum() const

   689 

   690   Subclasses of QTextCodec must reimplement this function.  It returns the

   691   MIBenum (see

   692   <a href="ftp://ftp.isi.edu/in-notes/iana/assignments/character-sets">

   693   the IANA character-sets encoding file</a> for more information).

   694   It is important that each QTextCodec subclass return the correct unique

   695   value for this function.

   696 */

   697 

   698 

   699 /*!

   700   \fn int QTextCodec::heuristicContentMatch(const char* chars, int len) const

   701 

   702   Subclasses of QTextCodec must reimplement this function.  It examines

   703   the first \a len bytes of \a chars and returns a value indicating how

   704   likely it is that the string is a prefix of text encoded in the

   705   encoding of the subclass.  Any negative return value indicates that the text

   706   is detectably not in the encoding (eg. it contains undefined characters).

   707   A return value of 0 indicates that the text should be decoded with this

   708   codec rather than as ASCII, but there

   709   is no particular evidence.  The value should range up to \a len.  Thus,

   710   most decoders will return -1, 0, or -\a len.

   711 

   712   The characters are not null terminated.

   713 

   714   \sa codecForContent().

   715 */

   716 

   717 

   718 /*!

   719   Creates a QTextDecoder which stores enough state to decode chunks

   720   of char* data to create chunks of Unicode data.  The default implementation

   721   creates a stateless decoder, which is sufficient for only the simplest

   722   encodings where each byte corresponds to exactly one Unicode character.

   723 

   724   The caller is responsible for deleting the returned object.

   725 */

   726 QTextDecoder* QTextCodec::makeDecoder() const

   727 {

   728     return new QTextStatelessDecoder(this);

   729 }

   730 

   731 

   732 /*!

   733   Creates a QTextEncoder which stores enough state to encode chunks

   734   of Unicode data as char* data.  The default implementation

   735   creates a stateless encoder, which is sufficient for only the simplest

   736   encodings where each Unicode character corresponds to exactly one char.

   737 

   738   The caller is responsible for deleting the returned object.

   739 */

   740 QTextEncoder* QTextCodec::makeEncoder() const

   741 {

   742     return new QTextStatelessEncoder(this);

   743 }

   744 

   745 

   746 /*!

   747   Subclasses of QTextCodec must reimplement this function or

   748   makeDecoder().  It converts the first \a len characters of \a chars

   749   to Unicode.

   750 

   751   The default implementation makes a decoder with makeDecoder() and

   752   converts the input with that.  Note that the default makeDecoder()

   753   implementation makes a decoder that simply calls

   754   this function, hence subclasses \e must reimplement one function or

   755   the other to avoid infinite recursion.

   756 */

   757 QString QTextCodec::toUnicode(const char* chars, int len) const

   758 {

   759     QTextDecoder* i = makeDecoder();

   760     QString result = i->toUnicode(chars,len);

   761     delete i;

   762     return result;

   763 }

   764 

   765 

   766 /*!

   767   Subclasses of QTextCodec must reimplement either this function or

   768   makeEncoder().  It converts the first \a lenInOut characters of \a

   769   uc from Unicode to the encoding of the subclass.  If \a lenInOut

   770   is negative or too large, the length of \a uc is used instead.

   771 

   772   The value returned is the property of the caller, which is

   773   responsible for deleting it with "delete []".  The length of the

   774   resulting Unicode character sequence is returned in \a lenInOut.

   775 

   776   The default implementation makes an encoder with makeEncoder() and

   777   converts the input with that.  Note that the default makeEncoder()

   778   implementation makes an encoder that simply calls

   779   this function, hence subclasses \e must reimplement one function or

   780   the other to avoid infinite recursion.

   781 */

   782 

   783 QCString QTextCodec::fromUnicode(const QString& uc, int& lenInOut) const

   784 {

   785     QTextEncoder* i = makeEncoder();

   786     QCString result = i->fromUnicode(uc, lenInOut);

   787     delete i;

   788     return result;

   789 }

   790 

   791 /*!

   792   \overload QCString QTextCodec::fromUnicode(const QString& uc) const

   793 */

   794 QCString QTextCodec::fromUnicode(const QString& uc) const

   795 {

   796     int l = uc.length();

   797     return fromUnicode(uc,l);

   798 }

   799 

   800 /*!

   801   \overload QString QTextCodec::toUnicode(const QByteArray& a, int len) const

   802 */

   803 QString QTextCodec::toUnicode(const QByteArray& a, int len) const

   804 {

   805     int l = a.size();

   806     if( l > 0 && a.data()[l - 1] == '\0' ) l--;

   807     l = QMIN( l, len );

   808     return toUnicode( a.data(), l );

   809 }

   810 

   811 /*!

   812   \overload QString QTextCodec::toUnicode(const QByteArray& a) const

   813 */

   814 QString QTextCodec::toUnicode(const QByteArray& a) const

   815 {

   816     int l = a.size();

   817     if( l > 0 && a.data()[l - 1] == '\0' ) l--;

   818     return toUnicode( a.data(), l );

   819 }

   820 

   821 /*!

   822   \overload QString QTextCodec::toUnicode(const char* chars) const

   823 */

   824 QString QTextCodec::toUnicode(const char* chars) const

   825 {

   826     return toUnicode(chars,qstrlen(chars));

   827 }

   828 

   829 /*!

   830   Returns TRUE if the unicode character \a ch can be fully encoded

   831   with this codec.  The default implementation tests if the result of

   832   toUnicode(fromUnicode(ch)) is the original \a ch. Subclasses may be

   833   able to improve the efficiency.

   834 */

   835 bool QTextCodec::canEncode( QChar ch ) const

   836 {

   837     return toUnicode(fromUnicode(ch)) == ch;

   838 }

   839 

   840 /*!

   841   Returns TRUE if the unicode string \a s can be fully encoded

   842   with this codec.  The default implementation tests if the result of

   843   toUnicode(fromUnicode(s)) is the original \a s. Subclasses may be

   844   able to improve the efficiency.

   845 */

   846 bool QTextCodec::canEncode( const QString& s ) const

   847 {

   848     return toUnicode(fromUnicode(s)) == s;

   849 }

   850 

   851 

   852 

   853 /*!

   854   \class QTextEncoder qtextcodec.h

   855   \brief State-based encoder

   856 

   857   A QTextEncoder converts Unicode into another format, remembering

   858   any state that is required between calls.

   859 

   860   \sa QTextCodec::makeEncoder()

   861 */

   862 

   863 /*!

   864   Destructs the encoder.

   865 */

   866 QTextEncoder::~QTextEncoder()

   867 {

   868 }

   869 /*!

   870   \fn QCString QTextEncoder::fromUnicode(const QString& uc, int& lenInOut)

   871 

   872   Converts \a lenInOut characters (not bytes) from \a uc, producing

   873   a QCString.  \a lenInOut will also be set to the

   874   \link QCString::length() length\endlink of the result (in bytes).

   875 

   876   The encoder is free to record state to use when subsequent calls are

   877   made to this function (for example, it might change modes with escape

   878   sequences if needed during the encoding of one string, then assume that

   879   mode applies when a subsequent call begins).

   880 */

   881 

   882 /*!

   883   \class QTextDecoder qtextcodec.h

   884   \brief State-based decoder

   885 

   886   A QTextEncoder converts a text format into Unicode, remembering

   887   any state that is required between calls.

   888 

   889   \sa QTextCodec::makeEncoder()

   890 */

   891 

   892 

   893 /*!

   894   Destructs the decoder.

   895 */

   896 QTextDecoder::~QTextDecoder()

   897 {

   898 }

   899 

   900 /*!

   901   \fn QString QTextDecoder::toUnicode(const char* chars, int len)

   902 

   903   Converts the first \a len bytes at \a chars to Unicode, returning the

   904   result.

   905 

   906   If not all characters are used (eg. only part of a multi-byte

   907   encoding is at the end of the characters), the decoder remembers

   908   enough state to continue with the next call to this function.

   909 */

   910 

   911 #define CHAINED 0xffff

   912 

   913 struct QMultiByteUnicodeTable {

   914     // If multibyte, ignore unicode and index into multibyte

   915     //  with the next character.

   916     QMultiByteUnicodeTable() : unicode(0xfffd), multibyte(0) { }

   917 

   918     ~QMultiByteUnicodeTable()

   919     {

   920         if ( multibyte )

   921             delete [] multibyte;

   922     }

   923 

   924     ushort unicode;

   925     QMultiByteUnicodeTable* multibyte;

   926 };

   927 

   928 #ifndef QT_NO_CODECS

   929 static int getByte(char* &cursor)

   930 {

   931     int byte = 0;

   932     if ( *cursor ) {

   933         if ( cursor[1] == 'x' )

   934             byte = strtol(cursor+2,&cursor,16);

   935         else if ( cursor[1] == 'd' )

   936             byte = strtol(cursor+2,&cursor,10);

   937         else

   938             byte = strtol(cursor+2,&cursor,8);

   939     }

   940     return byte&0xff;

   941 }

   942 

   943 class QTextCodecFromIOD;

   944 

   945 class QTextCodecFromIODDecoder : public QTextDecoder {

   946     const QTextCodecFromIOD* codec;

   947     QMultiByteUnicodeTable* mb;

   948 public:

   949     QTextCodecFromIODDecoder(const QTextCodecFromIOD* c);

   950     QString toUnicode(const char* chars, int len);

   951 };

   952 

   953 class QTextCodecFromIOD : public QTextCodec {

   954     friend class QTextCodecFromIODDecoder;

   955 

   956     QCString n;

   957 

   958     // If from_unicode_page[row()][cell()] is 0 and from_unicode_page_multibyte,

   959     //  use from_unicode_page_multibyte[row()][cell()] as string.

   960     char** from_unicode_page;

   961     char*** from_unicode_page_multibyte;

   962     char unkn;

   963 

   964     // Only one of these is used

   965     ushort* to_unicode;

   966     QMultiByteUnicodeTable* to_unicode_multibyte;

   967     int max_bytes_per_char;

   968     QStrList aliases;

   969 

   970     bool stateless() const { return !to_unicode_multibyte; }

   971 

   972 public:

   973     QTextCodecFromIOD(QIODevice* iod)

   974     {

   975         from_unicode_page = 0;

   976         to_unicode_multibyte = 0;

   977         to_unicode = 0;

   978         from_unicode_page_multibyte = 0;

   979         max_bytes_per_char = 1;

   980 

   981         const int maxlen=100;

   982         char line[maxlen];

   983         char esc='\\';

   984         char comm='%';

   985         bool incmap = FALSE;

   986         while (iod->readLine(line,maxlen) > 0) {

   987             if (0==qstrnicmp(line,"<code_set_name>",15))

   988                 n = line+15;

   989             else if (0==qstrnicmp(line,"<escape_char> ",14))

   990                 esc = line[14];

   991             else if (0==qstrnicmp(line,"<comment_char> ",15))

   992                 comm = line[15];

   993             else if (line[0]==comm && 0==qstrnicmp(line+1," alias ",7)) {

   994                 aliases.append(line+8);

   995             } else if (0==qstrnicmp(line,"CHARMAP",7)) {

   996                 if (!from_unicode_page) {

   997                     from_unicode_page = new char*[256];

   998                     for (int i=0; i<256; i++)

   999                         from_unicode_page[i]=0;

  1000                 }

  1001                 if (!to_unicode) {

  1002                     to_unicode = new ushort[256];

  1003                 }

  1004                 incmap = TRUE;

  1005             } else if (0==qstrnicmp(line,"END CHARMAP",11))

  1006                 break;

  1007             else if (incmap) {

  1008                 char* cursor = line;

  1009                 int byte,unicode=-1;

  1010                 ushort* mb_unicode=0;

  1011                 const int maxmb=8; // more -> we'll need to improve datastructures

  1012                 char mb[maxmb+1];

  1013                 int nmb=0;

  1014 

  1015                 while (*cursor) {

  1016                     if (cursor[0]=='<' && cursor[1]=='U' &&

  1017                         cursor[2]>='0' && cursor[2]<='9' &&

  1018                         cursor[3]>='0' && cursor[3]<='9') {

  1019 

  1020                         unicode = strtol(cursor+2,&cursor,16);

  1021 

  1022                     } else if (*cursor==esc) {

  1023 

  1024                         byte = getByte(cursor);

  1025 

  1026                         if ( *cursor == esc ) {

  1027                             if ( !to_unicode_multibyte ) {

  1028                                 to_unicode_multibyte =

  1029                                     new QMultiByteUnicodeTable[256];

  1030                                 for (int i=0; i<256; i++) {

  1031                                     to_unicode_multibyte[i].unicode =

  1032                                         to_unicode[i];

  1033                                     to_unicode_multibyte[i].multibyte = 0;

  1034                                 }

  1035                                 delete [] to_unicode;

  1036                                 to_unicode = 0;

  1037                             }

  1038                             QMultiByteUnicodeTable* mbut =

  1039                                 to_unicode_multibyte+byte;

  1040                             mb[nmb++] = byte;

  1041                             while ( nmb < maxmb && *cursor == esc ) {

  1042                                 // Always at least once

  1043 

  1044                                 mbut->unicode = CHAINED;

  1045                                 byte = getByte(cursor);

  1046                                 mb[nmb++] = byte;

  1047                                 if (!mbut->multibyte) {

  1048                                     mbut->multibyte =

  1049                                         new QMultiByteUnicodeTable[256];

  1050                                 }

  1051                                 mbut = mbut->multibyte+byte;

  1052                                 mb_unicode = & mbut->unicode;

  1053                             }

  1054 

  1055                             if ( nmb > max_bytes_per_char )

  1056                                 max_bytes_per_char = nmb;

  1057                         }

  1058                     } else {

  1059                         cursor++;

  1060                     }

  1061                 }

  1062 

  1063                 if (unicode >= 0 && unicode <= 0xffff)

  1064                 {

  1065                     QChar ch((ushort)unicode);

  1066                     if (!from_unicode_page[ch.row()]) {

  1067                         from_unicode_page[ch.row()] = new char[256];

  1068                         for (int i=0; i<256; i++)

  1069                             from_unicode_page[ch.row()][i]=0;

  1070                     }

  1071                     if ( mb_unicode ) {

  1072                         from_unicode_page[ch.row()][ch.cell()] = 0;

  1073                         if (!from_unicode_page_multibyte) {

  1074                             from_unicode_page_multibyte = new char**[256];

  1075                             for (int i=0; i<256; i++)

  1076                                 from_unicode_page_multibyte[i]=0;

  1077                         }

  1078                         if (!from_unicode_page_multibyte[ch.row()]) {

  1079                             from_unicode_page_multibyte[ch.row()] = new char*[256];

  1080                             for (int i=0; i<256; i++)

  1081                                 from_unicode_page_multibyte[ch.row()][i] = 0;

  1082                         }

  1083                         mb[nmb++] = 0;

  1084                         from_unicode_page_multibyte[ch.row()][ch.cell()]

  1085                             = qstrdup(mb);

  1086                         *mb_unicode = unicode;

  1087                     } else {

  1088                         from_unicode_page[ch.row()][ch.cell()] = (char)byte;

  1089                         if ( to_unicode )

  1090                             to_unicode[byte] = unicode;

  1091                         else

  1092                             to_unicode_multibyte[byte].unicode = unicode;

  1093                     }

  1094                 } else {

  1095                 }

  1096             }

  1097         }

  1098         n = n.stripWhiteSpace();

  1099 

  1100         unkn = '?'; // ##### Might be a bad choice.

  1101     }

  1102 

  1103     ~QTextCodecFromIOD()

  1104     {

  1105         if ( from_unicode_page ) {

  1106             for (int i=0; i<256; i++)

  1107                 if (from_unicode_page[i])

  1108                     delete [] from_unicode_page[i];

  1109         }

  1110         if ( from_unicode_page_multibyte ) {

  1111             for (int i=0; i<256; i++)

  1112                 if (from_unicode_page_multibyte[i])

  1113                     for (int j=0; j<256; j++)

  1114                         if (from_unicode_page_multibyte[i][j])

  1115                             delete [] from_unicode_page_multibyte[i][j];

  1116         }

  1117         if ( to_unicode )

  1118             delete [] to_unicode;

  1119         if ( to_unicode_multibyte )

  1120             delete [] to_unicode_multibyte;

  1121     }

  1122 

  1123     bool ok() const

  1124     {

  1125         return !!from_unicode_page;

  1126     }

  1127 

  1128     QTextDecoder* makeDecoder() const

  1129     {

  1130         if ( stateless() )

  1131             return QTextCodec::makeDecoder();

  1132         else

  1133             return new QTextCodecFromIODDecoder(this);

  1134     }

  1135