1 /****************************************************************************

     2 **

     3 ** Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies).

     4 ** All rights reserved.

     5 ** Contact: Nokia Corporation (qt-info@nokia.com)

     6 **

     7 ** This file is part of the Qt Linguist of the Qt Toolkit.

     8 **

     9 ** $QT_BEGIN_LICENSE:LGPL$

    10 ** No Commercial Usage

    11 ** This file contains pre-release code and may not be distributed.

    12 ** You may use this file in accordance with the terms and conditions

    13 ** contained in the Technology Preview License Agreement accompanying

    14 ** this package.

    15 **

    16 ** GNU Lesser General Public License Usage

    17 ** Alternatively, this file may be used under the terms of the GNU Lesser

    18 ** General Public License version 2.1 as published by the Free Software

    19 ** Foundation and appearing in the file LICENSE.LGPL included in the

    20 ** packaging of this file.  Please review the following information to

    21 ** ensure the GNU Lesser General Public License version 2.1 requirements

    22 ** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.

    23 **

    24 ** In addition, as a special exception, Nokia gives you certain additional

    25 ** rights.  These rights are described in the Nokia Qt LGPL Exception

    26 ** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.

    27 **

    28 ** If you have questions regarding the use of this file, please contact

    29 ** Nokia at qt-info@nokia.com.

    30 **

    31 **

    32 **

    33 **

    34 **

    35 **

    36 **

    37 **

    38 ** $QT_END_LICENSE$

    39 **

    40 ****************************************************************************/

    41 

    42 #include "translator.h"

    43 

    44 #include "simtexth.h"

    45 

    46 #include <stdio.h>

    47 

    48 #include <QtCore/QDebug>

    49 #include <QtCore/QDir>

    50 #include <QtCore/QFile>

    51 #include <QtCore/QFileInfo>

    52 #include <QtCore/QTextCodec>

    53 #include <QtCore/QTextStream>

    54 

    55 #include <private/qtranslator_p.h>

    56 

    57 QT_BEGIN_NAMESPACE

    58 

    59 Translator::Translator() :

    60     m_codecName("ISO-8859-1"),

    61     m_locationsType(AbsoluteLocations)

    62 {

    63 }

    64 

    65 void Translator::registerFileFormat(const FileFormat &format)

    66 {

    67     //qDebug() << "Translator: Registering format " << format.extension;

    68     QList<Translator::FileFormat> &formats = registeredFileFormats();

    69     for (int i = 0; i < formats.size(); ++i)

    70         if (format.fileType == formats[i].fileType && format.priority < formats[i].priority) {

    71             formats.insert(i, format);

    72             return;

    73         }

    74     formats.append(format);

    75 }

    76 

    77 QList<Translator::FileFormat> &Translator::registeredFileFormats()

    78 {

    79     static QList<Translator::FileFormat> theFormats;

    80     return theFormats;

    81 }

    82 

    83 void Translator::replace(const TranslatorMessage &msg)

    84 {

    85     int index = m_messages.indexOf(msg);

    86     if (index == -1)

    87         m_messages.append(msg);

    88     else

    89         m_messages[index] = msg;

    90 }

    91 

    92 void Translator::replaceSorted(const TranslatorMessage &msg)

    93 {

    94     int index = m_messages.indexOf(msg);

    95     if (index == -1)

    96         appendSorted(msg);

    97     else

    98         m_messages[index] = msg;

    99 }

   100 

   101 void Translator::extend(const TranslatorMessage &msg)

   102 {

   103     int index = m_messages.indexOf(msg);

   104     if (index == -1) {

   105         m_messages.append(msg);

   106     } else {

   107         TranslatorMessage &emsg = m_messages[index];

   108         emsg.addReferenceUniq(msg.fileName(), msg.lineNumber());

   109         if (!msg.extraComment().isEmpty()) {

   110             QString cmt = emsg.extraComment();

   111             if (!cmt.isEmpty())

   112                 cmt.append(QLatin1String("\n----------\n"));

   113             cmt.append(msg.extraComment());

   114             emsg.setExtraComment(cmt);

   115         }

   116         if (msg.isUtf8() != emsg.isUtf8()) {

   117             emsg.setUtf8(true);

   118             emsg.setNonUtf8(true);

   119         }

   120     }

   121 }

   122 

   123 void Translator::append(const TranslatorMessage &msg)

   124 {

   125     m_messages.append(msg);

   126 }

   127 

   128 void Translator::appendSorted(const TranslatorMessage &msg)

   129 {

   130     int msgLine = msg.lineNumber();

   131     if (msgLine < 0) {

   132         m_messages.append(msg);

   133         return;

   134     }

   135 

   136     int bestIdx = 0; // Best insertion point found so far

   137     int bestScore = 0; // Its category: 0 = no hit, 1 = pre or post, 2 = middle

   138     int bestSize = 0; // The length of the region. Longer is better within one category.

   139 

   140     // The insertion point to use should this region turn out to be the best one so far

   141     int thisIdx = 0;

   142     int thisScore = 0;

   143     int thisSize = 0;

   144     // Working vars

   145     int prevLine = 0;

   146     int curIdx = 0;

   147     foreach (const TranslatorMessage &mit, m_messages) {

   148         bool sameFile = mit.fileName() == msg.fileName();

   149         int curLine;

   150         if (sameFile && (curLine = mit.lineNumber()) >= prevLine) {

   151             if (msgLine >= prevLine && msgLine < curLine) {

   152                 thisIdx = curIdx;

   153                 thisScore = thisSize ? 2 : 1;

   154             }

   155             ++thisSize;

   156             prevLine = curLine;

   157         } else {

   158             if (thisSize) {

   159                 if (!thisScore) {

   160                     thisIdx = curIdx;

   161                     thisScore = 1;

   162                 }

   163                 if (thisScore > bestScore || (thisScore == bestScore && thisSize > bestSize)) {

   164                     bestIdx = thisIdx;

   165                     bestScore = thisScore;

   166                     bestSize = thisSize;

   167                 }

   168                 thisScore = 0;

   169                 thisSize = sameFile ? 1 : 0;

   170                 prevLine = 0;

   171             }

   172         }

   173         ++curIdx;

   174     }

   175     if (thisSize && !thisScore) {

   176         thisIdx = curIdx;

   177         thisScore = 1;

   178     }

   179     if (thisScore > bestScore || (thisScore == bestScore && thisSize > bestSize))

   180         m_messages.insert(thisIdx, msg);

   181     else if (bestScore)

   182         m_messages.insert(bestIdx, msg);

   183     else

   184         m_messages.append(msg);

   185 }

   186 

   187 static QString guessFormat(const QString &filename, const QString &format)

   188 {

   189     if (format != QLatin1String("auto"))

   190         return format;

   191 

   192     foreach (const Translator::FileFormat &fmt, Translator::registeredFileFormats()) {

   193         if (filename.endsWith(QLatin1Char('.') + fmt.extension, Qt::CaseInsensitive))

   194             return fmt.extension;

   195     }

   196 

   197     // the default format.

   198     // FIXME: change to something more widely distributed later.

   199     return QLatin1String("ts");

   200 }

   201 

   202 bool Translator::load(const QString &filename, ConversionData &cd, const QString &format)

   203 {

   204     cd.m_sourceDir = QFileInfo(filename).absoluteDir();

   205     cd.m_sourceFileName = filename;

   206 

   207     QFile file;

   208     if (filename.isEmpty() || filename == QLatin1String("-")) {

   209         if (!file.open(stdin, QIODevice::ReadOnly)) {

   210             cd.appendError(QString::fromLatin1("Cannot open stdin!? (%1)")

   211                 .arg(file.errorString()));

   212             return false;

   213         }

   214     } else {

   215         file.setFileName(filename);

   216         if (!file.open(QIODevice::ReadOnly)) {

   217             cd.appendError(QString::fromLatin1("Cannot open %1: %2")

   218                 .arg(filename, file.errorString()));

   219             return false;

   220         }

   221     }

   222 

   223     QString fmt = guessFormat(filename, format);

   224 

   225     foreach (const FileFormat &format, registeredFileFormats()) {

   226         if (fmt == format.extension) {

   227             if (format.loader)

   228                 return (*format.loader)(*this, file, cd);

   229             cd.appendError(QString(QLatin1String("No loader for format %1 found"))

   230                 .arg(fmt));

   231             return false;

   232         }

   233     }

   234 

   235     cd.appendError(QString(QLatin1String("Unknown format %1 for file %2"))

   236         .arg(format, filename));

   237     return false;

   238 }

   239 

   240 

   241 bool Translator::save(const QString &filename, ConversionData &cd, const QString &format) const

   242 {

   243     QFile file;

   244     if (filename.isEmpty() || filename == QLatin1String("-")) {

   245         if (!file.open(stdout, QIODevice::WriteOnly)) {

   246             cd.appendError(QString::fromLatin1("Cannot open stdout!? (%1)")

   247                 .arg(file.errorString()));

   248             return false;

   249         }

   250     } else {

   251         file.setFileName(filename);

   252         if (!file.open(QIODevice::WriteOnly)) {

   253             cd.appendError(QString::fromLatin1("Cannot create %1: %2")

   254                 .arg(filename, file.errorString()));

   255             return false;

   256         }

   257     }

   258 

   259     QString fmt = guessFormat(filename, format);

   260     cd.m_targetDir = QFileInfo(filename).absoluteDir();

   261 

   262     foreach (const FileFormat &format, registeredFileFormats()) {

   263         if (fmt == format.extension) {

   264             if (format.saver)

   265                 return (*format.saver)(*this, file, cd);

   266             cd.appendError(QString(QLatin1String("Cannot save %1 files")).arg(fmt));

   267             return false;

   268         }

   269     }

   270 

   271     cd.appendError(QString(QLatin1String("Unknown format %1 for file %2"))

   272         .arg(format).arg(filename));

   273     return false;

   274 }

   275 

   276 QString Translator::makeLanguageCode(QLocale::Language language, QLocale::Country country)

   277 {

   278     QLocale locale(language, country);

   279     if (country == QLocale::AnyCountry) {

   280         QString languageCode = locale.name().section(QLatin1Char('_'), 0, 0);

   281         if (languageCode.length() <= 3)

   282             return languageCode;

   283         return QString();

   284     } else {

   285         return locale.name();

   286     }

   287 }

   288 

   289 void Translator::languageAndCountry(const QString &languageCode,

   290     QLocale::Language *lang, QLocale::Country *country)

   291 {

   292     QLocale locale(languageCode);

   293     if (lang)

   294         *lang = locale.language();

   295 

   296     if (country) {

   297         if (languageCode.indexOf(QLatin1Char('_')) != -1)

   298             *country = locale.country();

   299         else

   300             *country = QLocale::AnyCountry;

   301     }

   302 }

   303 

   304 bool Translator::release(QFile *iod, ConversionData &cd) const

   305 {

   306     foreach (const FileFormat &format, registeredFileFormats()) {

   307         if (format.extension == QLatin1String("qm"))

   308             return (*format.saver)(*this, *iod, cd);

   309     }

   310     cd.appendError(QLatin1String("No .qm saver available."));

   311     return false;

   312 }

   313 

   314 bool Translator::contains(const QString &context,

   315     const QString &sourceText, const QString &comment) const

   316 {

   317     return m_messages.contains(TranslatorMessage(context, sourceText, comment,

   318         QString(), QString(), 0));

   319 }

   320 

   321 TranslatorMessage Translator::find(const QString &context,

   322     const QString &sourceText, const QString &comment) const

   323 {

   324     TranslatorMessage needle(context, sourceText, comment, QString(), QString(), 0);

   325     int index = m_messages.indexOf(needle);

   326     return index == -1 ? TranslatorMessage() : m_messages.at(index);

   327 }

   328 

   329 TranslatorMessage Translator::find(const QString &context,

   330     const QString &comment, const TranslatorMessage::References &refs) const

   331 {

   332     if (!refs.isEmpty()) {

   333         for (TMM::ConstIterator it = m_messages.constBegin(); it != m_messages.constEnd(); ++it) {

   334             if (it->context() == context && it->comment() == comment)

   335                 foreach (const TranslatorMessage::Reference &itref, it->allReferences())

   336                     foreach (const TranslatorMessage::Reference &ref, refs)

   337                         if (itref == ref)

   338                             return *it;

   339         }

   340     }

   341     return TranslatorMessage();

   342 }

   343 

   344 bool Translator::contains(const QString &context) const

   345 {

   346     foreach (const TranslatorMessage &msg, m_messages)

   347         if (msg.context() == context && msg.sourceText().isEmpty())

   348             return true;

   349     return false;

   350 }

   351 

   352 TranslatorMessage Translator::find(const QString &context) const

   353 {

   354     foreach (const TranslatorMessage &msg, m_messages)

   355         if (msg.context() == context && msg.sourceText().isEmpty())

   356             return msg;

   357     return TranslatorMessage();

   358 }

   359 

   360 void Translator::stripObsoleteMessages()

   361 {

   362     TMM newmm;

   363     for (TMM::ConstIterator it = m_messages.begin(); it != m_messages.end(); ++it)

   364         if (it->type() != TranslatorMessage::Obsolete)

   365             newmm.append(*it);

   366     m_messages = newmm;

   367 }

   368 

   369 void Translator::stripFinishedMessages()

   370 {

   371     TMM newmm;

   372     for (TMM::ConstIterator it = m_messages.begin(); it != m_messages.end(); ++it)

   373         if (it->type() != TranslatorMessage::Finished)

   374             newmm.append(*it);

   375     m_messages = newmm;

   376 }

   377 

   378 void Translator::stripEmptyContexts()

   379 {

   380     TMM newmm;

   381     for (TMM::ConstIterator it = m_messages.begin(); it != m_messages.end(); ++it)

   382         if (it->sourceText() != QLatin1String(ContextComment))

   383             newmm.append(*it);

   384     m_messages = newmm;

   385 }

   386 

   387 void Translator::stripNonPluralForms()

   388 {

   389     TMM newmm;

   390     for (TMM::ConstIterator it = m_messages.begin(); it != m_messages.end(); ++it)

   391         if (it->isPlural())

   392             newmm.append(*it);

   393     m_messages = newmm;

   394 }

   395 

   396 void Translator::stripIdenticalSourceTranslations()

   397 {

   398     TMM newmm;

   399     for (TMM::ConstIterator it = m_messages.begin(); it != m_messages.end(); ++it) {

   400         // we need to have just one translation, and it be equal to the source

   401         if (it->translations().count() != 1)

   402             newmm.append(*it);

   403         else if (it->translation() != it->sourceText())

   404             newmm.append(*it);

   405     }

   406     m_messages = newmm;

   407 }

   408 

   409 void Translator::dropTranslations()

   410 {

   411     for (TMM::Iterator it = m_messages.begin(); it != m_messages.end(); ++it) {

   412         if (it->type() == TranslatorMessage::Finished)

   413             it->setType(TranslatorMessage::Unfinished);

   414         it->setTranslation(QString());

   415     }

   416 }

   417 

   418 void Translator::dropUiLines()

   419 {

   420     QString uiXt = QLatin1String(".ui");

   421     QString juiXt = QLatin1String(".jui");

   422     for (TMM::Iterator it = m_messages.begin(); it != m_messages.end(); ++it) {

   423         QHash<QString, int> have;

   424         QList<TranslatorMessage::Reference> refs;

   425         foreach (const TranslatorMessage::Reference &itref, it->allReferences()) {

   426             const QString &fn = itref.fileName();

   427             if (fn.endsWith(uiXt) || fn.endsWith(juiXt)) {

   428                 if (++have[fn] == 1)

   429                     refs.append(TranslatorMessage::Reference(fn, -1));

   430             } else {

   431                 refs.append(itref);

   432             }

   433         }

   434         it->setReferences(refs);

   435     }

   436 }

   437 

   438 QSet<TranslatorMessagePtr> Translator::resolveDuplicates()

   439 {

   440     QSet<TranslatorMessagePtr> dups;

   441     QHash<TranslatorMessagePtr, int> refs;

   442     for (int i = 0; i < m_messages.count();) {

   443         const TranslatorMessage &msg = m_messages.at(i);

   444         QHash<TranslatorMessagePtr, int>::ConstIterator it = refs.constFind(msg);

   445         if (it != refs.constEnd()) {

   446             TranslatorMessage &omsg = m_messages[*it];

   447             if (omsg.isUtf8() != msg.isUtf8() && !omsg.isNonUtf8()) {

   448                 // Dual-encoded message

   449                 omsg.setUtf8(true);

   450                 omsg.setNonUtf8(true);

   451             } else {

   452                 // Duplicate

   453                 dups.insert(omsg);

   454             }

   455             if (!omsg.isTranslated() && msg.isTranslated())

   456                 omsg.setTranslations(msg.translations());

   457             m_messages.removeAt(i);

   458         } else {

   459             refs[msg] = i;

   460             ++i;

   461         }

   462     }

   463     return dups;

   464 }

   465 

   466 void Translator::reportDuplicates(const QSet<TranslatorMessagePtr> &dupes,

   467                                   const QString &fileName, bool verbose)

   468 {

   469     if (!dupes.isEmpty()) {

   470         if (!verbose) {

   471             qWarning("Warning: dropping duplicate messages in '%s'\n(try -verbose for more info).",

   472                      qPrintable(fileName));

   473         } else {

   474             qWarning("Warning: dropping duplicate messages in '%s':", qPrintable(fileName));

   475             foreach (const TranslatorMessagePtr &msg, dupes) {

   476                 qWarning("\n* Context: %s\n* Source: %s",

   477                         qPrintable(msg->context()),

   478                         qPrintable(msg->sourceText()));

   479                 if (!msg->comment().isEmpty())

   480                     qWarning("* Comment: %s", qPrintable(msg->comment()));

   481             }

   482             qWarning();

   483         }

   484     }

   485 }

   486 

   487 // Used by lupdate to be able to search using absolute paths during merging

   488 void Translator::makeFileNamesAbsolute(const QDir &originalPath)

   489 {

   490     TMM newmm;

   491     for (TMM::iterator it = m_messages.begin(); it != m_messages.end(); ++it) {

   492         TranslatorMessage msg = *it;

   493         msg.setReferences(TranslatorMessage::References());

   494         foreach (const TranslatorMessage::Reference &ref, it->allReferences()) {

   495             QString fileName = ref.fileName();

   496             QFileInfo fi (fileName);

   497             if (fi.isRelative())

   498                 fileName = originalPath.absoluteFilePath(fileName);

   499             msg.addReference(fileName, ref.lineNumber());

   500         }

   501         newmm.append(msg);

   502     }

   503     m_messages = newmm;

   504 }

   505 

   506 QList<TranslatorMessage> Translator::messages() const

   507 {

   508     return m_messages;

   509 }

   510 

   511 QList<TranslatorMessage> Translator::translatedMessages() const

   512 {

   513     TMM result;

   514     for (TMM::ConstIterator it = m_messages.begin(); it != m_messages.end(); ++it)

   515         if (it->type() == TranslatorMessage::Finished)

   516             result.append(*it);

   517     return result;

   518 }

   519 

   520 QStringList Translator::normalizedTranslations(const TranslatorMessage &msg, int numPlurals)

   521 {

   522     QStringList translations = msg.translations();

   523     int numTranslations = msg.isPlural() ? numPlurals : 1;

   524 

   525     // make sure that the stringlist always have the size of the

   526     // language's current numerus, or 1 if its not plural

   527     if (translations.count() > numTranslations) {

   528         for (int i = translations.count(); i > numTranslations; --i)

   529             translations.removeLast();

   530     } else if (translations.count() < numTranslations) {

   531         for (int i = translations.count(); i < numTranslations; ++i)

   532             translations.append(QString());

   533     }

   534     return translations;

   535 }

   536 

   537 void Translator::normalizeTranslations(ConversionData &cd)

   538 {

   539     bool truncated = false;

   540     QLocale::Language l;

   541     QLocale::Country c;

   542     languageAndCountry(languageCode(), &l, &c);

   543     int numPlurals = 1;

   544     if (l != QLocale::C) {

   545         QStringList forms;

   546         if (getNumerusInfo(l, c, 0, &forms))

   547             numPlurals = forms.count(); // includes singular

   548     }

   549     for (int i = 0; i < m_messages.count(); ++i) {

   550         const TranslatorMessage &msg = m_messages.at(i);

   551         QStringList tlns = msg.translations();

   552         int ccnt = msg.isPlural() ? numPlurals : 1;

   553         if (tlns.count() != ccnt) {

   554             while (tlns.count() < ccnt)

   555                 tlns.append(QString());

   556             while (tlns.count() > ccnt) {

   557                 tlns.removeLast();

   558                 truncated = true;

   559             }

   560             TranslatorMessage msg2(msg);

   561             msg2.setTranslations(tlns);

   562             m_messages[i] = msg2;

   563         }

   564     }

   565     if (truncated)

   566         cd.appendError(QLatin1String(

   567             "Removed plural forms as the target language has less "

   568             "forms.\nIf this sounds wrong, possibly the target language is "

   569             "not set or recognized.\n"));

   570 }

   571 

   572 QString Translator::guessLanguageCodeFromFileName(const QString &filename)

   573 {

   574     QString str = filename;

   575     foreach (const FileFormat &format, registeredFileFormats()) {

   576         if (str.endsWith(format.extension)) {

   577             str = str.left(str.size() - format.extension.size() - 1);

   578             break;

   579         }

   580     }

   581     static QRegExp re(QLatin1String("[\\._]"));

   582     while (true) {

   583         QLocale locale(str);

   584         //qDebug() << "LANGUAGE FROM " << str << "LANG: " << locale.language();

   585         if (locale.language() != QLocale::C) {

   586             //qDebug() << "FOUND " << locale.name();

   587             return locale.name();

   588         }

   589         int pos = str.indexOf(re);

   590         if (pos == -1)

   591             break;

   592         str = str.mid(pos + 1);

   593     }

   594     //qDebug() << "LANGUAGE GUESSING UNSUCCESSFUL";

   595     return QString();

   596 }

   597 

   598 bool Translator::hasExtra(const QString &key) const

   599 {

   600     return m_extra.contains(key);

   601 }

   602 

   603 QString Translator::extra(const QString &key) const

   604 {

   605     return m_extra[key];

   606 }

   607 

   608 void Translator::setExtra(const QString &key, const QString &value)

   609 {

   610     m_extra[key] = value;

   611 }

   612 

   613 void Translator::setCodecName(const QByteArray &name)

   614 {

   615     QTextCodec *codec = QTextCodec::codecForName(name);

   616     if (!codec) {

   617         if (!name.isEmpty())

   618             qWarning("No QTextCodec for %s available. Using Latin1\n", name.constData());

   619         m_codecName = "ISO-8859-1";

   620     } else {

   621         m_codecName = codec->name();

   622     }

   623 }

   624 

   625 void Translator::dump() const

   626 {

   627     for (int i = 0; i != messageCount(); ++i)

   628         message(i).dump();

   629 }

   630 

   631 QT_END_NAMESPACE