/****************************************************************************

**

** All rights reserved.

** Contact: Nokia Corporation (qt-info@nokia.com)

**

** This file is part of the Qt Linguist of the Qt Toolkit.

**

** $QT_BEGIN_LICENSE:LGPL$

** No Commercial Usage

** This file contains pre-release code and may not be distributed.

** You may use this file in accordance with the terms and conditions

** contained in the Technology Preview License Agreement accompanying

** this package.

**

** GNU Lesser General Public License Usage

** Alternatively, this file may be used under the terms of the GNU Lesser

** General Public License version 2.1 as published by the Free Software

** Foundation and appearing in the file LICENSE.LGPL included in the

** packaging of this file.  Please review the following information to

** ensure the GNU Lesser General Public License version 2.1 requirements

** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.

**

** In addition, as a special exception, Nokia gives you certain additional

** rights.  These rights are described in the Nokia Qt LGPL Exception

** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.

**

** If you have questions regarding the use of this file, please contact

** Nokia at qt-info@nokia.com.

**

**

**

**

**

**

**

**

** $QT_END_LICENSE$

**

****************************************************************************/

#include "translator.h"

#include <QtCore/QByteArray>

#include <QtCore/QDebug>

#include <QtCore/QTextCodec>

#include <QtCore/QTextStream>

#include <QtXml/QXmlStreamReader>

#include <QtXml/QXmlStreamAttribute>

#define STRINGIFY_INTERNAL(x) #x

#define STRINGIFY(x) STRINGIFY_INTERNAL(x)

#define STRING(s) static QString str##s(QLatin1String(STRINGIFY(s)))

QT_BEGIN_NAMESPACE

/*

 * The encodings are a total mess.

 * A Translator has a codecForTr(). Each message's text will be passed to tr()

 * in that encoding or as UTF-8 to trUtf8() if it is flagged as such.

 * For ts 2.0, the file content is always uniformly in UTF-8. The file stores

 * the codecForTr default and marks deviating messages accordingly.

 * For ts 1.1, the file content is in mixed encoding. Each message is encoded

 * the way it will be passed to tr() (with 8-bit characters encoded as numeric

 * entities) or trUtf8(). The file stores the encoding and codecForTr in one

 * attribute, for both the default and each deviating message.

 */

QDebug &operator<<(QDebug &d, const QXmlStreamAttribute &attr)

{

    return d << "[" << attr.name().toString() << "," << attr.value().toString() << "]";

}

class TSReader : public QXmlStreamReader

{

public:

    TSReader(QIODevice &dev, ConversionData &cd)

      : QXmlStreamReader(&dev), m_cd(cd)

    {}

    // the "real thing"

    bool read(Translator &translator);

private:

    bool elementStarts(const QString &str) const

    {

        return isStartElement() && name() == str;

    }

    bool isWhiteSpace() const

    {

        return isCharacters() && text().toString().trimmed().isEmpty();

    }

    // needed to expand <byte ... />

    QString readContents();

    // needed to join <lengthvariant>s

    QString readTransContents();

    void handleError();

    ConversionData &m_cd;

};

void TSReader::handleError()

{

    if (isComment())

        return;

    if (hasError() && error() == CustomError) // raised by readContents

        return;

    const QString loc = QString::fromLatin1("at %3:%1:%2")

        .arg(lineNumber()).arg(columnNumber()).arg(m_cd.m_sourceFileName);

    switch (tokenType()) {

    case NoToken: // Cannot happen

    default: // likewise

    case Invalid:

        raiseError(QString::fromLatin1("Parse error %1: %2").arg(loc, errorString()));

        break;

    case StartElement:

        raiseError(QString::fromLatin1("Unexpected tag <%1> %2").arg(name().toString(), loc));

        break;

    case Characters:

        {

            QString tok = text().toString();

            if (tok.length() > 30)

                tok = tok.left(30) + QLatin1String("[...]");

            raiseError(QString::fromLatin1("Unexpected characters '%1' %2").arg(tok, loc));

        }

        break;

    case EntityReference:

        raiseError(QString::fromLatin1("Unexpected entity '&%1;' %2").arg(name().toString(), loc));

        break;

    case ProcessingInstruction:

        raiseError(QString::fromLatin1("Unexpected processing instruction %1").arg(loc));

        break;

    }

}

static QString byteValue(QString value)

{

    int base = 10;

    if (value.startsWith(QLatin1String("x"))) {

        base = 16;

        value.remove(0, 1);

    }

    int n = value.toUInt(0, base);

    return (n != 0) ? QString(QChar(n)) : QString();

}

QString TSReader::readContents()

{

    STRING(byte);

    STRING(value);

    QString result;

    while (!atEnd()) {

        readNext();

        if (isEndElement()) {

            break;

        } else if (isCharacters()) {

            result += text();

        } else if (elementStarts(strbyte)) {

            // <byte value="...">

            result += byteValue(attributes().value(strvalue).toString());

            readNext();

            if (!isEndElement()) {

                handleError();

                break;

            }

        } else {

            handleError();

            break;

        }

    }

    //qDebug() << "TEXT: " << result;

    return result;

}

QString TSReader::readTransContents()

{

    STRING(lengthvariant);

    STRING(variants);

    STRING(yes);

    if (attributes().value(strvariants) == stryes) {

        QString result;

        while (!atEnd()) {

            readNext();

            if (isEndElement()) {

                break;

            } else if (isWhiteSpace()) {

                // ignore these, just whitespace

            } else if (elementStarts(strlengthvariant)) {

                if (!result.isEmpty())

                    result += QChar(Translator::BinaryVariantSeparator);

                result += readContents();

            } else {

                handleError();

                break;

            }

        }

        return result;

    } else {

        return readContents();

    }

}

bool TSReader::read(Translator &translator)

{

    STRING(both);

    STRING(byte);

    STRING(comment);

    STRING(context);

    STRING(defaultcodec);

    STRING(encoding);

    STRING(extracomment);

    STRING(filename);

    STRING(id);

    STRING(language);

    STRING(line);

    STRING(location);

    STRING(message);

    STRING(name);

    STRING(numerus);

    STRING(numerusform);

    STRING(obsolete);

    STRING(oldcomment);

    STRING(oldsource);

    STRING(source);

    STRING(sourcelanguage);

    STRING(translation);

    STRING(translatorcomment);

    STRING(true);

    STRING(TS);

    STRING(type);

    STRING(unfinished);

    STRING(userdata);

    STRING(utf8);

    STRING(value);

    //STRING(version);

    STRING(yes);

    static const QString strextrans(QLatin1String("extra-"));

    static const QString strUtf8(QLatin1String("UTF-8"));

    while (!atEnd()) {

        readNext();

        if (isStartDocument()) {

            // <!DOCTYPE TS>

            //qDebug() << attributes();

        } else if (isEndDocument()) {

            // <!DOCTYPE TS>

            //qDebug() << attributes();

        } else if (isDTD()) {

            // <!DOCTYPE TS>

            //qDebug() << tokenString();

        } else if (elementStarts(strTS)) {

            // <TS>

            //qDebug() << "TS " << attributes();

            QHash<QString, int> currentLine;

            QString currentFile;

            QXmlStreamAttributes atts = attributes();

            //QString version = atts.value(strversion).toString();

            translator.setLanguageCode(atts.value(strlanguage).toString());

            translator.setSourceLanguageCode(atts.value(strsourcelanguage).toString());

            while (!atEnd()) {

                readNext();

                if (isEndElement()) {

                    // </TS> found, finish local loop

                    break;

                } else if (isWhiteSpace()) {

                    // ignore these, just whitespace

                } else if (elementStarts(strdefaultcodec)) {

                    // <defaultcodec>

                    const QString &codec = readElementText();

                    if (!codec.isEmpty())

                        translator.setCodecName(codec.toLatin1());

                    // </defaultcodec>

                } else if (isStartElement()

                        && name().toString().startsWith(strextrans)) {

                    // <extra-...>

                    QString tag = name().toString();

                    translator.setExtra(tag.mid(6), readContents());

                    // </extra-...>

                } else if (elementStarts(strcontext)) {

                    // <context>

                    QString context;

                    while (!atEnd()) {

                        readNext();

                        if (isEndElement()) {

                            // </context> found, finish local loop

                            break;

                        } else if (isWhiteSpace()) {

                            // ignore these, just whitespace

                        } else if (elementStarts(strname)) {

                            // <name>

                            context = readElementText();

                            // </name>

                        } else if (elementStarts(strmessage)) {

                            // <message>

                            TranslatorMessage::References refs;

                            QString currentMsgFile = currentFile;

                            TranslatorMessage msg;

                            msg.setId(attributes().value(strid).toString());

                            msg.setContext(context);

                            msg.setType(TranslatorMessage::Finished);

                            msg.setPlural(attributes().value(strnumerus) == stryes);

                            const QStringRef &utf8Attr = attributes().value(strutf8);

                            msg.setNonUtf8(utf8Attr == strboth);

                            msg.setUtf8(msg.isNonUtf8() || utf8Attr == strtrue

                                 ||  attributes().value(strencoding) == strUtf8);

                            while (!atEnd()) {

                                readNext();

                                if (isEndElement()) {

                                    // </message> found, finish local loop

                                    msg.setReferences(refs);

                                    translator.append(msg);

                                    break;

                                } else if (isWhiteSpace()) {

                                    // ignore these, just whitespace

                                } else if (elementStarts(strsource)) {

                                    // <source>...</source>

                                    msg.setSourceText(readContents());

                                } else if (elementStarts(stroldsource)) {

                                    // <oldsource>...</oldsource>

                                    msg.setOldSourceText(readContents());

                                } else if (elementStarts(stroldcomment)) {

                                    // <oldcomment>...</oldcomment>

                                    msg.setOldComment(readContents());

                                } else if (elementStarts(strextracomment)) {

                                    // <extracomment>...</extracomment>

                                    msg.setExtraComment(readContents());

                                } else if (elementStarts(strtranslatorcomment)) {

                                    // <translatorcomment>...</translatorcomment>

                                    msg.setTranslatorComment(readContents());

                                } else if (elementStarts(strlocation)) {

                                    // <location/>

                                    QXmlStreamAttributes atts = attributes();

                                    QString fileName = atts.value(strfilename).toString();

                                    if (fileName.isEmpty()) {

                                        fileName = currentMsgFile;

                                    } else {

                                        if (refs.isEmpty())

                                            currentFile = fileName;

                                        currentMsgFile = fileName;

                                    }

                                    const QString lin = atts.value(strline).toString();

                                    if (lin.isEmpty()) {

                                        translator.setLocationsType(Translator::RelativeLocations);

                                        refs.append(TranslatorMessage::Reference(fileName, -1));

                                    } else {

                                        bool bOK;

                                        int lineNo = lin.toInt(&bOK);

                                        if (bOK) {

                                            if (lin.startsWith(QLatin1Char('+')) || lin.startsWith(QLatin1Char('-'))) {

                                                lineNo = (currentLine[fileName] += lineNo);

                                                translator.setLocationsType(Translator::RelativeLocations);

                                            } else {

                                                translator.setLocationsType(Translator::AbsoluteLocations);

                                            }

                                            refs.append(TranslatorMessage::Reference(fileName, lineNo));

                                        }

                                    }

                                    readContents();

                                } else if (elementStarts(strcomment)) {

                                    // <comment>...</comment>

                                    msg.setComment(readContents());

                                } else if (elementStarts(struserdata)) {

                                    // <userdata>...</userdata>

                                    msg.setUserData(readContents());

                                } else if (elementStarts(strtranslation)) {

                                    // <translation>

                                    QXmlStreamAttributes atts = attributes();

                                    QStringRef type = atts.value(strtype);

                                    if (type == strunfinished)

                                        msg.setType(TranslatorMessage::Unfinished);

                                    else if (type == strobsolete)

                                        msg.setType(TranslatorMessage::Obsolete);

                                    if (msg.isPlural()) {

                                        QStringList translations;

                                        while (!atEnd()) {

                                            readNext();

                                            if (isEndElement()) {

                                                break;

                                            } else if (isWhiteSpace()) {

                                                // ignore these, just whitespace

                                            } else if (elementStarts(strnumerusform)) {

                                                translations.append(readTransContents());

                                            } else {

                                                handleError();

                                                break;

                                            }

                                        }

                                        msg.setTranslations(translations);

                                    } else {

                                        msg.setTranslation(readTransContents());

                                    }

                                    // </translation>

                                } else if (isStartElement()

                                        && name().toString().startsWith(strextrans)) {

                                    // <extra-...>

                                    QString tag = name().toString();

                                    msg.setExtra(tag.mid(6), readContents());

                                    // </extra-...>

                                } else {

                                    handleError();

                                }

                            }

                            // </message>

                        } else {

                            handleError();

                        }

                    }

                    // </context>

                } else {

                    handleError();

                }

            } // </TS>

        } else {

            handleError();

        }

    }

    if (hasError()) {

        m_cd.appendError(errorString());

        return false;

    }

    return true;

}

static QString numericEntity(int ch)

{

    return QString(ch <= 0x20 ? QLatin1String("<byte value=\"x%1\"/>")

        : QLatin1String("&#x%1;")) .arg(ch, 0, 16);

}

static QString protect(const QString &str)

{

    QString result;

    result.reserve(str.length() * 12 / 10);

    for (int i = 0; i != str.size(); ++i) {

        uint c = str.at(i).unicode();

        switch (c) {

        case '\"':

            result += QLatin1String(""");

            break;

        case '&':

            result += QLatin1String("&");

            break;

        case '>':

            result += QLatin1String(">");

            break;

        case '<':

            result += QLatin1String("<");

            break;

        case '\'':

            result += QLatin1String("'");

            break;

        default:

            if (c < 0x20 && c != '\r' && c != '\n' && c != '\t')

                result += numericEntity(c);

            else // this also covers surrogates

                result += QChar(c);

        }

    }

    return result;

}

static QString evilBytes(const QString& str,

    bool isUtf8, int format, const QByteArray &codecName)

{

    //qDebug() << "EVIL: " << str << isUtf8 << format << codecName;

    if (isUtf8)

        return protect(str);

    if (format == 20)

        return protect(str);

    if (codecName == "UTF-8")

        return protect(str);

    QTextCodec *codec = QTextCodec::codecForName(codecName);

    if (!codec)

        return protect(str);

    QString t = QString::fromLatin1(codec->fromUnicode(protect(str)).data());

    int len = (int) t.length();

    QString result;

    // FIXME: Factor is sensible only for latin scripts, probably.

    result.reserve(t.length() * 2);

    for (int k = 0; k < len; k++) {

        if (t[k].unicode() >= 0x7f)

            result += numericEntity(t[k].unicode());

        else

            result += t[k];

    }

    return result;

}

static void writeExtras(QTextStream &t, const char *indent,

                        const TranslatorMessage::ExtraData &extras, const QRegExp &drops)

{

    for (Translator::ExtraData::ConstIterator it = extras.begin(); it != extras.end(); ++it) {

        if (!drops.exactMatch(it.key())) {

            t << indent << "<extra-" << it.key() << '>'

              << protect(it.value())

              << "</extra-" << it.key() << ">\n";

        }

    }

}

static void writeVariants(QTextStream &t, const char *indent, const QString &input)

{

    int offset;

    if ((offset = input.indexOf(QChar(Translator::BinaryVariantSeparator))) >= 0) {

        t << " variants=\"yes\">";

        int start = 0;

        forever {

            t << "\n    " << indent << "<lengthvariant>"

              << protect(input.mid(start, offset - start))

              << "</lengthvariant>";

            if (offset == input.length())

                break;

            start = offset + 1;

            offset = input.indexOf(QChar(Translator::BinaryVariantSeparator), start);

            if (offset < 0)