/****************************************************************************

**

** Copyright (C) 2010 Nokia Corporation and/or its subsidiary(-ies).

** All rights reserved.

** Contact: Nokia Corporation (qt-info@nokia.com)

**

** This file is part of the tools applications of the Qt Toolkit.

**

** $QT_BEGIN_LICENSE:LGPL$

** No Commercial Usage

** This file contains pre-release code and may not be distributed.

** You may use this file in accordance with the terms and conditions

** contained in the Technology Preview License Agreement accompanying

** this package.

**

** GNU Lesser General Public License Usage

** Alternatively, this file may be used under the terms of the GNU Lesser

** General Public License version 2.1 as published by the Free Software

** Foundation and appearing in the file LICENSE.LGPL included in the

** packaging of this file.  Please review the following information to

** ensure the GNU Lesser General Public License version 2.1 requirements

** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.

**

** In addition, as a special exception, Nokia gives you certain additional

** rights.  These rights are described in the Nokia Qt LGPL Exception

** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.

**

** If you have questions regarding the use of this file, please contact

** Nokia at qt-info@nokia.com.

**

**

**

**

**

**

**

**

** $QT_END_LICENSE$

**

****************************************************************************/

#include "config.h"

#include "doc.h"

#include "codemarker.h"

#include "editdistance.h"

#include "openedlist.h"

#include "quoter.h"

#include "text.h"

#include "tokenizer.h"

#include <qdatetime.h>

#include <qdebug.h>

#include <qfile.h>

#include <qfileinfo.h>

#include <qhash.h>

#include <qtextstream.h>

#include <qregexp.h>

#include <ctype.h>

#include <limits.h>

QT_BEGIN_NAMESPACE

Q_GLOBAL_STATIC(QSet<QString>, null_Set_QString)

Q_GLOBAL_STATIC(QStringList, null_QStringList)

Q_GLOBAL_STATIC(QList<Text>, null_QList_Text)

Q_GLOBAL_STATIC(QStringMap, null_QStringMap)

struct Macro

{

    QString defaultDef;

    Location defaultDefLocation;

    QStringMap otherDefs;

    int numParams;

};

enum {

    CMD_A, CMD_ABSTRACT, CMD_ANNOTATEDLIST, CMD_BADCODE,

    CMD_BASENAME, CMD_BOLD, CMD_BRIEF, CMD_C, CMD_CAPTION,

    CMD_CHAPTER, CMD_CODE, CMD_CODELINE, CMD_DOTS, CMD_ELSE,

    CMD_ENDABSTRACT, CMD_ENDCHAPTER, CMD_ENDCODE,

    CMD_ENDFOOTNOTE, CMD_ENDIF, CMD_ENDLEGALESE, CMD_ENDLINK,

    CMD_ENDLIST, CMD_ENDOMIT, CMD_ENDPART, CMD_ENDQUOTATION,

    CMD_ENDRAW, CMD_ENDSECTION1, CMD_ENDSECTION2,

    CMD_ENDSECTION3, CMD_ENDSECTION4, CMD_ENDSIDEBAR,

    CMD_ENDTABLE, CMD_EXPIRE, CMD_FOOTNOTE, CMD_GENERATELIST,

    CMD_GRANULARITY, CMD_HEADER, CMD_I, CMD_IF, CMD_IMAGE,

    CMD_INCLUDE, CMD_INLINEIMAGE, CMD_INDEX, CMD_KEYWORD,

    CMD_L, CMD_LEGALESE, CMD_LINK, CMD_LIST, CMD_META,

    CMD_NEWCODE, CMD_O, CMD_OLDCODE, CMD_OMIT, CMD_OMITVALUE,

    CMD_OVERLOAD, CMD_PART, CMD_PRINTLINE, CMD_PRINTTO,

    CMD_PRINTUNTIL, CMD_QUOTATION, CMD_QUOTEFILE,

    CMD_QUOTEFROMFILE, CMD_QUOTEFUNCTION, CMD_RAW, CMD_ROW,

    CMD_SA, CMD_SECTION1, CMD_SECTION2, CMD_SECTION3,

    CMD_SECTION4, CMD_SIDEBAR, CMD_SINCELIST, CMD_SKIPLINE,

    CMD_SKIPTO, CMD_SKIPUNTIL, CMD_SNIPPET, CMD_SUB, CMD_SUP,

    CMD_TABLE, CMD_TABLEOFCONTENTS, CMD_TARGET, CMD_TT,

    CMD_UNDERLINE, CMD_UNICODE, CMD_VALUE, CMD_WARNING,

#ifdef QDOC_QML    

    CMD_QML, CMD_ENDQML, CMD_CPP, CMD_ENDCPP, CMD_QMLTEXT,

    CMD_ENDQMLTEXT, CMD_CPPTEXT, CMD_ENDCPPTEXT,

#endif    

    NOT_A_CMD

};

static struct {

    const char *english;

    int no;

    QString *alias;

} cmds[] = {

    { "a", CMD_A, 0 },

    { "abstract", CMD_ABSTRACT, 0 },

    { "annotatedlist", CMD_ANNOTATEDLIST, 0 },

    { "badcode", CMD_BADCODE, 0 },

    { "basename", CMD_BASENAME, 0 }, // ### don't document for now

    { "bold", CMD_BOLD, 0 },

    { "brief", CMD_BRIEF, 0 },

    { "c", CMD_C, 0 },

    { "caption", CMD_CAPTION, 0 },

    { "chapter", CMD_CHAPTER, 0 },

    { "code", CMD_CODE, 0 },

    { "codeline", CMD_CODELINE, 0},

    { "dots", CMD_DOTS, 0 },

    { "else", CMD_ELSE, 0 },

    { "endabstract", CMD_ENDABSTRACT, 0 },

    { "endchapter", CMD_ENDCHAPTER, 0 },

    { "endcode", CMD_ENDCODE, 0 },

    { "endfootnote", CMD_ENDFOOTNOTE, 0 },

    { "endif", CMD_ENDIF, 0 },

    { "endlegalese", CMD_ENDLEGALESE, 0 },

    { "endlink", CMD_ENDLINK, 0 },

    { "endlist", CMD_ENDLIST, 0 },

    { "endomit", CMD_ENDOMIT, 0 },

    { "endpart", CMD_ENDPART, 0 },

    { "endquotation", CMD_ENDQUOTATION, 0 },

    { "endraw", CMD_ENDRAW, 0 },

    { "endsection1", CMD_ENDSECTION1, 0 },  // ### don't document for now

    { "endsection2", CMD_ENDSECTION2, 0 },  // ### don't document for now

    { "endsection3", CMD_ENDSECTION3, 0 },  // ### don't document for now

    { "endsection4", CMD_ENDSECTION4, 0 },  // ### don't document for now

    { "endsidebar", CMD_ENDSIDEBAR, 0 },

    { "endtable", CMD_ENDTABLE, 0 },

    { "expire", CMD_EXPIRE, 0 },

    { "footnote", CMD_FOOTNOTE, 0 },

    { "generatelist", CMD_GENERATELIST, 0 },

    { "granularity", CMD_GRANULARITY, 0 }, // ### don't document for now

    { "header", CMD_HEADER, 0 },

    { "i", CMD_I, 0 },

    { "if", CMD_IF, 0 },

    { "image", CMD_IMAGE, 0 },

    { "include", CMD_INCLUDE, 0 },

    { "inlineimage", CMD_INLINEIMAGE, 0 },

    { "index", CMD_INDEX, 0 }, // ### don't document for now

    { "keyword", CMD_KEYWORD, 0 },

    { "l", CMD_L, 0 },

    { "legalese", CMD_LEGALESE, 0 },

    { "link", CMD_LINK, 0 },

    { "list", CMD_LIST, 0 },

    { "meta", CMD_META, 0 },

    { "newcode", CMD_NEWCODE, 0 },

    { "o", CMD_O, 0 },

    { "oldcode", CMD_OLDCODE, 0 },

    { "omit", CMD_OMIT, 0 },

    { "omitvalue", CMD_OMITVALUE, 0 },

    { "overload", CMD_OVERLOAD, 0 },

    { "part", CMD_PART, 0 },

    { "printline", CMD_PRINTLINE, 0 },

    { "printto", CMD_PRINTTO, 0 },

    { "printuntil", CMD_PRINTUNTIL, 0 },

    { "quotation", CMD_QUOTATION, 0 },

    { "quotefile", CMD_QUOTEFILE, 0 },

    { "quotefromfile", CMD_QUOTEFROMFILE, 0 },

    { "quotefunction", CMD_QUOTEFUNCTION, 0 }, // ### don't document for now

    { "raw", CMD_RAW, 0 },

    { "row", CMD_ROW, 0 },

    { "sa", CMD_SA, 0 },

    { "section1", CMD_SECTION1, 0 },

    { "section2", CMD_SECTION2, 0 },

    { "section3", CMD_SECTION3, 0 },

    { "section4", CMD_SECTION4, 0 },

    { "sidebar", CMD_SIDEBAR, 0 }, // ### don't document for now

    { "sincelist", CMD_SINCELIST, 0 },

    { "skipline", CMD_SKIPLINE, 0 },

    { "skipto", CMD_SKIPTO, 0 },

    { "skipuntil", CMD_SKIPUNTIL, 0 },

    { "snippet", CMD_SNIPPET, 0 },

    { "sub", CMD_SUB, 0 },

    { "sup", CMD_SUP, 0 },

    { "table", CMD_TABLE, 0 },

    { "tableofcontents", CMD_TABLEOFCONTENTS, 0 },

    { "target", CMD_TARGET, 0 },

    { "tt", CMD_TT, 0 },

    { "underline", CMD_UNDERLINE, 0 },

    { "unicode", CMD_UNICODE, 0 },

    { "value", CMD_VALUE, 0 },

    { "warning", CMD_WARNING, 0 },

#ifdef QDOC_QML

    { "qml", CMD_QML, 0 },

    { "endqml", CMD_ENDQML, 0 },

    { "cpp", CMD_CPP, 0 },

    { "endcpp", CMD_ENDCPP, 0 },

    { "qmltext", CMD_QMLTEXT, 0 },

    { "endqmltext", CMD_ENDQMLTEXT, 0 },

    { "cpptext", CMD_CPPTEXT, 0 },

    { "endcpptext", CMD_ENDCPPTEXT, 0 },

#endif

    { 0, 0, 0 }

};

typedef QHash<QString, int> QHash_QString_int;

typedef QHash<QString, Macro> QHash_QString_Macro;

Q_GLOBAL_STATIC(QStringMap, aliasMap)

Q_GLOBAL_STATIC(QHash_QString_int, cmdHash)

Q_GLOBAL_STATIC(QHash_QString_Macro, macroHash)

class DocPrivateExtra

{

  public:

    QString             baseName;

    Doc::SectioningUnit granularity;

    Doc::SectioningUnit sectioningUnit; // ###

    QList<Atom*>        tableOfContents;

    QList<int>          tableOfContentsLevels;

    QList<Atom*>        keywords;

    QList<Atom*>        targets;

    QStringMap          metaMap;

    DocPrivateExtra()

	: granularity(Doc::Part) { }

};

struct Shared // ### get rid of

{

    Shared()

	: count(1) { }

    void ref() { ++count; }

    bool deref() { return (--count == 0); }

    int count;

};

static QString cleanLink(const QString &link)

{

    int colonPos = link.indexOf(':');

    if ((colonPos == -1) ||

        (!link.startsWith("file:") && !link.startsWith("mailto:")))

        return link;

    return link.mid(colonPos + 1).simplified();

}

class DocPrivate : public Shared

{

  public:

    DocPrivate(const Location& start = Location::null,

               const Location& end = Location::null,

               const QString& source = "");

    ~DocPrivate();

    void addAlso(const Text& also);

    void constructExtra();

    bool isEnumDocSimplifiable() const;

    // ### move some of this in DocPrivateExtra

    Location start_loc;

    Location end_loc;

    QString src;

    Text text;

    QSet<QString> params;

    QList<Text> alsoList;

    QStringList enumItemList;

    QStringList omitEnumItemList;

    QSet<QString> metacommandsUsed;

    QCommandMap metaCommandMap;

    bool hasLegalese : 1;

    bool hasSectioningUnits : 1;

    DocPrivateExtra *extra;

};

DocPrivate::DocPrivate(const Location& start,

                       const Location& end,

                       const QString& source)

    : start_loc(start),

      end_loc(end),

      src(source),

      hasLegalese(false),

      hasSectioningUnits(false),

      extra(0)

{

    // nothing.

}

DocPrivate::~DocPrivate()

{

    delete extra;

}

void DocPrivate::addAlso(const Text& also)

{

    alsoList.append(also);

}

void DocPrivate::constructExtra()

{

    if (extra == 0)

	extra = new DocPrivateExtra;

}

bool DocPrivate::isEnumDocSimplifiable() const

{

    bool justMetColon = false;

    int numValueTables = 0;

    const Atom *atom = text.firstAtom();

    while (atom) {

        if (atom->type() == Atom::AutoLink || atom->type() == Atom::String) {

            justMetColon = atom->string().endsWith(":");

        }

        else if ((atom->type() == Atom::ListLeft) &&

                   (atom->string() == ATOM_LIST_VALUE)) {

            if (justMetColon || numValueTables > 0)

                return false;

            ++numValueTables;

        }

        atom = atom->next();

    }

    return true;

}

class DocParser

{

  public:

    void parse(const QString &source,

               DocPrivate *docPrivate,

               const QSet<QString> &metaCommandSet);

    static int endCmdFor(int cmd);

    static QString cmdName(int cmd);

    static QString endCmdName(int cmd);

    static QString untabifyEtc(const QString& str);

    static int indentLevel(const QString& str);

    static QString unindent(int level, const QString& str);

    static QString slashed(const QString& str);

    static int tabSize;

    static QStringList exampleFiles;

    static QStringList exampleDirs;

    static QStringList sourceFiles;

    static QStringList sourceDirs;

    static bool quoting;

  private:

    Location& location();

    QString detailsUnknownCommand(const QSet<QString>& metaCommandSet,

				   const QString& str);

    void checkExpiry(const QString& date);

    void insertBaseName(const QString &baseName);

    void insertTarget(const QString& target, bool keyword);

    void include(const QString& fileName);

    void startFormat(const QString& format, int cmd);

    bool openCommand(int cmd);

    bool closeCommand(int endCmd);

    void startSection(Doc::SectioningUnit unit, int cmd);

    void endSection(int unit, int endCmd);

    void parseAlso();

    void append(Atom::Type type, const QString& string = "");

    void appendChar(QChar ch);

    void appendWord(const QString &word);

    void appendToCode(const QString &code);

    void startNewPara();

    void enterPara(Atom::Type leftType = Atom::ParaLeft,

		    Atom::Type rightType = Atom::ParaRight,

		    const QString& string = "");

    void leavePara();

    void leaveValue();

    void leaveValueList();

    void leaveTableRow();

    CodeMarker *quoteFromFile();

    void expandMacro(const QString& name, const QString& def, int numParams);

    Doc::SectioningUnit getSectioningUnit();

    QString getArgument(bool verbatim = false);

    QString getOptionalArgument();

    QString getRestOfLine();

    QString getMetaCommandArgument(const QString &cmdStr);

    QString getUntilEnd(int cmd);

    QString getCode(int cmd, CodeMarker *marker);

    QString getUnmarkedCode(int cmd);

    bool isBlankLine();

    bool isLeftBraceAhead();

    void skipSpacesOnLine();

    void skipSpacesOrOneEndl();

    void skipAllSpaces();

    void skipToNextPreprocessorCommand();

    QStack<int> openedInputs;

    QString in;

    int pos;

    int len;

    Location cachedLoc;

    int cachedPos;

    DocPrivate *priv;

    enum ParaState { OutsidePara, InsideSingleLinePara, InsideMultiLinePara };

    ParaState paraState;

    bool inTableHeader;

    bool inTableRow;

    bool inTableItem;

    bool indexStartedPara; // ### rename

    Atom::Type pendingParaLeftType;

    Atom::Type pendingParaRightType;

    QString pendingParaString;

    int braceDepth;

    int minIndent;

    Doc::SectioningUnit currentSectioningUnit;

    QMap<QString, Location> targetMap;

    QMap<int, QString> pendingFormats;

    QStack<int> openedCommands;

    QStack<OpenedList> openedLists;

    Quoter quoter;

};

int DocParser::tabSize;

QStringList DocParser::exampleFiles;

QStringList DocParser::exampleDirs;

QStringList DocParser::sourceFiles;

QStringList DocParser::sourceDirs;

bool DocParser::quoting;

/*!

  Parse the \a source string to build a Text data structure

  in \a docPrivate. The Text data structure is a linked list

  of Atoms.

  \a metaCommandSet is the set of metacommands that may be

  found in \a source. These metacommands are not markup text

  commands. They are topic commands and related metacommands.

 */

void DocParser::parse(const QString& source,

                      DocPrivate *docPrivate,

                      const QSet<QString>& metaCommandSet)

{

    in = source;

    pos = 0;

    len = in.length();

    cachedLoc = docPrivate->start_loc;

    cachedPos = 0;

    priv = docPrivate;

    priv->text << Atom::Nop;

    paraState = OutsidePara;

    inTableHeader = false;

    inTableRow = false;

    inTableItem = false;

    indexStartedPara = false;

    pendingParaLeftType = Atom::Nop;

    pendingParaRightType = Atom::Nop;

    braceDepth = 0;

    minIndent = INT_MAX;

    currentSectioningUnit = Doc::Book;

    openedCommands.push(CMD_OMIT);

    quoter.reset();

    CodeMarker *marker = 0;

    Atom *currentLinkAtom = 0;

    QString x;

    QStack<bool> preprocessorSkipping;

    int numPreprocessorSkipping = 0;

    while (pos < len) {

        QChar ch = in.at(pos);

        switch (ch.unicode()) {

        case '\\':

            {

                QString cmdStr;

                pos++;

                while (pos < len) {

                    ch = in.at(pos);

                    if (ch.isLetterOrNumber()) {

                        cmdStr += ch;

                        pos++;

                    }

                    else {

                        break;

                    }

                }

                if (cmdStr.isEmpty()) {

                    if (pos < len) {

                        enterPara();

                        if (in.at(pos).isSpace()) {

                            skipAllSpaces();

                            appendChar(QLatin1Char(' '));

                        }

                        else {

                            appendChar(in.at(pos++));

                        }

                    }

                }

                else {

                    int cmd = cmdHash()->value(cmdStr,NOT_A_CMD);

                    switch (cmd) {

                    case CMD_A:

                        enterPara();

                        x = getArgument();

                        append(Atom::FormattingLeft,ATOM_FORMATTING_PARAMETER);

                        append(Atom::String, x);

                        append(Atom::FormattingRight,ATOM_FORMATTING_PARAMETER);

                        priv->params.insert(x);

                        break;

                    case CMD_ABSTRACT:

                        if (openCommand(cmd)) {

                            leavePara();

                            append(Atom::AbstractLeft);

                        }

                        break;

                    case CMD_BADCODE:

                        leavePara();

                        append(Atom::CodeBad,getCode(CMD_BADCODE, marker));

                        break;

                    case CMD_BASENAME:

                        leavePara();

                        insertBaseName(getArgument());

                        break;

                    case CMD_BOLD:

                        startFormat(ATOM_FORMATTING_BOLD, cmd);

                        break;