/*

    This file is part of the KDE libraries

    Copyright (C) 1997 Martin Jones (mjones@kde.org)

              (C) 1997 Torben Weis (weis@kde.org)

              (C) 1998 Waldo Bastian (bastian@kde.org)

              (C) 1999 Lars Knoll (knoll@kde.org)

              (C) 1999 Antti Koivisto (koivisto@kde.org)

              (C) 2001 Dirk Mueller (mueller@kde.org)

    Copyright (C) 2004, 2005, 2006, 2007 Apple Inc.

    Copyright (C) 2005, 2006 Alexey Proskuryakov (ap@nypop.com)

    This library is free software; you can redistribute it and/or

    modify it under the terms of the GNU Library General Public

    License as published by the Free Software Foundation; either

    version 2 of the License, or (at your option) any later version.

    This library is distributed in the hope that it will be useful,

    but WITHOUT ANY WARRANTY; without even the implied warranty of

    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

    Library General Public License for more details.

    You should have received a copy of the GNU Library General Public License

    along with this library; see the file COPYING.LIB.  If not, write to

    the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,

    Boston, MA 02110-1301, USA.

*/

#include "config.h"

#include "HTMLTokenizer.h"

#include "CSSHelper.h"

#include "Cache.h"

#include "CachedScript.h"

#include "DocLoader.h"

#include "DocumentFragment.h"

#include "EventNames.h"

#include "Frame.h"

#include "FrameLoader.h"

#include "FrameView.h"

#include "HTMLElement.h"

#include "HTMLNames.h"

#include "HTMLParser.h"

#include "HTMLScriptElement.h"

#include "HTMLViewSourceDocument.h"

#if PRELOAD_SCANNER_ENABLED 

#include "PreloadScanner.h"

#endif

#include "Settings.h"

#include "SystemTime.h"

#include "kjs_proxy.h"

#if PRELOAD_SCANNER_ENABLED 

//#include <wtf/ASCIICType.h>

#endif

#include "HTMLEntityNames.c"

#if PLATFORM(SYMBIAN)

#include "oom.h"

#endif

// #define INSTRUMENT_LAYOUT_SCHEDULING 1

#if MOBILE

// The mobile device needs to be responsive, as such the tokenizer chunk size is reduced.

// This value is used to define how many characters the tokenizer will process before 

// yeilding control.

#define TOKENIZER_CHUNK_SIZE  256

#else

#define TOKENIZER_CHUNK_SIZE  4096

#endif

using namespace std;

#if PRELOAD_SCANNER_ENABLED 

using namespace WTF;

#endif

namespace WebCore {

using namespace HTMLNames;

using namespace EventNames;

#if MOBILE

// As the chunks are smaller (above), the tokenizer should not yield for as long a period, otherwise

// it will take way to long to load a page.

const double tokenizerTimeDelay = 0.300;

#else

// FIXME: We would like this constant to be 200ms.

// Yielding more aggressively results in increased responsiveness and better incremental rendering.

// It slows down overall page-load on slower machines, though, so for now we set a value of 500.

const double tokenizerTimeDelay = 0.500;

#endif

static const char commentStart [] = "<!--";

static const char scriptEnd [] = "</script";

static const char xmpEnd [] = "</xmp";

static const char styleEnd [] =  "</style";

static const char textareaEnd [] = "</textarea";

static const char titleEnd [] = "</title";

// Full support for MS Windows extensions to Latin-1.

// Technically these extensions should only be activated for pages

// marked "windows-1252" or "cp1252", but

// in the standard Microsoft way, these extensions infect hundreds of thousands

// of web pages.  Note that people with non-latin-1 Microsoft extensions

// are SOL.

//

// See: http://www.microsoft.com/globaldev/reference/WinCP.asp

//      http://www.bbsinc.com/iso8859.html

//      http://www.obviously.com/

//

// There may be better equivalents

// We only need this for entities. For non-entity text, we handle this in the text encoding.

static const UChar windowsLatin1ExtensionArray[32] = {

    0x20AC, 0x0081, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, // 80-87

    0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0x008D, 0x017D, 0x008F, // 88-8F

    0x0090, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, // 90-97

    0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0x009D, 0x017E, 0x0178  // 98-9F

};

static inline UChar fixUpChar(UChar c)

{

    if ((c & ~0x1F) != 0x0080)

        return c;

    return windowsLatin1ExtensionArray[c - 0x80];

}

static inline bool tagMatch(const char* s1, const UChar* s2, unsigned length)

{

    for (unsigned i = 0; i != length; ++i) {

        unsigned char c1 = s1[i];

        unsigned char uc1 = toupper(c1);

        UChar c2 = s2[i];

        if (c1 != c2 && uc1 != c2)

            return false;

    }

    return true;

}

inline void Token::addAttribute(Document* doc, AtomicString& attrName, const AtomicString& v, bool viewSourceMode)

{

    if (!attrName.isEmpty()) {

        ASSERT(!attrName.contains('/'));

        Attribute* a = new MappedAttribute(attrName, v);

        if (!attrs)

            attrs = new NamedMappedAttrMap(0);

        attrs->insertAttribute(a, viewSourceMode);

    }

    attrName = emptyAtom;

}

// ----------------------------------------------------------------------------

HTMLTokenizer::HTMLTokenizer(HTMLDocument* doc, bool reportErrors)

    : Tokenizer()

    , buffer(0)

    , scriptCode(0)

    , scriptCodeSize(0)

    , scriptCodeMaxSize(0)

    , scriptCodeResync(0)

    , m_executingScript(0)

    , m_requestingScript(false)

    , m_hasScriptsWaitingForStylesheets(false)

    , m_timer(this, &HTMLTokenizer::timerFired)

    , m_doc(doc)

    , parser(new HTMLParser(doc, reportErrors))

    , inWrite(false)

    , m_fragment(false)

#if PLATFORM(SYMBIAN)

    , executingExternalScript(false)

    , executingDocumentWrite(false)

#endif

{

    begin();

}

HTMLTokenizer::HTMLTokenizer(HTMLViewSourceDocument* doc)

    : Tokenizer(true)

    , buffer(0)

    , scriptCode(0)

    , scriptCodeSize(0)

    , scriptCodeMaxSize(0)

    , scriptCodeResync(0)

    , m_executingScript(0)

    , m_requestingScript(false)

    , m_hasScriptsWaitingForStylesheets(false)

    , m_timer(this, &HTMLTokenizer::timerFired)

    , m_doc(doc)

    , parser(0)

    , inWrite(false)

    , m_fragment(false)

#if PLATFORM(SYMBIAN)

    , executingExternalScript(false)

    , executingDocumentWrite(false)

#endif

{

    begin();

}

HTMLTokenizer::HTMLTokenizer(DocumentFragment* frag)

    : buffer(0)

    , scriptCode(0)

    , scriptCodeSize(0)

    , scriptCodeMaxSize(0)

    , scriptCodeResync(0)

    , m_executingScript(0)

    , m_requestingScript(false)

    , m_hasScriptsWaitingForStylesheets(false)

    , m_timer(this, &HTMLTokenizer::timerFired)

    , m_doc(frag->document())

    , inWrite(false)

    , m_fragment(true)

#if PLATFORM(SYMBIAN)

    , executingExternalScript(false)

    , executingDocumentWrite(false)

#endif

{

    parser = new HTMLParser(frag);

    begin();

}

void HTMLTokenizer::reset()

{

    ASSERT(m_executingScript == 0);

    while (!pendingScripts.isEmpty()) {

      CachedScript *cs = pendingScripts.dequeue();

      ASSERT(cache()->disabled() || cs->accessCount() > 0);

      cs->deref(this);

    }

    fastFree(buffer);

    buffer = dest = 0;

    size = 0;

    fastFree(scriptCode);

    scriptCode = 0;

    scriptCodeSize = scriptCodeMaxSize = scriptCodeResync = 0;

    m_timer.stop();

    m_state.setAllowYield(false);

    m_state.setForceSynchronous(false);

    currToken.reset();

}

void HTMLTokenizer::begin()

{

    m_executingScript = 0;

    m_requestingScript = false;

    m_hasScriptsWaitingForStylesheets = false;

    m_state.setLoadingExtScript(false);

    reset();

    size = 254;

    buffer = static_cast<UChar*>(fastMalloc(sizeof(UChar) * 254));

    dest = buffer;

    tquote = NoQuote;

    searchCount = 0;

    m_state.setEntityState(NoEntity);

    scriptSrc = String();

    pendingSrc.clear();

    currentPrependingSrc = 0;

    noMoreData = false;

    brokenComments = false;

    brokenServer = false;

    lineno = 0;

    scriptStartLineno = 0;

    tagStartLineno = 0;

    m_state.setForceSynchronous(false);

}

void HTMLTokenizer::setForceSynchronous(bool force)

{

    m_state.setForceSynchronous(force);

}

HTMLTokenizer::State HTMLTokenizer::processListing(SegmentedString list, State state)

{

    // This function adds the listing 'list' as

    // preformatted text-tokens to the token-collection

    while (!list.isEmpty()) {

        if (state.skipLF()) {

            state.setSkipLF(false);

            if (*list == '\n') {

                list.advance(0);

                continue;

            }

        }

        checkBuffer();

        if (*list == '\n' || *list == '\r') {

            if (state.discardLF())

                // Ignore this LF

                state.setDiscardLF(false); // We have discarded 1 LF

            else

                *dest++ = '\n';

            /* Check for MS-DOS CRLF sequence */

            if (*list == '\r')

                state.setSkipLF(true);

            list.advance(0);

        } else {

            state.setDiscardLF(false);

            *dest++ = *list;

            list.advance(0);

        }

    }

    return state;

}

HTMLTokenizer::State HTMLTokenizer::parseSpecial(SegmentedString &src, State state)

{

    ASSERT(state.inTextArea() || state.inTitle() || !state.hasEntityState());

    ASSERT(!state.hasTagState());

    ASSERT(state.inXmp() + state.inTextArea() + state.inTitle() + state.inStyle() + state.inScript() == 1 );

    if (state.inScript())

        scriptStartLineno = lineno;

    if (state.inComment()) 

        state = parseComment(src, state);

    while ( !src.isEmpty() ) {

        checkScriptBuffer();

        UChar ch = *src;

        if (!scriptCodeResync && !brokenComments && !state.inTextArea() && !state.inXmp() && ch == '-' && scriptCodeSize >= 3 && !src.escaped() && scriptCode[scriptCodeSize-3] == '<' && scriptCode[scriptCodeSize-2] == '!' && scriptCode[scriptCodeSize-1] == '-') {

            state.setInComment(true);

            state = parseComment(src, state);

            continue;

        }

        if (scriptCodeResync && !tquote && ch == '>') {

            src.advance(lineNumberPtr());

            scriptCodeSize = scriptCodeResync-1;

            scriptCodeResync = 0;

            scriptCode[ scriptCodeSize ] = scriptCode[ scriptCodeSize + 1 ] = 0;

            if (state.inScript())

                state = scriptHandler(state);

            else {

                state = processListing(SegmentedString(scriptCode, scriptCodeSize), state);

                processToken();

                if (state.inStyle()) { 

                    currToken.tagName = styleTag.localName(); 

                    currToken.beginTag = false; 

                } else if (state.inTextArea()) { 

                    currToken.tagName = textareaTag.localName(); 

                    currToken.beginTag = false; 

                } else if (state.inTitle()) { 

                    currToken.tagName = titleTag.localName(); 

                    currToken.beginTag = false; 

                } else if (state.inXmp()) {

                    currToken.tagName = xmpTag.localName(); 

                    currToken.beginTag = false; 

                }

                processToken();

                state.setInStyle(false);

                state.setInScript(false);

                state.setInTextArea(false);

                state.setInTitle(false);

                state.setInXmp(false);

                tquote = NoQuote;

                scriptCodeSize = scriptCodeResync = 0;

            }

            return state;

        }

        // possible end of tagname, lets check.

        if (!scriptCodeResync && !state.escaped() && !src.escaped() && (ch == '>' || ch == '/' || ch <= ' ') && ch &&

             scriptCodeSize >= searchStopperLen &&

             tagMatch( searchStopper, scriptCode+scriptCodeSize-searchStopperLen, searchStopperLen )) {

            scriptCodeResync = scriptCodeSize-searchStopperLen+1;

            tquote = NoQuote;

            continue;

        }

        if (scriptCodeResync && !state.escaped()) {

            if (ch == '\"')

                tquote = (tquote == NoQuote) ? DoubleQuote : ((tquote == SingleQuote) ? SingleQuote : NoQuote);

            else if (ch == '\'')

                tquote = (tquote == NoQuote) ? SingleQuote : (tquote == DoubleQuote) ? DoubleQuote : NoQuote;

            else if (tquote != NoQuote && (ch == '\r' || ch == '\n'))

                tquote = NoQuote;

        }

        state.setEscaped(!state.escaped() && ch == '\\');

        if (!scriptCodeResync && (state.inTextArea() || state.inTitle()) && !src.escaped() && ch == '&') {

            UChar* scriptCodeDest = scriptCode+scriptCodeSize;

            src.advance(lineNumberPtr());

            state = parseEntity(src, scriptCodeDest, state, m_cBufferPos, true, false);

            scriptCodeSize = scriptCodeDest-scriptCode;

        } else {

            scriptCode[scriptCodeSize++] = *src;

            src.advance(lineNumberPtr());

        }

    }

    return state;

}

HTMLTokenizer::State HTMLTokenizer::scriptHandler(State state)

{

    // We are inside a <script>

    bool doScriptExec = false;

    // (Bugzilla 3837) Scripts following a frameset element should not execute or, 

    // in the case of extern scripts, even load.

    bool followingFrameset = (m_doc->body() && m_doc->body()->hasTagName(framesetTag));

    CachedScript* cs = 0;

    // don't load external scripts for standalone documents (for now)

    if (!inViewSourceMode()) {

        if (!scriptSrc.isEmpty() && m_doc->frame()) {

            // forget what we just got; load from src url instead

            if (!parser->skipMode() && !followingFrameset) {

#ifdef INSTRUMENT_LAYOUT_SCHEDULING

                if (!m_doc->ownerElement())

                    printf("Requesting script at time %d\n", m_doc->elapsedTime());

#endif

                // The parser might have been stopped by for example a window.close call in an earlier script.

                // If so, we don't want to load scripts.

                if (!m_parserStopped && (cs = m_doc->docLoader()->requestScript(scriptSrc, scriptSrcCharset)))

                    pendingScripts.enqueue(cs);

                else

                    scriptNode = 0;

            } else

                scriptNode = 0;

            scriptSrc = String();

        } else {

#ifdef TOKEN_DEBUG

            kdDebug( 6036 ) << "---START SCRIPT---" << endl;

            kdDebug( 6036 ) << DeprecatedString(scriptCode, scriptCodeSize) << endl;

            kdDebug( 6036 ) << "---END SCRIPT---" << endl;

#endif

            // Parse scriptCode containing <script> info

#if USE(LOW_BANDWIDTH_DISPLAY)

            if (m_doc->inLowBandwidthDisplay()) {

                // ideal solution is only skipping internal JavaScript if there is external JavaScript.

                // but internal JavaScript can use document.write() to create an external JavaScript,

                // so we have to skip internal JavaScript all the time.

                m_doc->frame()->loader()->needToSwitchOutLowBandwidthDisplay();

                doScriptExec = false;

            } else

#endif

            doScriptExec = static_cast<HTMLScriptElement*>(scriptNode.get())->shouldExecuteAsJavaScript();

            scriptNode = 0;

        }

    }

    state = processListing(SegmentedString(scriptCode, scriptCodeSize), state);

    DeprecatedString exScript(reinterpret_cast<DeprecatedChar*>(buffer), dest - buffer);

    processToken();

    currToken.tagName = scriptTag.localName();

    currToken.beginTag = false;

    processToken();

    state.setInScript(false);

    // FIXME: The script should be syntax highlighted.

    if (inViewSourceMode())

        return state;

    SegmentedString *savedPrependingSrc = currentPrependingSrc;

    SegmentedString prependingSrc;

    currentPrependingSrc = &prependingSrc;

    scriptCodeSize = scriptCodeResync = 0;

    if (!parser->skipMode() && !followingFrameset) {

        if (cs) {

            if (savedPrependingSrc)

                savedPrependingSrc->append(src);

            else

                pendingSrc.prepend(src);

            setSrc(SegmentedString());

            // the ref() call below may call notifyFinished if the script is already in cache,

            // and that mucks with the state directly, so we must write it back to the object.

            m_state = state;

            bool savedRequestingScript = m_requestingScript;

            m_requestingScript = true;

            cs->ref(this);

            m_requestingScript = savedRequestingScript;

            state = m_state;

            // will be 0 if script was already loaded and ref() executed it

            if (!pendingScripts.isEmpty())

                state.setLoadingExtScript(true);

        } else if (!m_fragment && doScriptExec) {

            if (!m_executingScript)

                pendingSrc.prepend(src);

            else

                prependingSrc = src;

            setSrc(SegmentedString());

            state = scriptExecution(exScript, state, DeprecatedString::null, scriptStartLineno);

        }

    }

    if (!m_executingScript && !state.loadingExtScript()) {

        src.append(pendingSrc);

        pendingSrc.clear();

    } else if (!prependingSrc.isEmpty()) {

        // restore first so that the write appends in the right place

        // (does not hurt to do it again below)

        currentPrependingSrc = savedPrependingSrc;

        // we need to do this slightly modified bit of one of the write() cases

        // because we want to prepend to pendingSrc rather than appending

        // if there's no previous prependingSrc

        if (!pendingScripts.isEmpty()) {

            if (currentPrependingSrc) {

                currentPrependingSrc->append(prependingSrc);

            } else {

                pendingSrc.prepend(prependingSrc);

            }

        } else {

            m_state = state;

            write(prependingSrc, false);

            state = m_state;

        }