diff -r 000000000000 -r 1918ee327afb tools/assistant/lib/qhelpsearchindexreader_default.cpp
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tools/assistant/lib/qhelpsearchindexreader_default.cpp	Mon Jan 11 14:00:40 2010 +0000
@@ -0,0 +1,614 @@
+/****************************************************************************
+**
+** Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies).
+** All rights reserved.
+** Contact: Nokia Corporation (qt-info@nokia.com)
+**
+** This file is part of the Qt Assistant of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** No Commercial Usage
+** This file contains pre-release code and may not be distributed.
+** You may use this file in accordance with the terms and conditions
+** contained in the Technology Preview License Agreement accompanying
+** this package.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL included in the
+** packaging of this file.  Please review the following information to
+** ensure the GNU Lesser General Public License version 2.1 requirements
+** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Nokia gives you certain additional
+** rights.  These rights are described in the Nokia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+** If you have questions regarding the use of this file, please contact
+** Nokia at qt-info@nokia.com.
+**
+**
+**
+**
+**
+**
+**
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#include "qhelpenginecore.h"
+#include "qhelpsearchindexreader_default_p.h"
+
+#include <QtCore/QDir>
+#include <QtCore/QUrl>
+#include <QtCore/QFile>
+#include <QtCore/QVariant>
+#include <QtCore/QFileInfo>
+#include <QtCore/QDataStream>
+#include <QtCore/QTextStream>
+
+QT_BEGIN_NAMESPACE
+
+namespace qt {
+    namespace fulltextsearch {
+        namespace std {
+
+namespace {
+    QStringList split( const QString &str )
+    {
+        QStringList lst;
+        int j = 0;
+        int i = str.indexOf(QLatin1Char('*'), j );
+
+        if (str.startsWith(QLatin1String("*")))
+            lst << QLatin1String("*");
+
+        while ( i != -1 ) {
+            if ( i > j && i <= (int)str.length() ) {
+                lst << str.mid( j, i - j );
+                lst << QLatin1String("*");
+            }
+            j = i + 1;
+            i = str.indexOf(QLatin1Char('*'), j );
+        }
+
+        int l = str.length() - 1;
+        if ( str.mid( j, l - j + 1 ).length() > 0 )
+            lst << str.mid( j, l - j + 1 );
+
+        return lst;
+    }
+}
+
+
+Reader::Reader()
+    : indexPath(QString())
+    , indexFile(QString())
+    , documentFile(QString())
+{
+    termList.clear();
+    indexTable.clear();
+    searchIndexTable.clear();
+}
+
+Reader::~Reader()
+{
+    reset();
+    searchIndexTable.clear();
+}
+
+bool Reader::readIndex()
+{
+    if (indexTable.contains(indexFile))
+        return true;
+
+    QFile idxFile(indexFile);
+    if (!idxFile.open(QFile::ReadOnly))
+        return false;
+
+    QString key;
+    int numOfDocs;
+    EntryTable entryTable;
+    QVector<Document> docs;
+    QDataStream dictStream(&idxFile);
+    while (!dictStream.atEnd()) {
+        dictStream >> key;
+        dictStream >> numOfDocs;
+        docs.resize(numOfDocs);
+        dictStream >> docs;
+        entryTable.insert(key, new Entry(docs));
+    }
+    idxFile.close();
+
+    if (entryTable.isEmpty())
+        return false;
+
+    QFile docFile(documentFile);
+    if (!docFile.open(QFile::ReadOnly))
+        return false;
+
+    QString title, url;
+    DocumentList documentList;
+    QDataStream docStream(&docFile);
+    while (!docStream.atEnd()) {
+        docStream >> title;
+        docStream >> url;
+        documentList.append(QStringList(title) << url);
+    }
+    docFile.close();
+
+    if (documentList.isEmpty()) {
+        cleanupIndex(entryTable);
+        return false;
+    }
+
+    indexTable.insert(indexFile, Index(entryTable, documentList));
+    return true;
+}
+
+bool Reader::initCheck() const
+{
+    return !searchIndexTable.isEmpty();
+}
+
+void Reader::setIndexPath(const QString &path)
+{
+    indexPath = path;
+}
+
+void Reader::filterFilesForAttributes(const QStringList &attributes)
+{
+    searchIndexTable.clear();
+    for(IndexTable::ConstIterator it = indexTable.begin(); it != indexTable.end(); ++it) {
+        const QString fileName = it.key();
+        bool containsAll = true;
+        QStringList split = fileName.split(QLatin1String("@"));
+        foreach (const QString attribute, attributes) {
+            if (!split.contains(attribute, Qt::CaseInsensitive)) {
+                containsAll = false;
+                break;
+            }
+        }
+
+        if (containsAll)
+            searchIndexTable.insert(fileName, it.value());
+    }
+}
+
+void Reader::setIndexFile(const QString &namespaceName, const QString &attributes)
+{
+    QString extention = namespaceName + QLatin1String("@") + attributes;
+    indexFile = indexPath + QLatin1String("/indexdb40.") + extention;
+    documentFile = indexPath + QLatin1String("/indexdoc40.") + extention;
+}
+
+bool Reader::splitSearchTerm(const QString &searchTerm, QStringList *terms,
+                                  QStringList *termSeq, QStringList *seqWords)
+{
+    QString term = searchTerm;
+
+    term = term.simplified();
+    term = term.replace(QLatin1String("\'"), QLatin1String("\""));
+    term = term.replace(QLatin1String("`"), QLatin1String("\""));
+    term = term.replace(QLatin1String("-"), QLatin1String(" "));
+    term = term.replace(QRegExp(QLatin1String("\\s[\\S]?\\s")), QLatin1String(" "));
+
+    *terms = term.split(QLatin1Char(' '));
+    QStringList::iterator it = terms->begin();
+    for (; it != terms->end(); ++it) {
+        (*it) = (*it).simplified();
+        (*it) = (*it).toLower();
+        (*it) = (*it).replace(QLatin1String("\""), QLatin1String(""));
+    }
+
+    if (term.contains(QLatin1Char('\"'))) {
+        if ((term.count(QLatin1Char('\"')))%2 == 0) {
+            int beg = 0;
+            int end = 0;
+            QString s;
+            beg = term.indexOf(QLatin1Char('\"'), beg);
+            while (beg != -1) {
+                beg++;
+                end = term.indexOf(QLatin1Char('\"'), beg);
+                s = term.mid(beg, end - beg);
+                s = s.toLower();
+                s = s.simplified();
+                if (s.contains(QLatin1Char('*'))) {
+                    qWarning("Full Text Search, using a wildcard within phrases is not allowed.");
+                    return false;
+                }
+                *seqWords += s.split(QLatin1Char(' '));
+                *termSeq << s;
+                beg = term.indexOf(QLatin1Char('\"'), end + 1);
+            }
+        } else {
+            qWarning("Full Text Search, the closing quotation mark is missing.");
+            return false;
+        }
+    }
+
+    return true;
+}
+
+void Reader::searchInIndex(const QStringList &terms)
+{
+    foreach (const QString term, terms) {
+        QVector<Document> documents;
+
+        for(IndexTable::ConstIterator it = searchIndexTable.begin();
+            it != searchIndexTable.end(); ++it) {
+            EntryTable entryTable = it.value().first;
+            DocumentList documentList = it.value().second;
+
+            if (term.contains(QLatin1Char('*')))
+                documents = setupDummyTerm(getWildcardTerms(term, entryTable), entryTable);
+            else if (entryTable.value(term))
+                documents = entryTable.value(term)->documents;
+            else
+                continue;
+
+            if (!documents.isEmpty()) {
+                DocumentInfo info;
+                QString title, url;
+                QVector<DocumentInfo> documentsInfo;
+                foreach(const Document doc, documents) {
+                    info.docNumber = doc.docNumber;
+                    info.frequency = doc.frequency;
+                    info.documentUrl = documentList.at(doc.docNumber).at(1);
+                    info.documentTitle = documentList.at(doc.docNumber).at(0);
+                    documentsInfo.append(info);
+                }
+
+                bool found = false;
+                for(QList<TermInfo>::Iterator tit = termList.begin();
+                    tit != termList.end(); ++tit) {
+                    TermInfo *t = &(*tit);
+                    if(t->term == term) {
+                        t->documents += documentsInfo;
+                        t->frequency += documentsInfo.count();
+                        found = true; break;
+                    }
+                }
+                if (!found)
+                    termList.append(TermInfo(term, documentsInfo.count(), documentsInfo));
+            }
+        }
+    }
+    qSort(termList);
+}
+
+QVector<DocumentInfo> Reader::hits()
+{
+    QVector<DocumentInfo> documents;
+    if (!termList.count())
+        return documents;
+
+    documents = termList.takeFirst().documents;
+    for(QList<TermInfo>::Iterator it = termList.begin(); it != termList.end(); ++it) {
+        TermInfo *t = &(*it);
+        QVector<DocumentInfo> docs = t->documents;
+        for(QVector<DocumentInfo>::Iterator minDoc_it = documents.begin();
+            minDoc_it != documents.end(); ) {
+            bool found = false;
+            for (QVector<DocumentInfo>::ConstIterator doc_it = docs.constBegin();
+                doc_it != docs.constEnd(); ++doc_it ) {
+                if ( (*minDoc_it).docNumber == (*doc_it).docNumber ) {
+                    (*minDoc_it).frequency += (*doc_it).frequency;
+                    found = true;
+                    break;
+                }
+            }
+            if (!found)
+                minDoc_it = documents.erase(minDoc_it);
+            else
+                ++minDoc_it;
+        }
+    }
+
+    qSort(documents);
+    return documents;
+}
+
+bool Reader::searchForPattern(const QStringList &patterns, const QStringList &words,
+                                   const QByteArray &data)
+{
+    if (data.isEmpty())
+        return false;
+
+    for(QHash<QString, PosEntry*>::ConstIterator mit =
+        miniIndex.begin(); mit != miniIndex.end(); ++mit) {
+            delete mit.value();
+    }
+    miniIndex.clear();
+
+    wordNum = 3;
+    QStringList::ConstIterator cIt = words.begin();
+    for ( ; cIt != words.end(); ++cIt )
+        miniIndex.insert(*cIt, new PosEntry(0));
+
+    QTextStream s(data);
+    QString text = s.readAll();
+    bool valid = true;
+    const QChar *buf = text.unicode();
+    QChar str[64];
+    QChar c = buf[0];
+    int j = 0;
+    int i = 0;
+    while ( j < text.length() ) {
+        if ( c == QLatin1Char('<') || c == QLatin1Char('&') ) {
+            valid = false;
+            if ( i > 1 )
+                buildMiniIndex( QString(str,i) );
+            i = 0;
+            c = buf[++j];
+            continue;
+        }
+        if ( ( c == QLatin1Char('>') || c == QLatin1Char(';') ) && !valid ) {
+            valid = true;
+            c = buf[++j];
+            continue;
+        }
+        if ( !valid ) {
+            c = buf[++j];
+            continue;
+        }
+        if ( ( c.isLetterOrNumber() || c == QLatin1Char('_') ) && i < 63 ) {
+            str[i] = c.toLower();
+            ++i;
+        } else {
+            if ( i > 1 )
+                buildMiniIndex( QString(str,i) );
+            i = 0;
+        }
+        c = buf[++j];
+    }
+    if ( i > 1 )
+        buildMiniIndex( QString(str,i) );
+
+    QStringList::ConstIterator patIt = patterns.begin();
+    QStringList wordLst;
+    QList<uint> a, b;
+    QList<uint>::iterator aIt;
+    for ( ; patIt != patterns.end(); ++patIt ) {
+        wordLst = (*patIt).split(QLatin1Char(' '));
+        a = miniIndex[ wordLst[0] ]->positions;
+        for ( int j = 1; j < (int)wordLst.count(); ++j ) {
+            b = miniIndex[ wordLst[j] ]->positions;
+            aIt = a.begin();
+            while ( aIt != a.end() ) {
+                if ( b.contains( *aIt + 1 )) {
+                    (*aIt)++;
+                    ++aIt;
+                } else {
+                    aIt = a.erase( aIt );
+                }
+            }
+        }
+    }
+    if ( a.count() )
+        return true;
+    return false;
+}
+
+QVector<Document> Reader::setupDummyTerm(const QStringList &terms,
+                                              const EntryTable &entryTable)
+{
+    QList<Term> termList;
+    for (QStringList::ConstIterator it = terms.begin(); it != terms.end(); ++it) {
+        if (entryTable.value(*it)) {
+            Entry *e = entryTable.value(*it);
+            termList.append(Term(*it, e->documents.count(), e->documents ) );
+        }
+    }
+    QVector<Document> maxList(0);
+    if ( !termList.count() )
+        return maxList;
+    qSort(termList);
+
+    maxList = termList.takeLast().documents;
+    for(QList<Term>::Iterator it = termList.begin(); it != termList.end(); ++it) {
+        Term *t = &(*it);
+        QVector<Document> docs = t->documents;
+        for (QVector<Document>::iterator docIt = docs.begin(); docIt != docs.end(); ++docIt ) {
+            if ( maxList.indexOf( *docIt ) == -1 )
+                maxList.append( *docIt );
+        }
+    }
+    return maxList;
+}
+
+QStringList Reader::getWildcardTerms(const QString &term,
+                                          const EntryTable &entryTable)
+{
+    QStringList lst;
+    QStringList terms = split(term);
+    QStringList::Iterator iter;
+
+    for(EntryTable::ConstIterator it = entryTable.begin();
+        it != entryTable.end(); ++it) {
+        int index = 0;
+        bool found = false;
+        QString text( it.key() );
+        for ( iter = terms.begin(); iter != terms.end(); ++iter ) {
+            if ( *iter == QLatin1String("*") ) {
+                found = true;
+                continue;
+            }
+            if ( iter == terms.begin() && (*iter)[0] != text[0] ) {
+                found = false;
+                break;
+            }
+            index = text.indexOf( *iter, index );
+            if ( *iter == terms.last() && index != (int)text.length()-1 ) {
+                index = text.lastIndexOf( *iter );
+                if ( index != (int)text.length() - (int)(*iter).length() ) {
+                    found = false;
+                    break;
+                }
+            }
+            if ( index != -1 ) {
+                found = true;
+                index += (*iter).length();
+                continue;
+            } else {
+                found = false;
+                break;
+            }
+        }
+        if (found)
+            lst << text;
+    }
+
+    return lst;
+}
+
+void Reader::buildMiniIndex(const QString &string)
+{
+    if (miniIndex[string])
+        miniIndex[string]->positions.append(wordNum);
+    ++wordNum;
+}
+
+void Reader::reset()
+{
+    for(IndexTable::Iterator it = indexTable.begin();
+        it != indexTable.end(); ++it) {
+        cleanupIndex(it.value().first);
+        it.value().second.clear();
+    }
+}
+
+void Reader::cleanupIndex(EntryTable &entryTable)
+{
+    for(EntryTable::ConstIterator it =
+        entryTable.begin(); it != entryTable.end(); ++it) {
+            delete it.value();
+    }
+
+    entryTable.clear();
+}
+
+
+QHelpSearchIndexReaderDefault::QHelpSearchIndexReaderDefault()
+    : QHelpSearchIndexReader()
+{
+    // nothing todo
+}
+
+QHelpSearchIndexReaderDefault::~QHelpSearchIndexReaderDefault()
+{
+}
+
+void QHelpSearchIndexReaderDefault::run()
+{
+    mutex.lock();
+
+    if (m_cancel) {
+        mutex.unlock();
+        return;
+    }
+
+    const QList<QHelpSearchQuery> &queryList = this->m_query;
+    const QLatin1String key("DefaultSearchNamespaces");
+    const QString collectionFile(this->m_collectionFile);
+    const QString indexPath = m_indexFilesFolder;
+
+    mutex.unlock();
+
+    QString queryTerm;
+    foreach (const QHelpSearchQuery query, queryList) {
+        if (query.fieldName == QHelpSearchQuery::DEFAULT) {
+            queryTerm = query.wordList.at(0);
+            break;
+        }
+    }
+
+    if (queryTerm.isEmpty())
+        return;
+
+    QHelpEngineCore engine(collectionFile, 0);
+    if (!engine.setupData())
+        return;
+
+    const QStringList registeredDocs = engine.registeredDocumentations();
+    const QStringList indexedNamespaces = engine.customValue(key).toString().
+        split(QLatin1String("|"), QString::SkipEmptyParts);
+
+    emit searchingStarted();
+
+    // setup the reader
+    m_reader.setIndexPath(indexPath);
+    foreach(const QString namespaceName, registeredDocs) {
+        mutex.lock();
+        if (m_cancel) {
+            mutex.unlock();
+            searchingFinished(0);   // TODO: check this ???
+            return;
+        }
+        mutex.unlock();
+
+        const QList<QStringList> attributeSets =
+            engine.filterAttributeSets(namespaceName);
+
+        foreach (QStringList attributes, attributeSets) {
+            // read all index files
+            m_reader.setIndexFile(namespaceName, attributes.join(QLatin1String("@")));
+            if (!m_reader.readIndex()) {
+                qWarning("Full Text Search, could not read file for namespace: %s.",
+                    namespaceName.toUtf8().constData());
+            }
+        }
+    }
+
+    // get the current filter attributes and minimize the index files table
+    m_reader.filterFilesForAttributes(engine.filterAttributes(engine.currentFilter()));
+
+    hitList.clear();
+    QStringList terms, termSeq, seqWords;
+    if (m_reader.initCheck() && // check if we could read anything
+        m_reader.splitSearchTerm(queryTerm, &terms, &termSeq, &seqWords) ) {
+
+        // search for term(s)
+        m_reader.searchInIndex(terms);    // TODO: should this be interruptible as well ???
+
+        QVector<DocumentInfo> hits = m_reader.hits();
+        if (!hits.isEmpty()) {
+            if (termSeq.isEmpty()) {
+                foreach (const DocumentInfo docInfo, hits) {
+                    mutex.lock();
+                    if (m_cancel) {
+                        mutex.unlock();
+                        searchingFinished(0);   // TODO: check this, speed issue while locking???
+                        return;
+                    }
+                    mutex.unlock();
+                    hitList.append(qMakePair(docInfo.documentTitle, docInfo.documentUrl));
+                }
+            } else {
+                foreach (const DocumentInfo docInfo, hits) {
+                    mutex.lock();
+                    if (m_cancel) {
+                        mutex.unlock();
+                        searchingFinished(0);   // TODO: check this, speed issue while locking???
+                        return;
+                    }
+                    mutex.unlock();
+
+                    if (m_reader.searchForPattern(termSeq, seqWords, engine.fileData(docInfo.documentUrl))) // TODO: should this be interruptible as well ???
+                        hitList.append(qMakePair(docInfo.documentTitle, docInfo.documentUrl));
+                }
+            }
+        }
+    }
+
+    emit searchingFinished(hitList.count());
+}
+
+        }   // namespace std
+    }   // namespace fulltextsearch
+}   // namespace qt
+
+QT_END_NAMESPACE