--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/tools/linguist/shared/simtexth.cpp Mon Jan 11 14:00:40 2010 +0000
@@ -0,0 +1,277 @@
+/****************************************************************************
+**
+** Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies).
+** All rights reserved.
+** Contact: Nokia Corporation (qt-info@nokia.com)
+**
+** This file is part of the Qt Linguist of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** No Commercial Usage
+** This file contains pre-release code and may not be distributed.
+** You may use this file in accordance with the terms and conditions
+** contained in the Technology Preview License Agreement accompanying
+** this package.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL included in the
+** packaging of this file. Please review the following information to
+** ensure the GNU Lesser General Public License version 2.1 requirements
+** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Nokia gives you certain additional
+** rights. These rights are described in the Nokia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+** If you have questions regarding the use of this file, please contact
+** Nokia at qt-info@nokia.com.
+**
+**
+**
+**
+**
+**
+**
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#include "simtexth.h"
+#include "translator.h"
+
+#include <QtCore/QByteArray>
+#include <QtCore/QString>
+#include <QtCore/QList>
+
+
+QT_BEGIN_NAMESPACE
+
+typedef QList<TranslatorMessage> TML;
+
+/*
+ How similar are two texts? The approach used here relies on co-occurrence
+ matrices and is very efficient.
+
+ Let's see with an example: how similar are "here" and "hither"? The
+ co-occurrence matrix M for "here" is M[h,e] = 1, M[e,r] = 1, M[r,e] = 1, and 0
+ elsewhere; the matrix N for "hither" is N[h,i] = 1, N[i,t] = 1, ...,
+ N[h,e] = 1, N[e,r] = 1, and 0 elsewhere. The union U of both matrices is the
+ matrix U[i,j] = max { M[i,j], N[i,j] }, and the intersection V is
+ V[i,j] = min { M[i,j], N[i,j] }. The score for a pair of texts is
+
+ score = (sum of V[i,j] over all i, j) / (sum of U[i,j] over all i, j),
+
+ a formula suggested by Arnt Gulbrandsen. Here we have
+
+ score = 2 / 6,
+
+ or one third.
+
+ The implementation differs from this in a few details. Most importantly,
+ repetitions are ignored; for input "xxx", M[x,x] equals 1, not 2.
+*/
+
+/*
+ Every character is assigned to one of 20 buckets so that the co-occurrence
+ matrix requires only 20 * 20 = 400 bits, not 256 * 256 = 65536 bits or even
+ more if we want the whole Unicode. Which character falls in which bucket is
+ arbitrary.
+
+ The second half of the table is a replica of the first half, because of
+ laziness.
+*/
+static const int indexOf[256] = {
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+// ! " # $ % & ' ( ) * + , - . /
+ 0, 2, 6, 7, 10, 12, 15, 19, 2, 6, 7, 10, 12, 15, 19, 0,
+// 0 1 2 3 4 5 6 7 8 9 : ; < = > ?
+ 1, 3, 4, 5, 8, 9, 11, 13, 14, 16, 2, 6, 7, 10, 12, 15,
+// @ A B C D E F G H I J K L M N O
+ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 6, 10, 11, 12, 13, 14,
+// P Q R S T U V W X Y Z [ \ ] ^ _
+ 15, 12, 16, 17, 18, 19, 2, 10, 15, 7, 19, 2, 6, 7, 10, 0,
+// ` a b c d e f g h i j k l m n o
+ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 6, 10, 11, 12, 13, 14,
+// p q r s t u v w x y z { | } ~
+ 15, 12, 16, 17, 18, 19, 2, 10, 15, 7, 19, 2, 6, 7, 10, 0,
+
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 2, 6, 7, 10, 12, 15, 19, 2, 6, 7, 10, 12, 15, 19, 0,
+ 1, 3, 4, 5, 8, 9, 11, 13, 14, 16, 2, 6, 7, 10, 12, 15,
+ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 6, 10, 11, 12, 13, 14,
+ 15, 12, 16, 17, 18, 19, 2, 10, 15, 7, 19, 2, 6, 7, 10, 0,
+ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 6, 10, 11, 12, 13, 14,
+ 15, 12, 16, 17, 18, 19, 2, 10, 15, 7, 19, 2, 6, 7, 10, 0
+};
+
+/*
+ The entry bitCount[i] (for i between 0 and 255) is the number of bits used to
+ represent i in binary.
+*/
+static const int bitCount[256] = {
+ 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4,
+ 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
+ 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
+ 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+ 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
+ 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+ 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+ 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
+ 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
+ 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+ 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+ 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
+ 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+ 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
+ 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
+ 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8
+};
+
+struct CoMatrix
+{
+ /*
+ The matrix has 20 * 20 = 400 entries. This requires 50 bytes, or 13
+ words. Some operations are performed on words for more efficiency.
+ */
+ union {
+ quint8 b[52];
+ quint32 w[13];
+ };
+
+ CoMatrix() { memset( b, 0, 52 ); }
+
+ CoMatrix(const QString &str)
+ {
+ QByteArray ba = str.toUtf8();
+ const char *text = ba.constData();
+ char c = '\0', d;
+ memset( b, 0, 52 );
+ /*
+ The Knuth books are not in the office only for show; they help make
+ loops 30% faster and 20% as readable.
+ */
+ while ( (d = *text) != '\0' ) {
+ setCoOccurence( c, d );
+ if ( (c = *++text) != '\0' ) {
+ setCoOccurence( d, c );
+ text++;
+ }
+ }
+ }
+
+ void setCoOccurence( char c, char d ) {
+ int k = indexOf[(uchar) c] + 20 * indexOf[(uchar) d];
+ b[k >> 3] |= (1 << (k & 0x7));
+ }
+
+ int worth() const {
+ int w = 0;
+ for ( int i = 0; i < 50; i++ )
+ w += bitCount[b[i]];
+ return w;
+ }
+};
+
+static inline CoMatrix reunion(const CoMatrix &m, const CoMatrix &n)
+{
+ CoMatrix p;
+ for (int i = 0; i < 13; ++i)
+ p.w[i] = m.w[i] | n.w[i];
+ return p;
+}
+
+static inline CoMatrix intersection(const CoMatrix &m, const CoMatrix &n)
+{
+ CoMatrix p;
+ for (int i = 0; i < 13; ++i)
+ p.w[i] = m.w[i] & n.w[i];
+ return p;
+}
+
+StringSimilarityMatcher::StringSimilarityMatcher(const QString &stringToMatch)
+{
+ m_cm = new CoMatrix(stringToMatch);
+ m_length = stringToMatch.length();
+}
+
+int StringSimilarityMatcher::getSimilarityScore(const QString &strCandidate)
+{
+ CoMatrix cmTarget(strCandidate);
+ int delta = qAbs(m_length - strCandidate.size());
+ int score = ( (intersection(*m_cm, cmTarget).worth() + 1) << 10 ) /
+ ( reunion(*m_cm, cmTarget).worth() + (delta << 1) + 1 );
+ return score;
+}
+
+StringSimilarityMatcher::~StringSimilarityMatcher()
+{
+ delete m_cm;
+}
+
+/**
+ * Checks how similar two strings are.
+ * The return value is the score, and a higher score is more similar
+ * than one with a low score.
+ * Linguist considers a score over 190 to be a good match.
+ * \sa StringSimilarityMatcher
+ */
+int getSimilarityScore(const QString &str1, const QString &str2)
+{
+ CoMatrix cmTarget(str2);
+ CoMatrix cm(str1);
+ int delta = qAbs(str1.size() - str2.size());
+
+ int score = ( (intersection(cm, cmTarget).worth() + 1) << 10 )
+ / ( reunion(cm, cmTarget).worth() + (delta << 1) + 1 );
+
+ return score;
+}
+
+CandidateList similarTextHeuristicCandidates(const Translator *tor,
+ const QString &text, int maxCandidates)
+{
+ QList<int> scores;
+ CandidateList candidates;
+
+ TML all = tor->translatedMessages();
+
+ foreach (const TranslatorMessage &mtm, all) {
+ if (mtm.type() == TranslatorMessage::Unfinished
+ || mtm.translation().isEmpty())
+ continue;
+
+ QString s = mtm.sourceText();
+ int score = getSimilarityScore(s, text);
+
+ if (candidates.size() == maxCandidates && score > scores[maxCandidates - 1] )
+ candidates.removeLast();
+
+ if (candidates.size() < maxCandidates && score >= textSimilarityThreshold) {
+ Candidate cand( s, mtm.translation() );
+
+ int i;
+ for (i = 0; i < candidates.size(); i++) {
+ if (score >= scores.at(i)) {
+ if (score == scores.at(i)) {
+ if (candidates.at(i) == cand)
+ goto continue_outer_loop;
+ } else {
+ break;
+ }
+ }
+ }
+ scores.insert(i, score);
+ candidates.insert(i, cand);
+ }
+ continue_outer_loop:
+ ;
+ }
+ return candidates;
+}
+
+QT_END_NAMESPACE