--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/auto/selftests/tst_selftests.cpp Mon Jan 11 14:00:40 2010 +0000
@@ -0,0 +1,521 @@
+/****************************************************************************
+**
+** Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies).
+** All rights reserved.
+** Contact: Nokia Corporation (qt-info@nokia.com)
+**
+** This file is part of the test suite of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** No Commercial Usage
+** This file contains pre-release code and may not be distributed.
+** You may use this file in accordance with the terms and conditions
+** contained in the Technology Preview License Agreement accompanying
+** this package.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL included in the
+** packaging of this file. Please review the following information to
+** ensure the GNU Lesser General Public License version 2.1 requirements
+** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Nokia gives you certain additional
+** rights. These rights are described in the Nokia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+** If you have questions regarding the use of this file, please contact
+** Nokia at qt-info@nokia.com.
+**
+**
+**
+**
+**
+**
+**
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+
+#include <QtCore>
+#include <QtTest/QtTest>
+#include <QtXml/QXmlStreamReader>
+#include <private/cycle_p.h>
+
+class tst_Selftests: public QObject
+{
+ Q_OBJECT
+private slots:
+ void initTestCase();
+ void runSubTest_data();
+ void runSubTest();
+ void checkXML() const;
+ void checkXML_data();
+ void checkXunitxml() const;
+ void checkXunitxml_data();
+
+private:
+ QStringList m_checkXMLBlacklist;
+ QStringList m_checkXunitBlacklist;
+ void doRunSubTest(QString &subdir, QStringList &arguments );
+};
+
+struct BenchmarkResult
+{
+ qint64 total;
+ qint64 iterations;
+ QString unit;
+
+ inline QString toString() const
+ { return QString("total:%1, unit:%2, iterations:%3").arg(total).arg(unit).arg(iterations); }
+
+ static BenchmarkResult parse(QString const&, QString*);
+};
+
+QT_BEGIN_NAMESPACE
+namespace QTest
+{
+template <>
+inline bool qCompare
+ (BenchmarkResult const &r1, BenchmarkResult const &r2,
+ const char* actual, const char* expected, const char* file, int line)
+{
+ // First make sure the iterations and unit match.
+ if (r1.iterations != r2.iterations || r1.unit != r2.unit) {
+ /* Nope - compare whole string for best failure message */
+ return qCompare(r1.toString(), r2.toString(), actual, expected, file, line);
+ }
+
+ /*
+ Now check the value. Some variance is allowed, and how much depends on the
+ measured unit.
+ */
+ qreal variance = 0.;
+ if (r1.unit == "msec") {
+ variance = 0.1;
+ }
+ else if (r1.unit == "instr. loads") {
+ variance = 0.001;
+ }
+ else if (r1.unit == "ticks") {
+ variance = 0.001;
+ }
+ if (variance == 0.) {
+ /* No variance allowed - compare whole string */
+ return qCompare(r1.toString(), r2.toString(), actual, expected, file, line);
+ }
+
+ if (qAbs(qreal(r1.total) - qreal(r2.total)) <= qreal(r1.total)*variance) {
+ return compare_helper(true, "COMPARE()", file, line);
+ }
+
+ /* Whoops, didn't match. Compare the whole string for the most useful failure message. */
+ return qCompare(r1.toString(), r2.toString(), actual, expected, file, line);
+}
+}
+QT_END_NAMESPACE
+
+static QList<QByteArray> splitLines(QByteArray ba)
+{
+ ba.replace('\r', "");
+ return ba.split('\n');
+}
+
+static QList<QByteArray> expectedResult(const QString &subdir)
+{
+ QFile file(":/expected_" + subdir + ".txt");
+ if (!file.open(QIODevice::ReadOnly))
+ return QList<QByteArray>();
+ return splitLines(file.readAll());
+}
+
+void tst_Selftests::runSubTest_data()
+{
+ QTest::addColumn<QString>("subdir");
+ QTest::addColumn<QStringList>("arguments");
+
+ QTest::newRow("subtest") << "subtest" << QStringList();
+ QTest::newRow("warnings") << "warnings" << QStringList();
+ QTest::newRow("maxwarnings") << "maxwarnings" << QStringList();
+ QTest::newRow("cmptest") << "cmptest" << QStringList();
+// QTest::newRow("alive") << "alive" << QStringList(); // timer dependent
+ QTest::newRow("globaldata") << "globaldata" << QStringList();
+ QTest::newRow("skipglobal") << "skipglobal" << QStringList();
+ QTest::newRow("skip") << "skip" << QStringList();
+ QTest::newRow("strcmp") << "strcmp" << QStringList();
+ QTest::newRow("expectfail") << "expectfail" << QStringList();
+ QTest::newRow("sleep") << "sleep" << QStringList();
+ QTest::newRow("fetchbogus") << "fetchbogus" << QStringList();
+ QTest::newRow("crashes") << "crashes" << QStringList();
+ QTest::newRow("multiexec") << "multiexec" << QStringList();
+ QTest::newRow("failinit") << "failinit" << QStringList();
+ QTest::newRow("failinitdata") << "failinitdata" << QStringList();
+ QTest::newRow("skipinit") << "skipinit" << QStringList();
+ QTest::newRow("skipinitdata") << "skipinitdata" << QStringList();
+ QTest::newRow("datetime") << "datetime" << QStringList();
+ QTest::newRow("singleskip") << "singleskip" << QStringList();
+
+ //on windows assert does nothing in release mode and blocks execution with a popup window in debug mode
+#if !defined(Q_OS_WIN)
+ QTest::newRow("assert") << "assert" << QStringList();
+#endif
+
+ QTest::newRow("waitwithoutgui") << "waitwithoutgui" << QStringList();
+ QTest::newRow("differentexec") << "differentexec" << QStringList();
+#ifndef QT_NO_EXCEPTIONS
+ // The machine that run the intel autotests will popup a dialog
+ // with a warning that an uncaught exception was thrown.
+ // This will time out and falsely fail, therefore we disable the test for that platform.
+# if !defined(Q_CC_INTEL) || !defined(Q_OS_WIN)
+ QTest::newRow("exceptionthrow") << "exceptionthrow" << QStringList();
+# endif
+#endif
+ QTest::newRow("qexecstringlist") << "qexecstringlist" << QStringList();
+ QTest::newRow("datatable") << "datatable" << QStringList();
+ QTest::newRow("commandlinedata") << "commandlinedata" << QString("fiveTablePasses fiveTablePasses:fiveTablePasses_data1 -v2").split(' ');
+
+#if defined(__GNUC__) && defined(__i386) && defined(Q_OS_LINUX)
+ QTest::newRow("benchlibcallgrind") << "benchlibcallgrind" << QStringList("-callgrind");
+#endif
+ QTest::newRow("benchlibeventcounter") << "benchlibeventcounter" << QStringList("-eventcounter");
+ QTest::newRow("benchliboptions") << "benchliboptions" << QStringList("-eventcounter");
+ QTest::newRow("benchlibwalltime") << "benchlibwalltime" << QStringList();
+
+ //### This test is affected by the speed of the CPU and whether the tick counter is
+ //### monotonically increasing. It won't work on some machines so leave it off by default.
+ //### Feel free to uncomment for your own testing.
+#if 0
+ QTest::newRow("benchlibtickcounter") << "benchlibtickcounter" << QStringList("-tickcounter");
+#endif
+
+ QTest::newRow("xunit") << "xunit" << QStringList("-xunitxml");
+ QTest::newRow("longstring") << "longstring" << QStringList();
+
+}
+
+void tst_Selftests::doRunSubTest(QString &subdir, QStringList &arguments )
+{
+ QProcess proc;
+ proc.setEnvironment(QStringList(""));
+ proc.start(subdir + "/" + subdir, arguments);
+ QVERIFY2(proc.waitForFinished(), qPrintable(proc.errorString()));
+
+ const QByteArray out(proc.readAllStandardOutput());
+ const QByteArray err(proc.readAllStandardError());
+
+ /* Some platforms decides to output a message for uncaught exceptions. For instance,
+ * this is what windows platforms says:
+ * "This application has requested the Runtime to terminate it in an unusual way.
+ * Please contact the application's support team for more information." */
+ if(subdir != QLatin1String("exceptionthrow") && subdir != QLatin1String("fetchbogus")
+ && subdir != QLatin1String("xunit"))
+ QVERIFY2(err.isEmpty(), err.constData());
+
+ QList<QByteArray> res = splitLines(out);
+ QList<QByteArray> exp = expectedResult(subdir);
+
+ if (exp.count() == 0) {
+ QList<QList<QByteArray> > expArr;
+ int i = 1;
+ do {
+ exp = expectedResult(subdir + QString("_%1").arg(i++));
+ if (exp.count())
+ expArr += exp;
+ } while(exp.count());
+
+ for (int j = 0; j < expArr.count(); ++j) {
+ if (res.count() == expArr.at(j).count()) {
+ exp = expArr.at(j);
+ break;
+ }
+ }
+ } else {
+ QCOMPARE(res.count(), exp.count());
+ }
+
+ bool benchmark = false;
+ for (int i = 0; i < res.count(); ++i) {
+ QByteArray line = res.at(i);
+ if (line.startsWith("Config: Using QTest"))
+ continue;
+ // the __FILE__ __LINE__ output is compiler dependent, skip it
+ if (line.startsWith(" Loc: [") && line.endsWith(")]"))
+ continue;
+ if (line.endsWith(" : failure location"))
+ continue;
+
+ const QString output(QString::fromLatin1(line));
+ const QString expected(QString::fromLatin1(exp.at(i)));
+
+ if (line.contains("ASSERT") && output != expected)
+ QEXPECT_FAIL("assert", "QTestLib prints out the absolute path.", Continue);
+
+ /* On some platforms we compile without RTTI, and as a result we never throw an exception. */
+ if(expected.startsWith(QLatin1String("FAIL! : tst_Exception::throwException() Caught unhandled exce")) && expected != output)
+ QCOMPARE(output.simplified(), QString::fromLatin1("tst_Exception::throwException()").simplified());
+ else
+ {
+ if(output != expected && qstrcmp(QTest::currentDataTag(), "subtest") == 0)
+ {
+ /* The floating point formatting differs between platforms, so let's just skip it. */
+ continue;
+ }
+ else {
+ /*
+ Are we expecting this line to be a benchmark result?
+ If so, don't do a literal comparison, since results have some natural variance.
+ */
+ if (benchmark) {
+ QString error;
+
+ BenchmarkResult actualResult = BenchmarkResult::parse(output, &error);
+ QVERIFY2(error.isEmpty(), qPrintable(QString("Actual line didn't parse as benchmark result: %1\nLine: %2").arg(error).arg(output)));
+
+ BenchmarkResult expectedResult = BenchmarkResult::parse(expected, &error);
+ QVERIFY2(error.isEmpty(), qPrintable(QString("Expected line didn't parse as benchmark result: %1\nLine: %2").arg(error).arg(expected)));
+
+ QCOMPARE(actualResult, expectedResult);
+ }
+ else {
+ QCOMPARE(output, expected);
+ }
+ }
+ }
+
+ benchmark = line.startsWith("RESULT : ");
+ }
+}
+
+void tst_Selftests::runSubTest()
+{
+ QFETCH(QString, subdir);
+ QFETCH(QStringList, arguments);
+
+ doRunSubTest(subdir, arguments);
+}
+
+void tst_Selftests::initTestCase()
+{
+#if !defined(Q_OS_UNIX) || defined(Q_WS_MAC)
+ m_checkXMLBlacklist.append("crashes"); // This test crashes (XML valid on Unix only)
+#endif
+ m_checkXMLBlacklist.append("waitwithoutgui"); // This test is not a QTestLib test.
+
+ /* Output from several tests is broken with the XML output method,
+ * and it's quite heavy in the design. See task 155001. */
+ m_checkXMLBlacklist.append("multiexec");
+ m_checkXMLBlacklist.append("differentexec");
+ m_checkXMLBlacklist.append("qexecstringlist");
+ m_checkXMLBlacklist.append("benchliboptions");
+
+ /* These tests use printf and therefore corrupt the testlog */
+ m_checkXMLBlacklist.append("subtest");
+ m_checkXMLBlacklist.append("globaldata");
+ m_checkXMLBlacklist.append("warnings");
+
+ m_checkXunitBlacklist = m_checkXMLBlacklist;
+}
+
+void tst_Selftests::checkXML() const
+{
+ QFETCH(QString, subdir);
+ QFETCH(QStringList, arguments);
+
+ if(m_checkXMLBlacklist.contains(subdir))
+ return;
+
+ QStringList args;
+ /* Test both old (-flush) and new XML logger implementation */
+ for (int i = 0; i < 2; ++i) {
+ bool flush = i;
+ args = arguments;
+ args.prepend("-xml");
+ if (flush) args.prepend("-flush");
+
+ QProcess proc;
+ proc.setEnvironment(QStringList(""));
+ proc.start(subdir + "/" + subdir, args);
+ QVERIFY(proc.waitForFinished());
+
+ QByteArray out(proc.readAllStandardOutput());
+ QByteArray err(proc.readAllStandardError());
+
+ /* Some platforms decides to output a message for uncaught exceptions. For instance,
+ * this is what windows platforms says:
+ * "This application has requested the Runtime to terminate it in an unusual way.
+ * Please contact the application's support team for more information." */
+ if(subdir != QLatin1String("exceptionthrow") && subdir != QLatin1String("fetchbogus"))
+ QVERIFY2(err.isEmpty(), err.constData());
+
+ QXmlStreamReader reader(out);
+
+ while(!reader.atEnd())
+ reader.readNext();
+
+ QVERIFY2(!reader.error(), qPrintable(QString("(flush %0) line %1, col %2: %3")
+ .arg(flush)
+ .arg(reader.lineNumber())
+ .arg(reader.columnNumber())
+ .arg(reader.errorString())
+ ));
+ }
+}
+
+void tst_Selftests::checkXunitxml() const
+{
+ QFETCH(QString, subdir);
+ QFETCH(QStringList, arguments);
+
+ if(m_checkXunitBlacklist.contains(subdir))
+ return;
+
+ arguments.prepend("-xunitxml");
+ arguments.prepend("-flush");
+
+ QProcess proc;
+ proc.setEnvironment(QStringList(""));
+ proc.start(subdir + "/" + subdir, arguments);
+ QVERIFY(proc.waitForFinished());
+
+ QByteArray out(proc.readAllStandardOutput());
+ QByteArray err(proc.readAllStandardError());
+
+// qDebug()<<out;
+
+ /* Some platforms decides to output a message for uncaught exceptions. For instance,
+ * this is what windows platforms says:
+ * "This application has requested the Runtime to terminate it in an unusual way.
+ * Please contact the application's support team for more information." */
+ if(subdir != QLatin1String("exceptionthrow") && subdir != QLatin1String("fetchbogus"))
+ QVERIFY2(err.isEmpty(), err.constData());
+
+ QXmlStreamReader reader(out);
+
+ while(!reader.atEnd())
+ reader.readNext();
+
+ QVERIFY2(!reader.error(), qPrintable(QString("line %1, col %2: %3")
+ .arg(reader.lineNumber())
+ .arg(reader.columnNumber())
+ .arg(reader.errorString())
+ ));
+}
+
+void tst_Selftests::checkXunitxml_data()
+{
+ checkXML_data();
+}
+
+void tst_Selftests::checkXML_data()
+{
+ runSubTest_data();
+ QTest::newRow("badxml 1") << "badxml" << QStringList();
+ QTest::newRow("badxml 2") << "badxml" << (QStringList() << "-badstring" << "0");
+ QTest::newRow("badxml 3") << "badxml" << (QStringList() << "-badstring" << "1");
+ QTest::newRow("badxml 4") << "badxml" << (QStringList() << "-badstring" << "2");
+ QTest::newRow("badxml 5") << "badxml" << (QStringList() << "-badstring" << "3");
+}
+
+/* Parse line into the BenchmarkResult it represents. */
+BenchmarkResult BenchmarkResult::parse(QString const& line, QString* error)
+{
+ if (error) *error = QString();
+ BenchmarkResult out;
+
+ QString remaining = line.trimmed();
+
+ if (remaining.isEmpty()) {
+ if (error) *error = "Line is empty";
+ return out;
+ }
+
+ /* This code avoids using a QRegExp because QRegExp might be broken. */
+
+ /* Sample format: 4,000 msec per iteration (total: 4000, iterations: 1) */
+
+ QString sFirstNumber;
+ while (!remaining.isEmpty() && !remaining.at(0).isSpace()) {
+ sFirstNumber += remaining.at(0);
+ remaining.remove(0,1);
+ }
+ remaining = remaining.trimmed();
+
+ /* 4,000 -> 4000 */
+ sFirstNumber.remove(',');
+
+ /* Should now be parseable as floating point */
+ bool ok;
+ double firstNumber = sFirstNumber.toDouble(&ok);
+ if (!ok) {
+ if (error) *error = sFirstNumber + " (at beginning of line) is not a valid number";
+ return out;
+ }
+
+ /* Remaining: msec per iteration (total: 4000, iterations: 1) */
+ static const char periterbit[] = " per iteration (total: ";
+ QString unit;
+ while (!remaining.startsWith(periterbit) && !remaining.isEmpty()) {
+ unit += remaining.at(0);
+ remaining.remove(0,1);
+ }
+ if (remaining.isEmpty()) {
+ if (error) *error = "Could not find pattern: '<unit> per iteration (total: '";
+ return out;
+ }
+
+ remaining = remaining.mid(sizeof(periterbit)-1);
+
+ /* Remaining: 4000, iterations: 1) */
+ static const char itersbit[] = ", iterations: ";
+ QString sTotal;
+ while (!remaining.startsWith(itersbit) && !remaining.isEmpty()) {
+ sTotal += remaining.at(0);
+ remaining.remove(0,1);
+ }
+ if (remaining.isEmpty()) {
+ if (error) *error = "Could not find pattern: '<number>, iterations: '";
+ return out;
+ }
+
+ remaining = remaining.mid(sizeof(itersbit)-1);
+
+ qint64 total = sTotal.toLongLong(&ok);
+ if (!ok) {
+ if (error) *error = sTotal + " (total) is not a valid integer";
+ return out;
+ }
+
+ /* Remaining: 1) */
+ QString sIters;
+ while (remaining != QLatin1String(")") && !remaining.isEmpty()) {
+ sIters += remaining.at(0);
+ remaining.remove(0,1);
+ }
+ if (remaining.isEmpty()) {
+ if (error) *error = "Could not find pattern: '<num>)'";
+ return out;
+ }
+ qint64 iters = sIters.toLongLong(&ok);
+ if (!ok) {
+ if (error) *error = sIters + " (iterations) is not a valid integer";
+ return out;
+ }
+
+ double calcFirstNumber = double(total)/double(iters);
+ if (!qFuzzyCompare(firstNumber, calcFirstNumber)) {
+ if (error) *error = QString("total/iters is %1, but benchlib output result as %2").arg(calcFirstNumber).arg(firstNumber);
+ return out;
+ }
+
+ out.total = total;
+ out.unit = unit;
+ out.iterations = iters;
+ return out;
+}
+
+QTEST_MAIN(tst_Selftests)
+
+#include "tst_selftests.moc"