1 /****************************************************************************

     2 **

     3 ** Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies).

     4 ** All rights reserved.

     5 ** Contact: Nokia Corporation (qt-info@nokia.com)

     6 **

     7 ** This file is part of the test suite of the Qt Toolkit.

     8 **

     9 ** $QT_BEGIN_LICENSE:LGPL$

    10 ** No Commercial Usage

    11 ** This file contains pre-release code and may not be distributed.

    12 ** You may use this file in accordance with the terms and conditions

    13 ** contained in the Technology Preview License Agreement accompanying

    14 ** this package.

    15 **

    16 ** GNU Lesser General Public License Usage

    17 ** Alternatively, this file may be used under the terms of the GNU Lesser

    18 ** General Public License version 2.1 as published by the Free Software

    19 ** Foundation and appearing in the file LICENSE.LGPL included in the

    20 ** packaging of this file.  Please review the following information to

    21 ** ensure the GNU Lesser General Public License version 2.1 requirements

    22 ** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.

    23 **

    24 ** In addition, as a special exception, Nokia gives you certain additional

    25 ** rights.  These rights are described in the Nokia Qt LGPL Exception

    26 ** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.

    27 **

    28 ** If you have questions regarding the use of this file, please contact

    29 ** Nokia at qt-info@nokia.com.

    30 **

    31 **

    32 **

    33 **

    34 **

    35 **

    36 **

    37 **

    38 ** $QT_END_LICENSE$

    39 **

    40 ****************************************************************************/

    41 

    42 

    43 #include <QtCore>

    44 #include <QtTest/QtTest>

    45 #include <QtXml/QXmlStreamReader>

    46 #include <private/cycle_p.h>

    47 

    48 class tst_Selftests: public QObject

    49 {

    50     Q_OBJECT

    51 private slots:

    52     void initTestCase();

    53     void runSubTest_data();

    54     void runSubTest();

    55     void checkXML() const;

    56     void checkXML_data();

    57     void checkXunitxml() const;

    58     void checkXunitxml_data();

    59 

    60 private:

    61     QStringList m_checkXMLBlacklist;

    62     QStringList m_checkXunitBlacklist;

    63     void doRunSubTest(QString &subdir, QStringList &arguments );

    64 };

    65 

    66 struct BenchmarkResult

    67 {

    68     qint64  total;

    69     qint64  iterations;

    70     QString unit;

    71 

    72     inline QString toString() const

    73     { return QString("total:%1, unit:%2, iterations:%3").arg(total).arg(unit).arg(iterations); }

    74 

    75     static BenchmarkResult parse(QString const&, QString*);

    76 };

    77 

    78 QT_BEGIN_NAMESPACE

    79 namespace QTest

    80 {

    81 template <>

    82 inline bool qCompare

    83     (BenchmarkResult const &r1, BenchmarkResult const &r2,

    84      const char* actual, const char* expected, const char* file, int line)

    85 {

    86     // First make sure the iterations and unit match.

    87     if (r1.iterations != r2.iterations || r1.unit != r2.unit) {

    88         /* Nope - compare whole string for best failure message */

    89         return qCompare(r1.toString(), r2.toString(), actual, expected, file, line);

    90     }

    91 

    92     /*

    93         Now check the value.  Some variance is allowed, and how much depends on the

    94         measured unit.

    95     */

    96     qreal variance = 0.;

    97     if (r1.unit == "msec") {

    98         variance = 0.1;

    99     }

   100     else if (r1.unit == "instr. loads") {

   101         variance = 0.001;

   102     }

   103     else if (r1.unit == "ticks") {

   104         variance = 0.001;

   105     }

   106     if (variance == 0.) {

   107         /* No variance allowed - compare whole string */

   108         return qCompare(r1.toString(), r2.toString(), actual, expected, file, line);

   109     }

   110 

   111     if (qAbs(qreal(r1.total) - qreal(r2.total)) <= qreal(r1.total)*variance) {

   112         return compare_helper(true, "COMPARE()", file, line);

   113     }

   114 

   115     /* Whoops, didn't match.  Compare the whole string for the most useful failure message. */

   116     return qCompare(r1.toString(), r2.toString(), actual, expected, file, line);

   117 }

   118 }

   119 QT_END_NAMESPACE

   120 

   121 static QList<QByteArray> splitLines(QByteArray ba)

   122 {

   123     ba.replace('\r', "");

   124     return ba.split('\n');

   125 }

   126 

   127 static QList<QByteArray> expectedResult(const QString &subdir)

   128 {

   129     QFile file(":/expected_" + subdir + ".txt");

   130     if (!file.open(QIODevice::ReadOnly))

   131         return QList<QByteArray>();

   132     return splitLines(file.readAll());

   133 }

   134 

   135 void tst_Selftests::runSubTest_data()

   136 {

   137     QTest::addColumn<QString>("subdir");

   138     QTest::addColumn<QStringList>("arguments");

   139 

   140     QTest::newRow("subtest") << "subtest" << QStringList();

   141     QTest::newRow("warnings") << "warnings" << QStringList();

   142     QTest::newRow("maxwarnings") << "maxwarnings" << QStringList();

   143     QTest::newRow("cmptest") << "cmptest" << QStringList();

   144 //    QTest::newRow("alive") << "alive" << QStringList(); // timer dependent

   145     QTest::newRow("globaldata") << "globaldata" << QStringList();

   146     QTest::newRow("skipglobal") << "skipglobal" << QStringList();

   147     QTest::newRow("skip") << "skip" << QStringList();

   148     QTest::newRow("strcmp") << "strcmp" << QStringList();

   149     QTest::newRow("expectfail") << "expectfail" << QStringList();

   150     QTest::newRow("sleep") << "sleep" << QStringList();

   151     QTest::newRow("fetchbogus") << "fetchbogus" << QStringList();

   152     QTest::newRow("crashes") << "crashes" << QStringList();

   153     QTest::newRow("multiexec") << "multiexec" << QStringList();

   154     QTest::newRow("failinit") << "failinit" << QStringList();

   155     QTest::newRow("failinitdata") << "failinitdata" << QStringList();

   156     QTest::newRow("skipinit") << "skipinit" << QStringList();

   157     QTest::newRow("skipinitdata") << "skipinitdata" << QStringList();

   158     QTest::newRow("datetime") << "datetime" << QStringList();

   159     QTest::newRow("singleskip") << "singleskip" << QStringList();

   160 

   161     //on windows assert does nothing in release mode and blocks execution with a popup window in debug mode

   162 #if !defined(Q_OS_WIN)

   163     QTest::newRow("assert") << "assert" << QStringList();

   164 #endif

   165 

   166     QTest::newRow("waitwithoutgui") << "waitwithoutgui" << QStringList();

   167     QTest::newRow("differentexec") << "differentexec" << QStringList();

   168 #ifndef QT_NO_EXCEPTIONS

   169     // The machine that run the intel autotests will popup a dialog

   170     // with a warning that an uncaught exception was thrown.

   171     // This will time out and falsely fail, therefore we disable the test for that platform.

   172 # if !defined(Q_CC_INTEL) || !defined(Q_OS_WIN)

   173     QTest::newRow("exceptionthrow") << "exceptionthrow" << QStringList();

   174 # endif

   175 #endif

   176     QTest::newRow("qexecstringlist") << "qexecstringlist" << QStringList();

   177     QTest::newRow("datatable") << "datatable" << QStringList();

   178     QTest::newRow("commandlinedata") << "commandlinedata" << QString("fiveTablePasses fiveTablePasses:fiveTablePasses_data1 -v2").split(' ');

   179 

   180 #if defined(__GNUC__) && defined(__i386) && defined(Q_OS_LINUX)

   181     QTest::newRow("benchlibcallgrind") << "benchlibcallgrind" << QStringList("-callgrind");

   182 #endif

   183     QTest::newRow("benchlibeventcounter") << "benchlibeventcounter" << QStringList("-eventcounter");

   184     QTest::newRow("benchliboptions") << "benchliboptions" << QStringList("-eventcounter");

   185     QTest::newRow("benchlibwalltime") << "benchlibwalltime" << QStringList();

   186 

   187     //### This test is affected by the speed of the CPU and whether the tick counter is

   188     //### monotonically increasing. It won't work on some machines so leave it off by default.

   189     //### Feel free to uncomment for your own testing.

   190 #if 0

   191     QTest::newRow("benchlibtickcounter") << "benchlibtickcounter" << QStringList("-tickcounter");

   192 #endif

   193 

   194     QTest::newRow("xunit") << "xunit" << QStringList("-xunitxml");

   195     QTest::newRow("longstring") << "longstring" << QStringList();

   196 

   197 }

   198 

   199 void tst_Selftests::doRunSubTest(QString &subdir, QStringList &arguments )

   200 {

   201     QProcess proc;

   202     proc.setEnvironment(QStringList(""));

   203     proc.start(subdir + "/" + subdir, arguments);

   204     QVERIFY2(proc.waitForFinished(), qPrintable(proc.errorString()));

   205 

   206     const QByteArray out(proc.readAllStandardOutput());

   207     const QByteArray err(proc.readAllStandardError());

   208 

   209     /* Some platforms decides to output a message for uncaught exceptions. For instance,

   210      * this is what windows platforms says:

   211      * "This application has requested the Runtime to terminate it in an unusual way.

   212      * Please contact the application's support team for more information." */

   213     if(subdir != QLatin1String("exceptionthrow") && subdir != QLatin1String("fetchbogus")

   214         && subdir != QLatin1String("xunit"))

   215         QVERIFY2(err.isEmpty(), err.constData());

   216 

   217     QList<QByteArray> res = splitLines(out);

   218     QList<QByteArray> exp = expectedResult(subdir);

   219 

   220     if (exp.count() == 0) {

   221         QList<QList<QByteArray> > expArr;

   222         int i = 1;

   223         do {

   224             exp = expectedResult(subdir + QString("_%1").arg(i++));

   225             if (exp.count())

   226             expArr += exp;

   227         } while(exp.count());

   228 

   229         for (int j = 0; j < expArr.count(); ++j) {

   230             if (res.count() == expArr.at(j).count()) {

   231                 exp = expArr.at(j);

   232                 break;

   233             }

   234         }

   235     } else {

   236         QCOMPARE(res.count(), exp.count());

   237     }

   238 

   239     bool benchmark = false;

   240     for (int i = 0; i < res.count(); ++i) {

   241         QByteArray line = res.at(i);

   242         if (line.startsWith("Config: Using QTest"))

   243             continue;

   244         // the __FILE__ __LINE__ output is compiler dependent, skip it

   245         if (line.startsWith("   Loc: [") && line.endsWith(")]"))

   246             continue;

   247         if (line.endsWith(" : failure location"))

   248             continue;

   249 

   250         const QString output(QString::fromLatin1(line));

   251         const QString expected(QString::fromLatin1(exp.at(i)));

   252 

   253         if (line.contains("ASSERT") && output != expected)

   254             QEXPECT_FAIL("assert", "QTestLib prints out the absolute path.", Continue);

   255 

   256         /* On some platforms we compile without RTTI, and as a result we never throw an exception. */

   257         if(expected.startsWith(QLatin1String("FAIL!  : tst_Exception::throwException() Caught unhandled exce")) && expected != output)

   258             QCOMPARE(output.simplified(), QString::fromLatin1("tst_Exception::throwException()").simplified());

   259         else

   260         {

   261             if(output != expected && qstrcmp(QTest::currentDataTag(), "subtest") == 0)

   262             {

   263             /* The floating point formatting differs between platforms, so let's just skip it. */

   264             continue;

   265             }

   266             else {

   267                 /*

   268                    Are we expecting this line to be a benchmark result?

   269                    If so, don't do a literal comparison, since results have some natural variance.

   270                 */

   271                 if (benchmark) {

   272                     QString error;

   273 

   274                     BenchmarkResult actualResult = BenchmarkResult::parse(output, &error);

   275                     QVERIFY2(error.isEmpty(), qPrintable(QString("Actual line didn't parse as benchmark result: %1\nLine: %2").arg(error).arg(output)));

   276 

   277                     BenchmarkResult expectedResult = BenchmarkResult::parse(expected, &error);

   278                     QVERIFY2(error.isEmpty(), qPrintable(QString("Expected line didn't parse as benchmark result: %1\nLine: %2").arg(error).arg(expected)));

   279 

   280                     QCOMPARE(actualResult, expectedResult);

   281                 }

   282                 else {

   283                     QCOMPARE(output, expected);

   284                 }

   285             }

   286         }

   287 

   288         benchmark = line.startsWith("RESULT : ");

   289     }

   290 }

   291 

   292 void tst_Selftests::runSubTest()

   293 {

   294     QFETCH(QString, subdir);

   295     QFETCH(QStringList, arguments);

   296 

   297     doRunSubTest(subdir, arguments);

   298 }

   299 

   300 void tst_Selftests::initTestCase()

   301 {

   302 #if !defined(Q_OS_UNIX) || defined(Q_WS_MAC)

   303     m_checkXMLBlacklist.append("crashes"); // This test crashes (XML valid on Unix only)

   304 #endif

   305     m_checkXMLBlacklist.append("waitwithoutgui"); // This test is not a QTestLib test.

   306 

   307     /* Output from several tests is broken with the XML output method,

   308      * and it's quite heavy in the design. See task 155001. */

   309     m_checkXMLBlacklist.append("multiexec");

   310     m_checkXMLBlacklist.append("differentexec");

   311     m_checkXMLBlacklist.append("qexecstringlist");

   312     m_checkXMLBlacklist.append("benchliboptions");

   313 

   314     /* These tests use printf and therefore corrupt the testlog */

   315     m_checkXMLBlacklist.append("subtest");

   316     m_checkXMLBlacklist.append("globaldata");

   317     m_checkXMLBlacklist.append("warnings");

   318 

   319     m_checkXunitBlacklist = m_checkXMLBlacklist;

   320 }

   321 

   322 void tst_Selftests::checkXML() const

   323 {

   324     QFETCH(QString, subdir);

   325     QFETCH(QStringList, arguments);

   326 

   327     if(m_checkXMLBlacklist.contains(subdir))

   328         return;

   329 

   330     QStringList args;

   331     /* Test both old (-flush) and new XML logger implementation */

   332     for (int i = 0; i < 2; ++i) {

   333         bool flush = i;

   334         args = arguments;

   335         args.prepend("-xml");

   336         if (flush) args.prepend("-flush");

   337 

   338         QProcess proc;

   339         proc.setEnvironment(QStringList(""));

   340         proc.start(subdir + "/" + subdir, args);

   341         QVERIFY(proc.waitForFinished());

   342 

   343         QByteArray out(proc.readAllStandardOutput());

   344         QByteArray err(proc.readAllStandardError());

   345 

   346         /* Some platforms decides to output a message for uncaught exceptions. For instance,

   347          * this is what windows platforms says:

   348          * "This application has requested the Runtime to terminate it in an unusual way.

   349          * Please contact the application's support team for more information." */

   350         if(subdir != QLatin1String("exceptionthrow") && subdir != QLatin1String("fetchbogus"))

   351             QVERIFY2(err.isEmpty(), err.constData());

   352 

   353         QXmlStreamReader reader(out);

   354 

   355         while(!reader.atEnd())

   356             reader.readNext();

   357 

   358         QVERIFY2(!reader.error(), qPrintable(QString("(flush %0) line %1, col %2: %3")

   359             .arg(flush)

   360             .arg(reader.lineNumber())

   361             .arg(reader.columnNumber())

   362             .arg(reader.errorString())

   363         ));

   364     }

   365 }

   366 

   367 void tst_Selftests::checkXunitxml() const

   368 {

   369     QFETCH(QString, subdir);

   370     QFETCH(QStringList, arguments);

   371 

   372     if(m_checkXunitBlacklist.contains(subdir))

   373         return;

   374 

   375     arguments.prepend("-xunitxml");

   376     arguments.prepend("-flush");

   377 

   378     QProcess proc;

   379     proc.setEnvironment(QStringList(""));

   380     proc.start(subdir + "/" + subdir, arguments);

   381     QVERIFY(proc.waitForFinished());

   382 

   383     QByteArray out(proc.readAllStandardOutput());

   384     QByteArray err(proc.readAllStandardError());

   385 

   386 //    qDebug()<<out;

   387 

   388     /* Some platforms decides to output a message for uncaught exceptions. For instance,

   389      * this is what windows platforms says:

   390      * "This application has requested the Runtime to terminate it in an unusual way.

   391      * Please contact the application's support team for more information." */

   392     if(subdir != QLatin1String("exceptionthrow") && subdir != QLatin1String("fetchbogus"))

   393         QVERIFY2(err.isEmpty(), err.constData());

   394 

   395     QXmlStreamReader reader(out);

   396 

   397     while(!reader.atEnd())

   398         reader.readNext();

   399 

   400     QVERIFY2(!reader.error(), qPrintable(QString("line %1, col %2: %3")

   401         .arg(reader.lineNumber())

   402         .arg(reader.columnNumber())

   403         .arg(reader.errorString())

   404     ));

   405 }

   406 

   407 void tst_Selftests::checkXunitxml_data()

   408 {

   409     checkXML_data();

   410 }

   411 

   412 void tst_Selftests::checkXML_data()

   413 {

   414     runSubTest_data();

   415     QTest::newRow("badxml 1") << "badxml" << QStringList();

   416     QTest::newRow("badxml 2") << "badxml" << (QStringList() << "-badstring" << "0");

   417     QTest::newRow("badxml 3") << "badxml" << (QStringList() << "-badstring" << "1");

   418     QTest::newRow("badxml 4") << "badxml" << (QStringList() << "-badstring" << "2");

   419     QTest::newRow("badxml 5") << "badxml" << (QStringList() << "-badstring" << "3");

   420 }

   421 

   422 /* Parse line into the BenchmarkResult it represents. */

   423 BenchmarkResult BenchmarkResult::parse(QString const& line, QString* error)

   424 {

   425     if (error) *error = QString();

   426     BenchmarkResult out;

   427 

   428     QString remaining = line.trimmed();

   429 

   430     if (remaining.isEmpty()) {

   431         if (error) *error = "Line is empty";

   432         return out;

   433     }

   434 

   435     /* This code avoids using a QRegExp because QRegExp might be broken. */

   436 

   437     /* Sample format: 4,000 msec per iteration (total: 4000, iterations: 1) */

   438 

   439     QString sFirstNumber;

   440     while (!remaining.isEmpty() && !remaining.at(0).isSpace()) {

   441         sFirstNumber += remaining.at(0);

   442         remaining.remove(0,1);

   443     }

   444     remaining = remaining.trimmed();

   445 

   446     /* 4,000 -> 4000 */

   447     sFirstNumber.remove(',');

   448 

   449     /* Should now be parseable as floating point */

   450     bool ok;

   451     double firstNumber = sFirstNumber.toDouble(&ok);

   452     if (!ok) {

   453         if (error) *error = sFirstNumber + " (at beginning of line) is not a valid number";

   454         return out;

   455     }

   456 

   457     /* Remaining: msec per iteration (total: 4000, iterations: 1) */

   458     static const char periterbit[] = " per iteration (total: ";

   459     QString unit;

   460     while (!remaining.startsWith(periterbit) && !remaining.isEmpty()) {

   461         unit += remaining.at(0);

   462         remaining.remove(0,1);

   463     }

   464     if (remaining.isEmpty()) {

   465         if (error) *error = "Could not find pattern: '<unit> per iteration (total: '";

   466         return out;

   467     }

   468 

   469     remaining = remaining.mid(sizeof(periterbit)-1);

   470 

   471     /* Remaining: 4000, iterations: 1) */

   472     static const char itersbit[] = ", iterations: ";

   473     QString sTotal;

   474     while (!remaining.startsWith(itersbit) && !remaining.isEmpty()) {

   475         sTotal += remaining.at(0);

   476         remaining.remove(0,1);

   477     }

   478     if (remaining.isEmpty()) {

   479         if (error) *error = "Could not find pattern: '<number>, iterations: '";

   480         return out;

   481     }

   482 

   483     remaining = remaining.mid(sizeof(itersbit)-1);

   484 

   485     qint64 total = sTotal.toLongLong(&ok);

   486     if (!ok) {

   487         if (error) *error = sTotal + " (total) is not a valid integer";

   488         return out;

   489     }

   490 

   491     /* Remaining: 1) */

   492     QString sIters;

   493     while (remaining != QLatin1String(")") && !remaining.isEmpty()) {

   494         sIters += remaining.at(0);

   495         remaining.remove(0,1);

   496     }

   497     if (remaining.isEmpty()) {

   498         if (error) *error = "Could not find pattern: '<num>)'";

   499         return out;

   500     }

   501     qint64 iters = sIters.toLongLong(&ok);

   502     if (!ok) {

   503         if (error) *error = sIters + " (iterations) is not a valid integer";

   504         return out;

   505     }

   506 

   507     double calcFirstNumber = double(total)/double(iters);

   508     if (!qFuzzyCompare(firstNumber, calcFirstNumber)) {

   509         if (error) *error = QString("total/iters is %1, but benchlib output result as %2").arg(calcFirstNumber).arg(firstNumber);

   510         return out;

   511     }

   512 

   513     out.total = total;

   514     out.unit = unit;

   515     out.iterations = iters;

   516     return out;

   517 }

   518 

   519 QTEST_MAIN(tst_Selftests)

   520 

   521 #include "tst_selftests.moc"