55     void checkXML() const;

    54     void cleanupTestCase();

    56     void checkXML_data();

    57     void checkXunitxml() const;

    58     void checkXunitxml_data();

    61     QStringList m_checkXMLBlacklist;

    57     void doRunSubTest(QString const& subdir, QString const& logger, QStringList const& arguments );

    62     QStringList m_checkXunitBlacklist;

    63     void doRunSubTest(QString &subdir, QStringList &arguments );

   124     return ba.split('\n');

   118     QList<QByteArray> out = ba.split('\n');

   125 }

   119 

   126 

   120     // Replace any ` file="..."' in XML with a generic location.

   127 static QList<QByteArray> expectedResult(const QString &subdir)

   121     static const char marker[] = " file=\"";

   128 {

   122     for (int i = 0; i < out.size(); ++i) {

   129     QFile file(":/expected_" + subdir + ".txt");

   123         QByteArray& line = out[i];

   124         int index = line.indexOf(marker);

   125         if (index == -1) {

   126             continue;

   127         }

   128         int end = line.indexOf('"', index + sizeof(marker));

   129         if (end == -1) {

   130             continue;

   131         }

   132         line.replace(index, end-index, " file=\"__FILE__\"");

   133     }

   134 

   135     return out;

   136 }

   137 

   138 static QList<QByteArray> expectedResult(const QString &subdir, const QString &logger)

   139 {

   140     QString suffix = logger;

   141     if (suffix.isEmpty()) {

   142         suffix = "txt";

   143     }

   144     QFile file(":/expected_" + subdir + "." + suffix);

   140     QTest::newRow("subtest") << "subtest" << QStringList();

   183     QStringList tests = QStringList()

   141     QTest::newRow("warnings") << "warnings" << QStringList();

   184         << "subtest"

   142     QTest::newRow("maxwarnings") << "maxwarnings" << QStringList();

   185         << "warnings"

   143     QTest::newRow("cmptest") << "cmptest" << QStringList();

   186         << "maxwarnings"

   144 //    QTest::newRow("alive") << "alive" << QStringList(); // timer dependent

   187         << "cmptest"

   145     QTest::newRow("globaldata") << "globaldata" << QStringList();

   188 //        << "alive"    // timer dependent

   146     QTest::newRow("skipglobal") << "skipglobal" << QStringList();

   189         << "globaldata"

   147     QTest::newRow("skip") << "skip" << QStringList();

   190         << "skipglobal"

   148     QTest::newRow("strcmp") << "strcmp" << QStringList();

   191         << "skip"

   149     QTest::newRow("expectfail") << "expectfail" << QStringList();

   192         << "strcmp"

   150     QTest::newRow("sleep") << "sleep" << QStringList();

   193         << "expectfail"

   151     QTest::newRow("fetchbogus") << "fetchbogus" << QStringList();

   194         << "sleep"

   152     QTest::newRow("crashes") << "crashes" << QStringList();

   195         << "fetchbogus"

   153     QTest::newRow("multiexec") << "multiexec" << QStringList();

   196         << "crashes"

   154     QTest::newRow("failinit") << "failinit" << QStringList();

   197         << "multiexec"

   155     QTest::newRow("failinitdata") << "failinitdata" << QStringList();

   198         << "failinit"

   156     QTest::newRow("skipinit") << "skipinit" << QStringList();

   199         << "failinitdata"

   157     QTest::newRow("skipinitdata") << "skipinitdata" << QStringList();

   200         << "skipinit"

   158     QTest::newRow("datetime") << "datetime" << QStringList();

   201         << "skipinitdata"

   159     QTest::newRow("singleskip") << "singleskip" << QStringList();

   202         << "datetime"

   160 

   203         << "singleskip"

   161     //on windows assert does nothing in release mode and blocks execution with a popup window in debug mode

   204 

   205         //on windows assert does nothing in release mode and blocks execution with a popup window in debug mode

   163     QTest::newRow("assert") << "assert" << QStringList();

   207         << "assert"

   166     QTest::newRow("waitwithoutgui") << "waitwithoutgui" << QStringList();

   210         << "waitwithoutgui"

   167     QTest::newRow("differentexec") << "differentexec" << QStringList();

   211         << "differentexec"

   169     // The machine that run the intel autotests will popup a dialog

   213         // The machine that run the intel autotests will popup a dialog

   170     // with a warning that an uncaught exception was thrown.

   214         // with a warning that an uncaught exception was thrown.

   171     // This will time out and falsely fail, therefore we disable the test for that platform.

   215         // This will time out and falsely fail, therefore we disable the test for that platform.

   173     QTest::newRow("exceptionthrow") << "exceptionthrow" << QStringList();

   217         << "exceptionthrow"

   176     QTest::newRow("qexecstringlist") << "qexecstringlist" << QStringList();

   220         << "qexecstringlist"

   177     QTest::newRow("datatable") << "datatable" << QStringList();

   221         << "datatable"

   178     QTest::newRow("commandlinedata") << "commandlinedata" << QString("fiveTablePasses fiveTablePasses:fiveTablePasses_data1 -v2").split(' ');

   222         << "commandlinedata"

   181     QTest::newRow("benchlibcallgrind") << "benchlibcallgrind" << QStringList("-callgrind");

   225         << "benchlibcallgrind"

   183     QTest::newRow("benchlibeventcounter") << "benchlibeventcounter" << QStringList("-eventcounter");

   227         << "benchlibeventcounter"

   184     QTest::newRow("benchliboptions") << "benchliboptions" << QStringList("-eventcounter");

   228         << "benchliboptions"

   185 

   229 

   186     //### These tests are affected by timing and whether the CPU tick counter is

   230         //### These tests are affected by timing and whether the CPU tick counter is

   187     //### monotonically increasing. They won't work on some machines so leave them off by default.

   231         //### monotonically increasing. They won't work on some machines so leave them off by default.

   188     //### Feel free to uncomment for your own testing.

   232         //### Feel free to uncomment for your own testing.

   190     QTest::newRow("benchlibwalltime") << "benchlibwalltime" << QStringList();

   234         << "benchlibwalltime"

   191     QTest::newRow("benchlibtickcounter") << "benchlibtickcounter" << QStringList("-tickcounter");

   235         << "benchlibtickcounter"

   194     QTest::newRow("xunit") << "xunit" << QStringList("-xunitxml");

   238         << "xunit"

   195     QTest::newRow("longstring") << "longstring" << QStringList();

   239         << "longstring"

   196 

   240         << "badxml"

   197 }

   241     ;

   198 

   242 

   199 void tst_Selftests::doRunSubTest(QString &subdir, QStringList &arguments )

   243     foreach (Logger const& logger, allLoggers()) {

   200 {

   244         QString rowSuffix;

   245         if (logger.name != "plain") {

   246             rowSuffix = QString(" %1").arg(logger.name);

   247         }

   248 

   249         foreach (QString const& subtest, tests) {

   250             QStringList arguments = logger.arguments;

   251             if (subtest == "commandlinedata") {

   252                 arguments << QString("fiveTablePasses fiveTablePasses:fiveTablePasses_data1 -v2").split(' ');

   253             }

   254             else if (subtest == "benchlibcallgrind") {

   255                 arguments << "-callgrind";

   256             }

   257             else if (subtest == "benchlibeventcounter") {

   258                 arguments << "-eventcounter";

   259             }

   260             else if (subtest == "benchliboptions") {

   261                 arguments << "-eventcounter";

   262             }

   263             else if (subtest == "benchlibtickcounter") {

   264                 arguments << "-tickcounter";

   265             }

   266             else if (subtest == "badxml") {

   267                 arguments << "-eventcounter";

   268             }

   269 

   270             // These tests don't work right with loggers other than plain, usually because

   271             // they internally supply arguments to themselves.

   272             if (logger.name != "plain") {

   273                 if (subtest == "differentexec") {

   274                     continue;

   275                 }

   276                 if (subtest == "qexecstringlist") {

   277                     continue;

   278                 }

   279                 if (subtest == "benchliboptions") {

   280                     continue;

   281                 }

   282                 if (subtest == "waitwithoutgui") {

   283                     continue;

   284                 }

   285                 // `crashes' will not output valid XML on platforms without a crash handler

   286                 if (subtest == "crashes") {

   287                     continue;

   288                 }

   289                 // this test prints out some floats in the testlog and the formatting is

   290                 // platform-specific and hard to predict.

   291                 if (subtest == "subtest") {

   292                     continue;

   293                 }

   294             }

   295 

   296             QTest::newRow(qPrintable(QString("%1%2").arg(subtest).arg(rowSuffix)))

   297                 << subtest

   298                 << logger.testdata_suffix

   299                 << arguments

   300             ;

   301         }

   302     }

   303 }

   304 

   305 void tst_Selftests::doRunSubTest(QString const& subdir, QString const& logger, QStringList const& arguments )

   306 {

   307     // For the plain text logger, we'll read straight from standard output.

   308     // For all other loggers (XML), we'll tell testlib to redirect to a file.

   309     // The reason is that tests are allowed to print to standard output, and

   310     // that means the test log is no longer guaranteed to be valid XML.

   311     QStringList extraArguments;

   312     QString logfile;

   313     if (logger != "txt") {

   314         logfile = "test_output";

   315         extraArguments << "-o" << logfile;

   316     }

   317 

   203     proc.start(subdir + "/" + subdir, arguments);

   320     proc.start(subdir + "/" + subdir, QStringList() << arguments << extraArguments);

   206     const QByteArray out(proc.readAllStandardOutput());

   323     QByteArray out;

   324     if (logfile.isEmpty()) {

   325         out = proc.readAllStandardOutput();

   326     }

   327     else {

   328         QFile file(logfile);

   329         if (file.open(QIODevice::ReadOnly))

   330             out = file.readAll();

   331     }

   332 

   218     QList<QByteArray> exp = expectedResult(subdir);

   344     QList<QByteArray> exp = expectedResult(subdir, logger);

   224             exp = expectedResult(subdir + QString("_%1").arg(i++));

   350             exp = expectedResult(subdir + QString("_%1").arg(i++), logger);

   253         if (line.contains("ASSERT") && output != expected)

   401         if (line.contains("ASSERT") && output != expected) {

   254             QEXPECT_FAIL("assert", "QTestLib prints out the absolute path.", Continue);

   402             QEXPECT_FAIL("assert",          "QTestLib prints out the absolute path.", Continue);

   403             QEXPECT_FAIL("assert xml",      "QTestLib prints out the absolute path.", Continue);

   404             QEXPECT_FAIL("assert xml flush","QTestLib prints out the absolute path.", Continue);

   405             QEXPECT_FAIL("assert lightxml", "QTestLib prints out the absolute path.", Continue);

   406             QEXPECT_FAIL("assert xunitxml", "QTestLib prints out the absolute path.", Continue);

   407         }

   297     doRunSubTest(subdir, arguments);

   451     doRunSubTest(subdir, logger, arguments);

   298 }

   299 

   300 void tst_Selftests::initTestCase()

   301 {

   302 #if !defined(Q_OS_UNIX) || defined(Q_WS_MAC)

   303     m_checkXMLBlacklist.append("crashes"); // This test crashes (XML valid on Unix only)

   304 #endif

   305     m_checkXMLBlacklist.append("waitwithoutgui"); // This test is not a QTestLib test.

   306 

   307     /* Output from several tests is broken with the XML output method,

   308      * and it's quite heavy in the design. See task 155001. */

   309     m_checkXMLBlacklist.append("multiexec");

   310     m_checkXMLBlacklist.append("differentexec");

   311     m_checkXMLBlacklist.append("qexecstringlist");

   312     m_checkXMLBlacklist.append("benchliboptions");

   313 

   314     /* These tests use printf and therefore corrupt the testlog */

   315     m_checkXMLBlacklist.append("subtest");

   316     m_checkXMLBlacklist.append("globaldata");

   317     m_checkXMLBlacklist.append("warnings");

   318 

   319     m_checkXunitBlacklist = m_checkXMLBlacklist;

   320 }

   321 

   322 void tst_Selftests::checkXML() const

   323 {

   324     QFETCH(QString, subdir);

   325     QFETCH(QStringList, arguments);

   326 

   327     if(m_checkXMLBlacklist.contains(subdir))

   328         return;

   329 

   330     QStringList args;

   331     /* Test both old (-flush) and new XML logger implementation */

   332     for (int i = 0; i < 2; ++i) {

   333         bool flush = i;

   334         args = arguments;

   335         args.prepend("-xml");

   336         if (flush) args.prepend("-flush");

   337 

   338         QProcess proc;

   339         proc.setEnvironment(QStringList(""));

   340         proc.start(subdir + "/" + subdir, args);

   341         QVERIFY(proc.waitForFinished());

   342 

   343         QByteArray out(proc.readAllStandardOutput());

   344         QByteArray err(proc.readAllStandardError());

   345 

   346         /* Some platforms decides to output a message for uncaught exceptions. For instance,

   347          * this is what windows platforms says:

   348          * "This application has requested the Runtime to terminate it in an unusual way.

   349          * Please contact the application's support team for more information." */

   350         if(subdir != QLatin1String("exceptionthrow") && subdir != QLatin1String("fetchbogus"))

   351             QVERIFY2(err.isEmpty(), err.constData());

   352 

   353         QXmlStreamReader reader(out);

   354 

   355         while(!reader.atEnd())

   356             reader.readNext();

   357 

   358         QVERIFY2(!reader.error(), qPrintable(QString("(flush %0) line %1, col %2: %3")

   359             .arg(flush)

   360             .arg(reader.lineNumber())

   361             .arg(reader.columnNumber())

   362             .arg(reader.errorString())

   363         ));

   364     }

   365 }

   366 

   367 void tst_Selftests::checkXunitxml() const

   368 {

   369     QFETCH(QString, subdir);

   370     QFETCH(QStringList, arguments);

   371 

   372     if(m_checkXunitBlacklist.contains(subdir))

   373         return;

   374 

   375     arguments.prepend("-xunitxml");

   376     arguments.prepend("-flush");

   377 

   378     QProcess proc;

   379     proc.setEnvironment(QStringList(""));

   380     proc.start(subdir + "/" + subdir, arguments);

   381     QVERIFY(proc.waitForFinished());

   382 

   383     QByteArray out(proc.readAllStandardOutput());

   384     QByteArray err(proc.readAllStandardError());

   385 

   386 //    qDebug()<<out;

   387 

   388     /* Some platforms decides to output a message for uncaught exceptions. For instance,

   389      * this is what windows platforms says:

   390      * "This application has requested the Runtime to terminate it in an unusual way.

   391      * Please contact the application's support team for more information." */

   392     if(subdir != QLatin1String("exceptionthrow") && subdir != QLatin1String("fetchbogus"))

   393         QVERIFY2(err.isEmpty(), err.constData());

   394 

   395     QXmlStreamReader reader(out);

   396 

   397     while(!reader.atEnd())

   398         reader.readNext();

   399 

   400     QVERIFY2(!reader.error(), qPrintable(QString("line %1, col %2: %3")

   401         .arg(reader.lineNumber())

   402         .arg(reader.columnNumber())

   403         .arg(reader.errorString())

   404     ));

   405 }

   406 

   407 void tst_Selftests::checkXunitxml_data()

   408 {

   409     checkXML_data();

   410 }

   411 

   412 void tst_Selftests::checkXML_data()

   413 {

   414     runSubTest_data();

   415     QTest::newRow("badxml 1") << "badxml" << QStringList();

   416     QTest::newRow("badxml 2") << "badxml" << (QStringList() << "-badstring" << "0");

   417     QTest::newRow("badxml 3") << "badxml" << (QStringList() << "-badstring" << "1");

   418     QTest::newRow("badxml 4") << "badxml" << (QStringList() << "-badstring" << "2");

   419     QTest::newRow("badxml 5") << "badxml" << (QStringList() << "-badstring" << "3");