1 /****************************************************************************

     2 **

     3 ** Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies).

     4 ** All rights reserved.

     5 ** Contact: Nokia Corporation (qt-info@nokia.com)

     6 **

     7 ** This file is part of the test suite of the Qt Toolkit.

     8 **

     9 ** $QT_BEGIN_LICENSE:LGPL$

    10 ** No Commercial Usage

    11 ** This file contains pre-release code and may not be distributed.

    12 ** You may use this file in accordance with the terms and conditions

    13 ** contained in the Technology Preview License Agreement accompanying

    14 ** this package.

    15 **

    16 ** GNU Lesser General Public License Usage

    17 ** Alternatively, this file may be used under the terms of the GNU Lesser

    18 ** General Public License version 2.1 as published by the Free Software

    19 ** Foundation and appearing in the file LICENSE.LGPL included in the

    20 ** packaging of this file.  Please review the following information to

    21 ** ensure the GNU Lesser General Public License version 2.1 requirements

    22 ** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.

    23 **

    24 ** In addition, as a special exception, Nokia gives you certain additional

    25 ** rights.  These rights are described in the Nokia Qt LGPL Exception

    26 ** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.

    27 **

    28 ** If you have questions regarding the use of this file, please contact

    29 ** Nokia at qt-info@nokia.com.

    30 **

    31 **

    32 **

    33 **

    34 **

    35 **

    36 **

    37 **

    38 ** $QT_END_LICENSE$

    39 **

    40 ****************************************************************************/

    41 

    42 #include <QDir>

    43 #include <QEventLoop>

    44 #include <QPair>

    45 #include <QtDebug>

    46 

    47 #include "ExitCode.h"

    48 

    49 #include "Worker.h"

    50 

    51 using namespace QPatternistSDK;

    52 

    53 const char *const Worker::m_indent = "        ";

    54 

    55 Worker::Worker(QEventLoop &ev,

    56                const QFileInfo &baseline,

    57                const QFileInfo &result) : m_finishedCount(0)

    58                                         , m_baselineFile(baseline)

    59                                         , m_resultFile(result)

    60                                         , m_eventLoop(ev)

    61 {

    62 }

    63 

    64 void Worker::list(QTextStream &out, const QString &msg, QStringList &list)

    65 {

    66     Q_ASSERT(!msg.isEmpty());

    67 

    68     if(list.isEmpty())

    69         return;

    70 

    71     list.sort(); /* Make it pretty, and easy to read. */

    72 

    73     out << msg << ":\n";

    74 

    75     const QStringList::const_iterator end(list.constEnd());

    76     QStringList::const_iterator it(list.constBegin());

    77 

    78     for(; it != end; ++it)

    79         out << m_indent << qPrintable(*it) << '\n';

    80 }

    81 

    82 static inline int count(const ResultThreader::Hash &list, const TestResult::Status stat)

    83 {

    84     const ResultThreader::Hash::const_iterator end(list.constEnd());

    85     ResultThreader::Hash::const_iterator it(list.constBegin());

    86     int result = 0;

    87 

    88     for(; it != end; ++it)

    89     {

    90         if(it.value() == stat)

    91             ++result;

    92     }

    93 

    94     return result;

    95 }

    96 

    97 void Worker::threadFinished()

    98 {

    99     ++m_finishedCount;

   100     Q_ASSERT(m_finishedCount == 1 || m_finishedCount == 2);

   101 

   102     const ResultThreader *const handler = static_cast<ResultThreader *>(sender());

   103     Q_ASSERT(handler);

   104 

   105     switch(handler->type())

   106     {

   107         case ResultThreader::Baseline:

   108         {

   109             m_baseline = handler->result();

   110             break;

   111         }

   112         case ResultThreader::Result:

   113             m_result = handler->result();

   114     }

   115 

   116     if(m_finishedCount == 1) /* One thread's missing. */

   117         return;

   118 

   119     /* Ok, both threads have now finished, and we got their results in m_result and m_baseline. */

   120 

   121     /* No matter how this function exits, we want to delete this Worker. */

   122     deleteLater();

   123 

   124     ResultThreader::Hash::const_iterator itA(m_result.constBegin());

   125     ResultThreader::Hash::const_iterator itB(m_baseline.constBegin());

   126     const ResultThreader::Hash::const_iterator endA(m_result.constEnd());

   127     const ResultThreader::Hash::const_iterator endB(m_baseline.constEnd());

   128     const int baselineCount = m_baseline.count();

   129     const int resultCount = m_result.count();

   130 

   131     /* If you want useful output, change the QTextStream to use stderr. */

   132     //QTextStream err(stderr);

   133     QByteArray out;

   134     QTextStream err(&out);

   135 

   136     if(resultCount < baselineCount)

   137     {

   138         err << qPrintable(QString(QLatin1String("WARNING: Test result contains %1 reports, "

   139                                                 "but the baseline contains %2, a DECREASE "

   140                                                 "of %3 tests.\n"))

   141                                   .arg(resultCount)

   142                                   .arg(baselineCount)

   143                                   .arg(resultCount - baselineCount));

   144     }

   145     else if(resultCount > baselineCount)

   146     {

   147         err << qPrintable(QString(QLatin1String("NOTE: The number of tests run is more than what "

   148                                                 "the baseline specifies. Run was %1 test cases, the "

   149                                                 "baseline specifies %2; an increase of %3 tests.\n"))

   150                                   .arg(resultCount)

   151                                   .arg(baselineCount)

   152                                   .arg(resultCount - baselineCount));

   153     }

   154 

   155     for(; itA != endA; ++itA)

   156     {

   157         const TestResult::Status result = itA.value();

   158         const TestResult::Status baseline = m_baseline.value(itA.key());

   159 

   160         if(result == baseline) /* We have no change. */

   161         {

   162             if(result == TestResult::NotTested)

   163                 m_notTested.append(itA.key());

   164             else

   165                 continue;

   166         }

   167         else if(baseline == TestResult::Pass && result == TestResult::Fail)

   168             m_unexpectedFailures.append(itA.key());

   169         else if(baseline == TestResult::Fail && result == TestResult::Pass)

   170             m_unexpectedPasses.append(itA.key());

   171     }

   172 

   173     list(err, QLatin1String("Not tested"),           m_notTested);

   174     list(err, QLatin1String("Unexpected failures"),  m_unexpectedFailures);

   175     list(err, QLatin1String("Unexpected passes"),    m_unexpectedPasses);

   176 

   177     err << "SUMMARY:\n";

   178     typedef QPair<QString, int> Info;

   179     typedef QList<Info> InfoList;

   180     InfoList info;

   181 

   182     const int totFail       = count(m_result, TestResult::Fail);

   183     const int totPass       = count(m_result, TestResult::Pass);

   184     const int total         = resultCount;

   185     const int notTested     = m_notTested.count();

   186     const int percentage    = int((static_cast<double>(totPass) / total) * 100);

   187 

   188     Q_ASSERT_X(percentage >= 0 && percentage <= 100, Q_FUNC_INFO,

   189                qPrintable(QString(QLatin1String("Percentage was: %1")).arg(percentage)));

   190 

   191     info.append(Info(QLatin1String("Total"),                total));

   192     info.append(Info(QLatin1String("Failures"),             totFail));

   193     info.append(Info(QLatin1String("Passes"),               totPass));

   194     info.append(Info(QLatin1String("Not tested"),           notTested));

   195     info.append(Info(QLatin1String("Pass percentage(%)"),   percentage));

   196     info.append(Info(QLatin1String("Unexpected failures"),  m_unexpectedFailures.count()));

   197     info.append(Info(QLatin1String("Unexpected passes"),    m_unexpectedPasses.count()));

   198 

   199     const InfoList::const_iterator end(info.constEnd());

   200     InfoList::const_iterator it(info.constBegin());

   201 

   202     /* List the statistics nicely in a row with padded columns. */

   203     for(; it != end; ++it)

   204     {

   205         const QString result((((*it).first) + QLatin1Char(':')).leftJustified(22, QLatin1Char(' ')));

   206         err << m_indent << qPrintable(result) << (*it).second << '\n';

   207     }

   208 

   209     if(!m_unexpectedFailures.isEmpty())

   210     {

   211         err << "FAILURE: Regressions discovered, baseline was not updated.\n";

   212         err.flush();

   213         QTextStream(stderr) << out;

   214         m_eventLoop.exit(ExitCode::Regression);

   215         return;

   216     }

   217     else if(m_unexpectedPasses.isEmpty() && baselineCount == resultCount)

   218     {

   219         err << "Result was identical to the baseline, baseline was not updated.\n";

   220         m_eventLoop.exit(ExitCode::Success);

   221         return;

   222     }

   223 

   224     /* Ok, we got unexpected successes and no regressions: let's update the baseline. */

   225 

   226     QFile resultFile(m_resultFile.absoluteFilePath());

   227 

   228     /* Remove the old file, otherwise QFile::copy() will fail. */

   229     QDir baselineDir(m_baselineFile.absolutePath());

   230     baselineDir.remove(m_baselineFile.fileName());

   231 

   232     if(resultFile.copy(m_baselineFile.absoluteFilePath()))

   233     {

   234         /* Give a detailed message of what's going on. */

   235         if(resultCount > baselineCount)

   236             err << "More tests was run than specified in the baseline, updating the baseline.\n";

   237         else

   238             err << "Improvement, the baseline was updated.\n";

   239 

   240         /* We actually flag this as an error, because the new baseline must be submitted. */

   241         err.flush();

   242         QTextStream(stderr) << out;

   243         m_eventLoop.exit(ExitCode::Regression);

   244         return;

   245     }

   246     else

   247     {

   248         err << qPrintable(QString(QLatin1String("Encountered error when updating "

   249                                                 "the baseline: %1\n"))

   250                                   .arg(resultFile.errorString()));

   251         err.flush();

   252         QTextStream(stderr) << out;

   253         m_eventLoop.exit(ExitCode::WriteError);

   254         return;

   255     }

   256 }

   257 

   258 // vim: et:ts=4:sw=4:sts=4