1 /****************************************************************************

     2 **

     3 ** Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies).

     4 ** All rights reserved.

     5 ** Contact: Nokia Corporation (qt-info@nokia.com)

     6 **

     7 ** This file is part of the test suite of the Qt Toolkit.

     8 **

     9 ** $QT_BEGIN_LICENSE:LGPL$

    10 ** No Commercial Usage

    11 ** This file contains pre-release code and may not be distributed.

    12 ** You may use this file in accordance with the terms and conditions

    13 ** contained in the Technology Preview License Agreement accompanying

    14 ** this package.

    15 **

    16 ** GNU Lesser General Public License Usage

    17 ** Alternatively, this file may be used under the terms of the GNU Lesser

    18 ** General Public License version 2.1 as published by the Free Software

    19 ** Foundation and appearing in the file LICENSE.LGPL included in the

    20 ** packaging of this file.  Please review the following information to

    21 ** ensure the GNU Lesser General Public License version 2.1 requirements

    22 ** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.

    23 **

    24 ** In addition, as a special exception, Nokia gives you certain additional

    25 ** rights.  These rights are described in the Nokia Qt LGPL Exception

    26 ** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.

    27 **

    28 ** If you have questions regarding the use of this file, please contact

    29 ** Nokia at qt-info@nokia.com.

    30 **

    31 **

    32 **

    33 **

    34 **

    35 **

    36 **

    37 **

    38 ** $QT_END_LICENSE$

    39 **

    40 ****************************************************************************/

    41 

    42 

    43 #include <atWrapper.h>

    44 #include <datagenerator/datagenerator.h>

    45 

    46 #include <QString>

    47 #include <QHash>

    48 #include <QFile>

    49 #include <QFtp>

    50 #include <QObject>

    51 #include <QHostInfo>

    52 #include <QWidget>

    53 #include <QImage>

    54 #include <QtTest/QSignalSpy>

    55 #include <QLibraryInfo>

    56 

    57 static const char *ArthurDir = "../../arthur";

    58 

    59 #include <string.h>

    60 

    61 atWrapper::atWrapper()

    62 {

    63 

    64  //   initTests();

    65 

    66 }

    67 

    68 bool atWrapper::initTests(bool *haveBaseline)

    69 {

    70     qDebug() << "Running test on buildkey:" << QLibraryInfo::buildKey() << "  qt version:" << qVersion();

    71 

    72     qDebug( "Initializing tests..." );

    73 

    74     if (!loadConfig( QHostInfo::localHostName().split( "." ).first() + ".ini" ))

    75         return false;

    76 

    77     //Reset the FTP environment where the results are stored

    78     *haveBaseline = setupFTP();

    79 

    80     // Retrieve the latest test result baseline from the FTP server.

    81     downloadBaseline();

    82     return true;

    83 }

    84 

    85 void atWrapper::downloadBaseline()

    86 {

    87 

    88     qDebug() << "Now downloading baseline...";

    89 

    90     QFtp ftp;

    91 

    92     QObject::connect( &ftp, SIGNAL( listInfo( const QUrlInfo & ) ), this, SLOT( ftpMgetAddToList(const QUrlInfo & ) ) );

    93 

    94     //Making sure that the needed local directories exist.

    95 

    96     QHashIterator<QString, QString> j(enginesToTest);

    97 

    98     while ( j.hasNext() )

    99     {

   100         j.next();

   101 

   102         QDir dir( output );

   103 

   104         if ( !dir.cd( j.key() + ".baseline" ) )

   105             dir.mkdir( j.key() + ".baseline" );

   106 

   107     }

   108 

   109     //FTP to the host specified in the config file, and retrieve the test result baseline.

   110     ftp.connectToHost( ftpHost );

   111     ftp.login( ftpUser, ftpPass );

   112 

   113     ftp.cd( ftpBaseDir );

   114 

   115     QHashIterator<QString, QString> i(enginesToTest);

   116     while ( i.hasNext() )

   117     {

   118         i.next();

   119         mgetDirList.clear();

   120         mgetDirList << i.key() + ".baseline";

   121         ftp.cd( i.key() + ".baseline" );

   122         ftp.list();

   123         ftp.cd( ".." );

   124 

   125         while ( ftp.hasPendingCommands() )

   126             QCoreApplication::instance()->processEvents();

   127 

   128         ftpMgetDone( true );

   129     }

   130 

   131     ftp.close();

   132     ftp.close();

   133 

   134     while ( ftp.hasPendingCommands() )

   135         QCoreApplication::instance()->processEvents();

   136 

   137 }

   138 

   139 void atWrapper::ftpMgetAddToList( const QUrlInfo &urlInfo )

   140 {

   141     //Simply adding to the list of files to download.

   142     mgetDirList << urlInfo.name();

   143 

   144 }

   145 

   146 void atWrapper::ftpMgetDone( bool error)

   147 {

   148     Q_UNUSED( error );

   149 

   150     //Downloading the files listed in mgetDirList...

   151     QFtp ftp;

   152     ftp.connectToHost( ftpHost );

   153     ftp.login( ftpUser, ftpPass );

   154 

   155     QFile* file;

   156 

   157     if ( mgetDirList.size() > 1 )

   158         for ( int i = 1; i < mgetDirList.size(); ++i )

   159         {

   160             file = new QFile( QString( output ) + "/" + mgetDirList.at( 0 ) + "/" + mgetDirList.at( i ) );

   161             if (file->open(QIODevice::WriteOnly)) {

   162                 ftp.get( ftpBaseDir + "/" + mgetDirList.at( 0 ) + "/" + mgetDirList.at( i ), file );

   163                 ftp.list(); //Only there to fill up a slot in the pendingCommands queue.

   164                 while ( ftp.hasPendingCommands() )

   165                     QCoreApplication::instance()->processEvents();

   166                 file->close();

   167             } else {

   168                 qDebug() << "Couldn't open file for writing: " << file->fileName();

   169             }

   170         }

   171 

   172 

   173     while ( ftp.hasPendingCommands() )

   174         QCoreApplication::instance()->processEvents();

   175 }

   176 

   177 void atWrapper::uploadFailed( QString dir, QString filename, QByteArray filedata )

   178 {

   179     //Upload a failed test case image to the FTP server.

   180     QFtp ftp;

   181     ftp.connectToHost( ftpHost );

   182     ftp.login( ftpUser, ftpPass );

   183 

   184     ftp.cd( ftpBaseDir );

   185     ftp.cd( dir );

   186 

   187     ftp.put( filedata, filename, QFtp::Binary );

   188 

   189     ftp.close();

   190 

   191     while ( ftp.hasPendingCommands() )

   192         QCoreApplication::instance()->processEvents();

   193 }

   194 

   195 // returns false if no baseline exists

   196 bool atWrapper::setupFTP()

   197 {

   198     qDebug( "Setting up FTP environment" );

   199 

   200     QString dir = "";

   201     ftpMkDir( ftpBaseDir );

   202 

   203     ftpBaseDir += "/" + QLibraryInfo::buildKey();

   204 

   205     ftpMkDir( ftpBaseDir );

   206 

   207     ftpBaseDir += "/" + QString( qVersion() );

   208 

   209     ftpMkDir( ftpBaseDir );

   210 

   211     QHashIterator<QString, QString> i(enginesToTest);

   212     QHashIterator<QString, QString> j(enginesToTest);

   213 

   214     bool haveBaseline = true;

   215     //Creating the baseline directories for each engine

   216     while ( i.hasNext() )

   217     {

   218         i.next();

   219         //qDebug() << "Creating dir with key:" << i.key();

   220         ftpMkDir( ftpBaseDir + "/" +  QString( i.key() ) + ".failed" );

   221         ftpMkDir( ftpBaseDir + "/" +  QString( i.key() ) + ".diff" );

   222         if (!ftpMkDir( ftpBaseDir + "/" + QString( i.key() ) + ".baseline" ))

   223             haveBaseline = false;

   224     }

   225 

   226 

   227     QFtp ftp;

   228     ftp.connectToHost( ftpHost );

   229     ftp.login( ftpUser, ftpPass );

   230 

   231     ftp.cd( ftpBaseDir );

   232     //Deleting previous failed directory and all the files in it, then recreating it.

   233     while ( j.hasNext() )

   234     {

   235         j.next();

   236         rmDirList.clear();

   237         rmDirList << ftpBaseDir + "/" + j.key() + ".failed" + "/";

   238         ftpRmDir( j.key() + ".failed" );

   239         ftp.rmdir( j.key() + ".failed" );

   240         ftp.mkdir( j.key() + ".failed" );

   241         ftp.list();

   242 

   243         while ( ftp.hasPendingCommands() )

   244             QCoreApplication::instance()->processEvents();

   245 

   246         rmDirList.clear();

   247         rmDirList << ftpBaseDir + "/" + j.key() + ".diff" + "/";

   248         ftpRmDir( j.key() + ".diff" );

   249         ftp.rmdir( j.key() + ".diff" );

   250         ftp.mkdir( j.key() + ".diff" );

   251         ftp.list();

   252 

   253         while ( ftp.hasPendingCommands() )

   254             QCoreApplication::instance()->processEvents();

   255 

   256     }

   257 

   258     ftp.close();

   259 

   260     while ( ftp.hasPendingCommands() )

   261         QCoreApplication::instance()->processEvents();

   262 

   263     return haveBaseline;

   264 }

   265 

   266 void atWrapper::ftpRmDir( QString dir )

   267 {

   268     //Hack to remove a populated directory. (caveat: containing only files and empty dirs, not recursive!)

   269     qDebug() << "Now removing directory: " << dir;

   270     QFtp ftp;

   271     QObject::connect( &ftp, SIGNAL( listInfo( const QUrlInfo & ) ), this, SLOT( ftpRmDirAddToList(const QUrlInfo & ) ) );

   272     QObject::connect( &ftp, SIGNAL( done( bool ) ), this, SLOT( ftpRmDirDone( bool ) ) );

   273 

   274     ftp.connectToHost( ftpHost );

   275     ftp.login( ftpUser, ftpPass );

   276 

   277     ftp.list( ftpBaseDir + "/" +  dir );

   278     ftp.close();

   279     ftp.close();

   280 

   281     while ( ftp.hasPendingCommands() )

   282                 QCoreApplication::instance()->processEvents();

   283 }

   284 

   285 void atWrapper::ftpRmDirDone( bool error )

   286 {

   287     //Deleting each file in the directory listning, rmDirList.

   288     Q_UNUSED( error );

   289 

   290     QFtp ftp;

   291     ftp.connectToHost( ftpHost );

   292     ftp.login( ftpUser, ftpPass );

   293 

   294     if ( rmDirList.size() > 1 )

   295         for (int i = 1; i < rmDirList.size(); ++i)

   296             ftp.remove( rmDirList.at(0) + rmDirList.at( i ) );

   297 

   298     ftp.close();

   299 

   300     while ( ftp.hasPendingCommands() )

   301         QCoreApplication::instance()->processEvents();

   302 }

   303 

   304 // returns false if the directory already exists

   305 bool atWrapper::ftpMkDir( QString dir )

   306 {

   307     //Simply used to avoid QFTP from bailing out and loosing a queue of commands.

   308     // IE: conveniance.

   309     QFtp ftp;

   310 

   311     QSignalSpy commandSpy(&ftp, SIGNAL(commandFinished(int, bool)));

   312 

   313     ftp.connectToHost( ftpHost );

   314     ftp.login( ftpUser, ftpPass );

   315     const int command = ftp.mkdir( dir );

   316     ftp.close();

   317 

   318     while ( ftp.hasPendingCommands() )

   319         QCoreApplication::instance()->processEvents();

   320 

   321     for (int i = 0; i < commandSpy.count(); ++i)

   322         if (commandSpy.at(i).at(0) == command)

   323             return commandSpy.at(i).at(1).toBool();

   324 

   325     return false;

   326 }

   327 

   328 

   329 void atWrapper::ftpRmDirAddToList( const QUrlInfo &urlInfo )

   330 {

   331     //Just adding the file to the list for deletion

   332     rmDirList << urlInfo.name();

   333 }

   334 

   335 

   336 bool atWrapper::executeTests()

   337 {

   338     qDebug("Executing the tests...");

   339 

   340     QHashIterator<QString, QString> i(enginesToTest);

   341 

   342     DataGenerator generator;

   343 

   344     //Running datagenerator against all the frameworks specified in the config file.

   345     while ( i.hasNext() )

   346     {

   347 

   348         i.next();

   349 

   350         qDebug( "Now testing: " + i.key().toLatin1() );

   351 

   352         char* params[13];

   353         //./bin/datagenerator  -framework data/framework.ini  -engine OpenGL -suite 1.1 -output outtest

   354 

   355 

   356         QByteArray eng = i.key().toLatin1();

   357         QByteArray fwk = framework.toLatin1();

   358         QByteArray sut = suite.toLatin1();

   359         QByteArray out = output.toLatin1();

   360         QByteArray siz = size.toLatin1();

   361         QByteArray fill = fillColor.toLatin1();

   362 

   363         params[1] = "-framework";

   364         params[2] = fwk.data();

   365         params[3] = "-engine";

   366         params[4] = eng.data();

   367         params[5] = "-suite";

   368         params[6] = sut.data();

   369         params[7] = "-output";

   370         params[8] = out.data();

   371         params[9] = "-size";

   372         params[10] = siz.data();

   373         params[11] = "-fill";

   374         params[12] = fill.data();

   375 

   376         generator.run( 13, params );

   377     }

   378 

   379     return true;

   380 }

   381 

   382 void atWrapper::createBaseline()

   383 {

   384     qDebug( "Now uploading a baseline of only the latest test values" );

   385 

   386     QHashIterator<QString, QString> i(enginesToTest);

   387 

   388     QDir dir( output );

   389     QFtp ftp;

   390     ftp.connectToHost( ftpHost );

   391     ftp.login( ftpUser, ftpPass );

   392     ftp.cd( ftpBaseDir );

   393     //Upload all the latest test results to the FTP server's baseline directory.

   394     while ( i.hasNext() )

   395     {

   396 

   397         i.next();

   398         dir.cd( i.key() );

   399         ftp.cd( i.key() + ".baseline" );

   400         dir.setFilter(QDir::Files | QDir::Hidden | QDir::NoSymLinks);

   401         dir.setNameFilters( QStringList() << "*.png" );

   402         QFileInfoList list = dir.entryInfoList();

   403         dir.cd( ".." );

   404         for (int n = 0; n < list.size(); n++)

   405         {

   406             QFileInfo fileInfo = list.at( n );

   407             QFile file( QString( output ) + "/" + i.key() + "/" + fileInfo.fileName() );

   408             file.open( QIODevice::ReadOnly );

   409             QByteArray fileData = file.readAll();

   410             //qDebug() << "Sending up:" << fileInfo.fileName() << "with file size" << fileData.size();

   411             file.close();

   412             ftp.put( fileData, fileInfo.fileName(), QFtp::Binary );

   413         }

   414 

   415         ftp.cd( ".." );

   416     }

   417 

   418     ftp.close();

   419 

   420     while ( ftp.hasPendingCommands() )

   421         QCoreApplication::instance()->processEvents();

   422 }

   423 

   424 bool atWrapper::compare()

   425 {

   426     qDebug( "Now comparing the results to the baseline" );

   427 

   428     QHashIterator<QString, QString> i(enginesToTest);

   429 

   430     while ( i.hasNext() )

   431     {

   432         i.next();

   433 

   434         compareDirs( output , i.key() );

   435 

   436     }

   437 

   438     return true;

   439 }

   440 

   441 void atWrapper::compareDirs( QString basedir, QString target )

   442 {

   443 

   444     QDir dir( basedir );

   445 

   446     /* The following should be redundant now.

   447 

   448     if ( !dir.cd( target + ".failed" ) )

   449         dir.mkdir( target + ".failed" );

   450     else

   451         dir.cdUp();

   452 

   453     */

   454 

   455      if ( !dir.cd( target + ".diff" ) )

   456          dir.mkdir( target + ".diff" );

   457      else

   458          dir.cdUp();

   459 

   460 

   461 

   462     //Perform comparisons between the two directories.

   463 

   464     dir.setFilter(QDir::Files | QDir::Hidden | QDir::NoSymLinks);

   465     dir.setNameFilters( QStringList() << "*.png" );

   466     dir.cd( target + ".baseline" );

   467     QFileInfoList list = dir.entryInfoList();

   468 

   469     for (int i = 0; i < list.size(); ++i)

   470     {

   471         QFileInfo fileInfo = list.at(i);

   472         diff ( basedir, target, fileInfo.fileName() );

   473     }

   474 }

   475 

   476 bool atWrapper::diff( QString basedir, QString dir, QString target )

   477 {

   478     //Comparing the two specified files, and then uploading them to

   479     //the ftp server if they differ

   480 

   481     basedir += "/" + dir;

   482     QString one = basedir + ".baseline/" + target;

   483     QString two = basedir + "/" + target;

   484 

   485     QFile file( one );

   486 

   487     file.open( QIODevice::ReadOnly );

   488     QByteArray contentsOfOne = file.readAll();

   489     file.close();

   490 

   491     file.setFileName( two );

   492 

   493     file.open( QIODevice::ReadOnly );

   494     QByteArray contentsOfTwo = file.readAll();

   495     file.close();

   496 

   497     if ( contentsOfTwo.size() == 0 )

   498     {

   499         qDebug() << "No test result found for baseline: " << one;

   500         file.setFileName( one );

   501         file.open( QIODevice::ReadOnly );

   502         file.copy( basedir + ".failed/" + target + "_missing"  );

   503         uploadFailed( dir + ".failed", target + "_missing", contentsOfTwo );

   504         return false;

   505     }

   506 

   507 

   508     if ( ( memcmp( contentsOfOne, contentsOfTwo, contentsOfOne.size() ) ) == 0 )

   509         return true;

   510     else

   511     {

   512         qDebug() << "Sorry, the result did not match: " << one;

   513         file.setFileName( two );

   514         file.open( QIODevice::ReadOnly );

   515         file.copy( basedir + ".failed/" + target  );

   516         file.close();

   517         uploadFailed( dir + ".failed", target, contentsOfTwo );

   518         uploadDiff( basedir, dir, target );

   519         return false;

   520     }

   521 }

   522 

   523 void atWrapper::uploadDiff( QString basedir, QString dir, QString filename )

   524 {

   525 

   526     qDebug() << basedir;

   527     QImage im1( basedir + ".baseline/" + filename );

   528     QImage im2( basedir + "/" + filename );

   529 

   530     QImage im3(im1.size(), QImage::Format_ARGB32);

   531 

   532     im1 = im1.convertToFormat(QImage::Format_ARGB32);

   533     im2 = im2.convertToFormat(QImage::Format_ARGB32);

   534 

   535     for ( int y=0; y<im1.height(); ++y )

   536     {

   537         uint *s = (uint *) im1.scanLine(y);

   538         uint *d = (uint *) im2.scanLine(y);

   539         uint *w = (uint *) im3.scanLine(y);

   540 

   541         for ( int x=0; x<im1.width(); ++x )

   542         {

   543             if (*s != *d)

   544                 *w = 0xff000000;

   545             else

   546                 *w = 0xffffffff;

   547         w++;

   548         s++;

   549         d++;

   550         }

   551     }

   552 

   553     im3.save( basedir + ".diff/" + filename ,"PNG");

   554 

   555     QFile file( basedir + ".diff/" + filename );

   556     file.open( QIODevice::ReadOnly );

   557     QByteArray contents = file.readAll();

   558     file.close();

   559 

   560     uploadFailed( dir + ".diff", filename, contents );

   561 

   562 }

   563 

   564 bool atWrapper::loadConfig( QString path )

   565 {

   566     qDebug() << "Loading config file from ... " << path;

   567     configPath = path;

   568     //If there is no config file, dont proceed;

   569     if ( !QFile::exists( path ) )

   570     {

   571         return false;

   572     }

   573 

   574 

   575     QSettings settings( path, QSettings::IniFormat, this );

   576 

   577 

   578     //FIXME: Switch to QStringList or something, hash is not needed!

   579     int numEngines = settings.beginReadArray("engines");

   580 

   581     for ( int i = 0; i < numEngines; ++i )

   582     {

   583         settings.setArrayIndex(i);

   584         enginesToTest.insert( settings.value( "engine" ).toString(), "Info here please :p" );

   585     }

   586 

   587     settings.endArray();

   588 

   589     framework = QString(ArthurDir) + QDir::separator() + settings.value( "framework" ).toString();

   590     suite = settings.value( "suite" ).toString();

   591     output = settings.value( "output" ).toString();

   592     size = settings.value( "size", "480,360" ).toString();

   593     fillColor = settings.value( "fill", "white" ).toString();

   594     ftpUser = settings.value( "ftpUser" ).toString();

   595     ftpPass = settings.value( "ftpPass" ).toString();

   596     ftpHost = settings.value( "ftpHost" ).toString();

   597     ftpBaseDir = settings.value( "ftpBaseDir" ).toString();

   598 

   599 

   600     QDir::current().mkdir( output );

   601 

   602     output += "/" + QLibraryInfo::buildKey();

   603 

   604     QDir::current().mkdir( output );

   605 

   606     output += "/" + QString( qVersion() );

   607 

   608     QDir::current().mkdir( output );

   609 

   610 

   611     ftpBaseDir += "/" + QHostInfo::localHostName().split( "." ).first();

   612 

   613 

   614 /*

   615     framework = "data/framework.ini";

   616     suite = "1.1";

   617     output = "testresults";

   618     ftpUser = "anonymous";

   619     ftpPass = "anonymouspass";

   620     ftpHost = "kramer.troll.no";

   621     ftpBaseDir = "/arthurtest";

   622 */

   623     return true;

   624 }

   625 

   626 bool atWrapper::runAutoTests()

   627 {

   628     //SVG needs this widget...

   629     QWidget dummy;

   630 

   631     bool haveBaseline = false;

   632 

   633     if (!initTests(&haveBaseline))

   634         return false;

   635     executeTests();

   636 

   637     if ( !haveBaseline )

   638     {

   639         qDebug( " First run! Creating baseline..." );

   640         createBaseline();

   641     }

   642     else

   643     {

   644         qDebug( " Comparing results..." );

   645         compare();

   646     }

   647     return true;

   648 }