1 /****************************************************************************

     2 **

     3  * Copyright (c) 2010 Nokia Corporation and/or its subsidiary(-ies).

     4  *

     5  * This file is part of Qt Web Runtime.

     6  *

     7  * This library is free software; you can redistribute it and/or

     8  * modify it under the terms of the GNU Lesser General Public License

     9  * version 2.1 as published by the Free Software Foundation.

    10  *

    11  * This library is distributed in the hope that it will be useful,

    12  * but WITHOUT ANY WARRANTY; without even the implied warranty of

    13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU

    14  * Lesser General Public License for more details.

    15  *

    16  * You should have received a copy of the GNU Lesser General Public

    17  * License along with this library; if not, write to the Free Software

    18  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

    19  *

    20  */

    21 

    22 # include <QAbstractNetworkCache>

    23 # include <QNetworkCacheMetaData>

    24 # include <QDateTime>

    25 # include <QDir>

    26 # include <QDirIterator>

    27 # include <QFile>

    28 # include <QtGlobal>

    29 # include <QDebug>

    30 # include <QQueue>

    31 # include <featherweightcache.h>

    32 # include <featherweightcache_p.h>

    33 #if defined(Q_OS_SYMBIAN)

    34 #include <e32std.h>

    35 #endif

    36 

    37 //#define FEATHERWEIGHTCACHE_DEBUG

    38 

    39 #define CACHE_POSTFIX QLatin1String(".d")

    40 #define PREPARED_SLASH QLatin1String("prepared/")

    41 #define DATA_SLASH QLatin1String("data/")

    42 #define MAX_COMPRESSION_SIZE (1024 * 1024 * 3)

    43 

    44 namespace WRT {

    45 

    46 /*!

    47     \class FeatherWeightCache

    48 

    49     \brief The FeatherWeightCache class provides a very basic disk cache.

    50 

    51     FeatherWeightCache stores each url in its own file inside of the

    52     cacheDirectory using QDataStream.  Files with a text MimeType

    53     are compressed using qCompress.  Each cache file starts with "cache_"

    54     and ends in ".cache".  Data is written to disk only in insert()

    55     and updateMetaData().

    56 

    57     Currently you can not share the same cache files with more then

    58     one disk cache.

    59 

    60     FeatherWeightCache by default limits the amount of space that the cache will

    61     use on the system to 50MB.

    62 

    63     Note you have to set the cache directory before it will work.

    64 

    65     A network disk cache can be enabled by:

    66 

    67     \snippet doc/src/snippets/code/src_network_access_FeatherWeightCache.cpp 0

    68 

    69     When sending requests, to control the preference of when to use the cache

    70     and when to use the network, consider the following:

    71 

    72     \snippet doc/src/snippets/code/src_network_access_FeatherWeightCache.cpp 1

    73 

    74     To check whether the response came from the cache or from the network, the

    75     following can be applied:

    76 

    77     \snippet doc/src/snippets/code/src_network_access_FeatherWeightCache.cpp 2

    78 */

    79 

    80 /*!

    81     Creates a new disk cache. The \a parent argument is passed to

    82     QAbstractNetworkCache's constructor.

    83  */

    84 FeatherWeightCache::FeatherWeightCache(QObject *parent)

    85     : QAbstractNetworkCache(parent)

    86 {

    87 

    88     d = new FeatherWeightCachePrivate(this);

    89 }

    90 

    91 /*!

    92     Destroys the cache object.  This does not clear the disk cache.

    93  */

    94 FeatherWeightCache::~FeatherWeightCache()

    95 {

    96     QHashIterator<QIODevice*, CacheItem*> it(d->inserting);

    97     while (it.hasNext()) {

    98         it.next();

    99         delete it.value();

   100     }

   101 

   102 }

   103 

   104 /*!

   105     Returns the location where cached files will be stored.

   106 */

   107 QString FeatherWeightCache::cacheDirectory() const

   108 {

   109     return d->cacheDirectory;

   110 }

   111 

   112 /*!

   113     Sets the directory where cached files will be stored to \a cacheDir

   114 

   115     FeatherWeightCache will create this directory if it does not exists.

   116 

   117     Prepared cache items will be stored in the new cache directory when

   118     they are inserted.

   119 

   120     \sa QDesktopServices::CacheLocation

   121 */

   122 void FeatherWeightCache::setCacheDirectory(const QString &cacheDir)

   123 {

   124 #if defined(FEATHERWEIGHTCACHE_DEBUG)

   125     qDebug() << "FeatherWeightCache::setCacheDirectory()" << cacheDir;

   126 #endif

   127     if (cacheDir.isEmpty())

   128         return;

   129     d->cacheDirectory = cacheDir;

   130     QDir dir(d->cacheDirectory);

   131     d->cacheDirectory = dir.absolutePath();

   132     if (!d->cacheDirectory.endsWith(QLatin1Char('/')))

   133         d->cacheDirectory += QLatin1Char('/');

   134 

   135     d->prepareLayout();

   136 }

   137 

   138 /*!

   139     \reimp

   140 */

   141 qint64 FeatherWeightCache::cacheSize() const

   142 {

   143 #if defined(FEATHERWEIGHTCACHE_DEBUG)

   144     qDebug() << "FeatherWeightCache::cacheSize()";

   145 #endif

   146     if (d->cacheDirectory.isEmpty())

   147         return 0;

   148     if (d->currentCacheSize < 0) {

   149         FeatherWeightCache *that = const_cast<FeatherWeightCache*>(this);

   150         that->d->currentCacheSize = that->expire();

   151     }

   152     return d->currentCacheSize;

   153 }

   154 

   155 /*!

   156     \reimp

   157 */

   158 QIODevice *FeatherWeightCache::prepare(const QNetworkCacheMetaData &metaData)

   159 {

   160 #if defined(FEATHERWEIGHTCACHE_DEBUG)

   161     //qDebug() << "FeatherWeightCache::prepare()" << metaData.url();

   162 #endif

   163     if (!metaData.isValid() || !metaData.url().isValid() || !metaData.saveToDisk())

   164         return 0;

   165 

   166     if (d->cacheDirectory.isEmpty()) {

   167         qWarning() << "FeatherWeightCache::prepare() The cache directory is not set";

   168         return 0;

   169     }

   170 

   171     foreach (QNetworkCacheMetaData::RawHeader header, metaData.rawHeaders()) {

   172         if (header.first.toLower() == "content-length") {

   173             qint64 size = header.second.toInt();

   174             if (size > (maximumCacheSize() * 3)/4)

   175                 return 0;

   176             break;

   177         }

   178     }

   179     QScopedPointer<CacheItem> cacheItem(new CacheItem);

   180     cacheItem->metaData = metaData;

   181 

   182     QIODevice *device = 0;

   183     if (cacheItem->canCompress()) {

   184         cacheItem->data.open(QBuffer::ReadWrite);

   185         device = &(cacheItem->data);

   186     } else {

   187         QString templateName = d->tmpCacheFileName();

   188         QT_TRY {

   189             cacheItem->file = new QTemporaryFile(templateName, &cacheItem->data);

   190         } QT_CATCH(...) {

   191             cacheItem->file = 0;

   192         }

   193         if (!cacheItem->file || !cacheItem->file->open()) {

   194             qWarning() << "FeatherWeightCache::prepare() unable to open temporary file";

   195             cacheItem.reset();

   196             return 0;

   197         }

   198         cacheItem->writeHeader(cacheItem->file);

   199         device = cacheItem->file;

   200     }

   201     d->inserting[device] = cacheItem.take();

   202     return device;

   203 }

   204 

   205 /*!

   206     \reimp

   207 */

   208 void FeatherWeightCache::insert(QIODevice *device)

   209 {

   210 #if defined(FEATHERWEIGHTCACHE_DEBUG)

   211     //qDebug() << "FeatherWeightCache::insert()" << device;

   212 #endif

   213     QHash<QIODevice*, CacheItem*>::iterator it = d->inserting.find(device);

   214     if (it == d->inserting.end()) {

   215         qWarning() << "FeatherWeightCache::insert() called on a device we don't know about" << device;

   216         return;

   217     }

   218 

   219     d->storeItem(it.value());

   220     delete it.value();

   221     d->inserting.erase(it);

   222 }

   223 

   224 

   225 /*!

   226     Create subdirectories and other housekeeping on the filesystem.

   227     Prevents too many files from being present in any single directory.

   228 */

   229 void FeatherWeightCachePrivate::prepareLayout()

   230 {

   231     QDir prepared;

   232     prepared.mkpath(cacheDirectory + PREPARED_SLASH);

   233 

   234     QString path = cacheDirectory + DATA_SLASH;

   235     QDir dataDirectory(path);

   236 

   237     //Create directory and subdirectories 0-F

   238     dataDirectory.mkpath(path);

   239     for ( uint i = 0; i < 16 ; i++ ) {

   240         QString str = QString::number(i, 16);

   241         QString subdir = dataDirectory.path() + QDir::separator() + str;

   242         dataDirectory.mkdir(subdir);

   243     }

   244 

   245     // TODO: populate volumeInfo members here base on which disk/fileystem

   246     // you plan to write (a) temp ("prepared") files to (b) write final cache files too

   247     // volumeInfo->clusterSize = 1024;

   248     // volumeInfo->readBufSize = 16384;

   249     // volumeInfo->writeBufSize = 16384;

   250 #ifdef Q_OS_SYMBIAN

   251     //VolumeIOParam(TInt aDriveNo, TVolumeIOParamInfo &aParamInfo) const;

   252 #endif

   253 }

   254 

   255 // CRC32 implementation.

   256 // Could be made into new API QByteArray:qChecksum32()

   257 static const quint32 crc_tbl32[256] = {

   258     0x00000000, 0x77073096, 0xee0e612c, 0x990951ba,

   259     0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3,

   260     0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988,

   261     0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91,

   262     0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de,

   263     0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7,

   264     0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec,

   265     0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5,

   266     0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172,

   267     0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b,

   268     0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940,

   269     0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59,

   270     0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116,

   271     0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f,

   272     0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924,

   273     0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d,

   274     0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a,

   275     0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433,

   276     0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818,

   277     0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01,

   278     0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e,

   279     0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457,

   280     0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c,

   281     0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65,

   282     0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2,

   283     0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb,

   284     0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0,

   285     0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9,

   286     0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086,

   287     0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f,

   288     0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4,

   289     0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad,

   290     0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a,

   291     0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683,

   292     0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8,

   293     0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1,

   294     0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe,

   295     0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7,

   296     0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc,

   297     0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5,

   298     0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252,

   299     0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b,

   300     0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60,

   301     0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79,

   302     0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236,

   303     0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f,

   304     0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04,

   305     0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d,

   306     0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a,

   307     0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713,

   308     0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38,

   309     0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21,

   310     0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e,

   311     0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777,

   312     0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c,

   313     0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45,

   314     0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2,

   315     0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db,

   316     0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0,

   317     0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9,

   318     0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6,

   319     0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf,

   320     0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94,

   321     0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d

   322 };

   323 

   324 quint32 FeatherWeightCachePrivate::crc32(const char *data, uint len)

   325 {

   326     const uchar *p = reinterpret_cast<const uchar *>(data);

   327     const uchar *q = p + len;

   328     const quint32 init = 0xFFFFFFFFL;

   329 

   330     quint32 crc32 = init;

   331     while (p < q) {

   332         crc32 = (crc32 >> 8) ^ crc_tbl32[(crc32 ^ *p++) & 0xffL];

   333     }

   334     return crc32 ^ init ;

   335 }

   336 

   337 void FeatherWeightCachePrivate::storeItem(CacheItem *cacheItem)

   338 {

   339     Q_ASSERT(cacheItem->metaData.saveToDisk());

   340 

   341     QString fileName = cacheFileName(cacheItem->metaData.url());

   342     Q_ASSERT(!fileName.isEmpty());

   343 

   344 

   345     if (currentCacheSize > 0) {

   346         currentCacheSize += FILESYSTEMOVERHEAD + cacheItem->size();

   347     }

   348 

   349 

   350     //lut.insert( URL2HASH(cacheItem->metaData.url()), FILESYSTEMOVERHEAD + cacheItem->size() ) ;

   351 

   352     currentCacheSize = (reinterpret_cast<FeatherWeightCache *>(parent()))->expire();

   353 

   354 

   355 

   356     if (!cacheItem->file) {

   357         QString templateName = tmpCacheFileName();

   358         cacheItem->file = new QTemporaryFile(templateName, &cacheItem->data);

   359         if (cacheItem->file->open()) {

   360             cacheItem->writeHeader(cacheItem->file);

   361             cacheItem->writeCompressedData(cacheItem->file);

   362         }

   363     }

   364 

   365     if (cacheItem->file

   366         && cacheItem->file->isOpen()

   367         && cacheItem->file->error() == QFile::NoError) {

   368         cacheItem->file->setAutoRemove(false);

   369         // ### use atomic rename rather then remove & rename

   370         if (cacheItem->file->rename(fileName))

   371             currentCacheSize += cacheItem->file->size();

   372         else {

   373             // Presume that the destination file exists and/or is open. So try nuking.

   374             bool err1 = QFile::remove(fileName);

   375             Q_UNUSED(err1);

   376             bool err2 = cacheItem->file->rename(fileName);

   377             // You are hopeless. Don't persist

   378             if (!err2)  {

   379                 cacheItem->file->setAutoRemove(true);

   380 #if defined(FEATHERWEIGHTCACHE_DEBUG)

   381                 qWarning() << "FeatherWeightCache: couldn't replace the cache file " << fileName;

   382 #endif

   383             }

   384         }

   385     }

   386     if (cacheItem->metaData.url() == lastItem.metaData.url())

   387         lastItem.reset();

   388 }

   389 

   390 /*!

   391     \reimp

   392 */

   393 bool FeatherWeightCache::remove(const QUrl &url)

   394 {

   395 #if defined(FEATHERWEIGHTCACHE_DEBUG)

   396     //qDebug() << "FeatherWeightCache::remove()" << url;

   397 #endif

   398 

   399     // remove is also used to cancel insertions, not a common operation

   400     QHashIterator<QIODevice*, CacheItem*> it(d->inserting);

   401     while (it.hasNext()) {

   402         it.next();

   403         CacheItem *item = it.value();

   404         if (item && item->metaData.url() == url) {

   405             delete item;

   406             d->inserting.remove(it.key());

   407             return true;

   408         }

   409     }

   410 

   411     if (d->lastItem.metaData.url() == url)

   412         d->lastItem.reset();

   413     return d->removeFile(d->cacheFileName(url));

   414 }

   415 

   416 /*!

   417     Put all of the misc file removing into one function to be extra safe

   418  */

   419 bool FeatherWeightCachePrivate::removeFile(const QString &file)

   420 {

   421 #if defined(FEATHERWEIGHTCACHE_DEBUG)

   422     //qDebug() << "FeatherWeightCache::removFile()" << file;

   423 #endif

   424     if (file.isEmpty())

   425         return false;

   426     QFileInfo info(file);

   427     QString fileName = info.fileName();

   428     if (!fileName.endsWith(CACHE_POSTFIX))

   429         return false;

   430     qint64 size = info.size();

   431     if (QFile::remove(file)) {

   432         currentCacheSize -= size;

   433         return true;

   434     }

   435     return false;

   436 }

   437 

   438 /*!

   439     Use signal from worker thread to update disk usage awareness

   440  */

   441 void FeatherWeightCachePrivate::updateCacheSize(qint64 newSize)

   442 {

   443     currentCacheSize = newSize;

   444 #if defined(FEATHERWEIGHTCACHE_DEBUG)

   445     qDebug() << "FeatherWeightCachePrivate::updateCacheSize " << " new size " << currentCacheSize;

   446 #endif

   447 }

   448 

   449 /*!

   450     \reimp

   451 */

   452 QNetworkCacheMetaData FeatherWeightCache::metaData(const QUrl &url)

   453 {

   454 #if defined(FEATHERWEIGHTCACHE_DEBUG)

   455     //qDebug() << "FeatherWeightCache::metaData()" << url;

   456 #endif

   457     if (d->lastItem.metaData.url() == url)

   458         return d->lastItem.metaData;

   459     return fileMetaData(d->cacheFileName(url));

   460 }

   461 

   462 /*!

   463     Returns the QNetworkCacheMetaData for the cache file \a fileName.

   464 

   465     If \a fileName is not a cache file QNetworkCacheMetaData will be invalid.

   466  */

   467 QNetworkCacheMetaData FeatherWeightCache::fileMetaData(const QString &fileName) const

   468 {

   469 #if defined(FEATHERWEIGHTCACHE_DEBUG)

   470     //qDebug() << "FeatherWeightCache::fileMetaData()" << fileName;

   471 #endif

   472     QFile file(fileName);

   473     if (!file.open(QFile::ReadOnly))

   474         return QNetworkCacheMetaData();

   475     if (!d->lastItem.read(&file, false)) {

   476         file.close();

   477         FeatherWeightCachePrivate *that = const_cast<FeatherWeightCachePrivate*>(d);

   478         that->removeFile(fileName);

   479     }

   480     return d->lastItem.metaData;

   481 }

   482 

   483 /*!

   484     \reimp

   485 */

   486 QIODevice *FeatherWeightCache::data(const QUrl &url)

   487 {

   488 #if defined(FEATHERWEIGHTCACHE_DEBUG)

   489     //qDebug() << "FeatherWeightCache::data()" << url;

   490 #endif

   491 

   492     QScopedPointer<QBuffer> buffer;

   493     if (!url.isValid())

   494         return 0;

   495     if (d->lastItem.metaData.url() == url && d->lastItem.data.isOpen()) {

   496         buffer.reset(new QBuffer);

   497         buffer->setData(d->lastItem.data.data());

   498     } else {

   499         QScopedPointer<QFile> file(new QFile(d->cacheFileName(url)));

   500         if (!file->open(QFile::ReadOnly | QIODevice::Unbuffered))

   501             return 0;

   502 

   503         if (!d->lastItem.read(file.data(), true)) {

   504             file->close();

   505             remove(url);

   506             return 0;

   507         }

   508         if (d->lastItem.data.isOpen()) {

   509             // compressed

   510             buffer.reset(new QBuffer);

   511             buffer->setData(d->lastItem.data.data());

   512         } else {

   513             buffer.reset(new QBuffer);

   514             // ### verify that QFile uses the fd size and not the file name

   515             qint64 size = file->size() - file->pos();

   516             const uchar *p = 0;

   517 #ifndef Q_OS_WINCE

   518             p = file->map(file->pos(), size);

   519 #endif

   520             if (p) {

   521                 buffer->setData((const char *)p, size);

   522                 file.take()->setParent(buffer.data());

   523             } else {

   524                 buffer->setData(file->readAll());

   525             }

   526         }

   527     }

   528     buffer->open(QBuffer::ReadOnly);

   529     return buffer.take();

   530 }

   531 

   532 /*!

   533     \reimp

   534 */

   535 void FeatherWeightCache::updateMetaData(const QNetworkCacheMetaData &metaData)

   536 {

   537 #if defined(FEATHERWEIGHTCACHE_DEBUG)

   538     qDebug() << "FeatherWeightCache::updateMetaData()" << metaData.url();

   539 #endif

   540     QUrl url = metaData.url();

   541     QIODevice *oldDevice = data(url);

   542     if (!oldDevice) {

   543 #if defined(FEATHERWEIGHTCACHE_DEBUG)

   544         qDebug() << "FeatherWeightCache::updateMetaData(), no device!";

   545 #endif

   546         return;

   547     }

   548 

   549     QIODevice *newDevice = prepare(metaData);

   550     if (!newDevice) {

   551 #if defined(FEATHERWEIGHTCACHE_DEBUG)

   552         qDebug() << "FeatherWeightCache::updateMetaData(), no new device!" << url;

   553 #endif

   554         return;

   555     }

   556     //TODO: Optimize this somehow?

   557     char data[1024];

   558     while (!oldDevice->atEnd()) {

   559         qint64 s = oldDevice->read(data, 1024);

   560         newDevice->write(data, s);

   561     }

   562     delete oldDevice;

   563     insert(newDevice);

   564 }

   565 

   566 /*!

   567     Returns the current maximum size for the disk cache.

   568 

   569     \sa setMaximumCacheSize()

   570  */

   571 qint64 FeatherWeightCache::maximumCacheSize() const

   572 {

   573     return d->maximumCacheSize;

   574 }

   575 

   576 /*!

   577     Sets the maximum size of the disk cache to be \a size.

   578 

   579     If the new size is smaller then the current cache size then the cache will call expire().

   580 

   581     \sa maximumCacheSize()

   582  */

   583 void FeatherWeightCache::setMaximumCacheSize(qint64 size)

   584 {

   585 

   586     bool expireCache = (size < d->maximumCacheSize);

   587     d->maximumCacheSize = size;

   588     if (expireCache)

   589         d->currentCacheSize = expire();

   590 }

   591 

   592 /*!

   593     Cleans the cache so that its size is under the maximum cache size.

   594     Returns the current size of the cache.

   595 

   596     When the current size of the cache is greater than the maximumCacheSize()

   597     older cache files are removed until the total size is less then 90% of

   598     maximumCacheSize() starting with the oldest ones first using the file

   599     creation date to determine how old a cache file is.

   600 

   601     Subclasses can reimplement this function to change the order that cache

   602     files are removed taking into account information in the application

   603     knows about that FeatherWeightCache does not, for example the number of times

   604     a cache is accessed.

   605 

   606     Note: cacheSize() calls expire if the current cache size is unknown.

   607 

   608     \sa maximumCacheSize(), fileMetaData()

   609  */

   610 qint64 FeatherWeightCache::expire()

   611 {

   612 

   613     if (d->currentCacheSize >= 0 && d->currentCacheSize < maximumCacheSize())

   614         return d->currentCacheSize;

   615 

   616     if (cacheDirectory().isEmpty()) {

   617         qWarning() << "FeatherWeightCache::expire() The cache directory is not set";

   618         return 0;

   619     }

   620 

   621 #if defined(FEATHERWEIGHTCACHE_DEBUG)

   622     qDebug() << "Calling expire, size = " << d->currentCacheSize << " , max = " << maximumCacheSize() ;

   623 #endif

   624     return d->expire();

   625 }

   626 

   627 /*!

   628     \reimp

   629 */

   630 void FeatherWeightCache::clear()

   631 {

   632 #if defined(FEATHERWEIGHTCACHE_DEBUG)

   633     qDebug() << "FeatherWeightCache::clear()";

   634 #endif

   635 

   636     qint64 size = d->maximumCacheSize;

   637     d->maximumCacheSize = 0;

   638     d->currentCacheSize = expire();

   639     d->maximumCacheSize = size;

   640 }

   641 

   642 qint64 FeatherWeightCachePrivate::expire()

   643 {

   644 

   645     // ASYNC expiration via background thread

   646     beastOfBurden.expireLazily(cacheDirectory, maximumCacheSize);

   647 

   648     // Note: this cache size will not/cannot reflect the reduced

   649     // cache size due to the async nature of the expire() above.

   650     return currentCacheSize;

   651 }

   652 

   653 QByteArray FeatherWeightCachePrivate::generateId(const QUrl &url)

   654 {

   655     QUrl cleanUrl = url;

   656     cleanUrl.setPassword(QString());

   657     cleanUrl.setFragment(QString());

   658     QByteArray blob = cleanUrl.toEncoded();

   659 

   660     QByteArray hash;

   661     hash.setNum(crc32(blob.data(), blob.length()), 16);

   662     return hash;

   663 }

   664 

   665 QString FeatherWeightCachePrivate::tmpCacheFileName() const

   666 {

   667     //The subdirectory is presumed to be already read for use.

   668     return cacheDirectory + PREPARED_SLASH + QLatin1String("XXXXXX") + CACHE_POSTFIX;

   669 }

   670 

   671 /*!

   672     Genrates fully qualified path of cached resource from a URL.

   673  */

   674 QString FeatherWeightCachePrivate::cacheFileName(const QUrl &url) const

   675 {

   676     if (!url.isValid())

   677         return QString();

   678 

   679     // map URL to a unique enough signature

   680     const QByteArray unique(generateId(url));

   681 

   682     // generates <cache dir>/data/e/cache_beefcafe.cache

   683     // where 'e' is the last character of a hex string

   684     QString fullpath = cacheDirectory + DATA_SLASH

   685                        + QLatin1Char(unique.at(unique.length()-1)) + QLatin1String("/")

   686                        + QLatin1String(unique) + CACHE_POSTFIX;

   687 

   688     return  fullpath;

   689 }

   690 

   691 

   692 /* Important: This c'tor runs in the same thread as main cache */

   693 WorkerThread::WorkerThread()

   694 {

   695     abort = false;

   696 }

   697 

   698 /* Important: This d'tor runs in the same thread as main cache */

   699 WorkerThread::~WorkerThread()

   700 {

   701     // The destructor can be called at any point while the thread is active.

   702     // So we set abort to true to tell run() to stop running as soon as possible.

   703     mutex.lock();

   704     abort = true;

   705     condition.wakeOne(); // wake up thread if it has nothing to do

   706     mutex.unlock();

   707 

   708     wait(); // waits for run() to return

   709 }

   710 

   711 /* Important: This method runs in its own thread, unlike the c'tor and d'tor */

   712 void WorkerThread::run()

   713 {

   714 

   715 #if defined(Q_OS_SYMBIAN)

   716     // Remove this once QTBUG-10271 is fixed

   717     RThread myThread;

   718     myThread.SetPriority(EPriorityLess);

   719 #endif

   720 

   721     qint64 size = expireImpl();

   722     emit onDiskSizeChanged(size);

   723 

   724 #if defined(FEATHERWEIGHTCACHE_DEBUG)

   725     qDebug() << "New on-disk cache size: " << size <<  QThread::currentThreadId();

   726 #endif

   727 

   728 

   729 }

   730 

   731 qint64 WorkerThread::expireImpl()

   732 {

   733     QDir::Filters filters = QDir::AllDirs | QDir:: Files | QDir::NoDotAndDotDot;

   734     QDirIterator it(this->cacheDir, filters, QDirIterator::Subdirectories);

   735 

   736     QMultiMap<QDateTime, QString> cacheItems;

   737     qint64 totalSize = 0;

   738     while (it.hasNext()) {

   739         QString path = it.next();

   740         QFileInfo info = it.fileInfo();

   741         QString fileName = info.fileName();

   742         if (fileName.endsWith(CACHE_POSTFIX)) {

   743             cacheItems.insert(info.created(), path);

   744             totalSize += info.size();

   745         }

   746 

   747         // Interrupts this slow loop when d'tor is called

   748         if (abort) {

   749             // potentially incorrect, but can't do any better

   750             return totalSize;

   751         }

   752     }

   753 

   754     int removedFiles = 0;

   755     // this goal setting could be made smarter based on max cache size

   756     // e.g on desktop with large 50MB caches, freeing 10% is probably enough

   757     // but on mobile where caches are smaller (e.g 5MB) and disks are slow, you want

   758     // to free atleast 0.5-1MB if going through all this trouble.

   759     // Also TODO: Move to LRU algorithm

   760     qint64 goal = (this->maxCacheSize * 8) / 10;

   761     QMultiMap<QDateTime, QString>::const_iterator i = cacheItems.constBegin();

   762     while (i != cacheItems.constEnd()) {

   763         if (totalSize < goal)

   764             break;

   765         QString name = i.value();

   766         QFile file(name);

   767         qint64 size = file.size();

   768         file.remove();

   769         totalSize -= size;

   770         ++removedFiles;

   771         ++i;

   772 

   773         // Interrupts this slow loop when d'tor is called

   774         if (abort) {

   775             // potentially incorrect, but can't do any better

   776             return totalSize;

   777         }

   778 

   779     }

   780 #if defined(FEATHERWEIGHTCACHE_DEBUG)

   781     if (removedFiles > 0) {

   782         qDebug() << "FeatherWeightCache::expire()"

   783                 << "Removed:" << removedFiles

   784                 << "Kept:" << cacheItems.count() - removedFiles;

   785     }

   786 #endif

   787 

   788     //TODO: Why do we do this in the original

   789     //implementation? It isn't necessary that

   790     //running expiration logics caused last

   791     //insertion to become invalid?

   792     //if (removedFiles > 0)

   793     //    lastItem.reset();

   794 

   795     return totalSize;

   796 

   797 }

   798 

   799 /* Important: this function runs in the same thread as main cache */

   800 void WorkerThread::expireLazily(QString cacheDir, qint64 maxCacheSize)

   801 {

   802 

   803     //lock mutex. unlock automatically when locker goes out of scope

   804     QMutexLocker locker(&mutex);

   805 

   806     //make private copy so that other member functions can use this

   807     this->cacheDir = cacheDir;

   808     this->maxCacheSize = maxCacheSize;

   809 

   810     if (!isRunning()) {

   811 

   812 #if defined(FEATHERWEIGHTCACHE_DEBUG)

   813         qDebug() << "Starting worker thread a low priority";

   814 #endif

   815 

   816         start(LowPriority);

   817 

   818     } else {

   819 

   820 #if defined(FEATHERWEIGHTCACHE_DEBUG)

   821         qDebug() << "Waking up sleeping worker thread";

   822 #endif

   823 

   824         condition.wakeOne();

   825 

   826     }

   827 

   828 }

   829 

   830 

   831 /*!

   832     We compress small text and JavaScript files.

   833  */

   834 bool CacheItem::canCompress() const

   835 {

   836 #if 1

   837     bool sizeOk = false;

   838     bool typeOk = false;

   839     foreach (QNetworkCacheMetaData::RawHeader header, metaData.rawHeaders()) {

   840         if (header.first.toLower() == "content-length") {

   841             qint64 size = header.second.toLongLong();

   842             if (size > MAX_COMPRESSION_SIZE)

   843                 return false;

   844             else

   845                 sizeOk = true;

   846         }

   847 

   848         if (header.first.toLower() == "content-type") {

   849             QByteArray type = header.second;

   850             if (type.startsWith("text/")

   851                     || (type.startsWith("application/")

   852                         && (type.endsWith("javascript") || type.endsWith("ecmascript"))))

   853                 typeOk = true;

   854             else

   855                 return false;

   856         }

   857         if (sizeOk && typeOk)

   858             return true;

   859     }

   860     return false;

   861 #else

   862     return false;

   863 #endif

   864 

   865 }

   866 

   867 enum

   868 {

   869     CacheMagic = 0xe8,

   870     CurrentCacheVersion = 7

   871 };

   872 

   873 void CacheItem::writeHeader(QFile *device) const

   874 {

   875     QDataStream out(device);

   876 

   877     out << qint32(CacheMagic);

   878     out << qint32(CurrentCacheVersion);

   879     out << metaData;

   880     bool compressed = canCompress();

   881     out << compressed;

   882 }

   883 

   884 void CacheItem::writeCompressedData(QFile *device) const

   885 {

   886     QDataStream out(device);

   887 

   888     out << qCompress(data.data());

   889 }

   890 

   891 /*!

   892     Returns false if the file is a cache file,

   893     but is an older version and should be removed otherwise true.

   894  */

   895 bool CacheItem::read(QFile *device, bool readData)

   896 {

   897     reset();

   898 

   899     QDataStream in(device);

   900 

   901     qint32 marker;

   902     qint32 v;

   903     in >> marker;

   904     in >> v;

   905     if (marker != CacheMagic)

   906         return true;

   907 

   908     // If the cache magic is correct, but the version is not we should remove it

   909     if (v != CurrentCacheVersion)

   910         return false;

   911 

   912     bool compressed;

   913     QByteArray dataBA;

   914     in >> metaData;

   915     in >> compressed;

   916     if (readData && compressed) {

   917         in >> dataBA;

   918         data.setData(qUncompress(dataBA));

   919         data.open(QBuffer::ReadOnly);

   920     }

   921     return metaData.isValid();

   922 }

   923 

   924 } // namespace WRT