/*

** 2008 August 05

**

** The author disclaims copyright to this source code.  In place of

** a legal notice, here is a blessing:

**

**    May you do good and not evil.

**    May you find forgiveness for yourself and forgive others.

**    May you share freely, never taking more than you give.

**

*************************************************************************

** This file implements that page cache.

**

** @(#) $Id: pcache.c,v 1.33 2008/09/29 11:49:48 danielk1977 Exp $

*/

#include "sqliteInt.h"

/*

** A complete page cache is an instance of this structure.

**

** A cache may only be deleted by its owner and while holding the

** SQLITE_MUTEX_STATUS_LRU mutex.

*/

struct PCache {

  /*********************************************************************

  ** The first group of elements may be read or written at any time by

  ** the cache owner without holding the mutex.  No thread other than the

  ** cache owner is permitted to access these elements at any time.

  */

  PgHdr *pDirty, *pDirtyTail;         /* List of dirty pages in LRU order */

  PgHdr *pSynced;                     /* Last synced page in dirty page list */

  int nRef;                           /* Number of pinned pages */

  int nPinned;                        /* Number of pinned and/or dirty pages */

  int nMax;                           /* Configured cache size */

  int nMin;                           /* Configured minimum cache size */

  /**********************************************************************

  ** The next group of elements are fixed when the cache is created and

  ** may not be changed afterwards.  These elements can read at any time by

  ** the cache owner or by any thread holding the the mutex.  Non-owner

  ** threads must hold the mutex when reading these elements to prevent

  ** the entire PCache object from being deleted during the read.

  */

  int szPage;                         /* Size of every page in this cache */

  int szExtra;                        /* Size of extra space for each page */

  int bPurgeable;                     /* True if pages are on backing store */

  int (*xStress)(void*,PgHdr*);       /* Call to try make a page clean */

  void *pStress;                      /* Argument to xStress */

  /**********************************************************************

  ** The final group of elements can only be accessed while holding the

  ** mutex.  Both the cache owner and any other thread must hold the mutex

  ** to read or write any of these elements.

  */

  int nPage;                          /* Total number of pages in apHash */

  int nHash;                          /* Number of slots in apHash[] */

  PgHdr **apHash;                     /* Hash table for fast lookup by pgno */

  PgHdr *pClean;                      /* List of clean pages in use */

};

/*

** Free slots in the page block allocator

*/

typedef struct PgFreeslot PgFreeslot;

struct PgFreeslot {

  PgFreeslot *pNext;  /* Next free slot */

};

/*

** Global data for the page cache.

*/

static SQLITE_WSD struct PCacheGlobal {

  int isInit;                         /* True when initialized */

  sqlite3_mutex *mutex;               /* static mutex MUTEX_STATIC_LRU */

  int nMaxPage;                       /* Sum of nMaxPage for purgeable caches */

  int nMinPage;                       /* Sum of nMinPage for purgeable caches */

  int nCurrentPage;                   /* Number of purgeable pages allocated */

  PgHdr *pLruHead, *pLruTail;         /* LRU list of unused clean pgs */

  /* Variables related to SQLITE_CONFIG_PAGECACHE settings. */

  int szSlot;                         /* Size of each free slot */

  void *pStart, *pEnd;                /* Bounds of pagecache malloc range */

  PgFreeslot *pFree;                  /* Free page blocks */

} pcache = {0};

/*

** All code in this file should access the global pcache structure via the

** alias "pcache_g". This ensures that the WSD emulation is used when

** compiling for systems that do not support real WSD.

*/

#define pcache_g (GLOBAL(struct PCacheGlobal, pcache))

/*

** All global variables used by this module (all of which are grouped 

** together in global structure "pcache" above) are protected by the static 

** SQLITE_MUTEX_STATIC_LRU mutex. A pointer to this mutex is stored in

** variable "pcache.mutex".

**

** Some elements of the PCache and PgHdr structures are protected by the 

** SQLITE_MUTEX_STATUS_LRU mutex and other are not.  The protected

** elements are grouped at the end of the structures and are clearly

** marked.

**

** Use the following macros must surround all access (read or write)

** of protected elements.  The mutex is not recursive and may not be

** entered more than once.  The pcacheMutexHeld() macro should only be

** used within an assert() to verify that the mutex is being held.

*/

#define pcacheEnterMutex() sqlite3_mutex_enter(pcache_g.mutex)

#define pcacheExitMutex()  sqlite3_mutex_leave(pcache_g.mutex)

#define pcacheMutexHeld()  sqlite3_mutex_held(pcache_g.mutex)

/*

** Some of the assert() macros in this code are too expensive to run

** even during normal debugging.  Use them only rarely on long-running

** tests.  Enable the expensive asserts using the

** -DSQLITE_ENABLE_EXPENSIVE_ASSERT=1 compile-time option.

*/

#ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT

# define expensive_assert(X)  assert(X)

#else

# define expensive_assert(X)

#endif

/********************************** Linked List Management ********************/

#if !defined(NDEBUG) && defined(SQLITE_ENABLE_EXPENSIVE_ASSERT)

/*

** This routine verifies that the number of entries in the hash table

** is pCache->nPage.  This routine is used within assert() statements

** only and is therefore disabled during production builds.

*/

static int pcacheCheckHashCount(PCache *pCache){

  int i;

  int nPage = 0;

  for(i=0; i<pCache->nHash; i++){

    PgHdr *p;

    for(p=pCache->apHash[i]; p; p=p->pNextHash){

      nPage++;

    }

  }

  assert( nPage==pCache->nPage );

  return 1;

}

#endif /* !NDEBUG && SQLITE_ENABLE_EXPENSIVE_ASSERT */

#if !defined(NDEBUG) && defined(SQLITE_ENABLE_EXPENSIVE_ASSERT)

/*

** Based on the current value of PCache.nRef and the contents of the

** PCache.pDirty list, return the expected value of the PCache.nPinned

** counter. This is only used in debugging builds, as follows:

**

**   expensive_assert( pCache->nPinned==pcachePinnedCount(pCache) );

*/

static int pcachePinnedCount(PCache *pCache){

  PgHdr *p;

  int nPinned = pCache->nRef;

  for(p=pCache->pDirty; p; p=p->pNext){

    if( p->nRef==0 ){

      nPinned++;

    }

  }

  return nPinned;

}

#endif /* !NDEBUG && SQLITE_ENABLE_EXPENSIVE_ASSERT */

#if !defined(NDEBUG) && defined(SQLITE_ENABLE_EXPENSIVE_ASSERT)

/*

** Check that the pCache->pSynced variable is set correctly. If it

** is not, either fail an assert or return zero. Otherwise, return

** non-zero. This is only used in debugging builds, as follows:

**

**   expensive_assert( pcacheCheckSynced(pCache) );

*/

static int pcacheCheckSynced(PCache *pCache){

  PgHdr *p = pCache->pDirtyTail;

  for(p=pCache->pDirtyTail; p!=pCache->pSynced; p=p->pPrev){

    assert( p->nRef || (p->flags&PGHDR_NEED_SYNC) );

  }

  return (p==0 || p->nRef || (p->flags&PGHDR_NEED_SYNC)==0);

}

#endif /* !NDEBUG && SQLITE_ENABLE_EXPENSIVE_ASSERT */

/*

** Remove a page from its hash table (PCache.apHash[]).

*/

static void pcacheRemoveFromHash(PgHdr *pPage){

  assert( pcacheMutexHeld() );

  if( pPage->pPrevHash ){

    pPage->pPrevHash->pNextHash = pPage->pNextHash;

  }else{

    PCache *pCache = pPage->pCache;

    u32 h = pPage->pgno % pCache->nHash;

    assert( pCache->apHash[h]==pPage );

    pCache->apHash[h] = pPage->pNextHash;

  }

  if( pPage->pNextHash ){

    pPage->pNextHash->pPrevHash = pPage->pPrevHash;

  }

  pPage->pCache->nPage--;

  expensive_assert( pcacheCheckHashCount(pPage->pCache) );

}

/*

** Insert a page into the hash table

**

** The mutex must be held by the caller.

*/

static void pcacheAddToHash(PgHdr *pPage){

  PCache *pCache = pPage->pCache;

  u32 h = pPage->pgno % pCache->nHash;

  assert( pcacheMutexHeld() );

  pPage->pNextHash = pCache->apHash[h];

  pPage->pPrevHash = 0;

  if( pCache->apHash[h] ){

    pCache->apHash[h]->pPrevHash = pPage;

  }

  pCache->apHash[h] = pPage;

  pCache->nPage++;

  expensive_assert( pcacheCheckHashCount(pCache) );

}

/*

** Attempt to increase the size the hash table to contain

** at least nHash buckets.

*/

static int pcacheResizeHash(PCache *pCache, int nHash){

  PgHdr *p;

  PgHdr **pNew;

  assert( pcacheMutexHeld() );

#ifdef SQLITE_MALLOC_SOFT_LIMIT

  if( nHash*sizeof(PgHdr*)>SQLITE_MALLOC_SOFT_LIMIT ){

    nHash = SQLITE_MALLOC_SOFT_LIMIT/sizeof(PgHdr *);

  }

#endif

  pcacheExitMutex();

  pNew = (PgHdr **)sqlite3Malloc(sizeof(PgHdr*)*nHash);

  pcacheEnterMutex();

  if( !pNew ){

    return SQLITE_NOMEM;

  }

  memset(pNew, 0, sizeof(PgHdr *)*nHash);

  sqlite3_free(pCache->apHash);

  pCache->apHash = pNew;

  pCache->nHash = nHash;

  pCache->nPage = 0;

  for(p=pCache->pClean; p; p=p->pNext){

    pcacheAddToHash(p);

  }

  for(p=pCache->pDirty; p; p=p->pNext){

    pcacheAddToHash(p);

  }

  return SQLITE_OK;

}

/*

** Remove a page from a linked list that is headed by *ppHead.

** *ppHead is either PCache.pClean or PCache.pDirty.

*/

static void pcacheRemoveFromList(PgHdr **ppHead, PgHdr *pPage){

  int isDirtyList = (ppHead==&pPage->pCache->pDirty);

  assert( ppHead==&pPage->pCache->pClean || ppHead==&pPage->pCache->pDirty );

  assert( pcacheMutexHeld() || ppHead!=&pPage->pCache->pClean );

  if( pPage->pPrev ){

    pPage->pPrev->pNext = pPage->pNext;

  }else{

    assert( *ppHead==pPage );

    *ppHead = pPage->pNext;

  }

  if( pPage->pNext ){

    pPage->pNext->pPrev = pPage->pPrev;

  }

  if( isDirtyList ){

    PCache *pCache = pPage->pCache;

    assert( pPage->pNext || pCache->pDirtyTail==pPage );

    if( !pPage->pNext ){

      pCache->pDirtyTail = pPage->pPrev;

    }

    if( pCache->pSynced==pPage ){

      PgHdr *pSynced = pPage->pPrev;

      while( pSynced && (pSynced->flags&PGHDR_NEED_SYNC) ){

        pSynced = pSynced->pPrev;

      }

      pCache->pSynced = pSynced;

    }

  }

}

/*

** Add a page from a linked list that is headed by *ppHead.

** *ppHead is either PCache.pClean or PCache.pDirty.

*/

static void pcacheAddToList(PgHdr **ppHead, PgHdr *pPage){

  int isDirtyList = (ppHead==&pPage->pCache->pDirty);

  assert( ppHead==&pPage->pCache->pClean || ppHead==&pPage->pCache->pDirty );

  if( (*ppHead) ){

    (*ppHead)->pPrev = pPage;

  }

  pPage->pNext = *ppHead;

  pPage->pPrev = 0;

  *ppHead = pPage;

  if( isDirtyList ){

    PCache *pCache = pPage->pCache;

    if( !pCache->pDirtyTail ){

      assert( pPage->pNext==0 );

      pCache->pDirtyTail = pPage;

    }

    if( !pCache->pSynced && 0==(pPage->flags&PGHDR_NEED_SYNC) ){

      pCache->pSynced = pPage;

    }

  }

}

/*

** Remove a page from the global LRU list

*/

static void pcacheRemoveFromLruList(PgHdr *pPage){

  assert( sqlite3_mutex_held(pcache_g.mutex) );

  assert( (pPage->flags&PGHDR_DIRTY)==0 );

  if( pPage->pCache->bPurgeable==0 ) return;

  if( pPage->pNextLru ){

    assert( pcache_g.pLruTail!=pPage );

    pPage->pNextLru->pPrevLru = pPage->pPrevLru;

  }else{

    assert( pcache_g.pLruTail==pPage );

    pcache_g.pLruTail = pPage->pPrevLru;

  }

  if( pPage->pPrevLru ){

    assert( pcache_g.pLruHead!=pPage );

    pPage->pPrevLru->pNextLru = pPage->pNextLru;

  }else{

    assert( pcache_g.pLruHead==pPage );

    pcache_g.pLruHead = pPage->pNextLru;

  }

}

/*

** Add a page to the global LRU list.  The page is normally added

** to the front of the list so that it will be the last page recycled.

** However, if the PGHDR_REUSE_UNLIKELY bit is set, the page is added

** to the end of the LRU list so that it will be the next to be recycled.

*/

static void pcacheAddToLruList(PgHdr *pPage){

  assert( sqlite3_mutex_held(pcache_g.mutex) );

  assert( (pPage->flags&PGHDR_DIRTY)==0 );

  if( pPage->pCache->bPurgeable==0 ) return;

  if( pcache_g.pLruTail && (pPage->flags & PGHDR_REUSE_UNLIKELY)!=0 ){

    /* If reuse is unlikely.  Put the page at the end of the LRU list

    ** where it will be recycled sooner rather than later. 

    */

    assert( pcache_g.pLruHead );

    pPage->pNextLru = 0;

    pPage->pPrevLru = pcache_g.pLruTail;

    pcache_g.pLruTail->pNextLru = pPage;

    pcache_g.pLruTail = pPage;

    pPage->flags &= ~PGHDR_REUSE_UNLIKELY;

  }else{

    /* If reuse is possible. the page goes at the beginning of the LRU

    ** list so that it will be the last to be recycled.

    */

    if( pcache_g.pLruHead ){

      pcache_g.pLruHead->pPrevLru = pPage;

    }

    pPage->pNextLru = pcache_g.pLruHead;

    pcache_g.pLruHead = pPage;

    pPage->pPrevLru = 0;

    if( pcache_g.pLruTail==0 ){

      pcache_g.pLruTail = pPage;

    }

  }

}

/*********************************************** Memory Allocation ***********

**

** Initialize the page cache memory pool.

**

** This must be called at start-time when no page cache lines are

** checked out. This function is not threadsafe.

*/

void sqlite3PCacheBufferSetup(void *pBuf, int sz, int n){

  PgFreeslot *p;

  sz &= ~7;

  pcache_g.szSlot = sz;

  pcache_g.pStart = pBuf;

  pcache_g.pFree = 0;

  while( n-- ){

    p = (PgFreeslot*)pBuf;

    p->pNext = pcache_g.pFree;

    pcache_g.pFree = p;

    pBuf = (void*)&((char*)pBuf)[sz];

  }

  pcache_g.pEnd = pBuf;

}

/*

** Allocate a page cache line.  Look in the page cache memory pool first

** and use an element from it first if available.  If nothing is available

** in the page cache memory pool, go to the general purpose memory allocator.

*/

static void *pcacheMalloc(int sz, PCache *pCache){

  assert( sqlite3_mutex_held(pcache_g.mutex) );

  if( sz<=pcache_g.szSlot && pcache_g.pFree ){

    PgFreeslot *p = pcache_g.pFree;

    pcache_g.pFree = p->pNext;

    sqlite3StatusSet(SQLITE_STATUS_PAGECACHE_SIZE, sz);

    sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_USED, 1);

    return (void*)p;

  }else{

    void *p;

    /* Allocate a new buffer using sqlite3Malloc. Before doing so, exit the

    ** global pcache mutex and unlock the pager-cache object pCache. This is 

    ** so that if the attempt to allocate a new buffer causes the the 

    ** configured soft-heap-limit to be breached, it will be possible to

    ** reclaim memory from this pager-cache.

    */

    pcacheExitMutex();

    p = sqlite3Malloc(sz);

    pcacheEnterMutex();

    if( p ){

      sz = sqlite3MallocSize(p);

      sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_OVERFLOW, sz);

    }

    return p;

  }

}

void *sqlite3PageMalloc(int sz){

  void *p;

  pcacheEnterMutex();

  p = pcacheMalloc(sz, 0);

  pcacheExitMutex();

  return p;

}

/*

** Release a pager memory allocation

*/

static void pcacheFree(void *p){

  assert( sqlite3_mutex_held(pcache_g.mutex) );

  if( p==0 ) return;

  if( p>=pcache_g.pStart && p<pcache_g.pEnd ){

    PgFreeslot *pSlot;

    sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_USED, -1);

    pSlot = (PgFreeslot*)p;

    pSlot->pNext = pcache_g.pFree;

    pcache_g.pFree = pSlot;

  }else{

    int iSize = sqlite3MallocSize(p);

    sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_OVERFLOW, -iSize);

    sqlite3_free(p);

  }

}

void sqlite3PageFree(void *p){

  pcacheEnterMutex();

  pcacheFree(p);

  pcacheExitMutex();

}

/*

** Allocate a new page.

*/

static PgHdr *pcachePageAlloc(PCache *pCache){

  PgHdr *p;

  int sz = sizeof(*p) + pCache->szPage + pCache->szExtra;

  assert( sqlite3_mutex_held(pcache_g.mutex) );

  p = pcacheMalloc(sz, pCache);

  if( p==0 ) return 0;

  memset(p, 0, sizeof(PgHdr));

  p->pData = (void*)&p[1];

  p->pExtra = (void*)&((char*)p->pData)[pCache->szPage];

  if( pCache->bPurgeable ){

    pcache_g.nCurrentPage++;

  }

  return p;

}

/*

** Deallocate a page

*/

static void pcachePageFree(PgHdr *p){

  assert( sqlite3_mutex_held(pcache_g.mutex) );

  if( p->pCache->bPurgeable ){

    pcache_g.nCurrentPage--;

  }

  pcacheFree(p->apSave[0]);

  pcacheFree(p->apSave[1]);

  pcacheFree(p);

}

#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT

/*

** Return the number of bytes that will be returned to the heap when

** the argument is passed to pcachePageFree().

*/

static int pcachePageSize(PgHdr *p){

  assert( sqlite3_mutex_held(pcache_g.mutex) );

  assert( !pcache_g.pStart );

  assert( p->apSave[0]==0 );

  assert( p->apSave[1]==0 );

  assert( p && p->pCache );

  return sqlite3MallocSize(p);

}

#endif

/*

** Attempt to 'recycle' a page from the global LRU list. Only clean,

** unreferenced pages from purgeable caches are eligible for recycling.

**

** This function removes page pcache.pLruTail from the global LRU list,

** and from the hash-table and PCache.pClean list of the owner pcache.

** There should be no other references to the page.

**

** A pointer to the recycled page is returned, or NULL if no page is

** eligible for recycling.

*/

static PgHdr *pcacheRecyclePage(void){

  PgHdr *p = 0;

  assert( sqlite3_mutex_held(pcache_g.mutex) );

  if( (p=pcache_g.pLruTail) ){