FCL/sf/os/kernelhwsrv: comparison kernel/eka/memmodel/epoc/flexible/mmu/mpager.cpp

equal deleted inserted replaced

-:2d65c2f76d7b
+:947f0dc9f7a8
 #include "mmanager.h"
 #include "mptalloc.h"
 #include "mpagearray.h"
 #include "mswap.h"
 #include "mthrash.h"
+#include "mpagecleaner.h"
 #include "cache_maintenance.inl"
 const TUint16 KDefaultYoungOldRatio = 3;
 const TUint16 KDefaultMinPages = 256;
-#ifdef _USE_OLDEST_LISTS
 const TUint16 KDefaultOldOldestRatio = 3;
-#endif
 const TUint KMinOldPages = 1;
 /*	On a 32 bit system without PAE can't have more than 2^(32-KPageShift) pages.
 *	Subtract 1 so it doesn't overflow when converted to bytes.
 */
 const TUint	KAbsoluteMaxPageCount = (1u<<(32-KPageShift))-1u;
+/*
+Limit the maximum number of oldest pages to bound the time taken by SelectPagesToClean(), which is
+called with the MmuLock held.
+*/
+const TUint KMaxOldestPages = 32;
+static DMutex* ThePageCleaningLock = NULL;
 DPager ThePager;
 DPager::DPager()
 	: iMinimumPageCount(0), iMaximumPageCount(0), iYoungOldRatio(0),
-	  iYoungCount(0),iOldCount(0),iNumberOfFreePages(0)
+	  iYoungCount(0), iOldCount(0), iOldestCleanCount(0),
-	{
+	  iNumberOfFreePages(0), iReservePageCount(0), iMinimumPageLimit(0)
-	}
+#ifdef __DEMAND_PAGING_BENCHMARKS__
+	, iBenchmarkLock(TSpinLock::EOrderGenericIrqHigh3)
+#endif
-void DPager::Init2()
+	{
-	{
+	}
-	TRACEB(("DPager::Init2()"));
+void DPager::InitCache()
+	{
+	//
+	// This routine doesn't acquire any mutexes because it should be called before the system
+	// is fully up and running. I.e. called before another thread can preempt this.
+	//
+	TRACEB(("DPager::InitCache()"));
+	// If any pages have been reserved then they will have already been allocated and
+	// therefore should be counted as part of iMinimumPageCount.
+	__NK_ASSERT_DEBUG(iReservePageCount == iMinimumPageCount);
+	__NK_ASSERT_DEBUG(!CacheInitialised());
 #if defined(__CPU_ARM)
 /** Minimum number of young pages the demand paging live list may have.
 	Need at least 4 mapped pages to guarantee to be able to execute all ARM instructions,
 #error Unknown CPU
 #endif
 #ifdef __SMP__
 	// Adjust min page count so that all CPUs are guaranteed to make progress.
-	// NOTE: Can't use NKern::NumberOfCpus here because we haven't finished booting yet and will
+	TInt numberOfCpus = NKern::NumberOfCpus();
-	// always have only one CPU running at this point...
+	iMinYoungPages *= numberOfCpus;
-	// TODO: Before we can enable this the base test configuration needs
-	// updating to have a sufficient minimum page size...
-	//
-	// iMinYoungPages *= KMaxCpus;
 #endif
 	// A minimum young/old ratio of 1 means that we need at least twice iMinYoungPages pages...
 	iAbsoluteMinPageCount = 2*iMinYoungPages;
 	__NK_ASSERT_DEBUG(KMinOldPages<=iAbsoluteMinPageCount/2);
-	// initialise live list...
+	// Read any paging config data.
-	TUint minimumPageCount = 0;
-	TUint maximumPageCount = 0;
 	SDemandPagingConfig config = TheRomHeader().iDemandPagingConfig;
-	iMinimumPageCount = KDefaultMinPages;
+	// Set the list ratios...
-	if(minimumPageCount)
+	iYoungOldRatio = KDefaultYoungOldRatio;
-		iMinimumPageCount = minimumPageCount;
+	if(config.iYoungOldRatio)
+		iYoungOldRatio = config.iYoungOldRatio;
+	iOldOldestRatio = KDefaultOldOldestRatio;
+	if(config.iSpare[2])
+		iOldOldestRatio = config.iSpare[2];
+	// Set the minimum page counts...
+	iMinimumPageLimit = iMinYoungPages * (1 + iYoungOldRatio) / iYoungOldRatio
+									   + DPageReadRequest::ReservedPagesRequired();
+	if(iMinimumPageLimit < iAbsoluteMinPageCount)
+		iMinimumPageLimit = iAbsoluteMinPageCount;
+	if (K::MemModelAttributes & (EMemModelAttrRomPaging | EMemModelAttrCodePaging | EMemModelAttrDataPaging))
+	    iMinimumPageCount = KDefaultMinPages;
+	else
+		{// No paging is enabled so set the minimum cache size to the minimum
+		// allowable with the current young old ratio.
+	    iMinimumPageCount = iMinYoungPages * (iYoungOldRatio + 1);
+		}
 	if(config.iMinPages)
 		iMinimumPageCount = config.iMinPages;
-	if(iMinimumPageCount<iAbsoluteMinPageCount)
+	if(iMinimumPageCount < iAbsoluteMinPageCount)
 		iMinimumPageCount = iAbsoluteMinPageCount;
+	if (iMinimumPageLimit + iReservePageCount > iMinimumPageCount)
+		iMinimumPageCount = iMinimumPageLimit + iReservePageCount;
 	iInitMinimumPageCount = iMinimumPageCount;
+	// Set the maximum page counts...
 	iMaximumPageCount = KMaxTInt;
-	if(maximumPageCount)
-		iMaximumPageCount = maximumPageCount;
 	if(config.iMaxPages)
 		iMaximumPageCount = config.iMaxPages;
 	if (iMaximumPageCount > KAbsoluteMaxPageCount)
 		iMaximumPageCount = KAbsoluteMaxPageCount;
 	iInitMaximumPageCount = iMaximumPageCount;
-	iYoungOldRatio = KDefaultYoungOldRatio;
+	TRACEB(("DPager::InitCache() live list min=%d max=%d ratio=%d",iMinimumPageCount,iMaximumPageCount,iYoungOldRatio));
-	if(config.iYoungOldRatio)
-		iYoungOldRatio = config.iYoungOldRatio;
+	// Verify the page counts are valid.
+	__NK_ASSERT_ALWAYS(iMaximumPageCount >= iMinimumPageCount);
+	TUint minOldAndOldest = iMinimumPageCount / (1 + iYoungOldRatio);
+	__NK_ASSERT_ALWAYS(minOldAndOldest >= KMinOldPages);
+	__NK_ASSERT_ALWAYS(iMinimumPageCount >= minOldAndOldest);
+	// Need at least iMinYoungPages pages mapped to execute worst case CPU instruction
+	TUint minYoung = iMinimumPageCount - minOldAndOldest;
+	__NK_ASSERT_ALWAYS(minYoung >= iMinYoungPages);
+	// Verify that the young old ratio can be met even when there is only the
+	// minimum number of old pages.
 	TInt ratioLimit = (iMinimumPageCount-KMinOldPages)/KMinOldPages;
-	if(iYoungOldRatio>ratioLimit)
+	__NK_ASSERT_ALWAYS(iYoungOldRatio <= ratioLimit);
-		iYoungOldRatio = ratioLimit;
-#ifdef _USE_OLDEST_LISTS
-	iOldOldestRatio = KDefaultOldOldestRatio;
-	if(config.iSpare[2])
-		iOldOldestRatio = config.iSpare[2];
-#endif
-	iMinimumPageLimit = (iMinYoungPages * (1 + iYoungOldRatio)) / iYoungOldRatio;
-	if(iMinimumPageLimit<iAbsoluteMinPageCount)
-		iMinimumPageLimit = iAbsoluteMinPageCount;
-	TRACEB(("DPager::Init2() live list min=%d max=%d ratio=%d",iMinimumPageCount,iMaximumPageCount,iYoungOldRatio));
-	if(iMaximumPageCount<iMinimumPageCount)
-		__NK_ASSERT_ALWAYS(0);
-	//
-	// This routine doesn't acquire any mutexes because it should be called before the system
-	// is fully up and running. I.e. called before another thread can preempt this.
-	//
-	// Calculate page counts
-	TUint minOldAndOldest = iMinimumPageCount / (1 + iYoungOldRatio);
-	if(minOldAndOldest < KMinOldPages)
-		__NK_ASSERT_ALWAYS(0);
-	if (iMinimumPageCount < minOldAndOldest)
-		__NK_ASSERT_ALWAYS(0);
-	TUint minYoung = iMinimumPageCount - minOldAndOldest;
-	if(minYoung < iMinYoungPages)
-		__NK_ASSERT_ALWAYS(0); // Need at least iMinYoungPages pages mapped to execute worst case CPU instruction
-#ifdef _USE_OLDEST_LISTS
 	// There should always be enough old pages to allow the oldest lists ratio.
 	TUint oldestCount = minOldAndOldest / (1 + iOldOldestRatio);
-	if (!oldestCount)
+	__NK_ASSERT_ALWAYS(oldestCount);
-		__NK_ASSERT_ALWAYS(0);
-#endif
 	iNumberOfFreePages = 0;
 	iNumberOfDirtyPages = 0;
-	// Allocate RAM pages and put them all on the old list
+	// Allocate RAM pages and put them all on the old list.
+	// Reserved pages have already been allocated and already placed on the
+	// old list so don't allocate them again.
 	RamAllocLock::Lock();
 	iYoungCount = 0;
 	iOldCount = 0;
-#ifdef _USE_OLDEST_LISTS
-	iOldestCleanCount = 0;
 	iOldestDirtyCount = 0;
-#endif
+	__NK_ASSERT_DEBUG(iOldestCleanCount == iReservePageCount);
 	Mmu& m = TheMmu;
-	for(TUint i=0; i<iMinimumPageCount; i++)
+	for(TUint i = iReservePageCount; i < iMinimumPageCount; i++)
 		{
 		// Allocate a single page
 		TPhysAddr pagePhys;
 		TInt r = m.AllocRam(&pagePhys, 1,
 							(Mmu::TRamAllocFlags)(EMemAttNormalCached|Mmu::EAllocNoWipe|Mmu::EAllocNoPagerReclaim),
 		AddAsFreePage(SPageInfo::FromPhysAddr(pagePhys));
 		MmuLock::Unlock();
 		}
 	RamAllocLock::Unlock();
-#ifdef _USE_OLDEST_LISTS
+	__NK_ASSERT_DEBUG(CacheInitialised());
-	TRACEB(("DPager::Init2() end with young=%d old=%d oldClean=%d oldDirty=%d min=%d free=%d max=%d",iYoungCount,iOldCount,iOldestCleanCount,iOldestDirtyCount,iMinimumPageCount,iNumberOfFreePages,iMaximumPageCount));
+	TRACEB(("DPager::InitCache() end with young=%d old=%d oldClean=%d oldDirty=%d min=%d free=%d max=%d",iYoungCount,iOldCount,iOldestCleanCount,iOldestDirtyCount,iMinimumPageCount,iNumberOfFreePages,iMaximumPageCount));
-#else
-	TRACEB(("DPager::Init2() end with young=%d old=%d min=%d free=%d max=%d",iYoungCount,iOldCount,iMinimumPageCount,iNumberOfFreePages,iMaximumPageCount));
-#endif
 	}
 #ifdef _DEBUG
 TBool DPager::CheckLists()
 	__NK_ASSERT_DEBUG(aPageInfo->PagedState()==SPageInfo::EUnpaged);
 	TheMmu.PageFreed(aPageInfo);
 	__NK_ASSERT_DEBUG(aPageInfo->PagedState()==SPageInfo::EUnpaged);
 	// add as oldest page...
-#ifdef _USE_OLDEST_LISTS
 	aPageInfo->SetPagedState(SPageInfo::EPagedOldestClean);
 	iOldestCleanList.Add(&aPageInfo->iLink);
 	++iOldestCleanCount;
-#else
-	aPageInfo->SetPagedState(SPageInfo::EPagedOld);
-	iOldList.Add(&aPageInfo->iLink);
-	++iOldCount;
-#endif
 	Event(EEventPageInFree,aPageInfo);
 	}
 		__NK_ASSERT_DEBUG(iOldCount);
 		aPageInfo->iLink.Deque();
 		--iOldCount;
 		break;
-#ifdef _USE_OLDEST_LISTS
 	case SPageInfo::EPagedOldestClean:
 		__NK_ASSERT_DEBUG(iOldestCleanCount);
 		aPageInfo->iLink.Deque();
 		--iOldestCleanCount;
 		break;
 	case SPageInfo::EPagedOldestDirty:
 		__NK_ASSERT_DEBUG(iOldestDirtyCount);
 		aPageInfo->iLink.Deque();
 		--iOldestDirtyCount;
 		break;
-#endif
 	case SPageInfo::EPagedPinned:
 		// this can occur if a pinned mapping is being unmapped when memory is decommitted.
 		// the decommit will have succeeded because the the mapping no longer vetoes this,
 		// however the unpinning hasn't yet got around to changing the page state.
 		return KErrNotFound;
 		}
 	// Update the dirty page count as required...
 	if (aPageInfo->IsDirty())
+		{
+		aPageInfo->SetReadOnly();
 		SetClean(*aPageInfo);
+		}
 	// add as oldest page...
-#ifdef _USE_OLDEST_LISTS
 	aPageInfo->SetPagedState(SPageInfo::EPagedOldestClean);
 	iOldestCleanList.Add(&aPageInfo->iLink);
 	++iOldestCleanCount;
-#else
-	aPageInfo->SetPagedState(SPageInfo::EPagedOld);
-	iOldList.Add(&aPageInfo->iLink);
-	++iOldCount;
-#endif
 	return KErrNone;
 	}
 		__NK_ASSERT_DEBUG(iOldCount);
 		aPageInfo->iLink.Deque();
 		--iOldCount;
 		break;
-#ifdef _USE_OLDEST_LISTS
 	case SPageInfo::EPagedOldestClean:
 		__NK_ASSERT_DEBUG(iOldestCleanCount);
 		aPageInfo->iLink.Deque();
 		--iOldestCleanCount;
 		break;
 	case SPageInfo::EPagedOldestDirty:
 		__NK_ASSERT_DEBUG(iOldestDirtyCount);
 		aPageInfo->iLink.Deque();
 		--iOldestDirtyCount;
 		break;
-#endif
 	case SPageInfo::EPagedPinned:
 		__NK_ASSERT_DEBUG(0);
 	case SPageInfo::EUnpaged:
 #ifdef _DEBUG
 			break;
 		}
 	}
-SPageInfo* DPager::StealOldestPage()
+TInt DPager::TryStealOldestPage(SPageInfo*& aPageInfoOut)
 	{
 	__NK_ASSERT_DEBUG(RamAllocLock::IsHeld());
 	__NK_ASSERT_DEBUG(MmuLock::IsHeld());
+	// find oldest page in list...
+	SDblQueLink* link;
+	if (iOldestCleanCount)
+		{
+		__NK_ASSERT_DEBUG(!iOldestCleanList.IsEmpty());
+		link = iOldestCleanList.Last();
+		}
+	else if (iOldestDirtyCount)
+		{
+		__NK_ASSERT_DEBUG(!iOldestDirtyList.IsEmpty());
+		link = iOldestDirtyList.Last();
+		}
+	else if (iOldCount)
+		{
+		__NK_ASSERT_DEBUG(!iOldList.IsEmpty());
+		link = iOldList.Last();
+		}
+	else
+		{
+		__NK_ASSERT_DEBUG(iYoungCount);
+		__NK_ASSERT_ALWAYS(!iYoungList.IsEmpty());
+		link = iYoungList.Last();
+		}
+	SPageInfo* pageInfo = SPageInfo::FromLink(link);
+	if (pageInfo->IsDirty() && !PageCleaningLock::IsHeld())
+		return 1;
+	// try to steal it from owning object...
+	TInt r = StealPage(pageInfo);
+	if (r == KErrNone)
+		{
+		BalanceAges();
+		aPageInfoOut = pageInfo;
+		}
+	return r;
+	}
+SPageInfo* DPager::StealOldestPage()
+	{
+	__NK_ASSERT_DEBUG(MmuLock::IsHeld());
+	TBool pageCleaningLockHeld = EFalse;
 	for(;;)
 		{
-		// find oldest page in list...
+		SPageInfo* pageInfo = NULL;
-		SDblQueLink* link;
+		TInt r = TryStealOldestPage(pageInfo);
-#ifdef _USE_OLDEST_LISTS
-		if (iOldestCleanCount)
+		if (r == KErrNone)
 			{
-			__NK_ASSERT_DEBUG(!iOldestCleanList.IsEmpty());
+			if (pageCleaningLockHeld)
-			link = iOldestCleanList.Last();
+				{
-			}
+				MmuLock::Unlock();
-		else if (iOldestDirtyCount)
+				PageCleaningLock::Unlock();
-			{
+				MmuLock::Lock();
-			__NK_ASSERT_DEBUG(!iOldestDirtyList.IsEmpty());
+				}
-			link = iOldestDirtyList.Last();
+			return pageInfo;
 			}
-		else if (iOldCount)
+		else if (r == 1)
+			{
+			__NK_ASSERT_ALWAYS(!pageCleaningLockHeld);
+			MmuLock::Unlock();
+			PageCleaningLock::Lock();
+			MmuLock::Lock();
+			pageCleaningLockHeld = ETrue;
+			}
+		// else retry...
+		}
+	}
+#ifdef __CPU_CACHE_HAS_COLOUR
+template <class T, TInt maxObjects> class TSequentialColourSelector
+	{
+public:
+	static const TInt KMaxLength = maxObjects;
+	static const TInt KArrayLength = _ALIGN_UP(KMaxLength, KPageColourCount);
+	FORCE_INLINE TSequentialColourSelector()
+		{
+		memclr(this, sizeof(*this));
+		}
+	FORCE_INLINE TBool FoundLongestSequence()
+		{
+		return iLongestLength >= KMaxLength;
+		}
+	FORCE_INLINE void AddCandidate(T* aObject, TInt aColour)
+		{
+		// allocate objects to slots based on colour
+		for (TInt i = aColour ; i < KArrayLength ; i += KPageColourCount)
+			{
+			if (!iSlot[i])
+				{
+				iSlot[i] = aObject;
+				iSeqLength[i] = i == 0 ? 1 : iSeqLength[i - 1] + 1;
+				TInt j = i + 1;
+				while(j < KArrayLength && iSeqLength[j])
+					iSeqLength[j++] += iSeqLength[i];
+				TInt currentLength = iSeqLength[j - 1];
+				if (currentLength > iLongestLength)
+					{
+					iLongestLength = currentLength;
+					iLongestStart = j - currentLength;
+					}
+				break;
+				}
+			}
+		}
+	FORCE_INLINE TInt FindLongestRun(T** aObjectsOut)
+		{
+		if (iLongestLength == 0)
+			return 0;
+		if (iLongestLength < KMaxLength && iSlot[0] && iSlot[KArrayLength - 1])
+			{
+			// check possibility of wrapping
+			TInt i = 1;
+			while (iSlot[i]) ++i;  // find first hole
+			TInt wrappedLength = iSeqLength[KArrayLength - 1] + iSeqLength[i - 1];
+			if (wrappedLength > iLongestLength)
+				{
+				iLongestLength = wrappedLength;
+				iLongestStart = KArrayLength - iSeqLength[KArrayLength - 1];
+				}
+			}
+		iLongestLength = Min(iLongestLength, KMaxLength);
+		__NK_ASSERT_DEBUG(iLongestStart >= 0 && iLongestStart < KArrayLength);
+		__NK_ASSERT_DEBUG(iLongestStart + iLongestLength < 2 * KArrayLength);
+		TInt len = Min(iLongestLength, KArrayLength - iLongestStart);
+		wordmove(aObjectsOut, &iSlot[iLongestStart], len * sizeof(T*));
+		wordmove(aObjectsOut + len, &iSlot[0], (iLongestLength - len) * sizeof(T*));
+		return iLongestLength;
+		}
+private:
+	T* iSlot[KArrayLength];
+	TInt8 iSeqLength[KArrayLength];
+	TInt iLongestStart;
+	TInt iLongestLength;
+	};
+TInt DPager::SelectPagesToClean(SPageInfo** aPageInfosOut)
+	{
+	// select up to KMaxPagesToClean oldest dirty pages with sequential page colours
+	__NK_ASSERT_DEBUG(MmuLock::IsHeld());
+	TSequentialColourSelector<SPageInfo, KMaxPagesToClean> selector;
+	SDblQueLink* link = iOldestDirtyList.Last();
+	while (link != &iOldestDirtyList.iA)
+		{
+		SPageInfo* pi = SPageInfo::FromLink(link);
+		if (!pi->IsWritable())
+			{
+			// the page may be in the process of being restricted, stolen or decommitted, but don't
+			// check for this as it will occur infrequently and will be detected by CheckModified
+			// anyway
+			TInt colour = pi->Index() & KPageColourMask;
+			selector.AddCandidate(pi, colour);
+			if (selector.FoundLongestSequence())
+				break;
+			}
+		link = link->iPrev;
+		}
+	return selector.FindLongestRun(aPageInfosOut);
+	}
 #else
-		if (iOldCount)
+TInt DPager::SelectPagesToClean(SPageInfo** aPageInfosOut)
+	{
+	// no page colouring restrictions, so just take up to KMaxPagesToClean oldest dirty pages
+	__NK_ASSERT_DEBUG(MmuLock::IsHeld());
+	TInt pageCount = 0;
+	SDblQueLink* link = iOldestDirtyList.Last();
+	while (link != &iOldestDirtyList.iA && pageCount < KMaxPagesToClean)
+		{
+		SPageInfo* pi = SPageInfo::FromLink(link);
+		if (!pi->IsWritable())
+			{
+			// the page may be in the process of being restricted, stolen or decommitted, but don't
+			// check for this as it will occur infrequently and will be detected by CheckModified
+			// anyway
+			aPageInfosOut[pageCount++] = pi;
+			}
+		link = link->iPrev;
+		}
+	return pageCount;
+	}
 #endif
-			{
-			__NK_ASSERT_DEBUG(!iOldList.IsEmpty());
-			link = iOldList.Last();
+TInt DPager::CleanSomePages(TBool aBackground)
-			}
+	{
-		else
+	__NK_ASSERT_DEBUG(MmuLock::IsHeld());
-			{
+	__NK_ASSERT_DEBUG(PageCleaningLock::IsHeld());
-			__NK_ASSERT_DEBUG(iYoungCount);
+	// ram alloc lock may or may not be held
-			__NK_ASSERT_ALWAYS(!iYoungList.IsEmpty());
-			link = iYoungList.Last();
+	SPageInfo* pageInfos[KMaxPagesToClean];
-			}
+	TInt pageCount = SelectPagesToClean(&pageInfos[0]);
-		SPageInfo* pageInfo = SPageInfo::FromLink(link);
+	if (pageCount == 0)
-		// steal it from owning object...
+		return 0;
-		TInt r = StealPage(pageInfo);
+	TheDataPagedMemoryManager->CleanPages(pageCount, pageInfos, aBackground);
-		BalanceAges();
+	for (TInt i = 0 ; i < pageCount ; ++i)
-		if(r==KErrNone)
+		{
-			return pageInfo; // done
+		SPageInfo* pi = pageInfos[i];
+		if (pi)
-		// loop back and try again
+			{
-		}
+			__NK_ASSERT_DEBUG(pi->PagedState() == SPageInfo::EPagedOldestDirty && iOldestDirtyCount);
+			__NK_ASSERT_DEBUG(!pi->IsDirty() && !pi->IsWritable());
+			pi->iLink.Deque();
+			iOldestCleanList.AddHead(&pi->iLink);
+			--iOldestDirtyCount;
+			++iOldestCleanCount;
+			pi->SetPagedState(SPageInfo::EPagedOldestClean);
+			}
+		}
+	return pageCount;
+	}
+TBool DPager::HasPagesToClean()
+	{
+	__NK_ASSERT_DEBUG(MmuLock::IsHeld());
+	return iOldestDirtyCount > 0;
 	}
 TInt DPager::RestrictPage(SPageInfo* aPageInfo, TRestrictPagesType aRestriction)
 	{
 	TRACE(("DPager::StealPage returns %d",r));
 	return r;
 	}
+static TBool DiscardCanStealPage(SPageInfo* aOldPageInfo, TBool aBlockRest)
+	{
+	// If the page is pinned or if the page is dirty and a general defrag is being performed then
+	// don't attempt to steal it
+	return aOldPageInfo->Type() == SPageInfo::EUnused ||
+		(aOldPageInfo->PagedState() != SPageInfo::EPagedPinned && (!aBlockRest || !aOldPageInfo->IsDirty()));
+	}
 TInt DPager::DiscardPage(SPageInfo* aOldPageInfo, TUint aBlockZoneId, TBool aBlockRest)
 	{
+	// todo: assert MmuLock not released
+	TRACE(("> DPager::DiscardPage %08x", aOldPageInfo));
 	__NK_ASSERT_DEBUG(RamAllocLock::IsHeld());
 	__NK_ASSERT_DEBUG(MmuLock::IsHeld());
-	TInt r;
+	if (!DiscardCanStealPage(aOldPageInfo, aBlockRest))
-	// If the page is pinned or if the page is dirty and a general defrag is being
+		{
-	// performed then don't attempt to steal it.
+		// The page is pinned or is dirty and this is a general defrag so move the page.
-	if (aOldPageInfo->Type() != SPageInfo::EUnused &&
-		(aOldPageInfo->PagedState() == SPageInfo::EPagedPinned ||
-		(aBlockRest && aOldPageInfo->IsDirty())))
-		{// The page is pinned or is dirty and this is a general defrag so move the page.
 		DMemoryObject* memory = aOldPageInfo->Owner();
 		// Page must be managed if it is pinned or dirty.
 		__NK_ASSERT_DEBUG(aOldPageInfo->Type()==SPageInfo::EManaged);
 		__NK_ASSERT_DEBUG(memory);
 		MmuLock::Unlock();
 		TPhysAddr newAddr;
-		return memory->iManager->MovePage(memory, aOldPageInfo, newAddr, aBlockZoneId, aBlockRest);
+		TRACE2(("DPager::DiscardPage delegating pinned/dirty page to manager"));
-		}
+		TInt r = memory->iManager->MovePage(memory, aOldPageInfo, newAddr, aBlockZoneId, aBlockRest);
+		TRACE(("< DPager::DiscardPage %d", r));
-	if (!iNumberOfFreePages)
+		return r;
-		{
+		}
-		// Allocate a new page for the live list as it has reached its minimum size.
+	TInt r = KErrNone;
+	SPageInfo* newPageInfo = NULL;
+	TBool havePageCleaningLock = EFalse;
+	TBool needNewPage;
+	TBool needPageCleaningLock;
+	while(needNewPage = (iNumberOfFreePages == 0 && newPageInfo == NULL),
+		  needPageCleaningLock = (aOldPageInfo->IsDirty() && !havePageCleaningLock),
+		  needNewPage || needPageCleaningLock)
+		{
 		MmuLock::Unlock();
-		SPageInfo* newPageInfo = GetPageFromSystem((Mmu::TRamAllocFlags)(EMemAttNormalCached|Mmu::EAllocNoWipe),
-													aBlockZoneId, aBlockRest);
+		if (needNewPage)
-		 if (!newPageInfo)
+			{
-			return KErrNoMemory;
+			// Allocate a new page for the live list as it has reached its minimum size.
+			TUint flags = EMemAttNormalCached | Mmu::EAllocNoWipe;
+			newPageInfo = GetPageFromSystem((Mmu::TRamAllocFlags)flags, aBlockZoneId, aBlockRest);
+			if (!newPageInfo)
+				{
+				TRACE(("< DPager::DiscardPage KErrNoMemory"));
+				r = KErrNoMemory;
+				MmuLock::Lock();
+				break;
+				}
+			}
+		if (needPageCleaningLock)
+			{
+			// Acquire the page cleaning mutex so StealPage can clean it
+			PageCleaningLock::Lock();
+			havePageCleaningLock = ETrue;
+			}
 		// Re-acquire the mmulock and re-check that the page is not pinned or dirty.
 		MmuLock::Lock();
-		if (aOldPageInfo->Type() != SPageInfo::EUnused &&
+		if (!DiscardCanStealPage(aOldPageInfo, aBlockRest))
-			(aOldPageInfo->PagedState() == SPageInfo::EPagedPinned ||
+			{
-			(aBlockRest && aOldPageInfo->IsDirty())))
+			// Page is now pinned or dirty so give up as it is in use.
-			{// Page is now pinned or dirty so give up as it is inuse.
+			r = KErrInUse;
-			ReturnPageToSystem(*newPageInfo);
+			break;
-			MmuLock::Unlock();
+			}
-			return KErrInUse;
+		}
-			}
+	if (r == KErrNone)
+		{
 		// Attempt to steal the page
-		r = StealPage(aOldPageInfo);
+		r = StealPage(aOldPageInfo);  // temporarily releases MmuLock if page is dirty
-		__NK_ASSERT_DEBUG(MmuLock::IsHeld());
+		}
+	__NK_ASSERT_DEBUG(MmuLock::IsHeld());
-		if (r == KErrCompletion)
-			{// This was a page table that has been freed but added to the
+	if (r == KErrCompletion)
-			// live list as a free page.  Remove from live list and continue.
+		{// This was a page table that has been freed but added to the
-			__NK_ASSERT_DEBUG(!aOldPageInfo->IsDirty());
+		// live list as a free page.  Remove from live list and continue.
-			RemovePage(aOldPageInfo);
+		__NK_ASSERT_DEBUG(!aOldPageInfo->IsDirty());
-			r = KErrNone;
+		RemovePage(aOldPageInfo);
-			}
+		r = KErrNone;
+		}
-		if (r == KErrNone)
-			{// Add the new page to the live list as discarding the old page
+	if (r == KErrNone && iNumberOfFreePages == 0)
-			// will reduce the live list below the minimum.
+		{
+		if (newPageInfo)
+			{
+			// Add a new page to the live list if we have one as discarding the old page will reduce
+			// the live list below the minimum.
 			AddAsFreePage(newPageInfo);
-			// We've successfully discarded the page so return it to the free pool.
+			newPageInfo = NULL;
-			ReturnPageToSystem(*aOldPageInfo);
+			}
-			BalanceAges();
+		else
-			}
+			{
-		 else
+			// Otherwise the live list shrank when page was being cleaned so have to give up
-			{
+			AddAsFreePage(aOldPageInfo);
-			// New page not required so just return it to the system.  This is safe as
+			BalanceAges();                  // temporarily releases MmuLock
-			// iNumberOfFreePages will have this page counted but as it is not on the live list
+			r = KErrInUse;
-			// noone else can touch it.
+			}
-			ReturnPageToSystem(*newPageInfo);
+		}
-			}
-		}
+	if (r == KErrNone)
-	else
+		{
-		{
+		// We've successfully discarded the page and ensured the live list is large enough, so
-		// Attempt to steal the page
+		// return it to the free pool.
-		r = StealPage(aOldPageInfo);
+		ReturnPageToSystem(*aOldPageInfo);  // temporarily releases MmuLock
+		BalanceAges();                      // temporarily releases MmuLock
-		__NK_ASSERT_DEBUG(MmuLock::IsHeld());
+		}
-		if (r == KErrCompletion)
+	if (newPageInfo)
-			{// This was a page table that has been freed but added to the
+		{
-			// live list as a free page.  Remove from live list.
+		// New page not required so just return it to the system.  This is safe as
-			__NK_ASSERT_DEBUG(!aOldPageInfo->IsDirty());
+		// iNumberOfFreePages will have this page counted but as it is not on the live list noone
-			RemovePage(aOldPageInfo);
+		// else can touch it.
-			r = KErrNone;
+		if (iNumberOfFreePages == 0)
-			}
+			AddAsFreePage(newPageInfo);
+		else
-		if (r == KErrNone)
+			ReturnPageToSystem(*newPageInfo);   // temporarily releases MmuLock
-			{// We've successfully discarded the page so return it to the free pool.
+		}
-			ReturnPageToSystem(*aOldPageInfo);
-			BalanceAges();
+	if (havePageCleaningLock)
-			}
+		{
-		}
+		// Release the page cleaning mutex
+		MmuLock::Unlock();
+		PageCleaningLock::Unlock();
+		MmuLock::Lock();
+		}
 	MmuLock::Unlock();
+	TRACE(("< DPager::DiscardPage returns %d", r));
 	return r;
 	}
 TBool DPager::TryGrowLiveList()
 void DPager::ReturnPageToSystem(SPageInfo& aPageInfo)
 	{
 	__NK_ASSERT_DEBUG(RamAllocLock::IsHeld());
 	__NK_ASSERT_DEBUG(MmuLock::IsHeld());
+	// should be unpaged at this point, otherwise Mmu::FreeRam will just give it back to us
+	__NK_ASSERT_DEBUG(aPageInfo.PagedState() == SPageInfo::EUnpaged);
 	__NK_ASSERT_DEBUG(iNumberOfFreePages>0);
 	--iNumberOfFreePages;
 	MmuLock::Unlock();
 	}
 SPageInfo* DPager::PageInAllocPage(Mmu::TRamAllocFlags aAllocFlags)
 	{
+	TBool pageCleaningLockHeld = EFalse;
 	SPageInfo* pageInfo;
 	TPhysAddr pagePhys;
+	TInt r = KErrGeneral;
 	RamAllocLock::Lock();
 	MmuLock::Lock();
+find_a_page:
 	// try getting a free page from our live list...
-#ifdef _USE_OLDEST_LISTS
 	if (iOldestCleanCount)
 		{
 		pageInfo = SPageInfo::FromLink(iOldestCleanList.Last());
 		if(pageInfo->Type()==SPageInfo::EUnused)
-			goto get_oldest;
+			goto try_steal_oldest_page;
 		}
-#else
-	if(iOldCount)
-		{
-		pageInfo = SPageInfo::FromLink(iOldList.Last());
-		if(pageInfo->Type()==SPageInfo::EUnused)
-			goto get_oldest;
-		}
-#endif
 	// try getting a free page from the system pool...
 	if(!HaveMaximumPages())
 		{
 		MmuLock::Unlock();
 		if(pageInfo)
 			goto done;
 		MmuLock::Lock();
 		}
+	// try stealing a clean page...
+	if (iOldestCleanCount)
+		goto try_steal_oldest_page;
+	// see if we can clean multiple dirty pages in one go...
+	if (KMaxPagesToClean > 1 && iOldestDirtyCount > 1)
+		{
+		// if we don't hold the page cleaning mutex then temporarily release ram alloc mutex and
+		// acquire page cleaning mutex; if we hold it already just proceed
+		if (!pageCleaningLockHeld)
+			{
+			MmuLock::Unlock();
+			RamAllocLock::Unlock();
+			PageCleaningLock::Lock();
+			MmuLock::Lock();
+			}
+		// there may be clean pages now if we've waited on the page cleaning mutex, if so don't
+		// bother cleaning but just restart
+		if (iOldestCleanCount == 0)
+			CleanSomePages(EFalse);
+		if (!pageCleaningLockHeld)
+			{
+			MmuLock::Unlock();
+			PageCleaningLock::Unlock();
+			RamAllocLock::Lock();
+			MmuLock::Lock();
+			}
+		if (iOldestCleanCount > 0)
+			goto find_a_page;
+		}
 	// as a last resort, steal a page from the live list...
-get_oldest:
-#ifdef _USE_OLDEST_LISTS
+try_steal_oldest_page:
 	__NK_ASSERT_ALWAYS(iOldestCleanCount|iOldestDirtyCount|iOldCount|iYoungCount);
-#else
+	r = TryStealOldestPage(pageInfo);
-	__NK_ASSERT_ALWAYS(iOldCount|iYoungCount);
+	// if this fails we restart whole process
-#endif
+	if (r < KErrNone)
-	pageInfo = StealOldestPage();
+		goto find_a_page;
+	// if we need to clean, acquire page cleaning mutex for life of this function
+	if (r == 1)
+		{
+		__NK_ASSERT_ALWAYS(!pageCleaningLockHeld);
+		MmuLock::Unlock();
+		PageCleaningLock::Lock();
+		MmuLock::Lock();
+		pageCleaningLockHeld = ETrue;
+		goto find_a_page;
+		}
+	// otherwise we're done!
+	__NK_ASSERT_DEBUG(r == KErrNone);
 	MmuLock::Unlock();
 	// make page state same as a freshly allocated page...
 	pagePhys = pageInfo->PhysAddr();
 	TheMmu.PagesAllocated(&pagePhys,1,aAllocFlags);
 done:
+	if (pageCleaningLockHeld)
+		PageCleaningLock::Unlock();
 	RamAllocLock::Unlock();
 	return pageInfo;
 	}
 TBool DPager::GetFreePages(TInt aNumPages)
 			TheMmu.ChangePageType(pi, EPageMovable, EPageDiscard);
 			break;
 		case SPageInfo::EPagedYoung:
 		case SPageInfo::EPagedOld:
-#ifdef _USE_OLDEST_LISTS
 		case SPageInfo::EPagedOldestDirty:
 		case SPageInfo::EPagedOldestClean:
-#endif
 			continue; // discard already been allowed
 		case SPageInfo::EPagedPinned:
 			__NK_ASSERT_DEBUG(0);
 		default:
 		case SPageInfo::EUnpaged:
 			continue; // discard already been disallowed
 		case SPageInfo::EPagedYoung:
 		case SPageInfo::EPagedOld:
-#ifdef _USE_OLDEST_LISTS
 		case SPageInfo::EPagedOldestClean:
 		case SPageInfo::EPagedOldestDirty:
-#endif
 			changeType = ETrue;
 			break; // remove from live list
 		case SPageInfo::EPagedPinned:
 			__NK_ASSERT_DEBUG(0);
 	TheRomMemoryManager->Init3();
 	TheDataPagedMemoryManager->Init3();
 	TheCodePagedMemoryManager->Init3();
 	TInt r = Kern::AddHalEntry(EHalGroupVM, VMHalFunction, 0);
 	__NK_ASSERT_ALWAYS(r==KErrNone);
+	PageCleaningLock::Init();
 	}
 void DPager::Fault(TFault aFault)
 	{
 void DPager::BalanceAges()
 	{
 	__NK_ASSERT_DEBUG(MmuLock::IsHeld());
 	TBool restrictPage = EFalse;
 	SPageInfo* pageInfo = NULL;
-#ifdef _USE_OLDEST_LISTS
 	TUint oldestCount = iOldestCleanCount + iOldestDirtyCount;
 	if((iOldCount + oldestCount) * iYoungOldRatio < iYoungCount)
-#else
-	if (iOldCount * iYoungOldRatio < iYoungCount)
-#endif
 		{
 		// Need more old pages so make one young page into an old page...
 		__NK_ASSERT_DEBUG(!iYoungList.IsEmpty());
 		__NK_ASSERT_DEBUG(iYoungCount);
 		SDblQueLink* link = iYoungList.Last()->Deque();
 		Event(EEventPageAged,pageInfo);
 		// Delay restricting the page until it is safe to release the MmuLock.
 		restrictPage = ETrue;
 		}
-#ifdef _USE_OLDEST_LISTS
 	// Check we have enough oldest pages.
-	if (oldestCount * iOldOldestRatio < iOldCount)
+	if (oldestCount < KMaxOldestPages &&
+		oldestCount * iOldOldestRatio < iOldCount)
 		{
 		__NK_ASSERT_DEBUG(!iOldList.IsEmpty());
 		__NK_ASSERT_DEBUG(iOldCount);
 		SDblQueLink* link = iOldList.Last()->Deque();
 		--iOldCount;
 		if (oldestPageInfo->IsDirty())
 			{
 			oldestPageInfo->SetPagedState(SPageInfo::EPagedOldestDirty);
 			iOldestDirtyList.AddHead(link);
 			++iOldestDirtyCount;
+			PageCleaner::NotifyPagesToClean();
 			Event(EEventPageAgedDirty,oldestPageInfo);
 			}
 		else
 			{
 			oldestPageInfo->SetPagedState(SPageInfo::EPagedOldestClean);
 			iOldestCleanList.AddHead(link);
 			++iOldestCleanCount;
 			Event(EEventPageAgedClean,oldestPageInfo);
 			}
 		}
-#endif
 	if (restrictPage)
 		{
 		// Make the recently aged old page inaccessible.  This is done last as it
 		// will release the MmuLock and therefore the page counts may otherwise change.
 		RestrictPage(pageInfo,ERestrictPagesNoAccessForOldPage);
 	TRACE2(("DP: %O Rejuvenate PT 0x%08x 0x%08x",TheCurrentThread,pi->PhysAddr(),aPt));
 	switch(pi->PagedState())
 		{
 	case SPageInfo::EPagedYoung:
 	case SPageInfo::EPagedOld:
-#ifdef _USE_OLDEST_LISTS
 	case SPageInfo::EPagedOldestClean:
 	case SPageInfo::EPagedOldestDirty:
-#endif
 		RemovePage(pi);
 		AddAsYoungestPage(pi);
 		BalanceAges();
 		break;
 		__NK_ASSERT_DEBUG(0);
 		break;
 		}
 	}
 TInt DPager::PteAndInfoFromLinAddr(	TInt aOsAsid, TLinAddr aAddress, DMemoryMappingBase* aMapping,
 									TUint aMapInstanceCount, TPte*& aPte, SPageInfo*& aPageInfo)
 	{
 	__NK_ASSERT_DEBUG(MmuLock::IsHeld());
 	aPageInfo = pi;
 	return KErrNone;
 	}
 TInt DPager::TryRejuvenate(	TInt aOsAsid, TLinAddr aAddress, TUint aAccessPermissions, TLinAddr aPc,
 							DMemoryMappingBase* aMapping, TUint aMapInstanceCount, DThread* aThread,
 							TAny* aExceptionInfo)
 	{
 	__NK_ASSERT_DEBUG(MmuLock::IsHeld());
+	START_PAGING_BENCHMARK;
 	SPageInfo* pi;
 	TPte* pPte;
 	TPte pte;
 	TInt r = PteAndInfoFromLinAddr(aOsAsid, aAddress, aMapping, aMapInstanceCount, pPte, pi);
 	InvalidateTLBForPage((aAddress&~KPageMask)|aOsAsid);
 	Event(EEventPageRejuvenate,pi,aPc,aAddress,aAccessPermissions);
 	TBool balance = false;
-#ifdef _USE_OLDEST_LISTS
 	if(	state==SPageInfo::EPagedYoung || state==SPageInfo::EPagedOld ||
 		state==SPageInfo::EPagedOldestClean || state==SPageInfo::EPagedOldestDirty)
-#else
-	if(state==SPageInfo::EPagedYoung || state==SPageInfo::EPagedOld)
-#endif
 		{
 		RemovePage(pi);
 		AddAsYoungestPage(pi);
 		// delay BalanceAges because we don't want to release MmuLock until after
 		// RejuvenatePageTable has chance to look at the page table page...
 	RejuvenatePageTable(pPte);
 	if(balance)
 		BalanceAges();
+	END_PAGING_BENCHMARK(EPagingBmRejuvenate);
 	return KErrNone;
 	}
 TInt DPager::PageInAllocPages(TPhysAddr* aPages, TUint aCount, Mmu::TRamAllocFlags aAllocFlags)
 		SPageInfo* pi = SPageInfo::FromPhysAddr(aPages[aCount]);
 		switch(pi->PagedState())
 			{
 		case SPageInfo::EPagedYoung:
 		case SPageInfo::EPagedOld:
-#ifdef _USE_OLDEST_LISTS
 		case SPageInfo::EPagedOldestClean:
 		case SPageInfo::EPagedOldestDirty:
-#endif
 			RemovePage(pi);
 			// fall through...
 		case SPageInfo::EUnpaged:
 			AddAsFreePage(pi);
 			break;
 	__NK_ASSERT_DEBUG(MmuLock::IsHeld());
 	switch(aPageInfo->PagedState())
 		{
 	case SPageInfo::EPagedYoung:
 	case SPageInfo::EPagedOld:
-#ifdef _USE_OLDEST_LISTS
 	case SPageInfo::EPagedOldestClean:
 	case SPageInfo::EPagedOldestDirty:
-#endif
 		RemovePage(aPageInfo);
 		AddAsYoungestPage(aPageInfo);
 		BalanceAges();
 		break;
 		aPageInfo->iLink.Deque();
 		--iOldCount;
 		__NK_ASSERT_DEBUG(aPageInfo->PinCount()==1);
 		break;
-#ifdef _USE_OLDEST_LISTS
 	case SPageInfo::EPagedOldestClean:
 		__NK_ASSERT_DEBUG(iOldestCleanCount);
 		aPageInfo->iLink.Deque();
 		--iOldestCleanCount;
 		__NK_ASSERT_DEBUG(aPageInfo->PinCount()==1);
 		__NK_ASSERT_DEBUG(iOldestDirtyCount);
 		aPageInfo->iLink.Deque();
 		--iOldestDirtyCount;
 		__NK_ASSERT_DEBUG(aPageInfo->PinCount()==1);
 		break;
-#endif
 	case SPageInfo::EPagedPinned:
 		// nothing more to do...
 		__NK_ASSERT_DEBUG(aPageInfo->PinCount()>1);
 		return;
 TInt DPager::ResizeLiveList(TUint aMinimumPageCount, TUint aMaximumPageCount)
 	{
 	TRACE(("DPager::ResizeLiveList(%d,%d) current young=%d old=%d min=%d free=%d max=%d",aMinimumPageCount,aMaximumPageCount,iYoungCount,iOldCount,iMinimumPageCount,iNumberOfFreePages,iMaximumPageCount));
+	__NK_ASSERT_DEBUG(CacheInitialised());
 	if(!aMaximumPageCount)
 		{
 		aMinimumPageCount = iInitMinimumPageCount;
 		aMaximumPageCount = iInitMaximumPageCount;
 		}
 	NKern::ThreadEnterCS();
 	RamAllocLock::Lock();
 	MmuLock::Lock();
+	__NK_ASSERT_ALWAYS(iYoungOldRatio!=0);
 	// Make sure aMinimumPageCount is not less than absolute minimum we can cope with...
 	iMinimumPageLimit = iMinYoungPages * (1 + iYoungOldRatio) / iYoungOldRatio
 						+ DPageReadRequest::ReservedPagesRequired();
 	if(iMinimumPageLimit<iAbsoluteMinPageCount)
 void DPager::FlushAll()
 	{
 	NKern::ThreadEnterCS();
 	RamAllocLock::Lock();
+	PageCleaningLock::Lock();
 	TRACE(("DPager::FlushAll() live list young=%d old=%d min=%d free=%d max=%d",iYoungCount,iOldCount,iMinimumPageCount,iNumberOfFreePages,iMaximumPageCount));
 	// look at all RAM pages in the system, and unmap all those used for paging
 	const TUint32* piMap = (TUint32*)KPageInfoMap;
 				}
 			SPageInfo* piEnd = pi+KPageInfosPerPage;
 			do
 				{
 				SPageInfo::TPagedState state = pi->PagedState();
-#ifdef _USE_OLDEST_LISTS
 				if (state==SPageInfo::EPagedYoung || state==SPageInfo::EPagedOld ||
 					state==SPageInfo::EPagedOldestClean || state==SPageInfo::EPagedOldestDirty)
-#else
-				if(state==SPageInfo::EPagedYoung || state==SPageInfo::EPagedOld)
-#endif
 					{
 					if (pi->Type() != SPageInfo::EUnused)
 						{
 						TInt r = StealPage(pi);
 						if(r==KErrNone)
 	// reduce live page list to a minimum
 	while(GetFreePages(1)) {};
 	TRACE(("DPager::FlushAll() end with young=%d old=%d min=%d free=%d max=%d",iYoungCount,iOldCount,iMinimumPageCount,iNumberOfFreePages,iMaximumPageCount));
+	PageCleaningLock::Unlock();
 	RamAllocLock::Unlock();
 	NKern::ThreadLeaveCS();
 	}
 	case EVMHalGetPagingBenchmark:
 		{
 		TUint index = (TInt) a1;
 		if (index >= EMaxPagingBm)
 			return KErrNotFound;
-		NKern::LockSystem();
+		SPagingBenchmarkInfo info;
-		SPagingBenchmarkInfo info = ThePager.iBenchmarkInfo[index];
+		ThePager.ReadBenchmarkData((TPagingBenchmark)index, info);
-		NKern::UnlockSystem();
 		kumemput32(a2,&info,sizeof(info));
 		}
 		return KErrNone;
 	case EVMHalResetPagingBenchmark:
 		{
 		TUint index = (TInt) a1;
 		if (index >= EMaxPagingBm)
 			return KErrNotFound;
-		NKern::LockSystem();
 		ThePager.ResetBenchmarkData((TPagingBenchmark)index);
-		NKern::UnlockSystem();
 		}
 		return KErrNone;
 #endif
 	default:
 #ifdef __DEMAND_PAGING_BENCHMARKS__
 void DPager::ResetBenchmarkData(TPagingBenchmark aBm)
 {
 SPagingBenchmarkInfo& info = iBenchmarkInfo[aBm];
+	__SPIN_LOCK_IRQ(iBenchmarkLock);
 info.iCount = 0;
 info.iTotalTime = 0;
 info.iMaxTime = 0;
 info.iMinTime = KMaxTInt;
+	__SPIN_UNLOCK_IRQ(iBenchmarkLock);
 }
-void DPager::RecordBenchmarkData(TPagingBenchmark aBm, TUint32 aStartTime, TUint32 aEndTime)
+void DPager::RecordBenchmarkData(TPagingBenchmark aBm, TUint32 aStartTime, TUint32 aEndTime, TUint aCount)
 {
 SPagingBenchmarkInfo& info = iBenchmarkInfo[aBm];
-++info.iCount;
 #if !defined(HIGH_RES_TIMER) || defined(HIGH_RES_TIMER_COUNTS_UP)
 TInt64 elapsed = aEndTime - aStartTime;
 #else
 TInt64 elapsed = aStartTime - aEndTime;
 #endif
+	__SPIN_LOCK_IRQ(iBenchmarkLock);
+info.iCount +=  aCount;
 info.iTotalTime += elapsed;
 if (elapsed > info.iMaxTime)
 info.iMaxTime = elapsed;
 if (elapsed < info.iMinTime)
 info.iMinTime = elapsed;
+	__SPIN_UNLOCK_IRQ(iBenchmarkLock);
 }
+void DPager::ReadBenchmarkData(TPagingBenchmark aBm, SPagingBenchmarkInfo& aDataOut)
+	{
+	__SPIN_LOCK_IRQ(iBenchmarkLock);
+	aDataOut = iBenchmarkInfo[aBm];
+	__SPIN_UNLOCK_IRQ(iBenchmarkLock);
+	}
 #endif //__DEMAND_PAGING_BENCHMARKS__
 //
 //
 // DPagingRequest
 //
-DPagingRequest::DPagingRequest(DPagingRequestPool::TGroup& aPoolGroup)
+DPagingRequest::DPagingRequest()
-	: iPoolGroup(aPoolGroup), iUseRegionMemory(0), iUseRegionIndex(0), iUseRegionCount(0)
+	: iMutex(NULL), iUseRegionCount(0)
 	{
 	}
-FORCE_INLINE void DPagingRequest::SetUse(DMemoryObject* aMemory, TUint aIndex, TUint aCount)
+void DPagingRequest::SetUseContiguous(DMemoryObject* aMemory, TUint aIndex, TUint aCount)
 	{
 	__ASSERT_SYSTEM_LOCK;
-	iUseRegionMemory = aMemory;
+	__NK_ASSERT_DEBUG(iUseRegionCount == 0);
-	iUseRegionIndex = aIndex;
+	__NK_ASSERT_DEBUG(aCount > 0 && aCount <= EMaxPages);
+	for (TUint i = 0 ; i < aCount ; ++i)
+		{
+		iUseRegionMemory[i] = aMemory;
+		iUseRegionIndex[i] = aIndex + i;
+		}
 	iUseRegionCount = aCount;
 	}
-TBool DPagingRequest::CheckUse(DMemoryObject* aMemory, TUint aIndex, TUint aCount)
+void DPagingRequest::SetUseDiscontiguous(DMemoryObject** aMemory, TUint* aIndex, TUint aCount)
 	{
-	return aMemory==iUseRegionMemory
+	__ASSERT_SYSTEM_LOCK;
-		&& TUint(aIndex-iUseRegionIndex) < iUseRegionCount
+	__NK_ASSERT_DEBUG(iUseRegionCount == 0);
-		&& TUint(iUseRegionCount-TUint(aIndex-iUseRegionIndex)) <= aCount;
+	__NK_ASSERT_DEBUG(aCount > 0 && aCount <= EMaxPages);
-	}
+	for (TUint i = 0 ; i < aCount ; ++i)
+		{
+		iUseRegionMemory[i] = aMemory[i];
-void DPagingRequest::Release()
+		iUseRegionIndex[i] = aIndex[i];
+		}
+	iUseRegionCount = aCount;
+	}
+void DPagingRequest::ResetUse()
+	{
+	__ASSERT_SYSTEM_LOCK;
+	__NK_ASSERT_DEBUG(iUseRegionCount > 0);
+	iUseRegionCount = 0;
+	}
+TBool DPagingRequest::CheckUseContiguous(DMemoryObject* aMemory, TUint aIndex, TUint aCount)
+	{
+	if (iUseRegionCount != aCount)
+		return EFalse;
+	for (TUint i = 0 ; i < iUseRegionCount ; ++i)
+		{
+		if (iUseRegionMemory[i] != aMemory || iUseRegionIndex[i] != aIndex + i)
+			return EFalse;
+		}
+	return ETrue;
+	}
+TBool DPagingRequest::CheckUseDiscontiguous(DMemoryObject** aMemory, TUint* aIndex, TUint aCount)
+	{
+	if (iUseRegionCount != aCount)
+		return EFalse;
+	for (TUint i = 0 ; i < iUseRegionCount ; ++i)
+		{
+		if (iUseRegionMemory[i] != aMemory[i] || iUseRegionIndex[i] != aIndex[i])
+			return EFalse;
+		}
+	return ETrue;
+	}
+TBool DPagingRequest::IsCollisionContiguous(DMemoryObject* aMemory, TUint aIndex, TUint aCount)
+	{
+	// note this could be optimised as most of the time we will be checking read/read collusions,
+	// both of which will be contiguous
+	__ASSERT_SYSTEM_LOCK;
+	for (TUint i = 0 ; i < iUseRegionCount ; ++i)
+		{
+		if (iUseRegionMemory[i] == aMemory &&
+			TUint(iUseRegionIndex[i] - aIndex) < aCount)
+			return ETrue;
+		}
+	return EFalse;
+	}
+TLinAddr DPagingRequest::MapPages(TUint aColour, TUint aCount, TPhysAddr* aPages)
+	{
+	__NK_ASSERT_DEBUG(iMutex->iCleanup.iThread == &Kern::CurrentThread());
+	return iTempMapping.Map(aPages,aCount,aColour);
+	}
+void DPagingRequest::UnmapPages(TBool aIMBRequired)
+	{
+	__NK_ASSERT_DEBUG(iMutex->iCleanup.iThread == &Kern::CurrentThread());
+	iTempMapping.Unmap(aIMBRequired);
+	}
+//
+// DPoolPagingRequest
+//
+DPoolPagingRequest::DPoolPagingRequest(DPagingRequestPool::TGroup& aPoolGroup) :
+	iPoolGroup(aPoolGroup)
+	{
+	}
+void DPoolPagingRequest::Release()
 	{
 	NKern::LockSystem();
-	SetUse(0,0,0);
+	ResetUse();
 	Signal();
 	}
-void DPagingRequest::Wait()
+void DPoolPagingRequest::Wait()
 	{
 	__ASSERT_SYSTEM_LOCK;
 	++iUsageCount;
 	TInt r = iMutex->Wait();
 	__NK_ASSERT_ALWAYS(r == KErrNone);
 	}
-void DPagingRequest::Signal()
+void DPoolPagingRequest::Signal()
 	{
 	__ASSERT_SYSTEM_LOCK;
 	iPoolGroup.Signal(this);
 	}
-FORCE_INLINE TBool DPagingRequest::IsCollision(DMemoryObject* aMemory, TUint aIndex, TUint aCount)
-	{
-	__ASSERT_SYSTEM_LOCK;
-	DMemoryObject* memory = iUseRegionMemory;
-	TUint index = iUseRegionIndex;
-	TUint count = iUseRegionCount;
-	// note, this comparison would fail if either region includes page number KMaxTUint,
-	// but it isn't possible to create a memory object which is > KMaxTUint pages...
-	return memory == aMemory && index+count > aIndex && index < aIndex+aCount;
-	}
-TLinAddr DPagingRequest::MapPages(TUint aColour, TUint aCount, TPhysAddr* aPages)
-	{
-	__NK_ASSERT_DEBUG(iMutex->iCleanup.iThread == &Kern::CurrentThread());
-	return iTempMapping.Map(aPages,aCount,aColour);
-	}
-void DPagingRequest::UnmapPages(TBool aIMBRequired)
-	{
-	__NK_ASSERT_DEBUG(iMutex->iCleanup.iThread == &Kern::CurrentThread());
-	iTempMapping.Unmap(aIMBRequired);
-	}
 //
 // DPageReadRequest
 //
 TInt DPageReadRequest::iAllocNext = 0;
+DPageReadRequest::DPageReadRequest(DPagingRequestPool::TGroup& aPoolGroup) :
+	DPoolPagingRequest(aPoolGroup)
+	{
+	// allocate space for mapping pages whilst they're being loaded...
+	iTempMapping.Alloc(EMaxPages);
+	}
 TInt DPageReadRequest::Construct()
 	{
 	// allocate id and mutex...
 	TUint id = (TUint)__e32_atomic_add_ord32(&iAllocNext, 1);
 	TBuf<sizeof("PageReadRequest-")+10> mutexName(KLitPagingRequest);
 	mutexName.AppendNum(id);
 	TInt r = K::MutexCreate(iMutex, mutexName, NULL, EFalse, KMutexOrdPageIn);
 	if(r!=KErrNone)
 		return r;
-	// allocate space for mapping pages whilst they're being loaded...
-	iTempMapping.Alloc(EMaxPages);
 	// create memory buffer...
 	TUint bufferSize = EMaxPages+1;
 	DMemoryObject* bufferMemory;
 	r = MM::MemoryNew(bufferMemory,EMemoryObjectUnpaged,bufferSize,EMemoryCreateNoWipe);
 	r = MM::MappingNew(bufferMapping,bufferMemory,ESupervisorReadWrite,KKernelOsAsid);
 	if(r!=KErrNone)
 		return r;
 	iBuffer = MM::MappingBase(bufferMapping);
-	// ensure there are enough young pages to cope with new request object...
-	r = ThePager.ResizeLiveList();
-	if(r!=KErrNone)
-		return r;
 	return r;
 	}
 //
 // DPageWriteRequest
 //
-TInt DPageWriteRequest::iAllocNext = 0;
+DPageWriteRequest::DPageWriteRequest()
-TInt DPageWriteRequest::Construct()
+	{
-	{
+	iMutex = ThePageCleaningLock;
-	// allocate id and mutex...
-	TUint id = (TUint)__e32_atomic_add_ord32(&iAllocNext, 1);
-	_LIT(KLitPagingRequest,"PageWriteRequest-");
-	TBuf<sizeof("PageWriteRequest-")+10> mutexName(KLitPagingRequest);
-	mutexName.AppendNum(id);
-	TInt r = K::MutexCreate(iMutex, mutexName, NULL, EFalse, KMutexOrdPageOut);
-	if(r!=KErrNone)
-		return r;
 	// allocate space for mapping pages whilst they're being loaded...
-	iTempMapping.Alloc(EMaxPages);
+	iTempMapping.Alloc(KMaxPagesToClean);
+	}
-	return r;
+void DPageWriteRequest::Release()
+	{
+	NKern::LockSystem();
+	ResetUse();
+	NKern::UnlockSystem();
 	}
 //
 // DPagingRequestPool
 //
-DPagingRequestPool::DPagingRequestPool(TUint aNumPageReadRequest,TUint aNumPageWriteRequest)
+DPagingRequestPool::DPagingRequestPool(TUint aNumPageReadRequest, TBool aWriteRequest)
-	: iPageReadRequests(aNumPageReadRequest), iPageWriteRequests(aNumPageWriteRequest)
+	: iPageReadRequests(aNumPageReadRequest)
 	{
 	TUint i;
 	for(i=0; i<aNumPageReadRequest; ++i)
 		{
 		DPageReadRequest* req = new DPageReadRequest(iPageReadRequests);
 		__NK_ASSERT_ALWAYS(req);
 		TInt r = req->Construct();
 		__NK_ASSERT_ALWAYS(r==KErrNone);
 		iPageReadRequests.iRequests[i] = req;
 		iPageReadRequests.iFreeList.Add(req);
 		}
-	for(i=0; i<aNumPageWriteRequest; ++i)
+	if (aWriteRequest)
 		{
-		DPageWriteRequest* req = new DPageWriteRequest(iPageWriteRequests);
+		iPageWriteRequest = new DPageWriteRequest();
-		__NK_ASSERT_ALWAYS(req);
+		__NK_ASSERT_ALWAYS(iPageWriteRequest);
-		TInt r = req->Construct();
-		__NK_ASSERT_ALWAYS(r==KErrNone);
-		iPageWriteRequests.iRequests[i] = req;
-		iPageWriteRequests.iFreeList.Add(req);
 		}
 	}
 DPagingRequestPool::~DPagingRequestPool()
 DPageReadRequest* DPagingRequestPool::AcquirePageReadRequest(DMemoryObject* aMemory, TUint aIndex, TUint aCount)
 	{
 	NKern::LockSystem();
-	DPagingRequest* req;
+	DPoolPagingRequest* req;
-	// if we collide with page write operation...
+	// check for collision with existing write
-	req = iPageWriteRequests.FindCollision(aMemory,aIndex,aCount);
+	if(iPageWriteRequest && iPageWriteRequest->IsCollisionContiguous(aMemory,aIndex,aCount))
-	if(req)
+		{
-		{
+		NKern::UnlockSystem();
-		// wait until write completes...
+		PageCleaningLock::Lock();
-		req->Wait();
+		PageCleaningLock::Unlock();
-		req->Signal();
 		return 0; // caller expected to retry if needed
 		}
 	// get a request object to use...
 	req = iPageReadRequests.GetRequest(aMemory,aIndex,aCount);
-	// check no new requests collide with us...
+	// check no new read or write requests collide with us...
-	if(iPageWriteRequests.FindCollision(aMemory,aIndex,aCount)
+	if ((iPageWriteRequest && iPageWriteRequest->IsCollisionContiguous(aMemory,aIndex,aCount)) ||
-		|| iPageReadRequests.FindCollision(aMemory,aIndex,aCount))
+		iPageReadRequests.FindCollisionContiguous(aMemory,aIndex,aCount))
 		{
 		// another operation is colliding with this region, give up and retry...
 		req->Signal();
 		return 0; // caller expected to retry if needed
 		}
 	// we have a request object which we can use...
-	req->SetUse(aMemory,aIndex,aCount);
+	req->SetUseContiguous(aMemory,aIndex,aCount);
 	NKern::UnlockSystem();
 	return (DPageReadRequest*)req;
 	}
-DPageWriteRequest* DPagingRequestPool::AcquirePageWriteRequest(DMemoryObject* aMemory, TUint aIndex, TUint aCount)
+DPageWriteRequest* DPagingRequestPool::AcquirePageWriteRequest(DMemoryObject** aMemory, TUint* aIndex, TUint aCount)
 	{
+	__NK_ASSERT_DEBUG(iPageWriteRequest);
+	__NK_ASSERT_DEBUG(PageCleaningLock::IsHeld());
 	NKern::LockSystem();
-	DPagingRequest* req;
+	// Collision with existing read requests is not possible here.  For a page to be read it must
+	// not be present, and for it to be written it must be present and dirty.  There is no way for a
-	for(;;)
+	// page to go between these states without an intervening read on an uninitialised (freshly
-		{
+	// committed) page, which will wait on the first read request.  In other words something like
-		// get a request object to use...
+	// this:
-		req = iPageWriteRequests.GetRequest(aMemory,aIndex,aCount);
+	//
+	//   read (blocks), decommit, re-commit, read (waits on mutex), write (now no pending reads!)
-		if(iPageWriteRequests.FindCollision(aMemory,aIndex,aCount))
+	//
-			{
+	// Note that a read request can be outstanding and appear to collide with this write, but only
-			// another write operation is colliding with this region, give up and retry...
+	// in the case when the thread making the read has blocked just after acquiring the request but
-			req->Signal();
+	// before it checks whether the read is still necessasry.  This makes it difficult to assert
-			// Reacquire the system lock as Signal() above will release it.
+	// that no collisions take place.
-			NKern::LockSystem();
-			continue;
+	iPageWriteRequest->SetUseDiscontiguous(aMemory,aIndex,aCount);
-			}
-		break;
-		}
-	// we have a request object which we can use...
-	req->SetUse(aMemory,aIndex,aCount);
 	NKern::UnlockSystem();
-	return (DPageWriteRequest*)req;
+	return iPageWriteRequest;
 	}
 DPagingRequestPool::TGroup::TGroup(TUint aNumRequests)
 	{
 	iNumRequests = aNumRequests;
-	iRequests = new DPagingRequest*[aNumRequests];
+	iRequests = new DPoolPagingRequest*[aNumRequests];
 	__NK_ASSERT_ALWAYS(iRequests);
 	}
-DPagingRequest* DPagingRequestPool::TGroup::FindCollision(DMemoryObject* aMemory, TUint aIndex, TUint aCount)
+DPoolPagingRequest* DPagingRequestPool::TGroup::FindCollisionContiguous(DMemoryObject* aMemory, TUint aIndex, TUint aCount)
 	{
 	__ASSERT_SYSTEM_LOCK;
-	DPagingRequest** ptr = iRequests;
+	DPoolPagingRequest** ptr = iRequests;
-	DPagingRequest** ptrEnd = ptr+iNumRequests;
+	DPoolPagingRequest** ptrEnd = ptr+iNumRequests;
 	while(ptr<ptrEnd)
 		{
-		DPagingRequest* req = *ptr++;
+		DPoolPagingRequest* req = *ptr++;
-		if(req->IsCollision(aMemory,aIndex,aCount))
+		if(req->IsCollisionContiguous(aMemory,aIndex,aCount))
 			return req;
 		}
 	return 0;
 	}
 static TUint32 RandomSeed = 33333;
-DPagingRequest* DPagingRequestPool::TGroup::GetRequest(DMemoryObject* aMemory, TUint aIndex, TUint aCount)
+DPoolPagingRequest* DPagingRequestPool::TGroup::GetRequest(DMemoryObject* aMemory, TUint aIndex, TUint aCount)
 	{
 	__NK_ASSERT_DEBUG(iNumRequests > 0);
 	// try using an existing request which collides with this region...
-	DPagingRequest* req  = FindCollision(aMemory,aIndex,aCount);
+	DPoolPagingRequest* req  = FindCollisionContiguous(aMemory,aIndex,aCount);
 	if(!req)
 		{
 		// use a free request...
-		req = (DPagingRequest*)iFreeList.GetFirst();
+		req = (DPoolPagingRequest*)iFreeList.GetFirst();
 		if(req)
 			{
 			// free requests aren't being used...
 			__NK_ASSERT_DEBUG(req->iUsageCount == 0);
 			}
 	return req;
 	}
-void DPagingRequestPool::TGroup::Signal(DPagingRequest* aRequest)
+void DPagingRequestPool::TGroup::Signal(DPoolPagingRequest* aRequest)
 	{
 	// if there are no threads waiting on the mutex then return it to the free pool...
 	__NK_ASSERT_DEBUG(aRequest->iUsageCount > 0);
 	if (--aRequest->iUsageCount==0)
 		iFreeList.AddHead(aRequest);
 */
 EXPORT_C TInt Kern::InstallPagingDevice(DPagingDevice* aDevice)
 	{
 	TRACEB(("Kern::InstallPagingDevice(0x%08x) name='%s' type=%d",aDevice,aDevice->iName,aDevice->iType));
+	__NK_ASSERT_DEBUG(!ThePager.CacheInitialised());
 	__NK_ASSERT_ALWAYS(aDevice->iReadUnitShift <= KPageShift);
 	TInt r = KErrNotSupported;	// Will return this if unsupported device type is installed
 	// create the pools of page out and page in requests...
-	const TInt writeReqs = (aDevice->iType & DPagingDevice::EData) ? KPagingRequestsPerDevice : 0;
+	const TBool writeReq = (aDevice->iType & DPagingDevice::EData) != 0;
-	aDevice->iRequestPool = new DPagingRequestPool(KPagingRequestsPerDevice,writeReqs);
+	aDevice->iRequestPool = new DPagingRequestPool(KPagingRequestsPerDevice, writeReq);
 	if(!aDevice->iRequestPool)
 		{
 		r = KErrNoMemory;
 		goto exit;
 		}
 			goto exit;
 		}
 	if (K::MemModelAttributes & (EMemModelAttrRomPaging | EMemModelAttrCodePaging | EMemModelAttrDataPaging))
 		TheThrashMonitor.Start();
+	if (K::MemModelAttributes & EMemModelAttrDataPaging)
+		PageCleaner::Start();
 exit:
 	TRACEB(("Kern::InstallPagingDevice returns %d",r));
 	return r;
 	}
 	iLockedPageCount = 0;
 	NKern::ThreadLeaveCS();
 	}
+//
+// PageCleaningLock
+//
+_LIT(KLitPageCleaningLock,"PageCleaningLock");
+void PageCleaningLock::Init()
+	{
+	__NK_ASSERT_DEBUG(!ThePageCleaningLock);
+	TInt r = Kern::MutexCreate(ThePageCleaningLock, KLitPageCleaningLock, KMutexOrdPageOut);
+	__NK_ASSERT_ALWAYS(r == KErrNone);
+	}
+void PageCleaningLock::Lock()
+	{
+	Kern::MutexWait(*ThePageCleaningLock);
+	}
+void PageCleaningLock::Unlock()
+	{
+	Kern::MutexSignal(*ThePageCleaningLock);
+	}
+TBool PageCleaningLock::IsHeld()
+	{
+	return ThePageCleaningLock->iCleanup.iThread == &Kern::CurrentThread();
+	}

changeset 90	947f0dc9f7a8
parent 0	a41df078684a
child 102	ef2a444a7410
child 121	661475905584