--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/kernel/eka/memmodel/epoc/flexible/mmu/mpager.cpp	Mon Oct 19 15:55:17 2009 +0100
@@ -0,0 +1,2593 @@
+// Copyright (c) 2005-2009 Nokia Corporation and/or its subsidiary(-ies).
+// All rights reserved.
+// This component and the accompanying materials are made available
+// under the terms of the License "Eclipse Public License v1.0"
+// which accompanies this distribution, and is available
+// at the URL "http://www.eclipse.org/legal/epl-v10.html".
+//
+// Initial Contributors:
+// Nokia Corporation - initial contribution.
+//
+// Contributors:
+//
+// Description:
+//
+
+#include "memmodel.h"
+#include "mm.h"
+#include "mmu.h"
+
+#include "mpager.h"
+#include "mrom.h"
+#include "mobject.h"
+#include "mmapping.h"
+#include "maddressspace.h"
+#include "mmanager.h"
+#include "mptalloc.h"
+#include "mpagearray.h"
+#include "mswap.h"
+#include "mthrash.h"
+#include "cache_maintenance.inl"
+
+
+const TUint16 KDefaultYoungOldRatio = 3;
+const TUint16 KDefaultMinPages = 256;
+#ifdef _USE_OLDEST_LISTS
+const TUint16 KDefaultOldOldestRatio = 3;
+#endif
+
+const TUint KMinOldPages = 1;
+
+/*	On a 32 bit system without PAE can't have more than 2^(32-KPageShift) pages.
+ *	Subtract 1 so it doesn't overflow when converted to bytes.
+*/
+const TUint	KAbsoluteMaxPageCount = (1u<<(32-KPageShift))-1u;
+
+
+
+DPager ThePager;
+
+
+DPager::DPager()
+	: iMinimumPageCount(0), iMaximumPageCount(0), iYoungOldRatio(0),
+	  iYoungCount(0),iOldCount(0),iNumberOfFreePages(0)
+	{
+	}
+
+
+void DPager::Init2()
+	{
+	TRACEB(("DPager::Init2()"));
+
+#if defined(__CPU_ARM)
+
+/** Minimum number of young pages the demand paging live list may have.
+	Need at least 4 mapped pages to guarantee to be able to execute all ARM instructions,
+	plus enough pages for 4 page tables to map those pages, plus enough pages for the
+	page table info structures of those page tables.
+	(Worst case is a Thumb-2 STM instruction with both instruction and data straddling chunk
+	boundaries.)
+*/
+	iMinYoungPages = 4											// pages
+					+(4+KPtClusterSize-1)/KPtClusterSize		// page table pages
+					+(4+KPageTableInfosPerPage-1)/KPageTableInfosPerPage;	// page table info pages
+
+#elif defined(__CPU_X86)
+
+/*	Need at least 6 mapped pages to guarantee to be able to execute all ARM instructions,
+	plus enough pages for 6 page tables to map those pages, plus enough pages for the
+	page table info structures of those page tables.
+	(Worst case is (?) a MOV [X],[Y] instruction with instruction, 'X' and 'Y' all
+	straddling chunk boundaries.)
+*/
+	iMinYoungPages = 6											// pages
+					+(6+KPtClusterSize-1)/KPtClusterSize		// page table pages
+					+(6+KPageTableInfosPerPage-1)/KPageTableInfosPerPage;	// page table info pages
+
+#else
+#error Unknown CPU
+#endif
+
+#ifdef __SMP__
+	// Adjust min page count so that all CPUs are guaranteed to make progress.
+	// NOTE: Can't use NKern::NumberOfCpus here because we haven't finished booting yet and will
+	// always have only one CPU running at this point...
+
+	// TODO: Before we can enable this the base test configuration needs
+	// updating to have a sufficient minimum page size...
+	//
+	// iMinYoungPages *= KMaxCpus;
+#endif
+
+	// A minimum young/old ratio of 1 means that we need at least twice iMinYoungPages pages...
+	iAbsoluteMinPageCount = 2*iMinYoungPages;
+
+	__NK_ASSERT_DEBUG(KMinOldPages<=iAbsoluteMinPageCount/2);
+
+	// initialise live list...
+	TUint minimumPageCount = 0;
+	TUint maximumPageCount = 0;
+
+	SDemandPagingConfig config = TheRomHeader().iDemandPagingConfig;
+
+	iMinimumPageCount = KDefaultMinPages;
+	if(minimumPageCount)
+		iMinimumPageCount = minimumPageCount;
+	if(config.iMinPages)
+		iMinimumPageCount = config.iMinPages;
+	if(iMinimumPageCount<iAbsoluteMinPageCount)
+		iMinimumPageCount = iAbsoluteMinPageCount;
+	iInitMinimumPageCount = iMinimumPageCount;
+
+	iMaximumPageCount = KMaxTInt;
+	if(maximumPageCount)
+		iMaximumPageCount = maximumPageCount;
+	if(config.iMaxPages)
+		iMaximumPageCount = config.iMaxPages;
+	if (iMaximumPageCount > KAbsoluteMaxPageCount)
+		iMaximumPageCount = KAbsoluteMaxPageCount;
+	iInitMaximumPageCount = iMaximumPageCount;
+
+	iYoungOldRatio = KDefaultYoungOldRatio;
+	if(config.iYoungOldRatio)
+		iYoungOldRatio = config.iYoungOldRatio;
+	TInt ratioLimit = (iMinimumPageCount-KMinOldPages)/KMinOldPages;
+	if(iYoungOldRatio>ratioLimit)
+		iYoungOldRatio = ratioLimit;
+
+#ifdef _USE_OLDEST_LISTS
+	iOldOldestRatio = KDefaultOldOldestRatio;
+	if(config.iSpare[2])
+		iOldOldestRatio = config.iSpare[2];
+#endif
+
+	iMinimumPageLimit = (iMinYoungPages * (1 + iYoungOldRatio)) / iYoungOldRatio;
+	if(iMinimumPageLimit<iAbsoluteMinPageCount)
+		iMinimumPageLimit = iAbsoluteMinPageCount;
+
+	TRACEB(("DPager::Init2() live list min=%d max=%d ratio=%d",iMinimumPageCount,iMaximumPageCount,iYoungOldRatio));
+
+	if(iMaximumPageCount<iMinimumPageCount)
+		__NK_ASSERT_ALWAYS(0);
+
+	//
+	// This routine doesn't acquire any mutexes because it should be called before the system
+	// is fully up and running. I.e. called before another thread can preempt this.
+	//
+
+	// Calculate page counts
+	TUint minOldAndOldest = iMinimumPageCount / (1 + iYoungOldRatio);
+	if(minOldAndOldest < KMinOldPages)
+		__NK_ASSERT_ALWAYS(0);
+	if (iMinimumPageCount < minOldAndOldest)
+		__NK_ASSERT_ALWAYS(0);
+	TUint minYoung = iMinimumPageCount - minOldAndOldest;
+	if(minYoung < iMinYoungPages)
+		__NK_ASSERT_ALWAYS(0); // Need at least iMinYoungPages pages mapped to execute worst case CPU instruction
+#ifdef _USE_OLDEST_LISTS
+	// There should always be enough old pages to allow the oldest lists ratio.
+	TUint oldestCount = minOldAndOldest / (1 + iOldOldestRatio);
+	if (!oldestCount)
+		__NK_ASSERT_ALWAYS(0);
+#endif
+	iNumberOfFreePages = 0;
+	iNumberOfDirtyPages = 0;
+
+	// Allocate RAM pages and put them all on the old list
+	RamAllocLock::Lock();
+	iYoungCount = 0;
+	iOldCount = 0;
+#ifdef _USE_OLDEST_LISTS
+	iOldestCleanCount = 0;
+	iOldestDirtyCount = 0;
+#endif
+	Mmu& m = TheMmu;
+	for(TUint i=0; i<iMinimumPageCount; i++)
+		{
+		// Allocate a single page
+		TPhysAddr pagePhys;
+		TInt r = m.AllocRam(&pagePhys, 1, 
+							(Mmu::TRamAllocFlags)(EMemAttNormalCached|Mmu::EAllocNoWipe|Mmu::EAllocNoPagerReclaim), 
+							EPageDiscard);
+		if(r!=KErrNone)
+			__NK_ASSERT_ALWAYS(0);
+		MmuLock::Lock();
+		AddAsFreePage(SPageInfo::FromPhysAddr(pagePhys));
+		MmuLock::Unlock();
+		}
+	RamAllocLock::Unlock();
+
+#ifdef _USE_OLDEST_LISTS
+	TRACEB(("DPager::Init2() end with young=%d old=%d oldClean=%d oldDirty=%d min=%d free=%d max=%d",iYoungCount,iOldCount,iOldestCleanCount,iOldestDirtyCount,iMinimumPageCount,iNumberOfFreePages,iMaximumPageCount));
+#else
+	TRACEB(("DPager::Init2() end with young=%d old=%d min=%d free=%d max=%d",iYoungCount,iOldCount,iMinimumPageCount,iNumberOfFreePages,iMaximumPageCount));
+#endif
+	}
+
+
+#ifdef _DEBUG
+TBool DPager::CheckLists()
+	{
+#if 0
+	__NK_ASSERT_DEBUG(MmuLock::IsHeld());
+	SDblQueLink* head = &iOldList.iA;
+	TInt n = iOldCount;
+	SDblQueLink* link = head;
+	while(n--)
+		{
+		link = link->iNext;
+		if(link==head)
+			return false;
+		}
+	link = link->iNext;
+	if(link!=head)
+		return false;
+
+	head = &iYoungList.iA;
+	n = iYoungCount;
+	link = head;
+	while(n--)
+		{
+		link = link->iNext;
+		if(link==head)
+			return false;
+		}
+	link = link->iNext;
+	if(link!=head)
+		return false;
+
+//	TRACEP(("DP: y=%d o=%d f=%d",iYoungCount,iOldCount,iNumberOfFreePages));
+#endif
+//	TraceCounts();
+	return true;
+	}
+
+void DPager::TraceCounts()
+	{
+	TRACEP(("DP: y=%d o=%d f=%d min=%d max=%d ml=%d res=%d",
+		iYoungCount,iOldCount,iNumberOfFreePages,iMinimumPageCount,
+		iMaximumPageCount,iMinimumPageLimit,iReservePageCount));
+	}
+
+#endif
+
+
+TBool DPager::HaveTooManyPages()
+	{
+	__NK_ASSERT_DEBUG(RamAllocLock::IsHeld());
+	__NK_ASSERT_DEBUG(MmuLock::IsHeld());
+	return iMinimumPageCount+iNumberOfFreePages > iMaximumPageCount;
+	}
+
+
+TBool DPager::HaveMaximumPages()
+	{
+	__NK_ASSERT_DEBUG(RamAllocLock::IsHeld());
+	__NK_ASSERT_DEBUG(MmuLock::IsHeld());
+	return iMinimumPageCount+iNumberOfFreePages >= iMaximumPageCount;
+	}
+
+
+void DPager::AddAsYoungestPage(SPageInfo* aPageInfo)
+	{
+	__NK_ASSERT_DEBUG(MmuLock::IsHeld());
+	__NK_ASSERT_DEBUG(CheckLists());
+	__NK_ASSERT_DEBUG(aPageInfo->PagedState()==SPageInfo::EUnpaged);
+
+	aPageInfo->SetPagedState(SPageInfo::EPagedYoung);
+	iYoungList.AddHead(&aPageInfo->iLink);
+	++iYoungCount;
+	}
+
+
+void DPager::AddAsFreePage(SPageInfo* aPageInfo)
+	{
+	__NK_ASSERT_DEBUG(MmuLock::IsHeld());
+	__NK_ASSERT_DEBUG(CheckLists());
+
+	__NK_ASSERT_DEBUG(aPageInfo->PagedState()==SPageInfo::EUnpaged);
+	TheMmu.PageFreed(aPageInfo);
+	__NK_ASSERT_DEBUG(aPageInfo->PagedState()==SPageInfo::EUnpaged);
+
+	// add as oldest page...
+#ifdef _USE_OLDEST_LISTS
+	aPageInfo->SetPagedState(SPageInfo::EPagedOldestClean);
+	iOldestCleanList.Add(&aPageInfo->iLink);
+	++iOldestCleanCount;
+#else
+	aPageInfo->SetPagedState(SPageInfo::EPagedOld);
+	iOldList.Add(&aPageInfo->iLink);
+	++iOldCount;
+#endif
+
+	Event(EEventPageInFree,aPageInfo);
+	}
+
+
+TInt DPager::PageFreed(SPageInfo* aPageInfo)
+	{
+	__NK_ASSERT_DEBUG(MmuLock::IsHeld());
+	__NK_ASSERT_DEBUG(CheckLists());
+
+	switch(aPageInfo->PagedState())
+		{
+	case SPageInfo::EUnpaged:
+		return KErrNotFound;
+
+	case SPageInfo::EPagedYoung:
+		__NK_ASSERT_DEBUG(iYoungCount);
+		aPageInfo->iLink.Deque();
+		--iYoungCount;
+		break;
+
+	case SPageInfo::EPagedOld:
+		__NK_ASSERT_DEBUG(iOldCount);
+		aPageInfo->iLink.Deque();
+		--iOldCount;
+		break;
+
+#ifdef _USE_OLDEST_LISTS
+	case SPageInfo::EPagedOldestClean:
+		__NK_ASSERT_DEBUG(iOldestCleanCount);
+		aPageInfo->iLink.Deque();
+		--iOldestCleanCount;
+		break;
+
+	case SPageInfo::EPagedOldestDirty:
+		__NK_ASSERT_DEBUG(iOldestDirtyCount);
+		aPageInfo->iLink.Deque();
+		--iOldestDirtyCount;
+		break;
+#endif
+
+	case SPageInfo::EPagedPinned:
+		// this can occur if a pinned mapping is being unmapped when memory is decommitted.
+		// the decommit will have succeeded because the the mapping no longer vetoes this,
+		// however the unpinning hasn't yet got around to changing the page state.
+		// When the state change happens the page will be put back on the live list so
+		// we don't have to do anything now...
+		return KErrNone;
+
+	case SPageInfo::EPagedPinnedMoved:
+		// This page was pinned when it was moved but it has not been returned 
+		// to the free pool yet so make sure it is...
+		aPageInfo->SetPagedState(SPageInfo::EUnpaged);	// Must be unpaged before returned to free pool.
+		return KErrNotFound;
+
+	default:
+		__NK_ASSERT_DEBUG(0);
+		return KErrNotFound;
+		}
+
+	// Update the dirty page count as required...
+	if (aPageInfo->IsDirty())
+		SetClean(*aPageInfo);
+
+	// add as oldest page...
+#ifdef _USE_OLDEST_LISTS
+	aPageInfo->SetPagedState(SPageInfo::EPagedOldestClean);
+	iOldestCleanList.Add(&aPageInfo->iLink);
+	++iOldestCleanCount;
+#else
+	aPageInfo->SetPagedState(SPageInfo::EPagedOld);
+	iOldList.Add(&aPageInfo->iLink);
+	++iOldCount;
+#endif
+
+	return KErrNone;
+	}
+
+
+extern TBool IsPageTableUnpagedRemoveAllowed(SPageInfo* aPageInfo);
+
+void DPager::RemovePage(SPageInfo* aPageInfo)
+	{
+	__NK_ASSERT_DEBUG(MmuLock::IsHeld());
+	__NK_ASSERT_DEBUG(CheckLists());
+
+	switch(aPageInfo->PagedState())
+		{
+	case SPageInfo::EPagedYoung:
+		__NK_ASSERT_DEBUG(iYoungCount);
+		aPageInfo->iLink.Deque();
+		--iYoungCount;
+		break;
+
+	case SPageInfo::EPagedOld:
+		__NK_ASSERT_DEBUG(iOldCount);
+		aPageInfo->iLink.Deque();
+		--iOldCount;
+		break;
+
+#ifdef _USE_OLDEST_LISTS
+	case SPageInfo::EPagedOldestClean:
+		__NK_ASSERT_DEBUG(iOldestCleanCount);
+		aPageInfo->iLink.Deque();
+		--iOldestCleanCount;
+		break;
+
+	case SPageInfo::EPagedOldestDirty:
+		__NK_ASSERT_DEBUG(iOldestDirtyCount);
+		aPageInfo->iLink.Deque();
+		--iOldestDirtyCount;
+		break;
+#endif
+
+	case SPageInfo::EPagedPinned:
+		__NK_ASSERT_DEBUG(0);
+	case SPageInfo::EUnpaged:
+#ifdef _DEBUG
+		if (!IsPageTableUnpagedRemoveAllowed(aPageInfo))
+			__NK_ASSERT_DEBUG(0);
+		break;
+#endif
+	default:
+		__NK_ASSERT_DEBUG(0);
+		return;
+		}
+
+	aPageInfo->SetPagedState(SPageInfo::EUnpaged);
+	}
+
+
+void DPager::ReplacePage(SPageInfo& aOldPageInfo, SPageInfo& aNewPageInfo)
+	{
+	__NK_ASSERT_DEBUG(MmuLock::IsHeld());
+	__NK_ASSERT_DEBUG(CheckLists());
+
+	__NK_ASSERT_DEBUG(aOldPageInfo.PagedState() == aNewPageInfo.PagedState());
+	switch(aOldPageInfo.PagedState())
+		{
+		case SPageInfo::EPagedYoung:
+		case SPageInfo::EPagedOld:
+		case SPageInfo::EPagedOldestClean:
+		case SPageInfo::EPagedOldestDirty:
+			{// Update the list links point to the new page.
+			__NK_ASSERT_DEBUG(iYoungCount);
+			SDblQueLink* prevLink = aOldPageInfo.iLink.iPrev;
+#ifdef _DEBUG
+			SDblQueLink* nextLink = aOldPageInfo.iLink.iNext;
+			__NK_ASSERT_DEBUG(prevLink == aOldPageInfo.iLink.iPrev);
+			__NK_ASSERT_DEBUG(prevLink->iNext == &aOldPageInfo.iLink);
+			__NK_ASSERT_DEBUG(nextLink == aOldPageInfo.iLink.iNext);
+			__NK_ASSERT_DEBUG(nextLink->iPrev == &aOldPageInfo.iLink);
+#endif
+			aOldPageInfo.iLink.Deque();
+			aNewPageInfo.iLink.InsertAfter(prevLink);
+			aOldPageInfo.SetPagedState(SPageInfo::EUnpaged);
+#ifdef _DEBUG
+			__NK_ASSERT_DEBUG(prevLink == aNewPageInfo.iLink.iPrev);
+			__NK_ASSERT_DEBUG(prevLink->iNext == &aNewPageInfo.iLink);
+			__NK_ASSERT_DEBUG(nextLink == aNewPageInfo.iLink.iNext);
+			__NK_ASSERT_DEBUG(nextLink->iPrev == &aNewPageInfo.iLink);
+#endif
+			}
+			break;
+		case SPageInfo::EPagedPinned:
+			// Mark the page as 'pinned moved' so that when the page moving invokes 
+			// Mmu::FreeRam() it returns this page to the free pool.
+			aOldPageInfo.ClearPinCount();
+			aOldPageInfo.SetPagedState(SPageInfo::EPagedPinnedMoved);
+			break;
+		case SPageInfo::EPagedPinnedMoved:
+			// Shouldn't happen as the ram alloc mutex will be held for the 
+			// entire time the page's is paged state == EPagedPinnedMoved.
+		case SPageInfo::EUnpaged:
+			// Shouldn't happen as we only move pinned memory and unpinning will 
+			// atomically add the page to the live list and it can't be removed 
+			// from the live list without the ram alloc mutex.
+			__NK_ASSERT_DEBUG(0);
+			break;
+		}	
+	}
+
+
+SPageInfo* DPager::StealOldestPage()
+	{
+	__NK_ASSERT_DEBUG(RamAllocLock::IsHeld());
+	__NK_ASSERT_DEBUG(MmuLock::IsHeld());
+
+	for(;;)
+		{
+		// find oldest page in list...
+		SDblQueLink* link;
+#ifdef _USE_OLDEST_LISTS
+		if (iOldestCleanCount)
+			{
+			__NK_ASSERT_DEBUG(!iOldestCleanList.IsEmpty());
+			link = iOldestCleanList.Last();
+			}
+		else if (iOldestDirtyCount)
+			{
+			__NK_ASSERT_DEBUG(!iOldestDirtyList.IsEmpty());
+			link = iOldestDirtyList.Last();
+			}
+		else if (iOldCount)
+#else
+		if (iOldCount)
+#endif
+			{
+			__NK_ASSERT_DEBUG(!iOldList.IsEmpty());
+			link = iOldList.Last();
+			}
+		else
+			{
+			__NK_ASSERT_DEBUG(iYoungCount);
+			__NK_ASSERT_ALWAYS(!iYoungList.IsEmpty());
+			link = iYoungList.Last();
+			}
+		SPageInfo* pageInfo = SPageInfo::FromLink(link);
+
+		// steal it from owning object...
+		TInt r = StealPage(pageInfo);
+
+		BalanceAges();
+
+		if(r==KErrNone)
+			return pageInfo; // done
+
+		// loop back and try again
+		}
+	}
+
+
+TInt DPager::RestrictPage(SPageInfo* aPageInfo, TRestrictPagesType aRestriction)
+	{
+	TRACE(("DPager::RestrictPage(0x%08x,%d)",aPageInfo,aRestriction));
+	__NK_ASSERT_DEBUG(MmuLock::IsHeld());
+
+	TInt r;
+	if(aPageInfo->Type()==SPageInfo::EUnused)
+		{
+		// page was unused, so nothing to do...
+		r = KErrNone;
+		}
+	else
+		{
+		// get memory object which owns the page...
+		__NK_ASSERT_DEBUG(aPageInfo->Type()==SPageInfo::EManaged);
+		DMemoryObject* memory = aPageInfo->Owner();
+		memory->Open();
+
+		// try restricting access to page...
+		r = memory->iManager->RestrictPage(memory,aPageInfo,aRestriction);
+		__NK_ASSERT_DEBUG(r!=KErrNotSupported);
+
+		// close memory object...
+		MmuLock::Unlock();
+		memory->AsyncClose();
+		MmuLock::Lock();
+		}
+
+	TRACE(("DPager::RestrictPage returns %d",r));
+	return r;
+	}
+
+
+TInt DPager::StealPage(SPageInfo* aPageInfo)
+	{
+	TRACE(("DPager::StealPage(0x%08x)",aPageInfo));
+	__NK_ASSERT_DEBUG(RamAllocLock::IsHeld());
+	__NK_ASSERT_DEBUG(MmuLock::IsHeld());
+ 
+	__UNLOCK_GUARD_START(MmuLock);
+	RemovePage(aPageInfo);
+
+	TInt r;
+	if(aPageInfo->Type()==SPageInfo::EUnused)
+		{
+		// page was unused, so nothing to do...
+		r = KErrNone;
+		__UNLOCK_GUARD_END(MmuLock);
+		MmuLock::Unlock();
+		}
+	else
+		{
+		// get memory object which owns the page...
+		__NK_ASSERT_DEBUG(aPageInfo->Type()==SPageInfo::EManaged);
+		DMemoryObject* memory = aPageInfo->Owner();
+		memory->Open();
+
+		// try and steal page from memory object...
+		__UNLOCK_GUARD_END(MmuLock); // StealPage must be called without releasing the MmuLock
+		r = memory->iManager->StealPage(memory,aPageInfo);
+		__NK_ASSERT_DEBUG(r!=KErrNotSupported);
+
+		// close memory object...
+		MmuLock::Unlock();
+		memory->AsyncClose();
+		}
+
+	MmuLock::Lock();
+
+	if(r==KErrNone)
+		Event(EEventPageOut,aPageInfo);
+
+	TRACE(("DPager::StealPage returns %d",r));
+	return r;
+	}
+
+
+TInt DPager::DiscardPage(SPageInfo* aOldPageInfo, TUint aBlockZoneId, TBool aBlockRest)
+	{
+	__NK_ASSERT_DEBUG(RamAllocLock::IsHeld());
+	__NK_ASSERT_DEBUG(MmuLock::IsHeld());
+
+	TInt r;
+	// If the page is pinned or if the page is dirty and a general defrag is being 
+	// performed then don't attempt to steal it.
+	if (aOldPageInfo->Type() != SPageInfo::EUnused && 
+		(aOldPageInfo->PagedState() == SPageInfo::EPagedPinned ||
+		(aBlockRest && aOldPageInfo->IsDirty())))
+		{// The page is pinned or is dirty and this is a general defrag so move the page.
+		DMemoryObject* memory = aOldPageInfo->Owner();
+		// Page must be managed if it is pinned or dirty.
+		__NK_ASSERT_DEBUG(aOldPageInfo->Type()==SPageInfo::EManaged);
+		__NK_ASSERT_DEBUG(memory);
+		MmuLock::Unlock();
+		TPhysAddr newAddr;
+		return memory->iManager->MovePage(memory, aOldPageInfo, newAddr, aBlockZoneId, aBlockRest);
+		}
+
+	if (!iNumberOfFreePages)
+		{
+		// Allocate a new page for the live list as it has reached its minimum size.
+		MmuLock::Unlock();
+		SPageInfo* newPageInfo = GetPageFromSystem((Mmu::TRamAllocFlags)(EMemAttNormalCached|Mmu::EAllocNoWipe),
+													aBlockZoneId, aBlockRest);
+		 if (!newPageInfo)
+			return KErrNoMemory;
+
+		// Re-acquire the mmulock and re-check that the page is not pinned or dirty.
+		MmuLock::Lock();
+		if (aOldPageInfo->Type() != SPageInfo::EUnused && 
+			(aOldPageInfo->PagedState() == SPageInfo::EPagedPinned ||
+			(aBlockRest && aOldPageInfo->IsDirty())))
+			{// Page is now pinned or dirty so give up as it is inuse.
+			ReturnPageToSystem(*newPageInfo);
+			MmuLock::Unlock();
+			return KErrInUse;
+			}
+
+		// Attempt to steal the page
+		r = StealPage(aOldPageInfo);
+		__NK_ASSERT_DEBUG(MmuLock::IsHeld());
+
+		if (r == KErrCompletion)
+			{// This was a page table that has been freed but added to the 
+			// live list as a free page.  Remove from live list and continue.
+			__NK_ASSERT_DEBUG(!aOldPageInfo->IsDirty());
+			RemovePage(aOldPageInfo);
+			r = KErrNone;
+			}
+
+		if (r == KErrNone)
+			{// Add the new page to the live list as discarding the old page 
+			// will reduce the live list below the minimum.
+			AddAsFreePage(newPageInfo);
+			// We've successfully discarded the page so return it to the free pool.
+			ReturnPageToSystem(*aOldPageInfo);
+			BalanceAges();
+			}
+		 else
+			{
+			// New page not required so just return it to the system.  This is safe as 
+			// iNumberOfFreePages will have this page counted but as it is not on the live list
+			// noone else can touch it.
+			ReturnPageToSystem(*newPageInfo);
+			}
+		}
+	else
+		{
+		// Attempt to steal the page
+		r = StealPage(aOldPageInfo);
+
+		__NK_ASSERT_DEBUG(MmuLock::IsHeld());
+
+		if (r == KErrCompletion)
+			{// This was a page table that has been freed but added to the 
+			// live list as a free page.  Remove from live list.
+			__NK_ASSERT_DEBUG(!aOldPageInfo->IsDirty());
+			RemovePage(aOldPageInfo);
+			r = KErrNone;
+			}
+
+		if (r == KErrNone)
+			{// We've successfully discarded the page so return it to the free pool.
+			ReturnPageToSystem(*aOldPageInfo);
+			BalanceAges();
+			}
+		}
+	MmuLock::Unlock();
+	return r;	
+	}
+
+
+TBool DPager::TryGrowLiveList()
+	{
+	__NK_ASSERT_DEBUG(RamAllocLock::IsHeld());
+	__NK_ASSERT_DEBUG(MmuLock::IsHeld());
+
+	MmuLock::Unlock();
+	SPageInfo* sparePage = GetPageFromSystem((Mmu::TRamAllocFlags)(EMemAttNormalCached|Mmu::EAllocNoWipe));
+	MmuLock::Lock();
+
+	if(!sparePage)
+		return false;
+
+	// add page to live list...
+	AddAsFreePage(sparePage);
+	return true;
+	}
+
+
+SPageInfo* DPager::GetPageFromSystem(Mmu::TRamAllocFlags aAllocFlags, TUint aBlockZoneId, TBool aBlockRest)
+	{
+	__NK_ASSERT_DEBUG(RamAllocLock::IsHeld());
+
+	TPhysAddr pagePhys;
+	TInt r = TheMmu.AllocRam(&pagePhys, 1, 
+							(Mmu::TRamAllocFlags)(aAllocFlags|Mmu::EAllocNoPagerReclaim), 
+							EPageDiscard, aBlockZoneId, aBlockRest);
+	if(r!=KErrNone)
+		return NULL;
+
+	MmuLock::Lock();
+	++iNumberOfFreePages;
+	MmuLock::Unlock();
+
+	return SPageInfo::FromPhysAddr(pagePhys);
+	}
+
+
+void DPager::ReturnPageToSystem()
+	{
+	__NK_ASSERT_DEBUG(RamAllocLock::IsHeld());
+	__NK_ASSERT_DEBUG(MmuLock::IsHeld());
+
+	ReturnPageToSystem(*StealOldestPage());
+	}
+
+
+void DPager::ReturnPageToSystem(SPageInfo& aPageInfo)
+	{
+	__NK_ASSERT_DEBUG(RamAllocLock::IsHeld());
+	__NK_ASSERT_DEBUG(MmuLock::IsHeld());
+
+	__NK_ASSERT_DEBUG(iNumberOfFreePages>0);
+	--iNumberOfFreePages;
+
+	MmuLock::Unlock();
+
+	TPhysAddr pagePhys = aPageInfo.PhysAddr();
+	TheMmu.FreeRam(&pagePhys, 1, EPageDiscard);
+
+	MmuLock::Lock();
+	}
+
+
+SPageInfo* DPager::PageInAllocPage(Mmu::TRamAllocFlags aAllocFlags)
+	{
+	SPageInfo* pageInfo;
+	TPhysAddr pagePhys;
+
+	RamAllocLock::Lock();
+	MmuLock::Lock();
+
+	// try getting a free page from our live list...
+#ifdef _USE_OLDEST_LISTS
+	if (iOldestCleanCount)
+		{
+		pageInfo = SPageInfo::FromLink(iOldestCleanList.Last());
+		if(pageInfo->Type()==SPageInfo::EUnused)
+			goto get_oldest;
+		}
+#else
+	if(iOldCount)
+		{
+		pageInfo = SPageInfo::FromLink(iOldList.Last());
+		if(pageInfo->Type()==SPageInfo::EUnused)
+			goto get_oldest;
+		}
+#endif
+
+	// try getting a free page from the system pool...
+	if(!HaveMaximumPages())
+		{
+		MmuLock::Unlock();
+		pageInfo = GetPageFromSystem(aAllocFlags);
+		if(pageInfo)
+			goto done;
+		MmuLock::Lock();
+		}
+
+	// as a last resort, steal a page from the live list...
+get_oldest:
+#ifdef _USE_OLDEST_LISTS
+	__NK_ASSERT_ALWAYS(iOldestCleanCount|iOldestDirtyCount|iOldCount|iYoungCount);
+#else
+	__NK_ASSERT_ALWAYS(iOldCount|iYoungCount);
+#endif
+	pageInfo = StealOldestPage();
+	MmuLock::Unlock();
+
+	// make page state same as a freshly allocated page...
+	pagePhys = pageInfo->PhysAddr();
+	TheMmu.PagesAllocated(&pagePhys,1,aAllocFlags);
+
+done:
+	RamAllocLock::Unlock();
+	return pageInfo;
+	}
+
+
+TBool DPager::GetFreePages(TInt aNumPages)
+	{
+	TRACE(("DPager::GetFreePages(%d)",aNumPages));
+
+	__NK_ASSERT_DEBUG(RamAllocLock::IsHeld());
+
+	MmuLock::Lock();
+	while(aNumPages>0 && (TInt)NumberOfFreePages()>=aNumPages)
+		{
+		ReturnPageToSystem();
+		--aNumPages;
+		}
+	MmuLock::Unlock();
+
+	TRACE(("DPager::GetFreePages returns %d",!aNumPages));
+	return !aNumPages;
+	}
+
+
+void DPager::DonatePages(TUint aCount, TPhysAddr* aPages)
+	{
+	TRACE(("DPager::DonatePages(%d,?)",aCount));
+	__ASSERT_CRITICAL;
+	RamAllocLock::Lock();
+	MmuLock::Lock();
+
+	TPhysAddr* end = aPages+aCount;
+	while(aPages<end)
+		{
+		TPhysAddr pagePhys = *aPages++;
+		if(RPageArray::State(pagePhys)!=RPageArray::ECommitted)
+			continue; // page is not present
+
+#ifdef _DEBUG
+		SPageInfo* pi = SPageInfo::SafeFromPhysAddr(pagePhys&~KPageMask);
+		__NK_ASSERT_DEBUG(pi);
+#else
+		SPageInfo* pi = SPageInfo::FromPhysAddr(pagePhys);
+#endif
+		switch(pi->PagedState())
+			{
+		case SPageInfo::EUnpaged:
+			// Change the type of this page to discardable and 
+			// then add it to live list.
+			// Only the DDiscardableMemoryManager should be invoking this and
+			// its pages will be movable before they are donated.
+			__NK_ASSERT_DEBUG(pi->Owner()->iManager->PageType() == EPageMovable);
+			TheMmu.ChangePageType(pi, EPageMovable, EPageDiscard);
+			break;
+
+		case SPageInfo::EPagedYoung:
+		case SPageInfo::EPagedOld:
+#ifdef _USE_OLDEST_LISTS
+		case SPageInfo::EPagedOldestDirty:
+		case SPageInfo::EPagedOldestClean:
+#endif
+			continue; // discard already been allowed
+
+		case SPageInfo::EPagedPinned:
+			__NK_ASSERT_DEBUG(0);
+		default:
+			__NK_ASSERT_DEBUG(0);
+			continue;
+			}
+
+		// put page on live list...
+		AddAsYoungestPage(pi);
+		++iNumberOfFreePages;
+
+		Event(EEventPageDonate,pi);
+
+		// re-balance live list...
+		RemoveExcessPages();
+		BalanceAges();
+		}
+
+	MmuLock::Unlock();
+	RamAllocLock::Unlock();
+	}
+
+
+TInt DPager::ReclaimPages(TUint aCount, TPhysAddr* aPages)
+	{
+	TRACE(("DPager::ReclaimPages(%d,?)",aCount));
+	__ASSERT_CRITICAL;
+	RamAllocLock::Lock();
+	MmuLock::Lock();
+
+	TInt r = KErrNone;
+	TPhysAddr* end = aPages+aCount;
+	while(aPages<end)
+		{
+		TPhysAddr pagePhys = *aPages++;
+		TBool changeType = EFalse;
+
+		if(RPageArray::State(pagePhys)!=RPageArray::ECommitted)
+			{
+			r = KErrNotFound; // too late, page has gone
+			continue;
+			}
+
+#ifdef _DEBUG
+		SPageInfo* pi = SPageInfo::SafeFromPhysAddr(pagePhys&~KPageMask);
+		__NK_ASSERT_DEBUG(pi);
+#else
+		SPageInfo* pi = SPageInfo::FromPhysAddr(pagePhys);
+#endif
+		switch(pi->PagedState())
+			{
+		case SPageInfo::EUnpaged:
+			continue; // discard already been disallowed
+
+		case SPageInfo::EPagedYoung:
+		case SPageInfo::EPagedOld:
+#ifdef _USE_OLDEST_LISTS
+		case SPageInfo::EPagedOldestClean:
+		case SPageInfo::EPagedOldestDirty:
+#endif
+			changeType = ETrue;
+			break; // remove from live list
+
+		case SPageInfo::EPagedPinned:
+			__NK_ASSERT_DEBUG(0);
+		default:
+			__NK_ASSERT_DEBUG(0);
+			break;
+			}
+
+		// check paging list has enough pages before we remove one...
+		if(iNumberOfFreePages<1)
+			{
+			// need more pages so get a page from the system...
+			if(!TryGrowLiveList())
+				{
+				// out of memory...
+				r = KErrNoMemory;
+				break;
+				}
+			// retry the page reclaim...
+			--aPages;
+			continue;
+			}
+
+		if (changeType)
+			{// Change the type of this page to movable, wait until any retries
+			// have been attempted as we can't change a page's type twice.
+			// Only the DDiscardableMemoryManager should be invoking this and
+			// its pages should be movable once they are reclaimed.
+			__NK_ASSERT_DEBUG(pi->Owner()->iManager->PageType() == EPageMovable);
+			TheMmu.ChangePageType(pi, EPageDiscard, EPageMovable);
+			}
+
+		// remove page from paging list...
+		__NK_ASSERT_DEBUG(iNumberOfFreePages>0);
+		--iNumberOfFreePages;
+		RemovePage(pi);
+
+		Event(EEventPageReclaim,pi);
+
+		// re-balance live list...
+		BalanceAges();
+		}
+
+	// we may have added a spare free page to the live list without removing one,
+	// this could cause us to have too many pages, so deal with this...
+	RemoveExcessPages();
+
+	MmuLock::Unlock();
+	RamAllocLock::Unlock();
+	return r;
+	}
+
+
+TInt VMHalFunction(TAny*, TInt aFunction, TAny* a1, TAny* a2);
+
+void DPager::Init3()
+	{
+	TRACEB(("DPager::Init3()"));
+	TheRomMemoryManager->Init3();
+	TheDataPagedMemoryManager->Init3();
+	TheCodePagedMemoryManager->Init3();
+	TInt r = Kern::AddHalEntry(EHalGroupVM, VMHalFunction, 0);
+	__NK_ASSERT_ALWAYS(r==KErrNone);
+	}
+
+
+void DPager::Fault(TFault aFault)
+	{
+	Kern::Fault("DPager",aFault);
+	}
+
+
+void DPager::BalanceAges()
+	{
+	__NK_ASSERT_DEBUG(MmuLock::IsHeld());
+	TBool restrictPage = EFalse;
+	SPageInfo* pageInfo = NULL;
+#ifdef _USE_OLDEST_LISTS
+	TUint oldestCount = iOldestCleanCount + iOldestDirtyCount;
+	if((iOldCount + oldestCount) * iYoungOldRatio < iYoungCount)
+#else
+	if (iOldCount * iYoungOldRatio < iYoungCount)
+#endif
+		{
+		// Need more old pages so make one young page into an old page...
+		__NK_ASSERT_DEBUG(!iYoungList.IsEmpty());
+		__NK_ASSERT_DEBUG(iYoungCount);
+		SDblQueLink* link = iYoungList.Last()->Deque();
+		--iYoungCount;
+
+		pageInfo = SPageInfo::FromLink(link);
+		pageInfo->SetPagedState(SPageInfo::EPagedOld);
+
+		iOldList.AddHead(link);
+		++iOldCount;
+
+		Event(EEventPageAged,pageInfo);
+		// Delay restricting the page until it is safe to release the MmuLock.
+		restrictPage = ETrue;
+		}
+
+#ifdef _USE_OLDEST_LISTS
+	// Check we have enough oldest pages.
+	if (oldestCount * iOldOldestRatio < iOldCount)
+		{
+		__NK_ASSERT_DEBUG(!iOldList.IsEmpty());
+		__NK_ASSERT_DEBUG(iOldCount);
+		SDblQueLink* link = iOldList.Last()->Deque();
+		--iOldCount;
+
+		SPageInfo* oldestPageInfo = SPageInfo::FromLink(link);
+		if (oldestPageInfo->IsDirty())
+			{
+			oldestPageInfo->SetPagedState(SPageInfo::EPagedOldestDirty);
+			iOldestDirtyList.AddHead(link);
+			++iOldestDirtyCount;
+			Event(EEventPageAgedDirty,oldestPageInfo);
+			}
+		else
+			{
+			oldestPageInfo->SetPagedState(SPageInfo::EPagedOldestClean);
+			iOldestCleanList.AddHead(link);
+			++iOldestCleanCount;
+			Event(EEventPageAgedClean,oldestPageInfo);
+			}
+		}
+#endif
+	if (restrictPage)
+		{
+		// Make the recently aged old page inaccessible.  This is done last as it 
+		// will release the MmuLock and therefore the page counts may otherwise change.
+		RestrictPage(pageInfo,ERestrictPagesNoAccessForOldPage);
+		}
+	}
+
+
+void DPager::RemoveExcessPages()
+	{
+	__NK_ASSERT_DEBUG(RamAllocLock::IsHeld());
+	__NK_ASSERT_DEBUG(MmuLock::IsHeld());
+	while(HaveTooManyPages())
+		ReturnPageToSystem();
+	}
+
+
+void DPager::RejuvenatePageTable(TPte* aPt)
+	{
+	SPageInfo* pi = SPageInfo::FromPhysAddr(Mmu::PageTablePhysAddr(aPt));
+
+	SPageTableInfo* pti = SPageTableInfo::FromPtPtr(aPt);
+	if(!pti->IsDemandPaged())
+		{
+		__NK_ASSERT_DEBUG(pi->PagedState()==SPageInfo::EUnpaged);
+		return;
+		}
+
+	TRACE2(("DP: %O Rejuvenate PT 0x%08x 0x%08x",TheCurrentThread,pi->PhysAddr(),aPt));
+	switch(pi->PagedState())
+		{
+	case SPageInfo::EPagedYoung:
+	case SPageInfo::EPagedOld:
+#ifdef _USE_OLDEST_LISTS
+	case SPageInfo::EPagedOldestClean:
+	case SPageInfo::EPagedOldestDirty:
+#endif
+		RemovePage(pi);
+		AddAsYoungestPage(pi);
+		BalanceAges();
+		break;
+
+	case SPageInfo::EUnpaged:
+		AddAsYoungestPage(pi);
+		BalanceAges();
+		break;
+
+	case SPageInfo::EPagedPinned:
+		break;
+
+	default:
+		__NK_ASSERT_DEBUG(0);
+		break;
+		}
+	}
+
+TInt DPager::PteAndInfoFromLinAddr(	TInt aOsAsid, TLinAddr aAddress, DMemoryMappingBase* aMapping, 
+									TUint aMapInstanceCount, TPte*& aPte, SPageInfo*& aPageInfo)
+	{
+	__NK_ASSERT_DEBUG(MmuLock::IsHeld());	
+
+	// Verify the mapping is still mapped and has not been reused.
+	if (aMapInstanceCount != aMapping->MapInstanceCount() || aMapping->BeingDetached())
+		return KErrAbort;
+
+	aPte = Mmu::SafePtePtrFromLinAddr(aAddress,aOsAsid);
+	if(!aPte)
+		return KErrNotFound;
+
+	TPte pte = *aPte;
+	if(pte==KPteUnallocatedEntry)
+		return KErrNotFound;
+
+	SPageInfo* pi = SPageInfo::SafeFromPhysAddr(pte & ~KPageMask);
+	if(!pi)
+		return KErrNotFound;
+	aPageInfo = pi;
+
+	return KErrNone;
+	}
+
+TInt DPager::TryRejuvenate(	TInt aOsAsid, TLinAddr aAddress, TUint aAccessPermissions, TLinAddr aPc,
+							DMemoryMappingBase* aMapping, TUint aMapInstanceCount, DThread* aThread, 
+							TAny* aExceptionInfo)
+	{
+	__NK_ASSERT_DEBUG(MmuLock::IsHeld());
+
+	SPageInfo* pi;
+	TPte* pPte;
+	TPte pte;
+	TInt r = PteAndInfoFromLinAddr(aOsAsid, aAddress, aMapping, aMapInstanceCount, pPte, pi);
+	if (r != KErrNone)
+		{
+		if (aThread->IsRealtime())
+			{// This thread is real time so it shouldn't be accessing paged out paged memory
+			// unless there is a paging trap.
+			MmuLock::Unlock();
+			// Ensure that we abort when the thread is not allowed to access paged out pages.
+			if (CheckRealtimeThreadFault(aThread, aExceptionInfo) != KErrNone)
+				r = KErrAbort;
+			MmuLock::Lock();
+			}
+		return r;
+		}
+	pte = *pPte;
+	SPageInfo::TType type = pi->Type();
+	SPageInfo::TPagedState state = pi->PagedState();
+
+	if (aThread->IsRealtime() && 
+		state != SPageInfo::EPagedPinned && 
+		state != SPageInfo::EPagedPinnedMoved)
+		{// This thread is real time so it shouldn't be accessing unpinned paged memory
+		// unless there is a paging trap.
+		MmuLock::Unlock();
+		r = CheckRealtimeThreadFault(aThread, aExceptionInfo);
+		MmuLock::Lock();
+		if (r != KErrNone)
+			return r;
+		// We had to release the MmuLock have to reverify the status of the page and mappings.
+		r = PteAndInfoFromLinAddr(aOsAsid, aAddress, aMapping, aMapInstanceCount, pPte, pi);
+		if (r != KErrNone)
+			return r;
+		pte = *pPte;
+		type = pi->Type();
+		state = pi->PagedState();
+		}
+
+	if (type != SPageInfo::EManaged)
+		return KErrNotFound;
+
+	if(state==SPageInfo::EUnpaged)
+		return KErrNotFound;
+
+	DMemoryObject* memory = pi->Owner();
+	TUint index = pi->Index();
+
+	TPhysAddr page = memory->iPages.Page(index);
+	if(!RPageArray::IsPresent(page))
+		return KErrNotFound;
+
+	TPhysAddr physAddr = pi->PhysAddr();
+	if ((page^physAddr) >= (TPhysAddr)KPageSize)
+		{// Page array entry should contain same physical address as PTE unless the 
+		// page has or is being moved and this mapping accessed the page.
+		// Get the page info for the page that we should be using.
+		physAddr = page & ~KPageMask;
+		pi = SPageInfo::SafeFromPhysAddr(physAddr);
+		if(!pi)
+			return KErrNotFound;
+
+		type = pi->Type();
+		if (type!=SPageInfo::EManaged)
+			return KErrNotFound;
+
+		state = pi->PagedState();
+		if(state==SPageInfo::EUnpaged)
+			return KErrNotFound;
+
+		memory = pi->Owner();
+		index = pi->Index();
+
+		// Update pte to point to the correct physical address for this memory object's page.
+		pte = (pte & KPageMask) | physAddr;
+		}
+
+	if(aAccessPermissions&EReadWrite)
+		{// The mapping that took the fault permits writes and is still attached 
+		// to the memory object therefore the object can't be read only.
+		__NK_ASSERT_DEBUG(!memory->IsReadOnly());
+		SetWritable(*pi);
+		}
+
+	pte = Mmu::MakePteAccessible(pte,aAccessPermissions&EReadWrite);
+	TRACE2(("!PTE %x=%x",pPte,pte));
+	*pPte = pte;
+	CacheMaintenance::SinglePteUpdated((TLinAddr)pPte);
+	InvalidateTLBForPage((aAddress&~KPageMask)|aOsAsid);
+
+	Event(EEventPageRejuvenate,pi,aPc,aAddress,aAccessPermissions);
+
+	TBool balance = false;
+#ifdef _USE_OLDEST_LISTS
+	if(	state==SPageInfo::EPagedYoung || state==SPageInfo::EPagedOld || 
+		state==SPageInfo::EPagedOldestClean || state==SPageInfo::EPagedOldestDirty)
+#else
+	if(state==SPageInfo::EPagedYoung || state==SPageInfo::EPagedOld)
+#endif
+		{
+		RemovePage(pi);
+		AddAsYoungestPage(pi);
+		// delay BalanceAges because we don't want to release MmuLock until after
+		// RejuvenatePageTable has chance to look at the page table page...
+		balance = true;
+		}
+	else
+		{// Clear the modifier so that if this page is being moved then this 
+		// access is detected. For non-pinned pages the modifier is cleared 
+		// by RemovePage().
+		__NK_ASSERT_DEBUG(state==SPageInfo::EPagedPinned);
+		pi->SetModifier(0);
+		}
+
+	RejuvenatePageTable(pPte);
+
+	if(balance)
+		BalanceAges();
+
+	return KErrNone;
+	}
+
+
+TInt DPager::PageInAllocPages(TPhysAddr* aPages, TUint aCount, Mmu::TRamAllocFlags aAllocFlags)
+	{
+	TUint n = 0;
+	while(n<aCount)
+		{
+		SPageInfo* pi = PageInAllocPage(aAllocFlags);
+		if(!pi)
+			goto fail;
+		aPages[n++] = pi->PhysAddr();
+		}
+	return KErrNone;
+fail:
+	PageInFreePages(aPages,n);
+	return KErrNoMemory;
+	}
+
+
+void DPager::PageInFreePages(TPhysAddr* aPages, TUint aCount)
+	{
+	while(aCount--)
+		{
+		MmuLock::Lock();
+		SPageInfo* pi = SPageInfo::FromPhysAddr(aPages[aCount]);
+		switch(pi->PagedState())
+			{
+		case SPageInfo::EPagedYoung:
+		case SPageInfo::EPagedOld:
+#ifdef _USE_OLDEST_LISTS
+		case SPageInfo::EPagedOldestClean:
+		case SPageInfo::EPagedOldestDirty:
+#endif
+			RemovePage(pi);
+			// fall through...
+		case SPageInfo::EUnpaged:
+			AddAsFreePage(pi);
+			break;
+
+		case SPageInfo::EPagedPinned:
+			__NK_ASSERT_DEBUG(0);
+			break;
+		default:
+			__NK_ASSERT_DEBUG(0);
+			break;
+			}
+		MmuLock::Unlock();
+		}
+	}
+
+
+void DPager::PagedInUnneeded(SPageInfo* aPageInfo)
+	{
+	__NK_ASSERT_DEBUG(MmuLock::IsHeld());
+	Event(EEventPageInUnneeded,aPageInfo);
+	AddAsFreePage(aPageInfo);
+	}
+
+
+void DPager::PagedIn(SPageInfo* aPageInfo)
+	{
+	__NK_ASSERT_DEBUG(MmuLock::IsHeld());
+	switch(aPageInfo->PagedState())
+		{
+	case SPageInfo::EPagedYoung:
+	case SPageInfo::EPagedOld:
+#ifdef _USE_OLDEST_LISTS
+	case SPageInfo::EPagedOldestClean:
+	case SPageInfo::EPagedOldestDirty:
+#endif
+		RemovePage(aPageInfo);
+		AddAsYoungestPage(aPageInfo);
+		BalanceAges();
+		break;
+
+	case SPageInfo::EUnpaged:
+		AddAsYoungestPage(aPageInfo);
+		BalanceAges();
+		break;
+
+	case SPageInfo::EPagedPinned:
+		// Clear the modifier so that if this page is being moved then this 
+		// access is detected. For non-pinned pages the modifier is cleared by RemovePage().
+		aPageInfo->SetModifier(0);
+		break;
+
+	default:
+		__NK_ASSERT_DEBUG(0);
+		break;
+		}
+	}
+
+
+void DPager::PagedInPinned(SPageInfo* aPageInfo, TPinArgs& aPinArgs)
+	{
+	__NK_ASSERT_DEBUG(MmuLock::IsHeld());
+	Pin(aPageInfo,aPinArgs);
+	}
+
+
+void DPager::Pin(SPageInfo* aPageInfo, TPinArgs& aPinArgs)
+	{
+	__ASSERT_CRITICAL;
+	__NK_ASSERT_DEBUG(MmuLock::IsHeld());
+	__NK_ASSERT_DEBUG(aPinArgs.HaveSufficientPages(1));
+
+	aPageInfo->IncPinCount();
+	Event(EEventPagePin,aPageInfo);
+
+	// remove page from live list...
+	switch(aPageInfo->PagedState())
+		{
+	case SPageInfo::EPagedYoung:
+		__NK_ASSERT_DEBUG(iYoungCount);
+		aPageInfo->iLink.Deque();
+		--iYoungCount;
+		__NK_ASSERT_DEBUG(aPageInfo->PinCount()==1);
+		break;
+
+	case SPageInfo::EPagedOld:
+		__NK_ASSERT_DEBUG(iOldCount);
+		aPageInfo->iLink.Deque();
+		--iOldCount;
+		__NK_ASSERT_DEBUG(aPageInfo->PinCount()==1);
+		break;
+
+#ifdef _USE_OLDEST_LISTS
+	case SPageInfo::EPagedOldestClean:
+		__NK_ASSERT_DEBUG(iOldestCleanCount);
+		aPageInfo->iLink.Deque();
+		--iOldestCleanCount;
+		__NK_ASSERT_DEBUG(aPageInfo->PinCount()==1);
+		break;
+
+	case SPageInfo::EPagedOldestDirty:
+		__NK_ASSERT_DEBUG(iOldestDirtyCount);
+		aPageInfo->iLink.Deque();
+		--iOldestDirtyCount;
+		__NK_ASSERT_DEBUG(aPageInfo->PinCount()==1);
+		break;
+#endif
+
+	case SPageInfo::EPagedPinned:
+		// nothing more to do...
+		__NK_ASSERT_DEBUG(aPageInfo->PinCount()>1);
+		return;
+
+	case SPageInfo::EUnpaged:
+		__NK_ASSERT_DEBUG(aPageInfo->PinCount()==1);
+		TRACE2(("DPager::PinPage page was unpaged"));
+		// This could be a page in the process of being stolen.
+		// Could also be page for storing page table infos, which aren't necessarily
+		// on the live list.
+		break;
+
+	default:
+		__NK_ASSERT_DEBUG(0);
+		return;
+		}
+
+	// page has now been removed from the live list and is pinned...
+	aPageInfo->SetPagedState(SPageInfo::EPagedPinned);
+
+	if(aPinArgs.iReplacementPages==TPinArgs::EUseReserveForPinReplacementPages)
+		{
+		// pinned paged counts as coming from reserve pool...
+		aPageInfo->SetPinnedReserve();
+		}
+	else
+		{
+		// we used up a replacement page...
+		--aPinArgs.iReplacementPages;
+		}
+
+	BalanceAges();
+	}
+
+
+void DPager::Unpin(SPageInfo* aPageInfo, TPinArgs& aPinArgs)
+	{
+	__ASSERT_CRITICAL;
+	__NK_ASSERT_DEBUG(MmuLock::IsHeld());
+	__NK_ASSERT_DEBUG(aPageInfo->PagedState()==SPageInfo::EPagedPinned);
+	__NK_ASSERT_DEBUG(aPageInfo->PinCount()>0);
+
+	TUint pinCount = aPageInfo->DecPinCount();
+	Event(EEventPageUnpin,aPageInfo);
+
+	if(pinCount)
+		return;
+
+	aPageInfo->SetPagedState(SPageInfo::EUnpaged);
+
+	if(!aPageInfo->ClearPinnedReserve())
+		{
+		// was not a pinned reserve page, so we how have a spare replacement page,
+		// which can be used again or freed later ...
+		__NK_ASSERT_DEBUG(aPinArgs.iReplacementPages!=TPinArgs::EUseReserveForPinReplacementPages);
+		++aPinArgs.iReplacementPages;
+		}
+
+	AddAsYoungestPage(aPageInfo);
+	BalanceAges();
+	}
+
+
+TInt TPinArgs::AllocReplacementPages(TUint aNumPages)
+	{
+	if(iUseReserve)
+		{
+		__NK_ASSERT_DEBUG(iReplacementPages==0 || iReplacementPages==EUseReserveForPinReplacementPages);
+		iReplacementPages = EUseReserveForPinReplacementPages;
+		}
+	else
+		{
+		if(aNumPages>iReplacementPages)
+			{
+			if(!ThePager.AllocPinReplacementPages(aNumPages-iReplacementPages))
+				return KErrNoMemory;
+			iReplacementPages = aNumPages;
+			}
+		}
+	return KErrNone;
+	}
+
+
+void TPinArgs::FreeReplacementPages()
+	{
+	if(iReplacementPages!=0 && iReplacementPages!=EUseReserveForPinReplacementPages)
+		ThePager.FreePinReplacementPages(iReplacementPages);
+	iReplacementPages = 0;
+	}
+
+
+TBool DPager::AllocPinReplacementPages(TUint aNumPages)
+	{
+	TRACE2(("DPager::AllocPinReplacementPages(0x%x)",aNumPages));
+	__ASSERT_CRITICAL;
+	RamAllocLock::Lock();
+	MmuLock::Lock();
+
+	TBool ok = false;
+	do
+		{
+		if(iNumberOfFreePages>=aNumPages)
+			{
+			iNumberOfFreePages -= aNumPages;
+			ok = true;
+			break;
+			}
+		}
+	while(TryGrowLiveList());
+
+	MmuLock::Unlock();
+	RamAllocLock::Unlock();
+	return ok;
+	}
+
+
+void DPager::FreePinReplacementPages(TUint aNumPages)
+	{
+	TRACE2(("DPager::FreePinReplacementPage(0x%x)",aNumPages));
+	__ASSERT_CRITICAL;
+
+	RamAllocLock::Lock();
+	MmuLock::Lock();
+
+	iNumberOfFreePages += aNumPages;
+	RemoveExcessPages();
+
+	MmuLock::Unlock();
+	RamAllocLock::Unlock();
+	}
+
+
+TBool DPager::ReservePage()
+	{
+	__NK_ASSERT_DEBUG(RamAllocLock::IsHeld());
+	__NK_ASSERT_DEBUG(MmuLock::IsHeld());
+	__ASSERT_CRITICAL;
+	__NK_ASSERT_DEBUG(iMinimumPageCount >= iMinimumPageLimit+iReservePageCount);
+	while(iMinimumPageCount==iMinimumPageLimit+iReservePageCount && iNumberOfFreePages==0)
+		{
+		if(!TryGrowLiveList())
+			return false;
+		}
+	if(iMinimumPageCount==iMinimumPageLimit+iReservePageCount)
+		{
+		++iMinimumPageCount;
+		--iNumberOfFreePages;
+		if(iMinimumPageCount>iMaximumPageCount)
+			iMaximumPageCount = iMinimumPageCount;
+		}
+	++iReservePageCount;
+	__NK_ASSERT_DEBUG(iMinimumPageCount >= iMinimumPageLimit+iReservePageCount);
+	__NK_ASSERT_DEBUG(iMinimumPageCount+iNumberOfFreePages <= iMaximumPageCount);
+	return ETrue;
+	}
+
+
+TBool DPager::ReservePages(TUint aRequiredCount, TUint& aCount)
+	{
+	__ASSERT_CRITICAL;
+
+	RamAllocLock::Lock();
+	MmuLock::Lock();
+	while(aCount<aRequiredCount)
+		{
+		if(!ReservePage())
+			break;
+		++aCount;
+		MmuLock::Flash();
+		}
+	TBool enoughPages = aCount==aRequiredCount;
+	MmuLock::Unlock();
+	RamAllocLock::Unlock();
+
+	if(!enoughPages)
+		UnreservePages(aCount);
+
+	return enoughPages;
+	}
+
+
+void DPager::UnreservePages(TUint& aCount)
+	{
+	MmuLock::Lock();
+	iReservePageCount -= aCount;
+	aCount = 0;
+	MmuLock::Unlock();
+	}
+
+
+TInt DPager::CheckRealtimeThreadFault(DThread* aThread, TAny* aExceptionInfo)
+	{
+	// realtime threads shouldn't take paging faults...
+	DThread* client = aThread->iIpcClient;
+
+	// If iIpcClient is set then we are accessing the address space of a remote thread.  If we are
+	// in an IPC trap, this will contain information the local and remote addresses being accessed.
+	// If this is not set then we assume than any fault must be the fault of a bad remote address.
+	TIpcExcTrap* ipcTrap = (TIpcExcTrap*)aThread->iExcTrap;
+	if (ipcTrap && !ipcTrap->IsTIpcExcTrap())
+		ipcTrap = 0;
+	if (client && (!ipcTrap || ipcTrap->ExcLocation(aThread, aExceptionInfo) == TIpcExcTrap::EExcRemote))
+		{
+		// kill client thread...
+		if(K::IllegalFunctionForRealtimeThread(client,"Access to Paged Memory (by other thread)"))
+			{
+			// treat memory access as bad...
+			return KErrAbort;
+			}
+		// else thread is in 'warning only' state so allow paging...
+		}
+	else
+		{
+		// kill current thread...
+		if(K::IllegalFunctionForRealtimeThread(NULL,"Access to Paged Memory"))
+			{
+			// if current thread is in critical section, then the above kill will be deferred
+			// and we will continue executing. We will handle this by returning an error
+			// which means that the thread will take an exception (which hopefully is XTRAPed!)
+			return KErrAbort;
+			}
+		// else thread is in 'warning only' state so allow paging...
+		}
+	return KErrNone;
+	}
+
+
+TInt DPager::HandlePageFault(	TLinAddr aPc, TLinAddr aFaultAddress, TUint aFaultAsid, TUint aFaultIndex,
+								TUint aAccessPermissions, DMemoryObject* aMemory, DMemoryMapping* aMapping,
+								TUint aMapInstanceCount, DThread* aThread, TAny* aExceptionInfo)
+	{
+	MmuLock::Lock();
+	TInt r = TryRejuvenate(	aFaultAsid, aFaultAddress, aAccessPermissions, aPc, aMapping, aMapInstanceCount,
+							aThread, aExceptionInfo);
+	if(r == KErrNone || r == KErrAbort)
+		{
+		MmuLock::Unlock();
+		}
+	else
+		{
+		// rejuvenate failed, call memory manager to page in memory...
+		Event(EEventPageInStart, 0, aPc, aFaultAddress, aAccessPermissions);
+		MmuLock::Unlock();
+		TheThrashMonitor.NotifyStartPaging();
+
+		DMemoryManager* manager = aMemory->iManager;
+		r = manager->HandleFault(aMemory, aFaultIndex, aMapping, aMapInstanceCount, aAccessPermissions);
+
+		TheThrashMonitor.NotifyEndPaging();
+		}
+	return r;
+	}
+
+
+TInt DPager::ResizeLiveList()
+	{
+	MmuLock::Lock();
+	TUint min = iMinimumPageCount;
+	TUint max = iMaximumPageCount;
+	MmuLock::Unlock();
+	return ResizeLiveList(min,max);
+	}
+
+
+TInt DPager::ResizeLiveList(TUint aMinimumPageCount, TUint aMaximumPageCount)
+	{
+	TRACE(("DPager::ResizeLiveList(%d,%d) current young=%d old=%d min=%d free=%d max=%d",aMinimumPageCount,aMaximumPageCount,iYoungCount,iOldCount,iMinimumPageCount,iNumberOfFreePages,iMaximumPageCount));
+	if(!aMaximumPageCount)
+		{
+		aMinimumPageCount = iInitMinimumPageCount;
+		aMaximumPageCount = iInitMaximumPageCount;
+		}
+	if (aMaximumPageCount > KAbsoluteMaxPageCount)
+		aMaximumPageCount = KAbsoluteMaxPageCount;
+
+	// Min must not be greater than max...
+	if(aMinimumPageCount>aMaximumPageCount)
+		return KErrArgument;
+
+	NKern::ThreadEnterCS();
+	RamAllocLock::Lock();
+
+	MmuLock::Lock();
+
+	// Make sure aMinimumPageCount is not less than absolute minimum we can cope with...
+	iMinimumPageLimit = iMinYoungPages * (1 + iYoungOldRatio) / iYoungOldRatio
+						+ DPageReadRequest::ReservedPagesRequired();
+	if(iMinimumPageLimit<iAbsoluteMinPageCount)
+		iMinimumPageLimit = iAbsoluteMinPageCount;
+	if(aMinimumPageCount<iMinimumPageLimit+iReservePageCount)
+		aMinimumPageCount = iMinimumPageLimit+iReservePageCount;
+	if(aMaximumPageCount<aMinimumPageCount)
+		aMaximumPageCount=aMinimumPageCount;
+
+	// Increase iMaximumPageCount?
+	TInt extra = aMaximumPageCount-iMaximumPageCount;
+	if(extra>0)
+		iMaximumPageCount += extra;
+
+	// Reduce iMinimumPageCount?
+	TInt spare = iMinimumPageCount-aMinimumPageCount;
+	if(spare>0)
+		{
+		iMinimumPageCount -= spare;
+		iNumberOfFreePages += spare;
+		}
+
+	// Increase iMinimumPageCount?
+	TInt r=KErrNone;
+	while(iMinimumPageCount<aMinimumPageCount)
+		{
+		TUint newMin = aMinimumPageCount;
+		TUint maxMin = iMinimumPageCount+iNumberOfFreePages;
+		if(newMin>maxMin)
+			newMin = maxMin;
+
+		TUint delta = newMin-iMinimumPageCount;
+		if(delta)
+			{
+			iMinimumPageCount = newMin;
+			iNumberOfFreePages -= delta;
+			continue;
+			}
+
+		if(!TryGrowLiveList())
+			{
+			r=KErrNoMemory;
+			break;
+			}
+		}
+
+	// Reduce iMaximumPageCount?
+	while(iMaximumPageCount>aMaximumPageCount)
+		{
+		TUint newMax = aMaximumPageCount;
+		TUint minMax = iMinimumPageCount+iNumberOfFreePages;
+		if(newMax<minMax)
+			newMax = minMax;
+
+		TUint delta = iMaximumPageCount-newMax;
+		if(delta)
+			{
+			iMaximumPageCount = newMax;
+			continue;
+			}
+
+		ReturnPageToSystem();
+		}
+
+	TRACE(("DPager::ResizeLiveList end with young=%d old=%d min=%d free=%d max=%d",iYoungCount,iOldCount,iMinimumPageCount,iNumberOfFreePages,iMaximumPageCount));
+
+#ifdef BTRACE_KERNEL_MEMORY
+	BTrace4(BTrace::EKernelMemory,BTrace::EKernelMemoryDemandPagingCache,iMinimumPageCount << KPageShift);
+#endif
+
+	MmuLock::Unlock();
+
+	RamAllocLock::Unlock();
+	NKern::ThreadLeaveCS();
+
+	return r;
+	}
+
+
+void DPager::FlushAll()
+	{
+	NKern::ThreadEnterCS();
+	RamAllocLock::Lock();
+
+	TRACE(("DPager::FlushAll() live list young=%d old=%d min=%d free=%d max=%d",iYoungCount,iOldCount,iMinimumPageCount,iNumberOfFreePages,iMaximumPageCount));
+
+	// look at all RAM pages in the system, and unmap all those used for paging
+	const TUint32* piMap = (TUint32*)KPageInfoMap;
+	const TUint32* piMapEnd = piMap+(KNumPageInfoPages>>5);
+	SPageInfo* pi = (SPageInfo*)KPageInfoLinearBase;
+	MmuLock::Lock();
+	do
+		{
+		SPageInfo* piNext = pi+(KPageInfosPerPage<<5);
+		for(TUint32 piFlags=*piMap++; piFlags; piFlags>>=1)
+			{
+			if(!(piFlags&1))
+				{
+				pi += KPageInfosPerPage;
+				continue;
+				}
+			SPageInfo* piEnd = pi+KPageInfosPerPage;
+			do
+				{
+				SPageInfo::TPagedState state = pi->PagedState();
+#ifdef _USE_OLDEST_LISTS
+				if (state==SPageInfo::EPagedYoung || state==SPageInfo::EPagedOld ||
+					state==SPageInfo::EPagedOldestClean || state==SPageInfo::EPagedOldestDirty)
+#else
+				if(state==SPageInfo::EPagedYoung || state==SPageInfo::EPagedOld)
+#endif
+					{
+					if (pi->Type() != SPageInfo::EUnused)
+						{
+						TInt r = StealPage(pi);
+						if(r==KErrNone)
+							AddAsFreePage(pi);
+						MmuLock::Flash();
+						}
+					}
+				++pi;
+				if(((TUint)pi&(0xf<<KPageInfoShift))==0)
+					MmuLock::Flash(); // every 16 page infos
+				}
+			while(pi<piEnd);
+			}
+		pi = piNext;
+		}
+	while(piMap<piMapEnd);
+	MmuLock::Unlock();
+
+	// reduce live page list to a minimum
+	while(GetFreePages(1)) {}; 
+
+	TRACE(("DPager::FlushAll() end with young=%d old=%d min=%d free=%d max=%d",iYoungCount,iOldCount,iMinimumPageCount,iNumberOfFreePages,iMaximumPageCount));
+
+	RamAllocLock::Unlock();
+	NKern::ThreadLeaveCS();
+	}
+
+
+void DPager::GetLiveListInfo(SVMCacheInfo& aInfo)
+	{
+	MmuLock::Lock(); // ensure consistent set of values are read...
+	aInfo.iMinSize = iMinimumPageCount<<KPageShift;
+	aInfo.iMaxSize = iMaximumPageCount<<KPageShift;
+	aInfo.iCurrentSize = (iMinimumPageCount+iNumberOfFreePages)<<KPageShift;
+	aInfo.iMaxFreeSize = iNumberOfFreePages<<KPageShift;
+	MmuLock::Unlock();
+	}
+
+
+void DPager::GetEventInfo(SVMEventInfo& aInfoOut)
+	{
+	MmuLock::Lock(); // ensure consistent set of values are read...
+	aInfoOut = iEventInfo;
+	MmuLock::Unlock();
+	}
+
+
+void DPager::ResetEventInfo()
+	{
+	MmuLock::Lock();
+	memclr(&iEventInfo, sizeof(iEventInfo));
+	MmuLock::Unlock();
+	}
+
+
+TInt TestPageState(TLinAddr aAddr)
+	{
+	DMemModelProcess* process = (DMemModelProcess*)TheCurrentThread->iOwningProcess;
+	// Get the os asid of current thread's process so no need to open a reference on it.
+	TInt osAsid = process->OsAsid();
+	TPte* ptePtr = 0;
+	TPte pte = 0;
+	TInt r = 0;
+	SPageInfo* pageInfo = NULL;
+
+	NKern::ThreadEnterCS();
+
+	TUint offsetInMapping;
+	TUint mapInstanceCount;
+	DMemoryMapping* mapping = MM::FindMappingInAddressSpace(osAsid, aAddr, 1, offsetInMapping, mapInstanceCount);
+
+	MmuLock::Lock();
+
+	if(mapping)
+		{
+		DMemoryObject* memory = mapping->Memory();
+		if(mapInstanceCount == mapping->MapInstanceCount() && memory)
+			{
+			DMemoryManager* manager = memory->iManager;
+			if(manager==TheCodePagedMemoryManager)
+				r |= EPageStateInRamCode|EPageStatePaged;
+			}
+		}
+
+	ptePtr = Mmu::SafePtePtrFromLinAddr(aAddr,osAsid);
+	if (!ptePtr)
+		goto done;
+	pte = *ptePtr;
+	if (pte == KPteUnallocatedEntry)
+		goto done;		
+	r |= EPageStatePtePresent;
+	if (pte!=Mmu::MakePteInaccessible(pte,0))
+		r |= EPageStatePteValid;
+	
+	pageInfo = SPageInfo::SafeFromPhysAddr(pte&~KPageMask);
+	if(pageInfo)
+		{
+		r |= pageInfo->Type();
+		r |= pageInfo->PagedState()<<8;
+		}
+done:
+	MmuLock::Unlock();
+	if(mapping)
+		mapping->Close();
+	NKern::ThreadLeaveCS();
+	return r;
+	}
+
+
+
+TInt VMHalFunction(TAny*, TInt aFunction, TAny* a1, TAny* a2)
+	{
+	switch(aFunction)
+		{
+	case EVMHalFlushCache:
+		if(!TheCurrentThread->HasCapability(ECapabilityWriteDeviceData,__PLATSEC_DIAGNOSTIC_STRING("Checked by VMHalFunction(EVMHalFlushCache)")))
+			K::UnlockedPlatformSecurityPanic();
+		ThePager.FlushAll();
+		return KErrNone;
+
+	case EVMHalSetCacheSize:
+		{
+		if(!TheCurrentThread->HasCapability(ECapabilityWriteDeviceData,__PLATSEC_DIAGNOSTIC_STRING("Checked by VMHalFunction(EVMHalSetCacheSize)")))
+			K::UnlockedPlatformSecurityPanic();
+		TUint min = TUint(a1)>>KPageShift;
+		if(TUint(a1)&KPageMask)
+			++min;
+		TUint max = TUint(a2)>>KPageShift;
+		if(TUint(a2)&KPageMask)
+			++max;
+		return ThePager.ResizeLiveList(min,max);
+		}
+
+	case EVMHalGetCacheSize:
+		{
+		SVMCacheInfo info;
+		ThePager.GetLiveListInfo(info);
+		kumemput32(a1,&info,sizeof(info));
+		}
+		return KErrNone;
+
+	case EVMHalGetEventInfo:
+		{
+		SVMEventInfo info;
+		ThePager.GetEventInfo(info);
+		Kern::InfoCopy(*(TDes8*)a1,(TUint8*)&info,sizeof(info));
+		}
+		return KErrNone;
+
+	case EVMHalResetEventInfo:
+		ThePager.ResetEventInfo();
+		return KErrNone;
+
+#ifdef __SUPPORT_DEMAND_PAGING_EMULATION__
+	case EVMHalGetOriginalRomPages:
+		RomOriginalPages(*((TPhysAddr**)a1), *((TUint*)a2));
+		return KErrNone;
+#endif
+
+	case EVMPageState:
+		return TestPageState((TLinAddr)a1);
+
+	case EVMHalGetSwapInfo:
+		{
+		if ((K::MemModelAttributes & EMemModelAttrDataPaging) == 0)
+			return KErrNotSupported;
+		SVMSwapInfo info;
+		GetSwapInfo(info);
+		kumemput32(a1,&info,sizeof(info));
+		}
+		return KErrNone;
+
+	case EVMHalGetThrashLevel:
+		return TheThrashMonitor.ThrashLevel();
+
+	case EVMHalSetSwapThresholds:
+		{
+		if(!TheCurrentThread->HasCapability(ECapabilityWriteDeviceData,__PLATSEC_DIAGNOSTIC_STRING("Checked by VMHalFunction(EVMHalSetSwapThresholds)")))
+			K::UnlockedPlatformSecurityPanic();
+		if ((K::MemModelAttributes & EMemModelAttrDataPaging) == 0)
+			return KErrNotSupported;
+		SVMSwapThresholds thresholds;
+		kumemget32(&thresholds,a1,sizeof(thresholds));
+		return SetSwapThresholds(thresholds);
+		}
+
+	case EVMHalSetThrashThresholds:
+		if(!TheCurrentThread->HasCapability(ECapabilityWriteDeviceData,__PLATSEC_DIAGNOSTIC_STRING("Checked by VMHalFunction(EVMHalSetThrashThresholds)")))
+			K::UnlockedPlatformSecurityPanic();
+		return TheThrashMonitor.SetThresholds((TUint)a1, (TUint)a2);
+
+#ifdef __DEMAND_PAGING_BENCHMARKS__
+	case EVMHalGetPagingBenchmark:
+		{
+		TUint index = (TInt) a1;
+		if (index >= EMaxPagingBm)
+			return KErrNotFound;
+		NKern::LockSystem();
+		SPagingBenchmarkInfo info = ThePager.iBenchmarkInfo[index];
+		NKern::UnlockSystem();
+		kumemput32(a2,&info,sizeof(info));
+		}		
+		return KErrNone;
+		
+	case EVMHalResetPagingBenchmark:
+		{
+		TUint index = (TInt) a1;
+		if (index >= EMaxPagingBm)
+			return KErrNotFound;
+		NKern::LockSystem();
+		ThePager.ResetBenchmarkData((TPagingBenchmark)index);
+		NKern::UnlockSystem();
+		}
+		return KErrNone;
+#endif
+
+	default:
+		return KErrNotSupported;
+		}
+	}
+
+
+#ifdef __DEMAND_PAGING_BENCHMARKS__
+
+void DPager::ResetBenchmarkData(TPagingBenchmark aBm)
+    {
+    SPagingBenchmarkInfo& info = iBenchmarkInfo[aBm];
+    info.iCount = 0;
+    info.iTotalTime = 0;
+    info.iMaxTime = 0;
+    info.iMinTime = KMaxTInt;
+    }
+ 
+void DPager::RecordBenchmarkData(TPagingBenchmark aBm, TUint32 aStartTime, TUint32 aEndTime)
+    {
+    SPagingBenchmarkInfo& info = iBenchmarkInfo[aBm];
+    ++info.iCount;
+#if !defined(HIGH_RES_TIMER) || defined(HIGH_RES_TIMER_COUNTS_UP)
+    TInt64 elapsed = aEndTime - aStartTime;
+#else
+    TInt64 elapsed = aStartTime - aEndTime;
+#endif
+    info.iTotalTime += elapsed;
+    if (elapsed > info.iMaxTime)
+        info.iMaxTime = elapsed;
+    if (elapsed < info.iMinTime)
+        info.iMinTime = elapsed;
+    }
+
+#endif //__DEMAND_PAGING_BENCHMARKS__
+
+
+//
+// Paging request management...
+//
+
+//
+// DPagingRequest
+//
+
+DPagingRequest::DPagingRequest(DPagingRequestPool::TGroup& aPoolGroup)
+	: iPoolGroup(aPoolGroup), iUseRegionMemory(0), iUseRegionIndex(0), iUseRegionCount(0)
+	{
+	}
+
+
+FORCE_INLINE void DPagingRequest::SetUse(DMemoryObject* aMemory, TUint aIndex, TUint aCount)
+	{
+	__ASSERT_SYSTEM_LOCK;
+	iUseRegionMemory = aMemory;
+	iUseRegionIndex = aIndex;
+	iUseRegionCount = aCount;
+	}
+
+
+TBool DPagingRequest::CheckUse(DMemoryObject* aMemory, TUint aIndex, TUint aCount)
+	{
+	return aMemory==iUseRegionMemory
+		&& TUint(aIndex-iUseRegionIndex) < iUseRegionCount
+		&& TUint(iUseRegionCount-TUint(aIndex-iUseRegionIndex)) <= aCount;
+	}
+
+
+void DPagingRequest::Release()
+	{
+	NKern::LockSystem();
+	SetUse(0,0,0);
+	Signal();
+	}
+
+
+void DPagingRequest::Wait()
+	{
+	__ASSERT_SYSTEM_LOCK;
+	++iUsageCount;
+	TInt r = iMutex->Wait();
+	__NK_ASSERT_ALWAYS(r == KErrNone);
+	}
+
+
+void DPagingRequest::Signal()
+	{
+	__ASSERT_SYSTEM_LOCK;
+	iPoolGroup.Signal(this);
+	}
+
+
+FORCE_INLINE TBool DPagingRequest::IsCollision(DMemoryObject* aMemory, TUint aIndex, TUint aCount)
+	{
+	__ASSERT_SYSTEM_LOCK;
+	DMemoryObject* memory = iUseRegionMemory;
+	TUint index = iUseRegionIndex;
+	TUint count = iUseRegionCount;
+	// note, this comparison would fail if either region includes page number KMaxTUint,
+	// but it isn't possible to create a memory object which is > KMaxTUint pages...
+	return memory == aMemory && index+count > aIndex && index < aIndex+aCount;
+	}
+
+
+TLinAddr DPagingRequest::MapPages(TUint aColour, TUint aCount, TPhysAddr* aPages)
+	{
+	__NK_ASSERT_DEBUG(iMutex->iCleanup.iThread == &Kern::CurrentThread());
+	return iTempMapping.Map(aPages,aCount,aColour);
+	}
+
+
+void DPagingRequest::UnmapPages(TBool aIMBRequired)
+	{
+	__NK_ASSERT_DEBUG(iMutex->iCleanup.iThread == &Kern::CurrentThread());
+	iTempMapping.Unmap(aIMBRequired);
+	}
+
+
+//
+// DPageReadRequest
+//
+
+TInt DPageReadRequest::iAllocNext = 0;
+
+TInt DPageReadRequest::Construct()
+	{
+	// allocate id and mutex...
+	TUint id = (TUint)__e32_atomic_add_ord32(&iAllocNext, 1);
+	_LIT(KLitPagingRequest,"PageReadRequest-");
+	TBuf<sizeof("PageReadRequest-")+10> mutexName(KLitPagingRequest);
+	mutexName.AppendNum(id);
+	TInt r = K::MutexCreate(iMutex, mutexName, NULL, EFalse, KMutexOrdPageIn);
+	if(r!=KErrNone)
+		return r;
+
+	// allocate space for mapping pages whilst they're being loaded...
+	iTempMapping.Alloc(EMaxPages);
+
+	// create memory buffer...
+	TUint bufferSize = EMaxPages+1;
+	DMemoryObject* bufferMemory;
+	r = MM::MemoryNew(bufferMemory,EMemoryObjectUnpaged,bufferSize,EMemoryCreateNoWipe);
+	if(r!=KErrNone)
+		return r;
+	MM::MemorySetLock(bufferMemory,iMutex);
+	TPhysAddr physAddr;
+	r = MM::MemoryAllocContiguous(bufferMemory,0,bufferSize,0,physAddr);
+	(void)physAddr;
+	if(r!=KErrNone)
+		return r;
+	DMemoryMapping* bufferMapping;
+	r = MM::MappingNew(bufferMapping,bufferMemory,ESupervisorReadWrite,KKernelOsAsid);
+	if(r!=KErrNone)
+		return r;
+	iBuffer = MM::MappingBase(bufferMapping);
+
+	// ensure there are enough young pages to cope with new request object...
+	r = ThePager.ResizeLiveList();
+	if(r!=KErrNone)
+		return r;
+
+	return r;
+	}
+
+
+//
+// DPageWriteRequest
+//
+
+TInt DPageWriteRequest::iAllocNext = 0;
+
+TInt DPageWriteRequest::Construct()
+	{
+	// allocate id and mutex...
+	TUint id = (TUint)__e32_atomic_add_ord32(&iAllocNext, 1);
+	_LIT(KLitPagingRequest,"PageWriteRequest-");
+	TBuf<sizeof("PageWriteRequest-")+10> mutexName(KLitPagingRequest);
+	mutexName.AppendNum(id);
+	TInt r = K::MutexCreate(iMutex, mutexName, NULL, EFalse, KMutexOrdPageOut);
+	if(r!=KErrNone)
+		return r;
+
+	// allocate space for mapping pages whilst they're being loaded...
+	iTempMapping.Alloc(EMaxPages);
+
+	return r;
+	}
+
+
+//
+// DPagingRequestPool
+//
+
+DPagingRequestPool::DPagingRequestPool(TUint aNumPageReadRequest,TUint aNumPageWriteRequest)
+	: iPageReadRequests(aNumPageReadRequest), iPageWriteRequests(aNumPageWriteRequest)
+	{
+	TUint i;
+
+	for(i=0; i<aNumPageReadRequest; ++i)
+		{
+		DPageReadRequest* req = new DPageReadRequest(iPageReadRequests);
+		__NK_ASSERT_ALWAYS(req);
+		TInt r = req->Construct();
+		__NK_ASSERT_ALWAYS(r==KErrNone);
+		iPageReadRequests.iRequests[i] = req;
+		iPageReadRequests.iFreeList.Add(req);
+		}
+
+	for(i=0; i<aNumPageWriteRequest; ++i)
+		{
+		DPageWriteRequest* req = new DPageWriteRequest(iPageWriteRequests);
+		__NK_ASSERT_ALWAYS(req);
+		TInt r = req->Construct();
+		__NK_ASSERT_ALWAYS(r==KErrNone);
+		iPageWriteRequests.iRequests[i] = req;
+		iPageWriteRequests.iFreeList.Add(req);
+		}
+	}
+
+
+DPagingRequestPool::~DPagingRequestPool()
+	{
+	__NK_ASSERT_ALWAYS(0); // deletion not implemented
+	}
+
+
+DPageReadRequest* DPagingRequestPool::AcquirePageReadRequest(DMemoryObject* aMemory, TUint aIndex, TUint aCount)
+	{
+	NKern::LockSystem();
+
+	DPagingRequest* req;
+
+	// if we collide with page write operation...
+	req = iPageWriteRequests.FindCollision(aMemory,aIndex,aCount);
+	if(req)
+		{
+		// wait until write completes...
+		req->Wait();
+		req->Signal();
+		return 0; // caller expected to retry if needed
+		}
+
+	// get a request object to use...
+	req = iPageReadRequests.GetRequest(aMemory,aIndex,aCount);
+
+	// check no new requests collide with us...
+	if(iPageWriteRequests.FindCollision(aMemory,aIndex,aCount)
+		|| iPageReadRequests.FindCollision(aMemory,aIndex,aCount))
+		{
+		// another operation is colliding with this region, give up and retry...
+		req->Signal();
+		return 0; // caller expected to retry if needed
+		}
+
+	// we have a request object which we can use...
+	req->SetUse(aMemory,aIndex,aCount);
+
+	NKern::UnlockSystem();
+	return (DPageReadRequest*)req;
+	}
+
+
+DPageWriteRequest* DPagingRequestPool::AcquirePageWriteRequest(DMemoryObject* aMemory, TUint aIndex, TUint aCount)
+	{
+	NKern::LockSystem();
+
+	DPagingRequest* req;
+
+	for(;;)
+		{
+		// get a request object to use...
+		req = iPageWriteRequests.GetRequest(aMemory,aIndex,aCount);
+
+		if(iPageWriteRequests.FindCollision(aMemory,aIndex,aCount))
+			{
+			// another write operation is colliding with this region, give up and retry...
+			req->Signal();
+			// Reacquire the system lock as Signal() above will release it.
+			NKern::LockSystem();
+			continue;
+			}
+
+		break;
+		}
+
+	// we have a request object which we can use...
+	req->SetUse(aMemory,aIndex,aCount);
+
+	NKern::UnlockSystem();
+	return (DPageWriteRequest*)req;
+	}
+
+
+DPagingRequestPool::TGroup::TGroup(TUint aNumRequests)
+	{
+	iNumRequests = aNumRequests;
+	iRequests = new DPagingRequest*[aNumRequests];
+	__NK_ASSERT_ALWAYS(iRequests);
+	}
+
+
+DPagingRequest* DPagingRequestPool::TGroup::FindCollision(DMemoryObject* aMemory, TUint aIndex, TUint aCount)
+	{
+	__ASSERT_SYSTEM_LOCK;
+	DPagingRequest** ptr = iRequests;
+	DPagingRequest** ptrEnd = ptr+iNumRequests;
+	while(ptr<ptrEnd)
+		{
+		DPagingRequest* req = *ptr++;
+		if(req->IsCollision(aMemory,aIndex,aCount))
+			return req;
+		}
+	return 0;
+	}
+
+
+static TUint32 RandomSeed = 33333;
+
+DPagingRequest* DPagingRequestPool::TGroup::GetRequest(DMemoryObject* aMemory, TUint aIndex, TUint aCount)
+	{
+	__NK_ASSERT_DEBUG(iNumRequests > 0);
+
+	// try using an existing request which collides with this region...
+	DPagingRequest* req  = FindCollision(aMemory,aIndex,aCount);
+	if(!req)
+		{
+		// use a free request...
+		req = (DPagingRequest*)iFreeList.GetFirst();
+		if(req)
+			{
+			// free requests aren't being used...
+			__NK_ASSERT_DEBUG(req->iUsageCount == 0);
+			}
+		else
+			{
+			// pick a random request...
+			RandomSeed = RandomSeed*69069+1; // next 'random' number
+			TUint index = (TUint64(RandomSeed) * TUint64(iNumRequests)) >> 32;
+			req = iRequests[index];
+			__NK_ASSERT_DEBUG(req->iUsageCount > 0); // we only pick random when none are free
+			}
+		}
+
+	// wait for chosen request object...
+	req->Wait();
+
+	return req;
+	}
+
+
+void DPagingRequestPool::TGroup::Signal(DPagingRequest* aRequest)
+	{
+	// if there are no threads waiting on the mutex then return it to the free pool...
+	__NK_ASSERT_DEBUG(aRequest->iUsageCount > 0);
+	if (--aRequest->iUsageCount==0)
+		iFreeList.AddHead(aRequest);
+
+	aRequest->iMutex->Signal();
+	}
+
+
+/**
+Register the specified paging device with the kernel.
+
+@param aDevice	A pointer to the paging device to install
+
+@return KErrNone on success
+*/
+EXPORT_C TInt Kern::InstallPagingDevice(DPagingDevice* aDevice)
+	{
+	TRACEB(("Kern::InstallPagingDevice(0x%08x) name='%s' type=%d",aDevice,aDevice->iName,aDevice->iType));
+
+	__NK_ASSERT_ALWAYS(aDevice->iReadUnitShift <= KPageShift);
+
+	TInt r = KErrNotSupported;	// Will return this if unsupported device type is installed
+
+	// create the pools of page out and page in requests...
+	const TInt writeReqs = (aDevice->iType & DPagingDevice::EData) ? KPagingRequestsPerDevice : 0;
+	aDevice->iRequestPool = new DPagingRequestPool(KPagingRequestsPerDevice,writeReqs);
+	if(!aDevice->iRequestPool)
+		{
+		r = KErrNoMemory;
+		goto exit;
+		}
+
+	if(aDevice->iType & DPagingDevice::ERom)
+		{
+		r = TheRomMemoryManager->InstallPagingDevice(aDevice);
+		if(r!=KErrNone)
+			goto exit;
+		}
+
+	if(aDevice->iType & DPagingDevice::ECode)
+		{
+		r = TheCodePagedMemoryManager->InstallPagingDevice(aDevice);
+		if(r!=KErrNone)
+			goto exit;
+		}
+
+	if(aDevice->iType & DPagingDevice::EData)
+		{
+		r = TheDataPagedMemoryManager->InstallPagingDevice(aDevice);
+		if(r!=KErrNone)
+			goto exit;
+		}
+
+ 	if (K::MemModelAttributes & (EMemModelAttrRomPaging | EMemModelAttrCodePaging | EMemModelAttrDataPaging))
+		TheThrashMonitor.Start();
+
+exit:
+	TRACEB(("Kern::InstallPagingDevice returns %d",r));
+	return r;
+	}
+
+
+
+//
+// DDemandPagingLock
+//
+
+EXPORT_C DDemandPagingLock::DDemandPagingLock()
+	: iReservedPageCount(0), iLockedPageCount(0), iPinMapping(0)
+	{
+	}
+
+
+EXPORT_C TInt DDemandPagingLock::Alloc(TInt aSize)
+	{
+	TRACEP(("DDemandPagingLock[0x%08x]::Alloc(0x%x)",this,aSize));
+	iMaxPageCount = ((aSize-1+KPageMask)>>KPageShift)+1;
+
+	TInt r = KErrNoMemory;
+
+	NKern::ThreadEnterCS();
+
+	TUint maxPt = DVirtualPinMapping::MaxPageTables(iMaxPageCount);
+	// Note, we need to reserve whole pages even for page tables which are smaller
+	// because pinning can remove the page from live list...
+	TUint reserve = iMaxPageCount+maxPt*KNumPagesToPinOnePageTable;
+	if(ThePager.ReservePages(reserve,(TUint&)iReservedPageCount))
+		{
+		iPinMapping = DVirtualPinMapping::New(iMaxPageCount);
+		if(iPinMapping)
+			r = KErrNone;
+		else
+			ThePager.UnreservePages((TUint&)iReservedPageCount);
+		}
+
+	NKern::ThreadLeaveCS();
+	TRACEP(("DDemandPagingLock[0x%08x]::Alloc returns %d, iMaxPageCount=%d, iReservedPageCount=%d",this,r,iMaxPageCount,iReservedPageCount));
+	return r;
+	}
+
+
+EXPORT_C void DDemandPagingLock::Free()
+	{
+	TRACEP(("DDemandPagingLock[0x%08x]::Free()"));
+	Unlock();
+	NKern::ThreadEnterCS();
+	DVirtualPinMapping* pinMapping = (DVirtualPinMapping*)__e32_atomic_swp_ord_ptr(&iPinMapping, 0);
+	if (pinMapping)
+		pinMapping->Close();
+	NKern::ThreadLeaveCS();
+	ThePager.UnreservePages((TUint&)iReservedPageCount);
+	}
+
+
+EXPORT_C TInt DDemandPagingLock::Lock(DThread* aThread, TLinAddr aStart, TInt aSize)
+	{
+//	TRACEP(("DDemandPagingLock[0x%08x]::Lock(0x%08x,0x%08x,0x%08x)",this,aThread,aStart,aSize));
+	if(iLockedPageCount)
+		__NK_ASSERT_ALWAYS(0); // lock already used
+
+	// calculate the number of pages that need to be locked...
+	TUint mask=KPageMask;
+	TUint offset=aStart&mask;
+	TInt numPages = (aSize+offset+mask)>>KPageShift;
+	if(numPages>iMaxPageCount)
+		__NK_ASSERT_ALWAYS(0);
+
+	NKern::ThreadEnterCS();
+
+	// find mapping which covers the specified region...
+	TUint offsetInMapping;
+	TUint mapInstanceCount;
+	DMemoryMapping* mapping = MM::FindMappingInThread((DMemModelThread*)aThread, aStart, aSize, offsetInMapping, mapInstanceCount);
+	if(!mapping)
+		{
+		NKern::ThreadLeaveCS();
+		return KErrBadDescriptor;
+		}
+
+	MmuLock::Lock(); 
+	DMemoryObject* memory = mapping->Memory();
+	if(mapInstanceCount != mapping->MapInstanceCount() || !memory)
+		{// Mapping has been reused or no memory.
+		MmuLock::Unlock();
+		mapping->Close();
+		NKern::ThreadLeaveCS();
+		return KErrBadDescriptor;
+		}
+
+	if(!memory->IsDemandPaged())
+		{
+		// memory not demand paged, so we have nothing to do...
+		MmuLock::Unlock();
+		mapping->Close();
+		NKern::ThreadLeaveCS();
+		return KErrNone;
+		}
+
+	// Open a reference on the memory so it doesn't get deleted.
+	memory->Open();
+	MmuLock::Unlock();
+
+	// pin memory...
+	TUint index = (offsetInMapping>>KPageShift)+mapping->iStartIndex;
+	TUint count = ((offsetInMapping&KPageMask)+aSize+KPageMask)>>KPageShift;
+	TInt r = ((DVirtualPinMapping*)iPinMapping)->Pin(	memory,index,count,mapping->Permissions(),
+														mapping, mapInstanceCount);
+
+	if(r==KErrNotFound)
+		{
+		// some memory wasn't present, so treat this as an error...
+		memory->Close();
+		mapping->Close();
+		NKern::ThreadLeaveCS();
+		return KErrBadDescriptor;
+		}
+
+	// we can't fail to pin otherwise...
+	__NK_ASSERT_DEBUG(r!=KErrNoMemory); // separate OOM assert to aid debugging
+	__NK_ASSERT_ALWAYS(r==KErrNone);
+
+	// indicate that we have actually pinned...
+	__NK_ASSERT_DEBUG(iLockedPageCount==0);
+	iLockedPageCount = count;
+
+	// cleanup...
+	memory->Close();
+	mapping->Close();
+	NKern::ThreadLeaveCS();
+
+	return 1;
+	}
+
+
+EXPORT_C void DDemandPagingLock::DoUnlock()
+	{
+	NKern::ThreadEnterCS();
+	((DVirtualPinMapping*)iPinMapping)->Unpin();
+	__NK_ASSERT_DEBUG(iLockedPageCount);
+	iLockedPageCount = 0;
+	NKern::ThreadLeaveCS();
+	}
+
+