Fix for bug 2283 (RVCT 4.0 support is missing from PDK 3.0.h)
Have multiple extension sections in the bld.inf, one for each version
of the compiler. The RVCT version building the tools will build the
runtime libraries for its version, but make sure we extract all the other
versions from zip archives. Also add the archive for RVCT4.
// Copyright (c) 2005-2009 Nokia Corporation and/or its subsidiary(-ies).
// All rights reserved.
// This component and the accompanying materials are made available
// under the terms of the License "Eclipse Public License v1.0"
// which accompanies this distribution, and is available
// at the URL "http://www.eclipse.org/legal/epl-v10.html".
//
// Initial Contributors:
// Nokia Corporation - initial contribution.
//
// Contributors:
//
// Description:
//
#include "memmodel.h"
#include "mm.h"
#include "mmu.h"
#include "mpager.h"
#include "mrom.h"
#include "mobject.h"
#include "mmapping.h"
#include "maddressspace.h"
#include "mmanager.h"
#include "mptalloc.h"
#include "mpagearray.h"
#include "mswap.h"
#include "mthrash.h"
#include "cache_maintenance.inl"
const TUint16 KDefaultYoungOldRatio = 3;
const TUint16 KDefaultMinPages = 256;
#ifdef _USE_OLDEST_LISTS
const TUint16 KDefaultOldOldestRatio = 3;
#endif
const TUint KMinOldPages = 1;
/* On a 32 bit system without PAE can't have more than 2^(32-KPageShift) pages.
* Subtract 1 so it doesn't overflow when converted to bytes.
*/
const TUint KAbsoluteMaxPageCount = (1u<<(32-KPageShift))-1u;
DPager ThePager;
DPager::DPager()
: iMinimumPageCount(0), iMaximumPageCount(0), iYoungOldRatio(0),
iYoungCount(0),iOldCount(0),iNumberOfFreePages(0)
{
}
void DPager::Init2()
{
TRACEB(("DPager::Init2()"));
#if defined(__CPU_ARM)
/** Minimum number of young pages the demand paging live list may have.
Need at least 4 mapped pages to guarantee to be able to execute all ARM instructions,
plus enough pages for 4 page tables to map those pages, plus enough pages for the
page table info structures of those page tables.
(Worst case is a Thumb-2 STM instruction with both instruction and data straddling chunk
boundaries.)
*/
iMinYoungPages = 4 // pages
+(4+KPtClusterSize-1)/KPtClusterSize // page table pages
+(4+KPageTableInfosPerPage-1)/KPageTableInfosPerPage; // page table info pages
#elif defined(__CPU_X86)
/* Need at least 6 mapped pages to guarantee to be able to execute all ARM instructions,
plus enough pages for 6 page tables to map those pages, plus enough pages for the
page table info structures of those page tables.
(Worst case is (?) a MOV [X],[Y] instruction with instruction, 'X' and 'Y' all
straddling chunk boundaries.)
*/
iMinYoungPages = 6 // pages
+(6+KPtClusterSize-1)/KPtClusterSize // page table pages
+(6+KPageTableInfosPerPage-1)/KPageTableInfosPerPage; // page table info pages
#else
#error Unknown CPU
#endif
#ifdef __SMP__
// Adjust min page count so that all CPUs are guaranteed to make progress.
// NOTE: Can't use NKern::NumberOfCpus here because we haven't finished booting yet and will
// always have only one CPU running at this point...
// TODO: Before we can enable this the base test configuration needs
// updating to have a sufficient minimum page size...
//
// iMinYoungPages *= KMaxCpus;
#endif
// A minimum young/old ratio of 1 means that we need at least twice iMinYoungPages pages...
iAbsoluteMinPageCount = 2*iMinYoungPages;
__NK_ASSERT_DEBUG(KMinOldPages<=iAbsoluteMinPageCount/2);
// initialise live list...
TUint minimumPageCount = 0;
TUint maximumPageCount = 0;
SDemandPagingConfig config = TheRomHeader().iDemandPagingConfig;
iMinimumPageCount = KDefaultMinPages;
if(minimumPageCount)
iMinimumPageCount = minimumPageCount;
if(config.iMinPages)
iMinimumPageCount = config.iMinPages;
if(iMinimumPageCount<iAbsoluteMinPageCount)
iMinimumPageCount = iAbsoluteMinPageCount;
iInitMinimumPageCount = iMinimumPageCount;
iMaximumPageCount = KMaxTInt;
if(maximumPageCount)
iMaximumPageCount = maximumPageCount;
if(config.iMaxPages)
iMaximumPageCount = config.iMaxPages;
if (iMaximumPageCount > KAbsoluteMaxPageCount)
iMaximumPageCount = KAbsoluteMaxPageCount;
iInitMaximumPageCount = iMaximumPageCount;
iYoungOldRatio = KDefaultYoungOldRatio;
if(config.iYoungOldRatio)
iYoungOldRatio = config.iYoungOldRatio;
TInt ratioLimit = (iMinimumPageCount-KMinOldPages)/KMinOldPages;
if(iYoungOldRatio>ratioLimit)
iYoungOldRatio = ratioLimit;
#ifdef _USE_OLDEST_LISTS
iOldOldestRatio = KDefaultOldOldestRatio;
if(config.iSpare[2])
iOldOldestRatio = config.iSpare[2];
#endif
iMinimumPageLimit = (iMinYoungPages * (1 + iYoungOldRatio)) / iYoungOldRatio;
if(iMinimumPageLimit<iAbsoluteMinPageCount)
iMinimumPageLimit = iAbsoluteMinPageCount;
TRACEB(("DPager::Init2() live list min=%d max=%d ratio=%d",iMinimumPageCount,iMaximumPageCount,iYoungOldRatio));
if(iMaximumPageCount<iMinimumPageCount)
__NK_ASSERT_ALWAYS(0);
//
// This routine doesn't acquire any mutexes because it should be called before the system
// is fully up and running. I.e. called before another thread can preempt this.
//
// Calculate page counts
TUint minOldAndOldest = iMinimumPageCount / (1 + iYoungOldRatio);
if(minOldAndOldest < KMinOldPages)
__NK_ASSERT_ALWAYS(0);
if (iMinimumPageCount < minOldAndOldest)
__NK_ASSERT_ALWAYS(0);
TUint minYoung = iMinimumPageCount - minOldAndOldest;
if(minYoung < iMinYoungPages)
__NK_ASSERT_ALWAYS(0); // Need at least iMinYoungPages pages mapped to execute worst case CPU instruction
#ifdef _USE_OLDEST_LISTS
// There should always be enough old pages to allow the oldest lists ratio.
TUint oldestCount = minOldAndOldest / (1 + iOldOldestRatio);
if (!oldestCount)
__NK_ASSERT_ALWAYS(0);
#endif
iNumberOfFreePages = 0;
iNumberOfDirtyPages = 0;
// Allocate RAM pages and put them all on the old list
RamAllocLock::Lock();
iYoungCount = 0;
iOldCount = 0;
#ifdef _USE_OLDEST_LISTS
iOldestCleanCount = 0;
iOldestDirtyCount = 0;
#endif
Mmu& m = TheMmu;
for(TUint i=0; i<iMinimumPageCount; i++)
{
// Allocate a single page
TPhysAddr pagePhys;
TInt r = m.AllocRam(&pagePhys, 1,
(Mmu::TRamAllocFlags)(EMemAttNormalCached|Mmu::EAllocNoWipe|Mmu::EAllocNoPagerReclaim),
EPageDiscard);
if(r!=KErrNone)
__NK_ASSERT_ALWAYS(0);
MmuLock::Lock();
AddAsFreePage(SPageInfo::FromPhysAddr(pagePhys));
MmuLock::Unlock();
}
RamAllocLock::Unlock();
#ifdef _USE_OLDEST_LISTS
TRACEB(("DPager::Init2() end with young=%d old=%d oldClean=%d oldDirty=%d min=%d free=%d max=%d",iYoungCount,iOldCount,iOldestCleanCount,iOldestDirtyCount,iMinimumPageCount,iNumberOfFreePages,iMaximumPageCount));
#else
TRACEB(("DPager::Init2() end with young=%d old=%d min=%d free=%d max=%d",iYoungCount,iOldCount,iMinimumPageCount,iNumberOfFreePages,iMaximumPageCount));
#endif
}
#ifdef _DEBUG
TBool DPager::CheckLists()
{
#if 0
__NK_ASSERT_DEBUG(MmuLock::IsHeld());
SDblQueLink* head = &iOldList.iA;
TInt n = iOldCount;
SDblQueLink* link = head;
while(n--)
{
link = link->iNext;
if(link==head)
return false;
}
link = link->iNext;
if(link!=head)
return false;
head = &iYoungList.iA;
n = iYoungCount;
link = head;
while(n--)
{
link = link->iNext;
if(link==head)
return false;
}
link = link->iNext;
if(link!=head)
return false;
// TRACEP(("DP: y=%d o=%d f=%d",iYoungCount,iOldCount,iNumberOfFreePages));
#endif
// TraceCounts();
return true;
}
void DPager::TraceCounts()
{
TRACEP(("DP: y=%d o=%d f=%d min=%d max=%d ml=%d res=%d",
iYoungCount,iOldCount,iNumberOfFreePages,iMinimumPageCount,
iMaximumPageCount,iMinimumPageLimit,iReservePageCount));
}
#endif
TBool DPager::HaveTooManyPages()
{
__NK_ASSERT_DEBUG(RamAllocLock::IsHeld());
__NK_ASSERT_DEBUG(MmuLock::IsHeld());
return iMinimumPageCount+iNumberOfFreePages > iMaximumPageCount;
}
TBool DPager::HaveMaximumPages()
{
__NK_ASSERT_DEBUG(RamAllocLock::IsHeld());
__NK_ASSERT_DEBUG(MmuLock::IsHeld());
return iMinimumPageCount+iNumberOfFreePages >= iMaximumPageCount;
}
void DPager::AddAsYoungestPage(SPageInfo* aPageInfo)
{
__NK_ASSERT_DEBUG(MmuLock::IsHeld());
__NK_ASSERT_DEBUG(CheckLists());
__NK_ASSERT_DEBUG(aPageInfo->PagedState()==SPageInfo::EUnpaged);
aPageInfo->SetPagedState(SPageInfo::EPagedYoung);
iYoungList.AddHead(&aPageInfo->iLink);
++iYoungCount;
}
void DPager::AddAsFreePage(SPageInfo* aPageInfo)
{
__NK_ASSERT_DEBUG(MmuLock::IsHeld());
__NK_ASSERT_DEBUG(CheckLists());
__NK_ASSERT_DEBUG(aPageInfo->PagedState()==SPageInfo::EUnpaged);
TheMmu.PageFreed(aPageInfo);
__NK_ASSERT_DEBUG(aPageInfo->PagedState()==SPageInfo::EUnpaged);
// add as oldest page...
#ifdef _USE_OLDEST_LISTS
aPageInfo->SetPagedState(SPageInfo::EPagedOldestClean);
iOldestCleanList.Add(&aPageInfo->iLink);
++iOldestCleanCount;
#else
aPageInfo->SetPagedState(SPageInfo::EPagedOld);
iOldList.Add(&aPageInfo->iLink);
++iOldCount;
#endif
Event(EEventPageInFree,aPageInfo);
}
TInt DPager::PageFreed(SPageInfo* aPageInfo)
{
__NK_ASSERT_DEBUG(MmuLock::IsHeld());
__NK_ASSERT_DEBUG(CheckLists());
switch(aPageInfo->PagedState())
{
case SPageInfo::EUnpaged:
return KErrNotFound;
case SPageInfo::EPagedYoung:
__NK_ASSERT_DEBUG(iYoungCount);
aPageInfo->iLink.Deque();
--iYoungCount;
break;
case SPageInfo::EPagedOld:
__NK_ASSERT_DEBUG(iOldCount);
aPageInfo->iLink.Deque();
--iOldCount;
break;
#ifdef _USE_OLDEST_LISTS
case SPageInfo::EPagedOldestClean:
__NK_ASSERT_DEBUG(iOldestCleanCount);
aPageInfo->iLink.Deque();
--iOldestCleanCount;
break;
case SPageInfo::EPagedOldestDirty:
__NK_ASSERT_DEBUG(iOldestDirtyCount);
aPageInfo->iLink.Deque();
--iOldestDirtyCount;
break;
#endif
case SPageInfo::EPagedPinned:
// this can occur if a pinned mapping is being unmapped when memory is decommitted.
// the decommit will have succeeded because the the mapping no longer vetoes this,
// however the unpinning hasn't yet got around to changing the page state.
// When the state change happens the page will be put back on the live list so
// we don't have to do anything now...
return KErrNone;
case SPageInfo::EPagedPinnedMoved:
// This page was pinned when it was moved but it has not been returned
// to the free pool yet so make sure it is...
aPageInfo->SetPagedState(SPageInfo::EUnpaged); // Must be unpaged before returned to free pool.
return KErrNotFound;
default:
__NK_ASSERT_DEBUG(0);
return KErrNotFound;
}
// Update the dirty page count as required...
if (aPageInfo->IsDirty())
SetClean(*aPageInfo);
// add as oldest page...
#ifdef _USE_OLDEST_LISTS
aPageInfo->SetPagedState(SPageInfo::EPagedOldestClean);
iOldestCleanList.Add(&aPageInfo->iLink);
++iOldestCleanCount;
#else
aPageInfo->SetPagedState(SPageInfo::EPagedOld);
iOldList.Add(&aPageInfo->iLink);
++iOldCount;
#endif
return KErrNone;
}
extern TBool IsPageTableUnpagedRemoveAllowed(SPageInfo* aPageInfo);
void DPager::RemovePage(SPageInfo* aPageInfo)
{
__NK_ASSERT_DEBUG(MmuLock::IsHeld());
__NK_ASSERT_DEBUG(CheckLists());
switch(aPageInfo->PagedState())
{
case SPageInfo::EPagedYoung:
__NK_ASSERT_DEBUG(iYoungCount);
aPageInfo->iLink.Deque();
--iYoungCount;
break;
case SPageInfo::EPagedOld:
__NK_ASSERT_DEBUG(iOldCount);
aPageInfo->iLink.Deque();
--iOldCount;
break;
#ifdef _USE_OLDEST_LISTS
case SPageInfo::EPagedOldestClean:
__NK_ASSERT_DEBUG(iOldestCleanCount);
aPageInfo->iLink.Deque();
--iOldestCleanCount;
break;
case SPageInfo::EPagedOldestDirty:
__NK_ASSERT_DEBUG(iOldestDirtyCount);
aPageInfo->iLink.Deque();
--iOldestDirtyCount;
break;
#endif
case SPageInfo::EPagedPinned:
__NK_ASSERT_DEBUG(0);
case SPageInfo::EUnpaged:
#ifdef _DEBUG
if (!IsPageTableUnpagedRemoveAllowed(aPageInfo))
__NK_ASSERT_DEBUG(0);
break;
#endif
default:
__NK_ASSERT_DEBUG(0);
return;
}
aPageInfo->SetPagedState(SPageInfo::EUnpaged);
}
void DPager::ReplacePage(SPageInfo& aOldPageInfo, SPageInfo& aNewPageInfo)
{
__NK_ASSERT_DEBUG(MmuLock::IsHeld());
__NK_ASSERT_DEBUG(CheckLists());
__NK_ASSERT_DEBUG(aOldPageInfo.PagedState() == aNewPageInfo.PagedState());
switch(aOldPageInfo.PagedState())
{
case SPageInfo::EPagedYoung:
case SPageInfo::EPagedOld:
case SPageInfo::EPagedOldestClean:
case SPageInfo::EPagedOldestDirty:
{// Update the list links point to the new page.
__NK_ASSERT_DEBUG(iYoungCount);
SDblQueLink* prevLink = aOldPageInfo.iLink.iPrev;
#ifdef _DEBUG
SDblQueLink* nextLink = aOldPageInfo.iLink.iNext;
__NK_ASSERT_DEBUG(prevLink == aOldPageInfo.iLink.iPrev);
__NK_ASSERT_DEBUG(prevLink->iNext == &aOldPageInfo.iLink);
__NK_ASSERT_DEBUG(nextLink == aOldPageInfo.iLink.iNext);
__NK_ASSERT_DEBUG(nextLink->iPrev == &aOldPageInfo.iLink);
#endif
aOldPageInfo.iLink.Deque();
aNewPageInfo.iLink.InsertAfter(prevLink);
aOldPageInfo.SetPagedState(SPageInfo::EUnpaged);
#ifdef _DEBUG
__NK_ASSERT_DEBUG(prevLink == aNewPageInfo.iLink.iPrev);
__NK_ASSERT_DEBUG(prevLink->iNext == &aNewPageInfo.iLink);
__NK_ASSERT_DEBUG(nextLink == aNewPageInfo.iLink.iNext);
__NK_ASSERT_DEBUG(nextLink->iPrev == &aNewPageInfo.iLink);
#endif
}
break;
case SPageInfo::EPagedPinned:
// Mark the page as 'pinned moved' so that when the page moving invokes
// Mmu::FreeRam() it returns this page to the free pool.
aOldPageInfo.ClearPinCount();
aOldPageInfo.SetPagedState(SPageInfo::EPagedPinnedMoved);
break;
case SPageInfo::EPagedPinnedMoved:
// Shouldn't happen as the ram alloc mutex will be held for the
// entire time the page's is paged state == EPagedPinnedMoved.
case SPageInfo::EUnpaged:
// Shouldn't happen as we only move pinned memory and unpinning will
// atomically add the page to the live list and it can't be removed
// from the live list without the ram alloc mutex.
__NK_ASSERT_DEBUG(0);
break;
}
}
SPageInfo* DPager::StealOldestPage()
{
__NK_ASSERT_DEBUG(RamAllocLock::IsHeld());
__NK_ASSERT_DEBUG(MmuLock::IsHeld());
for(;;)
{
// find oldest page in list...
SDblQueLink* link;
#ifdef _USE_OLDEST_LISTS
if (iOldestCleanCount)
{
__NK_ASSERT_DEBUG(!iOldestCleanList.IsEmpty());
link = iOldestCleanList.Last();
}
else if (iOldestDirtyCount)
{
__NK_ASSERT_DEBUG(!iOldestDirtyList.IsEmpty());
link = iOldestDirtyList.Last();
}
else if (iOldCount)
#else
if (iOldCount)
#endif
{
__NK_ASSERT_DEBUG(!iOldList.IsEmpty());
link = iOldList.Last();
}
else
{
__NK_ASSERT_DEBUG(iYoungCount);
__NK_ASSERT_ALWAYS(!iYoungList.IsEmpty());
link = iYoungList.Last();
}
SPageInfo* pageInfo = SPageInfo::FromLink(link);
// steal it from owning object...
TInt r = StealPage(pageInfo);
BalanceAges();
if(r==KErrNone)
return pageInfo; // done
// loop back and try again
}
}
TInt DPager::RestrictPage(SPageInfo* aPageInfo, TRestrictPagesType aRestriction)
{
TRACE(("DPager::RestrictPage(0x%08x,%d)",aPageInfo,aRestriction));
__NK_ASSERT_DEBUG(MmuLock::IsHeld());
TInt r;
if(aPageInfo->Type()==SPageInfo::EUnused)
{
// page was unused, so nothing to do...
r = KErrNone;
}
else
{
// get memory object which owns the page...
__NK_ASSERT_DEBUG(aPageInfo->Type()==SPageInfo::EManaged);
DMemoryObject* memory = aPageInfo->Owner();
memory->Open();
// try restricting access to page...
r = memory->iManager->RestrictPage(memory,aPageInfo,aRestriction);
__NK_ASSERT_DEBUG(r!=KErrNotSupported);
// close memory object...
MmuLock::Unlock();
memory->AsyncClose();
MmuLock::Lock();
}
TRACE(("DPager::RestrictPage returns %d",r));
return r;
}
TInt DPager::StealPage(SPageInfo* aPageInfo)
{
TRACE(("DPager::StealPage(0x%08x)",aPageInfo));
__NK_ASSERT_DEBUG(RamAllocLock::IsHeld());
__NK_ASSERT_DEBUG(MmuLock::IsHeld());
__UNLOCK_GUARD_START(MmuLock);
RemovePage(aPageInfo);
TInt r;
if(aPageInfo->Type()==SPageInfo::EUnused)
{
// page was unused, so nothing to do...
r = KErrNone;
__UNLOCK_GUARD_END(MmuLock);
MmuLock::Unlock();
}
else
{
// get memory object which owns the page...
__NK_ASSERT_DEBUG(aPageInfo->Type()==SPageInfo::EManaged);
DMemoryObject* memory = aPageInfo->Owner();
memory->Open();
// try and steal page from memory object...
__UNLOCK_GUARD_END(MmuLock); // StealPage must be called without releasing the MmuLock
r = memory->iManager->StealPage(memory,aPageInfo);
__NK_ASSERT_DEBUG(r!=KErrNotSupported);
// close memory object...
MmuLock::Unlock();
memory->AsyncClose();
}
MmuLock::Lock();
if(r==KErrNone)
Event(EEventPageOut,aPageInfo);
TRACE(("DPager::StealPage returns %d",r));
return r;
}
TInt DPager::DiscardPage(SPageInfo* aOldPageInfo, TUint aBlockZoneId, TBool aBlockRest)
{
__NK_ASSERT_DEBUG(RamAllocLock::IsHeld());
__NK_ASSERT_DEBUG(MmuLock::IsHeld());
TInt r;
// If the page is pinned or if the page is dirty and a general defrag is being
// performed then don't attempt to steal it.
if (aOldPageInfo->Type() != SPageInfo::EUnused &&
(aOldPageInfo->PagedState() == SPageInfo::EPagedPinned ||
(aBlockRest && aOldPageInfo->IsDirty())))
{// The page is pinned or is dirty and this is a general defrag so move the page.
DMemoryObject* memory = aOldPageInfo->Owner();
// Page must be managed if it is pinned or dirty.
__NK_ASSERT_DEBUG(aOldPageInfo->Type()==SPageInfo::EManaged);
__NK_ASSERT_DEBUG(memory);
MmuLock::Unlock();
TPhysAddr newAddr;
return memory->iManager->MovePage(memory, aOldPageInfo, newAddr, aBlockZoneId, aBlockRest);
}
if (!iNumberOfFreePages)
{
// Allocate a new page for the live list as it has reached its minimum size.
MmuLock::Unlock();
SPageInfo* newPageInfo = GetPageFromSystem((Mmu::TRamAllocFlags)(EMemAttNormalCached|Mmu::EAllocNoWipe),
aBlockZoneId, aBlockRest);
if (!newPageInfo)
return KErrNoMemory;
// Re-acquire the mmulock and re-check that the page is not pinned or dirty.
MmuLock::Lock();
if (aOldPageInfo->Type() != SPageInfo::EUnused &&
(aOldPageInfo->PagedState() == SPageInfo::EPagedPinned ||
(aBlockRest && aOldPageInfo->IsDirty())))
{// Page is now pinned or dirty so give up as it is inuse.
ReturnPageToSystem(*newPageInfo);
MmuLock::Unlock();
return KErrInUse;
}
// Attempt to steal the page
r = StealPage(aOldPageInfo);
__NK_ASSERT_DEBUG(MmuLock::IsHeld());
if (r == KErrCompletion)
{// This was a page table that has been freed but added to the
// live list as a free page. Remove from live list and continue.
__NK_ASSERT_DEBUG(!aOldPageInfo->IsDirty());
RemovePage(aOldPageInfo);
r = KErrNone;
}
if (r == KErrNone)
{// Add the new page to the live list as discarding the old page
// will reduce the live list below the minimum.
AddAsFreePage(newPageInfo);
// We've successfully discarded the page so return it to the free pool.
ReturnPageToSystem(*aOldPageInfo);
BalanceAges();
}
else
{
// New page not required so just return it to the system. This is safe as
// iNumberOfFreePages will have this page counted but as it is not on the live list
// noone else can touch it.
ReturnPageToSystem(*newPageInfo);
}
}
else
{
// Attempt to steal the page
r = StealPage(aOldPageInfo);
__NK_ASSERT_DEBUG(MmuLock::IsHeld());
if (r == KErrCompletion)
{// This was a page table that has been freed but added to the
// live list as a free page. Remove from live list.
__NK_ASSERT_DEBUG(!aOldPageInfo->IsDirty());
RemovePage(aOldPageInfo);
r = KErrNone;
}
if (r == KErrNone)
{// We've successfully discarded the page so return it to the free pool.
ReturnPageToSystem(*aOldPageInfo);
BalanceAges();
}
}
MmuLock::Unlock();
return r;
}
TBool DPager::TryGrowLiveList()
{
__NK_ASSERT_DEBUG(RamAllocLock::IsHeld());
__NK_ASSERT_DEBUG(MmuLock::IsHeld());
MmuLock::Unlock();
SPageInfo* sparePage = GetPageFromSystem((Mmu::TRamAllocFlags)(EMemAttNormalCached|Mmu::EAllocNoWipe));
MmuLock::Lock();
if(!sparePage)
return false;
// add page to live list...
AddAsFreePage(sparePage);
return true;
}
SPageInfo* DPager::GetPageFromSystem(Mmu::TRamAllocFlags aAllocFlags, TUint aBlockZoneId, TBool aBlockRest)
{
__NK_ASSERT_DEBUG(RamAllocLock::IsHeld());
TPhysAddr pagePhys;
TInt r = TheMmu.AllocRam(&pagePhys, 1,
(Mmu::TRamAllocFlags)(aAllocFlags|Mmu::EAllocNoPagerReclaim),
EPageDiscard, aBlockZoneId, aBlockRest);
if(r!=KErrNone)
return NULL;
MmuLock::Lock();
++iNumberOfFreePages;
MmuLock::Unlock();
return SPageInfo::FromPhysAddr(pagePhys);
}
void DPager::ReturnPageToSystem()
{
__NK_ASSERT_DEBUG(RamAllocLock::IsHeld());
__NK_ASSERT_DEBUG(MmuLock::IsHeld());
ReturnPageToSystem(*StealOldestPage());
}
void DPager::ReturnPageToSystem(SPageInfo& aPageInfo)
{
__NK_ASSERT_DEBUG(RamAllocLock::IsHeld());
__NK_ASSERT_DEBUG(MmuLock::IsHeld());
__NK_ASSERT_DEBUG(iNumberOfFreePages>0);
--iNumberOfFreePages;
MmuLock::Unlock();
TPhysAddr pagePhys = aPageInfo.PhysAddr();
TheMmu.FreeRam(&pagePhys, 1, EPageDiscard);
MmuLock::Lock();
}
SPageInfo* DPager::PageInAllocPage(Mmu::TRamAllocFlags aAllocFlags)
{
SPageInfo* pageInfo;
TPhysAddr pagePhys;
RamAllocLock::Lock();
MmuLock::Lock();
// try getting a free page from our live list...
#ifdef _USE_OLDEST_LISTS
if (iOldestCleanCount)
{
pageInfo = SPageInfo::FromLink(iOldestCleanList.Last());
if(pageInfo->Type()==SPageInfo::EUnused)
goto get_oldest;
}
#else
if(iOldCount)
{
pageInfo = SPageInfo::FromLink(iOldList.Last());
if(pageInfo->Type()==SPageInfo::EUnused)
goto get_oldest;
}
#endif
// try getting a free page from the system pool...
if(!HaveMaximumPages())
{
MmuLock::Unlock();
pageInfo = GetPageFromSystem(aAllocFlags);
if(pageInfo)
goto done;
MmuLock::Lock();
}
// as a last resort, steal a page from the live list...
get_oldest:
#ifdef _USE_OLDEST_LISTS
__NK_ASSERT_ALWAYS(iOldestCleanCount|iOldestDirtyCount|iOldCount|iYoungCount);
#else
__NK_ASSERT_ALWAYS(iOldCount|iYoungCount);
#endif
pageInfo = StealOldestPage();
MmuLock::Unlock();
// make page state same as a freshly allocated page...
pagePhys = pageInfo->PhysAddr();
TheMmu.PagesAllocated(&pagePhys,1,aAllocFlags);
done:
RamAllocLock::Unlock();
return pageInfo;
}
TBool DPager::GetFreePages(TInt aNumPages)
{
TRACE(("DPager::GetFreePages(%d)",aNumPages));
__NK_ASSERT_DEBUG(RamAllocLock::IsHeld());
MmuLock::Lock();
while(aNumPages>0 && (TInt)NumberOfFreePages()>=aNumPages)
{
ReturnPageToSystem();
--aNumPages;
}
MmuLock::Unlock();
TRACE(("DPager::GetFreePages returns %d",!aNumPages));
return !aNumPages;
}
void DPager::DonatePages(TUint aCount, TPhysAddr* aPages)
{
TRACE(("DPager::DonatePages(%d,?)",aCount));
__ASSERT_CRITICAL;
RamAllocLock::Lock();
MmuLock::Lock();
TPhysAddr* end = aPages+aCount;
while(aPages<end)
{
TPhysAddr pagePhys = *aPages++;
if(RPageArray::State(pagePhys)!=RPageArray::ECommitted)
continue; // page is not present
#ifdef _DEBUG
SPageInfo* pi = SPageInfo::SafeFromPhysAddr(pagePhys&~KPageMask);
__NK_ASSERT_DEBUG(pi);
#else
SPageInfo* pi = SPageInfo::FromPhysAddr(pagePhys);
#endif
switch(pi->PagedState())
{
case SPageInfo::EUnpaged:
// Change the type of this page to discardable and
// then add it to live list.
// Only the DDiscardableMemoryManager should be invoking this and
// its pages will be movable before they are donated.
__NK_ASSERT_DEBUG(pi->Owner()->iManager->PageType() == EPageMovable);
TheMmu.ChangePageType(pi, EPageMovable, EPageDiscard);
break;
case SPageInfo::EPagedYoung:
case SPageInfo::EPagedOld:
#ifdef _USE_OLDEST_LISTS
case SPageInfo::EPagedOldestDirty:
case SPageInfo::EPagedOldestClean:
#endif
continue; // discard already been allowed
case SPageInfo::EPagedPinned:
__NK_ASSERT_DEBUG(0);
default:
__NK_ASSERT_DEBUG(0);
continue;
}
// put page on live list...
AddAsYoungestPage(pi);
++iNumberOfFreePages;
Event(EEventPageDonate,pi);
// re-balance live list...
RemoveExcessPages();
BalanceAges();
}
MmuLock::Unlock();
RamAllocLock::Unlock();
}
TInt DPager::ReclaimPages(TUint aCount, TPhysAddr* aPages)
{
TRACE(("DPager::ReclaimPages(%d,?)",aCount));
__ASSERT_CRITICAL;
RamAllocLock::Lock();
MmuLock::Lock();
TInt r = KErrNone;
TPhysAddr* end = aPages+aCount;
while(aPages<end)
{
TPhysAddr pagePhys = *aPages++;
TBool changeType = EFalse;
if(RPageArray::State(pagePhys)!=RPageArray::ECommitted)
{
r = KErrNotFound; // too late, page has gone
continue;
}
#ifdef _DEBUG
SPageInfo* pi = SPageInfo::SafeFromPhysAddr(pagePhys&~KPageMask);
__NK_ASSERT_DEBUG(pi);
#else
SPageInfo* pi = SPageInfo::FromPhysAddr(pagePhys);
#endif
switch(pi->PagedState())
{
case SPageInfo::EUnpaged:
continue; // discard already been disallowed
case SPageInfo::EPagedYoung:
case SPageInfo::EPagedOld:
#ifdef _USE_OLDEST_LISTS
case SPageInfo::EPagedOldestClean:
case SPageInfo::EPagedOldestDirty:
#endif
changeType = ETrue;
break; // remove from live list
case SPageInfo::EPagedPinned:
__NK_ASSERT_DEBUG(0);
default:
__NK_ASSERT_DEBUG(0);
break;
}
// check paging list has enough pages before we remove one...
if(iNumberOfFreePages<1)
{
// need more pages so get a page from the system...
if(!TryGrowLiveList())
{
// out of memory...
r = KErrNoMemory;
break;
}
// retry the page reclaim...
--aPages;
continue;
}
if (changeType)
{// Change the type of this page to movable, wait until any retries
// have been attempted as we can't change a page's type twice.
// Only the DDiscardableMemoryManager should be invoking this and
// its pages should be movable once they are reclaimed.
__NK_ASSERT_DEBUG(pi->Owner()->iManager->PageType() == EPageMovable);
TheMmu.ChangePageType(pi, EPageDiscard, EPageMovable);
}
// remove page from paging list...
__NK_ASSERT_DEBUG(iNumberOfFreePages>0);
--iNumberOfFreePages;
RemovePage(pi);
Event(EEventPageReclaim,pi);
// re-balance live list...
BalanceAges();
}
// we may have added a spare free page to the live list without removing one,
// this could cause us to have too many pages, so deal with this...
RemoveExcessPages();
MmuLock::Unlock();
RamAllocLock::Unlock();
return r;
}
TInt VMHalFunction(TAny*, TInt aFunction, TAny* a1, TAny* a2);
void DPager::Init3()
{
TRACEB(("DPager::Init3()"));
TheRomMemoryManager->Init3();
TheDataPagedMemoryManager->Init3();
TheCodePagedMemoryManager->Init3();
TInt r = Kern::AddHalEntry(EHalGroupVM, VMHalFunction, 0);
__NK_ASSERT_ALWAYS(r==KErrNone);
}
void DPager::Fault(TFault aFault)
{
Kern::Fault("DPager",aFault);
}
void DPager::BalanceAges()
{
__NK_ASSERT_DEBUG(MmuLock::IsHeld());
TBool restrictPage = EFalse;
SPageInfo* pageInfo = NULL;
#ifdef _USE_OLDEST_LISTS
TUint oldestCount = iOldestCleanCount + iOldestDirtyCount;
if((iOldCount + oldestCount) * iYoungOldRatio < iYoungCount)
#else
if (iOldCount * iYoungOldRatio < iYoungCount)
#endif
{
// Need more old pages so make one young page into an old page...
__NK_ASSERT_DEBUG(!iYoungList.IsEmpty());
__NK_ASSERT_DEBUG(iYoungCount);
SDblQueLink* link = iYoungList.Last()->Deque();
--iYoungCount;
pageInfo = SPageInfo::FromLink(link);
pageInfo->SetPagedState(SPageInfo::EPagedOld);
iOldList.AddHead(link);
++iOldCount;
Event(EEventPageAged,pageInfo);
// Delay restricting the page until it is safe to release the MmuLock.
restrictPage = ETrue;
}
#ifdef _USE_OLDEST_LISTS
// Check we have enough oldest pages.
if (oldestCount * iOldOldestRatio < iOldCount)
{
__NK_ASSERT_DEBUG(!iOldList.IsEmpty());
__NK_ASSERT_DEBUG(iOldCount);
SDblQueLink* link = iOldList.Last()->Deque();
--iOldCount;
SPageInfo* oldestPageInfo = SPageInfo::FromLink(link);
if (oldestPageInfo->IsDirty())
{
oldestPageInfo->SetPagedState(SPageInfo::EPagedOldestDirty);
iOldestDirtyList.AddHead(link);
++iOldestDirtyCount;
Event(EEventPageAgedDirty,oldestPageInfo);
}
else
{
oldestPageInfo->SetPagedState(SPageInfo::EPagedOldestClean);
iOldestCleanList.AddHead(link);
++iOldestCleanCount;
Event(EEventPageAgedClean,oldestPageInfo);
}
}
#endif
if (restrictPage)
{
// Make the recently aged old page inaccessible. This is done last as it
// will release the MmuLock and therefore the page counts may otherwise change.
RestrictPage(pageInfo,ERestrictPagesNoAccessForOldPage);
}
}
void DPager::RemoveExcessPages()
{
__NK_ASSERT_DEBUG(RamAllocLock::IsHeld());
__NK_ASSERT_DEBUG(MmuLock::IsHeld());
while(HaveTooManyPages())
ReturnPageToSystem();
}
void DPager::RejuvenatePageTable(TPte* aPt)
{
SPageInfo* pi = SPageInfo::FromPhysAddr(Mmu::PageTablePhysAddr(aPt));
SPageTableInfo* pti = SPageTableInfo::FromPtPtr(aPt);
if(!pti->IsDemandPaged())
{
__NK_ASSERT_DEBUG(pi->PagedState()==SPageInfo::EUnpaged);
return;
}
TRACE2(("DP: %O Rejuvenate PT 0x%08x 0x%08x",TheCurrentThread,pi->PhysAddr(),aPt));
switch(pi->PagedState())
{
case SPageInfo::EPagedYoung:
case SPageInfo::EPagedOld:
#ifdef _USE_OLDEST_LISTS
case SPageInfo::EPagedOldestClean:
case SPageInfo::EPagedOldestDirty:
#endif
RemovePage(pi);
AddAsYoungestPage(pi);
BalanceAges();
break;
case SPageInfo::EUnpaged:
AddAsYoungestPage(pi);
BalanceAges();
break;
case SPageInfo::EPagedPinned:
break;
default:
__NK_ASSERT_DEBUG(0);
break;
}
}
TInt DPager::PteAndInfoFromLinAddr( TInt aOsAsid, TLinAddr aAddress, DMemoryMappingBase* aMapping,
TUint aMapInstanceCount, TPte*& aPte, SPageInfo*& aPageInfo)
{
__NK_ASSERT_DEBUG(MmuLock::IsHeld());
// Verify the mapping is still mapped and has not been reused.
if (aMapInstanceCount != aMapping->MapInstanceCount() || aMapping->BeingDetached())
return KErrAbort;
aPte = Mmu::SafePtePtrFromLinAddr(aAddress,aOsAsid);
if(!aPte)
return KErrNotFound;
TPte pte = *aPte;
if(pte==KPteUnallocatedEntry)
return KErrNotFound;
SPageInfo* pi = SPageInfo::SafeFromPhysAddr(pte & ~KPageMask);
if(!pi)
return KErrNotFound;
aPageInfo = pi;
return KErrNone;
}
TInt DPager::TryRejuvenate( TInt aOsAsid, TLinAddr aAddress, TUint aAccessPermissions, TLinAddr aPc,
DMemoryMappingBase* aMapping, TUint aMapInstanceCount, DThread* aThread,
TAny* aExceptionInfo)
{
__NK_ASSERT_DEBUG(MmuLock::IsHeld());
SPageInfo* pi;
TPte* pPte;
TPte pte;
TInt r = PteAndInfoFromLinAddr(aOsAsid, aAddress, aMapping, aMapInstanceCount, pPte, pi);
if (r != KErrNone)
{
if (aThread->IsRealtime())
{// This thread is real time so it shouldn't be accessing paged out paged memory
// unless there is a paging trap.
MmuLock::Unlock();
// Ensure that we abort when the thread is not allowed to access paged out pages.
if (CheckRealtimeThreadFault(aThread, aExceptionInfo) != KErrNone)
r = KErrAbort;
MmuLock::Lock();
}
return r;
}
pte = *pPte;
SPageInfo::TType type = pi->Type();
SPageInfo::TPagedState state = pi->PagedState();
if (aThread->IsRealtime() &&
state != SPageInfo::EPagedPinned &&
state != SPageInfo::EPagedPinnedMoved)
{// This thread is real time so it shouldn't be accessing unpinned paged memory
// unless there is a paging trap.
MmuLock::Unlock();
r = CheckRealtimeThreadFault(aThread, aExceptionInfo);
MmuLock::Lock();
if (r != KErrNone)
return r;
// We had to release the MmuLock have to reverify the status of the page and mappings.
r = PteAndInfoFromLinAddr(aOsAsid, aAddress, aMapping, aMapInstanceCount, pPte, pi);
if (r != KErrNone)
return r;
pte = *pPte;
type = pi->Type();
state = pi->PagedState();
}
if (type != SPageInfo::EManaged)
return KErrNotFound;
if(state==SPageInfo::EUnpaged)
return KErrNotFound;
DMemoryObject* memory = pi->Owner();
TUint index = pi->Index();
TPhysAddr page = memory->iPages.Page(index);
if(!RPageArray::IsPresent(page))
return KErrNotFound;
TPhysAddr physAddr = pi->PhysAddr();
if ((page^physAddr) >= (TPhysAddr)KPageSize)
{// Page array entry should contain same physical address as PTE unless the
// page has or is being moved and this mapping accessed the page.
// Get the page info for the page that we should be using.
physAddr = page & ~KPageMask;
pi = SPageInfo::SafeFromPhysAddr(physAddr);
if(!pi)
return KErrNotFound;
type = pi->Type();
if (type!=SPageInfo::EManaged)
return KErrNotFound;
state = pi->PagedState();
if(state==SPageInfo::EUnpaged)
return KErrNotFound;
memory = pi->Owner();
index = pi->Index();
// Update pte to point to the correct physical address for this memory object's page.
pte = (pte & KPageMask) | physAddr;
}
if(aAccessPermissions&EReadWrite)
{// The mapping that took the fault permits writes and is still attached
// to the memory object therefore the object can't be read only.
__NK_ASSERT_DEBUG(!memory->IsReadOnly());
SetWritable(*pi);
}
pte = Mmu::MakePteAccessible(pte,aAccessPermissions&EReadWrite);
TRACE2(("!PTE %x=%x",pPte,pte));
*pPte = pte;
CacheMaintenance::SinglePteUpdated((TLinAddr)pPte);
InvalidateTLBForPage((aAddress&~KPageMask)|aOsAsid);
Event(EEventPageRejuvenate,pi,aPc,aAddress,aAccessPermissions);
TBool balance = false;
#ifdef _USE_OLDEST_LISTS
if( state==SPageInfo::EPagedYoung || state==SPageInfo::EPagedOld ||
state==SPageInfo::EPagedOldestClean || state==SPageInfo::EPagedOldestDirty)
#else
if(state==SPageInfo::EPagedYoung || state==SPageInfo::EPagedOld)
#endif
{
RemovePage(pi);
AddAsYoungestPage(pi);
// delay BalanceAges because we don't want to release MmuLock until after
// RejuvenatePageTable has chance to look at the page table page...
balance = true;
}
else
{// Clear the modifier so that if this page is being moved then this
// access is detected. For non-pinned pages the modifier is cleared
// by RemovePage().
__NK_ASSERT_DEBUG(state==SPageInfo::EPagedPinned);
pi->SetModifier(0);
}
RejuvenatePageTable(pPte);
if(balance)
BalanceAges();
return KErrNone;
}
TInt DPager::PageInAllocPages(TPhysAddr* aPages, TUint aCount, Mmu::TRamAllocFlags aAllocFlags)
{
TUint n = 0;
while(n<aCount)
{
SPageInfo* pi = PageInAllocPage(aAllocFlags);
if(!pi)
goto fail;
aPages[n++] = pi->PhysAddr();
}
return KErrNone;
fail:
PageInFreePages(aPages,n);
return KErrNoMemory;
}
void DPager::PageInFreePages(TPhysAddr* aPages, TUint aCount)
{
while(aCount--)
{
MmuLock::Lock();
SPageInfo* pi = SPageInfo::FromPhysAddr(aPages[aCount]);
switch(pi->PagedState())
{
case SPageInfo::EPagedYoung:
case SPageInfo::EPagedOld:
#ifdef _USE_OLDEST_LISTS
case SPageInfo::EPagedOldestClean:
case SPageInfo::EPagedOldestDirty:
#endif
RemovePage(pi);
// fall through...
case SPageInfo::EUnpaged:
AddAsFreePage(pi);
break;
case SPageInfo::EPagedPinned:
__NK_ASSERT_DEBUG(0);
break;
default:
__NK_ASSERT_DEBUG(0);
break;
}
MmuLock::Unlock();
}
}
void DPager::PagedInUnneeded(SPageInfo* aPageInfo)
{
__NK_ASSERT_DEBUG(MmuLock::IsHeld());
Event(EEventPageInUnneeded,aPageInfo);
AddAsFreePage(aPageInfo);
}
void DPager::PagedIn(SPageInfo* aPageInfo)
{
__NK_ASSERT_DEBUG(MmuLock::IsHeld());
switch(aPageInfo->PagedState())
{
case SPageInfo::EPagedYoung:
case SPageInfo::EPagedOld:
#ifdef _USE_OLDEST_LISTS
case SPageInfo::EPagedOldestClean:
case SPageInfo::EPagedOldestDirty:
#endif
RemovePage(aPageInfo);
AddAsYoungestPage(aPageInfo);
BalanceAges();
break;
case SPageInfo::EUnpaged:
AddAsYoungestPage(aPageInfo);
BalanceAges();
break;
case SPageInfo::EPagedPinned:
// Clear the modifier so that if this page is being moved then this
// access is detected. For non-pinned pages the modifier is cleared by RemovePage().
aPageInfo->SetModifier(0);
break;
default:
__NK_ASSERT_DEBUG(0);
break;
}
}
void DPager::PagedInPinned(SPageInfo* aPageInfo, TPinArgs& aPinArgs)
{
__NK_ASSERT_DEBUG(MmuLock::IsHeld());
Pin(aPageInfo,aPinArgs);
}
void DPager::Pin(SPageInfo* aPageInfo, TPinArgs& aPinArgs)
{
__ASSERT_CRITICAL;
__NK_ASSERT_DEBUG(MmuLock::IsHeld());
__NK_ASSERT_DEBUG(aPinArgs.HaveSufficientPages(1));
aPageInfo->IncPinCount();
Event(EEventPagePin,aPageInfo);
// remove page from live list...
switch(aPageInfo->PagedState())
{
case SPageInfo::EPagedYoung:
__NK_ASSERT_DEBUG(iYoungCount);
aPageInfo->iLink.Deque();
--iYoungCount;
__NK_ASSERT_DEBUG(aPageInfo->PinCount()==1);
break;
case SPageInfo::EPagedOld:
__NK_ASSERT_DEBUG(iOldCount);
aPageInfo->iLink.Deque();
--iOldCount;
__NK_ASSERT_DEBUG(aPageInfo->PinCount()==1);
break;
#ifdef _USE_OLDEST_LISTS
case SPageInfo::EPagedOldestClean:
__NK_ASSERT_DEBUG(iOldestCleanCount);
aPageInfo->iLink.Deque();
--iOldestCleanCount;
__NK_ASSERT_DEBUG(aPageInfo->PinCount()==1);
break;
case SPageInfo::EPagedOldestDirty:
__NK_ASSERT_DEBUG(iOldestDirtyCount);
aPageInfo->iLink.Deque();
--iOldestDirtyCount;
__NK_ASSERT_DEBUG(aPageInfo->PinCount()==1);
break;
#endif
case SPageInfo::EPagedPinned:
// nothing more to do...
__NK_ASSERT_DEBUG(aPageInfo->PinCount()>1);
return;
case SPageInfo::EUnpaged:
__NK_ASSERT_DEBUG(aPageInfo->PinCount()==1);
TRACE2(("DPager::PinPage page was unpaged"));
// This could be a page in the process of being stolen.
// Could also be page for storing page table infos, which aren't necessarily
// on the live list.
break;
default:
__NK_ASSERT_DEBUG(0);
return;
}
// page has now been removed from the live list and is pinned...
aPageInfo->SetPagedState(SPageInfo::EPagedPinned);
if(aPinArgs.iReplacementPages==TPinArgs::EUseReserveForPinReplacementPages)
{
// pinned paged counts as coming from reserve pool...
aPageInfo->SetPinnedReserve();
}
else
{
// we used up a replacement page...
--aPinArgs.iReplacementPages;
}
BalanceAges();
}
void DPager::Unpin(SPageInfo* aPageInfo, TPinArgs& aPinArgs)
{
__ASSERT_CRITICAL;
__NK_ASSERT_DEBUG(MmuLock::IsHeld());
__NK_ASSERT_DEBUG(aPageInfo->PagedState()==SPageInfo::EPagedPinned);
__NK_ASSERT_DEBUG(aPageInfo->PinCount()>0);
TUint pinCount = aPageInfo->DecPinCount();
Event(EEventPageUnpin,aPageInfo);
if(pinCount)
return;
aPageInfo->SetPagedState(SPageInfo::EUnpaged);
if(!aPageInfo->ClearPinnedReserve())
{
// was not a pinned reserve page, so we how have a spare replacement page,
// which can be used again or freed later ...
__NK_ASSERT_DEBUG(aPinArgs.iReplacementPages!=TPinArgs::EUseReserveForPinReplacementPages);
++aPinArgs.iReplacementPages;
}
AddAsYoungestPage(aPageInfo);
BalanceAges();
}
TInt TPinArgs::AllocReplacementPages(TUint aNumPages)
{
if(iUseReserve)
{
__NK_ASSERT_DEBUG(iReplacementPages==0 || iReplacementPages==EUseReserveForPinReplacementPages);
iReplacementPages = EUseReserveForPinReplacementPages;
}
else
{
if(aNumPages>iReplacementPages)
{
if(!ThePager.AllocPinReplacementPages(aNumPages-iReplacementPages))
return KErrNoMemory;
iReplacementPages = aNumPages;
}
}
return KErrNone;
}
void TPinArgs::FreeReplacementPages()
{
if(iReplacementPages!=0 && iReplacementPages!=EUseReserveForPinReplacementPages)
ThePager.FreePinReplacementPages(iReplacementPages);
iReplacementPages = 0;
}
TBool DPager::AllocPinReplacementPages(TUint aNumPages)
{
TRACE2(("DPager::AllocPinReplacementPages(0x%x)",aNumPages));
__ASSERT_CRITICAL;
RamAllocLock::Lock();
MmuLock::Lock();
TBool ok = false;
do
{
if(iNumberOfFreePages>=aNumPages)
{
iNumberOfFreePages -= aNumPages;
ok = true;
break;
}
}
while(TryGrowLiveList());
MmuLock::Unlock();
RamAllocLock::Unlock();
return ok;
}
void DPager::FreePinReplacementPages(TUint aNumPages)
{
TRACE2(("DPager::FreePinReplacementPage(0x%x)",aNumPages));
__ASSERT_CRITICAL;
RamAllocLock::Lock();
MmuLock::Lock();
iNumberOfFreePages += aNumPages;
RemoveExcessPages();
MmuLock::Unlock();
RamAllocLock::Unlock();
}
TBool DPager::ReservePage()
{
__NK_ASSERT_DEBUG(RamAllocLock::IsHeld());
__NK_ASSERT_DEBUG(MmuLock::IsHeld());
__ASSERT_CRITICAL;
__NK_ASSERT_DEBUG(iMinimumPageCount >= iMinimumPageLimit+iReservePageCount);
while(iMinimumPageCount==iMinimumPageLimit+iReservePageCount && iNumberOfFreePages==0)
{
if(!TryGrowLiveList())
return false;
}
if(iMinimumPageCount==iMinimumPageLimit+iReservePageCount)
{
++iMinimumPageCount;
--iNumberOfFreePages;
if(iMinimumPageCount>iMaximumPageCount)
iMaximumPageCount = iMinimumPageCount;
}
++iReservePageCount;
__NK_ASSERT_DEBUG(iMinimumPageCount >= iMinimumPageLimit+iReservePageCount);
__NK_ASSERT_DEBUG(iMinimumPageCount+iNumberOfFreePages <= iMaximumPageCount);
return ETrue;
}
TBool DPager::ReservePages(TUint aRequiredCount, TUint& aCount)
{
__ASSERT_CRITICAL;
RamAllocLock::Lock();
MmuLock::Lock();
while(aCount<aRequiredCount)
{
if(!ReservePage())
break;
++aCount;
MmuLock::Flash();
}
TBool enoughPages = aCount==aRequiredCount;
MmuLock::Unlock();
RamAllocLock::Unlock();
if(!enoughPages)
UnreservePages(aCount);
return enoughPages;
}
void DPager::UnreservePages(TUint& aCount)
{
MmuLock::Lock();
iReservePageCount -= aCount;
aCount = 0;
MmuLock::Unlock();
}
TInt DPager::CheckRealtimeThreadFault(DThread* aThread, TAny* aExceptionInfo)
{
// realtime threads shouldn't take paging faults...
DThread* client = aThread->iIpcClient;
// If iIpcClient is set then we are accessing the address space of a remote thread. If we are
// in an IPC trap, this will contain information the local and remote addresses being accessed.
// If this is not set then we assume than any fault must be the fault of a bad remote address.
TIpcExcTrap* ipcTrap = (TIpcExcTrap*)aThread->iExcTrap;
if (ipcTrap && !ipcTrap->IsTIpcExcTrap())
ipcTrap = 0;
if (client && (!ipcTrap || ipcTrap->ExcLocation(aThread, aExceptionInfo) == TIpcExcTrap::EExcRemote))
{
// kill client thread...
if(K::IllegalFunctionForRealtimeThread(client,"Access to Paged Memory (by other thread)"))
{
// treat memory access as bad...
return KErrAbort;
}
// else thread is in 'warning only' state so allow paging...
}
else
{
// kill current thread...
if(K::IllegalFunctionForRealtimeThread(NULL,"Access to Paged Memory"))
{
// if current thread is in critical section, then the above kill will be deferred
// and we will continue executing. We will handle this by returning an error
// which means that the thread will take an exception (which hopefully is XTRAPed!)
return KErrAbort;
}
// else thread is in 'warning only' state so allow paging...
}
return KErrNone;
}
TInt DPager::HandlePageFault( TLinAddr aPc, TLinAddr aFaultAddress, TUint aFaultAsid, TUint aFaultIndex,
TUint aAccessPermissions, DMemoryObject* aMemory, DMemoryMapping* aMapping,
TUint aMapInstanceCount, DThread* aThread, TAny* aExceptionInfo)
{
MmuLock::Lock();
TInt r = TryRejuvenate( aFaultAsid, aFaultAddress, aAccessPermissions, aPc, aMapping, aMapInstanceCount,
aThread, aExceptionInfo);
if(r == KErrNone || r == KErrAbort)
{
MmuLock::Unlock();
}
else
{
// rejuvenate failed, call memory manager to page in memory...
Event(EEventPageInStart, 0, aPc, aFaultAddress, aAccessPermissions);
MmuLock::Unlock();
TheThrashMonitor.NotifyStartPaging();
DMemoryManager* manager = aMemory->iManager;
r = manager->HandleFault(aMemory, aFaultIndex, aMapping, aMapInstanceCount, aAccessPermissions);
TheThrashMonitor.NotifyEndPaging();
}
return r;
}
TInt DPager::ResizeLiveList()
{
MmuLock::Lock();
TUint min = iMinimumPageCount;
TUint max = iMaximumPageCount;
MmuLock::Unlock();
return ResizeLiveList(min,max);
}
TInt DPager::ResizeLiveList(TUint aMinimumPageCount, TUint aMaximumPageCount)
{
TRACE(("DPager::ResizeLiveList(%d,%d) current young=%d old=%d min=%d free=%d max=%d",aMinimumPageCount,aMaximumPageCount,iYoungCount,iOldCount,iMinimumPageCount,iNumberOfFreePages,iMaximumPageCount));
if(!aMaximumPageCount)
{
aMinimumPageCount = iInitMinimumPageCount;
aMaximumPageCount = iInitMaximumPageCount;
}
if (aMaximumPageCount > KAbsoluteMaxPageCount)
aMaximumPageCount = KAbsoluteMaxPageCount;
// Min must not be greater than max...
if(aMinimumPageCount>aMaximumPageCount)
return KErrArgument;
NKern::ThreadEnterCS();
RamAllocLock::Lock();
MmuLock::Lock();
// Make sure aMinimumPageCount is not less than absolute minimum we can cope with...
iMinimumPageLimit = iMinYoungPages * (1 + iYoungOldRatio) / iYoungOldRatio
+ DPageReadRequest::ReservedPagesRequired();
if(iMinimumPageLimit<iAbsoluteMinPageCount)
iMinimumPageLimit = iAbsoluteMinPageCount;
if(aMinimumPageCount<iMinimumPageLimit+iReservePageCount)
aMinimumPageCount = iMinimumPageLimit+iReservePageCount;
if(aMaximumPageCount<aMinimumPageCount)
aMaximumPageCount=aMinimumPageCount;
// Increase iMaximumPageCount?
TInt extra = aMaximumPageCount-iMaximumPageCount;
if(extra>0)
iMaximumPageCount += extra;
// Reduce iMinimumPageCount?
TInt spare = iMinimumPageCount-aMinimumPageCount;
if(spare>0)
{
iMinimumPageCount -= spare;
iNumberOfFreePages += spare;
}
// Increase iMinimumPageCount?
TInt r=KErrNone;
while(iMinimumPageCount<aMinimumPageCount)
{
TUint newMin = aMinimumPageCount;
TUint maxMin = iMinimumPageCount+iNumberOfFreePages;
if(newMin>maxMin)
newMin = maxMin;
TUint delta = newMin-iMinimumPageCount;
if(delta)
{
iMinimumPageCount = newMin;
iNumberOfFreePages -= delta;
continue;
}
if(!TryGrowLiveList())
{
r=KErrNoMemory;
break;
}
}
// Reduce iMaximumPageCount?
while(iMaximumPageCount>aMaximumPageCount)
{
TUint newMax = aMaximumPageCount;
TUint minMax = iMinimumPageCount+iNumberOfFreePages;
if(newMax<minMax)
newMax = minMax;
TUint delta = iMaximumPageCount-newMax;
if(delta)
{
iMaximumPageCount = newMax;
continue;
}
ReturnPageToSystem();
}
TRACE(("DPager::ResizeLiveList end with young=%d old=%d min=%d free=%d max=%d",iYoungCount,iOldCount,iMinimumPageCount,iNumberOfFreePages,iMaximumPageCount));
#ifdef BTRACE_KERNEL_MEMORY
BTrace4(BTrace::EKernelMemory,BTrace::EKernelMemoryDemandPagingCache,iMinimumPageCount << KPageShift);
#endif
MmuLock::Unlock();
RamAllocLock::Unlock();
NKern::ThreadLeaveCS();
return r;
}
void DPager::FlushAll()
{
NKern::ThreadEnterCS();
RamAllocLock::Lock();
TRACE(("DPager::FlushAll() live list young=%d old=%d min=%d free=%d max=%d",iYoungCount,iOldCount,iMinimumPageCount,iNumberOfFreePages,iMaximumPageCount));
// look at all RAM pages in the system, and unmap all those used for paging
const TUint32* piMap = (TUint32*)KPageInfoMap;
const TUint32* piMapEnd = piMap+(KNumPageInfoPages>>5);
SPageInfo* pi = (SPageInfo*)KPageInfoLinearBase;
MmuLock::Lock();
do
{
SPageInfo* piNext = pi+(KPageInfosPerPage<<5);
for(TUint32 piFlags=*piMap++; piFlags; piFlags>>=1)
{
if(!(piFlags&1))
{
pi += KPageInfosPerPage;
continue;
}
SPageInfo* piEnd = pi+KPageInfosPerPage;
do
{
SPageInfo::TPagedState state = pi->PagedState();
#ifdef _USE_OLDEST_LISTS
if (state==SPageInfo::EPagedYoung || state==SPageInfo::EPagedOld ||
state==SPageInfo::EPagedOldestClean || state==SPageInfo::EPagedOldestDirty)
#else
if(state==SPageInfo::EPagedYoung || state==SPageInfo::EPagedOld)
#endif
{
if (pi->Type() != SPageInfo::EUnused)
{
TInt r = StealPage(pi);
if(r==KErrNone)
AddAsFreePage(pi);
MmuLock::Flash();
}
}
++pi;
if(((TUint)pi&(0xf<<KPageInfoShift))==0)
MmuLock::Flash(); // every 16 page infos
}
while(pi<piEnd);
}
pi = piNext;
}
while(piMap<piMapEnd);
MmuLock::Unlock();
// reduce live page list to a minimum
while(GetFreePages(1)) {};
TRACE(("DPager::FlushAll() end with young=%d old=%d min=%d free=%d max=%d",iYoungCount,iOldCount,iMinimumPageCount,iNumberOfFreePages,iMaximumPageCount));
RamAllocLock::Unlock();
NKern::ThreadLeaveCS();
}
void DPager::GetLiveListInfo(SVMCacheInfo& aInfo)
{
MmuLock::Lock(); // ensure consistent set of values are read...
aInfo.iMinSize = iMinimumPageCount<<KPageShift;
aInfo.iMaxSize = iMaximumPageCount<<KPageShift;
aInfo.iCurrentSize = (iMinimumPageCount+iNumberOfFreePages)<<KPageShift;
aInfo.iMaxFreeSize = iNumberOfFreePages<<KPageShift;
MmuLock::Unlock();
}
void DPager::GetEventInfo(SVMEventInfo& aInfoOut)
{
MmuLock::Lock(); // ensure consistent set of values are read...
aInfoOut = iEventInfo;
MmuLock::Unlock();
}
void DPager::ResetEventInfo()
{
MmuLock::Lock();
memclr(&iEventInfo, sizeof(iEventInfo));
MmuLock::Unlock();
}
TInt TestPageState(TLinAddr aAddr)
{
DMemModelProcess* process = (DMemModelProcess*)TheCurrentThread->iOwningProcess;
// Get the os asid of current thread's process so no need to open a reference on it.
TInt osAsid = process->OsAsid();
TPte* ptePtr = 0;
TPte pte = 0;
TInt r = 0;
SPageInfo* pageInfo = NULL;
NKern::ThreadEnterCS();
TUint offsetInMapping;
TUint mapInstanceCount;
DMemoryMapping* mapping = MM::FindMappingInAddressSpace(osAsid, aAddr, 1, offsetInMapping, mapInstanceCount);
MmuLock::Lock();
if(mapping)
{
DMemoryObject* memory = mapping->Memory();
if(mapInstanceCount == mapping->MapInstanceCount() && memory)
{
DMemoryManager* manager = memory->iManager;
if(manager==TheCodePagedMemoryManager)
r |= EPageStateInRamCode|EPageStatePaged;
}
}
ptePtr = Mmu::SafePtePtrFromLinAddr(aAddr,osAsid);
if (!ptePtr)
goto done;
pte = *ptePtr;
if (pte == KPteUnallocatedEntry)
goto done;
r |= EPageStatePtePresent;
if (pte!=Mmu::MakePteInaccessible(pte,0))
r |= EPageStatePteValid;
pageInfo = SPageInfo::SafeFromPhysAddr(pte&~KPageMask);
if(pageInfo)
{
r |= pageInfo->Type();
r |= pageInfo->PagedState()<<8;
}
done:
MmuLock::Unlock();
if(mapping)
mapping->Close();
NKern::ThreadLeaveCS();
return r;
}
TInt VMHalFunction(TAny*, TInt aFunction, TAny* a1, TAny* a2)
{
switch(aFunction)
{
case EVMHalFlushCache:
if(!TheCurrentThread->HasCapability(ECapabilityWriteDeviceData,__PLATSEC_DIAGNOSTIC_STRING("Checked by VMHalFunction(EVMHalFlushCache)")))
K::UnlockedPlatformSecurityPanic();
ThePager.FlushAll();
return KErrNone;
case EVMHalSetCacheSize:
{
if(!TheCurrentThread->HasCapability(ECapabilityWriteDeviceData,__PLATSEC_DIAGNOSTIC_STRING("Checked by VMHalFunction(EVMHalSetCacheSize)")))
K::UnlockedPlatformSecurityPanic();
TUint min = TUint(a1)>>KPageShift;
if(TUint(a1)&KPageMask)
++min;
TUint max = TUint(a2)>>KPageShift;
if(TUint(a2)&KPageMask)
++max;
return ThePager.ResizeLiveList(min,max);
}
case EVMHalGetCacheSize:
{
SVMCacheInfo info;
ThePager.GetLiveListInfo(info);
kumemput32(a1,&info,sizeof(info));
}
return KErrNone;
case EVMHalGetEventInfo:
{
SVMEventInfo info;
ThePager.GetEventInfo(info);
Kern::InfoCopy(*(TDes8*)a1,(TUint8*)&info,sizeof(info));
}
return KErrNone;
case EVMHalResetEventInfo:
ThePager.ResetEventInfo();
return KErrNone;
#ifdef __SUPPORT_DEMAND_PAGING_EMULATION__
case EVMHalGetOriginalRomPages:
RomOriginalPages(*((TPhysAddr**)a1), *((TUint*)a2));
return KErrNone;
#endif
case EVMPageState:
return TestPageState((TLinAddr)a1);
case EVMHalGetSwapInfo:
{
if ((K::MemModelAttributes & EMemModelAttrDataPaging) == 0)
return KErrNotSupported;
SVMSwapInfo info;
GetSwapInfo(info);
kumemput32(a1,&info,sizeof(info));
}
return KErrNone;
case EVMHalGetThrashLevel:
return TheThrashMonitor.ThrashLevel();
case EVMHalSetSwapThresholds:
{
if(!TheCurrentThread->HasCapability(ECapabilityWriteDeviceData,__PLATSEC_DIAGNOSTIC_STRING("Checked by VMHalFunction(EVMHalSetSwapThresholds)")))
K::UnlockedPlatformSecurityPanic();
if ((K::MemModelAttributes & EMemModelAttrDataPaging) == 0)
return KErrNotSupported;
SVMSwapThresholds thresholds;
kumemget32(&thresholds,a1,sizeof(thresholds));
return SetSwapThresholds(thresholds);
}
case EVMHalSetThrashThresholds:
if(!TheCurrentThread->HasCapability(ECapabilityWriteDeviceData,__PLATSEC_DIAGNOSTIC_STRING("Checked by VMHalFunction(EVMHalSetThrashThresholds)")))
K::UnlockedPlatformSecurityPanic();
return TheThrashMonitor.SetThresholds((TUint)a1, (TUint)a2);
#ifdef __DEMAND_PAGING_BENCHMARKS__
case EVMHalGetPagingBenchmark:
{
TUint index = (TInt) a1;
if (index >= EMaxPagingBm)
return KErrNotFound;
NKern::LockSystem();
SPagingBenchmarkInfo info = ThePager.iBenchmarkInfo[index];
NKern::UnlockSystem();
kumemput32(a2,&info,sizeof(info));
}
return KErrNone;
case EVMHalResetPagingBenchmark:
{
TUint index = (TInt) a1;
if (index >= EMaxPagingBm)
return KErrNotFound;
NKern::LockSystem();
ThePager.ResetBenchmarkData((TPagingBenchmark)index);
NKern::UnlockSystem();
}
return KErrNone;
#endif
default:
return KErrNotSupported;
}
}
#ifdef __DEMAND_PAGING_BENCHMARKS__
void DPager::ResetBenchmarkData(TPagingBenchmark aBm)
{
SPagingBenchmarkInfo& info = iBenchmarkInfo[aBm];
info.iCount = 0;
info.iTotalTime = 0;
info.iMaxTime = 0;
info.iMinTime = KMaxTInt;
}
void DPager::RecordBenchmarkData(TPagingBenchmark aBm, TUint32 aStartTime, TUint32 aEndTime)
{
SPagingBenchmarkInfo& info = iBenchmarkInfo[aBm];
++info.iCount;
#if !defined(HIGH_RES_TIMER) || defined(HIGH_RES_TIMER_COUNTS_UP)
TInt64 elapsed = aEndTime - aStartTime;
#else
TInt64 elapsed = aStartTime - aEndTime;
#endif
info.iTotalTime += elapsed;
if (elapsed > info.iMaxTime)
info.iMaxTime = elapsed;
if (elapsed < info.iMinTime)
info.iMinTime = elapsed;
}
#endif //__DEMAND_PAGING_BENCHMARKS__
//
// Paging request management...
//
//
// DPagingRequest
//
DPagingRequest::DPagingRequest(DPagingRequestPool::TGroup& aPoolGroup)
: iPoolGroup(aPoolGroup), iUseRegionMemory(0), iUseRegionIndex(0), iUseRegionCount(0)
{
}
FORCE_INLINE void DPagingRequest::SetUse(DMemoryObject* aMemory, TUint aIndex, TUint aCount)
{
__ASSERT_SYSTEM_LOCK;
iUseRegionMemory = aMemory;
iUseRegionIndex = aIndex;
iUseRegionCount = aCount;
}
TBool DPagingRequest::CheckUse(DMemoryObject* aMemory, TUint aIndex, TUint aCount)
{
return aMemory==iUseRegionMemory
&& TUint(aIndex-iUseRegionIndex) < iUseRegionCount
&& TUint(iUseRegionCount-TUint(aIndex-iUseRegionIndex)) <= aCount;
}
void DPagingRequest::Release()
{
NKern::LockSystem();
SetUse(0,0,0);
Signal();
}
void DPagingRequest::Wait()
{
__ASSERT_SYSTEM_LOCK;
++iUsageCount;
TInt r = iMutex->Wait();
__NK_ASSERT_ALWAYS(r == KErrNone);
}
void DPagingRequest::Signal()
{
__ASSERT_SYSTEM_LOCK;
iPoolGroup.Signal(this);
}
FORCE_INLINE TBool DPagingRequest::IsCollision(DMemoryObject* aMemory, TUint aIndex, TUint aCount)
{
__ASSERT_SYSTEM_LOCK;
DMemoryObject* memory = iUseRegionMemory;
TUint index = iUseRegionIndex;
TUint count = iUseRegionCount;
// note, this comparison would fail if either region includes page number KMaxTUint,
// but it isn't possible to create a memory object which is > KMaxTUint pages...
return memory == aMemory && index+count > aIndex && index < aIndex+aCount;
}
TLinAddr DPagingRequest::MapPages(TUint aColour, TUint aCount, TPhysAddr* aPages)
{
__NK_ASSERT_DEBUG(iMutex->iCleanup.iThread == &Kern::CurrentThread());
return iTempMapping.Map(aPages,aCount,aColour);
}
void DPagingRequest::UnmapPages(TBool aIMBRequired)
{
__NK_ASSERT_DEBUG(iMutex->iCleanup.iThread == &Kern::CurrentThread());
iTempMapping.Unmap(aIMBRequired);
}
//
// DPageReadRequest
//
TInt DPageReadRequest::iAllocNext = 0;
TInt DPageReadRequest::Construct()
{
// allocate id and mutex...
TUint id = (TUint)__e32_atomic_add_ord32(&iAllocNext, 1);
_LIT(KLitPagingRequest,"PageReadRequest-");
TBuf<sizeof("PageReadRequest-")+10> mutexName(KLitPagingRequest);
mutexName.AppendNum(id);
TInt r = K::MutexCreate(iMutex, mutexName, NULL, EFalse, KMutexOrdPageIn);
if(r!=KErrNone)
return r;
// allocate space for mapping pages whilst they're being loaded...
iTempMapping.Alloc(EMaxPages);
// create memory buffer...
TUint bufferSize = EMaxPages+1;
DMemoryObject* bufferMemory;
r = MM::MemoryNew(bufferMemory,EMemoryObjectUnpaged,bufferSize,EMemoryCreateNoWipe);
if(r!=KErrNone)
return r;
MM::MemorySetLock(bufferMemory,iMutex);
TPhysAddr physAddr;
r = MM::MemoryAllocContiguous(bufferMemory,0,bufferSize,0,physAddr);
(void)physAddr;
if(r!=KErrNone)
return r;
DMemoryMapping* bufferMapping;
r = MM::MappingNew(bufferMapping,bufferMemory,ESupervisorReadWrite,KKernelOsAsid);
if(r!=KErrNone)
return r;
iBuffer = MM::MappingBase(bufferMapping);
// ensure there are enough young pages to cope with new request object...
r = ThePager.ResizeLiveList();
if(r!=KErrNone)
return r;
return r;
}
//
// DPageWriteRequest
//
TInt DPageWriteRequest::iAllocNext = 0;
TInt DPageWriteRequest::Construct()
{
// allocate id and mutex...
TUint id = (TUint)__e32_atomic_add_ord32(&iAllocNext, 1);
_LIT(KLitPagingRequest,"PageWriteRequest-");
TBuf<sizeof("PageWriteRequest-")+10> mutexName(KLitPagingRequest);
mutexName.AppendNum(id);
TInt r = K::MutexCreate(iMutex, mutexName, NULL, EFalse, KMutexOrdPageOut);
if(r!=KErrNone)
return r;
// allocate space for mapping pages whilst they're being loaded...
iTempMapping.Alloc(EMaxPages);
return r;
}
//
// DPagingRequestPool
//
DPagingRequestPool::DPagingRequestPool(TUint aNumPageReadRequest,TUint aNumPageWriteRequest)
: iPageReadRequests(aNumPageReadRequest), iPageWriteRequests(aNumPageWriteRequest)
{
TUint i;
for(i=0; i<aNumPageReadRequest; ++i)
{
DPageReadRequest* req = new DPageReadRequest(iPageReadRequests);
__NK_ASSERT_ALWAYS(req);
TInt r = req->Construct();
__NK_ASSERT_ALWAYS(r==KErrNone);
iPageReadRequests.iRequests[i] = req;
iPageReadRequests.iFreeList.Add(req);
}
for(i=0; i<aNumPageWriteRequest; ++i)
{
DPageWriteRequest* req = new DPageWriteRequest(iPageWriteRequests);
__NK_ASSERT_ALWAYS(req);
TInt r = req->Construct();
__NK_ASSERT_ALWAYS(r==KErrNone);
iPageWriteRequests.iRequests[i] = req;
iPageWriteRequests.iFreeList.Add(req);
}
}
DPagingRequestPool::~DPagingRequestPool()
{
__NK_ASSERT_ALWAYS(0); // deletion not implemented
}
DPageReadRequest* DPagingRequestPool::AcquirePageReadRequest(DMemoryObject* aMemory, TUint aIndex, TUint aCount)
{
NKern::LockSystem();
DPagingRequest* req;
// if we collide with page write operation...
req = iPageWriteRequests.FindCollision(aMemory,aIndex,aCount);
if(req)
{
// wait until write completes...
req->Wait();
req->Signal();
return 0; // caller expected to retry if needed
}
// get a request object to use...
req = iPageReadRequests.GetRequest(aMemory,aIndex,aCount);
// check no new requests collide with us...
if(iPageWriteRequests.FindCollision(aMemory,aIndex,aCount)
|| iPageReadRequests.FindCollision(aMemory,aIndex,aCount))
{
// another operation is colliding with this region, give up and retry...
req->Signal();
return 0; // caller expected to retry if needed
}
// we have a request object which we can use...
req->SetUse(aMemory,aIndex,aCount);
NKern::UnlockSystem();
return (DPageReadRequest*)req;
}
DPageWriteRequest* DPagingRequestPool::AcquirePageWriteRequest(DMemoryObject* aMemory, TUint aIndex, TUint aCount)
{
NKern::LockSystem();
DPagingRequest* req;
for(;;)
{
// get a request object to use...
req = iPageWriteRequests.GetRequest(aMemory,aIndex,aCount);
if(iPageWriteRequests.FindCollision(aMemory,aIndex,aCount))
{
// another write operation is colliding with this region, give up and retry...
req->Signal();
// Reacquire the system lock as Signal() above will release it.
NKern::LockSystem();
continue;
}
break;
}
// we have a request object which we can use...
req->SetUse(aMemory,aIndex,aCount);
NKern::UnlockSystem();
return (DPageWriteRequest*)req;
}
DPagingRequestPool::TGroup::TGroup(TUint aNumRequests)
{
iNumRequests = aNumRequests;
iRequests = new DPagingRequest*[aNumRequests];
__NK_ASSERT_ALWAYS(iRequests);
}
DPagingRequest* DPagingRequestPool::TGroup::FindCollision(DMemoryObject* aMemory, TUint aIndex, TUint aCount)
{
__ASSERT_SYSTEM_LOCK;
DPagingRequest** ptr = iRequests;
DPagingRequest** ptrEnd = ptr+iNumRequests;
while(ptr<ptrEnd)
{
DPagingRequest* req = *ptr++;
if(req->IsCollision(aMemory,aIndex,aCount))
return req;
}
return 0;
}
static TUint32 RandomSeed = 33333;
DPagingRequest* DPagingRequestPool::TGroup::GetRequest(DMemoryObject* aMemory, TUint aIndex, TUint aCount)
{
__NK_ASSERT_DEBUG(iNumRequests > 0);
// try using an existing request which collides with this region...
DPagingRequest* req = FindCollision(aMemory,aIndex,aCount);
if(!req)
{
// use a free request...
req = (DPagingRequest*)iFreeList.GetFirst();
if(req)
{
// free requests aren't being used...
__NK_ASSERT_DEBUG(req->iUsageCount == 0);
}
else
{
// pick a random request...
RandomSeed = RandomSeed*69069+1; // next 'random' number
TUint index = (TUint64(RandomSeed) * TUint64(iNumRequests)) >> 32;
req = iRequests[index];
__NK_ASSERT_DEBUG(req->iUsageCount > 0); // we only pick random when none are free
}
}
// wait for chosen request object...
req->Wait();
return req;
}
void DPagingRequestPool::TGroup::Signal(DPagingRequest* aRequest)
{
// if there are no threads waiting on the mutex then return it to the free pool...
__NK_ASSERT_DEBUG(aRequest->iUsageCount > 0);
if (--aRequest->iUsageCount==0)
iFreeList.AddHead(aRequest);
aRequest->iMutex->Signal();
}
/**
Register the specified paging device with the kernel.
@param aDevice A pointer to the paging device to install
@return KErrNone on success
*/
EXPORT_C TInt Kern::InstallPagingDevice(DPagingDevice* aDevice)
{
TRACEB(("Kern::InstallPagingDevice(0x%08x) name='%s' type=%d",aDevice,aDevice->iName,aDevice->iType));
__NK_ASSERT_ALWAYS(aDevice->iReadUnitShift <= KPageShift);
TInt r = KErrNotSupported; // Will return this if unsupported device type is installed
// create the pools of page out and page in requests...
const TInt writeReqs = (aDevice->iType & DPagingDevice::EData) ? KPagingRequestsPerDevice : 0;
aDevice->iRequestPool = new DPagingRequestPool(KPagingRequestsPerDevice,writeReqs);
if(!aDevice->iRequestPool)
{
r = KErrNoMemory;
goto exit;
}
if(aDevice->iType & DPagingDevice::ERom)
{
r = TheRomMemoryManager->InstallPagingDevice(aDevice);
if(r!=KErrNone)
goto exit;
}
if(aDevice->iType & DPagingDevice::ECode)
{
r = TheCodePagedMemoryManager->InstallPagingDevice(aDevice);
if(r!=KErrNone)
goto exit;
}
if(aDevice->iType & DPagingDevice::EData)
{
r = TheDataPagedMemoryManager->InstallPagingDevice(aDevice);
if(r!=KErrNone)
goto exit;
}
if (K::MemModelAttributes & (EMemModelAttrRomPaging | EMemModelAttrCodePaging | EMemModelAttrDataPaging))
TheThrashMonitor.Start();
exit:
TRACEB(("Kern::InstallPagingDevice returns %d",r));
return r;
}
//
// DDemandPagingLock
//
EXPORT_C DDemandPagingLock::DDemandPagingLock()
: iReservedPageCount(0), iLockedPageCount(0), iPinMapping(0)
{
}
EXPORT_C TInt DDemandPagingLock::Alloc(TInt aSize)
{
TRACEP(("DDemandPagingLock[0x%08x]::Alloc(0x%x)",this,aSize));
iMaxPageCount = ((aSize-1+KPageMask)>>KPageShift)+1;
TInt r = KErrNoMemory;
NKern::ThreadEnterCS();
TUint maxPt = DVirtualPinMapping::MaxPageTables(iMaxPageCount);
// Note, we need to reserve whole pages even for page tables which are smaller
// because pinning can remove the page from live list...
TUint reserve = iMaxPageCount+maxPt*KNumPagesToPinOnePageTable;
if(ThePager.ReservePages(reserve,(TUint&)iReservedPageCount))
{
iPinMapping = DVirtualPinMapping::New(iMaxPageCount);
if(iPinMapping)
r = KErrNone;
else
ThePager.UnreservePages((TUint&)iReservedPageCount);
}
NKern::ThreadLeaveCS();
TRACEP(("DDemandPagingLock[0x%08x]::Alloc returns %d, iMaxPageCount=%d, iReservedPageCount=%d",this,r,iMaxPageCount,iReservedPageCount));
return r;
}
EXPORT_C void DDemandPagingLock::Free()
{
TRACEP(("DDemandPagingLock[0x%08x]::Free()"));
Unlock();
NKern::ThreadEnterCS();
DVirtualPinMapping* pinMapping = (DVirtualPinMapping*)__e32_atomic_swp_ord_ptr(&iPinMapping, 0);
if (pinMapping)
pinMapping->Close();
NKern::ThreadLeaveCS();
ThePager.UnreservePages((TUint&)iReservedPageCount);
}
EXPORT_C TInt DDemandPagingLock::Lock(DThread* aThread, TLinAddr aStart, TInt aSize)
{
// TRACEP(("DDemandPagingLock[0x%08x]::Lock(0x%08x,0x%08x,0x%08x)",this,aThread,aStart,aSize));
if(iLockedPageCount)
__NK_ASSERT_ALWAYS(0); // lock already used
// calculate the number of pages that need to be locked...
TUint mask=KPageMask;
TUint offset=aStart&mask;
TInt numPages = (aSize+offset+mask)>>KPageShift;
if(numPages>iMaxPageCount)
__NK_ASSERT_ALWAYS(0);
NKern::ThreadEnterCS();
// find mapping which covers the specified region...
TUint offsetInMapping;
TUint mapInstanceCount;
DMemoryMapping* mapping = MM::FindMappingInThread((DMemModelThread*)aThread, aStart, aSize, offsetInMapping, mapInstanceCount);
if(!mapping)
{
NKern::ThreadLeaveCS();
return KErrBadDescriptor;
}
MmuLock::Lock();
DMemoryObject* memory = mapping->Memory();
if(mapInstanceCount != mapping->MapInstanceCount() || !memory)
{// Mapping has been reused or no memory.
MmuLock::Unlock();
mapping->Close();
NKern::ThreadLeaveCS();
return KErrBadDescriptor;
}
if(!memory->IsDemandPaged())
{
// memory not demand paged, so we have nothing to do...
MmuLock::Unlock();
mapping->Close();
NKern::ThreadLeaveCS();
return KErrNone;
}
// Open a reference on the memory so it doesn't get deleted.
memory->Open();
MmuLock::Unlock();
// pin memory...
TUint index = (offsetInMapping>>KPageShift)+mapping->iStartIndex;
TUint count = ((offsetInMapping&KPageMask)+aSize+KPageMask)>>KPageShift;
TInt r = ((DVirtualPinMapping*)iPinMapping)->Pin( memory,index,count,mapping->Permissions(),
mapping, mapInstanceCount);
if(r==KErrNotFound)
{
// some memory wasn't present, so treat this as an error...
memory->Close();
mapping->Close();
NKern::ThreadLeaveCS();
return KErrBadDescriptor;
}
// we can't fail to pin otherwise...
__NK_ASSERT_DEBUG(r!=KErrNoMemory); // separate OOM assert to aid debugging
__NK_ASSERT_ALWAYS(r==KErrNone);
// indicate that we have actually pinned...
__NK_ASSERT_DEBUG(iLockedPageCount==0);
iLockedPageCount = count;
// cleanup...
memory->Close();
mapping->Close();
NKern::ThreadLeaveCS();
return 1;
}
EXPORT_C void DDemandPagingLock::DoUnlock()
{
NKern::ThreadEnterCS();
((DVirtualPinMapping*)iPinMapping)->Unpin();
__NK_ASSERT_DEBUG(iLockedPageCount);
iLockedPageCount = 0;
NKern::ThreadLeaveCS();
}