diff -r 2d65c2f76d7b -r 947f0dc9f7a8 kernel/eka/memmodel/epoc/flexible/mmu/mpager.cpp --- a/kernel/eka/memmodel/epoc/flexible/mmu/mpager.cpp Tue Feb 02 01:24:03 2010 +0200 +++ b/kernel/eka/memmodel/epoc/flexible/mmu/mpager.cpp Fri Apr 16 16:24:37 2010 +0300 @@ -27,14 +27,14 @@ #include "mpagearray.h" #include "mswap.h" #include "mthrash.h" +#include "mpagecleaner.h" + #include "cache_maintenance.inl" const TUint16 KDefaultYoungOldRatio = 3; const TUint16 KDefaultMinPages = 256; -#ifdef _USE_OLDEST_LISTS const TUint16 KDefaultOldOldestRatio = 3; -#endif const TUint KMinOldPages = 1; @@ -43,21 +43,39 @@ */ const TUint KAbsoluteMaxPageCount = (1u<<(32-KPageShift))-1u; - +/* +Limit the maximum number of oldest pages to bound the time taken by SelectPagesToClean(), which is +called with the MmuLock held. +*/ +const TUint KMaxOldestPages = 32; + +static DMutex* ThePageCleaningLock = NULL; DPager ThePager; DPager::DPager() : iMinimumPageCount(0), iMaximumPageCount(0), iYoungOldRatio(0), - iYoungCount(0),iOldCount(0),iNumberOfFreePages(0) + iYoungCount(0), iOldCount(0), iOldestCleanCount(0), + iNumberOfFreePages(0), iReservePageCount(0), iMinimumPageLimit(0) +#ifdef __DEMAND_PAGING_BENCHMARKS__ + , iBenchmarkLock(TSpinLock::EOrderGenericIrqHigh3) +#endif { } -void DPager::Init2() +void DPager::InitCache() { - TRACEB(("DPager::Init2()")); + // + // This routine doesn't acquire any mutexes because it should be called before the system + // is fully up and running. I.e. called before another thread can preempt this. + // + TRACEB(("DPager::InitCache()")); + // If any pages have been reserved then they will have already been allocated and + // therefore should be counted as part of iMinimumPageCount. + __NK_ASSERT_DEBUG(iReservePageCount == iMinimumPageCount); + __NK_ASSERT_DEBUG(!CacheInitialised()); #if defined(__CPU_ARM) @@ -90,13 +108,8 @@ #ifdef __SMP__ // Adjust min page count so that all CPUs are guaranteed to make progress. - // NOTE: Can't use NKern::NumberOfCpus here because we haven't finished booting yet and will - // always have only one CPU running at this point... - - // TODO: Before we can enable this the base test configuration needs - // updating to have a sufficient minimum page size... - // - // iMinYoungPages *= KMaxCpus; + TInt numberOfCpus = NKern::NumberOfCpus(); + iMinYoungPages *= numberOfCpus; #endif // A minimum young/old ratio of 1 means that we need at least twice iMinYoungPages pages... @@ -104,85 +117,83 @@ __NK_ASSERT_DEBUG(KMinOldPages<=iAbsoluteMinPageCount/2); - // initialise live list... - TUint minimumPageCount = 0; - TUint maximumPageCount = 0; - + // Read any paging config data. SDemandPagingConfig config = TheRomHeader().iDemandPagingConfig; - iMinimumPageCount = KDefaultMinPages; - if(minimumPageCount) - iMinimumPageCount = minimumPageCount; + // Set the list ratios... + iYoungOldRatio = KDefaultYoungOldRatio; + if(config.iYoungOldRatio) + iYoungOldRatio = config.iYoungOldRatio; + iOldOldestRatio = KDefaultOldOldestRatio; + if(config.iSpare[2]) + iOldOldestRatio = config.iSpare[2]; + + // Set the minimum page counts... + iMinimumPageLimit = iMinYoungPages * (1 + iYoungOldRatio) / iYoungOldRatio + + DPageReadRequest::ReservedPagesRequired(); + + if(iMinimumPageLimit < iAbsoluteMinPageCount) + iMinimumPageLimit = iAbsoluteMinPageCount; + + if (K::MemModelAttributes & (EMemModelAttrRomPaging | EMemModelAttrCodePaging | EMemModelAttrDataPaging)) + iMinimumPageCount = KDefaultMinPages; + else + {// No paging is enabled so set the minimum cache size to the minimum + // allowable with the current young old ratio. + iMinimumPageCount = iMinYoungPages * (iYoungOldRatio + 1); + } + if(config.iMinPages) iMinimumPageCount = config.iMinPages; - if(iMinimumPageCount iMinimumPageCount) + iMinimumPageCount = iMinimumPageLimit + iReservePageCount; + iInitMinimumPageCount = iMinimumPageCount; + // Set the maximum page counts... iMaximumPageCount = KMaxTInt; - if(maximumPageCount) - iMaximumPageCount = maximumPageCount; if(config.iMaxPages) iMaximumPageCount = config.iMaxPages; if (iMaximumPageCount > KAbsoluteMaxPageCount) iMaximumPageCount = KAbsoluteMaxPageCount; iInitMaximumPageCount = iMaximumPageCount; - iYoungOldRatio = KDefaultYoungOldRatio; - if(config.iYoungOldRatio) - iYoungOldRatio = config.iYoungOldRatio; + TRACEB(("DPager::InitCache() live list min=%d max=%d ratio=%d",iMinimumPageCount,iMaximumPageCount,iYoungOldRatio)); + + // Verify the page counts are valid. + __NK_ASSERT_ALWAYS(iMaximumPageCount >= iMinimumPageCount); + TUint minOldAndOldest = iMinimumPageCount / (1 + iYoungOldRatio); + __NK_ASSERT_ALWAYS(minOldAndOldest >= KMinOldPages); + __NK_ASSERT_ALWAYS(iMinimumPageCount >= minOldAndOldest); + + // Need at least iMinYoungPages pages mapped to execute worst case CPU instruction + TUint minYoung = iMinimumPageCount - minOldAndOldest; + __NK_ASSERT_ALWAYS(minYoung >= iMinYoungPages); + + // Verify that the young old ratio can be met even when there is only the + // minimum number of old pages. TInt ratioLimit = (iMinimumPageCount-KMinOldPages)/KMinOldPages; - if(iYoungOldRatio>ratioLimit) - iYoungOldRatio = ratioLimit; - -#ifdef _USE_OLDEST_LISTS - iOldOldestRatio = KDefaultOldOldestRatio; - if(config.iSpare[2]) - iOldOldestRatio = config.iSpare[2]; -#endif - - iMinimumPageLimit = (iMinYoungPages * (1 + iYoungOldRatio)) / iYoungOldRatio; - if(iMinimumPageLimitPagedState()==SPageInfo::EUnpaged); // add as oldest page... -#ifdef _USE_OLDEST_LISTS aPageInfo->SetPagedState(SPageInfo::EPagedOldestClean); iOldestCleanList.Add(&aPageInfo->iLink); ++iOldestCleanCount; -#else - aPageInfo->SetPagedState(SPageInfo::EPagedOld); - iOldList.Add(&aPageInfo->iLink); - ++iOldCount; -#endif Event(EEventPageInFree,aPageInfo); } @@ -326,7 +328,6 @@ --iOldCount; break; -#ifdef _USE_OLDEST_LISTS case SPageInfo::EPagedOldestClean: __NK_ASSERT_DEBUG(iOldestCleanCount); aPageInfo->iLink.Deque(); @@ -338,7 +339,6 @@ aPageInfo->iLink.Deque(); --iOldestDirtyCount; break; -#endif case SPageInfo::EPagedPinned: // this can occur if a pinned mapping is being unmapped when memory is decommitted. @@ -361,18 +361,15 @@ // Update the dirty page count as required... if (aPageInfo->IsDirty()) + { + aPageInfo->SetReadOnly(); SetClean(*aPageInfo); + } // add as oldest page... -#ifdef _USE_OLDEST_LISTS aPageInfo->SetPagedState(SPageInfo::EPagedOldestClean); iOldestCleanList.Add(&aPageInfo->iLink); ++iOldestCleanCount; -#else - aPageInfo->SetPagedState(SPageInfo::EPagedOld); - iOldList.Add(&aPageInfo->iLink); - ++iOldCount; -#endif return KErrNone; } @@ -399,7 +396,6 @@ --iOldCount; break; -#ifdef _USE_OLDEST_LISTS case SPageInfo::EPagedOldestClean: __NK_ASSERT_DEBUG(iOldestCleanCount); aPageInfo->iLink.Deque(); @@ -411,7 +407,6 @@ aPageInfo->iLink.Deque(); --iOldestDirtyCount; break; -#endif case SPageInfo::EPagedPinned: __NK_ASSERT_DEBUG(0); @@ -482,52 +477,253 @@ } -SPageInfo* DPager::StealOldestPage() +TInt DPager::TryStealOldestPage(SPageInfo*& aPageInfoOut) { __NK_ASSERT_DEBUG(RamAllocLock::IsHeld()); __NK_ASSERT_DEBUG(MmuLock::IsHeld()); + // find oldest page in list... + SDblQueLink* link; + if (iOldestCleanCount) + { + __NK_ASSERT_DEBUG(!iOldestCleanList.IsEmpty()); + link = iOldestCleanList.Last(); + } + else if (iOldestDirtyCount) + { + __NK_ASSERT_DEBUG(!iOldestDirtyList.IsEmpty()); + link = iOldestDirtyList.Last(); + } + else if (iOldCount) + { + __NK_ASSERT_DEBUG(!iOldList.IsEmpty()); + link = iOldList.Last(); + } + else + { + __NK_ASSERT_DEBUG(iYoungCount); + __NK_ASSERT_ALWAYS(!iYoungList.IsEmpty()); + link = iYoungList.Last(); + } + SPageInfo* pageInfo = SPageInfo::FromLink(link); + + if (pageInfo->IsDirty() && !PageCleaningLock::IsHeld()) + return 1; + + // try to steal it from owning object... + TInt r = StealPage(pageInfo); + if (r == KErrNone) + { + BalanceAges(); + aPageInfoOut = pageInfo; + } + + return r; + } + + +SPageInfo* DPager::StealOldestPage() + { + __NK_ASSERT_DEBUG(MmuLock::IsHeld()); + TBool pageCleaningLockHeld = EFalse; for(;;) { - // find oldest page in list... - SDblQueLink* link; -#ifdef _USE_OLDEST_LISTS - if (iOldestCleanCount) + SPageInfo* pageInfo = NULL; + TInt r = TryStealOldestPage(pageInfo); + + if (r == KErrNone) { - __NK_ASSERT_DEBUG(!iOldestCleanList.IsEmpty()); - link = iOldestCleanList.Last(); + if (pageCleaningLockHeld) + { + MmuLock::Unlock(); + PageCleaningLock::Unlock(); + MmuLock::Lock(); + } + return pageInfo; + } + else if (r == 1) + { + __NK_ASSERT_ALWAYS(!pageCleaningLockHeld); + MmuLock::Unlock(); + PageCleaningLock::Lock(); + MmuLock::Lock(); + pageCleaningLockHeld = ETrue; } - else if (iOldestDirtyCount) + // else retry... + } + } + +#ifdef __CPU_CACHE_HAS_COLOUR + +template class TSequentialColourSelector + { +public: + static const TInt KMaxLength = maxObjects; + static const TInt KArrayLength = _ALIGN_UP(KMaxLength, KPageColourCount); + + FORCE_INLINE TSequentialColourSelector() + { + memclr(this, sizeof(*this)); + } + + FORCE_INLINE TBool FoundLongestSequence() + { + return iLongestLength >= KMaxLength; + } + + FORCE_INLINE void AddCandidate(T* aObject, TInt aColour) + { + // allocate objects to slots based on colour + for (TInt i = aColour ; i < KArrayLength ; i += KPageColourCount) { - __NK_ASSERT_DEBUG(!iOldestDirtyList.IsEmpty()); - link = iOldestDirtyList.Last(); + if (!iSlot[i]) + { + iSlot[i] = aObject; + iSeqLength[i] = i == 0 ? 1 : iSeqLength[i - 1] + 1; + TInt j = i + 1; + while(j < KArrayLength && iSeqLength[j]) + iSeqLength[j++] += iSeqLength[i]; + TInt currentLength = iSeqLength[j - 1]; + if (currentLength > iLongestLength) + { + iLongestLength = currentLength; + iLongestStart = j - currentLength; + } + break; + } } - else if (iOldCount) -#else - if (iOldCount) -#endif + } + + FORCE_INLINE TInt FindLongestRun(T** aObjectsOut) + { + if (iLongestLength == 0) + return 0; + + if (iLongestLength < KMaxLength && iSlot[0] && iSlot[KArrayLength - 1]) { - __NK_ASSERT_DEBUG(!iOldList.IsEmpty()); - link = iOldList.Last(); - } - else + // check possibility of wrapping + + TInt i = 1; + while (iSlot[i]) ++i; // find first hole + TInt wrappedLength = iSeqLength[KArrayLength - 1] + iSeqLength[i - 1]; + if (wrappedLength > iLongestLength) + { + iLongestLength = wrappedLength; + iLongestStart = KArrayLength - iSeqLength[KArrayLength - 1]; + } + } + + iLongestLength = Min(iLongestLength, KMaxLength); + + __NK_ASSERT_DEBUG(iLongestStart >= 0 && iLongestStart < KArrayLength); + __NK_ASSERT_DEBUG(iLongestStart + iLongestLength < 2 * KArrayLength); + + TInt len = Min(iLongestLength, KArrayLength - iLongestStart); + wordmove(aObjectsOut, &iSlot[iLongestStart], len * sizeof(T*)); + wordmove(aObjectsOut + len, &iSlot[0], (iLongestLength - len) * sizeof(T*)); + + return iLongestLength; + } + +private: + T* iSlot[KArrayLength]; + TInt8 iSeqLength[KArrayLength]; + TInt iLongestStart; + TInt iLongestLength; + }; + +TInt DPager::SelectPagesToClean(SPageInfo** aPageInfosOut) + { + // select up to KMaxPagesToClean oldest dirty pages with sequential page colours + + __NK_ASSERT_DEBUG(MmuLock::IsHeld()); + + TSequentialColourSelector selector; + + SDblQueLink* link = iOldestDirtyList.Last(); + while (link != &iOldestDirtyList.iA) + { + SPageInfo* pi = SPageInfo::FromLink(link); + if (!pi->IsWritable()) { - __NK_ASSERT_DEBUG(iYoungCount); - __NK_ASSERT_ALWAYS(!iYoungList.IsEmpty()); - link = iYoungList.Last(); + // the page may be in the process of being restricted, stolen or decommitted, but don't + // check for this as it will occur infrequently and will be detected by CheckModified + // anyway + TInt colour = pi->Index() & KPageColourMask; + selector.AddCandidate(pi, colour); + if (selector.FoundLongestSequence()) + break; } - SPageInfo* pageInfo = SPageInfo::FromLink(link); - - // steal it from owning object... - TInt r = StealPage(pageInfo); - - BalanceAges(); - - if(r==KErrNone) - return pageInfo; // done - - // loop back and try again + link = link->iPrev; + } + + return selector.FindLongestRun(aPageInfosOut); + } + +#else + +TInt DPager::SelectPagesToClean(SPageInfo** aPageInfosOut) + { + // no page colouring restrictions, so just take up to KMaxPagesToClean oldest dirty pages + __NK_ASSERT_DEBUG(MmuLock::IsHeld()); + TInt pageCount = 0; + SDblQueLink* link = iOldestDirtyList.Last(); + while (link != &iOldestDirtyList.iA && pageCount < KMaxPagesToClean) + { + SPageInfo* pi = SPageInfo::FromLink(link); + if (!pi->IsWritable()) + { + // the page may be in the process of being restricted, stolen or decommitted, but don't + // check for this as it will occur infrequently and will be detected by CheckModified + // anyway + aPageInfosOut[pageCount++] = pi; + } + link = link->iPrev; } + return pageCount; + } + +#endif + + +TInt DPager::CleanSomePages(TBool aBackground) + { + __NK_ASSERT_DEBUG(MmuLock::IsHeld()); + __NK_ASSERT_DEBUG(PageCleaningLock::IsHeld()); + // ram alloc lock may or may not be held + + SPageInfo* pageInfos[KMaxPagesToClean]; + TInt pageCount = SelectPagesToClean(&pageInfos[0]); + + if (pageCount == 0) + return 0; + + TheDataPagedMemoryManager->CleanPages(pageCount, pageInfos, aBackground); + + for (TInt i = 0 ; i < pageCount ; ++i) + { + SPageInfo* pi = pageInfos[i]; + if (pi) + { + __NK_ASSERT_DEBUG(pi->PagedState() == SPageInfo::EPagedOldestDirty && iOldestDirtyCount); + __NK_ASSERT_DEBUG(!pi->IsDirty() && !pi->IsWritable()); + + pi->iLink.Deque(); + iOldestCleanList.AddHead(&pi->iLink); + --iOldestDirtyCount; + ++iOldestCleanCount; + pi->SetPagedState(SPageInfo::EPagedOldestClean); + } + } + + return pageCount; + } + + +TBool DPager::HasPagesToClean() + { + __NK_ASSERT_DEBUG(MmuLock::IsHeld()); + return iOldestDirtyCount > 0; } @@ -608,97 +804,144 @@ } +static TBool DiscardCanStealPage(SPageInfo* aOldPageInfo, TBool aBlockRest) + { + // If the page is pinned or if the page is dirty and a general defrag is being performed then + // don't attempt to steal it + return aOldPageInfo->Type() == SPageInfo::EUnused || + (aOldPageInfo->PagedState() != SPageInfo::EPagedPinned && (!aBlockRest || !aOldPageInfo->IsDirty())); + } + + TInt DPager::DiscardPage(SPageInfo* aOldPageInfo, TUint aBlockZoneId, TBool aBlockRest) { + // todo: assert MmuLock not released + + TRACE(("> DPager::DiscardPage %08x", aOldPageInfo)); + __NK_ASSERT_DEBUG(RamAllocLock::IsHeld()); __NK_ASSERT_DEBUG(MmuLock::IsHeld()); - TInt r; - // If the page is pinned or if the page is dirty and a general defrag is being - // performed then don't attempt to steal it. - if (aOldPageInfo->Type() != SPageInfo::EUnused && - (aOldPageInfo->PagedState() == SPageInfo::EPagedPinned || - (aBlockRest && aOldPageInfo->IsDirty()))) - {// The page is pinned or is dirty and this is a general defrag so move the page. + if (!DiscardCanStealPage(aOldPageInfo, aBlockRest)) + { + // The page is pinned or is dirty and this is a general defrag so move the page. DMemoryObject* memory = aOldPageInfo->Owner(); // Page must be managed if it is pinned or dirty. __NK_ASSERT_DEBUG(aOldPageInfo->Type()==SPageInfo::EManaged); __NK_ASSERT_DEBUG(memory); MmuLock::Unlock(); TPhysAddr newAddr; - return memory->iManager->MovePage(memory, aOldPageInfo, newAddr, aBlockZoneId, aBlockRest); + TRACE2(("DPager::DiscardPage delegating pinned/dirty page to manager")); + TInt r = memory->iManager->MovePage(memory, aOldPageInfo, newAddr, aBlockZoneId, aBlockRest); + TRACE(("< DPager::DiscardPage %d", r)); + return r; } - if (!iNumberOfFreePages) + TInt r = KErrNone; + SPageInfo* newPageInfo = NULL; + TBool havePageCleaningLock = EFalse; + + TBool needNewPage; + TBool needPageCleaningLock; + while(needNewPage = (iNumberOfFreePages == 0 && newPageInfo == NULL), + needPageCleaningLock = (aOldPageInfo->IsDirty() && !havePageCleaningLock), + needNewPage || needPageCleaningLock) { - // Allocate a new page for the live list as it has reached its minimum size. MmuLock::Unlock(); - SPageInfo* newPageInfo = GetPageFromSystem((Mmu::TRamAllocFlags)(EMemAttNormalCached|Mmu::EAllocNoWipe), - aBlockZoneId, aBlockRest); - if (!newPageInfo) - return KErrNoMemory; + + if (needNewPage) + { + // Allocate a new page for the live list as it has reached its minimum size. + TUint flags = EMemAttNormalCached | Mmu::EAllocNoWipe; + newPageInfo = GetPageFromSystem((Mmu::TRamAllocFlags)flags, aBlockZoneId, aBlockRest); + if (!newPageInfo) + { + TRACE(("< DPager::DiscardPage KErrNoMemory")); + r = KErrNoMemory; + MmuLock::Lock(); + break; + } + } + + if (needPageCleaningLock) + { + // Acquire the page cleaning mutex so StealPage can clean it + PageCleaningLock::Lock(); + havePageCleaningLock = ETrue; + } // Re-acquire the mmulock and re-check that the page is not pinned or dirty. MmuLock::Lock(); - if (aOldPageInfo->Type() != SPageInfo::EUnused && - (aOldPageInfo->PagedState() == SPageInfo::EPagedPinned || - (aBlockRest && aOldPageInfo->IsDirty()))) - {// Page is now pinned or dirty so give up as it is inuse. - ReturnPageToSystem(*newPageInfo); - MmuLock::Unlock(); - return KErrInUse; - } - - // Attempt to steal the page - r = StealPage(aOldPageInfo); - __NK_ASSERT_DEBUG(MmuLock::IsHeld()); - - if (r == KErrCompletion) - {// This was a page table that has been freed but added to the - // live list as a free page. Remove from live list and continue. - __NK_ASSERT_DEBUG(!aOldPageInfo->IsDirty()); - RemovePage(aOldPageInfo); - r = KErrNone; - } - - if (r == KErrNone) - {// Add the new page to the live list as discarding the old page - // will reduce the live list below the minimum. - AddAsFreePage(newPageInfo); - // We've successfully discarded the page so return it to the free pool. - ReturnPageToSystem(*aOldPageInfo); - BalanceAges(); - } - else + if (!DiscardCanStealPage(aOldPageInfo, aBlockRest)) { - // New page not required so just return it to the system. This is safe as - // iNumberOfFreePages will have this page counted but as it is not on the live list - // noone else can touch it. - ReturnPageToSystem(*newPageInfo); + // Page is now pinned or dirty so give up as it is in use. + r = KErrInUse; + break; } } - else + + if (r == KErrNone) { // Attempt to steal the page - r = StealPage(aOldPageInfo); - - __NK_ASSERT_DEBUG(MmuLock::IsHeld()); - - if (r == KErrCompletion) - {// This was a page table that has been freed but added to the - // live list as a free page. Remove from live list. - __NK_ASSERT_DEBUG(!aOldPageInfo->IsDirty()); - RemovePage(aOldPageInfo); - r = KErrNone; + r = StealPage(aOldPageInfo); // temporarily releases MmuLock if page is dirty + } + __NK_ASSERT_DEBUG(MmuLock::IsHeld()); + + if (r == KErrCompletion) + {// This was a page table that has been freed but added to the + // live list as a free page. Remove from live list and continue. + __NK_ASSERT_DEBUG(!aOldPageInfo->IsDirty()); + RemovePage(aOldPageInfo); + r = KErrNone; + } + + if (r == KErrNone && iNumberOfFreePages == 0) + { + if (newPageInfo) + { + // Add a new page to the live list if we have one as discarding the old page will reduce + // the live list below the minimum. + AddAsFreePage(newPageInfo); + newPageInfo = NULL; } - - if (r == KErrNone) - {// We've successfully discarded the page so return it to the free pool. - ReturnPageToSystem(*aOldPageInfo); - BalanceAges(); + else + { + // Otherwise the live list shrank when page was being cleaned so have to give up + AddAsFreePage(aOldPageInfo); + BalanceAges(); // temporarily releases MmuLock + r = KErrInUse; } } + + if (r == KErrNone) + { + // We've successfully discarded the page and ensured the live list is large enough, so + // return it to the free pool. + ReturnPageToSystem(*aOldPageInfo); // temporarily releases MmuLock + BalanceAges(); // temporarily releases MmuLock + } + + if (newPageInfo) + { + // New page not required so just return it to the system. This is safe as + // iNumberOfFreePages will have this page counted but as it is not on the live list noone + // else can touch it. + if (iNumberOfFreePages == 0) + AddAsFreePage(newPageInfo); + else + ReturnPageToSystem(*newPageInfo); // temporarily releases MmuLock + } + + if (havePageCleaningLock) + { + // Release the page cleaning mutex + MmuLock::Unlock(); + PageCleaningLock::Unlock(); + MmuLock::Lock(); + } + MmuLock::Unlock(); + TRACE(("< DPager::DiscardPage returns %d", r)); return r; } @@ -754,6 +997,9 @@ __NK_ASSERT_DEBUG(RamAllocLock::IsHeld()); __NK_ASSERT_DEBUG(MmuLock::IsHeld()); + // should be unpaged at this point, otherwise Mmu::FreeRam will just give it back to us + __NK_ASSERT_DEBUG(aPageInfo.PagedState() == SPageInfo::EUnpaged); + __NK_ASSERT_DEBUG(iNumberOfFreePages>0); --iNumberOfFreePages; @@ -768,28 +1014,22 @@ SPageInfo* DPager::PageInAllocPage(Mmu::TRamAllocFlags aAllocFlags) { + TBool pageCleaningLockHeld = EFalse; SPageInfo* pageInfo; TPhysAddr pagePhys; - + TInt r = KErrGeneral; + RamAllocLock::Lock(); MmuLock::Lock(); +find_a_page: // try getting a free page from our live list... -#ifdef _USE_OLDEST_LISTS if (iOldestCleanCount) { pageInfo = SPageInfo::FromLink(iOldestCleanList.Last()); if(pageInfo->Type()==SPageInfo::EUnused) - goto get_oldest; + goto try_steal_oldest_page; } -#else - if(iOldCount) - { - pageInfo = SPageInfo::FromLink(iOldList.Last()); - if(pageInfo->Type()==SPageInfo::EUnused) - goto get_oldest; - } -#endif // try getting a free page from the system pool... if(!HaveMaximumPages()) @@ -801,14 +1041,62 @@ MmuLock::Lock(); } + // try stealing a clean page... + if (iOldestCleanCount) + goto try_steal_oldest_page; + + // see if we can clean multiple dirty pages in one go... + if (KMaxPagesToClean > 1 && iOldestDirtyCount > 1) + { + // if we don't hold the page cleaning mutex then temporarily release ram alloc mutex and + // acquire page cleaning mutex; if we hold it already just proceed + if (!pageCleaningLockHeld) + { + MmuLock::Unlock(); + RamAllocLock::Unlock(); + PageCleaningLock::Lock(); + MmuLock::Lock(); + } + + // there may be clean pages now if we've waited on the page cleaning mutex, if so don't + // bother cleaning but just restart + if (iOldestCleanCount == 0) + CleanSomePages(EFalse); + + if (!pageCleaningLockHeld) + { + MmuLock::Unlock(); + PageCleaningLock::Unlock(); + RamAllocLock::Lock(); + MmuLock::Lock(); + } + + if (iOldestCleanCount > 0) + goto find_a_page; + } + // as a last resort, steal a page from the live list... -get_oldest: -#ifdef _USE_OLDEST_LISTS + +try_steal_oldest_page: __NK_ASSERT_ALWAYS(iOldestCleanCount|iOldestDirtyCount|iOldCount|iYoungCount); -#else - __NK_ASSERT_ALWAYS(iOldCount|iYoungCount); -#endif - pageInfo = StealOldestPage(); + r = TryStealOldestPage(pageInfo); + // if this fails we restart whole process + if (r < KErrNone) + goto find_a_page; + + // if we need to clean, acquire page cleaning mutex for life of this function + if (r == 1) + { + __NK_ASSERT_ALWAYS(!pageCleaningLockHeld); + MmuLock::Unlock(); + PageCleaningLock::Lock(); + MmuLock::Lock(); + pageCleaningLockHeld = ETrue; + goto find_a_page; + } + + // otherwise we're done! + __NK_ASSERT_DEBUG(r == KErrNone); MmuLock::Unlock(); // make page state same as a freshly allocated page... @@ -816,7 +1104,10 @@ TheMmu.PagesAllocated(&pagePhys,1,aAllocFlags); done: + if (pageCleaningLockHeld) + PageCleaningLock::Unlock(); RamAllocLock::Unlock(); + return pageInfo; } @@ -873,10 +1164,8 @@ case SPageInfo::EPagedYoung: case SPageInfo::EPagedOld: -#ifdef _USE_OLDEST_LISTS case SPageInfo::EPagedOldestDirty: case SPageInfo::EPagedOldestClean: -#endif continue; // discard already been allowed case SPageInfo::EPagedPinned: @@ -935,10 +1224,8 @@ case SPageInfo::EPagedYoung: case SPageInfo::EPagedOld: -#ifdef _USE_OLDEST_LISTS case SPageInfo::EPagedOldestClean: case SPageInfo::EPagedOldestDirty: -#endif changeType = ETrue; break; // remove from live list @@ -1004,6 +1291,7 @@ TheCodePagedMemoryManager->Init3(); TInt r = Kern::AddHalEntry(EHalGroupVM, VMHalFunction, 0); __NK_ASSERT_ALWAYS(r==KErrNone); + PageCleaningLock::Init(); } @@ -1018,12 +1306,8 @@ __NK_ASSERT_DEBUG(MmuLock::IsHeld()); TBool restrictPage = EFalse; SPageInfo* pageInfo = NULL; -#ifdef _USE_OLDEST_LISTS TUint oldestCount = iOldestCleanCount + iOldestDirtyCount; if((iOldCount + oldestCount) * iYoungOldRatio < iYoungCount) -#else - if (iOldCount * iYoungOldRatio < iYoungCount) -#endif { // Need more old pages so make one young page into an old page... __NK_ASSERT_DEBUG(!iYoungList.IsEmpty()); @@ -1042,9 +1326,9 @@ restrictPage = ETrue; } -#ifdef _USE_OLDEST_LISTS // Check we have enough oldest pages. - if (oldestCount * iOldOldestRatio < iOldCount) + if (oldestCount < KMaxOldestPages && + oldestCount * iOldOldestRatio < iOldCount) { __NK_ASSERT_DEBUG(!iOldList.IsEmpty()); __NK_ASSERT_DEBUG(iOldCount); @@ -1057,6 +1341,7 @@ oldestPageInfo->SetPagedState(SPageInfo::EPagedOldestDirty); iOldestDirtyList.AddHead(link); ++iOldestDirtyCount; + PageCleaner::NotifyPagesToClean(); Event(EEventPageAgedDirty,oldestPageInfo); } else @@ -1067,7 +1352,7 @@ Event(EEventPageAgedClean,oldestPageInfo); } } -#endif + if (restrictPage) { // Make the recently aged old page inaccessible. This is done last as it @@ -1102,10 +1387,8 @@ { case SPageInfo::EPagedYoung: case SPageInfo::EPagedOld: -#ifdef _USE_OLDEST_LISTS case SPageInfo::EPagedOldestClean: case SPageInfo::EPagedOldestDirty: -#endif RemovePage(pi); AddAsYoungestPage(pi); BalanceAges(); @@ -1125,6 +1408,7 @@ } } + TInt DPager::PteAndInfoFromLinAddr( TInt aOsAsid, TLinAddr aAddress, DMemoryMappingBase* aMapping, TUint aMapInstanceCount, TPte*& aPte, SPageInfo*& aPageInfo) { @@ -1150,11 +1434,13 @@ return KErrNone; } + TInt DPager::TryRejuvenate( TInt aOsAsid, TLinAddr aAddress, TUint aAccessPermissions, TLinAddr aPc, DMemoryMappingBase* aMapping, TUint aMapInstanceCount, DThread* aThread, TAny* aExceptionInfo) { __NK_ASSERT_DEBUG(MmuLock::IsHeld()); + START_PAGING_BENCHMARK; SPageInfo* pi; TPte* pPte; @@ -1250,12 +1536,8 @@ Event(EEventPageRejuvenate,pi,aPc,aAddress,aAccessPermissions); TBool balance = false; -#ifdef _USE_OLDEST_LISTS if( state==SPageInfo::EPagedYoung || state==SPageInfo::EPagedOld || state==SPageInfo::EPagedOldestClean || state==SPageInfo::EPagedOldestDirty) -#else - if(state==SPageInfo::EPagedYoung || state==SPageInfo::EPagedOld) -#endif { RemovePage(pi); AddAsYoungestPage(pi); @@ -1276,6 +1558,7 @@ if(balance) BalanceAges(); + END_PAGING_BENCHMARK(EPagingBmRejuvenate); return KErrNone; } @@ -1307,10 +1590,8 @@ { case SPageInfo::EPagedYoung: case SPageInfo::EPagedOld: -#ifdef _USE_OLDEST_LISTS case SPageInfo::EPagedOldestClean: case SPageInfo::EPagedOldestDirty: -#endif RemovePage(pi); // fall through... case SPageInfo::EUnpaged: @@ -1344,10 +1625,8 @@ { case SPageInfo::EPagedYoung: case SPageInfo::EPagedOld: -#ifdef _USE_OLDEST_LISTS case SPageInfo::EPagedOldestClean: case SPageInfo::EPagedOldestDirty: -#endif RemovePage(aPageInfo); AddAsYoungestPage(aPageInfo); BalanceAges(); @@ -1404,7 +1683,6 @@ __NK_ASSERT_DEBUG(aPageInfo->PinCount()==1); break; -#ifdef _USE_OLDEST_LISTS case SPageInfo::EPagedOldestClean: __NK_ASSERT_DEBUG(iOldestCleanCount); aPageInfo->iLink.Deque(); @@ -1418,7 +1696,6 @@ --iOldestDirtyCount; __NK_ASSERT_DEBUG(aPageInfo->PinCount()==1); break; -#endif case SPageInfo::EPagedPinned: // nothing more to do... @@ -1688,6 +1965,8 @@ TInt DPager::ResizeLiveList(TUint aMinimumPageCount, TUint aMaximumPageCount) { TRACE(("DPager::ResizeLiveList(%d,%d) current young=%d old=%d min=%d free=%d max=%d",aMinimumPageCount,aMaximumPageCount,iYoungCount,iOldCount,iMinimumPageCount,iNumberOfFreePages,iMaximumPageCount)); + __NK_ASSERT_DEBUG(CacheInitialised()); + if(!aMaximumPageCount) { aMinimumPageCount = iInitMinimumPageCount; @@ -1705,6 +1984,8 @@ MmuLock::Lock(); + __NK_ASSERT_ALWAYS(iYoungOldRatio!=0); + // Make sure aMinimumPageCount is not less than absolute minimum we can cope with... iMinimumPageLimit = iMinYoungPages * (1 + iYoungOldRatio) / iYoungOldRatio + DPageReadRequest::ReservedPagesRequired(); @@ -1789,6 +2070,7 @@ { NKern::ThreadEnterCS(); RamAllocLock::Lock(); + PageCleaningLock::Lock(); TRACE(("DPager::FlushAll() live list young=%d old=%d min=%d free=%d max=%d",iYoungCount,iOldCount,iMinimumPageCount,iNumberOfFreePages,iMaximumPageCount)); @@ -1811,12 +2093,8 @@ do { SPageInfo::TPagedState state = pi->PagedState(); -#ifdef _USE_OLDEST_LISTS if (state==SPageInfo::EPagedYoung || state==SPageInfo::EPagedOld || state==SPageInfo::EPagedOldestClean || state==SPageInfo::EPagedOldestDirty) -#else - if(state==SPageInfo::EPagedYoung || state==SPageInfo::EPagedOld) -#endif { if (pi->Type() != SPageInfo::EUnused) { @@ -1842,6 +2120,7 @@ TRACE(("DPager::FlushAll() end with young=%d old=%d min=%d free=%d max=%d",iYoungCount,iOldCount,iMinimumPageCount,iNumberOfFreePages,iMaximumPageCount)); + PageCleaningLock::Unlock(); RamAllocLock::Unlock(); NKern::ThreadLeaveCS(); } @@ -2016,9 +2295,8 @@ TUint index = (TInt) a1; if (index >= EMaxPagingBm) return KErrNotFound; - NKern::LockSystem(); - SPagingBenchmarkInfo info = ThePager.iBenchmarkInfo[index]; - NKern::UnlockSystem(); + SPagingBenchmarkInfo info; + ThePager.ReadBenchmarkData((TPagingBenchmark)index, info); kumemput32(a2,&info,sizeof(info)); } return KErrNone; @@ -2028,9 +2306,7 @@ TUint index = (TInt) a1; if (index >= EMaxPagingBm) return KErrNotFound; - NKern::LockSystem(); ThePager.ResetBenchmarkData((TPagingBenchmark)index); - NKern::UnlockSystem(); } return KErrNone; #endif @@ -2046,28 +2322,39 @@ void DPager::ResetBenchmarkData(TPagingBenchmark aBm) { SPagingBenchmarkInfo& info = iBenchmarkInfo[aBm]; + __SPIN_LOCK_IRQ(iBenchmarkLock); info.iCount = 0; info.iTotalTime = 0; info.iMaxTime = 0; info.iMinTime = KMaxTInt; + __SPIN_UNLOCK_IRQ(iBenchmarkLock); } -void DPager::RecordBenchmarkData(TPagingBenchmark aBm, TUint32 aStartTime, TUint32 aEndTime) +void DPager::RecordBenchmarkData(TPagingBenchmark aBm, TUint32 aStartTime, TUint32 aEndTime, TUint aCount) { SPagingBenchmarkInfo& info = iBenchmarkInfo[aBm]; - ++info.iCount; #if !defined(HIGH_RES_TIMER) || defined(HIGH_RES_TIMER_COUNTS_UP) TInt64 elapsed = aEndTime - aStartTime; #else TInt64 elapsed = aStartTime - aEndTime; #endif + __SPIN_LOCK_IRQ(iBenchmarkLock); + info.iCount += aCount; info.iTotalTime += elapsed; if (elapsed > info.iMaxTime) info.iMaxTime = elapsed; if (elapsed < info.iMinTime) info.iMinTime = elapsed; + __SPIN_UNLOCK_IRQ(iBenchmarkLock); } +void DPager::ReadBenchmarkData(TPagingBenchmark aBm, SPagingBenchmarkInfo& aDataOut) + { + __SPIN_LOCK_IRQ(iBenchmarkLock); + aDataOut = iBenchmarkInfo[aBm]; + __SPIN_UNLOCK_IRQ(iBenchmarkLock); + } + #endif //__DEMAND_PAGING_BENCHMARKS__ @@ -2079,62 +2366,86 @@ // DPagingRequest // -DPagingRequest::DPagingRequest(DPagingRequestPool::TGroup& aPoolGroup) - : iPoolGroup(aPoolGroup), iUseRegionMemory(0), iUseRegionIndex(0), iUseRegionCount(0) +DPagingRequest::DPagingRequest() + : iMutex(NULL), iUseRegionCount(0) { } -FORCE_INLINE void DPagingRequest::SetUse(DMemoryObject* aMemory, TUint aIndex, TUint aCount) +void DPagingRequest::SetUseContiguous(DMemoryObject* aMemory, TUint aIndex, TUint aCount) { __ASSERT_SYSTEM_LOCK; - iUseRegionMemory = aMemory; - iUseRegionIndex = aIndex; + __NK_ASSERT_DEBUG(iUseRegionCount == 0); + __NK_ASSERT_DEBUG(aCount > 0 && aCount <= EMaxPages); + for (TUint i = 0 ; i < aCount ; ++i) + { + iUseRegionMemory[i] = aMemory; + iUseRegionIndex[i] = aIndex + i; + } + iUseRegionCount = aCount; + } + + +void DPagingRequest::SetUseDiscontiguous(DMemoryObject** aMemory, TUint* aIndex, TUint aCount) + { + __ASSERT_SYSTEM_LOCK; + __NK_ASSERT_DEBUG(iUseRegionCount == 0); + __NK_ASSERT_DEBUG(aCount > 0 && aCount <= EMaxPages); + for (TUint i = 0 ; i < aCount ; ++i) + { + iUseRegionMemory[i] = aMemory[i]; + iUseRegionIndex[i] = aIndex[i]; + } iUseRegionCount = aCount; } -TBool DPagingRequest::CheckUse(DMemoryObject* aMemory, TUint aIndex, TUint aCount) - { - return aMemory==iUseRegionMemory - && TUint(aIndex-iUseRegionIndex) < iUseRegionCount - && TUint(iUseRegionCount-TUint(aIndex-iUseRegionIndex)) <= aCount; - } - - -void DPagingRequest::Release() - { - NKern::LockSystem(); - SetUse(0,0,0); - Signal(); - } - - -void DPagingRequest::Wait() +void DPagingRequest::ResetUse() { __ASSERT_SYSTEM_LOCK; - ++iUsageCount; - TInt r = iMutex->Wait(); - __NK_ASSERT_ALWAYS(r == KErrNone); + __NK_ASSERT_DEBUG(iUseRegionCount > 0); + iUseRegionCount = 0; } -void DPagingRequest::Signal() +TBool DPagingRequest::CheckUseContiguous(DMemoryObject* aMemory, TUint aIndex, TUint aCount) { - __ASSERT_SYSTEM_LOCK; - iPoolGroup.Signal(this); + if (iUseRegionCount != aCount) + return EFalse; + for (TUint i = 0 ; i < iUseRegionCount ; ++i) + { + if (iUseRegionMemory[i] != aMemory || iUseRegionIndex[i] != aIndex + i) + return EFalse; + } + return ETrue; } -FORCE_INLINE TBool DPagingRequest::IsCollision(DMemoryObject* aMemory, TUint aIndex, TUint aCount) +TBool DPagingRequest::CheckUseDiscontiguous(DMemoryObject** aMemory, TUint* aIndex, TUint aCount) { + if (iUseRegionCount != aCount) + return EFalse; + for (TUint i = 0 ; i < iUseRegionCount ; ++i) + { + if (iUseRegionMemory[i] != aMemory[i] || iUseRegionIndex[i] != aIndex[i]) + return EFalse; + } + return ETrue; + } + + + TBool DPagingRequest::IsCollisionContiguous(DMemoryObject* aMemory, TUint aIndex, TUint aCount) + { + // note this could be optimised as most of the time we will be checking read/read collusions, + // both of which will be contiguous __ASSERT_SYSTEM_LOCK; - DMemoryObject* memory = iUseRegionMemory; - TUint index = iUseRegionIndex; - TUint count = iUseRegionCount; - // note, this comparison would fail if either region includes page number KMaxTUint, - // but it isn't possible to create a memory object which is > KMaxTUint pages... - return memory == aMemory && index+count > aIndex && index < aIndex+aCount; + for (TUint i = 0 ; i < iUseRegionCount ; ++i) + { + if (iUseRegionMemory[i] == aMemory && + TUint(iUseRegionIndex[i] - aIndex) < aCount) + return ETrue; + } + return EFalse; } @@ -2151,6 +2462,38 @@ iTempMapping.Unmap(aIMBRequired); } +// +// DPoolPagingRequest +// + +DPoolPagingRequest::DPoolPagingRequest(DPagingRequestPool::TGroup& aPoolGroup) : + iPoolGroup(aPoolGroup) + { + } + + +void DPoolPagingRequest::Release() + { + NKern::LockSystem(); + ResetUse(); + Signal(); + } + + +void DPoolPagingRequest::Wait() + { + __ASSERT_SYSTEM_LOCK; + ++iUsageCount; + TInt r = iMutex->Wait(); + __NK_ASSERT_ALWAYS(r == KErrNone); + } + + +void DPoolPagingRequest::Signal() + { + __ASSERT_SYSTEM_LOCK; + iPoolGroup.Signal(this); + } // // DPageReadRequest @@ -2158,6 +2501,13 @@ TInt DPageReadRequest::iAllocNext = 0; +DPageReadRequest::DPageReadRequest(DPagingRequestPool::TGroup& aPoolGroup) : + DPoolPagingRequest(aPoolGroup) + { + // allocate space for mapping pages whilst they're being loaded... + iTempMapping.Alloc(EMaxPages); + } + TInt DPageReadRequest::Construct() { // allocate id and mutex... @@ -2169,9 +2519,6 @@ if(r!=KErrNone) return r; - // allocate space for mapping pages whilst they're being loaded... - iTempMapping.Alloc(EMaxPages); - // create memory buffer... TUint bufferSize = EMaxPages+1; DMemoryObject* bufferMemory; @@ -2190,11 +2537,6 @@ return r; iBuffer = MM::MappingBase(bufferMapping); - // ensure there are enough young pages to cope with new request object... - r = ThePager.ResizeLiveList(); - if(r!=KErrNone) - return r; - return r; } @@ -2203,23 +2545,20 @@ // DPageWriteRequest // -TInt DPageWriteRequest::iAllocNext = 0; - -TInt DPageWriteRequest::Construct() + +DPageWriteRequest::DPageWriteRequest() { - // allocate id and mutex... - TUint id = (TUint)__e32_atomic_add_ord32(&iAllocNext, 1); - _LIT(KLitPagingRequest,"PageWriteRequest-"); - TBuf mutexName(KLitPagingRequest); - mutexName.AppendNum(id); - TInt r = K::MutexCreate(iMutex, mutexName, NULL, EFalse, KMutexOrdPageOut); - if(r!=KErrNone) - return r; - + iMutex = ThePageCleaningLock; // allocate space for mapping pages whilst they're being loaded... - iTempMapping.Alloc(EMaxPages); - - return r; + iTempMapping.Alloc(KMaxPagesToClean); + } + + +void DPageWriteRequest::Release() + { + NKern::LockSystem(); + ResetUse(); + NKern::UnlockSystem(); } @@ -2227,11 +2566,10 @@ // DPagingRequestPool // -DPagingRequestPool::DPagingRequestPool(TUint aNumPageReadRequest,TUint aNumPageWriteRequest) - : iPageReadRequests(aNumPageReadRequest), iPageWriteRequests(aNumPageWriteRequest) +DPagingRequestPool::DPagingRequestPool(TUint aNumPageReadRequest, TBool aWriteRequest) + : iPageReadRequests(aNumPageReadRequest) { TUint i; - for(i=0; iConstruct(); - __NK_ASSERT_ALWAYS(r==KErrNone); - iPageWriteRequests.iRequests[i] = req; - iPageWriteRequests.iFreeList.Add(req); + iPageWriteRequest = new DPageWriteRequest(); + __NK_ASSERT_ALWAYS(iPageWriteRequest); } } @@ -2264,24 +2598,23 @@ { NKern::LockSystem(); - DPagingRequest* req; - - // if we collide with page write operation... - req = iPageWriteRequests.FindCollision(aMemory,aIndex,aCount); - if(req) + DPoolPagingRequest* req; + + // check for collision with existing write + if(iPageWriteRequest && iPageWriteRequest->IsCollisionContiguous(aMemory,aIndex,aCount)) { - // wait until write completes... - req->Wait(); - req->Signal(); + NKern::UnlockSystem(); + PageCleaningLock::Lock(); + PageCleaningLock::Unlock(); return 0; // caller expected to retry if needed } // get a request object to use... req = iPageReadRequests.GetRequest(aMemory,aIndex,aCount); - // check no new requests collide with us... - if(iPageWriteRequests.FindCollision(aMemory,aIndex,aCount) - || iPageReadRequests.FindCollision(aMemory,aIndex,aCount)) + // check no new read or write requests collide with us... + if ((iPageWriteRequest && iPageWriteRequest->IsCollisionContiguous(aMemory,aIndex,aCount)) || + iPageReadRequests.FindCollisionContiguous(aMemory,aIndex,aCount)) { // another operation is colliding with this region, give up and retry... req->Signal(); @@ -2289,61 +2622,57 @@ } // we have a request object which we can use... - req->SetUse(aMemory,aIndex,aCount); + req->SetUseContiguous(aMemory,aIndex,aCount); NKern::UnlockSystem(); return (DPageReadRequest*)req; } -DPageWriteRequest* DPagingRequestPool::AcquirePageWriteRequest(DMemoryObject* aMemory, TUint aIndex, TUint aCount) +DPageWriteRequest* DPagingRequestPool::AcquirePageWriteRequest(DMemoryObject** aMemory, TUint* aIndex, TUint aCount) { + __NK_ASSERT_DEBUG(iPageWriteRequest); + __NK_ASSERT_DEBUG(PageCleaningLock::IsHeld()); + NKern::LockSystem(); - DPagingRequest* req; - - for(;;) - { - // get a request object to use... - req = iPageWriteRequests.GetRequest(aMemory,aIndex,aCount); - - if(iPageWriteRequests.FindCollision(aMemory,aIndex,aCount)) - { - // another write operation is colliding with this region, give up and retry... - req->Signal(); - // Reacquire the system lock as Signal() above will release it. - NKern::LockSystem(); - continue; - } - - break; - } - - // we have a request object which we can use... - req->SetUse(aMemory,aIndex,aCount); - + // Collision with existing read requests is not possible here. For a page to be read it must + // not be present, and for it to be written it must be present and dirty. There is no way for a + // page to go between these states without an intervening read on an uninitialised (freshly + // committed) page, which will wait on the first read request. In other words something like + // this: + // + // read (blocks), decommit, re-commit, read (waits on mutex), write (now no pending reads!) + // + // Note that a read request can be outstanding and appear to collide with this write, but only + // in the case when the thread making the read has blocked just after acquiring the request but + // before it checks whether the read is still necessasry. This makes it difficult to assert + // that no collisions take place. + + iPageWriteRequest->SetUseDiscontiguous(aMemory,aIndex,aCount); NKern::UnlockSystem(); - return (DPageWriteRequest*)req; + + return iPageWriteRequest; } DPagingRequestPool::TGroup::TGroup(TUint aNumRequests) { iNumRequests = aNumRequests; - iRequests = new DPagingRequest*[aNumRequests]; + iRequests = new DPoolPagingRequest*[aNumRequests]; __NK_ASSERT_ALWAYS(iRequests); } -DPagingRequest* DPagingRequestPool::TGroup::FindCollision(DMemoryObject* aMemory, TUint aIndex, TUint aCount) +DPoolPagingRequest* DPagingRequestPool::TGroup::FindCollisionContiguous(DMemoryObject* aMemory, TUint aIndex, TUint aCount) { __ASSERT_SYSTEM_LOCK; - DPagingRequest** ptr = iRequests; - DPagingRequest** ptrEnd = ptr+iNumRequests; + DPoolPagingRequest** ptr = iRequests; + DPoolPagingRequest** ptrEnd = ptr+iNumRequests; while(ptrIsCollision(aMemory,aIndex,aCount)) + DPoolPagingRequest* req = *ptr++; + if(req->IsCollisionContiguous(aMemory,aIndex,aCount)) return req; } return 0; @@ -2352,16 +2681,16 @@ static TUint32 RandomSeed = 33333; -DPagingRequest* DPagingRequestPool::TGroup::GetRequest(DMemoryObject* aMemory, TUint aIndex, TUint aCount) +DPoolPagingRequest* DPagingRequestPool::TGroup::GetRequest(DMemoryObject* aMemory, TUint aIndex, TUint aCount) { __NK_ASSERT_DEBUG(iNumRequests > 0); // try using an existing request which collides with this region... - DPagingRequest* req = FindCollision(aMemory,aIndex,aCount); + DPoolPagingRequest* req = FindCollisionContiguous(aMemory,aIndex,aCount); if(!req) { // use a free request... - req = (DPagingRequest*)iFreeList.GetFirst(); + req = (DPoolPagingRequest*)iFreeList.GetFirst(); if(req) { // free requests aren't being used... @@ -2384,7 +2713,7 @@ } -void DPagingRequestPool::TGroup::Signal(DPagingRequest* aRequest) +void DPagingRequestPool::TGroup::Signal(DPoolPagingRequest* aRequest) { // if there are no threads waiting on the mutex then return it to the free pool... __NK_ASSERT_DEBUG(aRequest->iUsageCount > 0); @@ -2406,13 +2735,14 @@ { TRACEB(("Kern::InstallPagingDevice(0x%08x) name='%s' type=%d",aDevice,aDevice->iName,aDevice->iType)); + __NK_ASSERT_DEBUG(!ThePager.CacheInitialised()); __NK_ASSERT_ALWAYS(aDevice->iReadUnitShift <= KPageShift); TInt r = KErrNotSupported; // Will return this if unsupported device type is installed // create the pools of page out and page in requests... - const TInt writeReqs = (aDevice->iType & DPagingDevice::EData) ? KPagingRequestsPerDevice : 0; - aDevice->iRequestPool = new DPagingRequestPool(KPagingRequestsPerDevice,writeReqs); + const TBool writeReq = (aDevice->iType & DPagingDevice::EData) != 0; + aDevice->iRequestPool = new DPagingRequestPool(KPagingRequestsPerDevice, writeReq); if(!aDevice->iRequestPool) { r = KErrNoMemory; @@ -2442,6 +2772,9 @@ if (K::MemModelAttributes & (EMemModelAttrRomPaging | EMemModelAttrCodePaging | EMemModelAttrDataPaging)) TheThrashMonitor.Start(); + + if (K::MemModelAttributes & EMemModelAttrDataPaging) + PageCleaner::Start(); exit: TRACEB(("Kern::InstallPagingDevice returns %d",r)); @@ -2591,3 +2924,32 @@ } + +// +// PageCleaningLock +// + +_LIT(KLitPageCleaningLock,"PageCleaningLock"); + +void PageCleaningLock::Init() + { + __NK_ASSERT_DEBUG(!ThePageCleaningLock); + TInt r = Kern::MutexCreate(ThePageCleaningLock, KLitPageCleaningLock, KMutexOrdPageOut); + __NK_ASSERT_ALWAYS(r == KErrNone); + } + +void PageCleaningLock::Lock() + { + Kern::MutexWait(*ThePageCleaningLock); + } + + +void PageCleaningLock::Unlock() + { + Kern::MutexSignal(*ThePageCleaningLock); + } + +TBool PageCleaningLock::IsHeld() + { + return ThePageCleaningLock->iCleanup.iThread == &Kern::CurrentThread(); + }