diff -r 000000000000 -r a41df078684a kernel/eka/memmodel/epoc/mmubase/ramalloc.cpp --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/kernel/eka/memmodel/epoc/mmubase/ramalloc.cpp Mon Oct 19 15:55:17 2009 +0100 @@ -0,0 +1,2966 @@ +// Copyright (c) 1998-2009 Nokia Corporation and/or its subsidiary(-ies). +// All rights reserved. +// This component and the accompanying materials are made available +// under the terms of the License "Eclipse Public License v1.0" +// which accompanies this distribution, and is available +// at the URL "http://www.eclipse.org/legal/epl-v10.html". +// +// Initial Contributors: +// Nokia Corporation - initial contribution. +// +// Contributors: +// +// Description: +// e32\memmodel\epoc\mmubase\ramalloc.cpp +// +// + +/** + @file + @internalComponent +*/ +//#define __VERIFY_LEASTMOVDIS + +#include +#include +#include + +#ifndef __MEMMODEL_FLEXIBLE__ +#include +#else +#include "mdefrag.inl" +#endif //__MEMMODEL_FLEXIBLE__ + +DRamAllocator* DRamAllocator::New() + { + return new DRamAllocator; + } + +DRamAllocator* DRamAllocator::New(const SRamInfo& aInfo, const SRamZone* aZoneInfo, TRamZoneCallback aZoneCallback) + { + DRamAllocator* pA=New(); + if (!pA) + Panic(ECreateNoMemory); + // If this fails then it won't return but panic + pA->Create(aInfo,aZoneInfo, aZoneCallback); + return pA; + } + +void DRamAllocator::Panic(TPanic aPanic) + { + Kern::Fault("RAM-ALLOC", aPanic); + } + +#ifdef KMMU +void HexDump32(const TAny* a, TInt n, const char* s) + { + const TUint32* p=(const TUint32*)a; + Kern::Printf(s); + TInt i=0; + while(n) + { + TBuf8<80> b; + b.AppendNumFixedWidth(i,EHex,4); + b.Append(':'); + TInt m=Min(n,4); + n-=m; + i+=m; + while(m--) + { + b.Append(' '); + b.AppendNumFixedWidth(*p++,EHex,8); + } + Kern::Printf("%S",&b); + } + } + +void HexDump8(const TAny* a, TInt n, const char* s) + { + const TUint8* p=(const TUint8*)a; + Kern::Printf(s); + TInt i=0; + while(n) + { + TBuf8<80> b; + b.AppendNumFixedWidth(i,EHex,4); + b.Append(':'); + TInt m=Min(n,16); + n-=m; + i+=m; + while(m--) + { + b.Append(' '); + b.AppendNumFixedWidth(*p++,EHex,2); + } + Kern::Printf("%S",&b); + } + } + +void DRamAllocator::DebugDump() + { + Kern::Printf("PageSize=%08x PageShift=%d",KPageSize,KPageShift); + Kern::Printf("Total Pages=%x Total Free=%x",iTotalRamPages,iTotalFreeRamPages); + Kern::Printf("Number of zones=%d, PowerState=%016lx",iNumZones,iZonePwrState); + Kern::Printf("PhysAddrBase=%08x, PhysAddrTop=%08x",iPhysAddrBase,iPhysAddrTop); + + TUint i = 0; + Kern::Printf("Zone Info:"); + for (; iiNext) + { + SZone& zone = *_LOFF(link, SZone, iPrefLink); + Kern::Printf("ID0x%x rank0x%x", zone.iId, zone.iPrefRank); + } + SZone& zone = *_LOFF(iZoneLeastMovDis, SZone, iPrefLink); + Kern::Printf("iZoneLeastMovDis ID 0x%x rank 0x%x", zone.iId, iZoneLeastMovDisRank); + } +#endif + +TInt CountBanks(const SRamBank* aBankList) + { + TInt banks=0; + for (; aBankList->iSize; ++banks, ++aBankList); + return banks; + } + +TUint32 TotalBankSize(const SRamBank* aBankList) + { + TUint32 size=0; + for (; aBankList->iSize; ++aBankList) + size+=aBankList->iSize; + return size; + } + +/** +Count how many zones have been specified and do some basic checks on their layout: + Zones must be distinct, i.e. not overlap + Zone ID must be unique + Zones must be page size aligned + Zones must be big enough to cover all of the allocatable RAM +The end of the list is indicated by a SRamZone.iSize==0. +@param aZones The list of RAM zones to be setup +*/ +void DRamAllocator::CountZones(const SRamZone* aZones) + { + TUint32 totalSize = 0; + TUint32 pageMask = KPageSize-1; + // Check zones don't overlap each other and while running through the zones + // calculate how many there are + const SRamZone* pCurZ = aZones; + for (; pCurZ->iSize != 0; pCurZ++) + { + // Verify zone addresses and alignment + TUint32 curEnd = pCurZ->iBase + pCurZ->iSize - 1; + __KTRACE_OPT(KMMU,Kern::Printf("curBase %x curEnd %x pageMask %x",pCurZ->iBase,curEnd,pageMask)); + if (curEnd <= pCurZ->iBase || (((curEnd + 1) | pCurZ->iBase) & pageMask)) + { + Panic(EZonesAlignment); + } + + if (pCurZ->iId == KRamZoneInvalidId) + { + Panic(EZonesIDInvalid); + } + // Check the flags are not set to invalid values + if (pCurZ->iFlags & KRamZoneFlagInvalid) + { + Panic(EZonesFlagsInvalid); + } + + iNumZones++; + if (iNumZones > KMaxRamZones) + {// Too many zones specified + Panic(EZonesTooNumerousOrFew); + } + totalSize += pCurZ->iSize; + + // Verify this zone doesn't overlap any of the previous zones' address space + const SRamZone* pTmpZ = aZones; + for (; pTmpZ < pCurZ; pTmpZ++) + { + TUint32 tmpEnd = pTmpZ->iBase + pTmpZ->iSize - 1; + if (tmpEnd >= pCurZ->iBase && pTmpZ->iBase <= curEnd) + { + Panic(EZonesNotDistinct); + } + if(pTmpZ->iId == pCurZ->iId) + { + Panic(EZonesIDNotUnique); + } + } + } + __KTRACE_OPT(KMMU,Kern::Printf("iNumZones=%d, totalSize=%x",iNumZones,totalSize)); + if (!iNumZones) + {// no zones specified + Panic(EZonesTooNumerousOrFew); + } + + // Together all of the zones should cover the whole of the RAM + if (totalSize>>KPageShift < iTotalRamPages) + { + Panic(EZonesIncomplete); + } + } + + +/** +Get the zone from the ID +@param aId ID of zone to find +@return Pointer to the zone if zone of matching ID found, NULL otherwise +*/ +SZone* DRamAllocator::ZoneFromId(TUint aId) const + { + SZone* pZ = iZones; + const SZone* const pEndZone = iZones + iNumZones; + for (; pZ < pEndZone; pZ++) + { + if (aId == pZ->iId) + { + return pZ; + } + } + return NULL; + } + +/** Retrieve the physical base address and number of pages in the specified zone. + +@param aZoneId The ID of the zone +@param aPhysBaseAddr Receives the base address of the zone +@param aNumPages Receives the number of pages in the zone + +@return KErrNone if zone found, KErrArgument if zone couldn't be found +*/ +TInt DRamAllocator::GetZoneAddress(TUint aZoneId, TPhysAddr& aPhysBase, TUint& aNumPages) + { + SZone* zone = ZoneFromId(aZoneId); + if (zone == NULL) + { + return KErrArgument; + } + aPhysBase = zone->iPhysBase; + aNumPages = zone->iPhysPages; + return KErrNone; + } + +#ifdef __MEMMODEL_FLEXIBLE__ +/** +@param aAddr The address of page to find the zone of +@param aOffset The page offset from the start of the zone that the page is in +*/ +SZone* DRamAllocator::GetZoneAndOffset(TPhysAddr aAddr, TInt& aOffset) + { + // Get the zone from the SPageInfo of the page at aAddr + SPageInfo* pageInfo = SPageInfo::SafeFromPhysAddr(aAddr); + if (pageInfo == NULL) + { + return NULL; + } + + // Perform a binary search for the RAM zone, we know aAddr is within a RAM + // zone as pageInfo != NULL. + SZone* left = iZones; + SZone* mid = iZones + (iNumZones>>1); + SZone* top = iZones + iNumZones - 1; + + while (mid->iPhysEnd < aAddr || mid->iPhysBase > aAddr) + { + if (mid->iPhysEnd < aAddr) + left = mid + 1; + else + top = mid - 1; + mid = left + ((top - left) >> 1); + __ASSERT_DEBUG(left <= top && mid <= top && mid >= left, Panic(EAllocRamPagesInconsistent)); + } + __ASSERT_DEBUG(mid->iPhysBase <= aAddr && mid->iPhysEnd >= aAddr, Panic(EAllocRamPagesInconsistent)); + aOffset = (aAddr - mid->iPhysBase) >> KPageShift; + __ASSERT_DEBUG((TUint)aOffset < mid->iPhysPages, Panic(EAllocRamPagesInconsistent)); + return mid; + } +#else +/** +@param aAddr The address of page to find the zone of +@param aOffset The page offset from the start of the zone that the page is in +*/ +SZone* DRamAllocator::GetZoneAndOffset(TPhysAddr aAddr, TInt& aOffset) + { + // Get the zone from the SPageInfo of the page at aAddr + SPageInfo* pageInfo = SPageInfo::SafeFromPhysAddr(aAddr); + if (pageInfo == NULL) + { + return NULL; + } + SZone* z = iZones + pageInfo->Zone(); + aOffset = (aAddr - z->iPhysBase) >> KPageShift; + __ASSERT_DEBUG((TUint)aOffset < z->iPhysPages, Panic(EAllocRamPagesInconsistent)); + return z; + } +#endif +/** +@param aId ID of zone to get page count for +@param aPageData store for page counts +@return KErrNone if zone found, KErrArgument otherwise +*/ +TInt DRamAllocator::GetZonePageCount(TUint aId, SRamZonePageCount& aPageData) + { + // Search for the zone of ID aId + const SZone* zone = ZoneFromId(aId); + if (zone == NULL) + { + return KErrArgument; + } + aPageData.iFreePages = zone->iFreePages; + aPageData.iUnknownPages = zone->iAllocPages[EPageUnknown]; + aPageData.iFixedPages = zone->iAllocPages[EPageFixed]; + aPageData.iMovablePages = zone->iAllocPages[EPageMovable]; + aPageData.iDiscardablePages = zone->iAllocPages[EPageDiscard]; + + return KErrNone; + } + + +/** Update the count of free and allocated pages for the zone with +@param aZone The index of the zone whose counts are being updated +@param aCount The no of pages being allocated +@param aType The type of the pages being allocated +*/ +void DRamAllocator::ZoneAllocPages(SZone* aZone, TUint32 aCount, TZonePageType aType) + { +#ifdef _DEBUG + TUint32 free = aZone->iFreePages - aCount; + TUint32 alloc = aZone->iAllocPages[aType] + aCount; + TUint32 total_alloc = aZone->iAllocPages[EPageUnknown] + + aZone->iAllocPages[EPageDiscard] + + aZone->iAllocPages[EPageMovable] + + aZone->iAllocPages[EPageFixed] + aCount; + if (free > aZone->iFreePages || + alloc < aZone->iAllocPages[aType] || + free + total_alloc != aZone->iPhysPages || + iTotalFreeRamPages > iTotalRamPages) + { + __KTRACE_OPT(KMMU,Kern::Printf("TotalFree %x TotalPages %x",iTotalFreeRamPages, iTotalRamPages)); + __KTRACE_OPT(KMMU,Kern::Printf("ZoneFreePages - aCount %x free %x, alloc %x",aCount,free,alloc)); // counts rolled over + __KTRACE_OPT(KMMU,Kern::Printf("Alloc Unk %x Fx %x Mv %x Dis %x",aZone->iAllocPages[EPageUnknown], + aZone->iAllocPages[EPageFixed], aZone->iAllocPages[EPageMovable],aZone->iAllocPages[EPageDiscard])); + Panic(EZonesCountErr); + } + __ASSERT_DEBUG(free == (TUint32)aZone->iBma[KBmaAllPages]->iAvail, Panic(EAllocRamPagesInconsistent)); + __KTRACE_OPT(KMMU2,Kern::Printf("ZoneFreePages - aCount %x free %x, alloc %x",aCount,free,alloc)); + __KTRACE_OPT(KMMU2,Kern::Printf("Alloc Unk %x Fx %x Mv %x Dis %x",aZone->iAllocPages[EPageUnknown], + aZone->iAllocPages[EPageFixed], aZone->iAllocPages[EPageMovable],aZone->iAllocPages[EPageDiscard])); + + if (iAllowBmaVerify) + { + TBitMapAllocator& bmaType = *(aZone->iBma[(aType != EPageUnknown)? aType : EPageFixed]); + TUint allocPages; + if (aType == EPageFixed || aType == EPageUnknown) + allocPages = aZone->iAllocPages[EPageUnknown] + aZone->iAllocPages[EPageFixed]; + else + allocPages = aZone->iAllocPages[aType]; + allocPages += aCount; + __NK_ASSERT_DEBUG(aZone->iPhysPages - bmaType.iAvail == allocPages); + __NK_ASSERT_DEBUG((TUint)bmaType.iAvail >= aZone->iFreePages - aCount); + +//#define _FULL_VERIFY_TYPE_BMAS +#ifdef _FULL_VERIFY_TYPE_BMAS + TUint offset = 0; + TUint matchedPages = 0; + TInt r = KErrNone; + while (offset < aZone->iPhysPages && r == KErrNone) + { + r = NextAllocatedPage(aZone, offset, EPageTypes); + if (bmaType.NotFree(offset, 1)) + { + matchedPages++; + } + offset++; + } + __NK_ASSERT_DEBUG(matchedPages == allocPages); +#endif + } +#endif + + // Update counts + aZone->iAllocPages[aType] += aCount; + aZone->iFreePages -= aCount; + aZone->iFlags &= ~KRamZoneFlagMark; // clear the mark as this zone is active + + // Check if power state of zone needs to be changed + if (iZonePowerFunc && !(iZonePwrState & (((TUint64)1) << aZone - iZones))) + {//zone no longer empty so call variant to power RAM zone up if necessary + iZonePwrState |= (((TUint64)1) << aZone - iZones); + + if (iZoneCallbackInitSent) + { + TInt ret = (*iZonePowerFunc)(ERamZoneOp_PowerUp, (TAny*)aZone->iId, (TUint*)&iZonePwrState); + if (ret != KErrNone && ret != KErrNotSupported) + { + Panic(EZonesCallbackErr); + } + CHECK_PRECONDITIONS(MASK_THREAD_CRITICAL, "DRamAllocator::ZoneAllocPages"); + } + } + + // Re-order the zone preference list so that a RAM zone with more immovable pages + // is more preferable and secondary to that a RAM zone that is not empty is more + // preferable than one that is empty. + while (&aZone->iPrefLink != iZonePrefList.First()) + { + SZone* prevZ = _LOFF(aZone->iPrefLink.iPrev, SZone, iPrefLink); + __NK_ASSERT_DEBUG(K::Initialising || prevZ->iPrefRank == aZone->iPrefRank - 1); + if (prevZ->iPref == aZone->iPref && + (prevZ->iAllocPages[EPageFixed] + prevZ->iAllocPages[EPageUnknown] < + aZone->iAllocPages[EPageFixed] + aZone->iAllocPages[EPageUnknown] || + prevZ->iFreePages == prevZ->iPhysPages)) + { + __KTRACE_OPT(KMMU, Kern::Printf("a - Reorder aZone 0x%x free 0x%x before prevZ 0x%x free 0x%x", aZone->iId, aZone->iFreePages, prevZ->iId, prevZ->iFreePages)); + // Make this RAM zone more preferable. + aZone->iPrefLink.Deque(); + aZone->iPrefLink.InsertBefore(&prevZ->iPrefLink); + aZone->iPrefRank--; + prevZ->iPrefRank++; + + if (iZoneLeastMovDis == &prevZ->iPrefLink) + {// Ensure iZoneLeastMovDisRank is kept up to date. + iZoneLeastMovDisRank = prevZ->iPrefRank; + } + if (iZoneLeastMovDis == &aZone->iPrefLink) + {// Ensure iZoneLeastMovDisRank is kept up to date. + iZoneLeastMovDisRank = aZone->iPrefRank; + // aZone was the least preferable with movable and/or discardable so is it still? + if (prevZ->iAllocPages[EPageMovable] || prevZ->iAllocPages[EPageDiscard]) + {// prevZ is now the least preferable RAM zone with movable and/or discardable. + iZoneLeastMovDis = &prevZ->iPrefLink; + iZoneLeastMovDisRank = prevZ->iPrefRank; + __KTRACE_OPT(KMMU, Kern::Printf("aa - iZoneleastInUse ID 0x%x", (_LOFF(iZoneLeastMovDis, SZone, iPrefLink))->iId)); + } + __KTRACE_OPT(KMMU, Kern::Printf("iZoneLeastMovDisRank 0x%x", iZoneLeastMovDisRank)); + } + } + else + { + break; + } + } + + // Now that the preference list has been re-ordered check whether + // iZoneLeastMovDis needs updating. + if (aType >= EPageMovable && iZoneLeastMovDisRank < aZone->iPrefRank) + { + iZoneLeastMovDis = &aZone->iPrefLink; + iZoneLeastMovDisRank = aZone->iPrefRank; + __KTRACE_OPT(KMMU, Kern::Printf("a - iZoneleastInUse ID 0x%x", (_LOFF(iZoneLeastMovDis, SZone, iPrefLink))->iId)); + } + __NK_ASSERT_DEBUG( K::Initialising || + iZoneLeastMovDisRank == _LOFF(iZoneLeastMovDis, SZone, iPrefLink)->iPrefRank); +#ifdef __VERIFY_LEASTMOVDIS + if (!K::Initialising) + VerifyLeastPrefMovDis(); +#endif + } + + +/** Update the count of free and allocated pages for the zone with +@param aZone The index of the zone whose counts are being updated +@param aCount The no of pages being freed +@param aType The type of the pages being freed +*/ +void DRamAllocator::ZoneFreePages(SZone* aZone, TUint32 aCount, TZonePageType aType) + { +#ifdef _DEBUG + TUint32 alloc = aZone->iAllocPages[aType] - aCount; + TUint32 free = aZone->iFreePages + aCount; + TUint32 total_alloc = aZone->iAllocPages[EPageUnknown] + + aZone->iAllocPages[EPageDiscard] + + aZone->iAllocPages[EPageMovable] + + aZone->iAllocPages[EPageFixed] - aCount; + if (free < aZone->iFreePages || + alloc > aZone->iAllocPages[aType] || + free + total_alloc != aZone->iPhysPages || + iTotalFreeRamPages > iTotalRamPages) + { + __KTRACE_OPT(KMMU,Kern::Printf("TotalFree %x TotalPages %x",iTotalFreeRamPages, iTotalRamPages)); + __KTRACE_OPT(KMMU,Kern::Printf("ZoneFreePages - aCount %x free %x, alloc %x",aCount,free,alloc)); // counts rolled over + __KTRACE_OPT(KMMU,Kern::Printf("Alloc Unk %x Fx %x Mv %x Dis %x",aZone->iAllocPages[EPageUnknown], + aZone->iAllocPages[EPageFixed], aZone->iAllocPages[EPageMovable],aZone->iAllocPages[EPageDiscard])); + Panic(EZonesCountErr); + } + __ASSERT_DEBUG(free == (TUint32)aZone->iBma[KBmaAllPages]->iAvail, Panic(EAllocRamPagesInconsistent)); + __KTRACE_OPT(KMMU2,Kern::Printf("ZoneFreePages - aCount %x free %x, alloc %x",aCount,free,alloc)); + __KTRACE_OPT(KMMU2,Kern::Printf("Alloc Unk %x Fx %x Mv %x Dis %x",aZone->iAllocPages[EPageUnknown], + aZone->iAllocPages[EPageFixed], aZone->iAllocPages[EPageMovable],aZone->iAllocPages[EPageDiscard])); + + if (iAllowBmaVerify) + { + TBitMapAllocator& bmaType = *(aZone->iBma[(aType != EPageUnknown)? aType : EPageFixed]); + TUint allocPages; + if (aType == EPageFixed || aType == EPageUnknown) + allocPages = aZone->iAllocPages[EPageUnknown] + aZone->iAllocPages[EPageFixed]; + else + allocPages = aZone->iAllocPages[aType]; + allocPages -= aCount; + __NK_ASSERT_DEBUG(aZone->iPhysPages - bmaType.iAvail == allocPages); + __NK_ASSERT_DEBUG((TUint)bmaType.iAvail >= aZone->iFreePages + aCount); + +#ifdef _FULL_VERIFY_TYPE_BMAS + TUint offset = 0; + TUint matchedPages = 0; + TInt r = KErrNone; + while(offset < aZone->iPhysPages && r == KErrNone) + { + r = NextAllocatedPage(aZone, offset, EPageTypes); + if (bmaType.NotFree(offset, 1)) + { + matchedPages++; + } + offset++; + } + __NK_ASSERT_DEBUG(matchedPages == allocPages); +#endif + } +#endif + + // Update counts + aZone->iAllocPages[aType] -= aCount; + aZone->iFreePages += aCount; + aZone->iFlags &= ~KRamZoneFlagMark; // clear the mark as this zone is active + + // Check if power state of zone needs to be changed. + // Don't update iZonePwrState when a zone is being cleared to then be + // claimed as it shouldn't be powered off as it's about to be used. + if (iZonePowerFunc && !(aZone->iFlags & KRamZoneFlagClaiming) && + aZone->iFreePages == aZone->iPhysPages) + {// Zone is empty so call variant to power down RAM zone if desirable. + TUint64 pwrMask = ~(((TUint64)1) << aZone - iZones); + iZonePwrState &= pwrMask; + + // Don't invoke callback until Init callback sent. + if (iZoneCallbackInitSent) + { + TInt ret = (*iZonePowerFunc)(ERamZoneOp_PowerDown, (TAny*)aZone->iId, (TUint*)&iZonePwrState); + if (ret != KErrNone && ret != KErrNotSupported) + { + Panic(EZonesCallbackErr); + } + CHECK_PRECONDITIONS(MASK_THREAD_CRITICAL, "DRamAllocator::ZoneFreePages"); + } + } + + // Re-order the zone preference list so that a RAM zone with more immovable pages + // is more preferable and secondary to that a RAM zone that is not empty is more + // preferable than one that is empty. + while (&aZone->iPrefLink != iZonePrefList.Last()) + { + SZone* nextZ = _LOFF(aZone->iPrefLink.iNext, SZone, iPrefLink); + __NK_ASSERT_DEBUG(K::Initialising || nextZ->iPrefRank == aZone->iPrefRank + 1); + if (nextZ->iPref == aZone->iPref && + (nextZ->iAllocPages[EPageFixed] + nextZ->iAllocPages[EPageUnknown] > + aZone->iAllocPages[EPageFixed] + aZone->iAllocPages[EPageUnknown] || + (nextZ->iFreePages != nextZ->iPhysPages && + aZone->iFreePages == aZone->iPhysPages))) + { + __KTRACE_OPT(KMMU, Kern::Printf("f - Reorder aZone 0x%x free 0x%x after nextZ 0x%x free 0x%x", aZone->iId, aZone->iFreePages, nextZ->iId, nextZ->iFreePages)); + // Make this RAM zone less preferable. + aZone->iPrefLink.Deque(); + aZone->iPrefLink.InsertAfter(&nextZ->iPrefLink); + aZone->iPrefRank++; + nextZ->iPrefRank--; + + if (iZoneLeastMovDis == &aZone->iPrefLink) + {// Ensure iZoneLeastMovDisRank is kept up to date. + iZoneLeastMovDisRank = aZone->iPrefRank; + } + if (iZoneLeastMovDis == &nextZ->iPrefLink) + {// Ensure iZoneLeastMovDisRank is kept up to date. + iZoneLeastMovDisRank = nextZ->iPrefRank; + if (aZone->iAllocPages[EPageMovable] || aZone->iAllocPages[EPageDiscard]) + {// aZone is now the least preferable RAM zone with movable and/or discardable. + iZoneLeastMovDis = &aZone->iPrefLink; + iZoneLeastMovDisRank = aZone->iPrefRank; + __KTRACE_OPT(KMMU, Kern::Printf("aa - iZoneleastInUse ID 0x%x", (_LOFF(iZoneLeastMovDis, SZone, iPrefLink))->iId)); + } + __KTRACE_OPT(KMMU, Kern::Printf("iZoneLeastMovDis Rank 0x%x", iZoneLeastMovDisRank)); + } + } + else + { + break; + } + } + if (&aZone->iPrefLink == iZoneLeastMovDis && + !aZone->iAllocPages[EPageMovable] && !aZone->iAllocPages[EPageDiscard]) + {// This RAM zone no longer has movable or discardable and therefore it + // is also no longer the least preferable RAM zone with movable and/or + // discardable. + SZone* zonePrev; + do + { + iZoneLeastMovDis = iZoneLeastMovDis->iPrev; + iZoneLeastMovDisRank--; + if (iZoneLeastMovDis == iZonePrefList.First()) + {// This the most preferable RAM zone so can't go any further. + break; + } + zonePrev = _LOFF(iZoneLeastMovDis, SZone, iPrefLink); + __KTRACE_OPT(KMMU, Kern::Printf("f - iZoneLeastMovDis 0x%x", zonePrev->iId)); + } + while (!zonePrev->iAllocPages[EPageMovable] && !zonePrev->iAllocPages[EPageDiscard]); + + __NK_ASSERT_DEBUG( K::Initialising || + iZoneLeastMovDisRank == _LOFF(iZoneLeastMovDis, SZone, iPrefLink)->iPrefRank); + +#ifdef __VERIFY_LEASTMOVDIS + if (!K::Initialising) + VerifyLeastPrefMovDis(); +#endif + } + } + + +/** Calculate the physical address order of the zones and temporally store + the order in aZoneAddrOrder +*/ +inline void DRamAllocator::SortRamZones(const SRamZone* aZones, TUint8* aZoneAddrOrder) + { + const SRamZone* const endZone = aZones + iNumZones; + const SRamZone* zone = aZones; + for (; zone < endZone; zone++) + { + // zoneIdx is the number of zones that have a lower base address than the + // current zone and therefore it is the address index of the current zone + TInt zoneIdx = 0; + // search for any zones of lower base address + const SRamZone* zone2 = aZones; + for (; zone2 < endZone; zone2++) + { + if (zone2->iBase < zone->iBase) + { + zoneIdx++; // have another zone of lower base address + } + } + aZoneAddrOrder[zoneIdx] = zone - aZones; + } + } + + +/** Initialise SPageInfos for all pages in this zone with the +index of the zone. +@param aZone The zone the pages to be initialised are in +*/ +inline TUint DRamAllocator::InitSPageInfos(const SZone* aZone) + { + TUint pagesUpdated = 0; + if (aZone->iPhysBase > iPhysAddrTop || aZone->iPhysEnd < iPhysAddrBase) + {// None of the zone is in allocatable RAM + return pagesUpdated; + } + + // Mark each allocatable page in this zone with the index of the zone +#ifndef __MEMMODEL_FLEXIBLE__ + TUint8 zoneIndex = aZone - iZones; +#endif + TPhysAddr addr = aZone->iPhysBase; + for (; addr <= aZone->iPhysEnd; addr += KPageSize) + { + SPageInfo* pi = SPageInfo::SafeFromPhysAddr(addr); + if (pi) + { +#ifndef __MEMMODEL_FLEXIBLE__ // The FMM doesn't store zone indices in SPageInfos. + pi->SetZone(zoneIndex); +#endif + pagesUpdated++; + } + } + return pagesUpdated; + } + +/** HAL Function for the RAM allocator. +*/ +TInt DRamAllocator::HalFunction(TInt aFunction, TAny* a1, TAny* a2) + { + switch(aFunction) + { + case ERamHalGetZoneCount: + { + kumemput32(a1, &iNumZones, sizeof(iNumZones)); + return KErrNone; + } + + case ERamHalGetZoneConfig: + { + TUint zoneIndex = (TUint)a1; + if (zoneIndex < iNumZones) + { + SZone* pZone = iZones + zoneIndex; + struct SRamZoneConfig config; + NKern::ThreadEnterCS(); + M::RamAllocLock(); // get mutex to ensure consistent set of values are read... + config.iZoneId = pZone->iId; + config.iZoneIndex = zoneIndex; + config.iPhysBase = pZone->iPhysBase; + config.iPhysEnd = pZone->iPhysEnd; + config.iPhysPages = pZone->iPhysPages; + config.iPref = pZone->iPref; + config.iFlags = pZone->iFlags; + M::RamAllocUnlock(); + NKern::ThreadLeaveCS(); + kumemput32(a2,&config,sizeof(config)); + return KErrNone; + } + return KErrNotFound; + } + + case ERamHalGetZoneUtilisation: + { + TUint zoneIndex = (TUint)a1; + if (zoneIndex < iNumZones) + { + SZone* pZone = iZones + zoneIndex; + struct SRamZoneUtilisation config; + NKern::ThreadEnterCS(); + M::RamAllocLock(); // get mutex to ensure consistent set of values are read... + config.iZoneId = pZone->iId; + config.iZoneIndex = zoneIndex; + config.iPhysPages = pZone->iPhysPages; + config.iFreePages = pZone->iFreePages; + config.iAllocUnknown = pZone->iAllocPages[EPageUnknown]; + config.iAllocFixed = pZone->iAllocPages[EPageFixed]; + config.iAllocMovable = pZone->iAllocPages[EPageMovable]; + config.iAllocDiscardable = pZone->iAllocPages[EPageDiscard]; + config.iAllocOther = 0; + M::RamAllocUnlock(); + NKern::ThreadLeaveCS(); + kumemput32(a2,&config,sizeof(config)); + return KErrNone; + } + return KErrNotFound; + } + + default: + { + return KErrNotSupported; + } + } + } + +/** +Setup the ram allocator with information of the RAM available in the system that +comes from the bootstrap/superpage. This is intended to be called from +DRamAllocator::New(). +@internalComponent +@see DRamAllocator::New() +@param aInfo Two lists of SRamBanks for available and reserved banks in RAM, respectively +@param aZones A list of the ram zones in the system and their configuration/preferences +@param aZoneCallback Pointer to a base port call back function that will be invoked by this class +*/ +void DRamAllocator::Create(const SRamInfo& aInfo, const SRamZone* aZones, TRamZoneCallback aZoneCallback) + { + __KTRACE_OPT(KMMU,Kern::Printf("DRamAllocator::Create")); + + // SZone::iBma array assumes this and KBmaAllPages can't be the same as any + // allocatable page type. + __ASSERT_COMPILE(EPageFixed < KPageImmovable && EPageUnknown < KPageImmovable && + EPageDiscard >= KPageImmovable && EPageMovable >= KPageImmovable && + KBmaAllPages != EPageFixed && KBmaAllPages != EPageMovable && + KBmaAllPages != EPageDiscard); + // NoAllocOfPageType() requires this + __ASSERT_COMPILE( KRamZoneFlagNoFixed == 1 << (EPageFixed - KPageTypeAllocBase) && + KRamZoneFlagNoMovable == 1 << (EPageMovable - KPageTypeAllocBase) && + KRamZoneFlagNoDiscard == 1 << (EPageDiscard - KPageTypeAllocBase)); + + // SZone::iPhysEnd and iPhysAddrTop rely on this when checking contiguous zones etc. + __ASSERT_COMPILE(KPageShift != 0); + + /////////////////////////////////////////////////////////////////////////// + // Determine where all the allocatable RAM pages are, using the SRamBank + // data passed to the kernel by the bootstrap + ////////////////////////////////////////////////////////////////////////// + TUint num_boot_banks=CountBanks(aInfo.iBanks); + TUint32 total_ram_size=TotalBankSize(aInfo.iBanks); + __KTRACE_OPT(KMMU,Kern::Printf("#banks from bootstrap=%d",num_boot_banks)); + __KTRACE_OPT(KMMU,Kern::Printf("Total size=%08x",total_ram_size)); + iTotalRamPages=total_ram_size>>KPageShift; + // Assume all pages are allocated as unknown for now + iTotalFreeRamPages = 0; + __KTRACE_OPT(KMMU,Kern::Printf("Total size=%08x, total pages=%08x",total_ram_size,iTotalRamPages)); + + iPhysAddrBase=aInfo.iBanks[0].iBase; + const SRamBank& last_boot_bank=aInfo.iBanks[num_boot_banks-1]; + iPhysAddrTop = last_boot_bank.iBase + last_boot_bank.iSize - 1; + __KTRACE_OPT(KMMU,Kern::Printf("PA base=%08x, PA top=%08x",iPhysAddrBase,iPhysAddrTop)); + + __ASSERT_DEBUG(iPhysAddrTop > iPhysAddrBase, Panic(ECreateInvalidRamBanks)); + + + /////////////////////////////////////////////////////////////////////////// + // Determine how many zones are required and allocate all the + // data structures that will be required, permanent one first then + // temporary ones to avoid kernel heap fragmentation. + /////////////////////////////////////////////////////////////////////////// + // Stop any RAM zone callback operations until the initial one has been sent + iZoneCallbackInitSent = EFalse; + if (aZones) + { + CountZones(aZones); + iZonePowerFunc = aZoneCallback; + } + else + {// maximum number of zone is number of non-coalesced boot banks + iNumZones = num_boot_banks; + // No zones specified so don't worry about invoking callback function + iZonePowerFunc = NULL; + } + + // Permenant heap allocation #1 - may be resized if no zones specified + __KTRACE_OPT(KMMU,Kern::Printf("iNumZones=%d", iNumZones)); + iZones = (SZone*)Kern::AllocZ(iNumZones*sizeof(SZone)); + if (!iZones) + { + Panic(ECreateNoMemory); + } + + /////////////////////////////////////////////////////////////////////////// + // Coalesce contiguous boot banks + /////////////////////////////////////////////////////////////////////////// + SRamBank* physBanks = (SRamBank*)Kern::Alloc(num_boot_banks*sizeof(SRamBank)); + if (!physBanks) + { + Panic(ECreateNoMemory); + } + SRamBank* coalescedBank = physBanks; + const SRamBank* const lastBank = aInfo.iBanks + num_boot_banks; + TPhysAddr currentBase = aInfo.iBanks->iBase; + TPhysAddr currentEnd = aInfo.iBanks->iBase + aInfo.iBanks->iSize; + const SRamBank* nextBank = aInfo.iBanks + 1; + for (; nextBank <= lastBank; ++nextBank) + { + // Create new bank if the next bank isn't contiguous or if + // it is the last bank + if (nextBank == lastBank || nextBank->iBase != currentEnd) + { + coalescedBank->iBase = currentBase; + coalescedBank->iSize = currentEnd - currentBase; + // Mark all the SPageInfos for the pages in this bank as unused. + // Needs to be done here to allow SPageInfo::SafeFromPhysAddr to work + // which is used by InitSPageInfos() + SPageInfo* pi = SPageInfo::FromPhysAddr(coalescedBank->iBase); + SPageInfo* piBankEnd = pi + (coalescedBank->iSize >> KPageShift); + for (; pi < piBankEnd; pi++) + { + pi->SetUnused(); + } + ++coalescedBank; + __KTRACE_OPT(KMMU, Kern::Printf("Coalesced bank: %08x-%08x", currentBase, currentEnd)); + currentBase = nextBank->iBase; + currentEnd = currentBase + nextBank->iSize; + } + else + { + currentEnd += nextBank->iSize; + } + } + TUint num_coalesced_banks = coalescedBank - physBanks; + __KTRACE_OPT(KMMU, Kern::Printf("#Coalesced banks: %d", num_coalesced_banks)); + + /////////////////////////////////////////////////////////////////////////// + // Initialise the SZone objects and mark all the SPageInfos with the index + // of zone they are in. + ////////////////////////////////////////////////////////////////////////// + // Assume everything is off so base port will get notification every time the + // a new zone is required during the rest of boot process. + if (aZones != NULL) + { + SZone* newZone = iZones; // pointer to zone being created + + // Create and fill zoneAddrOrder with address ordered indices to aZones + TUint8* zoneAddrOrder = (TUint8*)Kern::Alloc(iNumZones); + if (!zoneAddrOrder) + { + Panic(ECreateNoMemory); + } + SortRamZones(aZones, zoneAddrOrder); + + // Now go through each SRamZone in address order initialising the SZone + // objects. + TUint i = 0; + TUint totalZonePages = 0; + for (; i < iNumZones; i++) + { + const SRamZone& ramZone = *(aZones + zoneAddrOrder[i]); + newZone->iPhysBase = ramZone.iBase; + newZone->iPhysEnd = ramZone.iBase + ramZone.iSize - 1; + newZone->iPhysPages = ramZone.iSize >> KPageShift; + newZone->iAllocPages[EPageUnknown] = newZone->iPhysPages; + newZone->iId = ramZone.iId; + newZone->iPref = ramZone.iPref; + newZone->iFlags = ramZone.iFlags; + totalZonePages += InitSPageInfos(newZone); + newZone++; + } + + // iZones now points to all the SZone objects stored in address order + Kern::Free(zoneAddrOrder); + if (totalZonePages != iTotalRamPages) + {// The zones don't cover all of the allocatable RAM. + Panic(EZonesIncomplete); + } + } + else + { + iNumZones = num_coalesced_banks; + iZones = (SZone*)Kern::ReAlloc((TAny*)iZones, iNumZones*sizeof(SZone)); + if (iZones == NULL) + { + Panic(ECreateNoMemory); + } + // Create a zone for each coalesced boot bank + SRamBank* bank = physBanks; + SRamBank* bankEnd = physBanks + num_coalesced_banks; + SZone* zone = iZones; + for (; bank < bankEnd; bank++, zone++) + { + zone->iPhysBase = bank->iBase; + zone->iPhysEnd = bank->iBase + bank->iSize - 1; + zone->iPhysPages = bank->iSize >> KPageShift; + zone->iAllocPages[EPageUnknown] = zone->iPhysPages; + zone->iId = (TUint)bank; // doesn't matter what it is as long as it is unique + InitSPageInfos(zone); + } + } + // Delete the coalesced banks as no longer required + Kern::Free(physBanks); + + ////////////////////////////////////////////////////////////////////////// + // Create each zones' bit map allocator now as no temporary heap + // cells still allocated at this point. + /////////////////////////////////////////////////////////////////////////// + const SZone* const endZone = iZones + iNumZones; + SZone* zone = iZones; + for (; zone < endZone; zone++) + {// Create each BMA with all pages allocated as unknown. + for (TUint i = 0; i < EPageTypes; i++) + { + // Only mark the all pages bma and fixed/unknown bma as allocated. + TBool notAllocated = (i >= (TUint)EPageMovable); + zone->iBma[i] = TBitMapAllocator::New(zone->iPhysPages, notAllocated); + if (!zone->iBma[i]) + { + Panic(ECreateNoMemory); + } + } + } + + /////////////////////////////////////////////////////////////////////////// + // Unallocate each page in each bank so that it can be allocated when required. + // Any page that exists outside a bank will remain allocated as EPageUnknown + // and will therefore not be touched by the allocator. + ////////////////////////////////////////////////////////////////////////// + // Temporarily fill preference list so SetPhysicalRamState can succeed +#ifdef _DEBUG + // Block bma verificaitons as bma and alloc counts aren't consistent yet. + iAllowBmaVerify = EFalse; +#endif + const SZone* const lastZone = iZones + iNumZones; + zone = iZones; + for (; zone < lastZone; zone++) + { + iZonePrefList.Add(&zone->iPrefLink); + } + const SRamBank* const lastPhysBank = aInfo.iBanks + num_boot_banks; + const SRamBank* bank = aInfo.iBanks; + for (; bank < lastPhysBank; bank++) + {// Free all the pages in this bank. + SetPhysicalRamState(bank->iBase, bank->iSize, ETrue, EPageUnknown); + } +#ifdef _DEBUG + // Only now is it safe to enable bma verifications + iAllowBmaVerify = ETrue; +#endif + + /////////////////////////////////////////////////////////////////////////// + // Sort the zones by preference and create a preference ordered linked list + /////////////////////////////////////////////////////////////////////////// + zone = iZones; + for (; zone < lastZone; zone++) + {// clear all the zones from the preference list as not in preference order + zone->iPrefLink.Deque(); + } + SZone** prefOrder = (SZone**)Kern::AllocZ(iNumZones * sizeof(SZone*)); + if (!prefOrder) + { + Panic(ECreateNoMemory); + } + zone = iZones; + for(; zone < lastZone; zone++) + { + TInt lowerZones = 0; + // Find how many zones that have a lower preference than this one + const SZone* zone2 = iZones; + for (; zone2 < lastZone; zone2++) + { + if (zone->iPref > zone2->iPref || + zone->iPref == zone2->iPref && zone->iFreePages > zone2->iFreePages) + { + lowerZones++; + } + } + while (prefOrder[lowerZones] != 0) + {// Zone(s) of this preference and size already exist so + // place this one after it/them + lowerZones++; + } + prefOrder[lowerZones] = zone; + } + // Fill preference ordered linked list + SZone** const lastPref = prefOrder + iNumZones; + SZone** prefZone = prefOrder; + TUint prefRank = 0; + for (; prefZone < lastPref; prefZone++, prefRank++) + { + SZone& zone = **prefZone; + iZonePrefList.Add(&zone.iPrefLink); + zone.iPrefRank = prefRank; + } + Kern::Free(prefOrder); // Remove temporary allocation + + /////////////////////////////////////////////////////////////////////////// + // Now mark any regions reserved by the base port as allocated and not + // for use by the RAM allocator. + /////////////////////////////////////////////////////////////////////////// + const SRamBank* pB = lastBank + 1; // first reserved block specifier + for (; pB->iSize; ++pB) + { + __KTRACE_OPT(KMMU, Kern::Printf("Reserve physical block %08x+%x", pB->iBase, pB->iSize)); + TInt r = SetPhysicalRamState(pB->iBase, pB->iSize, EFalse, EPageFixed); + __KTRACE_OPT(KMMU, Kern::Printf("Reserve returns %d", r)); + if (r!=KErrNone) + { + Panic(ECreateInvalidReserveBank); + } +#ifdef BTRACE_KERNEL_MEMORY + BTrace8(BTrace::EKernelMemory, BTrace::EKernelMemoryDrvPhysAlloc, pB->iSize, pB->iBase); + Epoc::DriverAllocdPhysRam += pB->iSize; +#endif +#ifndef __MEMMODEL_FLEXIBLE__ // Mmu::Init2Common() handles this in FMM. + // Synchronise the SPageInfo with any blocks that were reserved by + // marking any reserved regions as locked + TPhysAddr physAddrEnd = pB->iBase + pB->iSize; + TPhysAddr physAddr = pB->iBase; + for(; physAddr < physAddrEnd; physAddr += KPageSize) + { + SPageInfo* pi = SPageInfo::FromPhysAddr(physAddr); + pi->Lock(); + } +#endif + } + + ////////////////////////////////////////////////////////////////////////// + // Now that we have have the RAM zone preference list and know how many + // allocatable pages there are, set iZoneLeastMovDis to be the RAM zone + // that will be used when half of the RAM is in use. This a boot up + // optimisation to reduce the amount of moving and/or discarding fixed page + // allocations will have to make during boot. + ////////////////////////////////////////////////////////////////////////// + TUint halfAllocatablePages = iTotalFreeRamPages >> 1; + TUint pages = 0; + SDblQueLink* link = &iZonePrefList.iA; + do + { + link = link->iNext; + __NK_ASSERT_DEBUG(link != &iZonePrefList.iA); + SZone& zonePages = *_LOFF(link, SZone, iPrefLink); + pages += zonePages.iFreePages; + } + while(pages < halfAllocatablePages); + iZoneLeastMovDis = link; + iZoneLeastMovDisRank = _LOFF(link, SZone, iPrefLink)->iPrefRank; + + // Reset general defrag links. + iZoneGeneralPrefLink = NULL; + iZoneGeneralTmpLink = NULL; + + __KTRACE_OPT(KMMU,DebugDump()); + } + + +void DRamAllocator::MarkPagesAllocated(TPhysAddr aAddr, TInt aCount, TZonePageType aType) + { + __KTRACE_OPT(KMMU,Kern::Printf("DRamAllocator::MarkPagesAllocated(%x+%x)",aAddr,aCount)); + + M::RamAllocIsLocked(); + + // Don't allow unknown pages to be allocated, saves extra 'if' when + // creating bmaType. + __NK_ASSERT_DEBUG(aType != EPageUnknown); + + __ASSERT_DEBUG( !(TUint32(aAddr) & (KPageSize - 1)) && + (TUint32(aAddr) < TUint32(iPhysAddrTop)) && + (TUint32(aAddr) >= TUint32(iPhysAddrBase))&& + (TUint32((aCount << KPageShift) -1 + aAddr) <= TUint32(iPhysAddrTop)), + Panic(EDoMarkPagesAllocated1)); + + iTotalFreeRamPages-=aCount; + // Find the 1st zone the 1st set of allocations belong to + TInt offset = 0; + SZone* pZ = GetZoneAndOffset(aAddr,offset); + if (pZ == NULL) + {//aAddr not in RAM + Panic(EDoMarkPagesAllocated1); + } + while(aCount) + { + TBitMapAllocator& bmaAll = *(pZ->iBma[KBmaAllPages]); + TBitMapAllocator& bmaType = *(pZ->iBma[aType]); + TInt count = Min(bmaAll.iSize - offset, aCount); + bmaAll.Alloc(offset, count); + bmaType.Alloc(offset, count); + ZoneAllocPages(pZ, count, aType); + aCount -= count; + + // If spanning zones then ensure the next zone is contiguous. + __ASSERT_DEBUG(!aCount || ((pZ + 1)->iPhysBase != 0 && ((pZ + 1)->iPhysBase - 1) == pZ->iPhysEnd), Panic(EDoMarkPagesAllocated1)); + + pZ++; // zones in physical address order so move to next one + offset = 0; // and reset offset to start of the zone + } + } + +TInt DRamAllocator::MarkPageAllocated(TPhysAddr aAddr, TZonePageType aType) + { + __KTRACE_OPT(KMMU,Kern::Printf("DRamAllocator::MarkPageAllocated %08x",aAddr)); + + M::RamAllocIsLocked(); + + // Don't allow unknown pages to be allocated, saves extra 'if' when + // creating bmaType. + __NK_ASSERT_DEBUG(aType != EPageUnknown); + + TInt n; + SZone* z=GetZoneAndOffset(aAddr,n); + if (!z) + { + return KErrArgument; + } + __KTRACE_OPT(KMMU2,Kern::Printf("Zone index %d page index %04x",z-iZones,n)); + TBitMapAllocator& bmaAll = *(z->iBma[KBmaAllPages]); + TBitMapAllocator& bmaType = *(z->iBma[aType]); + if (bmaAll.NotFree(n,1)) + { + __KTRACE_OPT(KMMU,Kern::Printf("Page already allocated")); + return KErrAlreadyExists; // page is already allocated + } + bmaAll.Alloc(n,1); + bmaType.Alloc(n,1); + --iTotalFreeRamPages; + ZoneAllocPages(z, 1, aType); + __KTRACE_OPT(KMMU,Kern::Printf("Total free RAM pages now = %d",iTotalFreeRamPages)); + +#ifdef BTRACE_RAM_ALLOCATOR + BTrace8(BTrace::ERamAllocator, BTrace::ERamAllocMarkAllocated, aType, aAddr); +#endif + return KErrNone; + } + +TInt DRamAllocator::FreeRamPage(TPhysAddr aAddr, TZonePageType aType) + { + __KTRACE_OPT(KMMU,Kern::Printf("FreeRamPage %08x",aAddr)); + + M::RamAllocIsLocked(); + +#ifdef _DEBUG +#ifndef __MEMMODEL_FLEXIBLE__ + // Check lock counter of the page + if (aAddr != KPhysAddrInvalid) + { + SPageInfo* pi = SPageInfo::SafeFromPhysAddr(aAddr); + if(pi && pi->LockCount()) + Panic(EFreeingLockedPage); + } +#endif + // Don't allow unknown pages to be freed, saves extra 'if' when + // creating bmaType. + __NK_ASSERT_DEBUG(aType != EPageUnknown); +#endif + + TInt n; + SZone* z=GetZoneAndOffset(aAddr,n); + if (!z) + { + return KErrArgument; + } + __KTRACE_OPT(KMMU2,Kern::Printf("Zone index %d page index %04x",z-iZones,n)); + TBitMapAllocator& bmaAll = *(z->iBma[KBmaAllPages]); + TBitMapAllocator& bmaType = *(z->iBma[aType]); + bmaAll.Free(n); + bmaType.Free(n); + ++iTotalFreeRamPages; + ZoneFreePages(z, 1, aType); + +#ifdef BTRACE_RAM_ALLOCATOR + BTrace8(BTrace::ERamAllocator, BTrace::ERamAllocFreePage, aType, aAddr); +#endif + return KErrNone; + } + +void DRamAllocator::FreeRamPages(TPhysAddr* aPageList, TInt aNumPages, TZonePageType aType) + { + __KTRACE_OPT(KMMU,Kern::Printf("FreeRamPages count=%08x",aNumPages)); + + M::RamAllocIsLocked(); + +#if defined(_DEBUG) && !defined(__MEMMODEL_FLEXIBLE__) + // Check lock counter for each page that is about to be freed. + TInt pageNum = aNumPages; + TPhysAddr* pageList = aPageList; + while (pageNum--) + { + TPhysAddr pa = *pageList++; + if (pa == KPhysAddrInvalid) + continue; + SPageInfo* pi = SPageInfo::SafeFromPhysAddr(pa); + if(pi && pi->LockCount()) + Panic(EFreeingLockedPage); + } +#endif + + while(aNumPages--) + { + TPhysAddr first_pa = *aPageList++; + if (first_pa == KPhysAddrInvalid) + { + continue; + } + TInt ix; + SZone* z = GetZoneAndOffset(first_pa,ix); + if (!z) + { + continue; + } + TBitMapAllocator& bmaAll = *(z->iBma[KBmaAllPages]); + TInt zp_rem = bmaAll.iSize - ix; + __KTRACE_OPT(KMMU,Kern::Printf("1st PA=%08x Zone %d index %04x",first_pa,z-iZones,ix)); + TInt n = 1; + TPhysAddr pa = first_pa + KPageSize; + while (--zp_rem && aNumPages && *aPageList==pa) + { + ++n; + --aNumPages; + ++aPageList; + pa += KPageSize; + } + __KTRACE_OPT(KMMU2,Kern::Printf("%d consecutive pages, zp_rem=%x, %d remaining pages",n,zp_rem,aNumPages)); + bmaAll.Free(ix,n); + TBitMapAllocator& bmaType = *(z->iBma[aType]); + bmaType.Free(ix,n); + iTotalFreeRamPages += n; + ZoneFreePages(z, n, aType); +#ifdef BTRACE_RAM_ALLOCATOR + BTrace12(BTrace::ERamAllocator, BTrace::ERamAllocFreePages, aType, n, first_pa); +#endif + } +#ifdef BTRACE_RAM_ALLOCATOR + BTrace0(BTrace::ERamAllocator, BTrace::ERamAllocFreePagesEnd); +#endif + } + +/** + Attempt to clear upto the required amount of discardable or movable pages + from the RAM zone. + + @param aZone The RAM zone to clear. + @param aRequiredPages The maximum number of pages to clear. +*/ +void DRamAllocator::ZoneClearPages(SZone& aZone, TUint aRequiredPages) + { + __KTRACE_OPT(KMMU, + Kern::Printf("ZoneClearPages: ID 0x%x, req 0x%x", aZone.iId, aRequiredPages)); + // Discard the required number of discardable pages. + TUint offset = 0; + TInt r = NextAllocatedPage(&aZone, offset, EPageDiscard); + while (r == KErrNone && aRequiredPages) + { + TPhysAddr physAddr = (offset << KPageShift) + aZone.iPhysBase; + TInt discarded = M::DiscardPage(physAddr, aZone.iId, EFalse); + if (discarded == KErrNone) + {// The page was successfully discarded. + aRequiredPages--; + } + offset++; + r = NextAllocatedPage(&aZone, offset, EPageDiscard); + } + // Move the required number of movable pages. + offset = 0; + r = NextAllocatedPage(&aZone, offset, EPageMovable); + while(r == KErrNone && aRequiredPages) + { + TPhysAddr physAddr = (offset << KPageShift) + aZone.iPhysBase; + TPhysAddr newAddr = KPhysAddrInvalid; + if (M::MovePage(physAddr, newAddr, aZone.iId, EFalse) == KErrNone) + {// The page was successfully moved. +#ifdef _DEBUG + TInt newOffset = 0; + SZone* newZone = GetZoneAndOffset(newAddr, newOffset); + __NK_ASSERT_DEBUG(newZone != &aZone); +#endif + aRequiredPages--; + } + offset++; + r = NextAllocatedPage(&aZone, offset, EPageMovable); + } + } + +/** Attempt to allocate pages into a particular zone. Pages will not + always be contiguous. + + @param aPageList On return it will contain the addresses of any allocated pages + @param aZone The zone to allocate from + @param aNumPages The number of pages to allocate + @param aType The type of pages to allocate + @return The number of pages that were allocated +*/ +TUint32 DRamAllocator::ZoneFindPages(TPhysAddr*& aPageList, SZone& aZone, TUint32 aNumPages, TZonePageType aType) + { + // Don't allow unknown pages to be allocated, saves extra 'if' when + // creating bmaType. + __NK_ASSERT_DEBUG(aType != EPageUnknown); + + TBitMapAllocator& bmaAll = *aZone.iBma[KBmaAllPages]; + TBitMapAllocator& bmaType = *(aZone.iBma[aType]); + TPhysAddr zpb = aZone.iPhysBase; + TInt got = bmaAll.AllocList(aNumPages, (TInt*)aPageList); + if (got) + { + TPhysAddr* pE = aPageList + got; + while(aPageList < pE) + { + TInt ix = *aPageList; + *aPageList++ = zpb + (ix << KPageShift); + __KTRACE_OPT(KMMU,Kern::Printf("Got page @%08x",zpb + (ix << KPageShift))); + + // Mark the page allocated on the page type bit map. + bmaType.Alloc(ix, 1); + } + ZoneAllocPages(&aZone, got, aType); +#ifdef BTRACE_RAM_ALLOCATOR + BTrace12(BTrace::ERamAllocator, BTrace::ERamAllocRamPages, aType, got, *(pE-got)); +#endif + } + return got; + } + +/** +Allocate discontiguous pages. + +Fixed pages are always allocated into the most preferable RAM zone that has free, +movable or discardable pages in it. This is to avoid fixed pages being placed +in the less preferred RAM zones. + +Movable and discardable pages are allocated into the RAM zones currently in use. +An empty RAM zone will only be used (switched on) if there are not enough free +pages in the in use RAM zones. The pages will be allocated from the least +preferable RAM to be in use after the allocation to the more preferred RAM zones. + +If a valid zone is specified in aBlockedZoneId then that RAM zone will not be +allocated into. Also, if aBlockedZoneId and aBlockRest is set then the allocation +will stop if aBlockZoneId + +@param aPageList On success, will contain the address of each allocated page +@param aNumPages The number of the pages to allocate +@param aType The type of the pages to allocate +@param aBlockedZoneId The ID of the RAM zone that shouldn't be allocated into. + The default value has no effect. +@param aBlockRest Set to ETrue to stop this allocation using any currently empty + RAM zones, EFalse to allow empty RAM zones to be used. Only + effects movable and discardable allocations. + +@return 0 on success, the number of extra pages required to fulfill the request on failure. +*/ +TInt DRamAllocator::AllocRamPages(TPhysAddr* aPageList, TInt aNumPages, TZonePageType aType, TUint aBlockedZoneId, TBool aBlockRest) + { + __KTRACE_OPT(KMMU,Kern::Printf("AllocRamPages 0x%x type%d",aNumPages, aType)); + + M::RamAllocIsLocked(); + + // Should never allocate unknown pages. + __NK_ASSERT_DEBUG(aType != EPageUnknown); + + TPhysAddr* pageListBase = aPageList; + TUint32 numMissing = aNumPages; + + if (aType == EPageFixed) + {// Currently only a general defrag operation should set this and it won't + // allocate fixed pages. + __NK_ASSERT_DEBUG(!aBlockRest); + if ((TUint)aNumPages > iTotalFreeRamPages + M::NumberOfFreeDpPages()) + {// Not enough free space and not enough freeable pages. + goto exit; + } + + // Search through each zone in preference order until all pages allocated or + // have reached the end of the preference list + SDblQueLink* link = iZonePrefList.First(); + while (numMissing && link != &iZonePrefList.iA) + { + SZone& zone = *_LOFF(link, SZone, iPrefLink); + // Get the link to next zone before any potential reordering. + // Which would occur if previous zone is same preference and has + // more free space after this allocation. + link = link->iNext; + + if (zone.iId == aBlockedZoneId || NoAllocOfPageType(zone, aType)) + {// The flags disallow aType pages or all pages. + __KTRACE_OPT(KMMU2, Kern::Printf("ARP Flags 0x%08x", zone.iFlags)); + continue; + } + + numMissing -= ZoneFindPages(aPageList, zone, numMissing, aType); + __KTRACE_OPT(KMMU, Kern::Printf("zone.iId 0x%x", zone.iId)); + + if (numMissing && + (zone.iAllocPages[EPageMovable] || zone.iAllocPages[EPageDiscard])) + {// Not all the required pages where allocated and there are still some + // movable and discardable pages in this RAM zone. + ZoneClearPages(zone, numMissing); + + // Have discarded and moved everything required or possible so + // now allocate into the pages just freed. + numMissing -= ZoneFindPages(aPageList, zone, numMissing, aType); + } + } + } + else + { + if ((TUint)aNumPages > iTotalFreeRamPages) + {// Not enough free pages to fulfill this request so return amount required + return aNumPages - iTotalFreeRamPages; + } + + // Determine if there are enough free pages in the RAM zones in use. + TUint totalFreeInUse = 0; + SDblQueLink* link = iZoneLeastMovDis; + for(; link != &iZonePrefList.iA; link = link->iPrev) + { + SZone& zone = *_LOFF(link, SZone, iPrefLink); + if (zone.iId == aBlockedZoneId || NoAllocOfPageType(zone, aType) || + (aBlockRest && (zone.iFlags & KRamZoneFlagGenDefragBlock))) + {// The blocked RAM zone or flags disallow aType pages or all pages + __KTRACE_OPT(KMMU2, Kern::Printf("ARP Flags 0x%08x", zone.iFlags)); + continue; + } + totalFreeInUse += zone.iFreePages; + } + + if (aBlockRest && totalFreeInUse < (TUint)aNumPages) + {// Allocating as part of a general defragmentation and + // can't allocate without using a RAM zone less preferable than + // the current least prefeable RAM zone with movable and/or + //discardable. + __NK_ASSERT_DEBUG(numMissing); + goto exit; + } + + SDblQueLink* leastClearable = iZoneLeastMovDis; + while (totalFreeInUse < (TUint)aNumPages) + {// The amount of free pages in the RAM zones with movable + // and/or discardable isn't enough. + leastClearable = leastClearable->iNext; + if (leastClearable == &iZonePrefList.iA) + {// There are no more RAM zones to allocate into. + __NK_ASSERT_DEBUG(numMissing); + goto exit; + } + SZone& zone = *_LOFF(leastClearable, SZone, iPrefLink); + if (zone.iId == aBlockedZoneId || NoAllocOfPageType(zone, aType)) + {// The flags disallow aType pages or all pages + __KTRACE_OPT(KMMU2, Kern::Printf("ARP Flags 0x%08x", zone.iFlags)); + continue; + } + totalFreeInUse += zone.iFreePages; + } + // Now that we know exactly how many RAM zones will be required do + // the allocation. To reduce fixed allocations having to clear RAM + // zones, allocate from the least preferable RAM to be used + // to the most preferable RAM zone. + link = leastClearable; + while (numMissing) + { + __NK_ASSERT_DEBUG(link != &iZonePrefList.iA); + SZone& zone = *_LOFF(link, SZone, iPrefLink); + // Update the link before any reordering so we don't miss a RAM zone. + link = link->iPrev; + + if (zone.iId == aBlockedZoneId || NoAllocOfPageType(zone, aType) || + (aBlockRest && (zone.iFlags & KRamZoneFlagGenDefragBlock))) + {// The blocked RAM zone or flags disallow aType pages or all pages + __KTRACE_OPT(KMMU2, Kern::Printf("ARP Flags 0x%08x", zone.iFlags)); + continue; + } + + numMissing -= ZoneFindPages(aPageList, zone, numMissing, aType); + __KTRACE_OPT(KMMU, Kern::Printf("zone.iId 0x%x", zone.iId)); + } + __NK_ASSERT_DEBUG(!numMissing); + } + +exit: + // Update here so any call to FreeRamPages doesn't upset count + aNumPages -= numMissing; //set to number of pages that are allocated + iTotalFreeRamPages -= aNumPages; + + if (numMissing) + {// Couldn't allocate all required pages so free those that were allocated + FreeRamPages(pageListBase, aNumPages, aType); + } +#ifdef BTRACE_RAM_ALLOCATOR + else + { + BTrace0(BTrace::ERamAllocator, BTrace::ERamAllocRamPagesEnd); + } +#endif + return numMissing; + } + + +/** +Attempt to allocate discontiguous pages from the specified RAM zone. + +NOTE - This method only obeys the KRamZoneFlagNoAlloc and KRamZoneFlagClaiming +flags and not the others. +But as currently only EFixed pages will be allocated using this method that is +the desired behaviour. + +@param aZoneIdList An array of the IDs of the RAM zones to allocate from. +@param aZoneIdCount The number of IDs in aZoneIdList. +@param aPageList On success, will contain the address of each allocated page. +@param aNumPages The number of the pages to allocate. +@param aType The type of the pages to allocate. + +@return KErrNone on success, KErrNoMemory if allocation couldn't succeed or +the RAM zone has the KRamZoneFlagNoAlloc flag set, KErrArgument if a zone of +aZoneIdList doesn't exist or aNumPages is greater than the total pages in the zone. +*/ +TInt DRamAllocator::ZoneAllocRamPages(TUint* aZoneIdList, TUint aZoneIdCount, TPhysAddr* aPageList, TInt aNumPages, TZonePageType aType) + { + M::RamAllocIsLocked(); + __NK_ASSERT_DEBUG(aType == EPageFixed); + + + __KTRACE_OPT(KMMU,Kern::Printf("ZoneAllocRamPages 0x%x zones 0x%x",aNumPages, aZoneIdCount)); + + TInt r = KErrNone; + TUint* zoneIdPtr = aZoneIdList; + TUint* zoneIdEnd = zoneIdPtr + aZoneIdCount; + TUint numMissing = aNumPages; + TUint physicalPages = 0; + TPhysAddr* pageListBase = aPageList; + + // Always loop through all the RAM zones so that if an invalid ID is specified + // it is always detected whether all the specified RAM zones were required + // for the allocation or not. + for(; zoneIdPtr < zoneIdEnd; zoneIdPtr++) + { + SZone* zone = ZoneFromId(*zoneIdPtr); + + if (zone == NULL) + {// Invalid zone ID. + r = KErrArgument; + break; + } + + physicalPages += zone->iPhysPages; + + if (zone->iFlags & (KRamZoneFlagNoAlloc|KRamZoneFlagClaiming)) + {// If this RAM zone can't be allocated into then skip it. + continue; + } + + numMissing -= ZoneFindPages(aPageList, *zone, numMissing, aType); + + if (numMissing && aType == EPageFixed) + {// Remove up to required number of pages from the RAM zone + // and reattempt the allocation. + ZoneClearPages(*zone, numMissing); + numMissing -= ZoneFindPages(aPageList, *zone, numMissing, aType); + } + } + + // Update iTotalFreeRamPages here so that if allocation doesn't succeed then + // FreeRamPages() will keep it consistent. + TUint numAllocated = aNumPages - numMissing; + iTotalFreeRamPages -= numAllocated; + + if (r == KErrArgument || physicalPages < (TUint)aNumPages) + {// Invalid zone ID or the number of pages requested is too large. + // This should fail regardless of whether the allocation failed or not. + FreeRamPages(pageListBase, numAllocated, aType); + return KErrArgument; + } + + if (numMissing) + {// Couldn't allocate all required pages so free those that were allocated + FreeRamPages(pageListBase, numAllocated, aType); + return KErrNoMemory; + } + + // Have allocated all the required pages. +#ifdef BTRACE_RAM_ALLOCATOR + BTrace0(BTrace::ERamAllocator, BTrace::ERamAllocZoneRamPagesEnd); +#endif + return KErrNone; + } + + +/** +Will return zones one at a time in the following search patterns until a suitable +zone has been found or it is determined that there is no suitable zone: + - preference order + - address order +Before the first call for a new search sequence must set: + iZoneTmpAddrIndex = -1; + iZoneTmpPrefLink = iZonePrefList.First(); + +@param aZone On return this will be a pointer to the next zone to search. +@param aState The current search state, i.e. which of the zone orderings to follow. +It will be updated if necessary by this function. +@param aType The type of page to be allocated. +@param aBlockedZoneId The ID of a RAM zone to not allocate into. +@param aBlockRest ETrue if allocation should fail as soon as a blocked zone is reached, +EFalse otherwise. (Currently not used) +@return ETrue a sutiable zone is found, EFalse when the allocation is not possible. +*/ +TBool DRamAllocator::NextAllocZone(SZone*& aZone, TZoneSearchState& aState, TZonePageType aType, TUint aBlockedZoneId, TBool aBlockRest) + { + TUint currentState = aState; + TBool r = EFalse; + + for (; currentState < EZoneSearchEnd; currentState++) + { + if (currentState == EZoneSearchAddr) + { + iZoneTmpAddrIndex++; + for (; iZoneTmpAddrIndex < (TInt)iNumZones; iZoneTmpAddrIndex++) + { + aZone = iZones + iZoneTmpAddrIndex; + if (aBlockedZoneId != aZone->iId && !NoAllocOfPageType(*aZone, aType)) + { + r = ETrue; + goto exit; + } + } + } + else + { + while(iZoneTmpPrefLink != &iZonePrefList.iA) + { + aZone = _LOFF(iZoneTmpPrefLink, SZone, iPrefLink); + iZoneTmpPrefLink = iZoneTmpPrefLink->iNext; // Update before any re-ordering + if (aBlockedZoneId != aZone->iId && !NoAllocOfPageType(*aZone, aType)) + { + r = ETrue; + goto exit; + } + } + } + } +exit: + __NK_ASSERT_DEBUG((r && currentState < EZoneSearchEnd) || (!r && currentState == EZoneSearchEnd)); + + aState = (TZoneSearchState)currentState; + return r; + } + +/** +Search through the zones for the requested contiguous RAM, first in preference +order then, if that fails, in address order. + +@param aNumPages The number of contiguous pages to find +@param aPhysAddr Will contain the base address of any contiguous run if found +@param aType The page type of the memory to be allocated +@param aAlign Alignment specified as the alignment shift +@param aBlockedZoneId The ID of a zone that can't be allocated into, by default this has no effect +@param aBlockRest Set to ETrue to stop allocation as soon as aBlockedZoneId is reached +in preference ordering. EFalse otherwise. + +@return KErrNone on success, KErrNoMemory otherwise +*/ +TInt DRamAllocator::AllocContiguousRam(TUint aNumPages, TPhysAddr& aPhysAddr, TZonePageType aType, TInt aAlign, TUint aBlockedZoneId, TBool aBlockRest) + { + __KTRACE_OPT(KMMU,Kern::Printf("AllocContiguousRam size %08x align %d",aNumPages,aAlign)); + + M::RamAllocIsLocked(); + + // No support for non-fixed pages as this will discard and move + // pages if required. + __NK_ASSERT_DEBUG(aType == EPageFixed); + TInt alignWrtPage = Max(aAlign - KPageShift, 0); + TUint32 alignmask = (1u << alignWrtPage) - 1; + + // Attempt to find enough pages searching in preference order first then + // in address order + TZoneSearchState searchState = EZoneSearchPref; + SZone* zone; + SZone* prevZone = NULL; + TInt carryAll = 0; // Carry for all pages bma, clear to start new run. + TInt carryImmov = 0; // Carry for immovable pages bma, clear to start new run. + TInt base = 0; + TInt offset = 0; + iZoneTmpAddrIndex = -1; + iZoneTmpPrefLink = iZonePrefList.First(); + while (NextAllocZone(zone, searchState, aType, aBlockedZoneId, aBlockRest)) + { + // Be sure to start from scratch if zone not contiguous with previous zone + if (prevZone && (zone->iPhysBase == 0 || (zone->iPhysBase - 1) != prevZone->iPhysEnd)) + { + carryAll = 0; + carryImmov = 0; + } + prevZone = zone; + TBitMapAllocator& bmaAll = *(zone->iBma[KBmaAllPages]); + base = TInt(zone->iPhysBase >> KPageShift); + TInt runLength; + __KTRACE_OPT(KMMU,Kern::Printf("AllocAligned: base=%08x carryAll=%08x offset=%08x", base, carryAll, offset)); + offset = bmaAll.AllocAligned(aNumPages, alignWrtPage, base, EFalse, carryAll, runLength); + __KTRACE_OPT(KMMU,Kern::Printf("AllocAligned: offset=%08x", offset)); + + if (offset >= 0) + {// Have found enough contiguous pages so return address of physical page + // at the start of the region + aPhysAddr = TPhysAddr((base + offset - carryAll + alignmask) & ~alignmask) << KPageShift; + MarkPagesAllocated(aPhysAddr, aNumPages, aType); + + __KTRACE_OPT(KMMU,Kern::Printf("AllocContiguousRam returns %08x",aPhysAddr)); +#ifdef BTRACE_RAM_ALLOCATOR + BTrace12(BTrace::ERamAllocator, BTrace::ERamAllocContiguousRam, aType, aNumPages, aPhysAddr); +#endif + return KErrNone; + } + else + {// No run found when looking in just the free pages so see if this + // RAM zone could be used if pages where moved or discarded. + if (aNumPages > KMaxFreeableContiguousPages) + {// Can't move or discard any pages so move on to next RAM zone + // taking any run at the end of this RAM zone into account. + carryImmov = 0; + continue; + } + TBitMapAllocator& bmaImmov = *(zone->iBma[EPageFixed]); + offset = 0; // Clear so searches whole of fixed BMA on the first pass. + do + { + __KTRACE_OPT(KMMU,Kern::Printf("AllocAligned: base=%08x carryImmov=%08x offset=%08x", base, carryImmov, offset)); + offset = bmaImmov.AllocAligned(aNumPages, alignWrtPage, base, EFalse, carryImmov, runLength, offset); + __KTRACE_OPT(KMMU,Kern::Printf("AllocAligned: offset=%08x", offset)); + if (offset >= 0) + {// Have found a run in immovable page bma so attempt to clear + // it for the allocation. + TPhysAddr addrBase = TPhysAddr((base + offset - carryImmov + alignmask) & ~alignmask) << KPageShift; + TPhysAddr addrEnd = addrBase + (aNumPages << KPageShift); + + // Block the RAM zones containing the contiguous region + // from being allocated into when pages are moved or replaced. + TPhysAddr addr = addrBase; + TInt tmpOffset; + SZone* tmpZone = GetZoneAndOffset(addr, tmpOffset); + while (addr < addrEnd-1) + { + tmpZone->iFlags |= KRamZoneFlagTmpBlockAlloc; + addr = tmpZone->iPhysEnd; + tmpZone++; + } + + addr = addrBase; + TInt contigOffset = 0; + SZone* contigZone = GetZoneAndOffset(addr, contigOffset); + for (; addr != addrEnd; addr += KPageSize, contigOffset++) + { + if (contigZone->iPhysEnd < addr) + { + contigZone = GetZoneAndOffset(addr, contigOffset); + __NK_ASSERT_DEBUG(contigZone != NULL); + } +#ifdef _DEBUG // This page shouldn't be allocated as fixed, only movable or discardable. + __NK_ASSERT_DEBUG(contigZone != NULL); + __NK_ASSERT_DEBUG(contigZone->iBma[EPageFixed]->NotAllocated(contigOffset, 1)); + SPageInfo* pageInfo = SPageInfo::SafeFromPhysAddr(addr); + __NK_ASSERT_DEBUG(pageInfo != NULL); +#endif + TPhysAddr newAddr; + TInt moveRet = M::MovePage(addr, newAddr, contigZone->iId, EFalse); + if (moveRet != KErrNone && moveRet != KErrNotFound) + {// This page couldn't be moved or discarded so + // restart the search the page after this one. + __KTRACE_OPT(KMMU2, + Kern::Printf("ContigMov fail offset %x moveRet %d addr %x carryImmov %x", + offset, moveRet, addr, carryImmov)); + // Can't rely on RAM zone preference ordering being + // the same so clear carrys and restart search from + // within the current RAM zone or skip onto the next + // one if at the end of this one. + carryImmov = 0; + carryAll = 0; + offset = (addr < zone->iPhysBase)? 0 : contigOffset + 1; + __KTRACE_OPT(KMMU2, Kern::Printf("ContigMov fail offset %x", offset)); + break; + } + } + // Unblock the RAM zones containing the contiguous region. + TPhysAddr flagAddr = addrBase; + tmpZone = GetZoneAndOffset(flagAddr, tmpOffset); + while (flagAddr < addrEnd-1) + { + tmpZone->iFlags &= ~KRamZoneFlagTmpBlockAlloc; + flagAddr = tmpZone->iPhysEnd; + tmpZone++; + } + + if (addr == addrEnd) + {// Cleared all the required pages so allocate them. + // Return address of physical page at the start of the region. + aPhysAddr = addrBase; + MarkPagesAllocated(aPhysAddr, aNumPages, aType); + + __KTRACE_OPT(KMMU,Kern::Printf("AllocContiguousRam returns %08x",aPhysAddr)); +#ifdef BTRACE_RAM_ALLOCATOR + BTrace12(BTrace::ERamAllocator, BTrace::ERamAllocContiguousRam, aType, aNumPages, aPhysAddr); +#endif + return KErrNone; + } + } + } + // Keep searching immovable page bma of the current RAM zone until + // gone past end of RAM zone or no run can be found. + while (offset >= 0 && (TUint)offset < zone->iPhysPages); + } + } + return KErrNoMemory; + } + + +/** +Attempt to allocate the contiguous RAM from the specified zone. + +NOTE - This method only obeys the KRamZoneFlagNoAlloc and KRamZoneFlagClaiming +flags and not the others. +But as currently only EFixed pages will be allocated using this method that is +the desired behaviour. + +@param aZoneIdList An array of the IDs of the RAM zones to allocate from. +@param aZoneIdCount The number of the IDs listed by aZoneIdList. +@param aSize The number of contiguous bytes to find +@param aPhysAddr Will contain the base address of the contiguous run if found +@param aType The page type of the memory to be allocated +@param aAlign Alignment specified as the alignment shift + +@return KErrNone on success, KErrNoMemory if allocation couldn't succeed or +the RAM zone has the KRamZoneFlagNoAlloc flag set. KErrArgument if a zone of +aZoneIdList exists or if aSize is larger than the size of the zone. +*/ +TInt DRamAllocator::ZoneAllocContiguousRam(TUint* aZoneIdList, TUint aZoneIdCount, TInt aSize, TPhysAddr& aPhysAddr, TZonePageType aType, TInt aAlign) + { + __KTRACE_OPT(KMMU,Kern::Printf("ZoneAllocContiguousRam zones 0x%x size 0x%08x align %d",aZoneIdCount, aSize, aAlign)); + + M::RamAllocIsLocked(); + __NK_ASSERT_DEBUG(aType == EPageFixed); + + + TUint numPages = (aSize + KPageSize - 1) >> KPageShift; + TInt carry = 0; // must be zero as this is always the start of a new run + TInt alignWrtPage = Max(aAlign - KPageShift, 0); + TUint32 alignmask = (1u << alignWrtPage) - 1; + TInt offset = -1; + TInt base = 0; + + TUint physPages = 0; + TUint* zoneIdPtr = aZoneIdList; + TUint* zoneIdEnd = aZoneIdList + aZoneIdCount; + SZone* prevZone = NULL; + for (; zoneIdPtr < zoneIdEnd; zoneIdPtr++) + { + SZone* zone = ZoneFromId(*zoneIdPtr); + if (zone == NULL) + {// Couldn't find zone of this ID or it isn't large enough + return KErrArgument; + } + physPages += zone->iPhysPages; + + if (offset >= 0 || + (zone->iFlags & (KRamZoneFlagNoAlloc|KRamZoneFlagClaiming))) + {// Keep searching through the RAM zones if the allocation + // has succeeded, to ensure the ID list is always fully verified or + // if this zone is currently blocked for further allocations. + continue; + } + + // Be sure to start from scratch if zone not contiguous with previous zone + if (prevZone && (zone->iPhysBase == 0 || (zone->iPhysBase - 1) != prevZone->iPhysEnd)) + { + carry = 0; + } + prevZone = zone; + + TInt len; + TBitMapAllocator& bmaAll = *(zone->iBma[KBmaAllPages]); + base = TInt(zone->iPhysBase >> KPageShift); + + __KTRACE_OPT(KMMU,Kern::Printf("AllocAligned: aBase=%08x aCarry=%08x", base, carry)); + offset = bmaAll.AllocAligned(numPages, alignWrtPage, base, EFalse, carry, len); + __KTRACE_OPT(KMMU,Kern::Printf("AllocAligned: offset=%08x", offset)); + } + + if (physPages < numPages) + {// The allocation requested is too large for the specified RAM zones. + return KErrArgument; + } + + if (offset < 0) + {// The allocation failed. + return KErrNoMemory; + } + + // Have found enough contiguous pages so mark the pages allocated and + // return address of physical page at the start of the region. + aPhysAddr = TPhysAddr((base + offset - carry + alignmask) & ~alignmask) << KPageShift; + MarkPagesAllocated(aPhysAddr, numPages, aType); + + __KTRACE_OPT(KMMU,Kern::Printf("ZoneAllocContiguousRam returns %08x",aPhysAddr)); +#ifdef BTRACE_RAM_ALLOCATOR + BTrace12(BTrace::ERamAllocator, BTrace::ERamAllocZoneContiguousRam, aType, numPages, aPhysAddr); +#endif + return KErrNone; + } + + +/** +Attempt to set the specified contiguous block of RAM pages to be either +allocated or free. + +@param aBase The base address of the RAM to update. +@param aSize The number of contiguous bytes of RAM to update. +@param aState Set to ETrue to free the RAM, EFalse to allocate the RAM. +@param aType The type of the pages being updated. + +@return KErrNone on success, KErrArgument if aBase is an invalid address, +KErrGeneral if a page being marked free is already free, +KErrInUse if the page being marked allocated is already allocated. +*/ +TInt DRamAllocator::SetPhysicalRamState(TPhysAddr aBase, TInt aSize, TBool aState, TZonePageType aType) + { + M::RamAllocIsLocked(); + + __KTRACE_OPT(KMMU,Kern::Printf("SetPhysicalRamState(%08x,%x,%d)",aBase,aSize,aState?1:0)); + TUint32 pageMask = KPageSize-1; + aSize += (aBase & pageMask); + aBase &= ~pageMask; + TInt npages = (aSize + pageMask) >> KPageShift; + __KTRACE_OPT(KMMU,Kern::Printf("Rounded base %08x npages=%x",aBase,npages)); + TInt baseOffset; + SZone* baseZone = GetZoneAndOffset(aBase, baseOffset); + if (!baseZone || (TUint32)aSize > (iPhysAddrTop - aBase + 1)) + { + return KErrArgument; + } + SZone* zone = baseZone; + SZone* zoneEnd = iZones + iNumZones; + TPhysAddr base = aBase; + TInt pagesLeft = npages; + TInt offset = baseOffset; + TInt pageCount = -1; + __KTRACE_OPT(KMMU2,Kern::Printf("Zone %x page index %x z=%08x zE=%08x n=%x base=%08x",zone->iId, offset, zone, zoneEnd, pagesLeft, base)); + for (; pagesLeft && zone < zoneEnd; ++zone) + { + if (zone->iPhysBase + (offset << KPageShift) != base) + {// Zone not contiguous with current run of page, so have been + // asked to set the state of non-existent pages. + return KErrArgument; + } + + TBitMapAllocator& bmaAll = *(zone->iBma[KBmaAllPages]); + TInt zp_rem = bmaAll.iSize - offset; + pageCount = Min(pagesLeft, zp_rem); + __KTRACE_OPT(KMMU2,Kern::Printf("Zone %x pages %x+%x base %08x", zone->iId, offset, pageCount, base)); + if(aState) + { + if(bmaAll.NotAllocated(offset, pageCount)) + { + return KErrGeneral; + } + } + else + { + if(bmaAll.NotFree(offset, pageCount)) + { + return KErrInUse; + } + } + pagesLeft -= pageCount; + offset = 0; + base += (TPhysAddr(pageCount) << KPageShift); + } + if (pagesLeft) + { + return KErrArgument; // not all of the specified range exists + } + + iTotalFreeRamPages += (aState ? npages : -npages); + zone = baseZone; + offset = baseOffset; + for (pagesLeft = npages; pagesLeft; pagesLeft -= pageCount) + { + TBitMapAllocator& bmaAll = *(zone->iBma[KBmaAllPages]); + // Unknown and fixed pages share a bit map. + TBitMapAllocator& bmaType = *(zone->iBma[(aType != EPageUnknown)? aType : EPageFixed]); + TInt zp_rem = bmaAll.iSize - offset; + pageCount = Min(pagesLeft, zp_rem); + if (aState) + { + bmaAll.Free(offset, pageCount); + bmaType.Free(offset, pageCount); + ZoneFreePages(zone, pageCount, aType); + } + else + { + bmaAll.Alloc(offset, pageCount); + bmaType.Alloc(offset, pageCount); + ZoneAllocPages(zone, pageCount, aType); + } + __KTRACE_OPT(KMMU2,Kern::Printf("Zone %d pages %x+%x base %08x",zone-iZones, offset, pageCount, base)); + ++zone; + offset = 0; + } + return KErrNone; + } + +/** Update the allocated page counts for the zone that is page is allocated into. + +@param aAddr The physical address of the page +@param aOldPageType The type the page was allocated as +@param aNewPageType The type the page is changing to +*/ +void DRamAllocator::ChangePageType(SPageInfo* aPageInfo, TZonePageType aOldType, TZonePageType aNewType) + { + + TInt offset; + SZone* zone = GetZoneAndOffset(aPageInfo->PhysAddr(), offset); +#ifdef _DEBUG +// *********** System lock may be held while this is invoked so don't do******** +// *********** anything too slow and definitely don't call zone callback******** + M::RamAllocIsLocked(); + CHECK_PRECONDITIONS((MASK_THREAD_CRITICAL) & ~MASK_NO_FAST_MUTEX, "DRamAllocator::ChangePageType"); + + // Get zone page is in and on debug builds check that it is allocated + if (zone == NULL || zone->iBma[KBmaAllPages]->NotAllocated(offset, 1)) + { + Panic(EAllocRamPagesInconsistent); + } + + // Check if adjusting counts is valid, i.e. won't cause a roll over + if (zone->iAllocPages[aOldType] - 1 > zone->iAllocPages[aOldType] || + zone->iAllocPages[aNewType] + 1 < zone->iAllocPages[aNewType]) + { + __KTRACE_OPT(KMMU, Kern::Printf("ChangePageType Alloc Unk %x Fx %x Mv %x Dis %x",zone->iAllocPages[EPageUnknown], + zone->iAllocPages[EPageFixed], zone->iAllocPages[EPageMovable],zone->iAllocPages[EPageDiscard])); + Panic(EZonesCountErr); + } +#endif + + // Update the counts and bmas + zone->iAllocPages[aOldType]--; + zone->iBma[aOldType]->Free(offset); + zone->iAllocPages[aNewType]++; + zone->iBma[aNewType]->Alloc(offset, 1); + + __KTRACE_OPT(KMMU2, Kern::Printf("ChangePageType Alloc Unk %x Fx %x Mv %x Dis %x",zone->iAllocPages[EPageUnknown], + zone->iAllocPages[EPageFixed], zone->iAllocPages[EPageMovable],zone->iAllocPages[EPageDiscard])); +#ifdef BTRACE_RAM_ALLOCATOR + BTrace8(BTrace::ERamAllocator, BTrace::ERamAllocChangePageType, aNewType, aPageInfo->PhysAddr()); +#endif + } + +/** +Get the next page in this zone that is allocated after this one. + +@param aZone The zone to find the next allocated page in. +@param aOffset On entry this is the offset from which the next allocated + page in the zone should be found, on return it will be the offset + of the next allocated page. +@return KErrNone if a next allocated page could be found, KErrNotFound if no more pages in +the zone after aOffset are allocated, KErrArgument if aOffset is outside the zone. +*/ +TInt DRamAllocator::NextAllocatedPage(SZone* aZone, TUint& aOffset, TZonePageType aType) const + { + const TUint KWordAlignMask = KMaxTUint32 << 5; + + M::RamAllocIsLocked(); + + __NK_ASSERT_DEBUG(aZone - iZones < (TInt)iNumZones); + // Makes things simpler for bma selection. + __NK_ASSERT_DEBUG(aType != EPageUnknown); + + if (aOffset >= aZone->iPhysPages) + {// Starting point is outside the zone + return KErrArgument; + } + + TUint offset = aOffset; + TUint endOffset = aZone->iPhysPages; + TUint endOffsetAligned = endOffset & KWordAlignMask; + + // Select the BMA to search, + TUint bmaIndex = (aType == EPageTypes)? KBmaAllPages : aType; + TUint32* map = &(aZone->iBma[bmaIndex]->iMap[offset >> 5]); + TUint32 bits = *map++; + + // Set bits for pages before 'offset' (i.e. ones we want to ignore)... + bits |= ~(KMaxTUint32 >> (offset & ~KWordAlignMask)); + + // Find the first bit map word from aOffset in aZone with allocated pages + while (bits == KMaxTUint32 && offset < endOffsetAligned) + { + bits = *map++; + offset = (offset + 32) & KWordAlignMask; + } + + if (offset >= endOffsetAligned && endOffset != endOffsetAligned) + {// Have reached the last bit mask word so set the bits that are + // outside of the zone so that they are ignored. + bits |= KMaxTUint32 >> (endOffset - endOffsetAligned); + } + + if (bits == KMaxTUint32) + {// No allocated pages found after aOffset in aZone. + return KErrNotFound; + } + + // Now we have bits with allocated pages in it so determine the exact + // offset of the next allocated page + TUint32 mask = 0x80000000 >> (offset & ~KWordAlignMask); + while (bits & mask) + { + mask >>= 1; + offset++; + } + + if (offset >= endOffset) + {// Reached the end of the zone without finding an allocated page after aOffset + return KErrNotFound; + } + + // Should definitely have found an allocated page within aZone's pages + __NK_ASSERT_DEBUG(mask != 0 && !(bits & mask) && offset < aZone->iPhysPages); + + aOffset = offset; + return KErrNone; + } + +/** +See if any of the least preferable RAM zones can be emptied. If they can then +initialise the allocator for a general defragmentation operation. + +Stage 0 of the general defrag is to ensure that there are enough free +pages in the more preferable RAM zones to be in use after the general defrag +for the movable page allocations. This is achieved by discarding the +required amount of discardable pages from the more preferable RAM zones +to be in use after the general defrag. + + +@parm aInitialStage On return this will contain the stage the general + defragmentation should begin at. I.e. if no RAM + zones can be cleared then just perform the final + tidying stage. +@param aRequiredToBeDiscarded On return this will contain the number of + discardable pages that need to be discarded + from the RAM zones to be in use after the + general defrag. +@return Pointer to the RAM zone object that may potentially have pages + discarded by the general defrag. This will be NULL if no suitable + RAM zone could be found. +*/ +SZone* DRamAllocator::GeneralDefragStart0(TGenDefragStage& aStage, TUint& aRequiredToBeDiscarded) + { +#ifdef _DEBUG + if (!K::Initialising) + { + M::RamAllocIsLocked(); +#ifdef __VERIFY_LEASTMOVDIS + VerifyLeastPrefMovDis(); +#endif + } + // Any previous general defrag operation must have ended. + __NK_ASSERT_DEBUG(iZoneGeneralPrefLink == NULL); + __NK_ASSERT_DEBUG(iZoneGeneralTmpLink == NULL); +#endif + + if (iNumZones == 1) + { + // Only have one RAM zone so a defrag can't do anything. + return NULL; + } + + // Determine how many movable or discardable pages are required to be allocated. + TUint requiredPagesDis = 0; + TUint requiredPagesMov = 0; + TUint firstClearableInUseRank = 0; + SDblQueLink* link = iZoneLeastMovDis; + do + { + SZone& zone = *_LOFF(link, SZone, iPrefLink); + requiredPagesDis += zone.iAllocPages[EPageDiscard]; + requiredPagesMov += zone.iAllocPages[EPageMovable]; + + if (!firstClearableInUseRank && + (zone.iAllocPages[EPageMovable] || zone.iAllocPages[EPageDiscard]) && + !zone.iAllocPages[EPageFixed] && !zone.iAllocPages[EPageUnknown]) + {// This is the least preferable RAM zone that is has movable or + // discardable but may be clearable as it has no immovable pages. + firstClearableInUseRank = zone.iPrefRank; + } + + // Reset KRamZoneFlagGenDefrag flag bit for each RAM zone to be defraged. + zone.iFlags &= ~(KRamZoneFlagGenDefrag | KRamZoneFlagGenDefragBlock); + + link = link->iPrev; + } + while (link != &iZonePrefList.iA); + + // Adjust the number of discardable pages for those that are freeable. + // Dirty pages will be moved rather than discarded so they are not freeable + // and we must make sure that we have enough space in zones for these dirty + // paged pages. + __NK_ASSERT_DEBUG(requiredPagesDis >= (TUint)M::NumberOfFreeDpPages()); + requiredPagesDis -= M::NumberOfFreeDpPages(); + TUint totalDirtyPagesDis = M::NumberOfDirtyDpPages(); + if (requiredPagesDis < totalDirtyPagesDis) + requiredPagesDis = totalDirtyPagesDis; + + // Determine which is the least preferable RAM zone that needs to be + // in use for required number of movable and discardable pages. + TUint onlyPagesDis = 0; // Number of pages in RAM zones for discard only. + TUint onlyPagesMov = 0; // Number of pages in RAM zones for movable only. + TUint totalPagesDis = 0; // Total pages found so far for discardable pages. + TUint totalPagesMov = 0; // Total pages found so far for movable pages. + TUint totalCurrentDis = 0; // Number of allocated discardable pages found in + // RAM zones to be in use after the general defrag. + TUint totalCurrentMov = 0; // Number of allocated movable pages found in + // RAM zones to be in use after the general defrag. + TUint totalCurrentFree = 0; // The current number of free pages in the RAM zones + // to be in use after the general defrag. + iZoneGeneralPrefLink = &iZonePrefList.iA; + while (iZoneGeneralPrefLink != iZoneLeastMovDis && + (requiredPagesMov > totalPagesMov || + requiredPagesDis > totalPagesDis)) + { + iZoneGeneralPrefLink = iZoneGeneralPrefLink->iNext; + SZone& zone = *_LOFF(iZoneGeneralPrefLink, SZone, iPrefLink); + // Update the current totals. + totalCurrentDis += zone.iAllocPages[EPageDiscard]; + totalCurrentMov += zone.iAllocPages[EPageMovable]; + totalCurrentFree += zone.iFreePages; + + TBool onlyAllocDis = NoAllocOfPageType(zone, EPageMovable); + TBool onlyAllocMov = NoAllocOfPageType(zone, EPageDiscard); + if (!onlyAllocMov || !onlyAllocDis) + {// Either movable, discardable or both can be allocated in this zone. + TUint zonePagesFree = zone.iFreePages; + TUint zonePagesDis = zone.iAllocPages[EPageDiscard]; + TUint zonePagesMov = zone.iAllocPages[EPageMovable]; + // Total pages in this RAM zone that can be used for either + // discardable or movable pages. + TUint zonePagesGen = zonePagesDis + zonePagesMov + zonePagesFree; + if (onlyAllocMov) + { + if (requiredPagesDis > totalPagesDis) + {// No further discardable pages can be allocated into + // this RAM zone but consider any that already are. + TUint usedPages = Min( (TInt)zonePagesDis, + requiredPagesDis - totalPagesDis); + totalPagesDis += usedPages; + zonePagesDis -= usedPages; + } + TUint zoneOnlyMov = zonePagesDis + zonePagesMov + zonePagesFree; + onlyPagesMov += zoneOnlyMov; + totalPagesMov += zoneOnlyMov; + __KTRACE_OPT(KMMU2, Kern::Printf("onlyMov ID%x tot %x", + zone.iId, zoneOnlyMov)); + zonePagesGen = 0; // These pages aren't general purpose. + } + if (onlyAllocDis) + { + if (requiredPagesMov > totalPagesMov) + {// No further movable pages can be allocated into + // this RAM zone but consider any that already are. + TUint usedPages = Min( (TInt)zonePagesMov, + requiredPagesMov - totalPagesMov); + totalPagesMov += usedPages; + zonePagesMov -= usedPages; + } + TUint zoneOnlyDis = zonePagesDis + zonePagesMov + zonePagesFree; + onlyPagesDis += zoneOnlyDis; + totalPagesDis += zoneOnlyDis; + __KTRACE_OPT(KMMU2, Kern::Printf("onlyDis ID%x tot %x", + zone.iId, zoneOnlyDis)); + zonePagesGen = 0; // These pages aren't general purpose. + } + + if (requiredPagesDis > totalPagesDis) + {// Need some discardable pages so first steal any spare + // movable pages for discardable allocations. + if (totalPagesMov > requiredPagesMov) + {// Use any spare movable pages that can also be + // used for discardable allocations for discardable. + __NK_ASSERT_DEBUG(onlyPagesMov); + TUint spareMovPages = Min((TInt)(totalPagesMov - onlyPagesMov), + totalPagesMov - requiredPagesMov); + totalPagesMov -= spareMovPages; + totalPagesDis += spareMovPages; + __KTRACE_OPT(KMMU2, Kern::Printf("genDis Mov ID%x used%x", + zone.iId, spareMovPages)); + } + if (requiredPagesDis > totalPagesDis) + { + // Need more discardable pages but only grab those required. + TUint usedPages = Min( (TInt) zonePagesGen, + requiredPagesDis - totalPagesDis); + totalPagesDis += usedPages; + zonePagesGen -= usedPages; + __KTRACE_OPT(KMMU2, Kern::Printf("genDis ID%x used%x", + zone.iId, usedPages)); + } + } + if (requiredPagesMov > totalPagesMov) + {// Need some movable pages so first steal any spare + // discardable pages for movable allocations. + if (totalPagesDis > requiredPagesDis) + {// Use any spare discardable pages that can also be + // used for movable allocations for movable. + __NK_ASSERT_DEBUG(onlyPagesDis); + TUint spareDisPages = Min((TInt)(totalPagesDis - onlyPagesDis), + totalPagesDis - requiredPagesDis); + totalPagesDis -= spareDisPages; + totalPagesMov += spareDisPages; + __KTRACE_OPT(KMMU2, Kern::Printf("genMov Dis ID%x used%x", + zone.iId, spareDisPages)); + } + if (requiredPagesMov > totalPagesMov) + {// Still need some movable pages so grab them from this zone. + // Just grab all of the general pages left as discard pages will + // have already grabbed some if it had needed to. + totalPagesMov += zonePagesGen; + __KTRACE_OPT(KMMU2, Kern::Printf("genMov ID%x used%x", + zone.iId, zonePagesGen)); + } + } + } + } + + __KTRACE_OPT(KMMU, Kern::Printf("gen least in use ID 0x%x", + (_LOFF(iZoneGeneralPrefLink, SZone, iPrefLink))->iId)); + __NK_ASSERT_DEBUG(_LOFF(iZoneGeneralPrefLink, SZone, iPrefLink)->iPrefRank <= + iZoneLeastMovDisRank); + + if (iZoneGeneralPrefLink != iZoneLeastMovDis && + firstClearableInUseRank > _LOFF(iZoneGeneralPrefLink, SZone, iPrefLink)->iPrefRank) + {// We can reduce the number of RAM zones in use so block all the RAM + // zones not to be in use after the defrag from being allocated into + // by the general defrag. + link = iZoneLeastMovDis; + while (link != iZoneGeneralPrefLink) + { + SZone& zone = *_LOFF(link, SZone, iPrefLink); + zone.iFlags |= KRamZoneFlagGenDefragBlock; + link = link->iPrev; + } + + // Determine how many pages will need to be discarded to allow general + // defrag to succeed in using the minimum RAM zones required. + if (requiredPagesDis > totalCurrentDis) + {// Need to replace some discardable pages in RAM zones to be + // cleared with pages in the RAM zones to be in use after the + // general defrag. + __NK_ASSERT_DEBUG(totalCurrentFree >= requiredPagesDis - totalCurrentDis); + totalCurrentFree -= requiredPagesDis - totalCurrentDis; + } + TUint totalForMov = totalCurrentFree + totalCurrentMov; + if (requiredPagesMov > totalForMov) + {// Need to discard some pages from the least preferable RAM zone to be + // in use after the general for the movable pages to be moved to. + aRequiredToBeDiscarded = requiredPagesMov - totalForMov; + __NK_ASSERT_DEBUG(aRequiredToBeDiscarded <= totalCurrentDis); + __NK_ASSERT_DEBUG(totalCurrentDis - aRequiredToBeDiscarded >= requiredPagesDis); + } + + // This stage should discard pages from the least preferable RAM zones + // to be in use after the general defrag to save the pages having to + // be moved again by the final stage. + iZoneGeneralStage = EGenDefragStage0; + aStage = EGenDefragStage1; // Defrag::GeneralDefrag() requires this. + iZoneGeneralTmpLink = iZoneGeneralPrefLink; + return GeneralDefragNextZone0(); + } + + // General defrag can't clear any RAM zones so jump to tidying stage. + aStage = EGenDefragStage2; + iZoneGeneralStage = EGenDefragStage2; + return NULL; + } + + +/** +Find the next RAM zone that is suitable for stage 0 of a general defrag. +This should only be called after a preceeding call to +DRamAllocator::GeneralDefragStart0(). + +This goes through the RAM zones from the least preferable to be in use +after the general defrag to the most preferable RAM zone. It will +return each time it finds a RAM zone with discardable pages allocated into it. + +@return Pointer to the RAM zone object that may potentially have pages + discarded by the general defrag. This will be NULL if no suitable + RAM zone could be found. +*/ +SZone* DRamAllocator::GeneralDefragNextZone0() + { + M::RamAllocIsLocked(); + // Any previous general defrag operation must have ended. + __NK_ASSERT_DEBUG(iZoneGeneralPrefLink != NULL); + __NK_ASSERT_DEBUG(iZoneGeneralTmpLink != NULL); + __NK_ASSERT_DEBUG(iZoneGeneralStage == EGenDefragStage0); + + while (iZoneGeneralTmpLink != &iZonePrefList.iA) + { + SZone* zone = _LOFF(iZoneGeneralTmpLink, SZone, iPrefLink); + + // Save the RAM zone that is currently more preferable than this one + // before any reordering. + iZoneGeneralTmpLink = iZoneGeneralTmpLink->iPrev; + + if (zone->iFlags & KRamZoneFlagGenDefrag) + {// This zone has been selected for a general defrag already. + __KTRACE_OPT(KMMU, Kern::Printf("GenDefragNext0 zone ID 0x%x already defraged", + zone->iId)); + return NULL; + } + zone->iFlags |= KRamZoneFlagGenDefrag; + if (zone->iAllocPages[EPageDiscard]) + { + // A RAM zone that may have pages discarded by a general defrag has been found. + __KTRACE_OPT(KMMU, Kern::Printf("GenDefragNext0 zone ID 0x%x", zone->iId)); + return zone; + } + } + return NULL; + } + + +/** +Initialise this stage of a general defrag operation which will attempt +to clear all the RAM zones not to be in use once the general defrag +has completed. + +@return Pointer to the RAM zone object that may potentially be cleared + by the general defrag. This will be NULL if no suitable + RAM zone could be found. +*/ +SZone* DRamAllocator::GeneralDefragStart1() + { + M::RamAllocIsLocked(); + __NK_ASSERT_DEBUG(iNumZones == 1 || iZoneGeneralPrefLink != NULL); + + + if (iNumZones == 1) + {// On a device with one RAM zone can't do any defrag so return NULL. + return NULL; + } + + // Clear general defrag flags of each RAM zone to be defraged. + SDblQueLink* link = iZoneGeneralPrefLink; + for (; link != &iZonePrefList.iA; link = link->iPrev) + { + SZone& zone = *_LOFF(link, SZone, iPrefLink); + zone.iFlags &= ~KRamZoneFlagGenDefrag; + } + + // Flags cleared so now to start this stage from least preferable RAM zone + // currently in use. + iZoneGeneralTmpLink = iZoneLeastMovDis; + iZoneGeneralStage = EGenDefragStage1; + return GeneralDefragNextZone1(); + } + + +/** +Find the next RAM zone that is suitable for stage 1 of a general defrag. +This should only be called after a preceeding call to +DRamAllocator::GeneralDefragStart1(). + +This goes through the RAM zones from the least preferable currently +with movable or discardable pages allocated into it to the least +preferable RAM zone that is to be in use after the general defrag. +It will return each time it finds a RAM zone with movable and/or +discardable pages allocated into it. + +@return Pointer to the RAM zone object that may potentially be cleared by a + general defrag. This will be NULL if no suitable zone could be found. +*/ +SZone* DRamAllocator::GeneralDefragNextZone1() + { + M::RamAllocIsLocked(); + // Any previous general defrag operation must have ended. + __NK_ASSERT_DEBUG(iZoneGeneralPrefLink != NULL); + __NK_ASSERT_DEBUG(iZoneGeneralTmpLink != NULL); + __NK_ASSERT_DEBUG(iZoneGeneralStage == EGenDefragStage1); + + + // If we hit the target least preferable RAM zone to be in use once + // the defrag has completed then stop this stage of the general defrag. + + // Should never skip past iZoneGeneralPrefLink. + __NK_ASSERT_DEBUG(iZoneGeneralTmpLink != &iZonePrefList.iA); + + while (iZoneGeneralTmpLink != iZoneGeneralPrefLink) + { + SZone* zone = _LOFF(iZoneGeneralTmpLink, SZone, iPrefLink); + + // Save the RAM zone that is currently more preferable than this one + // before any reordering. + iZoneGeneralTmpLink = iZoneGeneralTmpLink->iPrev; + + if (zone->iFlags & KRamZoneFlagGenDefrag) + {// This zone has been selected for a general defrag already. + __KTRACE_OPT(KMMU, Kern::Printf("GenDefragNext1 zone ID 0x%x already defraged", + zone->iId)); + return NULL; + } + zone->iFlags |= KRamZoneFlagGenDefrag; + if (zone->iAllocPages[EPageMovable] || zone->iAllocPages[EPageDiscard]) + { + // A RAM zone that may be cleared by a general defrag has been found. + __KTRACE_OPT(KMMU, Kern::Printf("GenDefragNext1 zone ID 0x%x", zone->iId)); + return zone; + } + } + __KTRACE_OPT(KMMU, Kern::Printf("GenDefragNext1 reached general target")); + return NULL; + } + + +/** +Initialise stage 2 of a general defrag operation. + +Stage 2 creates room for fixed pages allocations in the more preferable RAM +zones in use by moving pages into the least preferable RAM zones in use. + +@return Pointer to the RAM zone object that may potentially be cleared of + movable and discardable pages by the general defrag. This will be + NULL if no suitable zone could be found. +*/ +SZone* DRamAllocator::GeneralDefragStart2() + { + M::RamAllocIsLocked(); + __NK_ASSERT_DEBUG(iNumZones == 1 || iZoneGeneralPrefLink != NULL); + + + if (iNumZones == 1) + {// On a device with one RAM zone can't do any defrag so return NULL. + return NULL; + } + + // Clear general defrag flags of each RAM zone to be defraged. + SDblQueLink* link = iZoneLeastMovDis; + for (; link != &iZonePrefList.iA; link = link->iPrev) + { + SZone& zone = *_LOFF(link, SZone, iPrefLink); + zone.iFlags &= ~(KRamZoneFlagGenDefrag | KRamZoneFlagGenDefragBlock); + } + + // Flags cleared so now to start 2nd stage from most preferable RAM zone. + iZoneGeneralTmpLink = iZonePrefList.First(); + iZoneGeneralStage = EGenDefragStage2; + return GeneralDefragNextZone2(); + } + + +/** +Find the next RAM zone that is suitable for this stage of general defrag. +This should only be called after a preceeding call to +DRamAllocator::GeneralDefragStart2(). + +This goes through the RAM zones from the most preferable to the least +preferable RAM zone that has movable and/or discardable pages allocated +into it. It will return each time it finds a RAM zone with movable and/or +discardable pages allocated into it. + +@return Pointer to the RAM zone object that may potentially be cleared of + movable and discardable pages by the general defrag. This will be + NULL if no suitable zone could be found. +*/ +SZone* DRamAllocator::GeneralDefragNextZone2() + { + M::RamAllocIsLocked(); + __NK_ASSERT_DEBUG(iZoneGeneralTmpLink != NULL); + __NK_ASSERT_DEBUG(iZoneGeneralStage == EGenDefragStage2); + + + while (iZoneGeneralTmpLink != iZoneLeastMovDis) + { + SZone* zone = _LOFF(iZoneGeneralTmpLink, SZone, iPrefLink); + + // Save the RAM zone that is currently less preferable than this one + // before any reordering. + iZoneGeneralTmpLink = iZoneGeneralTmpLink->iNext; + + if (zone->iFlags & KRamZoneFlagGenDefrag) + {// This zone has been selected for a general defrag already. + __KTRACE_OPT(KMMU, Kern::Printf("GenDefragNext2 zone ID 0x%x already defraged", zone->iId)); + return NULL; + } + zone->iFlags |= KRamZoneFlagGenDefrag | KRamZoneFlagGenDefragBlock; + if (zone->iAllocPages[EPageMovable] || zone->iAllocPages[EPageDiscard]) + {// A RAM zone that may be cleared by a general defrag has been found. + __KTRACE_OPT(KMMU, Kern::Printf("GenDefragNext2 zone ID 0x%x", zone->iId)); + return zone; + } + } + __KTRACE_OPT(KMMU, Kern::Printf("GenDefragNext2 reached general target")); + return NULL; + } + +/** +Inform the allocator that a general defragmentation operation has completed. + +*/ +void DRamAllocator::GeneralDefragEnd() + { +#ifdef _DEBUG + if (!K::Initialising) + { + M::RamAllocIsLocked(); +#ifdef __VERIFY_LEASTMOVDIS + VerifyLeastPrefMovDis(); +#endif + } +#endif + // Reset the general defrag preference link as it is no longer required. + iZoneGeneralPrefLink = NULL; + iZoneGeneralTmpLink = NULL; + } + + +/** +Calculate the number of free pages in all the RAM zones to be in use +once the general defragmentation operation has completed. + +@param aType The type of free pages to find in the higher priority zones. +@return The number of free pages in the RAM zones intended to be in use +after the general defrag operation has completed. +*/ +TUint DRamAllocator::GenDefragFreePages(TZonePageType aType) const + { + M::RamAllocIsLocked(); + + if (iZoneGeneralStage == EGenDefragStage2) + {// Second stage of general defrag where don't have to empty the RAM zone. + return KMaxTUint; + } + TUint totalFree = 0; + SDblQueLink* link = iZoneGeneralPrefLink; + for (; link != &iZonePrefList.iA; link = link->iPrev) + { + SZone& zone = *_LOFF(link, SZone, iPrefLink); + if (NoAllocOfPageType(zone, aType) || + zone.iFlags & KRamZoneFlagGenDefragBlock) + { + continue; + } + // This zone has free space for this type of page + totalFree += zone.iFreePages; + } + return totalFree; + } + + +/** Mark the RAM zone as being claimed to stop any further allocations. +@param aZone The zone to stop allocations to. + +@pre RamAlloc mutex held. +@post RamAlloc mutex held. +*/ +void DRamAllocator::ZoneClaimStart(SZone& aZone) + { + M::RamAllocIsLocked(); + __NK_ASSERT_DEBUG(!(aZone.iFlags & KRamZoneFlagClaiming)); + + aZone.iFlags |= KRamZoneFlagClaiming; + +#ifdef BTRACE_RAM_ALLOCATOR + BTrace8(BTrace::ERamAllocator, BTrace::ERamAllocZoneFlagsModified, aZone.iId, aZone.iFlags); +#endif + } + + +/** Mark the RAM zone as not being claimed to allow allocations. +@param aZone The zone to allow allocations into. + +@pre RamAlloc mutex held. +@post RamAlloc mutex held. +*/ +void DRamAllocator::ZoneClaimEnd(SZone& aZone) + { + M::RamAllocIsLocked(); + __NK_ASSERT_DEBUG(aZone.iFlags & KRamZoneFlagClaiming); + + aZone.iFlags &= ~KRamZoneFlagClaiming; + +#ifdef BTRACE_RAM_ALLOCATOR + BTrace8(BTrace::ERamAllocator, BTrace::ERamAllocZoneFlagsModified, aZone.iId, aZone.iFlags); +#endif + } + +/** Mark the RAM zone so that any allocation or frees from it can be detected. +Useful for defragging. +@param aZone The zone to mark. +@pre RamAlloc mutex held +@post RamAlloc mutex held +*/ +void DRamAllocator::ZoneMark(SZone& aZone) + { + M::RamAllocIsLocked(); + __NK_ASSERT_DEBUG(!(aZone.iFlags & KRamZoneFlagMark)); + + aZone.iFlags |= KRamZoneFlagMark; + +#ifdef BTRACE_RAM_ALLOCATOR + BTrace8(BTrace::ERamAllocator, BTrace::ERamAllocZoneFlagsModified, aZone.iId, aZone.iFlags); +#endif + } + +/** Unmark the RAM zone. +Useful for defragging. +@param aZone The zone to mark. +@return ETrue if the RAM zone is inactive, EFalse otherwise. +@pre RamAlloc mutex held +@post RamAlloc mutex held +*/ +TBool DRamAllocator::ZoneUnmark(SZone& aZone) + { + M::RamAllocIsLocked(); + + TInt r = aZone.iFlags & KRamZoneFlagMark; + aZone.iFlags &= ~KRamZoneFlagMark; + +#ifdef BTRACE_RAM_ALLOCATOR + BTrace8(BTrace::ERamAllocator, BTrace::ERamAllocZoneFlagsModified, aZone.iId, aZone.iFlags); +#endif + return r; + } + +/** Determine whether it is OK to allocate the specified page type +to the RAM zone. + +This should be used by all functions that search through the zones when +attempting to allocate pages. + +@return ETrue if this page type shouldn't be allocated into the RAM zone, +EFalse if it is OK to allocate that page type into the RAM zone. +*/ +TBool DRamAllocator::NoAllocOfPageType(SZone& aZone, TZonePageType aType) const + { + TUint8 flagMask = 1 << (aType - KPageTypeAllocBase); + return (aZone.iFlags & (KRamZoneFlagClaiming|KRamZoneFlagNoAlloc|KRamZoneFlagTmpBlockAlloc)) || + (aZone.iFlags & flagMask); + } + + +/** Updates the flags of the specified RAM zone. + +@param aId The ID of the RAM zone to modify. +@param aClearFlags The bit flags to clear. +@param aSetFlags The bit flags to set. + +@return KErrNone on success, KErrArgument if the RAM zone of aId not found or +aSetMask contains invalid flags. + +@pre RamAlloc mutex held +@post RamAlloc mutex held +*/ +TInt DRamAllocator::ModifyZoneFlags(TUint aId, TUint aClearMask, TUint aSetMask) + { + M::RamAllocIsLocked(); + + SZone* zone = ZoneFromId(aId); + if (zone == NULL || (aSetMask & KRamZoneFlagInvalid)) + {// aId invalid or an invalid flag bit was requested to be set. + return KErrArgument; + } + zone->iFlags &= ~aClearMask; + zone->iFlags |= aSetMask; + + __KTRACE_OPT(KMMU, Kern::Printf("Zone %x Flags %x", zone->iId, zone->iFlags)); + +#ifdef BTRACE_RAM_ALLOCATOR + BTrace8(BTrace::ERamAllocator, BTrace::ERamAllocZoneFlagsModified, zone->iId, zone->iFlags); +#endif + return KErrNone; + } + + +/** Invoke the RAM zone call back function to inform the variant of the RAM zones +in use so far by the system. +This is designed to only be invoked once during boot in MmuBase::Init2() +*/ +void DRamAllocator::InitialCallback() + { + __NK_ASSERT_DEBUG(iZoneCallbackInitSent == EFalse); + if (iZonePowerFunc) + { + TInt ret = (*iZonePowerFunc)(ERamZoneOp_Init, NULL, (TUint*)&iZonePwrState); + if (ret != KErrNone && ret != KErrNotSupported) + { + Panic(EZonesCallbackErr); + } + CHECK_PRECONDITIONS(MASK_THREAD_CRITICAL, "DRamAllocator::ZoneAllocPages"); + } + iZoneCallbackInitSent = ETrue; + } + + +#ifdef BTRACE_RAM_ALLOCATOR +/** +Structure for outputing zone information to BTrace that couldn't be fit into first +2 words of the BTraceN call +*/ +struct TRamAllocBtraceZone + { + TUint32 iId; + TUint8 iPref; + TUint8 iFlags; + TUint16 iReserved; + }; + +/** +This will be invoked when BTrace starts logging BTrace::ERamAllocator category +traces. +It outputs the zone configuration and the base addresses of any contiguous block +of allocated pages. +*/ +void DRamAllocator::SendInitialBtraceLogs(void) + { + M::RamAllocIsLocked(); + CHECK_PRECONDITIONS(MASK_THREAD_CRITICAL, "DRamAllocator::SendInitialBtraceLogs"); + + // Output the zone information + TRamAllocBtraceZone bZone; + BTrace4(BTrace::ERamAllocator, BTrace::ERamAllocZoneCount, iNumZones); + const SZone* zone = iZones; + const SZone* const endZone = iZones + iNumZones; + for (; zone < endZone; zone++) + { + bZone.iId = zone->iId; + bZone.iPref = zone->iPref; + bZone.iFlags = zone->iFlags; + BTraceN(BTrace::ERamAllocator, BTrace::ERamAllocZoneConfig, zone->iPhysPages, + zone->iPhysBase, &bZone, sizeof(TRamAllocBtraceZone)); + } + + // Search through zones and output each contiguous region of allocated pages + for (zone = iZones; zone < endZone; zone++) + { + if (zone->iFreePages != zone->iPhysPages) + { + TInt pageCount = 0; + TInt totalPages = 0; + TUint32 runStart = 0; + while ((TUint)totalPages != zone->iPhysPages - zone->iFreePages) + { + // find set of contiguous pages that have been allocated + // runStart will be set to first page of allocated run if one found + for (;runStart < zone->iPhysPages && zone->iBma[KBmaAllPages]->NotAllocated(runStart,1); runStart++); + + // find last allocated page of this run + TUint32 runEnd = runStart + 1; + for (;runEnd < zone->iPhysPages && zone->iBma[KBmaAllPages]->NotFree(runEnd,1); runEnd++); + + pageCount = runEnd - runStart; + if (pageCount > 0) + {// have a run of allocated pages so output BTrace + TPhysAddr baseAddr = (runStart << KPageShift) + zone->iPhysBase; + __KTRACE_OPT(KMMU2, Kern::Printf("offset %x physBase %x pages %x baseAddr %08x",runStart, zone->iPhysBase, pageCount, baseAddr)); + BTrace8(BTrace::ERamAllocator, BTrace::ERamAllocBootAllocation, pageCount, baseAddr); + runStart += pageCount; + totalPages += pageCount; + } + } + } + } + BTrace0(BTrace::ERamAllocator, BTrace::ERamAllocBootAllocationEnd); + } +#endif // BTRACE_RAM_ALLOCATOR + +TInt DRamAllocator::ClaimPhysicalRam(TPhysAddr aBase, TInt aSize) + { + TInt ret = SetPhysicalRamState(aBase,aSize,EFalse, EPageFixed); +#ifdef BTRACE_RAM_ALLOCATOR + if (ret == KErrNone) + { + BTrace8(BTrace::ERamAllocator, BTrace::ERamAllocClaimRam, aSize, aBase); + } +#endif + return ret; + } + +TInt DRamAllocator::FreePhysicalRam(TPhysAddr aBase, TInt aSize) + { + TInt ret = SetPhysicalRamState(aBase,aSize,ETrue, EPageFixed); +#ifdef BTRACE_RAM_ALLOCATOR + if (ret == KErrNone) + { + BTrace8(BTrace::ERamAllocator, BTrace::ERamAllocFreePhysical, aSize, aBase); + } +#endif + return ret; + } + + +TInt DRamAllocator::FreeRamInBytes() + { + return iTotalFreeRamPages<iNext; + while (tmpLink != &iZonePrefList.iA) + { + SZone& zone = *_LOFF(tmpLink, SZone, iPrefLink); + if (zone.iAllocPages[EPageMovable] != 0 || + zone.iAllocPages[EPageDiscard] != 0) + { + DebugDump(); + __NK_ASSERT_DEBUG(0); + } + tmpLink = tmpLink->iNext; + } + } +#endif