Fix for bug 2283 (RVCT 4.0 support is missing from PDK 3.0.h)
Have multiple extension sections in the bld.inf, one for each version
of the compiler. The RVCT version building the tools will build the
runtime libraries for its version, but make sure we extract all the other
versions from zip archives. Also add the archive for RVCT4.
// Copyright (c) 1997-2009 Nokia Corporation and/or its subsidiary(-ies).
// All rights reserved.
// This component and the accompanying materials are made available
// under the terms of the License "Eclipse Public License v1.0"
// which accompanies this distribution, and is available
// at the URL "http://www.eclipse.org/legal/epl-v10.html".
//
// Initial Contributors:
// Nokia Corporation - initial contribution.
//
// Contributors:
//
// Description:
// e32\memmodel\epoc\moving\arm\xmmu.cpp
//
//
#include "arm_mem.h"
#include <mmubase.inl>
#include <ramcache.h>
#include <demand_paging.h>
#include "execs.h"
#include <defrag.h>
#include "cache_maintenance.h"
extern void FlushTLBs();
#if defined(__CPU_SA1__)
const TPde KRomSectionPermissions = SECTION_PDE(KArmV45PermRORO, KArmV45MemAttWB, EDomainClient);
const TPde KShadowPdePerm = PT_PDE(EDomainClient);
const TPte KPtPtePerm = SP_PTE(KArmV45PermRWNO, KArmV45MemAttBuf); // page tables not cached
const TPte KRomPtePermissions = SP_PTE(KArmV45PermRORO, KArmV45MemAttWB); // ROM is cached, read-only for everyone
const TPte KShadowPtePerm = SP_PTE(KArmV45PermRWRO, KArmV45MemAttWB); // shadowed ROM is cached, supervisor writeable
#elif defined(__CPU_ARM710T__) || defined(__CPU_ARM720T__)
const TPde KRomSectionPermissions = SECTION_PDE(KArmV45PermRORO, KArmV45MemAttWB, EDomainClient);
const TPde KShadowPdePerm = PT_PDE(EDomainClient);
const TPte KPtPtePerm = SP_PTE(KArmV45PermRWNO, KArmV45MemAttWB); // page tables cached (write-through)
const TPte KRomPtePermissions = SP_PTE(KArmV45PermRORO, KArmV45MemAttWB); // ROM is cached, read-only for everyone
const TPte KShadowPtePerm = SP_PTE(KArmV45PermRWRO, KArmV45MemAttWB); // shadowed ROM is cached, supervisor writeable
#elif defined(__CPU_ARM920T__) || defined(__CPU_ARM925T__) || defined(__CPU_ARM926J__)
const TPde KRomSectionPermissions = SECTION_PDE(KArmV45PermRORO, KArmV45MemAttWT, EDomainClient);
const TPde KShadowPdePerm = PT_PDE(EDomainClient);
const TPte KPtPtePerm = SP_PTE(KArmV45PermRWNO, KArmV45MemAttWT); // page tables cached write through
const TPte KRomPtePermissions = SP_PTE(KArmV45PermRORO, KArmV45MemAttWT); // ROM is cached, read-only for everyone
const TPte KShadowPtePerm = SP_PTE(KArmV45PermRWRO, KArmV45MemAttWT); // shadowed ROM is cached, supervisor writeable
#elif defined(__CPU_XSCALE__)
#ifdef __CPU_XSCALE_MANZANO__
const TPde KRomSectionPermissions = SECTION_PDE(KArmV45PermRORO, KXScaleMemAttWTRA_WBWA, EDomainClient);
const TPde KShadowPdePerm = PT_PDE(EDomainClient);
const TPte KPtPtePerm = SP_PTE(KArmV45PermRWNO, KXScaleMemAttWTRA_WBWA); // page tables write-through cached
const TPte KRomPtePermissions = SP_PTE(KArmV45PermRORO, KXScaleMemAttWTRA_WBWA); // ROM is cached, read-only for everyone
const TPte KShadowPtePerm = SP_PTE(KArmV45PermRWRO, KXScaleMemAttWTRA_WBWA); // shadowed ROM is cached, supervisor writeable
#else
const TPde KRomSectionPermissions = SECTION_PDE(KArmV45PermRORO, KXScaleMemAttWTRA, EDomainClient);
const TPde KShadowPdePerm = PT_PDE(EDomainClient);
const TPte KPtPtePerm = SP_PTE(KArmV45PermRWNO, KXScaleMemAttWTRA); // page tables write-through cached
const TPte KRomPtePermissions = SP_PTE(KArmV45PermRORO, KXScaleMemAttWTRA); // ROM is cached, read-only for everyone
const TPte KShadowPtePerm = SP_PTE(KArmV45PermRWRO, KXScaleMemAttWTRA); // shadowed ROM is cached, supervisor writeable
#endif
#endif
const TPte KPtInfoPtePerm = KPtPtePerm;
const TPde KPtPdePerm = PT_PDE(EDomainClient);
// Permissions for each chunk type
enum TPTEProperties
{
ESupRo = SP_PTE(KArmV45PermRORO, KDefaultCaching),
ESupRw = SP_PTE(KArmV45PermRWNO, KDefaultCaching),
EUserRo = SP_PTE(KArmV45PermRWRO, KDefaultCaching),
EUserRw = SP_PTE(KArmV45PermRWRW, KDefaultCaching)
};
LOCAL_D const TPde ChunkPdePermissions[ENumChunkTypes] =
{
PT_PDE(EDomainClient), // EKernelData
PT_PDE(EDomainClient), // EKernelStack
PT_PDE(EDomainClient), // EKernelCode
PT_PDE(EDomainClient), // EDll
PT_PDE(EDomainClient), // EUserCode - user/ro & sup/rw everywhere
PT_PDE(EDomainClient), // ERamDrive - sup/rw accessed by domain change
// user data or self modifying code is sup/rw, user no access at home. It's user/rw & sup/rw when running
// note ARM MMU architecture prevents implementation of user read-only data
PT_PDE(EDomainClient), // EUserData
PT_PDE(EDomainClient), // EDllData
PT_PDE(EDomainClient), // EUserSelfModCode
PT_PDE(EDomainClient), // ESharedKernelSingle
PT_PDE(EDomainClient), // ESharedKernelMultiple
PT_PDE(EDomainClient), // ESharedIo
PT_PDE(EDomainClient), // ESharedKernelMirror (unused in this memory model)
PT_PDE(EDomainClient), // EKernelMessage
};
const TPde KUserDataRunningPermissions = PT_PDE(EDomainVarUserRun);
LOCAL_D const TPte ChunkPtePermissions[ENumChunkTypes] =
{
ESupRw, // EKernelData
ESupRw, // EKernelStack
ESupRw, // EKernelCode
EUserRo, // EDll
EUserRo, // EUserCode
ESupRw, // ERamDrive
ESupRw, // EUserData
ESupRw, // EDllData
ESupRw, // EUserSelfModCode
ESupRw, // ESharedKernelSingle
ESupRw, // ESharedKernelMultiple
ESupRw, // ESharedIo
ESupRw, // ESharedKernelMirror (unused in this memory model)
ESupRw, // EKernelMessage
};
const TPte KUserCodeLoadPte = (TPte)EUserRo;
const TPte KKernelCodeRunPte = (TPte)ESupRw;
// Inline functions for simple transformations
inline TLinAddr PageTableLinAddr(TInt aId)
{
return KPageTableBase + (aId<<KPageTableShift);
}
inline TPte* PageTable(TInt aId)
{
return (TPte*)(KPageTableBase+(aId<<KPageTableShift));
}
inline TPde* PageDirectoryEntry(TLinAddr aLinAddr)
{
return PageDirectory + (aLinAddr>>KChunkShift);
}
inline TBool IsPageTable(TPde aPde)
{
return ((aPde&KPdeTypeMask)==KArmV45PdePageTable);
}
inline TBool IsSectionDescriptor(TPde aPde)
{
return ((aPde&KPdeTypeMask)==KArmV45PdeSection);
}
inline TBool IsPresent(TPte aPte)
{
return (aPte&KPtePresentMask);
}
inline TPhysAddr PageTablePhysAddr(TPde aPde)
{
return aPde & KPdePageTableAddrMask;
}
inline TPhysAddr PhysAddrFromSectionDescriptor(TPde aPde)
{
return aPde & KPdeSectionAddrMask;
}
extern void InvalidateTLBForPage(TLinAddr /*aLinAddr*/);
void Mmu::SetupInitialPageInfo(SPageInfo* aPageInfo, TLinAddr aChunkAddr, TInt aPdeIndex)
{
__ASSERT_ALWAYS(aChunkAddr==0 || aChunkAddr>=KRamDriveEndAddress, Panic(EBadInitialPageAddr));
TLinAddr addr = aChunkAddr + (aPdeIndex<<KPageShift);
if (aPageInfo->Type()!=SPageInfo::EUnused)
return; // already set (page table)
if (addr == KPageTableInfoBase)
{
aPageInfo->SetPtInfo(0);
aPageInfo->Lock();
}
else if (addr>=KPageDirectoryBase && addr<(KPageDirectoryBase+KPageDirectorySize))
{
aPageInfo->SetPageDir(0,aPdeIndex);
aPageInfo->Lock();
}
else
aPageInfo->SetFixed();
}
void Mmu::SetupInitialPageTableInfo(TInt aId, TLinAddr aChunkAddr, TInt aNumPtes)
{
__ASSERT_ALWAYS(aChunkAddr==0 || aChunkAddr>=KRamDriveEndAddress, Panic(EBadInitialPageAddr));
SPageTableInfo& pti=PtInfo(aId);
pti.iCount=aNumPtes;
pti.SetGlobal(aChunkAddr>>KChunkShift);
}
TInt Mmu::GetPageTableId(TLinAddr aAddr)
{
TInt id=-1;
__KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::PageTableId(%08x)",aAddr));
TInt pdeIndex=aAddr>>KChunkShift;
TPde pde = PageDirectory[pdeIndex];
if (IsPageTable(pde))
{
SPageInfo* pi = SPageInfo::SafeFromPhysAddr(pde);
if (pi)
id = (pi->Offset()<<KPtClusterShift) | ((pde>>KPageTableShift)&KPtClusterMask);
}
__KTRACE_OPT(KMMU,Kern::Printf("ID=%d",id));
return id;
}
// Used only during boot for recovery of RAM drive
TInt ArmMmu::BootPageTableId(TLinAddr aAddr, TPhysAddr& aPtPhys)
{
TInt id=KErrNotFound;
__KTRACE_OPT(KMMU,Kern::Printf("ArmMmu:BootPageTableId(%08x,&)",aAddr));
TInt pdeIndex=aAddr>>KChunkShift;
TPde pde = PageDirectory[pdeIndex];
if (IsPageTable(pde))
{
aPtPhys = pde & KPdePageTableAddrMask;
SPageInfo* pi = SPageInfo::SafeFromPhysAddr(pde);
if (pi)
{
SPageInfo::TType type = pi->Type();
if (type == SPageInfo::EPageTable)
id = (pi->Offset()<<KPtClusterShift) | ((pde>>KPageTableShift)&KPtClusterMask);
else if (type == SPageInfo::EUnused)
id = KErrUnknown;
}
}
__KTRACE_OPT(KMMU,Kern::Printf("ID=%d",id));
return id;
}
TBool ArmMmu::PteIsPresent(TPte aPte)
{
return aPte & KPtePresentMask;
}
TPhysAddr ArmMmu::PtePhysAddr(TPte aPte, TInt aPteIndex)
{
TUint pte_type = aPte & KPteTypeMask;
if (pte_type == KArmV45PteLargePage)
return (aPte & KPteLargePageAddrMask) + (TPhysAddr(aPteIndex << KPageShift) & KLargePageMask);
else if (pte_type != 0)
return aPte & KPteSmallPageAddrMask;
return KPhysAddrInvalid;
}
TPhysAddr ArmMmu::PdePhysAddr(TLinAddr aAddr)
{
TPde pde = PageDirectory[aAddr>>KChunkShift];
if (IsSectionDescriptor(pde))
return PhysAddrFromSectionDescriptor(pde);
return KPhysAddrInvalid;
}
TPte* SafePageTableFromPde(TPde aPde)
{
if((aPde&KPdeTypeMask)==KArmV45PdePageTable)
{
SPageInfo* pi = SPageInfo::SafeFromPhysAddr(aPde);
if(pi)
{
TInt id = (pi->Offset()<<KPtClusterShift) | ((aPde>>KPageTableShift)&KPtClusterMask);
return PageTable(id);
}
}
return 0;
}
TPte* SafePtePtrFromLinAddr(TLinAddr aAddress)
{
TPde pde = PageDirectory[aAddress>>KChunkShift];
TPte* pt = SafePageTableFromPde(pde);
if(pt)
pt += (aAddress>>KPageShift)&(KChunkMask>>KPageShift);
return pt;
}
#ifdef __ARMCC__
__forceinline /* RVCT ignores normal inline qualifier :-( */
#else
inline
#endif
TPte* PtePtrFromLinAddr(TLinAddr aAddress)
{
TPde pde = PageDirectory[aAddress>>KChunkShift];
SPageInfo* pi = SPageInfo::FromPhysAddr(pde);
TInt id = (pi->Offset()<<KPtClusterShift) | ((pde>>KPageTableShift)&KPtClusterMask);
TPte* pt = PageTable(id);
pt += (aAddress>>KPageShift)&(KChunkMask>>KPageShift);
return pt;
}
TInt ArmMmu::LinearToPhysical(TLinAddr aLinAddr, TInt aSize, TPhysAddr& aPhysicalAddress, TPhysAddr* aPhysicalPageList)
{
TPhysAddr physStart = ArmMmu::LinearToPhysical(aLinAddr);
TPhysAddr nextPhys = physStart&~KPageMask;
TUint32* pageList = aPhysicalPageList;
TInt pageIndex = aLinAddr>>KPageShift;
TInt pagesLeft = ((aLinAddr+aSize-1)>>KPageShift)+1 - pageIndex;
TPde* pdePtr = &PageDirectory[aLinAddr>>KChunkShift];
while(pagesLeft)
{
pageIndex &= KChunkMask>>KPageShift;
TInt pagesLeftInChunk = (1<<(KChunkShift-KPageShift))-pageIndex;
if(pagesLeftInChunk>pagesLeft)
pagesLeftInChunk = pagesLeft;
pagesLeft -= pagesLeftInChunk;
TPhysAddr phys;
TPde pde = *pdePtr++;
TUint pdeType = pde&KPdeTypeMask;
if(pdeType==KArmV45PdeSection)
{
phys = (pde & KPdeSectionAddrMask) + (pageIndex*KPageSize);
__KTRACE_OPT(KMMU2,Kern::Printf("ArmMmu::LinearToPhysical Section phys=%8x",phys));
TInt n=pagesLeftInChunk;
phys==nextPhys ? nextPhys+=n*KPageSize : nextPhys=KPhysAddrInvalid;
if(pageList)
{
TUint32* pageEnd = pageList+n;
do
{
*pageList++ = phys;
phys+=KPageSize;
}
while(pageList<pageEnd);
}
}
else
{
TPte* pt = SafePageTableFromPde(pde);
if(!pt)
{
__KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::LinearToPhysical missing page table: PDE=%8x",pde));
return KErrNotFound;
}
pt += pageIndex;
for(;;)
{
TPte pte = *pt++;
TUint pte_type = pte & KPteTypeMask;
if (pte_type >= KArmV45PteSmallPage)
{
phys = (pte & KPteSmallPageAddrMask);
__KTRACE_OPT(KMMU2,Kern::Printf("ArmMmu::LinearToPhysical Small Page phys=%8x",phys));
phys==nextPhys ? nextPhys+=KPageSize : nextPhys=KPhysAddrInvalid;
if(pageList)
*pageList++ = phys;
if(--pagesLeftInChunk)
continue;
break;
}
if (pte_type == KArmV45PteLargePage)
{
--pt; // back up ptr
TUint pageOffset = ((TUint)pt>>2)&(KLargeSmallPageRatio-1);
phys = (pte & KPteLargePageAddrMask) + pageOffset*KPageSize;
__KTRACE_OPT(KMMU2,Kern::Printf("ArmMmu::LinearToPhysical Large Page phys=%8x",phys));
TInt n=KLargeSmallPageRatio-pageOffset;
if(n>pagesLeftInChunk)
n = pagesLeftInChunk;
phys==nextPhys ? nextPhys+=n*KPageSize : nextPhys=KPhysAddrInvalid;
if(pageList)
{
TUint32* pageEnd = pageList+n;
do
{
*pageList++ = phys;
phys+=KPageSize;
}
while(pageList<pageEnd);
}
pt += n;
if(pagesLeftInChunk-=n)
continue;
break;
}
__KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::LinearToPhysical bad PTE %8x",pte));
return KErrNotFound;
}
}
if(!pageList && nextPhys==KPhysAddrInvalid)
{
__KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::LinearToPhysical not contiguous"));
return KErrNotFound;
}
pageIndex = 0;
}
if(nextPhys==KPhysAddrInvalid)
{
// Memory is discontiguous...
aPhysicalAddress = KPhysAddrInvalid;
return 1;
}
else
{
// Memory is contiguous...
aPhysicalAddress = physStart;
return KErrNone;
}
}
TPhysAddr ArmMmu::LinearToPhysical(TLinAddr aLinAddr)
{
__KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::LinearToPhysical(%08x)",aLinAddr));
TPhysAddr phys = KPhysAddrInvalid;
TPde pde = PageDirectory[aLinAddr>>KChunkShift];
if (IsPageTable(pde))
{
SPageInfo* pi = SPageInfo::SafeFromPhysAddr(pde);
if (pi)
{
TInt id = (pi->Offset()<<KPtClusterShift) | ((pde>>KPageTableShift)&KPtClusterMask);
TInt pteIndex = (aLinAddr & KChunkMask)>>KPageShift;
TPte pte = PageTable(id)[pteIndex];
TUint pte_type = pte & KPteTypeMask;
if (pte_type == KArmV45PteLargePage)
{
phys = (pte & KPteLargePageAddrMask) + (aLinAddr & KLargePageMask);
__KTRACE_OPT(KMMU,Kern::Printf("Mapped with 64K page - returning %08x", phys));
}
else if (pte_type != 0)
{
phys = (pte & KPteSmallPageAddrMask) + (aLinAddr & KPageMask);
__KTRACE_OPT(KMMU,Kern::Printf("Mapped with 4K page - returning %08x", phys));
}
}
}
else if (IsSectionDescriptor(pde))
{
phys = (pde & KPdeSectionAddrMask) + (aLinAddr & KChunkMask);
__KTRACE_OPT(KMMU,Kern::Printf("Mapped with section - returning %08x", phys));
}
else
{
__KTRACE_OPT(KMMU,Kern::Printf("Address invalid"));
}
return phys;
}
TInt ArmMmu::PreparePagesForDMA(TLinAddr aLinAddr, TInt aSize, TPhysAddr* aPhysicalPageList)
//Returns the list of physical pages belonging to the specified memory space.
//Checks these pages belong to a chunk marked as being trusted.
//Locks these pages so they can not be moved by e.g. ram defragmenation.
{
SPageInfo* pi = NULL;
DChunk* chunk = NULL;
TInt err = KErrNone;
__KTRACE_OPT(KMMU2,Kern::Printf("ArmMmu::PreparePagesForDMA %08x+%08x, asid=%d",aLinAddr,aSize));
TUint32* pageList = aPhysicalPageList;
TInt pagesInList = 0; //The number of pages we put in the list so far
TInt pageIndex = (aLinAddr & KChunkMask) >> KPageShift; // Index of the page within the section
TInt pagesLeft = ((aLinAddr & KPageMask) + aSize + KPageMask) >> KPageShift;
MmuBase::Wait(); // RamAlloc Mutex for accessing page/directory tables.
NKern::LockSystem();// SystemlLock for accessing SPageInfo objects.
TPde* pdePtr = PageDirectory + (aLinAddr>>KChunkShift);
while(pagesLeft)
{
TInt pagesLeftInChunk = (1<<(KChunkShift-KPageShift))-pageIndex;
if(pagesLeftInChunk>pagesLeft)
pagesLeftInChunk = pagesLeft;
pagesLeft -= pagesLeftInChunk;
TPte* pt = SafePageTableFromPde(*pdePtr++);
if(!pt) { err = KErrNotFound; goto fail; }// Cannot get page table.
pt += pageIndex;
for(;pagesLeftInChunk--;)
{
TPhysAddr phys = (*pt++ & KPteSmallPageAddrMask);
pi = SPageInfo::SafeFromPhysAddr(phys);
if(!pi) { err = KErrNotFound; goto fail; }// Invalid address
__KTRACE_OPT(KMMU2,Kern::Printf("PageInfo: PA:%x T:%x S:%x O:%x C:%x",phys, pi->Type(), pi->State(), pi->Owner(), pi->LockCount()));
if (chunk==NULL)
{//This is the first page. Check 'trusted' bit.
if (pi->Type()!= SPageInfo::EChunk)
{ err = KErrAccessDenied; goto fail; }// The first page do not belong to chunk.
chunk = (DChunk*)pi->Owner();
if ( (chunk == NULL) || ((chunk->iAttributes & DChunk::ETrustedChunk)== 0) )
{ err = KErrAccessDenied; goto fail; } // Not a trusted chunk
}
pi->Lock();
*pageList++ = phys;
if ( (++pagesInList&127) == 0) //release system lock temporarily on every 512K
NKern::FlashSystem();
}
pageIndex = 0;
}
if (pi->Type()!= SPageInfo::EChunk)
{ err = KErrAccessDenied; goto fail; }// The last page do not belong to chunk.
if (chunk && (chunk != (DChunk*)pi->Owner()))
{ err = KErrArgument; goto fail; }//The first & the last page do not belong to the same chunk.
NKern::UnlockSystem();
MmuBase::Signal();
return KErrNone;
fail:
__KTRACE_OPT(KMMU2,Kern::Printf("ArmMmu::PreparePagesForDMA failed"));
NKern::UnlockSystem();
MmuBase::Signal();
ReleasePagesFromDMA(aPhysicalPageList, pagesInList);
return err;
}
TInt ArmMmu::ReleasePagesFromDMA(TPhysAddr* aPhysicalPageList, TInt aPageCount)
// Unlocks physical pages.
// @param aPhysicalPageList - points to the list of physical pages that should be released.
// @param aPageCount - the number of physical pages in the list.
{
NKern::LockSystem();
__KTRACE_OPT(KMMU2,Kern::Printf("ArmMmu::ReleasePagesFromDMA count:%d",aPageCount));
while (aPageCount--)
{
SPageInfo* pi = SPageInfo::SafeFromPhysAddr(*aPhysicalPageList++);
if(!pi)
{
NKern::UnlockSystem();
return KErrArgument;
}
__KTRACE_OPT(KMMU2,Kern::Printf("PageInfo: T:%x S:%x O:%x C:%x",pi->Type(), pi->State(), pi->Owner(), pi->LockCount()));
pi->Unlock();
}
NKern::UnlockSystem();
return KErrNone;
}
void ArmMmu::Init1()
{
__KTRACE_OPT2(KBOOT,KMMU,Kern::Printf("ArmMmu::Init1"));
// MmuBase data
iPageSize=KPageSize;
iPageMask=KPageMask;
iPageShift=KPageShift;
iChunkSize=KChunkSize;
iChunkMask=KChunkMask;
iChunkShift=KChunkShift;
iPageTableSize=KPageTableSize;
iPageTableMask=KPageTableMask;
iPageTableShift=KPageTableShift;
iPtClusterSize=KPtClusterSize;
iPtClusterMask=KPtClusterMask;
iPtClusterShift=KPtClusterShift;
iPtBlockSize=KPtBlockSize;
iPtBlockMask=KPtBlockMask;
iPtBlockShift=KPtBlockShift;
iPtGroupSize=KChunkSize/KPageTableSize;
iPtGroupMask=iPtGroupSize-1;
iPtGroupShift=iChunkShift-iPageTableShift;
//TInt* iPtBlockCount; // dynamically allocated - Init2
//TInt* iPtGroupCount; // dynamically allocated - Init2
iPtInfo=(SPageTableInfo*)KPageTableInfoBase;
iPageTableLinBase=KPageTableBase;
//iRamPageAllocator; // dynamically allocated - Init2
//iAsyncFreeList; // dynamically allocated - Init2
//iPageTableAllocator; // dynamically allocated - Init2
//iPageTableLinearAllocator;// dynamically allocated - Init2
iPtInfoPtePerm=KPtInfoPtePerm;
iPtPtePerm=KPtPtePerm;
iPtPdePerm=KPtPdePerm;
iTempAddr=KTempAddr;
iSecondTempAddr=KSecondTempAddr;
iMapSizes=KPageSize|KLargePageSize|KChunkSize;
iRomLinearBase = ::RomHeaderAddress;
iRomLinearEnd = KRomLinearEnd;
iShadowPtePerm = KShadowPtePerm;
iShadowPdePerm = KShadowPdePerm;
// Mmu data
TInt total_ram=TheSuperPage().iTotalRamSize;
#if defined(__HAS_EXTERNAL_CACHE__)
//L2 cache on ARMv5 is always in write-back mode => must be always purged
iDecommitThreshold = CacheMaintenance::SyncAllPerformanceThresholdPages();
#else
iDecommitThreshold = 0; ///no cache consistency issues on decommit
#endif
iDataSectionBase = KDataSectionBase;
iDataSectionEnd = KDataSectionEnd;
iMaxDllDataSize=Min(total_ram/2, 0x08000000); // phys RAM/2 up to 128Mb
iMaxDllDataSize=(iMaxDllDataSize+iChunkMask)&~iChunkMask; // round up to chunk size
iMaxUserCodeSize=Min(total_ram, 0x10000000); // phys RAM up to 256Mb
iMaxUserCodeSize=(iMaxUserCodeSize+iChunkMask)&~iChunkMask; // round up to chunk size
iMaxKernelCodeSize=Min(total_ram/2, 0x04000000); // phys RAM/2 up to 64Mb
iMaxKernelCodeSize=(iMaxKernelCodeSize+iChunkMask)&~iChunkMask; // round up to chunk size
iPdeBase=KPageDirectoryBase;
iUserCodeLoadPtePerm=KUserCodeLoadPte;
iKernelCodePtePerm=KKernelCodeRunPte;
iDllDataBase = KDataSectionEnd - iMaxDllDataSize;
iUserCodeBase = KPageInfoLinearBase - iMaxUserCodeSize;
iKernelCodeBase = iUserCodeBase - iMaxKernelCodeSize;
__KTRACE_OPT(KMMU,Kern::Printf("DDS %08x UCS %08x KCS %08x", iMaxDllDataSize, iMaxUserCodeSize, iMaxKernelCodeSize));
__KTRACE_OPT(KMMU,Kern::Printf("DDB %08x KCB %08x UCB %08x RLB %08x", iDllDataBase, iKernelCodeBase, iUserCodeBase, iRomLinearBase));
// ArmMmu data
// other
PP::MaxUserThreadStack=0x14000; // 80K - STDLIB asks for 64K for PosixServer!!!!
PP::UserThreadStackGuard=0x2000; // 8K
PP::MaxStackSpacePerProcess=0x200000; // 2Mb
K::SupervisorThreadStackSize=0x1000; // 4K
PP::SupervisorThreadStackGuard=0x1000; // 4K
K::MachineConfig=(TMachineConfig*)KMachineConfigLinAddr;
PP::RamDriveStartAddress=KRamDriveStartAddress;
PP::RamDriveRange=KRamDriveMaxSize;
PP::RamDriveMaxSize=KRamDriveMaxSize; // may be reduced later
__KTRACE_OPT(KBOOT,Kern::Printf("K::MaxMemCopyInOneGo=0x%x",K::MaxMemCopyInOneGo));
K::MemModelAttributes=EMemModelTypeMoving|EMemModelAttrNonExProt|EMemModelAttrKernProt|EMemModelAttrWriteProt|
EMemModelAttrVA|EMemModelAttrProcessProt|EMemModelAttrSameVA|EMemModelAttrSupportFixed|
EMemModelAttrSvKernProt|EMemModelAttrIPCKernProt;
Arm::DefaultDomainAccess=KDefaultDomainAccess;
// Domains 0-3 are preallocated
// 0=Variable user running, 1=Client, 2=Page tables, 3=RAM drive
Domains=(~(0xffffffffu<<ENumDomains))&0xfffffff0u;
iMaxPageTables = 1<<(32-KChunkShift); // possibly reduced when RAM size known
Mmu::Init1();
}
void ArmMmu::DoInit2()
{
__KTRACE_OPT2(KBOOT,KMMU,Kern::Printf("ArmMmu::DoInit2"));
iTempPte=PageTable(GetPageTableId(iTempAddr))+((iTempAddr&KChunkMask)>>KPageShift);
iSecondTempPte=PageTable(GetPageTableId(iSecondTempAddr))+((iSecondTempAddr&KChunkMask)>>KPageShift);
__KTRACE_OPT2(KBOOT,KMMU,Kern::Printf("iTempAddr=%08x, iTempPte=%08x, iSecondTempAddr=%08x, iSecondTempPte=%08x", iTempAddr, iTempPte, iSecondTempAddr, iSecondTempPte));
CreateKernelSection(iKernelCodeBase, KPageShift);
iHomePdeMap=(TUint32*)Kern::AllocZ(-KSuperPageLinAddr>>KChunkShift<<2);
iHomePdeMap=(TUint32*)((TUint32)iHomePdeMap-(KSuperPageLinAddr>>KChunkShift<<2)); //adjust the pointer so it's indexed by address>>20
#if defined(__CPU_WRITE_BACK_CACHE)
#if defined(__CPU_HAS_SINGLE_ENTRY_DCACHE_FLUSH)
if (InternalCache::Info[KCacheInfoD].iLineLength == 32)
iCopyPageFn = &::CopyPageForRemap32;
else if (InternalCache::Info[KCacheInfoD].iLineLength == 16)
iCopyPageFn = &::CopyPageForRemap16;
else
Panic(ENoCopyPageFunction);
#else
#error Write-back cache without single entry dcache flush is not supported
#endif
#else // !__CPU_HAS_WRITE_BACK_CACHE
iCopyPageFn = &::CopyPageForRemapWT;
#endif
Mmu::DoInit2();
}
#ifndef __MMU_MACHINE_CODED__
void ArmMmu::MapRamPages(TInt aId, SPageInfo::TType aType, TAny* aPtr, TUint32 aOffset, const TPhysAddr* aPageList, TInt aNumPages, TPte aPtePerm)
//
// Map a list of physical RAM pages into a specified page table with specified PTE permissions.
// Update the page information array.
// Call this with the system locked.
//
{
__KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::MapRamPages() id=%d type=%d ptr=%08x off=%08x n=%d perm=%08x",
aId, aType, aPtr, aOffset, aNumPages, aPtePerm));
SPageTableInfo& ptinfo=iPtInfo[aId];
ptinfo.iCount+=aNumPages;
aOffset>>=KPageShift;
TInt ptOffset=aOffset & KPagesInPDEMask; // entry number in page table
TPte* pPte=PageTable(aId)+ptOffset; // address of first PTE
while(aNumPages--)
{
TPhysAddr pa = *aPageList++;
*pPte++ = pa | aPtePerm; // insert PTE
__KTRACE_OPT(KMMU,Kern::Printf("Writing PTE %08x to %08x",pPte[-1],pPte-1));
if (aType!=SPageInfo::EInvalid)
{
SPageInfo* pi = SPageInfo::SafeFromPhysAddr(pa);
if(pi)
{
pi->Set(aType,aPtr,aOffset);
__KTRACE_OPT(KMMU,Kern::Printf("I: %d %08x %08x",aType,aPtr,aOffset));
++aOffset; // increment offset for next page
}
}
}
__DRAIN_WRITE_BUFFER;
}
void ArmMmu::MapPhysicalPages(TInt aId, SPageInfo::TType aType, TAny* aPtr, TUint32 aOffset, TPhysAddr aPhysAddr, TInt aNumPages, TPte aPtePerm)
//
// Map consecutive physical pages into a specified page table with specified PTE permissions.
// Update the page information array if RAM pages are being mapped.
// Call this with the system locked.
//
{
__KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::MapPhysicalPages() id=%d type=%d ptr=%08x off=%08x phys=%08x n=%d perm=%08x",
aId, aType, aPtr, aOffset, aPhysAddr, aNumPages, aPtePerm));
SPageTableInfo& ptinfo=iPtInfo[aId];
ptinfo.iCount+=aNumPages;
aOffset>>=KPageShift;
TInt ptOffset=aOffset & KPagesInPDEMask; // entry number in page table
TPte* pPte=PageTable(aId)+ptOffset; // address of first PTE
SPageInfo* pi;
if(aType==SPageInfo::EInvalid)
pi = NULL;
else
pi = SPageInfo::SafeFromPhysAddr(aPhysAddr);
while(aNumPages--)
{
*pPte++ = aPhysAddr|aPtePerm; // insert PTE
aPhysAddr+=KPageSize;
__KTRACE_OPT(KMMU,Kern::Printf("Writing PTE %08x to %08x",pPte[-1],pPte-1));
if (pi)
{
pi->Set(aType,aPtr,aOffset);
__KTRACE_OPT(KMMU,Kern::Printf("I: %d %08x %08x",aType,aPtr,aOffset));
++aOffset; // increment offset for next page
++pi;
}
}
__DRAIN_WRITE_BUFFER;
}
void ArmMmu::MapVirtual(TInt aId, TInt aNumPages)
//
// Called in place of MapRamPages or MapPhysicalPages to update mmu data structures when committing
// virtual address space to a chunk. No pages are mapped.
// Call this with the system locked.
//
{
SPageTableInfo& ptinfo=iPtInfo[aId];
ptinfo.iCount+=aNumPages;
}
void ArmMmu::RemapPage(TInt aId, TUint32 aAddr, TPhysAddr aOldAddr, TPhysAddr aNewAddr, TPte aPtePerm, DProcess* /*aProcess*/)
//
// Replace the mapping at address aAddr in page table aId.
// Update the page information array for both the old and new pages.
// Call this with the system locked.
//
{
__KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::RemapPage() id=%d addr=%08x old=%08x new=%08x perm=%08x", aId, aAddr, aOldAddr, aNewAddr, aPtePerm));
TInt ptOffset=(aAddr&KChunkMask)>>KPageShift; // entry number in page table
TPte* pPte=PageTable(aId)+ptOffset; // address of PTE
TPte pte=*pPte;
TUint pageType = (pte & KPteTypeMask);
if (pageType == KArmPteSmallPage || pageType == 0)
{
__ASSERT_ALWAYS((pte & KPteSmallPageAddrMask) == aOldAddr || pte==KPteNotPresentEntry, Panic(ERemapPageFailed));
SPageInfo* oldpi = SPageInfo::FromPhysAddr(aOldAddr);
__ASSERT_DEBUG(oldpi->LockCount()==0,Panic(ERemapPageFailed));
// remap page
*pPte = aNewAddr | aPtePerm; // overwrite PTE
__KTRACE_OPT(KMMU,Kern::Printf("Writing PTE %08x to %08x",*pPte,pPte));
__DRAIN_WRITE_BUFFER;
InvalidateTLBForPage(aAddr); // flush any corresponding TLB entry
// update new pageinfo, clear old
SPageInfo* pi = SPageInfo::FromPhysAddr(aNewAddr);
pi->Set(oldpi->Type(),oldpi->Owner(),oldpi->Offset());
oldpi->SetUnused();
}
else
{
__KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::RemapPage() called on a non-4K page!"));
Panic(ERemapPageFailed);
}
}
void ArmMmu::RemapKernelPage(TInt aId, TLinAddr aSrc, TLinAddr aDest, TPhysAddr aNewPhys, TPte aPtePerm)
//
// Replace the mapping at address aAddr in page table aId.
// Called with the system locked.
// MUST NOT INVOKE ANY TRACING - or do anything else that might touch the kernel heap
// We are depending on this not reintroducing any of the cache lines we previously
// invalidated.
//
{
TInt ptOffset=(aSrc&KChunkMask)>>KPageShift; // entry number in page table
TPte* pPte=PageTable(aId)+ptOffset; // address of PTE
TInt irq = NKern::DisableAllInterrupts();
CopyPageForRemap(aDest, aSrc);
*pPte = aNewPhys | aPtePerm; // overwrite PTE
__DRAIN_WRITE_BUFFER;
InvalidateTLBForPage(aSrc); // flush any corresponding TLB entry
NKern::RestoreInterrupts(irq);
}
TInt ArmMmu::UnmapPages(TInt aId, TUint32 aAddr, TInt aNumPages, TPhysAddr* aPageList, TBool aSetPagesFree, TInt& aNumPtes, TInt& aNumFree, DProcess*)
//
// Unmap a specified area at address aAddr in page table aId. Place physical addresses of unmapped
// pages into aPageList, and count of unmapped pages into aNumPtes.
// Return number of pages still mapped using this page table.
// Call this with the system locked.
// @param aId Identifies Page Table to unmap PTEs(Page Table Entries) from.
// @param aAddr Base Base Virtual Address of the region to unmap. It (indirectly) specifies the first PTE in this Page Table to unmap.
// @param aNumPages The number of consecutive PTEs to unmap.
// @param aPageList Points to pre-allocated array. On return, it is filled in with the list of physical addresses of the unmapped 4K
// memory blocks.
// @param aSetPagesFree If true, pages a placed in the free state and only mapped pages are added
// to aPageList.
// @param aNumPtes On return, indicates how many PTEs are unmapped.
// @param aNumFree On return, holds the number are freed 4K memory blocks. Not updated if aSetPagesFree is false.
// @return The number of PTEs still mapped in this Page Table (aId).
{
__KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::UnmapPages() id=%d addr=%08x n=%d pl=%08x set-free=%d",aId,aAddr,aNumPages,aPageList,aSetPagesFree));
TInt ptOffset=(aAddr&KChunkMask)>>KPageShift; // entry number in page table
TPte* pPte=PageTable(aId)+ptOffset; // address of first PTE
TInt np=0;
TInt nf=0;
while(aNumPages--)
{
TPte pte=*pPte; // get original PTE
*pPte++=0; // clear PTE
TUint pageType = (pte & KPteTypeMask);
if (pageType == KArmPteSmallPage)
InvalidateTLBForPage(aAddr); // flush any corresponding TLB entry
if (pageType == KArmPteSmallPage || (pageType == 0 && pte != KPteNotPresentEntry))
{
++np; // count unmapped pages
TPhysAddr pa=pte & KPteSmallPageAddrMask; // physical address of unmapped page
if (aSetPagesFree)
{
SPageInfo* pi = SPageInfo::FromPhysAddr(pa);
__NK_ASSERT_DEBUG(pageType == KArmPteSmallPage ||
(pi->Type()==SPageInfo::EPagedCode && pi->State()==SPageInfo::EStatePagedOld));
if(iRamCache->PageUnmapped(pi))
{
pi->SetUnused(); // mark page as unused
if (pi->LockCount()==0)
{
*aPageList++=pa; // store in page list
++nf; // count free pages
}
}
}
else
*aPageList++=pa; // store in page list
}
aAddr+=KPageSize;
}
aNumPtes=np;
aNumFree=nf;
SPageTableInfo& ptinfo=iPtInfo[aId];
TInt r=(ptinfo.iCount-=np);
__DRAIN_WRITE_BUFFER;
__KTRACE_OPT(KMMU,Kern::Printf("Unmapped %d; Freed: %d; Return %08x",np,nf,r));
return r; // return number of pages remaining in this page table
}
#endif
TInt ArmMmu::UnmapVirtual(TInt aId, TUint32 aAddr, TInt aNumPages, TPhysAddr* aPageList, TBool aSetPagesFree, TInt& aNumPtes, TInt& aNumFree, DProcess* aProcess)
//
// Unmap a specified area at address aAddr in page table aId. Place physical addresses of unmapped
// pages into aPageList, and count of unmapped pages into aNumPtes.
// Adjust the page table reference count as if aNumPages pages were unmapped.
// Return number of pages still mapped using this page table.
// Call this with the system locked.
//
{
SPageTableInfo& ptinfo=iPtInfo[aId];
TInt newCount = ptinfo.iCount - aNumPages;
UnmapPages(aId, aAddr, aNumPages, aPageList, aSetPagesFree, aNumPtes, aNumFree, aProcess);
ptinfo.iCount = newCount;
aNumPtes = aNumPages;
return newCount;
}
#ifndef __MMU_MACHINE_CODED__
void ArmMmu::DoAssignPageTable(TInt aId, TLinAddr aAddr, TPde aPdePerm)
//
// Assign an allocated page table to map a given linear address with specified permissions.
// This should be called with the system locked and the MMU mutex held.
//
{
__KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::DoAssignPageTable %d to %08x perm %08x",aId,aAddr,aPdePerm));
TLinAddr ptLin=PageTableLinAddr(aId);
TPhysAddr ptPhys=LinearToPhysical(ptLin);
TInt pdeIndex=TInt(aAddr>>KChunkShift);
PageDirectory[pdeIndex]=ptPhys|aPdePerm;
__KTRACE_OPT(KMMU,Kern::Printf("Writing PDE %08x to %08x", ptPhys|aPdePerm, PageDirectory+pdeIndex));
__DRAIN_WRITE_BUFFER;
}
void ArmMmu::RemapPageTable(TPhysAddr aOld, TPhysAddr aNew, TLinAddr aAddr)
//
// Replace a page table mapping the specified linear address.
// This should be called with the system locked and the MMU mutex held.
//
{
__KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::RemapPageTable %08x to %08x at %08x",aOld,aNew,aAddr));
TInt pdeIndex=TInt(aAddr>>KChunkShift);
TPde pde=PageDirectory[pdeIndex];
__ASSERT_ALWAYS((pde & KPdePageTableAddrMask) == aOld, Panic(ERemapPageTableFailed));
TPde newPde=aNew|(pde&~KPdePageTableAddrMask);
PageDirectory[pdeIndex]=newPde;
__KTRACE_OPT(KMMU,Kern::Printf("Writing PDE %08x to %08x", newPde, PageDirectory+pdeIndex));
__DRAIN_WRITE_BUFFER;
}
void ArmMmu::DoUnassignPageTable(TLinAddr aAddr)
//
// Unassign a now-empty page table currently mapping the specified linear address.
// We assume that TLB and/or cache flushing has been done when any RAM pages were unmapped.
// This should be called with the system locked and the MMU mutex held.
//
{
__KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::DoUnassignPageTable at %08x",aAddr));
TInt pdeIndex=TInt(aAddr>>KChunkShift);
PageDirectory[pdeIndex]=0;
__KTRACE_OPT(KMMU,Kern::Printf("Clearing PDE at %08x", PageDirectory+pdeIndex));
__DRAIN_WRITE_BUFFER;
}
#endif
// Initialise page table at physical address aXptPhys to be used as page table aXptId
// to expand the virtual address range used for mapping page tables. Map the page table
// at aPhysAddr as page table aId using the expanded range.
// Assign aXptPhys to kernel's Page Directory.
// Called with system unlocked and MMU mutex held.
void ArmMmu::BootstrapPageTable(TInt aXptId, TPhysAddr aXptPhys, TInt aId, TPhysAddr aPhysAddr)
{
__KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::BootstrapPageTable xptid=%04x, xptphys=%08x, id=%04x, phys=%08x",
aXptId, aXptPhys, aId, aPhysAddr));
// put in a temporary mapping for aXptPhys
// make it noncacheable
TPhysAddr pa=aXptPhys&~KPageMask;
*iTempPte = pa | SP_PTE(KArmV45PermRWNO, KMemAttNC);
__DRAIN_WRITE_BUFFER;
// clear XPT
TPte* xpt=(TPte*)(iTempAddr+(aXptPhys&KPageMask));
memclr(xpt, KPageTableSize);
// must in fact have aXptPhys and aPhysAddr in same physical page
__ASSERT_ALWAYS( TUint32(aXptPhys^aPhysAddr)<TUint32(KPageSize), MM::Panic(MM::EBootstrapPageTableBadAddr));
// so only need one mapping
xpt[(aXptId>>KPtClusterShift)&KPagesInPDEMask] = pa | KPtPtePerm;
// remove temporary mapping
*iTempPte=0;
__DRAIN_WRITE_BUFFER;
InvalidateTLBForPage(iTempAddr);
// initialise PtInfo...
TLinAddr xptAddr = PageTableLinAddr(aXptId);
iPtInfo[aXptId].SetGlobal(xptAddr>>KChunkShift);
// map xpt...
TInt pdeIndex=TInt(xptAddr>>KChunkShift);
NKern::LockSystem();
PageDirectory[pdeIndex]=aXptPhys|KPtPdePerm;
__DRAIN_WRITE_BUFFER;
NKern::UnlockSystem();
}
// Edit the self-mapping entry in page table aId, mapped at aTempMap, to
// change the physical address from aOld to aNew. Used when moving page
// tables which were created by BootstrapPageTable.
// Called with system locked and MMU mutex held.
void ArmMmu::FixupXPageTable(TInt aId, TLinAddr aTempMap, TPhysAddr aOld, TPhysAddr aNew)
{
__KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::FixupXPageTable id=%04x, tempmap=%08x, old=%08x, new=%08x",
aId, aTempMap, aOld, aNew));
// find correct page table inside the page
TPte* xpt=(TPte*)(aTempMap + ((aId & KPtClusterMask) << KPageTableShift));
// find the pte in that page table
xpt += (aId>>KPtClusterShift)&KPagesInPDEMask;
// switch the mapping
__ASSERT_ALWAYS((*xpt&~KPageMask)==aOld, Panic(EFixupXPTFailed));
*xpt = aNew | KPtPtePerm;
// invalidate the TLB entry for the self-mapping page table
// the PDE has not yet been changed, but since we hold the
// system lock, nothing should bring this back into the TLB.
InvalidateTLBForPage(PageTableLinAddr(aId));
}
// Set up a page table (specified by aId) to map a 1Mb section of ROM containing aRomAddr
// using ROM at aOrigPhys.
void ArmMmu::InitShadowPageTable(TInt aId, TLinAddr aRomAddr, TPhysAddr aOrigPhys)
{
__KTRACE_OPT(KMMU, Kern::Printf("ArmMmu:InitShadowPageTable id=%04x aRomAddr=%08x aOrigPhys=%08x",
aId, aRomAddr, aOrigPhys));
TPte* ppte = PageTable(aId);
TPte* ppte_End = ppte + KChunkSize/KPageSize;
TPhysAddr phys = aOrigPhys - (aRomAddr & KChunkMask);
for (; ppte<ppte_End; ++ppte, phys+=KPageSize)
*ppte = phys | KRomPtePermissions;
__DRAIN_WRITE_BUFFER;
}
// Copy the contents of ROM at aRomAddr to a shadow page at physical address aShadowPhys
void ArmMmu::InitShadowPage(TPhysAddr aShadowPhys, TLinAddr aRomAddr)
{
__KTRACE_OPT(KMMU, Kern::Printf("ArmMmu:InitShadowPage aShadowPhys=%08x aRomAddr=%08x",
aShadowPhys, aRomAddr));
// put in a temporary mapping for aShadowPhys
// make it noncacheable
*iTempPte = aShadowPhys | SP_PTE(KArmV45PermRWNO, KMemAttNC);
__DRAIN_WRITE_BUFFER;
// copy contents of ROM
wordmove( (TAny*)iTempAddr, (const TAny*)aRomAddr, KPageSize );
__DRAIN_WRITE_BUFFER; // make sure contents are written to memory
// remove temporary mapping
*iTempPte=0;
__DRAIN_WRITE_BUFFER;
InvalidateTLBForPage(iTempAddr);
}
// Assign a shadow page table to replace a ROM section mapping
// Enter and return with system locked
void ArmMmu::AssignShadowPageTable(TInt aId, TLinAddr aRomAddr)
{
__KTRACE_OPT(KMMU, Kern::Printf("ArmMmu:AssignShadowPageTable aId=%04x aRomAddr=%08x",
aId, aRomAddr));
TLinAddr ptLin=PageTableLinAddr(aId);
TPhysAddr ptPhys=LinearToPhysical(ptLin);
TPde* ppde = PageDirectory + (aRomAddr>>KChunkShift);
TPde newpde = ptPhys | KShadowPdePerm;
__KTRACE_OPT(KMMU,Kern::Printf("Writing PDE %08x to %08x", newpde, ppde));
TInt irq=NKern::DisableAllInterrupts();
*ppde = newpde; // map in the page table
__DRAIN_WRITE_BUFFER; // make sure new PDE written to main memory
FlushTLBs(); // flush both TLBs (no need to flush cache yet)
NKern::RestoreInterrupts(irq);
}
void ArmMmu::DoUnmapShadowPage(TInt aId, TLinAddr aRomAddr, TPhysAddr aOrigPhys)
{
__KTRACE_OPT(KMMU,Kern::Printf("ArmMmu:DoUnmapShadowPage, id=%04x lin=%08x origphys=%08x", aId, aRomAddr, aOrigPhys));
TPte* ppte = PageTable(aId) + ((aRomAddr & KChunkMask)>>KPageShift);
TPte newpte = aOrigPhys | KRomPtePermissions;
__KTRACE_OPT(KMMU,Kern::Printf("Writing PTE %08x to %08x", newpte, ppte));
TInt irq=NKern::DisableAllInterrupts();
*ppte = newpte;
__DRAIN_WRITE_BUFFER;
InvalidateTLBForPage(aRomAddr);
SyncCodeMappings();
CacheMaintenance::CodeChanged(aRomAddr, KPageSize, CacheMaintenance::EMemoryRemap);
CacheMaintenance::PageToReuse(aRomAddr, EMemAttNormalCached, KPhysAddrInvalid);
NKern::RestoreInterrupts(irq);
}
TInt ArmMmu::UnassignShadowPageTable(TLinAddr aRomAddr, TPhysAddr aOrigPhys)
{
__KTRACE_OPT(KMMU,Kern::Printf("ArmMmu:UnassignShadowPageTable, lin=%08x origphys=%08x", aRomAddr, aOrigPhys));
TPde* ppde = PageDirectory + (aRomAddr>>KChunkShift);
TPde newpde = (aOrigPhys &~ KChunkMask) | KRomSectionPermissions;
__KTRACE_OPT(KMMU,Kern::Printf("Writing PDE %08x to %08x", newpde, ppde));
TInt irq=NKern::DisableAllInterrupts();
*ppde = newpde; // revert to section mapping
__DRAIN_WRITE_BUFFER; // make sure new PDE written to main memory
FlushTLBs(); // flush both TLBs
NKern::RestoreInterrupts(irq);
return KErrNone;
}
void ArmMmu::DoFreezeShadowPage(TInt aId, TLinAddr aRomAddr)
{
__KTRACE_OPT(KMMU, Kern::Printf("ArmMmu:DoFreezeShadowPage aId=%04x aRomAddr=%08x",
aId, aRomAddr));
TPte* ppte = PageTable(aId) + ((aRomAddr & KChunkMask)>>KPageShift);
TPte newpte = (*ppte & KPteSmallPageAddrMask) | KRomPtePermissions;
__KTRACE_OPT(KMMU,Kern::Printf("Writing PTE %08x to %08x", newpte, ppte));
*ppte = newpte;
__DRAIN_WRITE_BUFFER;
InvalidateTLBForPage(aRomAddr);
}
void ArmMmu::Pagify(TInt aId, TLinAddr aLinAddr)
{
__KTRACE_OPT(KMMU, Kern::Printf("ArmMmu:Pagify aId=%04x aLinAddr=%08x", aId, aLinAddr));
TInt pteIndex = (aLinAddr & KChunkMask)>>KPageShift;
TPte* pte = PageTable(aId);
if ((pte[pteIndex] & KPteTypeMask) == KArmV45PteLargePage)
{
__KTRACE_OPT(KMMU,Kern::Printf("Converting 64K page to 4K pages"));
pteIndex &= ~0xf;
TPte source = pte[pteIndex];
source = (source & KPteLargePageAddrMask) | SP_PTE_FROM_LP_PTE(source);
pte += pteIndex;
for (TInt entry=0; entry<16; entry++)
{
pte[entry] = source | (entry<<12);
}
FlushTLBs();
}
}
void ArmMmu::FlushShadow(TLinAddr aRomAddr)
{
CacheMaintenance::CodeChanged(aRomAddr, KPageSize, CacheMaintenance::EMemoryRemap);
CacheMaintenance::PageToReuse(aRomAddr, EMemAttNormalCached, KPhysAddrInvalid);
InvalidateTLBForPage(aRomAddr); // remove all TLB references to original ROM page
SyncCodeMappings();
}
inline void ZeroPdes(TLinAddr aBase, TLinAddr aEnd)
{
memclr(PageDirectory+(aBase>>KChunkShift), ((aEnd-aBase)>>KChunkShift)*sizeof(TPde));
}
void ArmMmu::ClearPageTable(TInt aId, TInt aFirstIndex)
{
__KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::ClearPageTable(%d,%d)",aId,aFirstIndex));
TPte* pte=PageTable(aId);
memclr(pte+aFirstIndex, KPageTableSize-aFirstIndex*sizeof(TPte));
__DRAIN_WRITE_BUFFER;
}
void ArmMmu::ClearRamDrive(TLinAddr aStart)
{
// clear the page directory entries corresponding to the RAM drive
ZeroPdes(aStart, KRamDriveEndAddress);
__DRAIN_WRITE_BUFFER;
}
void ArmMmu::ApplyTopLevelPermissions(TLinAddr aAddr, TUint aChunkSize, TPde aPdePerm)
{
__KTRACE_OPT(KMMU,Kern::Printf("ApplyTopLevelPermissions at %x",aAddr));
TInt pdeIndex=aAddr>>KChunkShift;
TInt numPdes=(aChunkSize+KChunkMask)>>KChunkShift;
TPde* pPde=PageDirectory+pdeIndex;
while(numPdes--)
{
*pPde=(*pPde)?((*pPde & KPdePageTableAddrMask)|aPdePerm):0;
pPde++;
}
__DRAIN_WRITE_BUFFER;
}
void ArmMmu::ApplyPagePermissions(TInt aId, TInt aPageOffset, TInt aNumPages, TPte aPtePerm)
{
__KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::ApplyPagePermissions %04x:%03x+%03x perm %08x",
aId, aPageOffset, aNumPages, aPtePerm));
TPte* pPte=PageTable(aId)+aPageOffset;
TPde* pPteEnd=pPte+aNumPages;
NKern::LockSystem();
for (; pPte<pPteEnd; ++pPte)
{
TPte pte=*pPte;
if (pte)
*pPte = (pte&KPteSmallPageAddrMask)|aPtePerm;
}
NKern::UnlockSystem();
FlushTLBs();
__DRAIN_WRITE_BUFFER;
}
void ArmMmu::MoveChunk(TLinAddr aInitAddr, TUint aSize, TLinAddr aFinalAddr, TPde aPdePerm)
{
__KTRACE_OPT(KMMU,Kern::Printf("MoveChunk at %08x to %08x size %08x PdePerm %08x",
aInitAddr, aFinalAddr, aSize, aPdePerm));
TInt numPdes=(aSize+KChunkMask)>>KChunkShift;
TInt iS=aInitAddr>>KChunkShift;
TInt iD=aFinalAddr>>KChunkShift;
TPde* pS=PageDirectory+iS;
TPde* pD=PageDirectory+iD;
while(numPdes--)
{
*pD++=(*pS)?((*pS & KPdePageTableAddrMask)|aPdePerm):0;
*pS++=KPdeNotPresentEntry;
}
__DRAIN_WRITE_BUFFER;
}
void ArmMmu::MoveChunk(TLinAddr aInitAddr, TLinAddr aFinalAddr, TInt aNumPdes)
//
// Move a block of PDEs without changing permissions. Must work with overlapping initial and final
// regions. Call this with kernel locked.
//
{
__KTRACE_OPT(KMMU,Kern::Printf("MoveChunk at %08x to %08x numPdes %d", aInitAddr, aFinalAddr, aNumPdes));
if (aInitAddr==aFinalAddr || aNumPdes==0)
return;
TInt iS=aInitAddr>>KChunkShift;
TInt iD=aFinalAddr>>KChunkShift;
TBool forwardOverlap=(iS<iD && iD-iS<aNumPdes);
TBool backwardOverlap=(iS>iD && iS-iD<aNumPdes);
TInt iC=backwardOverlap?(iD+aNumPdes):iS; // first index to clear
TInt iZ=forwardOverlap?iD:(iS+aNumPdes); // last index to clear + 1
TPde* pS=PageDirectory+iS;
TPde* pD=PageDirectory+iD;
__KTRACE_OPT(KMMU,Kern::Printf("backwardOverlap=%d, forwardOverlap=%d",backwardOverlap,forwardOverlap));
__KTRACE_OPT(KMMU,Kern::Printf("first clear %03x, last clear %03x",iC,iZ));
wordmove(pD,pS,aNumPdes<<2); // move PDEs
pD=PageDirectory+iC; // pointer to first PDE to clear
iZ-=iC; // number of PDEs to clear
memclr(pD, iZ<<2); // clear PDEs
__DRAIN_WRITE_BUFFER;
}
TPde ArmMmu::PdePermissions(TChunkType aChunkType, TInt aChunkState)
{
if ((aChunkType==EUserData || aChunkType==EDllData || aChunkType==EUserSelfModCode
|| aChunkType==ESharedKernelSingle || aChunkType==ESharedKernelMultiple || aChunkType==ESharedIo)
&& aChunkState!=0)
return KUserDataRunningPermissions;
return ChunkPdePermissions[aChunkType];
}
TPte ArmMmu::PtePermissions(TChunkType aChunkType)
{
return ChunkPtePermissions[aChunkType];
}
const TUint FBLK=(EMapAttrFullyBlocking>>12);
const TUint BFNC=(EMapAttrBufferedNC>>12);
const TUint BUFC=(EMapAttrBufferedC>>12);
const TUint L1UN=(EMapAttrL1Uncached>>12);
const TUint WTRA=(EMapAttrCachedWTRA>>12);
const TUint WTWA=(EMapAttrCachedWTWA>>12);
const TUint WBRA=(EMapAttrCachedWBRA>>12);
const TUint WBWA=(EMapAttrCachedWBWA>>12);
const TUint AWTR=(EMapAttrAltCacheWTRA>>12);
const TUint AWTW=(EMapAttrAltCacheWTWA>>12);
const TUint AWBR=(EMapAttrAltCacheWBRA>>12);
const TUint AWBW=(EMapAttrAltCacheWBWA>>12);
const TUint MAXC=(EMapAttrL1CachedMax>>12);
const TUint L2UN=(EMapAttrL2Uncached>>12);
const TUint16 UNS=0xffffu; // Unsupported attribute
const TUint16 SPE=0xfffeu; // Special processing required
#if defined(__CPU_ARM710T__) || defined(__CPU_ARM720T__)
// Original definition of C B
static const TUint16 CacheBuffAttributes[16]=
{0x00,0x00,0x04,0x04,0x0C,0x0C,0x0C,0x0C, UNS, UNS, UNS, UNS, UNS, UNS, UNS,0x0C};
static const TUint8 CacheBuffActual[16]=
{FBLK,FBLK,BUFC,BUFC,WTRA,WTRA,WTRA,WTRA,FBLK,FBLK,FBLK,FBLK,FBLK,FBLK,FBLK,WTRA};
#elif defined(__CPU_ARM920T__) || defined(__CPU_ARM925T__) || defined(__CPU_ARM926J__)
// Newer definition of C B
static const TUint16 CacheBuffAttributes[16]=
{0x00,0x00,0x04,0x04,0x08,0x08,0x0C,0x0C, UNS, UNS, UNS, UNS, UNS, UNS, UNS,0x0C};
static const TUint8 CacheBuffActual[16]=
{FBLK,FBLK,BUFC,BUFC,WTRA,WTRA,WBRA,WBRA,FBLK,FBLK,FBLK,FBLK,FBLK,FBLK,FBLK,WBRA};
#elif defined(__CPU_SA1__)
// Special definition of C B
static const TUint16 CacheBuffAttributes[16]=
{0x00,0x00,0x04,0x04,0x04,0x04,0x0C,0x0C,0x04,0x04,0x08,0x08, UNS, UNS, UNS,0x0C};
static const TUint8 CacheBuffActual[16]=
{FBLK,FBLK,BUFC,BUFC,BUFC,BUFC,WBRA,WBRA,FBLK,FBLK,AWBR,AWBR,FBLK,FBLK,FBLK,WBRA};
#elif defined(__CPU_XSCALE__)
#ifdef __CPU_XSCALE_MANZANO__
#ifdef __HAS_EXTERNAL_CACHE__
// ***MANZANO with L2 cache****** //
//Specifies TEX::CB bits for different L1/L2 cache attributes
// ...876543201
// ...TEX..CB..
static const TUint16 CacheBuffAttributes[80]=
{ // L1CACHE:
// FBLK BFNC BUFC L1UN WTRA WTWA WBRA WBWA AWTR AWTW AWBR AWBT UNS UNS UNS MAX L2CACHE:
0x00, 0x44, 0x40, 0x40, 0x108, 0x108, 0x10c, 0x10c, SPE, SPE, SPE, SPE, UNS,UNS,UNS,0x10c, //NC
0x00, 0x44, 0x40, 0x40, 0x108, 0x108, 0x10c, 0x10c, SPE, SPE, SPE, SPE, UNS,UNS,UNS,0x10c, //WTRA
0x00, 0x44, 0x40, 0x40, 0x108, 0x108, 0x10c, 0x10c, SPE, SPE, SPE, SPE, UNS,UNS,UNS,0x10c, //WTWA
0x00, 0x44, 0x40, 0x140, 0x148, 0x148, 0x14c, 0x14c, SPE, SPE, SPE, SPE, UNS,UNS,UNS,0x14c, //WBRA
0x00, 0x44, 0x40, 0x140, 0x148, 0x148, 0x14c, 0x14c, SPE, SPE, SPE, SPE, UNS,UNS,UNS,0x14c, //WBWA
};
extern TUint MiniCacheConfig();
//Converts page table attributes(TEX:CB) into appropriate cache attributes.
TInt CacheAttributesActual(TUint& cacheL1, TUint& cacheL2, TUint cbatt)
{
switch (cbatt)
{
case 0: cacheL1 = FBLK; cacheL2 = L2UN; return KErrNone;
case 0x40: cacheL1 = L1UN; cacheL2 = L2UN; return KErrNone;
case 0x44: cacheL1 = BFNC; cacheL2 = L2UN; return KErrNone;
case 0x48: cacheL1 = MiniCacheConfig(); cacheL2 = L2UN; return KErrNone;
case 0x108: cacheL1 = WTRA; cacheL2 = L2UN; return KErrNone;
case 0x10c: cacheL1 = WBRA; cacheL2 = L2UN; return KErrNone;
case 0x140: cacheL1 = L1UN; cacheL2 = WBWA; return KErrNone;
case 0x148: cacheL1 = WTRA; cacheL2 = WBWA; return KErrNone;
case 0x14c: cacheL1 = WBRA; cacheL2 = WBWA; return KErrNone;
}
return KErrNotSupported;
}
#else //__HAS_EXTERNAL_CACHE__
// ***MANZANO without L2 cache****** //
static const TUint16 CacheBuffAttributes[16]=
// FBLK BFNC BUFC L1UN WTRA WTWA WBRA WBWA -----------AltCache-------- MAXC
{0x00,0x44,0x40,0x40,0x148,0x148,0x14C,0x14C,SPE,SPE,SPE,SPE,UNS,UNS,UNS,0x14C};
static const TUint8 CacheBuffActual[16]=
{FBLK,BFNC,BUFC,BUFC,WTRA,WTRA,WBRA,WBRA,FBLK,FBLK,FBLK,FBLK,FBLK,FBLK,FBLK,WBRA};
#endif //__HAS_EXTERNAL_CACHE__
#else
// ***XSCALE that is not MANZANO (no L2 cache)****** //
// X C B
static const TUint16 CacheBuffAttributes[16]=
{0x00,0x44,0x04,0x04,0x08,0x08,0x0C,0x4C,SPE,SPE,SPE,SPE,UNS,UNS,UNS,0x4C};
static const TUint8 CacheBuffActual[16]=
{FBLK,BFNC,BUFC,BUFC,WTRA,WTRA,WBRA,WBWA,FBLK,FBLK,FBLK,FBLK,FBLK,FBLK,FBLK,WBWA};
#endif
// ***Common code for all XSCALE cores*** //
extern TUint MiniCacheConfig();
void ProcessSpecialCacheAttr(TUint& cache, TUint& cbatt)
{
// If writeback requested, give writeback or writethrough
// If writethrough requested, give writethrough or uncached
// Give other allocation policy if necessary.
TUint mccfg=MiniCacheConfig();
__KTRACE_OPT(KMMU,Kern::Printf("MiniCacheConfig: %x",mccfg));
if (cache<AWBR && mccfg>=AWBR) // asked for WT but cache is set for WB
{
cache=BUFC; // so give uncached, buffered, coalescing
#if defined (__CPU_XSCALE_MANZANO__)
cbatt=0x40;
#else
cbatt=0x04;
#endif
}
else
{
cache=mccfg; // give whatever minicache is configured for
cbatt=0x48; // minicache attributes
}
}
#endif
static const TUint8 ActualReadPrivilegeLevel[4]={4,1,4,4}; // RORO,RWNO,RWRO,RWRW
static const TUint8 ActualWritePrivilegeLevel[4]={0,1,1,4}; // RORO,RWNO,RWRO,RWRW
/** Calculates cb attributes for page table and sets actual cache attributes*/
TInt GetCacheAttr(TUint& cacheL1, TUint& cacheL2, TUint& cbatt)
{
TInt r = KErrNone;
// Scale down L2 to 0-4 : NC, WTRA, WTWA, WBRA, WBWA
#if defined (__CPU_XSCALE_MANZANO__) && defined(__HAS_EXTERNAL_CACHE__)
if (cacheL2 == MAXC) cacheL2 = WBWA-3; // Scale down L2 cache attributes...
else if (cacheL2 > WBWA) return KErrNotSupported; // ... to 0-4 for...
else if (cacheL2 < WTRA) cacheL2 = L2UN; // ... L2UN to WBWA
else cacheL2-=3; //
#else
cacheL2 = 0; // Either no L2 cache or L2 cache attributes will be just a copy of L1 cache attributes.
#endif
//Get cb page attributes. (On some platforms, tex bits are includded as well.)
cbatt = CacheBuffAttributes[cacheL1 + (cacheL2<<4)];
__KTRACE_OPT(KMMU,Kern::Printf("GetCacheAttr, table returned:%x",cbatt));
#if defined(__CPU_XSCALE__)
//Check if altDCache/LLR cache attributes are defined
if (cbatt == SPE)
{
cacheL2 = 0; //Not L2 cached in such case
ProcessSpecialCacheAttr(cacheL1,cbatt);
__KTRACE_OPT(KMMU,Kern::Printf("GetCacheAttr, spec case returned:%x",cbatt));
}
#endif
if(cbatt == UNS)
return KErrNotSupported;
//W Got CB page attributes. Now, find out what are the actual cache attributes.
#if defined(__CPU_XSCALE_MANZANO__) && defined(__HAS_EXTERNAL_CACHE__)
r = CacheAttributesActual(cacheL1, cacheL2, cbatt);
#else
cacheL1 = CacheBuffActual[cacheL1];
#if defined(__HAS_EXTERNAL_CACHE__)
cacheL2 = cacheL1;
#else
cacheL2 = 0;
#endif
#endif
return r;
}
TInt ArmMmu::PdePtePermissions(TUint& aMapAttr, TPde& aPde, TPte& aPte)
{
__KTRACE_OPT(KMMU,Kern::Printf(">ArmMmu::PdePtePermissions, mapattr=%08x",aMapAttr));
TUint read=aMapAttr & EMapAttrReadMask;
TUint write=(aMapAttr & EMapAttrWriteMask)>>4;
TUint exec=(aMapAttr & EMapAttrExecMask)>>8;
// if execute access is greater than read, adjust read (since there are no separate execute permissions on ARM)
if (exec>read)
read=exec;
TUint ap;
if (write==0)
{
// read-only
if (read>=4)
ap=KArmV45PermRORO; // user and supervisor read-only
else
ap=KArmV45PermRWNO; // supervisor r/w user no access (since no RO/NO access is available)
}
else if (write<4)
{
// only supervisor can write
if (read>=4)
ap=KArmV45PermRWRO; // supervisor r/w user r/o
else
ap=KArmV45PermRWNO; // supervisor r/w user no access
}
else
ap=KArmV45PermRWRW; // supervisor r/w user r/w
read=ActualReadPrivilegeLevel[ap];
write=ActualWritePrivilegeLevel[ap];
#ifndef __CPU_USE_MMU_TEX_FIELD
ap|=(ap<<2);
ap|=(ap<<4); // replicate permissions in all four subpages
#endif
ap<<=4; // shift access permissions into correct position for PTE
ap|=KArmPteSmallPage; // add in mandatory small page bits
// Get cb atributes for the page table and the actual cache attributes
TUint cbatt;
TUint cacheL1=(aMapAttr & EMapAttrL1CacheMask)>>12;
TUint cacheL2=(aMapAttr & EMapAttrL2CacheMask)>>16;
TInt r = GetCacheAttr(cacheL1, cacheL2, cbatt);
if (r==KErrNone)
{
aPde=PT_PDE(EDomainClient);
aPte=ap|cbatt;
aMapAttr=read|(write<<4)|(read<<8)|(cacheL1<<12)|(cacheL2<<16);
}
__KTRACE_OPT(KMMU,Kern::Printf("<ArmMmu::PdePtePermissions, r=%d, mapattr=%08x, pde=%08x, pte=%08x",
r,aMapAttr,aPde,aPte));
return r;
}
void ArmMmu::Map(TLinAddr aLinAddr, TPhysAddr aPhysAddr, TInt aSize, TPde aPdePerm, TPte aPtePerm, TInt aMapShift)
//
// Map a region of physical addresses aPhysAddr to aPhysAddr+aSize-1 to virtual address aLinAddr.
// Use permissions specified by aPdePerm and aPtePerm. Use mapping sizes up to and including (1<<aMapShift).
// Assume any page tables required are already assigned.
// aLinAddr, aPhysAddr, aSize must be page-aligned.
//
{
__KTRACE_OPT(KMMU, Kern::Printf("ArmMmu::Map lin=%08x phys=%08x size=%08x", aLinAddr, aPhysAddr, aSize));
__KTRACE_OPT(KMMU, Kern::Printf("pde=%08x pte=%08x mapshift=%d", aPdePerm, aPtePerm, aMapShift));
TPde pt_pde=aPdePerm;
TPte sp_pte=aPtePerm;
TPde section_pde=SECTION_PDE_FROM_PDEPTE(pt_pde, sp_pte);
TPte lp_pte=LP_PTE_FROM_SP_PTE(sp_pte);
TLinAddr la=aLinAddr;
TPhysAddr pa=aPhysAddr;
TInt remain=aSize;
while (remain)
{
if (aMapShift>=KChunkShift && (la & KChunkMask)==0 && remain>=KChunkSize)
{
// use sections
TInt npdes = remain>>KChunkShift;
TPde* p_pde = PageDirectory + (la>>KChunkShift);
TPde* p_pde_E = p_pde + npdes;
TPde pde = pa|section_pde;
NKern::LockSystem();
for (; p_pde < p_pde_E; pde+=KChunkSize)
{
__ASSERT_DEBUG(*p_pde==0, MM::Panic(MM::EPdeAlreadyInUse));
__KTRACE_OPT(KMMU,Kern::Printf("Writing PDE %08x to %08x", pde, p_pde));
*p_pde++=pde;
}
NKern::UnlockSystem();
npdes<<=KChunkShift;
la+=npdes, pa+=npdes, remain-=npdes;
continue;
}
TInt block_size = Min(remain, KChunkSize-(la&KChunkMask));
TPte pa_mask=~KPageMask;
TPte pte_perm=sp_pte;
if (aMapShift>=KLargePageShift && block_size>=KLargePageSize)
{
if ((la & KLargePageMask)==0)
{
// use 64K large pages
pa_mask=~KLargePageMask;
pte_perm=lp_pte;
}
else
block_size = Min(remain, KLargePageSize-(la&KLargePageMask));
}
block_size &= pa_mask;
// use pages (large or small)
TInt id = PageTableId(la);
__ASSERT_DEBUG(id>=0, MM::Panic(MM::EMmuMapNoPageTable));
TPte* p_pte = PageTable(id) + ((la&KChunkMask)>>KPageShift);
TPte* p_pte_E = p_pte + (block_size>>KPageShift);
SPageTableInfo& ptinfo = iPtInfo[id];
NKern::LockSystem();
for (; p_pte < p_pte_E; pa+=KPageSize)
{
__ASSERT_DEBUG(*p_pte==0, MM::Panic(MM::EPteAlreadyInUse));
TPte pte = (pa & pa_mask) | pte_perm;
__KTRACE_OPT(KMMU,Kern::Printf("Writing PTE %08x to %08x", pte, p_pte));
*p_pte++=pte;
++ptinfo.iCount;
NKern::FlashSystem();
}
NKern::UnlockSystem();
la+=block_size, remain-=block_size;
}
}
void ArmMmu::Unmap(TLinAddr aLinAddr, TInt aSize)
//
// Remove all mappings in the specified range of addresses.
// Assumes there are only global mappings involved.
// Don't free page tables.
// aLinAddr, aSize must be page-aligned.
//
{
__KTRACE_OPT(KMMU, Kern::Printf("ArmMmu::Unmap lin=%08x size=%08x", aLinAddr, aSize));
TLinAddr a=aLinAddr;
TLinAddr end=a+aSize;
__KTRACE_OPT(KMMU,Kern::Printf("a=%08x end=%08x",a,end));
NKern::LockSystem();
while(a!=end)
{
TInt pdeIndex=a>>KChunkShift;
TLinAddr next=(pdeIndex<<KChunkShift)+KChunkSize;
TInt to_do = Min(TInt(end-a), TInt(next-a))>>KPageShift;
__KTRACE_OPT(KMMU,Kern::Printf("a=%08x next=%08x to_do=%d",a,next,to_do));
TPde pde = PageDirectory[pdeIndex];
if ( (pde&KPdePresentMask)==KArmV45PdeSection )
{
__ASSERT_DEBUG(!(a&KChunkMask), MM::Panic(MM::EUnmapBadAlignment));
PageDirectory[pdeIndex]=0;
InvalidateTLBForPage(a);
a=next;
NKern::FlashSystem();
continue;
}
TInt ptid = GetPageTableId(a);
SPageTableInfo& ptinfo=iPtInfo[ptid];
if (ptid>=0)
{
TPte* ppte = PageTable(ptid) + ((a&KChunkMask)>>KPageShift);
TPte* ppte_End = ppte + to_do;
for (; ppte<ppte_End; ++ppte, a+=KPageSize)
{
TUint pte_type = *ppte & KPteTypeMask;
if (pte_type && pte_type != KArmV45PteLargePage)
{
--ptinfo.iCount;
*ppte=0;
InvalidateTLBForPage(a);
}
else if (pte_type)
{
__ASSERT_DEBUG(!(a&KLargePageMask), MM::Panic(MM::EUnmapBadAlignment));
ptinfo.iCount-=KLargeSmallPageRatio;
memclr(ppte, KLargeSmallPageRatio*sizeof(TPte));
InvalidateTLBForPage(a);
a+=(KLargePageSize-KPageSize);
ppte+=(KLargeSmallPageRatio-1);
}
NKern::FlashSystem();
}
}
else
a += (to_do<<KPageShift);
}
NKern::UnlockSystem();
}
TInt ArmMmu::AllocDomain()
{
NKern::FMWait(&DomainLock);
TInt r=-1;
if (Domains)
{
r=__e32_find_ls1_32(Domains);
Domains &= ~(1<<r);
}
NKern::FMSignal(&DomainLock);
return r;
}
void ArmMmu::FreeDomain(TInt aDomain)
{
__ASSERT_ALWAYS(aDomain>=0 && aDomain<ENumDomains, MM::Panic(MM::EFreeInvalidDomain));
TUint32 m=1<<aDomain;
NKern::FMWait(&DomainLock);
__ASSERT_ALWAYS(!(Domains&m), MM::Panic(MM::EFreeDomainNotAllocated));
Domains|=m;
NKern::FMSignal(&DomainLock);
}
void ArmMmu::ClearPages(TInt aNumPages, TPhysAddr* aPageList, TUint8 aClearByte)
{
//map the pages at a temporary address, clear them and unmap
__ASSERT_MUTEX(RamAllocatorMutex);
while (--aNumPages >= 0)
{
TPhysAddr pa;
if((TInt)aPageList&1)
{
pa = (TPhysAddr)aPageList&~1;
*(TPhysAddr*)&aPageList += iPageSize;
}
else
pa = *aPageList++;
*iTempPte = pa | SP_PTE(KArmV45PermRWNO, KMemAttBuf);
__DRAIN_WRITE_BUFFER;
InvalidateTLBForPage(iTempAddr);
memset((TAny*)iTempAddr, aClearByte, iPageSize);
}
*iTempPte=0;
__DRAIN_WRITE_BUFFER;
InvalidateTLBForPage(iTempAddr);
}
TLinAddr DoMapTemp(TPhysAddr aPage, TBool aCached, TLinAddr aTempAddr, TPte* aTempPte)
{
__ASSERT_DEBUG(!*aTempPte,MM::Panic(MM::ETempMappingAlreadyInUse));
*aTempPte = (aPage&~KPageMask) | SP_PTE(KArmV45PermRWNO, aCached?KDefaultCaching:KMemAttBuf);
__DRAIN_WRITE_BUFFER;
return aTempAddr;
}
/**
Create a temporary mapping of a physical page.
The RamAllocatorMutex must be held before this function is called and not released
until after UnmapTemp has been called.
@param aPage The physical address of the page to be mapped.
@param aCached Whether to map the page cached or not.
@return The linear address of where the page has been mapped.
*/
TLinAddr ArmMmu::MapTemp(TPhysAddr aPage, TBool aCached)
{
__ASSERT_MUTEX(RamAllocatorMutex);
return DoMapTemp(aPage, aCached, iTempAddr, iTempPte);
}
/**
Create a temporary mapping of a physical page, distinct from that created by MapTemp.
The RamAllocatorMutex must be held before this function is called and not released
until after UnmapSecondTemp has been called.
@param aPage The physical address of the page to be mapped.
@param aCached Whether to map the page cached or not.
@return The linear address of where the page has been mapped.
*/
TLinAddr ArmMmu::MapSecondTemp(TPhysAddr aPage, TBool aCached)
{
__ASSERT_MUTEX(RamAllocatorMutex);
return DoMapTemp(aPage, aCached, iSecondTempAddr, iSecondTempPte);
}
void DoUnmapTemp(TLinAddr aTempAddr, TPte* aTempPte)
{
*aTempPte = 0;
__DRAIN_WRITE_BUFFER;
InvalidateTLBForPage(aTempAddr);
}
/**
Remove the temporary mapping created with MapTemp.
*/
void ArmMmu::UnmapTemp()
{
__ASSERT_MUTEX(RamAllocatorMutex);
DoUnmapTemp(iTempAddr, iTempPte);
}
/**
Remove the temporary mapping created with MapSecondTemp.
*/
void ArmMmu::UnmapSecondTemp()
{
__ASSERT_MUTEX(RamAllocatorMutex);
DoUnmapTemp(iSecondTempAddr, iSecondTempPte);
}
/*
* Performs cache maintenance on physical cache (VIPT & PIPT) for a page to be reused.
*/
void ArmMmu::CacheMaintenanceOnDecommit(TPhysAddr aAddr)
{
CacheMaintenance::PageToReusePhysicalCache(aAddr);
}
void ArmMmu::CacheMaintenanceOnDecommit(const TPhysAddr* aAddr, TInt aCount)
{
while (--aCount>=0)
ArmMmu::CacheMaintenanceOnDecommit(*aAddr++);
}
void ArmMmu::CacheMaintenanceOnPreserve(TPhysAddr, TUint)
{
//Not required for moving memory model
__ASSERT_ALWAYS(0, Panic(ECacheMaintenance));
}
void ArmMmu::CacheMaintenanceOnPreserve(const TPhysAddr*, TInt, TUint)
{
//Not required for moving memory model
__ASSERT_ALWAYS(0, Panic(ECacheMaintenance));
}
void ArmMmu::CacheMaintenanceOnPreserve(TPhysAddr , TInt , TLinAddr , TUint )
{
//Not required for moving memory model
__ASSERT_ALWAYS(0, Panic(ECacheMaintenance));
}
TInt ArmMmu::UnlockRamCachePages(TUint8* volatile & aBase, TInt aStartPage, TInt aNumPages)
{
NKern::LockSystem();
for(;;)
{
TInt page = ((TLinAddr)aBase>>KPageShift)+aStartPage;
TPde* pd = PageDirectory+(page>>(KChunkShift-KPageShift));
TPte* pt = SafePageTableFromPde(*pd++);
TInt pteIndex = page&(KChunkMask>>KPageShift);
if(!pt)
{
// whole page table has gone, so skip all pages in it...
TInt pagesInPt = (KChunkSize>>KPageShift)-pteIndex;
aNumPages -= pagesInPt;
aStartPage += pagesInPt;
if(aNumPages>0)
continue;
NKern::UnlockSystem();
return KErrNone;
}
pt += pteIndex;
do
{
TInt pagesInPt = (KChunkSize>>KPageShift)-pteIndex;
if(pagesInPt>aNumPages)
pagesInPt = aNumPages;
if(pagesInPt>KMaxPages)
pagesInPt = KMaxPages;
aNumPages -= pagesInPt;
aStartPage += pagesInPt;
do
{
TPte pte = *pt++;
if(pte!=KPteNotPresentEntry) // pte may be null if page has already been unlocked and reclaimed by system
iRamCache->DonateRamCachePage(SPageInfo::FromPhysAddr(pte));
}
while(--pagesInPt);
if(!aNumPages)
{
NKern::UnlockSystem();
return KErrNone;
}
pteIndex = aStartPage&(KChunkMask>>KPageShift);
}
while(!NKern::FlashSystem() && pteIndex);
}
}
TInt ArmMmu::LockRamCachePages(TUint8* volatile & aBase, TInt aStartPage, TInt aNumPages)
{
NKern::LockSystem();
for(;;)
{
TInt page = ((TLinAddr)aBase>>KPageShift)+aStartPage;
TPde* pd = PageDirectory+(page>>(KChunkShift-KPageShift));
TPte* pt = SafePageTableFromPde(*pd++);
TInt pteIndex = page&(KChunkMask>>KPageShift);
if(!pt)
goto not_found;
pt += pteIndex;
do
{
TInt pagesInPt = (KChunkSize>>KPageShift)-pteIndex;
if(pagesInPt>aNumPages)
pagesInPt = aNumPages;
if(pagesInPt>KMaxPages)
pagesInPt = KMaxPages;
aNumPages -= pagesInPt;
aStartPage += pagesInPt;
do
{
TPte pte = *pt++;
if(pte==KPteNotPresentEntry)
goto not_found;
if(!iRamCache->ReclaimRamCachePage(SPageInfo::FromPhysAddr(pte)))
goto not_found;
}
while(--pagesInPt);
if(!aNumPages)
{
NKern::UnlockSystem();
return KErrNone;
}
pteIndex = aStartPage&(KChunkMask>>KPageShift);
}
while(!NKern::FlashSystem() && pteIndex);
}
not_found:
NKern::UnlockSystem();
return KErrNotFound;
}
void RamCache::SetFree(SPageInfo* aPageInfo)
{
// Make a page free
SPageInfo::TType type = aPageInfo->Type();
if(type==SPageInfo::EPagedCache)
{
TInt offset = aPageInfo->Offset()<<KPageShift;
DArmPlatChunk* chunk = (DArmPlatChunk*)aPageInfo->Owner();
__NK_ASSERT_DEBUG(TUint(offset)<TUint(chunk->iMaxSize));
TLinAddr lin = ((TLinAddr)chunk->iBase)+offset;
TPte* pt = PtePtrFromLinAddr(lin);
*pt = KPteNotPresentEntry;
__DRAIN_WRITE_BUFFER;
InvalidateTLBForPage(lin);
((ArmMmu*)iMmu)->SyncCodeMappings();
CacheMaintenance::PageToReuseVirtualCache(lin);
// actually decommit it from chunk...
TInt ptid = ((TLinAddr)pt-KPageTableBase)>>KPageTableShift;
SPageTableInfo& ptinfo=((ArmMmu*)iMmu)->iPtInfo[ptid];
if(!--ptinfo.iCount)
{
((ArmMmu*)iMmu)->DoUnassignPageTable(lin);
chunk->RemovePde(offset);
NKern::UnlockSystem();
((ArmMmu*)iMmu)->FreePageTable(ptid);
NKern::LockSystem();
}
}
else
{
__KTRACE_OPT2(KPAGING,KPANIC,Kern::Printf("DP: SetFree() with bad page type = %d",aPageInfo->Type()));
Panic(EUnexpectedPageType);
}
}
//
// MemModelDemandPaging
//
class MemModelDemandPaging : public DemandPaging
{
public:
// From RamCacheBase
virtual void Init2();
virtual TInt Init3();
virtual TBool PageUnmapped(SPageInfo* aPageInfo);
// From DemandPaging
virtual TInt Fault(TAny* aExceptionInfo);
virtual void SetOld(SPageInfo* aPageInfo);
virtual void SetFree(SPageInfo* aPageInfo);
virtual void NotifyPageFree(TPhysAddr aPage);
virtual TInt EnsurePagePresent(TLinAddr aPage, DProcess* aProcess);
virtual TPhysAddr LinearToPhysical(TLinAddr aPage, DProcess* aProcess);
virtual void AllocLoadAddress(DPagingRequest& aReq, TInt aDeviceId);
virtual TInt PageState(TLinAddr aAddr);
virtual TBool NeedsMutexOrderCheck(TLinAddr aStartAddr, TUint aLength);
// New
inline ArmMmu& Mmu() { return (ArmMmu&)*iMmu; }
void InitRomPaging();
void InitCodePaging();
TInt HandleFault(TArmExcInfo& aExc, TLinAddr aFaultAddress, TBool aInRom);
TInt PageIn(TLinAddr aAddress, DMemModelCodeSegMemory* aCodeSegMemory);
private:
TLinAddr GetLinearAddress(SPageInfo* aPageInfo);
};
//
// MemModelDemandPaging
//
DemandPaging* DemandPaging::New()
{
return new MemModelDemandPaging();
}
void MemModelDemandPaging::Init2()
{
__KTRACE_OPT2(KPAGING,KBOOT,Kern::Printf(">MemModelDemandPaging::Init2"));
DemandPaging::Init2();
__KTRACE_OPT2(KPAGING,KBOOT,Kern::Printf("<MemModelDemandPaging::Init2"));
}
void MemModelDemandPaging::AllocLoadAddress(DPagingRequest& aReq, TInt aReqId)
{
aReq.iLoadAddr = iTempPages + aReqId * KPageSize;
aReq.iLoadPte = PtePtrFromLinAddr(aReq.iLoadAddr);
}
TInt MemModelDemandPaging::Init3()
{
TInt r=DemandPaging::Init3();
if(r!=KErrNone)
return r;
// Create a region for mapping pages during page in
DPlatChunkHw* chunk;
TInt chunkSize = KMaxPagingDevices * KPagingRequestsPerDevice * KPageSize;
DPlatChunkHw::DoNew(chunk, KPhysAddrInvalid, chunkSize, EMapAttrSupRw|EMapAttrFullyBlocking);
if(!chunk)
Panic(EInitialiseFailed);
iTempPages = chunk->iLinAddr;
if(RomPagingRequested())
InitRomPaging();
if (CodePagingRequested())
InitCodePaging();
__KTRACE_OPT2(KPAGING,KBOOT,Kern::Printf("<MemModelDemandPaging::Init3"));
return KErrNone;
}
void MemModelDemandPaging::InitRomPaging()
{
// Make page tables for demand paged part of ROM...
__KTRACE_OPT2(KPAGING,KBOOT,Kern::Printf("MemModelDemandPaging::Init3 making page tables for paged ROM"));
TLinAddr lin = iRomPagedLinearBase&~KChunkMask; // first chunk with paged ROM in
TLinAddr linEnd = iRomLinearBase+iRomSize;
while(lin<linEnd)
{
// Get a Page Table
TInt ptid = Mmu().PageTableId(lin);
if(ptid<0)
{
MmuBase::Wait();
ptid = Mmu().AllocPageTable();
MmuBase::Signal();
__NK_ASSERT_DEBUG(ptid>=0);
Mmu().PtInfo(ptid).SetGlobal(lin >> KChunkShift);
}
// Get new page table addresses
TPte* pt = PageTable(ptid);
TPhysAddr ptPhys=Mmu().LinearToPhysical((TLinAddr)pt);
// Pointer to page dirctory entry
TPde* ppde = PageDirectory + (lin>>KChunkShift);
// Fill in Page Table
TPte* ptEnd = pt+(1<<(KChunkShift-KPageShift));
pt += (lin&KChunkMask)>>KPageShift;
do
{
if(lin<iRomPagedLinearBase)
*pt++ = Mmu().LinearToPhysical(lin) | KRomPtePermissions;
else
*pt++ = KPteNotPresentEntry;
lin += KPageSize;
}
while(pt<ptEnd && lin<=linEnd);
__DRAIN_WRITE_BUFFER;
// Add new Page Table to the Page Directory
TPde newpde = ptPhys | KShadowPdePerm;
__KTRACE_OPT2(KPAGING,KMMU,Kern::Printf("Writing PDE %08x to %08x", newpde, ppde));
TInt irq=NKern::DisableAllInterrupts();
*ppde = newpde;
__DRAIN_WRITE_BUFFER;
FlushTLBs();
NKern::RestoreInterrupts(irq);
}
}
void MemModelDemandPaging::InitCodePaging()
{
// Initialise code paging info
iCodeLinearBase = Mmu().iUserCodeBase;
iCodeSize = Mmu().iMaxUserCodeSize;
}
/**
@return ETrue when the unmapped page should be freed, EFalse otherwise
*/
TBool MemModelDemandPaging::PageUnmapped(SPageInfo* aPageInfo)
{
SPageInfo::TType type = aPageInfo->Type();
if(type!=SPageInfo::EPagedCache && type!=SPageInfo::EPagedCode)
{
__NK_ASSERT_DEBUG(type!=SPageInfo::EPagedData); // not supported yet
return ETrue;
}
RemovePage(aPageInfo);
AddAsFreePage(aPageInfo);
// Return false to stop DMemModelChunk::DoDecommit from freeing this page
return EFalse;
}
TLinAddr MemModelDemandPaging::GetLinearAddress(SPageInfo* aPageInfo)
{
TInt offset = aPageInfo->Offset()<<KPageShift;
SPageInfo::TType type = aPageInfo->Type();
__NK_ASSERT_DEBUG(TUint(offset)<(type==SPageInfo::EPagedROM ? iRomSize : iCodeSize));
TLinAddr base = type==SPageInfo::EPagedROM ? iRomLinearBase : iCodeLinearBase;
return base + offset;
}
void MemModelDemandPaging::SetOld(SPageInfo* aPageInfo)
{
__NK_ASSERT_DEBUG(aPageInfo->State() == SPageInfo::EStatePagedOld);
SPageInfo::TType type = aPageInfo->Type();
if(type==SPageInfo::EPagedROM || type==SPageInfo::EPagedCode)
{
START_PAGING_BENCHMARK;
// get linear address of page...
TLinAddr lin = GetLinearAddress(aPageInfo);
// make page inaccessible...
TPte* pt = PtePtrFromLinAddr(lin);
*pt &= ~KPtePresentMask;
__DRAIN_WRITE_BUFFER;
InvalidateTLBForPage(lin);
Mmu().SyncCodeMappings();
if (type==SPageInfo::EPagedCode)
END_PAGING_BENCHMARK(this, EPagingBmSetCodePageOld);
}
else if(type==SPageInfo::EPagedCache)
{
// leave page accessible
}
else if(type!=SPageInfo::EPagedFree)
{
__KTRACE_OPT2(KPAGING,KPANIC,Kern::Printf("DP: SetOld() with bad page type = %d",aPageInfo->Type()));
Panic(EUnexpectedPageType);
}
NKern::FlashSystem();
}
void MemModelDemandPaging::SetFree(SPageInfo* aPageInfo)
{
__ASSERT_SYSTEM_LOCK;
__ASSERT_MUTEX(MmuBase::RamAllocatorMutex);
__NK_ASSERT_DEBUG(aPageInfo->State() == SPageInfo::EStatePagedDead);
if(aPageInfo->LockCount())
Panic(ERamPageLocked);
SPageInfo::TType type = aPageInfo->Type();
if(type==SPageInfo::EPagedROM || type==SPageInfo::EPagedCode)
{
START_PAGING_BENCHMARK;
// get linear address of page...
TLinAddr lin = GetLinearAddress(aPageInfo);
// unmap it...
TPte* pt = PtePtrFromLinAddr(lin);
*pt = KPteNotPresentEntry;
__DRAIN_WRITE_BUFFER;
InvalidateTLBForPage(lin);
Mmu().SyncCodeMappings();
if (type==SPageInfo::EPagedCode)
END_PAGING_BENCHMARK(this, EPagingBmSetCodePageFree);
#ifdef BTRACE_PAGING
TInt subCat = type==SPageInfo::EPagedCode ? BTrace::EPagingPageOutCode : BTrace::EPagingPageOutROM;
TPhysAddr phys = aPageInfo->PhysAddr();
BTraceContext8(BTrace::EPaging,subCat,phys,lin);
#endif
}
else if(type==SPageInfo::EPagedCache)
{
// get linear address of page...
TInt offset = aPageInfo->Offset()<<KPageShift;
DArmPlatChunk* chunk = (DArmPlatChunk*)aPageInfo->Owner();
__NK_ASSERT_DEBUG(TUint(offset)<TUint(chunk->iMaxSize));
TLinAddr lin = ((TLinAddr)chunk->iBase)+offset;
// unmap it...
TPte* pt = PtePtrFromLinAddr(lin);
*pt = KPteNotPresentEntry;
__DRAIN_WRITE_BUFFER;
InvalidateTLBForPage(lin);
Mmu().SyncCodeMappings();
NKern::UnlockSystem();
CacheMaintenance::PageToReuseVirtualCache(lin);
NKern::LockSystem();
// actually decommit it from chunk...
TInt ptid = ((TLinAddr)pt-KPageTableBase)>>KPageTableShift;
SPageTableInfo& ptinfo=((ArmMmu*)iMmu)->iPtInfo[ptid];
if(!--ptinfo.iCount)
{
((ArmMmu*)iMmu)->DoUnassignPageTable(lin);
chunk->RemovePde(offset);
NKern::UnlockSystem();
((ArmMmu*)iMmu)->FreePageTable(ptid);
NKern::LockSystem();
}
#ifdef BTRACE_PAGING
TPhysAddr phys = aPageInfo->PhysAddr();
BTraceContext8(BTrace::EPaging,BTrace::EPagingPageOutCache,phys,lin);
#endif
}
else if(type==SPageInfo::EPagedFree)
{
// already free...
#ifdef BTRACE_PAGING
TPhysAddr phys = aPageInfo->PhysAddr();
BTraceContext4(BTrace::EPaging,BTrace::EPagingPageOutFree,phys);
#endif
// external cache may not have been cleaned if PageUnmapped called
CacheMaintenance::PageToReusePhysicalCache(aPageInfo->PhysAddr());
}
else
{
__KTRACE_OPT2(KPAGING,KPANIC,Kern::Printf("DP: SetFree() with bad page type = %d",aPageInfo->Type()));
Panic(EUnexpectedPageType);
}
NKern::FlashSystem();
}
void MemModelDemandPaging::NotifyPageFree(TPhysAddr aPage)
{
MM::Panic(MM::EOperationNotImplemented);
}
/**
Return True if exception was caused by a memory write access.
This function can cause a paging exception!
*/
static TBool FaultDuringWrite(TArmExcInfo& aExc)
{
// We can't decode jazelle instruction to determine if they faulted during a read.
// Therefore we will treat them as writes (which will panic the thread)...
if(aExc.iCpsr&(1<<24))
return ETrue;
if(aExc.iCpsr&(1<<5))
{
// thumb
TUint32 op = *(TUint16*)aExc.iR15;
switch((op>>13)&7)
{
case 2:
if((op&0xfa00)==0x5000)
return ETrue; // STR (2) and STRB (2)
if((op&0xfe00)==0x5200)
return ETrue; // STRH (2)
return EFalse;
case 3:
return !(op&(1<<11)); // STR (1) and STRB (1)
case 4:
return !(op&(1<<11)); // STR (3) and STRH (1)
case 5:
return (op&0xfe00)==0xb400; // PUSH
case 6:
return (op&0xf800)==0xc000; // STMIA
}
}
else
{
// ARM
TUint32 op = *(TUint32*)aExc.iR15;
if(op<0xf0000000)
{
switch((op>>25)&7)
{
case 0:
if((op&0xf0)==(0xb0))
return !(op&(1<<20)); // load/store halfword
else if((op&0x0e1000f0)==(0x000000f0))
return ETrue; // store double
else if((op&0x0fb000f0) == 0x010000f0)
return ETrue; // swap instruction
else if((op&0x0ff000f0) == 0x01800090)
return ETrue; // strex
return EFalse;
case 2:
return !(op&(1<<20)); // load/store immediate
case 3:
if(!(op&0x10))
return !(op&(1<<20)); // load/store register offset
return EFalse;
case 4:
return !(op&(1<<20)); // load/store multiple
case 6:
return !(op&(1<<20)); // coproc store
}
}
else
{
switch((op>>25)&7)
{
case 4:
if((op&0xfe5f0f00)==(0xf84d0500))
return ETrue; // SRS instructions
return EFalse;
case 6:
return !(op&(1<<20)); // coproc store (STC2)
}
}
}
return EFalse;
}
TInt MemModelDemandPaging::Fault(TAny* aExceptionInfo)
{
TArmExcInfo& exc=*(TArmExcInfo*)aExceptionInfo;
// Get faulting address
TLinAddr faultAddress = exc.iFaultAddress;
if(exc.iExcCode==EArmExceptionDataAbort)
{
// Only handle page translation faults
if((exc.iFaultStatus&0xf)!=0x7)
return KErrUnknown;
// Let writes take an exception rather than page in any memory...
if(FaultDuringWrite(exc))
return KErrUnknown;
}
else if (exc.iExcCode != EArmExceptionPrefetchAbort)
return KErrUnknown; // Not prefetch or data abort
DThread* thread = TheCurrentThread;
// check which ragion fault occured in...
TBool inRom=ETrue;
if(TUint(faultAddress-iRomPagedLinearBase)<iRomPagedSize)
{
// in ROM
}
else if(TUint(faultAddress-iCodeLinearBase)<iCodeSize)
{
// in code
inRom=EFalse;
}
else
return KErrUnknown; // Not in pageable region
// Check if thread holds fast mutex and claim system lock
NFastMutex* fm = NKern::HeldFastMutex();
TPagingExcTrap* trap = thread->iPagingExcTrap;
if(!fm)
NKern::LockSystem();
else
{
if(!trap || fm!=&TheScheduler.iLock)
{
__KTRACE_OPT2(KPAGING,KPANIC,Kern::Printf("DP: Fault with FM Held! %x (%O pc=%x)",faultAddress,&Kern::CurrentThread(),exc.iR15));
Panic(EPageFaultWhilstFMHeld); // Not allowed to hold mutexes
}
// Current thread already has the system lock...
NKern::FlashSystem(); // Let someone else have a go with the system lock.
}
// System locked here
TInt r = KErrNone;
if(thread->IsRealtime())
r = CheckRealtimeThreadFault(thread, aExceptionInfo);
if (r == KErrNone)
r = HandleFault(exc, faultAddress, inRom);
// Restore system lock state
if (fm != NKern::HeldFastMutex())
{
if (fm)
NKern::LockSystem();
else
NKern::UnlockSystem();
}
// Deal with XTRAP_PAGING
if(r == KErrNone && trap)
{
trap->Exception(1); // Return from exception trap with result '1' (value>0)
// code doesn't continue beyond this point.
}
return r;
}
TInt MemModelDemandPaging::HandleFault(TArmExcInfo& aExc, TLinAddr aFaultAddress, TBool aInRom)
{
++iEventInfo.iPageFaultCount;
// get page table entry...
TPte* pt = SafePtePtrFromLinAddr(aFaultAddress);
if(!pt)
return KErrNotFound;
TPte pte = *pt;
// Do what is required to make page accessible...
if(pte&KPtePresentMask)
{
// PTE is present, so assume it has already been dealt with
#ifdef BTRACE_PAGING
BTraceContext12(BTrace::EPaging,BTrace::EPagingPageNop,pte&~KPageMask,aFaultAddress,aExc.iR15);
#endif
return KErrNone;
}
if(pte!=KPteNotPresentEntry)
{
// PTE alread has a page
SPageInfo* pageInfo = SPageInfo::FromPhysAddr(pte);
if(pageInfo->State()==SPageInfo::EStatePagedDead)
{
// page currently being unmapped, so do that here...
*pt = KPteNotPresentEntry; // Update page table
__DRAIN_WRITE_BUFFER;
}
else
{
// page just needs making young again...
*pt = TPte(pte|KArmPteSmallPage); // Update page table
__DRAIN_WRITE_BUFFER;
Rejuvenate(pageInfo);
#ifdef BTRACE_PAGING
BTraceContext12(BTrace::EPaging,BTrace::EPagingRejuvenate,pte&~KPageMask,aFaultAddress,aExc.iR15);
#endif
return KErrNone;
}
}
// PTE not present, so page it in...
// check if fault in a CodeSeg...
DMemModelCodeSegMemory* codeSegMemory = NULL;
if (aInRom)
NKern::ThreadEnterCS();
else
{
// find CodeSeg...
DMemModelCodeSeg* codeSeg = (DMemModelCodeSeg*)DCodeSeg::CodeSegsByAddress.Find(aFaultAddress);
if (!codeSeg)
return KErrNotFound;
codeSegMemory = codeSeg->Memory();
if (codeSegMemory==0 || !codeSegMemory->iIsDemandPaged)
return KErrNotFound;
// open reference on CodeSegMemory
NKern::ThreadEnterCS();
#ifdef _DEBUG
TInt r =
#endif
codeSegMemory->Open();
__NK_ASSERT_DEBUG(r==KErrNone);
NKern::FlashSystem();
}
#ifdef BTRACE_PAGING
BTraceContext8(BTrace::EPaging,BTrace::EPagingPageInBegin,aFaultAddress,aExc.iR15);
#endif
TInt r = PageIn(aFaultAddress,codeSegMemory);
NKern::UnlockSystem();
if(codeSegMemory)
codeSegMemory->Close();
NKern::ThreadLeaveCS();
return r;
}
TInt MemModelDemandPaging::PageIn(TLinAddr aAddress, DMemModelCodeSegMemory* aCodeSegMemory)
{
// Get a request object - this may block until one is available
DPagingRequest* req = AcquireRequestObject();
// Get page table entry
TPte* pt = SafePtePtrFromLinAddr(aAddress);
// Check page is still required...
if(!pt || *pt!=KPteNotPresentEntry)
{
#ifdef BTRACE_PAGING
BTraceContext0(BTrace::EPaging,BTrace::EPagingPageInUnneeded);
#endif
ReleaseRequestObject(req);
return pt ? KErrNone : KErrNotFound;
}
++iEventInfo.iPageInReadCount;
// Get a free page
SPageInfo* pageInfo = AllocateNewPage();
__NK_ASSERT_DEBUG(pageInfo);
// Get physical address of free page
TPhysAddr phys = pageInfo->PhysAddr();
__NK_ASSERT_DEBUG(phys!=KPhysAddrInvalid);
// Temporarily map free page
TLinAddr loadAddr = req->iLoadAddr;
pt = req->iLoadPte;
*pt = phys | SP_PTE(KArmV45PermRWNO, KMemAttTempDemandPaging);
__DRAIN_WRITE_BUFFER;
// Read page from backing store
aAddress &= ~KPageMask;
NKern::UnlockSystem();
TInt r;
if (!aCodeSegMemory)
r = ReadRomPage(req, aAddress);
else
{
r = ReadCodePage(req, aCodeSegMemory, aAddress);
if (r == KErrNone)
aCodeSegMemory->ApplyCodeFixups((TUint32*)loadAddr, aAddress);
}
if(r!=KErrNone)
Panic(EPageInFailed);
// make caches consistant (uncached memory is used for page loading)
__DRAIN_WRITE_BUFFER;
NKern::LockSystem();
// Invalidate temporary mapping
*pt = KPteNotPresentEntry;
__DRAIN_WRITE_BUFFER;
InvalidateTLBForPage(loadAddr);
ReleaseRequestObject(req);
// Get page table entry
pt = SafePtePtrFromLinAddr(aAddress);
// Check page still needs updating
TBool notNeeded = pt==0 || *pt!=KPteNotPresentEntry;
if(notNeeded)
{
// We don't need the new page after all, so put it on the active list as a free page
__KTRACE_OPT(KPAGING,Kern::Printf("DP: PageIn (New page not used)"));
#ifdef BTRACE_PAGING
BTraceContext0(BTrace::EPaging,BTrace::EPagingPageInUnneeded);
#endif
AddAsFreePage(pageInfo);
return pt ? KErrNone : KErrNotFound;
}
// Update page info
if (!aCodeSegMemory)
pageInfo->SetPagedROM((aAddress-iRomLinearBase)>>KPageShift);
else
pageInfo->SetPagedCode(aCodeSegMemory,(aAddress-Mmu().iUserCodeBase)>>KPageShift);
// Map page into final location
*pt = phys | (aCodeSegMemory ? KUserCodeLoadPte : KRomPtePermissions);
__DRAIN_WRITE_BUFFER;
#ifdef BTRACE_PAGING
TInt subCat = aCodeSegMemory ? BTrace::EPagingPageInCode : BTrace::EPagingPageInROM;
BTraceContext8(BTrace::EPaging,subCat,phys,aAddress);
#endif
AddAsYoungest(pageInfo);
BalanceAges();
return KErrNone;
}
inline TUint8 ReadByte(TLinAddr aAddress)
{ return *(volatile TUint8*)aAddress; }
TInt MemModelDemandPaging::EnsurePagePresent(TLinAddr aPage, DProcess* aProcess)
{
XTRAPD(exc,XT_DEFAULT,XTRAP_PAGING_RETRY(CHECK_PAGING_SAFE; ReadByte(aPage);));
return exc;
}
TPhysAddr MemModelDemandPaging::LinearToPhysical(TLinAddr aPage, DProcess* aProcess)
{
return Mmu().LinearToPhysical(aPage);
}
TInt MemModelDemandPaging::PageState(TLinAddr aAddr)
{
TPte* ptePtr = 0;
TPte pte = 0;
TInt r = 0;
SPageInfo* pageInfo = NULL;
NKern::LockSystem();
DMemModelCodeSegMemory* codeSegMemory = 0;
if(TUint(aAddr-iRomPagedLinearBase)<iRomPagedSize)
r |= EPageStateInRom;
else if (TUint(aAddr-iCodeLinearBase)<iCodeSize)
{
DMemModelCodeSeg* codeSeg = (DMemModelCodeSeg*)DCodeSeg::CodeSegsByAddress.Find(aAddr);
if(codeSeg)
codeSegMemory = codeSeg->Memory();
if (codeSegMemory)
{
r |= EPageStateInRamCode;
if (codeSegMemory->iIsDemandPaged)
r |= EPageStatePaged;
}
}
ptePtr = SafePtePtrFromLinAddr(aAddr);
if (!ptePtr)
goto done;
r |= EPageStatePageTablePresent;
pte = *ptePtr;
if (pte == KPteNotPresentEntry)
goto done;
r |= EPageStatePtePresent;
if (pte & KPtePresentMask)
r |= EPageStatePteValid;
pageInfo = SPageInfo::FromPhysAddr(pte);
r |= pageInfo->Type();
r |= pageInfo->State()<<8;
done:
NKern::UnlockSystem();
return r;
}
TBool MemModelDemandPaging::NeedsMutexOrderCheck(TLinAddr aStartAddr, TUint aLength)
{
// Don't check mutex order for reads from unpaged rom, kernel data area and kernel stack chunk
TLinAddr endAddr = aStartAddr + aLength;
TLinAddr stackBase = (TLinAddr)MM::SvStackChunk->Base();
TLinAddr stackEnd = stackBase + MM::SvStackChunk->iMaxSize;
TLinAddr unpagedRomEnd = iRomPagedLinearBase ? iRomPagedLinearBase : iRomLinearBase + iRomSize;
TBool rangeInUnpagedRom = aStartAddr >= iRomLinearBase && endAddr <= unpagedRomEnd;
TBool rangeInKernelData = aStartAddr >= KKernelDataBase && endAddr <= KKernelDataEnd;
TBool rangeInKernelStack = aStartAddr >= stackBase && endAddr <= stackEnd;
return !rangeInUnpagedRom && !rangeInKernelData && !rangeInKernelStack;
}
EXPORT_C TBool DDemandPagingLock::Lock(DThread* aThread, TLinAddr aStart, TInt aSize)
{
MemModelDemandPaging* pager = (MemModelDemandPaging*)iThePager;
if(pager)
{
ArmMmu& m = pager->Mmu();
TLinAddr end = aStart+aSize;
if ((aStart < TUint(pager->iRomPagedLinearBase+pager->iRomPagedSize) && end > pager->iRomPagedLinearBase) ||
(aStart < TUint(m.iUserCodeBase + m.iMaxUserCodeSize) && end > m.iUserCodeBase))
return pager->ReserveLock(aThread,aStart,aSize,*this);
}
return EFalse;
}
void ArmMmu::DisablePageModification(DMemModelChunk* aChunk, TInt aOffset)
//
// Mark the page at aOffset in aChunk inaccessible to prevent it being
// modified while defrag is in progress. Save the required information
// to allow the fault handler to deal with this.
// Flush the cache for the page so that it can be aliased elsewhere for
// copying.
// Call this with the system unlocked.
//
{
__KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::DisablePageModification() offset=%08x", aOffset));
// Acquire the system lock here for atomic access to aChunk->iBase as moving
// between the home and run addresses (a reschedule) may update aChunk->iBase.
NKern::LockSystem();
iDisabledAddr = (TLinAddr)(aChunk->iBase) + aOffset;
TInt ptid=GetPageTableId(iDisabledAddr);
if(ptid<0)
Panic(EDefragDisablePageFailed);
TPte* pPte = PageTable(ptid) + ((aOffset&KChunkMask)>>KPageShift);
TPte pte = *pPte;
if ((pte & KPteTypeMask) != KArmPteSmallPage)
Panic(EDefragDisablePageFailed);
iDisabledPte = pPte;
iDisabledOldVal = pte;
*pPte = 0;
__DRAIN_WRITE_BUFFER;
InvalidateTLBForPage(iDisabledAddr);
NKern::UnlockSystem();
CacheMaintenance::PageToPreserveAndReuseVirtualCache(iDisabledAddr);
__DRAIN_WRITE_BUFFER;
}
TBool FaultStatusFromLinAddr(TLinAddr aAddr, TBool aKernel, TUint32& aFaultStatus)
// Walk the page tables looking for the given linear address. If access
// would've caused a fault, return ETrue and fill in aFaultStatus with a
// FSR value. Otherwise, return EFalse. Assumes it was a read.
{
TPde pde = PageDirectory[aAddr>>KChunkShift];
TPde pdetype = pde & KPdeTypeMask;
if (pdetype == 0)
{
// section translation fault
aFaultStatus = 0x5;
return ETrue;
}
TPte pte=0;
TInt domain = (pde >> 5) & 0xf;
TUint32 dacr = Arm::Dacr();
TInt domaccess = (dacr >> (domain<<1)) & 0x3;
TInt ispage = (pdetype == KArmV45PdeSection) ? 0 : 0x2;
if (ispage)
{
pte = *PtePtrFromLinAddr(aAddr);
if ((pte & KPteTypeMask) == 0)
{
// page translation fault
aFaultStatus = 0x7;
return ETrue;
}
}
if (domaccess == 0x3)
{
// manager access
return EFalse;
}
if (domaccess == 0)
{
// domain fault
aFaultStatus = 0x9 | ispage;
return ETrue;
}
TInt perms;
if (ispage)
perms = (pte >> 4) & 0x3;
else
perms = (pde >> 10) & 0x3;
if (aKernel || perms != 0x1)
return EFalse;
// permission fault
aFaultStatus = 0xd | ispage;
return ETrue;
}
TInt ArmMmu::RamDefragFault(TAny* aExceptionInfo)
{
TArmExcInfo& exc=*(TArmExcInfo*)aExceptionInfo;
// Get faulting address
TLinAddr faultAddress;
TBool prefetch=EFalse;
if(exc.iExcCode==EArmExceptionDataAbort)
{
// Only handle page translation faults
if((exc.iFaultStatus & 0xf) != 0x7)
return KErrUnknown;
faultAddress = exc.iFaultAddress;
}
else if(exc.iExcCode==EArmExceptionPrefetchAbort)
{
prefetch = ETrue;
faultAddress = exc.iR15;
}
else
return KErrUnknown; // Not data/prefetch abort
TBool kernelmode = exc.iCpsr&EMaskMode != EUserMode;
// Take system lock if not already held
NFastMutex* fm = NKern::HeldFastMutex();
if(!fm)
NKern::LockSystem();
else if(fm!=&TheScheduler.iLock)
{
__KTRACE_OPT2(KMMU,KPANIC,Kern::Printf("Defrag: Fault with FM Held! %x (%O pc=%x)",faultAddress,TheCurrentThread,exc.iR15));
Panic(EDefragFaultWhilstFMHeld); // Not allowed to hold mutexes
}
TInt r = KErrUnknown;
// check if the mapping of the page has already been restored and retry if so
if (prefetch)
{
TUint32 fsr;
if (!FaultStatusFromLinAddr(faultAddress, kernelmode, fsr))
{
r = KErrNone;
goto leave;
}
}
else
{
TPte* pt = SafePtePtrFromLinAddr(faultAddress);
if(!pt)
{
r = KErrNotFound;
goto leave;
}
if ((*pt & 0x3) != 0)
{
r = KErrNone;
goto leave;
}
}
// check if the fault occurred in the page we are moving
if (iDisabledPte && TUint(faultAddress - iDisabledAddr) < TUint(KPageSize))
{
// restore access to the page
*iDisabledPte = iDisabledOldVal;
__DRAIN_WRITE_BUFFER;
InvalidateTLBForPage(iDisabledAddr);
iDisabledAddr = 0;
iDisabledPte = NULL;
iDisabledOldVal = 0;
r = KErrNone;
}
leave:
// Restore system lock state
if (!fm)
NKern::UnlockSystem();
return r;
}