--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/kernel/eka/memmodel/epoc/multiple/arm/xmmu.cpp Thu Dec 17 09:24:54 2009 +0200
@@ -0,0 +1,3905 @@
+// Copyright (c) 1997-2009 Nokia Corporation and/or its subsidiary(-ies).
+// All rights reserved.
+// This component and the accompanying materials are made available
+// under the terms of the License "Eclipse Public License v1.0"
+// which accompanies this distribution, and is available
+// at the URL "http://www.eclipse.org/legal/epl-v10.html".
+//
+// Initial Contributors:
+// Nokia Corporation - initial contribution.
+//
+// Contributors:
+//
+// Description:
+// e32\memmodel\epoc\multiple\arm\xmmu.cpp
+//
+//
+
+#include "arm_mem.h"
+#include <mmubase.inl>
+#include <ramcache.h>
+#include <demand_paging.h>
+#include "execs.h"
+#include <defrag.h>
+#include "cache_maintenance.inl"
+
+#undef __MMU_MACHINE_CODED__
+
+// SECTION_PDE(perm, attr, domain, execute, global)
+// PT_PDE(domain)
+// LP_PTE(perm, attr, execute, global)
+// SP_PTE(perm, attr, execute, global)
+
+const TInt KPageColourShift=2;
+const TInt KPageColourCount=(1<<KPageColourShift);
+const TInt KPageColourMask=KPageColourCount-1;
+
+
+const TPde KPdPdePerm=PT_PDE(0);
+const TPde KPtPdePerm=PT_PDE(0);
+const TPde KShadowPdePerm=PT_PDE(0);
+
+#if defined(__CPU_MEMORY_TYPE_REMAPPING)
+// ARM1176, ARM11MPCore, ARMv7 and later
+// __CPU_MEMORY_TYPE_REMAPPING means that only three bits (TEX0:C:B) in page table define
+// memory attributes. Kernel runs with a limited set of memory types: stronlgy ordered,
+// device, normal un-cached & and normal WBWA. Due to lack of write through mode, page tables are
+// write-back which means that cache has to be cleaned on every page/directory table update.
+const TPte KPdPtePerm= SP_PTE(KArmV6PermRWNO, EMemAttNormalCached, 0, 1);
+const TPte KPtPtePerm= SP_PTE(KArmV6PermRWNO, EMemAttNormalCached, 0, 1);
+const TPte KPtInfoPtePerm= SP_PTE(KArmV6PermRWNO, EMemAttNormalCached, 0, 1);
+const TPte KRomPtePerm= SP_PTE(KArmV6PermRORO, EMemAttNormalCached, 1, 1);
+const TPte KShadowPtePerm= SP_PTE(KArmV6PermRORO, EMemAttNormalCached, 1, 1);
+const TPde KRomSectionPermissions= SECTION_PDE(KArmV6PermRORO, EMemAttNormalCached, 0, 1, 1);
+const TPte KUserCodeLoadPte= SP_PTE(KArmV6PermRWNO, EMemAttNormalCached, 1, 0);
+const TPte KUserCodeRunPte= SP_PTE(KArmV6PermRORO, EMemAttNormalCached, 1, 0);
+const TPte KGlobalCodeRunPte= SP_PTE(KArmV6PermRORO, EMemAttNormalCached, 1, 1);
+const TPte KKernelCodeRunPte= SP_PTE(KArmV6PermRONO, EMemAttNormalCached, 1, 1);
+
+const TInt KNormalUncachedAttr = EMemAttNormalUncached;
+const TInt KNormalCachedAttr = EMemAttNormalCached;
+
+#else
+
+//ARM1136
+const TPte KPtInfoPtePerm=SP_PTE(KArmV6PermRWNO, KArmV6MemAttWBWAWBWA, 0, 1);
+#if defined (__CPU_WriteThroughDisabled)
+const TPte KPdPtePerm=SP_PTE(KArmV6PermRWNO, KArmV6MemAttWBWAWBWA, 0, 1);
+const TPte KPtPtePerm=SP_PTE(KArmV6PermRWNO, KArmV6MemAttWBWAWBWA, 0, 1);
+const TPte KRomPtePerm=SP_PTE(KArmV6PermRORO, KArmV6MemAttWBWAWBWA, 1, 1);
+const TPte KShadowPtePerm=SP_PTE(KArmV6PermRWRO, KArmV6MemAttWBWAWBWA, 1, 1);
+const TPde KRomSectionPermissions = SECTION_PDE(KArmV6PermRORO, KArmV6MemAttWBWAWBWA, 0, 1, 1);
+const TPte KUserCodeLoadPte=SP_PTE(KArmV6PermRWNO, KArmV6MemAttWBWAWBWA, 1, 0);
+const TPte KUserCodeRunPte=SP_PTE(KArmV6PermRWRO, KArmV6MemAttWBWAWBWA, 1, 0);
+const TPte KGlobalCodeRunPte=SP_PTE(KArmV6PermRWRO, KArmV6MemAttWBWAWBWA, 1, 1);
+const TInt KKernelCodeRunPteAttr = KArmV6MemAttWBWAWBWA;
+#else
+const TPte KPdPtePerm=SP_PTE(KArmV6PermRWNO, KArmV6MemAttWBRAWTRA, 0, 1);
+const TPte KPtPtePerm=SP_PTE(KArmV6PermRWNO, KArmV6MemAttWBRAWTRA, 0, 1);
+const TPte KRomPtePerm=SP_PTE(KArmV6PermRORO, KArmV6MemAttWTRAWTRA, 1, 1);
+const TPte KShadowPtePerm=SP_PTE(KArmV6PermRWRO, KArmV6MemAttWTRAWTRA, 1, 1);
+const TPde KRomSectionPermissions = SECTION_PDE(KArmV6PermRORO, KArmV6MemAttWTRAWTRA, 0, 1, 1);
+const TPte KUserCodeLoadPte=SP_PTE(KArmV6PermRWNO, KArmV6MemAttWTRAWTRA, 1, 0);
+const TPte KUserCodeRunPte=SP_PTE(KArmV6PermRWRO, KArmV6MemAttWTRAWTRA, 1, 0);
+const TPte KGlobalCodeRunPte=SP_PTE(KArmV6PermRWRO, KArmV6MemAttWTRAWTRA, 1, 1);
+const TInt KKernelCodeRunPteAttr = KArmV6MemAttWTRAWTRA;
+#endif
+
+
+#if defined(__CPU_ARM1136_ERRATUM_353494_FIXED)
+const TInt KKernelCodeRunPtePerm = KArmV6PermRONO;
+#else
+const TInt KKernelCodeRunPtePerm = KArmV6PermRORO;
+#endif
+const TPte KKernelCodeRunPte=SP_PTE(KKernelCodeRunPtePerm, KKernelCodeRunPteAttr, 1, 1);
+
+const TInt KNormalUncachedAttr = KArmV6MemAttNCNC;
+const TInt KNormalCachedAttr = KArmV6MemAttWBWAWBWA;
+
+#endif
+
+
+extern void __FlushBtb();
+
+#if defined(__CPU_ARM1136__) && !defined(__CPU_ARM1136_ERRATUM_353494_FIXED)
+extern void remove_and_invalidate_page(TPte* aPte, TLinAddr aAddr, TInt aAsid);
+extern void remove_and_invalidate_section(TPde* aPde, TLinAddr aAddr, TInt aAsid);
+#endif
+
+
+LOCAL_D const TPte ChunkPtePermissions[ENumChunkTypes] =
+ {
+#if defined(__CPU_MEMORY_TYPE_REMAPPING)
+// ARM1176, ARM11 mcore, ARMv7 and later
+ SP_PTE(KArmV6PermRWNO, EMemAttNormalCached, 0, 1), // EKernelData
+ SP_PTE(KArmV6PermRWNO, EMemAttNormalCached, 0, 1), // EKernelStack
+ SP_PTE(KArmV6PermRWNO, EMemAttNormalCached, 1, 1), // EKernelCode - loading
+ SP_PTE(KArmV6PermRWNO, EMemAttNormalCached, 1, 1), // EDll (used for global code) - loading
+ SP_PTE(KArmV6PermRORO, EMemAttNormalCached, 1, 0), // EUserCode - run
+ SP_PTE(KArmV6PermRWRW, EMemAttNormalCached, 0, 1), // ERamDrive
+ SP_PTE(KArmV6PermRWRW, EMemAttNormalCached, 0, 0), // EUserData
+ SP_PTE(KArmV6PermRWRW, EMemAttNormalCached, 0, 0), // EDllData
+ SP_PTE(KArmV6PermRWRW, EMemAttNormalCached, 1, 0), // EUserSelfModCode
+ SP_PTE(KArmV6PermRWRW, EMemAttNormalCached, 0, 0), // ESharedKernelSingle
+ SP_PTE(KArmV6PermRWRW, EMemAttNormalCached, 0, 0), // ESharedKernelMultiple
+ SP_PTE(KArmV6PermRWRW, EMemAttNormalCached, 0, 0), // ESharedIo
+ SP_PTE(KArmV6PermRWNO, EMemAttNormalCached, 0, 1), // ESharedKernelMirror
+ SP_PTE(KArmV6PermRWNO, EMemAttNormalCached, 0, 1), // EKernelMessage
+#else
+ SP_PTE(KArmV6PermRWNO, KArmV6MemAttWBWAWBWA, 0, 1), // EKernelData
+ SP_PTE(KArmV6PermRWNO, KArmV6MemAttWBWAWBWA, 0, 1), // EKernelStack
+#if defined (__CPU_WriteThroughDisabled)
+ SP_PTE(KArmV6PermRWNO, KArmV6MemAttWBWAWBWA, 1, 1), // EKernelCode - loading
+ SP_PTE(KArmV6PermRWNO, KArmV6MemAttWBWAWBWA, 1, 1), // EDll (used for global code) - loading
+ SP_PTE(KArmV6PermRWRO, KArmV6MemAttWBWAWBWA, 1, 0), // EUserCode - run
+#else
+ SP_PTE(KArmV6PermRWNO, KArmV6MemAttWTRAWTRA, 1, 1), // EKernelCode - loading
+ SP_PTE(KArmV6PermRWNO, KArmV6MemAttWTRAWTRA, 1, 1), // EDll (used for global code) - loading
+ SP_PTE(KArmV6PermRWRO, KArmV6MemAttWTRAWTRA, 1, 0), // EUserCode - run
+#endif
+ SP_PTE(KArmV6PermRWRW, KArmV6MemAttWBWAWBWA, 0, 1), // ERamDrive
+ SP_PTE(KArmV6PermRWRW, KArmV6MemAttWBWAWBWA, 0, 0), // EUserData
+ SP_PTE(KArmV6PermRWRW, KArmV6MemAttWBWAWBWA, 0, 0), // EDllData
+ SP_PTE(KArmV6PermRWRW, KArmV6MemAttWBWAWBWA, 1, 0), // EUserSelfModCode
+ SP_PTE(KArmV6PermRWRW, KArmV6MemAttWBWAWBWA, 0, 0), // ESharedKernelSingle
+ SP_PTE(KArmV6PermRWRW, KArmV6MemAttWBWAWBWA, 0, 0), // ESharedKernelMultiple
+ SP_PTE(KArmV6PermRWRW, KArmV6MemAttWBWAWBWA, 0, 0), // ESharedIo
+ SP_PTE(KArmV6PermRWNO, KArmV6MemAttWBWAWBWA, 0, 1), // ESharedKernelMirror
+ SP_PTE(KArmV6PermRWNO, KArmV6MemAttWBWAWBWA, 0, 1), // EKernelMessage
+#endif
+ };
+
+// The domain for each chunk is selected according to its type.
+// The RamDrive lives in a separate domain, to minimise the risk
+// of accidental access and corruption. User chunks may also be
+// located in a separate domain (15) in DEBUG builds.
+LOCAL_D const TPde ChunkPdePermissions[ENumChunkTypes] =
+ {
+ PT_PDE(0), // EKernelData
+ PT_PDE(0), // EKernelStack
+ PT_PDE(0), // EKernelCode
+ PT_PDE(0), // EDll
+ PT_PDE(USER_MEMORY_DOMAIN), // EUserCode
+ PT_PDE(1), // ERamDrive
+ PT_PDE(USER_MEMORY_DOMAIN), // EUserData
+ PT_PDE(USER_MEMORY_DOMAIN), // EDllData
+ PT_PDE(USER_MEMORY_DOMAIN), // EUserSelfModCode
+ PT_PDE(USER_MEMORY_DOMAIN), // ESharedKernelSingle
+ PT_PDE(USER_MEMORY_DOMAIN), // ESharedKernelMultiple
+ PT_PDE(0), // ESharedIo
+ PT_PDE(0), // ESharedKernelMirror
+ PT_PDE(0), // EKernelMessage
+ };
+
+// Inline functions for simple transformations
+inline TLinAddr PageTableLinAddr(TInt aId)
+ {
+ return (KPageTableBase+(aId<<KPageTableShift));
+ }
+
+inline TPte* PageTable(TInt aId)
+ {
+ return (TPte*)(KPageTableBase+(aId<<KPageTableShift));
+ }
+
+inline TPte* PageTableEntry(TInt aId, TLinAddr aAddress)
+ {
+ return PageTable(aId) + ((aAddress >> KPageShift) & (KChunkMask >> KPageShift));
+ }
+
+inline TLinAddr PageDirectoryLinAddr(TInt aOsAsid)
+ {
+ return (KPageDirectoryBase+(aOsAsid<<KPageDirectoryShift));
+ }
+
+inline TPde* PageDirectoryEntry(TInt aOsAsid, TLinAddr aAddress)
+ {
+ return PageDirectory(aOsAsid) + (aAddress >> KChunkShift);
+ }
+
+extern void InvalidateTLBForPage(TLinAddr /*aLinAddr*/, TInt /*aAsid*/);
+extern void FlushTLBs();
+extern TUint32 TTCR();
+
+TPte* SafePageTableFromPde(TPde aPde)
+ {
+ if((aPde&KPdeTypeMask)==KArmV6PdePageTable)
+ {
+ SPageInfo* pi = SPageInfo::SafeFromPhysAddr(aPde);
+ if(pi)
+ {
+ TInt id = (pi->Offset()<<KPtClusterShift) | ((aPde>>KPageTableShift)&KPtClusterMask);
+ return PageTable(id);
+ }
+ }
+ return 0;
+ }
+
+TPte* SafePtePtrFromLinAddr(TLinAddr aAddress, TInt aOsAsid=0)
+ {
+ if ((TInt)(aAddress>>KChunkShift)>=(TheMmu.iLocalPdSize>>2))
+ aOsAsid = 0;
+ TPde pde = PageDirectory(aOsAsid)[aAddress>>KChunkShift];
+ TPte* pt = SafePageTableFromPde(pde);
+ if(pt)
+ pt += (aAddress>>KPageShift)&(KChunkMask>>KPageShift);
+ return pt;
+ }
+
+#ifndef _DEBUG
+// inline in UREL builds...
+#ifdef __ARMCC__
+ __forceinline /* RVCT ignores normal inline qualifier :-( */
+#else
+ inline
+#endif
+#endif
+TPte* PtePtrFromLinAddr(TLinAddr aAddress, TInt aOsAsid=0)
+ {
+ // this function only works for process local memory addresses, or for kernel memory (asid==0).
+ __NK_ASSERT_DEBUG(aOsAsid==0 || (TInt)(aAddress>>KChunkShift)<(TheMmu.iLocalPdSize>>2));
+ TPde pde = PageDirectory(aOsAsid)[aAddress>>KChunkShift];
+ SPageInfo* pi = SPageInfo::FromPhysAddr(pde);
+ TInt id = (pi->Offset()<<KPtClusterShift) | ((pde>>KPageTableShift)&KPtClusterMask);
+ TPte* pt = PageTable(id);
+ pt += (aAddress>>KPageShift)&(KChunkMask>>KPageShift);
+ return pt;
+ }
+
+
+TInt ArmMmu::LinearToPhysical(TLinAddr aLinAddr, TInt aSize, TPhysAddr& aPhysicalAddress, TPhysAddr* aPhysicalPageList, TInt aOsAsid)
+ {
+ __KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::LinearToPhysical %08x+%08x, asid=%d",aLinAddr,aSize,aOsAsid));
+ TPhysAddr physStart = ArmMmu::LinearToPhysical(aLinAddr,aOsAsid);
+ TPhysAddr nextPhys = physStart&~KPageMask;
+
+ TUint32* pageList = aPhysicalPageList;
+
+ TInt pageIndex = aLinAddr>>KPageShift;
+ TInt pagesLeft = ((aLinAddr+aSize-1)>>KPageShift)+1 - pageIndex;
+ TInt pdeIndex = aLinAddr>>KChunkShift;
+ TPde* pdePtr = (pdeIndex<(iLocalPdSize>>2) || (iAsidInfo[aOsAsid]&1))
+ ? PageDirectory(aOsAsid)
+ : ::InitPageDirectory;
+ pdePtr += pdeIndex;
+ while(pagesLeft)
+ {
+ pageIndex &= KChunkMask>>KPageShift;
+ TInt pagesLeftInChunk = (1<<(KChunkShift-KPageShift))-pageIndex;
+ if(pagesLeftInChunk>pagesLeft)
+ pagesLeftInChunk = pagesLeft;
+ pagesLeft -= pagesLeftInChunk;
+
+ TPhysAddr phys;
+ TPde pde = *pdePtr++;
+ TUint pdeType = pde&KPdeTypeMask;
+ if(pdeType==KArmV6PdeSection)
+ {
+ phys = (pde & KPdeSectionAddrMask) + (pageIndex*KPageSize);
+ __KTRACE_OPT(KMMU2,Kern::Printf("ArmMmu::LinearToPhysical Section phys=%8x",phys));
+ TInt n=pagesLeftInChunk;
+ phys==nextPhys ? nextPhys+=n*KPageSize : nextPhys=KPhysAddrInvalid;
+ if(pageList)
+ {
+ TUint32* pageEnd = pageList+n;
+ do
+ {
+ *pageList++ = phys;
+ phys+=KPageSize;
+ }
+ while(pageList<pageEnd);
+ }
+ }
+ else
+ {
+ TPte* pt = SafePageTableFromPde(pde);
+ if(!pt)
+ {
+ __KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::LinearToPhysical missing page table: PDE=%8x",pde));
+ return KErrNotFound;
+ }
+ pt += pageIndex;
+ for(;;)
+ {
+ TPte pte = *pt++;
+ TUint pte_type = pte & KPteTypeMask;
+ if (pte_type >= KArmV6PteSmallPage)
+ {
+ phys = (pte & KPteSmallPageAddrMask);
+ __KTRACE_OPT(KMMU2,Kern::Printf("ArmMmu::LinearToPhysical Small Page phys=%8x",phys));
+ phys==nextPhys ? nextPhys+=KPageSize : nextPhys=KPhysAddrInvalid;
+ if(pageList)
+ *pageList++ = phys;
+ if(--pagesLeftInChunk)
+ continue;
+ break;
+ }
+ if (pte_type == KArmV6PteLargePage)
+ {
+ --pt; // back up ptr
+ TUint pageOffset = ((TUint)pt>>2)&(KLargeSmallPageRatio-1);
+ phys = (pte & KPteLargePageAddrMask) + pageOffset*KPageSize;
+ __KTRACE_OPT(KMMU2,Kern::Printf("ArmMmu::LinearToPhysical Large Page phys=%8x",phys));
+ TInt n=KLargeSmallPageRatio-pageOffset;
+ if(n>pagesLeftInChunk)
+ n = pagesLeftInChunk;
+ phys==nextPhys ? nextPhys+=n*KPageSize : nextPhys=KPhysAddrInvalid;
+ if(pageList)
+ {
+ TUint32* pageEnd = pageList+n;
+ do
+ {
+ *pageList++ = phys;
+ phys+=KPageSize;
+ }
+ while(pageList<pageEnd);
+ }
+ pt += n;
+ if(pagesLeftInChunk-=n)
+ continue;
+ break;
+ }
+ __KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::LinearToPhysical bad PTE %8x",pte));
+ return KErrNotFound;
+ }
+ }
+ if(!pageList && nextPhys==KPhysAddrInvalid)
+ {
+ __KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::LinearToPhysical not contiguous"));
+ return KErrNotFound;
+ }
+ pageIndex = 0;
+ }
+
+ if(nextPhys==KPhysAddrInvalid)
+ {
+ // Memory is discontiguous...
+ aPhysicalAddress = KPhysAddrInvalid;
+ return 1;
+ }
+ else
+ {
+ // Memory is contiguous...
+ aPhysicalAddress = physStart;
+ return KErrNone;
+ }
+ }
+
+TInt ArmMmu::PreparePagesForDMA(TLinAddr aLinAddr, TInt aSize, TInt aOsAsid, TPhysAddr* aPhysicalPageList)
+//Returns the list of physical pages belonging to the specified memory space.
+//Checks these pages belong to a chunk marked as being trusted.
+//Locks these pages so they can not be moved by e.g. ram defragmenation.
+ {
+ SPageInfo* pi = NULL;
+ DChunk* chunk = NULL;
+ TInt err = KErrNone;
+
+ __KTRACE_OPT(KMMU2,Kern::Printf("ArmMmu::PreparePagesForDMA %08x+%08x, asid=%d",aLinAddr,aSize,aOsAsid));
+
+ TUint32* pageList = aPhysicalPageList;
+ TInt pagesInList = 0; //The number of pages we put in the list so far
+
+ TInt pageIndex = (aLinAddr & KChunkMask) >> KPageShift; // Index of the page within the section
+ TInt pagesLeft = ((aLinAddr & KPageMask) + aSize + KPageMask) >> KPageShift;
+
+ TInt pdeIndex = aLinAddr>>KChunkShift;
+
+
+ MmuBase::Wait(); // RamAlloc Mutex for accessing page/directory tables.
+ NKern::LockSystem();// SystemlLock for accessing SPageInfo objects.
+
+ TPde* pdePtr = (pdeIndex<(iLocalPdSize>>2) || (iAsidInfo[aOsAsid]&1)) ? PageDirectory(aOsAsid) : ::InitPageDirectory;
+ pdePtr += pdeIndex;//This points to the first pde
+
+ while(pagesLeft)
+ {
+ TInt pagesLeftInChunk = (1<<(KChunkShift-KPageShift))-pageIndex;
+ if(pagesLeftInChunk>pagesLeft)
+ pagesLeftInChunk = pagesLeft;
+
+ pagesLeft -= pagesLeftInChunk;
+
+ TPte* pt = SafePageTableFromPde(*pdePtr++);
+ if(!pt) { err = KErrNotFound; goto fail; }// Cannot get page table.
+
+ pt += pageIndex;
+
+ for(;pagesLeftInChunk--;)
+ {
+ TPhysAddr phys = (*pt++ & KPteSmallPageAddrMask);
+ pi = SPageInfo::SafeFromPhysAddr(phys);
+ if(!pi) { err = KErrNotFound; goto fail; }// Invalid address
+
+ __KTRACE_OPT(KMMU2,Kern::Printf("PageInfo: PA:%x T:%x S:%x O:%x C:%x",phys, pi->Type(), pi->State(), pi->Owner(), pi->LockCount()));
+ if (chunk==NULL)
+ {//This is the first page. Check 'trusted' bit.
+ if (pi->Type()!= SPageInfo::EChunk)
+ { err = KErrAccessDenied; goto fail; }// The first page do not belong to chunk.
+
+ chunk = (DChunk*)pi->Owner();
+ if ( (chunk == NULL) || ((chunk->iAttributes & DChunk::ETrustedChunk)== 0) )
+ { err = KErrAccessDenied; goto fail; }// Not a trusted chunk
+ }
+ pi->Lock();
+
+ *pageList++ = phys;
+ if ( (++pagesInList&127) == 0) //release system lock temporarily on every 512K
+ NKern::FlashSystem();
+ }
+ pageIndex = 0;
+ }
+
+ if (pi->Type()!= SPageInfo::EChunk)
+ { err = KErrAccessDenied; goto fail; }// The last page do not belong to chunk.
+
+ if (chunk && (chunk != (DChunk*)pi->Owner()))
+ { err = KErrArgument; goto fail; }//The first & the last page do not belong to the same chunk.
+
+ NKern::UnlockSystem();
+ MmuBase::Signal();
+ return KErrNone;
+
+fail:
+ __KTRACE_OPT(KMMU2,Kern::Printf("ArmMmu::PreparePagesForDMA failed"));
+ NKern::UnlockSystem();
+ MmuBase::Signal();
+ ReleasePagesFromDMA(aPhysicalPageList, pagesInList);
+ return err;
+ }
+
+TInt ArmMmu::ReleasePagesFromDMA(TPhysAddr* aPhysicalPageList, TInt aPageCount)
+// Unlocks physical pages.
+// @param aPhysicalPageList - points to the list of physical pages that should be released.
+// @param aPageCount - the number of physical pages in the list.
+ {
+ NKern::LockSystem();
+ __KTRACE_OPT(KMMU2,Kern::Printf("ArmMmu::ReleasePagesFromDMA count:%d",aPageCount));
+
+ while (aPageCount--)
+ {
+ SPageInfo* pi = SPageInfo::SafeFromPhysAddr(*aPhysicalPageList++);
+ if(!pi)
+ {
+ NKern::UnlockSystem();
+ return KErrArgument;
+ }
+ __KTRACE_OPT(KMMU2,Kern::Printf("PageInfo: T:%x S:%x O:%x C:%x",pi->Type(), pi->State(), pi->Owner(), pi->LockCount()));
+ pi->Unlock();
+ }
+ NKern::UnlockSystem();
+ return KErrNone;
+ }
+
+TPhysAddr ArmMmu::LinearToPhysical(TLinAddr aLinAddr, TInt aOsAsid)
+//
+// Find the physical address corresponding to a given linear address in a specified OS
+// address space. Call with system locked.
+//
+ {
+ __KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::LinearToPhysical(%08x,%d)",aLinAddr,aOsAsid));
+ TInt pdeIndex=aLinAddr>>KChunkShift;
+ TPde pde = (pdeIndex<(iLocalPdSize>>2) || (iAsidInfo[aOsAsid]&1)) ? PageDirectory(aOsAsid)[pdeIndex] : ::InitPageDirectory[pdeIndex];
+ TPhysAddr pa=KPhysAddrInvalid;
+ if ((pde&KPdePresentMask)==KArmV6PdePageTable)
+ {
+ SPageInfo* pi = SPageInfo::SafeFromPhysAddr(pde);
+ if (pi)
+ {
+ TInt id = (pi->Offset()<<KPtClusterShift) | ((pde>>KPageTableShift)&KPtClusterMask);
+ TPte* pPte=PageTable(id);
+ TPte pte=pPte[(aLinAddr&KChunkMask)>>KPageShift];
+ if (pte & KArmV6PteSmallPage)
+ {
+ pa=(pte&KPteSmallPageAddrMask)+(aLinAddr&~KPteSmallPageAddrMask);
+ __KTRACE_OPT(KMMU,Kern::Printf("Mapped with small page - returning %08x",pa));
+ }
+ else if ((pte & KArmV6PteTypeMask) == KArmV6PteLargePage)
+ {
+ pa=(pte&KPteLargePageAddrMask)+(aLinAddr&~KPteLargePageAddrMask);
+ __KTRACE_OPT(KMMU,Kern::Printf("Mapped with large page - returning %08x",pa));
+ }
+ }
+ }
+ else if ((pde&KPdePresentMask)==KArmV6PdeSection)
+ {
+ pa=(pde&KPdeSectionAddrMask)|(aLinAddr&~KPdeSectionAddrMask);
+ __KTRACE_OPT(KMMU,Kern::Printf("Mapped with section - returning %08x",pa));
+ }
+ return pa;
+ }
+
+// permission table indexed by XN:APX:AP1:AP0
+static const TInt PermissionLookup[16]=
+ { //XN:APX:AP1:AP0
+ 0, //0 0 0 0 no access
+ EMapAttrWriteSup|EMapAttrReadSup|EMapAttrExecSup, //0 0 0 1 RW sup execute
+ EMapAttrWriteSup|EMapAttrReadUser|EMapAttrExecUser, //0 0 1 0 supRW usrR execute
+ EMapAttrWriteUser|EMapAttrReadUser|EMapAttrExecUser,//0 0 1 1 supRW usrRW execute
+ 0, //0 1 0 0 reserved
+ EMapAttrReadSup|EMapAttrExecSup, //0 1 0 1 supR execute
+ EMapAttrReadUser|EMapAttrExecUser, //0 1 1 0 supR usrR execute
+ 0, //0 1 1 1 reserved
+ 0, //1 0 0 0 no access
+ EMapAttrWriteSup|EMapAttrReadSup, //1 0 0 1 RW sup
+ EMapAttrWriteSup|EMapAttrReadUser, //1 0 1 0 supRW usrR
+ EMapAttrWriteUser|EMapAttrReadUser, //1 0 1 1 supRW usrRW
+ 0, //1 1 0 0 reserved
+ EMapAttrReadSup, //1 1 0 1 supR
+ EMapAttrReadUser, //1 1 1 0 supR usrR
+ EMapAttrReadUser, //1 1 1 1 supR usrR
+ };
+
+TInt ArmMmu::PageTableId(TLinAddr aAddr, TInt aOsAsid)
+ {
+ TInt id=-1;
+ __KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::PageTableId(%08x,%d)",aAddr,aOsAsid));
+ TInt pdeIndex=aAddr>>KChunkShift;
+ TPde pde = (pdeIndex<(iLocalPdSize>>2) || (iAsidInfo[aOsAsid]&1)) ? PageDirectory(aOsAsid)[pdeIndex] : ::InitPageDirectory[pdeIndex];
+ if ((pde&KArmV6PdeTypeMask)==KArmV6PdePageTable)
+ {
+ SPageInfo* pi = SPageInfo::SafeFromPhysAddr(pde);
+ if (pi)
+ id = (pi->Offset()<<KPtClusterShift) | ((pde>>KPageTableShift)&KPtClusterMask);
+ }
+ __KTRACE_OPT(KMMU,Kern::Printf("ID=%d",id));
+ return id;
+ }
+
+// Used only during boot for recovery of RAM drive
+TInt ArmMmu::BootPageTableId(TLinAddr aAddr, TPhysAddr& aPtPhys)
+ {
+ TInt id=KErrNotFound;
+ __KTRACE_OPT(KMMU,Kern::Printf("ArmMmu:BootPageTableId(%08x,&)",aAddr));
+ TPde* kpd=(TPde*)KPageDirectoryBase; // kernel page directory
+ TInt pdeIndex=aAddr>>KChunkShift;
+ TPde pde = kpd[pdeIndex];
+ if ((pde & KArmV6PdeTypeMask) == KArmV6PdePageTable)
+ {
+ aPtPhys = pde & KPdePageTableAddrMask;
+ SPageInfo* pi = SPageInfo::SafeFromPhysAddr(pde);
+ if (pi)
+ {
+ SPageInfo::TType type = pi->Type();
+ if (type == SPageInfo::EPageTable)
+ id = (pi->Offset()<<KPtClusterShift) | ((pde>>KPageTableShift)&KPtClusterMask);
+ else if (type == SPageInfo::EUnused)
+ id = KErrUnknown;
+ }
+ }
+ __KTRACE_OPT(KMMU,Kern::Printf("ID=%d",id));
+ return id;
+ }
+
+TBool ArmMmu::PteIsPresent(TPte aPte)
+ {
+ return aPte & KArmV6PteTypeMask;
+ }
+
+TPhysAddr ArmMmu::PtePhysAddr(TPte aPte, TInt aPteIndex)
+ {
+ TUint32 pte_type = aPte & KArmV6PteTypeMask;
+ if (pte_type == KArmV6PteLargePage)
+ return (aPte & KPteLargePageAddrMask) + (TPhysAddr(aPteIndex << KPageShift) & KLargePageMask);
+ else if (pte_type != 0)
+ return aPte & KPteSmallPageAddrMask;
+ return KPhysAddrInvalid;
+ }
+
+TPhysAddr ArmMmu::PdePhysAddr(TLinAddr aAddr)
+ {
+ TPde* kpd = (TPde*)KPageDirectoryBase; // kernel page directory
+ TPde pde = kpd[aAddr>>KChunkShift];
+ if ((pde & KPdePresentMask) == KArmV6PdeSection)
+ return pde & KPdeSectionAddrMask;
+ return KPhysAddrInvalid;
+ }
+
+void ArmMmu::Init1()
+ {
+ __KTRACE_OPT2(KBOOT,KMMU,Kern::Printf("ArmMmu::Init1"));
+
+ // MmuBase data
+ iPageSize=KPageSize;
+ iPageMask=KPageMask;
+ iPageShift=KPageShift;
+ iChunkSize=KChunkSize;
+ iChunkMask=KChunkMask;
+ iChunkShift=KChunkShift;
+ iPageTableSize=KPageTableSize;
+ iPageTableMask=KPageTableMask;
+ iPageTableShift=KPageTableShift;
+ iPtClusterSize=KPtClusterSize;
+ iPtClusterMask=KPtClusterMask;
+ iPtClusterShift=KPtClusterShift;
+ iPtBlockSize=KPtBlockSize;
+ iPtBlockMask=KPtBlockMask;
+ iPtBlockShift=KPtBlockShift;
+ iPtGroupSize=KChunkSize/KPageTableSize;
+ iPtGroupMask=iPtGroupSize-1;
+ iPtGroupShift=iChunkShift-iPageTableShift;
+ //TInt* iPtBlockCount; // dynamically allocated - Init2
+ //TInt* iPtGroupCount; // dynamically allocated - Init2
+ iPtInfo=(SPageTableInfo*)KPageTableInfoBase;
+ iPageTableLinBase=KPageTableBase;
+ //iRamPageAllocator; // dynamically allocated - Init2
+ //iAsyncFreeList; // dynamically allocated - Init2
+ //iPageTableAllocator; // dynamically allocated - Init2
+ //iPageTableLinearAllocator;// dynamically allocated - Init2
+ iPtInfoPtePerm=KPtInfoPtePerm;
+ iPtPtePerm=KPtPtePerm;
+ iPtPdePerm=KPtPdePerm;
+ iUserCodeLoadPtePerm=KUserCodeLoadPte;
+ iKernelCodePtePerm=KKernelCodeRunPte;
+ iTempAddr=KTempAddr;
+ iSecondTempAddr=KSecondTempAddr;
+ iMapSizes=KPageSize|KLargePageSize|KChunkSize;
+ iRomLinearBase = ::RomHeaderAddress;
+ iRomLinearEnd = KRomLinearEnd;
+ iShadowPtePerm = KShadowPtePerm;
+ iShadowPdePerm = KShadowPdePerm;
+
+ // Mmu data
+ TInt total_ram=TheSuperPage().iTotalRamSize;
+
+ // Large or small configuration?
+ // This is determined by the bootstrap based on RAM size
+ TUint32 ttcr=TTCR();
+ __NK_ASSERT_ALWAYS(ttcr==1 || ttcr==2);
+ TBool large = (ttcr==1);
+
+ // calculate cache colouring...
+ TInt iColourCount = 0;
+ TInt dColourCount = 0;
+ TUint32 ctr = InternalCache::TypeRegister();
+ __KTRACE_OPT2(KBOOT,KMMU,Kern::Printf("CacheTypeRegister = %08x",ctr));
+#ifdef __CPU_ARMV6
+ __NK_ASSERT_ALWAYS((ctr>>29)==0); // check ARMv6 format
+ if(ctr&0x800)
+ iColourCount = 4;
+ if(ctr&0x800000)
+ dColourCount = 4;
+#else
+ __KTRACE_OPT2(KBOOT,KMMU,Kern::Printf("CacheTypeRegister = %08x",ctr));
+ __NK_ASSERT_ALWAYS((ctr>>29)==4); // check ARMv7 format
+ TUint l1ip = (ctr>>14)&3; // L1 instruction cache indexing and tagging policy
+ __NK_ASSERT_ALWAYS(l1ip>=2); // check I cache is physically tagged
+
+ TUint32 clidr = InternalCache::LevelIDRegister();
+ __KTRACE_OPT2(KBOOT,KMMU,Kern::Printf("CacheLevelIDRegister = %08x",clidr));
+ TUint l1type = clidr&7;
+ if(l1type)
+ {
+ if(l1type==2 || l1type==3 || l1type==4)
+ {
+ // we have an L1 data cache...
+ TUint32 csir = InternalCache::SizeIdRegister(0,0);
+ TUint sets = ((csir>>13)&0x7fff)+1;
+ TUint ways = ((csir>>3)&0x3ff)+1;
+ TUint lineSizeShift = (csir&7)+4;
+ // assume L1 data cache is VIPT and alias checks broken and so we need data cache colouring...
+ dColourCount = (sets<<lineSizeShift)>>KPageShift;
+ if(l1type==4) // unified cache, so set instruction cache colour as well...
+ iColourCount = (sets<<lineSizeShift)>>KPageShift;
+ __KTRACE_OPT2(KBOOT,KMMU,Kern::Printf("L1DCache = 0x%x,0x%x,%d colourCount=%d",sets,ways,lineSizeShift,(sets<<lineSizeShift)>>KPageShift));
+ }
+
+ if(l1type==1 || l1type==3)
+ {
+ // we have a separate L1 instruction cache...
+ TUint32 csir = InternalCache::SizeIdRegister(1,0);
+ TUint sets = ((csir>>13)&0x7fff)+1;
+ TUint ways = ((csir>>3)&0x3ff)+1;
+ TUint lineSizeShift = (csir&7)+4;
+ iColourCount = (sets<<lineSizeShift)>>KPageShift;
+ __KTRACE_OPT2(KBOOT,KMMU,Kern::Printf("L1ICache = 0x%x,0x%x,%d colourCount=%d",sets,ways,lineSizeShift,(sets<<lineSizeShift)>>KPageShift));
+ }
+ }
+ if(l1ip==3)
+ {
+ // PIPT cache, so no colouring restrictions...
+ __KTRACE_OPT2(KBOOT,KMMU,Kern::Printf("L1ICache is PIPT"));
+ iColourCount = 0;
+ }
+ else
+ {
+ // VIPT cache...
+ __KTRACE_OPT2(KBOOT,KMMU,Kern::Printf("L1ICache is VIPT"));
+ }
+#endif
+ TUint colourShift = 0;
+ for(TUint colourCount=Max(iColourCount,dColourCount); colourCount!=0; colourCount>>=1)
+ ++colourShift;
+ iAliasSize=KPageSize<<colourShift;
+ __KTRACE_OPT2(KBOOT,KMMU,Kern::Printf("iAliasSize=0x%x",iAliasSize));
+ iAliasMask=iAliasSize-1;
+ iAliasShift=KPageShift+colourShift;
+
+ iDecommitThreshold = CacheMaintenance::SyncAllPerformanceThresholdPages();
+
+ iNumOsAsids=KArmV6NumAsids;
+ iNumGlobalPageDirs=1;
+ //iOsAsidAllocator; // dynamically allocated - Init2
+ iGlobalPdSize=KPageDirectorySize;
+ iGlobalPdShift=KPageDirectoryShift;
+ iAsidGroupSize=KChunkSize/KPageDirectorySize;
+ iAsidGroupMask=iAsidGroupSize-1;
+ iAsidGroupShift=KChunkShift-KPageDirectoryShift;
+ iUserLocalBase=KUserLocalDataBase;
+ iAsidInfo=(TUint32*)KAsidInfoBase;
+ iPdeBase=KPageDirectoryBase;
+ iPdPtePerm=KPdPtePerm;
+ iPdPdePerm=KPdPdePerm;
+ iRamDriveMask=0x00f00000;
+ iGlobalCodePtePerm=KGlobalCodeRunPte;
+#if defined(__CPU_MEMORY_TYPE_REMAPPING)
+ iCacheMaintenanceTempMapAttr = CacheMaintenance::TemporaryMapping();
+#else
+ switch(CacheMaintenance::TemporaryMapping())
+ {
+ case EMemAttNormalUncached:
+ iCacheMaintenanceTempMapAttr = KArmV6MemAttNCNC;
+ break;
+ case EMemAttNormalCached:
+ iCacheMaintenanceTempMapAttr = KArmV6MemAttWBWAWBWA;
+ break;
+ default:
+ Panic(ETempMappingFailed);
+ }
+#endif
+ iMaxDllDataSize=Min(total_ram/2, 0x08000000); // phys RAM/2 up to 128Mb
+ iMaxDllDataSize=(iMaxDllDataSize+iChunkMask)&~iChunkMask; // round up to chunk size
+ iMaxUserCodeSize=Min(total_ram, 0x10000000); // phys RAM up to 256Mb
+ iMaxUserCodeSize=(iMaxUserCodeSize+iChunkMask)&~iChunkMask; // round up to chunk size
+ if (large)
+ {
+ iLocalPdSize=KPageDirectorySize/2;
+ iLocalPdShift=KPageDirectoryShift-1;
+ iUserSharedBase=KUserSharedDataBase2GB;
+ iUserLocalEnd=iUserSharedBase-iMaxDllDataSize;
+ iUserSharedEnd=KUserSharedDataEnd2GB-iMaxUserCodeSize;
+ iDllDataBase=iUserLocalEnd;
+ iUserCodeBase=iUserSharedEnd;
+ }
+ else
+ {
+ iLocalPdSize=KPageDirectorySize/4;
+ iLocalPdShift=KPageDirectoryShift-2;
+ iUserSharedBase=KUserSharedDataBase1GB;
+ iUserLocalEnd=iUserSharedBase;
+ iDllDataBase=KUserSharedDataEnd1GB-iMaxDllDataSize;
+ iUserCodeBase=iDllDataBase-iMaxUserCodeSize;
+ iUserSharedEnd=iUserCodeBase;
+ }
+ __KTRACE_OPT(KMMU,Kern::Printf("LPD size %08x GPD size %08x Alias size %08x",
+ iLocalPdSize, iGlobalPdSize, iAliasSize));
+ __KTRACE_OPT(KMMU,Kern::Printf("ULB %08x ULE %08x USB %08x USE %08x",iUserLocalBase,iUserLocalEnd,
+ iUserSharedBase,iUserSharedEnd));
+ __KTRACE_OPT(KMMU,Kern::Printf("DDB %08x UCB %08x",iDllDataBase,iUserCodeBase));
+
+ // ArmMmu data
+
+ // other
+ PP::MaxUserThreadStack=0x14000; // 80K - STDLIB asks for 64K for PosixServer!!!!
+ PP::UserThreadStackGuard=0x2000; // 8K
+ PP::MaxStackSpacePerProcess=0x200000; // 2Mb
+ K::SupervisorThreadStackSize=0x1000; // 4K
+ PP::SupervisorThreadStackGuard=0x1000; // 4K
+ K::MachineConfig=(TMachineConfig*)KMachineConfigLinAddr;
+ PP::RamDriveStartAddress=KRamDriveStartAddress;
+ PP::RamDriveRange=KRamDriveMaxSize;
+ PP::RamDriveMaxSize=KRamDriveMaxSize; // may be reduced later
+ K::MemModelAttributes=EMemModelTypeMultiple|EMemModelAttrNonExProt|EMemModelAttrKernProt|EMemModelAttrWriteProt|
+ EMemModelAttrVA|EMemModelAttrProcessProt|EMemModelAttrSameVA|EMemModelAttrSvKernProt|
+ EMemModelAttrIPCKernProt|EMemModelAttrRamCodeProt;
+
+ Arm::DefaultDomainAccess=KDefaultDomainAccess;
+
+ Mmu::Init1();
+ }
+
+void ArmMmu::DoInit2()
+ {
+ __KTRACE_OPT2(KBOOT,KMMU,Kern::Printf("ArmMmu::DoInit2"));
+ iTempPte=PageTable(PageTableId(iTempAddr,0))+((iTempAddr&KChunkMask)>>KPageShift);
+ iSecondTempPte=PageTable(PageTableId(iSecondTempAddr,0))+((iSecondTempAddr&KChunkMask)>>KPageShift);
+ __KTRACE_OPT2(KBOOT,KMMU,Kern::Printf("iTempAddr=%08x, iTempPte=%08x, iSecondTempAddr=%08x, iSecondTempPte=%08x",
+ iTempAddr, iTempPte, iSecondTempAddr, iSecondTempPte));
+ CreateKernelSection(KKernelSectionEnd, iAliasShift);
+ CreateUserGlobalSection(KUserGlobalDataBase, KUserGlobalDataEnd);
+ Mmu::DoInit2();
+ }
+
+#ifndef __MMU_MACHINE_CODED__
+void ArmMmu::MapRamPages(TInt aId, SPageInfo::TType aType, TAny* aPtr, TUint32 aOffset, const TPhysAddr* aPageList, TInt aNumPages, TPte aPtePerm)
+//
+// Map a list of physical RAM pages into a specified page table with specified PTE permissions.
+// Update the page information array.
+// Call this with the system locked.
+//
+ {
+ __KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::MapRamPages() id=%d type=%d ptr=%08x off=%08x n=%d perm=%08x",
+ aId, aType, aPtr, aOffset, aNumPages, aPtePerm));
+
+ SPageTableInfo& ptinfo=iPtInfo[aId];
+ ptinfo.iCount+=aNumPages;
+ aOffset>>=KPageShift;
+ TInt ptOffset=aOffset & KPagesInPDEMask; // entry number in page table
+ TPte* pPte=PageTable(aId)+ptOffset; // address of first PTE
+
+ TLinAddr firstPte = (TLinAddr)pPte; //Will need this to clean page table changes in cache.
+
+ while(aNumPages--)
+ {
+ TPhysAddr pa = *aPageList++;
+ if(pa==KPhysAddrInvalid)
+ {
+ ++pPte;
+ __NK_ASSERT_DEBUG(aType==SPageInfo::EInvalid);
+ continue;
+ }
+ *pPte++ = pa | aPtePerm; // insert PTE
+ __KTRACE_OPT(KMMU,Kern::Printf("Writing PTE %08x to %08x",pPte[-1],pPte-1));
+ if (aType!=SPageInfo::EInvalid)
+ {
+ SPageInfo* pi = SPageInfo::SafeFromPhysAddr(pa);
+ if(pi)
+ {
+ pi->Set(aType,aPtr,aOffset);
+ __KTRACE_OPT(KMMU,Kern::Printf("I: %d %08x %08x",aType,aPtr,aOffset));
+ ++aOffset; // increment offset for next page
+ }
+ }
+ }
+ CacheMaintenance::MultiplePtesUpdated(firstPte, (TUint)pPte-firstPte);
+ }
+
+void ArmMmu::MapPhysicalPages(TInt aId, SPageInfo::TType aType, TAny* aPtr, TUint32 aOffset, TPhysAddr aPhysAddr, TInt aNumPages, TPte aPtePerm)
+//
+// Map consecutive physical pages into a specified page table with specified PTE permissions.
+// Update the page information array if RAM pages are being mapped.
+// Call this with the system locked.
+//
+ {
+ __KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::MapPhysicalPages() id=%d type=%d ptr=%08x off=%08x phys=%08x n=%d perm=%08x",
+ aId, aType, aPtr, aOffset, aPhysAddr, aNumPages, aPtePerm));
+ SPageTableInfo& ptinfo=iPtInfo[aId];
+ ptinfo.iCount+=aNumPages;
+ aOffset>>=KPageShift;
+ TInt ptOffset=aOffset & KPagesInPDEMask; // entry number in page table
+ TPte* pPte=(TPte*)(PageTableLinAddr(aId))+ptOffset; // address of first PTE
+
+ TLinAddr firstPte = (TLinAddr)pPte; //Will need this to clean page table changes in cache
+
+ SPageInfo* pi;
+ if(aType==SPageInfo::EInvalid)
+ pi = NULL;
+ else
+ pi = SPageInfo::SafeFromPhysAddr(aPhysAddr);
+ while(aNumPages--)
+ {
+ *pPte++ = aPhysAddr|aPtePerm; // insert PTE
+ aPhysAddr+=KPageSize;
+ __KTRACE_OPT(KMMU,Kern::Printf("Writing PTE %08x to %08x",pPte[-1],pPte-1));
+ if (pi)
+ {
+ pi->Set(aType,aPtr,aOffset);
+ __KTRACE_OPT(KMMU,Kern::Printf("I: %d %08x %08x",aType,aPtr,aOffset));
+ ++aOffset; // increment offset for next page
+ ++pi;
+ }
+ }
+
+ CacheMaintenance::MultiplePtesUpdated(firstPte, (TUint)pPte-firstPte);
+ }
+
+void ArmMmu::MapVirtual(TInt aId, TInt aNumPages)
+//
+// Called in place of MapRamPages or MapPhysicalPages to update mmu data structures when committing
+// virtual address space to a chunk. No pages are mapped.
+// Call this with the system locked.
+//
+ {
+ SPageTableInfo& ptinfo=iPtInfo[aId];
+ ptinfo.iCount+=aNumPages;
+ }
+
+void ArmMmu::RemapPage(TInt aId, TUint32 aAddr, TPhysAddr aOldAddr, TPhysAddr aNewAddr, TPte aPtePerm, DProcess* aProcess)
+//
+// Replace the mapping at address aAddr in page table aId.
+// Update the page information array for both the old and new pages.
+// Return physical address of old page if it is now ready to be freed.
+// Call this with the system locked.
+// May be called with interrupts disabled, do not enable/disable them.
+//
+ {
+ TInt ptOffset=(aAddr&KChunkMask)>>KPageShift; // entry number in page table
+ TPte* pPte=PageTable(aId)+ptOffset; // address of PTE
+ TPte pte=*pPte;
+ TInt asid=aProcess ? ((DMemModelProcess*)aProcess)->iOsAsid :
+ (aAddr<KRomLinearBase ? (TInt)UNKNOWN_MAPPING : (TInt)KERNEL_MAPPING );
+
+ if (pte & KArmV6PteSmallPage)
+ {
+ __ASSERT_ALWAYS((pte & KPteSmallPageAddrMask) == aOldAddr, Panic(ERemapPageFailed));
+ SPageInfo* oldpi = SPageInfo::FromPhysAddr(aOldAddr);
+ __ASSERT_DEBUG(oldpi->LockCount()==0,Panic(ERemapPageFailed));
+
+ // remap page
+ *pPte = aNewAddr | aPtePerm; // overwrite PTE
+ CacheMaintenance::SinglePteUpdated((TLinAddr)pPte);
+ InvalidateTLBForPage(aAddr,asid); // flush TLB entry
+
+ // update new pageinfo, clear old
+ SPageInfo* pi = SPageInfo::FromPhysAddr(aNewAddr);
+ pi->Set(oldpi->Type(),oldpi->Owner(),oldpi->Offset());
+ oldpi->SetUnused();
+ }
+ else
+ {
+ Panic(ERemapPageFailed);
+ }
+ }
+
+void ArmMmu::RemapPageByAsid(TBitMapAllocator* aOsAsids, TLinAddr aLinAddr, TPhysAddr aOldAddr, TPhysAddr aNewAddr, TPte aPtePerm)
+//
+// Replace the mapping at address aLinAddr in the relevant page table for all
+// ASIDs specified in aOsAsids, but only if the currently mapped address is
+// aOldAddr.
+// Update the page information array for both the old and new pages.
+// Call this with the system unlocked.
+//
+ {
+ __KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::RemapPageByAsid() linaddr=%08x oldaddr=%08x newaddr=%08x perm=%08x", aLinAddr, aOldAddr, aNewAddr, aPtePerm));
+
+ TInt asid = -1;
+ TInt lastAsid = KArmV6NumAsids - 1;
+ TUint32* ptr = aOsAsids->iMap;
+ NKern::LockSystem();
+ do
+ {
+ TUint32 bits = *ptr++;
+ do
+ {
+ ++asid;
+ if(bits & 0x80000000u)
+ {
+ // mapped in this address space, so update PTE...
+ TPte* pPte = PtePtrFromLinAddr(aLinAddr, asid);
+ TPte pte = *pPte;
+ if ((pte&~KPageMask) == aOldAddr)
+ {
+ *pPte = aNewAddr | aPtePerm;
+ __KTRACE_OPT(KMMU,Kern::Printf("Writing PTE %08x to %08x in asid %d",*pPte,pPte,asid));
+ CacheMaintenance::SinglePteUpdated((TLinAddr)pPte);
+ InvalidateTLBForPage(aLinAddr,asid); // flush TLB entry
+ }
+ }
+ }
+ while(bits<<=1);
+ NKern::FlashSystem();
+ asid |= 31;
+ }
+ while(asid<lastAsid);
+
+ // copy pageinfo attributes and mark old page unused
+ SPageInfo* oldpi = SPageInfo::FromPhysAddr(aOldAddr);
+ SPageInfo::FromPhysAddr(aNewAddr)->Set(oldpi->Type(),oldpi->Owner(),oldpi->Offset());
+ oldpi->SetUnused();
+
+ NKern::UnlockSystem();
+ }
+
+TInt ArmMmu::UnmapPages(TInt aId, TUint32 aAddr, TInt aNumPages, TPhysAddr* aPageList, TBool aSetPagesFree, TInt& aNumPtes, TInt& aNumFree, DProcess* aProcess)
+//
+// Unmap a specified area at address aAddr in page table aId. Place physical addresses of unmapped
+// pages into aPageList, and count of unmapped pages into aNumPtes.
+// Return number of pages still mapped using this page table.
+// Call this with the system locked.
+// On multiple memory model, do not call this method with aSetPagesFree false. Call UnmapUnownedPages instead.
+ {
+ __KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::UnmapPages() id=%d addr=%08x n=%d pl=%08x set-free=%d",aId,aAddr,aNumPages,aPageList,aSetPagesFree));
+ TInt ptOffset=(aAddr&KChunkMask)>>KPageShift; // entry number in page table
+ TPte* pPte=PageTable(aId)+ptOffset; // address of first PTE
+ TInt np=0;
+ TInt nf=0;
+ TUint32 ng=0;
+ TInt asid=aProcess ? ((DMemModelProcess*)aProcess)->iOsAsid :
+ (aAddr<KRomLinearBase ? (TInt)UNKNOWN_MAPPING : (TInt)KERNEL_MAPPING );
+
+
+ while(aNumPages--)
+ {
+ TPte pte=*pPte; // get original PTE
+#if defined(__CPU_ARM1136__) && !defined(__CPU_ARM1136_ERRATUM_353494_FIXED)
+ remove_and_invalidate_page(pPte, aAddr, asid);
+ ++pPte;
+#else
+ *pPte++=0; // clear PTE
+#endif
+
+ // We count all unmapped pages in np, including demand paged 'old' pages - but we don't pass
+ // these to PageUnmapped, as the page doesn't become free until it's unmapped from all
+ // processes
+ if (pte != KPteNotPresentEntry)
+ ++np;
+
+ if (pte & KArmV6PteSmallPage)
+ {
+ ng |= pte;
+#if !defined(__CPU_ARM1136__) || defined(__CPU_ARM1136_ERRATUM_353494_FIXED)
+ // Remove_and_invalidate_page will sort out cache and TLB.
+ // When __CPU_ARM1136_ERRATUM_353494_FIXED, we have to do it here.
+ CacheMaintenance::SinglePteUpdated((TLinAddr)(pPte-1));
+ if (asid >= 0) //otherwise, KUnmapPagesTLBFlushDeferred will be returned.
+ InvalidateTLBForPage(aAddr,asid); // flush any corresponding TLB entry
+#endif
+ TPhysAddr pa=pte & KPteSmallPageAddrMask; // physical address of unmapped page
+ if (aSetPagesFree)
+ {
+ SPageInfo* pi = SPageInfo::FromPhysAddr(pa);
+ if(iRamCache->PageUnmapped(pi))
+ {
+ pi->SetUnused(); // mark page as unused
+ if (pi->LockCount()==0)
+ {
+ *aPageList++=pa; // store in page list
+ ++nf; // count free pages
+ }
+ }
+ }
+ else
+ *aPageList++=pa; // store in page list
+ }
+ aAddr+=KPageSize;
+ }
+
+ aNumPtes=np;
+ aNumFree=nf;
+ SPageTableInfo& ptinfo=iPtInfo[aId];
+ TInt r=(ptinfo.iCount-=np);
+ if (asid<0)
+ r|=KUnmapPagesTLBFlushDeferred;
+
+
+ #if defined(__CPU_ARM1136__) && !defined(__CPU_ARM1136_ERRATUM_353494_FIXED)
+ __FlushBtb();
+ #endif
+
+ __KTRACE_OPT(KMMU,Kern::Printf("Unmapped %d; Freed: %d; Return %08x",np,nf,r));
+ return r; // return number of pages remaining in this page table
+ }
+
+TInt ArmMmu::UnmapVirtual(TInt aId, TUint32 aAddr, TInt aNumPages, TPhysAddr* aPageList, TBool aSetPagesFree, TInt& aNumPtes, TInt& aNumFree, DProcess* aProcess)
+//
+// Unmap a specified area at address aAddr in page table aId. Place physical addresses of unmapped
+// pages into aPageList, and count of unmapped pages into aNumPtes.
+// Adjust the page table reference count as if aNumPages pages were unmapped.
+// Return number of pages still mapped using this page table.
+// Call this with the system locked.
+// On multiple memory model, do not call this method with aSetPagesFree false. Call UnmapUnownedVirtual instead.
+//
+ {
+ SPageTableInfo& ptinfo=iPtInfo[aId];
+ TInt newCount = ptinfo.iCount - aNumPages;
+ UnmapPages(aId, aAddr, aNumPages, aPageList, aSetPagesFree, aNumPtes, aNumFree, aProcess);
+ ptinfo.iCount = newCount;
+ aNumPtes = aNumPages;
+ return newCount;
+ }
+
+TInt ArmMmu::UnmapUnownedPages(TInt aId, TUint32 aAddr, TInt aNumPages,
+ TPhysAddr* aPageList, TLinAddr* aLAPageList,TInt& aNumPtes, TInt& aNumFree, DProcess* aProcess)
+/*
+ * Unmaps specified area at address aAddr in page table aId.
+ * Places physical addresses of not-demaned-paged unmapped pages into aPageList.
+ * Corresponding linear addresses are placed into aLAPageList.
+ * 'Old' demand-paged pages (holds invalid PE entry with physucal address) are neither unmapped nor
+ * encountered in aPageList but are still counted in aNumPtes.
+ *
+ * This method should be called to decommit physical memory not owned by the chunk. As we do not know
+ * the origin of such memory, PtInfo could be invalid (or does't exist) so cache maintenance may not be
+ * able to obtain mapping colour. For that reason, this also returns former linear address of each page
+ * in aPageList.
+ *
+ * @pre All pages are mapped within a single page table identified by aId.
+ * @pre On entry, system locked is held and is not released during the execution.
+ *
+ * @arg aId Id of the page table that maps tha pages.
+ * @arg aAddr Linear address of the start of the area.
+ * @arg aNumPages The number of pages to unmap.
+ * @arg aProcess The owning process of the mamory area to unmap.
+ * @arg aPageList On exit, holds the list of unmapped pages.
+ * @arg aLAPageList On exit, holds the list of linear addresses of unmapped pages.
+ * @arg aNumFree On exit, holds the number of pages in aPageList.
+ * @arg aNumPtes On exit, holds the number of unmapped pages. This includes demand-paged 'old'
+ * pages (with invalid page table entry still holding the address of physical page.)
+ *
+ * @return The number of pages still mapped using this page table. It is orred by
+ * KUnmapPagesTLBFlushDeferred if TLB flush is not executed - which requires
+ * the caller to do global TLB flush.
+ */
+ {
+ __KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::UnmapUnownedPages() id=%d addr=%08x n=%d pl=%08x",aId,aAddr,aNumPages,aPageList));
+ TInt ptOffset=(aAddr&KChunkMask)>>KPageShift; // entry number in page table
+ TPte* pPte=PageTable(aId)+ptOffset; // address of first PTE
+ TInt np=0;
+ TInt nf=0;
+ TUint32 ng=0;
+ TInt asid=aProcess ? ((DMemModelProcess*)aProcess)->iOsAsid :
+ (aAddr<KRomLinearBase ? (TInt)UNKNOWN_MAPPING : (TInt)KERNEL_MAPPING );
+
+ while(aNumPages--)
+ {
+ TPte pte=*pPte; // get original PTE
+#if defined(__CPU_ARM1136__) && !defined(__CPU_ARM1136_ERRATUM_353494_FIXED)
+ remove_and_invalidate_page(pPte, aAddr, asid);
+ ++pPte;
+#else
+ *pPte++=0; // clear PTE
+#endif
+
+ // We count all unmapped pages in np, including demand paged 'old' pages - but we don't pass
+ // these to PageUnmapped, as the page doesn't become free until it's unmapped from all
+ // processes
+ if (pte != KPteNotPresentEntry)
+ ++np;
+
+ if (pte & KArmV6PteSmallPage)
+ {
+ ng |= pte;
+#if !defined(__CPU_ARM1136__) || defined(__CPU_ARM1136_ERRATUM_353494_FIXED)
+ // Remove_and_invalidate_page will sort out cache and TLB.
+ // When __CPU_ARM1136_ERRATUM_353494_FIXED, we have to do it here.
+ CacheMaintenance::SinglePteUpdated((TLinAddr)(pPte-1));
+ if (asid >= 0) //otherwise, KUnmapPagesTLBFlushDeferred will be returned.
+ InvalidateTLBForPage(aAddr,asid); // flush any corresponding TLB entry
+#endif
+ TPhysAddr pa=pte & KPteSmallPageAddrMask; // physical address of unmapped page
+ ++nf;
+ *aPageList++=pa; // store physical aaddress in page list
+ *aLAPageList++=aAddr; // store linear address in page list
+ }
+ aAddr+=KPageSize;
+ }
+
+ aNumPtes=np;
+ aNumFree=nf;
+ SPageTableInfo& ptinfo=iPtInfo[aId];
+ TInt r=(ptinfo.iCount-=np);
+ if (asid<0)
+ r|=KUnmapPagesTLBFlushDeferred;
+
+
+ #if defined(__CPU_ARM1136__) && !defined(__CPU_ARM1136_ERRATUM_353494_FIXED)
+ __FlushBtb();
+ #endif
+
+ __KTRACE_OPT(KMMU,Kern::Printf("Unmapped %d; Freed: %d; Return %08x",np,nf,r));
+ return r; // return number of pages remaining in this page table
+ }
+
+
+TInt ArmMmu::UnmapUnownedVirtual(TInt aId, TUint32 aAddr, TInt aNumPages,
+ TPhysAddr* aPageList, TLinAddr* aLAPageList,TInt& aNumPtes, TInt& aNumFree, DProcess* aProcess)
+//
+// Unmap a specified area at address aAddr in page table aId. Place physical addresses of unmapped
+// pages into aPageList, and count of unmapped pages into aNumPtes.
+// Adjust the page table reference count as if aNumPages pages were unmapped.
+// Return number of pages still mapped using this page table.
+// Call this with the system locked.
+//
+ {
+ SPageTableInfo& ptinfo=iPtInfo[aId];
+ TInt newCount = ptinfo.iCount - aNumPages;
+ UnmapUnownedPages(aId, aAddr, aNumPages, aPageList, aLAPageList, aNumPtes, aNumFree, aProcess);
+ ptinfo.iCount = newCount;
+ aNumPtes = aNumPages;
+ return newCount;
+ }
+
+void ArmMmu::DoAssignPageTable(TInt aId, TLinAddr aAddr, TPde aPdePerm, const TAny* aOsAsids)
+//
+// Assign an allocated page table to map a given linear address with specified permissions.
+// This should be called with the system unlocked and the MMU mutex held.
+//
+ {
+ __KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::DoAssignPageTable %d to %08x perm %08x asid %08x",aId,aAddr,aPdePerm,aOsAsids));
+ TLinAddr ptLin=PageTableLinAddr(aId);
+ TPhysAddr ptPhys=LinearToPhysical(ptLin,0);
+ TInt pdeIndex=TInt(aAddr>>KChunkShift);
+ TBool gpd=(pdeIndex>=(iLocalPdSize>>2));
+ TInt os_asid=(TInt)aOsAsids;
+ if (TUint32(os_asid)<TUint32(iNumOsAsids))
+ {
+ // single OS ASID
+ TPde* pageDir=PageDirectory(os_asid);
+ NKern::LockSystem();
+ pageDir[pdeIndex]=ptPhys|aPdePerm; // will blow up here if address is in global region aOsAsid doesn't have a global PD
+ CacheMaintenance::SinglePteUpdated((TLinAddr)(pageDir+pdeIndex));
+ NKern::UnlockSystem();
+
+ __KTRACE_OPT(KMMU,Kern::Printf("Writing PDE %08x to %08x",ptPhys|aPdePerm,pageDir+pdeIndex));
+ }
+ else if (os_asid==-1 && gpd)
+ {
+ // all OS ASIDs, address in global region
+ TInt num_os_asids=iNumGlobalPageDirs;
+ const TBitMapAllocator& b=*(const TBitMapAllocator*)iOsAsidAllocator;
+ for (os_asid=0; num_os_asids; ++os_asid)
+ {
+ if (!b.NotAllocated(os_asid,1) && (iAsidInfo[os_asid]&1))
+ {
+ // this OS ASID exists and has a global page directory
+ TPde* pageDir=PageDirectory(os_asid);
+ NKern::LockSystem();
+ pageDir[pdeIndex]=ptPhys|aPdePerm;
+ CacheMaintenance::SinglePteUpdated((TLinAddr)(pageDir+pdeIndex));
+ NKern::UnlockSystem();
+
+ __KTRACE_OPT(KMMU,Kern::Printf("Writing PDE %08x to %08x",ptPhys|aPdePerm,pageDir+pdeIndex));
+ --num_os_asids;
+ }
+ }
+ }
+ else
+ {
+ // selection of OS ASIDs or all OS ASIDs
+ const TBitMapAllocator* pB=(const TBitMapAllocator*)aOsAsids;
+ if (os_asid==-1)
+ pB=iOsAsidAllocator; // 0's in positions which exist
+ TInt num_os_asids=pB->iSize-pB->iAvail;
+ for (os_asid=0; num_os_asids; ++os_asid)
+ {
+ if (pB->NotAllocated(os_asid,1))
+ continue; // os_asid is not needed
+ TPde* pageDir=PageDirectory(os_asid);
+ NKern::LockSystem();
+ pageDir[pdeIndex]=ptPhys|aPdePerm;
+ CacheMaintenance::SinglePteUpdated((TLinAddr)(pageDir+pdeIndex));
+ NKern::UnlockSystem();
+
+ __KTRACE_OPT(KMMU,Kern::Printf("Writing PDE %08x to %08x",ptPhys|aPdePerm,pageDir+pdeIndex));
+ --num_os_asids;
+ }
+ }
+ }
+
+void ArmMmu::RemapPageTableSingle(TPhysAddr aOld, TPhysAddr aNew, TLinAddr aAddr, TInt aOsAsid)
+//
+// Replace a single page table mapping the specified linear address.
+// This should be called with the system locked and the MMU mutex held.
+//
+ {
+ __KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::RemapPageTableSingle %08x to %08x at %08x asid %d",aOld,aNew,aAddr,aOsAsid));
+ TPde* pageDir=PageDirectory(aOsAsid);
+ TInt pdeIndex=TInt(aAddr>>KChunkShift);
+ TPde pde=pageDir[pdeIndex];
+ __ASSERT_ALWAYS((pde & KPdePageTableAddrMask) == aOld, Panic(ERemapPageTableFailed));
+ TPde newPde=aNew|(pde&~KPdePageTableAddrMask);
+ pageDir[pdeIndex]=newPde; // will blow up here if address is in global region aOsAsid doesn't have a global PD
+ CacheMaintenance::SinglePteUpdated((TLinAddr)(pageDir+pdeIndex));
+
+ __KTRACE_OPT(KMMU,Kern::Printf("Writing PDE %08x to %08x",newPde,pageDir+pdeIndex));
+ }
+
+void ArmMmu::RemapPageTableGlobal(TPhysAddr aOld, TPhysAddr aNew, TLinAddr aAddr)
+//
+// Replace a global page table mapping the specified linear address.
+// This should be called with the system locked and the MMU mutex held.
+//
+ {
+ __KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::RemapPageTableGlobal %08x to %08x at %08x",aOld,aNew,aAddr));
+ TInt pdeIndex=TInt(aAddr>>KChunkShift);
+ TInt num_os_asids=iNumGlobalPageDirs;
+ const TBitMapAllocator& b=*(const TBitMapAllocator*)iOsAsidAllocator;
+ for (TInt os_asid=0; num_os_asids; ++os_asid)
+ {
+ if (!b.NotAllocated(os_asid,1) && (iAsidInfo[os_asid]&1))
+ {
+ // this OS ASID exists and has a global page directory
+ TPde* pageDir=PageDirectory(os_asid);
+ TPde pde=pageDir[pdeIndex];
+ if ((pde & KPdePageTableAddrMask) == aOld)
+ {
+ TPde newPde=aNew|(pde&~KPdePageTableAddrMask);
+ pageDir[pdeIndex]=newPde;
+ CacheMaintenance::SinglePteUpdated((TLinAddr)(pageDir+pdeIndex));
+
+ __KTRACE_OPT(KMMU,Kern::Printf("Writing PDE %08x to %08x",newPde,pageDir+pdeIndex));
+ }
+ --num_os_asids;
+ }
+ if ((os_asid&31)==31)
+ NKern::FlashSystem();
+ }
+ }
+
+void ArmMmu::RemapPageTableMultiple(TPhysAddr aOld, TPhysAddr aNew, TLinAddr aAddr, const TAny* aOsAsids)
+//
+// Replace multiple page table mappings of the specified linear address.
+// This should be called with the system locked and the MMU mutex held.
+//
+ {
+ __KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::RemapPageTableMultiple %08x to %08x at %08x asids %08x",aOld,aNew,aAddr,aOsAsids));
+ TInt pdeIndex=TInt(aAddr>>KChunkShift);
+ const TBitMapAllocator* pB=(const TBitMapAllocator*)aOsAsids;
+ if ((TInt)aOsAsids==-1)
+ pB=iOsAsidAllocator; // 0's in positions which exist
+
+ TInt asid = -1;
+ TInt lastAsid = KArmV6NumAsids - 1;
+ const TUint32* ptr = pB->iMap;
+ do
+ {
+ TUint32 bits = *ptr++;
+ do
+ {
+ ++asid;
+ if ((bits & 0x80000000u) == 0)
+ {
+ // mapped in this address space - bitmap is inverted
+ TPde* pageDir=PageDirectory(asid);
+ TPde pde=pageDir[pdeIndex];
+ if ((pde & KPdePageTableAddrMask) == aOld)
+ {
+ TPde newPde=aNew|(pde&~KPdePageTableAddrMask);
+ pageDir[pdeIndex]=newPde;
+ CacheMaintenance::SinglePteUpdated((TLinAddr)(pageDir+pdeIndex));
+
+ __KTRACE_OPT(KMMU,Kern::Printf("Writing PDE %08x to %08x",newPde,pageDir+pdeIndex));
+ }
+ }
+ }
+ while(bits<<=1);
+ NKern::FlashSystem();
+ asid |= 31;
+ }
+ while(asid<lastAsid);
+ }
+
+void ArmMmu::RemapPageTableAliases(TPhysAddr aOld, TPhysAddr aNew)
+//
+// Replace aliases of the specified page table.
+// This should be called with the system locked and the MMU mutex held.
+//
+ {
+ __KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::RemapPageTableAliases %08x to %08x",aOld,aNew));
+ SDblQue checkedList;
+ SDblQueLink* next;
+
+ while(!iAliasList.IsEmpty())
+ {
+ next = iAliasList.First()->Deque();
+ checkedList.Add(next);
+ DMemModelThread* thread = (DMemModelThread*)((TInt)next-_FOFF(DMemModelThread,iAliasLink));
+ TPde pde = thread->iAliasPde;
+ if ((pde & ~KPageMask) == aOld)
+ {
+ // a page table in this page is being aliased by the thread, so update it...
+ thread->iAliasPde = (pde & KPageMask) | aNew;
+ }
+ NKern::FlashSystem();
+ }
+
+ // copy checkedList back to iAliasList
+ iAliasList.MoveFrom(&checkedList);
+ }
+
+void ArmMmu::DoUnassignPageTable(TLinAddr aAddr, const TAny* aOsAsids)
+//
+// Unassign a now-empty page table currently mapping the specified linear address.
+// We assume that TLB and/or cache flushing has been done when any RAM pages were unmapped.
+// This should be called with the system unlocked and the MMU mutex held.
+//
+ {
+ __KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::DoUnassignPageTable at %08x a=%08x",aAddr,aOsAsids));
+ TInt pdeIndex=TInt(aAddr>>KChunkShift);
+ TBool gpd=(pdeIndex>=(iLocalPdSize>>2));
+ TInt os_asid=(TInt)aOsAsids;
+ TUint pde=0;
+
+ SDblQue checkedList;
+ SDblQueLink* next;
+
+ if (TUint32(os_asid)<TUint32(iNumOsAsids))
+ {
+ // single OS ASID
+ TPde* pageDir=PageDirectory(os_asid);
+ NKern::LockSystem();
+ pde = pageDir[pdeIndex];
+ pageDir[pdeIndex]=0;
+ CacheMaintenance::SinglePteUpdated((TLinAddr)(pageDir+pdeIndex));
+ __KTRACE_OPT(KMMU,Kern::Printf("Clearing PDE at %08x",pageDir+pdeIndex));
+
+ // remove any aliases of the page table...
+ TUint ptId = pde>>KPageTableShift;
+ while(!iAliasList.IsEmpty())
+ {
+ next = iAliasList.First()->Deque();
+ checkedList.Add(next);
+ DMemModelThread* thread = (DMemModelThread*)((TInt)next-_FOFF(DMemModelThread,iAliasLink));
+ if(thread->iAliasOsAsid==os_asid && (thread->iAliasPde>>KPageTableShift)==ptId)
+ {
+ // the page table is being aliased by the thread, so remove it...
+ thread->iAliasPde = 0;
+ }
+ NKern::FlashSystem();
+ }
+ }
+ else if (os_asid==-1 && gpd)
+ {
+ // all OS ASIDs, address in global region
+ TInt num_os_asids=iNumGlobalPageDirs;
+ const TBitMapAllocator& b=*(const TBitMapAllocator*)iOsAsidAllocator;
+ for (os_asid=0; num_os_asids; ++os_asid)
+ {
+ if (!b.NotAllocated(os_asid,1) && (iAsidInfo[os_asid]&1))
+ {
+ // this OS ASID exists and has a global page directory
+ TPde* pageDir=PageDirectory(os_asid);
+ NKern::LockSystem();
+ pageDir[pdeIndex]=0;
+ CacheMaintenance::SinglePteUpdated((TLinAddr)(pageDir+pdeIndex));
+ NKern::UnlockSystem();
+
+ __KTRACE_OPT(KMMU,Kern::Printf("Clearing PDE at %08x",pageDir+pdeIndex));
+ --num_os_asids;
+ }
+ }
+ // we don't need to look for aliases in this case, because these aren't
+ // created for page tables in the global region.
+ NKern::LockSystem();
+ }
+ else
+ {
+ // selection of OS ASIDs or all OS ASIDs
+ const TBitMapAllocator* pB=(const TBitMapAllocator*)aOsAsids;
+ if (os_asid==-1)
+ pB=iOsAsidAllocator; // 0's in positions which exist
+ TInt num_os_asids=pB->iSize-pB->iAvail;
+ for (os_asid=0; num_os_asids; ++os_asid)
+ {
+ if (pB->NotAllocated(os_asid,1))
+ continue; // os_asid is not needed
+ TPde* pageDir=PageDirectory(os_asid);
+ NKern::LockSystem();
+ pde = pageDir[pdeIndex];
+ pageDir[pdeIndex]=0;
+ CacheMaintenance::SinglePteUpdated((TLinAddr)(pageDir+pdeIndex));
+ NKern::UnlockSystem();
+
+ __KTRACE_OPT(KMMU,Kern::Printf("Clearing PDE at %08x",pageDir+pdeIndex));
+ --num_os_asids;
+ }
+
+ // remove any aliases of the page table...
+ TUint ptId = pde>>KPageTableShift;
+ NKern::LockSystem();
+ while(!iAliasList.IsEmpty())
+ {
+ next = iAliasList.First()->Deque();
+ checkedList.Add(next);
+ DMemModelThread* thread = (DMemModelThread*)((TInt)next-_FOFF(DMemModelThread,iAliasLink));
+ if((thread->iAliasPde>>KPageTableShift)==ptId && !pB->NotAllocated(thread->iAliasOsAsid,1))
+ {
+ // the page table is being aliased by the thread, so remove it...
+ thread->iAliasPde = 0;
+ }
+ NKern::FlashSystem();
+ }
+ }
+
+ // copy checkedList back to iAliasList
+ iAliasList.MoveFrom(&checkedList);
+
+ NKern::UnlockSystem();
+ }
+#endif
+
+// Initialise page table at physical address aXptPhys to be used as page table aXptId
+// to expand the virtual address range used for mapping page tables. Map the page table
+// at aPhysAddr as page table aId using the expanded range.
+// Assign aXptPhys to kernel's Page Directory.
+// Called with system unlocked and MMU mutex held.
+void ArmMmu::BootstrapPageTable(TInt aXptId, TPhysAddr aXptPhys, TInt aId, TPhysAddr aPhysAddr)
+ {
+ __KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::BootstrapPageTable xptid=%04x, xptphys=%08x, id=%04x, phys=%08x",
+ aXptId, aXptPhys, aId, aPhysAddr));
+
+ // put in a temporary mapping for aXptPhys
+ // make it noncacheable
+ TPhysAddr pa=aXptPhys&~KPageMask;
+ *iTempPte = pa | SP_PTE(KArmV6PermRWNO, KNormalUncachedAttr, 0, 1);
+ CacheMaintenance::SinglePteUpdated((TLinAddr)iTempPte);
+
+ // clear XPT
+ TPte* xpt=(TPte*)(iTempAddr+(aXptPhys&KPageMask));
+ memclr(xpt, KPageTableSize);
+
+ // must in fact have aXptPhys and aPhysAddr in same physical page
+ __ASSERT_ALWAYS( TUint32(aXptPhys^aPhysAddr)<TUint32(KPageSize), MM::Panic(MM::EBootstrapPageTableBadAddr));
+
+ // so only need one mapping
+ xpt[(aXptId>>KPtClusterShift)&KPagesInPDEMask] = pa | KPtPtePerm;
+ CacheMaintenance::MultiplePtesUpdated((TLinAddr)xpt, KPageTableSize);
+
+ // remove temporary mapping
+ *iTempPte=0;
+ CacheMaintenance::SinglePteUpdated((TLinAddr)iTempPte);
+
+ InvalidateTLBForPage(iTempAddr, KERNEL_MAPPING);
+
+ // initialise PtInfo...
+ TLinAddr xptAddr = PageTableLinAddr(aXptId);
+ iPtInfo[aXptId].SetGlobal(xptAddr>>KChunkShift);
+
+ // map xpt...
+ TInt pdeIndex=TInt(xptAddr>>KChunkShift);
+ TPde* pageDir=PageDirectory(0);
+ NKern::LockSystem();
+ pageDir[pdeIndex]=aXptPhys|KPtPdePerm;
+ CacheMaintenance::SinglePteUpdated((TLinAddr)(pageDir+pdeIndex));
+
+ NKern::UnlockSystem();
+ }
+
+// Edit the self-mapping entry in page table aId, mapped at aTempMap, to
+// change the physical address from aOld to aNew. Used when moving page
+// tables which were created by BootstrapPageTable.
+// Called with system locked and MMU mutex held.
+void ArmMmu::FixupXPageTable(TInt aId, TLinAddr aTempMap, TPhysAddr aOld, TPhysAddr aNew)
+ {
+ __KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::FixupXPageTable id=%04x, tempmap=%08x, old=%08x, new=%08x",
+ aId, aTempMap, aOld, aNew));
+
+ // find correct page table inside the page
+ TPte* xpt=(TPte*)(aTempMap + ((aId & KPtClusterMask) << KPageTableShift));
+ // find the pte in that page table
+ xpt += (aId>>KPtClusterShift)&KPagesInPDEMask;
+
+ // switch the mapping
+ __ASSERT_ALWAYS((*xpt&~KPageMask)==aOld, Panic(EFixupXPTFailed));
+ *xpt = aNew | KPtPtePerm;
+ // mapped with MapTemp, and thus not mapped as a PTE - have to do real cache clean.
+ CacheMaintenance::SinglePteUpdated((TLinAddr)xpt);
+ }
+
+TInt ArmMmu::NewPageDirectory(TInt aOsAsid, TBool aSeparateGlobal, TPhysAddr& aPhysAddr, TInt& aNumPages)
+ {
+ __KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::NewPageDirectory(%d,%d)",aOsAsid,aSeparateGlobal));
+ TInt r=0;
+ TInt nlocal=iLocalPdSize>>KPageShift;
+ aNumPages=aSeparateGlobal ? KPageDirectorySize/KPageSize : nlocal;
+ __KTRACE_OPT(KMMU,Kern::Printf("nlocal=%d, aNumPages=%d",nlocal,aNumPages));
+ if (aNumPages>1)
+ {
+ TInt align=aSeparateGlobal ? KPageDirectoryShift : KPageDirectoryShift-1;
+ r=AllocContiguousRam(aNumPages<<KPageShift, aPhysAddr, EPageFixed, align);
+ }
+ else
+ r=AllocRamPages(&aPhysAddr,1, EPageFixed);
+ __KTRACE_OPT(KMMU,Kern::Printf("r=%d, phys=%08x",r,aPhysAddr));
+ if (r!=KErrNone)
+ return r;
+#ifdef BTRACE_KERNEL_MEMORY
+ BTrace4(BTrace::EKernelMemory, BTrace::EKernelMemoryMiscAlloc, aNumPages<<KPageShift);
+ Epoc::KernelMiscPages += aNumPages;
+#endif
+ SPageInfo* pi = SPageInfo::FromPhysAddr(aPhysAddr);
+ NKern::LockSystem();
+ TInt i;
+ for (i=0; i<aNumPages; ++i)
+ pi[i].SetPageDir(aOsAsid,i);
+ NKern::UnlockSystem();
+ return KErrNone;
+ }
+
+inline void CopyPdes(TPde* aDest, const TPde* aSrc, TLinAddr aBase, TLinAddr aEnd)
+ {
+ memcpy(aDest+(aBase>>KChunkShift), aSrc+(aBase>>KChunkShift), ((aEnd-aBase)>>KChunkShift)*sizeof(TPde));
+ CacheMaintenance::MultiplePtesUpdated((TLinAddr)(aDest+(aBase>>KChunkShift)), ((aEnd-aBase)>>KChunkShift)*sizeof(TPde));
+ }
+
+inline void ZeroPdes(TPde* aDest, TLinAddr aBase, TLinAddr aEnd)
+ {
+ memclr(aDest+(aBase>>KChunkShift), ((aEnd-aBase)>>KChunkShift)*sizeof(TPde));
+ CacheMaintenance::MultiplePtesUpdated((TLinAddr)(aDest+(aBase>>KChunkShift)), ((aEnd-aBase)>>KChunkShift)*sizeof(TPde));
+ }
+
+void ArmMmu::InitPageDirectory(TInt aOsAsid, TBool aSeparateGlobal)
+ {
+ __KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::InitPageDirectory(%d,%d)",aOsAsid,aSeparateGlobal));
+ TPde* newpd=PageDirectory(aOsAsid); // new page directory
+ memclr(newpd, iLocalPdSize); // clear local page directory
+ CacheMaintenance::MultiplePtesUpdated((TLinAddr)newpd, iLocalPdSize);
+ if (aSeparateGlobal)
+ {
+ const TPde* kpd=(const TPde*)KPageDirectoryBase; // kernel page directory
+ if (iLocalPdSize==KPageSize)
+ ZeroPdes(newpd, KUserSharedDataEnd1GB, KUserSharedDataEnd2GB);
+ ZeroPdes(newpd, KRamDriveStartAddress, KRamDriveEndAddress); // don't copy RAM drive
+ CopyPdes(newpd, kpd, KRomLinearBase, KUserGlobalDataEnd); // copy ROM + user global
+ CopyPdes(newpd, kpd, KRamDriveEndAddress, 0x00000000); // copy kernel mappings
+ }
+ }
+
+void ArmMmu::ClearPageTable(TInt aId, TInt aFirstIndex)
+ {
+ __KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::ClearPageTable(%d,%d)",aId,aFirstIndex));
+ TPte* pte=PageTable(aId);
+ memclr(pte+aFirstIndex, KPageTableSize-aFirstIndex*sizeof(TPte));
+ CacheMaintenance::MultiplePtesUpdated((TLinAddr)(pte+aFirstIndex), KPageTableSize-aFirstIndex*sizeof(TPte));
+ }
+
+void ArmMmu::ApplyTopLevelPermissions(TLinAddr aAddr, TInt aOsAsid, TInt aNumPdes, TPde aPdePerm)
+ {
+ __KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::ApplyTopLevelPermissions %04x:%08x->%08x count %d",
+ aOsAsid, aAddr, aPdePerm, aNumPdes));
+ TInt ix=aAddr>>KChunkShift;
+ TPde* pPde=PageDirectory(aOsAsid)+ix;
+ TLinAddr firstPde = (TLinAddr)pPde; //Will need this to clean page table memory region in cache
+
+ TPde* pPdeEnd=pPde+aNumPdes;
+ NKern::LockSystem();
+ for (; pPde<pPdeEnd; ++pPde)
+ {
+ TPde pde=*pPde;
+ if (pde)
+ *pPde = (pde&KPdePageTableAddrMask)|aPdePerm;
+ }
+ CacheMaintenance::MultiplePtesUpdated(firstPde, aNumPdes*sizeof(TPde));
+ FlushTLBs();
+ NKern::UnlockSystem();
+ }
+
+void ArmMmu::ApplyPagePermissions(TInt aId, TInt aPageOffset, TInt aNumPages, TPte aPtePerm)
+ {
+ __KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::ApplyPagePermissions %04x:%03x+%03x perm %08x",
+ aId, aPageOffset, aNumPages, aPtePerm));
+ TPte* pPte=PageTable(aId)+aPageOffset;
+ TLinAddr firstPte = (TLinAddr)pPte; //Will need this to clean page table memory region in cache
+
+ TPde* pPteEnd=pPte+aNumPages;
+ NKern::LockSystem();
+ for (; pPte<pPteEnd; ++pPte)
+ {
+ TPte pte=*pPte;
+ if (pte)
+ *pPte = (pte&KPteSmallPageAddrMask)|aPtePerm;
+ }
+ CacheMaintenance::MultiplePtesUpdated(firstPte, aNumPages*sizeof(TPte));
+ FlushTLBs();
+ NKern::UnlockSystem();
+ }
+
+void ArmMmu::ClearRamDrive(TLinAddr aStart)
+ {
+ // clear the page directory entries corresponding to the RAM drive
+ TPde* kpd=(TPde*)KPageDirectoryBase; // kernel page directory
+ ZeroPdes(kpd, aStart, KRamDriveEndAddress);
+ }
+
+TPde ArmMmu::PdePermissions(TChunkType aChunkType, TBool aRO)
+ {
+// if (aChunkType==EUserData && aRO)
+// return KPdePtePresent|KPdePteUser;
+ return ChunkPdePermissions[aChunkType];
+ }
+
+TPte ArmMmu::PtePermissions(TChunkType aChunkType)
+ {
+ return ChunkPtePermissions[aChunkType];
+ }
+
+// Set up a page table (specified by aId) to map a 1Mb section of ROM containing aRomAddr
+// using ROM at aOrigPhys.
+void ArmMmu::InitShadowPageTable(TInt aId, TLinAddr aRomAddr, TPhysAddr aOrigPhys)
+ {
+ __KTRACE_OPT(KMMU, Kern::Printf("ArmMmu:InitShadowPageTable id=%04x aRomAddr=%08x aOrigPhys=%08x",
+ aId, aRomAddr, aOrigPhys));
+ TPte* ppte = PageTable(aId);
+ TLinAddr firstPte = (TLinAddr)ppte; //Will need this to clean page table memory region in cache
+
+ TPte* ppte_End = ppte + KChunkSize/KPageSize;
+ TPhysAddr phys = aOrigPhys - (aRomAddr & KChunkMask);
+ for (; ppte<ppte_End; ++ppte, phys+=KPageSize)
+ *ppte = phys | KRomPtePerm;
+ CacheMaintenance::MultiplePtesUpdated(firstPte, sizeof(TPte)*KChunkSize/KPageSize);
+ }
+
+// Copy the contents of ROM at aRomAddr to a shadow page at physical address aShadowPhys
+// It is assumed aShadowPage is not mapped, therefore any mapping colour is OK.
+void ArmMmu::InitShadowPage(TPhysAddr aShadowPhys, TLinAddr aRomAddr)
+ {
+ __KTRACE_OPT(KMMU, Kern::Printf("ArmMmu:InitShadowPage aShadowPhys=%08x aRomAddr=%08x",
+ aShadowPhys, aRomAddr));
+
+ // put in a temporary mapping for aShadowPhys
+ // make it noncacheable
+ *iTempPte = aShadowPhys | SP_PTE(KArmV6PermRWNO, KNormalUncachedAttr, 0, 1);
+ CacheMaintenance::SinglePteUpdated((TLinAddr)iTempPte);
+
+ // copy contents of ROM
+ wordmove( (TAny*)iTempAddr, (const TAny*)aRomAddr, KPageSize );
+ //Temp address is uncached. No need to clean cache, just flush write buffer
+ CacheMaintenance::MemoryToPreserveAndReuse((TLinAddr)iTempAddr, KPageSize, EMapAttrBufferedC);
+
+ // remove temporary mapping
+ *iTempPte=0;
+ CacheMaintenance::SinglePteUpdated((TLinAddr)iTempPte);
+ InvalidateTLBForPage(iTempAddr, KERNEL_MAPPING);
+ }
+
+// Assign a shadow page table to replace a ROM section mapping
+// Enter and return with system locked
+void ArmMmu::AssignShadowPageTable(TInt aId, TLinAddr aRomAddr)
+ {
+ __KTRACE_OPT(KMMU, Kern::Printf("ArmMmu:AssignShadowPageTable aId=%04x aRomAddr=%08x",
+ aId, aRomAddr));
+ TLinAddr ptLin=PageTableLinAddr(aId);
+ TPhysAddr ptPhys=LinearToPhysical(ptLin, 0);
+ TPde* ppde = ::InitPageDirectory + (aRomAddr>>KChunkShift);
+ TPde newpde = ptPhys | KShadowPdePerm;
+ __KTRACE_OPT(KMMU,Kern::Printf("Writing PDE %08x to %08x", newpde, ppde));
+ TInt irq=NKern::DisableAllInterrupts();
+ *ppde = newpde; // map in the page table
+ CacheMaintenance::SinglePteUpdated((TLinAddr)ppde);
+
+ FlushTLBs(); // flush both TLBs (no need to flush cache yet)
+ NKern::RestoreInterrupts(irq);
+ }
+
+void ArmMmu::DoUnmapShadowPage(TInt aId, TLinAddr aRomAddr, TPhysAddr aOrigPhys)
+ {
+ __KTRACE_OPT(KMMU,Kern::Printf("ArmMmu:DoUnmapShadowPage, id=%04x lin=%08x origphys=%08x", aId, aRomAddr, aOrigPhys));
+ TPte* ppte = PageTable(aId) + ((aRomAddr & KChunkMask)>>KPageShift);
+ TPte newpte = aOrigPhys | KRomPtePerm;
+ __KTRACE_OPT(KMMU,Kern::Printf("Writing PTE %08x to %08x", newpte, ppte));
+ TInt irq=NKern::DisableAllInterrupts();
+ *ppte = newpte;
+ CacheMaintenance::SinglePteUpdated((TLinAddr)ppte);
+
+ InvalidateTLBForPage(aRomAddr, KERNEL_MAPPING);
+ #if defined(__CPU_ARM1136__) && !defined(__CPU_ARM1136_ERRATUM_353494_FIXED)
+ __FlushBtb();
+ #endif
+
+ CacheMaintenance::CodeChanged(aRomAddr, KPageSize, CacheMaintenance::EMemoryRemap);
+ CacheMaintenance::PageToReuse(aRomAddr, EMemAttNormalCached, KPhysAddrInvalid);
+ NKern::RestoreInterrupts(irq);
+ }
+
+TInt ArmMmu::UnassignShadowPageTable(TLinAddr aRomAddr, TPhysAddr aOrigPhys)
+ {
+ __KTRACE_OPT(KMMU,Kern::Printf("ArmMmu:UnassignShadowPageTable, lin=%08x origphys=%08x", aRomAddr, aOrigPhys));
+ TPde* ppde = ::InitPageDirectory + (aRomAddr>>KChunkShift);
+ TPde newpde = (aOrigPhys &~ KChunkMask) | KRomSectionPermissions;
+ __KTRACE_OPT(KMMU,Kern::Printf("Writing PDE %08x to %08x", newpde, ppde));
+ TInt irq=NKern::DisableAllInterrupts();
+ *ppde = newpde; // revert to section mapping
+ CacheMaintenance::SinglePteUpdated((TLinAddr)ppde);
+
+ FlushTLBs(); // flush both TLBs
+ NKern::RestoreInterrupts(irq);
+ return KErrNone;
+ }
+
+
+#if defined(__CPU_MEMORY_TYPE_REMAPPING) // arm1176, arm11mcore, armv7, ...
+/**
+Shadow pages on platforms with remapping (mpcore, 1176, cortex...) are not writable.
+This will map the region into writable memory first.
+@pre No Fast Mutex held
+*/
+TInt ArmMmu::CopyToShadowMemory(TLinAddr aDest, TLinAddr aSrc, TUint32 aLength)
+ {
+ __KTRACE_OPT(KMMU, Kern::Printf("ArmMmu:CopyToShadowMemory aDest=%08x aSrc=%08x aLength=%08x", aDest, aSrc, aLength));
+
+ // Check that destination is ROM
+ if (aDest<iRomLinearBase || (aDest+aLength) > iRomLinearEnd)
+ {
+ __KTRACE_OPT(KMMU,Kern::Printf("ArmMmu:CopyToShadowMemory: Destination not entirely in ROM"));
+ return KErrArgument;
+ }
+ // do operation with RamAlloc mutex held (to prevent shadow pages from being released from under us)
+ MmuBase::Wait();
+
+
+ TInt r = KErrNone;
+ while (aLength)
+ {
+ // Calculate memory size to copy in this loop. A single page region will be copied per loop
+ TInt copySize = Min(aLength, iPageSize - (aDest&iPageMask));
+
+ // Get physical address
+ TPhysAddr physAddr = LinearToPhysical(aDest&~iPageMask, 0);
+ if (KPhysAddrInvalid==physAddr)
+ {
+ r = KErrArgument;
+ break;
+ }
+
+ //check whether it is shadowed rom
+ SPageInfo* pi = SPageInfo::SafeFromPhysAddr(physAddr);
+ if (pi==0 || pi->Type()!=SPageInfo::EShadow)
+ {
+ __KTRACE_OPT(KMMU,Kern::Printf("ArmMmu:CopyToShadowMemory: No shadow page at this address"));
+ r = KErrArgument;
+ break;
+ }
+
+ //Temporarily map into writable memory and copy data. RamAllocator DMutex is required
+ TLinAddr tempAddr = MapTemp (physAddr, aDest&~iPageMask);
+ __KTRACE_OPT(KMMU, Kern::Printf("ArmMmu:CopyToShadowMemory Copy aDest=%08x aSrc=%08x aSize=%08x", tempAddr+(aDest&iPageMask), aSrc, copySize));
+ memcpy ((TAny*)(tempAddr+(aDest&iPageMask)), (const TAny*)aSrc, copySize); //Kernel-to-Kernel copy is presumed
+ UnmapTemp();
+
+ //Update variables for the next loop/page
+ aDest+=copySize;
+ aSrc+=copySize;
+ aLength-=copySize;
+ }
+ MmuBase::Signal();
+ return r;
+ }
+#endif
+
+void ArmMmu::DoFreezeShadowPage(TInt aId, TLinAddr aRomAddr)
+ {
+#if defined(__CPU_MEMORY_TYPE_REMAPPING) //arm1176, arm11mcore, armv7 and later
+ __KTRACE_OPT(KMMU, Kern::Printf("ArmMmu:DoFreezeShadowPage not required with MEMORY_TYPE_REMAPPING"));
+#else
+ __KTRACE_OPT(KMMU, Kern::Printf("ArmMmu:DoFreezeShadowPage aId=%04x aRomAddr=%08x",
+ aId, aRomAddr));
+ TPte* ppte = PageTable(aId) + ((aRomAddr & KChunkMask)>>KPageShift);
+ TPte newpte = (*ppte & KPteSmallPageAddrMask) | KRomPtePerm;
+ __KTRACE_OPT(KMMU,Kern::Printf("Writing PTE %08x to %08x", newpte, ppte));
+ *ppte = newpte;
+ CacheMaintenance::SinglePteUpdated((TLinAddr)ppte);
+ InvalidateTLBForPage(aRomAddr, KERNEL_MAPPING);
+#endif
+ }
+
+/** Replaces large page(64K) entry in page table with small page(4K) entries.*/
+void ArmMmu::Pagify(TInt aId, TLinAddr aLinAddr)
+ {
+ __KTRACE_OPT(KMMU, Kern::Printf("ArmMmu:Pagify aId=%04x aLinAddr=%08x", aId, aLinAddr));
+
+ TInt pteIndex = (aLinAddr & KChunkMask)>>KPageShift;
+ TPte* pte = PageTable(aId);
+ if ((pte[pteIndex] & KArmV6PteTypeMask) == KArmV6PteLargePage)
+ {
+ __KTRACE_OPT(KMMU,Kern::Printf("Converting 64K page to 4K pages"));
+ pteIndex &= ~0xf;
+ TPte source = pte[pteIndex];
+ source = (source & KPteLargePageAddrMask) | SP_PTE_FROM_LP_PTE(source);
+ pte += pteIndex;
+ for (TInt entry=0; entry<16; entry++)
+ {
+ pte[entry] = source | (entry<<12);
+ }
+ CacheMaintenance::MultiplePtesUpdated((TLinAddr)pte, 16*sizeof(TPte));
+ FlushTLBs();
+ }
+ }
+
+void ArmMmu::FlushShadow(TLinAddr aRomAddr)
+ {
+ CacheMaintenance::CodeChanged(aRomAddr, KPageSize, CacheMaintenance::EMemoryRemap);
+ CacheMaintenance::PageToReuse(aRomAddr, EMemAttNormalCached, KPhysAddrInvalid);
+ InvalidateTLBForPage(aRomAddr, KERNEL_MAPPING); // remove all TLB references to original ROM page
+ }
+
+
+#if defined(__CPU_MEMORY_TYPE_REMAPPING) //arm1176, arm11mcore, armv7
+/**
+Calculates page directory/table entries for memory type described in aMapAttr.
+Global, small page (4KB) mapping is assumed.
+(All magic numbers come from ARM page table descriptions.)
+@param aMapAttr On entry, holds description(memory type, access permisions,...) of the memory.
+ It is made up of TMappingAttributes constants or TMappingAttributes2 object. If TMappingAttributes,
+ may be altered on exit to hold the actual cache attributes & access permissions.
+@param aPde On exit, holds page-table-entry for the 1st level descriptor
+ for given type of memory, with base address set to 0.
+@param aPte On exit, holds small-page-entry (4K) for the 2nd level descriptor
+ for given type of memory, with base address set to 0.
+@return KErrNotSupported If memory described in aMapAttr is not supported
+ KErrNone Otherwise
+*/
+TInt ArmMmu::PdePtePermissions(TUint& aMapAttr, TPde& aPde, TPte& aPte)
+ {
+ __KTRACE_OPT(KMMU,Kern::Printf(">ArmMmu::PdePtePermissions, mapattr=%08x",aMapAttr));
+
+ TMappingAttributes2& memory = (TMappingAttributes2&)aMapAttr;
+
+ if(memory.ObjectType2())
+ {
+//---------Memory described by TMappingAttributes2 object-----------------
+ aPde = KArmV6PdePageTable |
+ (memory.Parity() ? KArmV6PdeECCEnable : 0);
+#if defined(FAULTY_NONSHARED_DEVICE_MEMORY)
+ if(!memory.Shared() && (memory.Type() == EMemAttDevice ))
+ {
+ aMapAttr ^= EMapAttrBufferedNC;
+ aMapAttr |= EMapAttrFullyBlocking;
+ // Clear EMemAttDevice
+ aMapAttr ^= (EMemAttDevice << 26);
+ aMapAttr |= (EMemAttStronglyOrdered << 26);
+ }
+#endif
+ aPte = KArmV6PteSmallPage |
+ KArmV6PteAP0 | // AP0 bit always 1
+ ((memory.Type()&3)<<2) | ((memory.Type()&4)<<4) | // memory type
+ (memory.Executable() ? 0 : KArmV6PteSmallXN) | // eXecuteNever bit
+#if defined (__CPU_USE_SHARED_MEMORY)
+ KArmV6PteS | // Memory is always shared.
+#else
+ (memory.Shared() ? KArmV6PteS : 0) | // Shared bit
+#endif
+ (memory.Writable() ? 0 : KArmV6PteAPX) | // APX = !Writable
+ (memory.UserAccess() ? KArmV6PteAP1: 0); // AP1 = UserAccess
+ // aMapAttr remains the same
+ }
+ else
+ {
+//---------Memory described by TMappingAttributes bitmask-----------------
+#if defined(FAULTY_NONSHARED_DEVICE_MEMORY)
+ if(((aMapAttr & EMapAttrL1CacheMask) == EMapAttrBufferedNC) && !(aMapAttr & EMapAttrShared))
+ {
+ // Clear EMapAttrBufferedNC attribute
+ aMapAttr ^= EMapAttrBufferedNC;
+ aMapAttr |= EMapAttrFullyBlocking;
+ }
+#endif
+ // 1. Calculate TEX0:C:B bits in page table and actual cache attributes.
+ // Only L1 cache attribute from aMapAttr matters. Outer (L2) cache policy will be the same as inner one.
+ TUint l1cache=aMapAttr & EMapAttrL1CacheMask; // Inner cache attributes. May change to actual value.
+ TUint l2cache; // Will hold actual L2 cache attributes (in terms of TMappingAttributes constants)
+ TUint tex0_c_b; // Will hold TEX[0]:C:B value in page table
+
+ switch (l1cache)
+ {
+ case EMapAttrFullyBlocking:
+ tex0_c_b = EMemAttStronglyOrdered;
+ l2cache = EMapAttrL2Uncached;
+ break;
+ case EMapAttrBufferedNC:
+ tex0_c_b = EMemAttDevice;
+ l2cache = EMapAttrL2Uncached;
+ break;
+ case EMapAttrBufferedC:
+ case EMapAttrL1Uncached:
+ case EMapAttrCachedWTRA:
+ case EMapAttrCachedWTWA:
+ tex0_c_b = EMemAttNormalUncached;
+ l1cache = EMapAttrBufferedC;
+ l2cache = EMapAttrL2Uncached;
+ break;
+ case EMapAttrCachedWBRA:
+ case EMapAttrCachedWBWA:
+ case EMapAttrL1CachedMax:
+ tex0_c_b = EMemAttNormalCached;
+ l1cache = EMapAttrCachedWBWA;
+ l2cache = EMapAttrL2CachedWBWA;
+ break;
+ default:
+ return KErrNotSupported;
+ }
+
+ // 2. Step 2 has been removed :)
+
+ // 3. Calculate access permissions (apx:ap bits in page table + eXecute it)
+ TUint read=aMapAttr & EMapAttrReadMask;
+ TUint write=(aMapAttr & EMapAttrWriteMask)>>4;
+ TUint exec=(aMapAttr & EMapAttrExecMask)>>8;
+
+ read|=exec; // User/Sup execute access requires User/Sup read access.
+ if (exec) exec = 1; // There is a single eXecute bit in page table. Set to one if User or Sup exec is required.
+
+ TUint apxap=0;
+ if (write==0) // no write required
+ {
+ if (read>=4) apxap=KArmV6PermRORO; // user read required
+ else if (read==1) apxap=KArmV6PermRONO; // supervisor read required
+ else return KErrNotSupported; // no read required
+ }
+ else if (write<4) // supervisor write required
+ {
+ if (read<4) apxap=KArmV6PermRWNO; // user read not required
+ else return KErrNotSupported; // user read required
+ }
+ else // user & supervisor writes required
+ {
+ apxap=KArmV6PermRWRW;
+ }
+
+ // 4. Calculate page-table-entry for the 1st level (aka page directory) descriptor
+ aPde=((aMapAttr&EMapAttrUseECC)>>8)|KArmV6PdePageTable;
+
+ // 5. Calculate small-page-entry for the 2nd level (aka page table) descriptor
+ aPte=SP_PTE(apxap, tex0_c_b, exec, 1); // always global
+ if (aMapAttr&EMapAttrShared)
+ aPte |= KArmV6PteS;
+
+ // 6. Fix aMapAttr to hold the actual values for access permission & cache attributes
+ TUint xnapxap=((aPte<<3)&8)|((aPte>>7)&4)|((aPte>>4)&3);
+ aMapAttr &= ~(EMapAttrAccessMask|EMapAttrL1CacheMask|EMapAttrL2CacheMask);
+ aMapAttr |= PermissionLookup[xnapxap]; // Set actual access permissions
+ aMapAttr |= l1cache; // Set actual inner cache attributes
+ aMapAttr |= l2cache; // Set actual outer cache attributes
+ }
+
+ __KTRACE_OPT(KMMU,Kern::Printf("<ArmMmu::PdePtePermissions, mapattr=%08x, pde=%08x, pte=%08x", aMapAttr, aPde, aPte));
+ return KErrNone;
+ }
+
+#else //ARMv6 (arm1136)
+
+const TUint FBLK=(EMapAttrFullyBlocking>>12);
+const TUint BFNC=(EMapAttrBufferedNC>>12);
+//const TUint BUFC=(EMapAttrBufferedC>>12);
+const TUint L1UN=(EMapAttrL1Uncached>>12);
+const TUint WTRA=(EMapAttrCachedWTRA>>12);
+//const TUint WTWA=(EMapAttrCachedWTWA>>12);
+const TUint WBRA=(EMapAttrCachedWBRA>>12);
+const TUint WBWA=(EMapAttrCachedWBWA>>12);
+const TUint AWTR=(EMapAttrAltCacheWTRA>>12);
+//const TUint AWTW=(EMapAttrAltCacheWTWA>>12);
+//const TUint AWBR=(EMapAttrAltCacheWBRA>>12);
+const TUint AWBW=(EMapAttrAltCacheWBWA>>12);
+const TUint MAXC=(EMapAttrL1CachedMax>>12);
+
+const TUint L2UN=(EMapAttrL2Uncached>>16);
+
+const TUint8 UNS=0xffu; // Unsupported attribute
+
+//Maps L1 & L2 cache attributes into TEX[4:2]:CB[1:0]
+//ARMv6 doesn't do WTWA so we use WTRA instead
+
+#if !defined(__CPU_ARM1136_ERRATUM_399234_FIXED)
+// L1 Write-Through mode is outlawed, L1WT acts as L1UN.
+static const TUint8 CBTEX[40]=
+ { // L1CACHE:
+// FBLK BFNC BUFC L1UN WTRA WTWA WBRA WBWA L2CACHE:
+ 0x00, 0x01, 0x01, 0x04, 0x04, 0x04, 0x13, 0x11, //NC
+ 0x00, 0x01, 0x01, 0x18, 0x18, 0x18, 0x1b, 0x19, //WTRA
+ 0x00, 0x01, 0x01, 0x18, 0x18, 0x18, 0x1b, 0x19, //WTWA
+ 0x00, 0x01, 0x01, 0x1c, 0x1c, 0x1c, 0x1f, 0x1d, //WBRA
+ 0x00, 0x01, 0x01, 0x14, 0x14, 0x14, 0x17, 0x15 //WBWA
+ };
+#else
+static const TUint8 CBTEX[40]=
+ { // L1CACHE:
+// FBLK BFNC BUFC L1UN WTRA WTWA WBRA WBWA L2CACHE:
+ 0x00, 0x01, 0x01, 0x04, 0x12, 0x12, 0x13, 0x11, //NC
+ 0x00, 0x01, 0x01, 0x18, 0x02, 0x02, 0x1b, 0x19, //WTRA
+ 0x00, 0x01, 0x01, 0x18, 0x02, 0x02, 0x1b, 0x19, //WTWA
+ 0x00, 0x01, 0x01, 0x1c, 0x1e, 0x1e, 0x1f, 0x1d, //WBRA
+ 0x00, 0x01, 0x01, 0x14, 0x16, 0x16, 0x17, 0x15 //WBWA
+ };
+#endif
+
+//Maps TEX[4:2]:CB[1:0] value into L1 cache attributes
+static const TUint8 L1Actual[32]=
+ {
+//CB 00 01 10 11 //TEX
+ FBLK, BFNC, WTRA, WBRA, //000
+ L1UN, UNS, UNS, WBWA, //001
+ BFNC, UNS, UNS, UNS, //010
+ UNS, UNS, UNS, UNS, //011
+ L1UN, WBWA, WTRA, WBRA, //100
+ L1UN, WBWA, WTRA, WBRA, //101
+ L1UN, WBWA, WTRA, WBRA, //110
+ L1UN, WBWA, WTRA, WBRA //111
+ };
+
+//Maps TEX[4:2]:CB[1:0] value into L2 cache attributes
+static const TUint8 L2Actual[32]=
+ {
+//CB 00 01 10 11 //TEX
+ L2UN, L2UN, WTRA, WBRA, //000
+ L2UN, UNS, UNS, WBWA, //001
+ L2UN, UNS, UNS, UNS, //010
+ UNS, UNS, UNS, UNS, //011
+ L2UN, L2UN, L2UN, L2UN, //100
+ WBWA, WBWA, WBWA, WBWA, //101
+ WTRA, WTRA, WTRA, WTRA, //110
+ WBRA, WBRA, WBRA, WBRA //111
+ };
+
+TInt ArmMmu::PdePtePermissions(TUint& aMapAttr, TPde& aPde, TPte& aPte)
+ {
+ __KTRACE_OPT(KMMU,Kern::Printf(">ArmMmu::PdePtePermissions, mapattr=%08x",aMapAttr));
+
+ TUint read=aMapAttr & EMapAttrReadMask;
+ TUint write=(aMapAttr & EMapAttrWriteMask)>>4;
+ TUint exec=(aMapAttr & EMapAttrExecMask)>>8;
+ TUint l1cache=(aMapAttr & EMapAttrL1CacheMask)>>12;
+ TUint l2cache=(aMapAttr & EMapAttrL2CacheMask)>>16;
+ if (l1cache==MAXC) l1cache=WBRA; // map max cache to WBRA
+ if (l1cache>AWBW)
+ return KErrNotSupported; // undefined attribute
+ if (l1cache>=AWTR) l1cache-=4; // no alternate cache, so use normal cache
+ if (l1cache<L1UN) l2cache=0; // for blocking/device, don't cache L2
+ if (l2cache==MAXC) l2cache=WBRA; // map max cache to WBRA
+ if (l2cache>WBWA)
+ return KErrNotSupported; // undefined attribute
+ if (l2cache) l2cache-=(WTRA-1); // l2cache now in range 0-4
+ aPde=((aMapAttr&EMapAttrUseECC)>>8)|KArmV6PdePageTable;
+
+#if defined(__CPU_ARM1136__) && !defined(__CPU_ARM1136_ERRATUM_353494_FIXED)
+ // if broken 1136, can't have supervisor only code
+ if (exec)
+ exec = TUint(EMapAttrExecUser>>8);
+#endif
+
+ // if any execute access, must have read=execute
+ if (exec)
+ (void)(read>=exec || (read=exec)!=0), exec=1;
+
+ // l1cache between 0 and 7, l2cache between 0 and 4; look up CBTEX
+ TUint cbtex=CBTEX[(l2cache<<3)|l1cache];
+
+ // work out apx:ap
+ TUint apxap;
+ if (write==0)
+ apxap=(read>=4)?KArmV6PermRORO:(read?KArmV6PermRONO:KArmV6PermNONO);
+ else if (write<4)
+ apxap=(read>=4)?KArmV6PermRWRO:KArmV6PermRWNO;
+ else
+ apxap=KArmV6PermRWRW;
+ TPte pte=SP_PTE(apxap, cbtex, exec, 1); // always global
+ if (aMapAttr&EMapAttrShared)
+ pte |= KArmV6PteS;
+
+ // Translate back to get actual map attributes
+ TUint xnapxap=((pte<<3)&8)|((pte>>7)&4)|((pte>>4)&3);
+ cbtex=((pte>>4)&0x1c)|((pte>>2)&3); // = TEX[4:2]::CB[1:0]
+ aMapAttr &= ~(EMapAttrAccessMask|EMapAttrL1CacheMask|EMapAttrL2CacheMask);
+ aMapAttr |= PermissionLookup[xnapxap];
+ aMapAttr |= (L1Actual[cbtex]<<12);
+ aMapAttr |= (L2Actual[cbtex]<<16);
+ aPte=pte;
+ __KTRACE_OPT(KMMU,Kern::Printf("<ArmMmu::PdePtePermissions, mapattr=%08x, pde=%08x, pte=%08x",
+ aMapAttr, aPde, aPte));
+ return KErrNone;
+ }
+#endif
+
+void ArmMmu::Map(TLinAddr aLinAddr, TPhysAddr aPhysAddr, TInt aSize, TPde aPdePerm, TPte aPtePerm, TInt aMapShift)
+//
+// Map a region of physical addresses aPhysAddr to aPhysAddr+aSize-1 to virtual address aLinAddr.
+// Use permissions specified by aPdePerm and aPtePerm. Use mapping sizes up to and including (1<<aMapShift).
+// Assume any page tables required are already assigned.
+// aLinAddr, aPhysAddr, aSize must be page-aligned.
+//
+ {
+ __KTRACE_OPT(KMMU, Kern::Printf("ArmMmu::Map lin=%08x phys=%08x size=%08x", aLinAddr, aPhysAddr, aSize));
+ __KTRACE_OPT(KMMU, Kern::Printf("pde=%08x pte=%08x mapshift=%d", aPdePerm, aPtePerm, aMapShift));
+ TPde pt_pde=aPdePerm;
+ TPte sp_pte=aPtePerm;
+ TPde section_pde=SECTION_PDE_FROM_PDEPTE(pt_pde, sp_pte);
+ TPte lp_pte=LP_PTE_FROM_SP_PTE(sp_pte);
+ TLinAddr la=aLinAddr;
+ TPhysAddr pa=aPhysAddr;
+ TInt remain=aSize;
+ while (remain)
+ {
+ if (aMapShift>=KChunkShift && (la & KChunkMask)==0 && remain>=KChunkSize)
+ {
+ // use sections - ASSUMES ADDRESS IS IN GLOBAL REGION
+ TInt npdes=remain>>KChunkShift;
+ const TBitMapAllocator& b=*iOsAsidAllocator;
+ TInt num_os_asids=iNumGlobalPageDirs;
+ TInt os_asid=0;
+ for (; num_os_asids; ++os_asid)
+ {
+ if (b.NotAllocated(os_asid,1) || (iAsidInfo[os_asid]&1)==0)
+ continue; // os_asid is not needed
+ TPde* p_pde=PageDirectory(os_asid)+(la>>KChunkShift);
+ TPde* p_pde_E=p_pde+npdes;
+ TPde pde=pa|section_pde;
+ TLinAddr firstPde = (TLinAddr)p_pde; //Will need this to clean page table memory region from cache
+
+ NKern::LockSystem();
+ for (; p_pde < p_pde_E; pde+=KChunkSize)
+ {
+ __ASSERT_DEBUG(*p_pde==0, MM::Panic(MM::EPdeAlreadyInUse));
+ __KTRACE_OPT(KMMU,Kern::Printf("Writing PDE %08x to %08x", pde, p_pde));
+ *p_pde++=pde;
+ }
+ CacheMaintenance::MultiplePtesUpdated(firstPde, (TUint)p_pde-firstPde);
+ NKern::UnlockSystem();
+ --num_os_asids;
+ }
+ npdes<<=KChunkShift;
+ la+=npdes, pa+=npdes, remain-=npdes;
+ continue;
+ }
+ TInt block_size = Min(remain, KChunkSize-(la&KChunkMask));
+ TPte pa_mask=~KPageMask;
+ TPte pte_perm=sp_pte;
+ if (aMapShift>=KLargePageShift && block_size>=KLargePageSize)
+ {
+ if ((la & KLargePageMask)==0)
+ {
+ // use 64K large pages
+ pa_mask=~KLargePageMask;
+ pte_perm=lp_pte;
+ }
+ else
+ block_size = Min(remain, KLargePageSize-(la&KLargePageMask));
+ }
+ block_size &= pa_mask;
+
+ // use pages (large or small)
+ TInt id=PageTableId(la, 0);
+ __ASSERT_DEBUG(id>=0, MM::Panic(MM::EMmuMapNoPageTable));
+ TPte* p_pte=PageTable(id)+((la&KChunkMask)>>KPageShift);
+ TPte* p_pte_E=p_pte + (block_size>>KPageShift);
+ SPageTableInfo& ptinfo=iPtInfo[id];
+ TLinAddr firstPte = (TLinAddr)p_pte; //Will need this to clean page table memory region from cache
+
+ NKern::LockSystem();
+ for (; p_pte < p_pte_E; pa+=KPageSize)
+ {
+ __ASSERT_DEBUG(*p_pte==0, MM::Panic(MM::EPteAlreadyInUse));
+ TPte pte = (pa & pa_mask) | pte_perm;
+ __KTRACE_OPT(KMMU,Kern::Printf("Writing PTE %08x to %08x", pte, p_pte));
+ *p_pte++=pte;
+ ++ptinfo.iCount;
+ NKern::FlashSystem();
+ }
+ CacheMaintenance::MultiplePtesUpdated(firstPte, (TUint)p_pte-firstPte);
+ NKern::UnlockSystem();
+ la+=block_size, remain-=block_size;
+ }
+ }
+
+void ArmMmu::Unmap(TLinAddr aLinAddr, TInt aSize)
+//
+// Remove all mappings in the specified range of addresses.
+// Assumes there are only global mappings involved.
+// Don't free page tables.
+// aLinAddr, aSize must be page-aligned.
+//
+ {
+ __KTRACE_OPT(KMMU, Kern::Printf("ArmMmu::Unmap lin=%08x size=%08x", aLinAddr, aSize));
+ TLinAddr a=aLinAddr;
+ TLinAddr end=a+aSize;
+ __KTRACE_OPT(KMMU,Kern::Printf("a=%08x end=%08x",a,end));
+ NKern::LockSystem();
+ while(a!=end)
+ {
+ TInt pdeIndex=a>>KChunkShift;
+ TLinAddr next=(pdeIndex<<KChunkShift)+KChunkSize;
+ TInt to_do = Min(TInt(end-a), TInt(next-a))>>KPageShift;
+ __KTRACE_OPT(KMMU,Kern::Printf("a=%08x next=%08x to_do=%d",a,next,to_do));
+ TPde pde=::InitPageDirectory[pdeIndex];
+ if ( (pde&KArmV6PdeTypeMask)==KArmV6PdeSection )
+ {
+ __ASSERT_DEBUG(!(a&KChunkMask), MM::Panic(MM::EUnmapBadAlignment));
+#if defined(__CPU_ARM1136__) && !defined(__CPU_ARM1136_ERRATUM_353494_FIXED)
+ remove_and_invalidate_section(::InitPageDirectory + pdeIndex, a, KERNEL_MAPPING);
+#else
+ ::InitPageDirectory[pdeIndex]=0;
+ CacheMaintenance::SinglePteUpdated(TLinAddr(::InitPageDirectory + pdeIndex));
+ InvalidateTLBForPage(a, KERNEL_MAPPING); // ASID irrelevant since global
+#endif
+ a=next;
+ NKern::FlashSystem();
+ continue;
+ }
+ TInt ptid=PageTableId(a,0);
+ SPageTableInfo& ptinfo=iPtInfo[ptid];
+ if (ptid>=0)
+ {
+ TPte* ppte=PageTable(ptid)+((a&KChunkMask)>>KPageShift);
+ TPte* ppte_End=ppte+to_do;
+ for (; ppte<ppte_End; ++ppte, a+=KPageSize)
+ {
+ if (*ppte & KArmV6PteSmallPage)
+ {
+ --ptinfo.iCount;
+#if defined(__CPU_ARM1136__) && !defined(__CPU_ARM1136_ERRATUM_353494_FIXED)
+ remove_and_invalidate_page(ppte, a, KERNEL_MAPPING);
+#else
+ *ppte=0;
+ CacheMaintenance::SinglePteUpdated((TLinAddr)ppte);
+ InvalidateTLBForPage(a, KERNEL_MAPPING);
+#endif
+ }
+ else if ((*ppte & KArmV6PteTypeMask) == KArmV6PteLargePage)
+ {
+ __ASSERT_DEBUG(!(a&KLargePageMask), MM::Panic(MM::EUnmapBadAlignment));
+ ptinfo.iCount-=KLargeSmallPageRatio;
+#if defined(__CPU_ARM1136__) && !defined(__CPU_ARM1136_ERRATUM_353494_FIXED)
+ remove_and_invalidate_page(ppte, a, KERNEL_MAPPING);
+#else
+ memclr(ppte, KLargeSmallPageRatio*sizeof(TPte));
+ CacheMaintenance::MultiplePtesUpdated((TLinAddr)ppte, KLargeSmallPageRatio*sizeof(TPte));
+ InvalidateTLBForPage(a, KERNEL_MAPPING);
+#endif
+ a+=(KLargePageSize-KPageSize);
+ ppte+=(KLargeSmallPageRatio-1);
+ }
+ NKern::FlashSystem();
+ }
+ }
+ else
+ a += (to_do<<KPageShift);
+ }
+ NKern::UnlockSystem();
+ #if defined(__CPU_ARM1136__) && !defined(__CPU_ARM1136_ERRATUM_353494_FIXED)
+ __FlushBtb();
+ #endif
+ }
+
+
+void ArmMmu::ClearPages(TInt aNumPages, TPhysAddr* aPageList, TUint8 aClearByte)
+ {
+ //map the pages at a temporary address, clear them and unmap
+ __ASSERT_MUTEX(RamAllocatorMutex);
+ while (--aNumPages >= 0)
+ {
+ TPhysAddr pa;
+ if((TInt)aPageList&1)
+ {
+ pa = (TPhysAddr)aPageList&~1;
+ *(TPhysAddr*)&aPageList += iPageSize;
+ }
+ else
+ pa = *aPageList++;
+
+ *iTempPte = pa | SP_PTE(KArmV6PermRWNO, KNormalUncachedAttr, 0, 1);
+ CacheMaintenance::SinglePteUpdated((TLinAddr)iTempPte);
+ InvalidateTLBForPage(iTempAddr, KERNEL_MAPPING);
+ memset((TAny*)iTempAddr, aClearByte, iPageSize);
+ // This temporary mapping is noncached => No need to flush cache here.
+ // Still, we have to make sure that write buffer(s) are drained.
+ CacheMaintenance::MemoryToPreserveAndReuse((TLinAddr)iTempAddr, iPageSize, EMapAttrBufferedC);
+ }
+ *iTempPte=0;
+ CacheMaintenance::SinglePteUpdated((TLinAddr)iTempPte);
+ InvalidateTLBForPage(iTempAddr, KERNEL_MAPPING);
+ }
+
+
+/**
+Create a temporary mapping of one or more contiguous physical pages.
+Fully cached memory attributes apply.
+The RamAllocatorMutex must be held before this function is called and not released
+until after UnmapTemp has been called.
+
+@param aPage The physical address of the pages to be mapped.
+@param aLinAddr The linear address of any existing location where the page is mapped.
+ If the page isn't already mapped elsewhere as a cachable page then
+ this value irrelevent. (It is used for page colouring.)
+@param aPages Number of pages to map.
+
+@return The linear address of where the pages have been mapped.
+*/
+TLinAddr ArmMmu::MapTemp(TPhysAddr aPage,TLinAddr aLinAddr,TInt aPages)
+ {
+ __ASSERT_MUTEX(RamAllocatorMutex);
+ __ASSERT_DEBUG(!*iTempPte,MM::Panic(MM::ETempMappingAlreadyInUse));
+ iTempMapColor = (aLinAddr>>KPageShift)&KPageColourMask;
+ iTempMapCount = aPages;
+ if (aPages==1)
+ {
+ iTempPte[iTempMapColor] = (aPage&~KPageMask) | SP_PTE(KArmV6PermRWNO, KNormalCachedAttr, 0, 1);
+ CacheMaintenance::SinglePteUpdated((TLinAddr)(iTempPte+iTempMapColor));
+ }
+ else
+ {
+ __ASSERT_DEBUG(iTempMapColor+aPages<=KPageColourCount,MM::Panic(MM::ETempMappingNoRoom));
+ for (TInt i=0; i<aPages; i++)
+ iTempPte[iTempMapColor+i] = ((aPage&~KPageMask)+(i<<KPageShift)) | SP_PTE(KArmV6PermRWNO, KNormalCachedAttr, 0, 1);
+ CacheMaintenance::MultiplePtesUpdated((TLinAddr)(iTempPte+iTempMapColor), aPages*sizeof(TPte));
+ }
+ return iTempAddr+(iTempMapColor<<KPageShift);
+ }
+
+/**
+Create a temporary mapping of one or more contiguous physical pages.
+Memory attributes as specified by aMemType apply.
+@See ArmMmu::MapTemp(TPhysAddr aPage,TLinAddr aLinAddr,TInt aPages) for other details.
+*/
+TLinAddr ArmMmu::MapTemp(TPhysAddr aPage,TLinAddr aLinAddr,TInt aPages, TMemoryType aMemType)
+ {
+ __ASSERT_MUTEX(RamAllocatorMutex);
+ __ASSERT_DEBUG(!*iTempPte,MM::Panic(MM::ETempMappingAlreadyInUse));
+ iTempMapColor = (aLinAddr>>KPageShift)&KPageColourMask;
+ iTempMapCount = aPages;
+ TUint pte = SP_PTE(KArmV6PermRWNO, aMemType, 0, 1);
+ if (aPages==1)
+ {
+ iTempPte[iTempMapColor] = (aPage&~KPageMask) | SP_PTE(KArmV6PermRWNO, pte, 0, 1);
+ CacheMaintenance::SinglePteUpdated((TLinAddr)(iTempPte+iTempMapColor));
+ }
+ else
+ {
+ __ASSERT_DEBUG(iTempMapColor+aPages<=KPageColourCount,MM::Panic(MM::ETempMappingNoRoom));
+ for (TInt i=0; i<aPages; i++)
+ iTempPte[iTempMapColor+i] = ((aPage&~KPageMask)+(i<<KPageShift)) | SP_PTE(KArmV6PermRWNO, pte, 0, 1);
+ CacheMaintenance::MultiplePtesUpdated((TLinAddr)(iTempPte+iTempMapColor), aPages*sizeof(TPte));
+ }
+ return iTempAddr+(iTempMapColor<<KPageShift);
+ }
+
+/**
+Create a temporary mapping of one or more contiguous physical pages, distinct from
+that created by MapTemp.
+The RamAllocatorMutex must be held before this function is called and not released
+until after UnmapSecondTemp has been called.
+
+@param aPage The physical address of the pages to be mapped.
+@param aLinAddr The linear address of any existing location where the page is mapped.
+ If the page isn't already mapped elsewhere as a cachable page then
+ this value irrelevent. (It is used for page colouring.)
+@param aPages Number of pages to map.
+
+@return The linear address of where the pages have been mapped.
+*/
+TLinAddr ArmMmu::MapSecondTemp(TPhysAddr aPage,TLinAddr aLinAddr,TInt aPages)
+ {
+ __ASSERT_MUTEX(RamAllocatorMutex);
+ __ASSERT_DEBUG(!*iSecondTempPte,MM::Panic(MM::ETempMappingAlreadyInUse));
+ iSecondTempMapColor = (aLinAddr>>KPageShift)&KPageColourMask;
+ iSecondTempMapCount = aPages;
+ if (aPages==1)
+ {
+ iSecondTempPte[iSecondTempMapColor] = (aPage&~KPageMask) | SP_PTE(KArmV6PermRWNO, KNormalCachedAttr, 0, 1);
+ CacheMaintenance::SinglePteUpdated((TLinAddr)(iSecondTempPte+iSecondTempMapColor));
+ }
+ else
+ {
+ __ASSERT_DEBUG(iSecondTempMapColor+aPages<=KPageColourCount,MM::Panic(MM::ETempMappingNoRoom));
+ for (TInt i=0; i<aPages; i++)
+ iSecondTempPte[iSecondTempMapColor+i] = ((aPage&~KPageMask)+(i<<KPageShift)) | SP_PTE(KArmV6PermRWNO, KNormalCachedAttr, 0, 1);
+ CacheMaintenance::MultiplePtesUpdated((TLinAddr)(iSecondTempPte+iSecondTempMapColor), aPages*sizeof(TPte));
+ }
+ return iSecondTempAddr+(iSecondTempMapColor<<KPageShift);
+ }
+
+/**
+Remove the temporary mapping created with MapTemp.
+*/
+void ArmMmu::UnmapTemp()
+ {
+ __ASSERT_MUTEX(RamAllocatorMutex);
+ for (TInt i=0; i<iTempMapCount; i++)
+ {
+ iTempPte[iTempMapColor+i] = 0;
+ CacheMaintenance::SinglePteUpdated((TLinAddr)(iTempPte+iTempMapColor+i));
+ InvalidateTLBForPage(iTempAddr+((iTempMapColor+i)<<KPageShift), KERNEL_MAPPING);
+ }
+ }
+
+/**
+Remove the temporary mapping created with MapSecondTemp.
+*/
+void ArmMmu::UnmapSecondTemp()
+ {
+ __ASSERT_MUTEX(RamAllocatorMutex);
+ for (TInt i=0; i<iSecondTempMapCount; i++)
+ {
+ iSecondTempPte[iSecondTempMapColor+i] = 0;
+ CacheMaintenance::SinglePteUpdated((TLinAddr)(iSecondTempPte+iSecondTempMapColor+i));
+ InvalidateTLBForPage(iSecondTempAddr+((iSecondTempMapColor+i)<<KPageShift), KERNEL_MAPPING);
+ }
+ }
+
+
+TBool ArmMmu::ValidateLocalIpcAddress(TLinAddr aAddr,TInt aSize,TBool aWrite)
+ {
+ __NK_ASSERT_DEBUG(aSize<=KChunkSize);
+ TLinAddr end = aAddr+aSize-1;
+ if(end<aAddr)
+ end = ~0u;
+
+ if(TUint(aAddr^KIPCAlias)<TUint(KChunkSize) || TUint(end^KIPCAlias)<TUint(KChunkSize))
+ {
+ // local address is in alias region.
+ // remove alias...
+ NKern::LockSystem();
+ ((DMemModelThread*)TheCurrentThread)->RemoveAlias();
+ NKern::UnlockSystem();
+ // access memory, which will cause an exception...
+ if(!(TUint(aAddr^KIPCAlias)<TUint(KChunkSize)))
+ aAddr = end;
+ InvalidateTLBForPage(aAddr,((DMemModelProcess*)TheCurrentThread->iOwningProcess)->iOsAsid);
+ if(aWrite)
+ *(volatile TUint8*)aAddr = 0;
+ else
+ aWrite = *(volatile TUint8*)aAddr;
+ // can't get here
+ __NK_ASSERT_DEBUG(0);
+ }
+
+ TUint32 local_mask;
+ DMemModelProcess* process=(DMemModelProcess*)TheCurrentThread->iOwningProcess;
+ if(aWrite)
+ local_mask = process->iAddressCheckMaskW;
+ else
+ local_mask = process->iAddressCheckMaskR;
+ TUint32 mask = 2<<(end>>27);
+ mask -= 1<<(aAddr>>27);
+ if((local_mask&mask)!=mask)
+ return EFalse;
+
+ if(!aWrite)
+ return ETrue; // reads are ok
+
+ // writes need further checking...
+ TLinAddr userCodeStart = iUserCodeBase;
+ TLinAddr userCodeEnd = userCodeStart+iMaxUserCodeSize;
+ if(end>=userCodeStart && aAddr<userCodeEnd)
+ return EFalse; // trying to write to user code area
+
+ return ETrue;
+ }
+
+TInt DMemModelThread::Alias(TLinAddr aAddr, DMemModelProcess* aProcess, TInt aSize, TInt aPerm, TLinAddr& aAliasAddr, TInt& aAliasSize)
+//
+// Set up an alias mapping starting at address aAddr in specified process.
+// Check permissions aPerm.
+// Enter and return with system locked.
+// Note: Alias is removed if an exception if trapped by DThread::IpcExcHandler.
+//
+ {
+ __KTRACE_OPT(KMMU2,Kern::Printf("Thread %O Alias %08x+%x Process %O perm %x",this,aAddr,aSize,aProcess,aPerm));
+ __ASSERT_SYSTEM_LOCK
+
+ if(TUint(aAddr^KIPCAlias)<TUint(KChunkSize))
+ return KErrBadDescriptor; // prevent access to alias region
+
+ ArmMmu& m=::TheMmu;
+
+ // check if memory is in region which is safe to access with supervisor permissions...
+ TBool okForSupervisorAccess = aPerm&(EMapAttrReadSup|EMapAttrWriteSup) ? 1 : 0;
+ if(!okForSupervisorAccess)
+ {
+ TInt shift = aAddr>>27;
+ if(!(aPerm&EMapAttrWriteUser))
+ {
+ // reading with user permissions...
+ okForSupervisorAccess = (aProcess->iAddressCheckMaskR>>shift)&1;
+ }
+ else
+ {
+ // writing with user permissions...
+ okForSupervisorAccess = (aProcess->iAddressCheckMaskW>>shift)&1;
+ if(okForSupervisorAccess)
+ {
+ // check for user code, because this is supervisor r/w and so
+ // is not safe to write to access with supervisor permissions.
+ if(TUint(aAddr-m.iUserCodeBase)<TUint(m.iMaxUserCodeSize))
+ return KErrBadDescriptor; // prevent write to this...
+ }
+ }
+ }
+
+ TInt pdeIndex = aAddr>>KChunkShift;
+ if(pdeIndex>=(m.iLocalPdSize>>2))
+ {
+ // address is in global section, don't bother aliasing it...
+ if(iAliasLinAddr)
+ RemoveAlias();
+ aAliasAddr = aAddr;
+ TInt maxSize = KChunkSize-(aAddr&KChunkMask);
+ aAliasSize = aSize<maxSize ? aSize : maxSize;
+ __KTRACE_OPT(KMMU2,Kern::Printf("DMemModelThread::Alias() abandoned as memory is globaly mapped"));
+ return okForSupervisorAccess;
+ }
+
+ TInt asid = aProcess->iOsAsid;
+ TPde* pd = PageDirectory(asid);
+ TPde pde = pd[pdeIndex];
+ if ((TPhysAddr)(pde&~KPageMask) == AliasRemapOld)
+ pde = AliasRemapNew|(pde&KPageMask);
+ pde = PDE_IN_DOMAIN(pde, KIPCAliasDomain);
+ TLinAddr aliasAddr = KIPCAlias+(aAddr&(KChunkMask & ~KPageMask));
+ if(pde==iAliasPde && iAliasLinAddr)
+ {
+ // pde already aliased, so just update linear address...
+ iAliasLinAddr = aliasAddr;
+ }
+ else
+ {
+ // alias PDE changed...
+ iAliasPde = pde;
+ iAliasOsAsid = asid;
+ if(!iAliasLinAddr)
+ {
+ ArmMmu::UnlockAlias();
+ ::TheMmu.iAliasList.Add(&iAliasLink); // add to list if not already aliased
+ }
+ iAliasLinAddr = aliasAddr;
+ *iAliasPdePtr = pde;
+ CacheMaintenance::SinglePteUpdated((TLinAddr)iAliasPdePtr);
+ }
+
+ __KTRACE_OPT(KMMU2,Kern::Printf("DMemModelThread::Alias() PDEntry=%x, iAliasLinAddr=%x",pde, aliasAddr));
+ InvalidateTLBForPage(aliasAddr, ((DMemModelProcess*)iOwningProcess)->iOsAsid);
+ TInt offset = aAddr&KPageMask;
+ aAliasAddr = aliasAddr | offset;
+ TInt maxSize = KPageSize - offset;
+ aAliasSize = aSize<maxSize ? aSize : maxSize;
+ iAliasTarget = aAddr & ~KPageMask;
+ return okForSupervisorAccess;
+ }
+
+void DMemModelThread::RemoveAlias()
+//
+// Remove alias mapping (if present)
+// Enter and return with system locked.
+//
+ {
+ __KTRACE_OPT(KMMU2,Kern::Printf("Thread %O RemoveAlias", this));
+ __ASSERT_SYSTEM_LOCK
+ TLinAddr addr = iAliasLinAddr;
+ if(addr)
+ {
+ ArmMmu::LockAlias();
+ iAliasLinAddr = 0;
+ iAliasPde = 0;
+ *iAliasPdePtr = 0;
+ CacheMaintenance::SinglePteUpdated((TLinAddr)iAliasPdePtr);
+ InvalidateTLBForPage(addr, ((DMemModelProcess*)iOwningProcess)->iOsAsid);
+ iAliasLink.Deque();
+ }
+ }
+
+/*
+ * Performs cache maintenance for physical page that is going to be reused.
+ * Fully cached attributes are assumed.
+ */
+void ArmMmu::CacheMaintenanceOnDecommit(TPhysAddr a)
+ {
+ // purge a single page from the cache following decommit
+ ArmMmu& m=::TheMmu;
+ TInt colour = SPageInfo::FromPhysAddr(a)->Offset()&KPageColourMask;
+ TPte& pte=m.iTempPte[colour];
+ TLinAddr va=m.iTempAddr+(colour<<KPageShift);
+ pte=a|SP_PTE(KArmV6PermRWNO, iCacheMaintenanceTempMapAttr, 1, 1);
+ CacheMaintenance::SinglePteUpdated((TLinAddr)&pte);
+
+ CacheMaintenance::PageToReuse(va,EMemAttNormalCached, a);
+
+ pte=0;
+ CacheMaintenance::SinglePteUpdated((TLinAddr)&pte);
+ InvalidateTLBForPage(va,KERNEL_MAPPING);
+ }
+
+void ArmMmu::CacheMaintenanceOnDecommit(const TPhysAddr* al, TInt n)
+ {
+ // purge a list of pages from the cache following decommit
+ while (--n>=0)
+ ArmMmu::CacheMaintenanceOnDecommit(*al++);
+ }
+
+/*
+ * Performs cache maintenance to preserve physical page that is going to be reused.
+ */
+void ArmMmu::CacheMaintenanceOnPreserve(TPhysAddr a, TUint aMapAttr)
+ {
+ // purge a single page from the cache following decommit
+ ArmMmu& m=::TheMmu;
+ TInt colour = SPageInfo::FromPhysAddr(a)->Offset()&KPageColourMask;
+ TPte& pte=m.iTempPte[colour];
+ TLinAddr va=m.iTempAddr+(colour<<KPageShift);
+ pte=a|SP_PTE(KArmV6PermRWNO, iCacheMaintenanceTempMapAttr, 1, 1);
+ CacheMaintenance::SinglePteUpdated((TLinAddr)&pte);
+
+ CacheMaintenance::MemoryToPreserveAndReuse(va, KPageSize,aMapAttr);
+
+ pte=0;
+ CacheMaintenance::SinglePteUpdated((TLinAddr)&pte);
+ InvalidateTLBForPage(va,KERNEL_MAPPING);
+ }
+
+void ArmMmu::CacheMaintenanceOnPreserve(const TPhysAddr* al, TInt n, TUint aMapAttr)
+ {
+ // purge a list of pages from the cache following decommit
+ while (--n>=0)
+ ArmMmu::CacheMaintenanceOnPreserve(*al++, aMapAttr);
+ }
+
+/*
+ * Performs cache maintenance of physical memory that has been decommited and has to be preserved.
+ * Call this method for physical pages with no page info updated (or no page info at all).
+ * @arg aPhysAddr The address of contiguous physical memory to be preserved.
+ * @arg aSize The size of the region
+ * @arg aLinAddr Former linear address of the region. As said above, the physical memory is
+ * already remapped from this linear address.
+ * @arg aMapAttr Mapping attributes of the region when it was mapped in aLinAddr.
+ * @pre MMU mutex is held.
+ */
+void ArmMmu::CacheMaintenanceOnPreserve(TPhysAddr aPhysAddr, TInt aSize, TLinAddr aLinAddr, TUint aMapAttr )
+ {
+ __NK_ASSERT_DEBUG((aPhysAddr&KPageMask)==0);
+ __NK_ASSERT_DEBUG((aSize&KPageMask)==0);
+ __NK_ASSERT_DEBUG((aLinAddr&KPageMask)==0);
+
+ TPhysAddr pa = aPhysAddr;
+ TInt size = aSize;
+ TInt colour = (aLinAddr>>KPageShift)&KPageColourMask;
+ TPte* pte = &(iTempPte[colour]);
+ while (size)
+ {
+ pte=&(iTempPte[colour]);
+ TLinAddr va=iTempAddr+(colour<<KPageShift);
+ *pte=pa|SP_PTE(KArmV6PermRWNO, iCacheMaintenanceTempMapAttr, 1, 1);
+ CacheMaintenance::SinglePteUpdated((TLinAddr)pte);
+ CacheMaintenance::MemoryToPreserveAndReuse(va, KPageSize,aMapAttr);
+
+ *pte=0;
+ CacheMaintenance::SinglePteUpdated((TLinAddr)pte);
+ InvalidateTLBForPage(va,KERNEL_MAPPING);
+
+ colour = (colour+1)&KPageColourMask;
+ pa += KPageSize;
+ size -=KPageSize;
+ }
+ }
+
+TInt ArmMmu::UnlockRamCachePages(TLinAddr aLinAddr, TInt aNumPages, DProcess* aProcess)
+ {
+ TInt asid = ((DMemModelProcess*)aProcess)->iOsAsid;
+ TInt page = aLinAddr>>KPageShift;
+ NKern::LockSystem();
+ for(;;)
+ {
+ TPde* pd = PageDirectory(asid)+(page>>(KChunkShift-KPageShift));
+ TPte* pt = SafePageTableFromPde(*pd++);
+ TInt pteIndex = page&(KChunkMask>>KPageShift);
+ if(!pt)
+ {
+ // whole page table has gone, so skip all pages in it...
+ TInt pagesInPt = (KChunkSize>>KPageShift)-pteIndex;
+ aNumPages -= pagesInPt;
+ page += pagesInPt;
+ if(aNumPages>0)
+ continue;
+ NKern::UnlockSystem();
+ return KErrNone;
+ }
+ pt += pteIndex;
+ do
+ {
+ TInt pagesInPt = (KChunkSize>>KPageShift)-pteIndex;
+ if(pagesInPt>aNumPages)
+ pagesInPt = aNumPages;
+ if(pagesInPt>KMaxPages)
+ pagesInPt = KMaxPages;
+
+ aNumPages -= pagesInPt;
+ page += pagesInPt;
+
+ do
+ {
+ TPte pte = *pt++;
+ if(pte) // pte may be null if page has already been unlocked and reclaimed by system
+ iRamCache->DonateRamCachePage(SPageInfo::FromPhysAddr(pte));
+ }
+ while(--pagesInPt);
+
+ if(!aNumPages)
+ {
+ NKern::UnlockSystem();
+ return KErrNone;
+ }
+
+ pteIndex = page&(KChunkMask>>KPageShift);
+ }
+ while(!NKern::FlashSystem() && pteIndex);
+ }
+ }
+
+
+TInt ArmMmu::LockRamCachePages(TLinAddr aLinAddr, TInt aNumPages, DProcess* aProcess)
+ {
+ TInt asid = ((DMemModelProcess*)aProcess)->iOsAsid;
+ TInt page = aLinAddr>>KPageShift;
+ NKern::LockSystem();
+ for(;;)
+ {
+ TPde* pd = PageDirectory(asid)+(page>>(KChunkShift-KPageShift));
+ TPte* pt = SafePageTableFromPde(*pd++);
+ TInt pteIndex = page&(KChunkMask>>KPageShift);
+ if(!pt)
+ goto not_found;
+ pt += pteIndex;
+ do
+ {
+ TInt pagesInPt = (KChunkSize>>KPageShift)-pteIndex;
+ if(pagesInPt>aNumPages)
+ pagesInPt = aNumPages;
+ if(pagesInPt>KMaxPages)
+ pagesInPt = KMaxPages;
+
+ aNumPages -= pagesInPt;
+ page += pagesInPt;
+
+ do
+ {
+ TPte pte = *pt++;
+ if(pte==0)
+ goto not_found;
+ if(!iRamCache->ReclaimRamCachePage(SPageInfo::FromPhysAddr(pte)))
+ goto not_found;
+ }
+ while(--pagesInPt);
+
+ if(!aNumPages)
+ {
+ NKern::UnlockSystem();
+ return KErrNone;
+ }
+
+ pteIndex = page&(KChunkMask>>KPageShift);
+ }
+ while(!NKern::FlashSystem() && pteIndex);
+ }
+not_found:
+ NKern::UnlockSystem();
+ return KErrNotFound;
+ }
+
+
+void RamCache::SetFree(SPageInfo* aPageInfo)
+ {
+ ArmMmu& m=::TheMmu;
+ // Make a page free
+ SPageInfo::TType type = aPageInfo->Type();
+ if(type==SPageInfo::EPagedCache)
+ {
+ TInt offset = aPageInfo->Offset()<<KPageShift;
+ DMemModelChunk* chunk = (DMemModelChunk*)aPageInfo->Owner();
+ __NK_ASSERT_DEBUG(TUint(offset)<TUint(chunk->iMaxSize));
+ TLinAddr lin = ((TLinAddr)chunk->iBase)+offset;
+ TInt asid = ((DMemModelProcess*)chunk->iOwningProcess)->iOsAsid;
+ TPte* pt = PtePtrFromLinAddr(lin,asid);
+ TPhysAddr phys = (*pt)&~KPageMask;
+ *pt = KPteNotPresentEntry;
+ CacheMaintenance::SinglePteUpdated((TLinAddr)pt);
+ InvalidateTLBForPage(lin,asid);
+ m.CacheMaintenanceOnDecommit(phys);
+
+ // actually decommit it from chunk...
+ TInt ptid = ((TLinAddr)pt-KPageTableBase)>>KPageTableShift;
+ SPageTableInfo& ptinfo=((ArmMmu*)iMmu)->iPtInfo[ptid];
+ if(!--ptinfo.iCount)
+ {
+ chunk->iPageTables[offset>>KChunkShift] = 0xffff;
+ NKern::UnlockSystem();
+ ((ArmMmu*)iMmu)->DoUnassignPageTable(lin, (TAny*)asid);
+ ((ArmMmu*)iMmu)->FreePageTable(ptid);
+ NKern::LockSystem();
+ }
+ }
+ else
+ {
+ __KTRACE_OPT2(KPAGING,KPANIC,Kern::Printf("DP: SetFree() with bad page type = %d",aPageInfo->Type()));
+ Panic(EUnexpectedPageType);
+ }
+ }
+
+
+//
+// MemModelDemandPaging
+//
+
+class MemModelDemandPaging : public DemandPaging
+ {
+public:
+ // From RamCacheBase
+ virtual void Init2();
+ virtual TInt Init3();
+ virtual TBool PageUnmapped(SPageInfo* aPageInfo);
+ // From DemandPaging
+ virtual TInt Fault(TAny* aExceptionInfo);
+ virtual void SetOld(SPageInfo* aPageInfo);
+ virtual void SetFree(SPageInfo* aPageInfo);
+ virtual void NotifyPageFree(TPhysAddr aPage);
+ virtual TInt EnsurePagePresent(TLinAddr aPage, DProcess* aProcess);
+ virtual TPhysAddr LinearToPhysical(TLinAddr aPage, DProcess* aProcess);
+ virtual void AllocLoadAddress(DPagingRequest& aReq, TInt aDeviceId);
+ virtual TInt PageState(TLinAddr aAddr);
+ virtual TBool NeedsMutexOrderCheck(TLinAddr aStartAddr, TUint aLength);
+ // New
+ inline ArmMmu& Mmu() { return (ArmMmu&)*iMmu; }
+ void InitRomPaging();
+ void InitCodePaging();
+ TInt HandleFault(TArmExcInfo& aExc, TLinAddr aFaultAddress, TInt aAsid);
+ TInt PageIn(TLinAddr aAddress, TInt aAsid, DMemModelCodeSegMemory* aCodeSegMemory);
+public:
+ // use of the folowing members is protected by the system lock..
+ TPte* iPurgePte; // PTE used for temporary mappings during cache purge operations
+ TLinAddr iPurgeAddr; // address corresponding to iPurgePte
+ };
+
+extern void MakeGlobalPTEInaccessible(TPte* aPtePtr, TPte aNewPte, TLinAddr aLinAddr);
+extern void MakePTEInaccessible(TPte* aPtePtr, TPte aNewPte, TLinAddr aLinAddr, TInt aAsid);
+
+//
+// MemModelDemandPaging
+//
+
+
+DemandPaging* DemandPaging::New()
+ {
+ return new MemModelDemandPaging();
+ }
+
+
+void MemModelDemandPaging::Init2()
+ {
+ __KTRACE_OPT2(KPAGING,KBOOT,Kern::Printf(">MemModelDemandPaging::Init2"));
+ DemandPaging::Init2();
+
+ iPurgeAddr = KDemandPagingTempAddr;
+ iPurgePte = PtePtrFromLinAddr(iPurgeAddr);
+
+ __KTRACE_OPT2(KPAGING,KBOOT,Kern::Printf("<MemModelDemandPaging::Init2"));
+ }
+
+
+void MemModelDemandPaging::AllocLoadAddress(DPagingRequest& aReq, TInt aReqId)
+ {
+ aReq.iLoadAddr = iTempPages + aReqId * KPageSize * KPageColourCount;
+ aReq.iLoadPte = PtePtrFromLinAddr(aReq.iLoadAddr);
+ }
+
+
+TInt MemModelDemandPaging::Init3()
+ {
+ TInt r=DemandPaging::Init3();
+ if(r!=KErrNone)
+ return r;
+
+ // Create a region for mapping pages during page in
+ DPlatChunkHw* chunk;
+ TInt chunkSize = (KMaxPagingDevices * KPagingRequestsPerDevice + 1) * KPageColourCount * KPageSize;
+ DPlatChunkHw::DoNew(chunk, KPhysAddrInvalid, chunkSize, EMapAttrSupRw|EMapAttrFullyBlocking);
+ if(!chunk)
+ Panic(EInitialiseFailed);
+ TInt colourMask = KPageColourMask << KPageShift;
+ iTempPages = (chunk->iLinAddr + colourMask) & ~colourMask;
+
+ if(RomPagingRequested())
+ InitRomPaging();
+
+ if (CodePagingRequested())
+ InitCodePaging();
+
+ __KTRACE_OPT2(KPAGING,KBOOT,Kern::Printf("<MemModelDemandPaging::Init3"));
+ return KErrNone;
+ }
+
+void MemModelDemandPaging::InitRomPaging()
+ {
+ // Make page tables for demand paged part of ROM...
+ __KTRACE_OPT2(KPAGING,KBOOT,Kern::Printf("MemModelDemandPaging::Init3 making page tables for paged ROM"));
+ TLinAddr lin = iRomPagedLinearBase&~KChunkMask; // first chunk with paged ROM in
+ TLinAddr linEnd = iRomLinearBase+iRomSize;
+ while(lin<linEnd)
+ {
+ // Get a Page Table
+ TInt ptid = Mmu().PageTableId(lin,0);
+ if(ptid<0)
+ {
+ MmuBase::Wait();
+ ptid = Mmu().AllocPageTable();
+ MmuBase::Signal();
+ __NK_ASSERT_DEBUG(ptid>=0);
+ Mmu().PtInfo(ptid).SetGlobal(lin >> KChunkShift);
+ }
+
+ // Get new page table addresses
+ TPte* pt = PageTable(ptid);
+ TPhysAddr ptPhys=Mmu().LinearToPhysical((TLinAddr)pt,0);
+
+ // Pointer to page directory entry
+ TPde* ppde = ::InitPageDirectory + (lin>>KChunkShift);
+
+ // Fill in Page Table
+ TPte* ptEnd = pt+(1<<(KChunkShift-KPageShift));
+ pt += (lin&KChunkMask)>>KPageShift;
+ TLinAddr firstPte = (TLinAddr)pt; // Will need this to clean page table memory region from cache
+
+ do
+ {
+ if(lin<iRomPagedLinearBase)
+ *pt++ = Mmu().LinearToPhysical(lin,0) | KRomPtePerm;
+ else
+ {
+ MakeGlobalPTEInaccessible(pt, KPteNotPresentEntry, lin);
+ ++pt;
+ }
+ lin += KPageSize;
+ }
+ while(pt<ptEnd && lin<=linEnd);
+
+ CacheMaintenance::MultiplePtesUpdated((TLinAddr)firstPte, (TUint)pt-firstPte);
+
+ // Add new Page Table to the Page Directory
+ TPde newpde = ptPhys | KShadowPdePerm;
+ __KTRACE_OPT2(KPAGING,KMMU,Kern::Printf("Writing PDE %08x to %08x", newpde, ppde));
+ TInt irq=NKern::DisableAllInterrupts();
+ *ppde = newpde;
+ CacheMaintenance::SinglePteUpdated((TLinAddr)ppde);
+ FlushTLBs();
+ NKern::RestoreInterrupts(irq);
+ }
+ }
+
+
+void MemModelDemandPaging::InitCodePaging()
+ {
+ // Initialise code paging info
+ iCodeLinearBase = Mmu().iUserCodeBase;
+ iCodeSize = Mmu().iMaxUserCodeSize;
+ }
+
+
+/**
+@return ETrue when the unmapped page should be freed, EFalse otherwise
+*/
+TBool MemModelDemandPaging::PageUnmapped(SPageInfo* aPageInfo)
+ {
+ SPageInfo::TType type = aPageInfo->Type();
+
+ // Only have to deal with cache pages - pages containg code don't get returned to the system
+ // when they are decommitted from an individual process, only when the code segment is destroyed
+ if(type!=SPageInfo::EPagedCache)
+ {
+ __NK_ASSERT_DEBUG(type!=SPageInfo::EPagedCode); // shouldn't happen
+ __NK_ASSERT_DEBUG(type!=SPageInfo::EPagedData); // not supported yet
+ return ETrue;
+ }
+
+ RemovePage(aPageInfo);
+ AddAsFreePage(aPageInfo);
+ // Return false to stop DMemModelChunk::DoDecommit from freeing this page
+ return EFalse;
+ }
+
+
+void DoSetCodeOld(SPageInfo* aPageInfo, DMemModelCodeSegMemory* aCodeSegMemory, TLinAddr aLinAddr)
+ {
+ NThread* currentThread = NKern::CurrentThread();
+ aPageInfo->SetModifier(currentThread);
+ // scan all address spaces...
+ TInt asid = -1;
+ TInt lastAsid = KArmV6NumAsids-1;
+ TUint32* ptr = aCodeSegMemory->iOsAsids->iMap;
+ do
+ {
+ TUint32 bits = *ptr++;
+ do
+ {
+ ++asid;
+ if(bits&0x80000000u)
+ {
+ // codeseg is mapped in this address space, so update PTE...
+ TPte* pt = PtePtrFromLinAddr(aLinAddr,asid);
+ TPte pte = *pt;
+ if(pte&KPtePresentMask)
+ {
+ __NK_ASSERT_DEBUG((pte&~KPageMask) == aPageInfo->PhysAddr());
+ MakePTEInaccessible(pt, pte&~KPtePresentMask, aLinAddr, asid);
+ }
+ }
+ }
+ while(bits<<=1);
+ if(NKern::FlashSystem() && aPageInfo->CheckModified(currentThread))
+ return; // page was modified by another thread
+ asid |= 31;
+ }
+ while(asid<lastAsid);
+ }
+
+
+void MemModelDemandPaging::SetOld(SPageInfo* aPageInfo)
+ {
+ __ASSERT_SYSTEM_LOCK;
+ __NK_ASSERT_DEBUG(aPageInfo->State() == SPageInfo::EStatePagedOld);
+
+ SPageInfo::TType type = aPageInfo->Type();
+
+ if(type==SPageInfo::EPagedROM)
+ {
+ // get linear address of page...
+ TInt offset = aPageInfo->Offset()<<KPageShift;
+ __NK_ASSERT_DEBUG(TUint(offset)<iRomSize);
+
+ // make page inaccessible...
+ TLinAddr lin = iRomLinearBase+offset;
+ TPte* pt = PtePtrFromLinAddr(lin);
+ MakeGlobalPTEInaccessible(pt, *pt&~KPtePresentMask, lin);
+ }
+ else if(type==SPageInfo::EPagedCode)
+ {
+ START_PAGING_BENCHMARK;
+
+ // get linear address of page...
+ TInt offset = aPageInfo->Offset()<<KPageShift;
+ __NK_ASSERT_DEBUG(TUint(offset)<iCodeSize);
+ TLinAddr lin = iCodeLinearBase+offset;
+
+ // get CodeSegMemory...
+ DMemModelCodeSegMemory* codeSegMemory = (DMemModelCodeSegMemory*)aPageInfo->Owner();
+ __NK_ASSERT_DEBUG(codeSegMemory && codeSegMemory->iPages && codeSegMemory->iIsDemandPaged);
+
+#ifdef _DEBUG
+ TInt pageNumber = (lin - codeSegMemory->iRamInfo.iCodeRunAddr) >> KPageShift;
+ __NK_ASSERT_DEBUG(codeSegMemory->iPages[pageNumber] == aPageInfo->PhysAddr());
+#endif
+
+ // make page inaccessible...
+ DoSetCodeOld(aPageInfo,codeSegMemory,lin);
+
+ END_PAGING_BENCHMARK(this, EPagingBmSetCodePageOld);
+ }
+ else if(type==SPageInfo::EPagedCache)
+ {
+ // leave page accessible
+ }
+ else if(type!=SPageInfo::EPagedFree)
+ {
+ __KTRACE_OPT2(KPAGING,KPANIC,Kern::Printf("DP: SetOld() with bad page type = %d",aPageInfo->Type()));
+ Panic(EUnexpectedPageType);
+ }
+ NKern::FlashSystem();
+ }
+
+
+void DoSetCodeFree(SPageInfo* aPageInfo, TPhysAddr aPhysAddr, DMemModelCodeSegMemory* aCodeSegMemory, TLinAddr aLinAddr)
+ {
+ NThread* currentThread = NKern::CurrentThread();
+ aPageInfo->SetModifier(currentThread);
+ // scan all address spaces...
+ TInt asid = -1;
+ TInt lastAsid = KArmV6NumAsids-1;
+ TUint32* ptr = aCodeSegMemory->iOsAsids->iMap;
+ do
+ {
+ TUint32 bits = *ptr++;
+ do
+ {
+ ++asid;
+ if(bits&0x80000000u)
+ {
+ // codeseg is mapped in this address space, so update PTE...
+ TPte* pt = PtePtrFromLinAddr(aLinAddr,asid);
+ TPte pte = *pt;
+ if (pte!=KPteNotPresentEntry && (pte&~KPageMask) == aPhysAddr)
+ MakePTEInaccessible(pt, KPteNotPresentEntry, aLinAddr, asid);
+ }
+ }
+ while(bits<<=1);
+ if(NKern::FlashSystem())
+ {
+ // nobody else should modify page!
+ __NK_ASSERT_DEBUG(!aPageInfo->CheckModified(currentThread));
+ }
+ asid |= 31;
+ }
+ while(asid<lastAsid);
+ }
+
+
+void MemModelDemandPaging::SetFree(SPageInfo* aPageInfo)
+ {
+ __ASSERT_SYSTEM_LOCK;
+ __ASSERT_MUTEX(MmuBase::RamAllocatorMutex);
+ __NK_ASSERT_DEBUG(aPageInfo->State() == SPageInfo::EStatePagedDead);
+ if(aPageInfo->LockCount())
+ Panic(ERamPageLocked);
+
+ SPageInfo::TType type = aPageInfo->Type();
+ TPhysAddr phys = aPageInfo->PhysAddr();
+
+ if(type==SPageInfo::EPagedROM)
+ {
+ // get linear address of page...
+ TInt offset = aPageInfo->Offset()<<KPageShift;
+ __NK_ASSERT_DEBUG(TUint(offset)<iRomSize);
+ TLinAddr lin = iRomLinearBase+offset;
+
+ // unmap it...
+ TPte* pt = PtePtrFromLinAddr(lin);
+ MakeGlobalPTEInaccessible(pt, KPteNotPresentEntry, lin);
+
+#ifdef BTRACE_PAGING
+ BTraceContext8(BTrace::EPaging,BTrace::EPagingPageOutROM,phys,lin);
+#endif
+ }
+ else if(type==SPageInfo::EPagedCode)
+ {
+ START_PAGING_BENCHMARK;
+
+ // get linear address of page...
+ TInt offset = aPageInfo->Offset()<<KPageShift;
+ __NK_ASSERT_DEBUG(TUint(offset)<iCodeSize);
+ TLinAddr lin = iCodeLinearBase+offset;
+
+ // get CodeSegMemory...
+ // NOTE, this cannot die because we hold the RamAlloc mutex, and the CodeSegMemory
+ // destructor also needs this mutex to do it's cleanup...
+ DMemModelCodeSegMemory* codeSegMemory = (DMemModelCodeSegMemory*)aPageInfo->Owner();
+ __NK_ASSERT_DEBUG(codeSegMemory && codeSegMemory->iPages && codeSegMemory->iIsDemandPaged);
+
+ // remove page from CodeSegMemory (must come before System Lock is released)...
+ TInt pageNumber = (lin - codeSegMemory->iRamInfo.iCodeRunAddr) >> KPageShift;
+ __NK_ASSERT_DEBUG(codeSegMemory->iPages[pageNumber] == aPageInfo->PhysAddr());
+ codeSegMemory->iPages[pageNumber] = KPhysAddrInvalid;
+
+ // unmap page from all processes it's mapped into...
+ DoSetCodeFree(aPageInfo,phys,codeSegMemory,lin);
+
+ END_PAGING_BENCHMARK(this, EPagingBmSetCodePageFree);
+#ifdef BTRACE_PAGING
+ BTraceContext8(BTrace::EPaging,BTrace::EPagingPageOutCode,phys,lin);
+#endif
+ }
+ else if(type==SPageInfo::EPagedCache)
+ {
+ // get linear address of page...
+ TInt offset = aPageInfo->Offset()<<KPageShift;
+ DMemModelChunk* chunk = (DMemModelChunk*)aPageInfo->Owner();
+ __NK_ASSERT_DEBUG(TUint(offset)<TUint(chunk->iMaxSize));
+ TLinAddr lin = ((TLinAddr)chunk->iBase)+offset;
+
+ // unmap it...
+ TInt asid = ((DMemModelProcess*)chunk->iOwningProcess)->iOsAsid;
+ TPte* pt = PtePtrFromLinAddr(lin,asid);
+ *pt = KPteNotPresentEntry;
+ CacheMaintenance::SinglePteUpdated((TLinAddr)pt);
+
+ InvalidateTLBForPage(lin,asid);
+
+ // actually decommit it from chunk...
+ TInt ptid = ((TLinAddr)pt-KPageTableBase)>>KPageTableShift;
+ SPageTableInfo& ptinfo=Mmu().iPtInfo[ptid];
+ if(!--ptinfo.iCount)
+ {
+ chunk->iPageTables[offset>>KChunkShift] = 0xffff;
+ NKern::UnlockSystem();
+ Mmu().DoUnassignPageTable(lin, (TAny*)asid);
+ Mmu().FreePageTable(ptid);
+ NKern::LockSystem();
+ }
+
+#ifdef BTRACE_PAGING
+ BTraceContext8(BTrace::EPaging,BTrace::EPagingPageOutCache,phys,lin);
+#endif
+ }
+ else if(type==SPageInfo::EPagedFree)
+ {
+ // already free...
+#ifdef BTRACE_PAGING
+ BTraceContext4(BTrace::EPaging,BTrace::EPagingPageOutFree,phys);
+#endif
+ // fall through to cache purge code because cache may not have been
+ // cleaned for this page if PageUnmapped called
+ }
+ else
+ {
+ __KTRACE_OPT2(KPAGING,KPANIC,Kern::Printf("DP: SetFree() with bad page type = %d",aPageInfo->Type()));
+ Panic(EUnexpectedPageType);
+ return;
+ }
+
+ NKern::UnlockSystem();
+
+ // purge cache for page...
+ TInt colour = aPageInfo->Offset()&KPageColourMask;
+ TPte& pte=iPurgePte[colour];
+ TLinAddr va=iPurgeAddr+(colour<<KPageShift);
+ pte=phys|SP_PTE(KArmV6PermRWNO, TheMmu.iCacheMaintenanceTempMapAttr, 1, 1);
+ CacheMaintenance::SinglePteUpdated((TLinAddr)&pte);
+
+ CacheMaintenance::PageToReuse(va,EMemAttNormalCached, KPhysAddrInvalid);
+
+ pte=0;
+ CacheMaintenance::SinglePteUpdated((TLinAddr)&pte);
+ InvalidateTLBForPage(va,KERNEL_MAPPING);
+
+ NKern::LockSystem();
+ }
+
+
+void MemModelDemandPaging::NotifyPageFree(TPhysAddr aPage)
+ {
+ __KTRACE_OPT(KPAGING, Kern::Printf("MemModelDemandPaging::NotifyPageFree %08x", aPage));
+ __ASSERT_SYSTEM_LOCK;
+
+ SPageInfo* pageInfo = SPageInfo::FromPhysAddr(aPage);
+ __ASSERT_DEBUG(pageInfo->Type()==SPageInfo::EPagedCode, MM::Panic(MM::EUnexpectedPageType));
+ RemovePage(pageInfo);
+ SetFree(pageInfo);
+ AddAsFreePage(pageInfo);
+ }
+
+
+TInt MemModelDemandPaging::Fault(TAny* aExceptionInfo)
+ {
+ TArmExcInfo& exc=*(TArmExcInfo*)aExceptionInfo;
+
+ // Get faulting address
+ TLinAddr faultAddress = exc.iFaultAddress;
+ if(exc.iExcCode==EArmExceptionDataAbort)
+ {
+ // Let writes take an exception rather than page in any memory...
+ if(exc.iFaultStatus&(1<<11))
+ return KErrUnknown;
+ }
+ else if (exc.iExcCode != EArmExceptionPrefetchAbort)
+ return KErrUnknown; // Not prefetch or data abort
+
+ // Only handle page translation faults
+ if((exc.iFaultStatus & 0x40f) != 0x7)
+ return KErrUnknown;
+
+ DMemModelThread* thread = (DMemModelThread*)TheCurrentThread;
+
+ // check which ragion fault occured in...
+ TInt asid = 0; // asid != 0 => code paging fault
+ if(TUint(faultAddress-iRomPagedLinearBase)<iRomPagedSize)
+ {
+ // in ROM
+ }
+ else if(TUint(faultAddress-iCodeLinearBase)<iCodeSize)
+ {
+ // in code
+ asid = ((DMemModelProcess*)TheScheduler.iAddressSpace)->iOsAsid;
+ }
+ else if (thread->iAliasLinAddr && TUint(faultAddress - thread->iAliasLinAddr) < TUint(KPageSize))
+ {
+ // in aliased memory
+ faultAddress = (faultAddress - thread->iAliasLinAddr) + thread->iAliasTarget;
+ if(TUint(faultAddress-iCodeLinearBase)>=iCodeSize)
+ return KErrUnknown; // not in alias of code
+ asid = thread->iAliasOsAsid;
+ __NK_ASSERT_DEBUG(asid != 0);
+ }
+ else
+ return KErrUnknown; // Not in pageable region
+
+ // Check if thread holds fast mutex and claim system lock
+ NFastMutex* fm = NKern::HeldFastMutex();
+ TPagingExcTrap* trap = thread->iPagingExcTrap;
+ if(!fm)
+ NKern::LockSystem();
+ else
+ {
+ if(!trap || fm!=&TheScheduler.iLock)
+ {
+ __KTRACE_OPT2(KPAGING,KPANIC,Kern::Printf("DP: Fault with FM Held! %x (%O pc=%x)",faultAddress,&Kern::CurrentThread(),exc.iR15));
+ Panic(EPageFaultWhilstFMHeld); // Not allowed to hold mutexes
+ }
+ // restore address space on multiple memory model (because the trap will
+ // bypass any code which would have done this.)...
+ DMemModelThread::RestoreAddressSpace();
+
+ // Current thread already has the system lock...
+ NKern::FlashSystem(); // Let someone else have a go with the system lock.
+ }
+
+ // System locked here
+
+ TInt r = KErrNone;
+ if(thread->IsRealtime())
+ r = CheckRealtimeThreadFault(thread, aExceptionInfo);
+ if (r == KErrNone)
+ r = HandleFault(exc, faultAddress, asid);
+
+ // Restore system lock state
+ if (fm != NKern::HeldFastMutex())
+ {
+ if (fm)
+ NKern::LockSystem();
+ else
+ NKern::UnlockSystem();
+ }
+
+ // Deal with XTRAP_PAGING
+ if(r == KErrNone && trap)
+ {
+ trap->Exception(1); // Return from exception trap with result '1' (value>0)
+ // code doesn't continue beyond this point.
+ }
+
+ return r;
+ }
+
+
+
+TInt MemModelDemandPaging::HandleFault(TArmExcInfo& aExc, TLinAddr aFaultAddress, TInt aAsid)
+ {
+ ++iEventInfo.iPageFaultCount;
+
+ // get page table entry...
+ TPte* pt = SafePtePtrFromLinAddr(aFaultAddress, aAsid);
+ if(!pt)
+ return KErrNotFound;
+ TPte pte = *pt;
+
+ // Do what is required to make page accessible...
+
+ if(pte&KPtePresentMask)
+ {
+ // PTE is present, so assume it has already been dealt with
+#ifdef BTRACE_PAGING
+ BTraceContext12(BTrace::EPaging,BTrace::EPagingPageNop,pte&~KPageMask,aFaultAddress,aExc.iR15);
+#endif
+ return KErrNone;
+ }
+
+ if(pte!=KPteNotPresentEntry)
+ {
+ // PTE alread has a page
+ SPageInfo* pageInfo = SPageInfo::FromPhysAddr(pte);
+ if(pageInfo->State()==SPageInfo::EStatePagedDead)
+ {
+ // page currently being unmapped, so do that here...
+ MakePTEInaccessible(pt, KPteNotPresentEntry, aFaultAddress, aAsid);
+ }
+ else
+ {
+ // page just needs making young again...
+ *pt = TPte(pte|KArmV6PteSmallPage); // Update page table
+ CacheMaintenance::SinglePteUpdated((TLinAddr)pt);
+ Rejuvenate(pageInfo);
+#ifdef BTRACE_PAGING
+ BTraceContext12(BTrace::EPaging,BTrace::EPagingRejuvenate,pte&~KPageMask,aFaultAddress,aExc.iR15);
+#endif
+ return KErrNone;
+ }
+ }
+
+ // PTE not present, so page it in...
+ // check if fault in a CodeSeg...
+ DMemModelCodeSegMemory* codeSegMemory = NULL;
+ if (!aAsid)
+ NKern::ThreadEnterCS();
+ else
+ {
+ // find CodeSeg...
+ DMemModelCodeSeg* codeSeg = (DMemModelCodeSeg*)DCodeSeg::CodeSegsByAddress.Find(aFaultAddress);
+ if (!codeSeg)
+ return KErrNotFound;
+ codeSegMemory = codeSeg->Memory();
+ if (codeSegMemory==0 || !codeSegMemory->iIsDemandPaged || codeSegMemory->iOsAsids->NotFree(aAsid, 1))
+ return KErrNotFound;
+
+ // check if it's paged in but not yet mapped into this process...
+ TInt pageNumber = (aFaultAddress - codeSegMemory->iRamInfo.iCodeRunAddr) >> KPageShift;
+ TPhysAddr page = codeSegMemory->iPages[pageNumber];
+ if (page != KPhysAddrInvalid)
+ {
+ // map it into this process...
+ SPageInfo* pageInfo = SPageInfo::FromPhysAddr(page);
+ __NK_ASSERT_DEBUG(pageInfo->State()!=SPageInfo::EStatePagedDead);
+ *pt = page | (codeSegMemory->iCreator ? KUserCodeLoadPte : KUserCodeRunPte);
+ CacheMaintenance::SinglePteUpdated((TLinAddr)pt);
+ Rejuvenate(pageInfo);
+#ifdef BTRACE_PAGING
+ BTraceContext8(BTrace::EPaging,BTrace::EPagingMapCode,page,aFaultAddress);
+#endif
+ return KErrNone;
+ }
+
+ // open reference on CodeSegMemory
+ NKern::ThreadEnterCS();
+#ifdef _DEBUG
+ TInt r =
+#endif
+ codeSegMemory->Open();
+ __NK_ASSERT_DEBUG(r==KErrNone);
+ NKern::FlashSystem();
+ }
+
+#ifdef BTRACE_PAGING
+ BTraceContext8(BTrace::EPaging,BTrace::EPagingPageInBegin,aFaultAddress,aExc.iR15);
+#endif
+ TInt r = PageIn(aFaultAddress, aAsid, codeSegMemory);
+
+ NKern::UnlockSystem();
+
+ if(codeSegMemory)
+ codeSegMemory->Close();
+
+ NKern::ThreadLeaveCS();
+
+ return r;
+ }
+
+
+TInt MemModelDemandPaging::PageIn(TLinAddr aAddress, TInt aAsid, DMemModelCodeSegMemory* aCodeSegMemory)
+ {
+ // Get a request object - this may block until one is available
+ DPagingRequest* req = AcquireRequestObject();
+
+ // Get page table entry
+ TPte* pt = SafePtePtrFromLinAddr(aAddress, aAsid);
+
+ // Check page is still required...
+ if(!pt || *pt!=KPteNotPresentEntry)
+ {
+#ifdef BTRACE_PAGING
+ BTraceContext0(BTrace::EPaging,BTrace::EPagingPageInUnneeded);
+#endif
+ ReleaseRequestObject(req);
+ return pt ? KErrNone : KErrNotFound;
+ }
+
+ ++iEventInfo.iPageInReadCount;
+
+ // Get a free page
+ SPageInfo* pageInfo = AllocateNewPage();
+ __NK_ASSERT_DEBUG(pageInfo);
+
+ // Get physical address of free page
+ TPhysAddr phys = pageInfo->PhysAddr();
+ __NK_ASSERT_DEBUG(phys!=KPhysAddrInvalid);
+
+ // Temporarily map free page
+ TInt colour = (aAddress>>KPageShift)&KPageColourMask;
+ __NK_ASSERT_DEBUG((req->iLoadAddr & (KPageColourMask << KPageShift)) == 0);
+ req->iLoadAddr |= colour << KPageShift;
+ TLinAddr loadAddr = req->iLoadAddr;
+ pt = req->iLoadPte+colour;
+// *pt = phys | SP_PTE(KArmV6PermRWNO, KArmV6MemAttWTWAWTWA, 0, 1);
+ *pt = phys | SP_PTE(KArmV6PermRWNO, KNormalUncachedAttr, 0, 1);
+ CacheMaintenance::SinglePteUpdated((TLinAddr)pt);
+
+ // Read page from backing store
+ aAddress &= ~KPageMask;
+ NKern::UnlockSystem();
+
+ TInt r;
+ if (!aCodeSegMemory)
+ r = ReadRomPage(req, aAddress);
+ else
+ {
+ r = ReadCodePage(req, aCodeSegMemory, aAddress);
+ if (r == KErrNone)
+ aCodeSegMemory->ApplyCodeFixups((TUint32*)loadAddr, aAddress);
+ }
+ if(r!=KErrNone)
+ Panic(EPageInFailed);
+
+ // make caches consistant...
+// Cache::IMB_Range(loadAddr, KPageSize);
+ *pt = phys | SP_PTE(KArmV6PermRWNO, KNormalCachedAttr, 0, 1);
+ CacheMaintenance::SinglePteUpdated((TLinAddr)pt);
+ InvalidateTLBForPage(loadAddr,KERNEL_MAPPING);
+ CacheMaintenance::CodeChanged(loadAddr, KPageSize, CacheMaintenance::ECPUUncached);
+
+ NKern::LockSystem();
+
+ // Invalidate temporary mapping
+ MakeGlobalPTEInaccessible(pt, KPteNotPresentEntry, loadAddr);
+
+ // Release request object now we're finished with it
+ req->iLoadAddr &= ~(KPageColourMask << KPageShift);
+ ReleaseRequestObject(req);
+
+ // Get page table entry
+ pt = SafePtePtrFromLinAddr(aAddress, aAsid);
+
+ // Check page still needs updating
+ TBool notNeeded = pt==0 || *pt!=KPteNotPresentEntry;
+ if(aCodeSegMemory)
+ notNeeded |= aCodeSegMemory->iOsAsids->NotFree(aAsid, 1);
+ if(notNeeded)
+ {
+ // We don't need the new page after all, so put it on the active list as a free page
+ __KTRACE_OPT(KPAGING,Kern::Printf("DP: PageIn (New page not used)"));
+#ifdef BTRACE_PAGING
+ BTraceContext0(BTrace::EPaging,BTrace::EPagingPageInUnneeded);
+#endif
+ AddAsFreePage(pageInfo);
+ return pt ? KErrNone : KErrNotFound;
+ }
+
+ // Update page info
+ if (!aCodeSegMemory)
+ pageInfo->SetPagedROM((aAddress-iRomLinearBase)>>KPageShift);
+ else
+ {
+ // Check if page has been paged in and mapped into another process while we were waiting
+ TInt pageNumber = (aAddress - aCodeSegMemory->iRamInfo.iCodeRunAddr) >> KPageShift;
+ TPhysAddr page = aCodeSegMemory->iPages[pageNumber];
+ if (page != KPhysAddrInvalid)
+ {
+ // don't need page we've just paged in...
+ AddAsFreePage(pageInfo);
+
+ // map existing page into this process...
+ pageInfo = SPageInfo::FromPhysAddr(page);
+ __NK_ASSERT_DEBUG(pageInfo->State()!=SPageInfo::EStatePagedDead);
+ *pt = page | (aCodeSegMemory->iCreator ? KUserCodeLoadPte : KUserCodeRunPte);
+ CacheMaintenance::SinglePteUpdated((TLinAddr)pt);
+#ifdef BTRACE_PAGING
+ BTraceContext0(BTrace::EPaging,BTrace::EPagingPageInUnneeded);
+#endif
+ Rejuvenate(pageInfo);
+ return KErrNone;
+ }
+ aCodeSegMemory->iPages[pageNumber] = phys;
+
+ pageInfo->SetPagedCode(aCodeSegMemory,(aAddress-Mmu().iUserCodeBase)>>KPageShift);
+ }
+
+ // Map page into final location
+ *pt = phys | (aCodeSegMemory ? (aCodeSegMemory->iCreator ? KUserCodeLoadPte : KUserCodeRunPte) : KRomPtePerm);
+ CacheMaintenance::SinglePteUpdated((TLinAddr)pt);
+#ifdef BTRACE_PAGING
+ TInt subCat = aCodeSegMemory ? BTrace::EPagingPageInCode : BTrace::EPagingPageInROM;
+ BTraceContext8(BTrace::EPaging,subCat,phys,aAddress);
+#endif
+
+ AddAsYoungest(pageInfo);
+ BalanceAges();
+
+ return KErrNone;
+ }
+
+
+inline TUint8 ReadByte(TLinAddr aAddress)
+ { return *(volatile TUint8*)aAddress; }
+
+
+TInt MemModelDemandPaging::EnsurePagePresent(TLinAddr aPage, DProcess* aProcess)
+ {
+ TInt r = KErrBadDescriptor;
+ XTRAPD(exc,XT_DEFAULT,
+ if (!aProcess)
+ {
+ XTRAP_PAGING_RETRY(CHECK_PAGING_SAFE; ReadByte(aPage););
+ r = KErrNone;
+ }
+ else
+ {
+ DMemModelThread& t=*(DMemModelThread*)TheCurrentThread;
+ retry:
+ TInt pagingFault;
+ XTRAP_PAGING_START(pagingFault);
+ CHECK_PAGING_SAFE;
+ // make alias of page in this process
+ TLinAddr alias_src;
+ TInt alias_size;
+ TInt aliasResult = t.Alias(aPage, (DMemModelProcess*)aProcess, 1, EMapAttrReadUser, alias_src, alias_size);
+ if (aliasResult>=0)
+ {
+ // ensure page to be locked is mapped in, by reading from it...
+ ReadByte(alias_src);
+ r = KErrNone;
+ }
+ XTRAP_PAGING_END;
+ t.RemoveAlias();
+ if(pagingFault>0)
+ goto retry;
+ }
+ ); // end of XTRAPD
+ if(exc)
+ return KErrBadDescriptor;
+ return r;
+ }
+
+
+TPhysAddr MemModelDemandPaging::LinearToPhysical(TLinAddr aPage, DProcess* aProcess)
+ {
+ TInt asid = 0;
+ if (aProcess)
+ asid = ((DMemModelProcess*)aProcess)->iOsAsid;
+ return Mmu().LinearToPhysical(aPage, asid);
+ }
+
+
+TInt MemModelDemandPaging::PageState(TLinAddr aAddr)
+ {
+ DMemModelProcess* process = (DMemModelProcess*)TheCurrentThread->iOwningProcess;
+ TInt asid = 0;
+ TPte* ptePtr = 0;
+ TPte pte = 0;
+ TInt r = 0;
+ SPageInfo* pageInfo = NULL;
+
+ NKern::LockSystem();
+
+ DMemModelCodeSegMemory* codeSegMemory = 0;
+ if(TUint(aAddr-iRomPagedLinearBase)<iRomPagedSize)
+ r |= EPageStateInRom;
+ else if (TUint(aAddr-iCodeLinearBase)<iCodeSize)
+ {
+ DMemModelCodeSeg* codeSeg = (DMemModelCodeSeg*)DCodeSeg::CodeSegsByAddress.Find(aAddr);
+ if(codeSeg)
+ codeSegMemory = codeSeg->Memory();
+ asid = process->iOsAsid;
+ if (codeSegMemory && codeSegMemory->iOsAsids->NotAllocated(asid, 1))
+ {
+ r |= EPageStateInRamCode;
+ if (codeSegMemory->iIsDemandPaged)
+ r |= EPageStatePaged;
+ }
+ if(process->iCodeChunk)
+ r |= EPageStateCodeChunkPresent;
+ }
+
+ ptePtr = SafePtePtrFromLinAddr(aAddr,asid);
+ if (!ptePtr)
+ goto done;
+ r |= EPageStatePageTablePresent;
+ pte = *ptePtr;
+ if (pte == KPteNotPresentEntry)
+ goto done;
+ r |= EPageStatePtePresent;
+ if (pte & KPtePresentMask)
+ r |= EPageStatePteValid;
+
+ pageInfo = SPageInfo::FromPhysAddr(pte);
+ r |= pageInfo->Type();
+ r |= pageInfo->State()<<8;
+
+ if (codeSegMemory && codeSegMemory->iPages)
+ {
+ TPhysAddr phys = pte & ~KPageMask;
+ TInt pageNumber = (aAddr - codeSegMemory->iRamInfo.iCodeRunAddr) >> KPageShift;
+ if (codeSegMemory->iPages[pageNumber] == phys)
+ r |= EPageStatePhysAddrPresent;
+ }
+
+done:
+ NKern::UnlockSystem();
+ return r;
+ }
+
+
+TBool MemModelDemandPaging::NeedsMutexOrderCheck(TLinAddr aStartAddr, TUint aLength)
+ {
+ // Don't check mutex order for reads from global area, except for the paged part of rom
+ TBool rangeInGlobalArea = aStartAddr >= KRomLinearBase;
+ TBool rangeInPagedRom = iRomPagedLinearBase != 0 && aStartAddr < (iRomLinearBase + iRomSize) && (aStartAddr + aLength) > iRomPagedLinearBase;
+ return !rangeInGlobalArea || rangeInPagedRom;
+ }
+
+
+EXPORT_C TBool DDemandPagingLock::Lock(DThread* aThread, TLinAddr aStart, TInt aSize)
+ {
+ MemModelDemandPaging* pager = (MemModelDemandPaging*)iThePager;
+ if(pager)
+ {
+ ArmMmu& m = pager->Mmu();
+ TLinAddr end = aStart+aSize;
+
+ if ((aStart < TUint(pager->iRomPagedLinearBase+pager->iRomPagedSize) && end > pager->iRomPagedLinearBase) ||
+ (aStart < TUint(m.iUserCodeBase + m.iMaxUserCodeSize) && end > m.iUserCodeBase))
+ return pager->ReserveLock(aThread,aStart,aSize,*this);
+ }
+ return EFalse;
+ }
+
+void ArmMmu::DisablePageModification(DMemModelChunk* aChunk, TInt aOffset)
+//
+// Mark the page at aOffset in aChunk read-only to prevent it being
+// modified while defrag is in progress. Save the required information
+// to allow the fault handler to deal with this.
+// Call this with the system unlocked.
+//
+ {
+ __KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::DisablePageModification() offset=%08x", aOffset));
+
+ TInt ptid = aChunk->iPageTables[aOffset>>KChunkShift];
+ if(ptid == 0xffff)
+ Panic(EDefragDisablePageFailed);
+
+ NKern::LockSystem();
+ TPte* pPte = PageTable(ptid) + ((aOffset&KChunkMask)>>KPageShift);
+ TPte pte = *pPte;
+ if ((pte & KArmV6PteSmallPage) != KArmV6PteSmallPage
+ || SP_PTE_PERM_GET(pte) != (TUint)KArmV6PermRWRW)
+ Panic(EDefragDisablePageFailed);
+
+ iDisabledAddr = (TLinAddr)(aChunk->iBase) + aOffset;
+ if (aChunk->iOwningProcess)
+ iDisabledAddrAsid = ((DMemModelProcess*)(aChunk->iOwningProcess))->iOsAsid;
+ else
+ iDisabledAddrAsid = iDisabledAddr<KRomLinearBase ? UNKNOWN_MAPPING : KERNEL_MAPPING;
+ iDisabledPte = pPte;
+ iDisabledOldVal = pte;
+
+ *pPte = SP_PTE_PERM_SET(pte, KArmV6PermRORO);
+ CacheMaintenance::SinglePteUpdated((TLinAddr)pPte);
+ InvalidateTLBForPage(iDisabledAddr, iDisabledAddrAsid);
+ NKern::UnlockSystem();
+ }
+
+TInt ArmMmu::RamDefragFault(TAny* aExceptionInfo)
+ {
+ TArmExcInfo& exc=*(TArmExcInfo*)aExceptionInfo;
+
+ // Get faulting address
+ TLinAddr faultAddress;
+ if(exc.iExcCode==EArmExceptionDataAbort)
+ {
+ faultAddress = exc.iFaultAddress;
+ // Defrag can only cause writes to fault on multiple model
+ if(!(exc.iFaultStatus&(1<<11)))
+ return KErrUnknown;
+ }
+ else
+ return KErrUnknown; // Not data abort
+
+ // Only handle page permission faults
+ if((exc.iFaultStatus & 0x40f) != 0xf)
+ return KErrUnknown;
+
+ DMemModelThread* thread = (DMemModelThread*)TheCurrentThread;
+ TInt asid = ((DMemModelProcess*)TheScheduler.iAddressSpace)->iOsAsid;
+
+ TBool aliased = EFalse;
+ if (thread->iAliasLinAddr && TUint(faultAddress - thread->iAliasLinAddr) < TUint(KPageSize))
+ {
+ // in aliased memory
+ aliased = ETrue;
+ faultAddress = (faultAddress - thread->iAliasLinAddr) + thread->iAliasTarget;
+ asid = thread->iAliasOsAsid;
+ __NK_ASSERT_DEBUG(asid != 0);
+ }
+
+ // Take system lock if not already held
+ NFastMutex* fm = NKern::HeldFastMutex();
+ if(!fm)
+ NKern::LockSystem();
+ else if(fm!=&TheScheduler.iLock)
+ {
+ __KTRACE_OPT2(KMMU,KPANIC,Kern::Printf("Defrag: Fault with FM Held! %x (%O pc=%x)",faultAddress,&Kern::CurrentThread(),exc.iR15));
+ Panic(EDefragFaultWhilstFMHeld); // Not allowed to hold mutexes
+ }
+
+ TInt r = KErrUnknown;
+
+ // check if write access to the page has already been restored and retry if so
+ TPte* pt = SafePtePtrFromLinAddr(faultAddress, asid);
+ if(!pt)
+ {
+ r = KErrNotFound;
+ goto leave;
+ }
+ if (SP_PTE_PERM_GET(*pt) == (TUint)KArmV6PermRWRW)
+ {
+ r = KErrNone;
+ goto leave;
+ }
+
+ // check if the fault occurred in the page we are moving
+ if ( iDisabledPte
+ && TUint(faultAddress - iDisabledAddr) < TUint(KPageSize)
+ && (iDisabledAddrAsid < 0 || asid == iDisabledAddrAsid) )
+ {
+ // restore access to the page
+ *iDisabledPte = iDisabledOldVal;
+ CacheMaintenance::SinglePteUpdated((TLinAddr)iDisabledPte);
+ InvalidateTLBForPage(iDisabledAddr, iDisabledAddrAsid);
+ if (aliased)
+ InvalidateTLBForPage(exc.iFaultAddress, ((DMemModelProcess*)TheScheduler.iAddressSpace)->iOsAsid);
+ iDisabledAddr = 0;
+ iDisabledAddrAsid = -1;
+ iDisabledPte = NULL;
+ iDisabledOldVal = 0;
+ r = KErrNone;
+ }
+
+leave:
+ // Restore system lock state
+ if (!fm)
+ NKern::UnlockSystem();
+
+ return r;
+ }