diff -r 000000000000 -r a41df078684a kernel/eka/memmodel/epoc/moving/arm/xmmu.cpp --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/kernel/eka/memmodel/epoc/moving/arm/xmmu.cpp Mon Oct 19 15:55:17 2009 +0100 @@ -0,0 +1,2798 @@ +// Copyright (c) 1997-2009 Nokia Corporation and/or its subsidiary(-ies). +// All rights reserved. +// This component and the accompanying materials are made available +// under the terms of the License "Eclipse Public License v1.0" +// which accompanies this distribution, and is available +// at the URL "http://www.eclipse.org/legal/epl-v10.html". +// +// Initial Contributors: +// Nokia Corporation - initial contribution. +// +// Contributors: +// +// Description: +// e32\memmodel\epoc\moving\arm\xmmu.cpp +// +// + +#include "arm_mem.h" +#include +#include +#include +#include "execs.h" +#include +#include "cache_maintenance.h" + + +extern void FlushTLBs(); + +#if defined(__CPU_SA1__) +const TPde KRomSectionPermissions = SECTION_PDE(KArmV45PermRORO, KArmV45MemAttWB, EDomainClient); +const TPde KShadowPdePerm = PT_PDE(EDomainClient); +const TPte KPtPtePerm = SP_PTE(KArmV45PermRWNO, KArmV45MemAttBuf); // page tables not cached +const TPte KRomPtePermissions = SP_PTE(KArmV45PermRORO, KArmV45MemAttWB); // ROM is cached, read-only for everyone +const TPte KShadowPtePerm = SP_PTE(KArmV45PermRWRO, KArmV45MemAttWB); // shadowed ROM is cached, supervisor writeable + +#elif defined(__CPU_ARM710T__) || defined(__CPU_ARM720T__) +const TPde KRomSectionPermissions = SECTION_PDE(KArmV45PermRORO, KArmV45MemAttWB, EDomainClient); +const TPde KShadowPdePerm = PT_PDE(EDomainClient); +const TPte KPtPtePerm = SP_PTE(KArmV45PermRWNO, KArmV45MemAttWB); // page tables cached (write-through) +const TPte KRomPtePermissions = SP_PTE(KArmV45PermRORO, KArmV45MemAttWB); // ROM is cached, read-only for everyone +const TPte KShadowPtePerm = SP_PTE(KArmV45PermRWRO, KArmV45MemAttWB); // shadowed ROM is cached, supervisor writeable + +#elif defined(__CPU_ARM920T__) || defined(__CPU_ARM925T__) || defined(__CPU_ARM926J__) +const TPde KRomSectionPermissions = SECTION_PDE(KArmV45PermRORO, KArmV45MemAttWT, EDomainClient); +const TPde KShadowPdePerm = PT_PDE(EDomainClient); +const TPte KPtPtePerm = SP_PTE(KArmV45PermRWNO, KArmV45MemAttWT); // page tables cached write through +const TPte KRomPtePermissions = SP_PTE(KArmV45PermRORO, KArmV45MemAttWT); // ROM is cached, read-only for everyone +const TPte KShadowPtePerm = SP_PTE(KArmV45PermRWRO, KArmV45MemAttWT); // shadowed ROM is cached, supervisor writeable + +#elif defined(__CPU_XSCALE__) + #ifdef __CPU_XSCALE_MANZANO__ +const TPde KRomSectionPermissions = SECTION_PDE(KArmV45PermRORO, KXScaleMemAttWTRA_WBWA, EDomainClient); +const TPde KShadowPdePerm = PT_PDE(EDomainClient); +const TPte KPtPtePerm = SP_PTE(KArmV45PermRWNO, KXScaleMemAttWTRA_WBWA); // page tables write-through cached +const TPte KRomPtePermissions = SP_PTE(KArmV45PermRORO, KXScaleMemAttWTRA_WBWA); // ROM is cached, read-only for everyone +const TPte KShadowPtePerm = SP_PTE(KArmV45PermRWRO, KXScaleMemAttWTRA_WBWA); // shadowed ROM is cached, supervisor writeable + #else +const TPde KRomSectionPermissions = SECTION_PDE(KArmV45PermRORO, KXScaleMemAttWTRA, EDomainClient); +const TPde KShadowPdePerm = PT_PDE(EDomainClient); +const TPte KPtPtePerm = SP_PTE(KArmV45PermRWNO, KXScaleMemAttWTRA); // page tables write-through cached +const TPte KRomPtePermissions = SP_PTE(KArmV45PermRORO, KXScaleMemAttWTRA); // ROM is cached, read-only for everyone +const TPte KShadowPtePerm = SP_PTE(KArmV45PermRWRO, KXScaleMemAttWTRA); // shadowed ROM is cached, supervisor writeable + #endif +#endif + +const TPte KPtInfoPtePerm = KPtPtePerm; +const TPde KPtPdePerm = PT_PDE(EDomainClient); + +// Permissions for each chunk type +enum TPTEProperties + { + ESupRo = SP_PTE(KArmV45PermRORO, KDefaultCaching), + ESupRw = SP_PTE(KArmV45PermRWNO, KDefaultCaching), + EUserRo = SP_PTE(KArmV45PermRWRO, KDefaultCaching), + EUserRw = SP_PTE(KArmV45PermRWRW, KDefaultCaching) + }; + +LOCAL_D const TPde ChunkPdePermissions[ENumChunkTypes] = + { + PT_PDE(EDomainClient), // EKernelData + PT_PDE(EDomainClient), // EKernelStack + PT_PDE(EDomainClient), // EKernelCode + PT_PDE(EDomainClient), // EDll + PT_PDE(EDomainClient), // EUserCode - user/ro & sup/rw everywhere + PT_PDE(EDomainClient), // ERamDrive - sup/rw accessed by domain change + + // user data or self modifying code is sup/rw, user no access at home. It's user/rw & sup/rw when running + // note ARM MMU architecture prevents implementation of user read-only data + PT_PDE(EDomainClient), // EUserData + PT_PDE(EDomainClient), // EDllData + PT_PDE(EDomainClient), // EUserSelfModCode + PT_PDE(EDomainClient), // ESharedKernelSingle + PT_PDE(EDomainClient), // ESharedKernelMultiple + PT_PDE(EDomainClient), // ESharedIo + PT_PDE(EDomainClient), // ESharedKernelMirror (unused in this memory model) + PT_PDE(EDomainClient), // EKernelMessage + }; + +const TPde KUserDataRunningPermissions = PT_PDE(EDomainVarUserRun); + +LOCAL_D const TPte ChunkPtePermissions[ENumChunkTypes] = + { + ESupRw, // EKernelData + ESupRw, // EKernelStack + ESupRw, // EKernelCode + EUserRo, // EDll + EUserRo, // EUserCode + ESupRw, // ERamDrive + ESupRw, // EUserData + ESupRw, // EDllData + ESupRw, // EUserSelfModCode + ESupRw, // ESharedKernelSingle + ESupRw, // ESharedKernelMultiple + ESupRw, // ESharedIo + ESupRw, // ESharedKernelMirror (unused in this memory model) + ESupRw, // EKernelMessage + }; + +const TPte KUserCodeLoadPte = (TPte)EUserRo; +const TPte KKernelCodeRunPte = (TPte)ESupRw; + +// Inline functions for simple transformations +inline TLinAddr PageTableLinAddr(TInt aId) + { + return KPageTableBase + (aId<>KChunkShift); + } + +inline TBool IsPageTable(TPde aPde) + { + return ((aPde&KPdeTypeMask)==KArmV45PdePageTable); + } + +inline TBool IsSectionDescriptor(TPde aPde) + { + return ((aPde&KPdeTypeMask)==KArmV45PdeSection); + } + +inline TBool IsPresent(TPte aPte) + { + return (aPte&KPtePresentMask); + } + +inline TPhysAddr PageTablePhysAddr(TPde aPde) + { + return aPde & KPdePageTableAddrMask; + } + +inline TPhysAddr PhysAddrFromSectionDescriptor(TPde aPde) + { + return aPde & KPdeSectionAddrMask; + } + +extern void InvalidateTLBForPage(TLinAddr /*aLinAddr*/); + +void Mmu::SetupInitialPageInfo(SPageInfo* aPageInfo, TLinAddr aChunkAddr, TInt aPdeIndex) + { + __ASSERT_ALWAYS(aChunkAddr==0 || aChunkAddr>=KRamDriveEndAddress, Panic(EBadInitialPageAddr)); + TLinAddr addr = aChunkAddr + (aPdeIndex<Type()!=SPageInfo::EUnused) + return; // already set (page table) + if (addr == KPageTableInfoBase) + { + aPageInfo->SetPtInfo(0); + aPageInfo->Lock(); + } + else if (addr>=KPageDirectoryBase && addr<(KPageDirectoryBase+KPageDirectorySize)) + { + aPageInfo->SetPageDir(0,aPdeIndex); + aPageInfo->Lock(); + } + else + aPageInfo->SetFixed(); + } + +void Mmu::SetupInitialPageTableInfo(TInt aId, TLinAddr aChunkAddr, TInt aNumPtes) + { + __ASSERT_ALWAYS(aChunkAddr==0 || aChunkAddr>=KRamDriveEndAddress, Panic(EBadInitialPageAddr)); + SPageTableInfo& pti=PtInfo(aId); + pti.iCount=aNumPtes; + pti.SetGlobal(aChunkAddr>>KChunkShift); + } + +TInt Mmu::GetPageTableId(TLinAddr aAddr) + { + TInt id=-1; + __KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::PageTableId(%08x)",aAddr)); + TInt pdeIndex=aAddr>>KChunkShift; + TPde pde = PageDirectory[pdeIndex]; + if (IsPageTable(pde)) + { + SPageInfo* pi = SPageInfo::SafeFromPhysAddr(pde); + if (pi) + id = (pi->Offset()<>KPageTableShift)&KPtClusterMask); + } + __KTRACE_OPT(KMMU,Kern::Printf("ID=%d",id)); + return id; + } + +// Used only during boot for recovery of RAM drive +TInt ArmMmu::BootPageTableId(TLinAddr aAddr, TPhysAddr& aPtPhys) + { + TInt id=KErrNotFound; + __KTRACE_OPT(KMMU,Kern::Printf("ArmMmu:BootPageTableId(%08x,&)",aAddr)); + TInt pdeIndex=aAddr>>KChunkShift; + TPde pde = PageDirectory[pdeIndex]; + if (IsPageTable(pde)) + { + aPtPhys = pde & KPdePageTableAddrMask; + SPageInfo* pi = SPageInfo::SafeFromPhysAddr(pde); + if (pi) + { + SPageInfo::TType type = pi->Type(); + if (type == SPageInfo::EPageTable) + id = (pi->Offset()<>KPageTableShift)&KPtClusterMask); + else if (type == SPageInfo::EUnused) + id = KErrUnknown; + } + } + __KTRACE_OPT(KMMU,Kern::Printf("ID=%d",id)); + return id; + } + +TBool ArmMmu::PteIsPresent(TPte aPte) + { + return aPte & KPtePresentMask; + } + +TPhysAddr ArmMmu::PtePhysAddr(TPte aPte, TInt aPteIndex) + { + TUint pte_type = aPte & KPteTypeMask; + if (pte_type == KArmV45PteLargePage) + return (aPte & KPteLargePageAddrMask) + (TPhysAddr(aPteIndex << KPageShift) & KLargePageMask); + else if (pte_type != 0) + return aPte & KPteSmallPageAddrMask; + return KPhysAddrInvalid; + } + +TPhysAddr ArmMmu::PdePhysAddr(TLinAddr aAddr) + { + TPde pde = PageDirectory[aAddr>>KChunkShift]; + if (IsSectionDescriptor(pde)) + return PhysAddrFromSectionDescriptor(pde); + return KPhysAddrInvalid; + } + +TPte* SafePageTableFromPde(TPde aPde) + { + if((aPde&KPdeTypeMask)==KArmV45PdePageTable) + { + SPageInfo* pi = SPageInfo::SafeFromPhysAddr(aPde); + if(pi) + { + TInt id = (pi->Offset()<>KPageTableShift)&KPtClusterMask); + return PageTable(id); + } + } + return 0; + } + +TPte* SafePtePtrFromLinAddr(TLinAddr aAddress) + { + TPde pde = PageDirectory[aAddress>>KChunkShift]; + TPte* pt = SafePageTableFromPde(pde); + if(pt) + pt += (aAddress>>KPageShift)&(KChunkMask>>KPageShift); + return pt; + } + +#ifdef __ARMCC__ + __forceinline /* RVCT ignores normal inline qualifier :-( */ +#else + inline +#endif +TPte* PtePtrFromLinAddr(TLinAddr aAddress) + { + TPde pde = PageDirectory[aAddress>>KChunkShift]; + SPageInfo* pi = SPageInfo::FromPhysAddr(pde); + TInt id = (pi->Offset()<>KPageTableShift)&KPtClusterMask); + TPte* pt = PageTable(id); + pt += (aAddress>>KPageShift)&(KChunkMask>>KPageShift); + return pt; + } + + +TInt ArmMmu::LinearToPhysical(TLinAddr aLinAddr, TInt aSize, TPhysAddr& aPhysicalAddress, TPhysAddr* aPhysicalPageList) + { + TPhysAddr physStart = ArmMmu::LinearToPhysical(aLinAddr); + TPhysAddr nextPhys = physStart&~KPageMask; + + TUint32* pageList = aPhysicalPageList; + + TInt pageIndex = aLinAddr>>KPageShift; + TInt pagesLeft = ((aLinAddr+aSize-1)>>KPageShift)+1 - pageIndex; + TPde* pdePtr = &PageDirectory[aLinAddr>>KChunkShift]; + + while(pagesLeft) + { + pageIndex &= KChunkMask>>KPageShift; + TInt pagesLeftInChunk = (1<<(KChunkShift-KPageShift))-pageIndex; + if(pagesLeftInChunk>pagesLeft) + pagesLeftInChunk = pagesLeft; + pagesLeft -= pagesLeftInChunk; + + TPhysAddr phys; + TPde pde = *pdePtr++; + TUint pdeType = pde&KPdeTypeMask; + if(pdeType==KArmV45PdeSection) + { + phys = (pde & KPdeSectionAddrMask) + (pageIndex*KPageSize); + __KTRACE_OPT(KMMU2,Kern::Printf("ArmMmu::LinearToPhysical Section phys=%8x",phys)); + TInt n=pagesLeftInChunk; + phys==nextPhys ? nextPhys+=n*KPageSize : nextPhys=KPhysAddrInvalid; + if(pageList) + { + TUint32* pageEnd = pageList+n; + do + { + *pageList++ = phys; + phys+=KPageSize; + } + while(pageList= KArmV45PteSmallPage) + { + phys = (pte & KPteSmallPageAddrMask); + __KTRACE_OPT(KMMU2,Kern::Printf("ArmMmu::LinearToPhysical Small Page phys=%8x",phys)); + phys==nextPhys ? nextPhys+=KPageSize : nextPhys=KPhysAddrInvalid; + if(pageList) + *pageList++ = phys; + if(--pagesLeftInChunk) + continue; + break; + } + if (pte_type == KArmV45PteLargePage) + { + --pt; // back up ptr + TUint pageOffset = ((TUint)pt>>2)&(KLargeSmallPageRatio-1); + phys = (pte & KPteLargePageAddrMask) + pageOffset*KPageSize; + __KTRACE_OPT(KMMU2,Kern::Printf("ArmMmu::LinearToPhysical Large Page phys=%8x",phys)); + TInt n=KLargeSmallPageRatio-pageOffset; + if(n>pagesLeftInChunk) + n = pagesLeftInChunk; + phys==nextPhys ? nextPhys+=n*KPageSize : nextPhys=KPhysAddrInvalid; + if(pageList) + { + TUint32* pageEnd = pageList+n; + do + { + *pageList++ = phys; + phys+=KPageSize; + } + while(pageList>KChunkShift]; + if (IsPageTable(pde)) + { + SPageInfo* pi = SPageInfo::SafeFromPhysAddr(pde); + if (pi) + { + TInt id = (pi->Offset()<>KPageTableShift)&KPtClusterMask); + TInt pteIndex = (aLinAddr & KChunkMask)>>KPageShift; + TPte pte = PageTable(id)[pteIndex]; + TUint pte_type = pte & KPteTypeMask; + if (pte_type == KArmV45PteLargePage) + { + phys = (pte & KPteLargePageAddrMask) + (aLinAddr & KLargePageMask); + __KTRACE_OPT(KMMU,Kern::Printf("Mapped with 64K page - returning %08x", phys)); + } + else if (pte_type != 0) + { + phys = (pte & KPteSmallPageAddrMask) + (aLinAddr & KPageMask); + __KTRACE_OPT(KMMU,Kern::Printf("Mapped with 4K page - returning %08x", phys)); + } + } + } + else if (IsSectionDescriptor(pde)) + { + phys = (pde & KPdeSectionAddrMask) + (aLinAddr & KChunkMask); + __KTRACE_OPT(KMMU,Kern::Printf("Mapped with section - returning %08x", phys)); + } + else + { + __KTRACE_OPT(KMMU,Kern::Printf("Address invalid")); + } + return phys; + } + +TInt ArmMmu::PreparePagesForDMA(TLinAddr aLinAddr, TInt aSize, TPhysAddr* aPhysicalPageList) +//Returns the list of physical pages belonging to the specified memory space. +//Checks these pages belong to a chunk marked as being trusted. +//Locks these pages so they can not be moved by e.g. ram defragmenation. + { + SPageInfo* pi = NULL; + DChunk* chunk = NULL; + TInt err = KErrNone; + + __KTRACE_OPT(KMMU2,Kern::Printf("ArmMmu::PreparePagesForDMA %08x+%08x, asid=%d",aLinAddr,aSize)); + + TUint32* pageList = aPhysicalPageList; + TInt pagesInList = 0; //The number of pages we put in the list so far + + TInt pageIndex = (aLinAddr & KChunkMask) >> KPageShift; // Index of the page within the section + TInt pagesLeft = ((aLinAddr & KPageMask) + aSize + KPageMask) >> KPageShift; + + MmuBase::Wait(); // RamAlloc Mutex for accessing page/directory tables. + NKern::LockSystem();// SystemlLock for accessing SPageInfo objects. + + TPde* pdePtr = PageDirectory + (aLinAddr>>KChunkShift); + + while(pagesLeft) + { + TInt pagesLeftInChunk = (1<<(KChunkShift-KPageShift))-pageIndex; + if(pagesLeftInChunk>pagesLeft) + pagesLeftInChunk = pagesLeft; + + pagesLeft -= pagesLeftInChunk; + + TPte* pt = SafePageTableFromPde(*pdePtr++); + if(!pt) { err = KErrNotFound; goto fail; }// Cannot get page table. + + pt += pageIndex; + + for(;pagesLeftInChunk--;) + { + TPhysAddr phys = (*pt++ & KPteSmallPageAddrMask); + pi = SPageInfo::SafeFromPhysAddr(phys); + if(!pi) { err = KErrNotFound; goto fail; }// Invalid address + + __KTRACE_OPT(KMMU2,Kern::Printf("PageInfo: PA:%x T:%x S:%x O:%x C:%x",phys, pi->Type(), pi->State(), pi->Owner(), pi->LockCount())); + if (chunk==NULL) + {//This is the first page. Check 'trusted' bit. + if (pi->Type()!= SPageInfo::EChunk) + { err = KErrAccessDenied; goto fail; }// The first page do not belong to chunk. + + chunk = (DChunk*)pi->Owner(); + if ( (chunk == NULL) || ((chunk->iAttributes & DChunk::ETrustedChunk)== 0) ) + { err = KErrAccessDenied; goto fail; } // Not a trusted chunk + } + pi->Lock(); + + *pageList++ = phys; + if ( (++pagesInList&127) == 0) //release system lock temporarily on every 512K + NKern::FlashSystem(); + } + pageIndex = 0; + } + + if (pi->Type()!= SPageInfo::EChunk) + { err = KErrAccessDenied; goto fail; }// The last page do not belong to chunk. + + if (chunk && (chunk != (DChunk*)pi->Owner())) + { err = KErrArgument; goto fail; }//The first & the last page do not belong to the same chunk. + + NKern::UnlockSystem(); + MmuBase::Signal(); + return KErrNone; + +fail: + __KTRACE_OPT(KMMU2,Kern::Printf("ArmMmu::PreparePagesForDMA failed")); + NKern::UnlockSystem(); + MmuBase::Signal(); + ReleasePagesFromDMA(aPhysicalPageList, pagesInList); + return err; + } + +TInt ArmMmu::ReleasePagesFromDMA(TPhysAddr* aPhysicalPageList, TInt aPageCount) +// Unlocks physical pages. +// @param aPhysicalPageList - points to the list of physical pages that should be released. +// @param aPageCount - the number of physical pages in the list. + { + NKern::LockSystem(); + __KTRACE_OPT(KMMU2,Kern::Printf("ArmMmu::ReleasePagesFromDMA count:%d",aPageCount)); + + while (aPageCount--) + { + SPageInfo* pi = SPageInfo::SafeFromPhysAddr(*aPhysicalPageList++); + if(!pi) + { + NKern::UnlockSystem(); + return KErrArgument; + } + __KTRACE_OPT(KMMU2,Kern::Printf("PageInfo: T:%x S:%x O:%x C:%x",pi->Type(), pi->State(), pi->Owner(), pi->LockCount())); + pi->Unlock(); + } + NKern::UnlockSystem(); + return KErrNone; + } + + +void ArmMmu::Init1() + { + __KTRACE_OPT2(KBOOT,KMMU,Kern::Printf("ArmMmu::Init1")); + + // MmuBase data + iPageSize=KPageSize; + iPageMask=KPageMask; + iPageShift=KPageShift; + iChunkSize=KChunkSize; + iChunkMask=KChunkMask; + iChunkShift=KChunkShift; + iPageTableSize=KPageTableSize; + iPageTableMask=KPageTableMask; + iPageTableShift=KPageTableShift; + iPtClusterSize=KPtClusterSize; + iPtClusterMask=KPtClusterMask; + iPtClusterShift=KPtClusterShift; + iPtBlockSize=KPtBlockSize; + iPtBlockMask=KPtBlockMask; + iPtBlockShift=KPtBlockShift; + iPtGroupSize=KChunkSize/KPageTableSize; + iPtGroupMask=iPtGroupSize-1; + iPtGroupShift=iChunkShift-iPageTableShift; + //TInt* iPtBlockCount; // dynamically allocated - Init2 + //TInt* iPtGroupCount; // dynamically allocated - Init2 + iPtInfo=(SPageTableInfo*)KPageTableInfoBase; + iPageTableLinBase=KPageTableBase; + //iRamPageAllocator; // dynamically allocated - Init2 + //iAsyncFreeList; // dynamically allocated - Init2 + //iPageTableAllocator; // dynamically allocated - Init2 + //iPageTableLinearAllocator;// dynamically allocated - Init2 + iPtInfoPtePerm=KPtInfoPtePerm; + iPtPtePerm=KPtPtePerm; + iPtPdePerm=KPtPdePerm; + iTempAddr=KTempAddr; + iSecondTempAddr=KSecondTempAddr; + iMapSizes=KPageSize|KLargePageSize|KChunkSize; + iRomLinearBase = ::RomHeaderAddress; + iRomLinearEnd = KRomLinearEnd; + iShadowPtePerm = KShadowPtePerm; + iShadowPdePerm = KShadowPdePerm; + + // Mmu data + TInt total_ram=TheSuperPage().iTotalRamSize; + +#if defined(__HAS_EXTERNAL_CACHE__) + //L2 cache on ARMv5 is always in write-back mode => must be always purged + iDecommitThreshold = CacheMaintenance::SyncAllPerformanceThresholdPages(); +#else + iDecommitThreshold = 0; ///no cache consistency issues on decommit +#endif + + iDataSectionBase = KDataSectionBase; + iDataSectionEnd = KDataSectionEnd; + iMaxDllDataSize=Min(total_ram/2, 0x08000000); // phys RAM/2 up to 128Mb + iMaxDllDataSize=(iMaxDllDataSize+iChunkMask)&~iChunkMask; // round up to chunk size + iMaxUserCodeSize=Min(total_ram, 0x10000000); // phys RAM up to 256Mb + iMaxUserCodeSize=(iMaxUserCodeSize+iChunkMask)&~iChunkMask; // round up to chunk size + iMaxKernelCodeSize=Min(total_ram/2, 0x04000000); // phys RAM/2 up to 64Mb + iMaxKernelCodeSize=(iMaxKernelCodeSize+iChunkMask)&~iChunkMask; // round up to chunk size + iPdeBase=KPageDirectoryBase; + iUserCodeLoadPtePerm=KUserCodeLoadPte; + iKernelCodePtePerm=KKernelCodeRunPte; + iDllDataBase = KDataSectionEnd - iMaxDllDataSize; + iUserCodeBase = KPageInfoLinearBase - iMaxUserCodeSize; + iKernelCodeBase = iUserCodeBase - iMaxKernelCodeSize; + + __KTRACE_OPT(KMMU,Kern::Printf("DDS %08x UCS %08x KCS %08x", iMaxDllDataSize, iMaxUserCodeSize, iMaxKernelCodeSize)); + __KTRACE_OPT(KMMU,Kern::Printf("DDB %08x KCB %08x UCB %08x RLB %08x", iDllDataBase, iKernelCodeBase, iUserCodeBase, iRomLinearBase)); + + // ArmMmu data + + // other + PP::MaxUserThreadStack=0x14000; // 80K - STDLIB asks for 64K for PosixServer!!!! + PP::UserThreadStackGuard=0x2000; // 8K + PP::MaxStackSpacePerProcess=0x200000; // 2Mb + K::SupervisorThreadStackSize=0x1000; // 4K + PP::SupervisorThreadStackGuard=0x1000; // 4K + K::MachineConfig=(TMachineConfig*)KMachineConfigLinAddr; + PP::RamDriveStartAddress=KRamDriveStartAddress; + PP::RamDriveRange=KRamDriveMaxSize; + PP::RamDriveMaxSize=KRamDriveMaxSize; // may be reduced later + + __KTRACE_OPT(KBOOT,Kern::Printf("K::MaxMemCopyInOneGo=0x%x",K::MaxMemCopyInOneGo)); + K::MemModelAttributes=EMemModelTypeMoving|EMemModelAttrNonExProt|EMemModelAttrKernProt|EMemModelAttrWriteProt| + EMemModelAttrVA|EMemModelAttrProcessProt|EMemModelAttrSameVA|EMemModelAttrSupportFixed| + EMemModelAttrSvKernProt|EMemModelAttrIPCKernProt; + + Arm::DefaultDomainAccess=KDefaultDomainAccess; + + // Domains 0-3 are preallocated + // 0=Variable user running, 1=Client, 2=Page tables, 3=RAM drive + Domains=(~(0xffffffffu<>KPageShift); + iSecondTempPte=PageTable(GetPageTableId(iSecondTempAddr))+((iSecondTempAddr&KChunkMask)>>KPageShift); + __KTRACE_OPT2(KBOOT,KMMU,Kern::Printf("iTempAddr=%08x, iTempPte=%08x, iSecondTempAddr=%08x, iSecondTempPte=%08x", iTempAddr, iTempPte, iSecondTempAddr, iSecondTempPte)); + CreateKernelSection(iKernelCodeBase, KPageShift); + iHomePdeMap=(TUint32*)Kern::AllocZ(-KSuperPageLinAddr>>KChunkShift<<2); + iHomePdeMap=(TUint32*)((TUint32)iHomePdeMap-(KSuperPageLinAddr>>KChunkShift<<2)); //adjust the pointer so it's indexed by address>>20 +#if defined(__CPU_WRITE_BACK_CACHE) +#if defined(__CPU_HAS_SINGLE_ENTRY_DCACHE_FLUSH) + if (InternalCache::Info[KCacheInfoD].iLineLength == 32) + iCopyPageFn = &::CopyPageForRemap32; + else if (InternalCache::Info[KCacheInfoD].iLineLength == 16) + iCopyPageFn = &::CopyPageForRemap16; + else + Panic(ENoCopyPageFunction); +#else +#error Write-back cache without single entry dcache flush is not supported +#endif +#else // !__CPU_HAS_WRITE_BACK_CACHE + iCopyPageFn = &::CopyPageForRemapWT; +#endif + Mmu::DoInit2(); + } + +#ifndef __MMU_MACHINE_CODED__ +void ArmMmu::MapRamPages(TInt aId, SPageInfo::TType aType, TAny* aPtr, TUint32 aOffset, const TPhysAddr* aPageList, TInt aNumPages, TPte aPtePerm) +// +// Map a list of physical RAM pages into a specified page table with specified PTE permissions. +// Update the page information array. +// Call this with the system locked. +// + { + __KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::MapRamPages() id=%d type=%d ptr=%08x off=%08x n=%d perm=%08x", + aId, aType, aPtr, aOffset, aNumPages, aPtePerm)); + + SPageTableInfo& ptinfo=iPtInfo[aId]; + ptinfo.iCount+=aNumPages; + aOffset>>=KPageShift; + TInt ptOffset=aOffset & KPagesInPDEMask; // entry number in page table + TPte* pPte=PageTable(aId)+ptOffset; // address of first PTE + while(aNumPages--) + { + TPhysAddr pa = *aPageList++; + *pPte++ = pa | aPtePerm; // insert PTE + __KTRACE_OPT(KMMU,Kern::Printf("Writing PTE %08x to %08x",pPte[-1],pPte-1)); + if (aType!=SPageInfo::EInvalid) + { + SPageInfo* pi = SPageInfo::SafeFromPhysAddr(pa); + if(pi) + { + pi->Set(aType,aPtr,aOffset); + __KTRACE_OPT(KMMU,Kern::Printf("I: %d %08x %08x",aType,aPtr,aOffset)); + ++aOffset; // increment offset for next page + } + } + } + __DRAIN_WRITE_BUFFER; + } + +void ArmMmu::MapPhysicalPages(TInt aId, SPageInfo::TType aType, TAny* aPtr, TUint32 aOffset, TPhysAddr aPhysAddr, TInt aNumPages, TPte aPtePerm) +// +// Map consecutive physical pages into a specified page table with specified PTE permissions. +// Update the page information array if RAM pages are being mapped. +// Call this with the system locked. +// + { + __KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::MapPhysicalPages() id=%d type=%d ptr=%08x off=%08x phys=%08x n=%d perm=%08x", + aId, aType, aPtr, aOffset, aPhysAddr, aNumPages, aPtePerm)); + SPageTableInfo& ptinfo=iPtInfo[aId]; + ptinfo.iCount+=aNumPages; + aOffset>>=KPageShift; + TInt ptOffset=aOffset & KPagesInPDEMask; // entry number in page table + TPte* pPte=PageTable(aId)+ptOffset; // address of first PTE + SPageInfo* pi; + if(aType==SPageInfo::EInvalid) + pi = NULL; + else + pi = SPageInfo::SafeFromPhysAddr(aPhysAddr); + while(aNumPages--) + { + *pPte++ = aPhysAddr|aPtePerm; // insert PTE + aPhysAddr+=KPageSize; + __KTRACE_OPT(KMMU,Kern::Printf("Writing PTE %08x to %08x",pPte[-1],pPte-1)); + if (pi) + { + pi->Set(aType,aPtr,aOffset); + __KTRACE_OPT(KMMU,Kern::Printf("I: %d %08x %08x",aType,aPtr,aOffset)); + ++aOffset; // increment offset for next page + ++pi; + } + } + __DRAIN_WRITE_BUFFER; + } + +void ArmMmu::MapVirtual(TInt aId, TInt aNumPages) +// +// Called in place of MapRamPages or MapPhysicalPages to update mmu data structures when committing +// virtual address space to a chunk. No pages are mapped. +// Call this with the system locked. +// + { + SPageTableInfo& ptinfo=iPtInfo[aId]; + ptinfo.iCount+=aNumPages; + } + +void ArmMmu::RemapPage(TInt aId, TUint32 aAddr, TPhysAddr aOldAddr, TPhysAddr aNewAddr, TPte aPtePerm, DProcess* /*aProcess*/) +// +// Replace the mapping at address aAddr in page table aId. +// Update the page information array for both the old and new pages. +// Call this with the system locked. +// + { + __KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::RemapPages() id=%d addr=%08x old=%08x new=%08x perm=%08x", aId, aAddr, aOldAddr, aNewAddr, aPtePerm)); + + TInt ptOffset=(aAddr&KChunkMask)>>KPageShift; // entry number in page table + TPte* pPte=PageTable(aId)+ptOffset; // address of PTE + TPte pte=*pPte; + + TUint pageType = (pte & KPteTypeMask); + if (pageType == KArmPteSmallPage || pageType == 0) + { + __ASSERT_ALWAYS((pte & KPteSmallPageAddrMask) == aOldAddr || pte==KPteNotPresentEntry, Panic(ERemapPageFailed)); + SPageInfo* oldpi = SPageInfo::FromPhysAddr(aOldAddr); + __ASSERT_DEBUG(oldpi->LockCount()==0,Panic(ERemapPageFailed)); + + // remap page + *pPte = aNewAddr | aPtePerm; // overwrite PTE + __KTRACE_OPT(KMMU,Kern::Printf("Writing PTE %08x to %08x",*pPte,pPte)); + __DRAIN_WRITE_BUFFER; + InvalidateTLBForPage(aAddr); // flush any corresponding TLB entry + + // update new pageinfo, clear old + SPageInfo* pi = SPageInfo::FromPhysAddr(aNewAddr); + pi->Set(oldpi->Type(),oldpi->Owner(),oldpi->Offset()); + oldpi->SetUnused(); + } + else + { + __KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::RemapPages() called on a non-4K page!")); + Panic(ERemapPageFailed); + } + } + +void ArmMmu::RemapKernelPage(TInt aId, TLinAddr aSrc, TLinAddr aDest, TPhysAddr aNewPhys, TPte aPtePerm) +// +// Replace the mapping at address aAddr in page table aId. +// Called with the system locked. +// MUST NOT INVOKE ANY TRACING - or do anything else that might touch the kernel heap +// We are depending on this not reintroducing any of the cache lines we previously +// invalidated. +// + { + TInt ptOffset=(aSrc&KChunkMask)>>KPageShift; // entry number in page table + TPte* pPte=PageTable(aId)+ptOffset; // address of PTE + + TInt irq = NKern::DisableAllInterrupts(); + CopyPageForRemap(aDest, aSrc); + *pPte = aNewPhys | aPtePerm; // overwrite PTE + __DRAIN_WRITE_BUFFER; + InvalidateTLBForPage(aSrc); // flush any corresponding TLB entry + NKern::RestoreInterrupts(irq); + } + +TInt ArmMmu::UnmapPages(TInt aId, TUint32 aAddr, TInt aNumPages, TPhysAddr* aPageList, TBool aSetPagesFree, TInt& aNumPtes, TInt& aNumFree, DProcess*) +// +// Unmap a specified area at address aAddr in page table aId. Place physical addresses of unmapped +// pages into aPageList, and count of unmapped pages into aNumPtes. +// Return number of pages still mapped using this page table. +// Call this with the system locked. +// @param aId Identifies Page Table to unmap PTEs(Page Table Entries) from. +// @param aAddr Base Base Virtual Address of the region to unmap. It (indirectly) specifies the first PTE in this Page Table to unmap. +// @param aNumPages The number of consecutive PTEs to unmap. +// @param aPageList Points to pre-allocated array. On return, it is filled in with the list of physical addresses of the unmapped 4K +// memory blocks. +// @param aSetPagesFree If true, pages a placed in the free state and only mapped pages are added +// to aPageList. +// @param aNumPtes On return, indicates how many PTEs are unmapped. +// @param aNumFree On return, holds the number are freed 4K memory blocks. Not updated if aSetPagesFree is false. +// @return The number of PTEs still mapped in this Page Table (aId). + { + __KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::UnmapPages() id=%d addr=%08x n=%d pl=%08x set-free=%d",aId,aAddr,aNumPages,aPageList,aSetPagesFree)); + TInt ptOffset=(aAddr&KChunkMask)>>KPageShift; // entry number in page table + TPte* pPte=PageTable(aId)+ptOffset; // address of first PTE + TInt np=0; + TInt nf=0; + while(aNumPages--) + { + TPte pte=*pPte; // get original PTE + *pPte++=0; // clear PTE + TUint pageType = (pte & KPteTypeMask); + if (pageType == KArmPteSmallPage) + InvalidateTLBForPage(aAddr); // flush any corresponding TLB entry + if (pageType == KArmPteSmallPage || (pageType == 0 && pte != KPteNotPresentEntry)) + { + ++np; // count unmapped pages + TPhysAddr pa=pte & KPteSmallPageAddrMask; // physical address of unmapped page + if (aSetPagesFree) + { + SPageInfo* pi = SPageInfo::FromPhysAddr(pa); + __NK_ASSERT_DEBUG(pageType == KArmPteSmallPage || + (pi->Type()==SPageInfo::EPagedCode && pi->State()==SPageInfo::EStatePagedOld)); + if(iRamCache->PageUnmapped(pi)) + { + pi->SetUnused(); // mark page as unused + if (pi->LockCount()==0) + { + *aPageList++=pa; // store in page list + ++nf; // count free pages + } + } + } + else + *aPageList++=pa; // store in page list + } + aAddr+=KPageSize; + } + aNumPtes=np; + aNumFree=nf; + SPageTableInfo& ptinfo=iPtInfo[aId]; + TInt r=(ptinfo.iCount-=np); + __DRAIN_WRITE_BUFFER; + __KTRACE_OPT(KMMU,Kern::Printf("Unmapped %d; Freed: %d; Return %08x",np,nf,r)); + return r; // return number of pages remaining in this page table + } +#endif + +TInt ArmMmu::UnmapVirtual(TInt aId, TUint32 aAddr, TInt aNumPages, TPhysAddr* aPageList, TBool aSetPagesFree, TInt& aNumPtes, TInt& aNumFree, DProcess* aProcess) +// +// Unmap a specified area at address aAddr in page table aId. Place physical addresses of unmapped +// pages into aPageList, and count of unmapped pages into aNumPtes. +// Adjust the page table reference count as if aNumPages pages were unmapped. +// Return number of pages still mapped using this page table. +// Call this with the system locked. +// + { + SPageTableInfo& ptinfo=iPtInfo[aId]; + TInt newCount = ptinfo.iCount - aNumPages; + UnmapPages(aId, aAddr, aNumPages, aPageList, aSetPagesFree, aNumPtes, aNumFree, aProcess); + ptinfo.iCount = newCount; + aNumPtes = aNumPages; + return newCount; + } + + +#ifndef __MMU_MACHINE_CODED__ +void ArmMmu::DoAssignPageTable(TInt aId, TLinAddr aAddr, TPde aPdePerm) +// +// Assign an allocated page table to map a given linear address with specified permissions. +// This should be called with the system locked and the MMU mutex held. +// + { + __KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::DoAssignPageTable %d to %08x perm %08x",aId,aAddr,aPdePerm)); + TLinAddr ptLin=PageTableLinAddr(aId); + TPhysAddr ptPhys=LinearToPhysical(ptLin); + TInt pdeIndex=TInt(aAddr>>KChunkShift); + PageDirectory[pdeIndex]=ptPhys|aPdePerm; + __KTRACE_OPT(KMMU,Kern::Printf("Writing PDE %08x to %08x", ptPhys|aPdePerm, PageDirectory+pdeIndex)); + __DRAIN_WRITE_BUFFER; + } + +void ArmMmu::RemapPageTable(TPhysAddr aOld, TPhysAddr aNew, TLinAddr aAddr) +// +// Replace a page table mapping the specified linear address. +// This should be called with the system locked and the MMU mutex held. +// + { + __KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::RemapPageTable %08x to %08x at %08x",aOld,aNew,aAddr)); + TInt pdeIndex=TInt(aAddr>>KChunkShift); + TPde pde=PageDirectory[pdeIndex]; + __ASSERT_ALWAYS((pde & KPdePageTableAddrMask) == aOld, Panic(ERemapPageTableFailed)); + TPde newPde=aNew|(pde&~KPdePageTableAddrMask); + PageDirectory[pdeIndex]=newPde; + __KTRACE_OPT(KMMU,Kern::Printf("Writing PDE %08x to %08x", newPde, PageDirectory+pdeIndex)); + __DRAIN_WRITE_BUFFER; + } + +void ArmMmu::DoUnassignPageTable(TLinAddr aAddr) +// +// Unassign a now-empty page table currently mapping the specified linear address. +// We assume that TLB and/or cache flushing has been done when any RAM pages were unmapped. +// This should be called with the system locked and the MMU mutex held. +// + { + __KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::DoUnassignPageTable at %08x",aAddr)); + TInt pdeIndex=TInt(aAddr>>KChunkShift); + PageDirectory[pdeIndex]=0; + __KTRACE_OPT(KMMU,Kern::Printf("Clearing PDE at %08x", PageDirectory+pdeIndex)); + __DRAIN_WRITE_BUFFER; + } +#endif + +// Initialise page table at physical address aXptPhys to be used as page table aXptId +// to expand the virtual address range used for mapping page tables. Map the page table +// at aPhysAddr as page table aId using the expanded range. +// Assign aXptPhys to kernel's Page Directory. +// Called with system unlocked and MMU mutex held. +void ArmMmu::BootstrapPageTable(TInt aXptId, TPhysAddr aXptPhys, TInt aId, TPhysAddr aPhysAddr) + { + __KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::BootstrapPageTable xptid=%04x, xptphys=%08x, id=%04x, phys=%08x", + aXptId, aXptPhys, aId, aPhysAddr)); + + // put in a temporary mapping for aXptPhys + // make it noncacheable + TPhysAddr pa=aXptPhys&~KPageMask; + *iTempPte = pa | SP_PTE(KArmV45PermRWNO, KMemAttNC); + __DRAIN_WRITE_BUFFER; + + // clear XPT + TPte* xpt=(TPte*)(iTempAddr+(aXptPhys&KPageMask)); + memclr(xpt, KPageTableSize); + + // must in fact have aXptPhys and aPhysAddr in same physical page + __ASSERT_ALWAYS( TUint32(aXptPhys^aPhysAddr)>KPtClusterShift)&KPagesInPDEMask] = pa | KPtPtePerm; + + // remove temporary mapping + *iTempPte=0; + __DRAIN_WRITE_BUFFER; + InvalidateTLBForPage(iTempAddr); + + // initialise PtInfo... + TLinAddr xptAddr = PageTableLinAddr(aXptId); + iPtInfo[aXptId].SetGlobal(xptAddr>>KChunkShift); + + // map xpt... + TInt pdeIndex=TInt(xptAddr>>KChunkShift); + NKern::LockSystem(); + PageDirectory[pdeIndex]=aXptPhys|KPtPdePerm; + __DRAIN_WRITE_BUFFER; + NKern::UnlockSystem(); + } + +// Edit the self-mapping entry in page table aId, mapped at aTempMap, to +// change the physical address from aOld to aNew. Used when moving page +// tables which were created by BootstrapPageTable. +// Called with system locked and MMU mutex held. +void ArmMmu::FixupXPageTable(TInt aId, TLinAddr aTempMap, TPhysAddr aOld, TPhysAddr aNew) + { + __KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::FixupXPageTable id=%04x, tempmap=%08x, old=%08x, new=%08x", + aId, aTempMap, aOld, aNew)); + + // find correct page table inside the page + TPte* xpt=(TPte*)(aTempMap + ((aId & KPtClusterMask) << KPageTableShift)); + // find the pte in that page table + xpt += (aId>>KPtClusterShift)&KPagesInPDEMask; + + // switch the mapping + __ASSERT_ALWAYS((*xpt&~KPageMask)==aOld, Panic(EFixupXPTFailed)); + *xpt = aNew | KPtPtePerm; + + // invalidate the TLB entry for the self-mapping page table + // the PDE has not yet been changed, but since we hold the + // system lock, nothing should bring this back into the TLB. + InvalidateTLBForPage(PageTableLinAddr(aId)); + } + +// Set up a page table (specified by aId) to map a 1Mb section of ROM containing aRomAddr +// using ROM at aOrigPhys. +void ArmMmu::InitShadowPageTable(TInt aId, TLinAddr aRomAddr, TPhysAddr aOrigPhys) + { + __KTRACE_OPT(KMMU, Kern::Printf("ArmMmu:InitShadowPageTable id=%04x aRomAddr=%08x aOrigPhys=%08x", + aId, aRomAddr, aOrigPhys)); + TPte* ppte = PageTable(aId); + TPte* ppte_End = ppte + KChunkSize/KPageSize; + TPhysAddr phys = aOrigPhys - (aRomAddr & KChunkMask); + for (; ppte>KChunkShift); + TPde newpde = ptPhys | KShadowPdePerm; + __KTRACE_OPT(KMMU,Kern::Printf("Writing PDE %08x to %08x", newpde, ppde)); + TInt irq=NKern::DisableAllInterrupts(); + *ppde = newpde; // map in the page table + __DRAIN_WRITE_BUFFER; // make sure new PDE written to main memory + FlushTLBs(); // flush both TLBs (no need to flush cache yet) + NKern::RestoreInterrupts(irq); + } + +void ArmMmu::DoUnmapShadowPage(TInt aId, TLinAddr aRomAddr, TPhysAddr aOrigPhys) + { + __KTRACE_OPT(KMMU,Kern::Printf("ArmMmu:DoUnmapShadowPage, id=%04x lin=%08x origphys=%08x", aId, aRomAddr, aOrigPhys)); + TPte* ppte = PageTable(aId) + ((aRomAddr & KChunkMask)>>KPageShift); + TPte newpte = aOrigPhys | KRomPtePermissions; + __KTRACE_OPT(KMMU,Kern::Printf("Writing PTE %08x to %08x", newpte, ppte)); + TInt irq=NKern::DisableAllInterrupts(); + *ppte = newpte; + __DRAIN_WRITE_BUFFER; + InvalidateTLBForPage(aRomAddr); + SyncCodeMappings(); + CacheMaintenance::CodeChanged(aRomAddr, KPageSize, CacheMaintenance::EMemoryRemap); + CacheMaintenance::PageToReuse(aRomAddr, EMemAttNormalCached, KPhysAddrInvalid); + NKern::RestoreInterrupts(irq); + } + +TInt ArmMmu::UnassignShadowPageTable(TLinAddr aRomAddr, TPhysAddr aOrigPhys) + { + __KTRACE_OPT(KMMU,Kern::Printf("ArmMmu:UnassignShadowPageTable, lin=%08x origphys=%08x", aRomAddr, aOrigPhys)); + TPde* ppde = PageDirectory + (aRomAddr>>KChunkShift); + TPde newpde = (aOrigPhys &~ KChunkMask) | KRomSectionPermissions; + __KTRACE_OPT(KMMU,Kern::Printf("Writing PDE %08x to %08x", newpde, ppde)); + TInt irq=NKern::DisableAllInterrupts(); + *ppde = newpde; // revert to section mapping + __DRAIN_WRITE_BUFFER; // make sure new PDE written to main memory + FlushTLBs(); // flush both TLBs + NKern::RestoreInterrupts(irq); + return KErrNone; + } + +void ArmMmu::DoFreezeShadowPage(TInt aId, TLinAddr aRomAddr) + { + __KTRACE_OPT(KMMU, Kern::Printf("ArmMmu:DoFreezeShadowPage aId=%04x aRomAddr=%08x", + aId, aRomAddr)); + TPte* ppte = PageTable(aId) + ((aRomAddr & KChunkMask)>>KPageShift); + TPte newpte = (*ppte & KPteSmallPageAddrMask) | KRomPtePermissions; + __KTRACE_OPT(KMMU,Kern::Printf("Writing PTE %08x to %08x", newpte, ppte)); + *ppte = newpte; + __DRAIN_WRITE_BUFFER; + InvalidateTLBForPage(aRomAddr); + } + +void ArmMmu::Pagify(TInt aId, TLinAddr aLinAddr) + { + __KTRACE_OPT(KMMU, Kern::Printf("ArmMmu:Pagify aId=%04x aLinAddr=%08x", aId, aLinAddr)); + + TInt pteIndex = (aLinAddr & KChunkMask)>>KPageShift; + TPte* pte = PageTable(aId); + if ((pte[pteIndex] & KPteTypeMask) == KArmV45PteLargePage) + { + __KTRACE_OPT(KMMU,Kern::Printf("Converting 64K page to 4K pages")); + pteIndex &= ~0xf; + TPte source = pte[pteIndex]; + source = (source & KPteLargePageAddrMask) | SP_PTE_FROM_LP_PTE(source); + pte += pteIndex; + for (TInt entry=0; entry<16; entry++) + { + pte[entry] = source | (entry<<12); + } + FlushTLBs(); + } + } + +void ArmMmu::FlushShadow(TLinAddr aRomAddr) + { + CacheMaintenance::CodeChanged(aRomAddr, KPageSize, CacheMaintenance::EMemoryRemap); + CacheMaintenance::PageToReuse(aRomAddr, EMemAttNormalCached, KPhysAddrInvalid); + InvalidateTLBForPage(aRomAddr); // remove all TLB references to original ROM page + SyncCodeMappings(); + } + + +inline void ZeroPdes(TLinAddr aBase, TLinAddr aEnd) + { + memclr(PageDirectory+(aBase>>KChunkShift), ((aEnd-aBase)>>KChunkShift)*sizeof(TPde)); + } + +void ArmMmu::ClearPageTable(TInt aId, TInt aFirstIndex) + { + __KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::ClearPageTable(%d,%d)",aId,aFirstIndex)); + TPte* pte=PageTable(aId); + memclr(pte+aFirstIndex, KPageTableSize-aFirstIndex*sizeof(TPte)); + __DRAIN_WRITE_BUFFER; + } + +void ArmMmu::ClearRamDrive(TLinAddr aStart) + { + // clear the page directory entries corresponding to the RAM drive + ZeroPdes(aStart, KRamDriveEndAddress); + __DRAIN_WRITE_BUFFER; + } + +void ArmMmu::ApplyTopLevelPermissions(TLinAddr aAddr, TUint aChunkSize, TPde aPdePerm) + { + __KTRACE_OPT(KMMU,Kern::Printf("ApplyTopLevelPermissions at %x",aAddr)); + TInt pdeIndex=aAddr>>KChunkShift; + TInt numPdes=(aChunkSize+KChunkMask)>>KChunkShift; + TPde* pPde=PageDirectory+pdeIndex; + while(numPdes--) + { + *pPde=(*pPde)?((*pPde & KPdePageTableAddrMask)|aPdePerm):0; + pPde++; + } + __DRAIN_WRITE_BUFFER; + } + +void ArmMmu::ApplyPagePermissions(TInt aId, TInt aPageOffset, TInt aNumPages, TPte aPtePerm) + { + __KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::ApplyPagePermissions %04x:%03x+%03x perm %08x", + aId, aPageOffset, aNumPages, aPtePerm)); + TPte* pPte=PageTable(aId)+aPageOffset; + TPde* pPteEnd=pPte+aNumPages; + NKern::LockSystem(); + for (; pPte>KChunkShift; + TInt iS=aInitAddr>>KChunkShift; + TInt iD=aFinalAddr>>KChunkShift; + TPde* pS=PageDirectory+iS; + TPde* pD=PageDirectory+iD; + while(numPdes--) + { + *pD++=(*pS)?((*pS & KPdePageTableAddrMask)|aPdePerm):0; + *pS++=KPdeNotPresentEntry; + } + __DRAIN_WRITE_BUFFER; + } + +void ArmMmu::MoveChunk(TLinAddr aInitAddr, TLinAddr aFinalAddr, TInt aNumPdes) +// +// Move a block of PDEs without changing permissions. Must work with overlapping initial and final +// regions. Call this with kernel locked. +// + { + __KTRACE_OPT(KMMU,Kern::Printf("MoveChunk at %08x to %08x numPdes %d", aInitAddr, aFinalAddr, aNumPdes)); + if (aInitAddr==aFinalAddr || aNumPdes==0) + return; + TInt iS=aInitAddr>>KChunkShift; + TInt iD=aFinalAddr>>KChunkShift; + TBool forwardOverlap=(iSiD && iS-iD>12); +const TUint BFNC=(EMapAttrBufferedNC>>12); +const TUint BUFC=(EMapAttrBufferedC>>12); +const TUint L1UN=(EMapAttrL1Uncached>>12); +const TUint WTRA=(EMapAttrCachedWTRA>>12); +const TUint WTWA=(EMapAttrCachedWTWA>>12); +const TUint WBRA=(EMapAttrCachedWBRA>>12); +const TUint WBWA=(EMapAttrCachedWBWA>>12); +const TUint AWTR=(EMapAttrAltCacheWTRA>>12); +const TUint AWTW=(EMapAttrAltCacheWTWA>>12); +const TUint AWBR=(EMapAttrAltCacheWBRA>>12); +const TUint AWBW=(EMapAttrAltCacheWBWA>>12); +const TUint MAXC=(EMapAttrL1CachedMax>>12); + +const TUint L2UN=(EMapAttrL2Uncached>>12); + +const TUint16 UNS=0xffffu; // Unsupported attribute +const TUint16 SPE=0xfffeu; // Special processing required + +#if defined(__CPU_ARM710T__) || defined(__CPU_ARM720T__) +// Original definition of C B +static const TUint16 CacheBuffAttributes[16]= + {0x00,0x00,0x04,0x04,0x0C,0x0C,0x0C,0x0C, UNS, UNS, UNS, UNS, UNS, UNS, UNS,0x0C}; +static const TUint8 CacheBuffActual[16]= + {FBLK,FBLK,BUFC,BUFC,WTRA,WTRA,WTRA,WTRA,FBLK,FBLK,FBLK,FBLK,FBLK,FBLK,FBLK,WTRA}; + +#elif defined(__CPU_ARM920T__) || defined(__CPU_ARM925T__) || defined(__CPU_ARM926J__) +// Newer definition of C B +static const TUint16 CacheBuffAttributes[16]= + {0x00,0x00,0x04,0x04,0x08,0x08,0x0C,0x0C, UNS, UNS, UNS, UNS, UNS, UNS, UNS,0x0C}; +static const TUint8 CacheBuffActual[16]= + {FBLK,FBLK,BUFC,BUFC,WTRA,WTRA,WBRA,WBRA,FBLK,FBLK,FBLK,FBLK,FBLK,FBLK,FBLK,WBRA}; + +#elif defined(__CPU_SA1__) +// Special definition of C B +static const TUint16 CacheBuffAttributes[16]= + {0x00,0x00,0x04,0x04,0x04,0x04,0x0C,0x0C,0x04,0x04,0x08,0x08, UNS, UNS, UNS,0x0C}; +static const TUint8 CacheBuffActual[16]= + {FBLK,FBLK,BUFC,BUFC,BUFC,BUFC,WBRA,WBRA,FBLK,FBLK,AWBR,AWBR,FBLK,FBLK,FBLK,WBRA}; + +#elif defined(__CPU_XSCALE__) +#ifdef __CPU_XSCALE_MANZANO__ +#ifdef __HAS_EXTERNAL_CACHE__ +// ***MANZANO with L2 cache****** // + +//Specifies TEX::CB bits for different L1/L2 cache attributes +// ...876543201 +// ...TEX..CB.. +static const TUint16 CacheBuffAttributes[80]= + { // L1CACHE: +// FBLK BFNC BUFC L1UN WTRA WTWA WBRA WBWA AWTR AWTW AWBR AWBT UNS UNS UNS MAX L2CACHE: + 0x00, 0x44, 0x40, 0x40, 0x108, 0x108, 0x10c, 0x10c, SPE, SPE, SPE, SPE, UNS,UNS,UNS,0x10c, //NC + 0x00, 0x44, 0x40, 0x40, 0x108, 0x108, 0x10c, 0x10c, SPE, SPE, SPE, SPE, UNS,UNS,UNS,0x10c, //WTRA + 0x00, 0x44, 0x40, 0x40, 0x108, 0x108, 0x10c, 0x10c, SPE, SPE, SPE, SPE, UNS,UNS,UNS,0x10c, //WTWA + 0x00, 0x44, 0x40, 0x140, 0x148, 0x148, 0x14c, 0x14c, SPE, SPE, SPE, SPE, UNS,UNS,UNS,0x14c, //WBRA + 0x00, 0x44, 0x40, 0x140, 0x148, 0x148, 0x14c, 0x14c, SPE, SPE, SPE, SPE, UNS,UNS,UNS,0x14c, //WBWA + }; + +extern TUint MiniCacheConfig(); +//Converts page table attributes(TEX:CB) into appropriate cache attributes. +TInt CacheAttributesActual(TUint& cacheL1, TUint& cacheL2, TUint cbatt) + { + switch (cbatt) + { + case 0: cacheL1 = FBLK; cacheL2 = L2UN; return KErrNone; + case 0x40: cacheL1 = L1UN; cacheL2 = L2UN; return KErrNone; + case 0x44: cacheL1 = BFNC; cacheL2 = L2UN; return KErrNone; + case 0x48: cacheL1 = MiniCacheConfig(); cacheL2 = L2UN; return KErrNone; + case 0x108: cacheL1 = WTRA; cacheL2 = L2UN; return KErrNone; + case 0x10c: cacheL1 = WBRA; cacheL2 = L2UN; return KErrNone; + case 0x140: cacheL1 = L1UN; cacheL2 = WBWA; return KErrNone; + case 0x148: cacheL1 = WTRA; cacheL2 = WBWA; return KErrNone; + case 0x14c: cacheL1 = WBRA; cacheL2 = WBWA; return KErrNone; + } + return KErrNotSupported; + } +#else //__HAS_EXTERNAL_CACHE__ +// ***MANZANO without L2 cache****** // + +static const TUint16 CacheBuffAttributes[16]= +// FBLK BFNC BUFC L1UN WTRA WTWA WBRA WBWA -----------AltCache-------- MAXC + {0x00,0x44,0x40,0x40,0x148,0x148,0x14C,0x14C,SPE,SPE,SPE,SPE,UNS,UNS,UNS,0x14C}; +static const TUint8 CacheBuffActual[16]= + {FBLK,BFNC,BUFC,BUFC,WTRA,WTRA,WBRA,WBRA,FBLK,FBLK,FBLK,FBLK,FBLK,FBLK,FBLK,WBRA}; +#endif //__HAS_EXTERNAL_CACHE__ + +#else +// ***XSCALE that is not MANZANO (no L2 cache)****** // + +// X C B +static const TUint16 CacheBuffAttributes[16]= + {0x00,0x44,0x04,0x04,0x08,0x08,0x0C,0x4C,SPE,SPE,SPE,SPE,UNS,UNS,UNS,0x4C}; +static const TUint8 CacheBuffActual[16]= + {FBLK,BFNC,BUFC,BUFC,WTRA,WTRA,WBRA,WBWA,FBLK,FBLK,FBLK,FBLK,FBLK,FBLK,FBLK,WBWA}; +#endif + +// ***Common code for all XSCALE cores*** // + +extern TUint MiniCacheConfig(); +void ProcessSpecialCacheAttr(TUint& cache, TUint& cbatt) + { + // If writeback requested, give writeback or writethrough + // If writethrough requested, give writethrough or uncached + // Give other allocation policy if necessary. + TUint mccfg=MiniCacheConfig(); + __KTRACE_OPT(KMMU,Kern::Printf("MiniCacheConfig: %x",mccfg)); + + if (cache=AWBR) // asked for WT but cache is set for WB + { + cache=BUFC; // so give uncached, buffered, coalescing + #if defined (__CPU_XSCALE_MANZANO__) + cbatt=0x40; + #else + cbatt=0x04; + #endif + } + else + { + cache=mccfg; // give whatever minicache is configured for + cbatt=0x48; // minicache attributes + } + } +#endif + +static const TUint8 ActualReadPrivilegeLevel[4]={4,1,4,4}; // RORO,RWNO,RWRO,RWRW +static const TUint8 ActualWritePrivilegeLevel[4]={0,1,1,4}; // RORO,RWNO,RWRO,RWRW + +/** Calculates cb attributes for page table and sets actual cache attributes*/ +TInt GetCacheAttr(TUint& cacheL1, TUint& cacheL2, TUint& cbatt) + { + TInt r = KErrNone; + // Scale down L2 to 0-4 : NC, WTRA, WTWA, WBRA, WBWA +#if defined (__CPU_XSCALE_MANZANO__) && defined(__HAS_EXTERNAL_CACHE__) + if (cacheL2 == MAXC) cacheL2 = WBWA-3; // Scale down L2 cache attributes... + else if (cacheL2 > WBWA) return KErrNotSupported; // ... to 0-4 for... + else if (cacheL2 < WTRA) cacheL2 = L2UN; // ... L2UN to WBWA + else cacheL2-=3; // +#else + cacheL2 = 0; // Either no L2 cache or L2 cache attributes will be just a copy of L1 cache attributes. +#endif + + //Get cb page attributes. (On some platforms, tex bits are includded as well.) + cbatt = CacheBuffAttributes[cacheL1 + (cacheL2<<4)]; + __KTRACE_OPT(KMMU,Kern::Printf("GetCacheAttr, table returned:%x",cbatt)); + +#if defined(__CPU_XSCALE__) + //Check if altDCache/LLR cache attributes are defined + if (cbatt == SPE) + { + cacheL2 = 0; //Not L2 cached in such case + ProcessSpecialCacheAttr(cacheL1,cbatt); + __KTRACE_OPT(KMMU,Kern::Printf("GetCacheAttr, spec case returned:%x",cbatt)); + } +#endif + + if(cbatt == UNS) + return KErrNotSupported; + + //W Got CB page attributes. Now, find out what are the actual cache attributes. +#if defined(__CPU_XSCALE_MANZANO__) && defined(__HAS_EXTERNAL_CACHE__) + r = CacheAttributesActual(cacheL1, cacheL2, cbatt); +#else + cacheL1 = CacheBuffActual[cacheL1]; +#if defined(__HAS_EXTERNAL_CACHE__) + cacheL2 = cacheL1; +#else + cacheL2 = 0; +#endif +#endif + return r; + } + +TInt ArmMmu::PdePtePermissions(TUint& aMapAttr, TPde& aPde, TPte& aPte) + { + __KTRACE_OPT(KMMU,Kern::Printf(">ArmMmu::PdePtePermissions, mapattr=%08x",aMapAttr)); + TUint read=aMapAttr & EMapAttrReadMask; + TUint write=(aMapAttr & EMapAttrWriteMask)>>4; + TUint exec=(aMapAttr & EMapAttrExecMask)>>8; + + // if execute access is greater than read, adjust read (since there are no separate execute permissions on ARM) + if (exec>read) + read=exec; + TUint ap; + if (write==0) + { + // read-only + if (read>=4) + ap=KArmV45PermRORO; // user and supervisor read-only + else + ap=KArmV45PermRWNO; // supervisor r/w user no access (since no RO/NO access is available) + } + else if (write<4) + { + // only supervisor can write + if (read>=4) + ap=KArmV45PermRWRO; // supervisor r/w user r/o + else + ap=KArmV45PermRWNO; // supervisor r/w user no access + } + else + ap=KArmV45PermRWRW; // supervisor r/w user r/w + read=ActualReadPrivilegeLevel[ap]; + write=ActualWritePrivilegeLevel[ap]; +#ifndef __CPU_USE_MMU_TEX_FIELD + ap|=(ap<<2); + ap|=(ap<<4); // replicate permissions in all four subpages +#endif + ap<<=4; // shift access permissions into correct position for PTE + ap|=KArmPteSmallPage; // add in mandatory small page bits + + // Get cb atributes for the page table and the actual cache attributes + TUint cbatt; + TUint cacheL1=(aMapAttr & EMapAttrL1CacheMask)>>12; + TUint cacheL2=(aMapAttr & EMapAttrL2CacheMask)>>16; + TInt r = GetCacheAttr(cacheL1, cacheL2, cbatt); + + if (r==KErrNone) + { + aPde=PT_PDE(EDomainClient); + aPte=ap|cbatt; + aMapAttr=read|(write<<4)|(read<<8)|(cacheL1<<12)|(cacheL2<<16); + } + __KTRACE_OPT(KMMU,Kern::Printf("=KChunkShift && (la & KChunkMask)==0 && remain>=KChunkSize) + { + // use sections + TInt npdes = remain>>KChunkShift; + TPde* p_pde = PageDirectory + (la>>KChunkShift); + TPde* p_pde_E = p_pde + npdes; + TPde pde = pa|section_pde; + NKern::LockSystem(); + for (; p_pde < p_pde_E; pde+=KChunkSize) + { + __ASSERT_DEBUG(*p_pde==0, MM::Panic(MM::EPdeAlreadyInUse)); + __KTRACE_OPT(KMMU,Kern::Printf("Writing PDE %08x to %08x", pde, p_pde)); + *p_pde++=pde; + } + NKern::UnlockSystem(); + npdes<<=KChunkShift; + la+=npdes, pa+=npdes, remain-=npdes; + continue; + } + TInt block_size = Min(remain, KChunkSize-(la&KChunkMask)); + TPte pa_mask=~KPageMask; + TPte pte_perm=sp_pte; + if (aMapShift>=KLargePageShift && block_size>=KLargePageSize) + { + if ((la & KLargePageMask)==0) + { + // use 64K large pages + pa_mask=~KLargePageMask; + pte_perm=lp_pte; + } + else + block_size = Min(remain, KLargePageSize-(la&KLargePageMask)); + } + block_size &= pa_mask; + + // use pages (large or small) + TInt id = PageTableId(la); + __ASSERT_DEBUG(id>=0, MM::Panic(MM::EMmuMapNoPageTable)); + TPte* p_pte = PageTable(id) + ((la&KChunkMask)>>KPageShift); + TPte* p_pte_E = p_pte + (block_size>>KPageShift); + SPageTableInfo& ptinfo = iPtInfo[id]; + NKern::LockSystem(); + for (; p_pte < p_pte_E; pa+=KPageSize) + { + __ASSERT_DEBUG(*p_pte==0, MM::Panic(MM::EPteAlreadyInUse)); + TPte pte = (pa & pa_mask) | pte_perm; + __KTRACE_OPT(KMMU,Kern::Printf("Writing PTE %08x to %08x", pte, p_pte)); + *p_pte++=pte; + ++ptinfo.iCount; + NKern::FlashSystem(); + } + NKern::UnlockSystem(); + la+=block_size, remain-=block_size; + } + } + +void ArmMmu::Unmap(TLinAddr aLinAddr, TInt aSize) +// +// Remove all mappings in the specified range of addresses. +// Assumes there are only global mappings involved. +// Don't free page tables. +// aLinAddr, aSize must be page-aligned. +// + { + __KTRACE_OPT(KMMU, Kern::Printf("ArmMmu::Unmap lin=%08x size=%08x", aLinAddr, aSize)); + TLinAddr a=aLinAddr; + TLinAddr end=a+aSize; + __KTRACE_OPT(KMMU,Kern::Printf("a=%08x end=%08x",a,end)); + NKern::LockSystem(); + while(a!=end) + { + TInt pdeIndex=a>>KChunkShift; + TLinAddr next=(pdeIndex<>KPageShift; + __KTRACE_OPT(KMMU,Kern::Printf("a=%08x next=%08x to_do=%d",a,next,to_do)); + TPde pde = PageDirectory[pdeIndex]; + if ( (pde&KPdePresentMask)==KArmV45PdeSection ) + { + __ASSERT_DEBUG(!(a&KChunkMask), MM::Panic(MM::EUnmapBadAlignment)); + PageDirectory[pdeIndex]=0; + InvalidateTLBForPage(a); + a=next; + NKern::FlashSystem(); + continue; + } + TInt ptid = GetPageTableId(a); + SPageTableInfo& ptinfo=iPtInfo[ptid]; + if (ptid>=0) + { + TPte* ppte = PageTable(ptid) + ((a&KChunkMask)>>KPageShift); + TPte* ppte_End = ppte + to_do; + for (; ppte=0 && aDomain= 0) + { + TPhysAddr pa; + if((TInt)aPageList&1) + { + pa = (TPhysAddr)aPageList&~1; + *(TPhysAddr*)&aPageList += iPageSize; + } + else + pa = *aPageList++; + *iTempPte = pa | SP_PTE(KArmV45PermRWNO, KMemAttBuf); + __DRAIN_WRITE_BUFFER; + InvalidateTLBForPage(iTempAddr); + memset((TAny*)iTempAddr, aClearByte, iPageSize); + } + *iTempPte=0; + __DRAIN_WRITE_BUFFER; + InvalidateTLBForPage(iTempAddr); + } + +TLinAddr DoMapTemp(TPhysAddr aPage, TBool aCached, TLinAddr aTempAddr, TPte* aTempPte) + { + __ASSERT_DEBUG(!*aTempPte,MM::Panic(MM::ETempMappingAlreadyInUse)); + *aTempPte = (aPage&~KPageMask) | SP_PTE(KArmV45PermRWNO, aCached?KDefaultCaching:KMemAttBuf); + __DRAIN_WRITE_BUFFER; + return aTempAddr; + } + +/** +Create a temporary mapping of a physical page. +The RamAllocatorMutex must be held before this function is called and not released +until after UnmapTemp has been called. + +@param aPage The physical address of the page to be mapped. +@param aCached Whether to map the page cached or not. + +@return The linear address of where the page has been mapped. +*/ +TLinAddr ArmMmu::MapTemp(TPhysAddr aPage, TBool aCached) + { + __ASSERT_MUTEX(RamAllocatorMutex); + return DoMapTemp(aPage, aCached, iTempAddr, iTempPte); + } + +/** +Create a temporary mapping of a physical page, distinct from that created by MapTemp. +The RamAllocatorMutex must be held before this function is called and not released +until after UnmapSecondTemp has been called. + +@param aPage The physical address of the page to be mapped. +@param aCached Whether to map the page cached or not. + +@return The linear address of where the page has been mapped. +*/ +TLinAddr ArmMmu::MapSecondTemp(TPhysAddr aPage, TBool aCached) + { + __ASSERT_MUTEX(RamAllocatorMutex); + return DoMapTemp(aPage, aCached, iSecondTempAddr, iSecondTempPte); + } + +void DoUnmapTemp(TLinAddr aTempAddr, TPte* aTempPte) + { + *aTempPte = 0; + __DRAIN_WRITE_BUFFER; + InvalidateTLBForPage(aTempAddr); + } + +/** +Remove the temporary mapping created with MapTemp. +*/ +void ArmMmu::UnmapTemp() + { + __ASSERT_MUTEX(RamAllocatorMutex); + DoUnmapTemp(iTempAddr, iTempPte); + } + +/** +Remove the temporary mapping created with MapSecondTemp. +*/ +void ArmMmu::UnmapSecondTemp() + { + __ASSERT_MUTEX(RamAllocatorMutex); + DoUnmapTemp(iSecondTempAddr, iSecondTempPte); + } + +/* + * Performs cache maintenance on physical cache (VIPT & PIPT) for a page to be reused. + */ +void ArmMmu::CacheMaintenanceOnDecommit(TPhysAddr aAddr) + { + CacheMaintenance::PageToReusePhysicalCache(aAddr); + } + +void ArmMmu::CacheMaintenanceOnDecommit(const TPhysAddr* aAddr, TInt aCount) + { + while (--aCount>=0) + ArmMmu::CacheMaintenanceOnDecommit(*aAddr++); + } + +void ArmMmu::CacheMaintenanceOnPreserve(TPhysAddr, TUint) + { + //Not required for moving memory model + __ASSERT_ALWAYS(0, Panic(ECacheMaintenance)); + } + +void ArmMmu::CacheMaintenanceOnPreserve(const TPhysAddr*, TInt, TUint) + { + //Not required for moving memory model + __ASSERT_ALWAYS(0, Panic(ECacheMaintenance)); + } + +void ArmMmu::CacheMaintenanceOnPreserve(TPhysAddr , TInt , TLinAddr , TUint ) + { + //Not required for moving memory model + __ASSERT_ALWAYS(0, Panic(ECacheMaintenance)); + } + + +TInt ArmMmu::UnlockRamCachePages(TUint8* volatile & aBase, TInt aStartPage, TInt aNumPages) + { + NKern::LockSystem(); + for(;;) + { + TInt page = ((TLinAddr)aBase>>KPageShift)+aStartPage; + TPde* pd = PageDirectory+(page>>(KChunkShift-KPageShift)); + TPte* pt = SafePageTableFromPde(*pd++); + TInt pteIndex = page&(KChunkMask>>KPageShift); + if(!pt) + { + // whole page table has gone, so skip all pages in it... + TInt pagesInPt = (KChunkSize>>KPageShift)-pteIndex; + aNumPages -= pagesInPt; + aStartPage += pagesInPt; + if(aNumPages>0) + continue; + NKern::UnlockSystem(); + return KErrNone; + } + pt += pteIndex; + do + { + TInt pagesInPt = (KChunkSize>>KPageShift)-pteIndex; + if(pagesInPt>aNumPages) + pagesInPt = aNumPages; + if(pagesInPt>KMaxPages) + pagesInPt = KMaxPages; + + aNumPages -= pagesInPt; + aStartPage += pagesInPt; + + do + { + TPte pte = *pt++; + if(pte!=KPteNotPresentEntry) // pte may be null if page has already been unlocked and reclaimed by system + iRamCache->DonateRamCachePage(SPageInfo::FromPhysAddr(pte)); + } + while(--pagesInPt); + + if(!aNumPages) + { + NKern::UnlockSystem(); + return KErrNone; + } + + pteIndex = aStartPage&(KChunkMask>>KPageShift); + } + while(!NKern::FlashSystem() && pteIndex); + } + } + + +TInt ArmMmu::LockRamCachePages(TUint8* volatile & aBase, TInt aStartPage, TInt aNumPages) + { + NKern::LockSystem(); + for(;;) + { + TInt page = ((TLinAddr)aBase>>KPageShift)+aStartPage; + TPde* pd = PageDirectory+(page>>(KChunkShift-KPageShift)); + TPte* pt = SafePageTableFromPde(*pd++); + TInt pteIndex = page&(KChunkMask>>KPageShift); + if(!pt) + goto not_found; + pt += pteIndex; + do + { + TInt pagesInPt = (KChunkSize>>KPageShift)-pteIndex; + if(pagesInPt>aNumPages) + pagesInPt = aNumPages; + if(pagesInPt>KMaxPages) + pagesInPt = KMaxPages; + + aNumPages -= pagesInPt; + aStartPage += pagesInPt; + + do + { + TPte pte = *pt++; + if(pte==KPteNotPresentEntry) + goto not_found; + if(!iRamCache->ReclaimRamCachePage(SPageInfo::FromPhysAddr(pte))) + goto not_found; + } + while(--pagesInPt); + + if(!aNumPages) + { + NKern::UnlockSystem(); + return KErrNone; + } + + pteIndex = aStartPage&(KChunkMask>>KPageShift); + } + while(!NKern::FlashSystem() && pteIndex); + } +not_found: + NKern::UnlockSystem(); + return KErrNotFound; + } + + +void RamCache::SetFree(SPageInfo* aPageInfo) + { + // Make a page free + SPageInfo::TType type = aPageInfo->Type(); + if(type==SPageInfo::EPagedCache) + { + TInt offset = aPageInfo->Offset()<Owner(); + __NK_ASSERT_DEBUG(TUint(offset)iMaxSize)); + TLinAddr lin = ((TLinAddr)chunk->iBase)+offset; + TPte* pt = PtePtrFromLinAddr(lin); + *pt = KPteNotPresentEntry; + __DRAIN_WRITE_BUFFER; + InvalidateTLBForPage(lin); + ((ArmMmu*)iMmu)->SyncCodeMappings(); + CacheMaintenance::PageToReuseVirtualCache(lin); + // actually decommit it from chunk... + TInt ptid = ((TLinAddr)pt-KPageTableBase)>>KPageTableShift; + SPageTableInfo& ptinfo=((ArmMmu*)iMmu)->iPtInfo[ptid]; + if(!--ptinfo.iCount) + { + ((ArmMmu*)iMmu)->DoUnassignPageTable(lin); + chunk->RemovePde(offset); + NKern::UnlockSystem(); + ((ArmMmu*)iMmu)->FreePageTable(ptid); + NKern::LockSystem(); + } + } + else + { + __KTRACE_OPT2(KPAGING,KPANIC,Kern::Printf("DP: SetFree() with bad page type = %d",aPageInfo->Type())); + Panic(EUnexpectedPageType); + } + } + + +// +// MemModelDemandPaging +// + +class MemModelDemandPaging : public DemandPaging + { +public: + // From RamCacheBase + virtual void Init2(); + virtual TInt Init3(); + virtual TBool PageUnmapped(SPageInfo* aPageInfo); + // From DemandPaging + virtual TInt Fault(TAny* aExceptionInfo); + virtual void SetOld(SPageInfo* aPageInfo); + virtual void SetFree(SPageInfo* aPageInfo); + virtual void NotifyPageFree(TPhysAddr aPage); + virtual TInt EnsurePagePresent(TLinAddr aPage, DProcess* aProcess); + virtual TPhysAddr LinearToPhysical(TLinAddr aPage, DProcess* aProcess); + virtual void AllocLoadAddress(DPagingRequest& aReq, TInt aDeviceId); + virtual TInt PageState(TLinAddr aAddr); + virtual TBool NeedsMutexOrderCheck(TLinAddr aStartAddr, TUint aLength); + // New + inline ArmMmu& Mmu() { return (ArmMmu&)*iMmu; } + void InitRomPaging(); + void InitCodePaging(); + TInt HandleFault(TArmExcInfo& aExc, TLinAddr aFaultAddress, TBool aInRom); + TInt PageIn(TLinAddr aAddress, DMemModelCodeSegMemory* aCodeSegMemory); +private: + TLinAddr GetLinearAddress(SPageInfo* aPageInfo); + }; + + +// +// MemModelDemandPaging +// + + +DemandPaging* DemandPaging::New() + { + return new MemModelDemandPaging(); + } + + +void MemModelDemandPaging::Init2() + { + __KTRACE_OPT2(KPAGING,KBOOT,Kern::Printf(">MemModelDemandPaging::Init2")); + DemandPaging::Init2(); + __KTRACE_OPT2(KPAGING,KBOOT,Kern::Printf("iLinAddr; + + if(RomPagingRequested()) + InitRomPaging(); + + if (CodePagingRequested()) + InitCodePaging(); + + __KTRACE_OPT2(KPAGING,KBOOT,Kern::Printf("=0); + Mmu().PtInfo(ptid).SetGlobal(lin >> KChunkShift); + } + + // Get new page table addresses + TPte* pt = PageTable(ptid); + TPhysAddr ptPhys=Mmu().LinearToPhysical((TLinAddr)pt); + + // Pointer to page dirctory entry + TPde* ppde = PageDirectory + (lin>>KChunkShift); + + // Fill in Page Table + TPte* ptEnd = pt+(1<<(KChunkShift-KPageShift)); + pt += (lin&KChunkMask)>>KPageShift; + do + { + if(linType(); + + if(type!=SPageInfo::EPagedCache && type!=SPageInfo::EPagedCode) + { + __NK_ASSERT_DEBUG(type!=SPageInfo::EPagedData); // not supported yet + return ETrue; + } + + RemovePage(aPageInfo); + AddAsFreePage(aPageInfo); + // Return false to stop DMemModelChunk::DoDecommit from freeing this page + return EFalse; + } + + +TLinAddr MemModelDemandPaging::GetLinearAddress(SPageInfo* aPageInfo) + { + TInt offset = aPageInfo->Offset()<Type(); + __NK_ASSERT_DEBUG(TUint(offset)<(type==SPageInfo::EPagedROM ? iRomSize : iCodeSize)); + TLinAddr base = type==SPageInfo::EPagedROM ? iRomLinearBase : iCodeLinearBase; + return base + offset; + } + + +void MemModelDemandPaging::SetOld(SPageInfo* aPageInfo) + { + __NK_ASSERT_DEBUG(aPageInfo->State() == SPageInfo::EStatePagedOld); + SPageInfo::TType type = aPageInfo->Type(); + + if(type==SPageInfo::EPagedROM || type==SPageInfo::EPagedCode) + { + START_PAGING_BENCHMARK; + + // get linear address of page... + TLinAddr lin = GetLinearAddress(aPageInfo); + + // make page inaccessible... + TPte* pt = PtePtrFromLinAddr(lin); + *pt &= ~KPtePresentMask; + __DRAIN_WRITE_BUFFER; + InvalidateTLBForPage(lin); + Mmu().SyncCodeMappings(); + + if (type==SPageInfo::EPagedCode) + END_PAGING_BENCHMARK(this, EPagingBmSetCodePageOld); + } + else if(type==SPageInfo::EPagedCache) + { + // leave page accessible + } + else if(type!=SPageInfo::EPagedFree) + { + __KTRACE_OPT2(KPAGING,KPANIC,Kern::Printf("DP: SetOld() with bad page type = %d",aPageInfo->Type())); + Panic(EUnexpectedPageType); + } + NKern::FlashSystem(); + } + + +void MemModelDemandPaging::SetFree(SPageInfo* aPageInfo) + { + __ASSERT_SYSTEM_LOCK; + __ASSERT_MUTEX(MmuBase::RamAllocatorMutex); + __NK_ASSERT_DEBUG(aPageInfo->State() == SPageInfo::EStatePagedDead); + if(aPageInfo->LockCount()) + Panic(ERamPageLocked); + + SPageInfo::TType type = aPageInfo->Type(); + + if(type==SPageInfo::EPagedROM || type==SPageInfo::EPagedCode) + { + START_PAGING_BENCHMARK; + + // get linear address of page... + TLinAddr lin = GetLinearAddress(aPageInfo); + + // unmap it... + TPte* pt = PtePtrFromLinAddr(lin); + *pt = KPteNotPresentEntry; + __DRAIN_WRITE_BUFFER; + InvalidateTLBForPage(lin); + Mmu().SyncCodeMappings(); + + if (type==SPageInfo::EPagedCode) + END_PAGING_BENCHMARK(this, EPagingBmSetCodePageFree); +#ifdef BTRACE_PAGING + TInt subCat = type==SPageInfo::EPagedCode ? BTrace::EPagingPageOutCode : BTrace::EPagingPageOutROM; + TPhysAddr phys = aPageInfo->PhysAddr(); + BTraceContext8(BTrace::EPaging,subCat,phys,lin); +#endif + } + else if(type==SPageInfo::EPagedCache) + { + // get linear address of page... + TInt offset = aPageInfo->Offset()<Owner(); + __NK_ASSERT_DEBUG(TUint(offset)iMaxSize)); + TLinAddr lin = ((TLinAddr)chunk->iBase)+offset; + + // unmap it... + TPte* pt = PtePtrFromLinAddr(lin); + *pt = KPteNotPresentEntry; + __DRAIN_WRITE_BUFFER; + InvalidateTLBForPage(lin); + Mmu().SyncCodeMappings(); + NKern::UnlockSystem(); + CacheMaintenance::PageToReuseVirtualCache(lin); + NKern::LockSystem(); + + // actually decommit it from chunk... + TInt ptid = ((TLinAddr)pt-KPageTableBase)>>KPageTableShift; + SPageTableInfo& ptinfo=((ArmMmu*)iMmu)->iPtInfo[ptid]; + if(!--ptinfo.iCount) + { + ((ArmMmu*)iMmu)->DoUnassignPageTable(lin); + chunk->RemovePde(offset); + NKern::UnlockSystem(); + ((ArmMmu*)iMmu)->FreePageTable(ptid); + NKern::LockSystem(); + } + +#ifdef BTRACE_PAGING + TPhysAddr phys = aPageInfo->PhysAddr(); + BTraceContext8(BTrace::EPaging,BTrace::EPagingPageOutCache,phys,lin); +#endif + } + else if(type==SPageInfo::EPagedFree) + { + // already free... +#ifdef BTRACE_PAGING + TPhysAddr phys = aPageInfo->PhysAddr(); + BTraceContext4(BTrace::EPaging,BTrace::EPagingPageOutFree,phys); +#endif + // external cache may not have been cleaned if PageUnmapped called + CacheMaintenance::PageToReusePhysicalCache(aPageInfo->PhysAddr()); + } + else + { + __KTRACE_OPT2(KPAGING,KPANIC,Kern::Printf("DP: SetFree() with bad page type = %d",aPageInfo->Type())); + Panic(EUnexpectedPageType); + } + NKern::FlashSystem(); + } + + +void MemModelDemandPaging::NotifyPageFree(TPhysAddr aPage) + { + MM::Panic(MM::EOperationNotImplemented); + } + + +/** +Return True if exception was caused by a memory write access. +This function can cause a paging exception! +*/ +static TBool FaultDuringWrite(TArmExcInfo& aExc) + { + // We can't decode jazelle instruction to determine if they faulted during a read. + // Therefore we will treat them as writes (which will panic the thread)... + if(aExc.iCpsr&(1<<24)) + return ETrue; + + if(aExc.iCpsr&(1<<5)) + { + // thumb + TUint32 op = *(TUint16*)aExc.iR15; + switch((op>>13)&7) + { + case 2: + if((op&0xfa00)==0x5000) + return ETrue; // STR (2) and STRB (2) + if((op&0xfe00)==0x5200) + return ETrue; // STRH (2) + return EFalse; + case 3: + return !(op&(1<<11)); // STR (1) and STRB (1) + case 4: + return !(op&(1<<11)); // STR (3) and STRH (1) + case 5: + return (op&0xfe00)==0xb400; // PUSH + case 6: + return (op&0xf800)==0xc000; // STMIA + } + } + else + { + // ARM + TUint32 op = *(TUint32*)aExc.iR15; + if(op<0xf0000000) + { + switch((op>>25)&7) + { + case 0: + if((op&0xf0)==(0xb0)) + return !(op&(1<<20)); // load/store halfword + else if((op&0x0e1000f0)==(0x000000f0)) + return ETrue; // store double + else if((op&0x0fb000f0) == 0x010000f0) + return ETrue; // swap instruction + else if((op&0x0ff000f0) == 0x01800090) + return ETrue; // strex + return EFalse; + case 2: + return !(op&(1<<20)); // load/store immediate + case 3: + if(!(op&0x10)) + return !(op&(1<<20)); // load/store register offset + return EFalse; + case 4: + return !(op&(1<<20)); // load/store multiple + case 6: + return !(op&(1<<20)); // coproc store + } + } + else + { + switch((op>>25)&7) + { + case 4: + if((op&0xfe5f0f00)==(0xf84d0500)) + return ETrue; // SRS instructions + return EFalse; + case 6: + return !(op&(1<<20)); // coproc store (STC2) + } + } + } + return EFalse; + } + + +TInt MemModelDemandPaging::Fault(TAny* aExceptionInfo) + { + TArmExcInfo& exc=*(TArmExcInfo*)aExceptionInfo; + + // Get faulting address + TLinAddr faultAddress = exc.iFaultAddress; + if(exc.iExcCode==EArmExceptionDataAbort) + { + // Only handle page translation faults + if((exc.iFaultStatus&0xf)!=0x7) + return KErrUnknown; + // Let writes take an exception rather than page in any memory... + if(FaultDuringWrite(exc)) + return KErrUnknown; + } + else if (exc.iExcCode != EArmExceptionPrefetchAbort) + return KErrUnknown; // Not prefetch or data abort + + DThread* thread = TheCurrentThread; + + // check which ragion fault occured in... + TBool inRom=ETrue; + if(TUint(faultAddress-iRomPagedLinearBase)iPagingExcTrap; + if(!fm) + NKern::LockSystem(); + else + { + if(!trap || fm!=&TheScheduler.iLock) + { + __KTRACE_OPT2(KPAGING,KPANIC,Kern::Printf("DP: Fault with FM Held! %x (%O pc=%x)",faultAddress,&Kern::CurrentThread(),exc.iR15)); + Panic(EPageFaultWhilstFMHeld); // Not allowed to hold mutexes + } + + // Current thread already has the system lock... + NKern::FlashSystem(); // Let someone else have a go with the system lock. + } + + // System locked here + + TInt r = KErrNone; + if(thread->IsRealtime()) + r = CheckRealtimeThreadFault(thread, aExceptionInfo); + if (r == KErrNone) + r = HandleFault(exc, faultAddress, inRom); + + // Restore system lock state + if (fm != NKern::HeldFastMutex()) + { + if (fm) + NKern::LockSystem(); + else + NKern::UnlockSystem(); + } + + // Deal with XTRAP_PAGING + if(r == KErrNone && trap) + { + trap->Exception(1); // Return from exception trap with result '1' (value>0) + // code doesn't continue beyond this point. + } + + return r; + } + + +TInt MemModelDemandPaging::HandleFault(TArmExcInfo& aExc, TLinAddr aFaultAddress, TBool aInRom) + { + ++iEventInfo.iPageFaultCount; + + // get page table entry... + TPte* pt = SafePtePtrFromLinAddr(aFaultAddress); + if(!pt) + return KErrNotFound; + TPte pte = *pt; + + // Do what is required to make page accessible... + + if(pte&KPtePresentMask) + { + // PTE is present, so assume it has already been dealt with +#ifdef BTRACE_PAGING + BTraceContext12(BTrace::EPaging,BTrace::EPagingPageNop,pte&~KPageMask,aFaultAddress,aExc.iR15); +#endif + return KErrNone; + } + + if(pte!=KPteNotPresentEntry) + { + // PTE alread has a page + SPageInfo* pageInfo = SPageInfo::FromPhysAddr(pte); + if(pageInfo->State()==SPageInfo::EStatePagedDead) + { + // page currently being unmapped, so do that here... + *pt = KPteNotPresentEntry; // Update page table + __DRAIN_WRITE_BUFFER; + } + else + { + // page just needs making young again... + *pt = TPte(pte|KArmPteSmallPage); // Update page table + __DRAIN_WRITE_BUFFER; + Rejuvenate(pageInfo); +#ifdef BTRACE_PAGING + BTraceContext12(BTrace::EPaging,BTrace::EPagingRejuvenate,pte&~KPageMask,aFaultAddress,aExc.iR15); +#endif + return KErrNone; + } + } + + // PTE not present, so page it in... + // check if fault in a CodeSeg... + DMemModelCodeSegMemory* codeSegMemory = NULL; + if (aInRom) + NKern::ThreadEnterCS(); + else + { + // find CodeSeg... + DMemModelCodeSeg* codeSeg = (DMemModelCodeSeg*)DCodeSeg::CodeSegsByAddress.Find(aFaultAddress); + if (!codeSeg) + return KErrNotFound; + codeSegMemory = codeSeg->Memory(); + if (codeSegMemory==0 || !codeSegMemory->iIsDemandPaged) + return KErrNotFound; + // open reference on CodeSegMemory + NKern::ThreadEnterCS(); +#ifdef _DEBUG + TInt r = +#endif + codeSegMemory->Open(); + __NK_ASSERT_DEBUG(r==KErrNone); + NKern::FlashSystem(); + } + +#ifdef BTRACE_PAGING + BTraceContext8(BTrace::EPaging,BTrace::EPagingPageInBegin,aFaultAddress,aExc.iR15); +#endif + + TInt r = PageIn(aFaultAddress,codeSegMemory); + + NKern::UnlockSystem(); + + if(codeSegMemory) + codeSegMemory->Close(); + + NKern::ThreadLeaveCS(); + + return r; + } + + +TInt MemModelDemandPaging::PageIn(TLinAddr aAddress, DMemModelCodeSegMemory* aCodeSegMemory) + { + // Get a request object - this may block until one is available + DPagingRequest* req = AcquireRequestObject(); + + // Get page table entry + TPte* pt = SafePtePtrFromLinAddr(aAddress); + + // Check page is still required... + if(!pt || *pt!=KPteNotPresentEntry) + { +#ifdef BTRACE_PAGING + BTraceContext0(BTrace::EPaging,BTrace::EPagingPageInUnneeded); +#endif + ReleaseRequestObject(req); + return pt ? KErrNone : KErrNotFound; + } + + ++iEventInfo.iPageInReadCount; + + // Get a free page + SPageInfo* pageInfo = AllocateNewPage(); + __NK_ASSERT_DEBUG(pageInfo); + + // Get physical address of free page + TPhysAddr phys = pageInfo->PhysAddr(); + __NK_ASSERT_DEBUG(phys!=KPhysAddrInvalid); + + // Temporarily map free page + TLinAddr loadAddr = req->iLoadAddr; + pt = req->iLoadPte; + *pt = phys | SP_PTE(KArmV45PermRWNO, KMemAttTempDemandPaging); + __DRAIN_WRITE_BUFFER; + + // Read page from backing store + aAddress &= ~KPageMask; + NKern::UnlockSystem(); + + TInt r; + if (!aCodeSegMemory) + r = ReadRomPage(req, aAddress); + else + { + r = ReadCodePage(req, aCodeSegMemory, aAddress); + if (r == KErrNone) + aCodeSegMemory->ApplyCodeFixups((TUint32*)loadAddr, aAddress); + } + if(r!=KErrNone) + Panic(EPageInFailed); + + // make caches consistant (uncached memory is used for page loading) + __DRAIN_WRITE_BUFFER; + NKern::LockSystem(); + + // Invalidate temporary mapping + *pt = KPteNotPresentEntry; + __DRAIN_WRITE_BUFFER; + InvalidateTLBForPage(loadAddr); + + ReleaseRequestObject(req); + + // Get page table entry + pt = SafePtePtrFromLinAddr(aAddress); + + // Check page still needs updating + TBool notNeeded = pt==0 || *pt!=KPteNotPresentEntry; + if(notNeeded) + { + // We don't need the new page after all, so put it on the active list as a free page + __KTRACE_OPT(KPAGING,Kern::Printf("DP: PageIn (New page not used)")); +#ifdef BTRACE_PAGING + BTraceContext0(BTrace::EPaging,BTrace::EPagingPageInUnneeded); +#endif + AddAsFreePage(pageInfo); + return pt ? KErrNone : KErrNotFound; + } + + // Update page info + if (!aCodeSegMemory) + pageInfo->SetPagedROM((aAddress-iRomLinearBase)>>KPageShift); + else + pageInfo->SetPagedCode(aCodeSegMemory,(aAddress-Mmu().iUserCodeBase)>>KPageShift); + + // Map page into final location + *pt = phys | (aCodeSegMemory ? KUserCodeLoadPte : KRomPtePermissions); + __DRAIN_WRITE_BUFFER; +#ifdef BTRACE_PAGING + TInt subCat = aCodeSegMemory ? BTrace::EPagingPageInCode : BTrace::EPagingPageInROM; + BTraceContext8(BTrace::EPaging,subCat,phys,aAddress); +#endif + + AddAsYoungest(pageInfo); + BalanceAges(); + + return KErrNone; + } + + +inline TUint8 ReadByte(TLinAddr aAddress) + { return *(volatile TUint8*)aAddress; } + + +TInt MemModelDemandPaging::EnsurePagePresent(TLinAddr aPage, DProcess* aProcess) + { + XTRAPD(exc,XT_DEFAULT,XTRAP_PAGING_RETRY(CHECK_PAGING_SAFE; ReadByte(aPage);)); + return exc; + } + + +TPhysAddr MemModelDemandPaging::LinearToPhysical(TLinAddr aPage, DProcess* aProcess) + { + return Mmu().LinearToPhysical(aPage); + } + + +TInt MemModelDemandPaging::PageState(TLinAddr aAddr) + { + TPte* ptePtr = 0; + TPte pte = 0; + TInt r = 0; + SPageInfo* pageInfo = NULL; + + NKern::LockSystem(); + + DMemModelCodeSegMemory* codeSegMemory = 0; + if(TUint(aAddr-iRomPagedLinearBase)Memory(); + if (codeSegMemory) + { + r |= EPageStateInRamCode; + if (codeSegMemory->iIsDemandPaged) + r |= EPageStatePaged; + } + } + + ptePtr = SafePtePtrFromLinAddr(aAddr); + if (!ptePtr) + goto done; + r |= EPageStatePageTablePresent; + pte = *ptePtr; + if (pte == KPteNotPresentEntry) + goto done; + r |= EPageStatePtePresent; + if (pte & KPtePresentMask) + r |= EPageStatePteValid; + + pageInfo = SPageInfo::FromPhysAddr(pte); + r |= pageInfo->Type(); + r |= pageInfo->State()<<8; + +done: + NKern::UnlockSystem(); + return r; + } + + +TBool MemModelDemandPaging::NeedsMutexOrderCheck(TLinAddr aStartAddr, TUint aLength) + { + // Don't check mutex order for reads from unpaged rom, kernel data area and kernel stack chunk + TLinAddr endAddr = aStartAddr + aLength; + TLinAddr stackBase = (TLinAddr)MM::SvStackChunk->Base(); + TLinAddr stackEnd = stackBase + MM::SvStackChunk->iMaxSize; + TLinAddr unpagedRomEnd = iRomPagedLinearBase ? iRomPagedLinearBase : iRomLinearBase + iRomSize; + TBool rangeInUnpagedRom = aStartAddr >= iRomLinearBase && endAddr <= unpagedRomEnd; + TBool rangeInKernelData = aStartAddr >= KKernelDataBase && endAddr <= KKernelDataEnd; + TBool rangeInKernelStack = aStartAddr >= stackBase && endAddr <= stackEnd; + return !rangeInUnpagedRom && !rangeInKernelData && !rangeInKernelStack; + } + + +EXPORT_C TBool DDemandPagingLock::Lock(DThread* aThread, TLinAddr aStart, TInt aSize) + { + MemModelDemandPaging* pager = (MemModelDemandPaging*)iThePager; + if(pager) + { + ArmMmu& m = pager->Mmu(); + TLinAddr end = aStart+aSize; + + if ((aStart < TUint(pager->iRomPagedLinearBase+pager->iRomPagedSize) && end > pager->iRomPagedLinearBase) || + (aStart < TUint(m.iUserCodeBase + m.iMaxUserCodeSize) && end > m.iUserCodeBase)) + return pager->ReserveLock(aThread,aStart,aSize,*this); + } + + return EFalse; + } + +void ArmMmu::DisablePageModification(DMemModelChunk* aChunk, TInt aOffset) +// +// Mark the page at aOffset in aChunk inaccessible to prevent it being +// modified while defrag is in progress. Save the required information +// to allow the fault handler to deal with this. +// Flush the cache for the page so that it can be aliased elsewhere for +// copying. +// Call this with the system unlocked. +// + { + __KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::DisablePageModification() offset=%08x", aOffset)); + + // Acquire the system lock here for atomic access to aChunk->iBase as moving + // between the home and run addresses (a reschedule) may update aChunk->iBase. + NKern::LockSystem(); + + iDisabledAddr = (TLinAddr)(aChunk->iBase) + aOffset; + TInt ptid=GetPageTableId(iDisabledAddr); + if(ptid<0) + Panic(EDefragDisablePageFailed); + + TPte* pPte = PageTable(ptid) + ((aOffset&KChunkMask)>>KPageShift); + TPte pte = *pPte; + if ((pte & KPteTypeMask) != KArmPteSmallPage) + Panic(EDefragDisablePageFailed); + + iDisabledPte = pPte; + iDisabledOldVal = pte; + + *pPte = 0; + __DRAIN_WRITE_BUFFER; + InvalidateTLBForPage(iDisabledAddr); + NKern::UnlockSystem(); + + CacheMaintenance::PageToPreserveAndReuseVirtualCache(iDisabledAddr); + __DRAIN_WRITE_BUFFER; + } + +TBool FaultStatusFromLinAddr(TLinAddr aAddr, TBool aKernel, TUint32& aFaultStatus) + // Walk the page tables looking for the given linear address. If access + // would've caused a fault, return ETrue and fill in aFaultStatus with a + // FSR value. Otherwise, return EFalse. Assumes it was a read. + { + TPde pde = PageDirectory[aAddr>>KChunkShift]; + TPde pdetype = pde & KPdeTypeMask; + if (pdetype == 0) + { + // section translation fault + aFaultStatus = 0x5; + return ETrue; + } + + TPte pte=0; + TInt domain = (pde >> 5) & 0xf; + TUint32 dacr = Arm::Dacr(); + TInt domaccess = (dacr >> (domain<<1)) & 0x3; + TInt ispage = (pdetype == KArmV45PdeSection) ? 0 : 0x2; + + if (ispage) + { + pte = *PtePtrFromLinAddr(aAddr); + if ((pte & KPteTypeMask) == 0) + { + // page translation fault + aFaultStatus = 0x7; + return ETrue; + } + } + + if (domaccess == 0x3) + { + // manager access + return EFalse; + } + if (domaccess == 0) + { + // domain fault + aFaultStatus = 0x9 | ispage; + return ETrue; + } + + TInt perms; + if (ispage) + perms = (pte >> 4) & 0x3; + else + perms = (pde >> 10) & 0x3; + + if (aKernel || perms != 0x1) + return EFalse; + + // permission fault + aFaultStatus = 0xd | ispage; + return ETrue; + } + +TInt ArmMmu::RamDefragFault(TAny* aExceptionInfo) + { + TArmExcInfo& exc=*(TArmExcInfo*)aExceptionInfo; + + // Get faulting address + TLinAddr faultAddress; + TBool prefetch=EFalse; + if(exc.iExcCode==EArmExceptionDataAbort) + { + // Only handle page translation faults + if((exc.iFaultStatus & 0xf) != 0x7) + return KErrUnknown; + faultAddress = exc.iFaultAddress; + } + else if(exc.iExcCode==EArmExceptionPrefetchAbort) + { + prefetch = ETrue; + faultAddress = exc.iR15; + } + else + return KErrUnknown; // Not data/prefetch abort + + TBool kernelmode = exc.iCpsr&EMaskMode != EUserMode; + + // Take system lock if not already held + NFastMutex* fm = NKern::HeldFastMutex(); + if(!fm) + NKern::LockSystem(); + else if(fm!=&TheScheduler.iLock) + { + __KTRACE_OPT2(KMMU,KPANIC,Kern::Printf("Defrag: Fault with FM Held! %x (%O pc=%x)",faultAddress,TheCurrentThread,exc.iR15)); + Panic(EDefragFaultWhilstFMHeld); // Not allowed to hold mutexes + } + + TInt r = KErrUnknown; + + // check if the mapping of the page has already been restored and retry if so + if (prefetch) + { + TUint32 fsr; + if (!FaultStatusFromLinAddr(faultAddress, kernelmode, fsr)) + { + r = KErrNone; + goto leave; + } + } + else + { + TPte* pt = SafePtePtrFromLinAddr(faultAddress); + if(!pt) + { + r = KErrNotFound; + goto leave; + } + if ((*pt & 0x3) != 0) + { + r = KErrNone; + goto leave; + } + } + + // check if the fault occurred in the page we are moving + if (iDisabledPte && TUint(faultAddress - iDisabledAddr) < TUint(KPageSize)) + { + // restore access to the page + *iDisabledPte = iDisabledOldVal; + __DRAIN_WRITE_BUFFER; + InvalidateTLBForPage(iDisabledAddr); + iDisabledAddr = 0; + iDisabledPte = NULL; + iDisabledOldVal = 0; + r = KErrNone; + } + +leave: + // Restore system lock state + if (!fm) + NKern::UnlockSystem(); + + return r; + }