// Copyright (c) 1997-2009 Nokia Corporation and/or its subsidiary(-ies).

// All rights reserved.

// This component and the accompanying materials are made available

// under the terms of the License "Eclipse Public License v1.0"

// which accompanies this distribution, and is available

// at the URL "http://www.eclipse.org/legal/epl-v10.html".

//

// Initial Contributors:

// Nokia Corporation - initial contribution.

//

// Contributors:

//

// Description:

// e32\memmodel\epoc\moving\arm\xmmu.cpp

// 

//

#include "arm_mem.h"

#include <mmubase.inl>

#include <ramcache.h>

#include <demand_paging.h>

#include "execs.h"

#include <defrag.h>

#include "cache_maintenance.h"

extern void FlushTLBs();

#if defined(__CPU_SA1__)

const TPde KRomSectionPermissions	=	SECTION_PDE(KArmV45PermRORO, KArmV45MemAttWB, EDomainClient);

const TPde KShadowPdePerm			=	PT_PDE(EDomainClient);

const TPte KPtPtePerm				=	SP_PTE(KArmV45PermRWNO, KArmV45MemAttBuf);	// page tables not cached

const TPte KRomPtePermissions		=	SP_PTE(KArmV45PermRORO, KArmV45MemAttWB);	// ROM is cached, read-only for everyone

const TPte KShadowPtePerm			=	SP_PTE(KArmV45PermRWRO, KArmV45MemAttWB);	// shadowed ROM is cached, supervisor writeable

#elif defined(__CPU_ARM710T__) || defined(__CPU_ARM720T__)

const TPde KRomSectionPermissions	=	SECTION_PDE(KArmV45PermRORO, KArmV45MemAttWB, EDomainClient);

const TPde KShadowPdePerm			=	PT_PDE(EDomainClient);

const TPte KPtPtePerm				=	SP_PTE(KArmV45PermRWNO, KArmV45MemAttWB);	// page tables cached (write-through)

const TPte KRomPtePermissions		=	SP_PTE(KArmV45PermRORO, KArmV45MemAttWB);	// ROM is cached, read-only for everyone

const TPte KShadowPtePerm			=	SP_PTE(KArmV45PermRWRO, KArmV45MemAttWB);	// shadowed ROM is cached, supervisor writeable

#elif defined(__CPU_ARM920T__) || defined(__CPU_ARM925T__) || defined(__CPU_ARM926J__)

const TPde KRomSectionPermissions	=	SECTION_PDE(KArmV45PermRORO, KArmV45MemAttWT, EDomainClient);

const TPde KShadowPdePerm			=	PT_PDE(EDomainClient);

const TPte KPtPtePerm				=	SP_PTE(KArmV45PermRWNO, KArmV45MemAttWT);	// page tables cached write through

const TPte KRomPtePermissions		=	SP_PTE(KArmV45PermRORO, KArmV45MemAttWT);	// ROM is cached, read-only for everyone

const TPte KShadowPtePerm			=	SP_PTE(KArmV45PermRWRO, KArmV45MemAttWT);	// shadowed ROM is cached, supervisor writeable

#elif defined(__CPU_XSCALE__)

	#ifdef __CPU_XSCALE_MANZANO__

const TPde KRomSectionPermissions	=	SECTION_PDE(KArmV45PermRORO, KXScaleMemAttWTRA_WBWA, EDomainClient);

const TPde KShadowPdePerm			=	PT_PDE(EDomainClient);

const TPte KPtPtePerm				=	SP_PTE(KArmV45PermRWNO, KXScaleMemAttWTRA_WBWA);	// page tables write-through cached

const TPte KRomPtePermissions		=	SP_PTE(KArmV45PermRORO, KXScaleMemAttWTRA_WBWA);	// ROM is cached, read-only for everyone

const TPte KShadowPtePerm			=	SP_PTE(KArmV45PermRWRO, KXScaleMemAttWTRA_WBWA);	// shadowed ROM is cached, supervisor writeable

	#else

const TPde KRomSectionPermissions	=	SECTION_PDE(KArmV45PermRORO, KXScaleMemAttWTRA, EDomainClient);

const TPde KShadowPdePerm			=	PT_PDE(EDomainClient);

const TPte KPtPtePerm				=	SP_PTE(KArmV45PermRWNO, KXScaleMemAttWTRA);	// page tables write-through cached

const TPte KRomPtePermissions		=	SP_PTE(KArmV45PermRORO, KXScaleMemAttWTRA);	// ROM is cached, read-only for everyone

const TPte KShadowPtePerm			=	SP_PTE(KArmV45PermRWRO, KXScaleMemAttWTRA);	// shadowed ROM is cached, supervisor writeable

	#endif

#endif

const TPte KPtInfoPtePerm = KPtPtePerm;

const TPde KPtPdePerm = PT_PDE(EDomainClient);

// Permissions for each chunk type

enum TPTEProperties

	{

	ESupRo	=	SP_PTE(KArmV45PermRORO, KDefaultCaching),

	ESupRw	=	SP_PTE(KArmV45PermRWNO, KDefaultCaching),

	EUserRo	=	SP_PTE(KArmV45PermRWRO, KDefaultCaching),

	EUserRw	=	SP_PTE(KArmV45PermRWRW, KDefaultCaching)

	};

LOCAL_D const TPde ChunkPdePermissions[ENumChunkTypes] =

	{

	PT_PDE(EDomainClient),		// EKernelData

	PT_PDE(EDomainClient),		// EKernelStack

	PT_PDE(EDomainClient),		// EKernelCode

	PT_PDE(EDomainClient),		// EDll

	PT_PDE(EDomainClient),		// EUserCode - user/ro & sup/rw everywhere

	PT_PDE(EDomainClient),		// ERamDrive - sup/rw accessed by domain change

	// user data or self modifying code is sup/rw, user no access at home. It's user/rw & sup/rw when running

	// note ARM MMU architecture prevents implementation of user read-only data

	PT_PDE(EDomainClient),		// EUserData

	PT_PDE(EDomainClient),		// EDllData

	PT_PDE(EDomainClient),		// EUserSelfModCode

	PT_PDE(EDomainClient),		// ESharedKernelSingle

	PT_PDE(EDomainClient),		// ESharedKernelMultiple

	PT_PDE(EDomainClient),		// ESharedIo

	PT_PDE(EDomainClient),		// ESharedKernelMirror (unused in this memory model)

	PT_PDE(EDomainClient),		// EKernelMessage

	};

const TPde KUserDataRunningPermissions = PT_PDE(EDomainVarUserRun);

LOCAL_D const TPte ChunkPtePermissions[ENumChunkTypes] =

	{

	ESupRw,					// EKernelData

	ESupRw,					// EKernelStack

	ESupRw,					// EKernelCode

	EUserRo,				// EDll

	EUserRo,				// EUserCode

	ESupRw,					// ERamDrive

	ESupRw,					// EUserData

	ESupRw,					// EDllData

	ESupRw,					// EUserSelfModCode

	ESupRw,					// ESharedKernelSingle

	ESupRw,					// ESharedKernelMultiple

	ESupRw,					// ESharedIo

	ESupRw,					// ESharedKernelMirror (unused in this memory model)

	ESupRw,					// EKernelMessage

	};

const TPte KUserCodeLoadPte = (TPte)EUserRo;

const TPte KKernelCodeRunPte = (TPte)ESupRw;

// Inline functions for simple transformations

inline TLinAddr PageTableLinAddr(TInt aId)

	{

	return KPageTableBase + (aId<<KPageTableShift);

	}

inline TPte* PageTable(TInt aId)

	{

	return (TPte*)(KPageTableBase+(aId<<KPageTableShift));

	}

inline TPde* PageDirectoryEntry(TLinAddr aLinAddr)

	{

	return PageDirectory + (aLinAddr>>KChunkShift);

	}

inline TBool IsPageTable(TPde aPde)

	{

	return ((aPde&KPdeTypeMask)==KArmV45PdePageTable);

	}

inline TBool IsSectionDescriptor(TPde aPde)

	{

	return ((aPde&KPdeTypeMask)==KArmV45PdeSection);

	}

inline TBool IsPresent(TPte aPte)

	{

	return (aPte&KPtePresentMask);

	}

inline TPhysAddr PageTablePhysAddr(TPde aPde)

	{

	return aPde & KPdePageTableAddrMask;

	}

inline TPhysAddr PhysAddrFromSectionDescriptor(TPde aPde)

	{

	return aPde & KPdeSectionAddrMask;

	}

extern void InvalidateTLBForPage(TLinAddr /*aLinAddr*/);

void Mmu::SetupInitialPageInfo(SPageInfo* aPageInfo, TLinAddr aChunkAddr, TInt aPdeIndex)

	{

	__ASSERT_ALWAYS(aChunkAddr==0 || aChunkAddr>=KRamDriveEndAddress, Panic(EBadInitialPageAddr));

	TLinAddr addr = aChunkAddr + (aPdeIndex<<KPageShift);

	if (aPageInfo->Type()!=SPageInfo::EUnused)

		return;	// already set (page table)

	if (addr == KPageTableInfoBase)

		{

		aPageInfo->SetPtInfo(0);

		aPageInfo->Lock();

		}

	else if (addr>=KPageDirectoryBase && addr<(KPageDirectoryBase+KPageDirectorySize))

		{

		aPageInfo->SetPageDir(0,aPdeIndex);

		aPageInfo->Lock();

		}

	else

		aPageInfo->SetFixed();

	}

void Mmu::SetupInitialPageTableInfo(TInt aId, TLinAddr aChunkAddr, TInt aNumPtes)

	{

	__ASSERT_ALWAYS(aChunkAddr==0 || aChunkAddr>=KRamDriveEndAddress, Panic(EBadInitialPageAddr));

	SPageTableInfo& pti=PtInfo(aId);

	pti.iCount=aNumPtes;

	pti.SetGlobal(aChunkAddr>>KChunkShift);

	}

TInt Mmu::GetPageTableId(TLinAddr aAddr)

	{

	TInt id=-1;

	__KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::PageTableId(%08x)",aAddr));

	TInt pdeIndex=aAddr>>KChunkShift;

	TPde pde = PageDirectory[pdeIndex];

	if (IsPageTable(pde))

		{

		SPageInfo* pi = SPageInfo::SafeFromPhysAddr(pde);

		if (pi)

			id = (pi->Offset()<<KPtClusterShift) | ((pde>>KPageTableShift)&KPtClusterMask);

		}

	__KTRACE_OPT(KMMU,Kern::Printf("ID=%d",id));

	return id;

	}

// Used only during boot for recovery of RAM drive

TInt ArmMmu::BootPageTableId(TLinAddr aAddr, TPhysAddr& aPtPhys)

	{

	TInt id=KErrNotFound;

	__KTRACE_OPT(KMMU,Kern::Printf("ArmMmu:BootPageTableId(%08x,&)",aAddr));

	TInt pdeIndex=aAddr>>KChunkShift;

	TPde pde = PageDirectory[pdeIndex];

	if (IsPageTable(pde))

		{

		aPtPhys = pde & KPdePageTableAddrMask;

		SPageInfo* pi = SPageInfo::SafeFromPhysAddr(pde);

		if (pi)

			{

			SPageInfo::TType type = pi->Type();

			if (type == SPageInfo::EPageTable)

				id = (pi->Offset()<<KPtClusterShift) | ((pde>>KPageTableShift)&KPtClusterMask);

			else if (type == SPageInfo::EUnused)

				id = KErrUnknown;

			}

		}

	__KTRACE_OPT(KMMU,Kern::Printf("ID=%d",id));

	return id;

	}

TBool ArmMmu::PteIsPresent(TPte aPte)

	{

	return aPte & KPtePresentMask;

	}

TPhysAddr ArmMmu::PtePhysAddr(TPte aPte, TInt aPteIndex)

	{

	TUint pte_type = aPte & KPteTypeMask;

	if (pte_type == KArmV45PteLargePage)

		return (aPte & KPteLargePageAddrMask) + (TPhysAddr(aPteIndex << KPageShift) & KLargePageMask);

	else if (pte_type != 0)

		return aPte & KPteSmallPageAddrMask;

	return KPhysAddrInvalid;

	}

TPhysAddr ArmMmu::PdePhysAddr(TLinAddr aAddr)

	{

	TPde pde = PageDirectory[aAddr>>KChunkShift];

	if (IsSectionDescriptor(pde))

		return PhysAddrFromSectionDescriptor(pde);

	return KPhysAddrInvalid;

	}

TPte* SafePageTableFromPde(TPde aPde)

	{

	if((aPde&KPdeTypeMask)==KArmV45PdePageTable)

		{

		SPageInfo* pi = SPageInfo::SafeFromPhysAddr(aPde);

		if(pi)

			{

			TInt id = (pi->Offset()<<KPtClusterShift) | ((aPde>>KPageTableShift)&KPtClusterMask);

			return PageTable(id);

			}

		}

	return 0;

	}

TPte* SafePtePtrFromLinAddr(TLinAddr aAddress)

	{

	TPde pde = PageDirectory[aAddress>>KChunkShift];

	TPte* pt = SafePageTableFromPde(pde);

	if(pt)

		pt += (aAddress>>KPageShift)&(KChunkMask>>KPageShift);

	return pt;

	}

#ifdef __ARMCC__

	__forceinline /* RVCT ignores normal inline qualifier :-( */

#else

	inline

#endif

TPte* PtePtrFromLinAddr(TLinAddr aAddress)

	{

	TPde pde = PageDirectory[aAddress>>KChunkShift];

	SPageInfo* pi = SPageInfo::FromPhysAddr(pde);

	TInt id = (pi->Offset()<<KPtClusterShift) | ((pde>>KPageTableShift)&KPtClusterMask);

	TPte* pt = PageTable(id);

	pt += (aAddress>>KPageShift)&(KChunkMask>>KPageShift);

	return pt;

	}

TInt ArmMmu::LinearToPhysical(TLinAddr aLinAddr, TInt aSize, TPhysAddr& aPhysicalAddress, TPhysAddr* aPhysicalPageList)

	{

	TPhysAddr physStart = ArmMmu::LinearToPhysical(aLinAddr);

	TPhysAddr nextPhys = physStart&~KPageMask;

	TUint32* pageList = aPhysicalPageList;

	TInt pageIndex = aLinAddr>>KPageShift;

	TInt pagesLeft = ((aLinAddr+aSize-1)>>KPageShift)+1 - pageIndex;

	TPde* pdePtr = &PageDirectory[aLinAddr>>KChunkShift];

	while(pagesLeft)

		{

		pageIndex &= KChunkMask>>KPageShift;

		TInt pagesLeftInChunk = (1<<(KChunkShift-KPageShift))-pageIndex;

		if(pagesLeftInChunk>pagesLeft)

			pagesLeftInChunk = pagesLeft;

		pagesLeft -= pagesLeftInChunk;

		TPhysAddr phys;

		TPde pde = *pdePtr++;

		TUint pdeType = pde&KPdeTypeMask;

		if(pdeType==KArmV45PdeSection)

			{

			phys = (pde & KPdeSectionAddrMask) + (pageIndex*KPageSize);

			__KTRACE_OPT(KMMU2,Kern::Printf("ArmMmu::LinearToPhysical Section phys=%8x",phys));

			TInt n=pagesLeftInChunk;

			phys==nextPhys ? nextPhys+=n*KPageSize : nextPhys=KPhysAddrInvalid;

			if(pageList)

				{

				TUint32* pageEnd = pageList+n;

				do

					{

					*pageList++ = phys;

					phys+=KPageSize;

					}

				while(pageList<pageEnd);

				}

			}

		else

			{

			TPte* pt = SafePageTableFromPde(pde);

			if(!pt)

				{

				__KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::LinearToPhysical missing page table: PDE=%8x",pde));

				return KErrNotFound;

				}

			pt += pageIndex;

			for(;;)

				{

				TPte pte = *pt++;

				TUint pte_type = pte & KPteTypeMask;

				if (pte_type >= KArmV45PteSmallPage)

					{

					phys = (pte & KPteSmallPageAddrMask);

					__KTRACE_OPT(KMMU2,Kern::Printf("ArmMmu::LinearToPhysical Small Page phys=%8x",phys));

					phys==nextPhys ? nextPhys+=KPageSize : nextPhys=KPhysAddrInvalid;

					if(pageList)

						*pageList++ = phys;

					if(--pagesLeftInChunk)

						continue;

					break;

					}

				if (pte_type == KArmV45PteLargePage)

					{

					--pt; // back up ptr

					TUint pageOffset = ((TUint)pt>>2)&(KLargeSmallPageRatio-1);

					phys = (pte & KPteLargePageAddrMask) + pageOffset*KPageSize;

					__KTRACE_OPT(KMMU2,Kern::Printf("ArmMmu::LinearToPhysical Large Page phys=%8x",phys));

					TInt n=KLargeSmallPageRatio-pageOffset;

					if(n>pagesLeftInChunk)

						n = pagesLeftInChunk;

					phys==nextPhys ? nextPhys+=n*KPageSize : nextPhys=KPhysAddrInvalid;

					if(pageList)

						{

						TUint32* pageEnd = pageList+n;

						do

							{

							*pageList++ = phys;

							phys+=KPageSize;

							}

						while(pageList<pageEnd);

						}

					pt += n;

					if(pagesLeftInChunk-=n)

						continue;

					break;

					}

				__KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::LinearToPhysical bad PTE %8x",pte));

				return KErrNotFound;

				}

			}

		if(!pageList && nextPhys==KPhysAddrInvalid)

			{

			__KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::LinearToPhysical not contiguous"));

			return KErrNotFound;

			}

		pageIndex = 0;

		}

	if(nextPhys==KPhysAddrInvalid)

		{

		// Memory is discontiguous...

		aPhysicalAddress = KPhysAddrInvalid;

		return 1;

		}

	else

		{

		// Memory is contiguous...

		aPhysicalAddress = physStart;

		return KErrNone;

		}

	}

TPhysAddr ArmMmu::LinearToPhysical(TLinAddr aLinAddr)

	{

	__KTRACE_OPT(KMMU,Kern::Printf("ArmMmu::LinearToPhysical(%08x)",aLinAddr));

	TPhysAddr phys = KPhysAddrInvalid;

	TPde pde = PageDirectory[aLinAddr>>KChunkShift];

	if (IsPageTable(pde))

		{

		SPageInfo* pi = SPageInfo::SafeFromPhysAddr(pde);

		if (pi)

			{

			TInt id = (pi->Offset()<<KPtClusterShift) | ((pde>>KPageTableShift)&KPtClusterMask);

			TInt pteIndex = (aLinAddr & KChunkMask)>>KPageShift;

			TPte pte = PageTable(id)[pteIndex];

			TUint pte_type = pte & KPteTypeMask;

			if (pte_type == KArmV45PteLargePage)

				{

				phys = (pte & KPteLargePageAddrMask) + (aLinAddr & KLargePageMask);

				__KTRACE_OPT(KMMU,Kern::Printf("Mapped with 64K page - returning %08x", phys));

				}

			else if (pte_type != 0)

				{

				phys = (pte & KPteSmallPageAddrMask) + (aLinAddr & KPageMask);

				__KTRACE_OPT(KMMU,Kern::Printf("Mapped with 4K page - returning %08x", phys));

				}

			}

		}

	else if (IsSectionDescriptor(pde))

		{

		phys = (pde & KPdeSectionAddrMask) + (aLinAddr & KChunkMask);

		__KTRACE_OPT(KMMU,Kern::Printf("Mapped with section - returning %08x", phys));

		}

	else

		{

		__KTRACE_OPT(KMMU,Kern::Printf("Address invalid"));

		}

	return phys;

	}

TInt ArmMmu::PreparePagesForDMA(TLinAddr aLinAddr, TInt aSize, TPhysAddr* aPhysicalPageList)

//Returns the list of physical pages belonging to the specified memory space.

//Checks these pages belong to a chunk marked as being trusted. 

//Locks these pages so they can not be moved by e.g. ram defragmenation.

	{

	SPageInfo* pi = NULL;

	DChunk* chunk = NULL;

	TInt err = KErrNone;

	__KTRACE_OPT(KMMU2,Kern::Printf("ArmMmu::PreparePagesForDMA %08x+%08x, asid=%d",aLinAddr,aSize));

	TUint32* pageList = aPhysicalPageList;

	TInt pagesInList = 0;				//The number of pages we put in the list so far

	TInt pageIndex = (aLinAddr & KChunkMask) >> KPageShift;	// Index of the page within the section

	TInt pagesLeft = ((aLinAddr & KPageMask) + aSize + KPageMask) >> KPageShift;

	MmuBase::Wait(); 	// RamAlloc Mutex for accessing page/directory tables.

	NKern::LockSystem();// SystemlLock for accessing SPageInfo objects.

	TPde* pdePtr = PageDirectory + (aLinAddr>>KChunkShift);

	while(pagesLeft)

		{

		TInt pagesLeftInChunk = (1<<(KChunkShift-KPageShift))-pageIndex;

		if(pagesLeftInChunk>pagesLeft)

			pagesLeftInChunk = pagesLeft;

		pagesLeft -= pagesLeftInChunk;

		TPte* pt = SafePageTableFromPde(*pdePtr++);

		if(!pt) { err = KErrNotFound; goto fail; }// Cannot get page table.

		pt += pageIndex;

		for(;pagesLeftInChunk--;)

			{

			TPhysAddr phys = (*pt++ & KPteSmallPageAddrMask);

			pi =  SPageInfo::SafeFromPhysAddr(phys);

			if(!pi)	{ err = KErrNotFound; goto fail; }// Invalid address

			__KTRACE_OPT(KMMU2,Kern::Printf("PageInfo: PA:%x T:%x S:%x O:%x C:%x",phys, pi->Type(), pi->State(), pi->Owner(), pi->LockCount()));

			if (chunk==NULL)

				{//This is the first page. Check 'trusted' bit.

				if (pi->Type()!= SPageInfo::EChunk)

					{ err = KErrAccessDenied; goto fail; }// The first page do not belong to chunk.	

				chunk = (DChunk*)pi->Owner();

				if ( (chunk == NULL) || ((chunk->iAttributes & DChunk::ETrustedChunk)== 0) )

					{ err = KErrAccessDenied; goto fail; } // Not a trusted chunk

				}

			pi->Lock();

			*pageList++ = phys;

			if ( (++pagesInList&127) == 0) //release system lock temporarily on every 512K

				NKern::FlashSystem();

			}

		pageIndex = 0;

		}

	if (pi->Type()!= SPageInfo::EChunk)

		{ err = KErrAccessDenied; goto fail; }// The last page do not belong to chunk.	

	if (chunk && (chunk != (DChunk*)pi->Owner()))

		{ err = KErrArgument; goto fail; }//The first & the last page do not belong to the same chunk.

	NKern::UnlockSystem();

	MmuBase::Signal();

	return KErrNone;

fail:

	__KTRACE_OPT(KMMU2,Kern::Printf("ArmMmu::PreparePagesForDMA failed"));

	NKern::UnlockSystem();

	MmuBase::Signal();

	ReleasePagesFromDMA(aPhysicalPageList, pagesInList);

	return err;

	}

TInt ArmMmu::ReleasePagesFromDMA(TPhysAddr* aPhysicalPageList, TInt aPageCount)

// Unlocks physical pages.

// @param aPhysicalPageList - points to the list of physical pages that should be released.

// @param aPageCount		- the number of physical pages in the list.