// Copyright (c) 1997-2009 Nokia Corporation and/or its subsidiary(-ies).

// All rights reserved.

// This component and the accompanying materials are made available

// under the terms of the License "Eclipse Public License v1.0"

// which accompanies this distribution, and is available

// at the URL "http://www.eclipse.org/legal/epl-v10.html".

//

// Initial Contributors:

// Nokia Corporation - initial contribution.

//

// Contributors:

//

// Description:

//

#include "arm_mem.h"

#include "mm.h"

#include "mmu.h"

#include "mpager.h"

#include "cache_maintenance.inl"

#include "execs.h"

#ifdef BROADCAST_TLB_MAINTENANCE

class TTLBIPI : public TGenericIPI

	{

public:

	TTLBIPI();

	static void InvalidateIsr(TGenericIPI*);

	static void WaitAndInvalidateIsr(TGenericIPI*);

	void AddArg(TLinAddr aArg);

public:

	volatile TInt	iFlag;

	TLinAddr		iArg;

	};

TTLBIPI::TTLBIPI()

	:	iFlag(0), iArg(0)

	{

	}

void TTLBIPI::InvalidateIsr(TGenericIPI* aPtr)

	{

	TRACE2(("TLBInv"));

	TTLBIPI& a = *(TTLBIPI*)aPtr;

	TLinAddr arg = a.iArg;

	if (arg==0)

		LocalInvalidateTLB();

	else if (arg<256)

		LocalInvalidateTLBForAsid(arg);

	else

		LocalInvalidateTLBForPage(arg);

	}

void TTLBIPI::WaitAndInvalidateIsr(TGenericIPI* aPtr)

	{

	TRACE2(("TLBWtInv"));

	TTLBIPI& a = *(TTLBIPI*)aPtr;

	while (!a.iFlag)

		{ __chill(); }

	InvalidateIsr(aPtr);

	}

void TTLBIPI::AddArg(TLinAddr aArg)

	{

	iArg = aArg;

	NKern::Lock();

	InvalidateIsr(this);

	QueueAllOther(&InvalidateIsr);

	NKern::Unlock();

	WaitCompletion();

	}

void BroadcastInvalidateTLB(TLinAddr aLinAddrAndAsid)

	{

	TTLBIPI ipi;

	ipi.AddArg(aLinAddrAndAsid);

	}

#endif	// BROADCAST_TLB_MAINTENANCE

//

// Functions for class Mmu

//

/**

Return the physical address of the memory mapped by a Page Table Entry (PTE).

@param aPte			The value contained in the PTE.

@param aPteIndex	The index of the PTE within its page table.

*/

TPhysAddr Mmu::PtePhysAddr(TPte aPte, TUint aPteIndex)

	{

	if(aPte&KArmV6PteSmallPage)

		return aPte & KPteSmallPageAddrMask;

	if(aPte&KArmV6PteLargePage)

		return (aPte & KPteLargePageAddrMask) + (TPhysAddr(aPteIndex << KPageShift) & KLargePageMask);

	return KPhysAddrInvalid;

	}

/**

Return the virtual address of the page table referenced by the given

Page Directory Entry (PDE) \a aPde. If the PDE doesn't refer to a

page table then the null-pointer is returned.

If the page table was not one allocated by the kernel then the

results are unpredictable and may cause a system fault.

@pre #MmuLock held.

*/

TPte* Mmu::PageTableFromPde(TPde aPde)

	{

	if((aPde&KPdePresentMask)==KArmV6PdePageTable)

		{

		SPageInfo* pi = SPageInfo::FromPhysAddr(aPde);

		return (TPte*)(KPageTableBase+(pi->Index()<<KPageShift)+(aPde&(KPageMask&~KPageTableMask)));

		}

	return 0;

	}

/**

Perform the action of #PageTableFromPde but without the possibility of

a system fault caused the page table not being one allocated by the kernel.

@pre #MmuLock held.

*/

TPte* Mmu::SafePageTableFromPde(TPde aPde)

	{

	if((aPde&KPdeTypeMask)==KArmV6PdePageTable)

		{

		SPageInfo* pi = SPageInfo::SafeFromPhysAddr(aPde&~KPageMask);

		if(pi)

			return (TPte*)(KPageTableBase+(pi->Index()<<KPageShift)+(aPde&(KPageMask&~KPageTableMask)));

		}

	return 0;

	}

/**

Return the base phsical address of the section table referenced by the given

Page Directory Entry (PDE) \a aPde. If the PDE doesn't refer to a

section then KPhysAddrInvalid is returned.

@pre #MmuLock held.

*/

TPhysAddr Mmu::SectionBaseFromPde(TPde aPde)

	{

	if(PdeMapsSection(aPde))

		return aPde&KPdeSectionAddrMask;

	return KPhysAddrInvalid;

	}

/**

Return a pointer to the Page Table Entry (PTE) which maps the

virtual address \a aAddress in the address space \a aOsAsid.

If no page table exists or it was not one allocated by the kernel

then the results are unpredictable and may cause a system fault.

@pre #MmuLock held.

*/

TPte* Mmu::PtePtrFromLinAddr(TLinAddr aAddress, TInt aOsAsid)

	{

	TPde pde = PageDirectory(aOsAsid)[aAddress>>KChunkShift];

	SPageInfo* pi = SPageInfo::FromPhysAddr(pde);

	TPte* pt = (TPte*)(KPageTableBase+(pi->Index()<<KPageShift)+(pde&(KPageMask&~KPageTableMask)));

	pt += (aAddress>>KPageShift)&(KChunkMask>>KPageShift);

	return pt;

	}

/**

Perform the action of #PtePtrFromLinAddr but without the possibility

of a system fault. If the page table is not present or not one

allocated by the kernel then the null-pointer is returned.

@pre #MmuLock held.

*/

TPte* Mmu::SafePtePtrFromLinAddr(TLinAddr aAddress, TInt aOsAsid)

	{

	TPde pde = PageDirectory(aOsAsid)[aAddress>>KChunkShift];

	TPte* pt = SafePageTableFromPde(pde);

	if(pt)

		pt += (aAddress>>KPageShift)&(KChunkMask>>KPageShift);

	return pt;

	}

/**

Return the physical address for the page table whose virtual

address is \a aPt.

If the page table was not one allocated by the kernel then the

results are unpredictable and may cause a system fault.

@pre #MmuLock held.

*/

TPhysAddr Mmu::PageTablePhysAddr(TPte* aPt)

	{

	__NK_ASSERT_DEBUG(MmuLock::IsHeld() || PageTablesLockIsHeld());

	TInt pdeIndex = ((TLinAddr)aPt)>>KChunkShift;

	TPde pde = PageDirectory(KKernelOsAsid)[pdeIndex];

	__NK_ASSERT_DEBUG((pde&KPdePresentMask)==KArmV6PdePageTable);

	SPageInfo* pi = SPageInfo::FromPhysAddr(pde);

	TPte* pPte = (TPte*)(KPageTableBase+(pi->Index(true)<<KPageShift)+(pde&(KPageMask&~KPageTableMask)));

	TPte pte = pPte[(((TLinAddr)aPt)&KChunkMask)>>KPageShift];

	__NK_ASSERT_DEBUG(pte & KArmV6PteSmallPage);

	return (pte&KPteSmallPageAddrMask)|(((TLinAddr)aPt)&(KPageMask&~KPageTableMask));

	}

/**

Perform a page table walk to return the physical address of

the memory mapped at virtual address \a aLinAddr in the

address space \a aOsAsid.

If the page table used was not one allocated by the kernel

then the results are unpredictable and may cause a system fault.

Use of this function should be avoided, use instead Mmu::LinearToPhysical

which contains debug assertions for its preconditions.

@pre #MmuLock held.

*/

TPhysAddr Mmu::UncheckedLinearToPhysical(TLinAddr aLinAddr, TInt aOsAsid)

	{

	TRACE2(("Mmu::UncheckedLinearToPhysical(%08x,%d)",aLinAddr,aOsAsid));

	TInt pdeIndex = aLinAddr>>KChunkShift;

	TPde pde = PageDirectory(aOsAsid)[pdeIndex];

	if ((pde&KPdePresentMask)==KArmV6PdePageTable)

		{

		SPageInfo* pi = SPageInfo::FromPhysAddr(pde);

		TPte* pPte = (TPte*)(KPageTableBase+(pi->Index(true)<<KPageShift)+(pde&(KPageMask&~KPageTableMask)));

		TPte pte = pPte[(aLinAddr&KChunkMask)>>KPageShift];

		if (pte & KArmV6PteSmallPage)

			{

			TPhysAddr pa=(pte&KPteSmallPageAddrMask)|(aLinAddr&~KPteSmallPageAddrMask);

			__KTRACE_OPT(KMMU,Kern::Printf("Mapped with small page - returning %08x",pa));

			return pa;

			}

		else if (pte & KArmV6PteLargePage)

			{

			TPhysAddr pa=(pte&KPteLargePageAddrMask)|(aLinAddr&~KPteLargePageAddrMask);

			__KTRACE_OPT(KMMU,Kern::Printf("Mapped with large page - returning %08x",pa));

			return pa;

			}

		}

	else if ((pde&KPdePresentMask)==KArmV6PdeSection)

		{

		TPhysAddr pa=(pde&KPdeSectionAddrMask)|(aLinAddr&~KPdeSectionAddrMask);

		__KTRACE_OPT(KMMU,Kern::Printf("Mapped with section - returning %08x",pa));

		return pa;

		}

	return KPhysAddrInvalid;

	}

extern TUint32 TTCR();

extern TUint32 CPUID(TInt /*aRegNum*/);

void Mmu::Init1()

	{

	TRACEB(("Mmu::Init1"));

	// check page local/global page directory split is correct...

	__NK_ASSERT_ALWAYS(TTCR()==1);

	// check cache type is supported and consistent with compile time macros...

	TInt iColourCount = 0;

	TInt dColourCount = 0;

	TUint32 ctr = InternalCache::TypeRegister();

	TRACEB(("CacheTypeRegister = %08x",ctr));

#ifdef __CPU_ARMV6

	__NK_ASSERT_ALWAYS((ctr>>29)==0);	// check ARMv6 format

	if(ctr&0x800)

		iColourCount = 4;

	if(ctr&0x800000)

		dColourCount = 4;

#else

	__NK_ASSERT_ALWAYS((ctr>>29)==4);	// check ARMv7 format

	TUint l1ip = (ctr>>14)&3;			// L1 instruction cache indexing and tagging policy

	__NK_ASSERT_ALWAYS(l1ip>=2);		// check I cache is physically tagged

	TUint32 clidr = InternalCache::LevelIDRegister();

	TRACEB(("CacheLevelIDRegister = %08x",clidr));

	TUint l1type = clidr&7;

	if(l1type)

		{

		if(l1type==2 || l1type==3 || l1type==4)

			{

			// we have an L1 data cache...

			TUint32 csir = InternalCache::SizeIdRegister(0,0);

			TUint sets = ((csir>>13)&0x7fff)+1;

			TUint ways = ((csir>>3)&0x3ff)+1;

			TUint lineSizeShift = (csir&7)+4;

			// assume L1 data cache is VIPT and alias checks broken and so we need data cache colouring...

			dColourCount = (sets<<lineSizeShift)>>KPageShift;

			if(l1type==4) // unified cache, so set instruction cache colour as well...

				iColourCount = (sets<<lineSizeShift)>>KPageShift;

			TRACEB(("L1DCache = 0x%x,0x%x,%d colourCount=%d",sets,ways,lineSizeShift,(sets<<lineSizeShift)>>KPageShift));

			}

		if(l1type==1 || l1type==3)

			{

			// we have a separate L1 instruction cache...

			TUint32 csir = InternalCache::SizeIdRegister(1,0);

			TUint sets = ((csir>>13)&0x7fff)+1;

			TUint ways = ((csir>>3)&0x3ff)+1;

			TUint lineSizeShift = (csir&7)+4;

			iColourCount = (sets<<lineSizeShift)>>KPageShift;

			TRACEB(("L1ICache = 0x%x,0x%x,%d colourCount=%d",sets,ways,lineSizeShift,(sets<<lineSizeShift)>>KPageShift));

			}

		}

	if(l1ip==3)

		{

		// PIPT cache, so no colouring restrictions...

		TRACEB(("L1ICache is PIPT"));

		iColourCount = 0;

		}

	else

		{

		// VIPT cache...

		TRACEB(("L1ICache is VIPT"));

		}

#endif

	TRACEB(("page colouring counts I=%d, D=%d",iColourCount,dColourCount));

	__NK_ASSERT_ALWAYS(iColourCount<=KPageColourCount);

	__NK_ASSERT_ALWAYS(dColourCount<=KPageColourCount);

	#ifndef __CPU_I_CACHE_HAS_COLOUR

	__NK_ASSERT_ALWAYS(iColourCount==0);

	#endif

	#ifndef __CPU_D_CACHE_HAS_COLOUR

	__NK_ASSERT_ALWAYS(dColourCount==0);

	#endif

	#ifndef __CPU_CACHE_HAS_COLOUR

	__NK_ASSERT_ALWAYS(iColourCount==0);

	__NK_ASSERT_ALWAYS(dColourCount==0);

	#endif

	// check MMU attributes match our assumptions...

	if(((CPUID(-1)>>16)&0xf)==0xf) // if have new CPUID format....

		{

		TUint mmfr1 = CPUID(5);

		TRACEB(("mmfr1 = %08x",mmfr1));

		#ifdef __CPU_NEEDS_BTAC_FLUSH_AFTER_ASID_CHANGE

			__NK_ASSERT_ALWAYS(((mmfr1>>28)&0xf)==1); // Branch Predictor needs invalidating after ASID change

		#else

			__NK_ASSERT_ALWAYS(((mmfr1>>28)&0xf)>=2); // Branch Predictor doesn't needs invalidating after ASID change

		#endif

		TUint mmfr2 = CPUID(6);

		TRACEB(("mmfr2 = %08x",mmfr2));

		__NK_ASSERT_ALWAYS(((mmfr2>>20)&0xf)>=2); // check Mem Barrier instructions are supported in CP15

		TUint mmfr3 = CPUID(7);

		TRACEB(("mmfr3 = %08x",mmfr3));

		(void)mmfr3;

		#if defined(__SMP__) && !defined(__CPU_ARM11MP__)

			__NK_ASSERT_ALWAYS(((mmfr3>>12)&0xf)>=2); // check Maintenance Broadcast is for all cache and TLB operations

		#endif	

		#ifdef __CPU_SUPPORTS_PAGE_TABLE_WALK_TO_L1_CACHE

			__NK_ASSERT_ALWAYS(((mmfr3>>20)&0xf)>=1); // check Coherent Walk for page tables

		#endif	

		}

	Arm::DefaultDomainAccess = KDefaultDomainAccess;

#ifdef __SMP__

	TInt i;

	for (i=0; i<KMaxCpus; ++i)

		{

		TSubScheduler& ss = TheSubSchedulers[i];

		TLinAddr a = KIPCAlias + (i<<KChunkShift);

		ss.i_AliasLinAddr = (TAny*)a;

		ss.i_AliasPdePtr = (TAny*)(KPageDirectoryBase + (a>>KChunkShift)*sizeof(TPde));

		}

#endif

	Init1Common();

	}

void Mmu::Init2()

	{

	TRACEB(("Mmu::Init2"));

	Init2Common();

	}

DMemoryObject* ExceptionStacks;

void Mmu::Init2Final()

	{

	TRACEB(("Mmu::Init2Final"));

	Init2FinalCommon();

	// initialise memory object for exception stacks...

	TMappingCreateFlags mapFlags = (TMappingCreateFlags)(EMappingCreateFixedVirtual|EMappingCreateReserveAllResources);

	TMemoryAttributes memAttr = EMemoryAttributeStandard;

	TUint size = 4*2*KPageSize; // 4 exception stacks each of one guard page and one mapped page

	size |= 1; // lower bit of size is set if region to be claimed contains gaps

	TInt r = MM::InitFixedKernelMemory(ExceptionStacks, KExcptStacksLinearBase, KExcptStacksLinearEnd, size, EMemoryObjectUnpaged, EMemoryCreateNoWipe, memAttr, mapFlags);

	__NK_ASSERT_ALWAYS(r==KErrNone);

	}

/**

Return the page directory entry (PDE) value to use for when mapping page tables intended

to map memory with the given attributes.

The returned value has the physical address component being zero, so a page table's physical

address can be simply ORed in.

*/

TPde Mmu::BlankPde(TMemoryAttributes aAttributes)

	{

	TPde pde = KArmV6PdePageTable;

	if(aAttributes&EMemoryAttributeUseECC)

		pde |= 1<<9;

	TRACE2(("Mmu::BlankPde(%x) returns 0x%x",aAttributes,pde));

	return pde;

	}

/**

Return the page directory entry (PDE) value to use for when creating a section mapping for memory

with the given attributes and #TPteType.

The returned value has the physical address component being zero, so the section's physical address

can be simply ORed in.

*/

TPde Mmu::BlankSectionPde(TMemoryAttributes aAttributes, TUint aPteType)

	{

	// reuse existing functions rather than duplicating the logic

	TPde pde = BlankPde(aAttributes);

	return PageToSectionEntry(pte, pde);

	}

/**

Return the page table entry (PTE) to use when mapping memory pages

with the given attributes and #TPteType.

This value has the physical address component being zero, so a page's physical

address can be simply ORed in.

*/

TPte Mmu::BlankPte(TMemoryAttributes aAttributes, TUint aPteType)

	{

	TUint attr = CanonicalMemoryAttributes(aAttributes);

	// common PTE setup...

	TPte pte = KArmV6PteSmallPage|KArmV6PteAP0;

	if(aPteType&EPteTypeUserAccess)

		pte |= KArmV6PteAP1;					// AP1 = user access

	if((aPteType&EPteTypeWritable)==false)

		pte |= KArmV6PteAP2;					// AP2 = !writable

	if(attr&EMemoryAttributeShareable)

		pte |= KArmV6PteS;

	if((aPteType&EPteTypeGlobal)==false)

		pte |= KArmV6PteNG;

	if((aPteType&EPteTypeExecutable)==false)

		pte |= KArmV6PteSmallXN;

	#if defined(__CPU_MEMORY_TYPE_REMAPPING)

		// other PTE bits...

		if(pte&KArmV6PteSmallXN)

			pte |= KArmV6PteSmallTEX1;	// TEX1 is a copy of the XN

		// process memory type...

		TUint type = attr&EMemoryAttributeTypeMask;

		pte |= ((type&3)<<2) | ((type&4)<<4);

	#else

		// other PTE bits...

		if((pte&(KArmV6PteAP2|KArmV6PteAP1))==(KArmV6PteAP2|KArmV6PteAP1))

			pte &= ~KArmV6PteAP0;		// clear AP0 if user r/o

		// process memory type...

		TUint texcb;

		switch((TMemoryType)(attr&EMemoryAttributeTypeMask))

			{

		case EMemAttStronglyOrdered:

			texcb = KArmV6MemAttSO;

			break;

		case EMemAttDevice:

			if(attr&EMemoryAttributeShareable)

				texcb = KArmV6MemAttSD;

			else

				texcb = KArmV6MemAttSD;	// should be KArmV6MemAttNSD? (but this made H4 go bang)

			break;

		case EMemAttNormalUncached:

			texcb = KArmV6MemAttNCNC;

			break;

		case EMemAttNormalCached:

			texcb = KArmV6MemAttWBWAWBWA;

			break;

		default:

			__NK_ASSERT_ALWAYS(0);		// undefined memory type

			texcb = KArmV6MemAttSO;

			break;

			}

		pte |= ((texcb&0x1c)<<4) | ((texcb&0x03)<<2);

	#endif

	TRACE2(("Mmu::BlankPte(%x,%x) returns 0x%x",aAttributes,aPteType,pte));

	return pte;

	}

/**

Calculate PDE and PTE which represent a page table mapping for an existing

section mapping.

@param[in] aPde The PDE for the existing section mapping.

@param[out] aPde A PDE for a page table mapping, with physical address == 0.

@return The PTE value for the first entry in the page table.

*/