// Copyright (c) 2007-2009 Nokia Corporation and/or its subsidiary(-ies).

// All rights reserved.

// This component and the accompanying materials are made available

// under the terms of the License "Eclipse Public License v1.0"

// which accompanies this distribution, and is available

// at the URL "http://www.eclipse.org/legal/epl-v10.html".

//

// Initial Contributors:

// Nokia Corporation - initial contribution.

//

// Contributors:

//

// Description:

//

#include "cache_maintenance.inl"

/**

 @file

 @internalComponent

*/

#if defined(GNUC) && !defined(__MARM_ARM4__)

#define	__VOLATILE__	volatile

#else

#define __VOLATILE__

#endif

#if defined(__SMP__) && defined(__CPU_ARM11MP__) 

#define COARSE_GRAINED_TLB_MAINTENANCE

#define BROADCAST_TLB_MAINTENANCE

#endif

FORCE_INLINE void __arm_dmb()

	{

	#if defined(__CPU_ARMV6)

		// dmb instruction...

		#ifdef __GNUC__

			TInt zero = 0;

			asm __VOLATILE__ ("mcr p15, 0, %0, c7, c10, 5 " : : "r"(zero));

		#elif defined(__ARMCC__)

			TInt zero = 0;

			asm("mcr p15, 0, zero, c7, c10, 5 ");

		#elif defined(__GCCXML__)

			// empty

		#else

			#error Unknown compiler

		#endif

	#elif defined(__CPU_ARMV7)

		// deprecated CP15 version of DMB...

		#ifdef __GNUC__

			TInt zero = 0;

			asm __VOLATILE__ ("mcr p15, 0, %0, c7, c10, 5 " : : "r"(zero));

		#elif defined(__ARMCC__)

			TInt zero = 0;

			asm("mcr p15, 0, zero, c7, c10, 5 ");

		#elif defined(__GCCXML__)

			// empty

		#else

			#error Unknown compiler

		#endif

	#else

		// non inline version...

		__e32_memory_barrier();

	#endif

	}

FORCE_INLINE void __arm_dsb()

	{

	#if defined(__CPU_ARMV6)

		// drain write buffer...

		#ifdef __GNUC__

			TInt zero = 0;

			asm __VOLATILE__ ("mcr p15, 0, %0, c7, c10, 4 " : : "r"(zero));

		#elif defined(__ARMCC__)

			TInt zero = 0;

			asm("mcr p15, 0, zero, c7, c10, 4 ");

		#elif defined(__GCCXML__)

			// empty

		#else

			#error Unknown compiler

		#endif

	#elif defined(__CPU_ARMV7)

		// deprecated CP15 version of DSB...

		#ifdef __GNUC__

			TInt zero = 0;

			asm __VOLATILE__ ("mcr p15, 0, %0, c7, c10, 4 " : : "r"(zero));

		#elif defined(__ARMCC__)

			TInt zero = 0;

			asm("mcr p15, 0, zero, c7, c10, 4 ");

		#elif defined(__GCCXML__)

			// empty

		#else

			#error Unknown compiler

		#endif

	#else

		// non inline version...

		__e32_io_completion_barrier();

	#endif

	}

extern "C" void __e32_instruction_barrier();

FORCE_INLINE void __arm_isb()

	{

	#if defined(__CPU_ARMV6)

		// prefetch flush...

		#ifdef __GNUC__

			TInt zero = 0;

			asm __VOLATILE__ ("mcr p15, 0, %0, c7, c5, 4 " : : "r"(zero));

		#elif defined(__ARMCC__)

			TInt zero = 0;

			asm("mcr p15, 0, zero, c7, c5, 4 ");

		#elif defined(__GCCXML__)

			// empty

		#else

			#error Unknown compiler

		#endif

	#elif defined(__CPU_ARMV7)

		// deprecated CP15 version of ISB...

		#ifdef __GNUC__

			TInt zero = 0;

			asm __VOLATILE__ ("mcr p15, 0, %0, c7, c5, 4 " : : "r"(zero));

		#elif defined(__ARMCC__)

			TInt zero = 0;

			asm("mcr p15, 0, zero, c7, c5, 4 ");

		#elif defined(__GCCXML__)

			// empty

		#else

			#error Unknown compiler

		#endif

	#else

		// non inline version...

		__e32_instruction_barrier();

	#endif

	}

/**

Branch predictor invalidate all

*/

FORCE_INLINE void __arm_bpiall()

	{

	#ifdef __GNUC__

		TInt zero = 0;

		asm __VOLATILE__ ("mcr p15, 0, %0, c7, c5, 6 " : : "r"(zero));

	#elif defined(__ARMCC__)

		TInt zero = 0;

		asm("mcr p15, 0, zero, c7, c5, 6 ");

	#elif defined(__GCCXML__)

		// empty

	#else

		#error Unknown compiler

	#endif

	}

#ifdef __SMP__

/**

Branch predictor invalidate all inner-shareable

*/

FORCE_INLINE void __arm_bpiallis()

	{

	// branch predictor invalidate all inner-shareable

	#ifdef __GNUC__

		TInt zero = 0;

		asm __VOLATILE__ ("mcr p15, 0, %0, c7, c1, 6 " : : "r"(zero));

	#elif defined(__ARMCC__)

		TInt zero = 0;

		asm("mcr p15, 0, zero, c7, c1, 6 ");

	#elif defined(__GCCXML__)

		// empty

	#else

		#error Unknown compiler

	#endif

	}

#endif

/**	

This will make sure that the change in page directory is visible by H/W Page-Table Walk.  

Call this function when a single entry in page directory is changed.

*/

FORCE_INLINE void SinglePdeUpdated(TPde* aPde)

  	{

	CacheMaintenance::SinglePdeUpdated((TLinAddr)aPde);

  	}

#ifdef BROADCAST_TLB_MAINTENANCE

/**

Signal other CPU cores to perform TLB maintenance.

@param aLinAddrAndAsid	If == 0, then InvalidateTLB;

						if < KMmuAsidCount, then InvalidateTLBForAsid;

						else InvalidateTLBForPage.

*/

extern void BroadcastInvalidateTLB(TLinAddr aLinAddrAndAsid=0);

#endif

/**

Invalidate a single I+D TLB entry on this CPU core only.

@param aLinAddrAndAsid Virtual address of a page of memory ORed with the ASID value.

*/

FORCE_INLINE void LocalInvalidateTLBForPage(TLinAddr aLinAddrAndAsid)

	{

	#ifdef __GNUC__

		#if defined(__CPU_ARM11MP__) // why?...

			asm __VOLATILE__ ("mcr p15, 0, %0, c8, c7, 3 " : : "r"(aLinAddrAndAsid));

		#else

			asm __VOLATILE__ ("mcr p15, 0, %0, c8, c7, 1 " : : "r"(aLinAddrAndAsid));

		#endif

	#elif defined(__ARMCC__)

		#if defined(__CPU_ARM11MP__) // why?...

			asm("mcr p15, 0, aLinAddrAndAsid, c8, c7, 3 ");

		#else

			asm("mcr p15, 0, aLinAddrAndAsid, c8, c7, 1 ");

		#endif

	#elif defined(__GCCXML__)

		// empty

	#else

		#error Unknown compiler

	#endif

	__arm_bpiall();

	__arm_dsb();

	__arm_isb();

	}

/**

Invalidate a single I+D TLB entry on all CPU cores.

@param aLinAddrAndAsid Virtual address of a page of memory ORed with the ASID value.

*/

FORCE_INLINE void InvalidateTLBForPage(TLinAddr aLinAddrAndAsid)

	{

	#ifdef BROADCAST_TLB_MAINTENANCE

		BroadcastInvalidateTLB(aLinAddrAndAsid);

	#elif !defined(__SMP__)

		LocalInvalidateTLBForPage(aLinAddrAndAsid);

	#else // __SMP__

		// inner-shareable invalidate...

		#ifdef __GNUC__

			asm __VOLATILE__ ("mcr p15, 0, %0, c8, c3, 1 " : : "r"(aLinAddrAndAsid));

		#elif defined(__ARMCC__)

			asm("mcr p15, 0, aLinAddrAndAsid, c8, c3, 1 ");

		#elif defined(__GCCXML__)

			// empty

		#else

			#error Unknown compiler

		#endif

		__arm_bpiallis();

		__arm_dsb();

		__arm_isb();

	#endif

	}

/**

Invalidate entire TLB on this CPU only

*/

FORCE_INLINE void LocalInvalidateTLB()

	{

	#ifdef __GNUC__

		asm __VOLATILE__ ("mcr p15, 0, r0, c8, c7, 0 ");

	#elif defined(__ARMCC__)

		TInt dummy = 0; // damned RVCT

		asm("mcr p15, 0, dummy, c8, c7, 0 ");

	#elif defined(__GCCXML__)

		// empty

	#else

		#error Unknown compiler

	#endif

	__arm_bpiall();

	__arm_dsb();

	__arm_isb();

	}

/**

Invalidate entire TLB on all CPU cores.

*/

FORCE_INLINE void InvalidateTLB()

	{

	#ifdef BROADCAST_TLB_MAINTENANCE

		BroadcastInvalidateTLB(0);

	#elif !defined(__SMP__)

		LocalInvalidateTLB();

	#else // __SMP__

		// inner-shareable invalidate...

		#ifdef __GNUC__

			TInt zero = 0;

			asm __VOLATILE__ ("mcr p15, 0, %0, c8, c3, 0 " : : "r"(zero));

		#elif defined(__ARMCC__)

			TInt zero = 0;

			asm("mcr p15, 0, zero, c8, c3, 0 ");

		#elif defined(__GCCXML__)

			// empty

		#else

			#error Unknown compiler

		#endif

		__arm_bpiallis();

		__arm_dsb();

		__arm_isb();

	#endif

	}

#if defined(__CPU_ARM1136__) && !defined(__CPU_ARM1136_ERRATUM_424067_FIXED)

#define INVALIDATE_TLB_BY_ASID_BROKEN

#endif

#if defined(__CPU_ARM1176__) && !defined(__CPU_ARM1176_ERRATUM_424692_FIXED)

#define INVALIDATE_TLB_BY_ASID_BROKEN

#endif

__ASSERT_COMPILE(KKernelOsAsid==0); // InvalidateTLBForAsid assumes this

/**

Invalidate all TLB entries which match the given ASID value (current CPU only)

*/

FORCE_INLINE void LocalInvalidateTLBForAsid(TUint aAsid)

	{

#ifndef INVALIDATE_TLB_BY_ASID_BROKEN

	if(aAsid&=0xff)

		{

#if defined(__CPU_CORTEX_A9__) && !defined(__CPU_ARM_A9_ERRATUM_571618_FIXED)

		__arm_dmb();	// ARM Cortex-A9 MPCore erratum 571618 workaround

						// Execute memory barrier before interruptible CP15 operations

#endif

		// invalidate all I+D TLB entries for ASID...

		#ifdef __GNUC__

			asm __VOLATILE__ ("mcr p15, 0, %0, c8, c7, 2 " : : "r"(aAsid));

		#elif defined(__ARMCC__)

			asm("mcr p15, 0, aAsid, c8, c7, 2 ");

		#elif defined(__GCCXML__)

			// empty

		#else

			#error Unknown compiler

		#endif

		}

	else

		// ASID==0 means 'kernel' memory. We have to invalidate the entire TLB here

		// as this is the only way of getting rid of global entries...

#endif

		{

		// invalidate entire TLB...

		#ifdef __GNUC__

			asm __VOLATILE__ ("mcr p15, 0, r0, c8, c7, 0 ");

		#elif defined(__ARMCC__)

			TInt dummy = 0; // damned RVCT

			asm("mcr p15, 0, dummy, c8, c7, 0 ");

		#elif defined(__GCCXML__)

			// empty

		#else

			#error Unknown compiler

		#endif

		}

	__arm_bpiall();

	__arm_dsb();

	__arm_isb();

	}

/**

Invalidate all TLB entries which match the given ASID value on all CPU cores.

*/

FORCE_INLINE void InvalidateTLBForAsid(TUint aAsid)	

	{

	aAsid &= 0xff;

	#ifdef BROADCAST_TLB_MAINTENANCE

		BroadcastInvalidateTLB(aAsid);

	#elif !defined(__SMP__)

		LocalInvalidateTLBForAsid(aAsid);

	#else // __SMP__

		if(aAsid!=0)

			{

#if defined(__CPU_CORTEX_A9__) && !defined(__CPU_ARM_A9_ERRATUM_571618_FIXED)

			__arm_dmb();	// ARM Cortex-A9 MPCore erratum 571618 workaround

							// Execute memory barrier before interruptible CP15 operations

#endif

			// invalidate all I+D TLB entries for ASID...

			#ifdef __GNUC__

				asm __VOLATILE__ ("mcr p15, 0, %0, c8, c3, 2 " : : "r"(aAsid));

			#elif defined(__ARMCC__)

				asm("mcr p15, 0, aAsid, c8, c3, 2 ");

			#elif defined(__GCCXML__)

				// empty

			#else

				#error Unknown compiler

			#endif

			}

		else

			{

			// ASID==0 means 'kernel' memory. We have to invalidate the entire TLB here

			// as this is the only way of getting rid of global entries...

			#ifdef __GNUC__

				asm __VOLATILE__ ("mcr p15, 0, %0, c8, c3, 0 " : : "r"(aAsid));

			#elif defined(__ARMCC__)

				asm("mcr p15, 0, aAsid, c8, c3, 0 ");

			#elif defined(__GCCXML__)

				// empty

			#else

				#error Unknown compiler

			#endif

			}

		__arm_bpiallis();

		__arm_dsb();

		__arm_isb();

	#endif

	}

/**

Return the virtual address of the page directory used for address space

\a aOsAsid. Note, the global page directory is mapped after each

address space specific page director in a way which means that it

appears to be a single contiguous page directory which maps the

entire 32bit virtual address range. I.e. the returned page directory

address can be simply indexed by any virtual address without regard

to whether it belongs to the given address space or lies in the

global region.

*/

FORCE_INLINE TPde* Mmu::PageDirectory(TInt aOsAsid)

	{

	return (TPde*)(KPageDirectoryBase+(aOsAsid<<KPageDirectoryShift));

	}

/**

Return the virtual address of the Page Directory Entry (PDE) used to map

the region containing the virtual address \a aAddress in the address space

\a aOsAsid.

*/

FORCE_INLINE TPde* Mmu::PageDirectoryEntry(TInt aOsAsid, TLinAddr aAddress)

	{

	return PageDirectory(aOsAsid) + (aAddress>>KChunkShift);

	}

/**

Return the physical address mapped by the section mapping contained

in the given Page Directory Entry \a aPde. If \a aPde is not a

section mapping, then KPhysAddrInvalid is returned.

*/

FORCE_INLINE TPhysAddr Mmu::PdePhysAddr(TPde aPde)

	{

	if((aPde&KPdePresentMask)==KArmV6PdeSection)

		return aPde&KPdeSectionAddrMask;

	return KPhysAddrInvalid;

	}

#ifdef __CPU_MEMORY_TYPE_REMAPPING

/*

Bits in a PTE which represent access permissions...

AP2 AP1 AP0		usr	wr

0	0	x		n	y

0	1	x		y	y

1	0	x		n	n

1	1	x		y	n

*/

/**

Modify a Page Table Entry (PTE) value so it restricts access to the memory it maps.

The returned PTE value is the same as \a aPte but with its access permissions set

to read-only if \a aReadOnly is true, and set to allow no access if \a aReadOnly is false.

*/

FORCE_INLINE TPte Mmu::MakePteInaccessible(TPte aPte, TBool aReadOnly)

	{

	__NK_ASSERT_DEBUG((aPte&KArmV6PteTypeMask)!=KArmV6PteLargePage);

	if(aPte&KPtePresentMask)

		{

		__NK_ASSERT_DEBUG((bool)(aPte&KArmV6PteSmallTEX1)==(bool)(aPte&KArmV6PteSmallXN)); // TEX1 should be a copy of XN

		if(aReadOnly)

			aPte |= KArmV6PteAP2; // make read only

		else

			aPte &= ~KPtePresentMask; // make inaccessible

		}

	return aPte;

	}

/**

Modify a Page Table Entry (PTE) value so it allows greater access to the memory it maps.

The returned PTE value is the same as \a aPte but with its access permissions set

to read/write if \a aWrite is true, and set to read-only if \a aWrite is false.

*/

FORCE_INLINE TPte Mmu::MakePteAccessible(TPte aPte, TBool aWrite)

	{

	__NK_ASSERT_DEBUG((aPte&KArmV6PteTypeMask)!=KArmV6PteLargePage);

	if((aPte&KPtePresentMask)==0)

		{

		// wasn't accessible, make it so...

		if(aPte&KArmV6PteSmallTEX1)

			aPte |= KArmV6PteSmallXN; // restore XN by copying from TEX1

		aPte |= KArmV6PteSmallPage;

		aPte |= KArmV6PteAP2; // make read only

		}

	if(aWrite)

		aPte &= ~KArmV6PteAP2; // make writable

	return aPte;

	}

#else // not __CPU_MEMORY_TYPE_REMAPPING

/*

Bits in a PTE which represent access permissions...

AP2 AP1 AP0		usr	wr

0	0	0

0	0	1		n	y

0	1	0

0	1	1		y	y

1	0	0