// Copyright (c) 2005-2009 Nokia Corporation and/or its subsidiary(-ies).
// All rights reserved.
// This component and the accompanying materials are made available
// under the terms of the License "Eclipse Public License v1.0"
// which accompanies this distribution, and is available
// at the URL "http://www.eclipse.org/legal/epl-v10.html".
//
// Initial Contributors:
// Nokia Corporation - initial contribution.
//
// Contributors:
//
// Description:
// eka\kernel\arm\cachel2.cpp
//
// Support for the following external cache controllers:
//	- L210
//	- L220
//	- PL310
//

/* Errata status:
 *
 * General note that applies to all errata:
 * 	External cache configuration in bootstrap is implemented in baseport (usually HardwareInitialise
 *	procedure). Baseport should ensure errata fixes are properly implemented in bootstrap.
 *
 * L210 Errata status:
 *
 * 	- Incorrect IDLE assertion on BUSY to IDLE transaction transition on master ports.
 * 		There is no Software work-around for this erratum.
 * 
 *	- BWABT event only asserted if abort response is received on the last access of buffered write
 *	  or eviction transaction.
 *		There is no Software work-around for this erratum.
 * 
 *	- IDLE can be active while a transaction starts on a master port (HTRANSMx = NSEQ).
 *		There is no Software work-around for this erratum.
 * 
 *	- Abort response can be lost on a nonbufferable write access with master port in 32-bit configuration.
 *		There is no Software work-around for this erratum.
 * 
 *	- Evictions can cause write data from write buffer to be overwritten.
 *		The work-around for this erratum is to use the L210 as a write-through cache only, which
 *		is enforced in baseport.
 * 
 *	- Incorrect eviction buffer request deassertion can cause WAW hazard or outof-date data read.
 *		The work-around for this erratum is to use the L210 as a write-through cache only, which
 *		is enforced in baseport.
 * 
 *	- Read-after-write hazards can be incorrectly handled between write allocation linefills and
 *	  dirty data evictions in three master ports configuration.
 *		The software work-around for this erratum is to change memory regions attributes so that
 *		allocate on read miss only policy is used for all outer write-back regions.
 * 		Note:
 *			Write allocate is supported only for systems using ARMv6 extensions only
 *		Status:
 *			Fix not implemented.
 * 
 *	- Starting a background maintenance operation while a line is sitting in the writeallocate buffer
 *	  can prevent that line from being allocated
 *		The software work-around is to perform an explicit Cache Sync operation before initiating a
 *		background clean or clean-and-invalidate cache maintenance operation.
 *		Status:
 *			Fix implemented unconditionally because Kernel is using background maintenance
 *			operation only during soft restart and performance issue is minimal/nonexistant.
 * 
 *	- The Cache Sync operation does not guarantee that the Eviction Buffer is empty
 * 		From ARM Support:
 *			To insure the error does not occur, you can do a Cache Sync operation followed by a SWP
 *			to non-cacheable memory.
 *		Status:
 *			Fix implemented in Kernel when 
 *				__ARM_L210_ERRATA_CACHESYNC_DOESNT_FLUSH_EVICTION_BUFFER_FIXED
 *			is not defined in variant.mmh
 *
 * L220 Errata status:
 *
 *	- 484863: The Cache Sync operation does not guarantee that the Eviction Buffer is empty.
 *		The software workaround is to do a dummy STR to Non-cacheable Normal Memory before launching
 *		the Cache Sync operation. Baseport (in bootstrap) MUST ensure that memory mapping of
 *		BTP_Uncached entry in BootTable is either:
 * 			- MEMORY_UNCACHED (memory type remapping on) or,
 * 			- CACHE_BUFC	  (memory type remapping off)
 * 		Status:
 *			Fix implemented in Kernel when 
 *				__ARM_L220_ERRATUM_484863_FIXED
 *			is not defined in variant.mmh
 *
 * PL310 Errata status:
 *
 *	- 504326: Prefetching can be done on wrong address or wrong memory area.
 * 		Status:
 * 			No need for the fix in Kernel as it doesn't do prefetching.
 * 			When PL310 is configured in bootstrap, baseport should keep the Instruction
 * 			and Data Prefetch features disabled, as they are by default.
 * 
 * 	- 539766: Non Secure Invalidate by Way maintenance operation does not work properly.
 * 		Description:
 * 			Invalidate by Way in non-secure world may corrupt secure cache lines.
 * 		Status:
 *			Unconditionaly fixed. Kernal maintains only a single way in one go. 
 * 			When intialising PL310, baseport should make sure to invalide only a single
 * 			way in one go.
 * 
 *	- 540921: Parity must be disabled in exclusive cache configuration.
 * 		Status:
 * 			No need for the fix in Kernel as PL310 is initialised by baseport.
 * 			When intialising PL310, baseport should make sure not to enable both 
 * 			parity and exclusive mode.
 * 
 *  - 588369: Clean&Invalidate maintenance operations do not invalidate clean lines.
 * 		Status:
 *			Fix implemented in Kernel when 
 *				__ARM_PL310_ERRATUM_588369_FIXED
 *			is not defined in variant.mmh
 * 			CleanAndInvalidateByPA is replaced by CleanByPA and InvalidateByPA.
 * 			CleanAndInvalidateByIndexWay is made sure never to happen.
 * 			CleanAndInvalidateByWay is made sure never to happen.
 * 			Coherancy problem mentioned in the workaround is not relevant. 
 * 
 *	- 588371: Potential data corruption when an AXI locked access is received while a
 *	  		  prefetch is treated.
 * 		Status:
 * 			No need for the fix in Kernel as PL310 SWP instruction is depricated since ARMv6.
 * 
 *	- 501023: Address Filtering register content cannot be read.
 * 		Status:
 * 			No need for the fix in Kernel it doesn't access access filtering registers.
 * 
 *	- 502117: Address Filtering registers may not have appropriate values out of reset.
 * 		Status:
 * 			No need for the fix in Kernel it doesn't access access filtering registers.
 * 
 *	- 588375: Potential deadlock when evictions or store buffer writes are issued by
 * 			  different master ports.
 * 		Status:
 * 			Not fixed in Kernel as there is no software workaround for this erratum.
 */


#include <arm.h>
#include "cache_maintenance.h"
#include <nkern.h>

#ifdef __HAS_EXTERNAL_CACHE__

//L2 Cache globals
SCacheInfo ExternalCache::Info;
TLinAddr   ExternalCache::Base;
#if defined(__ARM_PL310_CACHE__)
#if defined(__SMP__)
TSpinLock   ExternalCache::iLock(TSpinLock::EOrderCacheMaintenance);
#else
// Any order will do here
TSpinLock   ExternalCache::iLock(TSpinLock::EOrderGenericIrqLow0);
#endif
#endif // defined(__ARM_PL310_CACHE__)


#if defined (__ARM_L220_CACHE__)
// These two macros that deal with disabling/restoring interrupts
// when L220 accesses controll register. They are defined here to
// improve readability of the code below.

#define L220_COMMAND_PREAMBLE                       \
    TInt irq = NKern::DisableAllInterrupts();       \
    while (*ctrlReg & 1) __chill();

#define L220_COMMAND_POSTAMBLE                      \
    NKern::RestoreInterrupts(irq);

#else  // defined (__ARM_L220_CACHE__)
#define L220_COMMAND_PREAMBLE
#define L220_COMMAND_POSTAMBLE
#endif // else defined (__ARM_L220_CACHE__)



#if defined(__ARM_PL310_CACHE__)
// These three macros that deal with pl310 spin lock are
// defined here to improve readability of the code below.
#define PL310_SPIN_LOCK                             \
    TInt lockCounter = KMaxCacheLinesPerSpinLock;   \
    TInt spinLockIrq = Lock();

#define PL310_SPIN_FLASH                            \
    if (!--lockCounter)                             \
        {                                           \
        lockCounter = KMaxCacheLinesPerSpinLock;    \
        FlashLock(spinLockIrq);                     \
        }

#define PL310_SPIN_UNLOCK                           \
    Unlock(spinLockIrq);
    
#else //defined(__ARM_PL310_CACHE__)
#define PL310_SPIN_LOCK
#define PL310_SPIN_FLASH
#define PL310_SPIN_UNLOCK
#endif // else defined(__ARM_PL310_CACHE__)



#define CACHEL2FAULT()	ExternalCacheFault(__LINE__)
void ExternalCacheFault(TInt aLine)
	{
	Kern::Fault("ExternalCacheFault",aLine);
	}

void ExternalCache::Init1()
	{
	TInt waySize = 0; // Number of bytes in a way

	Base = TheSuperPage().iArmL2CacheBase;
	TInt auxCtrl = *(TInt*)(Base+ARML2C_AuxiliaryControl);
	__KTRACE_OPT(KBOOT,Kern::Printf("ExternalCache::Init1 L2CacheBase=%x AuxCtrl reg=%x", Base, auxCtrl));

	//Calculate the number of ways
	switch((auxCtrl & ARML2C_WaySize_Mask) >> ARML2C_WaySize_Shift)
		{
		case 0:		waySize = 0x4000;  break;
		case 1:		waySize = 0x4000;  break;
		case 2:		waySize = 0x8000;  break;
		case 3:		waySize = 0x10000; break;
		case 4:		waySize = 0x20000; break;
#if defined (__ARM_L210_CACHE__) || defined (__ARM_L220_CACHE__)
		default:	waySize = 0x40000; break;
#elif defined(__ARM_PL310_CACHE__)
		case 5:		waySize = 0x40000; break;
		case 6:		waySize = 0x80000; break;
		default:	waySize = 0x80000; break;
#endif		
		}

#if defined (__ARM_L210_CACHE__) || defined (__ARM_L220_CACHE__)
	Info.iAssoc = (auxCtrl & ARML2C_Assoc_Mask)>> ARML2C_Assoc_Shift;
	if (Info.iAssoc > 8)
		Info.iAssoc = 8;
	
#elif defined(__ARM_PL310_CACHE__)
	Info.iAssoc = auxCtrl & ARML2C_Assoc_Mask ? 16 : 8;
#endif
		
	Info.iSize = (waySize * Info.iAssoc) >> 5;	// >>5 as iSize is counted in lines
	Info.iLineLength=32;						// It is always 32
	Info.iLineLenLog2=5;
	Info.iInvalidateThreshold=Info.iSize*8; //if invalidate region >=8*CacheSize, will flush entire cache
	Info.iCleanThreshold=Info.iSize*4; 		//if clean region >=4*CacheSize, will clean entire cache
	Info.iCleanAndInvalidateThreshold=Info.iSize*4; //if flush region >=4*CacheSize, will flush entire cache

	#ifdef _DEBUG
	SCacheInfo& c=Info;
	__KTRACE_OPT(KBOOT,Kern::Printf("External Cache:"));
	__KTRACE_OPT(KBOOT,Kern::Printf("Size %04x LineLength %04x Assoc %04x",c.iSize,c.iLineLength,c.iAssoc));
	__KTRACE_OPT(KBOOT,Kern::Printf("InvalidateThreshold %04x CleanThreshold %04x CleanAndInvalidateThreshold %04x",c.iInvalidateThreshold,c.iCleanThreshold,c.iCleanAndInvalidateThreshold));
	__KTRACE_OPT(KBOOT,Kern::Printf("LineLenLog2 %02x PreemptBlock %02x",c.iLineLenLog2,c.iPreemptBlock));
	#endif
	}

void ExternalCache::Clean(TLinAddr aBase, TUint aSize)
	{
	__KTRACE_OPT(KMMU, Kern::Printf("ExternalCache::Clean base=%xH, size=%xH", aBase, aSize));
	if (aSize>=Info.CleanThresholdBytes())
		Maintain_All((TInt*)(Base+ARML2C_CleanByIndexWay));
	else
		Maintain_Region(aBase, aSize, (TInt*)(Base+ARML2C_CleanLineByPA));

	DrainBuffers();
	}

void ExternalCache::Invalidate(TLinAddr aBase, TUint aSize)
	{
	__KTRACE_OPT(KMMU,Kern::Printf("ExternalCache::Purge base=%xH, size=%xH", aBase, aSize));

#if defined(__ARM_PL310_CACHE__) && !defined(__ARM_PL310_ERRATUM_588369_FIXED)
		//Cannot Clean&Invalidate all cache, so do not bother checking the threshold
		Maintain_Region(aBase, aSize, (TInt*)(Base+ARML2C_InvalidateLineByPA));
#else // defined(__ARM_PL310_CACHE__) && !defined(__ARM_PL310_ERRATUM_588369_FIXED)	
	
	if (aSize>=Info.InvalidateThresholdBytes())
		{
		Maintain_All((TInt*)(Base+ARML2C_CleanInvalidateByIndexWay));
		DrainBuffers();
		}
	else
		{
		Maintain_Region(aBase, aSize, (TInt*)(Base+ARML2C_InvalidateLineByPA));
		}
#endif //else defined(__ARM_PL310_CACHE__) && !defined(__ARM_PL310_ERRATUM_588369_FIXED)
	}

void ExternalCache::CleanAndInvalidate(TLinAddr aBase, TUint aSize)
	{
	__KTRACE_OPT(KMMU,Kern::Printf("ExternalCache::CleanAndInvalidate base=%xH, size=%xH", aBase, aSize));
#if defined(__ARM_PL310_CACHE__) && !defined(__ARM_PL310_ERRATUM_588369_FIXED)
    //Cannot Clean&Invalidate all cache, so do not bother checking the threshold
    Maintain_Region(aBase, aSize, (TInt*)(Base+ARML2C_CleanInvalidateLineByPA));
#else //defined(__ARM_PL310_CACHE__) && !defined(__ARM_PL310_ERRATUM_588369_FIXED)

	if (aSize>=Info.CleanAndInvalidateThresholdBytes())
		Maintain_All((TInt*)(Base+ARML2C_CleanInvalidateByIndexWay));
	else
		Maintain_Region(aBase, aSize, (TInt*)(Base+ARML2C_CleanInvalidateLineByPA));
#endif
	DrainBuffers();
	}

void ExternalCache::AtomicSync()
	{
	CHECK_PRECONDITIONS(MASK_INTERRUPTS_DISABLED,"ExternalCache::AtomicSync");
	__KTRACE_OPT(KMMU,Kern::Printf("ExternalCache::AtomicSync"));

#if defined(__ARM_PL310_CACHE__)
	// On Pl310, we hold the lock while maintaining cache. Therefore, we cannot
	// do that on a way basis as it takes too long to complete.

#if defined(__ARM_PL310_ERRATUM_588369_FIXED)
	Maintain_All((TInt*)(Base+ARML2C_CleanInvalidateByIndexWay));
#else //defined(__ARM_PL310_ERRATUM_588369_FIXED)
	//CleanAndInvalidate is broken. Just clean it.
	Maintain_All((TInt*)(Base+ARML2C_CleanByIndexWay));
#endif //else defined(__ARM_PL310_ERRATUM_588369_FIXED)
	
#else //defined(__ARM_PL310_CACHE__)

	#if defined (__ARM_L210_CACHE__)
	// Flush buffers before starting background clean+invalidate.
	// This is the fix for an erratum (see the top of the file).
	DrainBuffers();
	#endif

#if defined (__ARM_L220_CACHE__)
	{
	// Loop if there is ongoing background operation.
	volatile TInt* cacheSync = (volatile TInt*)(Base+ARML2C_CacheSync);
	while (*cacheSync & 1) __chill();
	}
#endif	
	
	int wayShift;
	// Clean and invalidate cache way-after-the-way. Maintenance of multiple ways at
	// the same time is the subject of numerous errata.
	for (wayShift=1; wayShift < 0x100; wayShift<<=1)//8 loops (for 8 ways), with wayShift values 1,2,4,...,0x80
		{
		volatile TInt* ctrlReg = (volatile TInt*)(Base+ARML2C_CleanInvalidateByWay);
		*ctrlReg = wayShift;
		while(*ctrlReg) __chill();
		}

	DrainBuffers();
#endif
	return;
	}

void ExternalCache::InvalidatePhysicalMemory(TPhysAddr aAddr, TUint aSize)
	{
	if (aSize == 0)
		return;
	
	volatile TInt* ctrlReg = (volatile TInt*)(Base+ARML2C_InvalidateLineByPA);
	TUint32 endAddr = aAddr+aSize; 	// (Exclusive)
	if (endAddr<aAddr)
	    endAddr = 0xffffffff; 

	aAddr &= ~(Info.iLineLength-1); // Align starting address to line length.

    PL310_SPIN_LOCK;
lineLoop:
    L220_COMMAND_PREAMBLE;
	*ctrlReg = aAddr;	//This will invalidate the line from the external cache
	L220_COMMAND_POSTAMBLE;
	PL310_SPIN_FLASH;
	
	aAddr+=Info.iLineLength;
	if (aAddr<endAddr) goto lineLoop;

	PL310_SPIN_UNLOCK;
	}

void ExternalCache::CleanPhysicalMemory(TPhysAddr aAddr, TUint aSize)
	{
	if (aSize == 0)
		return;
	
	volatile TInt* ctrlReg = (volatile TInt*)(Base+ARML2C_CleanLineByPA);
	TUint32 endAddr = aAddr+aSize; 	// (Exclusive)
    if (endAddr<aAddr)
        endAddr = 0xffffffff; 
	aAddr &= ~(Info.iLineLength-1); // Align starting address to line length.

    PL310_SPIN_LOCK;
lineLoop:
    L220_COMMAND_PREAMBLE;
	*ctrlReg = aAddr;	//This will clean the line from the external cache
    L220_COMMAND_POSTAMBLE;
    PL310_SPIN_FLASH;
	
	aAddr+=Info.iLineLength;
	if (aAddr<endAddr) goto lineLoop;
	
    PL310_SPIN_UNLOCK;
	DrainBuffers();
	}

void ExternalCache::CleanAndInvalidatePhysicalMemory(TPhysAddr aAddr, TUint aSize)
	{
	if (aSize == 0)
		return;
	
	TUint32 endAddr = aAddr+aSize; 	// (Exclusive)
    if (endAddr<aAddr)
        endAddr = 0xffffffff; 
	aAddr &= ~(Info.iLineLength-1); // Align starting address to line length.

    PL310_SPIN_LOCK;

#if defined(__ARM_PL310_CACHE__) && !defined(__ARM_PL310_ERRATUM_588369_FIXED)
    volatile TInt* cleanReg = (volatile TInt*)(Base+ARML2C_CleanLineByPA);
	volatile TInt* invalidateReg = (volatile TInt*)(Base+ARML2C_InvalidateLineByPA);

lineLoop:
    TInt ret = NKern::DisableAllInterrupts();
	*cleanReg = aAddr;
	*invalidateReg = aAddr;
    NKern::RestoreInterrupts(ret);

#else // #idefined(__ARM_PL310_CACHE__) && !defined(__ARM_PL310_ERRATUM_588369_FIXED)

	volatile TInt* ctrlReg = (volatile TInt*)(Base+ARML2C_CleanInvalidateLineByPA);
	
lineLoop:
    L220_COMMAND_PREAMBLE;
	*ctrlReg = aAddr;	//This will clean&invalidate the line from the external cache
    L220_COMMAND_POSTAMBLE;
#endif //else defined(__ARM_PL310_CACHE__) && !defined(__ARM_PL310_ERRATUM_588369_FIXED)

    PL310_SPIN_FLASH;
	
	aAddr+=Info.iLineLength;
	if (aAddr<endAddr) goto lineLoop;
	
    PL310_SPIN_UNLOCK;
	DrainBuffers();
	}

void ExternalCache::Maintain_Region(TLinAddr aBase, TUint aSize, TInt* aCtrlReg)
	{
	__KTRACE_OPT(KMMU,Kern::Printf("ExternalCache::Maintain_Region %08xH+%08xH, reg=%xH", aBase, aSize, aCtrlReg));

	if (aSize == 0)
		return;
	
	volatile TInt* ctrlReg = aCtrlReg;
	TInt lineLength = Info.iLineLength;
	//Round down aBase to line length and recalculate aSize accordingly.
	TInt diff = aBase & (lineLength-1);
	aBase -= diff;
	//Ensure size doesn't overflow
	if ( ((TInt)(aSize)<0) && (((TInt)(aSize)+diff)>0) )
		aSize = 0xffffffff;
	else
		aSize += diff;

	// Find the starting physical address. If VA is invalid, try the next page.
findStartingPA:	
	TPhysAddr physAddress = Epoc::LinearToPhysical(aBase);
	if (physAddress == KPhysAddrInvalid)
		{
		__KTRACE_OPT(KMMU,Kern::Printf("ExternalCache::Maintain_Region invalid VA:%xH", aBase));
		diff = KPageSize - (aBase&KPageMask);
		aBase+=diff;
		if ((TInt)aSize>0 && ((TInt)aSize<=diff))
		    return;
		aSize-=diff;
		goto findStartingPA;
		}

	__KTRACE_OPT(KMMU,Kern::Printf("ExternalCache::Maintain_Region VA:%xH > PA:%xH", aBase, physAddress));
	aBase &= ~KPageMask; //The rest of the function expects aBase to be page aligned.

    PL310_SPIN_LOCK;
lineLoop:

#if defined(__ARM_PL310_CACHE__) && !defined(__ARM_PL310_ERRATUM_588369_FIXED)
    if((TInt)aCtrlReg == Base+ARML2C_CleanInvalidateLineByPA)
        {
        // CleanInvalidateLineByPA is broken
        volatile TInt* cleanReg = (volatile TInt*)(Base+ARML2C_CleanLineByPA);
        volatile TInt* invalidateReg = (volatile TInt*)(Base+ARML2C_InvalidateLineByPA);
        TInt ret = NKern::DisableAllInterrupts();
        *cleanReg = physAddress;
        *invalidateReg = physAddress;
        NKern::RestoreInterrupts(ret);
        }
    else
        *ctrlReg = physAddress; //This will clean, purge or flush the line
        
#else
    L220_COMMAND_PREAMBLE;
    *ctrlReg = physAddress;	//This will clean, purge or flush the line
    L220_COMMAND_POSTAMBLE;
#endif
    PL310_SPIN_FLASH;
	
	if ((TInt)aSize>0 && ((TInt)aSize<=lineLength))
		goto endOfRegion;

	aSize=aSize-lineLength;
	physAddress+=lineLength;

	if ((physAddress & KPageMask))
		goto lineLoop;
	
	//Drop here when the end of the page is reached.
endOfPage:	
	aBase += KPageSize;
	physAddress = Epoc::LinearToPhysical(aBase);
	if (physAddress == KPhysAddrInvalid)
		{
		__KTRACE_OPT(KMMU,Kern::Printf("ExternalCache::Maintain_Region Invalid VA %xH", aBase));
		if ((TInt)aSize>0 && ((TInt)aSize<=KPageSize))
			goto endOfRegion;
		aSize-=KPageSize;
		goto endOfPage;
		}		
	__KTRACE_OPT(KMMU,Kern::Printf("ExternalCache::Maintain_Region VA:%xH > PA:%xH", aBase, physAddress));
	goto lineLoop;

endOfRegion:
    PL310_SPIN_UNLOCK;
	return;
	}

void ExternalCache::Maintain_All(TInt* aCtrlReg)
	{
	
	__KTRACE_OPT(KMMU,Kern::Printf("ExternalCache::Maintain_All %xH", aCtrlReg));

#if defined(__ARM_PL310_CACHE__) && !defined(__ARM_PL310_ERRATUM_588369_FIXED)
	//CleanAndInvalidateByIndexWay is broken
	__ASSERT_DEBUG((TInt)aCtrlReg != Base+ARML2C_CleanInvalidateByIndexWay, CACHEL2FAULT());
#endif
	
	TInt indexNo = Info.iSize>>3; // This is the number of cache lines in each way. Assoc is always 8 in this cache
	volatile TInt* ctrlReg = aCtrlReg;

    PL310_SPIN_LOCK;
	TInt way,index;
	for (way = 0 ; way <Info.iAssoc ; way++)
		{
		for (index = 0 ; index <indexNo ; index++)
			{
		    L220_COMMAND_PREAMBLE;
            *ctrlReg = (way<<29) | (index<<5); //this will clean,purge or flush cache line
            L220_COMMAND_POSTAMBLE;
			PL310_SPIN_FLASH;
			}
		}
    PL310_SPIN_UNLOCK;
	}

#if defined(__ARM_PL310_CACHE__)
TInt ExternalCache::Lock()
	{
	return __SPIN_LOCK_IRQSAVE(iLock);
	}

void ExternalCache::FlashLock(TInt aIrq)
	{
	__SPIN_FLASH_IRQRESTORE(iLock, aIrq);
	}
	
void ExternalCache::Unlock(TInt aIrq)
	{
	__SPIN_UNLOCK_IRQRESTORE(iLock, aIrq);
	}
#endif // defined(__ARM_PL310_CACHE__)

#endif // defined (__HAS_EXTERNAL_CACHE__)
