// Copyright (c) 1998-2009 Nokia Corporation and/or its subsidiary(-ies).

// All rights reserved.

// This component and the accompanying materials are made available

// under the terms of the License "Eclipse Public License v1.0"

// which accompanies this distribution, and is available

// at the URL "http://www.eclipse.org/legal/epl-v10.html".

//

// Initial Contributors:

// Nokia Corporation - initial contribution.

//

// Contributors:

//

// Description:

//

#include "memmodel.h"

#include "kernel/cache_maintenance.inl"

#include <kernel/cache.h>

#include <ramalloc.h>

#include <defrag.h>

#include "mm.h"

#include "mmu.h"

#include "mpager.h"

#include "mmapping.h"

#include "mobject.h"

#include "mmanager.h"

#include "mpagearray.h"

//

// SPageInfo

//

// check enough space for page infos...

__ASSERT_COMPILE((KPageInfoLinearEnd-KPageInfoLinearBase)/sizeof(SPageInfo)==(1<<(32-KPageShift)));

// check KPageInfoShift...

__ASSERT_COMPILE(sizeof(SPageInfo)==(1<<KPageInfoShift));

SPageInfo* SPageInfo::SafeFromPhysAddr(TPhysAddr aAddress)

	{

	__NK_ASSERT_DEBUG((aAddress&KPageMask)==0);

	TUint index = aAddress>>(KPageShift+KPageShift-KPageInfoShift);

	TUint flags = ((TUint8*)KPageInfoMap)[index>>3];

	TUint mask = 1<<(index&7);

	if(!(flags&mask))

		return 0; // no SPageInfo for aAddress

	SPageInfo* info = FromPhysAddr(aAddress);

	if(info->iType==SPageInfo::EInvalid)

		return 0;

	return info;

	}

#ifdef _DEBUG

void SPageInfo::CheckAccess(const char* aMessage, TUint aFlags)

	{

	if(K::Initialising || NKern::Crashed())

		return;

	if((aFlags&ECheckNotAllocated) && (iType!=EUnknown))

		{

		Kern::Printf("SPageInfo[0x%08x]::CheckAccess failed, PhysAddr()=0x%08x, iType==%d : %s",this,PhysAddr(),iType,aMessage);

		__NK_ASSERT_DEBUG(0);

		goto fail;

		}

	if((aFlags&ECheckNotUnused) && (iType==EUnused))

		{

		Kern::Printf("SPageInfo[0x%08x]::CheckAccess failed, PhysAddr()=0x%08x, iType==%d : %s",this,PhysAddr(),iType,aMessage);

		__NK_ASSERT_DEBUG(0);

		goto fail;

		}

	if((aFlags&ECheckUnused) && (iType!=EUnused))

		{

		Kern::Printf("SPageInfo[0x%08x]::CheckAccess failed, PhysAddr()=0x%08x, iType==%d : %s",this,PhysAddr(),iType,aMessage);

		__NK_ASSERT_DEBUG(0);

		goto fail;

		}

	if((aFlags&ECheckNotPaged) && (iPagedState!=EUnpaged))

		{

		Kern::Printf("SPageInfo[0x%08x]::CheckAccess failed, PhysAddr()=0x%08x, iPagedState=%d : %s",this,PhysAddr(),iPagedState,aMessage);

		__NK_ASSERT_DEBUG(0);

		goto fail;

		}

	if((aFlags&ECheckRamAllocLock) && !RamAllocLock::IsHeld())

		{

		Kern::Printf("SPageInfo[0x%08x]::CheckAccess failed, PhysAddr()=0x%08x, iType==%d : %s",this,PhysAddr(),iType,aMessage);

		__NK_ASSERT_DEBUG(0);

		goto fail;

		}

	if((aFlags&ENoCheckMmuLock) || MmuLock::IsHeld())

		return;

fail:

	Kern::Printf("SPageInfo[0x%08x]::CheckAccess failed, PhysAddr()=0x%08x : %s",this,PhysAddr(),aMessage);

	Mmu::Panic(Mmu::EUnsafePageInfoAccess);

	}

void SPageInfo::Dump()

	{

	Kern::Printf("SPageInfo for page %x = %d,%d,%02x,0x%08x,0x%x,%d",PhysAddr(),iType,iPagedState,iFlags,iOwner,iIndex,iPinCount);

	}

#endif

//

// SPageTableInfo

//

// check enough space for page table infos...

__ASSERT_COMPILE((KPageTableInfoEnd-KPageTableInfoBase)/sizeof(SPageTableInfo)

					>=(KPageTableEnd-KPageTableBase)/KPageTableSize);

// check KPtBlockShift...

__ASSERT_COMPILE((sizeof(SPageTableInfo)<<KPtBlockShift)==KPageSize);

#ifdef _DEBUG

TBool SPageTableInfo::CheckPageCount()

	{

	__NK_ASSERT_DEBUG(MmuLock::IsHeld());

	TPte* pt = PageTable();

	TUint realCount = 0;

	do if(*pt++) ++realCount;

	while(TLinAddr(pt)&(KPageTableMask/sizeof(TPte)*sizeof(TPte)));

	if(iPageCount==realCount)

		return true;

	Kern::Printf("CheckPageCount Failed: pt=0x%08x count=%d realCount=%d",TLinAddr(pt)-KPageTableSize,iPageCount,realCount);

	return false;

	}

void SPageTableInfo::CheckChangeUse(const char* aName)

	{

	if(K::Initialising)

		return;

	if(PageTablesLockIsHeld() && MmuLock::IsHeld())

		return;

	Kern::Printf("SPageTableInfo::CheckChangeUse failed : %s",aName);

	Mmu::Panic(Mmu::EUnsafePageTableInfoAccess);

	}

void SPageTableInfo::CheckCheckUse(const char* aName)

	{

	if(K::Initialising)

		return;

	if(PageTablesLockIsHeld() || MmuLock::IsHeld())

		return;

	Kern::Printf("SPageTableInfo::CheckCheckUse failed : %s",aName);

	Mmu::Panic(Mmu::EUnsafePageTableInfoAccess);

	}

void SPageTableInfo::CheckAccess(const char* aName)

	{

	if(K::Initialising)

		return;

	if(MmuLock::IsHeld())

		return;

	Kern::Printf("SPageTableInfo::CheckAccess failed : %s",aName);

	Mmu::Panic(Mmu::EUnsafePageTableInfoAccess);

	}

void SPageTableInfo::CheckInit(const char* aName)

	{

	if(K::Initialising)

		return;

	if(PageTablesLockIsHeld() && iType==EUnused)

		return;

	Kern::Printf("SPageTableInfo::CheckInit failed : %s",aName);

	Mmu::Panic(Mmu::EUnsafePageTableInfoAccess);

	}

#endif

//

// RamAllocLock

//

_LIT(KLitRamAlloc,"RamAlloc");

_LIT(KLitPhysMemSync,"PhysMemSync");

void RamAllocLock::Lock()

	{

	Mmu& m = TheMmu;

	Kern::MutexWait(*m.iRamAllocatorMutex);

	if(!m.iRamAllocLockCount++)

		{

		// first lock, so setup memory fail data...

		m.iRamAllocFailed = EFalse;

		__NK_ASSERT_DEBUG(m.iRamAllocInitialFreePages==m.FreeRamInPages()); // free RAM shouldn't have changed whilst lock was held

		}

	}

void RamAllocLock::Unlock()

	{

	Mmu& m = TheMmu;

	if(--m.iRamAllocLockCount)

		{

		Kern::MutexSignal(*m.iRamAllocatorMutex);

		return;

		}

	TBool failed = m.iRamAllocFailed;

	TUint initial = m.iRamAllocInitialFreePages;

	TUint final = m.FreeRamInPages();

	m.iRamAllocInitialFreePages = final; // new baseline value

	TUint changes = K::CheckFreeMemoryLevel(initial*KPageSize,final*KPageSize,failed);

	if(changes)

		{

		__KTRACE_OPT(KMMU,Kern::Printf("RamAllocLock::Unlock() changes=%x",changes));

		}

	Kern::MutexSignal(*m.iRamAllocatorMutex);

	}

TBool RamAllocLock::Flash()

	{

	Unlock();

	Lock();

	return true; // lock was released

	}

TBool RamAllocLock::IsHeld()

	{

	Mmu& m = TheMmu;

	return m.iRamAllocatorMutex->iCleanup.iThread == &Kern::CurrentThread() && m.iRamAllocLockCount;

	}

//

// MmuLock

//

#ifdef _DEBUG

TUint MmuLock::UnlockGuardNest =0;

TUint MmuLock::UnlockGuardFail =0;

#endif

NFastMutex MmuLock::iLock;

void MmuLock::Lock()

	{

	NKern::FMWait(&iLock);

	}

void MmuLock::Unlock()

	{

	UnlockGuardCheck();

	NKern::FMSignal(&iLock);

	}

TBool MmuLock::Flash()

	{

	UnlockGuardCheck();

	return NKern::FMFlash(&iLock);

	}

TBool MmuLock::IsHeld()

	{

	NFastMutex& m = iLock;

	return m.HeldByCurrentThread();

	}

//

// Initialisation

//

Mmu TheMmu;

void Mmu::Init1Common()

	{

	__KTRACE_OPT2(KBOOT,KMMU,Kern::Printf("Mmu::Init1Common"));

	// Mmu data

	TUint pteType = PteType(ESupervisorReadWrite,true);

	iTempPteCached = BlankPte((TMemoryAttributes)(EMemoryAttributeNormalCached|EMemoryAttributeDefaultShareable),pteType);

	iTempPteUncached = BlankPte((TMemoryAttributes)(EMemoryAttributeNormalUncached|EMemoryAttributeDefaultShareable),pteType);

	iTempPteCacheMaintenance = BlankPte((TMemoryAttributes)(CacheMaintenance::TemporaryMapping()|EMemoryAttributeDefaultShareable),pteType);

	// other

	PP::MaxUserThreadStack=0x14000;			// 80K - STDLIB asks for 64K for PosixServer!!!!

	PP::UserThreadStackGuard=0x2000;		// 8K

	PP::MaxStackSpacePerProcess=0x200000;	// 2Mb

	K::SupervisorThreadStackSize=0x1000;	// 4K

	PP::SupervisorThreadStackGuard=0x1000;	// 4K

	K::MachineConfig=(TMachineConfig*)KMachineConfigLinAddr;

	PP::RamDriveStartAddress=0;

	PP::RamDriveRange=0;

	PP::RamDriveMaxSize=0x20000000;	// 512MB, probably will be reduced later

	K::MemModelAttributes=EMemModelTypeFlexible|EMemModelAttrNonExProt|EMemModelAttrKernProt|EMemModelAttrWriteProt|

						EMemModelAttrVA|EMemModelAttrProcessProt|EMemModelAttrSameVA|EMemModelAttrSvKernProt|

						EMemModelAttrIPCKernProt|EMemModelAttrRamCodeProt;

	}

#ifdef FMM_VERIFY_RAM

// Attempt to write to each unused RAM page and verify the contents.

void Mmu::VerifyRam()

	{

	Kern::Printf("Mmu::VerifyRam() pass 1");

	RamAllocLock::Lock();

	TPhysAddr p = 0;

	do

		{

		SPageInfo* pi = SPageInfo::SafeFromPhysAddr(p);

		if(pi)

			{

			Kern::Printf("%08x %d",p,pi->Type());

			if(pi->Type()==SPageInfo::EUnused)

				{

				volatile TPhysAddr* b = (volatile TPhysAddr*)MapTemp(p,0);

				b[0] = p;

				b[1] = ~p;

				__NK_ASSERT_DEBUG(b[0]==p);

				__NK_ASSERT_DEBUG(b[1]==~p);

				UnmapTemp();

				}

			}

		p += KPageSize;

		}

	while(p);

	TBool fail = false;

	Kern::Printf("Mmu::VerifyRam() pass 2");

	do

		{

		SPageInfo* pi = SPageInfo::SafeFromPhysAddr(p);

		if(pi)

			{

			if(pi->Type()==SPageInfo::EUnused)

				{

				volatile TPhysAddr* b = (volatile TPhysAddr*)MapTemp(p,0);

				if(b[0]!=p || b[1]!=~p)

					{

					fail = true;

					Kern::Printf("%08x FAILED %x %x",b[0],b[1]);

					}

				UnmapTemp();

				}

			}

		p += KPageSize;

		}

	while(p);

	__NK_ASSERT_DEBUG(!fail);

	RamAllocLock::Unlock();

	}

#endif

void Mmu::Init2Common()

	{

	__KTRACE_OPT2(KBOOT,KMMU,Kern::Printf("Mmu::Init2Common"));

	// create allocator...

	const SRamInfo& info = *(const SRamInfo*)TheSuperPage().iRamBootData;

	iRamPageAllocator = DRamAllocator::New(info, iRamZones, iRamZoneCallback);

	// initialise all pages in banks as unused...

	const SRamBank* bank = info.iBanks;

	while(bank->iSize)

		{

		TUint32 base = bank->iBase;

		TUint32 size = bank->iSize;

		__KTRACE_OPT2(KBOOT,KMMU,Kern::Printf("Found RAM bank 0x%08x size %d",base,size));

		if(base+size<=base || ((base|size)&KPageMask))

			Panic(EInvalidRamBankAtBoot);

		SPageInfo* pi = SPageInfo::FromPhysAddr(base);

		SPageInfo* piEnd = pi+(size>>KPageShift);

		while(pi<piEnd)

			(pi++)->SetUnused();

		++bank;

		}

	// step over the last bank to get to the reserved banks.

	++bank;

	// mark any reserved regions as allocated...

	while(bank->iSize)

		{

		TUint32 base = bank->iBase;

		TUint32 size = bank->iSize;

		__KTRACE_OPT2(KBOOT,KMMU,Kern::Printf("Found reserved bank 0x%08x size %d",base,size));

		if(base+size<=base || ((base|size)&KPageMask))

			Panic(EInvalidReservedBankAtBoot);

		SPageInfo* pi = SPageInfo::FromPhysAddr(base);

		SPageInfo* piEnd = pi+(size>>KPageShift);

		while(pi<piEnd)

			(pi++)->SetPhysAlloc();

		++bank;

		}

	// Clear the inital (and only so far) page table info page so all unused

	// page tables infos will be marked as unused.

	__ASSERT_COMPILE(SPageTableInfo::EUnused == 0);

	memclr((TAny*)KPageTableInfoBase, KPageSize);

	// look for page tables - assume first page table maps page tables

	TPte* pPte = (TPte*)KPageTableBase;

	TInt i;

	for(i=0; i<KChunkSize/KPageSize; ++i)

		{

		TPte pte = *pPte++;

		if(pte==KPteUnallocatedEntry)	// after boot, page tables are contiguous

			break;

		TPhysAddr ptpgPhys = Mmu::PtePhysAddr(pte,i);

		__KTRACE_OPT(KBOOT,Kern::Printf("Page Table Group %08x -> Phys %08x", KPageTableBase+i*KPageSize, ptpgPhys));

		SPageInfo* pi = SPageInfo::SafeFromPhysAddr(ptpgPhys);

		__ASSERT_ALWAYS(pi, Panic(EInvalidPageTableAtBoot));

		pi->SetFixed(i); // this also sets the SPageInfo::iOffset so that linear-to-physical works

		}

	// look for mapped pages

	TPde* pd = Mmu::PageDirectory(KKernelOsAsid);

	for(i=0; i<(1<<(32-KChunkShift)); ++i)

		{

		TPde pde = pd[i];

		if(pde==KPdeUnallocatedEntry)

			continue;

		TPhysAddr pdePhys = Mmu::PdePhysAddr(pde);

		TPte* pt = 0;

		if(pdePhys!=KPhysAddrInvalid)

			{

			__KTRACE_OPT(KBOOT,Kern::Printf("Addr %08x -> Whole PDE Phys %08x", i<<KChunkShift, pdePhys));

			}

		else

			{

			pt = Mmu::PageTableFromPde(pde);

			__KTRACE_OPT(KBOOT,Kern::Printf("Addr %08x -> page table %08x", i<<KChunkShift, pt));

			__ASSERT_ALWAYS(pt,Panic(EInvalidPdeAtBoot)); // bad PDE

			}

		TInt j;

		TInt np = 0;

		for(j=0; j<KChunkSize/KPageSize; ++j)

			{

			TBool present = ETrue;	// all pages present if whole PDE mapping

			TPte pte = 0;

			if(pt)

				{

				pte = pt[j];

				present = pte!=KPteUnallocatedEntry;

				}

			if(present)

				{

				++np;

				TPhysAddr pa = pt ? Mmu::PtePhysAddr(pte,j) : (pdePhys + (j<<KPageShift));

				SPageInfo* pi = SPageInfo::SafeFromPhysAddr(pa);

				__KTRACE_OPT(KBOOT,Kern::Printf("Addr: %08x PA=%08x",

													(i<<KChunkShift)+(j<<KPageShift), pa));

				if(pi)	// ignore non-RAM mappings

					{

					TInt r = iRamPageAllocator->MarkPageAllocated(pa, EPageFixed);

					// allow KErrAlreadyExists since it's possible that a page is doubly mapped

					__ASSERT_ALWAYS(r==KErrNone || r==KErrAlreadyExists, Panic(EBadMappedPageAfterBoot));

					if(pi->Type()==SPageInfo::EUnused)

						pi->SetFixed();

#ifdef BTRACE_KERNEL_MEMORY

					if(r == KErrNone)

						++Epoc::KernelMiscPages;

#endif

					}

				}

			}

		__KTRACE_OPT(KBOOT,Kern::Printf("Addr: %08x #PTEs=%d",(i<<KChunkShift),np));

		if(pt)

			{

			SPageTableInfo* pti = SPageTableInfo::FromPtPtr(pt);

			pti->Boot(np);

			}

		}

	TInt r = K::MutexCreate(iRamAllocatorMutex, KLitRamAlloc, NULL, EFalse, KMutexOrdRamAlloc);

	if(r!=KErrNone)

		Panic(ERamAllocMutexCreateFailed);

	iRamAllocLockCount = 0;

	iRamAllocInitialFreePages = FreeRamInPages();

	__KTRACE_OPT2(KBOOT,KMMU,Kern::Printf("Mmu::DoInit2"));

	for(i=0; i<KNumTempMappingSlots; ++i)

		iTempMap[i].Alloc(1);

	iPhysMemSyncTemp.Alloc(1);

	r = K::MutexCreate(iPhysMemSyncMutex, KLitPhysMemSync, NULL, EFalse, KMutexOrdSyncPhysMem);

	if(r!=KErrNone)

		Panic(EPhysMemSyncMutexCreateFailed);

#ifdef FMM_VERIFY_RAM

	VerifyRam();

#endif

	}

void Mmu::Init2FinalCommon()

	{

	__KTRACE_OPT2(KBOOT,KMMU,Kern::Printf("Mmu::Init2FinalCommon"));

	// Reduce free memory to <2GB...

	while(FreeRamInPages()>=0x80000000/KPageSize)

		{

		TPhysAddr dummyPage;

		TInt r = iRamPageAllocator->AllocRamPages(&dummyPage,1, EPageFixed);

		__NK_ASSERT_ALWAYS(r==KErrNone);

		}

	// Reduce total RAM to <2GB...

	if(TheSuperPage().iTotalRamSize<0)

		TheSuperPage().iTotalRamSize = 0x80000000-KPageSize;