// Copyright (c) 2005-2009 Nokia Corporation and/or its subsidiary(-ies).

// All rights reserved.

// This component and the accompanying materials are made available

// under the terms of the License "Eclipse Public License v1.0"

// which accompanies this distribution, and is available

// at the URL "http://www.eclipse.org/legal/epl-v10.html".

//

// Initial Contributors:

// Nokia Corporation - initial contribution.

//

// Contributors:

//

// Description:

// e32\nkernsmp\sched.cpp

// 

//

// NThreadBase member data

#define __INCLUDE_NTHREADBASE_DEFINES__

// TDfc member data

#define __INCLUDE_TDFC_DEFINES__

#include "nk_priv.h"

#include <nk_irq.h>

TSpinLock	NEventHandler::TiedLock(TSpinLock::EOrderEventHandlerTied);

const TUint8 KClassFromPriority[KNumPriorities] =

	{

	0,	0,	0,	0,	0,	0,	0,	0,				// priorities 0-7

	0,	0,	0,	0,	1,	1,	1,	1,				// priorities 8-15

	2,	2,	2,	2,	2,	2,	2,	2,				// priorities 16-23

	2,	2,	2,	3,	3,	3,	3,	3,				// priorities 24-31

	3,	3,	3,	3,	3,	3,	3,	3,				// priorities 32-39

	3,	3,	3,	3,	3,	3,	3,	3,				// priorities 40-47

	3,	3,	3,	3,	3,	3,	3,	3,				// priorities 48-55

	3,	3,	3,	3,	3,	3,	3,	3				// priorities 56-63

	};

/******************************************************************************

 * TScheduler

 ******************************************************************************/

// TScheduler resides in .bss so other fields are zero-initialised

TScheduler::TScheduler()

	:	iThreadAcceptCpus(1),	// only boot CPU for now

		iIpiAcceptCpus(1),		// only boot CPU for now

		iGenIPILock(TSpinLock::EOrderGenericIPIList),

		iIdleBalanceLock(TSpinLock::EOrderEnumerate),

		iIdleSpinLock(TSpinLock::EOrderIdleDFCList),

		iCpusNotIdle(1),	// only boot CPU for now

		iEnumerateLock(TSpinLock::EOrderEnumerate),

		iBalanceListLock(TSpinLock::EOrderReadyList),

		iBalanceTimer(&BalanceTimerExpired, this, 1),

		iCCSyncIDFC(&CCSyncDone, 0),

		iCCReactivateDfc(&CCReactivateDfcFn, this, 3),

		iCCRequestLevel(1),		// only boot CPU for now

		iCCRequestDfc(&CCRequestDfcFn, this, 2),

		iCCPowerDownDfc(&CCIndirectPowerDown, this, 0),

	{

	TInt i;

	for (i=0; i<KMaxCpus; ++i)

		{

		TSubScheduler* s = TheSubSchedulers + i;

		iSub[i] = s;

		s->iScheduler = this;

		s->iCpuNum = TUint32(i);

		s->iCpuMask = 1u<<i;

		s->iLbCounter = TUint8(NSchedulable::ELbState_PerCpu + i);

		}

	iLbCounter = (TUint8)NSchedulable::ELbState_Global;

	iNeedBal = 1;	// stop anyone trying to kick rebalancer before it has been created

	}

/** Return a pointer to the scheduler

	Intended for use by the crash debugger, not for general device driver use.

	@return	Pointer to the scheduler object

	@internalTechnology

 */

EXPORT_C TScheduler* TScheduler::Ptr()

	{

	return &TheScheduler;

	}

/******************************************************************************

 * TSubScheduler

 ******************************************************************************/

// TSubScheduler resides in .bss so other fields are zero-initialised

TSubScheduler::TSubScheduler()

	:	iExIDfcLock(TSpinLock::EOrderExIDfcQ),

		iReadyListLock(TSpinLock::EOrderReadyList),

		iKernLockCount(1),

		iEventHandlerLock(TSpinLock::EOrderEventHandlerList)

	{

	}

void TSubScheduler::SSAddEntry(NSchedulable* aEntry)

	{

	if (aEntry->iParent!=aEntry || !((NThreadBase*)aEntry)->i_NThread_Initial)

		{

		TInt c = KClassFromPriority[aEntry->iPriority];

		++iPriClassThreadCount[c];

		++iRdyThreadCount;

		}

	iSSList.Add(aEntry);

	}

void TSubScheduler::SSAddEntryHead(NSchedulable* aEntry)

	{

	if (aEntry->iParent!=aEntry || !((NThreadBase*)aEntry)->i_NThread_Initial)

		{

		TInt c = KClassFromPriority[aEntry->iPriority];

		++iPriClassThreadCount[c];

		++iRdyThreadCount;

		}

	iSSList.AddHead(aEntry);

	}

void TSubScheduler::SSRemoveEntry(NSchedulable* aEntry)

	{

	if (aEntry->iParent!=aEntry || !((NThreadBase*)aEntry)->i_NThread_Initial)

		{

		TInt c = KClassFromPriority[aEntry->iPriority];

		--iPriClassThreadCount[c];

		--iRdyThreadCount;

		}

	iSSList.Remove(aEntry);

	}

void TSubScheduler::SSChgEntryP(NSchedulable* aEntry, TInt aNewPriority)

	{

	if (aEntry->iParent!=aEntry || !((NThreadBase*)aEntry)->i_NThread_Initial)

		{

		TInt c0 = KClassFromPriority[aEntry->iPriority];

		TInt c1 = KClassFromPriority[aNewPriority];

		if (c0 != c1)

			{

			--iPriClassThreadCount[c0];

			++iPriClassThreadCount[c1];

			}

		}

	iSSList.ChangePriority(aEntry, aNewPriority);

	}

/******************************************************************************

 * NSchedulable

 ******************************************************************************/

TUint32 NSchedulable::PreprocessCpuAffinity(TUint32 aAffinity)

	{

	if (!(aAffinity & NTHREADBASE_CPU_AFFINITY_MASK))

		return aAffinity;

	TUint32 x = aAffinity & ~NTHREADBASE_CPU_AFFINITY_MASK;

	if (x & (x-1))

		return aAffinity;

	return __e32_find_ls1_32(x);

	}

void NSchedulable::AcqSLock()

	{

	iSSpinLock.LockOnly();

	if (iParent!=this && iParent)

		iParent->AcqSLock();

	}

void NSchedulable::RelSLock()

	{

	if (iParent!=this && iParent)

		iParent->RelSLock();

	iSSpinLock.UnlockOnly();

	}

void NSchedulable::LAcqSLock()

	{

	NKern::Lock();

	AcqSLock();

	}

void NSchedulable::RelSLockU()

	{

	RelSLock();

	NKern::Unlock();

	}

void NSchedulable::UnPauseT()

	{

	CHECK_PRECONDITIONS(MASK_KERNEL_LOCKED|MASK_NOT_ISR,"NSchedulable::UnPauseT");

	__KTRACE_OPT(KNKERN,DEBUGPRINT("%T nUnPauseT",this));

	__NK_ASSERT_DEBUG(iPauseCount);

	if (--iPauseCount || iReady || iSuspended || (iParent && ((NThread*)this)->iWaitState.ThreadIsBlocked()))

		return;

	ReadyT(EUnPause);

	}

void NSchedulable::DeferredReadyIDfcFn(TAny* aPtr)

	{

	NSchedulable* a = (NSchedulable*)aPtr;

	a->AcqSLock();

	TUint32 evs = __e32_atomic_and_acq32(&a->iEventState, ~EDeferredReady);

	if (evs & EDeferredReady)

		{

		if (a->iParent)

			{

			// thread

			a->UnPauseT();

			}

		else

			{

			// thread group

			NThreadGroup* g = (NThreadGroup*)a;

			__KTRACE_OPT(KNKERN,DEBUGPRINT("%G nDeferredReady",g));

			__NK_ASSERT_DEBUG(g->iPauseCount);

			if (--g->iPauseCount && g->iNThreadList.NonEmpty())

				g->ReadyT(EUnPause);

			}

		}

	a->RelSLock();

	}

TInt NSchedulable::AddTiedEvent(NEventHandler* aEvent)

	{

	__KTRACE_OPT(KNKERN,DEBUGPRINT("%T AddEv %08x",this,aEvent));

	TInt r = KErrGeneral;

	NEventHandler::TiedLock.LockOnly();

	AcqSLock();

	if (iStopping)

		r = KErrDied;

	else if (!aEvent->iTied)

		{

		aEvent->iTied = this;

		iEvents.Add(&aEvent->iTiedLink);

		r = KErrNone;

		}

	RelSLock();

	NEventHandler::TiedLock.UnlockOnly();

	return r;

	}

void ipi_dummy(TGenericIPI*)

	{

	}

/** Detach and cancel any tied events attached to this thread/group

Call in a thread context with interrupts and preemption enabled.

Calling thread in critical section.

@internalComponent

*/

void NSchedulable::DetachTiedEvents()

	{

	__KTRACE_OPT(KNKERN,DEBUGPRINT("%T DetTiedEv",this));

	NKern::Lock();

	NEventHandler::TiedLock.LockOnly();

	AcqSLock();

	iStopping = TRUE;

	if (!iParent)

		{

		// can't destroy a group until all threads have detached from it

		NThreadGroup* g = (NThreadGroup*)this;

		__NK_ASSERT_ALWAYS(g->iThreadCount==0 && g->iNThreadList.IsEmpty());

		}

	RelSLock();

	NEventHandler::TiedLock.UnlockOnly();

	// send IPI to all processors to synchronise

	// after this, any tied IDFCs can only proceed to completion

	// they can't be queued again

	TGenericIPI ipi;

	ipi.QueueAllOther(&ipi_dummy);

	NKern::Unlock();

	ipi.WaitCompletion();

	FOREVER

		{

		NKern::Lock();

		NEventHandler::TiedLock.LockOnly();

		AcqSLock();

		NEventHandler* h = 0;

		TInt type = -1;

		if (!iEvents.IsEmpty())

			{

			h = _LOFF(iEvents.First()->Deque(), NEventHandler, iTiedLink);

			h->iTiedLink.iNext = 0;

			type = h->iHType;

			}

		RelSLock();

		if (type == NEventHandler::EEventHandlerNTimer)

			{

			// everything's easy for a timer since we can just cancel it here

			NTimer* tmr = (NTimer*)h;

			tmr->DoCancel(NTimer::ECancelDestroy);

			tmr->iTied = 0;

			}

		else if (type == NEventHandler::EEventHandlerIDFC)

			{

			// can just cancel the IDFC with TiedLock held

			// EndTiedEvent() may be delayed, but we wait for that further down

			// iTied will have been captured before the IDFC state is reset

			// Cancel() waits for the state to be reset

			TDfc* d = (TDfc*)h;

			d->Cancel();

			d->iHType = (TUint8)NEventHandler::EEventHandlerDummy;

			d->iTied = 0;

			}

		NEventHandler::TiedLock.UnlockOnly();

		NKern::Unlock();

		if (!h)

			break;

		switch (type)

			{

			case NEventHandler::EEventHandlerIrq:

				{

				NIrqHandler* pH = (NIrqHandler*)h;

				// pH can't have been freed since we dequeued it but left iTied set

				pH->Unbind(pH->iHandle, this);

				break;

				}

			case NEventHandler::EEventHandlerNTimer:

			case NEventHandler::EEventHandlerIDFC:

			case NEventHandler::EEventHandlerDummy:

				// nothing left to do

				break;

			default:

				__NK_ASSERT_ALWAYS(0);

				break;

			}

		}

	// Wait for any remaining tied event handlers to complete

	while (iEventState & EEventCountMask)

		{

		__chill();

		}

	}

/** Return the total CPU time so far used by the specified thread.

	@return The total CPU time in units of 1/NKern::CpuTimeMeasFreq().

*/

EXPORT_C TUint64 NKern::ThreadCpuTime(NThread* aThread)

	{

	NSchedulable::SCpuStats stats;

	NKern::Lock();

	aThread->GetCpuStats(NSchedulable::E_RunTime, stats);

	NKern::Unlock();

	return stats.iRunTime;

	}

void NSchedulable::GetCpuStats(TUint aMask, NSchedulable::SCpuStats& aOut)

	{

	AcqSLock();

	GetCpuStatsT(aMask, aOut);

	RelSLock();

	}

void NSchedulable::GetCpuStatsT(TUint aMask, NSchedulable::SCpuStats& aOut)

	{

	TSubScheduler* ss = 0;

	NThread* t = 0;

	TBool initial = FALSE;

	if (!IsGroup())

		t = (NThread*)this;

	if (t && t->i_NThread_Initial)

		ss = &TheSubSchedulers[iLastCpu], initial = TRUE;

	else if (iReady)

		{

		if (IsGroup())

			ss = &TheSubSchedulers[iReady & NSchedulable::EReadyCpuMask];

		else if (iParent->iReady)

			ss = &TheSubSchedulers[iParent->iReady & NSchedulable::EReadyCpuMask];

		}

	if (ss)

		ss->iReadyListLock.LockOnly();

	TUint64 now = NKern::Timestamp();

	if (aMask & (E_RunTime|E_RunTimeDelta))

		{

		aOut.iRunTime = iTotalCpuTime.i64;

		if (iCurrent || (initial && !ss->iCurrentThread))

			aOut.iRunTime += (now - ss->iLastTimestamp.i64);

		if (aMask & E_RunTimeDelta)

			{

			aOut.iRunTimeDelta = aOut.iRunTime - iSavedCpuTime.i64;

			iSavedCpuTime.i64 = aOut.iRunTime;

			}

		}

	if (aMask & (E_ActiveTime|E_ActiveTimeDelta))

		{

		aOut.iActiveTime = iTotalActiveTime.i64;

		if (iActiveState)

			aOut.iActiveTime += (now - iLastActivationTime.i64);

		if (aMask & E_ActiveTimeDelta)

			{

			aOut.iActiveTimeDelta = aOut.iActiveTime - iSavedActiveTime.i64;

			iSavedActiveTime.i64 = aOut.iActiveTime;

			}

		}

	if (aMask & E_LastRunTime)

		{

		if (iCurrent)

			aOut.iLastRunTime = 0;

		else

			aOut.iLastRunTime = now - iLastRunTime.i64;

		}

	if (aMask & E_LastActiveTime)

		{

		if (iActiveState)

			aOut.iLastActiveTime = 0;

		else

			aOut.iLastActiveTime = now - iLastRunTime.i64;

		}

	if (ss)

		ss->iReadyListLock.UnlockOnly();

	}

/******************************************************************************

 * NThreadGroup

 ******************************************************************************/

/******************************************************************************

 * NThreadBase

 ******************************************************************************/

/** Makes a nanothread ready.

	For use by RTOS personality layers.

	@pre	Kernel must be locked.

	@pre	Call either in a thread or an IDFC context.

	@pre	The thread being made ready must not be explicitly suspended

	@post	Kernel is locked.

 */

void NSchedulable::ReadyT(TUint aMode)

	{

	CHECK_PRECONDITIONS(MASK_KERNEL_LOCKED|MASK_NOT_ISR,"NSchedulable::ReadyT");

	__KTRACE_OPT(KNKERN,DEBUGPRINT("%T nReadyT(%x)",this,aMode));

	NThreadBase* t = (NThreadBase*)this;

	if (iParent && !iActiveState)

		{

		iActiveState=1;

		iLastActivationTime.i64 = NKern::Timestamp();

		if (iParent!=this && ++iParent->iActiveState==1)

			iParent->iLastActivationTime.i64 = iLastActivationTime.i64;

		}

#ifdef _DEBUG

	if (!iParent)

		t = (NThreadBase*)0xface0fff;

#endif

	__NK_ASSERT_DEBUG(!iReady && (!iParent || (!t->iWaitState.iWtC.iWtStFlags && !t->iSuspended)));

	TSubScheduler& ss0 = SubScheduler();

	TScheduler& s = TheScheduler;

	TBool reactivate = FALSE;

	TBool no_ipi = FALSE;

	NSchedulable* g = this;

	if (iParent != this && iParent)

		{

		NThreadGroup* tg = (NThreadGroup*)iParent;

		iReady = EReadyGroup;

		if (tg->iReady)

			{

			// extra thread added to group - change priority if necessary

			tg->iNThreadList.Add(this);

			TInt gp = tg->iPriority;

			TSubScheduler& ss = TheSubSchedulers[tg->iReady & EReadyCpuMask];

			ss.iReadyListLock.LockOnly();

			TInt hp = ss.HighestPriority();

			if (iPriority>gp)

				{

				ss.SSChgEntryP(tg, iPriority);

				}

			if (iPriority>hp || (iPriority==hp && ss.iCurrentThread && ss.iCurrentThread->iTime==0))

				{

				if (&ss == &ss0)

					RescheduleNeeded();					// reschedule on this processor

				else

					ss0.iReschedIPIs |= ss.iCpuMask;	// will kick the other CPU when this CPU reenables preemption

				}

			if ((aMode & ENewTimeslice) && t->iTime==0 && (iNext!=this || ss.EntryAtPriority(iPriority)) )

				t->iTime = t->iTimeslice;

			ss.iReadyListLock.UnlockOnly();

			ss0.iMadeReadyCounter++;

			return;

			}

		tg->iNThreadList.Add(this);

		tg->iPriority = iPriority;	// first in group

		g = tg;						// fall through to add group to subscheduler

		}

	TInt priClass = -1;

	TInt cpu = -1;

	TUint32 active = TheScheduler.iThreadAcceptCpus;

	if (g!=t || !t->i_NThread_Initial)

		priClass = KClassFromPriority[g->iPriority];

	if (g->iForcedCpu)

		{

		cpu = iForcedCpu & EReadyCpuMask;	// handles core cycling case (No.1 below)

		if (active & (1u<<cpu))

			goto cpu_ok;

		else

			goto single_cpu_reactivate;