// Copyright (c) 2007-2009 Nokia Corporation and/or its subsidiary(-ies).

// All rights reserved.

// This component and the accompanying materials are made available

// under the terms of the License "Eclipse Public License v1.0"

// which accompanies this distribution, and is available

// at the URL "http://www.eclipse.org/legal/epl-v10.html".

//

// Initial Contributors:

// Nokia Corporation - initial contribution.

//

// Contributors:

//

// Description:

// e32\nkernsmp\x86\ncsched.cia

// 

//

#include <x86.h>

#include <apic.h>

// SubSchedulerLookupTable	: global data, type: TSubScheduler* [256];

// BTraceLock				: global data, type: TSpinLock

const TLinAddr TScheduler_Reschedule = (TLinAddr)&TScheduler::Reschedule;

//const TLinAddr TheScheduler_iRescheduleNeededFlag = (TLinAddr)&TheScheduler.iRescheduleNeededFlag;

const TLinAddr NKern_FastCounter = (TLinAddr)&NKern::FastCounter;

const TLinAddr NKern_Lock = (TLinAddr)&NKern::Lock;

const TLinAddr NKern_Unlock = (TLinAddr)&NKern::Unlock;

const TLinAddr addressof_TheScheduler = (TLinAddr)&TheScheduler;

const TUint32 new_thread_trace_header = ((8<<BTrace::ESizeIndex) + (BTrace::EContextIdPresent<<BTrace::EFlagsIndex*8) + (BTrace::ECpuUsage<<BTrace::ECategoryIndex*8) + (BTrace::ENewThreadContext<<BTrace::ESubCategoryIndex*8));

extern "C" void __fastcall queue_dfcs(TSubScheduler* aS);

extern "C" NThreadBase* __fastcall select_next_thread(TSubScheduler* aS);

extern "C" void send_resched_ipis(TUint32 aMask);

extern "C" void __fastcall do_forced_exit(NThreadBase* aT);

extern "C" void NewThreadTrace(NThread* a);

/***************************************************************************

* Reschedule

* Enter with:

*		Kernel locked, interrupts enabled or disabled

* Return with:

*		Kernel unlocked, interrupts disabled

*		EAX=0 if no reschedule occurred, 1 if it did

*		ESI pointing to TSubScheduler for current CPU

*		EDI pointing to current NThread

***************************************************************************/

__NAKED__ void TScheduler::Reschedule()

	{

	asm("push 0 ");

	asm("mov eax, ds:[%0]" : : "i" (X86_LOCAL_APIC_BASE + X86_LOCAL_APIC_OFFSET_ID));	// OK since kernel locked

	asm("mov edi, %0" : : "i" (addressof_TheScheduler));

	asm("shr eax, 24 ");

	asm("mov esi, [eax*4+%0]" : : "i" (&SubSchedulerLookupTable));

	asm("cli ");

	asm("start_resched: ");

//	_asm cmp dword ptr [esi]TSubScheduler.iRescheduleNeededFlag, 10000h		VC6 ignores the "dword ptr"

	asm("lea eax, [esi+%0]" : : "i" _FOFF(TSubScheduler, iRescheduleNeededFlag));

	asm("cmp dword ptr [eax], 0x10000 ");

	asm("jb short resched_no_dfcs ");

	asm("mov ecx, esi ");

	asm("call %a0" : : "i" (&queue_dfcs));

	asm("resched_no_dfcs: ");

	asm("cmp byte ptr [esi+%0], 0" : : "i" _FOFF(TSubScheduler,iRescheduleNeededFlag));

	asm("jz resched_not_needed ");

	asm("sti ");

	asm("mov dword ptr [esp], 1 ");

	asm("mov ebp, [esi+%0]" : : "i"_FOFF(TSubScheduler, iCurrentThread));		// EBP -> original thread

	asm("mov eax, cr0");

	asm("push eax");

	asm("mov [ebp+%0], esp" : : "i" _FOFF(NThreadBase, iSavedSP));			// Save original thread stack pointer

	// We must move to a temporary stack before selecting the next thread.

	// This is because another CPU may begin executing this thread before the

	// select_next_thread() function returns and our stack would then be

	// corrupted. We use the stack belonging to this CPU's initial thread since

	// we are guaranteed that will never run on another CPU.

	asm("mov ecx, [esi+%0]" : : "i" _FOFF(TSubScheduler, iInitialThread));

	asm("mov esp, [ecx+%0]" : : "i" _FOFF(NThreadBase, iSavedSP));

	asm("select_thread:");

	asm("mov ecx, esi ");

	asm("call %a0" : : "i" (&select_next_thread));

	asm("mov ebx, eax ");

	asm("cmp ebx, 0 ");

	asm("jz no_thread ");

	asm("mov esp, [ebx+%0]" : : "i" _FOFF(NThreadBase, iSavedSP)); // move to new thread's stack

#ifdef BTRACE_CPU_USAGE

	asm("cmp byte ptr %a0, 0" : : "i" (&BTraceData.iFilter[4]));

	asm("jz short no_trace ");

	asm("push ebx ");

	asm("call %a0" : : "i" (NewThreadTrace));

	asm("pop ebx ");

	asm("no_trace: ");

#endif	// BTRACE_CPU_USAGE

	asm("cmp ebp, ebx ");

	asm("je same_thread ");

	asm("mov eax, [ebx+%0]" : : "i" _FOFF(NThreadBase, iStackBase));

	asm("add eax, [ebx+%0]" : : "i" _FOFF(NThreadBase, iStackSize));

	asm("mov [ecx+%0], eax" : : "i" _FOFF(TX86Tss, iEsp0));					// set ESP0 to top of new thread supervisor stack

	asm("test byte ptr [ebx+%0], 2" : : "i" _FOFF(NThreadBase,i_ThrdAttr));	// test for address space switch

	asm("jz short resched_no_as_switch ");

	asm("call [edi+%0]" : : "i" _FOFF(TScheduler, iProcessHandler));		// call handler with

														// EBX=pointer to new thread, EDI->scheduler, ESI->subscheduler

	asm("resched_no_as_switch: ");

	asm("same_thread: ");

	asm("pop eax ");

	asm("mov cr0, eax ");

	asm("cli ");

//	asm("cmp dword ptr [esi]TSubScheduler.iRescheduleNeededFlag, 0	VC6 ignores the "dword ptr"

	asm("lea eax, [esi+%0]" : : "i" _FOFF(TSubScheduler, iRescheduleNeededFlag));

	asm("cmp dword ptr [eax], 0 ");

	asm("jnz start_resched ");

	asm("resched_not_needed: ");

	asm("mov edi, [esi+%0]" : : "i" _FOFF(TSubScheduler, iCurrentThread));

	asm("cmp dword ptr [edi+%0], -3" : : "i" _FOFF(NThreadBase, iCsFunction)); // ECSDivertPending

	asm("je resched_thread_divert ");

	asm("mov dword ptr [esi+%0], 0" : : "i" _FOFF(TSubScheduler, iKernLockCount));

	asm("pop eax ");

	asm("ret ");

	asm("resched_thread_divert: ");

	asm("push edi ");

	asm("xor eax, eax ");

	asm("lock xchg eax, [esi+%0]" : : "i" _FOFF(TSubScheduler, iReschedIPIs));

	asm("test eax, eax ");

	asm("jz short no_resched_ipis ");

	asm("push eax ");

	asm("call %a0" : : "i" (&send_resched_ipis));

	asm("add esp, 4 ");

	asm("no_resched_ipis: ");

	asm("sti ");

	asm("mov ecx, [esp+12] ");			// SThreadReschedStack iReason 0 not run 1 unlock 2 IRQ

	asm("cmp ecx, 2 ");

	asm("ja short rtd_unknown ");		// unknown - die

	asm("shl ecx, 2 ");					// reason * 4

	asm("mov eax, 0xa1a ");

	asm("shr eax, cl ");

	asm("and eax, 15 ");

	asm("mov gs, [esp+eax*4+16] ");		// restore GS

	asm("pop ecx ");					// exiting thread pointer

	asm("call %a0" : : "i" (&do_forced_exit));

	asm("int 0xff ");					// should never get here

	asm("rtd_unknown: ");

	asm("int 0xff ");					// should never get here

	// There is no thread ready to run

	asm("no_thread: ");

	asm("cli ");

	asm("xor eax, eax ");

	asm("lock xchg eax, [esi+%0]" : : "i" _FOFF(TSubScheduler, iReschedIPIs));

	asm("test eax, eax ");

	asm("jz short no_resched_ipis2 ");

	asm("push eax ");

	asm("call %a0" : : "i" (&send_resched_ipis));

	asm("add esp, 4 ");

	asm("no_resched_ipis2: ");

	asm("sti ");

	asm("hlt ");

	asm("no_thread2: ");

//	_asm cmp dword ptr [esi]TSubScheduler.iRescheduleNeededFlag, 10000h		VC6 ignores the "dword ptr"

	asm("lea eax, [esi+%0]" : : "i" _FOFF(TSubScheduler, iRescheduleNeededFlag));

	asm("cmp dword ptr [eax], 0x10000 ");

	asm("jb short no_thread ");

	asm("mov ecx, esi ");

	asm("call %a0" : : "i" (&queue_dfcs));

	asm("cmp byte ptr [esi+%0], 0" : : "i" _FOFF(TSubScheduler, iRescheduleNeededFlag));

	asm("jz short no_thread2 ");

	asm("jmp select_thread ");

	}

/** Disable interrupts to the specified level

If aLevel = 0 does not affect interrupt state

If aLevel <>0 disables all maskable interrupts.

@param	aLevel level to which to disable

@return	Cookie to pass into RestoreInterrupts()

*/

EXPORT_C __NAKED__ TInt NKern::DisableInterrupts(TInt /*aLevel*/)

	{

	asm("pushfd");

	asm("mov ecx, [esp+4]");

	asm("pop eax");

	asm("and eax, 0x200");

	asm("test ecx, ecx");

	asm("jz disable_ints_0");

	asm("cli");

	asm("disable_ints_0:");

	asm("ret");

	}

/** Disable all maskable interrupts

@return	Cookie to pass into RestoreInterrupts()

*/

EXPORT_C __NAKED__ TInt NKern::DisableAllInterrupts()

	{

	asm("pushfd");

	asm("pop eax");

	asm("and eax, 0x200");

	asm("cli");

	asm("ret");

	}

/** Restore interrupt mask to state preceding a DisableInterrupts() call

@param	aLevel Cookie returned by Disable(All)Interrupts()

*/

EXPORT_C __NAKED__ void NKern::RestoreInterrupts(TInt aLevel)

	{

	asm("test byte ptr [esp+5], 2");	// test saved I flag

	asm("jz restore_irq_off");			// jump if clear

	asm("sti");							// else reenable interrupts

	asm("ret");

	asm("restore_irq_off:");

	asm("cli");

	asm("ret");

	}

/** Enable all maskable interrupts

@internalComponent

*/

EXPORT_C __NAKED__ void NKern::EnableAllInterrupts()

	{

	asm("sti");

	asm("ret");

	}

/**	Unlocks the kernel

	Decrements iKernCSLocked; if it becomes zero and IDFCs or a reschedule are

	pending, calls the scheduler to process them.

	@pre	Thread or IDFC context. Don't call from ISRs.

 */

EXPORT_C __NAKED__ void NKern::Unlock()

	{

	asm("mov eax, ds:[%0]" : : "i" (X86_LOCAL_APIC_BASE + X86_LOCAL_APIC_OFFSET_ID)); // OK since kernel locked

	asm("shr eax, 24 ");

	asm("push esi ");

	asm("mov esi, [eax*4+%0]" : : "i" (&SubSchedulerLookupTable));

#ifdef _DEBUG

	asm("cmp dword ptr [esi+%0], 0" : : "i" _FOFF(TSubScheduler, iKernLockCount));

	asm("jg short _dbg1 ");

	asm("int 0xff ");

	asm("_dbg1: ");

#endif

	asm("cli ");

	asm("dec dword ptr [esi+%0]" : : "i" _FOFF(TSubScheduler, iKernLockCount));

	asm("jnz short still_locked ");

//	asm("cmp dword ptr [esi]TSubScheduler.iRescheduleNeededFlag, 0	VC6 ignores the "dword ptr"

	asm("lea eax, [esi+%0]" : : "i" _FOFF(TSubScheduler, iRescheduleNeededFlag));

	asm("cmp dword ptr [eax], 0 ");

	asm("jz short no_resched ");

	asm("mov dword ptr [esi+%0], 1" : : "i" _FOFF(TSubScheduler, iKernLockCount));

	asm("push edi ");

	asm("push ebp ");

	asm("push ebx ");

	asm("push gs ");

	asm("push fs ");

	asm("sti ");

	// Reschedule - return with local interrupts disabled, iKernLockCount=0

	asm("push 1 ");

	asm("call %a0" : : "i" (TScheduler_Reschedule));

	asm("add esp, 4 ");

	asm("xor eax, eax ");

	asm("lock xchg eax, [esi+%0]" : : "i" _FOFF(TSubScheduler, iReschedIPIs));

	asm("test eax, eax ");

	asm("jz short no_resched_ipis_ul ");

	asm("unlock_do_resched_ipis: ");

	asm("push eax ");

	asm("call %a0" : : "i" (&send_resched_ipis));

	asm("add esp, 4 ");

	asm("no_resched_ipis_ul: ");

	asm("pop fs ");

	asm("pop gs ");

	asm("pop ebx ");

	asm("pop ebp ");

	asm("pop edi ");

	asm("still_locked: ");

	asm("sti ");

	asm("pop esi ");

	asm("ret ");

	asm("no_resched: ");

	asm("xor eax, eax ");

	asm("lock xchg eax, [esi+%0]" : : "i" _FOFF(TSubScheduler, iReschedIPIs));

	asm("test eax, eax ");

	asm("jz short still_locked ");

	asm("push edi ");

	asm("push ebp ");

	asm("push ebx ");

	asm("push gs ");

	asm("push fs ");

	asm("jmp short unlock_do_resched_ipis ");

	}

/**	Locks the kernel

	Defer IDFCs and preemption

	@pre	Thread or IDFC context. Don't call from ISRs.

 */

EXPORT_C __NAKED__ void NKern::Lock()

	{

	asm("cli");		// stop thread migration between reading APIC ID and subscheduler stuff

	asm("mov eax, ds:[%0]" : : "i"(X86_LOCAL_APIC_BASE + X86_LOCAL_APIC_OFFSET_ID));

	asm("shr eax, 24");

	asm("mov ecx, [eax*4+%0]" : : "i"(&SubSchedulerLookupTable));

	asm("inc dword ptr [ecx+%0]": : "i"_FOFF(TSubScheduler, iKernLockCount));

	asm("sti");

	asm("ret");

	}

/**	Locks the kernel and returns a pointer to the current thread

	Defer IDFCs and preemption

	@pre	Thread or IDFC context. Don't call from ISRs.

 */

EXPORT_C __NAKED__ NThread* NKern::LockC()

	{

	asm("cli");		// stop thread migration between reading APIC ID and subscheduler stuff

	asm("mov eax, ds:[%0]" : : "i"(X86_LOCAL_APIC_BASE + X86_LOCAL_APIC_OFFSET_ID));

	asm("shr eax, 24");

	asm("mov ecx, [eax*4+%0]" : : "i"(&SubSchedulerLookupTable));

	asm("inc dword ptr [ecx+%0]": : "i"_FOFF(TSubScheduler, iKernLockCount));

	asm("mov eax, [ecx+%0]" : : "i"_FOFF(TSubScheduler, iCurrentThread));

	asm("sti");

	asm("ret");

	}

/**	Allows IDFCs and rescheduling if they are pending.

	If IDFCs or a reschedule are pending and iKernCSLocked is exactly equal to 1

	calls the scheduler to process the IDFCs and possibly reschedule.

	@return	Nonzero if a reschedule actually occurred, zero if not.

	@pre	Thread or IDFC context. Don't call from ISRs.

 */

EXPORT_C __NAKED__ TInt NKern::PreemptionPoint()

	{

	asm("mov eax, ds:[%0]" : : "i"(X86_LOCAL_APIC_BASE + X86_LOCAL_APIC_OFFSET_ID));

	asm("shr eax, 24");

	asm("mov ecx, [eax*4+%0]" : : "i"(&SubSchedulerLookupTable));

#ifdef _DEBUG

	asm("cmp dword ptr [ecx+%0], 0": : "i"_FOFF(TSubScheduler, iKernLockCount));

	asm("jg _dbg1_pp");

	asm("int 0xff");

	asm("_dbg1_pp:");

#endif

	asm("cmp dword ptr [ecx+%0], 1": : "i"_FOFF(TSubScheduler, iKernLockCount));

	asm("jnz still_locked_pp");

//	asm("cmp dword ptr [ecx]TSubScheduler.iRescheduleNeededFlag, 0		VC6 ignores the "dword ptr"

	asm("lea eax, [ecx+%0]": : "i"_FOFF(TSubScheduler, iRescheduleNeededFlag));

	asm("cmp dword ptr [eax], 0");

	asm("jnz do_resched");

	asm("cli");

	asm("lock xchg eax, [ecx+%0]": : "i"_FOFF(TSubScheduler, iReschedIPIs));

	asm("test eax, eax");

	asm("jz pp_no_resched_ipis");

	asm("push eax");

	asm("call %a0": :"i"(&send_resched_ipis));

	asm("add esp, 4");

	asm("pp_no_resched_ipis:");

	asm("sti");

	asm("still_locked_pp:");

	asm("xor eax, eax");

	asm("ret");

	asm("do_resched:");

	asm("call %a0" : : "i"(NKern_Unlock));

	asm("call %a0" : : "i"(NKern_Lock));

	asm("mov eax, 1");

	asm("ret");

	}

/** Complete the saving of a thread's context

This saves the FPU registers if necessary once we know that we are definitely

switching threads.

@internalComponent

*/

__NAKED__ void NThread::CompleteContextSave()

	{

	THISCALL_PROLOG0()

	asm("mov edx, [ecx+%0]": : "i"_FOFF(NThreadBase,iSavedSP));	// EDX points to saved state on thread stack

	asm("test byte ptr [edx], 8");						// test thread's saved TS flag

	asm("jnz no_fpu");									// if set, thread did not use FPU

	asm("clts");

	asm("fnsave [ecx+%0]": : "i"_FOFF(NThread, iCoprocessorState));	// else thread did use FPU - save its state

	asm("or byte ptr [edx], 8");						// set TS flag so thread aborts next time it uses FPU

	asm("fwait");

	asm("no_fpu:");

	THISCALL_EPILOG0()

	}

/** Check if the kernel is locked the specified number of times.

	@param aCount	The number of times the kernel should be locked

					If zero, tests if it is locked at all

	@return TRUE if the tested condition is true.

	@internalTechnology

*/

EXPORT_C __NAKED__ TBool NKern::KernelLocked(TInt /*aCount*/)

	{

	asm("pushfd");

	asm("cli");

	asm("mov eax, ds:[%0]" : : "i"(X86_LOCAL_APIC_BASE + X86_LOCAL_APIC_OFFSET_ID));

	asm("shr eax, 24");

	asm("mov eax, [eax*4+%0]" : : "i"(&SubSchedulerLookupTable));

	asm("mov edx, [eax+%0]": : "i"_FOFF(TSubScheduler, iKernLockCount));

	asm("popfd");

	asm("cmp edx, 0");

	asm("jz not_locked");

	asm("mov eax, [esp+4]");

	asm("cmp eax, 0");

	asm("jz locked");

	asm("cmp eax, edx");

	asm("jnz not_locked");

	asm("locked:");

	asm("mov eax, 1");

	asm("ret");

	asm("not_locked:");

	asm("xor eax, eax");

	asm("ret");

	}

// Only call this if thread migration is disabled, i.e.

// interrupts disabled, kernel locked or current thread in 'freeze cpu' mode

extern "C" __NAKED__ TSubScheduler& SubScheduler()

	{

	asm("mov eax, ds:[%0]" : : "i"(X86_LOCAL_APIC_BASE + X86_LOCAL_APIC_OFFSET_ID));

	asm("shr eax, 24");

	asm("mov eax, [eax*4+%0]" : : "i"(&SubSchedulerLookupTable));

	asm("ret");

	}

/** Returns the NThread control block for the currently scheduled thread.

    Note that this is the calling thread if called from a thread context, or the

	interrupted thread if called from an interrupt context.

	@return A pointer to the NThread for the currently scheduled thread.

	@pre Call in any context.

*/

EXPORT_C __NAKED__ NThread* NKern::CurrentThread()

	{

	asm("pushfd");

	asm("cli");		// stop thread migration between reading APIC ID and thread pointer

	asm("mov eax, ds:[%0]" : : "i"(X86_LOCAL_APIC_BASE + X86_LOCAL_APIC_OFFSET_ID));

	asm("shr eax, 24");

	asm("mov eax, [eax*4+%0]" : : "i"(&SubSchedulerLookupTable));

	asm("cmp eax, 0");

	asm("jz done");

	asm("test al, 3");

	asm("jnz bad_ct");

	asm("mov eax, [eax+%0]": : "i"_FOFF(TSubScheduler, iCurrentThread));

	asm("done:");

	asm("popfd");

	asm("ret");

	asm("bad_ct:");

	asm("popfd");

	asm("xor eax, eax");

	asm("ret");

	}

/** Returns the NThread control block for the currently scheduled thread.

    Note that this is the calling thread if called from a thread context, or the

	interrupted thread if called from an interrupt context.

	@return A pointer to the NThread for the currently scheduled thread.

	@pre Call with migration disabled - i.e. from an ISR, IDFC, with interrupts

			disabled or with preemption disabled.

*/

extern "C" __NAKED__ NThread* NCurrentThreadL()

	{

	asm("mov eax, ds:[%0]" : : "i"(X86_LOCAL_APIC_BASE + X86_LOCAL_APIC_OFFSET_ID));

	asm("shr eax, 24");

	asm("mov eax, [eax*4+%0]" : : "i"(&SubSchedulerLookupTable));

	asm("mov eax, [eax+%0]": : "i"_FOFF(TSubScheduler, iCurrentThread));

	asm("ret");

	}

/** Returns the CPU number of the calling CPU.

	@return the CPU number of the calling CPU.

	@pre Call in any context.

*/

EXPORT_C __NAKED__ TInt NKern::CurrentCpu()

	{

	asm("xor eax, eax");

	asm("str ax");