// Copyright (c) 2007-2009 Nokia Corporation and/or its subsidiary(-ies).

// All rights reserved.

// This component and the accompanying materials are made available

// under the terms of the License "Eclipse Public License v1.0"

// which accompanies this distribution, and is available

// at the URL "http://www.eclipse.org/legal/epl-v10.html".

//

// Initial Contributors:

// Nokia Corporation - initial contribution.

//

// Contributors:

//

// Description:

// e32\nkernsmp\x86\vectors.cia

// 

//

#include <x86.h>

#include <apic.h>

#include "vectors.h"

#ifdef _DEBUG

#define __CHECK_LOCK_STATE__

#endif

void __X86VectorIrq();

void __X86VectorExc();

void __X86ExcFault(TAny*);

#ifdef __GCC32__

#define DECLARE_X86_INT(n)			GLDEF_C __NAKED__ void __X86Vector##n() { asm("push 0"); asm("push 0x"#n); asm("jmp %a0" : : "i" (&__X86VectorIrq)); }

#define DECLARE_X86_EXC_NOERR(n)	GLDEF_C __NAKED__ void __X86Vector##n() { asm("push 0"); asm("push 0x"#n); asm("jmp %a0" : : "i" (&__X86VectorExc)); }

#define DECLARE_X86_EXC_ERR(n)		GLDEF_C __NAKED__ void __X86Vector##n() { asm("push 0x"#n); asm("jmp %a0" : : "i" (&__X86VectorExc)); }

#else

#define DECLARE_X86_INT(n)			GLDEF_C __NAKED__ void __X86Vector##n() { _asm push 0 _asm push 0x##n _asm jmp __X86VectorIrq }

#define DECLARE_X86_EXC_NOERR(n)	GLDEF_C __NAKED__ void __X86Vector##n() { _asm push 0 _asm push 0x##n _asm jmp __X86VectorExc }

#define DECLARE_X86_EXC_ERR(n)		GLDEF_C __NAKED__ void __X86Vector##n() { _asm push 0x##n _asm jmp __X86VectorExc }

#endif

const TLinAddr NKern_WaitForAnyRequest = (TLinAddr)&NKern::WaitForAnyRequest;

const TLinAddr NKern_Lock = (TLinAddr)&NKern::Lock;

const TLinAddr NKern_LockSystem = (TLinAddr)&NKern::LockSystem;

const TLinAddr NKern_UnlockSystem = (TLinAddr)&NKern::UnlockSystem;

const TLinAddr NKern_Unlock = (TLinAddr)&NKern::Unlock;

const TLinAddr NKern_ThreadLeaveCS = (TLinAddr)&NKern::ThreadLeaveCS;

const TLinAddr TScheduler_Reschedule = (TLinAddr)&TScheduler::Reschedule;

const TLinAddr addressof_TheScheduler = (TLinAddr)&TheScheduler;

const TLinAddr addressof_CrashState = (TLinAddr)&::CrashState;

extern "C" void send_resched_ipis(TUint32 aMask);

extern "C" void run_user_mode_callbacks();

#ifdef __CHECK_LOCK_STATE__

/******************************************************************************

 * Check that the kernel is unlocked, no fast mutex is held and the thread

 * is not in a critical section when returning to user mode.

 ******************************************************************************/

extern "C" __NAKED__ void check_lock_state()

	{

	asm("pushfd ");

	asm("cli ");	// so we don't migrate between reading APIC ID and thread pointer

	asm("mov edx, ds:[%0]" : : "i" (X86_LOCAL_APIC_BASE + X86_LOCAL_APIC_OFFSET_ID));

	asm("shr edx, 24 ");

	asm("push esi ");

	asm("mov esi, [edx*4+%0]" : : "i"(&SubSchedulerLookupTable));

	asm("cmp dword ptr [esi+%0], 0" : : "i" _FOFF(TSubScheduler, iKernLockCount));

	asm("jnz short bad_lock_state1 ");

	asm("mov ecx, [esi+%0]" : : "i" _FOFF(TSubScheduler, iCurrentThread));

	asm("cmp dword ptr [ecx+%0], 0" : : "i" _FOFF(NThreadBase, iHeldFastMutex));

	asm("jne short bad_lock_state2 ");

	asm("cmp dword ptr [ecx+%0], 0" : : "i" _FOFF(NThreadBase, iCsCount));

	asm("jne short bad_lock_state3 ");

	asm("pop esi ");

	asm("popfd ");

	asm("ret ");

	asm("bad_lock_state1: ");

	asm("int 0xff ");

	asm("bad_lock_state2: ");

	asm("int 0xff ");

	asm("bad_lock_state3: ");

	asm("int 0xff ");

	}

#endif

/******************************************************************************

* Int 20h Handler - Fast Executive Calls

* Enter with:

*		Call number in EAX

*		Parameter in ECX if any

* On entry SS:ESP references current threads supervisor stack

* [ESP+0] = return EIP

* [ESP+4] = return CS

* [ESP+8] = return EFLAGS

* [ESP+12] = return ESP if privilege change occurred

* [ESP+16] = return SS if privilege change occurred

*******************************************************************************/

GLDEF_C __NAKED__ void __X86Vector20()

	{

	// Interrupts disabled on entry

	asm("cld ");

	asm("push 0 ");			// error code

	asm("push 0x20 ");		// vector number

	asm("push gs ");

	asm("push fs ");

	asm("push es ");

	asm("push ds ");

	asm("push eax ");

	asm("push ebp ");

	asm("push edi ");

	asm("push esi ");

	asm("push ebx ");

	asm("push edx ");

	asm("push ecx ");

	asm("mov cx, ds ");

	asm("mov dx, ss ");

	asm("mov ds, dx ");

	asm("mov gs, cx ");

	asm("mov es, dx ");

	asm("mov ecx, ds:[%0]" : : "i" (X86_LOCAL_APIC_BASE + X86_LOCAL_APIC_OFFSET_ID));

	asm("shr ecx, 24 ");

	asm("mov esi, [ecx*4+%0]" : : "i" (&SubSchedulerLookupTable));

	asm("mov edi, [esi+%0]" : : "i" _FOFF(TSubScheduler, iCurrentThread));

	asm("test eax, eax ");

	asm("je short wait_for_any_request ");

	asm("mov edx, [edi+%0]" : : "i" _FOFF(NThreadBase, iFastExecTable));

	asm("cmp eax, [edx] ");

	asm("jae short fast_exec_invalid ");

	asm("call [edx][eax*4] ");

	asm("fast_exec_exit: ");

	asm("test dword ptr [esp+%0], 3" : : "i" _FOFF(SThreadExcStack, iCs));	// returning to user mode?

	asm("jz short fast_exec_exit2 ");		// no so just return

#ifdef __CHECK_LOCK_STATE__

	asm("call %a0" : : "i" (&check_lock_state));

#endif

	// don't need to check for user mode callbacks here since

	// we couldn't have rescheduled

	asm("fast_exec_exit2: ");

	asm("pop ecx ");

	asm("pop edx ");

	asm("pop ebx ");

	asm("pop esi ");

	asm("pop edi ");

	asm("pop ebp ");

	asm("add esp, 4 ");

	asm("pop ds ");

	asm("pop es ");

	asm("pop fs ");

	asm("pop gs ");

	asm("add esp, 8 ");

	asm("iretd ");

	asm("wait_for_any_request: ");

	asm("sti ");

	asm("call %a0" : : "i" (NKern_WaitForAnyRequest));

	asm("cli ");

	asm("test dword ptr [esp+%0], 3" : : "i" _FOFF(SThreadExcStack, iCs));	// returning to user mode?

	asm("jz short fast_exec_exit2 ");		// no so just return

#ifdef __CHECK_LOCK_STATE__

	asm("call %a0" : : "i" (&check_lock_state));

#endif

	asm("cmp dword ptr [edi+%0], 4" : : "i" _FOFF(NThreadBase,iUserModeCallbacks));

	asm("jb short fast_exec_exit2 ");

	asm("call run_user_mode_callbacks ");

	asm("jmp short fast_exec_exit2 ");

	asm("fast_exec_invalid: ");

	asm("sti ");

	asm("mov esi, [edi+%0]" : : "i" _FOFF(NThreadBase,iSlowExecTable));	// esi=slow exec table base

	asm("call [esi-8] ");												// call invalid exec handler

	asm("cli ");

	asm("jmp short fast_exec_exit ");

	}

/******************************************************************************

 * Int 21h Handler - Slow Executive Calls

 * Enter with:

 *		Call number in EAX

 *		Parameters in ECX, EDX, EBX, ESI in that order

 * On entry SS:ESP references current threads supervisor stack

 * Must preserve EBX, EBP, ESI, EDI

 * [ESP+0] = return EIP

 * [ESP+4] = return CS

 * [ESP+8] = return EFLAGS

 * [ESP+12] = return ESP if privilege change occurred

 * [ESP+16] = return SS if privilege change occurred

 ******************************************************************************/

GLDEF_C __NAKED__ void __X86Vector21()

	{

	// Interrupts disabled on entry

	asm("cld ");

	asm("push 0 ");									// error code

	asm("push 0x21 ");								// vector number

	asm("sub esp, 32 ");							// reserve space for additional arguments

	asm("push gs ");

	asm("push fs ");

	asm("push es ");

	asm("push ds ");

	asm("push eax ");

	asm("push ebp ");

	asm("push edi ");

	asm("push esi ");

	asm("push ebx ");

	asm("push edx ");

	asm("push ecx ");

	asm("mov cx, ds ");

	asm("mov dx, ss ");

	asm("mov ds, dx ");

	asm("mov gs, cx ");

	asm("mov es, dx ");

	asm("mov edi, ds:[%0]" : : "i" (X86_LOCAL_APIC_BASE + X86_LOCAL_APIC_OFFSET_ID));

	asm("shr edi, 24 ");

	asm("mov esi, [edi*4+%0]" : : "i" (&SubSchedulerLookupTable));

	asm("mov edi, [esi+%0]" : : "i" _FOFF(TSubScheduler,iCurrentThread));	// edi=TheCurrentThread

	asm("sti");

	asm("mov esi, [edi+%0]" : : "i" _FOFF(NThreadBase, iSlowExecTable));	// esi=slow exec table base

	asm("lea ebp, [esi][eax*8] ");					// ebp points to exec table entry

	asm("cmp eax, [esi-12] ");

	asm("jae short slow_exec_invalid ");

	asm("mov ebx, [ebp] ");							// ebx=flags

	asm("test ebx, 0x1c000000 ");					// additional arguments required?

	asm("jz short slow_exec_no_extra_args ");

	asm("mov edx, [esp+8] ");						// edx points to additional args

	asm("lea eax, [esp+44] ");						// address of copied additional arguments

	asm("mov [esp+8], eax ");						// replace supplied address

	asm("mov ecx, ebx ");

	asm("shr ecx, 26 ");

	asm("and cl, 7 ");								// ecx=number of additional arguments-1

	asm("test edx, edx ");

	asm("jnz short slow_exec_extra_args_present ");	// if arg ptr not NULL, copy args

	asm("slow_exec_zero_args: ");

	asm("mov [esp+ecx*4+44], edx ");				// else zero args

	asm("dec ecx ");

	asm("jns short slow_exec_zero_args ");

	asm("jmp short slow_exec_no_extra_args ");

	asm("slow_exec_extra_args_present: ");

	asm("slow_exec_copy_args: ");

	asm("mov eax, gs:[edx+ecx*4] ");				// get argument

	asm("mov [esp+ecx*4+44], eax ");				// copy it

	asm("dec ecx ");

	asm("jns short slow_exec_copy_args ");

	asm("slow_exec_no_extra_args:");

	asm("test ebx, 0x80000000 ");					// test EClaim

	asm("jz short slow_exec_no_claim ");

	asm("call %a0" : : "i" (NKern_LockSystem));		// trashes eax, ecx, edx

	asm("slow_exec_no_claim: ");

	asm("test ebx, 0x20000000 ");					// test EPreprocess

	asm("jz short slow_exec_no_preprocess ");

	asm("mov eax, [esi-4] ");						// preprocess handler address

	asm("mov esi, edi ");							// save NThread pointer in ESI, also leave it in EDI

	asm("call eax ");								// trashes eax, ecx, edx, edi

	asm("mov edi, esi ");							// NThread pointer back into EDI

	asm("slow_exec_no_preprocess: ");

	asm("call [ebp+4] ");							// call exec function

	asm("mov [esp+%0], eax" : : "i" _FOFF(SThreadSlowExecStack, iEax));	// save return value

	asm("test ebx, 0x40000000 ");					// test ERelease

	asm("jz short slow_exec_no_release ");

	asm("call %a0" : : "i" (NKern_UnlockSystem));	// trashes eax, ecx, edx

	asm("slow_exec_no_release: ");

	asm("slow_exec_exit: ");

	asm("cli ");

	asm("test dword ptr [esp+%0], 3" : : "i" _FOFF(SThreadSlowExecStack, iCs));	// returning to user mode?

	asm("jz short slow_exec_exit2 ");				// no so just return

#ifdef __CHECK_LOCK_STATE__

	asm("call %a0" : : "i" (&check_lock_state));

#endif

	asm("cmp dword ptr [edi+%0], 4" : : "i" _FOFF(NThreadBase,iUserModeCallbacks));

	asm("jb short slow_exec_exit2 ");

	asm("call run_user_mode_callbacks ");

	asm("slow_exec_exit2: ");

	asm("pop ecx ");

	asm("pop edx ");

	asm("pop ebx ");

	asm("pop esi ");

	asm("pop edi ");

	asm("pop ebp ");

	asm("pop eax ");

	asm("pop ds ");

	asm("pop es ");

	asm("pop fs ");

	asm("pop gs ");

	asm("add esp, 40 ");

	asm("iretd ");

	asm("slow_exec_invalid: ");

	asm("call [esi-8] ");							// call invalid exec handler

	asm("jmp short slow_exec_exit ");

	}

__NAKED__ TUint32 __tr()

	{

	asm("xor eax, eax ");

	asm("str ax ");

	asm("ret ");

	}

extern "C" void _irqdebug(TUint a);

extern "C" void generic_ipi_isr(TSubScheduler* aS);

extern "C" void run_event_handlers(TSubScheduler* aS);

extern "C" void IrqStartTrace(TUint32 aVector);

extern "C" void IrqEndTrace();

/******************************************************************************

 * IRQ Preamble/Postamble Common Code

 * On entry SS:ESP references current threads supervisor stack

 * [ESP+0] = vector number

 * [ESP+4] = error code (=0)

 * [ESP+8] = return EIP

 * [ESP+12] = return CS

 * [ESP+16] = return EFLAGS

 * [ESP+20] = return ESP if privilege change occurred

 * [ESP+24] = return SS if privilege change occurred

 ******************************************************************************/

__NAKED__ void __X86VectorIrq()

	{

	// Interrupts disabled on entry

	asm("cld ");

	asm("push gs ");

	asm("push fs ");

	asm("push es ");

	asm("push ds ");

	asm("push eax ");

	asm("push ebp ");

	asm("push edi ");

	asm("push esi ");

	asm("push ebx ");

	asm("push edx ");

	asm("push ecx ");

	asm("mov ax, ss ");

	asm("mov ds, ax ");

	asm("mov es, ax ");

	asm("mov eax, ds:[%0]" : : "i" (X86_LOCAL_APIC_BASE + X86_LOCAL_APIC_OFFSET_ID));

	asm("shr eax, 24 ");

	asm("mov esi, [eax*4+%0]" : : "i" (&SubSchedulerLookupTable));

	asm("mov edi, esp ");								// edi points to saved stuff

	asm("inc dword ptr [esi+36+%0]" : : "i" _FOFF(TSubScheduler, iExtras));	// increment i_IrqCount

	asm("inc dword ptr [esi+52+%0]" : : "i" _FOFF(TSubScheduler, iExtras));	// nest count starts at -1, iExtras[13]

	asm("jnz short nested_irq_entry ");

	asm("mov esp, [esi+56+%0]" : : "i" _FOFF(TSubScheduler, iExtras));			// iExtras[14] = irq stack top

	asm("push edi ");

	asm("nested_irq_entry: ");

	asm("mov edx, [esi+%0]" : : "i" _FOFF(TSubScheduler, iCpuMask));

	asm("lock or [%a0], edx" : : "i" (&TheScheduler.iCpusNotIdle));

	asm("mov ebx, [edi+%0]" : : "i" _FOFF(SThreadExcStack, iVector));

#ifdef BTRACE_CPU_USAGE

	asm("cmp byte ptr %a0, 0" : : "i" (&BTraceData.iFilter[4/*BTrace::ECpuUsage*/]));

	asm("jz short no_trace ");

	asm("push ebx ");

	asm("call %a0" : : "i" (IrqStartTrace));

	asm("add esp, 4 ");

	asm("no_trace: ");

#endif

#ifdef _DEBUG

	asm("push ebx ");

	asm("call %a0 ": :"i" (&_irqdebug));

	asm("add esp, 4 ");

#endif

	asm("cmp ebx, 0x30 ");

	asm("jb short kernel_ipi ");

	asm("mov ecx, ebx ");

	asm("call [%a0]" : : "i" (&X86_IrqHandler));

	asm("jmp short postamble ");

	asm("kernel_ipi: ");

	asm("cmp ebx, %0" : : "i" (SPURIOUS_INTERRUPT_VECTOR));

	asm("je short postamble ");

	asm("xor eax, eax ");

	asm("mov ds:[%0], eax" : : "i" (X86_LOCAL_APIC_BASE + X86_LOCAL_APIC_OFFSET_EOI));

	asm("cmp ebx, %0" : : "i" (TRANSFERRED_IRQ_VECTOR));

	asm("je short postamble");

	asm("cmp ebx, %0" : : "i" (RESCHED_IPI_VECTOR));

	asm("je short resched_ipi ");

	asm("cmp ebx, %0" : : "i" (TIMESLICE_VECTOR));

	asm("jne short generic_ipi ");

	asm("resched_ipi: ");

	asm("mov byte ptr [esi+%0], 1" : : "i" _FOFF(TSubScheduler, iRescheduleNeededFlag));

	asm("jmp short postamble ");

	asm("generic_ipi:");

	asm("cmp ebx, %0" : : "i" (GENERIC_IPI_VECTOR));

	asm("jne short postamble ");

	asm("push esi ");

	asm("call %a0" : : "i" (&generic_ipi_isr));

	asm("add esp, 4 ");

	// Postamble. Interrupts disabled here.

	asm("postamble: ");

	asm("cli ");

	asm("dec dword ptr [esi+52+%0]" : : "i" _FOFF(TSubScheduler, iExtras));

	asm("jns short nested_irq_exit ");

	// Check for deferred/transferred IRQs

	asm("cmp byte ptr [esi+%0], 0 " : : "i" _FOFF(TSubScheduler,iEventHandlersPending));

	asm("je short no_event_handlers ");

	// increment i_IrqNestCount again since we are going to run more ISRs

	asm("inc dword ptr [esi+52+%0]" : : "i" _FOFF(TSubScheduler, iExtras));

	asm("push esi ");

	asm("call %a0" : : "i" (run_event_handlers));

	asm("add esp, 4 ");

	asm("dec dword ptr [esi+52+%0]" : : "i" _FOFF(TSubScheduler, iExtras));

	asm("no_event_handlers: ");

	asm("pop eax ");

	asm("mov esp, eax ");

	asm("cmp dword ptr [esi+%0], 0" : : "i" _FOFF(TSubScheduler, iKernLockCount));

	asm("jne short irq_kernel_locked_exit ");

//	asm("cmp dword ptr [esi]TSubScheduler.iRescheduleNeededFlag, 0	VC6 ignores the "dword ptr"

	asm("lea eax, [esi+%0]" : : "i" _FOFF(TSubScheduler, iRescheduleNeededFlag));

	asm("cmp dword ptr [eax], 0 ");

	asm("je short irq_kernel_locked_exit ");

	asm("mov dword ptr [esi+%0], 1" : : "i" _FOFF(TSubScheduler, iKernLockCount));

	asm("sti ");

	asm("push 2 ");

	asm("call %a0" : : "i" (TScheduler_Reschedule));	// returns with EDI -> current thread

	asm("add esp, 4 ");

	asm("xor eax, eax ");

	asm("lock xchg eax, [esi+%0]" : : "i" _FOFF(TSubScheduler, iReschedIPIs));

	asm("test eax, eax ");

	asm("jz short irq_user_check ");

	asm("push eax ");

	asm("call %a0" : : "i" (&send_resched_ipis));

	asm("add esp, 4 ");

	asm("irq_user_check: ");

	asm("test dword ptr [esp+%0], 3" : : "i" _FOFF(SThreadExcStack, iCs));	// returning to user mode?

	asm("jz short irq_exit ");		// no so just return

#ifdef __CHECK_LOCK_STATE__

	asm("call %a0" : : "i" (&check_lock_state));

#endif

	asm("cmp dword ptr [edi+%0], 4" : : "i" _FOFF(NThreadBase,iUserModeCallbacks));

	asm("jb short irq_exit ");		// no callbacks so just return

	asm("call run_user_mode_callbacks ");

	asm("jmp short irq_exit ");

	asm("irq_kernel_locked_exit: ");

	asm("nested_irq_exit: ");

#ifdef BTRACE_CPU_USAGE

	asm("cmp byte ptr %a0, 0" : : "i" (&BTraceData.iFilter[4/*BTrace::ECpuUsage*/]));

	asm("jz short no_trace2 ");

	asm("call %a0" : : "i" (IrqEndTrace));

	asm("no_trace2: ");

#endif

	asm("irq_exit: ");

	asm("pop ecx ");

	asm("pop edx ");

	asm("pop ebx ");

	asm("pop esi ");

	asm("pop edi ");

	asm("pop ebp ");

	asm("pop eax ");

	asm("pop ds ");

	asm("pop es ");

	asm("pop fs ");

	asm("pop gs ");

	asm("add esp, 8 ");

	asm("iretd ");

	}

/******************************************************************************

 * General Exception Handler

 * On entry SS:ESP references current threads supervisor stack

 * [ESP+0] = vector number

 * [ESP+4] = error code (filled with 0 for exceptions without error codes)

 * [ESP+8] = return EIP

 * [ESP+12] = return CS

 * [ESP+16] = return EFLAGS

 * [ESP+20] = return ESP if privilege change occurred

 * [ESP+24] = return SS if privilege change occurred

 ******************************************************************************/

GLDEF_C __NAKED__ void __X86VectorExc()

	{

	// Interrupts disabled on entry

	asm("cld ");

	asm("push gs ");

	asm("push fs ");

	asm("push es ");

	asm("push ds ");

	asm("push eax ");

	asm("push ebp ");

	asm("push edi ");

	asm("push esi ");

	asm("push ebx ");

	asm("push edx ");

	asm("push ecx ");

	asm("mov bp, ss ");

	asm("mov ds, bp ");

	asm("mov es, bp ");

	asm("mov eax, cr2 ");

	asm("push eax ");

	asm("sub esp, 8 ");

	asm("mov ebp, esp ");		// ebp points to exception info frame

	asm("mov esi, ds:[%0]" : : "i" (X86_LOCAL_APIC_BASE + X86_LOCAL_APIC_OFFSET_ID));

	asm("shr esi, 24 ");

	asm("mov esi, [esi*4+%0]" : : "i" (&SubSchedulerLookupTable));			// esi -> subscheduler

	asm("mov edi, [esi+%0]" : : "i" _FOFF(TSubScheduler, iCurrentThread));	// edi -> current thread

	asm("xor eax, eax ");

	asm("mov ax, ss ");

	asm("mov [ebp+4], eax ");	// SS

	asm("lea eax, [ebp+%0]" : : "i" _FOFF(TX86ExcInfo,iEsp3));	// EAX = ESP at point of exception if ring 0

	asm("test dword ptr [ebp+%0], 3 " : : "i" _FOFF(TX86ExcInfo,iCs));		// check if we came from kernel mode

	asm("jz short ring0_exception ");

	asm("add eax, 8 ");			// EAX = ESP at point of exception if ring 3

	asm("mov cx, %0" : : "i" (KRing0DS));

	asm("mov gs, cx ");			// exception in user mode -> GS = user mode DS

	asm("ring0_exception: ");

	asm("mov [ebp], eax ");

	asm("cmp dword ptr [esi+52+%0], -1 " : : "i" _FOFF(TSubScheduler, iExtras));

	asm("jnz short fatal_exception_irq ");

	asm("cmp dword ptr [esi+%0], 0 " : : "i" _FOFF(TSubScheduler, iKernLockCount));

	asm("jnz short fatal_exception_locked ");

	asm("sti ");

	asm("cmp dword ptr [ebp+%0], 7 " : : "i" _FOFF(TX86ExcInfo, iExcId)); // check for device not available

	asm("jne short not_fpu ");

	asm("call %a0" : : "i" (NKern_Lock));

	asm("clts ");