We need a way to pass flags to rombuilds in Raptor via extension flm interfaces, so that the CPP pass
of the rom input files can be informed what toolchain we are building with and conditionally
include or exclude files depending on whether the toolchain could build them.
// Copyright (c) 1998-2009 Nokia Corporation and/or its subsidiary(-ies).
// All rights reserved.
// This component and the accompanying materials are made available
// under the terms of the License "Eclipse Public License v1.0"
// which accompanies this distribution, and is available
// at the URL "http://www.eclipse.org/legal/epl-v10.html".
//
// Initial Contributors:
// Nokia Corporation - initial contribution.
//
// Contributors:
//
// Description:
// e32\nkern\arm\vectors.cia
//
//
#define __INCLUDE_NTHREADBASE_DEFINES__
#include <e32cia.h>
#include <arm.h>
void FastMutexNestAttempt();
void FastMutexSignalError();
extern "C" void ExcFault(TAny*);
#ifdef __CPU_HAS_MMU
#define __USE_CP15_FAULT_INFO__
#endif
#ifdef _DEBUG
#define __CHECK_LOCK_STATE__
#endif
//#define __FAULT_ON_FIQ__
#ifdef __CHECK_LOCK_STATE__
// Check that the kernel is unlocked, no fast mutexes are held and that the thread is not in a
// critical section. Called when returning to user mode
__NAKED__ void CheckLockState()
{
asm("stmfd sp!, {r14}");
asm("ldr r12, __TheScheduler ");
asm("ldr r14, [r12, #%a0]" : : "i" _FOFF(TScheduler,iKernCSLocked));
asm("cmp r14, #0 ");
asm("movne r12, #0xdd000000 ");
asm("strne r12, [r12, #1] ");
asm("ldr r12, [r12, #%a0]" : : "i" _FOFF(TScheduler,iCurrentThread));
asm("ldr r14, [r12, #%a0]" : : "i" _FOFF(NThread,iHeldFastMutex));
asm("cmp r14, #0 ");
__CPOPRET(eq, "");
asm("badLockState: ");
asm("mov r12, #0xd7 ");
asm("msr cpsr, r12 ");
asm("mov r12, #0xdd000000 ");
asm("str r12, [r12, #3] ");
}
#endif
__ASSERT_COMPILE(EUserModeCallbackRun == 0);
__NAKED__ void CallUserModeCallbacks()
{
// called with interrupts disabled
// preserves r0 and r1 in additional to usual registers
// leaves current thread in r2
// the vast majority of times this is called with zero or one callback pending
asm(".global callUserModeCallbacks ");
asm("callUserModeCallbacks: ");
asm("ldr ip, __TheScheduler ");
asm("ldr r2, [ip, #%a0]" : : "i" _FOFF(TScheduler, iCurrentThread));
asm("callUserModeCallbacks2: ");
USER_MEMORY_GUARD_ASSERT_ON(ip);
#ifdef __CHECK_LOCK_STATE__
asm("ldr ip, [r2, #%a0]" : : "i" _FOFF(NThread,iCsCount));
asm("cmp ip, #0 ");
asm("bne badLockState ");
#endif
asm("ldr ip, [r2, #%a0]" : : "i" _FOFF(NThread, iUserModeCallbacks));
asm("teq ip, #0");
asm("bne 1f");
__JUMP(,lr);
asm("1: ");
asm("stmfd sp!, {r0-r2, r4-r11, lr}");
asm("movs r4, r3");
// if r3 != 0 it is the user context type to set the thread to
asm("strneb r3, [r2, #%a0]" : : "i" _FOFF(NThread, iUserContextType));
// Remove first callback and enter critical section - we can just set iCsCount to 1 as we are
// guaranteed not be be in a critical section already
asm("ldmia ip, {r1, r3} "); // HARDCODED: TUserModeCallback layout
asm("mov r0, #1");
asm("str r0, [r2, #%a0]" : : "i" _FOFF(NThread, iCsCount));
asm("str r1, [r2, #%a0]" : : "i" _FOFF(NThread,iUserModeCallbacks));
// Re-enable interrupts and call callback
SET_MODE(r0, MODE_SVC, INTS_ALL_ON);
asm("mov r1, #%a0 " : : "i" ((TInt)KUserModeCallbackUnqueued));
asm("str r1, [ip, #%a0]" : : "i" _FOFF(TUserModeCallback, iNext));
asm("mov r0, ip");
asm("mov r1, #0 "); // 0 == EUserModeCallbackRun
__JUMPL(3);
SET_MODE(r0, MODE_SVC, INTS_ALL_OFF);
asm("movs r3, r4");
// Leave critical section, avoid calling NKern::ThreadLeaveCS unless we have to
asm("ldmfd sp!, {r0-r2, r4-r11, lr}");
// reset user context type to undefined if r3 != 0
asm("mov ip, #%a0" : : "i" (NThread::EContextUndefined));
asm("strneb ip, [r2, #%a0]" : : "i" _FOFF(NThread, iUserContextType));
asm("ldr ip, [r2, #%a0]" : : "i" _FOFF(NThread, iCsFunction));
asm("teq ip, #0");
asm("streq ip, [r2, #%a0]" : : "i" _FOFF(NThread, iCsCount));
asm("beq callUserModeCallbacks2 ");
asm("leaveCS:");
asm("sub sp, sp, #48 ");
SET_MODE(r0, MODE_SVC, INTS_ALL_ON);
asm("bl " CSM_ZN5NKern13ThreadLeaveCSEv);
SET_MODE(r0, MODE_SVC, INTS_ALL_OFF);
asm("ldmfd sp!, {r0-r2, r4-r11, lr}");
asm("b callUserModeCallbacks2 ");
}
/***************************************************************************
* SWI Handler
***************************************************************************/
extern "C" __NAKED__ void __ArmVectorSwi()
{
// IRQs disabled, FIQs enabled here
asm("ldr r12, [lr, #-4] "); // get SWI opcode
asm("stmfd sp!, {r11, lr} "); // save return address, r11 for 8 byte align
USER_MEMORY_GUARD_ON_IF_MODE_USR(r11);
asm("ldr r11, __TheScheduler ");
asm("adr lr, fast_swi_exit ");
asm("movs r12, r12, lsl #9 "); // 512*SWI number into r12
asm("bcc slow_swi "); // bit 23=0 for slow/unprot
asm("ldr r1, [r11, #%a0]" : : "i" _FOFF(TScheduler,iCurrentThread));
asm("beq wait_for_any_request "); // special case for Exec::WaitForAnyRequest
#ifdef __CPU_ARM_HAS_CPS
asm("ldr r2, [r1, #%a0]" : : "i" _FOFF(NThread,iFastExecTable));
CPSIDIF; // all interrupts off
asm("ldr r3, [r2], r12, lsr #7 "); // r3=limit, r2->dispatch table entry
asm("ldr r2, [r2] "); // r2->kernel function
asm("cmp r3, r12, lsr #9 "); // r3-SWI number
__JUMP(hi, r2); // if SWI number valid, call kernel function
#else
SET_INTS(r2, MODE_SVC, INTS_ALL_OFF);
asm("ldr r2, [r1, #%a0]" : : "i" _FOFF(NThread,iFastExecTable));
asm("ldr r3, [r2], r12, lsr #7 "); // r3=limit, r2->dispatch table entry
asm("cmp r3, r12, lsr #9 "); // r3-SWI number
asm("ldrhi pc, [r2] "); // if SWI number valid, call kernel function
#endif
asm("mvn r12, #0 "); // put invalid SWI number into r12
asm("b slow_swi "); // go through slow SWI routine to call invalid SWI handler
asm("fast_swi_exit: ");
#ifdef __CHECK_LOCK_STATE__
asm("mrs r12, spsr ");
asm("tst r12, #0x0f ");
asm("bleq " CSM_Z14CheckLockStatev);
#endif
USER_MEMORY_GUARD_OFF_IF_MODE_USR(r11);
ERRATUM_353494_MODE_CHANGE(,r11);
asm("ldmfd sp!, {r11, pc}^ "); // return and restore cpsr
asm("slow_swi: "); // IRQs off, FIQs on here
asm("stmfd sp!, {r3-r10} "); // save nonvolatile registers, r3 for 8 byte align
asm("ldr r9, [r11, #%a0]" : : "i" _FOFF(TScheduler,iCurrentThread)); // r9->current thread
SET_INTS(lr, MODE_SVC, INTS_ALL_ON); // all interrupts on
asm("mov r10, r11 "); // r10->scheduler
asm("ldr r4, [r9, #%a0]" : : "i" _FOFF(NThread,iSlowExecTable));
asm("mrs r11, spsr "); // spsr_svc into r11
asm("adr lr, slow_swi_exit ");
asm("add r6, r4, r12, lsr #6 "); // r6->dispatch table entry
asm("ldr r5, [r4, #-12] "); // r5=limit
SET_INTS_1(r7, MODE_SVC, INTS_ALL_OFF);
asm("cmp r5, r12, lsr #9 "); // r5-SWI number
asm("ldmhiia r6, {r5,r6} "); // if SWI number OK, flags into r5, function addr into r6
asm("ldrls pc, [r4, #-8] "); // if SWI number invalid, call invalid handler
asm("tst r5, #%a0" : : "i" ((TInt)KExecFlagExtraArgMask)); // extra arguments needed?
asm("addne r2, sp, #4 "); // if so, point r2 at saved registers on stack
asm("tst r5, #%a0" : : "i" ((TInt)KExecFlagClaim)); // claim system lock?
asm("beq slow_swi_no_wait "); // skip if not
SET_INTS_2(r7, MODE_SVC, INTS_ALL_OFF); // interrupts off
#ifdef _DEBUG
asm("ldr r12, [r9, #%a0]" : : "i" _FOFF(NThread,iHeldFastMutex));
asm("cmp r12, #0 ");
asm("bne " CSM_Z20FastMutexNestAttemptv); // debug check that current thread doesn't already hold a fast mutex
#endif
asm("ldr r12, [r10, #%a0]!" : : "i" _FOFF(TScheduler,iLock.iHoldingThread)); // r12=iLock.iHoldingThread
SET_INTS_1(r7, MODE_SVC, INTS_ALL_ON);
asm("cmp r12, #0 "); // is system lock already held?
asm("bne ss_fast_mutex_held "); // branch if it is
asm("ss_fast_mutex_obtained: ");
asm("str r10, [r9, #%a0]" : : "i" _FOFF(NThread,iHeldFastMutex)); // current thread->iHeldFastMutex=&iLock
asm("str r9, [r10], #-%a0" : : "i" _FOFF(TScheduler,iLock)); // iLock.iHoldingThread=current thread, r10->scheduler
#ifdef BTRACE_FAST_MUTEX
asm("ldrb r12, [r10,#%a0]" : : "i" (_FOFF(TScheduler,iFastMutexFilter)));
asm("cmp r12, #0");
asm("bne syslock_trace_wait");
asm("syslock_trace_wait_done:");
#endif
SET_INTS_2(r7, MODE_SVC, INTS_ALL_ON); // all interrupts on
asm("slow_swi_no_wait: ");
asm("tst r5, #%a0" : : "i" ((TInt)KExecFlagPreprocess)); // preprocess (handle lookup)? can use r4, r7, r8, r12, r0
asm("mov lr, pc ");
asm("ldrne pc, [r4, #-4] "); // call preprocess handler if required
asm("mov lr, pc ");
__JUMP(,r6); // call exec function, preserve r5,r11 if release syslock not required
// preserve r5,r9,r10,r11 if release required
asm("tst r5, #%a0" : : "i" ((TInt)KExecFlagRelease)); // release system lock?
asm("beq slow_swi_exit "); // skip if not
SET_INTS(r12, MODE_SVC, INTS_ALL_OFF); // disable interrupts
#ifdef _DEBUG
asm("add r8, r10, #%a0" : : "i" _FOFF(TScheduler,iLock));
asm("ldr r12, [r9, #%a0]" : : "i" _FOFF(NThread,iHeldFastMutex));
asm("cmp r12, r8 ");
asm("bne " CSM_Z20FastMutexSignalErrorv); // debug check that current thread holds system lock
#endif
#ifdef BTRACE_FAST_MUTEX
asm("ldrb r12, [r10,#%a0]" : : "i" (_FOFF(TScheduler,iFastMutexFilter)));
asm("cmp r12, #0");
asm("bne syslock_trace_signal");
asm("syslock_trace_signal_done:");
#endif
asm("mov r12, #0 ");
asm("str r12, [r10, #%a0]" : : "i" _FOFF(TScheduler,iLock.iHoldingThread)); // iLock.iHoldingThread=NULL
asm("str r12, [r9, #%a0]" : : "i" _FOFF(NThread,iHeldFastMutex)); // current thread->iHeldFastMutex=NULL
asm("ldr r3, [r10, #%a0]" : : "i" _FOFF(TScheduler,iLock.iWaiting)); // r3=iLock.iWaiting
asm("str r12, [r10, #%a0]" : : "i" _FOFF(TScheduler,iLock.iWaiting)); // iLock.iWaiting=0
SET_INTS_1(r8, MODE_SVC, INTS_ALL_ON);
asm("cmp r3, #0 "); // check waiting flag
asm("bne ss_signal_check "); // branch if set
asm("ss_signal_done: ");
SET_INTS_2(r8, MODE_SVC, INTS_ALL_ON); // otherwise reenable interrupts
asm("slow_swi_exit: ");
#ifdef __CHECK_LOCK_STATE__
asm("tst r11, #0x0f ");
asm("bleq " CSM_Z14CheckLockStatev);
#endif
SET_INTS(r12, MODE_SVC, INTS_ALL_OFF); // disable interrupts
asm("msr spsr, r11 "); // restore spsr_svc
asm("tst r11, #0x0f ");
asm("mov r3, #0 ");
#if defined(__CPU_CORTEX_A9__) && !defined(__CPU_ARM_A9_ERRATUM_571622_FIXED)
asm("nop "); // ARM Cortex-A9 MPCore erratum 571622 workaround
// Insert nops so branch doesn't occur in 2nd or 3rd position after a msr spsr
#endif
asm("bleq callUserModeCallbacks "); // call user-mode callbacks
USER_MEMORY_GUARD_OFF_IF_MODE_USR(r11);
ERRATUM_353494_MODE_CHANGE(,r11);
asm("ldmfd sp!, {r3-r11,pc}^ "); // return from EXEC function
// Come here if we need to wait for the system lock
// r9->current thread, r10=&iLock, r12=iLock.iHoldingThread
asm("ss_fast_mutex_held: ");
asm("mov r8, #1 ");
asm("str r8, [r10, #%a0]" : : "i" (_FOFF(TScheduler,iKernCSLocked)-_FOFF(TScheduler,iLock))); // lock the kernel
SET_INTS_2(r7, MODE_SVC, INTS_ALL_ON); // enable interrupts
asm("str r8, [r10, #4] "); // iWaiting=1
asm("str r10, [r9, #%a0]" : : "i" _FOFF(NThread,iWaitFastMutex)); // current thread->iWaitFastMutex=&iLock
asm("stmfd sp!, {r0-r3} "); // save exec call arguments
asm("mov r0, r12 "); // parameter for YieldTo
ASM_DEBUG1(NKFMWaitYield,r0);
asm("bl " CSM_ZN10TScheduler7YieldToEP11NThreadBase); // yield to the mutex holding thread
// will not return until the mutex is free
// on return r0=Scheduler,r1=0,r2!=0,r3=current thread, kernel unlocked, interrupts disabled
asm("str r1, [r9, #%a0]" : : "i" _FOFF(NThread,iWaitFastMutex)); // iWaitFastMutex=NULL
asm("ldmfd sp!, {r0-r3} "); // retrieve exec call arguments
asm("b ss_fast_mutex_obtained "); // branch back to main code path
// Come here if we need to reschedule after releasing the system lock
// kernel unlocked, interrupts enabled, r0 contains return value from Exec call
// r9->current thread, r10=&TheScheduler, r3=1, r8=0x13
asm("ss_signal_check: ");
asm("str r3, [r10, #%a0]" : : "i" _FOFF(TScheduler,iKernCSLocked)); // lock the kernel (assumes iWaiting always 0 or 1)
SET_INTS_2(r8, MODE_SVC, INTS_ALL_ON); // reenable interrupts
asm("strb r3, [r10, #%a0]" : : "i" _FOFF(TScheduler,iRescheduleNeededFlag));
asm("ldr r3, [r9, #%a0]" : : "i" _FOFF(NThread,iCsFunction)); // r3=current thread->iCsFunction
asm("ldr r2, [r9, #%a0]" : : "i" _FOFF(NThread,iCsCount)); // r2=current thread->iCsCount
asm("mov r4, r0 "); // save return value
asm("cmp r3, #0 "); // outstanding CS function?
asm("beq 2f "); // branch if not
asm("cmp r2, #0 "); // iCsCount!=0 ?
asm("moveq r0, r9 "); // if iCsCount=0, DoCsFunction()
asm("bleq " CSM_ZN11NThreadBase12DoCsFunctionEv);
asm("2: ");
asm("bl " CSM_ZN10TScheduler10RescheduleEv); // reschedule to allow waiting thread in
asm("mov r0, r4 "); // recover return value
asm("b ss_signal_done "); // branch back to main code path
#ifdef BTRACE_FAST_MUTEX
asm("syslock_trace_wait:");
asm("ldr r12, [sp,#9*4]"); // r12 = return address from SWI
asm("mov r8, r3"); // save r3
asm("stmdb sp!,{r0-r2,r12}"); // 4th item on stack is PC value for trace
asm("ldr r0, fmwait_trace_header");
asm("mov r2, r9"); // current thread
asm("add r3, r10, #%a0" : : "i" _FOFF(TScheduler,iLock));
asm("mov lr, pc");
asm("ldr pc, [r10, #%a0]" : : "i" _FOFF(TScheduler,iBTraceHandler));
asm("ldmia sp!,{r0-r2,r12}");
asm("mov r3, r8"); // restore r3
asm("b syslock_trace_wait_done");
asm("syslock_trace_signal:");
asm("ldr r12, [sp,#9*4]"); // r12 = return address from SWI
asm("stmdb sp!,{r0-r2,r12}"); // 4th item on stack is PC value for trace
asm("ldr r0, fmsignal_trace_header");
asm("mov r2, r9"); // current thread
asm("add r3, r10, #%a0" : : "i" _FOFF(TScheduler,iLock));
asm("mov lr, pc");
asm("ldr pc, [r10, #%a0]" : : "i" _FOFF(TScheduler,iBTraceHandler));
asm("ldmia sp!,{r0-r2,r12}");
asm("b syslock_trace_signal_done");
asm("fmsignal_trace_header:");
asm(".word %a0" : : "i" ((TInt)(16<<BTrace::ESizeIndex) + ((BTrace::EContextIdPresent|BTrace::EPcPresent) << BTrace::EFlagsIndex*8) + (BTrace::EFastMutex<< BTrace::ECategoryIndex*8) + (BTrace::EFastMutexSignal << BTrace::ESubCategoryIndex*8)) );
asm("fmwait_trace_header:");
asm(".word %a0" : : "i" ((TInt)(16<<BTrace::ESizeIndex) + ((BTrace::EContextIdPresent|BTrace::EPcPresent) << BTrace::EFlagsIndex*8) + (BTrace::EFastMutex << BTrace::ECategoryIndex*8) + (BTrace::EFastMutexWait << BTrace::ESubCategoryIndex*8)) );
#endif
}
/***************************************************************************
* IRQ Postamble
* This routine is called after the IRQ has been dispatched
* spsr_irq, r4-r11 are unmodified
* spsr_irq,r0-r3,r12,return address are on the top of the IRQ stack
***************************************************************************/
extern "C" __NAKED__ void __ArmVectorIrq()
{
// FIQs enabled here but not IRQs
asm("ldr r1, __TheScheduler ");
asm("mrs r0, spsr "); // check interrupted mode
asm("add r12, sp, #%a0 " : : "i" (4*(6+USER_MEMORY_GUARD_SAVE_WORDS))); // r12=sp_irq+6 or 8 words
asm("and r2, r0, #0x1f ");
asm("ldr r3, [r1, #%a0]" : : "i" _FOFF(TScheduler,iKernCSLocked)); // r3=KernCSLocked
asm("cmp r2, #0x10 "); // check for mode_usr
asm("cmpne r2, #0x13 "); // or mode_svc
asm("cmpeq r3, #0 "); // and then check if kernel locked
asm("bne IrqExit0 "); // if wrong mode or locked, return immediately
SET_INTS(r2, MODE_IRQ, INTS_ALL_OFF); // disable FIQs before we check for reschedule
asm("ldr r2, [r1, #%a0]" : : "i" _FOFF(TScheduler,iRescheduleNeededFlag)); // r2=DfcPendingFlag/RescheduleNeededFlag
asm("add r3, r3, #1 ");
SET_MODE_1(lr, MODE_SVC, INTS_ALL_ON);
asm("cmp r2, #0 "); // check if reschedule needed
asm("beq IrqExit0 "); // if not, return immediately
asm("str r3, [r1, #%a0]" : : "i" _FOFF(TScheduler,iKernCSLocked)); // lock the kernel
SET_MODE_2(lr, MODE_SVC, INTS_ALL_ON); // mode_svc, interrupts back on
asm("ldmdb r12!, {r1-r3} "); // move saved registers (r0-r3,r12,pc) over to mode_svc stack
asm("stmfd sp!, {r1-r3} ");
asm("ldmdb r12!, {r1-r3} ");
asm("stmfd sp!, {r1-r3} ");
asm("stmfd sp!, {r0,lr} "); // store lr_svc and interrupted cpsr on current mode_svc stack
#ifdef __USER_MEMORY_GUARDS_ENABLED__
asm("ldmdb r12, {r1-r2} ");
asm("stmfd sp!, {r1-r2} "); // move user guard over to mode_svc stack
#endif
SET_MODE_1(r2, MODE_SVC, INTS_ALL_ON);
SET_MODE(lr, MODE_IRQ, INTS_IRQ_OFF); // mode_irq, IRQs off
asm("add sp, r12, #24 "); // restore mode_irq stack balance
SET_MODE_2(r2, MODE_SVC, INTS_ALL_ON); // back to mode_svc, IRQs on
// reschedule - this also switches context if necessary
// enter this function in mode_svc, interrupts on, kernel locked
// exit this function in mode_svc, all interrupts off, kernel unlocked
asm("irq_do_resched: ");
asm("bl " CSM_ZN10TScheduler10RescheduleEv);
asm(".global irq_resched_return ");
asm("irq_resched_return: ");
SET_MODE(r2, MODE_SVC, INTS_ALL_OFF); // all interrupts off
asm("ldr r1, [sp, #%a0] " : : "i" (4*USER_MEMORY_GUARD_SAVE_WORDS)); // get interrupted cpsr, don't unbalance stack
#ifdef __CHECK_LOCK_STATE__
asm("mov r2, r12 ");
asm("tst r1, #0x0f ");
asm("bleq " CSM_Z14CheckLockStatev);
asm("mov r12, r2 ");
#endif
asm("tst r1, #0x0f ");
asm("mov r3, #%a0 " : : "i" (NThread::EContextUserIntrCallback));
asm("bleq callUserModeCallbacks "); // call user-mode callbacks
#ifdef __USER_MEMORY_GUARDS_ENABLED__
asm("ldr r1, [sp], #%a0 " : : "i" (4*USER_MEMORY_GUARD_SAVE_WORDS)); // pop saved DACR, adjust sp
USER_MEMORY_GUARD_RESTORE(r1,lr);
#endif
asm("ldmfd sp!, {r1, lr} "); // restore lr_svc
asm("add sp, sp, #24 "); // restore mode_svc stack balance
asm("mov r12, sp "); // r12=address of remaining saved registers
SET_MODE(r2, MODE_IRQ, INTS_ALL_OFF); // back into mode_irq, all interrupts off
asm("msr spsr, r1 "); // interrupted cpsr into spsr_irq
ERRATUM_353494_MODE_CHANGE(,r12);
asm("ldmdb r12, {r0-r3,r12,pc}^ "); // return from interrupt
asm("IrqExit0: ");
#ifdef __CHECK_LOCK_STATE__
asm("tst r0, #0x0f ");
asm("bleq " CSM_Z14CheckLockStatev);
#endif
asm("IrqExit1: "); // entry point for __ArmVectorIrqPostambleNoResched()
#ifdef __USER_MEMORY_GUARDS_ENABLED__
asm("ldr lr, [sp], #%a0 " : : "i" (4*USER_MEMORY_GUARD_SAVE_WORDS)); // pop saved DACR, adjust sp
USER_MEMORY_GUARD_RESTORE(lr,r12);
#endif
#ifdef BTRACE_CPU_USAGE
asm("ldrb r2, [r1,#%a0]" : : "i" _FOFF(TScheduler,iCpuUsageFilter));
asm("mov r0, #%a0" : : "i" ((TInt)4 ) );
asm("add r0, r0, #%a0" : : "i" ((TInt)(BTrace::ECpuUsage<<BTrace::ECategoryIndex*8)+(BTrace::EIrqEnd<<BTrace::ESubCategoryIndex*8)) );
asm("cmp r2, #0");
asm("movne lr, pc");
asm("ldrne pc, [r1,#%a0]" : : "i" _FOFF(TScheduler,iBTraceHandler));
#endif
ERRATUM_353494_MODE_CHANGE(,r12);
asm("ldmfd sp!, {r0-r3,r12,pc}^ "); // return from interrupt
}
/***************************************************************************
* IRQ Postamble which will not reschedule (can be returned to by co-resident OS).
* This routine is called after the IRQ has been dispatched
* spsr_irq, r4-r11 are unmodified
* spsr_irq,r0-r3,r12,return address are on the top of the IRQ stack
***************************************************************************/
extern "C" EXPORT_C __NAKED__ void __ArmVectorIrqPostambleNoResched()
{
// FIQs enabled here but not IRQs
asm("ldr r1, __TheScheduler ");
asm("b IrqExit1 ");
}
/***************************************************************************
* FIQ Postamble
* This routine is called after the FIQ has been dispatched
* spsr_fiq, r0-r3 are unmodified
* Return address is on the top of the FIQ stack
***************************************************************************/
extern "C" __NAKED__ void __ArmVectorFiq()
{
#ifdef __FAULT_ON_FIQ__
asm(".word 0xe7f10f10 ");
#endif
// IRQs and FIQs disabled here
// r0-r7 are unaltered from when FIQ occurred
asm("ldr r9, __TheScheduler ");
#ifdef __USER_MEMORY_GUARDS_ENABLED__
asm("ldr r12, [sp], #4 "); // pop saved DACR
#endif
asm("mrs r8, spsr "); // check interrupted mode
asm("and r10, r8, #0x1f ");
asm("cmp r10, #0x10 "); // check for mode_usr
asm("ldr r11, [r9, #%a0]" : : "i" _FOFF(TScheduler,iKernCSLocked));
asm("cmpne r10, #0x13 "); // or mode_svc
asm("ldreq r10, [r9, #%a0]" : : "i" _FOFF(TScheduler,iRescheduleNeededFlag));
asm("cmpeq r11, #0 "); // and check if kernel locked
asm("bne FiqExit0 "); // if wrong mode or kernel locked, return immediately
asm("cmp r10, #0 "); // check if reschedule needed
asm("beq FiqExit0 "); // if not, return from interrupt
// we interrupted mode_usr or mode_svc, kernel unlocked, reschedule needed
asm("add r11, r11, #1 ");
asm("str r11, [r9, #%a0]" : : "i" _FOFF(TScheduler,iKernCSLocked)); // lock the kernel
asm("stmfd sp!, {r1-r3} "); // save interrupted r1-r3 on FIQ stack
asm("mov r1, r8 "); // r1=interrupted cpsr
asm("mov r3, sp "); // r3 points to saved registers
#ifdef __USER_MEMORY_GUARDS_ENABLED__
asm("mov r2, r12 "); // saved DACR into R2
#endif
SET_MODE(lr, MODE_SVC, INTS_ALL_ON); // switch to mode_svc, IRQs and FIQs back on
#ifdef __USER_MEMORY_GUARDS_ENABLED__
asm("str r2, [sp, #%a0]! " : : "i" (-4*(8+USER_MEMORY_GUARD_SAVE_WORDS))); // save DACR and leave room for spare, cpsr, lr_svc, r0-r3, r12, pc
#else
asm("sub sp, sp, #32 "); // make room for saved registers on mode_svc stack
#endif
asm("ldr r2, [r3, #12] "); // r2=return address
asm("str r12, [sp, #%a0] " : : "i" (4*(6+USER_MEMORY_GUARD_SAVE_WORDS))); // save r12 on mode_svc stack
asm("str r2, [sp, #%a0] " : : "i" (4*(7+USER_MEMORY_GUARD_SAVE_WORDS))); // save return address on mode_svc stack
asm("add r12, sp, #%a0 " : : "i" (4*(USER_MEMORY_GUARD_SAVE_WORDS)));
asm("stmia r12!, {r1,lr} "); // save interrupted cpsr and lr_svc
asm("ldmia r3, {r1,r2,lr} "); // retrieve original r1-r3 from mode_fiq stack
asm("stmia r12, {r0-r2,lr} "); // save original r0-r3 - saved register order is now cpsr,lr_svc,r0-r3,r12,pc
SET_MODE_1(r2, MODE_SVC, INTS_ALL_ON);
SET_MODE(lr, MODE_FIQ, INTS_ALL_OFF); // mode_fiq, IRQs and FIQs off
asm("add sp, r3, #16 "); // restore mode_fiq stack balance
SET_MODE_2(r2, MODE_SVC, INTS_ALL_ON); // back to mode_svc, IRQs on
asm("adr lr, irq_resched_return ");
asm("b " CSM_ZN10TScheduler10RescheduleEv); // do reschedule and return to irq_resched_return
asm("FiqExit0:"); // also entry point for __ArmVectorFiqPostambleNoResched()
USER_MEMORY_GUARD_RESTORE(r12,lr);
#ifndef BTRACE_CPU_USAGE
ERRATUM_353494_MODE_CHANGE(,r11);
asm("ldmfd sp!, {pc}^ "); // return from interrupt
#else
asm("ldrb r8, [r9,#%a0]" : : "i" _FOFF(TScheduler,iCpuUsageFilter));
asm("mov r10, #%a0" : : "i" ((TInt)(BTrace::ECpuUsage<<BTrace::ECategoryIndex*8)+(BTrace::EFiqEnd<<BTrace::ESubCategoryIndex*8)) );
asm("adr lr, FiqTraceExit0");
asm("cmp r8, #0");
ERRATUM_353494_MODE_CHANGE(eq,r8);
asm("ldmeqfd sp!, {pc}^ "); // return from interrupt if trace not enabled
asm("stmfd sp!, {r0-r3} ");
asm("add r0, r10, #%a0" : : "i" ((TInt)4 ) );
asm("ldr pc, [r9,#%a0]" : : "i" _FOFF(TScheduler,iBTraceHandler));
asm("FiqTraceExit0:");
ERRATUM_353494_MODE_CHANGE(,r3);
asm("ldmfd sp!, {r0-r3,pc}^ "); // return from interrupt
#endif
asm("__TheScheduler: ");
asm(".word TheScheduler ");
}
/***************************************************************************
* FIQ Postamble which will not reschedule (can be returned to by co-resident OS).
* This routine is called after the FIQ has been dispatched
* spsr_fiq, r0-r3 are unmodified
* Return address is on the top of the FIQ stack
***************************************************************************/
extern "C" EXPORT_C __NAKED__ void __ArmVectorFiqPostambleNoResched()
{
#ifdef __FAULT_ON_FIQ__
asm(".word 0xe7f10f10 ");
#endif
// IRQs and FIQs disabled here
// r0-r7 are unaltered from when FIQ occurred
asm("ldr r9, __TheScheduler ");
#ifdef __USER_MEMORY_GUARDS_ENABLED__
asm("ldr r12, [sp], #4 "); // pop saved DACR
#endif
asm("b FiqExit0 ");
}
extern "C" __NAKED__ void __ArmVectorAbortData()
//
// Data abort
//
{
#if defined(__CPU_CORTEX_A8__) && (!defined(__CPU_ARM_A8_ERRATUM_447862_FIXED) || !defined(__CPU_ARM_A8_ERRATUM_451027_FIXED))
ARM_DMBSH; // ARM Cortex-A8 erratum 447862/451027 workaround
#endif
asm("sub lr, lr, #8"); // lr now points to aborted instruction
asm("stmfd sp!, {r0-r4,r12,lr}"); // save it along with r0-r4,r12
#if defined(__CPU_ARM_HAS_WORKING_CLREX)
CLREX // reset exclusive monitor
#elif defined(__CPU_ARM_HAS_LDREX_STREX)
STREX(12,0,13); // dummy STREX to reset exclusivity monitor
#endif
asm("mov r1, #%a0 " : : "i" ((TInt)EArmExceptionDataAbort));
// generic exception handler
// come here with r1=exception code, lr points to aborted instruction, r0-r4,r12,lr saved
asm("handle_exception: ");
asm("mrs r0, spsr "); // r0=value of cpsr when abort occurred
asm("handle_exception2: ");
asm("mrs r12, cpsr ");
asm("and r3, r0, #0x1f "); // r3=processor mode when abort occurred
asm("bic r12, r12, #0xc0 ");
asm("cmp r3, #0x10 "); // aborted in user mode?
asm("cmpne r3, #0x13 "); // if not, aborted in mode_svc?
asm("bne fatal_exception_mode "); // if neither, fault
asm("msr cpsr, r12 "); // reenable interrupts - rescheduling disabled by mode_abt/mode_und
asm("ldr r2, __TheScheduler ");
asm("mov r3, sp "); // r3 points to saved registers
asm("ldr r4, [r2, #%a0]" : : "i" _FOFF(TScheduler,iKernCSLocked));
asm("cmp r4, #0 "); // exception with kernel locked?
asm("bne fatal_exception_mode "); // if so, fault
asm("add r4, r4, #1 "); // lock the kernel
asm("str r4, [r2, #%a0]" : : "i" _FOFF(TScheduler,iKernCSLocked));
asm("mov r4, #0x13 ");
asm("msr cpsr, r4 "); // mode_svc, interrupts on, kernel locked
asm("ldr r4, [r2, #%a0]" : : "i" _FOFF(TScheduler,iCurrentThread));
asm("tst r0, #0x0f "); // check if exception in mode_usr
asm("mov r2, #%a0 " : : "i" ((TInt)NThread::EContextException));
asm("streqb r2, [r4, #%a0]" : : "i" _FOFF(NThread,iSpare3)); // if so, set iUserContextType = EContextException
asm("add r4, r4, #%a0" : : "i" _FOFF(NThread,iStackBase));
asm("ldmia r4, {r2,r4} "); // r2=supervisor stack area base, r4=size
asm("subs r2, sp, r2 "); // r2=amount of mode_svc stack remaining
asm("blo fatal_exception_stack "); // if stack pointer invalid, fault
asm("cmp r2, r4 ");
asm("bhi fatal_exception_stack ");
asm("cmp r2, #128 "); // check enough stack to handle exception
asm("blo fatal_exception_stack "); // if not, fault
// At this point we are in mode_svc with interrupts enabled and the kernel locked.
// We know the supervisor stack is valid and has enough free space to store the exception info.
// Registers: R0=aborted cpsr, R1=exception type, R2,R4 scratch, R3 points to saved registers
// on mode_abt or mode_und stack, R12 holds mode of exception (mode_abt or mode_und).
asm("ldr r4, [r3, #16] "); // restore original r4
asm("mov r2, sp "); // r2=sp_svc when abort occurred
asm("sub sp, sp, #92 "); // push 23 words onto mode_svc stack
asm("stmia sp, {r0-r2,r4-r11,lr} "); // save cpsr, exc id, sp_svc, r4-r11, lr_svc
asm("ldmia r3!, {r4-r10} "); // get registers from mode_abt or mode_und stack
asm("stmdb r2!, {r4-r7,r9,r10} "); // transfer saved registers from exception stack except r4
asm("stmdb r2, {r13,r14}^ "); // save sp_usr and lr_usr
asm("sub r2, r2, #20 ");
// Set r0 = fault address and r1 = fault status.
// For prefetch aborts use IFAR if it exists otherwise use the return address.
#ifdef __USE_CP15_FAULT_INFO__
asm("cmp r1, #%a0 " : : "i" ((TInt)EArmExceptionPrefetchAbort));
#ifdef __CPU_ARM_HAS_SPLIT_FSR
asm("mrcne p15, 0, r1, c5, c0, 0"); // r1 = data fault status
asm("mrcne p15, 0, r0, c6, c0, 0"); // r0 = DFAR fault address
asm("mrceq p15, 0, r1, c5, c0, 1"); // r1 = instruction fault status
#ifdef __CPU_ARM_HAS_CP15_IFAR
asm("mrceq p15, 0, r0, c6, c0, 2"); // r0 = IFAR fault address
#else
asm("moveq r0, r10"); // r0 = return address.
#endif // __CPU_ARM_HAS_CP15_IFAR
#else
asm("mrcne p15, 0, r0, c6, c0"); // r0 = fault address
asm("moveq r0, r10"); // r0 = return address.
asm("mrc p15, 0, r1, c5, c0"); // r1 = fault status
#endif // __CPU_ARM_HAS_SPLIT_FSR
#endif // __USE_CP15_FAULT_INFO__
asm("mrs r3, spsr "); // r3=spsr_svc
asm("stmia r2, {r0,r1,r3} "); // save these
asm("msr cpsr, r12 "); // back into exception mode
asm("add sp, sp, #28 "); // restore exception stack balance
asm("mov r5, #0x13 ");
asm("msr cpsr, r5 "); // back into mode_svc
// Now we can unlock the kernel and process the exception
asm("bl " CSM_ZN10TScheduler10RescheduleEv);
asm("msr cpsr, r5 "); // enable interrupts
// call the exception dispatcher, r3 is the current thread
asm("ldr r12, [r3, #%a0]" : : "i" _FOFF(NThread,iHandlers));
asm("mov r1, r3 ");
asm("mov r0, sp "); // r0 points to saved exception information
asm("sub sp, sp, #4 "); // make room for r0
asm("bic sp, sp, #4 "); // align stack to 8 byte boundary
asm("str r0, [sp] "); // save original stack pointer
USER_MEMORY_GUARD_ON(,r11,lr);
asm("ldr r12, [r12, #%a0]" : : "i" _FOFF(SNThreadHandlers,iExceptionHandler));
asm("mov lr, pc ");
__JUMP(,r12); // call exception handler
USER_MEMORY_GUARD_RESTORE(r11,lr);
asm("ldr sp, [sp, #0] "); // restore stack pointer
// return from exception
asm("ldr r0, __TheScheduler ");
asm("mov r3, sp ");
asm("ldr r2, [r0, #%a0]" : : "i" _FOFF(TScheduler,iCurrentThread));
asm("ldr r0, [r3], #12 "); // r0=cpsr, skip exc id and sp_svc
asm("ldmfd r3!, {r4-r11,lr} "); // restore r4-r11 and lr_svc
asm("ldr r12, [r3, #8]! "); // skip fault address and fault status, r12=spsr_svc
asm("ldmib r3, {r13,r14}^ "); // restore sp_usr and lr_usr
asm("add r1, r3, #12 "); // r3 points to saved r0-r3,r12,pc
asm("mov r3, #0xd3 ");
asm("msr cpsr, r3 "); // mode_svc, all interrupts off
asm("msr spsr, r12 "); // restore spsr_svc
asm("tst r0, #0x0f "); // check if exception in mode_usr
#if defined(__CPU_CORTEX_A9__) && !defined(__CPU_ARM_A9_ERRATUM_571622_FIXED)
asm("nop "); // ARM Cortex-A9 MPCore erratum 571622 workaround
asm("nop "); // Insert nops so branch doesn't occur in 2nd or 3rd position after a msr spsr
#endif
#ifdef __CHECK_LOCK_STATE__
asm("bleq " CSM_Z14CheckLockStatev);
asm("tst r0, #0x0f "); // recheck if exception in mode_usr
#endif
asm("bne 1f ");
#ifdef __USER_MEMORY_GUARDS_ENABLED__
USER_MEMORY_GUARD_ON(,lr,r12);
asm("tst lr, #0xc0000000 "); // user memory enabled?
asm("adrne lr, 2f "); // yes - enable it after callbacks
#endif
asm("adreq lr, 1f "); // no - leave it disabled after callbacks
asm("mov r3, #0 ");
asm("b callUserModeCallbacks2 "); // call user-mode callbacks
asm("2: ");
USER_MEMORY_GUARD_OFF(,lr,lr);
asm("1: ");
asm("tst r0, #0x0f "); // check if exception in mode_usr
asm("mov r3, #%a0 " : : "i" ((TInt)NThread::EContextUndefined));
asm("streqb r3, [r2, #%a0]" : : "i" _FOFF(NThread,iSpare3)); // if so, set iUserContextType = EContextUndefined
asm("add sp, r1, #24 "); // restore mode_svc stack balance
asm("mov r2, #0xd7 ");
asm("msr cpsr, r2 "); // mode_abt, all interrupts off
asm("msr spsr, r0 "); // spsr_abt=aborted cpsr
ERRATUM_353494_MODE_CHANGE(,r12);
asm("ldmia r1, {r0-r3,r12,pc}^ "); // restore r0-r3,r12 and return from exception
// get here if exception occurred in mode other than usr or svc
// we are in mode_abt or mode_und with IRQs disabled
asm("fatal_exception_mode: ");
asm("ldr r2, __TheScheduler ");
asm("ldr lr, [r2, #%a0]" : : "i" _FOFF(TScheduler,iMonitorExceptionHandler));
asm("cmp lr, #0 ");
__JUMP(ne,lr); // if crash debugger running, let it handle exception
// get here if mode_svc stack has overflowed
// we are in mode_svc with interrupts enabled and the kernel locked
// R0=original CPSR R1=exc code R12=mode of exception
asm("fatal_exception_stack: ");
asm("orr r3, r12, #0xC0 ");
asm("msr cpsr, r3 "); // back to exception mode, all interrupts off
asm("mov r2, r0 ");
asm("ldr r0, __TheScheduler ");
asm("ldr r0, [r0, #%a0]" : : "i" _FOFF(TScheduler,i_Regs)); // pass in address of stored registers
asm("bl " CSM_ZN3Arm9SaveStateER14SFullArmRegSet );
asm("str r1, [r0, #%a0]" : : "i" _FOFF(SFullArmRegSet,iExcCode));
asm("str r2, [r0, #%a0]" : : "i" _FOFF(SFullArmRegSet,iN.iFlags));
asm("ldmia sp!, {r3-r7} "); // get original R0-R4
asm("stmia r0, {r1-r5} "); // save original R0-R4
asm("ldmia sp!, {r6,r7} "); // get original R12 and aborted instruction address
asm("str r6, [r0, #%a0]" : : "i" _FOFF(SFullArmRegSet,iN.iR12));
asm("str r7, [r0, #%a0]" : : "i" _FOFF(SFullArmRegSet,iN.iR15));
asm("mov r1, #13 "); // r1 = regnum
asm("mrs r2, cpsr "); // r2 = mode
asm("mov r4, r0 ");
asm("bl " CSM_ZN3Arm3RegER14SFullArmRegSetim ); // r0 = pointer to exception mode R13
asm("str sp, [r0] "); // save correct original value for exception mode R13
// call the exception fault dispatcher
asm("mov r0, #0 ");
asm("b ExcFault ");
}
extern "C" __NAKED__ void __ArmVectorAbortPrefetch()
//
// Prefetch abort
//
{
asm("sub lr, lr, #4"); // lr now points to instruction whose prefetch was aborted
asm("stmfd sp!, {r0-r4,r12,lr}"); // save it along with r0-r4,r12
#if defined(__CPU_ARM_HAS_WORKING_CLREX)
CLREX // reset exclusive monitor
#elif defined(__CPU_ARM_HAS_LDREX_STREX)
STREX(12,0,13); // dummy STREX to reset exclusivity monitor
#endif
asm("mov r1, #%a0 " : : "i" ((TInt)EArmExceptionPrefetchAbort));
asm("b handle_exception ");
}
extern "C" __NAKED__ void __ArmVectorUndef()
//
// Undefined instruction exception
//
{
asm("sub lr, lr, #4"); // lr now points to undefined instruction
asm("stmfd sp!, {r0-r4,r12,lr}"); // save it along with r0-r4,r12
#if defined(__CPU_ARM_HAS_WORKING_CLREX)
CLREX // reset exclusive monitor
#elif defined(__CPU_ARM_HAS_LDREX_STREX)
STREX(12,0,13); // dummy STREX to reset exclusivity monitor
#endif
asm("mrs r0, spsr "); // r0=CPSR at time of exception
asm("mov r1, #%a0 " : : "i" ((TInt)EArmExceptionUndefinedOpcode));
asm("tst r0, #0x20 "); // exception in THUMB mode?
asm("addne lr, lr, #2 "); // if so, correct saved return address
asm("strne lr, [sp, #24] ");
asm("b handle_exception2 ");
}