diff -r 2d65c2f76d7b -r 947f0dc9f7a8 kernel/eka/nkernsmp/arm/ncsched.cia --- a/kernel/eka/nkernsmp/arm/ncsched.cia Tue Feb 02 01:24:03 2010 +0200 +++ b/kernel/eka/nkernsmp/arm/ncsched.cia Fri Apr 16 16:24:37 2010 +0300 @@ -54,6 +54,7 @@ extern "C" void NewThreadTrace(NThread* a); extern "C" void send_accumulated_resched_ipis(); +extern "C" void wake_up_for_ipi(TSubScheduler*, TInt); __NAKED__ void TScheduler::Reschedule() @@ -71,7 +72,7 @@ __ASM_CLI(); // interrupts off asm("ldr r1, [r0, #%a0]" : : "i" (_FOFF(TSubScheduler,iDfcPendingFlag)&~3)); // check iDfcPendingFlag and iExIDfcPendingFlag asm("mov r11, r0 "); // r11->TSubScheduler - asm("ldr r10, [r0, #%a0]" : : "i" _FOFF(TSubScheduler,i_LocalTimerAddr)); // r10->CPU local timer + asm("ldr r10, [r0, #%a0]" : : "i" _FOFF(TSubScheduler,iSSX.iLocalTimerAddr)); // r10->CPU local timer asm("start_resched: "); asm("movs r1, r1, lsr #16 "); // check if IDFCs or ExIDFCs pending @@ -829,11 +830,13 @@ } -extern "C" __NAKED__ void send_irq_ipi(TSubScheduler*) +extern "C" __NAKED__ void send_irq_ipi(TSubScheduler*, TInt) { + asm("tst r1, #%a0" : : "i" ((TInt)EQueueEvent_WakeUp) ); + asm("bne " CSM_CFUNC(wake_up_for_ipi)); __DATA_SYNC_BARRIER_Z__(r3); // need DSB before sending any IPI asm("ldr r3, [r0, #%a0]" : : "i" _FOFF(TSubScheduler, iCpuMask)); - asm("ldr r2, [r0, #%a0]" : : "i" _FOFF(TSubScheduler, i_GicDistAddr)); // we assume i_GicDistAddr is the same for all CPUs + asm("ldr r2, [r0, #%a0]" : : "i" _FOFF(TSubScheduler, iSSX.iGicDistAddr)); // we assume i_GicDistAddr is the same for all CPUs asm("mov r1, #%a0" : : "i" ((TInt)TRANSFERRED_IRQ_VECTOR)); asm("orr r1, r1, r3, lsl #16 "); asm("str r1, [r2, #%a0]" : : "i" _FOFF(GicDistributor, iSoftIrq)); // trigger IPIs @@ -845,7 +848,12 @@ // Return with R0 unaltered. extern "C" __NAKED__ void send_accumulated_resched_ipis() { - asm("ldr r2, [r0, #%a0]" : : "i" _FOFF(TSubScheduler, i_GicDistAddr)); + asm("ldr r3, [r0, #%a0]" : : "i" _FOFF(TSubScheduler, iScheduler)); + asm("ldr r2, [r0, #%a0]" : : "i" _FOFF(TSubScheduler, iSSX.iGicDistAddr)); + asm("ldr r1, [r3, #%a0]" : : "i" _FOFF(TScheduler, iThreadAcceptCpus)); + asm("bics r1, r12, r1 "); + asm("bne 2f "); + asm("1: "); asm("mov r1, #0 "); asm("str r1, [r0, #%a0]" : : "i" _FOFF(TSubScheduler,iReschedIPIs)); __DATA_SYNC_BARRIER__(r1); // need DSB before sending any IPI @@ -853,6 +861,16 @@ // asm("orr r1, r1, #%a0" : : "i" ((TInt)RESCHED_IPI_VECTOR)); RESCHED_IPI_VECTOR=0 asm("str r1, [r2, #%a0]" : : "i" _FOFF(GicDistributor, iSoftIrq)); // trigger IPIs __JUMP(,lr); + + asm("2: "); + asm("stmfd sp!, {r0,lr} "); + asm("mov r0, r3 "); + asm("mov r1, r12 "); + asm("bl ReschedInactiveCpus__10TSchedulerUl "); + asm("mov r12, r0 "); + asm("ldmfd sp!, {r0,lr} "); + asm("ldr r2, [r0, #%a0]" : : "i" _FOFF(TSubScheduler, iSSX.iGicDistAddr)); + asm("b 1b "); } // Send a reschedule IPI to the specified CPU @@ -860,7 +878,7 @@ { GET_RWNO_TID(,r3); __DATA_SYNC_BARRIER_Z__(r2); // need DSB before sending any IPI - asm("ldr r2, [r3, #%a0]" : : "i" _FOFF(TSubScheduler, i_GicDistAddr)); // we assume i_GicDistAddr is the same for all CPUs + asm("ldr r2, [r3, #%a0]" : : "i" _FOFF(TSubScheduler, iSSX.iGicDistAddr)); // we assume i_GicDistAddr is the same for all CPUs ASM_DEBUG1(SendReschedIPI,r0); asm("mov r1, #0x10000 "); asm("mov r1, r1, lsl r0 "); // 0x10000< wait asm("cmp r2, #0 "); @@ -930,6 +934,7 @@ asm(".word TheSubSchedulers "); } + /* If the current thread is subject to timeslicing, update its remaining time from the current CPU's local timer. Don't stop the timer. If the remaining time is negative, save it as zero. @@ -938,49 +943,57 @@ { asm("ldr r3, [r1, #%a0]" : : "i" _FOFF(NThreadBase,iTime)); asm("ldrb r12, [r1, #%a0]" : : "i" _FOFF(NThreadBase,i_NThread_Initial)); - asm("ldr r2, [r0, #%a0]" : : "i" _FOFF(TSubScheduler,i_LocalTimerAddr)); + asm("ldr r2, [r0, #%a0]" : : "i" _FOFF(TSubScheduler,iSSX.iLocalTimerAddr)); asm("cmp r3, #0 "); asm("ble 0f "); // thread isn't timesliced or timeslice already expired so skip asm("cmp r12, #0 "); asm("bne 0f "); // initial (i.e. idle) thread, so skip asm("ldr r3, [r2, #%a0]" : : "i" _FOFF(ArmLocalTimer,iTimerCount)); - asm("ldr r12, [r0, #%a0]" : : "i" _FOFF(TSubScheduler,i_TimerMultI)); + asm("ldr r12, [r0, #%a0]" : : "i" _FOFF(TSubScheduler,iSSX.iTimerPeriodM)); + asm("ldr r2, [r0, #%a0]" : : "i" _FOFF(TSubScheduler,iSSX.iTimerPeriodS)); asm("cmp r3, #0 "); asm("movmi r0, #0 "); // if timer count is negative, save zero asm("bmi 1f "); - asm("umull r0, r3, r12, r3 "); // scale up to max timer clock - asm("adds r0, r0, #0x00800000 "); - asm("adcs r3, r3, #0 "); - asm("mov r0, r0, lsr #24 "); - asm("orr r0, r0, r3, lsl #8 "); + asm("umull r0, r3, r12, r3 "); // scale up to max timer clock (R3:R0) + asm("rsb r12, r2, #32 "); + asm("movs r0, r0, lsr r2 "); // r0 >>= iSSX.iTimerPeriodS, C = last bit shifted off (rounding) + asm("orr r0, r0, r3, lsl r12 "); // bottom bits from r12 into top bits of r0 + asm("adcs r0, r0, #0 "); // round using last bit shifted off asm("1: "); asm("str r0, [r1, #%a0]" : : "i" _FOFF(NThreadBase,iTime)); asm("0: "); __JUMP(,lr); } + +#if defined(__UTT_MACHINE_CODED__) +#if defined(__NKERN_TIMESTAMP_USE_LOCAL_TIMER__) +#error Use of local timer for NKern::Timestamp() no longer supported +#else + /* Update aOld's execution time and set up the timer for aNew Update this CPU's timestamp value if (!aOld) aOld=iInitialThread if (!aNew) aNew=iInitialThread - newcount = aNew->iTime>0 ? Max(aNew->iTime*i_TimerMultF/2^32, 1) : 2^31-1 + newcount = aNew->iTime>0 ? Max(aNew->iTime*iSSX.iTimerFreqM/2^(32+iTimerFreqS), 1) : 2^31-1 cli() oldcount = timer count if (oldcount<=0 || aOld!=aNew) { timer count = newcount - elapsed = i_LastTimerSet - oldcount - i_LastTimerSet = newcount - elapsed = elapsed * i_TimerMultI / 2^24 - aOld->iTotalCpuTime64 += elapsed - correction = i_TimestampError; - if (correction > i_MaxCorrection) - correction = i_MaxCorrection - else if (correction < -i_MaxCorrection) - correction = -i_MaxCorrection - i_TimestampError -= correction - i_LastTimestamp += elapsed + i_TimerGap - correction + iSSX.iLastTimerSet = newcount + if (aOld!=aNew) + { + TUint64 now = NKern::Timestamp(); + elapsed = iLastTimestamp -= now; + iLastTimestamp = now; + aOld->iTotalCpuTime.i64 += elapsed; + if (!aOld->iActiveState) + aOld->iTotalActiveTime.i64 += (now - aOld->iLastActivationTime.i64); + ++iReschedCount.i64; + ++aNew->iRunCount.i64; + } } sti() */ @@ -988,77 +1001,73 @@ { asm("cmp r2, #0 "); asm("ldreq r2, [r0, #%a0]" : : "i" _FOFF(TSubScheduler,iInitialThread)); - asm("ldr r12, [r0, #%a0]" : : "i" _FOFF(TSubScheduler,i_TimerMultF)); + asm("ldr r12, [r0, #%a0]" : : "i" _FOFF(TSubScheduler,iSSX.iTimerFreqM)); asm("cmp r1, #0 "); asm("ldreq r1, [r0, #%a0]" : : "i" _FOFF(TSubScheduler,iInitialThread)); asm("ldr r3, [r2, #%a0]" : : "i" _FOFF(NThreadBase,iTime)); asm("stmfd sp!, {r4-r7} "); - asm("ldr r6, [r2, #%a0]" : : "i" _FOFF(NThreadBase,iRunCount64)); - asm("ldr r7, [r2, #%a0]" : : "i" (_FOFF(NThreadBase,iRunCount64)+4)); asm("cmp r1, r2 "); asm("beq 2f "); + asm("ldr r6, [r2, #%a0]" : : "i" _FOFF(NThreadBase,iRunCount.i32[0])); + asm("ldr r7, [r2, #%a0]" : : "i" _FOFF(NThreadBase,iRunCount.i32[1])); asm("adds r6, r6, #1 "); - asm("str r6, [r2, #%a0]" : : "i" _FOFF(NThreadBase,iRunCount64)); - asm("ldr r4, [r0, #%a0]" : : "i" _FOFF(TSubScheduler,iReschedCount64)); - asm("ldr r6, [r0, #%a0]" : : "i" (_FOFF(TSubScheduler,iReschedCount64)+4)); + asm("str r6, [r2, #%a0]" : : "i" _FOFF(NThreadBase,iRunCount.i32[0])); + asm("ldr r4, [r0, #%a0]" : : "i" _FOFF(TSubScheduler,iReschedCount.i32[0])); + asm("ldr r6, [r0, #%a0]" : : "i" _FOFF(TSubScheduler,iReschedCount.i32[1])); asm("adcs r7, r7, #0 "); - asm("str r7, [r2, #%a0]" : : "i" (_FOFF(NThreadBase,iRunCount64)+4)); + asm("str r7, [r2, #%a0]" : : "i" _FOFF(NThreadBase,iRunCount.i32[1])); asm("adds r4, r4, #1 "); asm("adcs r6, r6, #0 "); - asm("str r4, [r0, #%a0]" : : "i" _FOFF(TSubScheduler,iReschedCount64)); - asm("str r6, [r0, #%a0]" : : "i" (_FOFF(TSubScheduler,iReschedCount64)+4)); + asm("str r4, [r0, #%a0]" : : "i" _FOFF(TSubScheduler,iReschedCount.i32[0])); + asm("str r6, [r0, #%a0]" : : "i" _FOFF(TSubScheduler,iReschedCount.i32[1])); asm("2: "); + asm("ldr r6, [r0, #%a0]" : : "i" _FOFF(TSubScheduler,iSSX.iLocalTimerAddr)); asm("cmp r3, #1 "); // aNew->iTime > 0 ? - asm("umullge r4, r3, r12, r3 "); - asm("ldr r5, [r0, #%a0]" : : "i" _FOFF(TSubScheduler,i_LocalTimerAddr)); - asm("movlt r3, #0x7fffffff "); - asm("addges r3, r3, r4, lsr #31 "); // round up top 32 bits if bit 31 set - asm("moveq r3, #1 "); // if result zero, limit to 1 - asm("ldr r12, [r0, #%a0]" : : "i" _FOFF(TSubScheduler,i_LastTimerSet)); - __ASM_CLI(); - asm("ldr r4, [r5, #%a0]" : : "i" _FOFF(ArmLocalTimer,iTimerCount)); - asm("cmp r1, r2 "); - asm("bne 1f "); - asm("cmp r4, #0 "); - asm("bgt 0f "); // same thread, timeslice not expired -> leave timer alone - asm("1: "); - asm("str r3, [r5, #%a0]" : : "i" _FOFF(ArmLocalTimer,iTimerCount)); // set new timeslice value in timer - asm("ldr r5, [r0, #%a0]" : : "i" _FOFF(TSubScheduler,i_TimerMultI)); - asm("str r3, [r0, #%a0]" : : "i" _FOFF(TSubScheduler,i_LastTimerSet)); - asm("sub r12, r12, r4 "); // r12 = elapsed (actual timer ticks) - asm("umull r4, r5, r12, r5 "); - asm("ldr r3, [r1, #%a0]!" : : "i" _FOFF(NThreadBase,iTotalCpuTime64)); + asm("movlt r3, #0x7fffffff "); // if not, use 2^31-1 + asm("blt 3f "); + asm("cmp r1, r2 "); // different thread? + asm("beq 0f "); // no - finish + asm("ldr r5, [r0, #%a0]" : : "i" _FOFF(TSubScheduler,iSSX.iTimerFreqS)); + asm("umull r4, r3, r12, r3 "); // r3:r4 = aNew->iTime * iTimerFreqM + asm("adds r4, r4, r4 "); // bit 31 into C + asm("teq r5, #0 "); // check for iTimerFreqS=0 without changing C + asm("movnes r3, r3, lsr r5 "); // if not, r3>>=iTimerFreqS, last bit shifted out into C + asm("adcs r3, r3, #0 "); // round using last bit shifted off + asm("3: "); + asm("str r3, [r0, #%a0]" : : "i" _FOFF(TSubScheduler,iSSX.iLastTimerSet)); + asm("str r3, [r6, #%a0]" : : "i" _FOFF(ArmLocalTimer,iTimerCount)); // set new timeslice value in timer + + asm("ldr r6, [r0, #%a0]" : : "i" _FOFF(TSubScheduler,iLastTimestamp.i32[0])); + asm("ldr r7, [r0, #%a0]" : : "i" _FOFF(TSubScheduler,iLastTimestamp.i32[1])); + asm("stmfd sp!, {r0-r2,lr} "); + asm("bl Timestamp__5NKern "); // R1:R0 = current time + asm("mov r4, r0 "); + asm("mov r5, r1 "); // R5:R4 = current time + asm("ldmfd sp!, {r0-r2,lr} "); + asm("str r4, [r0, #%a0]" : : "i" _FOFF(TSubScheduler,iLastTimestamp.i32[0])); + asm("ldr r3, [r1, #%a0]!" : : "i" _FOFF(NThreadBase,iTotalCpuTime.i64)); asm("ldr r12, [r1, #4] "); - asm("adds r4, r4, #0x00800000 "); - asm("adcs r5, r5, #0 "); - asm("mov r4, r4, lsr #24 "); - asm("orr r4, r4, r5, lsl #8 "); // r4 = elapsed - asm("adds r3, r3, r4 "); - asm("adcs r12, r12, #0 "); - asm("stmia r1, {r3,r12} "); // aOld->iTotalCpuTime64 += elapsed - asm("ldr r3, [r0, #%a0]" : : "i" _FOFF(TSubScheduler,i_TimestampError)); - asm("ldr r5, [r0, #%a0]" : : "i" _FOFF(TSubScheduler,i_MaxCorrection)); - asm("ldr r1, [r0, #%a0]" : : "i" _FOFF(TSubScheduler,iLastTimestamp64)); - asm("ldr r2, [r0, #%a0]" : : "i" (_FOFF(TSubScheduler,iLastTimestamp64)+4)); - asm("mov r12, r3 "); - asm("cmp r3, r5 "); - asm("movgt r3, r5 "); // if (correction>i_MaxCorrection) correction=i_MaxCorrection - asm("cmn r3, r5 "); - asm("rsblt r3, r5, #0 "); // if (correction+i_MaxCorrection<0) correction=-i_MaxCorrection - asm("ldr r5, [r0, #%a0]" : : "i" _FOFF(TSubScheduler,i_TimerGap)); - asm("sub r12, r12, r3 "); - asm("str r12, [r0, #%a0]" : : "i" _FOFF(TSubScheduler,i_TimestampError)); - asm("add r4, r4, r5 "); // r4 = elapsed + i_TimerGap - asm("adds r1, r1, r4 "); - asm("adcs r2, r2, #0 "); // iLastTimestamp64 + (elapsed + i_TimerGap) - asm("subs r1, r1, r3 "); - asm("sbcs r1, r1, r3, asr #32 "); // iLastTimestamp64 + (elapsed + i_TimerGap - correction) - asm("str r1, [r0, #%a0]" : : "i" _FOFF(TSubScheduler,iLastTimestamp64)); - asm("str r2, [r0, #%a0]" : : "i" (_FOFF(TSubScheduler,iLastTimestamp64)+4)); + asm("str r5, [r0, #%a0]" : : "i" _FOFF(TSubScheduler,iLastTimestamp.i32[1])); + asm("stmdb r1, {r4-r5} "); // aOld->iLastRunTime + asm("ldrb r2, [r1, #%a0]" : : "i" (_FOFF(NSchedulable,iActiveState)-_FOFF(NThreadBase,iTotalCpuTime.i64))); + asm("subs r6, r4, r6 "); + asm("sbcs r7, r5, r7 "); // R7:R6 = time since last reschedule + asm("adds r3, r3, r6 "); + asm("adcs r12, r12, r7 "); // total CPU time of old thread + asm("stmia r1!, {r3,r12} "); // store, r1=&aOld.iLastActivationTime + asm("cmp r2, #0 "); // old thread still active? + asm("bne 0f "); // yes - done + asm("ldmia r1!, {r2,r3,r6,r7} "); // R3:R2 = last activation time, R7:R6=total active time + asm("subs r2, r4, r2 "); + asm("sbcs r3, r5, r3 "); // R3:R2 = time since last activation + asm("adds r6, r6, r2 "); + asm("adcs r7, r7, r3 "); // R7:R6 = new total active time + asm("stmdb r1, {r6,r7} "); + asm("0: "); - __ASM_STI(); asm("ldmfd sp!, {r4-r7} "); __JUMP(,lr); } - +#endif +#endif // __UTT_MACHINE_CODED__