diff -r 000000000000 -r a41df078684a kernel/eka/nkernsmp/dfcs.cpp --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/kernel/eka/nkernsmp/dfcs.cpp Mon Oct 19 15:55:17 2009 +0100 @@ -0,0 +1,930 @@ +// Copyright (c) 2006-2009 Nokia Corporation and/or its subsidiary(-ies). +// All rights reserved. +// This component and the accompanying materials are made available +// under the terms of the License "Eclipse Public License v1.0" +// which accompanies this distribution, and is available +// at the URL "http://www.eclipse.org/legal/epl-v10.html". +// +// Initial Contributors: +// Nokia Corporation - initial contribution. +// +// Contributors: +// +// Description: +// e32\nkernsmp\dfcs.cpp +// DFCs +// +// + +// NThreadBase member data +#define __INCLUDE_NTHREADBASE_DEFINES__ + +// TDfc member data +#define __INCLUDE_TDFC_DEFINES__ + +#include "nk_priv.h" + +extern "C" void send_self_resched_ipi(); + +/** Construct an IDFC + + @param aFunction = function to call + @param aPtr = parameter to be passed to function + */ +EXPORT_C TDfc::TDfc(TDfcFn aFunction, TAny* aPtr) + { + iPtr = aPtr; + iFn = aFunction; + iTied = 0; + iHType = EEventHandlerIDFC; + i8888.iHState0 = 0; + i8888.iHState1 = 0; + i8888.iHState2 = 0; + iTiedLink.iNext = 0; + } + + +/** Construct an IDFC tied to a thread or group + + @param aTied = pointer to thread or group to which IDFC should be tied + @param aFunction = function to call + @param aPtr = parameter to be passed to function + + @pre Call in thread context, interrupts enabled + */ +EXPORT_C TDfc::TDfc(NSchedulable* aTied, TDfcFn aFunction, TAny* aPtr) + { + iPtr = aPtr; + iFn = aFunction; + iTied = 0; + iHType = EEventHandlerIDFC; + i8888.iHState0 = 0; + i8888.iHState1 = 0; + i8888.iHState2 = 0; + iTiedLink.iNext = 0; + if (aTied) + { + SetTied(aTied); + } + } + + +/** Construct a DFC without specifying a DFC queue. + The DFC queue must be set before the DFC may be queued. + + @param aFunction = function to call + @param aPtr = parameter to be passed to function + @param aPriority = priority of DFC within the queue (0 to 7, where 7 is highest) + */ +EXPORT_C TDfc::TDfc(TDfcFn aFunction, TAny* aPtr, TInt aPriority) + { + __NK_ASSERT_DEBUG((TUint)aPriority<(TUint)KNumDfcPriorities); + iPtr = aPtr; + iFn = aFunction; + iTied = 0; + iHType = TUint8(aPriority); + i8888.iHState0 = 0; + i8888.iHState1 = 0; + i8888.iHState2 = 0; + iTiedLink.iNext = 0; + } + + +/** Construct a DFC specifying a DFC queue. + + @param aFunction = function to call + @param aPtr = parameter to be passed to function + @param aDfcQ = pointer to DFC queue which this DFC should use + @param aPriority = priority of DFC within the queue (0-7) + */ +EXPORT_C TDfc::TDfc(TDfcFn aFunction, TAny* aPtr, TDfcQue* aDfcQ, TInt aPriority) + { + __NK_ASSERT_DEBUG((TUint)aPriority<(TUint)KNumDfcPriorities); + iPtr = aPtr; + iFn = aFunction; + iDfcQ = aDfcQ; + iHType = TUint8(aPriority); + i8888.iHState0 = 0; + i8888.iHState1 = 0; + i8888.iHState2 = 0; + iTiedLink.iNext = 0; + } + + +/** Tie an IDFC to a thread or group + + @param aTied = pointer to thread or group to which IDFC should be tied + @return KErrNone if successful + @return KErrDied if thread has exited or group has been destroyed. + + @pre Call in thread context, interrupts enabled + @pre Must be IDFC not DFC + @pre IDFC must not be queued or running + @pre IDFC must not already be tied + */ +EXPORT_C TInt TDfc::SetTied(NSchedulable* aTied) + { + __NK_ASSERT_ALWAYS(IsIDFC() && i8816.iHState16==0); + __NK_ASSERT_ALWAYS(aTied && !iTied); + NKern::Lock(); + TInt r = aTied->AddTiedEvent(this); + __NK_ASSERT_ALWAYS(r==KErrNone || r==KErrDied); + NKern::Unlock(); + return r; + } + + +/** Destroy a DFC or IDFC + + @pre Call from thread context with interrupts and preemption enabled + @pre Calling thread holds no fast mutex + @pre Calling thread in critical section + */ +EXPORT_C TDfc::~TDfc() + { + CHECK_PRECONDITIONS(MASK_THREAD_CRITICAL,"TDfc::~TDfc"); + NKern::Lock(); + NEventHandler::TiedLock.LockOnly(); + NSchedulable* tied = iTied; + if (IsDFC() || (IsIDFC() && !tied)) + { + Cancel(); + iHType = (TUint8)EEventHandlerDummy; + } + if (IsIDFC()) + { + __NK_ASSERT_ALWAYS(tied!=0); + tied->AcqSLock(); + if (iTiedLink.iNext) + { + iTiedLink.Deque(); + iTiedLink.iNext = 0; + } + tied->RelSLock(); + Cancel(); + iHType = (TUint8)EEventHandlerDummy; + iTied = 0; + } + NEventHandler::TiedLock.UnlockOnly(); + NKern::Unlock(); + } + + +/** Construct a DFC queue + Kern::DfcQInit() should be called on the new DFC queue before it can be used. + */ +EXPORT_C TDfcQue::TDfcQue() + : iThread(NULL) + {} + + + +/** Queue an IDFC or a DFC from an ISR + + This function is the only way to queue an IDFC and is the only way to queue + a DFC from an ISR. To queue a DFC from an IDFC or a thread either Enque() + or DoEnque() should be used. + + This function does nothing if the IDFC/DFC is already queued. + + @pre Call only from ISR, IDFC or thread with preemption disabled. + @pre Do not call from thread with preemption enabled. + @return TRUE if DFC was actually queued by this call + FALSE if DFC was already queued on entry so this call did nothing + + @see TDfc::DoEnque() + @see TDfc::Enque() + */ +EXPORT_C TBool TDfc::Add() + { + __ASSERT_DEBUG(NKern::CurrentContext()!=NKern::EThread || NKern::KernelLocked(), *(int*)0xdfcadd01=0); + __ASSERT_DEBUG(IsIDFC() || (IsDFC() && iDfcQ), *(int*)0xdfcadd03=0); +// __ASSERT_WITH_MESSAGE_DEBUG( NKern::CurrentContext()!=NKern::EThread || NKern::KernelLocked(),"Do not call from thread with preemption enabled","TDfc::Add"); +// __ASSERT_WITH_MESSAGE_DEBUG( IsIDFC() || (IsDFC() && iDfcQ), "DFC queue not set", "TDfc::Add"); +#ifdef __WINS__ + __NK_ASSERT_ALWAYS(Interrupt.InInterrupt() || NKern::KernelLocked()); +#endif + TInt irq = NKern::DisableAllInterrupts(); + TSubScheduler& ss = SubScheduler(); + TUint32 orig = 0xFF00; + + // Transition the state to 'on normal IDFC queue' + // 0000->008n + // 00Cn->00En + // All other states unchanged + // Return original state + if (IsValid()) // don't add if tied and tied thread/group is being/has been destroyed + orig = AddStateChange(); + if (orig==0) + { + // wasn't already queued + i8888.iHState0 = 0; // BeginTiedEvent() not done + ss.iDfcs.Add(this); + ss.iDfcPendingFlag = 1; +#ifdef _DEBUG + TUint32 st8 = DFC_STATE(this) & 0xFF; + if (st8 != (0x80|ss.iCpuNum)) + __crash(); +#endif + } + NKern::RestoreInterrupts(irq); + return (orig==0 || (orig&0xFFE0)==0x00C0); + } + + +/** Queue an IDFC or a DFC from any context + + This function is identical to TDfc::Add() but no checks are performed for correct usage, + and it contains no instrumentation code. + + @return TRUE if DFC was actually queued by this call + FALSE if DFC was already queued on entry so this call did nothing + + @see TDfc::DoEnque() + @see TDfc::Enque() + @see TDfc::Add() + */ +EXPORT_C TBool TDfc::RawAdd() + { + TInt irq = NKern::DisableAllInterrupts(); + TSubScheduler& ss = SubScheduler(); + TUint32 orig = 0xFF00; + if (IsValid()) // don't add if tied and tied thread/group is being/has been destroyed + orig = AddStateChange(); + if (orig==0) + { + // wasn't already queued + i8888.iHState0 = 0; // BeginTiedEvent() not done + ss.iDfcs.Add(this); + ss.iDfcPendingFlag = 1; + send_self_resched_ipi(); // ensure current CPU runs the DFC +#ifdef _DEBUG + TUint32 st8 = DFC_STATE(this) & 0xFF; + if (st8 != (0x80|ss.iCpuNum)) + __crash(); +#endif + // FIXME: Need to wait to ensure IRQ is active before reenabling interrupts + } + NKern::RestoreInterrupts(irq); + return (orig==0 || (orig&0xFFE0)==0x00C0); + } + + +/** Queue a DFC (not an IDFC) from an IDFC or thread with preemption disabled. + + This function is the preferred way to queue a DFC from an IDFC. It should not + be used to queue an IDFC - use TDfc::Add() for this. + + This function does nothing if the DFC is already queued. + + @pre Call only from IDFC or thread with preemption disabled. + @pre Do not call from ISR or thread with preemption enabled. + @return TRUE if DFC was actually queued by this call + FALSE if DFC was already queued on entry so this call did nothing + + @see TDfc::Add() + @see TDfc::Enque() + */ +EXPORT_C TBool TDfc::DoEnque() + { + __ASSERT_WITH_MESSAGE_DEBUG( (NKern::CurrentContext()==NKern::EIDFC )||( NKern::CurrentContext()==NKern::EThread && NKern::KernelLocked()),"Do not call from ISR or thread with preemption enabled","TDfc::DoEnque"); + __NK_ASSERT_DEBUG(IsDFC()); + __ASSERT_WITH_MESSAGE_DEBUG(iDfcQ, "DFC queue not set", "TDfc::DoEnque"); + + // Check not already queued and then mark queued to prevent ISRs touching this DFC + TDfcQue* q = iDfcQ; + NThreadBase* t = q->iThread; + t->AcqSLock(); // also protects DFC queue + TUint16 expect = 0; + TBool ok = __e32_atomic_cas_acq16(&iDfcState, &expect, 1); + if (ok) + { + // wasn't already queued, now marked as on final queue, which means + // attempts to cancel will block on the thread spin lock + TUint present = q->iPresent[0]; + q->Add((TPriListLink*)this); + if (!present) + t->iWaitState.UnBlockT(NThreadBase::EWaitDfc, q, KErrNone); + } + t->RelSLock(); // also protects DFC queue + return ok; + } + +void TDfcQue::ThreadFunction(TAny* aDfcQ) + { + TDfcQue& q = *(TDfcQue*)aDfcQ; + NThreadBase* t = NKern::CurrentThread(); + FOREVER + { + NKern::Lock(); + t->AcqSLock(); // also protects DFC queue + if (q.IsEmpty()) + { + t->iWaitState.SetUpWait(NThreadBase::EWaitDfc, 0, &q); + RescheduleNeeded(); + t->RelSLock(); // also protects DFC queue + NKern::Unlock(); + } + else + { + TDfc* d = q.First(); + q.Remove((TPriListLink*)d); + TDfcFn f = d->iFn; + TAny* p = d->iPtr; + d->ResetState(); + t->RelSLock(); // also protects DFC queue + NKern::Unlock(); + (*f)(p); + } + } + } + + + +void TCancelIPI::Send(TDfc* aDfc, TInt aCpu) + { + iDfc = aDfc; + Queue(&Isr, 1u<iDfc; + if (d->iNext) + { + // QueueDfcs() hasn't dequeued it yet + // just dequeue it here and reset the state - QueueDfcs() will never see it + // Note that this means we have to release the tied thread/group if necessary + // BeginTiedEvent() has occurred if iHState0 is set and it's actually an IDFC not an NTimer + NSchedulable* tied = (d->iHType==NEventHandler::EEventHandlerIDFC && d->i8888.iHState0) ? d->iTied : 0; + d->Deque(); + d->ResetState(); + if (tied) + tied->EndTiedEvent(); + } + else + { + // QueueDfcs() has already dequeued it + // state transition: + // XXYY->XX00 + // XX00->0000 + // QueueDfcs() will take care of the tied thread/group + d->CancelFinalStateChange(); + } + } + + +/** Cancels an IDFC or DFC. + + This function does nothing if the IDFC or DFC is not queued. + + For any DFC or IDFC the following identity holds: + Number of times Add() is called and returns TRUE + + Number of times DoEnque() is called and returns TRUE + + Number of times Enque() is called and returns TRUE + + Number of times QueueOnIdle() is called and returns TRUE + = Number of times Cancel() is called and returns TRUE + + Number of times the DFC/IDFC function executes + + @pre IDFC or thread context. Do not call from ISRs. + + @pre If the DFC function accesses the DFC object itself, the user must ensure that + Cancel() cannot be called while the DFC function is running. + + @return TRUE if the DFC was actually dequeued by this call - i.e. an + instance of the DFC's execution has been prevented. It + is still possible that a previous execution is still in + progress. + FALSE if the DFC was not queued on entry to the call, or was in + the process of being executed or cancelled. In this case + it is possible that the DFC executes after this call + returns. + + @post However in either case it is safe to delete the DFC object on + return from this call provided only that the DFC function does not + refer to the DFC object itself. + */ +EXPORT_C TBool TDfc::Cancel() + { + enum TAction { EDeque=1, EReset=2, EIdleUnlock=4, ESendIPI=8, EWait=16 }; + + CHECK_PRECONDITIONS(MASK_NOT_ISR|MASK_INTERRUPTS_ENABLED,"TDfc::Cancel"); + if (!iDfcState) + return FALSE; + TUint action = EIdleUnlock; + TBool ret = FALSE; + TInt cpu = -1; + NSchedulable* tied = 0; + TDfcQue* q = 0; + NThreadBase* t = 0; + NKern::Lock(); + TSubScheduler& ss0 = SubScheduler(); + if (IsDFC()) + q = iDfcQ, t = q->iThread, t->AcqSLock(); + TInt irq = NKern::DisableAllInterrupts(); + TheScheduler.iIdleSpinLock.LockOnly(); + + // 0000->0000, XX00->ZZ00, xxYY->zzYY + TUint state = CancelInitialStateChange(); + TUint stt = state >> 5; + if (state & 0xFF00) + { + // someone else cancelling at the same time - just wait for them to finish + action = EWait|EIdleUnlock; + goto end; + } + if (state == 0) // DFC not active + goto end; + + // possible states here are 0001, 002g, 006m, 008m, 00Am, 00Cm, 00Em + ret = (stt!=6); // if running but not pending, Cancel() will not have prevented an execution + if (state == TUint(TheScheduler.iIdleGeneration | 0x20)) + { + // was on idle queue, BeginTiedEvent() isn't called until QueueDfcs() runs + action = EDeque|EReset|EIdleUnlock; + goto end; + } + if (state == 1) + { + // was on final queue, must be DFC not IDFC + q->Remove((TPriListLink*)this); + action = EReset|EIdleUnlock; + goto end; + } + + // possible states here are 002g (spilled), 006m, 008m, 00Am, 00Cm, 00Em + // i.e. either on IDFC queue, ExIDFC queue or running + // For IDFCs, tied thread/group is now in play. + cpu = state & 0x1f; // CPU it's on for states 006m, 008m, 00Am, 00Cm, 00Em + if (stt==3 || stt==6 || stt==7) + { + // It's actually running - must be IDFC. A re-queue may also be pending. + TheScheduler.iIdleSpinLock.UnlockOnly(); + TSubScheduler* ss = TheSubSchedulers + cpu; + TDfc* expect = this; + TBool done = __e32_atomic_cas_acq_ptr(&ss->iCurrentIDFC, &expect, 0); + if (done) + { + // We cleared iCurrentIDFC so QueueDfcs() won't touch this again - we reset the state and finish up + // We must also release the tied thread/group + tied = iTied; + action = EReset; + goto end; + } + // QueueDfcs() got to iCurrentIDFC before we did, so we interlock with it + // and we can leave the EndTiedEvent to it as well + // State transition: + // XXAm->XX00, wait + // XX00->0000, don't wait + TUint32 orig = CancelFinalStateChange() & 0xFF; + __NK_ASSERT_ALWAYS(orig==0 || orig==state); + action = orig ? EWait : 0; + goto end; + } + + // possible states here 002g (propagated), 008m, 00Am so it's either on the endogenous or exogenous IDFC queue + if (stt==5) + { + // it's on the exogenous IDFC queue + TheScheduler.iIdleSpinLock.UnlockOnly(); + TSubScheduler* ss = TheSubSchedulers + cpu; + ss->iExIDfcLock.LockOnly(); + if (iNext) + { + // we got to it before QueueDfcs() on the other CPU so we can finish up here + // QueueDfcs() will never see it again so we must release tied thread/group + Deque(); + tied = iTied; + ss->iExIDfcLock.UnlockOnly(); + action = EReset; + goto end; + } + // QueueDfcs() on other CPU has already dequeued it - we must now interlock with RunIDFCStateChange() + ss->iExIDfcLock.UnlockOnly(); + // State transition: + // XXAm->XX00, wait + // XX00->0000, don't wait + // QueueDfcs() will take care of tied thread/group + TUint32 orig = CancelFinalStateChange() & 0xFF; + __NK_ASSERT_ALWAYS(orig==0 || orig==state); + action = orig ? EWait : 0; + goto end; + } + + // possible states here 002g (propagated idle) or 008m (IDFC or DFC on endogenous DFC queue) + if (stt==1) // propagated idle + cpu = TheScheduler.iIdleSpillCpu; + + // if it's on this CPU's IDFC queue we can just remove it and reset the state here + // otherwise we send a cancel IPI to the CPU it's on + // We are guaranteed to dequeue the DFC before it executes since the + // QueueDfcs() on the target CPU will notice that a cancel is in progress and + // so will not run the DFC even if it dequeues it. + // QueueDfcs() takes care of the tied thread/group if it sees the DFC/IDFC again, otherwise + // we must do it here. + if (TUint(cpu) == ss0.iCpuNum) + { + if (IsIDFC()) + tied = iTied; + action = EDeque|EReset|EIdleUnlock; + } + else + action = EIdleUnlock|ESendIPI|EWait; + +end: + // Common exit point + if (action & EDeque) + Deque(); + if (action & EReset) + { + ResetState(); + } + if (action & EIdleUnlock) + TheScheduler.iIdleSpinLock.UnlockOnly(); + NKern::RestoreInterrupts(irq); + if (t) + t->RelSLock(); + + // on another CPU's IDFC queue so send IPI to remove it + if (action & ESendIPI) + { + TCancelIPI ipi; + ipi.Send(this, cpu); + ipi.WaitCompletion(); + tied = 0; + } + + // wait for cancel to complete + if (action & EWait) + { + TUint n = 0x01000000; + while ((iDfcState>>8) & ss0.iCpuMask) + { + __chill(); + if (!--n) + __crash(); + } + } + + // release tied thread/group if waiting for IDFC to complete + if (tied) + tied->EndTiedEvent(); + NKern::Unlock(); + return ret; + } + + +/** Queues a DFC (not an IDFC) from a thread. + + Does nothing if DFC is already queued. + + NOTE: Although this can be called in an IDFC context, it is more efficient to call + DoEnque() in this case. + + @pre Call either in a thread or an IDFC context. + @pre Do not call from an ISR. + @return TRUE if DFC was actually queued by this call + FALSE if DFC was already queued on entry so this call did nothing + */ +EXPORT_C TBool TDfc::Enque() + { + CHECK_PRECONDITIONS(MASK_NOT_ISR,"TDfc::Enque()"); + NKern::Lock(); + TBool ret = DoEnque(); + NKern::Unlock(); + return ret; + } + + +/** Queue a DFC (not an IDFC) from a thread and also signals a fast mutex. + + The DFC is unaffected if it is already queued. + + The fast mutex is signalled before preemption is reenabled to avoid potential + scheduler thrashing. + + @param aMutex = pointer to fast mutex to be signalled; + NULL means system lock mutex. + @return TRUE if DFC was actually queued by this call + FALSE if DFC was already queued on entry so this call did nothing + @pre Call in a thread context. + @pre Kernel must be unlocked. + @pre Do not call from an ISR. + @pre Do not call from an IDFC. + */ +EXPORT_C TBool TDfc::Enque(NFastMutex* aMutex) + { + CHECK_PRECONDITIONS(MASK_KERNEL_UNLOCKED|MASK_NOT_ISR|MASK_NOT_IDFC,"TDfc::Enque(NFastMutex* aMutex)"); + if (!aMutex) + aMutex=&TheScheduler.iLock; + NKern::Lock(); + TBool ret = DoEnque(); + aMutex->Signal(); + NKern::Unlock(); + return ret; + } + + +/** Returns a pointer to the thread on which a DFC runs + + @return If this is a DFC and the DFC queue has been set, a pointer to the + thread which will run the DFC. + NULL if this is an IDFC or the DFC queue has not been set. + */ +EXPORT_C NThreadBase* TDfc::Thread() + { + if (!IsDFC()) + return 0; + return iDfcQ ? iDfcQ->iThread : 0; + } + + +/****************************************************************************** + * Idle notification + ******************************************************************************/ + +/** Register an IDFC or a DFC to be called when the system goes idle + + This function does nothing if the IDFC/DFC is already queued. + + @return TRUE if DFC was actually queued by this call + FALSE if DFC was already queued on entry so this call did nothing + */ +EXPORT_C TBool TDfc::QueueOnIdle() + { + TInt irq = TheScheduler.iIdleSpinLock.LockIrqSave(); + TUint32 orig = 0xFF00; + + // Transition the state to 'on normal idle queue' + // 0000->002g + // 00Cn->006n + // All other states unchanged + // Return original state + if (IsValid()) // don't add if tied and tied thread/group is being/has been destroyed + orig = QueueOnIdleStateChange(); + if (orig==0) + { + i8888.iHState0 = 0; // BeginTiedEvent() not done + TheScheduler.iIdleDfcs.Add(this); + } + + TheScheduler.iIdleSpinLock.UnlockIrqRestore(irq); + return (orig==0 || (orig&0xFFE0)==0x00C0); + } + + +/****************************************************************************** + * Scheduler IDFC/DFC Processing + ******************************************************************************/ + +void TSubScheduler::QueueDfcs() +// +// Enter with interrupts off and kernel locked +// Leave with interrupts off and kernel locked +// +// In state descriptions: +// XX=8 bits not all zero (bitmask representing cancelling CPUs) +// xx=8 bits (bitmask representing cancelling CPUs) +// YY=8 bits not all zero +// ZZ=XX with an additional bit set corresponding to the current CPU +// zz=xx with an additional bit set corresponding to the current CPU +// n = current CPU number +// m = another CPU number +// g = idle generation number + { + __KTRACE_OPT(KSCHED2,DEBUGPRINT("^")); + iInIDFC = TRUE; + BTrace0(BTrace::ECpuUsage, BTrace::EIDFCStart); + TDfc* d = 0; + NSchedulable* tied = 0; + FOREVER + { + NKern::DisableAllInterrupts(); + // remove from pending queue with interrupts disabled + d = (TDfc*)iDfcs.GetFirst(); + if (d) + { + d->iNext = 0; +#ifdef _DEBUG + TUint32 st8 = DFC_STATE(d) & 0xFF; + if (st8 != TUint(0x80|iCpuNum) && st8 != TUint(0x21^TheScheduler.iIdleGeneration)) + __crash(); +#endif + if (d->IsDFC()) // also true for mutating NTimer + { + NKern::EnableAllInterrupts(); + TDfcQue* q = d->iDfcQ; + NThreadBase* t = q->iThread; + t->AcqSLock(); // also protects DFC queue + + // transition to 'final queue' state + // 002g->0001, ok=TRUE + // 008n->0001, ok=TRUE + // XXYY->XX00, ok=FALSE + // XX00->0000, ok=FALSE + // other starting states invalid + TUint32 orig = d->MoveToFinalQStateChange() >> 5; + if (orig==1 || orig==4) + { + // wasn't being cancelled, now marked as on final queue, which means + // attempts to cancel will block on the thread spin lock + TUint present = q->iPresent[0]; + q->Add((TPriListLink*)d); + if (!present) + t->iWaitState.UnBlockT(NThreadBase::EWaitDfc, q, KErrNone); + } + t->RelSLock(); // also protects DFC queue + continue; + } + // endogenous IDFC - could be tied in which case may need to be punted over to another CPU + // can't be mutating NTimer since that would have gone into IsDFC() path + tied = d->iTied; + if (tied && !d->i8888.iHState0) // if tied and BeginTiedEvent() not already done + { + d->i8888.iHState0 = 1; // flag that BeginTiedEvent() done + TInt cpu = tied->BeginTiedEvent(); + if (TUint(cpu) != iCpuNum) + { + // punt over to other CPU + TBool kick = FALSE; + TSubScheduler* ss = TheSubSchedulers + cpu; + ss->iExIDfcLock.LockOnly(); + // transition state here to handle cancel + // XXYY->XX00, ok=FALSE + // XX00->0000, ok=FALSE + // 008n->00Am, ok=TRUE + // 002g->00Am, ok=TRUE + // other starting states invalid + TUint32 orig = d->TransferIDFCStateChange(cpu) >> 5; + if (orig==1 || orig==4) + { + kick = !ss->iExIDfcPendingFlag; + ss->iExIDfcPendingFlag = TRUE; + ss->iExIDfcs.Add(d); + } + ss->iExIDfcLock.UnlockOnly(); + if (kick) + send_resched_ipi(cpu); + NKern::EnableAllInterrupts(); // let interrupts in + if (orig >= 8) + tied->EndTiedEvent(); // IDFC cancelled so release tied thread/group + continue; + } + } + } + else + { + if (!iExIDfcPendingFlag) + break; + iExIDfcLock.LockOnly(); + d = (TDfc*)iExIDfcs.GetFirst(); + if (!d) + { + iExIDfcPendingFlag = 0; + iExIDfcLock.UnlockOnly(); + break; + } + d->iNext = 0; + tied = d->iTied; + __NK_ASSERT_ALWAYS(d->IsIDFC() && tied); // only tied IDFCs should get here +#ifdef _DEBUG + TUint32 st8 = DFC_STATE(d) & 0xFF; + if (st8 != (0xA0|iCpuNum)) + __crash(); +#endif + iExIDfcLock.UnlockOnly(); + } + + // endogenous or exogenous IDFC + // if tied, we are on correct CPU + TDfcFn f = d->iFn; + TAny* p = d->iPtr; + + // If Cancel() finds the IDFC in the running state (00Cn or 00En) it will do the following + // atomic { if (iCurrentIDFC==d) iCurrentIDFC=0; } + // We must guarantee that the following access is observed before the state change in RunIDFCStateChange() + // We assume the latter has full barrier semantics to guarantee this. + iCurrentIDFC = d; + + // transition to running state + // 002g->00Cn, ok=TRUE + // 008n->00Cn, ok=TRUE + // 00An->00Cn, ok=TRUE + // XXYY->XX00, ok=FALSE + // XX00->0000, ok=FALSE + // other starting states invalid + TUint32 orig = d->RunIDFCStateChange() >> 5; + NKern::EnableAllInterrupts(); + if (orig==1 || orig==4 || orig==5) + { + (*f)(p); + + // transition to idle state or rerun if necessary + // first swap iCurrentIDFC with 0 - if original value != d, don't touch d again, return 0xFFFFFFFF + // 00Cn->0000 + // 00En->008n + // 006n->006n + // XXCn->XX00 + // XXEn->XX00 + // XX6n->XX00 + // other starting states invalid + // return original state + NKern::DisableAllInterrupts(); + TUint32 orig = d->EndIDFCStateChange(this); + if ((orig>>5)==7) + { + iDfcs.Add(d); +#ifdef _DEBUG + TUint32 st8 = DFC_STATE(d) & 0xFF; + if (st8 != (0x80|iCpuNum)) + __crash(); +#endif + continue; + } + else if ((orig>>5)==3) + { + TheScheduler.iIdleSpinLock.LockOnly(); + // 006n->002g + // XX6n->XX00 + orig = d->EndIDFCStateChange2(); + if ((orig>>5)==3) + TheScheduler.iIdleDfcs.Add(d); + TheScheduler.iIdleSpinLock.UnlockOnly(); + } + NKern::EnableAllInterrupts(); + if (tied && orig<0x10000) + tied->EndTiedEvent(); // if we set iCurrentIDFC back to 0, we release the tied thread/group + } + else + { + iCurrentIDFC = 0; + if (tied) + tied->EndTiedEvent(); // IDFC cancelled so release tied thread/group + } + } + iDfcPendingFlag = 0; + BTrace0(BTrace::ECpuUsage, BTrace::EIDFCEnd); + iInIDFC = 0; + __KTRACE_OPT(KSCHED2,DEBUGPRINT("~")); + } + + +/****************************************************************************** + * Kernel-side asynchronous request DFCs + ******************************************************************************/ + +EXPORT_C TAsyncRequest::TAsyncRequest(TDfcFn aFunction, TDfcQue* aDfcQ, TInt aPriority) + : TDfc(aFunction, this, aDfcQ, aPriority), iCompletionObject(0), iCancel(0), iResult(0) + { + } + + +EXPORT_C void TAsyncRequest::Send(TDfc* aCompletionDfc) + { + __NK_ASSERT_DEBUG(!iCompletionObject); + iCancel = EFalse; + iCompletionObject = (TAny*)((TLinAddr)aCompletionDfc|1); + TDfc::Enque(); + } + + +EXPORT_C void TAsyncRequest::Send(NFastSemaphore* aCompletionSemaphore) + { + __NK_ASSERT_DEBUG(!iCompletionObject); + iCancel = EFalse; + iCompletionObject = aCompletionSemaphore; + TDfc::Enque(); + } + + +EXPORT_C TInt TAsyncRequest::SendReceive() + { + NFastSemaphore signal; + NKern::FSSetOwner(&signal, 0); + Send(&signal); + NKern::FSWait(&signal); + return iResult; + } + + +EXPORT_C void TAsyncRequest::Cancel() + { + iCancel = ETrue; + if(TDfc::Cancel()) + Complete(KErrCancel); + } + + +EXPORT_C void TAsyncRequest::Complete(TInt aResult) + { + TLinAddr signal = (TLinAddr)__e32_atomic_swp_ord_ptr(&iCompletionObject, 0); + if(signal) + { + iResult = aResult; + if(signal&1) + ((TDfc*)(signal&~1))->Enque(); + else + NKern::FSSignal((NFastSemaphore*)signal); + } + }