diff -r a179b74831c9 -r c1f20ce4abcf kernel/eka/nkernsmp/nkern.cpp --- a/kernel/eka/nkernsmp/nkern.cpp Thu Aug 19 11:14:22 2010 +0300 +++ b/kernel/eka/nkernsmp/nkern.cpp Tue Aug 31 16:34:26 2010 +0300 @@ -98,7 +98,10 @@ pC->iLinkedObj = this; pC->iWaitState.SetUpWait(NThreadBase::EWaitFastMutex, NThreadWaitState::EWtStObstructed, this); pC->iWaitLink.iPriority = pC->iPriority; - iWaitQ.Add(&pC->iWaitLink); + if (waited) + iWaitQ.AddHead(&pC->iWaitLink); // we were next at this priority + else + iWaitQ.Add(&pC->iWaitLink); pC->RelSLock(); if (pH) pH->SetMutexPriority(this); @@ -880,6 +883,9 @@ iCsFunction = ECSDivertPending; iSuspendCount = 0; iSuspended = 0; + + // If thread is killed before first resumption, set iACount=1 + __e32_atomic_tau_ord8(&iACount, 1, 0, 1); if (aS) aS->iReadyListLock.UnlockOnly(); DoReleaseT(KErrDied,0); @@ -894,7 +900,7 @@ TBool NThreadBase::SuspendOrKill(TInt aCount) { __KTRACE_OPT(KNKERN,DEBUGPRINT("%T nSuspendOrKill %d", this, aCount)); - if (aCount==0) + if (aCount==0 || i_NThread_Initial) return FALSE; TBool result = FALSE; TBool concurrent = FALSE; @@ -1047,6 +1053,10 @@ { result = TRUE; iSuspended = 0; + + // On first resumption set iACount=1 + // From then on the thread must be killed before being deleted + __e32_atomic_tau_ord8(&iACount, 1, 0, 1); if (!iPauseCount && !iReady && !iWaitState.iWtC.iWtStFlags) ReadyT(0); } @@ -1212,6 +1222,9 @@ if (xh) pD = (*xh)((NThread*)this); // call exit handler + // if CPU freeze still active, remove it + NKern::EndFreezeCpu(0); + // detach any tied events DetachTiedEvents(); @@ -1283,34 +1296,65 @@ KCpuAny if it should be able to run on any CPU. @return The previous affinity mask. */ -TUint32 NThreadBase::SetCpuAffinity(TUint32 aAffinity) +TUint32 NSchedulable::SetCpuAffinityT(TUint32 aAffinity) { // check aAffinity is valid - AcqSLock(); - TUint32 old_aff = iParent->iCpuAffinity; - TBool migrate = FALSE; + NThreadBase* t = 0; + NThreadGroup* g = 0; + NSchedulable* p = iParent; + if (!p) + g = (NThreadGroup*)this, p=g; + else + t = (NThreadBase*)this; + if (iParent && iParent!=this) + g = (NThreadGroup*)iParent; + TUint32 old_aff = p->iCpuAffinity; TBool make_ready = FALSE; TSubScheduler* ss0 = &SubScheduler(); TSubScheduler* ss = 0; - __KTRACE_OPT(KNKERN,DEBUGPRINT("%T nSetCpu %08x->%08x, F:%d R:%02x PR:%02x",this,iParent->iCpuAffinity,aAffinity,iParent->iFreezeCpu,iReady,iParent->iReady)); - if (i_NThread_Initial) +#ifdef KNKERN + if (iParent) + { + __KTRACE_OPT(KNKERN,DEBUGPRINT("%T nSetCpu %08x->%08x, F:%d R:%02x PR:%02x",this,iParent->iCpuAffinity,aAffinity,iParent->iFreezeCpu,iReady,iParent->iReady)); + } + else + { + __KTRACE_OPT(KNKERN,DEBUGPRINT("%G nSetCpu %08x->%08x, F:%d R:%02x",this,iCpuAffinity,aAffinity,iFreezeCpu,iReady)); + } +#endif + if (t && t->i_NThread_Initial) goto done; // can't change affinity of initial thread - iParent->iCpuAffinity = aAffinity; // set new affinity, might not take effect yet - if (!iParent->iReady) + if (aAffinity == NTHREADBASE_CPU_AFFINITY_MASK) + { + p->iTransientCpu = 0; + } + else if ( (aAffinity & (KCpuAffinityPref|NTHREADBASE_CPU_AFFINITY_MASK)) == KCpuAffinityPref) + { + p->iTransientCpu = 0; + p->iPreferredCpu = TUint8((aAffinity & (EReadyCpuMask|EReadyCpuSticky)) | EReadyOffset); + } + else if ( (aAffinity & (KCpuAffinityTransient|KCpuAffinityPref|NTHREADBASE_CPU_AFFINITY_MASK)) == KCpuAffinityTransient) + { + p->iTransientCpu = TUint8(aAffinity & EReadyCpuMask) | EReadyOffset; + } + else + p->iCpuAffinity = NSchedulable::PreprocessCpuAffinity(aAffinity); // set new affinity, might not take effect yet + if (!p->iReady) goto done; // thread/group not currently on a ready list so can just change affinity - migrate = !CheckCpuAgainstAffinity(iParent->iReady & EReadyCpuMask, aAffinity); // TRUE if thread's current CPU is incompatible with the new affinity - if (!migrate) + + // Check if the thread needs to migrate or can stay where it is + if (!p->ShouldMigrate(p->iReady & EReadyCpuMask)) goto done; // don't need to move thread, so just change affinity - ss = TheSubSchedulers + (iParent->iReady & EReadyCpuMask); + ss = TheSubSchedulers + (p->iReady & EReadyCpuMask); ss->iReadyListLock.LockOnly(); - if (iParent->iCurrent) + if (p->iCurrent) { - iParent->iCpuChange = TRUE; // mark CPU change pending + p->iCpuChange = TRUE; // mark CPU change pending if (ss == ss0) RescheduleNeeded(); else // kick other CPU now so migration happens before acquisition of fast mutex - send_resched_ipi_and_wait(iParent->iReady & EReadyCpuMask); + send_resched_ipi_and_wait(p->iReady & EReadyCpuMask); } else { @@ -1318,23 +1362,81 @@ // This is handled by the scheduler - when a thread belonging to a group is context switched // out while holding a fast mutex its iFastMutexDefer is set to 1 and the group's iFreezeCpu // is incremented. - if (iParent->iFreezeCpu || (iParent==this && CheckFastMutexDefer())) - iParent->iCpuChange = TRUE; // CPU frozen or fast mutex held so just mark deferred CPU migration + if (p->iFreezeCpu || (iParent==this && t->CheckFastMutexDefer())) + p->iCpuChange = TRUE; // CPU frozen or fast mutex held so just mark deferred CPU migration else { - ss->Remove(iParent); - iParent->iReady = 0; + ss->SSRemoveEntry(p); + p->iReady = 0; make_ready = TRUE; } } ss->iReadyListLock.UnlockOnly(); if (make_ready) - iParent->ReadyT(0); + p->ReadyT(0); done: - RelSLock(); return old_aff; } +/** Force the current thread onto a particular CPU + + @pre Kernel must not be locked. + @pre Call in a thread context. + @pre Current thread must not be in a group + @pre Current thread must not hold a fast mutex + @pre Current thread must have an active CPU freeze + @pre Current thread must not be an initial thread + + @param The number of the CPU to which this thread should be moved +*/ +void NKern::JumpTo(TInt aCpu) + { + // check aAffinity is valid + NThreadBase* t = NKern::CurrentThread(); + __KTRACE_OPT(KNKERN,DEBUGPRINT("%T NJumpTo %d", t, aCpu)); + if (NKern::HeldFastMutex()) + __crash(); + t->LAcqSLock(); + if (t->iParent!=t) + __crash(); + if (!t->iFreezeCpu) + __crash(); + if (t->i_NThread_Initial) + __crash(); + if (TUint(aCpu) >= (TUint)NKern::NumberOfCpus()) + __crash(); + TUint8 fc = (TUint8)(aCpu | NSchedulable::EReadyOffset); + if (t->iCurrent != fc) + { + t->iForcedCpu = fc; + t->iCpuChange = TRUE; + RescheduleNeeded(); + } + t->RelSLockU(); // reschedules and jumps to new CPU + } + +TBool NSchedulable::ShouldMigrate(TInt aCpu) + { + // Check if the thread's current CPU is compatible with the new affinity + TUint32 active = TheScheduler.iThreadAcceptCpus; + + // If it can't stay where it is, migrate + if (!CheckCpuAgainstAffinity(aCpu, iCpuAffinity, active)) + return TRUE; + + TInt cpu = iTransientCpu ? iTransientCpu : iPreferredCpu; + + // No preferred or transient CPU, so can stay where it is + if (!cpu) + return FALSE; + + // If thread isn't on preferred CPU but could be, migrate + cpu &= EReadyCpuMask; + if (cpu!=aCpu && CheckCpuAgainstAffinity(cpu, iCpuAffinity, active)) + return TRUE; + return FALSE; + } + /****************************************************************************** * Thread wait state @@ -1412,7 +1514,7 @@ { NThreadBase* t = Thread(); if (!t->iPauseCount && !t->iSuspended) - t->ReadyT(0); + t->ReadyT(oldws64 & EWtStObstructed); } return KErrNone; } @@ -1756,6 +1858,24 @@ } +/** Changes the nominal priority of a thread. + + This function is intended to be used by the EPOC layer and personality layers. + Do not use this function directly on a Symbian OS thread - use Kern::ThreadSetPriority(). + + @param aThread Thread to receive the new priority. + @param aPriority New inherited priority for aThread. + + @see Kern::SetThreadPriority() +*/ +void NKern::ThreadSetNominalPriority(NThread* aThread, TInt aPriority) + { + NKern::Lock(); + aThread->SetNominalPriority(aPriority); + NKern::Unlock(); + } + + /** Atomically signals the request semaphore of a nanothread and a fast mutex. This function is intended to be used by the EPOC layer and personality @@ -1947,6 +2067,7 @@ return 1; } pC->iFreezeCpu = 1; + __e32_atomic_add_rlx32(&ss.iDeferShutdown, 1); if (pC->iParent != pC) { pC->AcqSLock(); @@ -1986,6 +2107,7 @@ } else if (pC->iCpuChange) // deferred CPU change? RescheduleNeeded(); + __e32_atomic_add_rlx32(&ss.iDeferShutdown, TUint32(-1)); } NKern::Unlock(); } @@ -2000,9 +2122,9 @@ */ EXPORT_C TUint32 NKern::ThreadSetCpuAffinity(NThread* aThread, TUint32 aAffinity) { - NKern::Lock(); - TUint32 r = aThread->SetCpuAffinity(aAffinity); - NKern::Unlock(); + aThread->LAcqSLock(); + TUint32 r = aThread->SetCpuAffinityT(aAffinity); + aThread->RelSLockU(); return r; } @@ -2327,7 +2449,15 @@ pC->UnReadyT(); pC->iParent = pC; g->iCurrent = 0; // since current thread is no longer in g - ss.AddHead(pC); + TUint64 now = NKern::Timestamp(); + g->iLastRunTime.i64 = now; + g->iTotalCpuTime.i64 += (now - g->iLastStartTime.i64); + if (--g->iActiveState == 0) + { + // group no longer active + g->iTotalActiveTime.i64 += (now - g->iLastActivationTime.i64); + } + ss.SSAddEntryHead(pC); pC->iReady = TUint8(ss.iCpuNum | NSchedulable::EReadyOffset); pC->iCpuAffinity = g->iCpuAffinity; // keep same CPU affinity // if we're frozen, the group's freeze count was incremented @@ -2351,7 +2481,7 @@ g->iCpuChange = FALSE; if (g->iReady) { - ss.Remove(g); + ss.SSRemoveEntry(g); g->iReady = 0; make_group_ready = TRUE; } @@ -2393,7 +2523,9 @@ __KTRACE_OPT(KNKERN,DEBUGPRINT("NJoinGroup %T->%G",pC,aGroup)); pC->AcqSLock(); aGroup->AcqSLock(); - TBool migrate = !CheckCpuAgainstAffinity(ss.iCpuNum, aGroup->iCpuAffinity); // TRUE if thread's current CPU is incompatible with the group's affinity + + // Check if current CPU is compatible with group's affinity + TBool migrate = !CheckCpuAgainstAffinity(ss.iCpuNum, aGroup->iCpuAffinity); if (!aGroup->iReady || aGroup->iReady==pC->iReady) { // group not ready or ready on this CPU @@ -2406,17 +2538,19 @@ if (!aGroup->iReady) { aGroup->iPriority = pC->iPriority; - ss.AddHead(aGroup); + ss.SSAddEntryHead(aGroup); aGroup->iReady = TUint8(ss.iCpuNum | NSchedulable::EReadyOffset); } else if (pC->iPriority > aGroup->iPriority) - { - ss.ChangePriority(aGroup, pC->iPriority); - } + ss.SSChgEntryP(aGroup, pC->iPriority); pC->iReady = NSchedulable::EReadyGroup; aGroup->iCurrent = aGroup->iReady; ss.iReadyListLock.UnlockOnly(); ++aGroup->iThreadCount; + TUint64 now = NKern::Timestamp(); + aGroup->iLastStartTime.i64 = now; + if (++aGroup->iActiveState == 1) + aGroup->iLastActivationTime.i64 = now; goto done; } } @@ -2444,6 +2578,45 @@ /****************************************************************************** + * Iterable Doubly Linked List + ******************************************************************************/ +TInt SIterDQIterator::Step(SIterDQLink*& aObj, TInt aMaxSteps) + { + if (aMaxSteps <= 0) + aMaxSteps = KMaxCpus + 3; + SIterDQLink* p = Next(); + SIterDQLink* q = p; + __NK_ASSERT_DEBUG(p!=0); + for(; p->IsIterator() && --aMaxSteps>0; p=p->Next()) + {} + if (p->IsObject()) + { + // found object + Deque(); + InsertAfter(p); + aObj = p; + return KErrNone; + } + if (p->IsAnchor()) + { + // reached end of list + if (p != q) + { + Deque(); + InsertBefore(p); // put at the end + } + aObj = 0; + return KErrEof; + } + // Maximum allowed number of other iterators skipped + Deque(); + InsertAfter(p); + aObj = 0; + return KErrGeneral; + } + + +/****************************************************************************** * Priority Lists ******************************************************************************/ @@ -2454,8 +2627,6 @@ */ EXPORT_C TInt TPriListBase::HighestPriority() { -// TUint64 present = MAKE_TUINT64(iPresent[1], iPresent[0]); -// return __e32_find_ms1_64(present); return __e32_find_ms1_64(iPresent64); } @@ -2564,28 +2735,22 @@ * Generic IPIs ******************************************************************************/ -TGenIPIList::TGenIPIList() - : iGenIPILock(TSpinLock::EOrderGenericIPIList) - { - } - -TGenIPIList GenIPIList; - extern "C" { extern void send_generic_ipis(TUint32); void generic_ipi_isr(TSubScheduler* aS) { + TScheduler& s = TheScheduler; TGenericIPI* ipi = aS->iNextIPI; if (!ipi) return; TUint32 m = aS->iCpuMask; - SDblQueLink* anchor = &GenIPIList.iA; + SDblQueLink* anchor = &s.iGenIPIList.iA; while (ipi != anchor) { __e32_atomic_and_acq32(&ipi->iCpusIn, ~m); (*ipi->iFunc)(ipi); - TInt irq = GenIPIList.iGenIPILock.LockIrqSave(); + TInt irq = s.iGenIPILock.LockIrqSave(); TGenericIPI* n = (TGenericIPI*)ipi->iNext; ipi->iCpusOut &= ~m; if (ipi->iCpusOut == 0) @@ -2599,7 +2764,7 @@ ipi = (TGenericIPI*)ipi->iNext; if (ipi == anchor) aS->iNextIPI = 0; - GenIPIList.iGenIPILock.UnlockIrqRestore(irq); + s.iGenIPILock.UnlockIrqRestore(irq); } } } @@ -2611,13 +2776,13 @@ TScheduler& s = TheScheduler; TInt i; TUint32 ipis = 0; - TInt irq = GenIPIList.iGenIPILock.LockIrqSave(); + TInt irq = s.iGenIPILock.LockIrqSave(); if (aCpuMask & 0x80000000u) { if (aCpuMask==0xffffffffu) - aCpuMask = s.iActiveCpus2; + aCpuMask = s.iIpiAcceptCpus; else if (aCpuMask==0xfffffffeu) - aCpuMask = s.iActiveCpus2 &~ SubScheduler().iCpuMask; + aCpuMask = s.iIpiAcceptCpus &~ SubScheduler().iCpuMask; else aCpuMask = 0; } @@ -2625,11 +2790,11 @@ iCpusOut = aCpuMask; if (!aCpuMask) { - GenIPIList.iGenIPILock.UnlockIrqRestore(irq); + s.iGenIPILock.UnlockIrqRestore(irq); iNext = 0; return; } - GenIPIList.Add(this); + s.iGenIPIList.Add(this); for (i=0; iiFlag) + while (!__e32_atomic_load_acq32(&s->iFlag)) { __chill(); } + __e32_io_completion_barrier(); } +/****************************************************************************** + * TCoreCycler - general method to execute something on all active cores + ******************************************************************************/ +TCoreCycler::TCoreCycler() + { + iCores = 0; + iG = 0; + } + +void TCoreCycler::Init() + { + CHECK_PRECONDITIONS(MASK_THREAD_STANDARD,"TCoreCycler::Init()"); + TScheduler& s = TheScheduler; + NKern::ThreadEnterCS(); + iG = NKern::LeaveGroup(); + NThread* t = NKern::CurrentThread(); + if (t->iCoreCycling) + { + __crash(); + } + t->iCoreCycling = TRUE; + + // Stop any cores powering up or down for now + // A core already on the way down will stop just before the transition to SHUTDOWN_FINAL + // A core already on the way up will carry on powering up + TInt irq = s.iGenIPILock.LockIrqSave(); + ++s.iCCDeferCount; // stops bits in iIpiAcceptCpus being cleared, but doesn't stop them being set + // but iIpiAcceptCpus | s.iCpusComingUp is constant + TUint32 act2 = s.iIpiAcceptCpus; // CPUs still accepting IPIs + TUint32 cu = s.iCpusComingUp; // CPUs powering up + TUint32 gd = s.iCpusGoingDown; // CPUs no longer accepting IPIs on the way down + s.iGenIPILock.UnlockIrqRestore(irq); + if (gd) + { + // wait for CPUs going down to reach INACTIVE state + TUint32 remain = gd; + FOREVER + { + TInt i; + for (i=0; iDetached()) + remain &= ~(1u<iCoreCycling = FALSE; + if (iG) + NKern::JoinGroup(iG); + NKern::ThreadLeaveCS(); + return KErrEof; + }