65 EThreadCpuShift=8u, |
88 EThreadCpuShift=8u, |
66 EThreadCpuMask=0x1f00u, |
89 EThreadCpuMask=0x1f00u, |
67 EDeferredReady=0x4000u, |
90 EDeferredReady=0x4000u, |
68 EEventParent=0x8000u, |
91 EEventParent=0x8000u, |
69 }; |
92 }; |
70 public: |
93 |
71 NSchedulable(); |
94 /** |
72 void AcqSLock(); |
95 @internalComponent |
73 void RelSLock(); |
96 */ |
74 void LAcqSLock(); |
97 enum NLbState |
75 void RelSLockU(); |
98 { |
76 void ReadyT(TUint aMode); // make ready, assumes lock held |
99 ELbState_Inactive = 0x00u, // not currently involved in load balancing |
77 TInt BeginTiedEvent(); |
100 ELbState_Global = 0x01u, // flag indicating this is on global load balance list |
78 void EndTiedEvent(); |
101 ELbState_Temp = 0x02u, // flag indicating this is on a temporary load balance list |
79 TInt AddTiedEvent(NEventHandler* aEvent); |
102 ELbState_CpuMask = 0x1Fu, // mask of bits indicating CPU if on per-CPU list |
80 TBool TiedEventReadyInterlock(TInt aCpu); |
103 ELbState_PerCpu = 0x20u, // flag indicating this is on per-CPU load balance list |
81 void UnPauseT(); // decrement pause count and make ready if necessary |
104 ELbState_ExtraRef = 0x40u, // flag indicating extra reference has been taken after thread/group died |
82 static void DeferredReadyIDfcFn(TAny*); |
105 ELbState_Generation = 0x80u, // 1 bit generation number |
83 void DetachTiedEvents(); |
106 }; |
84 public: |
107 |
85 inline TBool IsGroup() {return !iParent;} |
108 /** |
86 inline TBool IsLoneThread() {return iParent==this;} |
109 @internalComponent |
87 inline TBool IsGroupThread() {return iParent && iParent!=this;} |
110 */ |
|
111 enum NCpuStatsSelect |
|
112 { |
|
113 E_RunTime=0x01u, |
|
114 E_RunTimeDelta=0x02u, |
|
115 E_ActiveTime=0x04u, |
|
116 E_ActiveTimeDelta=0x08u, |
|
117 E_LastRunTime=0x10u, |
|
118 E_LastActiveTime=0x20u, |
|
119 |
|
120 E_AllStats = 0x3fu |
|
121 }; |
|
122 |
|
123 /** |
|
124 @internalComponent |
|
125 */ |
|
126 struct SCpuStats |
|
127 { |
|
128 TUint64 iRunTime; // total run time |
|
129 TUint64 iRunTimeDelta; // run time since we last asked |
|
130 TUint64 iActiveTime; // total active time |
|
131 TUint64 iActiveTimeDelta; // active time since we last asked |
|
132 TUint64 iLastRunTime; // how long ago this last ran |
|
133 TUint64 iLastActiveTime; // how long ago this was last active |
|
134 }; |
|
135 public: |
|
136 NSchedulable(); /**< @internalComponent */ |
|
137 void AcqSLock(); /**< @internalComponent */ |
|
138 void RelSLock(); /**< @internalComponent */ |
|
139 void LAcqSLock(); /**< @internalComponent */ |
|
140 void RelSLockU(); /**< @internalComponent */ |
|
141 void ReadyT(TUint aMode); /**< @internalComponent */ // make ready, assumes lock held |
|
142 TInt BeginTiedEvent(); /**< @internalComponent */ |
|
143 void EndTiedEvent(); /**< @internalComponent */ |
|
144 TInt AddTiedEvent(NEventHandler* aEvent); /**< @internalComponent */ |
|
145 TBool TiedEventReadyInterlock(TInt aCpu); /**< @internalComponent */ |
|
146 void UnPauseT(); /**< @internalComponent */ // decrement pause count and make ready if necessary |
|
147 static void DeferredReadyIDfcFn(TAny*); /**< @internalComponent */ |
|
148 void DetachTiedEvents(); /**< @internalComponent */ |
|
149 TBool TakeRef(); /**< @internalComponent */ |
|
150 TBool DropRef(); /**< @internalComponent */ |
|
151 void LbUnlink(); /**< @internalComponent */ |
|
152 void LbTransfer(SDblQue& aDestQ); /**< @internalComponent */ |
|
153 void RemoveFromEnumerateList(); /**< @internalComponent */ |
|
154 void GetCpuStats(TUint aMask, SCpuStats& aOut); /**< @internalComponent */ |
|
155 void GetCpuStatsT(TUint aMask, SCpuStats& aOut); /**< @internalComponent */ |
|
156 void GetLbStats(TUint64 aTime); /**< @internalComponent */ |
|
157 void LbDone(TUint aFlags); /**< @internalComponent */ |
|
158 TUint32 SetCpuAffinityT(TUint32 aAffinity); /**< @internalComponent */ |
|
159 TBool ShouldMigrate(TInt aCpu); /**< @internalComponent */ |
|
160 void InitLbInfo(); /**< @internalComponent */ |
|
161 void NominalPriorityChanged(); /**< @internalComponent */ |
|
162 void AddToEnumerateList(); /**< @internalComponent */ |
|
163 void SetEventCpu(); /**< @internalComponent */ |
|
164 public: |
|
165 static TUint32 PreprocessCpuAffinity(TUint32 aAffinity); /**< @internalComponent */ |
|
166 inline TBool IsGroup() {return !iParent;} /**< @internalComponent */ |
|
167 inline TBool IsLoneThread() {return iParent==this;} /**< @internalComponent */ |
|
168 inline TBool IsGroupThread() {return iParent && iParent!=this;} /**< @internalComponent */ |
88 public: |
169 public: |
89 // TUint8 iReady; /**< @internalComponent */ // flag indicating thread on ready list = cpu number | EReadyOffset |
170 // TUint8 iReady; /**< @internalComponent */ // flag indicating thread on ready list = cpu number | EReadyOffset |
90 // TUint8 iCurrent; /**< @internalComponent */ // flag indicating thread is running |
171 // TUint8 iCurrent; /**< @internalComponent */ // flag indicating thread is running |
91 // TUint8 iLastCpu; /**< @internalComponent */ // CPU on which this thread last ran |
172 // TUint8 iLastCpu; /**< @internalComponent */ // CPU on which this thread last ran |
92 TUint8 iPauseCount; /**< @internalComponent */ // count of externally requested pauses extending a voluntary wait |
173 TUint8 iPauseCount; /**< @internalComponent */ // count of externally requested pauses extending a voluntary wait |
93 TUint8 iSuspended; /**< @internalComponent */ // flag indicating active external suspend (Not used for groups) |
174 TUint8 iSuspended; /**< @internalComponent */ // flag indicating active external suspend (Not used for groups) |
94 TUint8 iNSchedulableSpare1; /**< @internalComponent */ |
175 TUint8 iACount; /**< @internalComponent */ // access count |
95 TUint8 iNSchedulableSpare2; /**< @internalComponent */ |
176 TUint8 iPreferredCpu; /**< @internalComponent */ |
|
177 |
|
178 TInt iActiveState; /**< @internalComponent */ |
|
179 TUint8 i_NSchedulable_Spare2; /**< @internalComponent */ |
|
180 TUint8 iForcedCpu; /**< @internalComponent */ |
|
181 TUint8 iTransientCpu; /**< @internalComponent */ |
|
182 TUint8 iLbState; /**< @internalComponent */ |
96 |
183 |
97 TUint8 iCpuChange; /**< @internalComponent */ // flag showing CPU migration outstanding |
184 TUint8 iCpuChange; /**< @internalComponent */ // flag showing CPU migration outstanding |
98 TUint8 iStopping; /**< @internalComponent */ // thread is exiting, thread group is being destroyed |
185 TUint8 iStopping; /**< @internalComponent */ // thread is exiting, thread group is being destroyed |
99 TUint16 iFreezeCpu; /**< @internalComponent */ // flag set if CPU frozen - count for groups |
186 TUint16 iFreezeCpu; /**< @internalComponent */ // flag set if CPU frozen - count for groups |
100 NSchedulable* iParent; /**< @internalComponent */ // Pointer to group containing thread, =this for normal thread, =0 for group |
187 NSchedulable* iParent; /**< @internalComponent */ // Pointer to group containing thread, =this for normal thread, =0 for group |
107 SDblQue iEvents; /**< @internalComponent */ // doubly-linked list of tied events |
194 SDblQue iEvents; /**< @internalComponent */ // doubly-linked list of tied events |
108 |
195 |
109 TUint32 i_IDfcMem[sizeof(TDfc)/sizeof(TUint32)]; /**< @internalComponent */ // IDFC used to make thread ready after last tied event completes |
196 TUint32 i_IDfcMem[sizeof(TDfc)/sizeof(TUint32)]; /**< @internalComponent */ // IDFC used to make thread ready after last tied event completes |
110 // TDfc iDeferredReadyIDfc; /**< @internalComponent */ // IDFC used to make thread ready after last tied event completes |
197 // TDfc iDeferredReadyIDfc; /**< @internalComponent */ // IDFC used to make thread ready after last tied event completes |
111 |
198 |
112 union |
199 union { |
113 { |
200 TUint64HL iRunCount; /**< @internalComponent */ // number of times this thread has run |
114 TUint64 iRunCount64; |
201 TUint64HL iLastStartTime; /**< @internalComponent */ // last start time for groups |
115 TUint32 iRunCount32[2]; |
202 }; |
116 }; |
203 TUint64HL iLastRunTime; /**< @internalComponent */ // time when this thread last ran |
117 union |
204 TUint64HL iTotalCpuTime; /**< @internalComponent */ // total CPU time used by this thread |
118 { |
205 TUint64HL iLastActivationTime; /**< @internalComponent */ // time when this thread last became active |
119 TUint64 iTotalCpuTime64; /**< @internalComponent */ // total time spent running, in hi-res timer ticks |
206 TUint64HL iTotalActiveTime; /**< @internalComponent */ // total time this thread has been active |
120 TUint32 iTotalCpuTime32[2]; /**< @internalComponent */ // total time spent running, in hi-res timer ticks |
207 TUint64HL iSavedCpuTime; /**< @internalComponent */ // Total CPU time used at last check |
|
208 TUint64HL iSavedActiveTime; /**< @internalComponent */ // Total active time at last check |
|
209 SDblQueLink iLbLink; /**< @internalComponent */ // Link into queue of tasks requiring load balancing |
|
210 SIterDQLink iEnumerateLink; /**< @internalComponent */ |
|
211 |
|
212 enum {EMaxLbInfoSize = 48}; /**< @internalComponent */ |
|
213 union { |
|
214 TUint64 i__Dummy[EMaxLbInfoSize/sizeof(TUint64)]; /**< @internalComponent */ |
|
215 SLbInfo iLbInfo; /**< @internalComponent */ |
121 }; |
216 }; |
122 }; |
217 }; |
123 |
218 |
124 __ASSERT_COMPILE(!(_FOFF(NSchedulable,iSSpinLock)&7)); |
219 __ASSERT_COMPILE(!(_FOFF(NSchedulable,iSSpinLock)&7)); |
125 __ASSERT_COMPILE(!(_FOFF(NSchedulable,iRunCount64)&7)); |
220 __ASSERT_COMPILE(!(_FOFF(NSchedulable,iRunCount)&7)); |
126 __ASSERT_COMPILE(!(_FOFF(NSchedulable,iTotalCpuTime64)&7)); |
221 __ASSERT_COMPILE(!(_FOFF(NSchedulable,iTotalCpuTime)&7)); |
|
222 __ASSERT_COMPILE(!(_FOFF(NSchedulable,iLbInfo)&7)); |
|
223 __ASSERT_COMPILE(sizeof(SLbInfo) <= NSchedulable::EMaxLbInfoSize); |
127 __ASSERT_COMPILE(!(sizeof(NSchedulable)&7)); |
224 __ASSERT_COMPILE(!(sizeof(NSchedulable)&7)); |
128 |
225 |
129 |
226 |
130 /** |
227 /** |
131 @internalComponent |
228 @internalComponent |
259 ETimeoutPostamble=1, |
356 ETimeoutPostamble=1, |
260 ETimeoutSpurious=2, |
357 ETimeoutSpurious=2, |
261 }; |
358 }; |
262 public: |
359 public: |
263 NThreadBase(); |
360 NThreadBase(); |
264 TInt Create(SNThreadCreateInfo& anInfo, TBool aInitial); |
361 TInt Create(SNThreadCreateInfo& anInfo, TBool aInitial); /**< @internalComponent */ |
265 void UnReadyT(); |
362 void UnReadyT(); /**< @internalComponent */ |
266 TBool SuspendOrKill(TInt aCount); |
363 TBool SuspendOrKill(TInt aCount); /**< @internalComponent */ |
267 TBool DoSuspendOrKillT(TInt aCount, TSubScheduler* aS); |
364 TBool DoSuspendOrKillT(TInt aCount, TSubScheduler* aS); /**< @internalComponent */ |
268 TBool CancelTimerT(); |
365 TBool CancelTimerT(); /**< @internalComponent */ |
269 void DoReleaseT(TInt aReturnCode, TUint aMode); |
366 void DoReleaseT(TInt aReturnCode, TUint aMode); /**< @internalComponent */ |
270 TBool CheckFastMutexDefer(); |
367 TBool CheckFastMutexDefer(); /**< @internalComponent */ |
271 void DoCsFunctionT(); |
368 void DoCsFunctionT(); /**< @internalComponent */ |
272 TBool Resume(TBool aForce); |
369 TBool Resume(TBool aForce); /**< @internalComponent */ |
273 IMPORT_C TBool Suspend(TInt aCount); /**< @internalComponent */ |
370 IMPORT_C TBool Suspend(TInt aCount); /**< @internalComponent */ |
274 IMPORT_C TBool Resume(); /**< @internalComponent */ |
371 IMPORT_C TBool Resume(); /**< @internalComponent */ |
275 IMPORT_C TBool ForceResume(); /**< @internalComponent */ |
372 IMPORT_C TBool ForceResume(); /**< @internalComponent */ |
276 IMPORT_C void Release(TInt aReturnCode, TUint aMode); /**< @internalComponent */ |
373 IMPORT_C void Release(TInt aReturnCode, TUint aMode); /**< @internalComponent */ |
277 IMPORT_C void RequestSignal(); /**< @internalComponent */ |
374 IMPORT_C void RequestSignal(); /**< @internalComponent */ |
278 IMPORT_C void SetPriority(TInt aPriority); /**< @internalComponent */ |
375 IMPORT_C void SetPriority(TInt aPriority); /**< @internalComponent */ |
279 void SetMutexPriority(NFastMutex* aMutex); |
376 void SetNominalPriority(TInt aPriority); /**< @internalComponent */ |
280 void LoseInheritedPriorityT(); |
377 void SetMutexPriority(NFastMutex* aMutex); /**< @internalComponent */ |
281 void ChangeReadyThreadPriority(); |
378 void LoseInheritedPriorityT(); /**< @internalComponent */ |
282 TUint32 SetCpuAffinity(TUint32 aAffinity); |
379 void ChangeReadyThreadPriority(); /**< @internalComponent */ |
283 TBool TiedEventLeaveInterlock(); |
380 TBool TiedEventLeaveInterlock(); /**< @internalComponent */ |
284 TBool TiedEventJoinInterlock(); |
381 TBool TiedEventJoinInterlock(); /**< @internalComponent */ |
285 IMPORT_C void Kill(); /**< @internalComponent */ |
382 IMPORT_C void Kill(); /**< @internalComponent */ |
286 void Exit(); |
383 void Exit(); /**< @internalComponent */ |
287 // hooks for platform-specific code |
384 // hooks for platform-specific code |
288 void OnKill(); |
385 void OnKill(); /**< @internalComponent */ |
289 void OnExit(); |
386 void OnExit(); /**< @internalComponent */ |
290 public: |
387 public: |
291 static void TimerExpired(TAny* aPtr); |
388 static void TimerExpired(TAny* aPtr); /**< @internalComponent */ |
292 |
389 |
293 /** @internalComponent */ |
390 /** @internalComponent */ |
294 inline void UnknownState(TInt aOp, TInt aParam) |
391 inline void UnknownState(TInt aOp, TInt aParam) |
295 { (*iHandlers->iStateHandler)((NThread*)this,aOp,aParam); } |
392 { (*iHandlers->iStateHandler)((NThread*)this,aOp,aParam); } |
296 |
393 |
319 { return iWaitState.ThreadIsDead(); } |
416 { return iWaitState.ThreadIsDead(); } |
320 public: |
417 public: |
321 TPriListLink iWaitLink; /**< @internalComponent */ // used to link thread into a wait queue |
418 TPriListLink iWaitLink; /**< @internalComponent */ // used to link thread into a wait queue |
322 // TUint8 iBasePri; /**< @internalComponent */ // priority with no fast mutex held |
419 // TUint8 iBasePri; /**< @internalComponent */ // priority with no fast mutex held |
323 // TUint8 iMutexPri; /**< @internalComponent */ // priority from held fast mutex |
420 // TUint8 iMutexPri; /**< @internalComponent */ // priority from held fast mutex |
324 // TUint8 iInitial; /**< @internalComponent */ // TRUE if this is an initial thread |
421 // TUint8 iNominalPri; /**< @internalComponent */ // nominal priority of thread (excluding effect of higher level inheritance) |
325 TUint8 iLinkedObjType; |
422 TUint8 iLinkedObjType; |
326 TUint8 i_ThrdAttr; /**< @internalComponent */ |
423 TUint8 i_ThrdAttr; /**< @internalComponent */ |
327 TUint8 iNThreadBaseSpare10; |
424 TUint8 iInitial; /**< @internalComponent */ // TRUE if this is an initial thread |
328 TUint8 iFastMutexDefer; /**< @internalComponent */ |
425 TUint8 iFastMutexDefer; /**< @internalComponent */ |
329 |
426 |
330 NFastSemaphore iRequestSemaphore; /**< @internalComponent */ |
427 NFastSemaphore iRequestSemaphore; /**< @internalComponent */ |
331 |
428 |
332 TInt iTime; /**< @internalComponent */ // time remaining, 0 if expired |
429 TInt iTime; /**< @internalComponent */ // time remaining, 0 if expired |
355 TInt iStackSize; /**< @internalComponent */ |
452 TInt iStackSize; /**< @internalComponent */ |
356 |
453 |
357 TAny* iExtraContext; /**< @internalComponent */ // parent FPSCR value (iExtraContextSize == -1), coprocessor context (iExtraContextSize > 0) or NULL |
454 TAny* iExtraContext; /**< @internalComponent */ // parent FPSCR value (iExtraContextSize == -1), coprocessor context (iExtraContextSize > 0) or NULL |
358 TInt iExtraContextSize; /**< @internalComponent */ // +ve=dynamically allocated, 0=none, -1=iExtraContext stores parent FPSCR value |
455 TInt iExtraContextSize; /**< @internalComponent */ // +ve=dynamically allocated, 0=none, -1=iExtraContext stores parent FPSCR value |
359 |
456 |
|
457 TUint8 iCoreCycling; /**< @internalComponent */ // this thread is currently cycling through all active cores |
|
458 TUint8 iRebalanceAttr; /**< @internalComponent */ // behaviour of load balancing wrt this thread |
|
459 TUint8 iNThreadBaseSpare4c; /**< @internalComponent */ // spare to allow growth while preserving BC |
|
460 TUint8 iNThreadBaseSpare4d; /**< @internalComponent */ // spare to allow growth while preserving BC |
|
461 TUint32 iNThreadBaseSpare5; /**< @internalComponent */ // spare to allow growth while preserving BC |
360 TUint32 iNThreadBaseSpare6; /**< @internalComponent */ // spare to allow growth while preserving BC |
462 TUint32 iNThreadBaseSpare6; /**< @internalComponent */ // spare to allow growth while preserving BC |
361 TUint32 iNThreadBaseSpare7; /**< @internalComponent */ // spare to allow growth while preserving BC |
463 TUint32 iNThreadBaseSpare7; /**< @internalComponent */ // spare to allow growth while preserving BC |
362 TUint32 iNThreadBaseSpare8; /**< @internalComponent */ // spare to allow growth while preserving BC |
464 TUint32 iNThreadBaseSpare8; /**< @internalComponent */ // spare to allow growth while preserving BC |
363 TUint32 iNThreadBaseSpare9; /**< @internalComponent */ // spare to allow growth while preserving BC |
465 TUint32 iNThreadBaseSpare9; /**< @internalComponent */ // spare to allow growth while preserving BC |
364 |
|
365 // For EMI support - HOPEFULLY THIS CAN DIE |
|
366 TUint32 iTag; /**< @internalComponent */ // User defined set of bits which is ANDed with a mask when the thread is scheduled, and indicates if a DFC should be scheduled. |
|
367 TAny* iVemsData; /**< @internalComponent */ // This pointer can be used by any VEMS to store any data associated with the thread. This data must be clean up before the Thread Exit Monitor completes. |
|
368 }; |
466 }; |
369 |
467 |
370 __ASSERT_COMPILE(!(_FOFF(NThreadBase,iWaitLink)&7)); |
468 __ASSERT_COMPILE(!(_FOFF(NThreadBase,iWaitLink)&7)); |
371 __ASSERT_COMPILE(!(sizeof(NThreadBase)&7)); |
469 __ASSERT_COMPILE(!(sizeof(NThreadBase)&7)); |
372 |
470 |
405 { |
505 { |
406 public: |
506 public: |
407 NThreadGroup(); |
507 NThreadGroup(); |
408 public: |
508 public: |
409 TInt iThreadCount; /**< @internalComponent */ |
509 TInt iThreadCount; /**< @internalComponent */ |
|
510 TDfc* iDestructionDfc; /**< @internalComponent */ |
410 TPriList<NThreadBase, KNumPriorities> iNThreadList; /**< @internalComponent */ |
511 TPriList<NThreadBase, KNumPriorities> iNThreadList; /**< @internalComponent */ |
411 }; |
512 }; |
412 |
513 |
413 /******************************************** |
514 /******************************************** |
414 * Scheduler |
515 * Scheduler |
415 ********************************************/ |
516 ********************************************/ |
416 |
517 |
|
518 #include <nk_plat.h> |
|
519 |
|
520 /** |
|
521 @internalComponent |
|
522 */ |
|
523 enum |
|
524 { |
|
525 EQueueEvent_Kick=1, |
|
526 EQueueEvent_WakeUp=2, |
|
527 }; |
|
528 |
417 /** |
529 /** |
418 @internalComponent |
530 @internalComponent |
419 */ |
531 */ |
420 class TScheduler; |
532 class TScheduler; |
421 class NThread; |
533 class NThread; |
422 class NIrqHandler; |
534 class NIrqHandler; |
423 class TSubScheduler : public TPriListBase |
535 struct SIdlePullThread; |
|
536 class TSubScheduler |
424 { |
537 { |
425 public: |
538 public: |
426 TSubScheduler(); |
539 TSubScheduler(); |
427 void QueueDfcs(); |
540 void QueueDfcs(); |
428 void RotateReadyList(TInt aPriority); |
541 void RotateReadyList(TInt aPriority); |
429 NThread* SelectNextThread(); |
542 NThread* SelectNextThread(); |
430 TBool QueueEvent(NEventHandler* aEvent); |
543 TInt QueueEvent(NEventHandler* aEvent); |
431 void QueueEventAndKick(NEventHandler* aEvent); |
544 void QueueEventAndKick(NEventHandler* aEvent); |
432 void SaveTimesliceTimer(NThreadBase* aThread); |
545 void SaveTimesliceTimer(NThreadBase* aThread); |
433 void UpdateThreadTimes(NThreadBase* aOld, NThreadBase* aNew); |
546 void UpdateThreadTimes(NThreadBase* aOld, NThreadBase* aNew); |
|
547 void SSAddEntry(NSchedulable* aEntry); |
|
548 void SSAddEntryHead(NSchedulable* aEntry); |
|
549 void SSRemoveEntry(NSchedulable* aEntry); |
|
550 void SSChgEntryP(NSchedulable* aEntry, TInt aNewPriority); |
|
551 void IdlePullSearch(SIdlePullThread& a, TSubScheduler* aDest); |
|
552 void GetLbThreads(SDblQue& aQ); |
|
553 TBool Detached(); // platform specific |
|
554 |
|
555 inline TInt HighestPriority() |
|
556 { return iSSList.HighestPriority(); } |
|
557 inline NSchedulable* EntryAtPriority(TInt aPri) |
|
558 { return (NSchedulable*)iSSList.iQueue[aPri]; } |
434 private: |
559 private: |
435 SDblQueLink* iExtraQueues[KNumPriorities-1]; |
560 TPriList<NSchedulable, KNumPriorities> iSSList; |
436 public: |
561 public: |
437 TSpinLock iExIDfcLock; // lock to protect exogenous IDFC queue |
562 TSpinLock iExIDfcLock; // lock to protect exogenous IDFC queue |
438 |
563 |
439 SDblQue iExIDfcs; // list of pending exogenous IDFCs (i.e. ones punted over from another CPU) |
564 SDblQue iExIDfcs; // list of pending exogenous IDFCs (i.e. ones punted over from another CPU) |
440 |
565 |
454 volatile TUint8 iExIDfcPendingFlag; // TRUE if an exogenous IDFC is pending |
579 volatile TUint8 iExIDfcPendingFlag; // TRUE if an exogenous IDFC is pending |
455 TInt iKernLockCount; // how many times the current CPU has locked the kernel |
580 TInt iKernLockCount; // how many times the current CPU has locked the kernel |
456 |
581 |
457 TUint8 iInIDFC; // TRUE if IDFCs are currently being run on this CPU |
582 TUint8 iInIDFC; // TRUE if IDFCs are currently being run on this CPU |
458 volatile TUint8 iEventHandlersPending; // TRUE if an event handler is pending on this CPU |
583 volatile TUint8 iEventHandlersPending; // TRUE if an event handler is pending on this CPU |
459 TUint8 iSubSchedulerSpare4; |
584 TUint8 iCCSyncPending; |
460 TUint8 iSubSchedulerSpare5; |
585 TUint8 iLbCounter; |
461 TAny* iAddressSpace; |
586 TAny* iAddressSpace; |
462 |
587 |
463 TUint32 iReschedIPIs; |
588 TUint32 iReschedIPIs; |
464 TScheduler* iScheduler; |
589 TScheduler* iScheduler; |
465 |
590 |
466 union |
591 TInt iDeferShutdown; // counts reasons why this CPU can't shut down |
467 { |
592 TInt iRdyThreadCount; // number of ready threads excluding idle thread |
468 TUint64 iLastTimestamp64; // NKern::Timestamp() value at last reschedule or timestamp sync |
593 TUint16 iPriClassThreadCount[KNumPriClasses]; |
469 TUint32 iLastTimestamp32[2]; |
594 |
470 }; |
595 TUint64HL iLastTimestamp; // timestamp at which last reschedule occurred |
471 union |
596 TUint64HL iReschedCount; |
472 { |
|
473 TUint64 iReschedCount64; |
|
474 TUint32 iReschedCount32[2]; |
|
475 }; |
|
476 |
|
477 TAny* iExtras[24]; // Space for platform-specific extras |
|
478 |
597 |
479 TGenericIPI* iNextIPI; // next generic IPI to run on this CPU |
598 TGenericIPI* iNextIPI; // next generic IPI to run on this CPU |
480 NThread* iInitialThread; // Initial (idle) thread on this CPU |
599 NThread* iInitialThread; // Initial (idle) thread on this CPU |
481 |
600 |
482 TSpinLock iEventHandlerLock; // lock to protect event handler queue |
601 TSpinLock iEventHandlerLock; // lock to protect event handler queue |
483 |
602 |
484 SDblQue iEventHandlers; // queue of pending event handlers on this CPU |
603 SDblQue iEventHandlers; // queue of pending event handlers on this CPU |
485 |
604 |
486 TUint64 iSpinLockOrderCheck; // bitmask showing which spinlock orders currently held |
605 TUint64 iSpinLockOrderCheck; // bitmask showing which spinlock orders currently held |
487 |
606 |
488 TUint32 iSubSchedulerPadding[8]; |
607 TSubSchedulerX iSSX; // platform specific extras |
|
608 |
|
609 volatile TAny* iUncached; // points to platform specific uncached data structure |
|
610 TUint iMadeReadyCounter; // Number of times this core made a thread ready. |
|
611 |
|
612 TUint iMadeUnReadyCounter; // Number of times this core made a thread unready. |
|
613 TUint iTimeSliceExpireCounter; // Number of times this core hass reschedualed due to time slice exireation. |
|
614 |
|
615 TUint32 iSubSchedulerPadding[70]; |
|
616 SDblQue iLbQ; // threads to be considered by subsequent periodic load balance |
|
617 |
|
618 TAny* iSubSchedScratch[16]; // For use by code outside NKern |
489 }; |
619 }; |
|
620 |
|
621 const TInt KSubSchedulerShift = 10; // log2(sizeof(TSubScheduler)) |
490 |
622 |
491 __ASSERT_COMPILE(!(_FOFF(TSubScheduler,iExIDfcLock)&7)); |
623 __ASSERT_COMPILE(!(_FOFF(TSubScheduler,iExIDfcLock)&7)); |
492 __ASSERT_COMPILE(!(_FOFF(TSubScheduler,iEventHandlerLock)&7)); |
624 __ASSERT_COMPILE(!(_FOFF(TSubScheduler,iEventHandlerLock)&7)); |
493 __ASSERT_COMPILE(!(_FOFF(TSubScheduler,iReadyListLock)&7)); |
625 __ASSERT_COMPILE(!(_FOFF(TSubScheduler,iReadyListLock)&7)); |
494 __ASSERT_COMPILE(!(_FOFF(TSubScheduler,iLastTimestamp64)&7)); |
626 __ASSERT_COMPILE(!(_FOFF(TSubScheduler,iLastTimestamp)&7)); |
495 __ASSERT_COMPILE(!(_FOFF(TSubScheduler,iReschedCount64)&7)); |
627 __ASSERT_COMPILE(!(_FOFF(TSubScheduler,iReschedCount)&7)); |
496 __ASSERT_COMPILE(sizeof(TSubScheduler)==512); // make it a nice power of 2 size for easy indexing |
628 __ASSERT_COMPILE(sizeof(TSubSchedulerX)==256); |
|
629 __ASSERT_COMPILE(sizeof(TSubScheduler)==(1<<KSubSchedulerShift)); // make it a nice power of 2 size for easy indexing |
|
630 |
|
631 struct SCoreControlAction; |
|
632 struct SVariantInterfaceBlock; |
497 |
633 |
498 /** |
634 /** |
499 @internalComponent |
635 @internalComponent |
500 */ |
636 */ |
501 class TScheduler |
637 class TScheduler |
503 public: |
639 public: |
504 TScheduler(); |
640 TScheduler(); |
505 static void Reschedule(); |
641 static void Reschedule(); |
506 IMPORT_C static TScheduler* Ptr(); |
642 IMPORT_C static TScheduler* Ptr(); |
507 inline void SetProcessHandler(TLinAddr aHandler) {iProcessHandler=aHandler;} |
643 inline void SetProcessHandler(TLinAddr aHandler) {iProcessHandler=aHandler;} |
|
644 void PeriodicBalance(); |
|
645 TBool ReBalance(SDblQue& aQ, TBool aCC); |
|
646 void CCReactivate(TUint32 aMore); |
|
647 void CCIpiReactivate(); |
|
648 void CCRequest(); |
|
649 void GetLbThreads(SDblQue& aQ); |
|
650 void CCUnDefer(); |
|
651 void ChangeThreadAcceptCpus(TUint32 aNewMask); |
|
652 TUint32 ReschedInactiveCpus(TUint32 aMask); |
|
653 void InitCCAction(SCoreControlAction& aA); |
|
654 TUint32 ModifyCCState(TUint32 aAnd, TUint32 aXor); |
|
655 TUint32 CpuShuttingDown(TSubScheduler& aSS); |
|
656 void AllCpusIdle(); |
|
657 void FirstBackFromIdle(); |
|
658 |
|
659 void InitLB(); |
|
660 void StartRebalanceTimer(TBool aRestart); |
|
661 void StopRebalanceTimer(TBool aTemp); |
|
662 static void BalanceTimerExpired(TAny*); |
|
663 static void StartPeriodicBalancing(); |
|
664 static void CCSyncDone(TAny*); |
|
665 static void CCReactivateDfcFn(TAny*); |
|
666 static void CCRequestDfcFn(TAny*); |
|
667 static void CCIpiReactivateFn(TAny*); |
|
668 static TDfcQue* RebalanceDfcQ(); |
|
669 static NThread* LBThread(); |
|
670 static TBool CoreControlSupported(); |
|
671 static void CCInitiatePowerUp(TUint32 aCores); |
|
672 static void CCIndirectPowerDown(TAny*); |
|
673 static void DoFrequencyChanged(TAny*); |
508 public: |
674 public: |
509 TLinAddr iMonitorExceptionHandler; |
675 TLinAddr iMonitorExceptionHandler; |
510 TLinAddr iProcessHandler; |
676 TLinAddr iProcessHandler; |
511 |
677 |
|
678 volatile TUint32 iThreadAcceptCpus; // bit n set if CPU n is accepting unlocked threads |
|
679 volatile TUint32 iIpiAcceptCpus; // bit n set if CPU n is accepting generic IPIs |
|
680 volatile TUint32 iCpusComingUp; // bit n set if CPU n is in the process of powering up |
|
681 volatile TUint32 iCpusGoingDown; // bit n set if CPU n is in the process of powering down and is no longer accepting IPIs |
|
682 volatile TInt iCCDeferCount; // >0 means CPUs on the way down will stop just before the 'point of no return' |
|
683 volatile TUint32 iCCSyncCpus; // bit n set if CPU n has not yet observed a change to iThreadAcceptCpus |
|
684 volatile TUint32 iCCReactivateCpus; |
|
685 volatile TUint32 iCCState; |
|
686 |
|
687 TInt iNumCpus; // number of CPUs under the kernel's control |
512 TLinAddr iRescheduleHook; |
688 TLinAddr iRescheduleHook; |
513 TUint32 iActiveCpus1; // bit n set if CPU n is accepting unlocked threads |
689 |
514 |
690 SDblQue iGenIPIList; // list of active generic IPIs |
515 TUint32 iActiveCpus2; // bit n set if CPU n is accepting generic IPIs |
691 TSpinLock iGenIPILock; // spin lock protects iGenIPIList, also iIpiAcceptCpus, iCpusComingUp, iCpusGoingDown, iCCDeferCount |
516 TInt iNumCpus; // number of CPUs under the kernel's control |
|
517 |
692 |
518 TSubScheduler* iSub[KMaxCpus]; // one subscheduler per CPU |
693 TSubScheduler* iSub[KMaxCpus]; // one subscheduler per CPU |
519 |
694 |
520 TAny* iExtras[24]; // Space for platform-specific extras |
695 TAny* iSchedScratch[16]; // for use by code outside NKern |
|
696 |
|
697 TSchedulerX iSX; // platform specific extras |
521 |
698 |
522 NFastMutex iLock; // the 'system lock' fast mutex |
699 NFastMutex iLock; // the 'system lock' fast mutex |
|
700 |
|
701 TSpinLock iIdleBalanceLock; |
523 |
702 |
524 TSpinLock iIdleSpinLock; // lock to protect list of DFCs to be run on idle |
703 TSpinLock iIdleSpinLock; // lock to protect list of DFCs to be run on idle |
525 |
704 |
526 SDblQue iIdleDfcs; // list of DFCs to run when all CPUs go idle |
705 SDblQue iIdleDfcs; // list of DFCs to run when all CPUs go idle |
527 |
706 |
528 TUint32 iCpusNotIdle; // bitmask - Bit n set => CPU n is not idle |
707 TUint32 iCpusNotIdle; // bitmask - Bit n set => CPU n is not idle |
529 TUint8 iIdleGeneration; // Toggles between 0 and 1 each time iIdleDfcs list is spilled to a CPU IDFC queue |
708 TUint8 iIdleGeneration; // Toggles between 0 and 1 each time iIdleDfcs list is spilled to a CPU IDFC queue |
530 TUint8 iIdleSpillCpu; // Which CPU last spilled the iIdleDfcs list to its IDFC queue |
709 TUint8 iIdleSpillCpu; // Which CPU last spilled the iIdleDfcs list to its IDFC queue |
531 TUint8 iTSchedulerSpare1; |
710 TUint8 iLbCounter; |
532 TUint8 iTSchedulerSpare2; |
711 volatile TUint8 iNeedBal; |
533 |
712 |
534 TUint32 iIdleGenerationCount; // Incremented each time iIdleDfcs list is spilled to a CPU IDFC queue |
713 TUint32 iIdleGenerationCount; // Incremented each time iIdleDfcs list is spilled to a CPU IDFC queue |
535 TUint32 i_Scheduler_Padding[3]; |
714 TDfcQue* iRebalanceDfcQ; |
536 |
715 |
537 // For EMI support - HOPEFULLY THIS CAN DIE |
716 TSpinLock iEnumerateLock; // lock to protect iAllThreads, iAllGroups |
538 NThread* iSigma; |
717 SIterDQ iAllThreads; // list of all nanokernel threads in order of creation |
539 TDfc* iEmiDfc; |
718 SIterDQ iAllGroups; // list of all thread groups in order of creation |
540 TUint32 iEmiMask; |
719 TSpinLock iBalanceListLock; // lock to protect iBalanceList |
541 TUint32 iEmiState; |
720 TUint64 iLastBalanceTime; // time at which last rebalance occurred |
542 TUint32 iEmiDfcTrigger; |
721 SDblQue iBalanceList; // list of threads/groups for load balancing |
543 TBool iLogging; |
722 NTimer iBalanceTimer; // triggers periodic rebalancing |
544 TAny* iBufferStart; |
723 TDfc iCCSyncIDFC; // runs when a change to iThreadAcceptCpus has been observed by all CPUs |
545 TAny* iBufferEnd; |
724 TDfc iCCReactivateDfc; // runs when a reschedule IPI is targeted to an inactive CPU |
546 TAny* iBufferTail; |
725 |
547 TAny* iBufferHead; |
726 TUint32 iCCRequestLevel; // Number of active cores last requested |
|
727 volatile TUint32 iCCIpiReactivate; // Cores to be woken up because of IPIs |
|
728 |
|
729 TDfc iCCRequestDfc; // runs when a request is made to change the number of active cores |
|
730 TDfc iCCPowerDownDfc; // runs when indirect power down of core(s) is required |
|
731 TDfc iCCIpiReactIDFC; // runs when an IPI needs to wake up a core |
|
732 TDfc iFreqChgDfc; // runs when frequency changes required |
|
733 |
|
734 TSubScheduler* iPoweringOff; // CPU last to power off |
|
735 TUint32 iDetachCount; // detach count before power off |
|
736 |
|
737 SVariantInterfaceBlock* iVIB; |
|
738 TUint32 i_Scheduler_Padding[29]; |
548 }; |
739 }; |
549 |
740 |
|
741 __ASSERT_COMPILE(!(_FOFF(TScheduler,iGenIPILock)&7)); |
550 __ASSERT_COMPILE(!(_FOFF(TScheduler,iIdleSpinLock)&7)); |
742 __ASSERT_COMPILE(!(_FOFF(TScheduler,iIdleSpinLock)&7)); |
551 __ASSERT_COMPILE(sizeof(TScheduler)==512); |
743 __ASSERT_COMPILE(!(_FOFF(TScheduler,iIdleBalanceLock)&7)); |
|
744 __ASSERT_COMPILE(!(_FOFF(TScheduler,iEnumerateLock)&7)); |
|
745 __ASSERT_COMPILE(!(_FOFF(TScheduler,iBalanceListLock)&7)); |
|
746 __ASSERT_COMPILE(sizeof(TSchedulerX)==32*4); |
|
747 __ASSERT_COMPILE(sizeof(TScheduler)==1024); |
552 |
748 |
553 extern TScheduler TheScheduler; |
749 extern TScheduler TheScheduler; |
554 extern TSubScheduler TheSubSchedulers[KMaxCpus]; |
750 extern TSubScheduler TheSubSchedulers[KMaxCpus]; |
555 |
751 |
556 #ifdef __USE_BTRACE_LOCK__ |
752 #ifdef __USE_BTRACE_LOCK__ |
692 void Add(NTimer* aTimer); |
894 void Add(NTimer* aTimer); |
693 void AddFinal(NTimer* aTimer); |
895 void AddFinal(NTimer* aTimer); |
694 public: |
896 public: |
695 STimerQ iTickQ[ENumTimerQueues]; /**< @internalComponent */ // NOTE: the order of member data is important |
897 STimerQ iTickQ[ENumTimerQueues]; /**< @internalComponent */ // NOTE: the order of member data is important |
696 TUint32 iPresent; /**< @internalComponent */ // The assembler code relies on it |
898 TUint32 iPresent; /**< @internalComponent */ // The assembler code relies on it |
697 TUint32 iMsCount; /**< @internalComponent */ |
899 |
|
900 /** |
|
901 This member is intended for use by ASSP/variant interrupt code as a convenient |
|
902 location to store the value of a free running counter at the point where the |
|
903 system tick is started. |
|
904 @publishedPartner |
|
905 @prototype |
|
906 */ |
|
907 TUint32 iFRCOffset; |
|
908 |
|
909 union { |
|
910 TUint32 iMsCount; /**< @internalComponent */ |
|
911 TUint64 iMsCount64; /**< @internalComponent */ |
|
912 }; |
698 SDblQue iHoldingQ; /**< @internalComponent */ |
913 SDblQue iHoldingQ; /**< @internalComponent */ |
699 SDblQue iOrderedQ; /**< @internalComponent */ |
914 SDblQue iOrderedQ; /**< @internalComponent */ |
700 SDblQue iCompletedQ; /**< @internalComponent */ |
915 SDblQue iCompletedQ; /**< @internalComponent */ |
701 TDfc iDfc; /**< @internalComponent */ |
916 TDfc iDfc; /**< @internalComponent */ |
702 TUint8 iTransferringCancelled; /**< @internalComponent */ |
917 TUint8 iTransferringCancelled; /**< @internalComponent */ |
795 #ifdef _DEBUG |
999 #ifdef _DEBUG |
796 |
1000 |
797 /** |
1001 /** |
798 @internalComponent |
1002 @internalComponent |
799 */ |
1003 */ |
800 #define MASK_NO_FAST_MUTEX 0x1 |
1004 #define MASK_NO_FAST_MUTEX 0x1 |
801 #define MASK_CRITICAL 0x2 |
1005 #define MASK_CRITICAL 0x2 |
802 #define MASK_NO_CRITICAL 0x4 |
1006 #define MASK_NO_CRITICAL 0x4 |
803 #define MASK_KERNEL_LOCKED 0x8 |
1007 #define MASK_KERNEL_LOCKED 0x8 |
804 #define MASK_KERNEL_UNLOCKED 0x10 |
1008 #define MASK_KERNEL_UNLOCKED 0x10 |
805 #define MASK_KERNEL_LOCKED_ONCE 0x20 |
1009 #define MASK_KERNEL_LOCKED_ONCE 0x20 |
806 #define MASK_INTERRUPTS_ENABLED 0x40 |
1010 #define MASK_INTERRUPTS_ENABLED 0x40 |
807 #define MASK_INTERRUPTS_DISABLED 0x80 |
1011 #define MASK_INTERRUPTS_DISABLED 0x80 |
808 #define MASK_SYSTEM_LOCKED 0x100 |
1012 #define MASK_SYSTEM_LOCKED 0x100 |
809 #define MASK_NOT_ISR 0x400 |
1013 #define MASK_NOT_ISR 0x400 |
810 #define MASK_NOT_IDFC 0x800 |
1014 #define MASK_NOT_IDFC 0x800 |
811 #define MASK_NOT_THREAD 0x1000 |
1015 #define MASK_NOT_THREAD 0x1000 |
812 #define MASK_NO_CRITICAL_IF_USER 0x2000 |
1016 #define MASK_NO_CRITICAL_IF_USER 0x2000 |
813 #define MASK_ALWAYS_FAIL 0x4000 |
1017 #define MASK_ALWAYS_FAIL 0x4000 |
814 #define MASK_NO_RESCHED 0x8000 |
1018 #define MASK_NO_RESCHED 0x8000 |
815 #define MASK_NO_KILL_OR_SUSPEND 0x10000 |
1019 #define MASK_NO_KILL_OR_SUSPEND 0x10000 |
816 |
1020 #define MASK_THREAD_STANDARD ( MASK_NO_FAST_MUTEX | MASK_KERNEL_UNLOCKED | MASK_INTERRUPTS_ENABLED | MASK_NOT_ISR | MASK_NOT_IDFC ) |
817 #define MASK_THREAD_STANDARD ( MASK_NO_FAST_MUTEX | MASK_KERNEL_UNLOCKED | MASK_INTERRUPTS_ENABLED | MASK_NOT_ISR | MASK_NOT_IDFC ) |
1021 #define MASK_THREAD_CRITICAL ( MASK_THREAD_STANDARD | MASK_CRITICAL ) |
818 #define MASK_THREAD_CRITICAL ( MASK_THREAD_STANDARD | MASK_CRITICAL ) |
|
819 |
1022 |
820 #if defined(__STANDALONE_NANOKERNEL__) || (!defined (__KERNEL_APIS_CONTEXT_CHECKS_WARNING__)&&!defined (__KERNEL_APIS_CONTEXT_CHECKS_FAULT__)) |
1023 #if defined(__STANDALONE_NANOKERNEL__) || (!defined (__KERNEL_APIS_CONTEXT_CHECKS_WARNING__)&&!defined (__KERNEL_APIS_CONTEXT_CHECKS_FAULT__)) |
821 #define CHECK_PRECONDITIONS(mask,function) |
1024 #define CHECK_PRECONDITIONS(mask,function) |
822 #define __ASSERT_WITH_MESSAGE_DEBUG(cond,message,function) |
1025 #define __ASSERT_WITH_MESSAGE_DEBUG(cond,message,function) |
823 |
1026 |