diff --git a/src/system/kernel/scheduler/scheduler_cpu.cpp b/src/system/kernel/scheduler/scheduler_cpu.cpp index feac66e1eb..0ff1739eee 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.cpp +++ b/src/system/kernel/scheduler/scheduler_cpu.cpp @@ -39,7 +39,16 @@ class Scheduler::DebugDumper { public: static void DumpCPURunQueue(CPUEntry* cpu); static void DumpCoreRunQueue(CoreEntry* core); + static void DumpCoreLoadHeapEntry(CoreEntry* core); static void DumpIdleCoresInPackage(PackageEntry* package); + +private: + struct CoreThreadsData { + CoreEntry* fCore; + int32 fLoad; + }; + + static void _AnalyzeCoreThreads(Thread* thread, void* data); }; @@ -357,12 +366,15 @@ CoreEntry::CoreEntry() fThreadCount(0), fActiveTime(0), fLoad(0), + fCurrentLoad(0), + fLoadMeasurementEpoch(0), fHighLoad(false), fLastLoadUpdate(0) { B_INITIALIZE_SPINLOCK(&fCPULock); B_INITIALIZE_SPINLOCK(&fQueueLock); B_INITIALIZE_SEQLOCK(&fActiveTimeLock); + B_INITIALIZE_RW_SPINLOCK(&fLoadLock); } @@ -413,67 +425,6 @@ CoreEntry::Remove(ThreadData* thread) } -void -CoreEntry::UpdateLoad(int32 delta) -{ - SCHEDULER_ENTER_FUNCTION(); - - ASSERT(gTrackCoreLoad); - - if (fCPUCount <= 0) - return; - - atomic_add(&fLoad, delta); - - bigtime_t now = system_time(); - if (now < kLoadMeasureInterval + fLastLoadUpdate) - return; - if (!try_acquire_write_spinlock(&gCoreHeapsLock)) - return; - WriteSpinLocker coreLocker(gCoreHeapsLock, true); - - fLastLoadUpdate = now; - - int32 newKey = GetLoad(); - int32 oldKey = CoreLoadHeap::GetKey(this); - - ASSERT(oldKey >= 0); - ASSERT(newKey >= 0); - - if (oldKey == newKey) - return; - - if (newKey > kHighLoad) { - if (!fHighLoad) { - gCoreLoadHeap.ModifyKey(this, -1); - ASSERT(gCoreLoadHeap.PeekMinimum() == this); - gCoreLoadHeap.RemoveMinimum(); - - gCoreHighLoadHeap.Insert(this, newKey); - - fHighLoad = true; - } else - gCoreHighLoadHeap.ModifyKey(this, newKey); - } else if (newKey < kMediumLoad) { - if (fHighLoad) { - gCoreHighLoadHeap.ModifyKey(this, -1); - ASSERT(gCoreHighLoadHeap.PeekMinimum() == this); - gCoreHighLoadHeap.RemoveMinimum(); - - gCoreLoadHeap.Insert(this, newKey); - - fHighLoad = false; - } else - gCoreLoadHeap.ModifyKey(this, newKey); - } else { - if (fHighLoad) - gCoreHighLoadHeap.ModifyKey(this, newKey); - else - gCoreLoadHeap.ModifyKey(this, newKey); - } -} - - void CoreEntry::AddCPU(CPUEntry* cpu) { @@ -484,6 +435,7 @@ CoreEntry::AddCPU(CPUEntry* cpu) if (fCPUCount++ == 0) { // core has been reenabled fLoad = 0; + fCurrentLoad = 0; fHighLoad = false; gCoreLoadHeap.Insert(this, 0); @@ -540,6 +492,69 @@ CoreEntry::RemoveCPU(CPUEntry* cpu, ThreadProcessing& threadPostProcessing) } +void +CoreEntry::_UpdateLoad() +{ + SCHEDULER_ENTER_FUNCTION(); + + if (fCPUCount <= 0) + return; + + bigtime_t now = system_time(); + if (now < kLoadMeasureInterval + fLastLoadUpdate) + return; + if (!try_acquire_write_spinlock(&gCoreHeapsLock)) + return; + WriteSpinLocker coreLocker(gCoreHeapsLock, true); + WriteSpinLocker locker(fLoadLock); + + int32 newKey = GetLoad(); + int32 oldKey = CoreLoadHeap::GetKey(this); + + ASSERT(oldKey >= 0); + ASSERT(newKey >= 0); + + ASSERT(fCurrentLoad >= 0); + ASSERT(fLoad >= fCurrentLoad); + + fLoad = fCurrentLoad; + fLoadMeasurementEpoch++; + fLastLoadUpdate = now; + + if (oldKey == newKey) + return; + + if (newKey > kHighLoad) { + if (!fHighLoad) { + gCoreLoadHeap.ModifyKey(this, -1); + ASSERT(gCoreLoadHeap.PeekMinimum() == this); + gCoreLoadHeap.RemoveMinimum(); + + gCoreHighLoadHeap.Insert(this, newKey); + + fHighLoad = true; + } else + gCoreHighLoadHeap.ModifyKey(this, newKey); + } else if (newKey < kMediumLoad) { + if (fHighLoad) { + gCoreHighLoadHeap.ModifyKey(this, -1); + ASSERT(gCoreHighLoadHeap.PeekMinimum() == this); + gCoreHighLoadHeap.RemoveMinimum(); + + gCoreLoadHeap.Insert(this, newKey); + + fHighLoad = false; + } else + gCoreLoadHeap.ModifyKey(this, newKey); + } else { + if (fHighLoad) + gCoreHighLoadHeap.ModifyKey(this, newKey); + else + gCoreLoadHeap.ModifyKey(this, newKey); + } +} + + /* static */ void CoreEntry::_UnassignThread(Thread* thread, void* data) { @@ -565,8 +580,7 @@ CoreLoadHeap::Dump() while (entry) { int32 key = GetKey(entry); - kprintf("%4" B_PRId32 " %3" B_PRId32 "%% %7" B_PRId32 "\n", entry->ID(), - entry->GetLoad() / 10, entry->ThreadCount()); + DebugDumper::DumpCoreLoadHeapEntry(entry); RemoveMinimum(); sDebugCoreHeap.Insert(entry, key); @@ -644,6 +658,21 @@ DebugDumper::DumpCoreRunQueue(CoreEntry* core) } +/* static */ void +DebugDumper::DumpCoreLoadHeapEntry(CoreEntry* entry) +{ + CoreThreadsData threadsData; + threadsData.fCore = entry; + threadsData.fLoad = 0; + thread_map(DebugDumper::_AnalyzeCoreThreads, &threadsData); + + kprintf("%4" B_PRId32 " %11" B_PRId32 "%% %11" B_PRId32 "%% %11" B_PRId32 + "%% %7" B_PRId32 " %5" B_PRIu32 "\n", entry->ID(), entry->fLoad / 10, + entry->fCurrentLoad / 10, threadsData.fLoad, entry->ThreadCount(), + entry->fLoadMeasurementEpoch); +} + + /* static */ void DebugDumper::DumpIdleCoresInPackage(PackageEntry* package) { @@ -663,6 +692,15 @@ DebugDumper::DumpIdleCoresInPackage(PackageEntry* package) } +/* static */ void +DebugDumper::_AnalyzeCoreThreads(Thread* thread, void* data) +{ + CoreThreadsData* threadsData = static_cast(data); + if (thread->scheduler_data->Core() == threadsData->fCore) + threadsData->fLoad += thread->scheduler_data->GetLoad(); +} + + static int dump_run_queue(int /* argc */, char** /* argv */) { @@ -684,7 +722,7 @@ dump_run_queue(int /* argc */, char** /* argv */) static int dump_cpu_heap(int /* argc */, char** /* argv */) { - kprintf("core load threads\n"); + kprintf("core average_load current_load threads_load threads epoch\n"); gCoreLoadHeap.Dump(); kprintf("\n"); gCoreHighLoadHeap.Dump(); diff --git a/src/system/kernel/scheduler/scheduler_cpu.h b/src/system/kernel/scheduler/scheduler_cpu.h index f8664e7f65..364d3eac7b 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.h +++ b/src/system/kernel/scheduler/scheduler_cpu.h @@ -146,7 +146,13 @@ public: bigtime_t activeTime); inline int32 GetLoad() const; - void UpdateLoad(int32 delta); + inline uint32 LoadMeasurementEpoch() const + { return fLoadMeasurementEpoch; } + + inline void AddLoad(int32 load, uint32 epoch, + bool updateLoad); + inline uint32 RemoveLoad(int32 load, bool force); + inline void ChangeLoad(int32 delta); inline int32 StarvationCounter() const; inline int32 StarvationCounterIdle() const; @@ -162,6 +168,8 @@ public: static inline CoreEntry* GetCore(int32 cpu); private: + void _UpdateLoad(); + static void _UnassignThread(Thread* thread, void* core); @@ -184,8 +192,11 @@ private: mutable seqlock fActiveTimeLock; int32 fLoad; + int32 fCurrentLoad; + uint32 fLoadMeasurementEpoch; bool fHighLoad; bigtime_t fLastLoadUpdate; + rw_spinlock fLoadLock; friend class DebugDumper; } CACHE_LINE_ALIGN; @@ -388,6 +399,62 @@ CoreEntry::GetLoad() const } +inline void +CoreEntry::AddLoad(int32 load, uint32 epoch, bool updateLoad) +{ + SCHEDULER_ENTER_FUNCTION(); + + ASSERT(gTrackCoreLoad); + ASSERT(load >= 0 && load <= kMaxLoad); + + ReadSpinLocker locker(fLoadLock); + atomic_add(&fCurrentLoad, load); + if (fLoadMeasurementEpoch != epoch) + atomic_add(&fLoad, load); + locker.Unlock(); + + if (updateLoad) + _UpdateLoad(); +} + + +inline uint32 +CoreEntry::RemoveLoad(int32 load, bool force) +{ + SCHEDULER_ENTER_FUNCTION(); + + ASSERT(gTrackCoreLoad); + ASSERT(load >= 0 && load <= kMaxLoad); + + ReadSpinLocker locker(fLoadLock); + atomic_add(&fCurrentLoad, -load); + if (force) { + atomic_add(&fLoad, -load); + locker.Unlock(); + + _UpdateLoad(); + } + return fLoadMeasurementEpoch; +} + + +inline void +CoreEntry::ChangeLoad(int32 delta) +{ + SCHEDULER_ENTER_FUNCTION(); + + ASSERT(gTrackCoreLoad); + ASSERT(delta >= -kMaxLoad && delta <= kMaxLoad); + + ReadSpinLocker locker(fLoadLock); + atomic_add(&fCurrentLoad, delta); + atomic_add(&fLoad, delta); + locker.Unlock(); + + _UpdateLoad(); +} + + inline int32 CoreEntry::StarvationCounter() const { diff --git a/src/system/kernel/scheduler/scheduler_thread.cpp b/src/system/kernel/scheduler/scheduler_thread.cpp index 6ce5a78705..b0d69b5b41 100644 --- a/src/system/kernel/scheduler/scheduler_thread.cpp +++ b/src/system/kernel/scheduler/scheduler_thread.cpp @@ -102,6 +102,7 @@ ThreadData::Init() Thread* currentThread = thread_get_current_thread(); ThreadData* currentThreadData = currentThread->scheduler_data; fCore = currentThreadData->fCore; + fLoadMeasurementEpoch = fCore->LoadMeasurementEpoch() - 1; if (!IsRealTime()) { fPriorityPenalty = std::min(currentThreadData->fPriorityPenalty, @@ -170,9 +171,13 @@ ThreadData::ChooseCoreAndCPU(CoreEntry*& targetCore, CPUEntry*& targetCPU) ASSERT(targetCore != NULL); ASSERT(targetCPU != NULL); - if (fReady && fCore != targetCore && fCore != NULL) { - fCore->UpdateLoad(-fNeededLoad); - targetCore->UpdateLoad(fNeededLoad); + if (fCore != targetCore) { + fLoadMeasurementEpoch = targetCore->LoadMeasurementEpoch() - 1; + if (fReady) { + if (fCore != NULL) + fCore->RemoveLoad(fNeededLoad, true); + targetCore->AddLoad(fNeededLoad, fLoadMeasurementEpoch, true); + } } fCore = targetCore; @@ -262,14 +267,14 @@ void ThreadData::_ComputeNeededLoad() { SCHEDULER_ENTER_FUNCTION(); + ASSERT(!IsIdle()); int32 oldLoad = compute_load(fLastMeasureAvailableTime, fMeasureAvailableActiveTime, fNeededLoad, fMeasureAvailableTime); if (oldLoad < 0 || oldLoad == fNeededLoad) return; - int32 delta = fNeededLoad - oldLoad; - fCore->UpdateLoad(delta); + fCore->ChangeLoad(fNeededLoad - oldLoad); } diff --git a/src/system/kernel/scheduler/scheduler_thread.h b/src/system/kernel/scheduler/scheduler_thread.h index 254d8c3494..415b86eb55 100644 --- a/src/system/kernel/scheduler/scheduler_thread.h +++ b/src/system/kernel/scheduler/scheduler_thread.h @@ -131,6 +131,7 @@ private: bigtime_t fLastMeasureAvailableTime; int32 fNeededLoad; + uint32 fLoadMeasurementEpoch; CoreEntry* fCore; }; @@ -377,7 +378,7 @@ ThreadData::GoesAway() fWentSleepCountIdle = fCore->StarvationCounterIdle(); if (gTrackCoreLoad) - fCore->UpdateLoad(-fNeededLoad); + fLoadMeasurementEpoch = fCore->RemoveLoad(fNeededLoad, false); fReady = false; } @@ -389,7 +390,7 @@ ThreadData::Dies() ASSERT(fReady); if (gTrackCoreLoad) - fCore->UpdateLoad(-fNeededLoad); + fCore->RemoveLoad(fNeededLoad, true); fReady = false; } @@ -428,8 +429,10 @@ ThreadData::Enqueue() if (!fReady) { if (gTrackCoreLoad) { bigtime_t timeSlept = system_time() - fWentSleep; - fCore->UpdateLoad(fNeededLoad); - if (timeSlept > 0) { + bool updateLoad = timeSlept > 0; + + fCore->AddLoad(fNeededLoad, fLoadMeasurementEpoch, !updateLoad); + if (updateLoad) { fMeasureAvailableTime += timeSlept; _ComputeNeededLoad(); }