From 6155ab7b25c5f0b32bb014bab5916f4594dbb685 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Fri, 31 Jan 2014 01:02:33 +0100 Subject: [PATCH] scheduler: Provide more stable core load statistics Originially, core load was a sum of eastimated loads of all currently running or ready threads on a given core. Such value is changing very rapidly preventing the thread migration logic from making any reasonable decisions. This patch changes the way core load is computed to make it more stable thus improving the qualitiy of decisions made by the thread migration logic. Currently core load is a sum of estimated loads of all threads that have been ready during last load measurement interval and haven't been migrated or killed. --- src/system/kernel/scheduler/scheduler_cpu.cpp | 166 +++++++++++------- src/system/kernel/scheduler/scheduler_cpu.h | 69 +++++++- .../kernel/scheduler/scheduler_thread.cpp | 15 +- .../kernel/scheduler/scheduler_thread.h | 11 +- 4 files changed, 187 insertions(+), 74 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler_cpu.cpp b/src/system/kernel/scheduler/scheduler_cpu.cpp index feac66e1eb..0ff1739eee 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.cpp +++ b/src/system/kernel/scheduler/scheduler_cpu.cpp @@ -39,7 +39,16 @@ class Scheduler::DebugDumper { public: static void DumpCPURunQueue(CPUEntry* cpu); static void DumpCoreRunQueue(CoreEntry* core); + static void DumpCoreLoadHeapEntry(CoreEntry* core); static void DumpIdleCoresInPackage(PackageEntry* package); + +private: + struct CoreThreadsData { + CoreEntry* fCore; + int32 fLoad; + }; + + static void _AnalyzeCoreThreads(Thread* thread, void* data); }; @@ -357,12 +366,15 @@ CoreEntry::CoreEntry() fThreadCount(0), fActiveTime(0), fLoad(0), + fCurrentLoad(0), + fLoadMeasurementEpoch(0), fHighLoad(false), fLastLoadUpdate(0) { B_INITIALIZE_SPINLOCK(&fCPULock); B_INITIALIZE_SPINLOCK(&fQueueLock); B_INITIALIZE_SEQLOCK(&fActiveTimeLock); + B_INITIALIZE_RW_SPINLOCK(&fLoadLock); } @@ -413,67 +425,6 @@ CoreEntry::Remove(ThreadData* thread) } -void -CoreEntry::UpdateLoad(int32 delta) -{ - SCHEDULER_ENTER_FUNCTION(); - - ASSERT(gTrackCoreLoad); - - if (fCPUCount <= 0) - return; - - atomic_add(&fLoad, delta); - - bigtime_t now = system_time(); - if (now < kLoadMeasureInterval + fLastLoadUpdate) - return; - if (!try_acquire_write_spinlock(&gCoreHeapsLock)) - return; - WriteSpinLocker coreLocker(gCoreHeapsLock, true); - - fLastLoadUpdate = now; - - int32 newKey = GetLoad(); - int32 oldKey = CoreLoadHeap::GetKey(this); - - ASSERT(oldKey >= 0); - ASSERT(newKey >= 0); - - if (oldKey == newKey) - return; - - if (newKey > kHighLoad) { - if (!fHighLoad) { - gCoreLoadHeap.ModifyKey(this, -1); - ASSERT(gCoreLoadHeap.PeekMinimum() == this); - gCoreLoadHeap.RemoveMinimum(); - - gCoreHighLoadHeap.Insert(this, newKey); - - fHighLoad = true; - } else - gCoreHighLoadHeap.ModifyKey(this, newKey); - } else if (newKey < kMediumLoad) { - if (fHighLoad) { - gCoreHighLoadHeap.ModifyKey(this, -1); - ASSERT(gCoreHighLoadHeap.PeekMinimum() == this); - gCoreHighLoadHeap.RemoveMinimum(); - - gCoreLoadHeap.Insert(this, newKey); - - fHighLoad = false; - } else - gCoreLoadHeap.ModifyKey(this, newKey); - } else { - if (fHighLoad) - gCoreHighLoadHeap.ModifyKey(this, newKey); - else - gCoreLoadHeap.ModifyKey(this, newKey); - } -} - - void CoreEntry::AddCPU(CPUEntry* cpu) { @@ -484,6 +435,7 @@ CoreEntry::AddCPU(CPUEntry* cpu) if (fCPUCount++ == 0) { // core has been reenabled fLoad = 0; + fCurrentLoad = 0; fHighLoad = false; gCoreLoadHeap.Insert(this, 0); @@ -540,6 +492,69 @@ CoreEntry::RemoveCPU(CPUEntry* cpu, ThreadProcessing& threadPostProcessing) } +void +CoreEntry::_UpdateLoad() +{ + SCHEDULER_ENTER_FUNCTION(); + + if (fCPUCount <= 0) + return; + + bigtime_t now = system_time(); + if (now < kLoadMeasureInterval + fLastLoadUpdate) + return; + if (!try_acquire_write_spinlock(&gCoreHeapsLock)) + return; + WriteSpinLocker coreLocker(gCoreHeapsLock, true); + WriteSpinLocker locker(fLoadLock); + + int32 newKey = GetLoad(); + int32 oldKey = CoreLoadHeap::GetKey(this); + + ASSERT(oldKey >= 0); + ASSERT(newKey >= 0); + + ASSERT(fCurrentLoad >= 0); + ASSERT(fLoad >= fCurrentLoad); + + fLoad = fCurrentLoad; + fLoadMeasurementEpoch++; + fLastLoadUpdate = now; + + if (oldKey == newKey) + return; + + if (newKey > kHighLoad) { + if (!fHighLoad) { + gCoreLoadHeap.ModifyKey(this, -1); + ASSERT(gCoreLoadHeap.PeekMinimum() == this); + gCoreLoadHeap.RemoveMinimum(); + + gCoreHighLoadHeap.Insert(this, newKey); + + fHighLoad = true; + } else + gCoreHighLoadHeap.ModifyKey(this, newKey); + } else if (newKey < kMediumLoad) { + if (fHighLoad) { + gCoreHighLoadHeap.ModifyKey(this, -1); + ASSERT(gCoreHighLoadHeap.PeekMinimum() == this); + gCoreHighLoadHeap.RemoveMinimum(); + + gCoreLoadHeap.Insert(this, newKey); + + fHighLoad = false; + } else + gCoreLoadHeap.ModifyKey(this, newKey); + } else { + if (fHighLoad) + gCoreHighLoadHeap.ModifyKey(this, newKey); + else + gCoreLoadHeap.ModifyKey(this, newKey); + } +} + + /* static */ void CoreEntry::_UnassignThread(Thread* thread, void* data) { @@ -565,8 +580,7 @@ CoreLoadHeap::Dump() while (entry) { int32 key = GetKey(entry); - kprintf("%4" B_PRId32 " %3" B_PRId32 "%% %7" B_PRId32 "\n", entry->ID(), - entry->GetLoad() / 10, entry->ThreadCount()); + DebugDumper::DumpCoreLoadHeapEntry(entry); RemoveMinimum(); sDebugCoreHeap.Insert(entry, key); @@ -644,6 +658,21 @@ DebugDumper::DumpCoreRunQueue(CoreEntry* core) } +/* static */ void +DebugDumper::DumpCoreLoadHeapEntry(CoreEntry* entry) +{ + CoreThreadsData threadsData; + threadsData.fCore = entry; + threadsData.fLoad = 0; + thread_map(DebugDumper::_AnalyzeCoreThreads, &threadsData); + + kprintf("%4" B_PRId32 " %11" B_PRId32 "%% %11" B_PRId32 "%% %11" B_PRId32 + "%% %7" B_PRId32 " %5" B_PRIu32 "\n", entry->ID(), entry->fLoad / 10, + entry->fCurrentLoad / 10, threadsData.fLoad, entry->ThreadCount(), + entry->fLoadMeasurementEpoch); +} + + /* static */ void DebugDumper::DumpIdleCoresInPackage(PackageEntry* package) { @@ -663,6 +692,15 @@ DebugDumper::DumpIdleCoresInPackage(PackageEntry* package) } +/* static */ void +DebugDumper::_AnalyzeCoreThreads(Thread* thread, void* data) +{ + CoreThreadsData* threadsData = static_cast(data); + if (thread->scheduler_data->Core() == threadsData->fCore) + threadsData->fLoad += thread->scheduler_data->GetLoad(); +} + + static int dump_run_queue(int /* argc */, char** /* argv */) { @@ -684,7 +722,7 @@ dump_run_queue(int /* argc */, char** /* argv */) static int dump_cpu_heap(int /* argc */, char** /* argv */) { - kprintf("core load threads\n"); + kprintf("core average_load current_load threads_load threads epoch\n"); gCoreLoadHeap.Dump(); kprintf("\n"); gCoreHighLoadHeap.Dump(); diff --git a/src/system/kernel/scheduler/scheduler_cpu.h b/src/system/kernel/scheduler/scheduler_cpu.h index f8664e7f65..364d3eac7b 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.h +++ b/src/system/kernel/scheduler/scheduler_cpu.h @@ -146,7 +146,13 @@ public: bigtime_t activeTime); inline int32 GetLoad() const; - void UpdateLoad(int32 delta); + inline uint32 LoadMeasurementEpoch() const + { return fLoadMeasurementEpoch; } + + inline void AddLoad(int32 load, uint32 epoch, + bool updateLoad); + inline uint32 RemoveLoad(int32 load, bool force); + inline void ChangeLoad(int32 delta); inline int32 StarvationCounter() const; inline int32 StarvationCounterIdle() const; @@ -162,6 +168,8 @@ public: static inline CoreEntry* GetCore(int32 cpu); private: + void _UpdateLoad(); + static void _UnassignThread(Thread* thread, void* core); @@ -184,8 +192,11 @@ private: mutable seqlock fActiveTimeLock; int32 fLoad; + int32 fCurrentLoad; + uint32 fLoadMeasurementEpoch; bool fHighLoad; bigtime_t fLastLoadUpdate; + rw_spinlock fLoadLock; friend class DebugDumper; } CACHE_LINE_ALIGN; @@ -388,6 +399,62 @@ CoreEntry::GetLoad() const } +inline void +CoreEntry::AddLoad(int32 load, uint32 epoch, bool updateLoad) +{ + SCHEDULER_ENTER_FUNCTION(); + + ASSERT(gTrackCoreLoad); + ASSERT(load >= 0 && load <= kMaxLoad); + + ReadSpinLocker locker(fLoadLock); + atomic_add(&fCurrentLoad, load); + if (fLoadMeasurementEpoch != epoch) + atomic_add(&fLoad, load); + locker.Unlock(); + + if (updateLoad) + _UpdateLoad(); +} + + +inline uint32 +CoreEntry::RemoveLoad(int32 load, bool force) +{ + SCHEDULER_ENTER_FUNCTION(); + + ASSERT(gTrackCoreLoad); + ASSERT(load >= 0 && load <= kMaxLoad); + + ReadSpinLocker locker(fLoadLock); + atomic_add(&fCurrentLoad, -load); + if (force) { + atomic_add(&fLoad, -load); + locker.Unlock(); + + _UpdateLoad(); + } + return fLoadMeasurementEpoch; +} + + +inline void +CoreEntry::ChangeLoad(int32 delta) +{ + SCHEDULER_ENTER_FUNCTION(); + + ASSERT(gTrackCoreLoad); + ASSERT(delta >= -kMaxLoad && delta <= kMaxLoad); + + ReadSpinLocker locker(fLoadLock); + atomic_add(&fCurrentLoad, delta); + atomic_add(&fLoad, delta); + locker.Unlock(); + + _UpdateLoad(); +} + + inline int32 CoreEntry::StarvationCounter() const { diff --git a/src/system/kernel/scheduler/scheduler_thread.cpp b/src/system/kernel/scheduler/scheduler_thread.cpp index 6ce5a78705..b0d69b5b41 100644 --- a/src/system/kernel/scheduler/scheduler_thread.cpp +++ b/src/system/kernel/scheduler/scheduler_thread.cpp @@ -102,6 +102,7 @@ ThreadData::Init() Thread* currentThread = thread_get_current_thread(); ThreadData* currentThreadData = currentThread->scheduler_data; fCore = currentThreadData->fCore; + fLoadMeasurementEpoch = fCore->LoadMeasurementEpoch() - 1; if (!IsRealTime()) { fPriorityPenalty = std::min(currentThreadData->fPriorityPenalty, @@ -170,9 +171,13 @@ ThreadData::ChooseCoreAndCPU(CoreEntry*& targetCore, CPUEntry*& targetCPU) ASSERT(targetCore != NULL); ASSERT(targetCPU != NULL); - if (fReady && fCore != targetCore && fCore != NULL) { - fCore->UpdateLoad(-fNeededLoad); - targetCore->UpdateLoad(fNeededLoad); + if (fCore != targetCore) { + fLoadMeasurementEpoch = targetCore->LoadMeasurementEpoch() - 1; + if (fReady) { + if (fCore != NULL) + fCore->RemoveLoad(fNeededLoad, true); + targetCore->AddLoad(fNeededLoad, fLoadMeasurementEpoch, true); + } } fCore = targetCore; @@ -262,14 +267,14 @@ void ThreadData::_ComputeNeededLoad() { SCHEDULER_ENTER_FUNCTION(); + ASSERT(!IsIdle()); int32 oldLoad = compute_load(fLastMeasureAvailableTime, fMeasureAvailableActiveTime, fNeededLoad, fMeasureAvailableTime); if (oldLoad < 0 || oldLoad == fNeededLoad) return; - int32 delta = fNeededLoad - oldLoad; - fCore->UpdateLoad(delta); + fCore->ChangeLoad(fNeededLoad - oldLoad); } diff --git a/src/system/kernel/scheduler/scheduler_thread.h b/src/system/kernel/scheduler/scheduler_thread.h index 254d8c3494..415b86eb55 100644 --- a/src/system/kernel/scheduler/scheduler_thread.h +++ b/src/system/kernel/scheduler/scheduler_thread.h @@ -131,6 +131,7 @@ private: bigtime_t fLastMeasureAvailableTime; int32 fNeededLoad; + uint32 fLoadMeasurementEpoch; CoreEntry* fCore; }; @@ -377,7 +378,7 @@ ThreadData::GoesAway() fWentSleepCountIdle = fCore->StarvationCounterIdle(); if (gTrackCoreLoad) - fCore->UpdateLoad(-fNeededLoad); + fLoadMeasurementEpoch = fCore->RemoveLoad(fNeededLoad, false); fReady = false; } @@ -389,7 +390,7 @@ ThreadData::Dies() ASSERT(fReady); if (gTrackCoreLoad) - fCore->UpdateLoad(-fNeededLoad); + fCore->RemoveLoad(fNeededLoad, true); fReady = false; } @@ -428,8 +429,10 @@ ThreadData::Enqueue() if (!fReady) { if (gTrackCoreLoad) { bigtime_t timeSlept = system_time() - fWentSleep; - fCore->UpdateLoad(fNeededLoad); - if (timeSlept > 0) { + bool updateLoad = timeSlept > 0; + + fCore->AddLoad(fNeededLoad, fLoadMeasurementEpoch, !updateLoad); + if (updateLoad) { fMeasureAvailableTime += timeSlept; _ComputeNeededLoad(); }