scheduler: Provide more stable core load statistics
Originially, core load was a sum of eastimated loads of all currently running or ready threads on a given core. Such value is changing very rapidly preventing the thread migration logic from making any reasonable decisions. This patch changes the way core load is computed to make it more stable thus improving the qualitiy of decisions made by the thread migration logic. Currently core load is a sum of estimated loads of all threads that have been ready during last load measurement interval and haven't been migrated or killed.
This commit is contained in:
Pawel Dziepak
@@ -39,7 +39,16 @@ class Scheduler::DebugDumper {
|
||||
public:
|
||||
static void DumpCPURunQueue(CPUEntry* cpu);
|
||||
static void DumpCoreRunQueue(CoreEntry* core);
|
||||
static void DumpCoreLoadHeapEntry(CoreEntry* core);
|
||||
static void DumpIdleCoresInPackage(PackageEntry* package);
|
||||
|
||||
private:
|
||||
struct CoreThreadsData {
|
||||
CoreEntry* fCore;
|
||||
int32 fLoad;
|
||||
};
|
||||
|
||||
static void _AnalyzeCoreThreads(Thread* thread, void* data);
|
||||
};
|
||||
|
||||
|
||||
@@ -357,12 +366,15 @@ CoreEntry::CoreEntry()
|
||||
fThreadCount(0),
|
||||
fActiveTime(0),
|
||||
fLoad(0),
|
||||
fCurrentLoad(0),
|
||||
fLoadMeasurementEpoch(0),
|
||||
fHighLoad(false),
|
||||
fLastLoadUpdate(0)
|
||||
{
|
||||
B_INITIALIZE_SPINLOCK(&fCPULock);
|
||||
B_INITIALIZE_SPINLOCK(&fQueueLock);
|
||||
B_INITIALIZE_SEQLOCK(&fActiveTimeLock);
|
||||
B_INITIALIZE_RW_SPINLOCK(&fLoadLock);
|
||||
}
|
||||
|
||||
|
||||
@@ -413,67 +425,6 @@ CoreEntry::Remove(ThreadData* thread)
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
CoreEntry::UpdateLoad(int32 delta)
|
||||
{
|
||||
SCHEDULER_ENTER_FUNCTION();
|
||||
|
||||
ASSERT(gTrackCoreLoad);
|
||||
|
||||
if (fCPUCount <= 0)
|
||||
return;
|
||||
|
||||
atomic_add(&fLoad, delta);
|
||||
|
||||
bigtime_t now = system_time();
|
||||
if (now < kLoadMeasureInterval + fLastLoadUpdate)
|
||||
return;
|
||||
if (!try_acquire_write_spinlock(&gCoreHeapsLock))
|
||||
return;
|
||||
WriteSpinLocker coreLocker(gCoreHeapsLock, true);
|
||||
|
||||
fLastLoadUpdate = now;
|
||||
|
||||
int32 newKey = GetLoad();
|
||||
int32 oldKey = CoreLoadHeap::GetKey(this);
|
||||
|
||||
ASSERT(oldKey >= 0);
|
||||
ASSERT(newKey >= 0);
|
||||
|
||||
if (oldKey == newKey)
|
||||
return;
|
||||
|
||||
if (newKey > kHighLoad) {
|
||||
if (!fHighLoad) {
|
||||
gCoreLoadHeap.ModifyKey(this, -1);
|
||||
ASSERT(gCoreLoadHeap.PeekMinimum() == this);
|
||||
gCoreLoadHeap.RemoveMinimum();
|
||||
|
||||
gCoreHighLoadHeap.Insert(this, newKey);
|
||||
|
||||
fHighLoad = true;
|
||||
} else
|
||||
gCoreHighLoadHeap.ModifyKey(this, newKey);
|
||||
} else if (newKey < kMediumLoad) {
|
||||
if (fHighLoad) {
|
||||
gCoreHighLoadHeap.ModifyKey(this, -1);
|
||||
ASSERT(gCoreHighLoadHeap.PeekMinimum() == this);
|
||||
gCoreHighLoadHeap.RemoveMinimum();
|
||||
|
||||
gCoreLoadHeap.Insert(this, newKey);
|
||||
|
||||
fHighLoad = false;
|
||||
} else
|
||||
gCoreLoadHeap.ModifyKey(this, newKey);
|
||||
} else {
|
||||
if (fHighLoad)
|
||||
gCoreHighLoadHeap.ModifyKey(this, newKey);
|
||||
else
|
||||
gCoreLoadHeap.ModifyKey(this, newKey);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
CoreEntry::AddCPU(CPUEntry* cpu)
|
||||
{
|
||||
@@ -484,6 +435,7 @@ CoreEntry::AddCPU(CPUEntry* cpu)
|
||||
if (fCPUCount++ == 0) {
|
||||
// core has been reenabled
|
||||
fLoad = 0;
|
||||
fCurrentLoad = 0;
|
||||
fHighLoad = false;
|
||||
gCoreLoadHeap.Insert(this, 0);
|
||||
|
||||
@@ -540,6 +492,69 @@ CoreEntry::RemoveCPU(CPUEntry* cpu, ThreadProcessing& threadPostProcessing)
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
CoreEntry::_UpdateLoad()
|
||||
{
|
||||
SCHEDULER_ENTER_FUNCTION();
|
||||
|
||||
if (fCPUCount <= 0)
|
||||
return;
|
||||
|
||||
bigtime_t now = system_time();
|
||||
if (now < kLoadMeasureInterval + fLastLoadUpdate)
|
||||
return;
|
||||
if (!try_acquire_write_spinlock(&gCoreHeapsLock))
|
||||
return;
|
||||
WriteSpinLocker coreLocker(gCoreHeapsLock, true);
|
||||
WriteSpinLocker locker(fLoadLock);
|
||||
|
||||
int32 newKey = GetLoad();
|
||||
int32 oldKey = CoreLoadHeap::GetKey(this);
|
||||
|
||||
ASSERT(oldKey >= 0);
|
||||
ASSERT(newKey >= 0);
|
||||
|
||||
ASSERT(fCurrentLoad >= 0);
|
||||
ASSERT(fLoad >= fCurrentLoad);
|
||||
|
||||
fLoad = fCurrentLoad;
|
||||
fLoadMeasurementEpoch++;
|
||||
fLastLoadUpdate = now;
|
||||
|
||||
if (oldKey == newKey)
|
||||
return;
|
||||
|
||||
if (newKey > kHighLoad) {
|
||||
if (!fHighLoad) {
|
||||
gCoreLoadHeap.ModifyKey(this, -1);
|
||||
ASSERT(gCoreLoadHeap.PeekMinimum() == this);
|
||||
gCoreLoadHeap.RemoveMinimum();
|
||||
|
||||
gCoreHighLoadHeap.Insert(this, newKey);
|
||||
|
||||
fHighLoad = true;
|
||||
} else
|
||||
gCoreHighLoadHeap.ModifyKey(this, newKey);
|
||||
} else if (newKey < kMediumLoad) {
|
||||
if (fHighLoad) {
|
||||
gCoreHighLoadHeap.ModifyKey(this, -1);
|
||||
ASSERT(gCoreHighLoadHeap.PeekMinimum() == this);
|
||||
gCoreHighLoadHeap.RemoveMinimum();
|
||||
|
||||
gCoreLoadHeap.Insert(this, newKey);
|
||||
|
||||
fHighLoad = false;
|
||||
} else
|
||||
gCoreLoadHeap.ModifyKey(this, newKey);
|
||||
} else {
|
||||
if (fHighLoad)
|
||||
gCoreHighLoadHeap.ModifyKey(this, newKey);
|
||||
else
|
||||
gCoreLoadHeap.ModifyKey(this, newKey);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/* static */ void
|
||||
CoreEntry::_UnassignThread(Thread* thread, void* data)
|
||||
{
|
||||
@@ -565,8 +580,7 @@ CoreLoadHeap::Dump()
|
||||
while (entry) {
|
||||
int32 key = GetKey(entry);
|
||||
|
||||
kprintf("%4" B_PRId32 " %3" B_PRId32 "%% %7" B_PRId32 "\n", entry->ID(),
|
||||
entry->GetLoad() / 10, entry->ThreadCount());
|
||||
DebugDumper::DumpCoreLoadHeapEntry(entry);
|
||||
|
||||
RemoveMinimum();
|
||||
sDebugCoreHeap.Insert(entry, key);
|
||||
@@ -644,6 +658,21 @@ DebugDumper::DumpCoreRunQueue(CoreEntry* core)
|
||||
}
|
||||
|
||||
|
||||
/* static */ void
|
||||
DebugDumper::DumpCoreLoadHeapEntry(CoreEntry* entry)
|
||||
{
|
||||
CoreThreadsData threadsData;
|
||||
threadsData.fCore = entry;
|
||||
threadsData.fLoad = 0;
|
||||
thread_map(DebugDumper::_AnalyzeCoreThreads, &threadsData);
|
||||
|
||||
kprintf("%4" B_PRId32 " %11" B_PRId32 "%% %11" B_PRId32 "%% %11" B_PRId32
|
||||
"%% %7" B_PRId32 " %5" B_PRIu32 "\n", entry->ID(), entry->fLoad / 10,
|
||||
entry->fCurrentLoad / 10, threadsData.fLoad, entry->ThreadCount(),
|
||||
entry->fLoadMeasurementEpoch);
|
||||
}
|
||||
|
||||
|
||||
/* static */ void
|
||||
DebugDumper::DumpIdleCoresInPackage(PackageEntry* package)
|
||||
{
|
||||
@@ -663,6 +692,15 @@ DebugDumper::DumpIdleCoresInPackage(PackageEntry* package)
|
||||
}
|
||||
|
||||
|
||||
/* static */ void
|
||||
DebugDumper::_AnalyzeCoreThreads(Thread* thread, void* data)
|
||||
{
|
||||
CoreThreadsData* threadsData = static_cast<CoreThreadsData*>(data);
|
||||
if (thread->scheduler_data->Core() == threadsData->fCore)
|
||||
threadsData->fLoad += thread->scheduler_data->GetLoad();
|
||||
}
|
||||
|
||||
|
||||
static int
|
||||
dump_run_queue(int /* argc */, char** /* argv */)
|
||||
{
|
||||
@@ -684,7 +722,7 @@ dump_run_queue(int /* argc */, char** /* argv */)
|
||||
static int
|
||||
dump_cpu_heap(int /* argc */, char** /* argv */)
|
||||
{
|
||||
kprintf("core load threads\n");
|
||||
kprintf("core average_load current_load threads_load threads epoch\n");
|
||||
gCoreLoadHeap.Dump();
|
||||
kprintf("\n");
|
||||
gCoreHighLoadHeap.Dump();
|
||||
|
||||
@@ -146,7 +146,13 @@ public:
|
||||
bigtime_t activeTime);
|
||||
|
||||
inline int32 GetLoad() const;
|
||||
void UpdateLoad(int32 delta);
|
||||
inline uint32 LoadMeasurementEpoch() const
|
||||
{ return fLoadMeasurementEpoch; }
|
||||
|
||||
inline void AddLoad(int32 load, uint32 epoch,
|
||||
bool updateLoad);
|
||||
inline uint32 RemoveLoad(int32 load, bool force);
|
||||
inline void ChangeLoad(int32 delta);
|
||||
|
||||
inline int32 StarvationCounter() const;
|
||||
inline int32 StarvationCounterIdle() const;
|
||||
@@ -162,6 +168,8 @@ public:
|
||||
static inline CoreEntry* GetCore(int32 cpu);
|
||||
|
||||
private:
|
||||
void _UpdateLoad();
|
||||
|
||||
static void _UnassignThread(Thread* thread,
|
||||
void* core);
|
||||
|
||||
@@ -184,8 +192,11 @@ private:
|
||||
mutable seqlock fActiveTimeLock;
|
||||
|
||||
int32 fLoad;
|
||||
int32 fCurrentLoad;
|
||||
uint32 fLoadMeasurementEpoch;
|
||||
bool fHighLoad;
|
||||
bigtime_t fLastLoadUpdate;
|
||||
rw_spinlock fLoadLock;
|
||||
|
||||
friend class DebugDumper;
|
||||
} CACHE_LINE_ALIGN;
|
||||
@@ -388,6 +399,62 @@ CoreEntry::GetLoad() const
|
||||
}
|
||||
|
||||
|
||||
inline void
|
||||
CoreEntry::AddLoad(int32 load, uint32 epoch, bool updateLoad)
|
||||
{
|
||||
SCHEDULER_ENTER_FUNCTION();
|
||||
|
||||
ASSERT(gTrackCoreLoad);
|
||||
ASSERT(load >= 0 && load <= kMaxLoad);
|
||||
|
||||
ReadSpinLocker locker(fLoadLock);
|
||||
atomic_add(&fCurrentLoad, load);
|
||||
if (fLoadMeasurementEpoch != epoch)
|
||||
atomic_add(&fLoad, load);
|
||||
locker.Unlock();
|
||||
|
||||
if (updateLoad)
|
||||
_UpdateLoad();
|
||||
}
|
||||
|
||||
|
||||
inline uint32
|
||||
CoreEntry::RemoveLoad(int32 load, bool force)
|
||||
{
|
||||
SCHEDULER_ENTER_FUNCTION();
|
||||
|
||||
ASSERT(gTrackCoreLoad);
|
||||
ASSERT(load >= 0 && load <= kMaxLoad);
|
||||
|
||||
ReadSpinLocker locker(fLoadLock);
|
||||
atomic_add(&fCurrentLoad, -load);
|
||||
if (force) {
|
||||
atomic_add(&fLoad, -load);
|
||||
locker.Unlock();
|
||||
|
||||
_UpdateLoad();
|
||||
}
|
||||
return fLoadMeasurementEpoch;
|
||||
}
|
||||
|
||||
|
||||
inline void
|
||||
CoreEntry::ChangeLoad(int32 delta)
|
||||
{
|
||||
SCHEDULER_ENTER_FUNCTION();
|
||||
|
||||
ASSERT(gTrackCoreLoad);
|
||||
ASSERT(delta >= -kMaxLoad && delta <= kMaxLoad);
|
||||
|
||||
ReadSpinLocker locker(fLoadLock);
|
||||
atomic_add(&fCurrentLoad, delta);
|
||||
atomic_add(&fLoad, delta);
|
||||
locker.Unlock();
|
||||
|
||||
_UpdateLoad();
|
||||
}
|
||||
|
||||
|
||||
inline int32
|
||||
CoreEntry::StarvationCounter() const
|
||||
{
|
||||
|
||||
@@ -102,6 +102,7 @@ ThreadData::Init()
|
||||
Thread* currentThread = thread_get_current_thread();
|
||||
ThreadData* currentThreadData = currentThread->scheduler_data;
|
||||
fCore = currentThreadData->fCore;
|
||||
fLoadMeasurementEpoch = fCore->LoadMeasurementEpoch() - 1;
|
||||
|
||||
if (!IsRealTime()) {
|
||||
fPriorityPenalty = std::min(currentThreadData->fPriorityPenalty,
|
||||
@@ -170,9 +171,13 @@ ThreadData::ChooseCoreAndCPU(CoreEntry*& targetCore, CPUEntry*& targetCPU)
|
||||
ASSERT(targetCore != NULL);
|
||||
ASSERT(targetCPU != NULL);
|
||||
|
||||
if (fReady && fCore != targetCore && fCore != NULL) {
|
||||
fCore->UpdateLoad(-fNeededLoad);
|
||||
targetCore->UpdateLoad(fNeededLoad);
|
||||
if (fCore != targetCore) {
|
||||
fLoadMeasurementEpoch = targetCore->LoadMeasurementEpoch() - 1;
|
||||
if (fReady) {
|
||||
if (fCore != NULL)
|
||||
fCore->RemoveLoad(fNeededLoad, true);
|
||||
targetCore->AddLoad(fNeededLoad, fLoadMeasurementEpoch, true);
|
||||
}
|
||||
}
|
||||
|
||||
fCore = targetCore;
|
||||
@@ -262,14 +267,14 @@ void
|
||||
ThreadData::_ComputeNeededLoad()
|
||||
{
|
||||
SCHEDULER_ENTER_FUNCTION();
|
||||
ASSERT(!IsIdle());
|
||||
|
||||
int32 oldLoad = compute_load(fLastMeasureAvailableTime,
|
||||
fMeasureAvailableActiveTime, fNeededLoad, fMeasureAvailableTime);
|
||||
if (oldLoad < 0 || oldLoad == fNeededLoad)
|
||||
return;
|
||||
|
||||
int32 delta = fNeededLoad - oldLoad;
|
||||
fCore->UpdateLoad(delta);
|
||||
fCore->ChangeLoad(fNeededLoad - oldLoad);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -131,6 +131,7 @@ private:
|
||||
bigtime_t fLastMeasureAvailableTime;
|
||||
|
||||
int32 fNeededLoad;
|
||||
uint32 fLoadMeasurementEpoch;
|
||||
|
||||
CoreEntry* fCore;
|
||||
};
|
||||
@@ -377,7 +378,7 @@ ThreadData::GoesAway()
|
||||
fWentSleepCountIdle = fCore->StarvationCounterIdle();
|
||||
|
||||
if (gTrackCoreLoad)
|
||||
fCore->UpdateLoad(-fNeededLoad);
|
||||
fLoadMeasurementEpoch = fCore->RemoveLoad(fNeededLoad, false);
|
||||
fReady = false;
|
||||
}
|
||||
|
||||
@@ -389,7 +390,7 @@ ThreadData::Dies()
|
||||
|
||||
ASSERT(fReady);
|
||||
if (gTrackCoreLoad)
|
||||
fCore->UpdateLoad(-fNeededLoad);
|
||||
fCore->RemoveLoad(fNeededLoad, true);
|
||||
fReady = false;
|
||||
}
|
||||
|
||||
@@ -428,8 +429,10 @@ ThreadData::Enqueue()
|
||||
if (!fReady) {
|
||||
if (gTrackCoreLoad) {
|
||||
bigtime_t timeSlept = system_time() - fWentSleep;
|
||||
fCore->UpdateLoad(fNeededLoad);
|
||||
if (timeSlept > 0) {
|
||||
bool updateLoad = timeSlept > 0;
|
||||
|
||||
fCore->AddLoad(fNeededLoad, fLoadMeasurementEpoch, !updateLoad);
|
||||
if (updateLoad) {
|
||||
fMeasureAvailableTime += timeSlept;
|
||||
_ComputeNeededLoad();
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user