From ef8e55a1d09185c714afac7b5d00f28064af3428 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Sat, 28 Dec 2013 21:47:35 +0100 Subject: [PATCH] scheduler: Use single ended heap for CPU heap --- src/system/kernel/scheduler/low_latency.cpp | 2 +- src/system/kernel/scheduler/power_saving.cpp | 4 +- src/system/kernel/scheduler/scheduler_cpu.cpp | 68 ++++++++++++------- src/system/kernel/scheduler/scheduler_cpu.h | 9 ++- .../kernel/scheduler/scheduler_thread.cpp | 2 +- 5 files changed, 55 insertions(+), 30 deletions(-) diff --git a/src/system/kernel/scheduler/low_latency.cpp b/src/system/kernel/scheduler/low_latency.cpp index f76673be88..1e0c79d267 100644 --- a/src/system/kernel/scheduler/low_latency.cpp +++ b/src/system/kernel/scheduler/low_latency.cpp @@ -138,7 +138,7 @@ rebalance_irqs(bool idle) other = gCoreHighLoadHeap.PeekMinimum(); coreLocker.Unlock(); - int32 newCPU = other->CPUHeap()->PeekMinimum()->ID(); + int32 newCPU = other->CPUHeap()->PeekRoot()->ID(); ASSERT(other != NULL); diff --git a/src/system/kernel/scheduler/power_saving.cpp b/src/system/kernel/scheduler/power_saving.cpp index c1da765748..4239fd150b 100644 --- a/src/system/kernel/scheduler/power_saving.cpp +++ b/src/system/kernel/scheduler/power_saving.cpp @@ -174,7 +174,7 @@ pack_irqs() irq_assignment* irq = (irq_assignment*)list_get_first_item(&cpu->irqs); locker.Unlock(); - int32 newCPU = smallTaskCore->CPUHeap()->PeekMinimum()->ID(); + int32 newCPU = smallTaskCore->CPUHeap()->PeekRoot()->ID(); if (newCPU != cpu->cpu_num) assign_io_interrupt_to_cpu(irq->irq, newCPU); @@ -219,7 +219,7 @@ rebalance_irqs(bool idle) coreLocker.Unlock(); if (other == NULL) return; - int32 newCPU = other->CPUHeap()->PeekMinimum()->ID(); + int32 newCPU = other->CPUHeap()->PeekRoot()->ID(); CoreEntry* core = CoreEntry::GetCore(smp_get_current_cpu()); if (other == core) diff --git a/src/system/kernel/scheduler/scheduler_cpu.cpp b/src/system/kernel/scheduler/scheduler_cpu.cpp index ef6c9bc75b..753f9b30bd 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.cpp +++ b/src/system/kernel/scheduler/scheduler_cpu.cpp @@ -146,22 +146,15 @@ CPUEntry::UpdatePriority(int32 priority) if (gCPU[fCPUNumber].disabled) return; - CPUPriorityHeap* cpuHeap = fCore->CPUHeap(); - int32 corePriority = CPUPriorityHeap::GetKey(cpuHeap->PeekMaximum()); - cpuHeap->ModifyKey(this, priority); - - if (gSingleCore) + int32 oldPriority = CPUPriorityHeap::GetKey(this); + if (oldPriority == priority) return; + fCore->CPUHeap()->ModifyKey(this, priority); - int32 maxPriority = CPUPriorityHeap::GetKey(cpuHeap->PeekMaximum()); - if (corePriority == maxPriority) - return; - - PackageEntry* packageEntry = fCore->Package(); - if (maxPriority == B_IDLE_PRIORITY) - packageEntry->CoreGoesIdle(fCore); - else if (corePriority == B_IDLE_PRIORITY) - packageEntry->CoreWakesUp(fCore); + if (oldPriority == B_IDLE_PRIORITY) + fCore->CPUWakesUp(this); + else if (priority == B_IDLE_PRIORITY) + fCore->CPUGoesIdle(this); } @@ -292,7 +285,7 @@ CPUEntry::_RequestPerformanceLevel(ThreadData* threadData) CPUPriorityHeap::CPUPriorityHeap(int32 cpuCount) : - MinMaxHeap(cpuCount) + Heap(cpuCount) { } @@ -301,25 +294,25 @@ void CPUPriorityHeap::Dump() { kprintf("cpu priority load\n"); - CPUEntry* entry = PeekMinimum(); + CPUEntry* entry = PeekRoot(); while (entry) { int32 cpu = entry->ID(); int32 key = GetKey(entry); kprintf("%3" B_PRId32 " %8" B_PRId32 " %3" B_PRId32 "%%\n", cpu, key, entry->GetLoad() / 10); - RemoveMinimum(); + RemoveRoot(); sDebugCPUHeap.Insert(entry, key); - entry = PeekMinimum(); + entry = PeekRoot(); } - entry = sDebugCPUHeap.PeekMinimum(); + entry = sDebugCPUHeap.PeekRoot(); while (entry) { int32 key = GetKey(entry); - sDebugCPUHeap.RemoveMinimum(); + sDebugCPUHeap.RemoveRoot(); Insert(entry, key); - entry = sDebugCPUHeap.PeekMinimum(); + entry = sDebugCPUHeap.PeekRoot(); } } @@ -327,6 +320,7 @@ CPUPriorityHeap::Dump() CoreEntry::CoreEntry() : fCPUCount(0), + fCPUIdleCount(0), fStarvationCounter(0), fThreadCount(0), fActiveTime(0), @@ -449,10 +443,33 @@ CoreEntry::UpdateLoad(int32 delta) } +inline void +CoreEntry::CPUGoesIdle(CPUEntry* /* cpu */) +{ + ASSERT(fCPUIdleCount < fCPUCount); + + if (++fCPUIdleCount == fCPUCount) + fPackage->CoreGoesIdle(this); +} + + +inline void +CoreEntry::CPUWakesUp(CPUEntry* /* cpu */) +{ + ASSERT(fCPUIdleCount > 0); + + if (fCPUIdleCount-- == fCPUCount) + fPackage->CoreWakesUp(this); +} + + void CoreEntry::AddCPU(CPUEntry* cpu) { ASSERT(fCPUCount >= 0); + ASSERT(fCPUIdleCount >= 0); + + fCPUIdleCount++; if (fCPUCount++ == 0) { // core has been reenabled fLoad = 0; @@ -470,6 +487,9 @@ void CoreEntry::RemoveCPU(CPUEntry* cpu, ThreadProcessing& threadPostProcessing) { ASSERT(fCPUCount > 0); + ASSERT(fCPUIdleCount > 0); + + fCPUIdleCount--; if (--fCPUCount == 0) { // core has been disabled if (fHighLoad) { @@ -499,9 +519,9 @@ CoreEntry::RemoveCPU(CPUEntry* cpu, ThreadProcessing& threadPostProcessing) fThreadCount = 0; } - fCPUHeap.ModifyKey(cpu, THREAD_MAX_SET_PRIORITY + 1); - ASSERT(fCPUHeap.PeekMaximum() == cpu); - fCPUHeap.RemoveMaximum(); + fCPUHeap.ModifyKey(cpu, -1); + ASSERT(fCPUHeap.PeekRoot() == cpu); + fCPUHeap.RemoveRoot(); ASSERT(cpu->GetLoad() >= 0 && cpu->GetLoad() <= kMaxLoad); fLoad -= cpu->GetLoad(); diff --git a/src/system/kernel/scheduler/scheduler_cpu.h b/src/system/kernel/scheduler/scheduler_cpu.h index 9115f67fbd..2ea2774068 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.h +++ b/src/system/kernel/scheduler/scheduler_cpu.h @@ -11,6 +11,7 @@ #include #include #include +#include #include @@ -41,7 +42,7 @@ public: void Dump() const; }; -class CPUEntry : public MinMaxHeapLinkImpl { +class CPUEntry : public HeapLinkImpl { public: CPUEntry(); @@ -99,7 +100,7 @@ private: friend class DebugDumper; } CACHE_LINE_ALIGN; -class CPUPriorityHeap : public MinMaxHeap { +class CPUPriorityHeap : public Heap { public: CPUPriorityHeap() { } CPUPriorityHeap(int32 cpuCount); @@ -146,6 +147,9 @@ public: inline int32 StarvationCounter() const; + inline void CPUGoesIdle(CPUEntry* cpu); + inline void CPUWakesUp(CPUEntry* cpu); + void AddCPU(CPUEntry* cpu); void RemoveCPU(CPUEntry* cpu, ThreadProcessing& @@ -161,6 +165,7 @@ private: PackageEntry* fPackage; int32 fCPUCount; + int32 fCPUIdleCount; CPUPriorityHeap fCPUHeap; spinlock fCPULock; diff --git a/src/system/kernel/scheduler/scheduler_thread.cpp b/src/system/kernel/scheduler/scheduler_thread.cpp index de6a64bb75..4db6516a04 100644 --- a/src/system/kernel/scheduler/scheduler_thread.cpp +++ b/src/system/kernel/scheduler/scheduler_thread.cpp @@ -205,7 +205,7 @@ ThreadData::_ChooseCPU(CoreEntry* core, bool& rescheduleNeeded) const } CoreCPUHeapLocker _(core); - CPUEntry* cpu = core->CPUHeap()->PeekMinimum(); + CPUEntry* cpu = core->CPUHeap()->PeekRoot(); ASSERT(cpu != NULL); if (CPUPriorityHeap::GetKey(cpu) < threadPriority) {