scheduler: Use single ended heap for CPU heap

src/system/kernel/scheduler/low_latency.cpp

@@ -138,7 +138,7 @@ rebalance_irqs(bool idle)
 		other = gCoreHighLoadHeap.PeekMinimum();
 	coreLocker.Unlock();
 
-	int32 newCPU = other->CPUHeap()->PeekMinimum()->ID();
+	int32 newCPU = other->CPUHeap()->PeekRoot()->ID();
 
 	ASSERT(other != NULL);
 

src/system/kernel/scheduler/power_saving.cpp

@@ -174,7 +174,7 @@ pack_irqs()
 		irq_assignment* irq = (irq_assignment*)list_get_first_item(&cpu->irqs);
 		locker.Unlock();
 
-		int32 newCPU = smallTaskCore->CPUHeap()->PeekMinimum()->ID();
+		int32 newCPU = smallTaskCore->CPUHeap()->PeekRoot()->ID();
 
 		if (newCPU != cpu->cpu_num)
 			assign_io_interrupt_to_cpu(irq->irq, newCPU);
@@ -219,7 +219,7 @@ rebalance_irqs(bool idle)
 	coreLocker.Unlock();
 	if (other == NULL)
 		return;
-	int32 newCPU = other->CPUHeap()->PeekMinimum()->ID();
+	int32 newCPU = other->CPUHeap()->PeekRoot()->ID();
 
 	CoreEntry* core = CoreEntry::GetCore(smp_get_current_cpu());
 	if (other == core)

src/system/kernel/scheduler/scheduler_cpu.cpp

@@ -146,22 +146,15 @@ CPUEntry::UpdatePriority(int32 priority)
 	if (gCPU[fCPUNumber].disabled)
 		return;
 
-	CPUPriorityHeap* cpuHeap = fCore->CPUHeap();
+	int32 oldPriority = CPUPriorityHeap::GetKey(this);
-	int32 corePriority = CPUPriorityHeap::GetKey(cpuHeap->PeekMaximum());
+	if (oldPriority == priority)
-	cpuHeap->ModifyKey(this, priority);
-
-	if (gSingleCore)
 		return;
+	fCore->CPUHeap()->ModifyKey(this, priority);
 
-	int32 maxPriority = CPUPriorityHeap::GetKey(cpuHeap->PeekMaximum());
+	if (oldPriority == B_IDLE_PRIORITY)
-	if (corePriority == maxPriority)
+		fCore->CPUWakesUp(this);
-		return;
+	else if (priority == B_IDLE_PRIORITY)
-
+		fCore->CPUGoesIdle(this);
-	PackageEntry* packageEntry = fCore->Package();
-	if (maxPriority == B_IDLE_PRIORITY)
-		packageEntry->CoreGoesIdle(fCore);
-	else if (corePriority == B_IDLE_PRIORITY)
-		packageEntry->CoreWakesUp(fCore);
 }
 
 
@@ -292,7 +285,7 @@ CPUEntry::_RequestPerformanceLevel(ThreadData* threadData)
 
 CPUPriorityHeap::CPUPriorityHeap(int32 cpuCount)
 	:
-	MinMaxHeap<CPUEntry, int32>(cpuCount)
+	Heap<CPUEntry, int32>(cpuCount)
 {
 }
 
@@ -301,25 +294,25 @@ void
 CPUPriorityHeap::Dump()
 {
 	kprintf("cpu priority load\n");
-	CPUEntry* entry = PeekMinimum();
+	CPUEntry* entry = PeekRoot();
 	while (entry) {
 		int32 cpu = entry->ID();
 		int32 key = GetKey(entry);
 		kprintf("%3" B_PRId32 " %8" B_PRId32 " %3" B_PRId32 "%%\n", cpu, key,
 			entry->GetLoad() / 10);
 
-		RemoveMinimum();
+		RemoveRoot();
 		sDebugCPUHeap.Insert(entry, key);
 
-		entry = PeekMinimum();
+		entry = PeekRoot();
 	}
 
-	entry = sDebugCPUHeap.PeekMinimum();
+	entry = sDebugCPUHeap.PeekRoot();
 	while (entry) {
 		int32 key = GetKey(entry);
-		sDebugCPUHeap.RemoveMinimum();
+		sDebugCPUHeap.RemoveRoot();
 		Insert(entry, key);
-		entry = sDebugCPUHeap.PeekMinimum();
+		entry = sDebugCPUHeap.PeekRoot();
 	}
 }
 
@@ -327,6 +320,7 @@ CPUPriorityHeap::Dump()
 CoreEntry::CoreEntry()
 	:
 	fCPUCount(0),
+	fCPUIdleCount(0),
 	fStarvationCounter(0),
 	fThreadCount(0),
 	fActiveTime(0),
@@ -449,10 +443,33 @@ CoreEntry::UpdateLoad(int32 delta)
 }
 
 
+inline void
+CoreEntry::CPUGoesIdle(CPUEntry* /* cpu */)
+{
+	ASSERT(fCPUIdleCount < fCPUCount);
+
+	if (++fCPUIdleCount == fCPUCount)
+		fPackage->CoreGoesIdle(this);
+}
+
+
+inline void
+CoreEntry::CPUWakesUp(CPUEntry* /* cpu */)
+{
+	ASSERT(fCPUIdleCount > 0);
+
+	if (fCPUIdleCount-- == fCPUCount)
+		fPackage->CoreWakesUp(this);
+}
+
+
 void
 CoreEntry::AddCPU(CPUEntry* cpu)
 {
 	ASSERT(fCPUCount >= 0);
+	ASSERT(fCPUIdleCount >= 0);
+
+	fCPUIdleCount++;
 	if (fCPUCount++ == 0) {
 		// core has been reenabled
 		fLoad = 0;
@@ -470,6 +487,9 @@ void
 CoreEntry::RemoveCPU(CPUEntry* cpu, ThreadProcessing& threadPostProcessing)
 {
 	ASSERT(fCPUCount > 0);
+	ASSERT(fCPUIdleCount > 0);
+
+	fCPUIdleCount--;
 	if (--fCPUCount == 0) {
 		// core has been disabled
 		if (fHighLoad) {
@@ -499,9 +519,9 @@ CoreEntry::RemoveCPU(CPUEntry* cpu, ThreadProcessing& threadPostProcessing)
 		fThreadCount = 0;
 	}
 
-	fCPUHeap.ModifyKey(cpu, THREAD_MAX_SET_PRIORITY + 1);
+	fCPUHeap.ModifyKey(cpu, -1);
-	ASSERT(fCPUHeap.PeekMaximum() == cpu);
+	ASSERT(fCPUHeap.PeekRoot() == cpu);
-	fCPUHeap.RemoveMaximum();
+	fCPUHeap.RemoveRoot();
 
 	ASSERT(cpu->GetLoad() >= 0 && cpu->GetLoad() <= kMaxLoad);
 	fLoad -= cpu->GetLoad();

src/system/kernel/scheduler/scheduler_cpu.h

@@ -11,6 +11,7 @@
 #include <thread.h>
 #include <util/AutoLock.h>
 #include <util/MinMaxHeap.h>
+#include <util/Heap.h>
 
 #include <cpufreq.h>
 
@@ -41,7 +42,7 @@ public:
 						void			Dump() const;
 };
 
-class CPUEntry : public MinMaxHeapLinkImpl<CPUEntry, int32> {
+class CPUEntry : public HeapLinkImpl<CPUEntry, int32> {
 public:
 										CPUEntry();
 
@@ -99,7 +100,7 @@ private:
 						friend class DebugDumper;
 } CACHE_LINE_ALIGN;
 
-class CPUPriorityHeap : public MinMaxHeap<CPUEntry, int32> {
+class CPUPriorityHeap : public Heap<CPUEntry, int32> {
 public:
 										CPUPriorityHeap() { }
 										CPUPriorityHeap(int32 cpuCount);
@@ -146,6 +147,9 @@ public:
 
 	inline				int32			StarvationCounter() const;
 
+	inline				void			CPUGoesIdle(CPUEntry* cpu);
+	inline				void			CPUWakesUp(CPUEntry* cpu);
+
 						void			AddCPU(CPUEntry* cpu);
 						void			RemoveCPU(CPUEntry* cpu,
 											ThreadProcessing&
@@ -161,6 +165,7 @@ private:
 						PackageEntry*	fPackage;
 
 						int32			fCPUCount;
+						int32			fCPUIdleCount;
 						CPUPriorityHeap	fCPUHeap;
 						spinlock		fCPULock;
 

src/system/kernel/scheduler/scheduler_thread.cpp

@@ -205,7 +205,7 @@ ThreadData::_ChooseCPU(CoreEntry* core, bool& rescheduleNeeded) const
 	}
 
 	CoreCPUHeapLocker _(core);
-	CPUEntry* cpu = core->CPUHeap()->PeekMinimum();
+	CPUEntry* cpu = core->CPUHeap()->PeekRoot();
 	ASSERT(cpu != NULL);
 
 	if (CPUPriorityHeap::GetKey(cpu) < threadPriority) {