scheduler: Improve recognition of CPU bound threads

src/system/kernel/scheduler/low_latency.cpp

@@ -160,13 +160,11 @@ rebalance_irqs(bool idle)
 scheduler_mode_operations gSchedulerLowLatencyMode = {
 	"low latency",
 
-	true,
-
 	2000,
-	700,
+	100,
-	{ 2, 30 },
+	{ 2, 25 },
 
-	60000,
+	50000,
 
 	switch_to_mode,
 	set_cpu_enabled,

src/system/kernel/scheduler/power_saving.cpp

@@ -236,8 +236,6 @@ rebalance_irqs(bool idle)
 scheduler_mode_operations gSchedulerPowerSavingMode = {
 	"power saving",
 
-	false,
-
 	3000,
 	1000,
 	{ 3, 60 },

src/system/kernel/scheduler/scheduler.cpp

@@ -404,8 +404,6 @@ reschedule(int32 nextState)
 
 			if (oldThreadData->HasQuantumEnded(oldThread->cpu->preempted,
 					oldThread->has_yielded)) {
-				oldThreadData->IncreasePenalty();
-
 				TRACE("enqueueing thread %ld into run queue priority = %ld\n",
 					oldThread->id, oldThreadData->GetEffectivePriority());
 				putOldThreadAtBack = true;
@@ -419,7 +417,6 @@ reschedule(int32 nextState)
 		case THREAD_STATE_FREE_ON_RESCHED:
 			break;
 		default:
-			oldThreadData->IncreasePenalty();
 			oldThreadData->GoesAway();
 			TRACE("not enqueueing thread %ld into run queue next_state = %ld\n",
 				oldThread->id, nextState);

src/system/kernel/scheduler/scheduler_cpu.cpp

@@ -21,7 +21,6 @@ public:
 	static	void		DumpCPURunQueue(CPUEntry* cpu);
 	static	void		DumpCoreRunQueue(CoreEntry* core);
 	static	void		DumpIdleCoresInPackage(PackageEntry* package);
-
 };
 
 
@@ -43,7 +42,8 @@ ThreadRunQueue::Dump() const
 
 			kprintf("%p  %-7" B_PRId32 " %-8" B_PRId32 " %-8" B_PRId32 " %s\n",
 				thread, thread->id, thread->priority,
-				threadData->GetEffectivePriority(), thread->name);
+				thread->priority - threadData->GetEffectivePriority(),
+				thread->name);
 		}
 	}
 }
@@ -159,9 +159,6 @@ CPUEntry::UpdatePriority(int32 priority)
 		return;
 	fCore->CPUHeap()->ModifyKey(this, priority);
 
-	if (gSingleCore)
-		return;
-
 	if (oldPriority == B_IDLE_PRIORITY)
 		fCore->CPUWakesUp(this);
 	else if (priority == B_IDLE_PRIORITY)
@@ -345,6 +342,7 @@ CoreEntry::CoreEntry()
 	fCPUCount(0),
 	fCPUIdleCount(0),
 	fStarvationCounter(0),
+	fStarvationCounterIdle(0),
 	fThreadCount(0),
 	fActiveTime(0),
 	fLoad(0),
@@ -380,8 +378,6 @@ CoreEntry::PushBack(ThreadData* thread, int32 priority)
 	SCHEDULER_ENTER_FUNCTION();
 
 	fRunQueue.PushBack(thread, priority);
-	fThreadList.Insert(thread);
-
 	atomic_add(&fThreadCount, 1);
 }
 
@@ -393,12 +389,13 @@ CoreEntry::Remove(ThreadData* thread)
 
 	ASSERT(thread->IsEnqueued());
 	thread->SetDequeued();
-	if (thread_is_idle_thread(thread->GetThread())
+
-		|| fThreadList.Head() == thread) {
+	ASSERT(!thread_is_idle_thread(thread->GetThread()));
+	if (thread->GetEffectivePriority() == B_LOWEST_ACTIVE_PRIORITY
+		|| thread->IsCPUBound()) {
 		atomic_add(&fStarvationCounter, 1);
 	}
-	if (thread->WentSleepCount() == 0)
+
-		fThreadList.Remove(thread);
 	fRunQueue.Remove(thread);
 	atomic_add(&fThreadCount, -1);
 }
@@ -548,8 +545,8 @@ CoreLoadHeap::Dump()
 	CoreEntry* entry = PeekMinimum();
 	while (entry) {
 		int32 key = GetKey(entry);
-		kprintf("%4" B_PRId32 " %3" B_PRId32 "%%\n", entry->ID(),
+		kprintf("%4" B_PRId32 " %3" B_PRId32 "%% %7" B_PRId32 "\n", entry->ID(),
-			entry->GetLoad() / 10);
+			entry->GetLoad() / 10, entry->ThreadCount());
 
 		RemoveMinimum();
 		sDebugCoreHeap.Insert(entry, key);
@@ -647,7 +644,7 @@ DebugDumper::DumpIdleCoresInPackage(PackageEntry* package)
 
 
 static int
-dump_run_queue(int argc, char **argv)
+dump_run_queue(int /* argc */, char** /* argv */)
 {
 	int32 cpuCount = smp_get_num_cpus();
 	int32 coreCount = gCoreCount;
@@ -665,9 +662,9 @@ dump_run_queue(int argc, char **argv)
 
 
 static int
-dump_cpu_heap(int argc, char** argv)
+dump_cpu_heap(int /* argc */, char** /* argv */)
 {
-	kprintf("core load\n");
+	kprintf("core load threads\n");
 	gCoreLoadHeap.Dump();
 	kprintf("\n");
 	gCoreHighLoadHeap.Dump();
@@ -685,7 +682,7 @@ dump_cpu_heap(int argc, char** argv)
 
 
 static int
-dump_idle_cores(int argc, char** argv)
+dump_idle_cores(int /* argc */, char** /* argv */)
 {
 	kprintf("Idle packages:\n");
 	IdlePackageList::ReverseIterator idleIterator

src/system/kernel/scheduler/scheduler_cpu.h

@@ -150,6 +150,7 @@ public:
 						void			UpdateLoad(int32 delta);
 
 	inline				int32			StarvationCounter() const;
+	inline				int32			StarvationCounterIdle() const;
 
 	inline				void			CPUGoesIdle(CPUEntry* cpu);
 	inline				void			CPUWakesUp(CPUEntry* cpu);
@@ -174,7 +175,7 @@ private:
 						spinlock		fCPULock;
 
 						int32			fStarvationCounter;
-						DoublyLinkedList<ThreadData>	fThreadList;
+						int32			fStarvationCounterIdle;
 
 						int32			fThreadCount;
 						ThreadRunQueue	fRunQueue;
@@ -387,6 +388,14 @@ CoreEntry::StarvationCounter() const
 }
 
 
+inline int32
+CoreEntry::StarvationCounterIdle() const
+{
+	SCHEDULER_ENTER_FUNCTION();
+	return fStarvationCounterIdle;
+}
+
+
 /* PackageEntry::CoreGoesIdle and PackageEntry::CoreWakesUp have to be defined
    before CoreEntry::CPUGoesIdle and CoreEntry::CPUWakesUp. If they weren't
    GCC2 wouldn't inline them as, apparently, it doesn't do enough optimization
@@ -437,8 +446,13 @@ PackageEntry::CoreWakesUp(CoreEntry* core)
 inline void
 CoreEntry::CPUGoesIdle(CPUEntry* /* cpu */)
 {
-	ASSERT(fCPUIdleCount < fCPUCount);
+	atomic_add(&fStarvationCounter, 1);
+	atomic_add(&fStarvationCounterIdle, 1);
 
+	if (gSingleCore)
+		return;
+
+	ASSERT(fCPUIdleCount < fCPUCount);
 	if (++fCPUIdleCount == fCPUCount)
 		fPackage->CoreGoesIdle(this);
 }
@@ -447,8 +461,10 @@ CoreEntry::CPUGoesIdle(CPUEntry* /* cpu */)
 inline void
 CoreEntry::CPUWakesUp(CPUEntry* /* cpu */)
 {
-	ASSERT(fCPUIdleCount > 0);
+	if (gSingleCore)
+		return;
 
+	ASSERT(fCPUIdleCount > 0);
 	if (fCPUIdleCount-- == fCPUCount)
 		fPackage->CoreWakesUp(this);
 }

src/system/kernel/scheduler/scheduler_modes.h

@@ -13,8 +13,6 @@
 struct scheduler_mode_operations {
 	const char*				name;
 
-	bool					avoid_boost;
-
 	bigtime_t				base_quantum;
 	bigtime_t				minimal_quantum;
 	bigtime_t				quantum_multipliers[2];

src/system/kernel/scheduler/scheduler_thread.cpp

@@ -19,6 +19,9 @@ ThreadData::_InitBase()
 	fAdditionalPenalty = 0;
 	fEffectivePriority = fThread->priority;
 
+	fReceivedPenalty = false;
+	fHasSlept = false;
+
 	fTimeLeft = 0;
 	fStolenTime = 0;
 
@@ -28,7 +31,8 @@ ThreadData::_InitBase()
 
 	fWentSleep = 0;
 	fWentSleepActive = 0;
-	fWentSleepCount = -1;
+	fWentSleepCount = 0;
+	fWentSleepCountIdle = 0;
 
 	fEnqueued = false;
 }
@@ -124,7 +128,25 @@ ThreadData::Dump() const
 		additionalPenalty = fAdditionalPenalty % kMinimalPriority;
 	kprintf("\tadditional_penalty:\t%" B_PRId32 " (%" B_PRId32 ")\n",
 		additionalPenalty, fAdditionalPenalty);
-	kprintf("\tstolen_time:\t\t%" B_PRId64 "\n", fStolenTime);
+	kprintf("\teffective_priority:\t%" B_PRId32 "\n", GetEffectivePriority());
+
+	kprintf("\treceived_penalty:\t%s\n", fReceivedPenalty ? "true" : "false");
+	kprintf("\thas_slept:\t\t%s\n", fHasSlept ? "true" : "false");
+
+	bigtime_t quantum = _GetBaseQuantum();
+	if (fThread->priority < B_FIRST_REAL_TIME_PRIORITY) {
+		int32 threadCount = (fCore->ThreadCount() + 1) / fCore->CPUCount();
+		threadCount = max_c(threadCount, 1);
+
+		quantum
+			= std::min(gCurrentMode->maximum_latency / threadCount, quantum);
+		quantum = std::max(quantum,	gCurrentMode->minimal_quantum);
+	}
+	kprintf("\ttime_left:\t\t%" B_PRId64 " us (quantum: %" B_PRId64 " us)\n",
+		fTimeLeft, quantum);
+
+	kprintf("\tstolen_time:\t\t%" B_PRId64 " us\n", fStolenTime);
+	kprintf("\tquantum_start:\t\t%" B_PRId64 " us\n", fQuantumStart);
 	kprintf("\tload:\t\t\t%" B_PRId32 "%%\n", fLoad / 10);
 	kprintf("\twent_sleep:\t\t%" B_PRId64 "\n", fWentSleep);
 	kprintf("\twent_sleep_active:\t%" B_PRId64 "\n", fWentSleepActive);

src/system/kernel/scheduler/scheduler_thread.h

@@ -44,10 +44,11 @@ public:
 
 	inline	int32		GetEffectivePriority() const;
 
-	inline	void		IncreasePenalty();
 	inline	void		CancelPenalty();
 	inline	bool		ShouldCancelPenalty() const;
 
+	inline	bool		IsCPUBound() const	{ return fAdditionalPenalty != 0; }
+
 			bool		ChooseCoreAndCPU(CoreEntry*& targetCore,
 							CPUEntry*& targetCPU);
 
@@ -58,7 +59,6 @@ public:
 	inline	void		GoesAway();
 	inline	bigtime_t	WentSleep() const	{ return fWentSleep; }
 	inline	bigtime_t	WentSleepActive() const	{ return fWentSleepActive; }
-	inline	bigtime_t	WentSleepCount() const	{ return fWentSleepCount; }
 
 	inline	void		PutBack();
 	inline	void		Enqueue();
@@ -83,6 +83,7 @@ public:
 	static	void		ComputeQuantumLengths();
 
 private:
+	inline	void		_IncreasePenalty();
 	inline	int32		_GetPenalty() const;
 
 			void		_ComputeEffectivePriority() const;
@@ -98,6 +99,7 @@ private:
 			bigtime_t	fWentSleep;
 			bigtime_t	fWentSleepActive;
 			int32		fWentSleepCount;
+			int32		fWentSleepCountIdle;
 
 			bool		fEnqueued;
 
@@ -105,6 +107,8 @@ private:
 
 			int32		fPriorityPenalty;
 			int32		fAdditionalPenalty;
+			bool		fReceivedPenalty;
+			bool		fHasSlept;
 
 	mutable	int32		fEffectivePriority;
 
@@ -167,7 +171,7 @@ ThreadData::GetEffectivePriority() const
 
 
 inline void
-ThreadData::IncreasePenalty()
+ThreadData::_IncreasePenalty()
 {
 	SCHEDULER_ENTER_FUNCTION();
 
@@ -178,6 +182,7 @@ ThreadData::IncreasePenalty()
 
 	TRACE("increasing thread %ld penalty\n", fThread->id);
 
+	fReceivedPenalty = true;
 	int32 oldPenalty = fPriorityPenalty++;
 
 	ASSERT(fThread->priority - oldPenalty >= B_LOWEST_ACTIVE_PRIORITY);
@@ -216,9 +221,16 @@ ThreadData::ShouldCancelPenalty() const
 
 	if (fCore == NULL)
 		return false;
+	if (system_time() - fWentSleep > gCurrentMode->minimal_quantum * 2)
+		return false;
 
-	return fCore->StarvationCounter() != fWentSleepCount
+	if (GetEffectivePriority() != B_LOWEST_ACTIVE_PRIORITY
-		&& system_time() - fWentSleep > gCurrentMode->base_quantum;
+		&& !IsCPUBound()) {
+		if (fCore->StarvationCounter() != fWentSleepCount)
+			return true;
+	}
+
+	return fCore->StarvationCounterIdle() != fWentSleepCountIdle;
 }
 
 
@@ -238,10 +250,15 @@ ThreadData::GoesAway()
 {
 	SCHEDULER_ENTER_FUNCTION();
 
+	if (!fReceivedPenalty)
+		_IncreasePenalty();
+	fHasSlept = true;
+
 	fLastInterruptTime = 0;
 
 	fWentSleep = system_time();
 	fWentSleepCount = fCore->StarvationCounter();
+	fWentSleepCountIdle = fCore->StarvationCounterIdle();
 	fWentSleepActive = fCore->GetActiveTime();
 }
 
@@ -253,7 +270,6 @@ ThreadData::PutBack()
 
 	if (gTrackLoad)
 		ComputeLoad();
-	fWentSleepCount = -1;
 
 	int32 priority = GetEffectivePriority();
 
@@ -285,7 +301,6 @@ ThreadData::Enqueue()
 
 	if (gTrackLoad)
 		ComputeLoad();
-	fWentSleepCount = 0;
 
 	int32 priority = GetEffectivePriority();
 
@@ -328,7 +343,7 @@ ThreadData::Dequeue()
 	CoreRunQueueLocker _(fCore);
 	if (!fEnqueued)
 		return false;
-	ASSERT(fWentSleepCount < 1);
+
 	fCore->Remove(this);
 	ASSERT(!fEnqueued);
 	return true;
@@ -354,15 +369,25 @@ ThreadData::HasQuantumEnded(bool wasPreempted, bool hasYielded)
 	}
 
 	bigtime_t timeUsed = system_time() - fQuantumStart;
-	fTimeLeft -= timeUsed;
+	if (timeUsed > 0);
+		fTimeLeft -= timeUsed;
 	fTimeLeft = std::max(fTimeLeft, bigtime_t(0));
 
 	// too little time left, it's better make the next quantum a bit longer
-	if (wasPreempted || fTimeLeft <= gCurrentMode->minimal_quantum) {
+	int32 skipTime = gCurrentMode->minimal_quantum;
+	skipTime -= skipTime / 10;
+	if (wasPreempted || fTimeLeft <= skipTime) {
 		fStolenTime += fTimeLeft;
 		fTimeLeft = 0;
 	}
 
+	if (fTimeLeft == 0) {
+		if (!fReceivedPenalty && !fHasSlept)
+			_IncreasePenalty();
+		fReceivedPenalty = false;
+		fHasSlept = false;
+	}
+
 	return fTimeLeft == 0;
 }