scheduler: Improve latencies

src/system/kernel/scheduler/low_latency.cpp

@@ -161,10 +161,10 @@ scheduler_mode_operations gSchedulerLowLatencyMode = {
 	"low latency",
 
 	1000,
-	50,
+	100,
-	{ 2, 50 },
+	{ 2, 5 },
 
-	50000,
+	5000,
 
 	switch_to_mode,
 	set_cpu_enabled,

src/system/kernel/scheduler/power_saving.cpp

@@ -236,11 +236,11 @@ rebalance_irqs(bool idle)
 scheduler_mode_operations gSchedulerPowerSavingMode = {
 	"power saving",
 
-	3000,
+	2000,
-	1000,
+	500,
-	{ 3, 60 },
+	{ 3, 10 },
 
-	200000,
+	20000,
 
 	switch_to_mode,
 	set_cpu_enabled,

src/system/kernel/scheduler/scheduler.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright 2013, Paweł Dziepak, [email protected].
+ * Copyright 2013-2014, Paweł Dziepak, [email protected].
  * Copyright 2009, Rene Gollent, [email protected].
  * Copyright 2008-2011, Ingo Weinhold, [email protected].
  * Copyright 2002-2010, Axel Dörfler, [email protected].
@@ -484,7 +484,7 @@ reschedule(int32 nextState)
 
 		oldThread->cpu->preempted = false;
 		if (!thread_is_idle_thread(nextThread)) {
-			bigtime_t quantum = nextThreadData->ComputeQuantum();
+			bigtime_t quantum = nextThreadData->GetQuantumLeft();
 			add_timer(quantumTimer, &reschedule_event, quantum,
 				B_ONE_SHOT_RELATIVE_TIMER);
 

src/system/kernel/scheduler/scheduler_thread.cpp

@@ -19,10 +19,7 @@ ThreadData::_InitBase()
 	fAdditionalPenalty = 0;
 	fEffectivePriority = fThread->priority;
 
-	fReceivedPenalty = false;
+	fTimeUsed = 0;
-	fHasSlept = false;
-
-	fTimeLeft = 0;
 	fStolenTime = 0;
 
 	fMeasureAvailableActiveTime = 0;
@@ -111,11 +108,6 @@ ThreadData::Init()
 	fCore = currentThreadData->fCore;
 
 	if (fThread->priority < B_FIRST_REAL_TIME_PRIORITY) {
-		if (!thread_is_idle_thread(currentThread)
-			&& currentThread->priority < B_FIRST_REAL_TIME_PRIORITY) {
-			currentThreadData->_IncreasePenalty(false);
-		}
-
 		fPriorityPenalty = std::min(currentThreadData->fPriorityPenalty,
 				std::max(fThread->priority - _GetMinimalPriority(), int32(0)));
 		fAdditionalPenalty = currentThreadData->fAdditionalPenalty;
@@ -148,21 +140,8 @@ ThreadData::Dump() const
 		additionalPenalty, fAdditionalPenalty);
 	kprintf("\teffective_priority:\t%" B_PRId32 "\n", GetEffectivePriority());
 
-	kprintf("\treceived_penalty:\t%s\n", fReceivedPenalty ? "true" : "false");
+	kprintf("\ttime_used:\t\t%" B_PRId64 " us (quantum: %" B_PRId64 " us)\n",
-	kprintf("\thas_slept:\t\t%s\n", fHasSlept ? "true" : "false");
+		fTimeUsed, ComputeQuantum());
-
-	bigtime_t quantum = _GetBaseQuantum();
-	if (fThread->priority < B_FIRST_REAL_TIME_PRIORITY) {
-		int32 threadCount = (fCore->ThreadCount() + 1) / fCore->CPUCount();
-		threadCount = max_c(threadCount, 1);
-
-		quantum
-			= std::min(gCurrentMode->maximum_latency / threadCount, quantum);
-		quantum = std::max(quantum,	gCurrentMode->minimal_quantum);
-	}
-	kprintf("\ttime_left:\t\t%" B_PRId64 " us (quantum: %" B_PRId64 " us)\n",
-		fTimeLeft, quantum);
-
 	kprintf("\tstolen_time:\t\t%" B_PRId64 " us\n", fStolenTime);
 	kprintf("\tquantum_start:\t\t%" B_PRId64 " us\n", fQuantumStart);
 	kprintf("\tneeded_load:\t\t%" B_PRId32 "%%\n", fNeededLoad / 10);
@@ -206,22 +185,14 @@ ThreadData::ChooseCoreAndCPU(CoreEntry*& targetCore, CPUEntry*& targetCPU)
 
 
 bigtime_t
-ThreadData::ComputeQuantum()
+ThreadData::ComputeQuantum() const
 {
 	SCHEDULER_ENTER_FUNCTION();
 
-	bigtime_t quantum;
+	bigtime_t quantum = _GetBaseQuantum();
-	if (fTimeLeft != 0)
-		quantum = fTimeLeft;
-	else
-		quantum = _GetBaseQuantum();
-
 	if (fThread->priority >= B_FIRST_REAL_TIME_PRIORITY)
 		return quantum;
 
-	quantum += fStolenTime;
-	fStolenTime = 0;
-
 	int32 threadCount = fCore->ThreadCount() / fCore->CPUCount();
 	if (threadCount >= 1) {
 		quantum
@@ -229,7 +200,6 @@ ThreadData::ComputeQuantum()
 		quantum = std::max(quantum,	gCurrentMode->minimal_quantum);
 	}
 
-	fTimeLeft = quantum;
 	return quantum;
 }
 
@@ -266,14 +236,7 @@ inline int32
 ThreadData::_GetPenalty() const
 {
 	SCHEDULER_ENTER_FUNCTION();
-
+	return fPriorityPenalty;
-	int32 penalty = fPriorityPenalty;
-
-	const int kMinimalPriority = _GetMinimalPriority();
-	if (kMinimalPriority > 0)
-		penalty += fAdditionalPenalty % kMinimalPriority;
-
-	return penalty;
 }
 
 
@@ -304,6 +267,8 @@ ThreadData::_ComputeEffectivePriority() const
 	else {
 		fEffectivePriority = fThread->priority;
 		fEffectivePriority -= _GetPenalty();
+		if (fEffectivePriority > 0)
+			fEffectivePriority -= fAdditionalPenalty % fEffectivePriority;
 
 		ASSERT(fEffectivePriority < B_FIRST_REAL_TIME_PRIORITY);
 		ASSERT(fEffectivePriority >= B_LOWEST_ACTIVE_PRIORITY);

src/system/kernel/scheduler/scheduler_thread.h

@@ -56,6 +56,11 @@ public:
 							{ fLastInterruptTime = interruptTime; }
 	inline	void		SetStolenInterruptTime(bigtime_t interruptTime);
 
+			bigtime_t	ComputeQuantum() const;
+	inline	bigtime_t	GetQuantumLeft();
+	inline	void		StartQuantum();
+	inline	bool		HasQuantumEnded(bool wasPreempted, bool hasYielded);
+
 	inline	void		Continues();
 	inline	void		GoesAway();
 	inline	void		Dies();
@@ -69,10 +74,6 @@ public:
 
 	inline	void		UpdateActivity(bigtime_t active);
 
-	inline	bool		HasQuantumEnded(bool wasPreempted, bool hasYielded);
-			bigtime_t	ComputeQuantum();
-	inline	void		StartQuantum();
-
 	inline	bool		IsEnqueued() const	{ return fEnqueued; }
 	inline	void		SetDequeued()	{ fEnqueued = false; }
 
@@ -85,7 +86,7 @@ public:
 	static	void		ComputeQuantumLengths();
 
 private:
-	inline	void		_IncreasePenalty(bool strong);
+	inline	void		_IncreasePenalty();
 	inline	int32		_GetPenalty() const;
 
 			void		_ComputeNeededLoad();
@@ -112,12 +113,10 @@ private:
 
 			int32		fPriorityPenalty;
 			int32		fAdditionalPenalty;
-			bool		fReceivedPenalty;
-			bool		fHasSlept;
 
 	mutable	int32		fEffectivePriority;
 
-			bigtime_t	fTimeLeft;
+			bigtime_t	fTimeUsed;
 
 			bigtime_t	fMeasureAvailableActiveTime;
 			bigtime_t	fMeasureAvailableTime;
@@ -178,7 +177,7 @@ ThreadData::GetEffectivePriority() const
 
 
 inline void
-ThreadData::_IncreasePenalty(bool strong)
+ThreadData::_IncreasePenalty()
 {
 	SCHEDULER_ENTER_FUNCTION();
 
@@ -189,18 +188,10 @@ ThreadData::_IncreasePenalty(bool strong)
 
 	TRACE("increasing thread %ld penalty\n", fThread->id);
 
-	fReceivedPenalty = true;
+	int32 oldPenalty = fPriorityPenalty++;
-	int32 oldPenalty = fPriorityPenalty;
-	if (strong)
-		fPriorityPenalty++;
-
-	ASSERT(fThread->priority - oldPenalty >= B_LOWEST_ACTIVE_PRIORITY);
-
 	const int kMinimalPriority = _GetMinimalPriority();
-	if (!strong || fThread->priority - oldPenalty <= kMinimalPriority) {
+	if (fThread->priority - oldPenalty <= kMinimalPriority)
 		fPriorityPenalty = oldPenalty;
-		fAdditionalPenalty++;
-	}
 
 	_ComputeEffectivePriority();
 }
@@ -210,7 +201,7 @@ inline bool
 ThreadData::IsCPUBound() const
 {
 	SCHEDULER_ENTER_FUNCTION();
-	return fAdditionalPenalty != 0 && fPriorityPenalty != 0;
+	return GetThread()->priority - fPriorityPenalty == _GetMinimalPriority();
 }
 
 
@@ -220,8 +211,6 @@ ThreadData::CancelPenalty()
 	SCHEDULER_ENTER_FUNCTION();
 
 	int32 oldPenalty = fPriorityPenalty;
-
-	fAdditionalPenalty = 0;
 	fPriorityPenalty = 0;
 
 	if (oldPenalty != 0) {
@@ -239,8 +228,7 @@ ThreadData::ShouldCancelPenalty() const
 	if (fCore == NULL)
 		return false;
 
-	if (GetEffectivePriority() != B_LOWEST_ACTIVE_PRIORITY
+	if (GetEffectivePriority() != B_LOWEST_ACTIVE_PRIORITY && !IsCPUBound()) {
-		&& !IsCPUBound()) {
 		if (fCore->StarvationCounter() != fWentSleepCount)
 			return true;
 	}
@@ -259,6 +247,61 @@ ThreadData::SetStolenInterruptTime(bigtime_t interruptTime)
 }
 
 
+inline bigtime_t
+ThreadData::GetQuantumLeft()
+{
+	SCHEDULER_ENTER_FUNCTION();
+
+	bigtime_t stolenTime = std::min(fStolenTime, gCurrentMode->minimal_quantum);
+	ASSERT(stolenTime >= 0);
+	fStolenTime -= stolenTime;
+
+	bigtime_t quantum = ComputeQuantum() - fTimeUsed;
+	quantum += stolenTime;
+	quantum = std::max(quantum, gCurrentMode->minimal_quantum);
+
+	return quantum;
+}
+
+
+inline void
+ThreadData::StartQuantum()
+{
+	SCHEDULER_ENTER_FUNCTION();
+	fQuantumStart = system_time();
+}
+
+
+inline bool
+ThreadData::HasQuantumEnded(bool wasPreempted, bool hasYielded)
+{
+	SCHEDULER_ENTER_FUNCTION();
+
+	bigtime_t timeUsed = system_time() - fQuantumStart;
+	ASSERT(timeUsed >= 0);
+	fTimeUsed += timeUsed;
+
+	bigtime_t timeLeft = ComputeQuantum() - fTimeUsed;
+	timeLeft = std::max(bigtime_t(0), timeLeft);
+
+	// too little time left, it's better make the next quantum a bit longer
+	bigtime_t skipTime = gCurrentMode->minimal_quantum / 2;
+	if (hasYielded || wasPreempted || timeLeft <= skipTime) {
+		fStolenTime += timeLeft;
+		timeLeft = 0;
+	}
+
+	if (timeLeft == 0) {
+		fAdditionalPenalty++;
+		_IncreasePenalty();
+		fTimeUsed = 0;
+		return true;
+	}
+
+	return false;
+}
+
+
 inline void
 ThreadData::Continues()
 {
@@ -277,20 +320,17 @@ ThreadData::GoesAway()
 
 	ASSERT(fReady);
 
-	bigtime_t timeUsed = system_time() - fQuantumStart;
+	if (!HasQuantumEnded(false, false)) {
-	ASSERT(timeUsed >= 0);
+		fAdditionalPenalty++;
-	fTimeLeft -= timeUsed;
+		_ComputeEffectivePriority();
-
+	}
-	if (!fReceivedPenalty)
-		_IncreasePenalty(false);
-	fHasSlept = true;
 
 	fLastInterruptTime = 0;
 
 	fWentSleep = system_time();
+	fWentSleepActive = fCore->GetActiveTime();
 	fWentSleepCount = fCore->StarvationCounter();
 	fWentSleepCountIdle = fCore->StarvationCounterIdle();
-	fWentSleepActive = fCore->GetActiveTime();
 
 	if (gTrackCoreLoad)
 		fCore->UpdateLoad(-fNeededLoad);
@@ -415,46 +455,6 @@ ThreadData::UpdateActivity(bigtime_t active)
 }
 
 
-inline bool
-ThreadData::HasQuantumEnded(bool wasPreempted, bool hasYielded)
-{
-	SCHEDULER_ENTER_FUNCTION();
-
-	bigtime_t timeUsed = system_time() - fQuantumStart;
-	ASSERT(timeUsed >= 0);
-	fTimeLeft -= timeUsed;
-
-	if (hasYielded) {
-		fTimeLeft = 0;
-		return true;
-	}
-
-	// too little time left, it's better make the next quantum a bit longer
-	int32 skipTime = gCurrentMode->minimal_quantum / 2;
-	if (wasPreempted || fTimeLeft <= skipTime) {
-		fStolenTime += fTimeLeft;
-		fTimeLeft = 0;
-	}
-
-	if (fTimeLeft == 0) {
-		if (!fReceivedPenalty && !fHasSlept)
-			_IncreasePenalty(true);
-		fReceivedPenalty = false;
-		fHasSlept = false;
-	}
-
-	return fTimeLeft == 0;
-}
-
-
-inline void
-ThreadData::StartQuantum()
-{
-	SCHEDULER_ENTER_FUNCTION();
-	fQuantumStart = system_time();
-}
-
-
 inline void
 ThreadData::UnassignCore(bool running)
 {