kernel: Propagate scheduler modes to cpu{freq, idle} modules

headers/os/drivers/cpufreq.h

@@ -8,6 +8,8 @@
 
 #include <module.h>
 
+#include <scheduler.h>
+
 
 #define CPUFREQ_MODULES_PREFIX	"power/cpufreq"
 
@@ -19,8 +21,10 @@ typedef struct cpufreq_module_info {
 
 	float			rank;
 
-	status_t		(*increase_performance)(int delta, bool allowBoost);
+	void			(*cpufreq_set_scheduler_mode)(enum scheduler_mode mode);
-	status_t		(*decrease_performance)(int delta);
+
+	status_t		(*cpufreq_increase_performance)(int delta);
+	status_t		(*cpufreq_decrease_performance)(int delta);
 } cpufreq_module_info;
 
 

headers/os/drivers/cpuidle.h

@@ -8,6 +8,8 @@
 
 #include <module.h>
 
+#include <scheduler.h>
+
 
 #define CPUIDLE_MODULES_PREFIX	"power/cpuidle"
 
@@ -17,8 +19,10 @@ typedef struct cpuidle_module_info {
 
 	float			rank;
 
-	void			(*idle)(void);
+	void			(*cpuidle_set_scheduler_mode)(enum scheduler_mode mode);
-	void			(*wait)(int32* variable, int32 test);
+
+	void			(*cpuidle_idle)(void);
+	void			(*cpuidle_wait)(int32* variable, int32 test);
 } cpuidle_module_info;
 
 

headers/private/kernel/cpu.h

@@ -16,6 +16,8 @@
 #include <timer.h>
 #include <arch/cpu.h>
 
+#include <scheduler.h>
+
 
 struct kernel_args;
 
@@ -107,7 +109,9 @@ extern inline cpu_ent *get_cpu_struct(void) { return &gCPU[smp_get_current_cpu()
 status_t cpu_build_topology_tree(void);
 const cpu_topology_node* get_cpu_topology(void);
 
-status_t increase_cpu_performance(int delta, bool allowBoost);
+void cpu_set_scheduler_mode(enum scheduler_mode mode);
+
+status_t increase_cpu_performance(int delta);
 status_t decrease_cpu_performance(int delta);
 
 void cpu_idle(void);

headers/private/kernel/kscheduler.h

@@ -18,12 +18,6 @@ struct scheduling_analysis;
 struct SchedulerListener;
 
 
-enum scheduler_mode {
-	SCHEDULER_MODE_LOW_LATENCY,
-	SCHEDULER_MODE_POWER_SAVING,
-};
-
-
 #ifdef __cplusplus
 extern "C" {
 #endif

src/add-ons/kernel/power/cpufreq/intel_pstates/intel_pstates.cpp

@@ -25,6 +25,8 @@ static uint16 sMinPState;
 static uint16 sMaxPState;
 static uint16 sBoostPState;
 
+static bool sAvoidBoost;
+
 
 struct CPUEntry {
 				CPUEntry();
@@ -32,7 +34,7 @@ struct CPUEntry {
 	uint16		fCurrentPState;
 
 	bigtime_t	fLastUpdate;
-};
+} CACHE_LINE_ALIGN;
 static CPUEntry* sCPUEntries;
 
 
@@ -44,6 +46,13 @@ CPUEntry::CPUEntry()
 }
 
 
+static void
+pstates_set_scheduler_mode(scheduler_mode mode)
+{
+	sAvoidBoost = mode == SCHEDULER_MODE_POWER_SAVING;
+}
+
+
 static int
 measure_pstate(CPUEntry* entry)
 {
@@ -83,7 +92,7 @@ set_pstate(uint16 pstate)
 
 
 static status_t
-increase_performance(int delta, bool allowBoost)
+pstates_increase_performance(int delta)
 {
 	CPUEntry* entry = &sCPUEntries[smp_get_current_cpu()];
 
@@ -93,7 +102,7 @@ increase_performance(int delta, bool allowBoost)
 	int pState = measure_pstate(entry);
 	pState += (sBoostPState - pState) * delta / kCPUPerformanceScaleMax;
 
-	if (!allowBoost)
+	if (sAvoidBoost && pState < (sMaxPState + sBoostPState) / 2)
 		pState = min_c(pState, sMaxPState);
 
 	set_pstate(pState);
@@ -102,7 +111,7 @@ increase_performance(int delta, bool allowBoost)
 
 
 static status_t
-decrease_performance(int delta)
+pstates_decrease_performance(int delta)
 {
 	CPUEntry* entry = &sCPUEntries[smp_get_current_cpu()];
 
@@ -184,6 +193,8 @@ init_pstates()
 	if (sCPUEntries == NULL)
 		return B_NO_MEMORY;
 
+	pstates_set_scheduler_mode(SCHEDULER_MODE_LOW_LATENCY);
+
 	call_all_cpus_sync(set_normal_pstate, NULL);
 	return B_OK;
 }
@@ -224,8 +235,10 @@ static cpufreq_module_info sIntelPStates = {
 
 	1.0f,
 
-	increase_performance,
+	pstates_set_scheduler_mode,
-	decrease_performance,
+
+	pstates_increase_performance,
+	pstates_decrease_performance,
 };
 
 

src/add-ons/kernel/power/cpuidle/intel_cstates/intel_cstates.cpp

@@ -27,17 +27,19 @@
 
 #define INTEL_CSTATES_MODULE_NAME	CPUIDLE_MODULES_PREFIX "/intel_cstates/v1"
 
-#define TIME_STEP					250
+#define BASE_TIME_STEP				500
 
 struct CState {
 	uint32	fCode;
 	int		fSubStatesCount;
-	int		fSubStatesStep;
 };
 
 static CState sCStates[CPUIDLE_CSTATE_MAX];
 static int sCStateCount;
 
+static int sTimeStep = BASE_TIME_STEP;
+static bool sEnableWait = false;
+
 static bigtime_t* sIdleTime;
 
 
@@ -56,18 +58,31 @@ x86_mwait(uint32 eax, uint32 ecx)
 
 
 static void
-idle(void)
+cstates_set_scheduler_mode(scheduler_mode mode)
+{
+	if (mode == SCHEDULER_MODE_POWER_SAVING) {
+		sTimeStep = BASE_TIME_STEP / 4;
+		sEnableWait = true;
+	} else {
+		sTimeStep = BASE_TIME_STEP;
+		sEnableWait = false;
+	}
+}
+
+
+static void
+cstates_idle(void)
 {
 	ASSERT(thread_get_current_thread()->pinned_to_cpu > 0);
 	int32 cpu = smp_get_current_cpu();
 
 	bigtime_t idleTime = sIdleTime[cpu];
-	int state = min_c(idleTime / TIME_STEP, sCStateCount - 1);
+	int state = min_c(idleTime / sTimeStep, sCStateCount - 1);
 
 	ASSERT(state >= 0 && state < sCStateCount);
 
-	int subState = idleTime % TIME_STEP;
+	int subState = idleTime % sTimeStep;
-	subState /= sCStates[state].fSubStatesStep;
+	subState /= sTimeStep / sCStates[state].fSubStatesCount;
 
 	ASSERT(subState >= 0 && subState < sCStates[state].fSubStatesCount);
 
@@ -84,8 +99,11 @@ idle(void)
 
 
 static void
-wait(int32* variable, int32 test)
+cstates_wait(int32* variable, int32 test)
 {
+	if (!sEnableWait)
+		return;
+
 	InterruptsLocker _;
 	x86_monitor(variable, 0, 0);
 	if (*variable != test)
@@ -133,7 +151,6 @@ init_cstates()
 
 		sCStates[sCStateCount].fCode = sCStateCount * 0x10;
 		sCStates[sCStateCount].fSubStatesCount = subStates;
-		sCStates[sCStateCount].fSubStatesStep = TIME_STEP / subStates;
 		sCStateCount++;
 	}
 
@@ -145,6 +162,8 @@ init_cstates()
 		return B_NO_MEMORY;
 	memset(sIdleTime, 0, sizeof(bigtime_t) * smp_get_num_cpus());
 
+	cstates_set_scheduler_mode(SCHEDULER_MODE_LOW_LATENCY);
+
 	dprintf("using Intel C-States: C0%s\n", cStates);
 	return B_OK;
 }
@@ -183,8 +202,10 @@ static cpuidle_module_info sIntelCStates = {
 
 	0.8f,
 
-	idle,
+	cstates_set_scheduler_mode,
-	wait
+
+	cstates_idle,
+	cstates_wait
 };
 
 

src/system/kernel/cpu.cpp

@@ -281,11 +281,21 @@ get_cpu_topology(void)
 }
 
 
-status_t
+void
-increase_cpu_performance(int delta, bool allowBoost)
+cpu_set_scheduler_mode(enum scheduler_mode mode)
 {
 	if (sCPUPerformanceModule != NULL)
-		return sCPUPerformanceModule->increase_performance(delta, allowBoost);
+		sCPUPerformanceModule->cpufreq_set_scheduler_mode(mode);
+	if (sCPUIdleModule != NULL)
+		sCPUIdleModule->cpuidle_set_scheduler_mode(mode);
+}
+
+
+status_t
+increase_cpu_performance(int delta)
+{
+	if (sCPUPerformanceModule != NULL)
+		return sCPUPerformanceModule->cpufreq_increase_performance(delta);
 	return B_NOT_SUPPORTED;
 }
 
@@ -294,7 +304,7 @@ status_t
 decrease_cpu_performance(int delta)
 {
 	if (sCPUPerformanceModule != NULL)
-		return sCPUPerformanceModule->decrease_performance(delta);
+		return sCPUPerformanceModule->cpufreq_decrease_performance(delta);
 	return B_NOT_SUPPORTED;
 }
 
@@ -308,7 +318,7 @@ cpu_idle(void)
 #endif
 
 	if (sCPUIdleModule != NULL)
-		sCPUIdleModule->idle();
+		sCPUIdleModule->cpuidle_idle();
 	else
 		arch_cpu_idle();
 }
@@ -318,7 +328,7 @@ void
 cpu_wait(int32* variable, int32 test)
 {
 	if (sCPUIdleModule != NULL)
-		sCPUIdleModule->wait(variable, test);
+		sCPUIdleModule->cpuidle_wait(variable, test);
 	else
 		arch_cpu_pause();
 }

src/system/kernel/scheduler/scheduler.cpp

@@ -996,7 +996,11 @@ _user_estimate_max_scheduling_latency(thread_id id)
 status_t
 _user_set_scheduler_mode(int32 mode)
 {
-	return scheduler_set_operation_mode(static_cast<scheduler_mode>(mode));
+	scheduler_mode schedulerMode = static_cast<scheduler_mode>(mode);
+	status_t error = scheduler_set_operation_mode(schedulerMode);
+	if (error == B_OK)
+		cpu_set_scheduler_mode(schedulerMode);
+	return error;
 }
 
 

src/system/kernel/scheduler/scheduler_cpu.cpp

@@ -215,13 +215,11 @@ CPUEntry::_RequestPerformanceLevel(ThreadData* threadData)
 
 		decrease_cpu_performance(delta);
 	} else {
-		bool allowBoost = !gCurrentMode->avoid_boost;
-
 		int32 delta = load - kTargetLoad;
 		delta *= kMaxLoad - kTargetLoad;
 		delta /= kCPUPerformanceScaleMax;
 
-		increase_cpu_performance(delta, allowBoost);
+		increase_cpu_performance(delta);
 	}
 }