i2p/haiku-beta6
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

kernel: Rework cpuidle module

Browse Source 
* Create new interface for cpuidle modules (similar to the cpufreq
   interface)
 * Generic cpuidle module is no longer needed
 * Fix and update Intel C-State module
This commit is contained in:
Pawel Dziepak
2013-11-25 23:50:27 +01:00
parent 0e94a12f8e
commit 7db89e8dc3
22 changed files with 356 additions and 339 deletions
Download Patch File Download Diff File Expand all files Collapse all files
build/jam/packages/Haiku
+1
View File
@@ -53,6 +53,7 @@ AddFilesToPackage add-ons kernel partitioning_systems
: amiga_rdb apple efi_gpt intel session ;
AddFilesToPackage add-ons kernel interrupt_controllers : openpic@ppc ;
AddFilesToPackage add-ons kernel power cpufreq : intel_pstates@x86,x86_64 ;
AddFilesToPackage add-ons kernel power cpuidle : intel_cstates@x86,x86_64 ;
if $(TARGET_ARCH) = x86 || $(TARGET_ARCH) = x86_64 {
AddFilesToPackage add-ons kernel cpu : generic_x86 ;
build/jam/packages/HaikuBootstrap
+1
View File
@@ -52,6 +52,7 @@ AddFilesToPackage add-ons kernel partitioning_systems
: amiga_rdb apple efi_gpt intel session ;
AddFilesToPackage add-ons kernel interrupt_controllers : openpic@ppc ;
AddFilesToPackage add-ons kernel power cpufreq : intel_pstates@x86,x86_64 ;
AddFilesToPackage add-ons kernel power cpuidle : intel_cstates@x86,x86_64 ;
if $(TARGET_ARCH) = x86 || $(TARGET_ARCH) = x86_64 {
AddFilesToPackage add-ons kernel cpu : generic_x86 ;
headers/os/drivers/cpuidle.h
+7 -39
View File
@@ -8,50 +8,18 @@
#include <module.h>
#ifdef __cplusplus
extern "C" {
#endif
#define CPUIDLE_CSTATE_MAX 8
#define CSTATE_NAME_LENGTH 32
#define B_CPUIDLE_MODULE_NAME "generic/cpuidle/v1"
#define CPUIDLE_MODULES_PREFIX "power/cpuidle"
struct CpuidleStat {
uint64 usageCount;
bigtime_t usageTime;
};
typedef struct cpuidle_module_info {
module_info info;
float rank;
struct CpuidleInfo {
int32 cstateSleep;
CpuidleStat stats[CPUIDLE_CSTATE_MAX];
};
void (*idle)(void);
void (*wait)(int32* variable, int32 test);
} cpuidle_module_info;
struct CpuidleDevice;
struct CpuidleCstate {
char name[CSTATE_NAME_LENGTH];
int32 latency;
int32 (*EnterIdle)(int32 state, CpuidleDevice *device);
void *pData;
};
struct CpuidleDevice {
CpuidleCstate cStates[CPUIDLE_CSTATE_MAX];
int32 cStateCount;
};
struct CpuidleModuleInfo {
module_info info;
status_t (*AddDevice)(CpuidleDevice *device);
};
#ifdef __cplusplus
}
#endif
#endif // _CPUIDLE_MODULE_H
headers/private/kernel/arch/cpu.h
-1
View File
@@ -41,7 +41,6 @@ ssize_t arch_cpu_user_strlcpy(char *to, const char *from, size_t size,
status_t arch_cpu_user_memset(void *s, char c, size_t count,
addr_t *faultHandler);
void arch_cpu_idle(void);
void arch_cpu_sync_icache(void *address, size_t length);
void arch_cpu_memory_read_barrier(void);
headers/private/kernel/arch/x86/arch_cpu.h
+26 -3
View File
@@ -185,6 +185,10 @@
| IA32_FEATURE_AMD_EXT_RDTSCP \
| IA32_FEATURE_AMD_EXT_LONG)
// x86 defined features from cpuid eax 5, ecx register
#define IA32_FEATURE_POWER_MWAIT (1 << 0)
#define IA32_FEATURE_INTERRUPT_MWAIT (1 << 1)
// x86 defined features from cpuid eax 6, eax register
// reference http://www.intel.com/Assets/en_US/PDF/appnote/241618.pdf (Table 5-11)
#define IA32_FEATURE_DTS (1 << 0) //Digital Thermal Sensor
@@ -199,6 +203,9 @@
#define IA32_FEATURE_APERFMPERF (1 << 0) //IA32_APERF, IA32_MPERF
#define IA32_FEATURE_EPB (1 << 3) //IA32_ENERGY_PERF_BIAS
// x86 defined features from cpuid eax 0x80000007, edx register
#define IA32_FEATURE_INVARIANT_TSC (1 << 8)
// cr4 flags
#define IA32_CR4_PAE (1UL << 5)
#define IA32_CR4_GLOBAL_PAGES (1UL << 7)
@@ -282,8 +289,10 @@ enum x86_feature_type {
FEATURE_EXT, // cpuid eax=1, edx register
FEATURE_EXT_AMD_ECX, // cpuid eax=0x80000001, ecx register (AMD)
FEATURE_EXT_AMD, // cpuid eax=0x80000001, edx register (AMD)
FEATURE_5_ECX, // cpuid eax=5, ecx register
FEATURE_6_EAX, // cpuid eax=6, eax registers
FEATURE_6_ECX, // cpuid eax=6, ecx registers
FEATURE_EXT_7_EDX, // cpuid eax=0x80000007, edx register
FEATURE_NUM
};
@@ -330,9 +339,6 @@ typedef struct arch_cpu_info {
} arch_cpu_info;
#undef PAUSE
#define PAUSE() asm volatile ("pause;")
#define nop() __asm__ ("nop"::)
#define x86_read_cr0() ({ \
@@ -426,6 +432,9 @@ typedef struct arch_cpu_info {
})
extern void (*gCpuIdleFunc)(void);
#ifdef __cplusplus
extern "C" {
#endif
@@ -478,6 +487,20 @@ void x86_fnsave_swap(void* oldFpuState, const void* newFpuState);
#endif
static inline void
arch_cpu_idle(void)
{
gCpuIdleFunc();
}
static inline void
arch_cpu_pause(void)
{
asm volatile("pause");
}
#ifdef __cplusplus
} // extern "C" {
#endif
headers/private/kernel/cpu.h
+11 -6
View File
@@ -17,12 +17,6 @@
#include <arch/cpu.h>
// define PAUSE, if not done in arch/cpu.h
#ifndef PAUSE
# define PAUSE()
#endif
struct kernel_args;
namespace BKernel {
@@ -114,6 +108,17 @@ cpu_topology_node* get_cpu_topology(void);
status_t increase_cpu_performance(int delta, bool allowBoost);
status_t decrease_cpu_performance(int delta);
void cpu_idle(void);
void cpu_wait(int32* variable, int32 test);
static inline void
cpu_pause(void)
{
arch_cpu_pause();
}
void _user_clear_caches(void *address, size_t length, uint32 flags);
bool _user_cpu_enabled(int32 cpu);
status_t _user_set_cpu_enabled(int32 cpu, bool enabled);
src/add-ons/kernel/drivers/power/x86_cpuidle/intel_cpuidle.cpp
-121
View File
@@ -1,121 +0,0 @@
/*
* Copyright 2012, Haiku, Inc. All Rights Reserved.
* Distributed under the terms of the MIT License.
*
* Authors:
* Yongcong Du <[email protected]>
*/
#include <KernelExport.h>
#include <cpu.h>
#include <arch_system_info.h>
#include <stdio.h>
#include <cpuidle.h>
#include "x86_cpuidle.h"
static CpuidleDevice sIntelDevice;
static void *kMwaitEax[] = {
// C0, we never use it
(void *)0x00,
// MWAIT C1
(void *)0x00,
// MWAIT C2
(void *)0x10,
// MWAIT C3
(void *)0x20,
// MWAIT C4
(void *)0x30,
// MWAIT C5
(void *)0x40,
// MWAIT C6, 0x2 is used to fully shrink L2 cache
(void *)0x52
};
static int32
IntelCstateIdleEnter(int32 state, CpuidleDevice *device)
{
cpu_ent *cpu = get_cpu_struct();
if (cpu->invoke_scheduler)
return 0;
CpuidleCstate *cState = &device->cStates[state];
x86_monitor((void *)&cpu->invoke_scheduler, 0, 0);
if (!cpu->invoke_scheduler)
x86_mwait((unsigned long)cState->pData, 1);
return state;
}
static CpuidleCstate sSnbcStates[CPUIDLE_CSTATE_MAX] = {
{},
{
"C1-SNB",
1,
IntelCstateIdleEnter,
},
{
"C3-SNB",
80,
IntelCstateIdleEnter,
},
{
"C6-SNB",
104,
IntelCstateIdleEnter,
},
{
"C7-SNB",
109,
IntelCstateIdleEnter,
},
};
status_t
intel_cpuidle_init(void)
{
dprintf("intel idle init\n");
cpu_ent *cpu = get_cpu_struct();
if (cpu->arch.vendor != VENDOR_INTEL || cpu->arch.family != 6)
return B_ERROR;
// Calculated Model Value: M = (Extended Model << 4) + Model
uint32 model = (cpu->arch.extended_model << 4) + cpu->arch.model;
if (model != 0x2a && model != 0x2d)
return B_ERROR;
cpuid_info cpuid;
get_current_cpuid(&cpuid, 5, 0);
/* ecx[0] monitor/mwait extension supported
* ecx[1] support for treating interrupts as break-events for mwait
* edx number of sub-states
*/
if ((cpuid.regs.ecx & 0x1) == 0 ||
(cpuid.regs.ecx & 0x2) == 0 ||
cpuid.regs.edx == 0) {
return B_ERROR;
}
sIntelDevice.cStateCount = 1;
for (int32 i = 1; i < CPUIDLE_CSTATE_MAX; i++) {
int32 subStates = (cpuid.regs.edx >> ((i) * 4)) & 0xf;
// no sub-states means the state is not available
if (!subStates)
continue;
sIntelDevice.cStates[sIntelDevice.cStateCount] =
sSnbcStates[i];
sIntelDevice.cStates[sIntelDevice.cStateCount].pData =
kMwaitEax[i];
sIntelDevice.cStateCount++;
}
status_t status = gIdle->AddDevice(&sIntelDevice);
if (status == B_OK)
dprintf("using intel idle\n");
return status;
}
src/add-ons/kernel/generic/Jamfile
-1
View File
@@ -3,7 +3,6 @@ SubDir HAIKU_TOP src add-ons kernel generic ;
SubInclude HAIKU_TOP src add-ons kernel generic ata_adapter ;
SubInclude HAIKU_TOP src add-ons kernel generic atomizer ;
SubInclude HAIKU_TOP src add-ons kernel generic bios ;
SubInclude HAIKU_TOP src add-ons kernel generic cpuidle ;
SubInclude HAIKU_TOP src add-ons kernel generic dpc ;
SubInclude HAIKU_TOP src add-ons kernel generic ide_adapter ;
SubInclude HAIKU_TOP src add-ons kernel generic locked_pool ;
src/add-ons/kernel/generic/cpuidle/Jamfile
-7
View File
@@ -1,7 +0,0 @@
SubDir HAIKU_TOP src add-ons kernel generic cpuidle ;
UsePrivateKernelHeaders ;
KernelAddon cpuidle :
cpuidle.cpp
;
src/add-ons/kernel/generic/cpuidle/cpuidle.cpp
-105
View File
@@ -1,105 +0,0 @@
/*
* Copyright 2012, Haiku, Inc. All Rights Reserved.
* Distributed under the terms of the MIT License.
*
* Authors:
* Yongcong Du <ycdu.vmcore@gmail.com>
*/
#include <KernelExport.h>
#include <stdio.h>
#include <string.h>
#include <smp.h>
#include <cpuidle.h>
static CpuidleInfo sPerCPU[B_MAX_CPU_COUNT];
static CpuidleDevice *sDevice;
extern void (*gCpuIdleFunc)(void);
/*
* next cstate selection algorithm is based on NetBSD's
* it's simple, stupid
*/
static int32
SelectCstate(CpuidleInfo *info)
{
static const int32 csFactor = 3;
for (int32 i = sDevice->cStateCount - 1; i > 0; i--) {
CpuidleCstate *cState = &sDevice->cStates[i];
if (info->cstateSleep > cState->latency * csFactor)
return i;
}
/* Choose C1 if there's no state found */
return 1;
}
static inline void
EnterCstate(int32 state, CpuidleInfo *info)
{
CpuidleCstate *cstate = &sDevice->cStates[state];
bigtime_t now = system_time();
int32 finalState = cstate->EnterIdle(state, sDevice);
if (finalState > 0) {
bigtime_t diff = system_time() - now;
info->cstateSleep = diff;
info->stats[finalState].usageCount++;
info->stats[finalState].usageTime += diff;
} else {
info->cstateSleep = 0;
}
}
static void
CpuCstateIdle(void)
{
CpuidleInfo *info = &sPerCPU[smp_get_current_cpu()];
int32 state = SelectCstate(info);
EnterCstate(state, info);
}
static status_t
std_ops(int32 op, ...)
{
switch (op) {
case B_MODULE_INIT:
return B_OK;
case B_MODULE_UNINIT:
return B_OK;
}
return B_ERROR;
}
status_t AddDevice(CpuidleDevice *device)
{
sDevice = device;
memory_write_barrier();
gCpuIdleFunc = CpuCstateIdle;
return B_OK;
}
static CpuidleModuleInfo sCpuidleModule = {
{
B_CPUIDLE_MODULE_NAME,
0,
std_ops
},
AddDevice,
};
module_info *modules[] = {
(module_info *)&sCpuidleModule,
NULL
};
src/add-ons/kernel/power/Jamfile
+1
View File
@@ -1,4 +1,5 @@
SubDir HAIKU_TOP src add-ons kernel power ;
SubInclude HAIKU_TOP src add-ons kernel power cpufreq ;
SubInclude HAIKU_TOP src add-ons kernel power cpuidle ;
src/add-ons/kernel/power/cpuidle/Jamfile
+4
View File
@@ -0,0 +1,4 @@
SubDir HAIKU_TOP src add-ons kernel power cpuidle ;
SubInclude HAIKU_TOP src add-ons kernel power cpuidle intel_cstates ;
src/add-ons/kernel/power/cpuidle/intel_cstates/Jamfile
+8
View File
@@ -0,0 +1,8 @@
SubDir HAIKU_TOP src add-ons kernel power cpuidle intel_cstates ;
UsePrivateKernelHeaders ;
KernelAddon intel_cstates :
intel_cstates.cpp
;
src/add-ons/kernel/power/cpuidle/intel_cstates/intel_cstates.cpp
+195
View File
@@ -0,0 +1,195 @@
/*
* Copyright 2012-2013, Haiku, Inc. All Rights Reserved.
* Distributed under the terms of the MIT License.
*
* Authors:
* Yongcong Du <ycdu.vmcore@gmail.com>
* Paweł Dziepak, <pdziepak@quarnos.org>
*/
#include <arch_system_info.h>
#include <cpu.h>
#include <debug.h>
#include <smp.h>
#include <thread.h>
#include <util/AutoLock.h>
#include <cpuidle.h>
#include <KernelExport.h>
#include <stdio.h>
#define CPUIDLE_CSTATE_MAX 8
#define MWAIT_INTERRUPTS_BREAK (1 << 0)
#define INTEL_CSTATES_MODULE_NAME CPUIDLE_MODULES_PREFIX "/intel_cstates/v1"
#define TIME_STEP 250
struct CState {
uint32 fCode;
int fSubStatesCount;
int fSubStatesStep;
};
static CState sCStates[CPUIDLE_CSTATE_MAX];
static int sCStateCount;
static bigtime_t* sIdleTime;
static inline void
x86_monitor(void* address, uint32 ecx, uint32 edx)
{
asm volatile("monitor" : : "a" (address), "c" (ecx), "d"(edx));
}
static inline void
x86_mwait(uint32 eax, uint32 ecx)
{
asm volatile("mwait" : : "a" (eax), "c" (ecx));
}
static void
idle(void)
{
ASSERT(thread_get_current_thread()->pinned_to_cpu > 0);
int32 cpu = smp_get_current_cpu();
bigtime_t idleTime = sIdleTime[cpu];
int state = min_c(idleTime / TIME_STEP, sCStateCount - 1);
ASSERT(state >= 0 && state < sCStateCount);
int subState = idleTime % TIME_STEP;
subState /= sCStates[state].fSubStatesStep;
ASSERT(subState >= 0 && subState < sCStates[state].fSubStatesCount);
InterruptsLocker locker;
int dummy;
bigtime_t start = system_time();
x86_monitor(&dummy, 0, 0);
x86_mwait(sCStates[state].fCode | subState, MWAIT_INTERRUPTS_BREAK);
bigtime_t delta = system_time() - start;
locker.Unlock();
sIdleTime[cpu] = (idleTime + delta) / 2;
}
static void
wait(int32* variable, int32 test)
{
InterruptsLocker _;
x86_monitor(variable, 0, 0);
if (*variable != test)
x86_mwait(sCStates[0].fCode, MWAIT_INTERRUPTS_BREAK);
}
static status_t
init_cstates()
{
if (!x86_check_feature(IA32_FEATURE_EXT_MONITOR, FEATURE_EXT))
return B_ERROR;
if (!x86_check_feature(IA32_FEATURE_POWER_MWAIT, FEATURE_5_ECX))
return B_ERROR;
if (!x86_check_feature(IA32_FEATURE_INTERRUPT_MWAIT, FEATURE_5_ECX))
return B_ERROR;
// we need invariant TSC
if (!x86_check_feature(IA32_FEATURE_INVARIANT_TSC, FEATURE_EXT_7_EDX))
return B_ERROR;
// get C-state data
cpuid_info cpuid;
get_current_cpuid(&cpuid, 0, 0);
uint32 maxBasicLeaf = cpuid.eax_0.max_eax;
if (maxBasicLeaf < 5)
return B_ERROR;
get_current_cpuid(&cpuid, 5, 0);
if ((cpuid.regs.eax & 0xffff) < sizeof(int32))
return B_ERROR;
char cStates[64];
unsigned int offset = 0;
for (int32 i = 1; i < CPUIDLE_CSTATE_MAX; i++) {
int32 subStates = (cpuid.regs.edx >> (i * 4)) & 0xf;
// no sub-states means the state is not available
if (subStates == 0)
continue;
if (offset < sizeof(cStates)) {
offset += snprintf(cStates + offset, sizeof(cStates) - offset,
", C%" B_PRId32, i);
}
sCStates[sCStateCount].fCode = sCStateCount * 0x10;
sCStates[sCStateCount].fSubStatesCount = subStates;
sCStates[sCStateCount].fSubStatesStep = TIME_STEP / subStates;
sCStateCount++;
}
if (sCStateCount == 0)
return B_ERROR;
sIdleTime = new(std::nothrow) bigtime_t[smp_get_num_cpus()];
if (sIdleTime == NULL)
return B_NO_MEMORY;
memset(sIdleTime, 0, sizeof(bigtime_t) * smp_get_num_cpus());
dprintf("using Intel C-States: C0%s\n", cStates);
return B_OK;
}
static status_t
uninit_cstates()
{
delete[] sIdleTime;
return B_OK;
}
static status_t
std_ops(int32 op, ...)
{
switch (op) {
case B_MODULE_INIT:
return init_cstates();
case B_MODULE_UNINIT:
uninit_cstates();
return B_OK;
}
return B_ERROR;
}
static cpuidle_module_info sIntelCStates = {
{
INTEL_CSTATES_MODULE_NAME,
0,
std_ops,
},
0.8f,
idle,
wait
};
module_info* modules[] = {
(module_info*)&sIntelCStates,
NULL
};
src/system/kernel/UserTimer.cpp
+1 -1
View File
@@ -196,7 +196,7 @@ UserTimer::HandleTimerHook(struct timer* timer)
while (!locked && atomic_get(&userTimer->fSkip) == 0) {
locked = try_acquire_write_seqlock(&sUserTimerLock);
if (!locked)
PAUSE();
cpu_pause();
}
if (locked) {
src/system/kernel/arch/x86/arch_cpu.cpp
+11 -26
View File
@@ -1,5 +1,6 @@
/*
* Copyright 2002-2010, Axel Dörfler, axeld@pinc-software.de.
* Copyright 2013, Paweł Dziepak, pdziepak@quarnos.org.
* Copyright 2012, Alex Smith, alex@alex-smith.me.uk.
* Distributed under the terms of the MIT License.
*
@@ -69,16 +70,6 @@ static const struct cpu_vendor_info vendor_info[VENDOR_NUM] = {
#define K8_CMPHALT (K8_SMIONCMPHALT | K8_C1EONCMPHALT)
/*
* 0 favors highest performance while 15 corresponds to the maximum energy
* savings. 7 means balance between performance and energy savings.
* Refer to Section 14.3.4 in <Intel 64 and IA-32 Architectures Software
* Developer's Manual Volume 3> for details
*/
#define ENERGY_PERF_BIAS_PERFORMANCE 0
#define ENERGY_PERF_BIAS_BALANCE 7
#define ENERGY_PERF_BIAS_POWERSAVE 15
struct set_mtrr_parameter {
int32 index;
uint64 base;
@@ -929,12 +920,22 @@ detect_cpu(int currentCPU)
cpu->arch.feature[FEATURE_EXT_AMD] &= IA32_FEATURES_INTEL_EXT;
}
if (maxBasicLeaf >= 5) {
get_current_cpuid(&cpuid, 5, 0);
cpu->arch.feature[FEATURE_5_ECX] = cpuid.regs.ecx;
}
if (maxBasicLeaf >= 6) {
get_current_cpuid(&cpuid, 6, 0);
cpu->arch.feature[FEATURE_6_EAX] = cpuid.regs.eax;
cpu->arch.feature[FEATURE_6_ECX] = cpuid.regs.ecx;
}
if (maxExtendedLeaf >= 0x80000007) {
get_current_cpuid(&cpuid, 0x80000007, 0);
cpu->arch.feature[FEATURE_EXT_7_EDX] = cpuid.regs.edx;
}
detect_cpu_topology(currentCPU, cpu, maxBasicLeaf, maxExtendedLeaf);
#if DUMP_FEATURE_STRING
@@ -1060,15 +1061,6 @@ arch_cpu_init_percpu(kernel_args* args, int cpu)
gCpuIdleFunc = halt_idle;
}
if (x86_check_feature(IA32_FEATURE_EPB, FEATURE_6_ECX)) {
uint64 msr = x86_read_msr(IA32_MSR_ENERGY_PERF_BIAS);
if ((msr & 0xf) == ENERGY_PERF_BIAS_PERFORMANCE) {
msr &= ~0xf;
msr |= ENERGY_PERF_BIAS_BALANCE;
x86_write_msr(IA32_MSR_ENERGY_PERF_BIAS, msr);
}
}
return B_OK;
}
@@ -1287,13 +1279,6 @@ arch_cpu_shutdown(bool rebootSystem)
}
void
arch_cpu_idle(void)
{
gCpuIdleFunc();
}
void
arch_cpu_sync_icache(void* address, size_t length)
{
src/system/kernel/arch/x86/arch_debug_console.cpp
+2 -2
View File
@@ -316,7 +316,7 @@ arch_debug_blue_screen_getchar(void)
if (c >= 0)
return (char)c;
PAUSE();
arch_cpu_pause();
}
}
@@ -352,7 +352,7 @@ arch_debug_serial_getchar(void)
if ((lineStatus & 0x1) != 0)
break;
PAUSE();
arch_cpu_pause();
}
return in8(sSerialBasePort + SERIAL_RECEIVE_BUFFER);
src/system/kernel/cpu.cpp
+64 -1
View File
@@ -16,6 +16,7 @@
#include <string.h>
#include <cpufreq.h>
#include <cpuidle.h>
#include <boot/kernel_args.h>
#include <kscheduler.h>
@@ -30,6 +31,7 @@ uint32 gCPUCacheLevelCount;
static cpu_topology_node sCPUTopology;
static cpufreq_module_info* sCPUPerformanceModule;
static cpuidle_module_info* sCPUIdleModule;
static spinlock sSetCpuLock;
@@ -89,6 +91,40 @@ load_cpufreq_module()
}
static void
load_cpuidle_module()
{
void* cookie = open_module_list(CPUIDLE_MODULES_PREFIX);
while (true) {
char name[B_FILE_NAME_LENGTH];
size_t nameLength = sizeof(name);
cpuidle_module_info* current = NULL;
if (read_next_module_name(cookie, name, &nameLength) != B_OK)
break;
if (get_module(name, (module_info**)&current) == B_OK) {
dprintf("found cpuidle module: %s\n", name);
if (sCPUIdleModule != NULL) {
if (sCPUIdleModule->rank < current->rank) {
put_module(sCPUIdleModule->info.name);
sCPUIdleModule = current;
} else
put_module(name);
} else
sCPUIdleModule = current;
}
}
close_module_list(cookie);
if (sCPUIdleModule == NULL)
dprintf("no valid cpuidle module found\n");
}
status_t
cpu_init_post_modules(kernel_args *args)
{
@@ -97,6 +133,7 @@ cpu_init_post_modules(kernel_args *args)
return result;
load_cpufreq_module();
load_cpuidle_module();
return B_OK;
}
@@ -232,6 +269,8 @@ cpu_build_topology_tree(void)
int32 lastID[CPU_TOPOLOGY_LEVELS];
memset(&lastID, 0, sizeof(lastID));
cpu_rebuild_topology_tree(&sCPUTopology, lastID);
return B_OK;
}
@@ -260,6 +299,31 @@ decrease_cpu_performance(int delta)
}
void
cpu_idle(void)
{
#if KDEBUG
if (!are_interrupts_enabled())
panic("cpu_idle() called with interrupts disabled.");
#endif
if (sCPUIdleModule != NULL)
sCPUIdleModule->idle();
else
arch_cpu_idle();
}
void
cpu_wait(int32* variable, int32 test)
{
if (sCPUIdleModule != NULL)
sCPUIdleModule->wait(variable, test);
else
arch_cpu_pause();
}
// #pragma mark -
@@ -283,7 +347,6 @@ _user_cpu_enabled(int32 cpu)
status_t
_user_set_cpu_enabled(int32 cpu, bool enabled)
{
cpu_status state;
int32 i, count;
if (cpu < 0 || cpu >= smp_get_num_cpus())
src/system/kernel/debug/debug.cpp
+2 -2
View File
@@ -703,7 +703,7 @@ kgetc(void)
}
}
PAUSE();
cpu_pause();
}
}
@@ -1008,7 +1008,7 @@ hand_over_kernel_debugger()
// initiated by the user, it is harmless, though.
sHandOverKDL = true;
while (atomic_get(&sHandOverKDLToCPU) >= 0)
PAUSE();
cpu_wait(&sHandOverKDLToCPU, -1);
}
src/system/kernel/main.cpp
+1 -1
View File
@@ -262,7 +262,7 @@ _start(kernel_args *bootKernelArgs, int currentCPU)
#endif
for (;;)
arch_cpu_idle();
cpu_idle();
return 0;
}
src/system/kernel/smp.cpp
+17 -17
View File
@@ -358,7 +358,7 @@ acquire_spinlock(spinlock* lock)
}
process_all_pending_ici(currentCPU);
PAUSE();
cpu_wait(lock, 0);
}
if (atomic_get_and_set((int32*)lock, 1) == 0)
break;
@@ -411,7 +411,7 @@ acquire_spinlock_nocheck(spinlock *lock)
count = 0;
}
PAUSE();
cpu_wait(lock, 0);
}
if (atomic_get_and_set((int32*)lock, 1) == 0)
@@ -455,7 +455,7 @@ acquire_spinlock_cpu(int32 currentCPU, spinlock *lock)
}
process_all_pending_ici(currentCPU);
PAUSE();
cpu_wait(lock, 0);
}
if (atomic_get_and_set((int32*)lock, 1) == 0)
break;
@@ -570,7 +570,7 @@ acquire_write_spinlock(rw_spinlock* lock)
}
process_all_pending_ici(currentCPU);
PAUSE();
cpu_wait(&lock->lock, 0);
}
}
}
@@ -583,7 +583,7 @@ release_write_spinlock(rw_spinlock* lock)
uint32 previous = atomic_get_and_set(&lock->lock, 0);
if ((previous & 1u << 31) == 0) {
panic("release_write_spinlock: lock %p was already released (value: "
"%x)\n", lock, previous);
"%#" B_PRIx32 ")\n", lock, previous);
}
#else
atomic_set(&lock->lock, 0);
@@ -639,7 +639,7 @@ acquire_read_spinlock(rw_spinlock* lock)
}
process_all_pending_ici(currentCPU);
PAUSE();
cpu_wait(&lock->lock, 0);
}
}
}
@@ -652,7 +652,7 @@ release_read_spinlock(rw_spinlock* lock)
uint32 previous = atomic_add(&lock->lock, -1);
if ((previous & 1 << 31) != 0) {
panic("release_read_spinlock: lock %p was already released (value:"
" %x)\n", lock, previous);
" %#" B_PRIx32 ")\n", lock, previous);
}
#else
atomic_add(&lock->lock, -1);
@@ -695,7 +695,7 @@ release_read_seqlock(seqlock* lock, uint32 count) {
uint32 current = atomic_get((int32*)&lock->count);
if (count % 2 == 1 || current != count) {
PAUSE();
cpu_pause();
return false;
}
@@ -719,7 +719,7 @@ retry:
state = disable_interrupts();
process_all_pending_ici(smp_get_current_cpu());
restore_interrupts(state);
PAUSE();
cpu_pause();
}
state = disable_interrupts();
acquire_spinlock(&sFreeMessageSpinlock);
@@ -757,7 +757,7 @@ find_free_message_interrupts_disabled(int32 currentCPU,
while (sFreeMessageCount <= 0) {
release_spinlock(&sFreeMessageSpinlock);
process_all_pending_ici(currentCPU);
PAUSE();
cpu_pause();
acquire_spinlock_cpu(currentCPU, &sFreeMessageSpinlock);
}
@@ -1014,7 +1014,7 @@ call_all_cpus_early(void (*function)(void*, int), void* cookie)
// wait for all CPUs to finish
while ((atomic_get(&sEarlyCPUCall) & cpuMask) != 0)
PAUSE();
cpu_pause();
}
function(cookie, 0);
@@ -1085,7 +1085,7 @@ smp_send_ici(int32 targetCPU, int32 message, addr_t data, addr_t data2,
// if the message is sync after it has removed it from the mailbox
while (msg->done == false) {
process_all_pending_ici(currentCPU);
PAUSE();
cpu_pause();
}
// for SYNC messages, it's our responsibility to put it
// back into the free list
@@ -1148,7 +1148,7 @@ smp_send_multicast_ici(cpu_mask_t cpuMask, int32 message, addr_t data,
// if the message is sync after it has removed it from the mailbox
while (msg->done == false) {
process_all_pending_ici(currentCPU);
PAUSE();
cpu_pause();
}
// for SYNC messages, it's our responsibility to put it
@@ -1210,7 +1210,7 @@ smp_send_broadcast_ici(int32 message, addr_t data, addr_t data2, addr_t data3,
while (msg->done == false) {
process_all_pending_ici(currentCPU);
PAUSE();
cpu_pause();
}
TRACE(("smp_send_broadcast_ici: returning message to free list\n"));
@@ -1274,7 +1274,7 @@ smp_send_broadcast_ici_interrupts_disabled(int32 currentCPU, int32 message,
while (msg->done == false) {
process_all_pending_ici(currentCPU);
PAUSE();
cpu_pause();
}
TRACE(("smp_send_broadcast_ici_interrupts_disabled %ld: returning "
@@ -1310,7 +1310,7 @@ smp_trap_non_boot_cpus(int32 cpu, uint32* rendezVous)
if ((atomic_get(&sEarlyCPUCall) & (1 << cpu)) != 0)
process_early_cpu_call(cpu);
PAUSE();
cpu_pause();
}
return false;
@@ -1346,7 +1346,7 @@ smp_cpu_rendezvous(uint32* var, int current_cpu)
uint32 allReady = ((uint32)1 << sNumCPUs) - 1;
while ((uint32)atomic_get((int32*)var) != allReady)
PAUSE();
cpu_wait((int32*)var, allReady);
}
src/system/kernel/timer.cpp
+4 -6
View File
@@ -447,9 +447,8 @@ cancel_timer(timer* event)
if (cpu != smp_get_current_cpu()) {
spinLocker.Unlock();
while (atomic_get(&cpuData.current_event_in_progress) == 1) {
PAUSE();
}
while (atomic_get(&cpuData.current_event_in_progress) == 1)
cpu_wait(&cpuData.current_event_in_progress, 0);
}
return true;
@@ -461,7 +460,6 @@ spin(bigtime_t microseconds)
{
bigtime_t time = system_time();
while ((system_time() - time) < microseconds) {
PAUSE();
}
while ((system_time() - time) < microseconds)
cpu_pause();
}